@bitkyc08/opencodex 2.11.1 → 2.12.0

package/src/lab/conformance/observation.ts

@@ -0,0 +1,355 @@
+import type {
+  CaseRecord,
+  NormalizedEvent,
+  NormalizedObservation,
+  ToolCallProjection,
+} from "./types";
+import { normalizeSseBytes } from "./sse-normalize";
+
+export function emptyObservation(): NormalizedObservation {
+  return {
+    client: {
+      request: { status: 0, headers: {}, json: null, rawBytes: 0 },
+      response: {
+        status: 0,
+        headers: {},
+        json: null,
+        events: [],
+        toolCalls: [],
+        mcpCalls: [],
+        terminal: null,
+        normalizedText: "",
+      },
+    },
+    upstream: { requests: [], responses: [] },
+    process: { exitCode: null },
+    verifiers: {},
+  };
+}
+
+export function recordUpstreamRequest(
+  observation: NormalizedObservation,
+  json: unknown,
+  status = 0,
+): void {
+  const body = JSON.stringify(json ?? null);
+  observation.upstream.requests.push({
+    status,
+    headers: {},
+    json,
+    rawBytes: new TextEncoder().encode(body).byteLength,
+  });
+}
+
+export function setClientResponse(
+  observation: NormalizedObservation,
+  patch: Partial<NormalizedObservation["client"]["response"]>,
+): void {
+  observation.client.response = { ...observation.client.response, ...patch };
+}
+
+function parseToolArguments(raw: unknown, kind: "function" | "custom"): unknown {
+  if (kind === "custom") return typeof raw === "string" ? raw : "";
+  if (typeof raw === "string") {
+    try { return JSON.parse(raw); } catch { return null; }
+  }
+  return raw;
+}
+
+interface ToolCallProjectionResult {
+  calls: ToolCallProjection[];
+  sawCallItems: boolean;
+  duplicateIds: boolean;
+}
+
+function projectToolCallsDetailed(output: unknown[]): ToolCallProjectionResult {
+  const calls: ToolCallProjection[] = [];
+  let ordinal = 0;
+  let sawCallItems = false;
+  for (const item of output) {
+    if (!item || typeof item !== "object") continue;
+    const rec = item as Record<string, unknown>;
+    if (rec.type === "function_call") {
+      sawCallItems = true;
+      const id = rec.call_id ?? rec.id;
+      const name = rec.name;
+      if (typeof id !== "string" || id.length === 0 || typeof name !== "string" || name.length === 0) continue;
+      const args = parseToolArguments(rec.arguments, "function");
+      if (args === null) continue;
+      calls.push({ id, name, arguments: args, kind: "function", ordinal: ordinal++ });
+    } else if (rec.type === "custom_tool_call") {
+      sawCallItems = true;
+      const id = rec.call_id ?? rec.id;
+      const name = rec.name;
+      if (typeof id !== "string" || id.length === 0 || typeof name !== "string" || name.length === 0) continue;
+      calls.push({
+        id,
+        name,
+        arguments: parseToolArguments(rec.input, "custom"),
+        kind: "custom",
+        ordinal: ordinal++,
+      });
+    }
+  }
+  const ids = calls.map((call) => call.id);
+  const duplicateIds = new Set(ids).size !== ids.length;
+  return { calls: duplicateIds ? [] : calls, sawCallItems, duplicateIds };
+}
+
+/** Build toolCalls projection from Responses output items or SSE events (manifest §5). */
+export function projectToolCallsFromOutput(output: unknown[]): ToolCallProjection[] {
+  return projectToolCallsDetailed(output).calls;
+}
+
+function projectToolCallsFromEventsDetailed(events: NormalizedEvent[]): ToolCallProjectionResult {
+  const output: unknown[] = [];
+  for (const ev of events) {
+    if (ev.event === "response.output_item.done" && ev.data && typeof ev.data === "object") {
+      const data = ev.data as Record<string, unknown>;
+      const item = data.item;
+      if (item && typeof item === "object") output.push(item);
+    }
+  }
+  return projectToolCallsDetailed(output);
+}
+
+export function projectToolCallsFromEvents(events: NormalizedEvent[]): ToolCallProjection[] {
+  return projectToolCallsFromEventsDetailed(events).calls;
+}
+
+export function projectMcpCalls(toolCalls: ToolCallProjection[]): Array<{ namespace: string; name: string }> {
+  const out: Array<{ namespace: string; name: string }> = [];
+  for (const call of toolCalls) {
+    if (!call.name.startsWith("mcp__")) continue;
+    const idx = call.name.lastIndexOf("__");
+    if (idx <= 0 || idx >= call.name.length - 2) continue;
+    const namespace = call.name.slice(0, idx);
+    const name = call.name.slice(idx + 2);
+    if (!namespace || !name) continue;
+    if (new TextEncoder().encode(namespace).byteLength > 64 || new TextEncoder().encode(name).byteLength > 64) continue;
+    out.push({ namespace, name });
+  }
+  return out;
+}
+
+export function filterAnthropicEvents(events: ReturnType<typeof normalizeSseBytes>): ReturnType<typeof normalizeSseBytes> {
+  return events.filter((e) => e.event !== "ping");
+}
+
+function deriveTerminal(events: NormalizedEvent[]): string | null {
+  if (events.some((e) => e.event === "error")) return "failed";
+  if (events.some((e) => e.event === "response.failed")) return "failed";
+  if (events.some((e) => e.event === "response.completed")) return "completed";
+  if (events.some((e) => e.event === "message_stop")) return "message_stop";
+  if (events.some((e) => e.event === "response.incomplete")) return "incomplete";
+  return null;
+}
+
+function extractOutputText(json: Record<string, unknown>): string {
+  let text = "";
+  const output = json.output;
+  if (!Array.isArray(output)) return text;
+  for (const item of output) {
+    if (!item || typeof item !== "object") continue;
+    const content = (item as { content?: unknown[] }).content;
+    if (!Array.isArray(content)) continue;
+    for (const part of content) {
+      if (part && typeof part === "object" && (part as { type?: string }).type === "output_text") {
+        text += String((part as { text?: string }).text ?? "");
+      }
+    }
+  }
+  return text;
+}
+
+export function deriveNormalizedText(events: NormalizedEvent[], json: unknown): string {
+  if (json && typeof json === "object" && !Array.isArray(json)) {
+    const text = extractOutputText(json as Record<string, unknown>);
+    if (text) return text;
+  }
+  let text = "";
+  for (const ev of events) {
+    if (ev.event === "response.output_text.delta" && ev.data && typeof ev.data === "object") {
+      text += String((ev.data as { delta?: string }).delta ?? "");
+    }
+    if (ev.event === "content_block_delta" && ev.data && typeof ev.data === "object") {
+      const delta = (ev.data as { delta?: { text?: string } }).delta;
+      if (delta && typeof delta.text === "string") text += delta.text;
+    }
+  }
+  return text;
+}
+
+export function finalizeObservation(
+  observation: NormalizedObservation,
+  events: NormalizedEvent[],
+  json: unknown = null,
+  status = 200,
+): void {
+  const eventProjection = projectToolCallsFromEventsDetailed(events);
+  const jsonProjection = projectToolCallsDetailed(
+    json && typeof json === "object" && !Array.isArray(json)
+      ? ((json as { output?: unknown[] }).output ?? [])
+      : [],
+  );
+  const selectedProjection = eventProjection.sawCallItems ? eventProjection : jsonProjection;
+  const resolvedToolCalls = selectedProjection.calls;
+  const terminal = deriveTerminal(events);
+  setClientResponse(observation, {
+    events,
+    toolCalls: resolvedToolCalls,
+    mcpCalls: projectMcpCalls(resolvedToolCalls),
+    terminal,
+    normalizedText: deriveNormalizedText(events, json),
+    json,
+    status,
+  });
+  observation.verifiers.duplicate_tool_call_ids = selectedProjection.duplicateIds;
+}
+
+export function attachVerifiers(observation: NormalizedObservation, caseRecord: CaseRecord): void {
+  observation.verifiers = { ...observation.verifiers, ...buildVerifiers(observation, caseRecord) };
+}
+
+function buildVerifiers(observation: NormalizedObservation, caseRecord: CaseRecord): Record<string, unknown> {
+  const verifiers: Record<string, unknown> = {};
+  const toolCalls = observation.client.response.toolCalls;
+
+  verifiers.nonoverlap_order = (() => {
+    const ids: string[] = [];
+    for (let i = 0; i < toolCalls.length; i++) {
+      const call = toolCalls[i];
+      if (!call.id || call.arguments === null) return [];
+      if (call.ordinal !== i) return [];
+      ids.push(call.id);
+    }
+    const unique = new Set(ids);
+    return unique.size === ids.length ? ids : [];
+  })();
+
+  verifiers.call_result_order = evaluateCallResultOrder(observation);
+
+  if (caseRecord.id === "codex-core.protocol.compaction-and-special-items") {
+    verifiers.compaction_replayed = evaluateCompactionReplayed(caseRecord);
+    verifiers.local_shell_correlated = evaluateLocalShellCorrelated(caseRecord);
+    verifiers.tool_search_error = evaluateToolSearchError(caseRecord);
+  }
+
+  if (caseRecord.id === "responses-core.protocol.json-sse-equivalence") {
+    verifiers.json_sse_equivalence = evaluateJsonSseEquivalence(caseRecord);
+  }
+
+  if (caseRecord.id === "vision-core.protocol.modality-gate") {
+    verifiers.modality_path = evaluateModalityPath(caseRecord);
+    verifiers.silent_image_drop = evaluateSilentImageDrop(caseRecord);
+  }
+
+  return verifiers;
+}
+
+function evaluateCallResultOrder(observation: NormalizedObservation): string {
+  const request = observation.upstream.requests[0]?.json as { input?: unknown[] } | undefined;
+  const input = request?.input;
+  if (!Array.isArray(input)) return "fail";
+  const pendingCallIds = new Set<string>();
+  let callCount = 0;
+  let resultCount = 0;
+  for (const item of input) {
+    if (!item || typeof item !== "object") continue;
+    const record = item as { type?: string; call_id?: unknown };
+    if (record.type === "function_call") {
+      if (typeof record.call_id !== "string" || record.call_id.length === 0) return "fail";
+      if (pendingCallIds.has(record.call_id)) return "fail";
+      pendingCallIds.add(record.call_id);
+      callCount++;
+      continue;
+    }
+    if (record.type === "function_call_output") {
+      if (typeof record.call_id !== "string" || record.call_id.length === 0) return "fail";
+      if (!pendingCallIds.delete(record.call_id)) return "fail";
+      resultCount++;
+    }
+  }
+  return callCount > 0 && resultCount === callCount && pendingCallIds.size === 0 ? "pass" : "fail";
+}
+
+function evaluateCompactionReplayed(caseRecord: CaseRecord): boolean {
+  const vector = JSON.parse(caseRecord.fixture.bytesUtf8) as { input?: unknown[] };
+  const input = vector.input;
+  if (!Array.isArray(input)) return false;
+  const compaction = input.find((i) => i && typeof i === "object" && (i as { type?: string }).type === "context_compaction");
+  if (!compaction) return false;
+  return typeof (compaction as { encrypted_content?: string }).encrypted_content === "string";
+}
+
+function evaluateLocalShellCorrelated(caseRecord: CaseRecord): boolean {
+  const vector = JSON.parse(caseRecord.fixture.bytesUtf8) as { input?: unknown[] };
+  const input = vector.input;
+  if (!Array.isArray(input)) return false;
+  let shellId: string | undefined;
+  for (const item of input) {
+    if (!item || typeof item !== "object") continue;
+    const type = (item as { type?: string }).type;
+    if (type === "local_shell_call") {
+      shellId = String((item as { call_id?: string }).call_id ?? "");
+      continue;
+    }
+    if (type === "function_call_output" && shellId) {
+      return String((item as { call_id?: string }).call_id ?? "") === shellId;
+    }
+  }
+  return false;
+}
+
+function evaluateToolSearchError(caseRecord: CaseRecord): string | null {
+  const vector = JSON.parse(caseRecord.fixture.bytesUtf8) as { input?: unknown[] };
+  const input = vector.input;
+  if (!Array.isArray(input)) return null;
+  const failed = input.filter((i) => i && typeof i === "object" && (i as { type?: string }).type === "tool_search_output"
+    && (i as { status?: string }).status === "failed");
+  if (failed.length !== 1) return null;
+  return String((failed[0] as { error?: string }).error ?? "");
+}
+
+function evaluateModalityPath(caseRecord: CaseRecord): string {
+  const vector = JSON.parse(caseRecord.fixture.bytesUtf8) as Record<string, unknown>;
+  const requestHasImage = Boolean(vector.requestHasImage);
+  const modalities = vector.modelInputModalities as string[] | undefined;
+  const sidecar = vector.visionSidecar as { enabled?: boolean } | undefined;
+  if (requestHasImage && Array.isArray(modalities) && modalities.includes("image")) return "native";
+  if (sidecar?.enabled) return "sidecar";
+  return "unsupported";
+}
+
+function evaluateSilentImageDrop(caseRecord: CaseRecord): boolean {
+  const vector = JSON.parse(caseRecord.fixture.bytesUtf8) as Record<string, unknown>;
+  if (!Boolean(vector.requestHasImage)) return false;
+  // Protocol V1's closed modality-gate vector treats the explicit `unsupported` path as a
+  // typed rejection, not as an omitted image. Native/sidecar paths likewise preserve it.
+  return !["native", "sidecar", "unsupported"].includes(evaluateModalityPath(caseRecord));
+}
+
+function evaluateJsonSseEquivalence(caseRecord: CaseRecord): string {
+  const vector = JSON.parse(caseRecord.fixture.bytesUtf8) as { json?: Record<string, unknown>; sse?: string };
+  const json = vector.json;
+  const sse = vector.sse ?? "";
+  if (!json) return "fail";
+  const jsonProjection = {
+    text: extractOutputText(json),
+    terminal: String(json.status ?? ""),
+  };
+  const events = normalizeSseBytes(new TextEncoder().encode(sse), "openai-responses");
+  let sseText = "";
+  let sseTerminal = "";
+  for (const ev of events) {
+    if (ev.event === "response.output_text.delta" && ev.data && typeof ev.data === "object") {
+      sseText += String((ev.data as { delta?: string }).delta ?? "");
+    }
+    if (ev.event === "response.completed" && ev.data && typeof ev.data === "object") {
+      const response = (ev.data as { response?: { status?: string } }).response;
+      sseTerminal = String(response?.status ?? "");
+    }
+  }
+  const sseProjection = { text: sseText, terminal: sseTerminal };
+  return JSON.stringify(jsonProjection) === JSON.stringify(sseProjection) ? "pass" : "fail";
+}

package/src/lab/conformance/runner.ts

@@ -0,0 +1,57 @@
+import { discoverScenarios, loadCaseAuthority } from "./manifest";
+import { runScenario } from "./executor";
+import { buildNegativeControls } from "./negative-controls";
+import type { ScenarioRunResult } from "./types";
+import { CL01_SUITES } from "./types";
+
+export interface ConformanceRunSummary {
+  /** Number of scenarios executed. */
+  total: number;
+  /** Number of scenarios that met their expected outcome. */
+  passed: number;
+  /** Number of scenarios that did not meet their expected outcome. */
+  failed: number;
+  results: ScenarioRunResult[];
+}
+
+export interface NegativeControlRunSummary extends ConformanceRunSummary {
+  /** Number of deliberately defective controls correctly rejected by the harness. */
+  rejected: number;
+}
+
+export async function runConformanceSuite(
+  suites: readonly string[] = CL01_SUITES,
+): Promise<ConformanceRunSummary> {
+  const authority = loadCaseAuthority();
+  const scenarios = discoverScenarios(authority, suites);
+  if (scenarios.length === 0) throw new Error("harness_failure: no CL-01 scenarios discovered");
+  const results: ScenarioRunResult[] = [];
+  for (const scenario of scenarios) {
+    results.push(await runScenario(scenario));
+  }
+  const passed = results.filter((r) => r.passed).length;
+  return { total: results.length, passed, failed: results.length - passed, results };
+}
+
+export async function runNegativeControls(): Promise<NegativeControlRunSummary> {
+  const authority = loadCaseAuthority();
+  const scenarios = buildNegativeControls(discoverScenarios(authority));
+  if (scenarios.length === 0) throw new Error("harness_failure: no negative controls discovered");
+  const results: ScenarioRunResult[] = [];
+  for (const scenario of scenarios) {
+    results.push(await runScenario(scenario));
+  }
+  const rejected = results.filter((r) => !r.passed).length;
+  return {
+    total: results.length,
+    passed: rejected,
+    rejected,
+    failed: results.length - rejected,
+    results,
+  };
+}
+
+export function listScenarioIds(suites: readonly string[] = CL01_SUITES): string[] {
+  const authority = loadCaseAuthority();
+  return discoverScenarios(authority, suites).map((s) => s.id);
+}

package/src/lab/conformance/sse-normalize.ts

@@ -0,0 +1,59 @@
+import { sseFieldValue } from "../../lib/sse-decoder";
+import type { NormalizedEvent } from "./types";
+
+/** CL-00 §5 SSE normalization for assertion observations. */
+export function normalizeSseBytes(bytes: Uint8Array, sourceProtocol: string): NormalizedEvent[] {
+  let text = new TextDecoder("utf-8", { fatal: true }).decode(bytes);
+  if (text.charCodeAt(0) === 0xfeff) text = text.slice(1);
+  text = text.replace(/\r\n/g, "\n").replace(/\r/g, "\n");
+
+  const events: NormalizedEvent[] = [];
+  let ordinal = 0;
+  const frames = text.split("\n\n");
+  for (const rawFrame of frames) {
+    if (!rawFrame.trim()) continue;
+    const lines = rawFrame.split("\n");
+    const dataLines: string[] = [];
+    let eventName: string | undefined;
+    for (const line of lines) {
+      if (line.startsWith(":")) continue;
+      const eventValue = sseFieldValue(line, "event");
+      if (eventValue !== null) {
+        eventName = eventValue;
+        continue;
+      }
+      const dataValue = sseFieldValue(line, "data");
+      if (dataValue !== null) dataLines.push(dataValue);
+    }
+    if (dataLines.length === 0) continue;
+    const joined = dataLines.join("\n");
+    if (sourceProtocol === "openai-chat" && joined === "[DONE]") {
+      events.push({ event: "[DONE]", data: "[DONE]", ordinal: ordinal++ });
+      continue;
+    }
+    let parsed: unknown;
+    try {
+      parsed = JSON.parse(joined);
+    } catch {
+      events.push({ event: eventName ?? "malformed", data: joined, ordinal: ordinal++ });
+      continue;
+    }
+    // Protocol V1 treats null, scalar, and array data values as padding.
+    if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) continue;
+    const inferred = eventName ?? (typeof (parsed as { type?: unknown }).type === "string"
+      ? (parsed as { type: string }).type
+      : "message");
+    events.push({ event: inferred, data: parsed, ordinal: ordinal++ });
+  }
+  return events;
+}
+
+export function eventsFromBridgeFrames(
+  frames: Array<{ event?: string; data: Record<string, unknown> }>,
+): NormalizedEvent[] {
+  return frames.map((frame, ordinal) => ({
+    event: frame.event ?? (typeof frame.data.type === "string" ? frame.data.type : "message"),
+    data: frame.data,
+    ordinal,
+  }));
+}

package/src/lab/conformance/suite-manifest.ts

@@ -0,0 +1,78 @@
+import { scenarioManifestDigest, suiteManifestDigest } from "../digest";
+import type { CaseAuthority, CaseRecord, VerificationRole } from "./types";
+import { expandScenario } from "./manifest";
+
+export interface SuiteScenarioRefV1 {
+  id: string;
+  version: string;
+  role: VerificationRole;
+  manifestDigest: string;
+}
+
+export interface SuiteManifestV1 {
+  schemaVersion: 1;
+  id: string;
+  version: string;
+  evidenceLayer: string;
+  capability: string;
+  assertionDslVersion: string;
+  evidenceSchemaVersion: string;
+  freshness: { maxAgeMs: number | null };
+  contradictionRule: string;
+  scenarios: SuiteScenarioRefV1[];
+  verificationRule: string;
+}
+
+/** Expand the canonical suite manifest for one suite ID from CL-01 authority. */
+export function expandSuiteManifest(
+  suiteId: string,
+  authority: CaseAuthority,
+): SuiteManifestV1 {
+  const cases = authority.cases.filter((c) => c.suite === suiteId);
+  if (cases.length === 0) {
+    throw new Error(`unknown suite ${suiteId}`);
+  }
+  const defaults = authority.manifestDefaults;
+  const capability = cases[0]!.capability;
+  if (cases.some((caseRecord) => caseRecord.capability !== capability)) {
+    throw new Error(`suite ${suiteId} declares mixed capabilities`);
+  }
+  const scenarios: SuiteScenarioRefV1[] = cases
+    .map((caseRecord) => suiteScenarioRef(caseRecord, authority))
+    .sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0));
+
+  return {
+    schemaVersion: 1,
+    id: suiteId,
+    version: String(defaults.suiteVersion),
+    evidenceLayer: defaults.evidenceLayer,
+    capability,
+    assertionDslVersion: authority.assertionDslVersion,
+    evidenceSchemaVersion: authority.evidenceSchemaVersion,
+    freshness: defaults.freshness ?? { maxAgeMs: null },
+    contradictionRule: "newest-required-observation-v1",
+    scenarios,
+    verificationRule: "all-applicable-required-pass-v1",
+  };
+}
+
+function suiteScenarioRef(caseRecord: CaseRecord, authority: CaseAuthority): SuiteScenarioRefV1 {
+  const expanded = expandScenario(caseRecord, authority);
+  return {
+    id: caseRecord.id,
+    version: String(authority.manifestDefaults.version),
+    role: caseRecord.verificationRole ?? authority.manifestDefaults.verificationRole,
+    manifestDigest: scenarioManifestDigest(expanded),
+  };
+}
+
+export function suiteManifestObjectForCase(
+  caseRecord: CaseRecord,
+  authority: CaseAuthority,
+): Record<string, unknown> {
+  return expandSuiteManifest(caseRecord.suite, authority) as unknown as Record<string, unknown>;
+}
+
+export function suiteManifestDigestForCase(caseRecord: CaseRecord, authority: CaseAuthority): string {
+  return suiteManifestDigest(expandSuiteManifest(caseRecord.suite, authority) as unknown as Record<string, unknown>);
+}