@bitkyc08/opencodex 2.11.1 → 2.12.0

package/src/lab/artifacts/store.ts

@@ -0,0 +1,310 @@
+import { MAX_BYTES_PER_ARTIFACT, MAX_AGGREGATE_ARTIFACT_BYTES, MAX_ARTIFACTS_PER_RUN, ARTIFACT_FILENAME_EXT, type ArtifactClass, type ContractArtifactClass } from "../constants";
+import {
+  artifactBytesDigest,
+  claimSourceManifestDigest,
+  fixtureDigest,
+  isSha256Hex,
+  jcsStringify,
+  scenarioManifestDigest,
+  suiteManifestDigest,
+} from "../digest";
+import type { ArtifactRefV1, ClaimSourceManifestV1 } from "../events/types";
+import { artifactClassMediaType, validateClaimSourceManifest } from "../events/validate";
+import {
+  closeTrustedArtifactDir,
+  ArtifactFsError,
+  deleteArtifactBytes,
+  openTrustedArtifactDir,
+  putArtifactBytes,
+  putNamedDigestBytes,
+  readArtifactBytes,
+  type TrustedArtifactDir,
+} from "./secure-fs";
+import { redactForArtifact, sanitizeDiagnostic } from "./sanitize";
+
+export { ArtifactFsError, openTrustedArtifactDir };
+export type { TrustedArtifactDir };
+
+export interface PutArtifactInput {
+  artifactClass: ArtifactClass;
+  /** Pre-redaction payload; sanitizer runs before hash/write. */
+  payload: Uint8Array | string | unknown;
+  mediaType?: string;
+  redactionPolicy?: string;
+  /** For contract artifacts whose digest uses a domain other than artifact-bytes. */
+  expectedDigest?: string;
+}
+
+export interface ArtifactReadOptions {
+  expectedByteCount?: number;
+  artifactClass?: ArtifactClass;
+}
+
+export interface ArtifactStore {
+  dir: TrustedArtifactDir;
+  put(input: PutArtifactInput): ArtifactRefV1;
+  get(digest: string, expectedByteCountOrOpts?: number | ArtifactReadOptions): Uint8Array;
+  getVerified(
+    digest: string,
+    expectedByteCountOrOpts?: number | ArtifactReadOptions,
+  ): { bytes: Uint8Array; digest: string };
+  remove(digest: string): void;
+  close(): void;
+}
+
+function toBytes(payload: Uint8Array | string | unknown): Uint8Array {
+  if (payload instanceof Uint8Array) return payload;
+  if (typeof payload === "string") return new TextEncoder().encode(payload);
+  return new TextEncoder().encode(jcsStringify(payload));
+}
+
+function normalizeReadOptions(value?: number | ArtifactReadOptions): ArtifactReadOptions {
+  return typeof value === "number" ? { expectedByteCount: value } : value ?? {};
+}
+
+function parseContractJson(bytes: Uint8Array): unknown {
+  return JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(bytes));
+}
+
+function jsonDigest(
+  digest: (value: Record<string, unknown>) => string,
+): (bytes: Uint8Array) => string {
+  return (bytes) => {
+    try {
+      return digest(parseContractJson(bytes) as Record<string, unknown>);
+    } catch (err) {
+      if (err instanceof ArtifactFsError) throw err;
+      throw new ArtifactFsError("artifact_mismatch", "artifact content is not valid contract JSON");
+    }
+  };
+}
+
+function digestForArtifactClass(artifactClass: ArtifactClass): (bytes: Uint8Array) => string {
+  switch (artifactClass) {
+    case "fixture":
+      return fixtureDigest;
+    case "scenario_manifest":
+      return jsonDigest(scenarioManifestDigest);
+    case "suite_manifest":
+      return jsonDigest(suiteManifestDigest);
+    case "claim_source_manifest":
+      return (bytes) => {
+        try {
+          const parsed = parseContractJson(bytes);
+          return claimSourceManifestDigest(validateClaimSourceManifest(parsed).manifest);
+        } catch (err) {
+          if (err instanceof ArtifactFsError) throw err;
+          throw new ArtifactFsError("artifact_mismatch", "claim-source artifact failed validation");
+        }
+      };
+    default:
+      return artifactBytesDigest;
+  }
+}
+
+export function createArtifactStore(artifactsDir: string): ArtifactStore {
+  const dir = openTrustedArtifactDir(artifactsDir);
+  let aggregateBytes = 0;
+  let putCount = 0;
+
+  const getVerified = (
+    digest: string,
+    expectedByteCountOrOpts?: number | ArtifactReadOptions,
+  ): { bytes: Uint8Array; digest: string } => {
+    const opts = normalizeReadOptions(expectedByteCountOrOpts);
+    const candidates = opts.artifactClass
+      ? [digestForArtifactClass(opts.artifactClass)]
+      : [
+          artifactBytesDigest,
+          fixtureDigest,
+          jsonDigest(scenarioManifestDigest),
+          jsonDigest(suiteManifestDigest),
+          digestForArtifactClass("claim_source_manifest"),
+        ];
+
+    let lastErr: unknown;
+    for (const contentDigest of candidates) {
+      try {
+        const got = readArtifactBytes(dir, digest, {
+          expectedByteCount: opts.expectedByteCount,
+          contentDigest,
+        });
+        return { bytes: got.bytes, digest: got.digest };
+      } catch (err) {
+        lastErr = err;
+        if (
+          err instanceof ArtifactFsError &&
+          err.code !== "artifact_mismatch" &&
+          !err.message.includes("mismatch")
+        ) {
+          throw err;
+        }
+      }
+    }
+    throw lastErr instanceof Error
+      ? lastErr
+      : new ArtifactFsError("artifact_mismatch", "artifact digest verification failed");
+  };
+
+  return {
+    dir,
+    put(input: PutArtifactInput): ArtifactRefV1 {
+      if (putCount >= MAX_ARTIFACTS_PER_RUN) {
+        throw new ArtifactFsError("budget_exhausted", "maximum artifacts per run exceeded");
+      }
+      if (input.payload instanceof Uint8Array && input.payload.byteLength > MAX_BYTES_PER_ARTIFACT) {
+        throw new ArtifactFsError("budget_exhausted", `artifact exceeds ${MAX_BYTES_PER_ARTIFACT} bytes`);
+      }
+      if (typeof input.payload === "string" && new TextEncoder().encode(input.payload).byteLength > MAX_BYTES_PER_ARTIFACT) {
+        throw new ArtifactFsError("budget_exhausted", `artifact exceeds ${MAX_BYTES_PER_ARTIFACT} bytes`);
+      }
+      const redacted = redactForArtifact(input.artifactClass, input.payload);
+      const bytes = toBytes(redacted);
+      if (bytes.byteLength > MAX_BYTES_PER_ARTIFACT) {
+        throw new ArtifactFsError("budget_exhausted", `artifact exceeds ${MAX_BYTES_PER_ARTIFACT} bytes`);
+      }
+      if (aggregateBytes + bytes.byteLength > MAX_AGGREGATE_ARTIFACT_BYTES) {
+        throw new ArtifactFsError("budget_exhausted", "aggregate artifact ceiling exceeded");
+      }
+
+      let stored;
+      if (isContractClass(input.artifactClass)) {
+        const contractClass = input.artifactClass;
+        let computedDigest: string;
+        try {
+          computedDigest = computeContractDigest(contractClass, bytes, redacted);
+        } catch (err) {
+          if (err instanceof ArtifactFsError) throw err;
+          throw new ArtifactFsError(
+            "artifact_mismatch",
+            err instanceof Error ? err.message : "contract artifact failed validation",
+          );
+        }
+        if (input.expectedDigest !== undefined && computedDigest !== input.expectedDigest) {
+          throw new ArtifactFsError("artifact_mismatch", "contract artifact digest mismatch");
+        }
+        stored = putNamedDigestBytes(
+          dir,
+          computedDigest,
+          bytes,
+          digestForArtifactClass(contractClass),
+        );
+      } else {
+        stored = putArtifactBytes(dir, bytes, input.expectedDigest);
+      }
+
+      putCount += 1;
+      aggregateBytes += stored.byteCount;
+      return {
+        digest: stored.digest,
+        mediaType: input.mediaType ?? artifactClassMediaType(input.artifactClass),
+        byteCount: stored.byteCount,
+        redactionPolicy: input.redactionPolicy ?? defaultRedactionPolicy(input.artifactClass),
+        relativePath: `${stored.digest}${ARTIFACT_FILENAME_EXT}`,
+        artifactClass: input.artifactClass,
+      };
+    },
+    get(digest: string, expectedByteCountOrOpts?: number | ArtifactReadOptions): Uint8Array {
+      return getVerified(digest, expectedByteCountOrOpts).bytes;
+    },
+    getVerified,
+    remove(digest: string): void {
+      deleteArtifactBytes(dir, digest);
+    },
+    close(): void {
+      closeTrustedArtifactDir(dir);
+    },
+  };
+}
+
+function isContractClass(c: ArtifactClass): c is ContractArtifactClass {
+  return (
+    c === "scenario_manifest" ||
+    c === "suite_manifest" ||
+    c === "fixture" ||
+    c === "claim_source_manifest"
+  );
+}
+
+function computeContractDigest(
+  artifactClass: ContractArtifactClass,
+  bytes: Uint8Array,
+  redacted: unknown,
+): string {
+  switch (artifactClass) {
+    case "fixture":
+      return fixtureDigest(bytes);
+    case "scenario_manifest":
+      return scenarioManifestDigest(
+        typeof redacted === "object" && redacted ? (redacted as Record<string, unknown>) : parseContractJson(bytes) as Record<string, unknown>,
+      );
+    case "suite_manifest":
+      return suiteManifestDigest(
+        typeof redacted === "object" && redacted ? (redacted as Record<string, unknown>) : parseContractJson(bytes) as Record<string, unknown>,
+      );
+    case "claim_source_manifest": {
+      const parsed = typeof redacted === "object" && redacted
+        ? redacted
+        : parseContractJson(bytes);
+      return claimSourceManifestDigest(validateClaimSourceManifest(parsed).manifest);
+    }
+    default: {
+      const _never: never = artifactClass;
+      return _never;
+    }
+  }
+}
+
+function defaultRedactionPolicy(artifactClass: ArtifactClass): string {
+  switch (artifactClass) {
+    case "scenario_manifest":
+    case "suite_manifest":
+    case "fixture":
+    case "claim_source_manifest":
+      return "contract_canonical_v1";
+    default:
+      // v2 marks the sanitizer that also redacts network and account
+      // identifiers. The field records which algorithm produced the bytes, so
+      // it moves whenever those semantics change.
+      return "sanitized_evidence_v2";
+  }
+}
+
+export function putClaimSourceManifest(
+  store: ArtifactStore,
+  manifest: ClaimSourceManifestV1,
+): ArtifactRefV1 {
+  const { manifest: validated, digest } = validateClaimSourceManifest(manifest);
+  return store.put({
+    artifactClass: "claim_source_manifest",
+    payload: validated,
+    expectedDigest: digest,
+  });
+}
+
+export type LoadClaimSourceManifestResult =
+  | { ok: true; manifest: ClaimSourceManifestV1; corruption?: undefined }
+  | { ok: false; manifest: ClaimSourceManifestV1 | null; corruption: string };
+
+export function loadClaimSourceManifest(
+  store: ArtifactStore,
+  digest: string,
+  expected: { subjectId: string; capability: string },
+): LoadClaimSourceManifestResult {
+  if (!isSha256Hex(digest)) return { ok: false, manifest: null, corruption: "invalid digest" };
+  try {
+    const bytes = store.get(digest, { artifactClass: "claim_source_manifest" });
+    const parsed = parseContractJson(bytes);
+    const { manifest, digest: recomputed } = validateClaimSourceManifest(parsed);
+    if (recomputed !== digest) return { ok: false, manifest, corruption: "claim-source digest mismatch" };
+    if (manifest.subjectId !== expected.subjectId) return { ok: false, manifest, corruption: "claim-source subjectId mismatch" };
+    if (manifest.capability !== expected.capability) return { ok: false, manifest, corruption: "claim-source capability mismatch" };
+    return { ok: true, manifest };
+  } catch (err) {
+    return {
+      ok: false,
+      manifest: null,
+      corruption: sanitizeDiagnostic(err),
+    };
+  }
+}

package/src/lab/conformance/assertion.ts

@@ -0,0 +1,325 @@
+import { jcsEqual } from "./jcs";
+import { pointerExists, resolveJsonPointer } from "./json-pointer";
+import type { AssertionResult, AssertionSpec, NormalizedObservation } from "./types";
+
+const ID_GRAMMARS: Record<string, RegExp> = {
+  responses_message: /^msg_[A-Za-z0-9_-]{1,128}$/,
+  responses_reasoning: /^rs_[A-Za-z0-9_-]{1,128}$/,
+  responses_call: /^call_[A-Za-z0-9_-]{1,128}$/,
+  nonempty_128: /^[^\s]{1,128}$/,
+};
+
+export function evaluateAssertion(
+  assertion: AssertionSpec,
+  observation: NormalizedObservation,
+): AssertionResult {
+  const base: AssertionResult = {
+    id: assertion.id,
+    operator: assertion.operator,
+    required: assertion.required,
+    passed: false,
+    observedSummary: "",
+  };
+
+  try {
+    switch (assertion.operator) {
+      case "http_status_equals":
+        return evaluateEquals(assertion, observation, (obs) => obs.client.response.status);
+      case "json_path_equals":
+        return evaluateJsonPathEquals(assertion, observation);
+      case "json_path_present":
+        return evaluatePresence(assertion, observation, true);
+      case "json_path_absent":
+        return evaluatePresence(assertion, observation, false);
+      case "sse_event_sequence":
+        return evaluateEventSequence(assertion, observation);
+      case "sse_event_count":
+        return evaluateEventCount(assertion, observation);
+      case "terminal_signal_equals":
+        return evaluateEquals(assertion, observation, (obs) => obs.client.response.terminal);
+      case "id_matches":
+        return evaluateIdMatches(assertion, observation);
+      case "id_stable_across_events":
+        return evaluateIdStable(assertion, observation);
+      case "id_correlates":
+        return evaluateIdCorrelates(assertion, observation);
+      case "tool_call_equals":
+        return evaluateJsonPathEquals(assertion, observation);
+      case "tool_result_correlates":
+        return evaluateToolResultCorrelates(assertion, observation);
+      case "normalized_text_equals":
+        return evaluateEquals(assertion, observation, (obs) => obs.client.response.normalizedText);
+      case "verifier_result_equals":
+        return evaluateJsonPathEquals(assertion, observation);
+      default:
+        return {
+          ...base,
+          passed: false,
+          observedSummary: `unknown operator ${assertion.operator}`,
+          reason: "unknown_operator",
+        };
+    }
+  } catch (error) {
+    return {
+      ...base,
+      passed: false,
+      observedSummary: String(error),
+      reason: "evaluation_error",
+    };
+  }
+}
+
+function evaluateEquals(
+  assertion: AssertionSpec,
+  observation: NormalizedObservation,
+  pick: (obs: NormalizedObservation) => unknown,
+): AssertionResult {
+  const observed = pick(observation);
+  const passed = jcsEqual(observed, assertion.expected);
+  return {
+    id: assertion.id,
+    operator: assertion.operator,
+    required: assertion.required,
+    passed,
+    observedSummary: summarize(observed),
+    reason: passed ? undefined : "value_mismatch",
+  };
+}
+
+function evaluateJsonPathEquals(assertion: AssertionSpec, observation: NormalizedObservation): AssertionResult {
+  const resolved = resolveJsonPointer(observation, assertion.selector);
+  if (!resolved.ok) {
+    return {
+      id: assertion.id,
+      operator: assertion.operator,
+      required: assertion.required,
+      passed: false,
+      observedSummary: resolved.reason,
+      reason: resolved.reason,
+    };
+  }
+  const passed = jcsEqual(resolved.value, assertion.expected);
+  return {
+    id: assertion.id,
+    operator: assertion.operator,
+    required: assertion.required,
+    passed,
+    observedSummary: summarize(resolved.value),
+    reason: passed ? undefined : "value_mismatch",
+  };
+}
+
+function evaluatePresence(
+  assertion: AssertionSpec,
+  observation: NormalizedObservation,
+  shouldExist: boolean,
+): AssertionResult {
+  const exists = pointerExists(observation, assertion.selector);
+  const passed = shouldExist ? exists : !exists;
+  return {
+    id: assertion.id,
+    operator: assertion.operator,
+    required: assertion.required,
+    passed,
+    observedSummary: exists ? "present" : "absent",
+    reason: passed ? undefined : shouldExist ? "selector_missing" : "selector_present",
+  };
+}
+
+function evaluateEventSequence(assertion: AssertionSpec, observation: NormalizedObservation): AssertionResult {
+  const events = observation.client.response.events.map((e) => e.event);
+  const expected = assertion.expected as string[];
+  const passed = Array.isArray(expected)
+    && events.length === expected.length
+    && events.every((e, i) => e === expected[i]);
+  return {
+    id: assertion.id,
+    operator: assertion.operator,
+    required: assertion.required,
+    passed,
+    observedSummary: summarize(events),
+    reason: passed ? undefined : "event_sequence_mismatch",
+  };
+}
+
+function evaluateEventCount(assertion: AssertionSpec, observation: NormalizedObservation): AssertionResult {
+  const spec = assertion.expected as { event: string; count: number };
+  const count = observation.client.response.events.filter((e) => e.event === spec.event).length;
+  const passed = count === spec.count;
+  return {
+    id: assertion.id,
+    operator: assertion.operator,
+    required: assertion.required,
+    passed,
+    observedSummary: String(count),
+    reason: passed ? undefined : "event_count_mismatch",
+  };
+}
+
+function evaluateIdMatches(assertion: AssertionSpec, observation: NormalizedObservation): AssertionResult {
+  const resolved = resolveJsonPointer(observation, assertion.selector);
+  if (!resolved.ok) {
+    return {
+      id: assertion.id,
+      operator: assertion.operator,
+      required: assertion.required,
+      passed: false,
+      observedSummary: resolved.reason,
+      reason: resolved.reason,
+    };
+  }
+  const grammar = ID_GRAMMARS[String(assertion.expected)];
+  const value = typeof resolved.value === "string" ? resolved.value : "";
+  const passed = grammar ? grammar.test(value) : false;
+  return {
+    id: assertion.id,
+    operator: assertion.operator,
+    required: assertion.required,
+    passed,
+    observedSummary: value,
+    reason: passed ? undefined : "id_grammar_mismatch",
+  };
+}
+
+function evaluateIdStable(assertion: AssertionSpec, observation: NormalizedObservation): AssertionResult {
+  const pointers = assertion.expected;
+  if (!Array.isArray(pointers) || pointers.length < 2 || !pointers.every((p) => typeof p === "string")) {
+    return {
+      id: assertion.id,
+      operator: assertion.operator,
+      required: assertion.required,
+      passed: false,
+      observedSummary: "expected at least two pointers",
+      reason: "invalid_expected",
+    };
+  }
+  const values: string[] = [];
+  for (const pointer of pointers) {
+    const resolved = resolveJsonPointer(observation, pointer);
+    if (!resolved.ok) {
+      return {
+        id: assertion.id,
+        operator: assertion.operator,
+        required: assertion.required,
+        passed: false,
+        observedSummary: resolved.reason,
+        reason: resolved.reason,
+      };
+    }
+    if (typeof resolved.value !== "string") {
+      return {
+        id: assertion.id,
+        operator: assertion.operator,
+        required: assertion.required,
+        passed: false,
+        observedSummary: "identifier must be a string",
+        reason: "selector_type_mismatch",
+      };
+    }
+    values.push(resolved.value);
+  }
+  const passed = values.every((v) => v === values[0]);
+  return {
+    id: assertion.id,
+    operator: assertion.operator,
+    required: assertion.required,
+    passed,
+    observedSummary: summarize(values),
+    reason: passed ? undefined : "id_not_stable",
+  };
+}
+
+function correlatedIds(left: unknown, right: unknown): boolean {
+  return typeof left === "string"
+    && typeof right === "string"
+    && left.length > 0
+    && right.length > 0
+    && left === right;
+}
+
+function evaluateIdCorrelates(assertion: AssertionSpec, observation: NormalizedObservation): AssertionResult {
+  const pointers = assertion.expected;
+  if (!Array.isArray(pointers) || pointers.length !== 2 || !pointers.every((p) => typeof p === "string")) {
+    return {
+      id: assertion.id,
+      operator: assertion.operator,
+      required: assertion.required,
+      passed: false,
+      observedSummary: "expected two pointers",
+      reason: "invalid_expected",
+    };
+  }
+  const left = resolveJsonPointer(observation, pointers[0]);
+  const right = resolveJsonPointer(observation, pointers[1]);
+  if (!left.ok || !right.ok) {
+    return {
+      id: assertion.id,
+      operator: assertion.operator,
+      required: assertion.required,
+      passed: false,
+      observedSummary: left.ok ? (right as { reason: string }).reason : (left as { reason: string }).reason,
+      reason: "selector_missing",
+    };
+  }
+  const passed = correlatedIds(left.value, right.value);
+  return {
+    id: assertion.id,
+    operator: assertion.operator,
+    required: assertion.required,
+    passed,
+    observedSummary: `${summarize(left.value)} vs ${summarize(right.value)}`,
+    reason: passed ? undefined : "id_correlation_mismatch",
+  };
+}
+
+function evaluateToolResultCorrelates(assertion: AssertionSpec, observation: NormalizedObservation): AssertionResult {
+  const spec = assertion.expected as { call?: unknown; result?: unknown };
+  if (!spec || typeof spec.call !== "string" || typeof spec.result !== "string") {
+    return {
+      id: assertion.id,
+      operator: assertion.operator,
+      required: assertion.required,
+      passed: false,
+      observedSummary: "expected call/result pointers",
+      reason: "invalid_expected",
+    };
+  }
+  const call = resolveJsonPointer(observation, spec.call);
+  const result = resolveJsonPointer(observation, spec.result);
+  if (!call.ok || !result.ok) {
+    return {
+      id: assertion.id,
+      operator: assertion.operator,
+      required: assertion.required,
+      passed: false,
+      observedSummary: call.ok ? (result as { reason: string }).reason : (call as { reason: string }).reason,
+      reason: "selector_missing",
+    };
+  }
+  const passed = correlatedIds(call.value, result.value);
+  return {
+    id: assertion.id,
+    operator: assertion.operator,
+    required: assertion.required,
+    passed,
+    observedSummary: `${summarize(call.value)} -> ${summarize(result.value)}`,
+    reason: passed ? undefined : "tool_result_correlation_mismatch",
+  };
+}
+
+function summarize(value: unknown): string {
+  if (typeof value === "string") return value.length > 120 ? value.slice(0, 120) + "…" : value;
+  try {
+    const text = JSON.stringify(value);
+    return text.length > 200 ? text.slice(0, 200) + "…" : text;
+  } catch {
+    return String(value);
+  }
+}
+
+export function evaluateAssertions(
+  assertions: AssertionSpec[],
+  observation: NormalizedObservation,
+): AssertionResult[] {
+  return assertions.map((a) => evaluateAssertion(a, observation));
+}

package/src/lab/conformance/digest.ts

@@ -0,0 +1,22 @@
+import { createHash } from "node:crypto";
+import { jcsStringify } from "./jcs";
+
+function domainHash(domain: string, payload: Uint8Array | string): string {
+  const hash = createHash("sha256");
+  hash.update(new TextEncoder().encode(`${domain}\0`));
+  if (typeof payload === "string") hash.update(new TextEncoder().encode(payload));
+  else hash.update(payload);
+  return hash.digest("hex");
+}
+
+export function fixtureDigest(bytes: Uint8Array): string {
+  return domainHash("ocx-lab:fixture:v1", bytes);
+}
+
+export function scenarioManifestDigest(expandedScenario: Record<string, unknown>): string {
+  return domainHash("ocx-lab:scenario-manifest:v1", jcsStringify(expandedScenario));
+}
+
+export function suiteManifestDigest(expandedSuite: Record<string, unknown>): string {
+  return domainHash("ocx-lab:suite-manifest:v1", jcsStringify(expandedSuite));
+}