@iislee/opencodex 2.11.0 → 2.31.0

package/src/lab/conformance/types.ts

@@ -0,0 +1,214 @@
+/** CL-01 deterministic protocol conformance harness types (CL-00 contract). */
+
+export const SYNTHETIC_MARKER = "ocx-lab-synthetic-v1";
+
+export type EvidenceLayer = "protocol_conformance" | "live_route_compatibility" | "task_effectiveness";
+
+export type VerificationRole = "required" | "supplemental" | "negative_control";
+
+export type FailureClassification =
+  | "harness_failure"
+  | "timeout"
+  | "inactivity_timeout"
+  | "budget_exhausted"
+  | "sandbox_violation"
+  | "malformed_producer_outcome"
+  | "layer_subject_mismatch"
+  | "protocol_failure"
+  | "capability_failure"
+  | "behavioral_failure"
+  | "authentication_blocked"
+  | "quota_blocked"
+  | "region_blocked"
+  | "network_failure"
+  | "provider_transient"
+  | "inconclusive";
+
+export const FAILURE_CLASSIFICATIONS = [
+  "harness_failure",
+  "timeout",
+  "inactivity_timeout",
+  "budget_exhausted",
+  "sandbox_violation",
+  "malformed_producer_outcome",
+  "layer_subject_mismatch",
+  "protocol_failure",
+  "capability_failure",
+  "behavioral_failure",
+  "authentication_blocked",
+  "quota_blocked",
+  "region_blocked",
+  "network_failure",
+  "provider_transient",
+  "inconclusive",
+] as const satisfies readonly FailureClassification[];
+
+export interface FixtureRecord {
+  id: string;
+  role: "client_request" | "upstream_response" | "adapter_vector" | "synthetic_tool";
+  mediaType: string;
+  bytesUtf8: string;
+  digest: string;
+}
+
+export interface AssertionSpec {
+  id: string;
+  operator: string;
+  selector: string;
+  expected: unknown;
+  required: boolean;
+}
+
+export interface ExpectedFailureSpec {
+  controlKind: "conformance_negative_control" | "capability_absence_control";
+  expectedClass: FailureClassification;
+  expectedCode: string;
+  assertionIds: string[];
+  onMatch: "pass" | "unsupported";
+  onMismatch: "fail" | "inconclusive";
+}
+
+export interface CaseRecord {
+  id: string;
+  suite: string;
+  capability: string;
+  verificationRole?: VerificationRole;
+  requirements: {
+    inboundProtocols: string[];
+    upstreamProtocols: string[];
+    surfaces: string[];
+    requiredClaims: string[];
+    requiredHarnessFeatures: string[];
+    platforms: string[];
+    routePreconditions: string[];
+  };
+  fixture: FixtureRecord;
+  initiatingRequest?: FixtureRecord;
+  assertions: AssertionSpec[];
+  expectedFailure?: ExpectedFailureSpec;
+}
+
+export interface FailureRule {
+  id: string;
+  match: string[];
+  classification: FailureClassification;
+  secondaryCode?: string;
+  verdictEffect: "none" | "degraded" | "unsupported";
+  retry: "never" | "bounded" | "after_precondition_change";
+  expected: boolean;
+}
+
+export interface CaseAuthority {
+  schemaVersion: number;
+  sourceCommit: string;
+  assertionDslVersion: string;
+  evidenceSchemaVersion: string;
+  failureRuleSets: Record<string, FailureRule[]>;
+  expectedFailureRuleTemplate: Pick<FailureRule, "id" | "match" | "retry" | "expected">;
+  manifestDefaults: {
+    version: string;
+    suiteVersion: string;
+    evidenceLayer: EvidenceLayer;
+    verificationRole: VerificationRole;
+    executionMode: string;
+    freshness: { maxAgeMs: number | null };
+    executionLimits: Record<string, number>;
+    artifactPolicy: Record<string, unknown>;
+    failureRuleSet: string;
+  };
+  cases: CaseRecord[];
+}
+
+export interface NormalizedEvent {
+  event: string;
+  data: unknown;
+  ordinal: number;
+}
+
+export interface ToolCallProjection {
+  id: string;
+  name: string;
+  arguments: unknown;
+  kind: "function" | "custom";
+  ordinal: number;
+}
+
+export interface McpCallProjection {
+  namespace: string;
+  name: string;
+}
+
+export interface NormalizedObservation {
+  client: {
+    request: { status: number; headers: Record<string, string>; json: unknown; rawBytes: number };
+    response: {
+      status: number;
+      headers: Record<string, string>;
+      json: unknown;
+      events: NormalizedEvent[];
+      toolCalls: ToolCallProjection[];
+      mcpCalls: McpCallProjection[];
+      terminal: string | null;
+      normalizedText: string;
+    };
+  };
+  upstream: {
+    requests: Array<{ status: number; headers: Record<string, string>; json: unknown; rawBytes: number }>;
+    responses: unknown[];
+  };
+  process: { exitCode: number | null };
+  verifiers: Record<string, unknown>;
+}
+
+export interface AssertionResult {
+  id: string;
+  operator: string;
+  required: boolean;
+  passed: boolean;
+  observedSummary: string;
+  reason?: string;
+}
+
+export interface ProtocolExecutionContextV1 {
+  inboundProtocol: string;
+  upstreamProtocol: string;
+  surface: string;
+}
+
+export interface ScenarioRunResult {
+  scenarioId: string;
+  suite: string;
+  startedAt: number;
+  completedAt: number;
+  passed: boolean;
+  classification: FailureClassification;
+  secondaryCode?: string;
+  assertionResults: AssertionResult[];
+  expectedFailureMatched?: boolean;
+  diagnostics: string[];
+  executionContext: ProtocolExecutionContextV1;
+}
+
+/** All eight live-route compatibility suites frozen by CL-03 Live V1. */
+export const CL03_LIVE_SUITES = [
+  "responses-core",
+  "chat-core",
+  "anthropic-core",
+  "tools-core",
+  "codex-core",
+  "vision-core",
+  "reasoning-core",
+  "mcp-core",
+] as const;
+
+/** All eight protocol-conformance suites frozen by CL-00 Protocol V1. */
+export const CL01_SUITES = [
+  "responses-core",
+  "chat-core",
+  "anthropic-core",
+  "tools-core",
+  "codex-core",
+  "vision-core",
+  "reasoning-core",
+  "mcp-core",
+] as const;

package/src/lab/constants.ts

@@ -0,0 +1,126 @@
+/** CL-02 projection / ledger constants frozen from CL-00 contracts. */
+
+export const LAB_EVENT_SCHEMA_VERSION = 1;
+export const LAB_PROJECTION_SPEC_VERSION = "cl-02.v1";
+export const LAB_PRODUCER = "opencodex-lab";
+export const LAB_PRODUCER_VERSION = "2.10.2";
+
+export const MAX_INVALIDATION_TARGETS = 1024;
+export const MAX_BYTES_PER_ARTIFACT = 256 * 1024;
+export const MAX_AGGREGATE_ARTIFACT_BYTES = 1024 * 1024;
+export const MAX_ARTIFACTS_PER_RUN = 16;
+export const MAX_SERIALIZED_EVENT_BYTES = 64 * 1024;
+export const MAX_SANITIZED_STRING_FIELD = 4 * 1024;
+export const MAX_EVENT_NESTING_DEPTH = 8;
+export const MAX_OBJECT_KEYS_PER_EVENT = 64;
+export const MAX_ARRAY_ELEMENTS_PER_EVENT = 256;
+
+export const OBSERVATION_LIMIT_NAMES = [
+  "totalTimeoutMs",
+  "connectTimeoutMs",
+  "firstByteTimeoutMs",
+  "inactivityTimeoutMs",
+  "maxRequests",
+  "maxInputBytes",
+  "maxOutputBytes",
+  "maxOutputTokens",
+  "maxToolCalls",
+  "maxArtifactBytes",
+] as const;
+
+export type ObservationLimitName = (typeof OBSERVATION_LIMIT_NAMES)[number];
+
+export const EVENT_KINDS = [
+  "observation",
+  "claim_snapshot",
+  "invalidation",
+  "purge_tombstone",
+] as const;
+
+export type LabEventKind = (typeof EVENT_KINDS)[number];
+
+export const EVIDENCE_LAYERS = [
+  "protocol_conformance",
+  "live_route_compatibility",
+  "task_effectiveness",
+] as const;
+
+export type EvidenceLayer = (typeof EVIDENCE_LAYERS)[number];
+
+export const VERDICTS = [
+  "UNKNOWN",
+  "CLAIMED",
+  "PROBED",
+  "VERIFIED",
+  "DEGRADED",
+  "BLOCKED",
+  "UNSUPPORTED",
+] as const;
+
+export type CompatibilityVerdict = (typeof VERDICTS)[number];
+
+export const INVALIDATION_REASONS = [
+  "harness_defect",
+  "fixture_defect",
+  "redaction_defect",
+  "integrity_defect",
+  "contract_artifact_missing",
+  "manual_correction",
+] as const;
+
+export type InvalidationReason = (typeof INVALIDATION_REASONS)[number];
+
+export const PURGE_ACTIONS = [
+  "ledger",
+  "sqlite",
+  "artifact",
+  "scratch",
+  "export",
+] as const;
+
+export type PurgeAction = (typeof PURGE_ACTIONS)[number];
+
+export const CLAIM_SOURCE_KINDS = [
+  "provider_config",
+  "provider_registry",
+  "cached_catalog",
+  "native_metadata",
+  "adapter_inference",
+] as const;
+
+export type ClaimSourceKind = (typeof CLAIM_SOURCE_KINDS)[number];
+
+export const ARTIFACT_CLASSES = [
+  "scenario_manifest",
+  "suite_manifest",
+  "fixture",
+  "claim_source_manifest",
+  "assertion_report",
+  "request_shape",
+  "response_shape",
+  "event_trace",
+  "error_taxonomy",
+  "verifier_summary",
+] as const;
+
+export type ArtifactClass = (typeof ARTIFACT_CLASSES)[number];
+
+export const CONTRACT_ARTIFACT_CLASSES = [
+  "scenario_manifest",
+  "suite_manifest",
+  "fixture",
+  "claim_source_manifest",
+] as const;
+
+export type ContractArtifactClass = (typeof CONTRACT_ARTIFACT_CLASSES)[number];
+
+export const EXECUTION_MODES = ["fixture", "live", "fabric"] as const;
+export type ExecutionMode = (typeof EXECUTION_MODES)[number];
+
+export const OUTCOMES = ["pass", "fail", "blocked", "inconclusive"] as const;
+export type ObservationOutcome = (typeof OUTCOMES)[number];
+
+export const CLAIM_POLARITIES = ["supported", "not_supported", "withdrawn"] as const;
+export type ClaimPolarity = (typeof CLAIM_POLARITIES)[number];
+
+export const ARTIFACT_FILENAME_EXT = ".bin";

package/src/lab/digest.ts

@@ -0,0 +1,64 @@
+import { createHash, createHmac } from "node:crypto";
+import { jcsStringify } from "./conformance/jcs";
+import {
+  fixtureDigest,
+  scenarioManifestDigest,
+  suiteManifestDigest,
+} from "./conformance/digest";
+
+export { fixtureDigest, scenarioManifestDigest, suiteManifestDigest, jcsStringify };
+export { jcsEqual } from "./conformance/jcs";
+
+/** Domain-separated SHA-256: UTF-8(domain) || NUL || payload. */
+export function domainHash(domain: string, payload: Uint8Array | string): string {
+  const hash = createHash("sha256");
+  hash.update(new TextEncoder().encode(`${domain}\0`));
+  if (typeof payload === "string") hash.update(new TextEncoder().encode(payload));
+  else hash.update(payload);
+  return hash.digest("hex");
+}
+
+export const LAB_DOMAIN = {
+  event: "ocx-lab:event:v1",
+  subject: "ocx-lab:subject:v1",
+  claimSourceManifest: "ocx-lab:claim-source-manifest:v1",
+  scenarioManifest: "ocx-lab:scenario-manifest:v1",
+  suiteManifest: "ocx-lab:suite-manifest:v1",
+  fixture: "ocx-lab:fixture:v1",
+  localFingerprint: "ocx-lab:local-fingerprint:v1",
+  artifactBytes: "ocx-lab:artifact-bytes:v1",
+} as const;
+
+export function eventIdForPayload(eventWithoutId: unknown): string {
+  return domainHash(LAB_DOMAIN.event, jcsStringify(eventWithoutId));
+}
+
+export function subjectIdForSubject(subject: unknown): string {
+  return domainHash(LAB_DOMAIN.subject, jcsStringify(subject));
+}
+
+export function claimSourceManifestDigest(manifest: unknown): string {
+  return domainHash(LAB_DOMAIN.claimSourceManifest, jcsStringify(manifest));
+}
+
+/** Content digest for already-redacted artifact bytes (not contract fixture domain). */
+export function artifactBytesDigest(bytes: Uint8Array): string {
+  return domainHash(LAB_DOMAIN.artifactBytes, bytes);
+}
+
+export function localFingerprint(
+  fieldName: string,
+  value: unknown,
+  salt: Uint8Array | string,
+): string {
+  const key = typeof salt === "string" ? new TextEncoder().encode(salt) : salt;
+  const preimage = `${LAB_DOMAIN.localFingerprint}\0${fieldName}\0`;
+  return createHmac("sha256", key)
+    .update(new TextEncoder().encode(preimage))
+    .update(new TextEncoder().encode(jcsStringify(value)))
+    .digest("hex");
+}
+
+export function isSha256Hex(value: string): boolean {
+  return /^[0-9a-f]{64}$/.test(value);
+}

package/src/lab/events/errors.ts

@@ -0,0 +1,9 @@
+export class LabValidationError extends Error {
+  readonly code: string;
+
+  constructor(code: string, message: string) {
+    super(message);
+    this.name = "LabValidationError";
+    this.code = code;
+  }
+}

package/src/lab/events/limits.ts

@@ -0,0 +1,117 @@
+import {
+  MAX_ARRAY_ELEMENTS_PER_EVENT,
+  MAX_EVENT_NESTING_DEPTH,
+  MAX_OBJECT_KEYS_PER_EVENT,
+  MAX_SANITIZED_STRING_FIELD,
+} from "../constants";
+import { LabValidationError } from "./errors";
+
+const FORBIDDEN_KEY_RE =
+  /(?:^|_)(?:secret|token|apikey|api_key|password|credential|authorization|cookie|bearer|prompt|repository|filepath|file_path|baseurl|base_url|hostname|rawrequest|raw_request)(?:$|_)/i;
+
+const FORBIDDEN_EXACT_KEYS = new Set([
+  "authorization",
+  "apiKey",
+  "api_key",
+  "password",
+  "secret",
+  "token",
+  "cookie",
+  "cookies",
+  "headers",
+  "prompt",
+  "path",
+  "filepath",
+  "filePath",
+  "repository",
+  "repo",
+  "baseUrl",
+  "url",
+  "rawRequest",
+  "rawBytes",
+]);
+
+const RAW_POSIX_PATH_RE =
+  /(?:^|[^A-Za-z0-9._~/])\/(?:(?=$|[^A-Za-z0-9._~/])|(?!\/)(?![ \t\r\n])(?:\/|[^/\0\r\n]+)+\/?(?=$|[^A-Za-z0-9._~/]))/u;
+
+function fieldPath(base: string, key: string | number): string {
+  return base ? `${base}.${String(key)}` : String(key);
+}
+
+function assertAllowedValueType(value: unknown, path: string): void {
+  if (value === null) return;
+  const t = typeof value;
+  if (t === "string" || t === "boolean") return;
+  if (t === "number") {
+    if (!Number.isFinite(value as number)) {
+      throw new LabValidationError("non_finite", `${path} must be finite number`);
+    }
+    return;
+  }
+  if (Array.isArray(value) || (t === "object" && value !== null)) return;
+  throw new LabValidationError("unsupported_type", `${path} has unsupported type`);
+}
+
+function assertKeyAllowed(key: string, path: string): void {
+  if (FORBIDDEN_EXACT_KEYS.has(key) || FORBIDDEN_KEY_RE.test(key)) {
+    throw new LabValidationError("forbidden_field", `${path} forbidden`);
+  }
+  if (key.includes("\0")) {
+    throw new LabValidationError("nul_forbidden", `${path} contains NUL`);
+  }
+}
+
+/** Recursive deterministic CL-00 structural/privacy ceilings before JSONL admission. */
+export function enforceEventStructureLimits(
+  value: unknown,
+  path = "",
+  depth = 0,
+): void {
+  assertAllowedValueType(value, path || "event");
+  if (value === null) return;
+
+  if (typeof value === "string") {
+    const bytes = new TextEncoder().encode(value).byteLength;
+    if (bytes > MAX_SANITIZED_STRING_FIELD) {
+      throw new LabValidationError("field_too_large", `${path} exceeds ${MAX_SANITIZED_STRING_FIELD} bytes`);
+    }
+    if (/sk-[a-z0-9]{10,}/i.test(value) || /Bearer\s+\S+/i.test(value)) {
+      throw new LabValidationError("secret_pattern", `${path} contains secret-shaped data`);
+    }
+    if (
+      /^[A-Za-z]:\\/.test(value) ||
+      RAW_POSIX_PATH_RE.test(value) ||
+      value.includes("\\Users\\")
+    ) {
+      throw new LabValidationError("raw_path", `${path} contains raw filesystem path`);
+    }
+    return;
+  }
+
+  if (typeof value === "number" || typeof value === "boolean") return;
+
+  if (Array.isArray(value)) {
+    if (depth >= MAX_EVENT_NESTING_DEPTH) {
+      throw new LabValidationError("nesting_depth", `${path} exceeds nesting depth ${MAX_EVENT_NESTING_DEPTH}`);
+    }
+    if (value.length > MAX_ARRAY_ELEMENTS_PER_EVENT) {
+      throw new LabValidationError("array_too_large", `${path} exceeds ${MAX_ARRAY_ELEMENTS_PER_EVENT} elements`);
+    }
+    for (let i = 0; i < value.length; i++) {
+      enforceEventStructureLimits(value[i], fieldPath(path, i), depth + 1);
+    }
+    return;
+  }
+
+  if (depth >= MAX_EVENT_NESTING_DEPTH) {
+    throw new LabValidationError("nesting_depth", `${path} exceeds nesting depth ${MAX_EVENT_NESTING_DEPTH}`);
+  }
+  const keys = Object.keys(value as Record<string, unknown>);
+  if (keys.length > MAX_OBJECT_KEYS_PER_EVENT) {
+    throw new LabValidationError("too_many_keys", `${path} exceeds ${MAX_OBJECT_KEYS_PER_EVENT} keys`);
+  }
+  for (const key of keys) {
+    assertKeyAllowed(key, fieldPath(path, key));
+    enforceEventStructureLimits((value as Record<string, unknown>)[key], fieldPath(path, key), depth + 1);
+  }
+}