@iislee/opencodex 2.11.0 → 2.35.0

package/src/lab/automation/recovery.ts

@@ -0,0 +1,24 @@
+import type { LabAutomationPolicyV1, LabAutomationStateV1 } from "./types";
+import { transitionRun } from "./queue";
+import { setCooldown } from "./cooldown";
+import { LAB_AUTOMATION_HARD_MAX } from "./constants";
+
+/** Reconcile runs left running after process restart. */
+export function recoverLabAutomationState(
+  policy: LabAutomationPolicyV1,
+  state: LabAutomationStateV1,
+  now: number,
+): LabAutomationStateV1 {
+  let next = { ...state };
+  for (const run of state.runs) {
+    if (run.state !== "running") continue;
+    next = transitionRun(next, run.runId, "abandoned", now, "process_restart");
+    next = setCooldown(
+      next,
+      run.runKey,
+      now + Math.min(policy.failureCooldownMs, LAB_AUTOMATION_HARD_MAX.failureCooldownMs),
+      now,
+    );
+  }
+  return next;
+}

package/src/lab/automation/route-context.ts

@@ -0,0 +1,21 @@
+import type { LabRouteContext } from "../live/types";
+import type { ResolvedPolicyRouteSubject } from "../../routing/compatibility/subject";
+
+/** Harness features the production trusted-route automation contract actually provides. */
+export const CL08_TRUSTED_LIVE_HARNESS_FEATURES = Object.freeze([
+  "live_transport",
+]);
+
+/** Build the CL-03 route context automation dispatch and planning share. */
+export function buildAutomationLiveRouteContext(
+  resolved: ResolvedPolicyRouteSubject,
+  allowPrivateNetwork = false,
+): LabRouteContext {
+  return {
+    ...resolved.routeContext,
+    labRunApproval: true,
+    allowPrivateNetwork,
+    requiredClaims: resolved.routeContext.requiredClaims ?? [],
+    availableHarnessFeatures: [...CL08_TRUSTED_LIVE_HARNESS_FEATURES],
+  };
+}

package/src/lab/automation/run-key.ts

@@ -0,0 +1,44 @@
+import { createHash } from "node:crypto";
+import { jcsStringify } from "../digest";
+import type { LabAutomationLayer } from "./types";
+
+export interface RunKeyInput {
+  evidenceLayer: LabAutomationLayer;
+  subjectId: string;
+  suiteId: string;
+  suiteVersion: string;
+  suiteManifestDigest: string;
+  scenarioId: string;
+  scenarioVersion: string;
+  scenarioManifestDigest: string;
+  executionContractDigest: string;
+}
+
+/** Deterministic deduplication key for orchestration runs. */
+export function buildLabAutomationRunKey(input: RunKeyInput): string {
+  const payload = {
+    evidenceLayer: input.evidenceLayer,
+    subjectId: input.subjectId,
+    suiteId: input.suiteId,
+    suiteVersion: input.suiteVersion,
+    suiteManifestDigest: input.suiteManifestDigest,
+    scenarioId: input.scenarioId,
+    scenarioVersion: input.scenarioVersion,
+    scenarioManifestDigest: input.scenarioManifestDigest,
+    executionContractDigest: input.executionContractDigest,
+  };
+  return createHash("sha256").update(jcsStringify(payload)).digest("hex");
+}
+
+/** Contract identity digest for protocol/live/fabric execution boundaries. */
+export function protocolExecutionContractDigest(): string {
+  return createHash("sha256").update(jcsStringify({ contract: "cl-01-protocol-harness-v1" })).digest("hex");
+}
+
+export function liveExecutionContractDigest(): string {
+  return createHash("sha256").update(jcsStringify({ contract: "cl-03-live-trusted-route-v1" })).digest("hex");
+}
+
+export function fabricExecutionContractDigest(): string {
+  return createHash("sha256").update(jcsStringify({ contract: "cl-07-fabric-trusted-route-v1" })).digest("hex");
+}

package/src/lab/automation/runs-query.ts

@@ -0,0 +1,34 @@
+import type { LabAutomationRunRecordV1, LabAutomationStateV1 } from "./types";
+import { LAB_AUTOMATION_HARD_MAX } from "./constants";
+import { LabAutomationError } from "./types";
+
+export interface LabAutomationRunsPage {
+  items: LabAutomationRunRecordV1[];
+  hasMore: boolean;
+  nextCursor?: string;
+}
+
+export function listLabAutomationRuns(
+  state: LabAutomationStateV1,
+  limit = 50,
+  cursor?: string,
+): LabAutomationRunsPage {
+  const capped = Math.min(Math.max(1, limit), LAB_AUTOMATION_HARD_MAX.maxPersistedRuns);
+  const sorted = [...state.runs].sort((a, b) => {
+    if (a.createdAt !== b.createdAt) return b.createdAt - a.createdAt;
+    return a.runId < b.runId ? -1 : 1;
+  });
+  let start = 0;
+  if (cursor) {
+    const index = sorted.findIndex((row) => row.runId === cursor);
+    if (index < 0) throw new LabAutomationError("invalid or expired run cursor", "invalid_cursor");
+    start = index + 1;
+  }
+  const slice = sorted.slice(start, start + capped);
+  const hasMore = start + capped < sorted.length;
+  return {
+    items: slice,
+    hasMore,
+    ...(hasMore && slice.length > 0 ? { nextCursor: slice[slice.length - 1]!.runId } : {}),
+  };
+}

package/src/lab/automation/types.ts

@@ -0,0 +1,160 @@
+import type { EvidenceLayer } from "../constants";
+
+export type LabAutomationLayer =
+  | "protocol_conformance"
+  | "live_route_compatibility"
+  | "task_effectiveness";
+
+export interface LabAutomationPolicyV1 {
+  schemaVersion: 1;
+  enabled: boolean;
+  layers: {
+    protocolConformance: boolean;
+    liveRouteCompatibility: boolean;
+    taskEffectiveness: boolean;
+  };
+  refreshBeforeStaleMs: number;
+  maxConcurrentRuns: number;
+  maxConcurrentLiveRuns: number;
+  maxConcurrentRunsPerRoute: number;
+  maxRunsPerHour: number;
+  maxLiveRequestsPerHour: number;
+  failureCooldownMs: number;
+  blockedCooldownMs: number;
+  taskEffectivenessBackgroundEnabled: boolean;
+}
+
+export type LabAutomationPlanReason =
+  | "automation_disabled"
+  | "layer_disabled"
+  | "fresh"
+  | "refresh_due"
+  | "missing"
+  | "cooldown"
+  | "budget_blocked"
+  | "already_queued"
+  | "already_running"
+  | "route_ineligible"
+  | "scenario_inapplicable"
+  | "task_background_disabled";
+
+export interface PlannedLabRunV1 {
+  runKey: string;
+  evidenceLayer: LabAutomationLayer;
+  suiteId: string;
+  suiteVersion: string;
+  suiteManifestDigest: string;
+  scenarioId: string;
+  scenarioVersion: string;
+  scenarioManifestDigest: string;
+  subjectId: string;
+  reason: LabAutomationPlanReason;
+  priority: number;
+  eligibleAt: number;
+  providerName?: string;
+  modelId?: string;
+}
+
+export type LabAutomationRunState =
+  | "queued"
+  | "running"
+  | "completed"
+  | "blocked"
+  | "failed"
+  | "cancelled"
+  | "abandoned";
+
+export type LabAutomationRunTrigger = "scheduled" | "manual";
+
+export interface LabAutomationRunRecordV1 {
+  runId: string;
+  runKey: string;
+  state: LabAutomationRunState;
+  evidenceLayer: LabAutomationLayer;
+  suiteId: string;
+  suiteVersion: string;
+  suiteManifestDigest: string;
+  scenarioId: string;
+  scenarioVersion: string;
+  scenarioManifestDigest: string;
+  subjectId: string;
+  reason: LabAutomationPlanReason;
+  priority: number;
+  eligibleAt: number;
+  trigger: LabAutomationRunTrigger;
+  createdAt: number;
+  updatedAt: number;
+  startedAt?: number;
+  completedAt?: number;
+  terminalCode?: string;
+  providerName?: string;
+  modelId?: string;
+}
+
+export interface LabAutomationRouteRefV1 {
+  providerName: string;
+  modelId: string;
+}
+
+export interface LabAutomationRoutesV1 {
+  schemaVersion: 1;
+  routes: readonly LabAutomationRouteRefV1[];
+}
+
+export interface LabAutomationStateV1 {
+  schemaVersion: 1;
+  runs: LabAutomationRunRecordV1[];
+  budgetWindowStartedAt: number;
+  runsThisHour: number;
+  liveRequestsThisHour: number;
+  cooldownUntilByKey: Record<string, number>;
+}
+
+export interface LabAutomationStatusV1 {
+  policy: LabAutomationPolicyV1;
+  routes: LabAutomationRoutesV1;
+  counters: {
+    queued: number;
+    running: number;
+    completedLastHour: number;
+    blockedLastHour: number;
+    remainingRunBudget: number;
+    remainingLiveRequestBudget: number;
+  };
+  schedulerRunning: boolean;
+}
+
+export interface AutomationDispatchDeps {
+  configDir?: string;
+  routeExecutor?: import("../live/types").TrustedLabRouteExecutor;
+  patchExecutor?: import("../fabric/types").TrustedFabricPatchExecutor;
+  loadConfig?: () => import("../../types").OcxConfig;
+  abortSignal?: AbortSignal;
+  /** Orchestrator-set guard that requires the queued identity to match current frozen contracts. */
+  enforceRunIdentity?: boolean;
+  /** Test-only DNS seam; production uses destination policy resolution when unset. */
+  resolve?: import("../live/types").DnsResolver;
+}
+
+export class LabAutomationError extends Error {
+  override readonly name = "LabAutomationError";
+  constructor(
+    message: string,
+    readonly code: string,
+  ) {
+    super(message);
+  }
+}
+
+export function evidenceLayerToAutomationLayer(layer: EvidenceLayer): LabAutomationLayer | null {
+  switch (layer) {
+    case "protocol_conformance":
+      return "protocol_conformance";
+    case "live_route_compatibility":
+      return "live_route_compatibility";
+    case "task_effectiveness":
+      return "task_effectiveness";
+    default:
+      return null;
+  }
+}

package/src/lab/conformance/assertion.ts

@@ -0,0 +1,325 @@
+import { jcsEqual } from "./jcs";
+import { pointerExists, resolveJsonPointer } from "./json-pointer";
+import type { AssertionResult, AssertionSpec, NormalizedObservation } from "./types";
+
+const ID_GRAMMARS: Record<string, RegExp> = {
+  responses_message: /^msg_[A-Za-z0-9_-]{1,128}$/,
+  responses_reasoning: /^rs_[A-Za-z0-9_-]{1,128}$/,
+  responses_call: /^call_[A-Za-z0-9_-]{1,128}$/,
+  nonempty_128: /^[^\s]{1,128}$/,
+};
+
+export function evaluateAssertion(
+  assertion: AssertionSpec,
+  observation: NormalizedObservation,
+): AssertionResult {
+  const base: AssertionResult = {
+    id: assertion.id,
+    operator: assertion.operator,
+    required: assertion.required,
+    passed: false,
+    observedSummary: "",
+  };
+
+  try {
+    switch (assertion.operator) {
+      case "http_status_equals":
+        return evaluateEquals(assertion, observation, (obs) => obs.client.response.status);
+      case "json_path_equals":
+        return evaluateJsonPathEquals(assertion, observation);
+      case "json_path_present":
+        return evaluatePresence(assertion, observation, true);
+      case "json_path_absent":
+        return evaluatePresence(assertion, observation, false);
+      case "sse_event_sequence":
+        return evaluateEventSequence(assertion, observation);
+      case "sse_event_count":
+        return evaluateEventCount(assertion, observation);
+      case "terminal_signal_equals":
+        return evaluateEquals(assertion, observation, (obs) => obs.client.response.terminal);
+      case "id_matches":
+        return evaluateIdMatches(assertion, observation);
+      case "id_stable_across_events":
+        return evaluateIdStable(assertion, observation);
+      case "id_correlates":
+        return evaluateIdCorrelates(assertion, observation);
+      case "tool_call_equals":
+        return evaluateJsonPathEquals(assertion, observation);
+      case "tool_result_correlates":
+        return evaluateToolResultCorrelates(assertion, observation);
+      case "normalized_text_equals":
+        return evaluateEquals(assertion, observation, (obs) => obs.client.response.normalizedText);
+      case "verifier_result_equals":
+        return evaluateJsonPathEquals(assertion, observation);
+      default:
+        return {
+          ...base,
+          passed: false,
+          observedSummary: `unknown operator ${assertion.operator}`,
+          reason: "unknown_operator",
+        };
+    }
+  } catch (error) {
+    return {
+      ...base,
+      passed: false,
+      observedSummary: String(error),
+      reason: "evaluation_error",
+    };
+  }
+}
+
+function evaluateEquals(
+  assertion: AssertionSpec,
+  observation: NormalizedObservation,
+  pick: (obs: NormalizedObservation) => unknown,
+): AssertionResult {
+  const observed = pick(observation);
+  const passed = jcsEqual(observed, assertion.expected);
+  return {
+    id: assertion.id,
+    operator: assertion.operator,
+    required: assertion.required,
+    passed,
+    observedSummary: summarize(observed),
+    reason: passed ? undefined : "value_mismatch",
+  };
+}
+
+function evaluateJsonPathEquals(assertion: AssertionSpec, observation: NormalizedObservation): AssertionResult {
+  const resolved = resolveJsonPointer(observation, assertion.selector);
+  if (!resolved.ok) {
+    return {
+      id: assertion.id,
+      operator: assertion.operator,
+      required: assertion.required,
+      passed: false,
+      observedSummary: resolved.reason,
+      reason: resolved.reason,
+    };
+  }
+  const passed = jcsEqual(resolved.value, assertion.expected);
+  return {
+    id: assertion.id,
+    operator: assertion.operator,
+    required: assertion.required,
+    passed,
+    observedSummary: summarize(resolved.value),
+    reason: passed ? undefined : "value_mismatch",
+  };
+}
+
+function evaluatePresence(
+  assertion: AssertionSpec,
+  observation: NormalizedObservation,
+  shouldExist: boolean,
+): AssertionResult {
+  const exists = pointerExists(observation, assertion.selector);
+  const passed = shouldExist ? exists : !exists;
+  return {
+    id: assertion.id,
+    operator: assertion.operator,
+    required: assertion.required,
+    passed,
+    observedSummary: exists ? "present" : "absent",
+    reason: passed ? undefined : shouldExist ? "selector_missing" : "selector_present",
+  };
+}
+
+function evaluateEventSequence(assertion: AssertionSpec, observation: NormalizedObservation): AssertionResult {
+  const events = observation.client.response.events.map((e) => e.event);
+  const expected = assertion.expected as string[];
+  const passed = Array.isArray(expected)
+    && events.length === expected.length
+    && events.every((e, i) => e === expected[i]);
+  return {
+    id: assertion.id,
+    operator: assertion.operator,
+    required: assertion.required,
+    passed,
+    observedSummary: summarize(events),
+    reason: passed ? undefined : "event_sequence_mismatch",
+  };
+}
+
+function evaluateEventCount(assertion: AssertionSpec, observation: NormalizedObservation): AssertionResult {
+  const spec = assertion.expected as { event: string; count: number };
+  const count = observation.client.response.events.filter((e) => e.event === spec.event).length;
+  const passed = count === spec.count;
+  return {
+    id: assertion.id,
+    operator: assertion.operator,
+    required: assertion.required,
+    passed,
+    observedSummary: String(count),
+    reason: passed ? undefined : "event_count_mismatch",
+  };
+}
+
+function evaluateIdMatches(assertion: AssertionSpec, observation: NormalizedObservation): AssertionResult {
+  const resolved = resolveJsonPointer(observation, assertion.selector);
+  if (!resolved.ok) {
+    return {
+      id: assertion.id,
+      operator: assertion.operator,
+      required: assertion.required,
+      passed: false,
+      observedSummary: resolved.reason,
+      reason: resolved.reason,
+    };
+  }
+  const grammar = ID_GRAMMARS[String(assertion.expected)];
+  const value = typeof resolved.value === "string" ? resolved.value : "";
+  const passed = grammar ? grammar.test(value) : false;
+  return {
+    id: assertion.id,
+    operator: assertion.operator,
+    required: assertion.required,
+    passed,
+    observedSummary: value,
+    reason: passed ? undefined : "id_grammar_mismatch",
+  };
+}
+
+function evaluateIdStable(assertion: AssertionSpec, observation: NormalizedObservation): AssertionResult {
+  const pointers = assertion.expected;
+  if (!Array.isArray(pointers) || pointers.length < 2 || !pointers.every((p) => typeof p === "string")) {
+    return {
+      id: assertion.id,
+      operator: assertion.operator,
+      required: assertion.required,
+      passed: false,
+      observedSummary: "expected at least two pointers",
+      reason: "invalid_expected",
+    };
+  }
+  const values: string[] = [];
+  for (const pointer of pointers) {
+    const resolved = resolveJsonPointer(observation, pointer);
+    if (!resolved.ok) {
+      return {
+        id: assertion.id,
+        operator: assertion.operator,
+        required: assertion.required,
+        passed: false,
+        observedSummary: resolved.reason,
+        reason: resolved.reason,
+      };
+    }
+    if (typeof resolved.value !== "string") {
+      return {
+        id: assertion.id,
+        operator: assertion.operator,
+        required: assertion.required,
+        passed: false,
+        observedSummary: "identifier must be a string",
+        reason: "selector_type_mismatch",
+      };
+    }
+    values.push(resolved.value);
+  }
+  const passed = values.every((v) => v === values[0]);
+  return {
+    id: assertion.id,
+    operator: assertion.operator,
+    required: assertion.required,
+    passed,
+    observedSummary: summarize(values),
+    reason: passed ? undefined : "id_not_stable",
+  };
+}
+
+function correlatedIds(left: unknown, right: unknown): boolean {
+  return typeof left === "string"
+    && typeof right === "string"
+    && left.length > 0
+    && right.length > 0
+    && left === right;
+}
+
+function evaluateIdCorrelates(assertion: AssertionSpec, observation: NormalizedObservation): AssertionResult {
+  const pointers = assertion.expected;
+  if (!Array.isArray(pointers) || pointers.length !== 2 || !pointers.every((p) => typeof p === "string")) {
+    return {
+      id: assertion.id,
+      operator: assertion.operator,
+      required: assertion.required,
+      passed: false,
+      observedSummary: "expected two pointers",
+      reason: "invalid_expected",
+    };
+  }
+  const left = resolveJsonPointer(observation, pointers[0]);
+  const right = resolveJsonPointer(observation, pointers[1]);
+  if (!left.ok || !right.ok) {
+    return {
+      id: assertion.id,
+      operator: assertion.operator,
+      required: assertion.required,
+      passed: false,
+      observedSummary: left.ok ? (right as { reason: string }).reason : (left as { reason: string }).reason,
+      reason: "selector_missing",
+    };
+  }
+  const passed = correlatedIds(left.value, right.value);
+  return {
+    id: assertion.id,
+    operator: assertion.operator,
+    required: assertion.required,
+    passed,
+    observedSummary: `${summarize(left.value)} vs ${summarize(right.value)}`,
+    reason: passed ? undefined : "id_correlation_mismatch",
+  };
+}
+
+function evaluateToolResultCorrelates(assertion: AssertionSpec, observation: NormalizedObservation): AssertionResult {
+  const spec = assertion.expected as { call?: unknown; result?: unknown };
+  if (!spec || typeof spec.call !== "string" || typeof spec.result !== "string") {
+    return {
+      id: assertion.id,
+      operator: assertion.operator,
+      required: assertion.required,
+      passed: false,
+      observedSummary: "expected call/result pointers",
+      reason: "invalid_expected",
+    };
+  }
+  const call = resolveJsonPointer(observation, spec.call);
+  const result = resolveJsonPointer(observation, spec.result);
+  if (!call.ok || !result.ok) {
+    return {
+      id: assertion.id,
+      operator: assertion.operator,
+      required: assertion.required,
+      passed: false,
+      observedSummary: call.ok ? (result as { reason: string }).reason : (call as { reason: string }).reason,
+      reason: "selector_missing",
+    };
+  }
+  const passed = correlatedIds(call.value, result.value);
+  return {
+    id: assertion.id,
+    operator: assertion.operator,
+    required: assertion.required,
+    passed,
+    observedSummary: `${summarize(call.value)} -> ${summarize(result.value)}`,
+    reason: passed ? undefined : "tool_result_correlation_mismatch",
+  };
+}
+
+function summarize(value: unknown): string {
+  if (typeof value === "string") return value.length > 120 ? value.slice(0, 120) + "…" : value;
+  try {
+    const text = JSON.stringify(value);
+    return text.length > 200 ? text.slice(0, 200) + "…" : text;
+  } catch {
+    return String(value);
+  }
+}
+
+export function evaluateAssertions(
+  assertions: AssertionSpec[],
+  observation: NormalizedObservation,
+): AssertionResult[] {
+  return assertions.map((a) => evaluateAssertion(a, observation));
+}

package/src/lab/conformance/digest.ts

@@ -0,0 +1,22 @@
+import { createHash } from "node:crypto";
+import { jcsStringify } from "./jcs";
+
+function domainHash(domain: string, payload: Uint8Array | string): string {
+  const hash = createHash("sha256");
+  hash.update(new TextEncoder().encode(`${domain}\0`));
+  if (typeof payload === "string") hash.update(new TextEncoder().encode(payload));
+  else hash.update(payload);
+  return hash.digest("hex");
+}
+
+export function fixtureDigest(bytes: Uint8Array): string {
+  return domainHash("ocx-lab:fixture:v1", bytes);
+}
+
+export function scenarioManifestDigest(expandedScenario: Record<string, unknown>): string {
+  return domainHash("ocx-lab:scenario-manifest:v1", jcsStringify(expandedScenario));
+}
+
+export function suiteManifestDigest(expandedSuite: Record<string, unknown>): string {
+  return domainHash("ocx-lab:suite-manifest:v1", jcsStringify(expandedSuite));
+}