@iislee/opencodex 2.11.0 → 2.31.0

package/src/lab/live/mcp-loopback.ts

@@ -0,0 +1,57 @@
+export interface McpStubTool {
+  namespace: string;
+  name: string;
+  inputSchema?: Record<string, unknown>;
+}
+
+export interface McpStubCall {
+  namespace: string;
+  name: string;
+  arguments: Record<string, unknown>;
+}
+
+export interface McpStubResult {
+  content: Array<{ type: "text"; text: string }>;
+}
+
+const tools = new Map<string, McpStubTool>();
+const invocations: McpStubCall[] = [];
+
+function toolKey(namespace: string, name: string): string {
+  return `${namespace}::${name}`;
+}
+
+/** Register a lab-owned MCP stub tool (in-memory only). */
+export function registerMcpStubTool(tool: McpStubTool): void {
+  tools.set(toolKey(tool.namespace, tool.name), { ...tool });
+}
+
+export function clearMcpStub(): void {
+  tools.clear();
+  invocations.length = 0;
+}
+
+export function listMcpStubTools(): McpStubTool[] {
+  return [...tools.values()];
+}
+
+/** Invoke the in-memory MCP stub with fixed pure results. */
+export function invokeMcpStub(
+  namespace: string,
+  name: string,
+  args: Record<string, unknown>,
+  result: McpStubResult,
+): McpStubResult {
+  const key = toolKey(namespace, name);
+  if (!tools.has(key)) throw new Error(`MCP stub tool not registered: ${namespace}/${name}`);
+  invocations.push({ namespace, name, arguments: { ...args } });
+  return result;
+}
+
+export function mcpStubInvocations(): readonly McpStubCall[] {
+  return invocations;
+}
+
+export function wireName(namespace: string, name: string): string {
+  return `mcp__${namespace}__${name}`;
+}

package/src/lab/live/runner.ts

@@ -0,0 +1,19 @@
+import { CL03_LIVE_SUITES } from "../conformance/types";
+import { isLiveCaseApplicableToRoute, runLiveScenario, type LiveExecutorOptions } from "./executor";
+import { discoverLiveScenarios, loadLiveCaseAuthority } from "./manifest";
+import type { LabRouteContext, LiveScenarioRunResult } from "./types";
+
+export interface LiveRunSummary { total: number; passed: number; failed: number; results: LiveScenarioRunResult[] }
+
+export async function runLiveSuite(routeContext: LabRouteContext, suites: readonly string[] = CL03_LIVE_SUITES, opts: LiveExecutorOptions = {}): Promise<LiveRunSummary> {
+  const authority = loadLiveCaseAuthority();
+  const scenarios = discoverLiveScenarios(authority, suites).filter((scenario) => isLiveCaseApplicableToRoute(scenario, routeContext));
+  const results: LiveScenarioRunResult[] = [];
+  for (const scenario of scenarios) results.push(await runLiveScenario(scenario, routeContext, opts));
+  const passed = results.filter((row) => row.passed).length;
+  return { total: results.length, passed, failed: results.length - passed, results };
+}
+
+export function listLiveScenarioIds(suites: readonly string[] = CL03_LIVE_SUITES): string[] {
+  return discoverLiveScenarios(loadLiveCaseAuthority(), suites).map((scenario) => scenario.id);
+}

package/src/lab/live/sandbox.ts

@@ -0,0 +1,61 @@
+import type { LiveRunConfig } from "./types";
+
+const PROXY_ENV_VARS = ["HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY", "NO_PROXY", "http_proxy", "https_proxy", "all_proxy", "no_proxy"] as const;
+const ALLOWED_ENV = { TZ: "UTC", NO_COLOR: "1" } as const;
+
+export class LabSandboxError extends Error {
+  override readonly name = "LabSandboxError";
+  constructor(message: string, readonly code: string) { super(message); }
+}
+
+export function rejectProxyEnvironment(env: NodeJS.ProcessEnv = process.env): void {
+  for (const name of PROXY_ENV_VARS) {
+    const value = env[name];
+    if (value !== undefined && value !== "") throw new LabSandboxError(`proxy environment variable ${name} is forbidden`, "harness_failure");
+  }
+}
+
+export function labSandboxEnvironment(): Readonly<Record<string, string>> { return Object.freeze({ ...ALLOWED_ENV }); }
+
+export interface SandboxResourceState {
+  requests: number;
+  inputBytes: number;
+  outputBytes: number;
+  outputTokens: number;
+  toolCalls: number;
+  artifacts: number;
+  artifactBytes: number;
+  childProcesses: number;
+}
+
+export function createSandboxResourceState(): SandboxResourceState {
+  return { requests: 0, inputBytes: 0, outputBytes: 0, outputTokens: 0, toolCalls: 0, artifacts: 0, artifactBytes: 0, childProcesses: 0 };
+}
+
+/**
+ * Counter-based ceilings enforce the resources observable in this process. The hard resident-memory
+ * boundary is NOT inferred from the hosting Bun test/server process RSS; a trusted exact-route
+ * executor must attest/enforce that isolated boundary before its result is evidence-eligible.
+ */
+export function enforceSandboxLimits(state: SandboxResourceState, limits: LiveRunConfig, delta: Partial<SandboxResourceState> = {}): void {
+  const next: SandboxResourceState = {
+    requests: state.requests + (delta.requests ?? 0), inputBytes: state.inputBytes + (delta.inputBytes ?? 0),
+    outputBytes: state.outputBytes + (delta.outputBytes ?? 0), outputTokens: state.outputTokens + (delta.outputTokens ?? 0),
+    toolCalls: state.toolCalls + (delta.toolCalls ?? 0), artifacts: state.artifacts + (delta.artifacts ?? 0),
+    artifactBytes: state.artifactBytes + (delta.artifactBytes ?? 0), childProcesses: state.childProcesses + (delta.childProcesses ?? 0),
+  };
+  if (next.childProcesses > limits.maxChildProcesses) throw new LabSandboxError("child process limit exceeded", "child_process_limit");
+  if (next.requests > limits.maxRequests) throw new LabSandboxError("request limit exceeded", "request_limit");
+  if (next.inputBytes > limits.maxInputBytes) throw new LabSandboxError("input byte limit exceeded", "input_byte_limit");
+  if (next.outputBytes > limits.maxOutputBytes) throw new LabSandboxError("output byte limit exceeded", "output_byte_limit");
+  if (next.outputTokens > limits.maxOutputTokens) throw new LabSandboxError("output token limit exceeded", "output_token_limit");
+  if (next.toolCalls > limits.maxToolCalls) throw new LabSandboxError("tool call limit exceeded", "tool_call_limit");
+  if (next.artifacts > limits.maxArtifacts) throw new LabSandboxError("artifact limit exceeded", "artifact_byte_limit");
+  if (next.artifactBytes > limits.aggregateArtifactBytes) throw new LabSandboxError("aggregate artifact limit exceeded", "artifact_byte_limit");
+  Object.assign(state, next);
+}
+
+export function prepareLiveSandbox(env: NodeJS.ProcessEnv = process.env): Readonly<Record<string, string>> {
+  rejectProxyEnvironment(env);
+  return labSandboxEnvironment();
+}

package/src/lab/live/suite-manifest.ts

@@ -0,0 +1,41 @@
+import { scenarioManifestDigest, suiteManifestDigest } from "../digest";
+import type { CaseAuthority, CaseRecord, VerificationRole } from "../conformance/types";
+import { expandLiveScenario } from "./manifest";
+
+export interface LiveSuiteScenarioRefV1 { id: string; version: string; role: VerificationRole; manifestDigest: string }
+export interface LiveSuiteManifestV1 {
+  schemaVersion: 1; id: string; version: string; evidenceLayer: string; capability: string;
+  assertionDslVersion: string; evidenceSchemaVersion: string; freshness: { maxAgeMs: number | null };
+  contradictionRule: string; scenarios: LiveSuiteScenarioRefV1[]; verificationRule: string;
+}
+
+function casesForSuiteManifest(suiteId: string, authority: CaseAuthority): { own: CaseRecord[]; referenced: CaseRecord[] } {
+  const own = authority.cases.filter((row) => row.suite === suiteId);
+  if (own.length === 0) throw new Error(`unknown live suite ${suiteId}`);
+  if (suiteId !== "codex-core") return { own, referenced: own };
+  const prerequisite = authority.cases.find((row) => row.id === "responses-core.live.basic-turn");
+  if (!prerequisite) throw new Error("harness_failure: codex live prerequisite missing");
+  return { own, referenced: [...own, prerequisite] };
+}
+
+export function expandLiveSuiteManifest(suiteId: string, authority: CaseAuthority): LiveSuiteManifestV1 {
+  const { own, referenced } = casesForSuiteManifest(suiteId, authority);
+  const defaults = authority.manifestDefaults;
+  const scenarios = referenced.map((caseRecord) => ({
+    id: caseRecord.id, version: String(defaults.version), role: caseRecord.verificationRole ?? defaults.verificationRole,
+    manifestDigest: scenarioManifestDigest(expandLiveScenario(caseRecord, authority)),
+  })).sort((a, b) => a.id < b.id ? -1 : a.id > b.id ? 1 : 0);
+  return {
+    schemaVersion: 1, id: suiteId, version: String(defaults.suiteVersion), evidenceLayer: defaults.evidenceLayer,
+    capability: own[0]!.capability, assertionDslVersion: authority.assertionDslVersion, evidenceSchemaVersion: authority.evidenceSchemaVersion,
+    freshness: defaults.freshness ?? { maxAgeMs: null }, contradictionRule: "newest-required-observation-v1", scenarios,
+    verificationRule: "all-applicable-required-pass-v1",
+  };
+}
+
+export function liveSuiteManifestObjectForCase(caseRecord: CaseRecord, authority: CaseAuthority): Record<string, unknown> {
+  return expandLiveSuiteManifest(caseRecord.suite, authority) as unknown as Record<string, unknown>;
+}
+export function liveSuiteManifestDigestForCase(caseRecord: CaseRecord, authority: CaseAuthority): string {
+  return suiteManifestDigest(expandLiveSuiteManifest(caseRecord.suite, authority) as unknown as Record<string, unknown>);
+}

package/src/lab/live/transport.ts

@@ -0,0 +1,118 @@
+import type { FailureClassification } from "../conformance/types";
+import { assertLeaseScope, LabCredentialError } from "./credential-lease";
+import type {
+  LabCredentialLeaseV1, LabDestinationV1, LabPinnedSender, LabTransport, LabTransportRequest,
+  LabTransportResponse, LiveRunConfig, TransportErrorCode,
+} from "./types";
+
+export interface MockTransportEntry {
+  matchPath?: string | RegExp;
+  status: number;
+  headers?: Record<string, string>;
+  body: string;
+  error?: TransportErrorCode;
+}
+export interface MockTransportOptions { entries?: MockTransportEntry[]; onRequest?: (req: LabTransportRequest) => void }
+
+/** Test seam only. No live runner path creates this implicitly. */
+export function createMockTransport(opts: MockTransportOptions = {}): LabTransport {
+  let callIndex = 0;
+  return {
+    async request(req): Promise<LabTransportResponse> {
+      opts.onRequest?.(req);
+      const entry = opts.entries?.[callIndex] ?? opts.entries?.[opts.entries.length - 1];
+      callIndex += 1;
+      if (!entry) throw new TransportError("harness_failure", "no mock transport entry");
+      if (entry.error) throw new TransportError(entry.error, entry.error);
+      if (entry.matchPath) {
+        const ok = typeof entry.matchPath === "string" ? req.path.includes(entry.matchPath) : entry.matchPath.test(req.path);
+        if (!ok) throw new TransportError("harness_failure", "mock path mismatch");
+      }
+      if (entry.status >= 300 && entry.status < 400) throw new TransportError("redirect_blocked", "redirect response");
+      return { status: entry.status, headers: { "content-type": "application/json", ...(entry.headers ?? {}) }, body: entry.body };
+    },
+  };
+}
+
+export interface PinnedTransportOptions {
+  destination: LabDestinationV1;
+  lease: LabCredentialLeaseV1;
+  sender: LabPinnedSender;
+  limits: LiveRunConfig;
+  transportId?: string;
+}
+
+function assertRelativeRequestPath(path: string): void {
+  if (!path.startsWith("/") || path.startsWith("//") || path.includes("://") || /[\r\n]/.test(path)) {
+    throw new TransportError("harness_failure", "request path attempted to widen destination");
+  }
+}
+
+function selectedPinnedAddress(destination: LabDestinationV1): { address: string; family: 4 | 6 } {
+  const row = destination.addresses.find((a) => a.family === 4) ?? destination.addresses[0];
+  if (!row) throw new TransportError("harness_failure", "approved destination has no address");
+  return { address: row.address, family: row.family };
+}
+
+/** Uses only the frozen approved address set. Secret injection is owned by the trusted sender, outside Lab. */
+export function createPinnedTransport(opts: PinnedTransportOptions): LabTransport {
+  const transportId = opts.transportId ?? "default";
+  return {
+    async request(req): Promise<LabTransportResponse> {
+      assertRelativeRequestPath(req.path);
+      assertLeaseScope(opts.lease, opts.destination, transportId);
+      const inputBytes = new TextEncoder().encode(req.body ?? "").byteLength;
+      if (inputBytes > opts.limits.maxInputBytes) throw new TransportError("input_byte_limit", "request exceeds input byte budget");
+      opts.lease.consume();
+      const controller = new AbortController();
+      const timer = setTimeout(() => controller.abort(new TransportError("total_timeout", "live request total timeout")), opts.limits.totalTimeoutMs);
+      const onAbort = () => controller.abort(req.signal?.reason);
+      req.signal?.addEventListener("abort", onAbort, { once: true });
+      try {
+        const response = await opts.sender(opts.lease, opts.destination, selectedPinnedAddress(opts.destination), req, controller.signal, opts.limits);
+        const outputBytes = new TextEncoder().encode(response.body).byteLength;
+        if (outputBytes > opts.limits.maxOutputBytes) throw new TransportError("output_byte_limit", "response exceeds output byte budget");
+        if (response.status >= 300 && response.status < 400) throw new TransportError("redirect_blocked", "redirect response");
+        if (response.status === 401 || response.status === 403) throw new TransportError("auth_blocked", `HTTP ${response.status}`);
+        if (response.status === 429) throw new TransportError("quota_blocked", "HTTP 429");
+        if (response.status === 451) throw new TransportError("region_blocked", "HTTP 451");
+        if (response.status >= 500) throw new TransportError("provider_transient", `HTTP ${response.status}`);
+        return response;
+      } catch (error) {
+        if (error instanceof LabCredentialError) {
+          throw new TransportError(error.code === "budget_exhausted" ? "request_limit" : "harness_failure", error.message);
+        }
+        throw error;
+      } finally {
+        clearTimeout(timer);
+        req.signal?.removeEventListener("abort", onAbort);
+      }
+    },
+  };
+}
+
+export class TransportError extends Error {
+  override readonly name = "TransportError";
+  constructor(readonly code: TransportErrorCode, message: string) { super(message); }
+}
+
+export function classifyTransportError(error: unknown): { classification: FailureClassification; secondaryCode: string } {
+  const code = error instanceof TransportError ? error.code : undefined;
+  switch (code) {
+    case "auth_blocked": return { classification: "authentication_blocked", secondaryCode: code };
+    case "quota_blocked": return { classification: "quota_blocked", secondaryCode: code };
+    case "network_blocked": return { classification: "network_failure", secondaryCode: code };
+    case "region_blocked": return { classification: "region_blocked", secondaryCode: code };
+    case "provider_transient": return { classification: "provider_transient", secondaryCode: code };
+    case "connect_timeout": case "first_byte_timeout": case "total_timeout":
+      return { classification: "timeout", secondaryCode: code };
+    case "inactivity_timeout":
+      return { classification: "inactivity_timeout", secondaryCode: code };
+    case "request_limit": case "input_byte_limit": case "output_byte_limit": case "output_token_limit":
+    case "tool_call_limit": case "artifact_byte_limit": case "memory_limit": case "child_process_limit":
+      return { classification: "budget_exhausted", secondaryCode: code };
+    case "live_transport_required": case "untrusted_route_executor": case "redirect_blocked": case "destination_mismatch": case "host_sni_mismatch": case "harness_failure":
+      return { classification: "harness_failure", secondaryCode: code };
+    default: return { classification: "harness_failure", secondaryCode: "execution_error" };
+  }
+}

package/src/lab/live/types.ts

@@ -0,0 +1,197 @@
+import type { FailureClassification, NormalizedObservation } from "../conformance/types";
+import type { RouteDependencyV1, RouteSubjectV1 } from "../events/types";
+
+export interface LabDestinationV1 {
+  readonly scheme: "http" | "https";
+  readonly host: string;
+  readonly port: number;
+  readonly basePath: string;
+  readonly sniHost: string;
+  readonly addresses: ReadonlyArray<{ readonly address: string; readonly family: 4 | 6 }>;
+  readonly privateNetwork: boolean;
+  readonly fingerprint: string;
+}
+
+export type LabBehaviorSource =
+  | "request"
+  | "model_override"
+  | "provider_config"
+  | "registry_runtime_default"
+  | "generated_model_metadata"
+  | "global_config"
+  | "adapter_default"
+  | "lab_forced";
+
+export interface LabBehaviorValue {
+  source: LabBehaviorSource;
+  value: unknown;
+}
+
+/** Authoritative effective values emitted by the production route/model/adapter resolver. */
+export type LabBehaviorValues = Record<string, LabBehaviorValue>;
+
+export interface LabRouteContext {
+  providerId: string;
+  /** Config-owner identity only; HMACed before it enters evidence. */
+  providerInstanceKey: string;
+  clientModelId: string;
+  upstreamModelId: string;
+  effectiveAdapter: string;
+  inboundProtocol: string;
+  upstreamProtocol: string;
+  surface: string;
+  baseUrl: string;
+  /** Generated compatibility hash, not package marketing version. */
+  opencodexCompatibilityVersion: string;
+  /** Closed, effective behavior inputs from the production resolver. */
+  behaviorValues: LabBehaviorValues;
+  allowPrivateNetwork?: boolean;
+  labRunApproval?: boolean;
+  requiredClaims?: string[];
+  availableHarnessFeatures?: string[];
+  dependencies?: RouteDependencyV1[];
+}
+
+export interface LiveRunConfig {
+  totalTimeoutMs: number;
+  connectTimeoutMs: number;
+  firstByteTimeoutMs: number;
+  inactivityTimeoutMs: number;
+  maxRequests: number;
+  maxInputBytes: number;
+  maxOutputBytes: number;
+  maxOutputTokens: number;
+  maxToolCalls: number;
+  maxMemoryBytes: number;
+  maxChildProcesses: number;
+  maxArtifacts: number;
+  perArtifactBytes: number;
+  aggregateArtifactBytes: number;
+}
+
+export type DnsResolver = (hostname: string) => Promise<Array<{ address: string; family: 4 | 6 }>>;
+
+export type TransportErrorCode =
+  | "auth_blocked"
+  | "quota_blocked"
+  | "network_blocked"
+  | "region_blocked"
+  | "provider_transient"
+  | "redirect_blocked"
+  | "destination_mismatch"
+  | "host_sni_mismatch"
+  | "connect_timeout"
+  | "first_byte_timeout"
+  | "inactivity_timeout"
+  | "total_timeout"
+  | "request_limit"
+  | "input_byte_limit"
+  | "output_byte_limit"
+  | "output_token_limit"
+  | "tool_call_limit"
+  | "artifact_byte_limit"
+  | "memory_limit"
+  | "child_process_limit"
+  | "live_transport_required"
+  | "untrusted_route_executor"
+  | "harness_failure";
+
+export interface LabTransportRequest {
+  method: "GET" | "POST";
+  path: string;
+  headers?: Record<string, string>;
+  body?: string;
+  signal?: AbortSignal;
+}
+
+export interface LabTransportResponse {
+  status: number;
+  headers: Record<string, string>;
+  body: string;
+}
+
+export type TransportRequest = LabTransportRequest;
+export type TransportResponse = LabTransportResponse;
+
+export interface LabTransport {
+  request(req: LabTransportRequest): Promise<LabTransportResponse>;
+}
+
+export const LAB_CREDENTIAL_LEASE = Symbol("LabCredentialLeaseV1");
+
+/** Opaque capability. Scope metadata and secret material are deliberately not public fields. */
+export interface LabCredentialLeaseV1 {
+  readonly [LAB_CREDENTIAL_LEASE]: true;
+  readonly remainingRequests: number;
+  consume(): void;
+}
+
+export interface LabPinnedAddress {
+  address: string;
+  family: 4 | 6;
+}
+
+/** Trusted provider/credential transport seam. Implementations own secret injection outside Lab. */
+export type LabPinnedSender = (
+  lease: LabCredentialLeaseV1,
+  destination: LabDestinationV1,
+  pinned: LabPinnedAddress,
+  request: LabTransportRequest,
+  signal: AbortSignal,
+  limits: LiveRunConfig,
+) => Promise<LabTransportResponse>;
+
+export interface LabRouteExecutorInput {
+  routeContext: LabRouteContext;
+  destination: LabDestinationV1;
+  routeSubject: RouteSubjectV1;
+  scenarioId: string;
+  initiatingRequest?: string;
+  limits: LiveRunConfig;
+  signal: AbortSignal;
+  environment: Readonly<Record<string, string>>;
+}
+
+/** Trusted exact-route execution function. The wrapper capability is issued only by the host integration. */
+export type LabRouteExecutor = (input: LabRouteExecutorInput) => Promise<NormalizedObservation>;
+
+export const REQUIRED_LAB_SANDBOX_BOUNDARIES = [
+  "destination_pinning",
+  "credential_binding",
+  "wall_clock",
+  "connect_timeout",
+  "first_byte_timeout",
+  "inactivity_timeout",
+  "request_budget",
+  "input_byte_budget",
+  "output_byte_budget",
+  "output_token_budget",
+  "tool_call_budget",
+  "memory_limit",
+  "child_process_limit",
+  "artifact_budget",
+] as const;
+export type LabSandboxBoundary = typeof REQUIRED_LAB_SANDBOX_BOUNDARIES[number];
+
+/** Opaque host-issued capability; runtime trust is established outside Lab. */
+export interface TrustedLabRouteExecutor {
+  execute(input: LabRouteExecutorInput): Promise<NormalizedObservation>;
+  readonly enforcedBoundaries: readonly LabSandboxBoundary[];
+}
+
+export type LiveExecutionAuthority = "trusted_route" | "test_transport" | "none";
+
+export interface LiveScenarioRunResult {
+  scenarioId: string;
+  suite: string;
+  startedAt: number;
+  completedAt: number;
+  passed: boolean;
+  classification: FailureClassification;
+  secondaryCode?: string;
+  assertionResults: import("../conformance/types").AssertionResult[];
+  diagnostics: string[];
+  routeSubject?: RouteSubjectV1;
+  transportError?: TransportErrorCode;
+  executionAuthority: LiveExecutionAuthority;
+}