@bitkyc08/opencodex 2.11.1 → 2.12.0

package/src/lab/index.ts

@@ -0,0 +1,36 @@
+export * from "./constants";
+export * from "./digest";
+export * from "./paths";
+export * from "./events/types";
+export * from "./events/validate";
+export * from "./artifacts/secure-fs";
+export * from "./artifacts/store";
+export * from "./artifacts/sanitize";
+export * from "./ledger/store";
+export * from "./ledger/invalidation";
+export * from "./ledger/purge";
+export * from "./projection/schema";
+export * from "./projection/verdicts";
+export * from "./projection/rebuild";
+export * from "./events/limits";
+export * from "./conformance/suite-manifest";
+export * from "./ledger/artifact-refs";
+export * from "./projection/verification";
+export * from "./observe/from-conformance";
+export * from "./observe/from-live";
+export * from "./live/manifest";
+export * from "./live/runner";
+export * from "./live/executor";
+export * from "./live/types";
+export * from "./live/destination";
+export * from "./live/sandbox";
+export * from "./live/transport";
+export * from "./live/credential-lease";
+export * from "./live/inert-tools";
+export * from "./live/mcp-loopback";
+export * from "./live/suite-manifest";
+export * from "./subject/route-subject";
+export * from "./subject/behavior-fingerprint";
+export * from "./subject/installation-salt";
+export { CL03_LIVE_SUITES } from "./conformance/types";
+export * from "./query";

package/src/lab/ledger/artifact-refs.ts

@@ -0,0 +1,127 @@
+import type { ClaimSnapshotEvent, LabEvent, ObservationEvent } from "../events/types";
+import type { InvalidationIndex } from "./invalidation";
+
+/** Collect artifacts that the frozen CL-00 retention rules still require. */
+export function collectReferencedArtifactDigests(
+  events: LabEvent[],
+  index: InvalidationIndex,
+  opts: { excludeEventIds?: Set<string> } = {},
+): Set<string> {
+  const refs = new Set<string>();
+  const exclude = opts.excludeEventIds ?? new Set<string>();
+
+  for (const event of events) {
+    if (exclude.has(event.eventId)) continue;
+    if (index.purgedEventIds.has(event.eventId)) continue;
+
+    if (event.eventKind === "observation") {
+      // CL-00 releases observation artifacts after invalidation or purge.
+      if (index.invalidatedBy.has(event.eventId)) continue;
+      addObservationArtifacts(event, refs);
+      continue;
+    }
+    if (event.eventKind === "claim_snapshot") {
+      // CL-00 retains claim source manifests while any non-purged claim references them.
+      refs.add(event.sourceManifestDigest);
+    }
+  }
+  return refs;
+}
+
+function addObservationArtifacts(obs: ObservationEvent, refs: Set<string>): void {
+  refs.add(obs.scenarioManifestDigest);
+  refs.add(obs.suiteManifestDigest);
+  for (const digest of obs.fixtureDigests) refs.add(digest);
+  for (const ref of obs.artifactRefs) refs.add(ref.digest);
+}
+
+export function eventReferencesArtifactDigest(event: LabEvent, digest: string): boolean {
+  if (event.eventKind === "observation") {
+    if (event.scenarioManifestDigest === digest) return true;
+    if (event.suiteManifestDigest === digest) return true;
+    if (event.fixtureDigests.includes(digest)) return true;
+    return event.artifactRefs.some((ref) => ref.digest === digest);
+  }
+  if (event.eventKind === "claim_snapshot") {
+    return event.sourceManifestDigest === digest;
+  }
+  return false;
+}
+
+/**
+ * Expand purge targets so explicitly sensitive artifact digests cannot survive
+ * while any retained ledger evidence line still references them.
+ */
+export function expandSensitiveArtifactEventTargets(
+  events: LabEvent[],
+  index: InvalidationIndex,
+  targetEventIds: Set<string>,
+  explicitArtifactDigests: Set<string>,
+): Set<string> {
+  const expanded = new Set(targetEventIds);
+  if (explicitArtifactDigests.size === 0) return expanded;
+
+  for (const event of events) {
+    if (expanded.has(event.eventId)) continue;
+    if (index.purgedEventIds.has(event.eventId)) continue;
+    for (const digest of explicitArtifactDigests) {
+      if (eventReferencesArtifactDigest(event, digest)) {
+        expanded.add(event.eventId);
+        break;
+      }
+    }
+  }
+  return expanded;
+}
+
+/** Artifacts still required by surviving evidence after excluding purge targets. */
+export function artifactsStillRequired(
+  events: LabEvent[],
+  index: InvalidationIndex,
+  excludeEventIds: Set<string>,
+): Set<string> {
+  return collectReferencedArtifactDigests(events, index, { excludeEventIds });
+}
+
+export interface ArtifactDeletionPlan {
+  deletable: string[];
+  /** Explicitly sensitive digests that remain pinned by surviving evidence. */
+  retainedExplicit: string[];
+}
+
+/** Plan physical artifact deletion for a purge without silently retaining explicit targets. */
+export function artifactDeletionPlan(
+  events: LabEvent[],
+  index: InvalidationIndex,
+  targetEventIds: Set<string>,
+  explicitArtifactDigests: string[],
+): ArtifactDeletionPlan {
+  const stillRequired = artifactsStillRequired(events, index, targetEventIds);
+  const candidates = new Set<string>(explicitArtifactDigests);
+
+  for (const event of events) {
+    if (!targetEventIds.has(event.eventId)) continue;
+    if (event.eventKind === "observation") {
+      const scratch = new Set<string>();
+      addObservationArtifacts(event, scratch);
+      for (const digest of scratch) candidates.add(digest);
+    } else if (event.eventKind === "claim_snapshot") {
+      candidates.add(event.sourceManifestDigest);
+    }
+  }
+
+  return {
+    deletable: [...candidates].filter((digest) => !stillRequired.has(digest)).sort(),
+    retainedExplicit: explicitArtifactDigests.filter((digest) => stillRequired.has(digest)).sort(),
+  };
+}
+
+export function observationArtifactDigests(obs: ObservationEvent): string[] {
+  const out = new Set<string>();
+  addObservationArtifacts(obs, out);
+  return [...out].sort();
+}
+
+export function claimArtifactDigests(claim: ClaimSnapshotEvent): string[] {
+  return [claim.sourceManifestDigest];
+}

package/src/lab/ledger/invalidation.ts

@@ -0,0 +1,136 @@
+import type {
+  ClaimSnapshotEvent,
+  InvalidationEvent,
+  LabEvent,
+  LedgerCorruption,
+  ObservationEvent,
+  PurgeTombstoneEvent,
+} from "../events/types";
+import { LabValidationError } from "../events/validate";
+
+export interface InvalidationIndex {
+  /** eventId -> invalidation eventIds that targeted it */
+  invalidatedBy: Map<string, string[]>;
+  purgedEventIds: Set<string>;
+  purgedArtifactDigests: Set<string>;
+  corruptions: LedgerCorruption[];
+}
+
+type EventPosition = { kind: LabEvent["eventKind"]; index: number };
+
+/**
+ * Apply purge tombstones before ordinary invalidations.
+ * Invalidation target lists are atomic: any bad target rejects the whole event.
+ */
+export function buildInvalidationIndex(events: LabEvent[]): InvalidationIndex {
+  const purgedEventIds = new Set<string>();
+  const purgedArtifactDigests = new Set<string>();
+  const invalidatedBy = new Map<string, string[]>();
+  const corruptions: LedgerCorruption[] = [];
+
+  const validEvidenceIds = new Map<string, { kind: "observation" | "claim_snapshot"; index: number }>();
+  const allEventIds = new Map<string, EventPosition>();
+
+  // First pass: record event positions and evidence positions; apply purges as encountered.
+  for (let i = 0; i < events.length; i++) {
+    const event = events[i]!;
+    allEventIds.set(event.eventId, { kind: event.eventKind, index: i });
+    if (event.eventKind === "observation" || event.eventKind === "claim_snapshot") {
+      validEvidenceIds.set(event.eventId, { kind: event.eventKind, index: i });
+      continue;
+    }
+    if (event.eventKind === "purge_tombstone") {
+      applyPurge(event, purgedEventIds, purgedArtifactDigests);
+    }
+  }
+
+  // Second pass: validate and apply invalidations against earlier evidence.
+  for (let i = 0; i < events.length; i++) {
+    const event = events[i]!;
+    if (event.eventKind !== "invalidation") continue;
+    try {
+      validateInvalidationTargets(event, i, validEvidenceIds, allEventIds);
+      for (const target of event.targetEventIds) {
+        const list = invalidatedBy.get(target) ?? [];
+        list.push(event.eventId);
+        invalidatedBy.set(target, list);
+      }
+    } catch (err) {
+      corruptions.push({
+        kind: "invalid_reference",
+        eventId: event.eventId,
+        detail: err instanceof Error ? err.message : String(err),
+      });
+    }
+  }
+
+  return { invalidatedBy, purgedEventIds, purgedArtifactDigests, corruptions };
+}
+
+function applyPurge(
+  event: PurgeTombstoneEvent,
+  purgedEventIds: Set<string>,
+  purgedArtifactDigests: Set<string>,
+): void {
+  for (const id of event.targetEventIds) purgedEventIds.add(id);
+  for (const digest of event.targetArtifactDigests) purgedArtifactDigests.add(digest);
+}
+
+function validateInvalidationTargets(
+  event: InvalidationEvent,
+  index: number,
+  validEvidenceIds: Map<string, { kind: "observation" | "claim_snapshot"; index: number }>,
+  allEventIds: Map<string, EventPosition>,
+): void {
+  for (const target of event.targetEventIds) {
+    if (target === event.eventId) {
+      throw new LabValidationError("self_target", "invalidation cannot target itself");
+    }
+    const meta = validEvidenceIds.get(target);
+    if (!meta) {
+      const other = allEventIds.get(target);
+      if (
+        other &&
+        other.index < index &&
+        (other.kind === "invalidation" || other.kind === "purge_tombstone")
+      ) {
+        throw new LabValidationError("bad_target_kind", `cannot invalidate ${other.kind}`);
+      }
+      throw new LabValidationError("unknown_target", `unknown target ${target}`);
+    }
+    if (meta.index >= index) {
+      throw new LabValidationError("future_target", `target ${target} is not earlier`);
+    }
+    // Purged targets may no longer have a ledger line. A previously valid evidence
+    // ID remains addressable because validEvidenceIds records its original position.
+  }
+}
+
+export function isEventExcluded(
+  eventId: string,
+  index: InvalidationIndex,
+): boolean {
+  if (index.purgedEventIds.has(eventId)) return true;
+  if (index.invalidatedBy.has(eventId)) return true;
+  return false;
+}
+
+export function usableObservations(
+  events: LabEvent[],
+  index: InvalidationIndex,
+): ObservationEvent[] {
+  return events.filter(
+    (e): e is ObservationEvent =>
+      e.eventKind === "observation" && !isEventExcluded(e.eventId, index),
+  );
+}
+
+export function usableClaims(
+  events: LabEvent[],
+  index: InvalidationIndex,
+): ClaimSnapshotEvent[] {
+  return events.filter(
+    (e): e is ClaimSnapshotEvent =>
+      e.eventKind === "claim_snapshot" && !isEventExcluded(e.eventId, index),
+  );
+}

package/src/lab/ledger/purge.ts

@@ -0,0 +1,241 @@
+import {
+  closeTrustedArtifactDir,
+  deleteArtifactBytes,
+  openTrustedArtifactDir,
+  type TrustedArtifactDir,
+} from "../artifacts/secure-fs";
+import { ArtifactFsError } from "../artifacts/secure-fs";
+import {
+  LAB_EVENT_SCHEMA_VERSION,
+  LAB_PRODUCER,
+  LAB_PRODUCER_VERSION,
+  PURGE_ACTIONS,
+} from "../constants";
+import type { LabEvent, PurgeTombstoneEvent } from "../events/types";
+import { assignEventId, validateLabEvent } from "../events/validate";
+import {
+  artifactDeletionPlan,
+  expandSensitiveArtifactEventTargets,
+} from "./artifact-refs";
+import { buildInvalidationIndex } from "./invalidation";
+import { appendLabEvent, replayLabLedger } from "./store";
+import { ensureLabDirs } from "../paths";
+import { rebuildLabProjection } from "../projection/rebuild";
+import { jcsStringify } from "../digest";
+import {
+  closeSync,
+  existsSync,
+  fsyncSync,
+  openSync,
+  readdirSync,
+  renameSync,
+  rmSync,
+  unlinkSync,
+  writeSync,
+} from "node:fs";
+import { dirname, join } from "node:path";
+
+export class PurgeError extends Error {
+  readonly code: string;
+  readonly completedActions: string[];
+  constructor(code: string, message: string, completedActions: string[] = []) {
+    super(message);
+    this.name = "PurgeError";
+    this.code = code;
+    this.completedActions = [...completedActions];
+  }
+}
+
+export interface SensitivePurgeRequest {
+  configDir?: string;
+  targetEventIds?: string[];
+  targetArtifactDigests?: string[];
+  purgeActions?: Array<(typeof PURGE_ACTIONS)[number]>;
+  recordedAt?: number;
+  producerVersion?: string;
+}
+
+function writeAll(fd: number, bytes: Uint8Array): void {
+  let offset = 0;
+  while (offset < bytes.byteLength) {
+    const n = writeSync(fd, bytes, offset, bytes.byteLength - offset);
+    if (n <= 0) throw new PurgeError("short_write", "ledger rewrite short write");
+    offset += n;
+  }
+}
+
+function atomicRewriteLedger(ledgerPath: string, events: LabEvent[]): void {
+  const body = events.map((e) => jcsStringify(e)).join("\n") + (events.length ? "\n" : "");
+  const bytes = new TextEncoder().encode(body);
+  const parent = dirname(ledgerPath);
+  const tmpPath = join(parent, `.purge-${process.pid}-${Date.now()}.jsonl.tmp`);
+  let renamed = false;
+  try {
+    const fd = openSync(tmpPath, "wx", 0o600);
+    try {
+      writeAll(fd, bytes);
+      fsyncSync(fd);
+    } finally {
+      closeSync(fd);
+    }
+    renameSync(tmpPath, ledgerPath);
+    renamed = true;
+  } catch (err) {
+    if (!renamed) {
+      try {
+        unlinkSync(tmpPath);
+      } catch {
+        // Preserve the original failure.
+      }
+    }
+    throw err;
+  }
+
+  if (process.platform !== "win32") {
+    try {
+      const dirFd = openSync(parent, "r");
+      try {
+        fsyncSync(dirFd);
+      } finally {
+        closeSync(dirFd);
+      }
+    } catch (err) {
+      // The rename is already committed and visible. Report durability failure
+      // without pretending the ledger action can be rolled back.
+      throw new PurgeError(
+        "ledger_durability_failed",
+        `ledger rewrite committed but directory fsync failed: ${err instanceof Error ? err.message : String(err)}`,
+        ["ledger"],
+      );
+    }
+  }
+}
+
+function deleteArtifactsFailClosed(dir: TrustedArtifactDir, digests: string[]): void {
+  const errors: string[] = [];
+  for (const digest of digests) {
+    try {
+      deleteArtifactBytes(dir, digest);
+    } catch (err) {
+      if (err instanceof ArtifactFsError && err.code === "artifact_missing") continue;
+      errors.push(err instanceof Error ? err.message : String(err));
+    }
+  }
+  if (errors.length > 0) {
+    throw new PurgeError("artifact_delete_failed", errors.join("; "));
+  }
+}
+
+function purgeBoundedDirectory(dirPath: string): void {
+  if (!existsSync(dirPath)) return;
+  const entries = readdirSync(dirPath, { withFileTypes: true });
+  for (const entry of entries) {
+    const full = join(dirPath, entry.name);
+    try {
+      rmSync(full, { recursive: entry.isDirectory(), force: true });
+    } catch (err) {
+      throw new PurgeError(
+        "scratch_export_delete_failed",
+        err instanceof Error ? err.message : String(err),
+      );
+    }
+  }
+}
+
+/**
+ * Exceptional sensitive-evidence purge:
+ * physically remove targeted JSONL lines and artifacts, append purge_tombstone,
+ * rebuild SQLite. Fails closed when required sensitive bytes cannot be removed.
+ */
+export function purgeSensitiveEvidence(req: SensitivePurgeRequest): PurgeTombstoneEvent {
+  const paths = ensureLabDirs(req.configDir);
+  const targetEventIds = [...(req.targetEventIds ?? [])].sort();
+  const targetArtifactDigests = [...(req.targetArtifactDigests ?? [])].sort();
+  const purgeActions = [...(req.purgeActions ?? PURGE_ACTIONS)].sort();
+  const explicitSensitive = new Set(targetArtifactDigests);
+
+  const replay = replayLabLedger(paths.ledgerPath);
+  const index = buildInvalidationIndex(replay.events);
+  const removeIds = expandSensitiveArtifactEventTargets(
+    replay.events,
+    index,
+    new Set(targetEventIds),
+    explicitSensitive,
+  );
+
+  const tombstonePayload = {
+    schemaVersion: LAB_EVENT_SCHEMA_VERSION,
+    eventKind: "purge_tombstone" as const,
+    recordedAt: req.recordedAt ?? Date.now(),
+    producer: LAB_PRODUCER,
+    producerVersion: req.producerVersion ?? LAB_PRODUCER_VERSION,
+    targetEventIds: [...removeIds].sort(),
+    targetArtifactDigests,
+    reason: "sensitive_evidence" as const,
+    purgeActions,
+  };
+  const tombstone = validateLabEvent(assignEventId(tombstonePayload)) as PurgeTombstoneEvent;
+
+  const deletionPlan = purgeActions.includes("artifact")
+    ? artifactDeletionPlan(replay.events, index, removeIds, targetArtifactDigests)
+    : { deletable: [], retainedExplicit: [] };
+
+  if (deletionPlan.retainedExplicit.length > 0) {
+    throw new PurgeError(
+      "sensitive_bytes_retained",
+      `explicit sensitive artifacts remain required: ${deletionPlan.retainedExplicit.join(",")}`,
+    );
+  }
+
+  let dir: TrustedArtifactDir | null = null;
+  const completed: string[] = [];
+  try {
+    if (purgeActions.includes("scratch")) {
+      purgeBoundedDirectory(paths.scratchDir);
+      completed.push("scratch");
+    }
+    if (purgeActions.includes("export")) {
+      purgeBoundedDirectory(paths.exportDir);
+      completed.push("export");
+    }
+
+    if (purgeActions.includes("artifact")) {
+      if (deletionPlan.deletable.length > 0) {
+        dir = openTrustedArtifactDir(paths.artifactsDir);
+        deleteArtifactsFailClosed(dir, deletionPlan.deletable);
+      }
+      completed.push("artifact");
+    }
+
+    if (purgeActions.includes("ledger")) {
+      const kept: LabEvent[] = [];
+      for (const event of replay.events) {
+        if (removeIds.has(event.eventId)) continue;
+        kept.push(event);
+      }
+      kept.push(tombstone);
+      atomicRewriteLedger(paths.ledgerPath, kept);
+    } else {
+      appendLabEvent(paths.ledgerPath, tombstone);
+    }
+    completed.push("ledger");
+
+    if (purgeActions.includes("sqlite")) {
+      rebuildLabProjection(req.configDir);
+      completed.push("sqlite");
+    }
+
+    return tombstone;
+  } catch (err) {
+    if (err instanceof PurgeError) {
+      throw new PurgeError(err.code, err.message, [...completed, ...err.completedActions]);
+    }
+    throw new PurgeError(
+      "purge_failed",
+      err instanceof Error ? err.message : String(err),
+      completed,
+    );
+  } finally {
+    if (dir) closeTrustedArtifactDir(dir);
+  }
+}