@iislee/opencodex 2.11.0 → 2.31.0

package/src/integrations/config-io.ts

@@ -0,0 +1,269 @@
+/**
+ * The filesystem seam shared by the reader and the writer.
+ *
+ * It lives in WP2 rather than beside the writer because `readIntegrationState`
+ * needs it too, and a reader that disagreed with the writer about what counts
+ * as an absent file is exactly how an unreadable config gets overwritten.
+ *
+ * Design of record: devlog/_fin/260802_client_toggle_api/021 §5-6.
+ */
+import { mkdirSync, readFileSync, rmSync, statSync } from "node:fs";
+import type { ConfigFormat } from "../clients/config-export";
+import { MAX_JSON_NESTING } from "./serialize";
+import { atomicWriteFile } from "../config";
+import type { JournalEntry } from "./journal";
+import type { OwnershipRecord } from "./ownership";
+import type { IntegrationClientId } from "./registry";
+
+/**
+ * A parse failure sentinel. This is deliberately ONE symbol exported from ONE
+ * module: two `Symbol("parse-failed")` calls produce different values, so a
+ * second declaration would not duplicate code — it would make every comparison
+ * against it fail, and an unparseable config would be classified `absent` and
+ * then overwritten.
+ */
+export const PARSE_FAILED = Symbol("parse-failed");
+
+/**
+ * What `JSON.parse` has already discarded by the time we hold the parsed value,
+ * and a rewrite would therefore silently change. Both classes are invisible in
+ * the parsed object, which is why this scans the RAW text — same reasoning as
+ * the TOML inf/nan guard below.
+ *
+ * Numbers: `1e999` overflows to Infinity (a rewrite bakes in `null` — the merge
+ * layer's JSON clone does it even before the serializer could refuse), `1e-9999`
+ * underflows to `+0`, an integer literal may have been rounded (a rewrite then
+ * hands consumers that read JSON integers exactly — python, jq, BigInt revivers
+ * — a different value), and `-0` re-serializes as `0`. Only literals whose value
+ * actually changed are refused: `1e21`, `1e-320` or 2^54 round-trip exactly and
+ * stay usable.
+ *
+ * Duplicate members: `{"a":1,"a":2}` parses to a single `a`, so rewriting the
+ * document DELETES the earlier member. That is content loss, not the formatting
+ * normalization we promise, and this classifier is what makes the rewrite of a
+ * user-edited file reachable at all — so it fails closed here.
+ *
+ * Depth: the same pass counts container nesting against MAX_JSON_NESTING
+ * (shared with the serializer, see serialize.ts). JSON.parse handles hundreds
+ * of thousands of levels iteratively, but the downstream merge and
+ * JSON.stringify recurse — a 100KB file nested 50k deep sailed through parse
+ * and guard, then blew up serialization with a raw RangeError after a
+ * multi-GB allocation spike. A hand-written scan rather than a JSON.parse
+ * source-access reviver on purpose: the reviver walk recurses internally, so
+ * its depth limit would be an unpredictable stack size instead of this
+ * deterministic ceiling.
+ */
+function jsonTextSafeToRewrite(text: string): boolean {
+  /**
+   * One frame per open container; a Set of member names for objects, null for
+   * arrays. Its length is the current nesting depth.
+   */
+  const containers: Array<Set<string> | null> = [];
+  /** The most recent string literal — the member name if a `:` follows. */
+  let lastString: string | null = null;
+  let inString = false;
+  let escaped = false;
+  let stringStart = 0;
+  for (let i = 0; i < text.length; i += 1) {
+    const ch = text[i]!;
+    if (inString) {
+      if (escaped) escaped = false;
+      else if (ch === "\\") escaped = true;
+      else if (ch === "\"") {
+        inString = false;
+        lastString = text.slice(stringStart, i + 1);
+      }
+      continue;
+    }
+    if (ch === "\"") { inString = true; stringStart = i; continue; }
+    if (ch === "{" || ch === "[") {
+      containers.push(ch === "{" ? new Set<string>() : null);
+      if (containers.length > MAX_JSON_NESTING) return false;
+      lastString = null;
+      continue;
+    }
+    if (ch === "}" || ch === "]") { containers.pop(); lastString = null; continue; }
+    if (ch === ":") {
+      const members = containers[containers.length - 1];
+      if (members && lastString !== null) {
+        /*
+         * Decoded, not raw: `"a"` and `"a"` are spellings of ONE member,
+         * and JSON.parse keeps only the last of them.
+         */
+        let name: string;
+        try { name = JSON.parse(lastString) as string; } catch { return false; }
+        if (members.has(name)) return false;
+        members.add(name);
+      }
+      lastString = null;
+      continue;
+    }
+    if (ch !== "-" && (ch < "0" || ch > "9")) continue;
+    let end = i + 1;
+    while (end < text.length && /[0-9+\-.eE]/.test(text[end]!)) end += 1;
+    const literal = text.slice(i, end);
+    i = end - 1;
+    const value = Number(literal);
+    if (!Number.isFinite(value)) return false;
+    if (value === 0) {
+      /*
+       * `-0` (re-serializes as `0`) and underflow: `1e-9999` is a nonzero
+       * value the parse already flattened to `+0`. The significand alone
+       * decides, so genuine zero spellings (`0`, `0.0`, `0e10`) stay usable.
+       */
+      if (literal.startsWith("-") || /[1-9]/.test(literal.split(/[eE]/)[0]!)) return false;
+    }
+    /*
+     * Deliberately plain digit runs only. They are the one spelling real
+     * consumers read with exact integer semantics (python's json yields an
+     * arbitrary-precision int, jq preserves big integer literals), so baking
+     * in the rounded double changes what those consumers extract. Decimal or
+     * exponent spellings of the same value (`9007199254740993e0`, `…3.0`) are
+     * float semantics for every consumer — they round identically before and
+     * after a rewrite, and shortest-round-trip stringify preserves what any
+     * reader can observe, so refusing them would only manufacture dead ends
+     * (`1e308` is not exactly representable either, yet rewrites losslessly
+     * for every possible reader).
+     */
+    const digits = literal[0] === "-" ? literal.slice(1) : literal;
+    if (/^[0-9]{16,}$/.test(digits) && BigInt(literal) !== BigInt(value)) return false;
+  }
+  return true;
+}
+
+/** Parse a client config, tolerating absence. PARSE_FAILED on garbage. */
+export function parseConfig(text: string | null, format: ConfigFormat): unknown | typeof PARSE_FAILED {
+  if (text === null || text.trim().length === 0) return {};
+  try {
+    switch (format) {
+      case "json": {
+        /*
+         * Scanned BEFORE parsing: the guard is text-only, and refusing first
+         * means a hostile document is never materialized — the depth ceiling
+         * would otherwise cap the rewrite only after JSON.parse had already
+         * built the 50k-deep object graph. The outcome is unchanged: invalid
+         * JSON still returns PARSE_FAILED, from the catch below.
+         */
+        if (!jsonTextSafeToRewrite(text)) return PARSE_FAILED;
+        return JSON.parse(text);
+      }
+      case "json5": return Bun.JSON5.parse(text);
+      case "yaml": return Bun.YAML.parse(text);
+      case "toml": {
+        /*
+         * Bun's TOML parser mangles the special floats before we ever see the
+         * document: `inf` comes back as the STRING "inf", `-inf` as the number
+         * 0, and `nan` as "nan". Re-serializing that would write those
+         * corruptions back to the user's file while reporting success — a
+         * silent value change is worse than a refusal, so a document
+         * containing them is treated as one we cannot safely rewrite.
+         *
+         * Detected on the raw text, because by the time it is parsed the
+         * evidence is gone.
+         */
+        if (/(^|[\s,[=])[-+]?(?:inf|nan)(?=[\s,\]]|$)/mi.test(text)) return PARSE_FAILED;
+        return Bun.TOML.parse(text);
+      }
+    }
+  } catch {
+    return PARSE_FAILED;
+  }
+}
+
+export type ReadResult =
+  | { kind: "text"; text: string }
+  | { kind: "missing" }
+  | { kind: "failed"; code?: string };
+
+export type StatKind = "file" | "dir" | "other" | "missing" | "failed";
+
+export interface IntegrationIO {
+  /**
+   * ONLY a missing file yields `missing`. Every other failure (EACCES, EPERM,
+   * EISDIR, …) yields `failed`, which callers must treat as unsafe.
+   */
+  readText: (path: string) => ReadResult;
+  /** `failed` is distinct from `missing` for the same reason. */
+  statKind: (path: string) => StatKind;
+  writeText: (path: string, text: string) => void;
+  removeFile: (path: string) => void;
+  mkdirp: (path: string) => void;
+  now: () => number;
+  /** Bookkeeping seams, bound to one store so a test can redirect them. */
+  appendJournal: (entry: JournalEntry) => void;
+  putRecord: (record: OwnershipRecord) => void;
+  dropRecord: (clientId: IntegrationClientId) => void;
+}
+
+export type TargetState =
+  | { ok: true; before: string | null }
+  | { ok: false; why: "not-regular-file" | "read-failed" };
+
+/**
+ * Read the target and classify the three failure shapes correctly.
+ *
+ * The subtle case is a `stat` that succeeds as a file and a `read` that then
+ * fails: that is a real file we cannot see, and treating it as absence is how
+ * an unreadable config gets clobbered.
+ */
+export function loadTarget(io: IntegrationIO, configPath: string): TargetState {
+  const kind = io.statKind(configPath);
+  if (kind === "missing") return { ok: true, before: null };
+  if (kind === "failed") return { ok: false, why: "read-failed" };
+  if (kind !== "file") return { ok: false, why: "not-regular-file" };
+  const read = io.readText(configPath);
+  if (read.kind === "text") return { ok: true, before: read.text };
+  if (read.kind === "failed") return { ok: false, why: "read-failed" };
+  // Raced deletion between the stat and the read.
+  return { ok: true, before: null };
+}
+
+/**
+ * Filesystem half of the seam, split from the store-bound half so there is one
+ * place that decides what a failed read or stat MEANS, and one place that
+ * decides WHERE bookkeeping goes.
+ */
+export function fileIO(): Omit<IntegrationIO, "appendJournal" | "putRecord" | "dropRecord"> {
+  return {
+    readText: path => {
+      try {
+        return { kind: "text", text: readFileSync(path, "utf8") };
+      } catch (error) {
+        const code = (error as NodeJS.ErrnoException).code;
+        return code === "ENOENT" ? { kind: "missing" } : { kind: "failed", ...(code ? { code } : {}) };
+      }
+    },
+    statKind: path => {
+      try {
+        const stats = statSync(path);
+        return stats.isFile() ? "file" : stats.isDirectory() ? "dir" : "other";
+      } catch (error) {
+        return (error as NodeJS.ErrnoException).code === "ENOENT" ? "missing" : "failed";
+      }
+    },
+    writeText: (path, text) => atomicWriteFile(path, text),
+    removeFile: path => rmSync(path, { force: true }),
+    mkdirp: path => mkdirSync(path, { recursive: true, mode: 0o700 }),
+    now: () => Date.now(),
+  };
+}
+
+/**
+ * The full seam: filesystem behavior plus bookkeeping bound to ONE store.
+ *
+ * Kept together so the IO a writer uses and the store it journals into can
+ * never point at different roots — that split is what let an "isolated"
+ * operation mutate the real state while a test believed otherwise.
+ */
+export function defaultIntegrationIO(store: {
+  appendJournal: IntegrationIO["appendJournal"];
+  putRecord: IntegrationIO["putRecord"];
+  dropRecord: IntegrationIO["dropRecord"];
+}): IntegrationIO {
+  return {
+    ...fileIO(),
+    appendJournal: entry => store.appendJournal(entry),
+    putRecord: record => store.putRecord(record),
+    dropRecord: clientId => store.dropRecord(clientId),
+  };
+}

package/src/integrations/journal.ts

@@ -0,0 +1,315 @@
+/**
+ * The operation log, its snapshots, and the retention bookkeeping around them.
+ *
+ * Two ordering rules carry the safety here:
+ *
+ * 1. `appendOperation` commits the row and NOTHING else. Pruning runs after,
+ *    best-effort, because a GC failure that looked like an append failure would
+ *    make the writer compensate for an operation that already succeeded.
+ * 2. Snapshots can hold the user's own credentials — we copy their file
+ *    verbatim — so they go through `atomicWriteFile`, which applies 0600 plus
+ *    Windows ACL hardening.
+ *
+ * Design of record: devlog/_fin/260802_client_toggle_api/021 §4.
+ */
+import { randomUUID } from "node:crypto";
+import { appendFileSync, existsSync, readFileSync, readdirSync, rmSync } from "node:fs";
+import { join, resolve, sep } from "node:path";
+import { atomicWriteFile } from "../config";
+import { ensureDir, fingerprint, integrationsDir, type OwnershipRecord } from "./ownership";
+import { isIntegrationClientId, type IntegrationClientId } from "./registry";
+
+export type OperationKind = "apply" | "disable" | "refresh" | "restore";
+
+/**
+ * Tagged so "the file did not exist" and "the snapshot was collected" stay
+ * distinguishable. `null` conflated them, and only one of the two is
+ * recoverable — restoring an operation that created a file means deleting it.
+ */
+export type SnapshotRef =
+  | { kind: "none" }
+  | { kind: "stored"; relPath: string }
+  | { kind: "expired" };
+
+export interface JournalEntry {
+  opId: string;
+  clientId: IntegrationClientId;
+  kind: OperationKind;
+  at: string;
+  configPath: string;
+  snapshot: SnapshotRef;
+  /** Fingerprint of the file AFTER this op; "" when the op left no file. */
+  resultFingerprint: string;
+  /** True when the op's result was file absence — restore then means "delete". */
+  resultAbsent: boolean;
+  /**
+   * Ownership as it stood BEFORE this operation. Restore puts this back
+   * alongside the bytes, so provenance always describes the file it came with
+   * and is never re-derived from a provider-id prefix — which would silently
+   * adopt a user's own `opencodex/...` entry.
+   */
+  priorRecord: OwnershipRecord | null;
+}
+
+export const SNAPSHOT_RETENTION = 10;
+
+/**
+ * Does the file on disk still hold what this operation left behind?
+ *
+ * The one place that answers it, because two places answered differently. An
+ * operation whose result was ABSENCE records `resultAbsent: true` and an empty
+ * `resultFingerprint`; the route compared a missing file to `""` and called it
+ * a match, while restore hashed `""` into a real digest and called the same
+ * unchanged absence a drift. So the journal offered Undo and the restore
+ * route then demanded a confirmation for edits nobody had made.
+ *
+ * `currentText` is `null` for a missing file — not `""`, which is a file that
+ * exists and is empty. The distinction is the whole point.
+ */
+export function matchesOperationResult(entry: JournalEntry, currentText: string | null): boolean {
+  if (entry.resultAbsent) return currentText === null;
+  if (currentText === null) return false;
+  return fingerprint(currentText) === entry.resultFingerprint;
+}
+
+export function newOpId(): string {
+  return randomUUID();
+}
+
+function journalPath(dir: string): string {
+  return join(dir, "journal.jsonl");
+}
+
+/**
+ * A snapshot file name must be ONE path component we produced, never a value
+ * that walks anywhere. `opId` comes from `randomUUID()` in normal operation,
+ * but it also arrives from a persisted journal row, and `relPath` is read
+ * straight off disk — a row carrying `../../escaped` would otherwise let
+ * capture write outside the store and read-back read an arbitrary file.
+ */
+function assertSafeComponent(value: string, what: string): string {
+  if (value.length === 0 || value === "." || value === ".."
+    || value.includes("/") || value.includes("\\") || value.includes("\0")) {
+    throw new Error(`unsafe ${what}: ${JSON.stringify(value)}`);
+  }
+  return value;
+}
+
+/** Resolve `relative` under `root`, refusing anything that escapes it. */
+function containedPath(root: string, ...segments: string[]): string {
+  const target = resolve(root, ...segments);
+  const base = resolve(root);
+  if (target !== base && !target.startsWith(base + sep)) {
+    throw new Error(`path escapes the integration store: ${target}`);
+  }
+  return target;
+}
+
+function snapshotDir(clientId: IntegrationClientId, dir: string): string {
+  return join(dir, "snapshots", clientId);
+}
+
+export function captureSnapshot(
+  clientId: IntegrationClientId,
+  opId: string,
+  text: string | null,
+  dir: string = integrationsDir(),
+): SnapshotRef {
+  if (text === null) return { kind: "none" };
+  const target = containedPath(dir, "snapshots", assertSafeComponent(clientId, "clientId"), assertSafeComponent(opId, "opId"));
+  ensureDir(target);
+  atomicWriteFile(target, text);
+  return { kind: "stored", relPath: join("snapshots", clientId, opId) };
+}
+
+/** Commit the row. Pruning is post-commit and can never fail the append. */
+export function appendOperation(entry: JournalEntry, dir: string = integrationsDir()): void {
+  ensureDir(journalPath(dir));
+  appendFileSync(journalPath(dir), `${JSON.stringify(entry)}\n`, { encoding: "utf8", mode: 0o600 });
+  try {
+    const pruned = pruneSnapshots(entry.clientId, dir);
+    if (pruned.ok) clearPruneFailure(entry.clientId, dir);
+    else markPruneFailure(entry.clientId, pruned.error, dir);
+  } catch (error) {
+    console.error(`[integrations] post-commit maintenance failed: ${String(error)}`);
+  }
+}
+
+/** Newest first. A torn final line (crash mid-append) is skipped, not thrown. */
+export function listOperations(
+  clientId?: IntegrationClientId,
+  limit = 50,
+  dir: string = integrationsDir(),
+): JournalEntry[] {
+  let raw: string;
+  try {
+    raw = readFileSync(journalPath(dir), "utf8");
+  } catch {
+    return [];
+  }
+  const rows: JournalEntry[] = [];
+  for (const line of raw.split("\n")) {
+    if (!line.trim()) continue;
+    try {
+      const parsed = JSON.parse(line) as JournalEntry;
+      if (!clientId || parsed.clientId === clientId) rows.push(parsed);
+    } catch {
+      // Torn line from an interrupted append; the rest of the log is still good.
+    }
+  }
+  return rows.reverse().slice(0, limit);
+}
+
+export function findOperation(opId: string, dir: string = integrationsDir()): JournalEntry | null {
+  return listOperations(undefined, Number.MAX_SAFE_INTEGER, dir).find(row => row.opId === opId) ?? null;
+}
+
+/** Resolves the tag against what is actually on disk now. */
+export function readSnapshot(
+  entry: JournalEntry,
+  dir: string = integrationsDir(),
+): { kind: "none" } | { kind: "stored"; text: string; path: string } | { kind: "expired" } {
+  if (entry.snapshot.kind === "none") return { kind: "none" };
+  if (entry.snapshot.kind === "expired") return { kind: "expired" };
+  // Derive the path from validated identifiers rather than trusting the
+  // persisted relPath, then verify containment either way.
+  const abs = containedPath(dir, entry.snapshot.relPath);
+  if (!existsSync(abs)) return { kind: "expired" };
+  return { kind: "stored", text: readFileSync(abs, "utf8"), path: abs };
+}
+
+/**
+ * Snapshot files retained right now — the witness for `retentionDegraded`.
+ *
+ * `null` means "cannot inspect", which is NOT the same as zero: reporting an
+ * unreadable snapshot directory as a healthy empty one would hide the
+ * unbounded, credential-bearing pile this field exists to disclose.
+ */
+export function countSnapshots(
+  clientId: IntegrationClientId,
+  dir: string = integrationsDir(),
+): number | null {
+  try {
+    return readdirSync(snapshotDir(clientId, dir)).length;
+  } catch (error) {
+    return (error as NodeJS.ErrnoException).code === "ENOENT" ? 0 : null;
+  }
+}
+
+/**
+ * Keep the newest N snapshot files per client; journal rows always survive.
+ * Structured rather than throwing or swallowing: a swallowed failure would let
+ * credential-bearing snapshots pile up while every operation reported success.
+ */
+export function pruneSnapshots(
+  clientId: IntegrationClientId,
+  dir: string = integrationsDir(),
+): { ok: true } | { ok: false; error: string } {
+  /*
+   * Filter to rows that HAVE a snapshot first, then take the newest N.
+   *
+   * Taking the newest N operations and filtering afterwards counted rows that
+   * never stored anything: an apply-to-absent records `snapshot: none`, so a
+   * client whose history alternates stored and none kept only half the backups
+   * the contract promises. The docs say ten backups per client, and this is
+   * the code that has to make that true.
+   */
+  const keep = new Set(
+    listOperations(clientId, Number.MAX_SAFE_INTEGER, dir)
+      .filter(row => row.snapshot.kind === "stored")
+      .slice(0, SNAPSHOT_RETENTION)
+      .map(row => row.opId),
+  );
+  let names: string[];
+  try {
+    names = readdirSync(snapshotDir(clientId, dir));
+  } catch (error) {
+    const code = (error as NodeJS.ErrnoException).code;
+    if (code === "ENOENT") return { ok: true };
+    return { ok: false, error: String(error) };
+  }
+  for (const name of names) {
+    if (keep.has(name)) continue;
+    try {
+      rmSync(join(snapshotDir(clientId, dir), name), { force: true });
+    } catch (error) {
+      return { ok: false, error: String(error) };
+    }
+  }
+  return { ok: true };
+}
+
+export interface MaintenanceState {
+  pruneFailures: Partial<Record<IntegrationClientId, { at: string; error: string }>>;
+}
+
+function maintenancePath(dir: string): string {
+  return join(dir, "maintenance.json");
+}
+
+/**
+ * Validates rather than casts: `{}` parses fine and would leave `pruneFailures`
+ * undefined, so the next mark/clear would throw AFTER the journal row was
+ * committed — producing the phantom row the write ordering exists to prevent.
+ */
+export function readMaintenance(dir: string = integrationsDir()): MaintenanceState {
+  try {
+    const parsed: unknown = JSON.parse(readFileSync(maintenancePath(dir), "utf8"));
+    if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
+      const raw = (parsed as { pruneFailures?: unknown }).pruneFailures;
+      const failures: MaintenanceState["pruneFailures"] = {};
+      if (raw && typeof raw === "object" && !Array.isArray(raw)) {
+        for (const [key, value] of Object.entries(raw as Record<string, unknown>)) {
+          if (!isIntegrationClientId(key)) continue;
+          if (!value || typeof value !== "object") continue;
+          const { at, error } = value as { at?: unknown; error?: unknown };
+          if (typeof at !== "string" || typeof error !== "string") continue;
+          failures[key] = { at, error };
+        }
+      }
+      return { pruneFailures: failures };
+    }
+  } catch {
+    // Absent or corrupt: no pending retries known.
+  }
+  return { pruneFailures: {} };
+}
+
+function writeMaintenance(state: MaintenanceState, dir: string): void {
+  try {
+    ensureDir(maintenancePath(dir));
+    atomicWriteFile(maintenancePath(dir), `${JSON.stringify(state, null, 2)}\n`);
+  } catch (error) {
+    // The marker is an optimization for scheduling retries. `retentionDegraded`
+    // is derived from the snapshot count, so losing it costs a retry, not the
+    // claim — a durable promise must not depend on a write that can fail.
+    console.error(`[integrations] could not record maintenance state: ${String(error)}`);
+  }
+}
+
+export function markPruneFailure(
+  clientId: IntegrationClientId,
+  error: string,
+  dir: string = integrationsDir(),
+): void {
+  const state = readMaintenance(dir);
+  state.pruneFailures[clientId] = { at: new Date().toISOString(), error };
+  writeMaintenance(state, dir);
+}
+
+export function clearPruneFailure(
+  clientId: IntegrationClientId,
+  dir: string = integrationsDir(),
+): void {
+  const state = readMaintenance(dir);
+  if (!(clientId in state.pruneFailures)) return;
+  delete state.pruneFailures[clientId];
+  writeMaintenance(state, dir);
+}
+
+/** Re-attempt every marked client. */
+export function retryPendingPrunes(dir: string = integrationsDir()): void {
+  for (const clientId of Object.keys(readMaintenance(dir).pruneFailures) as IntegrationClientId[]) {
+    if (pruneSnapshots(clientId, dir).ok) clearPruneFailure(clientId, dir);
+  }
+}