@crewhaus/citation-tracker 0.1.4 → 0.1.6

package/dist/index.d.ts

@@ -0,0 +1,60 @@
+import { CrewhausError } from "@crewhaus/errors";
+export declare const DEFAULT_ROOT_DIR = ".crewhaus/research";
+export type RunId = string;
+export type FetchRecord = {
+    readonly version: 1;
+    readonly url: string;
+    readonly retrievedAt: string;
+    readonly sha256: string;
+    readonly contentBytes: number;
+    /** Branch (sub-question) that initiated this fetch, if known. */
+    readonly branchId?: string;
+};
+export type Citation = {
+    readonly version: 1;
+    readonly url: string;
+    readonly snippet: string;
+    readonly retrievedAt: string;
+    readonly sha256: string;
+    readonly branchId?: string;
+    /** Free-form supporting-claim label (e.g., "supports: target shapes list"). */
+    readonly supportingClaim?: string;
+};
+export type RecordFetchInput = {
+    readonly url: string;
+    readonly content: string;
+    readonly branchId?: string;
+};
+export type RecordCitationInput = {
+    readonly url: string;
+    readonly snippet: string;
+    readonly branchId?: string;
+    readonly supportingClaim?: string;
+};
+export interface CitationTracker {
+    readonly runId: RunId;
+    /** True iff the tracker already has a content body for this URL. */
+    hasFetched(url: string): boolean;
+    /** Cached content body for a URL (matched by exact string). */
+    getFetchedContent(url: string): string | undefined;
+    /** Cached fetch metadata (sha256 + retrievedAt) for a URL. */
+    getFetchRecord(url: string): FetchRecord | undefined;
+    /** Idempotent: re-recording the same URL is a no-op (preserves first sha256). */
+    recordFetch(input: RecordFetchInput): FetchRecord;
+    recordCitation(input: RecordCitationInput): Citation;
+    /** Citations in append-order (deterministic across re-runs). */
+    listCitationsOrdered(): ReadonlyArray<Citation>;
+    /** All fetch records in append-order. */
+    listFetches(): ReadonlyArray<FetchRecord>;
+}
+export type CreateCitationTrackerOptions = {
+    readonly runId?: RunId;
+    readonly rootDir?: string;
+    readonly now?: () => Date;
+};
+export declare function newRunId(): RunId;
+export declare class CitationTrackerError extends CrewhausError {
+    readonly name = "CitationTrackerError";
+    constructor(message: string, cause?: unknown);
+}
+export declare function createCitationTracker(opts?: CreateCitationTrackerOptions): CitationTracker;

package/dist/index.js

@@ -0,0 +1,189 @@
+/**
+ * Catalog R-orch `citation-tracker` — Section 23 RES.
+ *
+ * Persistent per-research-run record of every URL the crawler has
+ * touched, plus every fact the agent has cited. The same instance does
+ * two jobs that the kickoff treats as one:
+ *
+ *   1. **Fetch dedup** — `hasFetched(url)` lets the crawler skip a HTTP
+ *      round-trip on resume. `recordFetch({url, content})` stores the
+ *      sha256 + the content body so a kill-and-resume returns identical
+ *      bytes from cache rather than re-pulling the URL.
+ *   2. **Citation registry** — `recordCitation({url, snippet, branchId?})`
+ *      appends a fact to the run's citation log. `listCitationsOrdered()`
+ *      returns citations in first-seen order (deterministic across
+ *      re-runs because of the JSONL append-only file).
+ *
+ * Storage layout:
+ *   `<rootDir>/<runId>/fetches.jsonl`   one line per URL ever fetched
+ *   `<rootDir>/<runId>/citations.jsonl` one line per `recordCitation` call
+ *
+ * Both files are append-only, mode 0o600, atomic-per-line on POSIX
+ * (mirrors `event-log`). On boot the tracker reads each file once into
+ * an in-memory cache; subsequent calls hit memory.
+ *
+ * The smoke test's "byte-identical citation block across re-runs"
+ * invariant is anchored here: citations are ordered by their first
+ * append timestamp, NOT by retrievedAt, so even if the model takes
+ * different routes on the second pass, the citation list still ends up
+ * in stable order keyed off the FIRST-seen URL.
+ */
+import { createHash, randomUUID } from "node:crypto";
+import { appendFileSync, existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { CrewhausError, RuntimeError } from "@crewhaus/errors";
+export const DEFAULT_ROOT_DIR = ".crewhaus/research";
+const RUN_ID_RE = /^run_[0-9a-f]{16}$/;
+/**
+ * A sha256 digest is always exactly 64 lowercase hex chars. We re-validate it
+ * before interpolating into the cache file path because the value can originate
+ * from a JSONL line read off disk (boot-time replay), and a corrupted or
+ * tampered `fetches.jsonl` must not be able to steer a read outside `cache/`
+ * (e.g. a `sha256` of "../../../etc/passwd"). Legitimate records always match.
+ */
+const SHA256_RE = /^[0-9a-f]{64}$/;
+function validateRunId(runId) {
+    if (!RUN_ID_RE.test(runId)) {
+        throw new RuntimeError(`citation-tracker: invalid runId "${runId}" — expected run_<16 hex>`);
+    }
+}
+export function newRunId() {
+    // 16 hex chars. UUID is 32 hex (with dashes); strip dashes and take 16.
+    const id = randomUUID().replace(/-/g, "").slice(0, 16);
+    return `run_${id}`;
+}
+export class CitationTrackerError extends CrewhausError {
+    name = "CitationTrackerError";
+    constructor(message, cause) {
+        super("runtime", message, cause);
+    }
+}
+export function createCitationTracker(opts = {}) {
+    const runId = opts.runId ?? newRunId();
+    validateRunId(runId);
+    const rootDir = opts.rootDir ?? DEFAULT_ROOT_DIR;
+    const runDir = join(rootDir, runId);
+    const fetchesPath = join(runDir, "fetches.jsonl");
+    const citationsPath = join(runDir, "citations.jsonl");
+    const now = opts.now ?? (() => new Date());
+    mkdirSync(runDir, { recursive: true });
+    // Boot-time replay of prior records so a resume sees the same dedup state.
+    const fetchByUrl = new Map();
+    const fetchOrder = [];
+    const citations = [];
+    if (existsSync(fetchesPath)) {
+        for (const raw of readFileSync(fetchesPath, "utf8").split("\n")) {
+            const line = raw.trim();
+            if (line.length === 0)
+                continue;
+            try {
+                const rec = JSON.parse(line);
+                if (typeof rec.url === "string" && !fetchByUrl.has(rec.url)) {
+                    fetchByUrl.set(rec.url, rec);
+                    fetchOrder.push(rec);
+                }
+            }
+            catch (err) {
+                throw new CitationTrackerError(`malformed JSON in ${fetchesPath}`, err);
+            }
+        }
+    }
+    if (existsSync(citationsPath)) {
+        for (const raw of readFileSync(citationsPath, "utf8").split("\n")) {
+            const line = raw.trim();
+            if (line.length === 0)
+                continue;
+            try {
+                const rec = JSON.parse(line);
+                citations.push(rec);
+            }
+            catch (err) {
+                throw new CitationTrackerError(`malformed JSON in ${citationsPath}`, err);
+            }
+        }
+    }
+    // Per-fetch content cache — written separately under <runDir>/cache/ so the
+    // JSONL stays small (citations only carry sha256 + snippet, not full bodies).
+    const cacheDir = join(runDir, "cache");
+    mkdirSync(cacheDir, { recursive: true });
+    function cachePathFor(sha256) {
+        return join(cacheDir, `${sha256}.txt`);
+    }
+    function tryReadCache(rec) {
+        // Defend the cache read against a tampered fetches.jsonl: a sha256 that is
+        // not a clean 64-hex digest could otherwise traverse out of cache/.
+        if (typeof rec.sha256 !== "string" || !SHA256_RE.test(rec.sha256))
+            return undefined;
+        const p = cachePathFor(rec.sha256);
+        if (!existsSync(p))
+            return undefined;
+        return readFileSync(p, "utf8");
+    }
+    return {
+        runId,
+        hasFetched(url) {
+            return fetchByUrl.has(url);
+        },
+        getFetchedContent(url) {
+            const rec = fetchByUrl.get(url);
+            if (rec === undefined)
+                return undefined;
+            return tryReadCache(rec);
+        },
+        getFetchRecord(url) {
+            return fetchByUrl.get(url);
+        },
+        recordFetch(input) {
+            const existing = fetchByUrl.get(input.url);
+            if (existing !== undefined)
+                return existing;
+            const sha256 = createHash("sha256").update(input.content).digest("hex");
+            const rec = {
+                version: 1,
+                url: input.url,
+                retrievedAt: now().toISOString(),
+                sha256,
+                contentBytes: Buffer.byteLength(input.content, "utf8"),
+                ...(input.branchId !== undefined ? { branchId: input.branchId } : {}),
+            };
+            fetchByUrl.set(input.url, rec);
+            fetchOrder.push(rec);
+            appendFileSync(fetchesPath, `${JSON.stringify(rec)}\n`, { mode: 0o600 });
+            // Cache content for resume by sha256 — same content across runs ends up
+            // as the same file, so a re-fetch into a different tracker still hits
+            // the same cache directory if the rootDir is shared.
+            writeFileSync(cachePathFor(sha256), input.content, { mode: 0o600 });
+            return rec;
+        },
+        recordCitation(input) {
+            // Resolve sha256: prefer the matching fetch record's hash so a citation
+            // anchors to a specific content version. If no matching fetch, hash
+            // the snippet itself.
+            const fetch = fetchByUrl.get(input.url);
+            const sha256 = fetch !== undefined
+                ? fetch.sha256
+                : createHash("sha256").update(input.snippet).digest("hex");
+            const c = {
+                version: 1,
+                url: input.url,
+                snippet: input.snippet,
+                retrievedAt: fetch !== undefined ? fetch.retrievedAt : now().toISOString(),
+                sha256,
+                ...(input.branchId !== undefined ? { branchId: input.branchId } : {}),
+                ...(input.supportingClaim !== undefined ? { supportingClaim: input.supportingClaim } : {}),
+            };
+            citations.push(c);
+            appendFileSync(citationsPath, `${JSON.stringify(c)}\n`, { mode: 0o600 });
+            return c;
+        },
+        listCitationsOrdered() {
+            // Stable order: append order. Same-URL citations are kept (the agent
+            // may cite multiple snippets from one source); the report-writer's
+            // numbering layer dedups by URL.
+            return [...citations];
+        },
+        listFetches() {
+            return [...fetchOrder];
+        },
+    };
+}

package/package.json

@@ -1,18 +1,21 @@
 {
   "name": "@crewhaus/citation-tracker",
-  "version": "0.1.4",
+  "version": "0.1.6",
   "type": "module",
   "description": "Per-research-run citation persistence + URL dedup (Section 23 RES)",
-  "main": "src/index.ts",
-  "types": "src/index.ts",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
   "exports": {
-    ".": "./src/index.ts"
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    }
   },
   "scripts": {
     "test": "bun test src"
   },
   "dependencies": {
-    "@crewhaus/errors": "0.1.4"
+    "@crewhaus/errors": "0.1.6"
   },
   "license": "Apache-2.0",
   "author": {
@@ -32,5 +35,5 @@
   "publishConfig": {
     "access": "public"
   },
-  "files": ["src", "README.md", "LICENSE", "NOTICE"]
+  "files": ["dist", "README.md", "LICENSE", "NOTICE"]
 }

package/src/index.test.ts

@@ -1,290 +0,0 @@
-import { describe, expect, test } from "bun:test";
-import { appendFileSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
-import { tmpdir } from "node:os";
-import { join } from "node:path";
-import { CitationTrackerError, createCitationTracker, newRunId } from "./index.js";
-
-function newRoot(): string {
-  return mkdtempSync(join(tmpdir(), "citation-tracker-"));
-}
-
-describe("createCitationTracker", () => {
-  test("hasFetched returns false for fresh tracker, true after recordFetch", () => {
-    const root = newRoot();
-    try {
-      const t = createCitationTracker({ rootDir: root });
-      expect(t.hasFetched("https://example.com/a")).toBe(false);
-      t.recordFetch({ url: "https://example.com/a", content: "hello world" });
-      expect(t.hasFetched("https://example.com/a")).toBe(true);
-    } finally {
-      rmSync(root, { recursive: true, force: true });
-    }
-  });
-
-  test("recordFetch is idempotent — re-recording same URL returns first record", () => {
-    const root = newRoot();
-    try {
-      const t = createCitationTracker({ rootDir: root });
-      const first = t.recordFetch({ url: "u", content: "v1" });
-      const second = t.recordFetch({ url: "u", content: "v2-different" });
-      // First write wins; the second record is the same as the first.
-      expect(second).toEqual(first);
-      expect(t.getFetchedContent("u")).toBe("v1");
-    } finally {
-      rmSync(root, { recursive: true, force: true });
-    }
-  });
-
-  test("getFetchedContent returns the original body verbatim (T4 cache hit)", () => {
-    const root = newRoot();
-    try {
-      const t = createCitationTracker({ rootDir: root });
-      const body = "some\nmulti\nline\ncontent";
-      t.recordFetch({ url: "u", content: body });
-      expect(t.getFetchedContent("u")).toBe(body);
-    } finally {
-      rmSync(root, { recursive: true, force: true });
-    }
-  });
-
-  test("kill-and-resume: a new tracker against the same runId/rootDir picks up prior fetches (T4)", () => {
-    const root = newRoot();
-    const runId = newRunId();
-    try {
-      const t1 = createCitationTracker({ runId, rootDir: root });
-      t1.recordFetch({ url: "u1", content: "body-1" });
-      t1.recordCitation({ url: "u1", snippet: "snippet-1", branchId: "b1" });
-
-      // Simulate kill by dropping t1; new tracker reads files back.
-      const t2 = createCitationTracker({ runId, rootDir: root });
-      expect(t2.hasFetched("u1")).toBe(true);
-      expect(t2.getFetchedContent("u1")).toBe("body-1");
-      const cs = t2.listCitationsOrdered();
-      expect(cs).toHaveLength(1);
-      expect(cs[0]?.url).toBe("u1");
-      expect(cs[0]?.snippet).toBe("snippet-1");
-    } finally {
-      rmSync(root, { recursive: true, force: true });
-    }
-  });
-
-  test("listCitationsOrdered returns citations in append order — deterministic across re-runs", () => {
-    const root = newRoot();
-    const runId = newRunId();
-    try {
-      const t = createCitationTracker({ runId, rootDir: root });
-      t.recordFetch({ url: "a", content: "ka" });
-      t.recordFetch({ url: "b", content: "kb" });
-      t.recordCitation({ url: "a", snippet: "fact a-1" });
-      t.recordCitation({ url: "b", snippet: "fact b-1" });
-      t.recordCitation({ url: "a", snippet: "fact a-2" });
-
-      const reopened = createCitationTracker({ runId, rootDir: root });
-      const order = reopened.listCitationsOrdered().map((c) => c.snippet);
-      expect(order).toEqual(["fact a-1", "fact b-1", "fact a-2"]);
-    } finally {
-      rmSync(root, { recursive: true, force: true });
-    }
-  });
-
-  test("recordCitation pulls sha256 from the matching fetch record when present", () => {
-    const root = newRoot();
-    try {
-      const t = createCitationTracker({ rootDir: root });
-      const f = t.recordFetch({ url: "u", content: "the body text" });
-      const c = t.recordCitation({ url: "u", snippet: "body" });
-      expect(c.sha256).toBe(f.sha256);
-      expect(c.retrievedAt).toBe(f.retrievedAt);
-    } finally {
-      rmSync(root, { recursive: true, force: true });
-    }
-  });
-
-  test("recordCitation with no prior fetch hashes the snippet itself", () => {
-    const root = newRoot();
-    try {
-      const t = createCitationTracker({ rootDir: root });
-      const c = t.recordCitation({ url: "u", snippet: "abc" });
-      expect(c.sha256).toMatch(/^[0-9a-f]{64}$/);
-    } finally {
-      rmSync(root, { recursive: true, force: true });
-    }
-  });
-
-  test("invalid runId throws", () => {
-    const root = newRoot();
-    try {
-      expect(() => createCitationTracker({ runId: "bad", rootDir: root })).toThrow(/invalid runId/);
-    } finally {
-      rmSync(root, { recursive: true, force: true });
-    }
-  });
-
-  test("malformed jsonl on disk surfaces as a CitationTrackerError", async () => {
-    const root = newRoot();
-    const runId = newRunId();
-    const path = join(root, runId, "fetches.jsonl");
-    try {
-      // Create the dir + a malformed file.
-      const t = createCitationTracker({ runId, rootDir: root });
-      t.recordFetch({ url: "u", content: "ok" });
-      // Append garbage and re-open.
-      const fs = await import("node:fs");
-      fs.appendFileSync(path, "not-json\n");
-      expect(() => createCitationTracker({ runId, rootDir: root })).toThrow(CitationTrackerError);
-    } finally {
-      rmSync(root, { recursive: true, force: true });
-    }
-  });
-
-  test("malformed jsonl in citations.jsonl surfaces as a CitationTrackerError", () => {
-    const root = newRoot();
-    const runId = newRunId();
-    const path = join(root, runId, "citations.jsonl");
-    try {
-      const t = createCitationTracker({ runId, rootDir: root });
-      t.recordCitation({ url: "u", snippet: "s" });
-      appendFileSync(path, "{not valid json\n");
-      expect(() => createCitationTracker({ runId, rootDir: root })).toThrow(CitationTrackerError);
-    } finally {
-      rmSync(root, { recursive: true, force: true });
-    }
-  });
-
-  test("getFetchRecord returns metadata for a known URL and undefined otherwise", () => {
-    const root = newRoot();
-    try {
-      const t = createCitationTracker({ rootDir: root });
-      expect(t.getFetchRecord("missing")).toBeUndefined();
-      const rec = t.recordFetch({ url: "u", content: "the body", branchId: "b7" });
-      const got = t.getFetchRecord("u");
-      expect(got).toEqual(rec);
-      expect(got?.sha256).toMatch(/^[0-9a-f]{64}$/);
-      expect(got?.contentBytes).toBe(Buffer.byteLength("the body", "utf8"));
-      expect(got?.branchId).toBe("b7");
-    } finally {
-      rmSync(root, { recursive: true, force: true });
-    }
-  });
-
-  test("getFetchedContent returns undefined for an unknown URL", () => {
-    const root = newRoot();
-    try {
-      const t = createCitationTracker({ rootDir: root });
-      expect(t.getFetchedContent("never-fetched")).toBeUndefined();
-    } finally {
-      rmSync(root, { recursive: true, force: true });
-    }
-  });
-
-  test("listFetches returns all fetch records in append order", () => {
-    const root = newRoot();
-    try {
-      const t = createCitationTracker({ rootDir: root });
-      expect(t.listFetches()).toEqual([]);
-      t.recordFetch({ url: "first", content: "1" });
-      t.recordFetch({ url: "second", content: "2" });
-      // Idempotent re-record must NOT add a second entry.
-      t.recordFetch({ url: "first", content: "1-again" });
-      const urls = t.listFetches().map((r) => r.url);
-      expect(urls).toEqual(["first", "second"]);
-    } finally {
-      rmSync(root, { recursive: true, force: true });
-    }
-  });
-
-  test("listFetches survives kill-and-resume in append order", () => {
-    const root = newRoot();
-    const runId = newRunId();
-    try {
-      const t1 = createCitationTracker({ runId, rootDir: root });
-      t1.recordFetch({ url: "a", content: "ka" });
-      t1.recordFetch({ url: "b", content: "kb" });
-      const t2 = createCitationTracker({ runId, rootDir: root });
-      expect(t2.listFetches().map((r) => r.url)).toEqual(["a", "b"]);
-    } finally {
-      rmSync(root, { recursive: true, force: true });
-    }
-  });
-
-  test("recordCitation carries branchId and supportingClaim through to disk", () => {
-    const root = newRoot();
-    const runId = newRunId();
-    try {
-      const t = createCitationTracker({ runId, rootDir: root });
-      const c = t.recordCitation({
-        url: "u",
-        snippet: "s",
-        branchId: "branch-9",
-        supportingClaim: "supports: target shapes list",
-      });
-      expect(c.branchId).toBe("branch-9");
-      expect(c.supportingClaim).toBe("supports: target shapes list");
-
-      // Round-trips verbatim through the JSONL on resume.
-      const reopened = createCitationTracker({ runId, rootDir: root });
-      const persisted = reopened.listCitationsOrdered()[0];
-      expect(persisted?.branchId).toBe("branch-9");
-      expect(persisted?.supportingClaim).toBe("supports: target shapes list");
-    } finally {
-      rmSync(root, { recursive: true, force: true });
-    }
-  });
-
-  test("now() injection makes retrievedAt deterministic", () => {
-    const root = newRoot();
-    try {
-      const fixed = new Date("2030-01-02T03:04:05.000Z");
-      const t = createCitationTracker({ rootDir: root, now: () => fixed });
-      const f = t.recordFetch({ url: "u", content: "x" });
-      expect(f.retrievedAt).toBe("2030-01-02T03:04:05.000Z");
-      // Citation with no prior fetch also stamps via now().
-      const c = t.recordCitation({ url: "other", snippet: "s" });
-      expect(c.retrievedAt).toBe("2030-01-02T03:04:05.000Z");
-    } finally {
-      rmSync(root, { recursive: true, force: true });
-    }
-  });
-
-  test("newRunId produces well-formed, accepted run ids", () => {
-    const root = newRoot();
-    try {
-      const id = newRunId();
-      expect(id).toMatch(/^run_[0-9a-f]{16}$/);
-      // A freshly-minted id must be accepted by the validator.
-      const t = createCitationTracker({ runId: id, rootDir: root });
-      expect(t.runId).toBe(id);
-    } finally {
-      rmSync(root, { recursive: true, force: true });
-    }
-  });
-
-  test("security: a tampered sha256 in fetches.jsonl cannot traverse out of the cache dir", () => {
-    const root = newRoot();
-    const runId = newRunId();
-    try {
-      // Boot once to create the run dir, then overwrite fetches.jsonl with a
-      // record whose sha256 is a path-traversal payload. A secret file lives
-      // two levels up (the runId dir) at a guessable name.
-      createCitationTracker({ runId, rootDir: root });
-      const secretPath = join(root, runId, "stolen.txt");
-      writeFileSync(secretPath, "TOP SECRET CONTENTS");
-      // cache/<sha256>.txt -> cache/../stolen.txt resolves to the secret file.
-      const evilRecord = {
-        version: 1,
-        url: "https://victim.example/x",
-        retrievedAt: "2030-01-01T00:00:00.000Z",
-        sha256: "../stolen",
-        contentBytes: 19,
-      };
-      writeFileSync(join(root, runId, "fetches.jsonl"), `${JSON.stringify(evilRecord)}\n`);
-
-      const t = createCitationTracker({ runId, rootDir: root });
-      expect(t.hasFetched("https://victim.example/x")).toBe(true);
-      // The guard must refuse the malformed sha256 rather than read the secret.
-      expect(t.getFetchedContent("https://victim.example/x")).toBeUndefined();
-    } finally {
-      rmSync(root, { recursive: true, force: true });
-    }
-  });
-});

package/src/index.ts

@@ -1,262 +0,0 @@
-/**
- * Catalog R-orch `citation-tracker` — Section 23 RES.
- *
- * Persistent per-research-run record of every URL the crawler has
- * touched, plus every fact the agent has cited. The same instance does
- * two jobs that the kickoff treats as one:
- *
- *   1. **Fetch dedup** — `hasFetched(url)` lets the crawler skip a HTTP
- *      round-trip on resume. `recordFetch({url, content})` stores the
- *      sha256 + the content body so a kill-and-resume returns identical
- *      bytes from cache rather than re-pulling the URL.
- *   2. **Citation registry** — `recordCitation({url, snippet, branchId?})`
- *      appends a fact to the run's citation log. `listCitationsOrdered()`
- *      returns citations in first-seen order (deterministic across
- *      re-runs because of the JSONL append-only file).
- *
- * Storage layout:
- *   `<rootDir>/<runId>/fetches.jsonl`   one line per URL ever fetched
- *   `<rootDir>/<runId>/citations.jsonl` one line per `recordCitation` call
- *
- * Both files are append-only, mode 0o600, atomic-per-line on POSIX
- * (mirrors `event-log`). On boot the tracker reads each file once into
- * an in-memory cache; subsequent calls hit memory.
- *
- * The smoke test's "byte-identical citation block across re-runs"
- * invariant is anchored here: citations are ordered by their first
- * append timestamp, NOT by retrievedAt, so even if the model takes
- * different routes on the second pass, the citation list still ends up
- * in stable order keyed off the FIRST-seen URL.
- */
-import { createHash, randomUUID } from "node:crypto";
-import { appendFileSync, existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
-import { join } from "node:path";
-import { CrewhausError, RuntimeError } from "@crewhaus/errors";
-
-export const DEFAULT_ROOT_DIR = ".crewhaus/research";
-const RUN_ID_RE = /^run_[0-9a-f]{16}$/;
-/**
- * A sha256 digest is always exactly 64 lowercase hex chars. We re-validate it
- * before interpolating into the cache file path because the value can originate
- * from a JSONL line read off disk (boot-time replay), and a corrupted or
- * tampered `fetches.jsonl` must not be able to steer a read outside `cache/`
- * (e.g. a `sha256` of "../../../etc/passwd"). Legitimate records always match.
- */
-const SHA256_RE = /^[0-9a-f]{64}$/;
-
-export type RunId = string;
-
-export type FetchRecord = {
-  readonly version: 1;
-  readonly url: string;
-  readonly retrievedAt: string;
-  readonly sha256: string;
-  readonly contentBytes: number;
-  /** Branch (sub-question) that initiated this fetch, if known. */
-  readonly branchId?: string;
-};
-
-export type Citation = {
-  readonly version: 1;
-  readonly url: string;
-  readonly snippet: string;
-  readonly retrievedAt: string;
-  readonly sha256: string;
-  readonly branchId?: string;
-  /** Free-form supporting-claim label (e.g., "supports: target shapes list"). */
-  readonly supportingClaim?: string;
-};
-
-export type RecordFetchInput = {
-  readonly url: string;
-  readonly content: string;
-  readonly branchId?: string;
-};
-
-export type RecordCitationInput = {
-  readonly url: string;
-  readonly snippet: string;
-  readonly branchId?: string;
-  readonly supportingClaim?: string;
-};
-
-export interface CitationTracker {
-  readonly runId: RunId;
-
-  /** True iff the tracker already has a content body for this URL. */
-  hasFetched(url: string): boolean;
-  /** Cached content body for a URL (matched by exact string). */
-  getFetchedContent(url: string): string | undefined;
-  /** Cached fetch metadata (sha256 + retrievedAt) for a URL. */
-  getFetchRecord(url: string): FetchRecord | undefined;
-  /** Idempotent: re-recording the same URL is a no-op (preserves first sha256). */
-  recordFetch(input: RecordFetchInput): FetchRecord;
-
-  recordCitation(input: RecordCitationInput): Citation;
-  /** Citations in append-order (deterministic across re-runs). */
-  listCitationsOrdered(): ReadonlyArray<Citation>;
-  /** All fetch records in append-order. */
-  listFetches(): ReadonlyArray<FetchRecord>;
-}
-
-export type CreateCitationTrackerOptions = {
-  readonly runId?: RunId;
-  readonly rootDir?: string;
-  readonly now?: () => Date;
-};
-
-function validateRunId(runId: string): void {
-  if (!RUN_ID_RE.test(runId)) {
-    throw new RuntimeError(`citation-tracker: invalid runId "${runId}" — expected run_<16 hex>`);
-  }
-}
-
-export function newRunId(): RunId {
-  // 16 hex chars. UUID is 32 hex (with dashes); strip dashes and take 16.
-  const id = randomUUID().replace(/-/g, "").slice(0, 16);
-  return `run_${id}`;
-}
-
-export class CitationTrackerError extends CrewhausError {
-  override readonly name = "CitationTrackerError";
-  constructor(message: string, cause?: unknown) {
-    super("runtime", message, cause);
-  }
-}
-
-export function createCitationTracker(opts: CreateCitationTrackerOptions = {}): CitationTracker {
-  const runId = opts.runId ?? newRunId();
-  validateRunId(runId);
-  const rootDir = opts.rootDir ?? DEFAULT_ROOT_DIR;
-  const runDir = join(rootDir, runId);
-  const fetchesPath = join(runDir, "fetches.jsonl");
-  const citationsPath = join(runDir, "citations.jsonl");
-  const now = opts.now ?? (() => new Date());
-
-  mkdirSync(runDir, { recursive: true });
-
-  // Boot-time replay of prior records so a resume sees the same dedup state.
-  const fetchByUrl = new Map<string, FetchRecord>();
-  const fetchOrder: FetchRecord[] = [];
-  const citations: Citation[] = [];
-
-  if (existsSync(fetchesPath)) {
-    for (const raw of readFileSync(fetchesPath, "utf8").split("\n")) {
-      const line = raw.trim();
-      if (line.length === 0) continue;
-      try {
-        const rec = JSON.parse(line) as FetchRecord;
-        if (typeof rec.url === "string" && !fetchByUrl.has(rec.url)) {
-          fetchByUrl.set(rec.url, rec);
-          fetchOrder.push(rec);
-        }
-      } catch (err) {
-        throw new CitationTrackerError(`malformed JSON in ${fetchesPath}`, err);
-      }
-    }
-  }
-  if (existsSync(citationsPath)) {
-    for (const raw of readFileSync(citationsPath, "utf8").split("\n")) {
-      const line = raw.trim();
-      if (line.length === 0) continue;
-      try {
-        const rec = JSON.parse(line) as Citation;
-        citations.push(rec);
-      } catch (err) {
-        throw new CitationTrackerError(`malformed JSON in ${citationsPath}`, err);
-      }
-    }
-  }
-
-  // Per-fetch content cache — written separately under <runDir>/cache/ so the
-  // JSONL stays small (citations only carry sha256 + snippet, not full bodies).
-  const cacheDir = join(runDir, "cache");
-  mkdirSync(cacheDir, { recursive: true });
-
-  function cachePathFor(sha256: string): string {
-    return join(cacheDir, `${sha256}.txt`);
-  }
-
-  function tryReadCache(rec: FetchRecord): string | undefined {
-    // Defend the cache read against a tampered fetches.jsonl: a sha256 that is
-    // not a clean 64-hex digest could otherwise traverse out of cache/.
-    if (typeof rec.sha256 !== "string" || !SHA256_RE.test(rec.sha256)) return undefined;
-    const p = cachePathFor(rec.sha256);
-    if (!existsSync(p)) return undefined;
-    return readFileSync(p, "utf8");
-  }
-
-  return {
-    runId,
-
-    hasFetched(url) {
-      return fetchByUrl.has(url);
-    },
-
-    getFetchedContent(url) {
-      const rec = fetchByUrl.get(url);
-      if (rec === undefined) return undefined;
-      return tryReadCache(rec);
-    },
-
-    getFetchRecord(url) {
-      return fetchByUrl.get(url);
-    },
-
-    recordFetch(input) {
-      const existing = fetchByUrl.get(input.url);
-      if (existing !== undefined) return existing;
-      const sha256 = createHash("sha256").update(input.content).digest("hex");
-      const rec: FetchRecord = {
-        version: 1,
-        url: input.url,
-        retrievedAt: now().toISOString(),
-        sha256,
-        contentBytes: Buffer.byteLength(input.content, "utf8"),
-        ...(input.branchId !== undefined ? { branchId: input.branchId } : {}),
-      };
-      fetchByUrl.set(input.url, rec);
-      fetchOrder.push(rec);
-      appendFileSync(fetchesPath, `${JSON.stringify(rec)}\n`, { mode: 0o600 });
-      // Cache content for resume by sha256 — same content across runs ends up
-      // as the same file, so a re-fetch into a different tracker still hits
-      // the same cache directory if the rootDir is shared.
-      writeFileSync(cachePathFor(sha256), input.content, { mode: 0o600 });
-      return rec;
-    },
-
-    recordCitation(input) {
-      // Resolve sha256: prefer the matching fetch record's hash so a citation
-      // anchors to a specific content version. If no matching fetch, hash
-      // the snippet itself.
-      const fetch = fetchByUrl.get(input.url);
-      const sha256 =
-        fetch !== undefined
-          ? fetch.sha256
-          : createHash("sha256").update(input.snippet).digest("hex");
-      const c: Citation = {
-        version: 1,
-        url: input.url,
-        snippet: input.snippet,
-        retrievedAt: fetch !== undefined ? fetch.retrievedAt : now().toISOString(),
-        sha256,
-        ...(input.branchId !== undefined ? { branchId: input.branchId } : {}),
-        ...(input.supportingClaim !== undefined ? { supportingClaim: input.supportingClaim } : {}),
-      };
-      citations.push(c);
-      appendFileSync(citationsPath, `${JSON.stringify(c)}\n`, { mode: 0o600 });
-      return c;
-    },
-
-    listCitationsOrdered() {
-      // Stable order: append order. Same-URL citations are kept (the agent
-      // may cite multiple snippets from one source); the report-writer's
-      // numbering layer dedups by URL.
-      return [...citations];
-    },
-
-    listFetches() {
-      return [...fetchOrder];
-    },
-  };
-}