@crewhaus/citation-tracker 0.1.0

package/package.json

@@ -0,0 +1,41 @@
+{
+  "name": "@crewhaus/citation-tracker",
+  "version": "0.1.0",
+  "type": "module",
+  "description": "Per-research-run citation persistence + URL dedup (Section 23 RES)",
+  "main": "src/index.ts",
+  "types": "src/index.ts",
+  "exports": {
+    ".": "./src/index.ts"
+  },
+  "scripts": {
+    "test": "bun test src"
+  },
+  "dependencies": {
+    "@crewhaus/errors": "0.0.0"
+  },
+  "license": "Apache-2.0",
+  "author": {
+    "name": "Max Meier",
+    "email": "max@studiomax.io",
+    "url": "https://studiomax.io"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/crewhaus/factory.git",
+    "directory": "packages/citation-tracker"
+  },
+  "homepage": "https://github.com/crewhaus/factory/tree/main/packages/citation-tracker#readme",
+  "bugs": {
+    "url": "https://github.com/crewhaus/factory/issues"
+  },
+  "publishConfig": {
+    "access": "restricted"
+  },
+  "files": [
+    "src",
+    "README.md",
+    "LICENSE",
+    "NOTICE"
+  ]
+}

package/src/index.test.ts

@@ -0,0 +1,139 @@
+import { describe, expect, test } from "bun:test";
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { CitationTrackerError, createCitationTracker, newRunId } from "./index.js";
+
+function newRoot(): string {
+  return mkdtempSync(join(tmpdir(), "citation-tracker-"));
+}
+
+describe("createCitationTracker", () => {
+  test("hasFetched returns false for fresh tracker, true after recordFetch", () => {
+    const root = newRoot();
+    try {
+      const t = createCitationTracker({ rootDir: root });
+      expect(t.hasFetched("https://example.com/a")).toBe(false);
+      t.recordFetch({ url: "https://example.com/a", content: "hello world" });
+      expect(t.hasFetched("https://example.com/a")).toBe(true);
+    } finally {
+      rmSync(root, { recursive: true, force: true });
+    }
+  });
+
+  test("recordFetch is idempotent — re-recording same URL returns first record", () => {
+    const root = newRoot();
+    try {
+      const t = createCitationTracker({ rootDir: root });
+      const first = t.recordFetch({ url: "u", content: "v1" });
+      const second = t.recordFetch({ url: "u", content: "v2-different" });
+      // First write wins; the second record is the same as the first.
+      expect(second).toEqual(first);
+      expect(t.getFetchedContent("u")).toBe("v1");
+    } finally {
+      rmSync(root, { recursive: true, force: true });
+    }
+  });
+
+  test("getFetchedContent returns the original body verbatim (T4 cache hit)", () => {
+    const root = newRoot();
+    try {
+      const t = createCitationTracker({ rootDir: root });
+      const body = "some\nmulti\nline\ncontent";
+      t.recordFetch({ url: "u", content: body });
+      expect(t.getFetchedContent("u")).toBe(body);
+    } finally {
+      rmSync(root, { recursive: true, force: true });
+    }
+  });
+
+  test("kill-and-resume: a new tracker against the same runId/rootDir picks up prior fetches (T4)", () => {
+    const root = newRoot();
+    const runId = newRunId();
+    try {
+      const t1 = createCitationTracker({ runId, rootDir: root });
+      t1.recordFetch({ url: "u1", content: "body-1" });
+      t1.recordCitation({ url: "u1", snippet: "snippet-1", branchId: "b1" });
+
+      // Simulate kill by dropping t1; new tracker reads files back.
+      const t2 = createCitationTracker({ runId, rootDir: root });
+      expect(t2.hasFetched("u1")).toBe(true);
+      expect(t2.getFetchedContent("u1")).toBe("body-1");
+      const cs = t2.listCitationsOrdered();
+      expect(cs).toHaveLength(1);
+      expect(cs[0]?.url).toBe("u1");
+      expect(cs[0]?.snippet).toBe("snippet-1");
+    } finally {
+      rmSync(root, { recursive: true, force: true });
+    }
+  });
+
+  test("listCitationsOrdered returns citations in append order — deterministic across re-runs", () => {
+    const root = newRoot();
+    const runId = newRunId();
+    try {
+      const t = createCitationTracker({ runId, rootDir: root });
+      t.recordFetch({ url: "a", content: "ka" });
+      t.recordFetch({ url: "b", content: "kb" });
+      t.recordCitation({ url: "a", snippet: "fact a-1" });
+      t.recordCitation({ url: "b", snippet: "fact b-1" });
+      t.recordCitation({ url: "a", snippet: "fact a-2" });
+
+      const reopened = createCitationTracker({ runId, rootDir: root });
+      const order = reopened.listCitationsOrdered().map((c) => c.snippet);
+      expect(order).toEqual(["fact a-1", "fact b-1", "fact a-2"]);
+    } finally {
+      rmSync(root, { recursive: true, force: true });
+    }
+  });
+
+  test("recordCitation pulls sha256 from the matching fetch record when present", () => {
+    const root = newRoot();
+    try {
+      const t = createCitationTracker({ rootDir: root });
+      const f = t.recordFetch({ url: "u", content: "the body text" });
+      const c = t.recordCitation({ url: "u", snippet: "body" });
+      expect(c.sha256).toBe(f.sha256);
+      expect(c.retrievedAt).toBe(f.retrievedAt);
+    } finally {
+      rmSync(root, { recursive: true, force: true });
+    }
+  });
+
+  test("recordCitation with no prior fetch hashes the snippet itself", () => {
+    const root = newRoot();
+    try {
+      const t = createCitationTracker({ rootDir: root });
+      const c = t.recordCitation({ url: "u", snippet: "abc" });
+      expect(c.sha256).toMatch(/^[0-9a-f]{64}$/);
+    } finally {
+      rmSync(root, { recursive: true, force: true });
+    }
+  });
+
+  test("invalid runId throws", () => {
+    const root = newRoot();
+    try {
+      expect(() => createCitationTracker({ runId: "bad", rootDir: root })).toThrow(/invalid runId/);
+    } finally {
+      rmSync(root, { recursive: true, force: true });
+    }
+  });
+
+  test("malformed jsonl on disk surfaces as a CitationTrackerError", async () => {
+    const root = newRoot();
+    const runId = newRunId();
+    const path = join(root, runId, "fetches.jsonl");
+    try {
+      // Create the dir + a malformed file.
+      const t = createCitationTracker({ runId, rootDir: root });
+      t.recordFetch({ url: "u", content: "ok" });
+      // Append garbage and re-open.
+      const fs = await import("node:fs");
+      fs.appendFileSync(path, "not-json\n");
+      expect(() => createCitationTracker({ runId, rootDir: root })).toThrow(CitationTrackerError);
+    } finally {
+      rmSync(root, { recursive: true, force: true });
+    }
+  });
+});

package/src/index.ts

@@ -0,0 +1,251 @@
+/**
+ * Catalog R-orch `citation-tracker` — Section 23 RES.
+ *
+ * Persistent per-research-run record of every URL the crawler has
+ * touched, plus every fact the agent has cited. The same instance does
+ * two jobs that the kickoff treats as one:
+ *
+ *   1. **Fetch dedup** — `hasFetched(url)` lets the crawler skip a HTTP
+ *      round-trip on resume. `recordFetch({url, content})` stores the
+ *      sha256 + the content body so a kill-and-resume returns identical
+ *      bytes from cache rather than re-pulling the URL.
+ *   2. **Citation registry** — `recordCitation({url, snippet, branchId?})`
+ *      appends a fact to the run's citation log. `listCitationsOrdered()`
+ *      returns citations in first-seen order (deterministic across
+ *      re-runs because of the JSONL append-only file).
+ *
+ * Storage layout:
+ *   `<rootDir>/<runId>/fetches.jsonl`   one line per URL ever fetched
+ *   `<rootDir>/<runId>/citations.jsonl` one line per `recordCitation` call
+ *
+ * Both files are append-only, mode 0o600, atomic-per-line on POSIX
+ * (mirrors `event-log`). On boot the tracker reads each file once into
+ * an in-memory cache; subsequent calls hit memory.
+ *
+ * The smoke test's "byte-identical citation block across re-runs"
+ * invariant is anchored here: citations are ordered by their first
+ * append timestamp, NOT by retrievedAt, so even if the model takes
+ * different routes on the second pass, the citation list still ends up
+ * in stable order keyed off the FIRST-seen URL.
+ */
+import { createHash, randomUUID } from "node:crypto";
+import { appendFileSync, existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { CrewhausError, RuntimeError } from "@crewhaus/errors";
+
+export const DEFAULT_ROOT_DIR = ".crewhaus/research";
+const RUN_ID_RE = /^run_[0-9a-f]{16}$/;
+
+export type RunId = string;
+
+export type FetchRecord = {
+  readonly version: 1;
+  readonly url: string;
+  readonly retrievedAt: string;
+  readonly sha256: string;
+  readonly contentBytes: number;
+  /** Branch (sub-question) that initiated this fetch, if known. */
+  readonly branchId?: string;
+};
+
+export type Citation = {
+  readonly version: 1;
+  readonly url: string;
+  readonly snippet: string;
+  readonly retrievedAt: string;
+  readonly sha256: string;
+  readonly branchId?: string;
+  /** Free-form supporting-claim label (e.g., "supports: target shapes list"). */
+  readonly supportingClaim?: string;
+};
+
+export type RecordFetchInput = {
+  readonly url: string;
+  readonly content: string;
+  readonly branchId?: string;
+};
+
+export type RecordCitationInput = {
+  readonly url: string;
+  readonly snippet: string;
+  readonly branchId?: string;
+  readonly supportingClaim?: string;
+};
+
+export interface CitationTracker {
+  readonly runId: RunId;
+
+  /** True iff the tracker already has a content body for this URL. */
+  hasFetched(url: string): boolean;
+  /** Cached content body for a URL (matched by exact string). */
+  getFetchedContent(url: string): string | undefined;
+  /** Cached fetch metadata (sha256 + retrievedAt) for a URL. */
+  getFetchRecord(url: string): FetchRecord | undefined;
+  /** Idempotent: re-recording the same URL is a no-op (preserves first sha256). */
+  recordFetch(input: RecordFetchInput): FetchRecord;
+
+  recordCitation(input: RecordCitationInput): Citation;
+  /** Citations in append-order (deterministic across re-runs). */
+  listCitationsOrdered(): ReadonlyArray<Citation>;
+  /** All fetch records in append-order. */
+  listFetches(): ReadonlyArray<FetchRecord>;
+}
+
+export type CreateCitationTrackerOptions = {
+  readonly runId?: RunId;
+  readonly rootDir?: string;
+  readonly now?: () => Date;
+};
+
+function validateRunId(runId: string): void {
+  if (!RUN_ID_RE.test(runId)) {
+    throw new RuntimeError(`citation-tracker: invalid runId "${runId}" — expected run_<16 hex>`);
+  }
+}
+
+export function newRunId(): RunId {
+  // 16 hex chars. UUID is 32 hex (with dashes); strip dashes and take 16.
+  const id = randomUUID().replace(/-/g, "").slice(0, 16);
+  return `run_${id}`;
+}
+
+export class CitationTrackerError extends CrewhausError {
+  override readonly name = "CitationTrackerError";
+  constructor(message: string, cause?: unknown) {
+    super("runtime", message, cause);
+  }
+}
+
+export function createCitationTracker(opts: CreateCitationTrackerOptions = {}): CitationTracker {
+  const runId = opts.runId ?? newRunId();
+  validateRunId(runId);
+  const rootDir = opts.rootDir ?? DEFAULT_ROOT_DIR;
+  const runDir = join(rootDir, runId);
+  const fetchesPath = join(runDir, "fetches.jsonl");
+  const citationsPath = join(runDir, "citations.jsonl");
+  const now = opts.now ?? (() => new Date());
+
+  mkdirSync(runDir, { recursive: true });
+
+  // Boot-time replay of prior records so a resume sees the same dedup state.
+  const fetchByUrl = new Map<string, FetchRecord>();
+  const fetchOrder: FetchRecord[] = [];
+  const citations: Citation[] = [];
+
+  if (existsSync(fetchesPath)) {
+    for (const raw of readFileSync(fetchesPath, "utf8").split("\n")) {
+      const line = raw.trim();
+      if (line.length === 0) continue;
+      try {
+        const rec = JSON.parse(line) as FetchRecord;
+        if (typeof rec.url === "string" && !fetchByUrl.has(rec.url)) {
+          fetchByUrl.set(rec.url, rec);
+          fetchOrder.push(rec);
+        }
+      } catch (err) {
+        throw new CitationTrackerError(`malformed JSON in ${fetchesPath}`, err);
+      }
+    }
+  }
+  if (existsSync(citationsPath)) {
+    for (const raw of readFileSync(citationsPath, "utf8").split("\n")) {
+      const line = raw.trim();
+      if (line.length === 0) continue;
+      try {
+        const rec = JSON.parse(line) as Citation;
+        citations.push(rec);
+      } catch (err) {
+        throw new CitationTrackerError(`malformed JSON in ${citationsPath}`, err);
+      }
+    }
+  }
+
+  // Per-fetch content cache — written separately under <runDir>/cache/ so the
+  // JSONL stays small (citations only carry sha256 + snippet, not full bodies).
+  const cacheDir = join(runDir, "cache");
+  mkdirSync(cacheDir, { recursive: true });
+
+  function cachePathFor(sha256: string): string {
+    return join(cacheDir, `${sha256}.txt`);
+  }
+
+  function tryReadCache(rec: FetchRecord): string | undefined {
+    const p = cachePathFor(rec.sha256);
+    if (!existsSync(p)) return undefined;
+    return readFileSync(p, "utf8");
+  }
+
+  return {
+    runId,
+
+    hasFetched(url) {
+      return fetchByUrl.has(url);
+    },
+
+    getFetchedContent(url) {
+      const rec = fetchByUrl.get(url);
+      if (rec === undefined) return undefined;
+      return tryReadCache(rec);
+    },
+
+    getFetchRecord(url) {
+      return fetchByUrl.get(url);
+    },
+
+    recordFetch(input) {
+      const existing = fetchByUrl.get(input.url);
+      if (existing !== undefined) return existing;
+      const sha256 = createHash("sha256").update(input.content).digest("hex");
+      const rec: FetchRecord = {
+        version: 1,
+        url: input.url,
+        retrievedAt: now().toISOString(),
+        sha256,
+        contentBytes: Buffer.byteLength(input.content, "utf8"),
+        ...(input.branchId !== undefined ? { branchId: input.branchId } : {}),
+      };
+      fetchByUrl.set(input.url, rec);
+      fetchOrder.push(rec);
+      appendFileSync(fetchesPath, `${JSON.stringify(rec)}\n`, { mode: 0o600 });
+      // Cache content for resume by sha256 — same content across runs ends up
+      // as the same file, so a re-fetch into a different tracker still hits
+      // the same cache directory if the rootDir is shared.
+      writeFileSync(cachePathFor(sha256), input.content, { mode: 0o600 });
+      return rec;
+    },
+
+    recordCitation(input) {
+      // Resolve sha256: prefer the matching fetch record's hash so a citation
+      // anchors to a specific content version. If no matching fetch, hash
+      // the snippet itself.
+      const fetch = fetchByUrl.get(input.url);
+      const sha256 =
+        fetch !== undefined
+          ? fetch.sha256
+          : createHash("sha256").update(input.snippet).digest("hex");
+      const c: Citation = {
+        version: 1,
+        url: input.url,
+        snippet: input.snippet,
+        retrievedAt: fetch !== undefined ? fetch.retrievedAt : now().toISOString(),
+        sha256,
+        ...(input.branchId !== undefined ? { branchId: input.branchId } : {}),
+        ...(input.supportingClaim !== undefined ? { supportingClaim: input.supportingClaim } : {}),
+      };
+      citations.push(c);
+      appendFileSync(citationsPath, `${JSON.stringify(c)}\n`, { mode: 0o600 });
+      return c;
+    },
+
+    listCitationsOrdered() {
+      // Stable order: append order. Same-URL citations are kept (the agent
+      // may cite multiple snippets from one source); the report-writer's
+      // numbering layer dedups by URL.
+      return [...citations];
+    },
+
+    listFetches() {
+      return [...fetchOrder];
+    },
+  };
+}