@staix/agent-research 0.0.0-stage → 0.2.1

package/src/exp/snapshot.ts

@@ -0,0 +1,147 @@
+import { createHash } from "node:crypto";
+import { lstatSync, mkdirSync, mkdtempSync, opendirSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { dirname, join, posix } from "node:path";
+import { runOnExecutor, spawnRunner, validateExecutor, type Executor, type ProcessResult, type ProcessRunner } from "./executor.ts";
+import { TASK_BYTES, TASK_FILES } from "./protocol.ts";
+import { readBytesBounded, sha256 } from "../study.ts";
+
+export const SNAPSHOT_BYTES = 256 * 1024 * 1024;
+function fullCommit(value: string): string {
+  if (!/^[a-fA-F0-9]{40}$/.test(value)) throw new Error("snapshot commit: full 40-hex SHA required; pass the commit SHA from git rev-parse HEAD");
+  return value.toLowerCase();
+}
+function repository(value: string): string {
+  if (!value || value.length > 4096 || value.includes("\0")) throw new Error("snapshot repo: invalid path; pass a local Git repository path of 1..4096 characters without NUL");
+  return value;
+}
+function passed(result: ProcessResult): boolean { return result.exitCode === 0 && !result.timedOut && !result.truncated; }
+function gitFailure(step: string): Error { return new Error(`snapshot Git ${step} failed; check --repo and that --commit names an existing commit, then retry ares attempt run`); }
+
+/** One pending chunk, with explicit cancellation instead of stream callback ordering. */
+class ArchiveChannel implements AsyncIterableIterator<Uint8Array> {
+  private pending?: { chunk: Uint8Array; resolve: () => void; reject: (error: Error) => void };
+  private waiting?: { resolve: (value: IteratorResult<Uint8Array>) => void; reject: (error: Error) => void };
+  private ended = false;
+  private failure?: Error;
+  [Symbol.asyncIterator](): AsyncIterableIterator<Uint8Array> { return this; }
+  write(chunk: Uint8Array): Promise<void> {
+    if (this.failure || this.ended) return Promise.reject(this.failure ?? new Error("archive channel closed"));
+    if (this.waiting) { const waiting = this.waiting; this.waiting = undefined; waiting.resolve({ value: chunk, done: false }); return Promise.resolve(); }
+    return new Promise((resolve, reject) => { this.pending = { chunk, resolve, reject }; });
+  }
+  next(): Promise<IteratorResult<Uint8Array>> {
+    if (this.pending) { const pending = this.pending; this.pending = undefined; pending.resolve(); return Promise.resolve({ value: pending.chunk, done: false }); }
+    if (this.failure) return Promise.reject(this.failure);
+    if (this.ended) return Promise.resolve({ value: undefined, done: true });
+    return new Promise((resolve, reject) => { this.waiting = { resolve, reject }; });
+  }
+  end(error?: Error): void {
+    if (this.ended) return;
+    this.ended = true; this.failure = error;
+    if (this.pending) { this.pending.reject(error ?? new Error("archive channel closed")); this.pending = undefined; }
+    if (this.waiting) { if (error) this.waiting.reject(error); else this.waiting.resolve({ value: undefined, done: true }); this.waiting = undefined; }
+  }
+  return(): Promise<IteratorResult<Uint8Array>> { this.end(new Error("archive consumer stopped")); return Promise.resolve({ value: undefined, done: true }); }
+}
+
+/** Resolve only an existing commit object, independent of the worktree or an annotated tag. */
+export async function resolveSnapshot(repoArg: string, commitArg: string, runner: ProcessRunner = spawnRunner): Promise<{ commit: string; treeId: string }> {
+  const repo = repository(repoArg), commit = fullCommit(commitArg);
+  let type: ProcessResult, tree: ProcessResult;
+  try {
+    type = await runner({ command: "git", args: ["-C", repo, "cat-file", "-t", commit] });
+    if (!passed(type) || type.stdout.trim() !== "commit") throw gitFailure("commit validation");
+  } catch { throw gitFailure("commit validation"); }
+  try { tree = await runner({ command: "git", args: ["-C", repo, "rev-parse", "--verify", `${commit}^{tree}`] }); }
+  catch { throw gitFailure("tree resolution"); }
+  const treeId = tree.stdout.trim();
+  if (!passed(tree) || !/^[a-f0-9]{40}$/.test(treeId)) throw gitFailure("tree resolution");
+  return { commit, treeId };
+}
+
+/** Stream raw archive bytes with backpressure; nothing buffers or decodes the whole archive. */
+export async function transferSnapshot(repoArg: string, commitArg: string, executorArg: Executor, destination: string, runner: ProcessRunner = spawnRunner): Promise<{ sha256: string; bytes: number }> {
+  const repo = repository(repoArg), commit = fullCommit(commitArg);
+  return transferArchive("snapshot", { command: "git", args: ["-C", repo, "archive", "--format=tar", commit] }, SNAPSHOT_BYTES, executorArg, destination, runner);
+}
+
+/** Capture each file once; the hash binds the same bytes copied into private immutable evaluator staging. */
+export async function transferTask(taskDir: string, executor: Executor, destination: string, runner: ProcessRunner = spawnRunner, expectedTreeHash?: string): Promise<{ sha256: string; bytes: number; treeHash: string }> {
+  repository(taskDir);
+  if (expectedTreeHash !== undefined && !/^[a-f0-9]{64}$/.test(expectedTreeHash)) throw new Error("task tree hash: invalid; use the approved taskTreeHash before transferring evaluator files");
+  const staging = mkdtempSync(join(tmpdir(), "ares-task-")), entries: [string, string][] = [];
+  let total = 0, directories = 0;
+  try {
+    const evaluator = lstatSync(join(taskDir, "evaluator"));
+    if (!evaluator.isDirectory() || evaluator.isSymbolicLink()) throw new Error("task/evaluator: regular directory required; restore evaluator files before ares attempt run");
+    mkdirSync(join(staging, "evaluator"), { mode: 0o700 });
+    function capture(relative: string, depth: number): void {
+      const source = relative ? join(taskDir, relative) : taskDir, label = relative ? `task/${relative}` : "task/", stat = lstatSync(source);
+      if (stat.isSymbolicLink()) throw new Error(`${label}: symlink refused; replace it with regular task files before ares attempt run`);
+      if (stat.isDirectory()) {
+        if (++directories > 1000 || depth > 64) throw new Error(`${label}: task traversal limit exceeded; reduce task/ to 1000 directories and 64 levels`);
+        if (relative.startsWith("evaluator/")) mkdirSync(join(staging, relative), { recursive: true, mode: 0o700 });
+        const directory = opendirSync(source);
+        try { for (let entry = directory.readSync(); entry; entry = directory.readSync()) capture(relative ? `${relative}/${entry.name}` : entry.name, depth + 1); }
+        finally { directory.closeSync(); }
+      } else {
+        if (!stat.isFile()) throw new Error(`${label}: regular file required; remove special files before ares attempt run`);
+        if (entries.length >= TASK_FILES || stat.size > TASK_BYTES - total) throw new Error("task/: task size limit exceeded; reduce task/ to 500 files and 64 MiB before ares attempt run");
+        let content: Buffer;
+        try { content = readBytesBounded(source, TASK_BYTES - total); }
+        catch { throw new Error(`${label}: bounded read failed; restore a readable regular file within the 64 MiB task limit`); }
+        total += content.length; entries.push([relative, sha256(content)]);
+        if (relative.startsWith("evaluator/")) {
+          const target = join(staging, relative); mkdirSync(dirname(target), { recursive: true, mode: 0o700 });
+          writeFileSync(target, content, { mode: stat.mode & 0o777 });
+        }
+      }
+    }
+    capture("", 0);
+    const treeHash = sha256(JSON.stringify(entries.sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0)));
+    if (expectedTreeHash !== undefined && treeHash !== expectedTreeHash) throw new Error("task/: content changed since approval; set the protocol, approve design again, and retry ares attempt run");
+    // Bounded tar headers and long-path metadata supplement the 64 MiB task payload cap.
+    const transferred = await transferArchive("task", { command: "tar", args: ["-C", staging, "-cf", "-", "evaluator"] }, TASK_BYTES + 16 * 1024 * 1024, executor, destination, runner);
+    return { ...transferred, treeHash };
+  } catch (error) {
+    if (error instanceof Error && (error.message.startsWith("task/") || error.message.startsWith("executor "))) throw error;
+    throw new Error("task archive capture failed; restore readable task/evaluator files, then retry ares attempt run");
+  } finally { rmSync(staging, { recursive: true, force: true }); }
+}
+
+async function transferArchive(kind: "snapshot" | "task", source: { command: string; args: string[] }, limit: number, executorArg: Executor, destination: string, runner: ProcessRunner): Promise<{ sha256: string; bytes: number }> {
+  const executor = validateExecutor(executorArg);
+  if (!destination.startsWith("/") || destination.length > 4096 || /[\0\r\n,"]/.test(destination) || posix.normalize(destination) === "/") throw new Error("snapshot destination: invalid directory; use a private absolute non-root directory without NUL, newlines, commas or double quotes");
+  const stream = new ArchiveChannel(), controller = new AbortController(), hash = createHash("sha256");
+  let bytes = 0, oversized = false, failedSide: "archive" | "extract" | undefined;
+  const archive = (async () => {
+    try {
+      const result = await runner({ ...source, signal: controller.signal, timeoutMs: 300_000,
+        stdoutSink: async chunk => {
+          if (bytes + chunk.byteLength > limit) { oversized = true; throw new Error("archive size limit"); }
+          bytes += chunk.byteLength; hash.update(chunk);
+          await stream.write(chunk);
+        } });
+      if (!passed(result)) throw gitFailure("archive");
+      stream.end();
+    } catch (error) { failedSide ??= "archive"; stream.end(error as Error); throw error; }
+  })();
+  const extract = (async () => {
+    try {
+      const result = await runOnExecutor(executor, "tar", ["--no-same-owner", "-xf", "-", "-C", destination], runner, { stdin: stream, signal: controller.signal, timeoutMs: 300_000 });
+      if (!passed(result)) throw new Error("archive extraction failed");
+    } catch (error) { failedSide ??= "extract"; throw error; }
+  })();
+  try {
+    await Promise.all([archive, extract]);
+    if (bytes === 0) { failedSide = "archive"; throw gitFailure("empty archive"); }
+    return { sha256: hash.digest("hex"), bytes };
+  } catch {
+    controller.abort(); stream.end(new Error("archive transfer cancelled"));
+    await Promise.allSettled([archive, extract]);
+    if (oversized) throw new Error(`${kind} archive: larger than ${limit} bytes; reduce the ${kind === "snapshot" ? "committed snapshot" : "task evaluator"} before ares attempt run`);
+    if (failedSide === "archive") throw kind === "snapshot" ? gitFailure("archive") : new Error("task archive failed; restore readable task/evaluator files, then retry ares attempt run");
+    throw new Error(`executor ${executor.kind === "ssh" ? executor.host : "local"}: snapshot extraction failed; check the private destination directory and tar on the executor host, then retry ares attempt run`);
+  }
+}

package/src/http.ts

@@ -0,0 +1,106 @@
+import { version } from "../package.json";
+
+export type FetchLike = (url: string, init?: { headers?: Record<string, string>; signal?: AbortSignal }) => Promise<Response>;
+export type HttpErrorKind = "network" | "rate" | "server" | "client" | "format";
+/** Errors name the host and status only: request URLs can carry API keys. */
+export class HttpError extends Error {
+  constructor(readonly kind: HttpErrorKind, readonly host: string, readonly status?: number) { super(`${host}: ${kind}${status ? ` ${status}` : ""}`); }
+}
+export interface HttpResult<T> { status: number; body: T | undefined }
+export interface Http {
+  json(url: string, headers?: Record<string, string>): Promise<HttpResult<unknown>>;
+  text(url: string, headers?: Record<string, string>): Promise<HttpResult<string>>;
+  env: NodeJS.ProcessEnv;
+}
+const TIMEOUT_MS = 20_000, MAX_BYTES = 5 * 1024 * 1024;
+const EMAIL = /^[^\s@<>()]{1,64}@[^\s@<>()]{1,190}$/;
+
+export const S2_START_INTERVAL_MS = 1_100;
+export interface HttpPacingClock {
+  now(): number;
+  sleep(ms: number): Promise<void>;
+}
+export interface HttpPacing {
+  start<T>(request: () => Promise<T>): Promise<T>;
+}
+
+/** Serialize admissions only: an in-flight response never holds the next start. */
+export function createHttpPacing(clock: HttpPacingClock = {
+  now: () => performance.now(),
+  sleep: ms => new Promise(resolve => setTimeout(resolve, ms)),
+}): HttpPacing {
+  let tail = Promise.resolve(), lastStart: number | undefined;
+  return {
+    start<T>(request: () => Promise<T>): Promise<T> {
+      const admitted = tail.then(async () => {
+        if (lastStart !== undefined) {
+          let remaining = S2_START_INTERVAL_MS - (clock.now() - lastStart);
+          while (remaining > 0) {
+            await clock.sleep(remaining);
+            remaining = S2_START_INTERVAL_MS - (clock.now() - lastStart);
+          }
+        }
+        // Measure after invocation, conservatively spacing actual fetch starts.
+        let response: Promise<T>;
+        try { response = request(); } finally { lastStart = clock.now(); }
+        // Box the promise so admission releases without waiting for the response.
+        // A later admission must not delay handling a rejected response.
+        void response.catch(() => {});
+        return { response };
+      });
+      tail = admitted.then(() => {}, () => {});
+      return admitted.then(({ response }) => response);
+    },
+  };
+}
+
+const processS2Pacing = createHttpPacing();
+
+/** Read the body while counting bytes; past the limit the stream is cancelled, never buffered whole. */
+async function readBounded(response: Response, host: string): Promise<string> {
+  if (!response.body) return "";
+  const reader = response.body.getReader(), chunks: Uint8Array[] = [];
+  let size = 0;
+  try {
+    for (;;) {
+      const { done, value } = await reader.read();
+      if (done) break;
+      size += value.byteLength;
+      if (size > MAX_BYTES) { await reader.cancel().catch(() => {}); throw new HttpError("format", host); }
+      chunks.push(value);
+    }
+  } catch (error) { if (error instanceof HttpError) throw error; throw new HttpError("network", host); }
+  try { return new TextDecoder("utf-8", { fatal: true }).decode(Buffer.concat(chunks)); } catch { throw new HttpError("format", host); }
+}
+
+export function userAgent(env: NodeJS.ProcessEnv): string {
+  const email = env.ARES_CONTACT_EMAIL?.trim();
+  return `agent-research/${version}${email && EMAIL.test(email) ? ` (mailto:${email})` : ""}`;
+}
+
+/** 200 returns the body, 404 returns `body: undefined`; 429, 5xx, other statuses and network failures throw HttpError. */
+export function createHttp(env: NodeJS.ProcessEnv = process.env, fetchImpl: FetchLike = fetch as FetchLike, pacing: HttpPacing = processS2Pacing): Http {
+  async function get(url: string, headers: Record<string, string> = {}): Promise<HttpResult<string>> {
+    const host = new URL(url).host;
+    let response: Response;
+    try {
+      const request = () => fetchImpl(url, { headers: { "user-agent": userAgent(env), ...headers }, signal: AbortSignal.timeout(TIMEOUT_MS) });
+      response = await (new URL(url).hostname === "api.semanticscholar.org" ? pacing.start(request) : request());
+    }
+    catch { throw new HttpError("network", host); }
+    if (response.status === 404) return { status: 404, body: undefined };
+    if (response.status === 429) throw new HttpError("rate", host, 429);
+    if (response.status >= 500) throw new HttpError("server", host, response.status);
+    if (response.status !== 200) throw new HttpError("client", host, response.status);
+    return { status: 200, body: await readBounded(response, host) };
+  }
+  return {
+    env,
+    text: get,
+    async json(url, headers) {
+      const result = await get(url, headers);
+      if (result.body === undefined) return { status: result.status, body: undefined };
+      try { return { status: 200, body: JSON.parse(result.body) }; } catch { throw new HttpError("format", new URL(url).host); }
+    },
+  };
+}

package/src/index.ts

@@ -0,0 +1,4 @@
+export * as ares from "./commands.ts";
+export { createHttp, HttpError, type Http, type FetchLike } from "./http.ts";
+export { detectActor, type Actor, type Study, type LedgerEvent, type GateState } from "./study.ts";
+export type { Candidate, Verification, Registry } from "./lit/candidates.ts";

package/src/lit/bibtex.ts

@@ -0,0 +1,61 @@
+import { isVerified, type Candidate } from "./candidates.ts";
+
+const STOP = new Set(["the", "a", "an", "on", "of", "for", "and", "in", "to", "with", "towards", "toward", "from", "via", "by", "at", "is", "are"]);
+const ascii = (text: string) => text.normalize("NFKD").replace(/[\u0300-\u036f]/g, "").replace(/[^A-Za-z]/g, "").toLowerCase();
+
+/** "Family, Given" stays; "Given Family" becomes "Family, Given". */
+export function bibName(name: string): string {
+  if (name.includes(",")) return name.trim();
+  const parts = name.trim().split(/\s+/);
+  return parts.length < 2 ? parts[0] ?? "" : `${parts.at(-1)}, ${parts.slice(0, -1).join(" ")}`;
+}
+
+export function baseKey(c: Candidate): string {
+  const meta = c.verify?.registry, first = (meta?.authors ?? c.authors)[0];
+  const family = first ? ascii(bibName(first).split(",")[0]!) : "";
+  const year = meta?.year ?? c.year;
+  const word = (meta?.title ?? c.title).split(/\s+/).map(ascii).find(w => w.length >= 3 && !STOP.has(w)) ?? "";
+  const key = `${family}${year ?? ""}${word}`;
+  return /[a-z]/.test(key) ? key : `ref${year ?? ""}`;
+}
+
+/** Assign keys once, in id order, with a, b, ... suffixes on collision. Assigned keys never change. */
+export function assignCitekeys(list: Candidate[], eligible: Candidate[]): number {
+  const taken = new Set(list.map(c => c.citekey).filter((k): k is string => !!k));
+  let assigned = 0;
+  for (const c of [...eligible].sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0))) {
+    if (c.citekey) continue;
+    const base = baseKey(c);
+    let key = base;
+    for (let n = 0; taken.has(key); n++) key = `${base}${n < 26 ? String.fromCharCode(97 + n) : n}`;
+    c.citekey = key; taken.add(key); assigned++;
+  }
+  return assigned;
+}
+
+export function escapeBib(text: string): string {
+  // A private-use placeholder keeps the backslash replacement from being escaped again.
+  return text.replace(/\uE000/g, "").replace(/\\/g, "\uE000").replace(/([{}&%$#_])/g, "\\$1").replace(/~/g, "\\textasciitilde{}").replace(/\^/g, "\\textasciicircum{}").replace(/\uE000/g, "\\textbackslash{}");
+}
+
+export function bibEntry(c: Candidate): string {
+  const meta = c.verify?.registry, type = meta?.type ?? "";
+  const kind = type === "journal-article" ? "article" : type === "proceedings-article" ? "inproceedings" : "misc";
+  const fields: [string, string][] = [];
+  const authors = meta?.authors.length ? meta.authors : c.authors;
+  if (authors.length) fields.push(["author", authors.map(a => escapeBib(bibName(a))).join(" and ")]);
+  fields.push(["title", `{${escapeBib(meta?.title ?? c.title)}}`]);
+  const year = meta?.year ?? c.year;
+  if (year !== undefined) fields.push(["year", String(year)]);
+  const venue = meta?.venue ?? c.venue;
+  if (venue) fields.push([kind === "article" ? "journal" : kind === "inproceedings" ? "booktitle" : "publisher", escapeBib(venue)]);
+  if (c.doi) fields.push(["doi", escapeBib(c.doi)]);
+  if (c.arxivId) { fields.push(["eprint", escapeBib(c.arxivId)], ["archiveprefix", "arXiv"]); if (!c.doi) fields.push(["url", `https://arxiv.org/abs/${c.arxivId}`]); }
+  return `@${kind}{${c.citekey},\n${fields.map(([k, v]) => `  ${k} = {${v}}`).join(",\n")}\n}\n`;
+}
+
+/** Only included and verified candidates are written. */
+export function renderBib(list: Candidate[]): { text: string; entries: number } {
+  const eligible = list.filter(c => c.status === "included" && isVerified(c) && c.citekey).sort((a, b) => (a.citekey! < b.citekey! ? -1 : 1));
+  return { text: `% Generated by ares (agent-research). Do not edit: rerun ares lit bib.\n\n${eligible.map(bibEntry).join("\n")}`, entries: eligible.length };
+}

package/src/lit/candidates.ts

@@ -0,0 +1,148 @@
+import { existsSync } from "node:fs";
+import { join } from "node:path";
+import { plain, readJsonFile, sha256, writeAtomic, type Actor, type FileWrite } from "../study.ts";
+
+export interface Registry { title: string; authors: string[]; year?: number; venue?: string; type?: string }
+export interface Verification {
+  syntax: "ok" | "none" | "bad"; exists: "yes" | "no" | "unknown"; match: "yes" | "partial" | "no" | "unknown";
+  via?: "crossref" | "datacite" | "arxiv"; checkedAt: string; error?: string; registry?: Registry;
+  /** Hash of the identifiers, title and year that were checked; see verifySubject. */
+  subject?: string;
+}
+export interface RawCandidate { title: string; authors: string[]; year?: number; venue?: string; doi?: string; arxivId?: string; url?: string; abstract?: string; source: string }
+export interface Candidate extends Omit<RawCandidate, "source"> {
+  id: string; sources: string[]; addedAt: string;
+  status: "new" | "included" | "excluded"; reason?: string; decidedBy?: Actor; decidedAt?: string;
+  citekey?: string; verify?: Verification;
+  /** Other candidates that a record linked to this one but could not be absorbed (screened or keyed). */
+  duplicates?: string[];
+}
+
+const ABSTRACT_MAX = 2000, TITLE_MAX = 1000, AUTHORS_MAX = 100, AUTHOR_MAX = 300, CANDIDATES_BYTES = 64 * 1024 * 1024;
+const ARXIV_DOI = /^10\.48550\/arxiv\.(.+)$/i;
+
+export function normTitle(title: string): string {
+  return title.normalize("NFKC").toLowerCase().replace(/[^\p{L}\p{N}]+/gu, " ").trim();
+}
+export function normDoi(doi: string | undefined): string | undefined {
+  if (!doi) return undefined;
+  const value = doi.trim().replace(/^https?:\/\/(dx\.)?doi\.org\//i, "").replace(/^doi:/i, "").toLowerCase();
+  return value || undefined;
+}
+export function normArxiv(id: string | undefined): string | undefined {
+  if (!id) return undefined;
+  const value = id.trim().replace(/^https?:\/\/arxiv\.org\/abs\//i, "").replace(/^arxiv:/i, "").replace(/v\d+$/, "");
+  return value || undefined;
+}
+
+/** Clean one source record: bounded fields, arXiv DOIs mapped to the arXiv id. */
+export function cleanRaw(raw: RawCandidate): RawCandidate | undefined {
+  const title = raw.title.replace(/\s+/g, " ").trim().slice(0, TITLE_MAX);
+  if (!title) return undefined;
+  let doi = normDoi(raw.doi), arxivId = normArxiv(raw.arxivId);
+  const arxivDoi = doi && ARXIV_DOI.exec(doi);
+  if (arxivDoi) { arxivId ??= normArxiv(arxivDoi[1]); doi = undefined; }
+  return {
+    title, authors: raw.authors.map(a => a.replace(/\s+/g, " ").trim().slice(0, AUTHOR_MAX)).filter(Boolean).slice(0, AUTHORS_MAX), source: raw.source,
+    ...(raw.year !== undefined && Number.isSafeInteger(raw.year) ? { year: raw.year } : {}),
+    ...(raw.venue?.trim() ? { venue: raw.venue.trim().slice(0, 500) } : {}),
+    ...(doi ? { doi } : {}), ...(arxivId ? { arxivId } : {}),
+    ...(raw.url?.trim() ? { url: raw.url.trim().slice(0, 2000) } : {}),
+    ...(raw.abstract?.trim() ? { abstract: raw.abstract.replace(/\s+/g, " ").trim().slice(0, ABSTRACT_MAX) } : {}),
+  };
+}
+
+export function candidateId(raw: RawCandidate): string {
+  if (raw.doi) return `doi:${raw.doi}`;
+  if (raw.arxivId) return `arxiv:${raw.arxivId}`;
+  return `title:${sha256(`${normTitle(raw.title)}|${raw.year ?? ""}`).slice(0, 12)}`;
+}
+
+const titleKey = (c: { title: string; year?: number }) => `${normTitle(c.title)}|${c.year ?? ""}`;
+
+/** Groups of candidates that are the same work by DOI, arXiv id, title plus year, or a recorded duplicate link. */
+export function sameWork(list: Candidate[]): string[][] {
+  const parent = new Map(list.map(c => [c.id, c.id]));
+  const root = (id: string): string => { const p = parent.get(id)!; return p === id ? id : root(p); };
+  const join = (a: string, b: string) => { if (parent.has(a) && parent.has(b)) parent.set(root(a), root(b)); };
+  const seen = new Map<string, string>();
+  for (const c of list) {
+    for (const key of [c.doi && `doi:${c.doi}`, c.arxivId && `arxiv:${c.arxivId}`, `title:${titleKey(c)}`]) {
+      if (!key) continue;
+      const first = seen.get(key);
+      if (first) join(first, c.id); else seen.set(key, c.id);
+    }
+    for (const d of c.duplicates ?? []) join(c.id, d);
+  }
+  const groups = new Map<string, string[]>();
+  for (const c of list) { const r = root(c.id); groups.set(r, [...(groups.get(r) ?? []), c.id]); }
+  return [...groups.values()].filter(g => g.length > 1);
+}
+
+/**
+ * Merge source records into the list. A record joins an existing candidate on DOI, arXiv id or normalized
+ * title plus year; the candidate keeps its first id so screening decisions and citekeys stay attached.
+ */
+function absorb(into: Candidate, from: Omit<RawCandidate, "source"> & { sources?: string[]; source?: string }): void {
+  for (const source of from.sources ?? (from.source ? [from.source] : [])) if (!into.sources.includes(source)) into.sources.push(source);
+  for (const key of ["doi", "arxivId", "year", "venue", "url", "abstract"] as const) if (into[key] === undefined && from[key] !== undefined) (into as unknown as Record<string, unknown>)[key] = from[key];
+  if (!into.authors.length && from.authors.length) into.authors = from.authors;
+}
+function link(a: Candidate, b: Candidate): void {
+  a.duplicates = [...new Set([...(a.duplicates ?? []), b.id])];
+  b.duplicates = [...new Set([...(b.duplicates ?? []), a.id])];
+}
+
+export function mergeCandidates(list: Candidate[], incoming: RawCandidate[], limit: number, now = new Date().toISOString()): { added: number; merged: number; dropped: number; linked: number } {
+  let added = 0, merged = 0, dropped = 0, linked = 0;
+  for (const raw of incoming) {
+    const clean = cleanRaw(raw);
+    if (!clean) { dropped++; continue; }
+    const matches = list.filter(c => (clean.doi && c.doi === clean.doi) || (clean.arxivId && c.arxivId === clean.arxivId) || titleKey(c) === titleKey(clean));
+    const [primary, ...others] = matches;
+    if (primary) {
+      // A record that links several candidates folds the unscreened, unkeyed ones into the first; the rest are flagged.
+      for (const other of others) {
+        if (other.status === "new" && !other.citekey) {
+          absorb(primary, other);
+          for (const twin of other.duplicates ?? []) { const t = list.find(c => c.id === twin); if (t && t !== primary) { t.duplicates = t.duplicates!.filter(d => d !== other.id); link(primary, t); } }
+          list.splice(list.indexOf(other), 1);
+        }
+        else { link(primary, other); linked++; }
+      }
+      absorb(primary, clean);
+      merged++;
+      continue;
+    }
+    if (list.length >= limit) { dropped++; continue; }
+    const { source, ...rest } = clean;
+    list.push({ ...rest, id: candidateId(clean), sources: [source], addedAt: now, status: "new" });
+    added++;
+  }
+  return { added, merged, dropped, linked };
+}
+
+export function candidatesPath(dir: string): string { return join(dir, "literature", "candidates.json"); }
+export function loadCandidates(dir: string): Candidate[] {
+  const path = candidatesPath(dir);
+  if (!existsSync(path)) return [];
+  const raw = readJsonFile(path, CANDIDATES_BYTES);
+  if (!Array.isArray(raw) || !raw.every(c => plain(c) && typeof c.id === "string" && typeof c.title === "string" && Array.isArray(c.sources)))
+    throw new Error("literature/candidates.json: malformed");
+  return raw as Candidate[];
+}
+export function candidatesFile(dir: string, list: Candidate[]): FileWrite { return [candidatesPath(dir), JSON.stringify(list, null, 1) + "\n", CANDIDATES_BYTES]; }
+export function saveCandidates(dir: string, list: Candidate[]): void { writeAtomic(...candidatesFile(dir, list)); }
+
+/** The fields a verification checked. A later merge that adds a DOI, arXiv id or year makes an older verification stale. */
+export function verifySubject(c: Pick<Candidate, "doi" | "arxivId" | "title" | "year">): string {
+  return sha256(JSON.stringify([c.doi ?? null, c.arxivId ?? null, normTitle(c.title), c.year ?? null]));
+}
+export function isVerified(c: Candidate): boolean {
+  return c.verify?.syntax === "ok" && c.verify.exists === "yes" && c.verify.match === "yes" && c.verify.subject === verifySubject(c);
+}
+/** The screen gate binds to the included set. */
+export function includedHash(list: Candidate[]): string | undefined {
+  const ids = list.filter(c => c.status === "included").map(c => c.id).sort();
+  return ids.length ? sha256(ids.join("\n")) : undefined;
+}

package/src/lit/cards.ts

@@ -0,0 +1,21 @@
+import { join } from "node:path";
+import { boundedText, onlyKeys, plain, sha256 } from "../study.ts";
+
+export interface Card { summary: string; claims: string[]; methods?: string; findings?: string; limitations?: string; relevance?: string }
+const OPTIONAL = ["methods", "findings", "limitations", "relevance"] as const;
+
+export function validateCard(raw: unknown): Card {
+  if (!plain(raw)) throw new Error("card: JSON object required");
+  onlyKeys(raw, ["summary", "claims", ...OPTIONAL], "card");
+  const claims = raw.claims ?? [];
+  if (!Array.isArray(claims) || claims.length > 50) throw new Error("card.claims: list of at most 50");
+  const card: Card = { summary: boundedText(raw.summary, "card.summary", 4000), claims: claims.map((c, i) => boundedText(c, `card.claims[${i}]`, 1000)) };
+  for (const key of OPTIONAL) if (raw[key] !== undefined) card[key] = boundedText(raw[key], `card.${key}`, 4000);
+  return card;
+}
+
+/** A readable, collision-free file name for a candidate id such as `doi:10.1/x`. */
+export function cardPath(dir: string, id: string): string {
+  const readable = id.toLowerCase().replace(/[^a-z0-9.-]+/g, "_").slice(0, 80);
+  return join(dir, "literature", "cards", `${readable}-${sha256(id).slice(0, 8)}.json`);
+}

package/src/lit/cite.ts

@@ -0,0 +1,26 @@
+import { isVerified, type Candidate } from "./candidates.ts";
+
+/** Keys cited as `[@key]` or `[see @a, p. 3; @b]` (Pandoc style). */
+export function citationOccurrences(text: string): { key: string; start: number; end: number; groupStart: number; groupEnd: number }[] {
+  const result: { key: string; start: number; end: number; groupStart: number; groupEnd: number }[] = [];
+  for (const group of text.matchAll(/\[([^\[\]]*?@[^\[\]]*?)\]/g)) {
+    for (const match of group[1]!.matchAll(/(?:^|[\s;,(-])@([A-Za-z0-9_](?:[A-Za-z0-9_:.#$%&+?<>~/-]*[A-Za-z0-9_])?)/g)) {
+      const start = group.index + 1 + match.index + match[0].indexOf("@");
+      result.push({ key: match[1]!, start, end: start + match[1]!.length + 1, groupStart: group.index, groupEnd: group.index + group[0].length });
+    }
+  }
+  return result;
+}
+export function citedKeys(text: string): { count: number; keys: string[] } {
+  const occurrences = citationOccurrences(text);
+  return { count: occurrences.length, keys: [...new Set(occurrences.map(hit => hit.key))] };
+}
+
+export interface CiteReport { ok: boolean; citations: number; keys: string[]; unknown: string[]; notVerified: string[] }
+export function checkCitations(text: string, list: Candidate[]): CiteReport {
+  const { count, keys } = citedKeys(text);
+  const byKey = new Map(list.filter(c => c.citekey).map(c => [c.citekey!, c]));
+  const unknown = keys.filter(k => !byKey.has(k));
+  const notVerified = keys.filter(k => { const c = byKey.get(k); return !!c && !(c.status === "included" && isVerified(c)); });
+  return { ok: !unknown.length && !notVerified.length, citations: count, keys, unknown, notVerified };
+}

package/src/lit/sources.ts

@@ -0,0 +1,122 @@
+import { HttpError, type Http } from "../http.ts";
+import type { RawCandidate } from "./candidates.ts";
+
+export type SourceName = "openalex" | "s2" | "arxiv";
+export const SOURCES: readonly SourceName[] = ["openalex", "s2", "arxiv"];
+export interface SearchOptions { limit: number; yearFrom?: number; yearTo?: number }
+export interface SourceResult { source: SourceName; ok: boolean; records: RawCandidate[]; error?: string }
+
+const record = (v: unknown): Record<string, unknown> => (v && typeof v === "object" && !Array.isArray(v) ? v as Record<string, unknown> : {});
+const str = (v: unknown): string | undefined => (typeof v === "string" && v.trim() ? v : undefined);
+const int = (v: unknown): number | undefined => (Number.isSafeInteger(v) ? v as number : typeof v === "string" && /^\d{4}$/.test(v) ? Number(v) : undefined);
+
+/** OpenAlex returns abstracts as an inverted index: word -> positions. */
+export function invertedAbstract(index: unknown): string | undefined {
+  const words: string[] = [];
+  for (const [word, positions] of Object.entries(record(index))) {
+    if (!Array.isArray(positions)) continue;
+    for (const p of positions) if (Number.isSafeInteger(p) && p >= 0 && p < 5000) words[p] = word;
+  }
+  const text = words.filter(Boolean).join(" ");
+  return text || undefined;
+}
+
+export function parseOpenAlex(body: unknown): RawCandidate[] {
+  const results = record(body).results;
+  if (!Array.isArray(results)) throw new Error("openalex: unexpected response");
+  return results.map(raw => {
+    const r = record(raw), ids = record(r.ids), location = record(r.primary_location), source = record(location.source);
+    const doi = str(r.doi), title = str(r.title) ?? str(r.display_name) ?? "";
+    const authors = Array.isArray(r.authorships) ? r.authorships.map(a => str(record(record(a).author).display_name)).filter((a): a is string => !!a) : [];
+    const arxiv = str(ids.arxiv);
+    return {
+      source: "openalex", title, authors, year: int(r.publication_year), venue: str(source.display_name), doi, arxivId: arxiv,
+      url: str(location.landing_page_url) ?? str(r.id), abstract: invertedAbstract(r.abstract_inverted_index),
+    };
+  });
+}
+
+export function parseS2(body: unknown): RawCandidate[] {
+  const data = record(body).data;
+  if (!Array.isArray(data)) throw new Error("s2: unexpected response");
+  return data.map(raw => {
+    const r = record(raw), ext = record(r.externalIds);
+    return {
+      source: "s2", title: str(r.title) ?? "", authors: Array.isArray(r.authors) ? r.authors.map(a => str(record(a).name)).filter((a): a is string => !!a) : [],
+      year: int(r.year), venue: str(r.venue), doi: str(ext.DOI), arxivId: str(ext.ArXiv), url: str(r.url), abstract: str(r.abstract),
+    };
+  });
+}
+
+const ENTITY: Record<string, string> = { amp: "&", lt: "<", gt: ">", quot: '"', apos: "'" };
+export function xmlText(text: string): string {
+  return text.replace(/<!\[CDATA\[([\s\S]*?)\]\]>/g, "$1").replace(/&(#x[0-9a-f]+|#\d+|[a-z]+);/gi, (whole, name: string) => {
+    if (name[0] === "#") { const code = name[1]?.toLowerCase() === "x" ? parseInt(name.slice(2), 16) : parseInt(name.slice(1), 10); return Number.isFinite(code) && code > 0 && code <= 0x10ffff ? String.fromCodePoint(code) : whole; }
+    return ENTITY[name.toLowerCase()] ?? whole;
+  }).replace(/\s+/g, " ").trim();
+}
+const tag = (xml: string, name: string) => { const m = new RegExp(`<${name}(?:\\s[^>]*)?>([\\s\\S]*?)</${name}>`).exec(xml); return m ? xmlText(m[1]!) : undefined; };
+
+export interface ArxivEntry extends RawCandidate { arxivId: string }
+/**
+ * Parse an arXiv Atom feed. `complete` means the feed closed and every entry closed; `errors` counts the error
+ * entries arXiv reports under /api/errors. Absence may be concluded only from a complete, error-free feed.
+ */
+export function parseArxiv(xml: string): { total: number | undefined; entries: ArxivEntry[]; errors: number; complete: boolean } {
+  if (!/<feed[\s>]/.test(xml)) throw new Error("arxiv: unexpected response");
+  const totalText = /<opensearch:totalResults[^>]*>(\d+)</.exec(xml)?.[1];
+  const entries: ArxivEntry[] = [];
+  let errors = 0;
+  for (const m of xml.matchAll(/<entry>([\s\S]*?)<\/entry>/g)) {
+    const body = m[1]!, idUrl = tag(body, "id") ?? "";
+    const id = /arxiv\.org\/abs\/(.+?)(v\d+)?$/.exec(idUrl)?.[1];
+    if (!id) { errors++; continue; }
+    const authors = [...body.matchAll(/<author>[\s\S]*?<name>([\s\S]*?)<\/name>[\s\S]*?<\/author>/g)].map(a => xmlText(a[1]!));
+    entries.push({
+      source: "arxiv", arxivId: id, title: tag(body, "title") ?? "", authors, year: int(tag(body, "published")?.slice(0, 4)),
+      venue: tag(body, "arxiv:journal_ref"), doi: tag(body, "arxiv:doi"), url: `https://arxiv.org/abs/${id}`, abstract: tag(body, "summary"),
+    });
+  }
+  const opened = xml.match(/<entry>/g)?.length ?? 0, closed = xml.match(/<\/entry>/g)?.length ?? 0;
+  const complete = /<\/feed>\s*$/.test(xml) && opened === closed;
+  return { total: totalText === undefined ? undefined : Number(totalText), entries, errors, complete };
+}
+
+function terms(query: string): string[] { return query.split(/\s+/).map(t => t.replace(/[^\p{L}\p{N}-]/gu, "")).filter(Boolean).slice(0, 20); }
+
+export function sourceUrl(source: SourceName, query: string, options: SearchOptions, env: NodeJS.ProcessEnv): string {
+  const q = encodeURIComponent(query);
+  if (source === "openalex") {
+    const filters = [options.yearFrom ? `from_publication_date:${options.yearFrom}-01-01` : "", options.yearTo ? `to_publication_date:${options.yearTo}-12-31` : ""].filter(Boolean);
+    const key = env.OPENALEX_API_KEY?.trim();
+    return `https://api.openalex.org/works?search=${q}&per-page=${options.limit}${filters.length ? `&filter=${filters.join(",")}` : ""}${key ? `&api_key=${encodeURIComponent(key)}` : ""}`;
+  }
+  if (source === "s2") {
+    const years = options.yearFrom || options.yearTo ? `&year=${options.yearFrom ?? ""}-${options.yearTo ?? ""}` : "";
+    return `https://api.semanticscholar.org/graph/v1/paper/search?query=${q}&limit=${options.limit}&fields=title,year,authors,venue,externalIds,abstract,url${years}`;
+  }
+  const parts = terms(query).map(t => `all:${t}`);
+  if (options.yearFrom || options.yearTo) parts.push(`submittedDate:[${options.yearFrom ?? 1991}01010000 TO ${options.yearTo ?? 2100}12312359]`);
+  return `https://export.arxiv.org/api/query?search_query=${encodeURIComponent(parts.join(" AND "))}&max_results=${options.limit}`;
+}
+
+/** Search one source. Failures are reported per source and never abort the others. */
+export async function searchSource(http: Http, source: SourceName, query: string, options: SearchOptions): Promise<SourceResult> {
+  const url = sourceUrl(source, query, options, http.env);
+  try {
+    if (source === "arxiv") {
+      const result = await http.text(url);
+      const feed = result.body === undefined ? undefined : parseArxiv(result.body);
+      if (feed?.errors) throw new Error("arxiv: query error");
+      if (feed && !feed.complete) throw new Error("arxiv: incomplete response");
+      return { source, ok: true, records: feed?.entries ?? [] };
+    }
+    const key = http.env.S2_API_KEY?.trim();
+    const result = await http.json(url, source === "s2" && key ? { "x-api-key": key } : {});
+    if (result.body === undefined) return { source, ok: true, records: [] };
+    return { source, ok: true, records: source === "openalex" ? parseOpenAlex(result.body) : parseS2(result.body) };
+  } catch (error) {
+    const message = error instanceof HttpError ? error.message : error instanceof Error ? error.message.slice(0, 200) : "failed";
+    return { source, ok: false, records: [], error: message };
+  }
+}