@staix/agent-hub 0.20.0 → 0.22.0

package/src/hub/snapshots.ts

@@ -1,5 +1,5 @@
 import { Database } from "bun:sqlite";
-import { closeSync, constants, copyFileSync, existsSync, fstatSync, lstatSync, mkdirSync, mkdtempSync, openSync, readSync, readdirSync, rmSync, statSync, utimesSync, writeFileSync, type Stats } from "node:fs";
+import { closeSync, constants, copyFileSync, existsSync, fstatSync, lstatSync, mkdirSync, mkdtempSync, openSync, readFileSync, readSync, readdirSync, rmSync, statSync, utimesSync, writeFileSync, type Stats } from "node:fs";
 import { createHash } from "node:crypto";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
@@ -26,6 +26,15 @@ export const budgetNow = (budget: SnapshotBudget): number => budget.now ? budget
 export class SnapshotBudgetExceeded extends Error {
   constructor() { super("snapshot burst budget exhausted; boundary evidence is unknown"); }
 }
+const SNAPSHOT_TIMEOUT_MS = 10_000;
+/**
+ * The wall-clock timeout of one git child. Production (no `budget.now`): the remaining budget, between 1 ms and the 10 s ceiling, so a child is killed at the budget.
+ * ponytail: a host-held clock (`budget.now`, only the host-only snapshotClock seam sets it) holds the deadline, so the child's timeout is the 10 s ceiling, not the remaining budget:
+ * with the clock fixed the remaining budget never shrinks, and 250 ms of real time per child timed a slow git out under load (#554). Production never sets `now`, so its child is still killed
+ * at the budget. A child past the 10 s ceiling still reports ETIMEDOUT and the evidence stays unknown. Upgrade path: a fake spawn seam, if a held clock ever needs to model a slow child.
+ */
+export const childTimeoutMs = (budget?: SnapshotBudget): number =>
+  budget?.now ? SNAPSHOT_TIMEOUT_MS : Math.max(1, Math.min(SNAPSHOT_TIMEOUT_MS, Math.floor(budget ? budget.deadline - budgetNow(budget) : SNAPSHOT_TIMEOUT_MS)));
 const git = (top: string | Repo, args: string[], env: Record<string, string> = {}, budget?: SnapshotBudget, maxBuffer?: number, input?: string | Buffer) => {
   const argv = repoGitArgs(top, args);
   const remaining = budget ? budget.deadline - budgetNow(budget) : SNAPSHOT_TIMEOUT_MS;
@@ -33,12 +42,11 @@ const git = (top: string | Repo, args: string[], env: Record<string, string> = {
   const result = hubGitSync(argv, {
     encoding: "utf8", env: { ...repoGitEnv(top), ...env }, ...(maxBuffer === undefined ? {} : { maxBuffer }),
     ...(input === undefined ? {} : { input }),
-    timeout: Math.max(1, Math.min(SNAPSHOT_TIMEOUT_MS, Math.floor(remaining))),
+    timeout: childTimeoutMs(budget),
   });
   if (budget && (budgetNow(budget) >= budget.deadline || (result.error as NodeJS.ErrnoException | undefined)?.code === "ETIMEDOUT")) throw new SnapshotBudgetExceeded();
   return result;
 };
-const SNAPSHOT_TIMEOUT_MS = 10_000;
 
 /** A repository and where the project sits in it: `prefix` is "" at the top level, else "sub/dir/". */
 export interface Repo {
@@ -543,6 +551,8 @@ export interface TurnRecord {
   start_tree: string | null;
   end_tree: string | null;
   changed: string[];
+  /** Dirty submodules at the turn's end, whose contents nothing here observes (#798 D2). Absent: not recorded (an older turn, or git could not answer), which is unknown, never none. */
+  unobserved?: string[];
   native: string | null;
 }
 
@@ -606,6 +616,43 @@ export function dirtySubmodules(repo: Repo, budget?: SnapshotBudget): string[] |
   return result.status === 0 ? parseDirtySubmodules(result.stdout) : undefined;
 }
 
+/**
+ * One git-config value as git reads it: an unquoted `#` or `;` starts a comment, a quoted run keeps them, quotes are dropped, and `\"` and `\\` are unescaped inside a quoted
+ * run. ponytail: other escapes (`\t`, octal) are kept as the character after the backslash; use `git config -f .gitmodules --get-regexp path` if a name needs them.
+ */
+function configValue(raw: string): string {
+  let out = "", keep = 0, quoted = false;
+  for (let i = 0; i < raw.length; i++) {
+    const c = raw[i]!;
+    if (quoted && c === "\\" && i + 1 < raw.length) { out += raw[++i]; keep = out.length; continue; }
+    if (c === '"') { quoted = !quoted; keep = out.length; continue; }
+    if (!quoted && (c === "#" || c === ";")) break;
+    if (!quoted && out.length === 0 && /\s/.test(c)) continue; // leading blanks
+    out += c;
+    if (quoted || !/\s/.test(c)) keep = out.length;       // trailing blanks outside quotes are not part of the value
+  }
+  return out.slice(0, keep);
+}
+
+/**
+ * The submodule paths a project declares, relative to the project (#798 D1), read from `.gitmodules` with no git call. Empty without the file, and for a shadow
+ * repository. A path outside the project's own directory in a larger repository is not the project's. A path git quotes (`"a\tb"`) is taken as written, never
+ * unescaped beyond `\"` and `\\`: see `configValue`.
+ */
+export function submodulePaths(repo: Repo): string[] {
+  if (repo.shadow) return [];
+  let text: string;
+  try { text = readFileSync(join(repo.top, ".gitmodules"), "utf8"); } catch { return []; }
+  const found = new Set<string>();
+  for (const line of text.split(/\r?\n/)) {
+    const match = /^\s*path\s*=(.*)$/.exec(line);
+    let path = match ? configValue(match[1]!) : "";
+    path = path.replace(/\/+$/, "");
+    if (path && !path.startsWith("/") && !path.split("/").includes("..") && path.startsWith(repo.prefix)) found.add(path.slice(repo.prefix.length));
+  }
+  return [...found].sort();
+}
+
 /** Turn records in hub.db: the daemon writes them, `ahub turns` and `ahub undo` read them. */
 export class Turns {
   private readonly db: Database;
@@ -623,6 +670,7 @@ export class Turns {
     this.db.run("CREATE TABLE IF NOT EXISTS touches (task INTEGER NOT NULL, peer TEXT NOT NULL, path TEXT NOT NULL, at INTEGER NOT NULL, PRIMARY KEY (task, peer, path))");
     this.db.run("CREATE INDEX IF NOT EXISTS touches_path ON touches (path)"); // the PreToolUse hook looks up one path
     this.db.run("CREATE TABLE IF NOT EXISTS turn_workspaces (turn TEXT PRIMARY KEY, root TEXT NOT NULL, start_tree TEXT, end_tree TEXT, changed TEXT NOT NULL DEFAULT '[]')");
+    this.db.run("CREATE TABLE IF NOT EXISTS turn_unobserved (turn TEXT PRIMARY KEY, paths TEXT NOT NULL)");
     this.db.run("CREATE TABLE IF NOT EXISTS task_snapshots (task INTEGER NOT NULL, proposed_at INTEGER NOT NULL, start_tree TEXT, PRIMARY KEY (task, proposed_at))");
   }
   /** A proposal's original project snapshot; task ids may be reused after reset. Never repair a missing baseline later. */
@@ -665,8 +713,9 @@ export class Turns {
   native(id: string, native: string): void {
     this.db.query("UPDATE turns SET native = ? WHERE id = ? AND native IS NULL").run(native, id);
   }
-  end(id: string, endTree: string | undefined, changed: string[], keep: number): void {
+  end(id: string, endTree: string | undefined, changed: string[], keep: number, unobserved?: string[]): void {
     this.db.transaction(() => {
+      if (unobserved !== undefined) this.db.query("INSERT OR REPLACE INTO turn_unobserved (turn, paths) VALUES (?, ?)").run(id, JSON.stringify(unobserved));
       const scoped = this.workspace(id) !== undefined;
       // Older CLIs do not know workspace receipts. Leave their canonical-checkout snapshot view unknown.
       this.db.query("UPDATE turns SET ended = ?, end_tree = ?, changed = ? WHERE id = ?").run(Date.now(), scoped ? null : endTree ?? null, scoped ? "[]" : JSON.stringify(changed), id);
@@ -675,18 +724,35 @@ export class Turns {
     const peer = (this.db.query("SELECT peer FROM turns WHERE id = ?").get(id) as { peer: string } | null)?.peer;
     if (peer) this.db.query("DELETE FROM turns WHERE peer = ? AND id NOT IN (SELECT id FROM turns WHERE peer = ? ORDER BY started DESC, rowid DESC LIMIT ?)").run(peer, peer, keep);
     this.db.run("DELETE FROM turn_workspaces WHERE turn NOT IN (SELECT id FROM turns)");
+    this.db.run("DELETE FROM turn_unobserved WHERE turn NOT IN (SELECT id FROM turns)");
+  }
+  /** The dirty submodules recorded when the turn ended; undefined when none were recorded (unknown, not "none"). */
+  unobserved(id: string): string[] | undefined {
+    try {
+      const row = this.db.query("SELECT paths FROM turn_unobserved WHERE turn = ?").get(id) as { paths: string } | null;
+      if (!row) return undefined;
+      const paths = JSON.parse(row.paths);
+      return Array.isArray(paths) && paths.every(path => typeof path === "string") ? paths : undefined;
+    } catch (error) {
+      if (/no such table/.test((error as Error).message)) return undefined;
+      throw error;
+    }
+  }
+  private withUnobserved<T extends { id: string }>(record: T): T & { unobserved?: string[] } {
+    const unobserved = this.unobserved(record.id);
+    return unobserved === undefined ? record : { ...record, unobserved };
   }
   list(peer?: string, limit = 20): TurnRecord[] {
     const rows = (peer
       ? this.db.query("SELECT * FROM turns WHERE peer = ? ORDER BY started DESC, rowid DESC LIMIT ?").all(peer, limit)
       : this.db.query("SELECT * FROM turns ORDER BY started DESC, rowid DESC LIMIT ?").all(limit)) as (Omit<TurnRecord, "changed"> & { changed: string })[];
-    return rows.map(row => this.snapshotFields(row)).map((r) => ({ ...r, changed: JSON.parse(r.changed) as string[] }));
+    return rows.map(row => this.snapshotFields(row)).map((r) => this.withUnobserved({ ...r, changed: JSON.parse(r.changed) as string[] }));
   }
   get(id: string): TurnRecord | undefined {
     const r = this.db.query("SELECT * FROM turns WHERE id = ?").get(id) as (Omit<TurnRecord, "changed"> & { changed: string }) | null;
     if (!r) return undefined;
     const record = this.snapshotFields(r);
-    return { ...record, changed: JSON.parse(record.changed) as string[] };
+    return this.withUnobserved({ ...record, changed: JSON.parse(record.changed) as string[] });
   }
   /**
    * Other peers' turns that overlapped this one in time: their changes are in this turn's diff too. `unknown` lists

package/src/hub/source-read.ts

@@ -0,0 +1,160 @@
+import { sanitize } from "./envelope.ts";
+import { PreEffectToolRefusal } from "./tool-refusal.ts";
+import { validSourceId } from "./sources.ts";
+import type { SourceMeta } from "./sources.ts";
+
+export const MAX_SOURCE_READ_BYTES = 32 * 1024;
+const CUT_MARKER = " [cut]";
+const REFUSAL = "source read unavailable";
+
+export interface SourceReadRegistry {
+  get(id: string): SourceMeta | undefined;
+  readText(id: string): string | undefined;
+  markPrivate(id: string): SourceMeta | undefined;
+}
+
+export interface SourceReadPolicy {
+  /** Screen the complete stored text. True means clear; false means private or unknown. */
+  screen(text: string, meta: SourceMeta): Promise<boolean>;
+  /** A synchronous, fresh authorization check. */
+  valid(): boolean;
+  /** True only for an authorized on-prem worker inside a PII task. */
+  canReadPrivate(): Promise<boolean>;
+}
+
+export interface SourceReadArgs { id: string; range?: { offset: number; length: number } }
+export type SourceReadResult =
+  | { id: string; kind: SourceMeta["kind"]; private: true }
+  | { id: string; kind: SourceMeta["kind"]; private: true; text: string; next: number }
+  | { id: string; kind: SourceMeta["kind"]; private: false; text: string; next: number };
+
+function refuse(): never { throw new PreEffectToolRefusal(REFUSAL); }
+
+/** Validate the exact tool shape before touching the registry or privacy policy. */
+export function validateSourceRead(value: unknown): SourceReadArgs {
+  if (!value || typeof value !== "object" || Array.isArray(value)) refuse();
+  const input = value as Record<string, unknown>;
+  if (Object.keys(input).some(key => key !== "id" && key !== "range") || !validSourceId(input.id)) refuse();
+  if (input.range === undefined) return { id: input.id };
+  if (!input.range || typeof input.range !== "object" || Array.isArray(input.range)) refuse();
+  const range = input.range as Record<string, unknown>;
+  if (Object.keys(range).length !== 2 || Object.keys(range).some(key => key !== "offset" && key !== "length")
+    || !Number.isSafeInteger(range.offset) || (range.offset as number) < 0
+    || !Number.isSafeInteger(range.length) || (range.length as number) < 1
+    || !Number.isSafeInteger((range.offset as number) + (range.length as number))) refuse();
+  return { id: input.id, range: { offset: range.offset as number, length: range.length as number } };
+}
+
+function sameRecord(a: SourceMeta, b: SourceMeta): boolean {
+  return a.id === b.id && a.kind === b.kind && a.origin === b.origin && a.sha256 === b.sha256 && a.bytes === b.bytes
+    && a.added_at === b.added_at && a.private === b.private && JSON.stringify(a.origins) === JSON.stringify(b.origins);
+}
+function sameContent(a: SourceMeta, b: SourceMeta): boolean {
+  return a.id === b.id && a.kind === b.kind && a.sha256 === b.sha256 && a.bytes === b.bytes;
+}
+
+async function privateResult(registry: SourceReadRegistry, id: string, meta: SourceMeta, range: SourceReadArgs["range"], policy: SourceReadPolicy): Promise<SourceReadResult> {
+  const authorized = await policy.canReadPrivate();
+  if (!policy.valid()) refuse();
+  const afterAuth = registry.get(id);
+  if (!afterAuth || !afterAuth.private || !sameRecord(meta, afterAuth)) refuse();
+  if (!authorized) {
+    if (!policy.valid()) refuse();
+    return { id, kind: afterAuth.kind, private: true };
+  }
+  const text = registry.readText(id), finalMeta = registry.get(id);
+  if (text === undefined || !finalMeta?.private || !sameRecord(afterAuth, finalMeta)) refuse();
+  if (!policy.valid()) refuse();
+  return render(id, finalMeta.kind, text, range, true);
+}
+
+/** Screen the whole source before selecting a bounded excerpt; re-read identity and privacy after async policy boundaries. */
+export async function sourceRead(registry: SourceReadRegistry, value: unknown, policy: SourceReadPolicy): Promise<SourceReadResult> {
+  const args = validateSourceRead(value);
+  try {
+    if (!policy.valid()) refuse();
+    let meta = registry.get(args.id);
+    if (!meta) refuse();
+    if (meta.private) {
+      const result = await privateResult(registry, args.id, meta, args.range, policy);
+      if (!policy.valid()) refuse();
+      return result;
+    }
+
+    let fullText = registry.readText(args.id);
+    if (fullText === undefined) refuse();
+    let clear = false;
+    try { clear = await policy.screen(fullText, meta); }
+    catch { clear = false; }
+    if (!policy.valid()) refuse();
+    const afterScreen = registry.get(args.id), afterScreenText = registry.readText(args.id);
+    if (!afterScreen || afterScreenText === undefined || !sameContent(meta, afterScreen) || afterScreenText !== fullText) refuse();
+    if (afterScreen.private) {
+      const result = await privateResult(registry, args.id, afterScreen, args.range, policy);
+      if (!policy.valid()) refuse();
+      return result;
+    }
+    if (!sameRecord(meta, afterScreen)) refuse();
+    meta = afterScreen;
+    if (!clear) {
+      if (!policy.valid()) refuse();
+      meta = registry.markPrivate(args.id) ?? registry.get(args.id);
+      if (!meta?.private) refuse();
+      const result = await privateResult(registry, args.id, meta, args.range, policy);
+      if (!policy.valid()) refuse();
+      return result;
+    }
+
+    if (!policy.valid()) refuse();
+    const current = registry.get(args.id);
+    if (!current || !sameRecord(meta, current) || current.private) refuse();
+    fullText = registry.readText(args.id);
+    if (fullText === undefined || fullText !== afterScreenText) refuse();
+    if (!policy.valid()) refuse();
+    return render(args.id, current.kind, fullText, args.range, false);
+  } catch {
+    refuse();
+  }
+}
+
+/** Walk only the requested, byte-bounded prefix. Sanitizer growth is measured by a bounded binary search over that prefix. */
+function render(id: string, kind: SourceMeta["kind"], fullText: string, range: SourceReadArgs["range"], isPrivate: boolean): SourceReadResult {
+  const offset = range?.offset ?? 0, limit = range?.length ?? Number.MAX_SAFE_INTEGER;
+  const iterator = fullText[Symbol.iterator]();
+  let skipped = 0;
+  while (skipped < offset) {
+    if (iterator.next().done) return isPrivate
+      ? { id, kind, private: true, text: "", next: offset }
+      : { id, kind, private: false, text: "", next: offset };
+    skipped++;
+  }
+
+  const chars: string[] = [];
+  let rawBytes = 0, exhausted = false, stoppedAtByteLimit = false;
+  while (chars.length < limit) {
+    const next = iterator.next();
+    if (next.done) { exhausted = true; break; }
+    const point = next.value;
+    const size = Buffer.byteLength(point, "utf8");
+    if (rawBytes + size > MAX_SOURCE_READ_BYTES) { stoppedAtByteLimit = true; break; }
+    chars.push(point);
+    rawBytes += size;
+  }
+  const hasMoreAfterSelection = stoppedAtByteLimit || (!exhausted && chars.length >= limit && !iterator.next().done);
+  const fits = (count: number): boolean => {
+    const candidate = sanitize(chars.slice(0, count).join(""));
+    const needsMarker = hasMoreAfterSelection || count < chars.length;
+    return Buffer.byteLength(candidate, "utf8") + (needsMarker ? Buffer.byteLength(CUT_MARKER, "utf8") : 0) <= MAX_SOURCE_READ_BYTES;
+  };
+  let low = 0, high = chars.length;
+  while (low < high) {
+    const middle = Math.ceil((low + high) / 2);
+    if (fits(middle)) low = middle; else high = middle - 1;
+  }
+  const consumed = low;
+  let text = sanitize(chars.slice(0, consumed).join(""));
+  const hasMore = hasMoreAfterSelection || consumed < chars.length;
+  if (hasMore) text += CUT_MARKER;
+  const next = offset + consumed;
+  return isPrivate ? { id, kind, private: true, text, next } : { id, kind, private: false, text, next };
+}

package/src/hub/sources.ts

@@ -0,0 +1,311 @@
+import { createHash } from "node:crypto";
+import { chmodSync, closeSync, constants, existsSync, fstatSync, linkSync, lstatSync, mkdirSync, openSync, readFileSync, readSync, readdirSync, renameSync, rmSync, unlinkSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { processLiveness, processSignature } from "@staix/agent-mem/platform";
+import { MAX_PROJECTION_INPUT } from "./text-projection.ts";
+
+/**
+ * #508 (AC1, AC2, AC5): the source registry. A source is a snapshot of bytes a person registered, named by its content, kept under the hub state dir.
+ * Pure over the filesystem: no daemon, no network (the URL fetch is a separate function the CLI calls), no model. Every refusal names its cap.
+ */
+export const MAX_SOURCE_BYTES = MAX_PROJECTION_INPUT;
+export const MAX_REDIRECTS = 5;
+export const MAX_SOURCES = 10_000;
+export const MAX_ORIGINS = 256;
+export const MAX_ORIGIN_CHARS = 2000;
+/**
+ * One cap per file, used by the writer AND the reader, so what `add` accepts can always be read back (a record the reader cannot read is treated as damaged,
+ * and damage must never be mistaken for absence). `meta.json` holds at most MAX_ORIGINS origins of MAX_ORIGIN_CHARS characters, each up to 3 UTF-8 bytes
+ * per UTF-16 unit, twice over (the first origin is also the record's own), plus indentation: under 1.7 MiB, so 2 MiB. Text is the projection of the bytes:
+ * a converter's is at most 256 KiB, a plain file's is its own decoded bytes, at most the raw cap.
+ */
+export const MAX_META_BYTES = 2 * 1024 * 1024;
+export const MAX_SOURCE_TEXT = MAX_SOURCE_BYTES;
+/** How long a registration waits for another process's registration before it gives up. A lock is never taken from its holder. */
+export const LOCK_WAIT_MS = 10_000;
+const ID = /^src:[0-9a-f]{16}$/;
+export type SourceKind = "file" | "url";
+export interface SourceOrigin { kind: SourceKind; origin: string; added_at: number; private: boolean }
+/** `meta.json`: the spec's `{id, kind, origin, sha256, bytes, added_at, private}` for the first registration, plus every origin ever registered for these bytes. */
+export interface SourceMeta { id: string; kind: SourceKind; origin: string; sha256: string; bytes: number; added_at: number; private: boolean; origins: SourceOrigin[] }
+export interface SourceAdd { id: string; created: boolean; private: boolean; upgraded: boolean; text: boolean }
+
+export const sourceId = (sha256: string): string => {
+  if (!/^[0-9a-f]{64}$/.test(sha256)) throw new Error("source id needs a SHA-256 hex digest");
+  return `src:${sha256.slice(0, 16)}`;
+};
+export const sha256Of = (bytes: Buffer): string => createHash("sha256").update(bytes).digest("hex");
+export const validSourceId = (value: unknown): value is string => typeof value === "string" && ID.test(value);
+/** `src:` is not a safe path segment on every filesystem, so the directory drops the colon. */
+const dirName = (id: string): string => id.replace(":", "-");
+
+function atomic(path: string, data: string | Buffer, mode = 0o600): void {
+  const temporary = `${path}.${crypto.randomUUID()}.tmp`;
+  try { writeFileSync(temporary, data, { mode, flag: "wx" }); chmodSync(temporary, mode); renameSync(temporary, path); }
+  catch (error) { rmSync(temporary, { force: true }); throw error; }
+}
+/** A regular, single-link file read through one descriptor with a hard size cap; anything else is no record. */
+function boundedRead(path: string, cap: number): Buffer {
+  const fd = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
+  try {
+    const st = fstatSync(fd);
+    if (!st.isFile() || st.nlink !== 1 || st.size > cap) throw new Error("not a bounded regular file");
+    const bytes = Buffer.alloc(st.size); let got = 0;
+    while (got < bytes.length) { const n = readSync(fd, bytes, got, bytes.length - got, got); if (!n) break; got += n; }
+    if (got !== st.size) throw new Error("file changed while read");
+    return bytes;
+  } finally { closeSync(fd); }
+}
+const cleanOrigin = (value: unknown): value is string => typeof value === "string" && value.length > 0 && value.length <= MAX_ORIGIN_CHARS && !/[\x00-\x1f\x7f]/.test(value);
+function parseMeta(text: string, id: string): SourceMeta | undefined {
+  try {
+    const raw = JSON.parse(text) as Record<string, unknown>;
+    const origins = raw.origins;
+    if (raw.id !== id || !["file", "url"].includes(raw.kind as string) || !cleanOrigin(raw.origin) || typeof raw.sha256 !== "string" || sourceId(raw.sha256) !== id
+      || !Number.isSafeInteger(raw.bytes) || (raw.bytes as number) < 0 || (raw.bytes as number) > MAX_SOURCE_BYTES || !Number.isSafeInteger(raw.added_at) || typeof raw.private !== "boolean"
+      || !Array.isArray(origins) || !origins.length || origins.length > MAX_ORIGINS) return undefined;
+    const kept: SourceOrigin[] = [];
+    for (const item of origins as Record<string, unknown>[]) {
+      if (!item || !["file", "url"].includes(item.kind as string) || !cleanOrigin(item.origin) || !Number.isSafeInteger(item.added_at) || typeof item.private !== "boolean") return undefined;
+      kept.push({ kind: item.kind as SourceKind, origin: item.origin as string, added_at: item.added_at as number, private: item.private });
+    }
+    // The flag is the OR of every registration; a record whose flag is lower than one of its origins is damaged, not trusted.
+    if (kept.some(item => item.private) && raw.private !== true) return undefined;
+    return { id, kind: raw.kind as SourceKind, origin: raw.origin as string, sha256: raw.sha256, bytes: raw.bytes as number, added_at: raw.added_at as number, private: raw.private, origins: kept };
+  } catch { return undefined; }
+}
+
+/**
+ * `sources/<id>/` under the hub state dir: `meta.json`, `raw` (the exact bytes) and `text` (the projection, only when one exists).
+ * The raw cap is the text-projection input cap, and nothing is truncated: an input over it is refused with the cap in the reason.
+ */
+export class SourceRegistry {
+  readonly root: string;
+  private readonly lockWaitMs: number;
+  constructor(stateDir: string, options: { lockWaitMs?: number } = {}) { this.root = join(stateDir, "sources"); this.lockWaitMs = options.lockWaitMs ?? LOCK_WAIT_MS; }
+
+  private dir(id: string): string { return join(this.root, dirName(id)); }
+
+  get(id: string): SourceMeta | undefined {
+    if (!validSourceId(id)) return undefined;
+    try { return parseMeta(boundedRead(join(this.dir(id), "meta.json"), MAX_META_BYTES).toString("utf8"), id); } catch { return undefined; }
+  }
+
+  list(): SourceMeta[] {
+    let names: string[];
+    try { names = readdirSync(this.root); } catch { return []; }
+    const rows: SourceMeta[] = [];
+    for (const name of names) {
+      const match = /^src-([0-9a-f]{16})$/.exec(name);
+      const meta = match ? this.get(`src:${match[1]}`) : undefined;
+      if (meta) {
+        if (rows.length === MAX_SOURCES) throw new Error(`the registry exceeds ${MAX_SOURCES} sources; nothing was omitted silently (inspect ${this.root})`);
+        rows.push(meta);
+      }
+    }
+    return rows.sort((a, b) => a.added_at - b.added_at || a.id.localeCompare(b.id));
+  }
+
+  /** The stored bytes of a source, re-hashed: a snapshot that no longer matches its id is no source. */
+  readRaw(id: string): Buffer | undefined {
+    const meta = this.get(id);
+    if (!meta) return undefined;
+    try { const bytes = boundedRead(join(this.dir(id), "raw"), MAX_SOURCE_BYTES); return sha256Of(bytes) === meta.sha256 ? bytes : undefined; } catch { return undefined; }
+  }
+  readText(id: string): string | undefined {
+    if (!this.get(id)) return undefined;
+    try { return boundedRead(join(this.dir(id), "text"), MAX_SOURCE_TEXT).toString("utf8"); } catch { return undefined; }
+  }
+
+  /**
+   * The registry-wide lock, and the only way a registration is serialized. It is published whole by `link(2)` of a fully written temp file (atomic, fails with
+   * EEXIST when held), so a lock file never exists without its owner's pid and process signature. It is removed only by its own holder, when the registration ends.
+   * It is NEVER removed on someone else's behalf: "check that the holder is gone, then unlink" is two steps, and a second recoverer that already passed the check
+   * would delete the first recoverer's fresh lock and run beside it, so a stale public write could overwrite a private upgrade. No filesystem call removes a file only
+   * if it is still the one that was inspected, so a lock whose holder is gone fails closed and the person repairs it (the error says how).
+   */
+  private locked<T>(action: () => T): T {
+    const lock = join(this.root, ".lock"), temp = `${lock}.${crypto.randomUUID()}.tmp`;
+    const deadline = Date.now() + this.lockWaitMs;
+    writeFileSync(temp, JSON.stringify({ pid: process.pid, signature: processSignature(process.pid) ?? null }), { mode: 0o600, flag: "wx" });
+    let held = false;
+    try {
+      for (;;) {
+        try { linkSync(temp, lock); held = true; break; }
+        catch (error) {
+          if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error;
+          let owner: { pid?: unknown; signature?: unknown } | undefined;
+          let observed: { dev: number; ino: number } | undefined, ownerText: string | undefined;
+          try {
+            const st = lstatSync(lock); observed = { dev: st.dev, ino: st.ino };
+            ownerText = readFileSync(lock, "utf8"); owner = JSON.parse(ownerText);
+          } catch { owner = undefined; }
+          const state = owner ? processLiveness(owner.pid, typeof owner.signature === "string" ? owner.signature : null) : "unknown";
+          if (state === "gone") {
+            // The normal holder may have released/replaced this lock while its
+            // process was inspected. Refuse only the same still-present record.
+            let unchanged = false;
+            try {
+              const current = lstatSync(lock);
+              unchanged = current.dev === observed?.dev && current.ino === observed?.ino && readFileSync(lock, "utf8") === ownerText;
+            } catch { /* released: retry acquisition without removing anything */ }
+            if (unchanged) throw new Error(`${lock} was left by a registration that is gone (pid ${String(owner?.pid)}); it is never removed automatically. Once no \`ahub source\` runs, delete that file and try again`);
+          }
+          if (Date.now() >= deadline) throw new Error(`another ahub source registration holds ${lock}${owner ? ` (pid ${String(owner.pid)})` : ""}; try again in a moment`);
+          Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 25);
+        }
+      }
+    } finally { try { unlinkSync(temp); } catch { /* the temp name is ours alone */ } }
+    try { return action(); }
+    finally { if (held) { try { unlinkSync(lock); } catch { /* already gone */ } } }
+  }
+
+  /** Host classification is monotonic and does not register a new origin or consume its cap. */
+  markPrivate(id: string): SourceMeta {
+    if (!validSourceId(id)) throw new Error("source classification unavailable");
+    return this.locked(() => {
+      const meta = this.get(id);
+      if (!meta) throw new Error("source classification unavailable");
+      if (meta.private) return meta;
+      const next = { ...meta, private: true };
+      atomic(join(this.dir(id), "meta.json"), JSON.stringify(next, null, 2) + "\n");
+      return next;
+    });
+  }
+
+  /**
+   * Register bytes. Equal bytes share one id, so a second registration returns the existing id. `private` is the OR over every registration of
+   * those bytes: re-adding from a private path, or with text the PII screen flags, upgrades the record; re-adding from a public path never
+   * downgrades it. Every origin is kept. `text` is the projection of the bytes (undefined when no converter handles them).
+   */
+  add(input: { bytes: Buffer; kind: SourceKind; origin: string; private: boolean; text?: string; now?: number }): SourceAdd {
+    if (!Buffer.isBuffer(input.bytes)) throw new Error("a source is bytes");
+    if (input.text !== undefined && Buffer.byteLength(input.text) > MAX_SOURCE_TEXT) throw new Error(`the text is over the ${MAX_SOURCE_TEXT} byte cap; nothing was stored`);
+    if (input.bytes.length > MAX_SOURCE_BYTES) throw new Error(`source is ${input.bytes.length} bytes, over the ${MAX_SOURCE_BYTES} byte cap (MAX_PROJECTION_INPUT); nothing was stored`);
+    if (!cleanOrigin(input.origin)) throw new Error(`source origin must be 1 to ${MAX_ORIGIN_CHARS} printable characters`);
+    const sha256 = sha256Of(input.bytes), id = sourceId(sha256), now = input.now ?? Date.now(), directory = this.dir(id);
+    mkdirSync(this.root, { recursive: true, mode: 0o700 });
+    // One registration at a time per registry, across processes: the read of the record, the decision and the write are one step, so two CLIs adding the
+    // same bytes (one private, one public) can never each read the old flag and the later write lose the other's upgrade.
+    return this.locked(() => this.addLocked(input, sha256, id, now, directory));
+  }
+
+  private addLocked(input: { bytes: Buffer; kind: SourceKind; origin: string; private: boolean; text?: string }, sha256: string, id: string, now: number, directory: string): SourceAdd {
+    const existing = this.get(id);
+    // A directory that exists but whose record cannot be read is DAMAGED, not absent: replacing it could erase a private record and the bytes with it,
+    // so a damaged record is refused and left for the person, never silently rebuilt as public.
+    if (!existing && existsSync(join(directory, "meta.json"))) throw new Error(`the record for ${id} exists but cannot be read; it was left as it is (inspect ${directory})`);
+    if (existing) {
+      const origin: SourceOrigin = { kind: input.kind, origin: input.origin, added_at: now, private: input.private };
+      const known = existing.origins.some(item => item.kind === origin.kind && item.origin === origin.origin && item.private === origin.private);
+      if (!known && existing.origins.length >= MAX_ORIGINS) throw new Error(`the source holds ${MAX_ORIGINS} origins already; this registration was refused and the record was left unchanged`);
+      const privateNow = existing.private || input.private;
+      const origins = known ? existing.origins : [...existing.origins, origin];
+      if (privateNow === existing.private && origins === existing.origins) return { id, created: false, private: existing.private, upgraded: false, text: this.readText(id) !== undefined };
+      atomic(join(directory, "meta.json"), JSON.stringify({ ...existing, private: privateNow, origins }, null, 2) + "\n");
+      return { id, created: false, private: privateNow, upgraded: privateNow && !existing.private, text: this.readText(id) !== undefined };
+    }
+    // Count validated records, not directory entries or a target's mere existence.
+    // A crash partial at this id is still a NEW source and cannot bypass capacity.
+    if (this.list().length >= MAX_SOURCES) throw new Error(`the registry holds ${MAX_SOURCES} sources already; nothing was stored`);
+    if (existsSync(directory)) rmSync(directory, { recursive: true, force: true }); // no meta.json (checked above): a half-written directory of an earlier crash, never a record
+    const staging = `${directory}.${crypto.randomUUID()}.partial`;
+    try {
+      mkdirSync(staging, { recursive: true, mode: 0o700 });
+      writeFileSync(join(staging, "raw"), input.bytes, { mode: 0o600 });
+      if (input.text !== undefined) writeFileSync(join(staging, "text"), input.text, { mode: 0o600 });
+      const meta: SourceMeta = { id, kind: input.kind, origin: input.origin, sha256, bytes: input.bytes.length, added_at: now, private: input.private, origins: [{ kind: input.kind, origin: input.origin, added_at: now, private: input.private }] };
+      writeFileSync(join(staging, "meta.json"), JSON.stringify(meta, null, 2) + "\n", { mode: 0o600 });
+      // The directory appears whole or not at all: a reader never sees raw without its meta.
+      renameSync(staging, directory);
+    } catch (error) { rmSync(staging, { recursive: true, force: true }); throw error; }
+    return { id, created: true, private: input.private, upgraded: false, text: input.text !== undefined };
+  }
+}
+
+/** Anything a person could paste that is not a plain https URL is refused before a connection is made. */
+export function checkSourceUrl(value: string): URL {
+  let url: URL;
+  try { url = new URL(value); } catch { throw new Error("not a URL"); }
+  if (url.protocol !== "https:") throw new Error("only https URLs can be registered");
+  if (url.username || url.password) throw new Error("a URL with credentials is refused");
+  if (!url.hostname) throw new Error("a URL needs a host");
+  return url;
+}
+
+type FetchLike = (input: string, init: { redirect: "manual"; signal: AbortSignal; headers: Record<string, string> }) => Promise<Response>;
+/**
+ * Fetched once, by the person's CLI process (never a hub tool, agent or sandbox): https only, at most MAX_REDIRECTS redirects followed by hand (each one
+ * re-checked as https without credentials), and a running byte cap on the body, so a huge or endless response is cut off at the cap with the cap in the
+ * reason, never buffered whole and never truncated silently.
+ */
+export async function fetchSourceUrl(value: string, options: { fetch?: FetchLike; timeoutMs?: number; maxBytes?: number } = {}): Promise<{ bytes: Buffer; finalUrl: string; redirects: number }> {
+  const cap = options.maxBytes ?? MAX_SOURCE_BYTES, doFetch: FetchLike = options.fetch ?? ((input, init) => fetch(input, init));
+  const signal = AbortSignal.timeout(options.timeoutMs ?? 30_000);
+  let url = checkSourceUrl(value), redirects = 0;
+  for (;;) {
+    const response = await doFetch(url.toString(), { redirect: "manual", signal, headers: { accept: "*/*", "user-agent": "agent-hub-source/1" } });
+    if (response.status >= 300 && response.status < 400 && response.headers.get("location")) {
+      if (++redirects > MAX_REDIRECTS) throw new Error(`more than ${MAX_REDIRECTS} redirects`);
+      try { url = checkSourceUrl(new URL(response.headers.get("location")!, url).toString()); }
+      catch (error) { throw new Error(`redirect refused: ${(error as Error).message}`); }
+      await response.body?.cancel().catch(() => {});
+      continue;
+    }
+    if (!response.ok) { await response.body?.cancel().catch(() => {}); throw new Error(`HTTP ${response.status}`); }
+    const declared = Number(response.headers.get("content-length"));
+    if (Number.isFinite(declared) && declared > cap) { await response.body?.cancel().catch(() => {}); throw new Error(`the response is ${declared} bytes, over the ${cap} byte cap (MAX_PROJECTION_INPUT); nothing was stored`); }
+    const chunks: Uint8Array[] = []; let total = 0;
+    const reader = response.body?.getReader();
+    if (!reader) throw new Error("the response has no body");
+    for (;;) {
+      const { done, value: chunk } = await reader.read();
+      if (done) break;
+      total += chunk.byteLength;
+      if (total > cap) { await reader.cancel().catch(() => {}); throw new Error(`the response is over the ${cap} byte cap (MAX_PROJECTION_INPUT); nothing was stored`); }
+      chunks.push(chunk);
+    }
+    return { bytes: Buffer.concat(chunks, total), finalUrl: url.toString(), redirects };
+  }
+}
+
+/** Why a registration is private, in fixed words: the reason a person reads, never the matched text. */
+export type PrivateReason = "path under pii.paths" | "text matches signals.pii_patterns" | "text could not be screened";
+export interface RegisterPolicy {
+  /** `pii.paths` of the project config. */
+  piiPaths: readonly string[];
+  /** `routing.signals.pii_patterns`: the same regular expressions the hub applies to task text. */
+  piiPatterns: readonly string[];
+  /** Project root the pii.paths are relative to. */
+  cwd: string;
+  /** The in-memory projection of the bytes (the existing text_projection converters), or undefined when none handles them. */
+  project?: (bytes: Buffer) => string | undefined;
+  matchesPolicy: (cwd: string, path: string, paths: string[]) => boolean;
+}
+/**
+ * One decision, one place: whether a registration is private and what its text is. A path under `pii.paths` is private; so is bytes whose projected
+ * (or, for a plain text file, decoded) text matches a configured PII pattern. A pattern that cannot be evaluated counts as flagged, never as clear.
+ */
+export function screenRegistration(input: { bytes: Buffer; kind: SourceKind; path?: string; typedPath?: string }, policy: RegisterPolicy): { private: boolean; reasons: PrivateReason[]; text?: string } {
+  const reasons: PrivateReason[] = [];
+  // `path` is the resolved name and `typedPath` the one the person gave (#552, the #541 class): Bun's realpath can name another hardlink of the same file, so a name under pii.paths counts on either.
+  const named = [input.path, input.typedPath].filter((name): name is string => name !== undefined);
+  if (input.kind === "file" && named.some(name => policy.matchesPolicy(policy.cwd, name, [...policy.piiPaths]))) reasons.push("path under pii.paths");
+  let text: string | undefined;
+  try { text = policy.project?.(input.bytes); } catch { text = undefined; }
+  // A plain text file is its own text when no converter handles it and it is valid UTF-8 without NUL bytes; anything else has no text to screen.
+  if (text === undefined && !input.bytes.includes(0)) {
+    const decoded = new TextDecoder("utf-8", { fatal: false }).decode(input.bytes);
+    if (!decoded.includes("\uFFFD")) text = decoded;
+  }
+  if (text !== undefined && text.length) {
+    try { if (policy.piiPatterns.some(pattern => new RegExp(pattern).test(text!))) reasons.push("text matches signals.pii_patterns"); }
+    catch { reasons.push("text could not be screened"); }
+  }
+  return { private: reasons.length > 0, reasons, ...(text !== undefined ? { text } : {}) };
+}
+
+/** The origin a registration is shown with: a private record shows none to anyone but the person (the CLI is the person). */
+export const publicSourceView = (meta: SourceMeta): { id: string; kind: SourceKind; private: true } | SourceMeta => meta.private ? { id: meta.id, kind: meta.kind, private: true } : meta;
+/** Existence check that never throws: used to tell a fixture from a corrupt directory. */
+export const sourceDirExists = (stateDir: string, id: string): boolean => { try { return lstatSync(join(stateDir, "sources", dirName(id))).isDirectory(); } catch { return false; } };