failproofai 1.0.10 → 1.0.11-beta.0

package/src/hooks/backfill-cli.ts

@@ -29,13 +29,17 @@
  * on a real machine: the CLI reported success, and the actual failure sat in the
  * journal for twenty minutes while batches parked.
  */
-import { existsSync, mkdirSync, writeFileSync, readdirSync, statSync } from "node:fs";
-import { dirname, join } from "node:path";
+import { existsSync } from "node:fs";
 
 import { failproofaiHome } from "./fp-home";
 import { readConfig } from "./fp-config";
 import { readIngestCredential } from "./collector-config";
-import { daemonServiceStatus, isDaemonSupportedPlatform } from "./daemon-service";
+import { daemonServiceStatus, daemonVersionSkew, isDaemonSupportedPlatform } from "./daemon-service";
+import { backfillRequestPath, writeBackfillRequest } from "./backfill-request";
+import { getIntegration } from "./integrations";
+import { INTEGRATION_TYPES, type IntegrationType } from "./types";
+import { screenKit, type RenderOpts } from "./tui";
+import { getAdapter } from "../audit/cli-adapters";
 
 /** Default window. `--since` widens it. */
 export const DEFAULT_BACKFILL_DAYS = 30;
@@ -43,10 +47,21 @@ export const DEFAULT_BACKFILL_DAYS = 30;
 export interface BackfillOptions {
   /** Epoch ms. Everything modified at or after this is re-read. */
   sinceMs?: number;
+  /**
+   * `--since` as typed (`6m`, `2026-04-13`), repeated in the dry run's "run it
+   * for real" command. Without it the window is given as the date it began.
+   */
+  since?: string;
   /** Report what would be re-read and write nothing. */
   dryRun?: boolean;
+  /** `--agents`: a subset of the traced agents. Unvalidated ids, as typed. */
+  agents?: string[];
+  /** How the screen is drawn. */
+  render?: RenderOpts;
   /** Injected for tests. */
   now?: number;
+  /** Injected for tests: sessions an agent has at or after `sinceMs`. */
+  countSessions?: (cli: IntegrationType, sinceMs: number) => Promise<number>;
 }
 
 export interface BackfillResult {
@@ -54,194 +69,193 @@ export interface BackfillResult {
   lines: string[];
 }
 
-/**
- * `~/.failproofai/state/backfill-request.json` — mirrored from the daemon's
- * `paths::backfill_request_path()`. Two processes, one path; the comment there
- * says why that is written down twice rather than derived.
- */
-export function backfillRequestPath(home?: string): string {
-  return join(failproofaiHome(home), "state", "backfill-request.json");
-}
+export { backfillRequestPath };
 
 /**
- * Where each agent CLI keeps its transcripts, so the command can say what it is
- * about to re-read rather than asking for trust.
- *
- * Deliberately a SURVEY, not the source of truth. The daemon decides what is
- * actually collected from its own config; this list only produces the sentence
- * a person reads before agreeing. A directory missing here means a quieter
- * message, never a missed backfill.
+ * Agents whose history lives in a database the collector reads by row, with
+ * no time window: a backfill re-sends all of it, whatever `--since` says.
+ * Hermes keeps one database per profile, and the daemon does not rewind those
+ * at all until it can bound them (D7).
  */
-const SESSION_DIRS: Array<{ cli: string; dir: string }> = [
-  { cli: "Claude Code", dir: ".claude/projects" },
-  { cli: "OpenAI Codex", dir: ".codex/sessions" },
-  { cli: "Cursor", dir: ".cursor/chats" },
-  { cli: "GitHub Copilot", dir: ".copilot/history-session-state" },
-  { cli: "OpenCode", dir: ".local/share/opencode/storage" },
-  { cli: "Factory Droid", dir: ".factory/sessions" },
-  { cli: "Antigravity", dir: ".gemini/antigravity-cli/brain" },
-];
-
-interface Survey {
-  cli: string;
-  files: number;
-  newest: number;
-}
+const WHOLE_HISTORY: ReadonlySet<IntegrationType> = new Set(["goose", "opencode", "devin"]);
+const NOT_RESENT: ReadonlySet<IntegrationType> = new Set(["hermes"]);
 
-/** What is on disk and inside the window, per CLI. */
-function surveySessions(sinceMs: number, homeDir: string): Survey[] {
-  const out: Survey[] = [];
-  for (const { cli, dir } of SESSION_DIRS) {
-    const root = join(homeDir, dir);
-    if (!existsSync(root)) continue;
-    let files = 0;
-    let newest = 0;
-    const walk = (d: string, depth: number) => {
-      // Bounded: these trees are date-nested a few levels deep, and an
-      // unbounded walk on a symlinked home is a way to hang a CLI command.
-      if (depth > 6) return;
-      let entries;
-      try {
-        entries = readdirSync(d, { withFileTypes: true });
-      } catch {
-        return;
-      }
-      for (const e of entries) {
-        const p = join(d, e.name);
-        if (e.isDirectory()) {
-          walk(p, depth + 1);
-          continue;
-        }
-        try {
-          const st = statSync(p);
-          if (st.mtimeMs >= sinceMs) {
-            files += 1;
-            newest = Math.max(newest, st.mtimeMs);
-          }
-        } catch {
-          /* raced away mid-walk */
-        }
-      }
-    };
-    walk(root, 0);
-    if (files > 0) out.push({ cli, files, newest });
+/** Sessions an agent has in the window, from the same adapters the audit reads. */
+async function adapterSessionCount(cli: IntegrationType, sinceMs: number): Promise<number> {
+  try {
+    return (await getAdapter(cli).listTranscripts({ sinceMs })).length;
+  } catch {
+    // An agent whose store cannot be read has nothing to count; the daemon
+    // decides what is actually sent either way.
+    return 0;
   }
-  return out;
 }
 
-export function runBackfillCommand(opts: BackfillOptions = {}): BackfillResult {
+const plural = (n: number, one: string, many = `${one}s`): string => `${n.toLocaleString("en-US")} ${n === 1 ? one : many}`;
+
+/** "A", "A and B", "A, B and C". */
+function joinNames(names: string[]): string {
+  return names.length <= 1 ? (names[0] ?? "") : `${names.slice(0, -1).join(", ")} and ${names[names.length - 1]}`;
+}
+
+export async function runBackfillCommand(opts: BackfillOptions = {}): Promise<BackfillResult> {
+  const kit = screenKit(opts.render ?? {});
   const now = opts.now ?? Date.now();
   const sinceMs = opts.sinceMs ?? now - DEFAULT_BACKFILL_DAYS * 24 * 60 * 60 * 1000;
+  const sinceLabel = new Date(sinceMs).toISOString().slice(0, 10);
   const home = failproofaiHome();
+  const fail = (text: string, ...fix: string[]): BackfillResult => ({ exitCode: 1, lines: [kit.fail(text), ...fix] });
 
   if (!existsSync(home)) {
-    return {
-      exitCode: 1,
-      lines: [`No failproofai home at ${home}. Run \`failproofai config\` first.`],
-    };
+    return fail(`There is no failproofai home at ${home}.`, `  Set this machine up first:  ${kit.cmd("failproofai config")}`);
   }
 
   // Checked HERE, not left to the daemon. Handing off an impossible request and
   // reporting success is the failure this command is partly a response to.
-  const ingest = readIngestCredential();
-  if (!ingest) {
-    return {
-      exitCode: 1,
-      lines: [
-        "This machine is not connected, so there is nowhere to send history.",
-        "Connect first: `failproofai config --token <key>`.",
-      ],
-    };
+  if (!readIngestCredential()) {
+    return fail(
+      "This machine isn't connected, so there is nowhere to send history.",
+      `  Connect it:  ${kit.cmd("failproofai config")}`,
+    );
   }
 
   let sessions = false;
   let hooks = false;
+  let selected: IntegrationType[] | null = null;
   try {
     const cfg = readConfig();
     sessions = cfg.collector?.sessions ?? false;
     hooks = cfg.collector?.hooks ?? false;
+    if (cfg.agents) selected = INTEGRATION_TYPES.filter((id) => cfg.agents?.selected.includes(id));
   } catch {
     /* an unreadable config is reported as "nothing enabled" below */
   }
   if (!sessions && !hooks) {
-    return {
-      exitCode: 1,
-      lines: [
-        "Collection is switched off, so a backfill would re-read history and send",
-        'none of it. Enable it in ~/.failproofai/config.json under "collector".',
-      ],
-    };
+    return fail(
+      "Collection is switched off, so a backfill would send nothing.",
+      `  Turn it on under "collector" in  ${kit.cmd("~/.failproofai/config.json")}`,
+    );
   }
 
-  const found = surveySessions(sinceMs, dirname(home));
-  const totalFiles = found.reduce((n, f) => n + f.files, 0);
-  const sinceLabel = new Date(sinceMs).toISOString().slice(0, 10);
+  // Traced agents only. With no saved selection every agent is traced, which
+  // is how the daemon reads the same file.
+  const traced = selected ?? [...INTEGRATION_TYPES];
+  let target = traced;
+  if (opts.agents && opts.agents.length > 0) {
+    const unknown = opts.agents.filter((id) => !(INTEGRATION_TYPES as readonly string[]).includes(id));
+    if (unknown.length > 0) {
+      return fail(`Not an agent: ${unknown.join(", ")}.`, `  Agents:  ${INTEGRATION_TYPES.join(", ")}`);
+    }
+    const asked = INTEGRATION_TYPES.filter((id) => opts.agents?.includes(id));
+    const untraced = asked.filter((id) => !traced.includes(id));
+    if (untraced.length > 0) {
+      const names = joinNames(untraced.map((id) => getIntegration(id).displayName));
+      return fail(
+        `${names} ${untraced.length === 1 ? "isn't" : "aren't"} traced, so ${untraced.length === 1 ? "it" : "they"} can't be backfilled.`,
+        `  Trace ${untraced.length === 1 ? "it" : "them"} first with  ${kit.cmd("failproofai config")}  or  ${kit.cmd(
+          `failproofai config --agents ${[...new Set([...traced, ...untraced])].filter((id) => INTEGRATION_TYPES.includes(id)).join(",")}`,
+        )}`,
+      );
+    }
+    // A daemon older than the scoped request reads every request as "rewind
+    // everything", so naming agents to it would re-send all of them. Refused on
+    // any mismatch: which side is newer cannot be told from here reliably, and
+    // `update` brings both to one version either way.
+    const skew = daemonVersionSkew();
+    if (skew) {
+      return fail(
+        `failproofaid ${skew.installed} is running, but this CLI ships ${skew.expected}; naming agents needs them to match.`,
+        `  Bring them in line:  ${kit.cmd("failproofai update")}`,
+      );
+    }
+    target = asked;
+  }
 
-  const lines: string[] = [
-    `Re-reading everything modified since ${sinceLabel}, and sending it again.`,
-    "",
-  ];
-  // Named honestly: only what the config actually enables, because that is all
-  // the daemon will ship. Promising transcripts on a machine with
-  // `sessions = false` would be a lie the user could only catch by waiting.
-  lines.push(
-    `  streams: ${[sessions && "session transcripts", hooks && "hook decisions"]
-      .filter(Boolean)
-      .join(" + ")}`,
-  );
+  const count = opts.countSessions ?? adapterSessionCount;
+  const rows: Array<[string, string]> = [];
+  let total = 0;
+  let agentsWithSessions = 0;
   if (sessions) {
-    if (found.length === 0) {
-      lines.push("  sessions: none found on disk in this window");
-    } else {
-      for (const f of found) {
-        lines.push(`  ${f.cli}: ${f.files} file${f.files === 1 ? "" : "s"}`);
+    for (const id of target) {
+      const name = getIntegration(id).displayName;
+      if (NOT_RESENT.has(id)) {
+        rows.push([name, "not re-sent: its sessions have no time window yet"]);
+        continue;
       }
+      const n = await count(id, WHOLE_HISTORY.has(id) ? 0 : sinceMs);
+      if (n === 0) continue;
+      total += n;
+      agentsWithSessions += 1;
+      rows.push([name, WHOLE_HISTORY.has(id) ? `all ${plural(n, "session")}, whatever the window` : plural(n, "session")]);
     }
   }
+  const skipped = INTEGRATION_TYPES.filter((id) => !traced.includes(id)).map((id) => getIntegration(id).displayName);
 
   if (opts.dryRun) {
-    lines.push("", "--dry-run: nothing was requested.");
+    const lines = [kit.header("Backfill"), "", kit.head("Would re-send", `since ${sinceLabel}, traced agents only`)];
+    if (!sessions) lines.push(`${"  "}Hook decisions only: session collection is off.`);
+    else if (rows.length === 0) lines.push("  No sessions on disk in this window.");
+    else lines.push(...kit.rows(rows, 14));
+    if (skipped.length > 0) lines.push(`  Not traced, skipped: ${skipped.join(", ")}`);
+    // Every option the preview was given, or the command it hands over re-sends
+    // a different window: without --since a real run takes the default 30 days.
+    const forReal = ["failproofai backfill"];
+    if (opts.sinceMs !== undefined) {
+      // A date Date.parse reads can hold spaces ("April 13 2026"); quoted, so
+      // the line still runs as printed.
+      const since = opts.since ?? sinceLabel;
+      forReal.push(`--since ${/\s/.test(since) ? `"${since}"` : since}`);
+    }
+    if (opts.agents) forReal.push(`--agents ${target.join(",")}`);
+    lines.push("", `Run it for real with ${kit.cmd(forReal.join(" "))}`);
     return { exitCode: 0, lines };
   }
 
   try {
-    const path = backfillRequestPath();
-    mkdirSync(dirname(path), { recursive: true });
-    // Whole-file write of a small JSON object: the daemon deletes it before
-    // acting, so a partially-written file can only ever be read once, and an
-    // unparseable one is discarded rather than retried.
-    writeFileSync(path, `${JSON.stringify({ sinceMs, requestedAtMs: now }, null, 2)}\n`, {
-      mode: 0o600,
+    // Merged with anything still pending rather than written over it, and
+    // atomically: see backfill-request.ts. Without --agents the request names
+    // nobody, which the daemon reads as "every traced agent" at the moment it
+    // acts — a selection changed in between is honoured, not a stale copy.
+    writeBackfillRequest({
+      kind: "user",
+      sinceMs,
+      requestedAtMs: now,
+      ...(opts.agents && opts.agents.length > 0 ? { agents: target } : {}),
     });
   } catch (err) {
     return {
       exitCode: 2,
-      lines: [
-        ...lines,
-        "",
-        `Could not write the request: ${err instanceof Error ? err.message : String(err)}`,
-      ],
+      lines: [kit.fail(`Could not write the request: ${err instanceof Error ? err.message : String(err)}`)],
     };
   }
 
-  lines.push("", "Requested. The daemon picks this up within a few seconds.");
-
-  // A daemon that is not running will act on the request whenever it next
-  // starts — the request is a file, deliberately, so it survives that. Saying so
-  // is the difference between "nothing happened" and "nothing happened yet".
+  const what = !sessions
+    ? `hook decisions since ${sinceLabel}`
+    : total === 0
+      ? `everything since ${sinceLabel}`
+      : `${plural(total, "session")} from ${plural(agentsWithSessions, "traced agent")} since ${sinceLabel}`;
+  const lines = [kit.header("Backfill"), "", kit.ok(`Asked the daemon to re-send ${what}.`)];
+  // Named honestly: only what the config enables, because that is all the
+  // daemon ships.
+  if (sessions && !hooks) lines.push("  Session transcripts only: hook activity collection is off.");
+  if (!sessions) lines.push("  Session transcripts are not sent: session collection is off.");
+  // A daemon that is not running acts on the request when it next starts — the
+  // request is a file, deliberately, so it survives that.
+  let waiting = false;
   if (isDaemonSupportedPlatform()) {
     const status = daemonServiceStatus();
     if (status !== "running") {
+      waiting = true;
       lines.push(
-        `failproofaid is ${status}, so nothing moves until it starts. The request is`,
-        "on disk and will be honoured then.",
+        kit.caution(`failproofaid is ${status}, so nothing moves until it starts. The request waits for it.`),
+        `  Check it:  ${kit.cmd("failproofai config --status")}`,
       );
     }
   }
-  lines.push(
-    "",
-    `Watch it land:  ${totalFiles > 0 ? `~${totalFiles} files queued · ` : ""}` +
-      "`failproofai config --status`",
-  );
+  if (!waiting) {
+    // When, not "now": the daemon re-reads first, so `flush --wait` would only
+    // empty a queue that is still being refilled.
+    lines.push(`  They reach the dashboard over the next few minutes. Watch them land:  ${kit.cmd("failproofai config --status")}`);
+  }
   return { exitCode: 0, lines };
 }

package/src/hooks/backfill-request.ts

@@ -0,0 +1,149 @@
+/**
+ * The one writer of `~/.failproofai/state/backfill-request.json`.
+ *
+ * Two kinds of request go in it: `failproofai backfill` writes a `user`
+ * request (re-send what was modified since `sinceMs`), and every path that
+ * traces an agent again writes an `added` one through `requestFreshStart`
+ * (`failproofai config`, and `policies --install`, `policies add` and the
+ * dashboard's agent panel through `keepAgentSelectionTrue`): agents whose old
+ * cursors must not be resumed. The file is a single slot, so each writer MERGES with whatever is
+ * still pending instead of replacing it — otherwise config's request would
+ * quietly cancel a backfill somebody asked for and the daemon has not picked up
+ * yet. The daemon's reader is `crates/failproofaid/src/backfill.rs`.
+ *
+ * Merging keeps the stronger request. A pending `user` request stays `user`:
+ * its window is the one somebody chose, and an added agent swept into it is
+ * re-read over that window instead of the default seven days. A request with no
+ * `agents` reaches every traced agent, so a merge with one has no list either.
+ *
+ * An `added` request never carries `sinceMs`. A daemon older than `kind` reads
+ * every request as a backfill of `sinceMs`, so one written with it would
+ * rewind every agent on such a machine; without it, the old daemon drops it.
+ */
+import { existsSync, readFileSync } from "node:fs";
+import { join } from "node:path";
+
+import { writeJsonAtomically } from "../../lib/atomic-write";
+import { readIngestCredential } from "./collector-config";
+import { readConfig } from "./fp-config";
+import { failproofaiHome } from "./fp-home";
+import { hookLogWarn } from "./hook-logger";
+
+/** How long the daemon acts on an `added` request; `ADDED_REQUEST_MAX_AGE` there. */
+export const ADDED_REQUEST_MAX_AGE_MS = 10 * 60 * 1000;
+
+export type BackfillRequest =
+  | { kind: "user"; sinceMs: number; requestedAtMs: number; agents?: string[] }
+  | { kind: "added"; requestedAtMs: number; agents: string[] };
+
+/**
+ * `~/.failproofai/state/backfill-request.json` — mirrored from the daemon's
+ * `paths::backfill_request_path()`. Two processes, one path; the comment there
+ * says why that is written down twice rather than derived.
+ */
+export function backfillRequestPath(home?: string): string {
+  return join(failproofaiHome(home), "state", "backfill-request.json");
+}
+
+/** A pending request as the daemon would read it, or null when it would drop it. */
+export function readPendingRequest(raw: unknown, now: number): BackfillRequest | null {
+  if (typeof raw !== "object" || raw === null || Array.isArray(raw)) return null;
+  const r = raw as Record<string, unknown>;
+  let agents: string[] | undefined;
+  if (Array.isArray(r.agents)) {
+    agents = [...new Set(r.agents.filter((a): a is string => typeof a === "string" && a.trim() !== ""))];
+  } else if (r.agents !== undefined && r.agents !== null) {
+    return null;
+  }
+  const requestedAtMs = typeof r.requestedAtMs === "number" ? r.requestedAtMs : undefined;
+  const kind = r.kind === undefined || r.kind === null ? "user" : r.kind;
+  if (kind === "user") {
+    if (typeof r.sinceMs !== "number") return null;
+    return { kind: "user", sinceMs: r.sinceMs, requestedAtMs: requestedAtMs ?? now, ...(agents ? { agents } : {}) };
+  }
+  if (kind === "added") {
+    if (requestedAtMs === undefined || !agents) return null;
+    if (now - requestedAtMs > ADDED_REQUEST_MAX_AGE_MS) return null;
+    return { kind: "added", requestedAtMs, agents };
+  }
+  return null;
+}
+
+/** What `next` becomes when `pending` is still waiting. Pure, for tests. */
+export function mergeBackfillRequests(pending: BackfillRequest | null, next: BackfillRequest): BackfillRequest {
+  if (!pending) return next;
+  const union = (a?: string[], b?: string[]): string[] | undefined =>
+    a === undefined || b === undefined ? undefined : [...new Set([...a, ...b])];
+  if (pending.kind === "added" && next.kind === "added") {
+    return { kind: "added", requestedAtMs: next.requestedAtMs, agents: [...new Set([...pending.agents, ...next.agents])] };
+  }
+  const sinces = [pending, next].flatMap((r) => (r.kind === "user" ? [r.sinceMs] : []));
+  const agents = union(pending.agents, next.agents);
+  return {
+    kind: "user",
+    sinceMs: Math.min(...sinces),
+    requestedAtMs: next.requestedAtMs,
+    ...(agents ? { agents } : {}),
+  };
+}
+
+/** Write `next`, merged with whatever is still pending. Returns what was written. Throws on I/O failure. */
+export function writeBackfillRequest(next: BackfillRequest, home?: string): BackfillRequest {
+  const path = backfillRequestPath(home);
+  let pending: BackfillRequest | null = null;
+  if (existsSync(path)) {
+    try {
+      pending = readPendingRequest(JSON.parse(readFileSync(path, "utf8")), next.requestedAtMs);
+    } catch {
+      // Unparseable: the daemon discards it, and so does this.
+      pending = null;
+    }
+  }
+  const merged = mergeBackfillRequests(pending, next);
+  writeJsonAtomically(path, merged);
+  return merged;
+}
+
+/**
+ * Whether this machine collects session transcripts now: an ingest key on disk
+ * and `collector.sessions` on. A machine that is about to connect is the
+ * caller's to know — `failproofai config` decides that itself.
+ */
+export function machineCollectsSessions(): boolean {
+  try {
+    return readIngestCredential() !== null && readConfig().collector.sessions === true;
+  } catch {
+    // Unreadable credentials or config: the daemon cannot collect from them either.
+    return false;
+  }
+}
+
+/**
+ * Agents traced again start fresh: their old cursors are forgotten and the
+ * collector reads the default window, exactly as for an agent never traced.
+ * Without this, an agent unticked while its files kept growing resumes the old
+ * offsets once it is traced again, and ships what was written while nobody
+ * asked for it — the job the `added` request exists for (backfill.rs).
+ *
+ * The ONE place that decides it, so every path that re-traces an agent behaves
+ * the same. Call it BEFORE writing the selection that traces them: the daemon
+ * handles a pending request ahead of its config compare, so one landing a tick
+ * later would let the agent resume its stale cursors first.
+ *
+ * Only when something will be collecting (`collecting`): on an open-source
+ * machine the request would wait for a connection that may never come, and the
+ * daemon drops one that old anyway. Never throws; a failure is logged, and the
+ * caller's own write goes ahead. Returns whether a request was written.
+ */
+export function requestFreshStart(agents: readonly string[], collecting: boolean, now = Date.now()): boolean {
+  if (agents.length === 0 || !collecting) return false;
+  try {
+    writeBackfillRequest({ kind: "added", agents: [...agents], requestedAtMs: now });
+    return true;
+  } catch (err) {
+    hookLogWarn(
+      `could not write the backfill request for re-added agents: ${err instanceof Error ? err.message : String(err)}`,
+    );
+    return false;
+  }
+}

package/src/hooks/cloud-connection.ts

@@ -414,7 +414,7 @@ writeCloudCredentials(creds);
  * this machine now, and — the one case worth a second line — why an existing
  * `jev.json` was not touched.
  */
-function jevLines(outcome: ConnectOutcome): string[] {
+export function jevLines(outcome: ConnectOutcome): string[] {
   const jev = outcome.jev;
   if (!jev) return [];
   if (jev.unconfirmed === "cleared") {