backpass 0.1.0

package/src/discovery/adapters/sqlite.js

@@ -0,0 +1,50 @@
+import fs from "node:fs";
+
+/**
+ * `node:sqlite` is still flagged experimental, so Node prints a warning on first use.
+ * backpass opens several read-only stores; the warning is noise the user cannot act on,
+ * so it is filtered here (and only here) rather than suppressed process-wide.
+ */
+let DatabaseSync = null;
+let loadError = null;
+
+function filterExperimentalWarning() {
+  const listeners = process.listeners("warning");
+  process.removeAllListeners("warning");
+  process.on("warning", (w) => {
+    if (w.name === "ExperimentalWarning" && /SQLite/i.test(w.message)) return;
+    for (const listener of listeners) listener(w);
+    if (!listeners.length) console.error(w.stack || String(w));
+  });
+}
+
+async function loadDriver() {
+  if (DatabaseSync || loadError) return;
+  filterExperimentalWarning();
+  try {
+    ({ DatabaseSync } = await import("node:sqlite"));
+  } catch (err) {
+    loadError = err;
+  }
+}
+
+/**
+ * Open a SQLite file read-only. Returns null (never throws) when the store is missing,
+ * locked, or the driver is unavailable - discovery is fail-soft per harness.
+ */
+export async function openReadOnly(file) {
+  if (!fs.existsSync(file)) return null;
+  await loadDriver();
+  if (!DatabaseSync) {
+    throw new Error(`node:sqlite unavailable (${loadError?.message || "unknown"}); Node >= 22.5 is required`);
+  }
+  return new DatabaseSync(file, { readOnly: true });
+}
+
+export function safeJsonParse(text) {
+  try {
+    return JSON.parse(text);
+  } catch {
+    return null;
+  }
+}

package/src/discovery/association.js

@@ -0,0 +1,100 @@
+import fs from "node:fs";
+import path from "node:path";
+
+import { normalizeRemote } from "../repo.js";
+
+/**
+ * Three-tier repo/worktree association (design section 2.1).
+ *
+ *   tier 1  deterministic  - transcript cwd is (or sits under) a live worktree path
+ *   tier 2  deterministic  - a recorded git remote matches one of the repo's remotes;
+ *                            survives worktree deletion (codex, grok)
+ *   tier 3  best-effort    - dead cwd whose last segment is the repo dir name, or that
+ *                            matches a user-supplied worktree glob; excluded by --strict
+ *
+ * Returns null when the transcript belongs to some other repo.
+ */
+
+function realpathOrResolve(p) {
+  try {
+    return fs.realpathSync(p);
+  } catch {
+    return path.resolve(p);
+  }
+}
+
+function isUnder(child, parent) {
+  if (child === parent) return true;
+  return child.startsWith(parent.endsWith(path.sep) ? parent : parent + path.sep);
+}
+
+/** Glob support is intentionally minimal: `*` (one segment) and `**` (many). */
+export function globToRegExp(glob) {
+  const expanded = glob.startsWith("~/") ? path.join(process.env.HOME || "", glob.slice(2)) : glob;
+  let out = "";
+  for (let i = 0; i < expanded.length; i += 1) {
+    const c = expanded[i];
+    if (c === "*") {
+      if (expanded[i + 1] === "*") {
+        out += ".*";
+        i += 1;
+        if (expanded[i + 1] === "/") i += 1;
+      } else {
+        out += "[^/]*";
+      }
+    } else if ("\\^$.|?+()[]{}".includes(c)) {
+      out += `\\${c}`;
+    } else {
+      out += c;
+    }
+  }
+  return new RegExp(`^${out}/?$`);
+}
+
+export function associate({ cwd, remotes = [], gitRoot = null }, repo, options = {}) {
+  const globs = options.worktreeGlobs || [];
+  const candidates = [cwd, gitRoot].filter(Boolean);
+
+  // Tier 1 - live path under a known worktree.
+  for (const candidate of candidates) {
+    const real = realpathOrResolve(candidate);
+    for (const worktree of repo.worktrees) {
+      if (real === worktree) {
+        return { tier: 1, confidence: "exact", reason: `cwd is worktree ${worktree}` };
+      }
+      if (isUnder(real, worktree)) {
+        return { tier: 1, confidence: "nested", reason: `cwd is inside worktree ${worktree}` };
+      }
+    }
+  }
+
+  // Tier 2 - recorded remote, valid even when the worktree is long gone.
+  const repoRemotes = new Set(repo.remotes);
+  for (const remote of remotes) {
+    const norm = normalizeRemote(remote);
+    if (norm && repoRemotes.has(norm)) {
+      return { tier: 2, confidence: "remote", reason: `recorded remote ${norm}` };
+    }
+  }
+
+  // Tier 3 - best-effort, only for paths that no longer exist.
+  for (const candidate of candidates) {
+    const resolved = path.resolve(candidate);
+    if (fs.existsSync(resolved)) continue;
+    if (path.basename(resolved.replace(/\/+$/, "")) === repo.name) {
+      return { tier: 3, confidence: "path", reason: `dead path ending in /${repo.name}` };
+    }
+    for (const glob of globs) {
+      if (globToRegExp(glob).test(resolved)) {
+        return { tier: 3, confidence: "glob", reason: `dead path matches glob ${glob}` };
+      }
+    }
+  }
+
+  return null;
+}
+
+export function passesStrict(association, strict) {
+  if (!association) return false;
+  return strict ? association.tier <= 2 : true;
+}

package/src/discovery/index.js

@@ -0,0 +1,226 @@
+import * as claude from "./adapters/claude.js";
+import * as codex from "./adapters/codex.js";
+import * as pi from "./adapters/pi.js";
+import * as grok from "./adapters/grok.js";
+import * as opencode from "./adapters/opencode.js";
+import * as cursorCli from "./adapters/cursor-cli.js";
+import * as cursorIde from "./adapters/cursor-ide.js";
+
+import { associate, passesStrict } from "./association.js";
+import { isSelfSession } from "./self.js";
+import { sinceCutoff } from "../config.js";
+import { emitProgress } from "../progress.js";
+import { warn } from "../logger.js";
+
+export const ADAPTERS = {
+  claude,
+  codex,
+  pi,
+  grok,
+  opencode,
+  cursor: cursorCli,
+  "cursor-ide": cursorIde,
+};
+
+export function getAdapter(harness) {
+  return ADAPTERS[harness] || null;
+}
+
+/**
+ * Discovery (design section 2).
+ *
+ * For file-backed stores the expensive step is reading each transcript's header, so
+ * results are memoised in `.backpass/scan-cache.json` keyed by path + mtime + size.
+ * Re-scans are then O(new files) - which matters: codex alone had 10,317 rollouts on
+ * the machine this was designed against.
+ *
+ * SQLite-backed stores (opencode, cursor IDE) answer the same question with one indexed
+ * query, so they skip the cache entirely.
+ *
+ * Every harness is fail-soft: a store that is missing, unreadable, or has drifted into
+ * an unrecognised format produces a named warning and is skipped, never a failed run.
+ *
+ * Sessions backpass itself created (its analysis and synthesis calls, which the harness
+ * files under this repo's cwd) are excluded after association and counted in
+ * `perHarness[h].self` - see `./self.js`.
+ */
+export async function discoverTranscripts({ repo, config, strict = false, harnesses = null, now = Date.now() }) {
+  const cutoffMs = sinceCutoff(config.discovery.since, now);
+  const selected = harnesses || config.discovery.harnesses;
+  const cache = config.state.readScanCache();
+
+  const transcripts = [];
+  const perHarness = {};
+  let cacheDirty = false;
+
+  emitProgress("discover:start", { harnesses: selected.filter((h) => getAdapter(h)) });
+
+  for (const harness of selected) {
+    const adapter = getAdapter(harness);
+    if (!adapter) {
+      warn(`no adapter for harness "${harness}" - skipped`);
+      continue;
+    }
+
+    const stats = { scanned: 0, matched: 0, cached: 0, skipped: 0, self: 0, error: null };
+    perHarness[harness] = stats;
+    emitProgress("discover:harness:start", { harness });
+
+    try {
+      const found = adapter.discover
+        ? await discoverDirect(adapter, { repo, config, cutoffMs, strict, stats })
+        : discoverFiles(adapter, {
+            repo,
+            config,
+            cutoffMs,
+            strict,
+            stats,
+            cache,
+            markDirty: () => {
+              cacheDirty = true;
+            },
+          });
+      transcripts.push(...found);
+      stats.matched = found.length;
+      emitProgress("discover:harness:done", {
+        harness,
+        scanned: stats.scanned,
+        cached: stats.cached,
+        matched: stats.matched,
+        self: stats.self,
+        tiers: tierCounts(found),
+      });
+    } catch (err) {
+      stats.error = err.message;
+      warn(`${harness}: transcript store unreadable (${err.message}) - harness skipped`);
+      emitProgress("discover:harness:done", { harness, error: err.message });
+    }
+  }
+
+  if (cacheDirty) config.state.writeScanCache(cache);
+
+  transcripts.sort((a, b) => (b.mtimeMs || 0) - (a.mtimeMs || 0));
+  emitProgress("discover:done", { total: transcripts.length });
+  return { transcripts, perHarness, cutoffMs };
+}
+
+function tierCounts(found) {
+  const tiers = {};
+  for (const transcript of found) {
+    const tier = transcript.association?.tier;
+    if (tier) tiers[tier] = (tiers[tier] || 0) + 1;
+  }
+  return tiers;
+}
+
+async function discoverDirect(adapter, { repo, config, cutoffMs, strict, stats }) {
+  const rows = await adapter.discover({ cutoffMs, repo, config });
+  const out = [];
+  for (const row of rows) {
+    stats.scanned += 1;
+    const association = associate({ cwd: row.cwd, remotes: row.remotes || [], gitRoot: row.gitRoot }, repo, {
+      worktreeGlobs: config.discovery.worktreeGlobs,
+    });
+    if (!passesStrict(association, strict)) {
+      stats.skipped += 1;
+      continue;
+    }
+    const transcript = toTranscript(adapter, row, association, row.id);
+    if (isSelfSession(transcript)) {
+      stats.self += 1;
+      continue;
+    }
+    out.push(transcript);
+  }
+  return out;
+}
+
+function discoverFiles(adapter, { repo, config, cutoffMs, strict, stats, cache, markDirty }) {
+  const candidates = adapter.enumerate({ cutoffMs, repo, config });
+  const out = [];
+
+  for (const candidate of candidates) {
+    if (cutoffMs && candidate.mtimeMs < cutoffMs) continue;
+    stats.scanned += 1;
+    // Large stores (codex holds 10k+ session files) get a live scan tick; the
+    // classify loop is synchronous, so this is the only paint opportunity.
+    if (stats.scanned % 25 === 0) {
+      emitProgress("discover:harness:tick", {
+        harness: adapter.name,
+        scanned: stats.scanned,
+        total: candidates.length,
+        matched: out.length,
+      });
+    }
+
+    const cacheKey = `${adapter.name}:${candidate.key}`;
+    const cached = cache.entries[cacheKey];
+    let descriptor;
+
+    if (cached && cached.mtimeMs === candidate.mtimeMs && cached.bytes === candidate.bytes) {
+      stats.cached += 1;
+      descriptor = cached.descriptor;
+    } else {
+      descriptor = adapter.classify(candidate, { repo, config }) || null;
+      cache.entries[cacheKey] = { mtimeMs: candidate.mtimeMs, bytes: candidate.bytes, descriptor };
+      markDirty();
+    }
+
+    if (!descriptor) {
+      stats.skipped += 1;
+      continue;
+    }
+
+    const association = associate(
+      { cwd: descriptor.cwd, remotes: descriptor.remotes || [], gitRoot: descriptor.gitRoot },
+      repo,
+      { worktreeGlobs: config.discovery.worktreeGlobs },
+    );
+    if (!passesStrict(association, strict)) {
+      stats.skipped += 1;
+      continue;
+    }
+
+    const transcript = toTranscript(adapter, { ...candidate, ...descriptor }, association, descriptor.id);
+    // backpass's own acpx runs land in this store under this cwd; drop them here so
+    // they never reach sampling or analysis (see ./self.js).
+    if (isSelfSession(transcript)) {
+      stats.self += 1;
+      continue;
+    }
+    out.push(transcript);
+  }
+
+  return out;
+}
+
+function toTranscript(adapter, row, association, id) {
+  return {
+    harness: adapter.name,
+    id: `${adapter.name}-${id}`,
+    nativeId: id,
+    path: row.path,
+    cwd: row.cwd || null,
+    gitBranch: row.gitBranch || null,
+    title: row.title || null,
+    model: row.model || null,
+    startedAt: row.startedAt || null,
+    mtimeMs: row.mtimeMs || 0,
+    bytes: row.bytes || 0,
+    experimental: Boolean(adapter.experimental),
+    association,
+    extra: row.extra || {},
+  };
+}
+
+/** Read one transcript through its adapter and normalize it to distiller events. */
+export async function readTranscript(transcript) {
+  const adapter = getAdapter(transcript.harness);
+  if (!adapter) throw new Error(`no adapter for harness ${transcript.harness}`);
+  const result = await adapter.read(transcript);
+  return {
+    events: result.events || [],
+    model: result.model || transcript.model || null,
+    rawPath: adapter.rawPath ? adapter.rawPath(transcript) : transcript.path,
+  };
+}

package/src/discovery/self.js

@@ -0,0 +1,62 @@
+import fs from "node:fs";
+
+import { SELF_SESSION_SENTINEL } from "../prompts.js";
+
+/**
+ * Self-session exclusion.
+ *
+ * backpass's own model calls run through acpx with `--cwd <repo root>`, so the harness
+ * files each one in its normal store under this repo's cwd - a tier-1 exact match for
+ * discovery. Left alone, the next run would analyze backpass talking to itself (and a
+ * synthesis session quotes the memory file verbatim, so the "evidence" would be
+ * backpass's own text). Every prompt backpass writes begins with
+ * `SELF_SESSION_SENTINEL`; a transcript whose first user message starts with it is
+ * backpass's and is dropped here, before sampling, so it never takes a slot or a
+ * model call.
+ *
+ * The check reads only the head of the file and keys on the JSON-encoded user text as
+ * every file-backed harness records it (`"text":"..."` / `"content":"..."` /
+ * `"message":"..."`). SQLite-backed stores (opencode, cursor IDE) have no file to
+ * inspect and acpx does not drive them, so they are passed through.
+ */
+
+const HEAD_BYTES = 256 * 1024;
+
+const SENTINEL_PATTERN = new RegExp(`"(?:text|content|message)":"${escapeRegExp(jsonInner(SELF_SESSION_SENTINEL))}`);
+
+function jsonInner(text) {
+  return JSON.stringify(text).slice(1, -1);
+}
+
+function escapeRegExp(text) {
+  return text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+
+/** True when `text` (the head of a transcript file) opens a user message with the sentinel. */
+export function headHasSelfSentinel(text) {
+  return SENTINEL_PATTERN.test(text || "");
+}
+
+/**
+ * @param {{ path?: string | null }} transcript
+ * @returns {boolean}
+ */
+export function isSelfSession(transcript) {
+  const file = transcript?.path;
+  if (!file) return false;
+  let fd;
+  try {
+    fd = fs.openSync(file, "r");
+    const stat = fs.fstatSync(fd);
+    if (!stat.isFile()) return false;
+    const length = Math.min(stat.size, HEAD_BYTES);
+    const buffer = Buffer.alloc(length);
+    fs.readSync(fd, buffer, 0, length, 0);
+    return headHasSelfSentinel(buffer.toString("utf8"));
+  } catch {
+    // Unreadable here means unreadable for the adapter too; let the adapter report it.
+    return false;
+  } finally {
+    if (fd !== undefined) fs.closeSync(fd);
+  }
+}

package/src/distill.js

@@ -0,0 +1,182 @@
+import { redact } from "./redact.js";
+import { estimateTokens } from "./tokens.js";
+
+/**
+ * Stage 0 of the pipeline (design section 3): turn a raw session log into a distilled
+ * markdown trace, with no model involved.
+ *
+ * The point is cheap-first analysis. Raw transcripts on this machine run to megabytes,
+ * almost all of it tool-call noise. Distillation keeps what carries the loss signal -
+ * what the human asked, what the agent said, and a one-line shape of each tool call -
+ * and drops the rest. The trace ends with the raw transcript path so the analysis agent
+ * can open the original when (and only when) a claim needs it.
+ *
+ * Adapters produce a normalized event stream; everything below is shared.
+ */
+
+const TOOL_INPUT_CHARS = 160;
+const TOOL_OUTPUT_CHARS = 200;
+const MESSAGE_CHARS = 6000;
+
+function oneLine(text, limit) {
+  const flat = String(text ?? "")
+    .replace(/\s+/g, " ")
+    .trim();
+  return flat.length > limit ? `${flat.slice(0, limit)}...` : flat;
+}
+
+function clampMessage(text) {
+  const trimmed = String(text ?? "").trim();
+  if (trimmed.length <= MESSAGE_CHARS) return trimmed;
+  const head = trimmed.slice(0, MESSAGE_CHARS - 1200);
+  const tail = trimmed.slice(-1000);
+  return `${head}\n\n[... ${trimmed.length - MESSAGE_CHARS} chars elided ...]\n\n${tail}`;
+}
+
+function describeToolInput(input) {
+  if (input === null || input === undefined) return "";
+  if (typeof input === "string") return oneLine(input, TOOL_INPUT_CHARS);
+  // Prefer the field a human would recognise for the common tools.
+  for (const key of ["command", "cmd", "file_path", "path", "pattern", "query", "url", "description"]) {
+    if (typeof input[key] === "string" && input[key].trim()) {
+      return oneLine(input[key], TOOL_INPUT_CHARS);
+    }
+  }
+  try {
+    return oneLine(JSON.stringify(input), TOOL_INPUT_CHARS);
+  } catch {
+    return "";
+  }
+}
+
+function describeToolResult(result) {
+  if (result === null || result === undefined) return "";
+  const text = typeof result === "string" ? result : safeStringify(result);
+  const bytes = Buffer.byteLength(text, "utf8");
+  const summary = oneLine(text, TOOL_OUTPUT_CHARS);
+  return bytes > TOOL_OUTPUT_CHARS ? `${summary} (output ${formatBytes(bytes)}, truncated)` : summary;
+}
+
+function safeStringify(value) {
+  try {
+    return JSON.stringify(value);
+  } catch {
+    return String(value);
+  }
+}
+
+function formatBytes(bytes) {
+  if (bytes < 1024) return `${bytes}B`;
+  if (bytes < 1024 * 1024) return `${Math.round(bytes / 1024)}KB`;
+  return `${(bytes / (1024 * 1024)).toFixed(1)}MB`;
+}
+
+/**
+ * Injected harness scaffolding (system reminders, environment dumps, plugin lists) is
+ * not user intent and is identical across every session; keeping it would drown the
+ * real signal and inflate every prompt.
+ */
+const BOILERPLATE = [
+  /^<system-reminder>/,
+  /^<user_info>/,
+  /^<recommended_plugins>/,
+  /^<permissions instructions>/,
+  /^<env>/,
+  /^Caveat: The messages below were generated/,
+];
+
+export function isBoilerplate(text) {
+  const trimmed = String(text ?? "").trim();
+  if (!trimmed) return true;
+  return BOILERPLATE.some((re) => re.test(trimmed));
+}
+
+/**
+ * Normalized event shapes accepted from adapters:
+ *   { kind: 'message', role: 'user'|'assistant', text }
+ *   { kind: 'tool', name, input, result, status }
+ *   { kind: 'thinking', text }   (dropped - reasoning traces are noise for this purpose)
+ */
+export function distill(events, meta, options = {}) {
+  const maxTraceTokens = options.maxTraceTokens ?? 12000;
+  const lines = [];
+  let userTurns = 0;
+  let assistantTurns = 0;
+  let toolCalls = 0;
+  let turn = 0;
+
+  for (const event of events) {
+    if (!event) continue;
+    if (event.kind === "message") {
+      const text = clampMessage(redact(event.text));
+      if (!text || isBoilerplate(text)) continue;
+      turn += 1;
+      if (event.role === "user") userTurns += 1;
+      else assistantTurns += 1;
+      lines.push(`### turn ${turn} · ${event.role}`);
+      lines.push(text);
+      lines.push("");
+    } else if (event.kind === "tool") {
+      toolCalls += 1;
+      const input = redact(describeToolInput(event.input));
+      const result = redact(describeToolResult(event.result));
+      const status = event.status && event.status !== "completed" ? ` [${event.status}]` : "";
+      const arrow = result ? ` -> ${result}` : "";
+      lines.push(`tool: ${event.name || "unknown"}${input ? ` ${JSON.stringify(input)}` : ""}${status}${arrow}`);
+    }
+  }
+
+  const header = [
+    `# session ${meta.id}`,
+    `harness: ${meta.harness}`,
+    meta.model ? `model: ${meta.model}` : null,
+    meta.startedAt ? `date: ${new Date(meta.startedAt).toISOString()}` : null,
+    meta.cwd ? `cwd: ${meta.cwd}` : null,
+    meta.gitBranch ? `branch: ${meta.gitBranch}` : null,
+    `association: tier ${meta.association?.tier} (${meta.association?.confidence})`,
+    "",
+  ]
+    .filter((l) => l !== null)
+    .join("\n");
+
+  const footer = [
+    "",
+    "---",
+    `raw transcript: ${meta.rawPath}`,
+    "Tool calls above are one-line summaries and tool output is truncated. Open the raw",
+    "transcript only if a specific claim needs the full text.",
+  ].join("\n");
+
+  const { body, elided } = capTrace(lines.join("\n").trim(), maxTraceTokens);
+  const trace = `${header}\n${body}\n${footer}\n`;
+
+  return {
+    trace,
+    stats: {
+      userTurns,
+      assistantTurns,
+      toolCalls,
+      elided,
+      distilledTokens: estimateTokens(trace),
+    },
+  };
+}
+
+/**
+ * Long sessions still blow past what a cheap analysis pass should read. Keep the head
+ * (the task as stated) and the tail (how it actually ended) and elide the middle - the
+ * raw transcript path in the footer remains the escape hatch for anything in between.
+ */
+function capTrace(body, maxTraceTokens) {
+  if (estimateTokens(body) <= maxTraceTokens) return { body, elided: false };
+  const budgetChars = maxTraceTokens * 4;
+  const headChars = Math.floor(budgetChars * 0.45);
+  const tailChars = Math.floor(budgetChars * 0.45);
+  const head = body.slice(0, headChars);
+  const tail = body.slice(-tailChars);
+  const droppedTokens = estimateTokens(body) - estimateTokens(head) - estimateTokens(tail);
+  return {
+    body: `${head}\n\n[... middle of session elided: ~${droppedTokens} tokens. Open the raw transcript below if a claim needs it ...]\n\n${tail}`,
+    elided: true,
+  };
+}