memhouse 0.18.12 → 0.20.1

package/memhouse/shipper/dedupe-v3.js

@@ -0,0 +1,1027 @@
+// Row-level dedupe of a session's converted legacy rows against its v3 capture.
+// docs/design/source-identity.md draft 5 §7 step 5, as decided on 2026-10-14:
+// THERE MUST NEVER BE DUPLICATE RECORDS.
+//
+// A session converted from the legacy rooms (§7 step 4) holds `legacy:` rows: message_id
+// 'legacy:<origin>:<epoch>:<seq>', call_id 'legacy:<origin>:<epoch>:<idx>', rev 0. When the
+// capture first reads the same session from disk, it writes the same messages again under their
+// native ids. Before this module the engine tombstoned the legacy ship rows only when the WHOLE
+// session reconciled; a session that did not kept both copies. Now every legacy row is decided
+// on its own:
+//
+//   matched    a captured row (or run of rows) holds what the legacy row held → the legacy row
+//              is tombstoned (a FULL copy, rev raised, is_deleted = 1 — §2.2; a key-only
+//              tombstone is resurrected by 26.7's skip indexes under FINAL)
+//   unmatched  the transcript no longer has it → the legacy row stays live, and is reported
+//   import     'legacy:import:' rows (and any origin but ship) are never tombstoned (§7 step 5)
+//
+// THE MATCH. Both sides are put in one order and aligned, keeping that order:
+//
+//   legacy     one row per Claude Code record; one row per Codex turn, its parts '\n'-joined:
+//              assistant text, '[thinking] <summary>', '[tool-call: <name>(<arg keys>)]',
+//              '[tool-result: <name>] <one-line preview>', or just '[assistant activity]'.
+//              Sub-agent rows lead with '[subagent] '. Text capped at 50,000 chars (ship.js).
+//   captured   one row per content block (`part`), in file order (`ord` = byte offset).
+//
+//   A legacy row is parsed into PIECES (a plain line, a thinking line, a call, a result, an
+//   activity marker); it matches a captured RUN starting at atom s when every piece is
+//   consumed in order by the atoms of one unit: plain and thinking lines equal (whitespace
+//   collapsed) the lines of text/thinking atoms, a call needs a tool_call atom of that name, a
+//   result a tool_result atom (its preview is a quality signal, not a gate), an activity a usage
+//   atom. The unit is the record (Claude Code: one message_id) or, for an assistant row of a
+//   turn-folding harness (Codex), the turn (response_id) — a run never crosses into another
+//   unit of its class or into a user message. Atoms the legacy parser never rendered (tool
+//   results of a Claude user record, reasoning without a summary, empty blocks) are skipped.
+//
+//   Each (legacy row, start) that matches is a candidate interval; the alignment is the
+//   heaviest chain of candidates increasing in BOTH orders with disjoint runs (weighted interval
+//   chaining, Fenwick tree, O(c log c)). Weight: 1000 per matched row first — the count always
+//   wins — then same agent, whole-atom consumption, fewer skipped atoms, agreeing previews. Two
+//   identical legacy rows therefore need two captured runs (multiplicity), and a repeated text
+//   cannot pull the alignment out of order.
+//
+//   Claude Code legacy rows carry the record's own timestamp (ship.js messageTs uses _ts), so a
+//   candidate must have the same ts (`tsExact`); Codex legacy timestamps were interpolated and
+//   are ignored. Two follow-ups: (1) tsExact — a legacy row identical in agent, role, ts and text
+//   to a matched one is a verbatim re-append of the same record (§2.5; the capture collapses
+//   it), so it is matched too; (2) turn units — an '[assistant activity]' row with no usage atom
+//   of its own (tokens the legacy parser flushed between two parts of one turn) is matched when
+//   the legacy row before it matched into a turn and the row after it is a user row, is absent,
+//   or matched into the same turn.
+//
+//   A third, for turn units: an '[assistant activity]' row between two user messages with no
+//   captured assistant row between them may be a turn aborted before any output, where Codex only
+//   repeated the last token_count (the same running total) and the 0.18 parser counted it again
+//   (`rereport`): its tokens are the previous turn's, already in the capture. Matched only on the
+//   transcript's proof (o.repeats, codexRepeats): such a repeat with the row's input and output,
+//   each repeat proving one row. Without it the row stays live (report.rereport_unproven).
+//
+//   CAPTURED FORMS. The captured side is the 0.20 engines' rows or the harness-session-adapters package's
+//   (records-to-rows.js), read alike: a call `name(args)` or `name <args JSON>`; `reasoning` is
+//   thinking; a compaction `summary` and an `event` (an API error, a notice) are text, as 0.18
+//   rendered them; a tool message is on the user side, and in a turn unit a tool result outside any
+//   response joins the turn it follows; a custom tool's input as a bare JSON string is { input }.
+//   A converted profile row (step B) is read through the same shapes.
+//
+//   Calls: the same chaining over (legacy calls, captured calls): the tool name must be equal and
+//   the arguments compatible (equal JSON; or the legacy object contained in the captured one —
+//   the Codex custom-tool preview; or an empty legacy object — the '[tool-call: X]' fallback);
+//   a call whose message matched into a run holding the captured call weighs more. A phantom
+//   (the 0.18 conversion parsed '[tool-call: X]' out of prose) is retired without a match: see
+//   matchCalls.
+//
+// DEAD ROWS RESERVE. The alignment runs over EVERY legacy ship row — live and already
+// tombstoned (the newest version per key; a tombstone is a full copy, so it still has its
+// text). A tombstoned row keeps the captured run it had, so a re-run (a crash before the
+// marker) decides the same way, never hands its run to a live twin, and never resurrects it:
+// legacy keys are only ever written as tombstones here.
+//   The reserve does not survive a tombstone cleanup (maintenance.tombstoneCleanup), which drops
+//   tombstones with their originals. A re-plan after it (--redupe) can give a remaining live row
+//   the run a dropped copy held — only where the 0.18 parse holds more copies of a text than the
+//   transcript; Claude Code's echo rule matches those copies anyway, so one extra copy can be
+//   retired for Codex and the bridge profile. Accepted (2026-10-23).
+//
+// TOTALS. The session's `sessions` row = Σ files.contrib (the capture) + the contribution of
+// every legacy row still live (unmatched ship rows and imports), which the engine stores as a
+// `files` row of its own (file_key LEGACY_FILE_KEY, role 'legacy') — so `sessions` stays a sum
+// over files rows and a pass still reads no conversation rows. Legacy tokens stay as they were
+// (per record); `usage_basis` is 'mixed' while any legacy row is live, else 'per_response'.
+// That files row is also the marker: a session that has it is deduped and is never matched
+// again; later passes only append.
+//
+// Pure: no I/O. capture.js reads the rows and writes what this returns.
+
+'use strict';
+
+const { capture: base } = require('../../editors/base');
+
+const LEGACY_FILE_KEY = 'legacy:rows';
+const LEGACY_ROLE = 'legacy';
+const LEGACY_TEXT_MAX = 50000;        // ship.js TEXT_MAX: a row at the cap may end mid-line
+const SUBAGENT_TAG = '[subagent] ';
+const SYSTEM_MARKER = /^\[(info|error|warning)\] /;
+// Gemini CLI thoughts are { subject, description }: the normaliser writes '**subject** description',
+// the 0.18 reader wrote the description alone.
+const unsubject = (line) => line.replace(/^\*\*[^*]+\*\* ?/, '');
+const MATCH = 1000;                   // weight of one matched row: the count always comes first
+const CANDIDATES_PER_ROW = 4000;      // a text repeated thousands of times: the nearest are enough
+const ZERO_SHA = '0'.repeat(64);
+const EPOCH_TS = '1970-01-01 00:00:00.000';
+
+/** How a harness's legacy rows relate to its capture. */
+const PROFILES = {
+  // One legacy row per record, stamped with the record's own timestamp.
+  'claude-code': { tsExact: true, unit: 'record' },
+  // One legacy row per turn (split again at turn_context and user messages); timestamps interpolated.
+  codex: { tsExact: false, unit: 'turn' },
+};
+// Every other harness is captured through the bridge (editors/bridge.js): its snapshot lines carry
+// exactly the text 0.18 stored for each message, from the same reader, so a legacy row and a captured
+// row are the same message when their role and text are equal. Matched one-to-one by that, with
+// multiplicity (two equal legacy rows need two captured rows); timestamps are not compared, because
+// 0.18 interpolated them over a span that may have moved since.
+const BRIDGE_PROFILE = { tsExact: false, unit: 'message', byText: true };
+// What byText leaves is matched piece by piece: since layout 3.2 these harnesses are captured through
+// the normalisers (editors/normalise), one row per part, where the bridge held 0.18's message text.
+const PIECE_PROFILE = { tsExact: false, unit: 'message', systemMarkers: true };
+function profileFor(harness) { return PROFILES[harness] || BRIDGE_PROFILE; }
+
+// Since layout 3.2 the capture keeps every text whole (base.js TEXT_CAP, TOOL_RESULT_CAP): two texts are
+// the same message only when they are equal. A row cut at a cap compares as a prefix: a 0.18 row at
+// LEGACY_TEXT_MAX (the capture continues it), or a row an older build captured cut at its caps (the
+// 0.18 row continues it).
+const OLD_CAPS = [2000, 50000];
+const cutAtOldCap = (t) => OLD_CAPS.some((n) => t.length === n || t.length === n - 1);
+const textKey = (t) => norm(t);
+/** byText: legacy turns and calls matched to captured ones by (role class, normalized text) / (tool, args). */
+function matchByText(shipT, shipC, capT, capC) {
+  const pool = new Map(); const cut = [];
+  const put = (k, v) => { if (!pool.has(k)) pool.set(k, []); pool.get(k).push(v); };
+  for (const r of [...capT].sort((a, b) => cmp([String(a.ts), Number(a.ord) || 0], [String(b.ts), Number(b.ord) || 0]))) {
+    const t = String(r.text ?? ''); const k = `${roleClass(r.role)}\u0000${textKey(t)}`;
+    put(k, r);
+    if (cutAtOldCap(t)) cut.push({ k, r });
+  }
+  const takeRow = (k, r) => { const list = pool.get(k) || []; const i = list.indexOf(r); if (i < 0) return null; list.splice(i, 1); return r; };
+  // Every exact match first: a prefix may take only what no equal text claims.
+  const rows = [...shipT].sort((a, b) => seqOf(a.message_id) - seqOf(b.message_id)).map((r) => {
+    // 0.18 stored a tool message with role 'tool'; the bridge files it as a user-side row (bridge.js bRole).
+    const t = stripTag(String(r.text ?? ''));
+    const key = `${r.role === 'tool' ? 'user' : roleClass(r.role)}\u0000${textKey(t)}`;
+    const q = pool.get(key);
+    return { r, t, key, m: q && q.length ? q.shift() : null };
+  });
+  for (const x of rows) {
+    if (x.m) continue;
+    if (x.t.length >= LEGACY_TEXT_MAX - 1) for (const [k, list] of pool) if (list.length && k.startsWith(x.key)) { x.m = list.shift(); break; }
+    if (!x.m) for (const c of cut) if (x.key.startsWith(c.k) && (x.m = takeRow(c.k, c.r))) break;
+  }
+  const LT = rows.map(({ r, m }) => ({ row: r, match: m ? [{ mid: String(m.message_id), part: Number(m.part_ord) || 0 }] : null, how: 'aligned' }));
+  const cpool = new Map();
+  for (const c of capC) { const k = `${c.tool_name}\u0000${String(c.args ?? '').replace(/\s+/g, '')}`; if (!cpool.has(k)) cpool.set(k, []); cpool.get(k).push(c); }
+  const LC = [...shipC].sort((a, b) => seqOf(a.call_id) - seqOf(b.call_id)).map((r) => {
+    const q = cpool.get(`${r.tool_name}\u0000${String(r.args ?? '').replace(/\s+/g, '')}`);
+    let m = q && q.length ? q.shift() : null;
+    if (!m) { // 0.18 parsed '[tool-call: X]' markers without arguments: any captured call of that tool
+      for (const [k, list] of cpool) if (list.length && k.startsWith(`${r.tool_name}\u0000`) && String(r.args ?? '').replace(/\s+/g, '') === '{}') { m = list.shift(); break; }
+    }
+    return { row: r, match: m };
+  });
+  return { LT, LC };
+}
+
+// ── text ─────────────────────────────────────────────────────────────────────────────────
+const norm = (s) => String(s ?? '').replace(/\s+/g, ' ').trim();
+function linesOf(text) {
+  const out = [];
+  for (const l of String(text ?? '').split('\n')) { const n = norm(l); if (n) out.push(n); }
+  return out;
+}
+function roleClass(role) { return role === 'user' ? 'user' : (role === 'system' ? 'system' : 'assistant'); }
+const isShip = (id) => String(id).startsWith('legacy:ship:');
+const isLegacy = (id) => String(id).startsWith('legacy:');
+function stripTag(text) { const t = String(text ?? ''); return t.startsWith(SUBAGENT_TAG) ? t.slice(SUBAGENT_TAG.length) : t; }
+
+/**
+ * A legacy row's pieces. Markers exist only in assistant rows (the legacy parsers added them
+ * there); a user or system row is plain lines, whatever it starts with.
+ *   { t: 'line', x } | { t: 'thinking', x } | { t: 'call', name, keys } | { t: 'result', name, x } | { t: 'activity' }
+ */
+function legacyPieces(text, cls, { systemMarkers = false } = {}) {
+  const out = [];
+  const lines = stripTag(text).split('\n');
+  for (const raw of lines) {
+    if (cls !== 'assistant') {
+      // The bridge-era readers marked a harness's own notices '[info] ' / '[error] ' (system rows).
+      const n = norm(systemMarkers && cls === 'system' ? raw.replace(SYSTEM_MARKER, '') : raw);
+      if (n) out.push({ t: 'line', x: n });
+      continue;
+    }
+    const tail = raw.trimEnd();
+    if (tail.trim() === '[assistant activity]') { out.push({ t: 'activity' }); continue; }
+    if (raw.startsWith('[thinking]')) { out.push({ t: 'thinking', x: norm(raw.slice(10)) }); continue; }
+    if (raw.startsWith('[tool-call: ') && tail.endsWith(']')) {
+      const inner = tail.slice(12, -1);
+      const p = inner.indexOf('(');
+      const name = (p === -1 ? inner : inner.slice(0, p)).trim();
+      const keys = p !== -1 && inner.endsWith(')') ? inner.slice(p + 1, -1).split(',').map((k) => k.trim()).filter(Boolean) : null;
+      out.push({ t: 'call', name, keys });
+      continue;
+    }
+    if (raw.startsWith('[tool-result: ')) {
+      const close = raw.indexOf(']', 14);
+      out.push({ t: 'result', name: close === -1 ? '' : raw.slice(14, close), x: close === -1 ? '' : norm(raw.slice(close + 1)).replace(/…$/, '') });
+      continue;
+    }
+    const n = norm(raw);
+    if (n) out.push({ t: 'line', x: n });
+  }
+  return out;
+}
+
+/**
+ * A captured tool_call row's text → { name, keys } (keys null when the args do not parse). The 0.20
+ * engines wrote `name(args)`; the harness-session-adapters package (records-to-rows.js) writes
+ * `name <args JSON>`. A tool name holds neither a space nor a parenthesis, so whichever of the two
+ * comes first tells the forms apart, also when the arguments contain the other.
+ */
+function callOfText(text) {
+  const s = String(text ?? '');
+  const p = s.indexOf('(');
+  const sp = s.indexOf(' ');
+  const keysOf = (j) => {
+    try { const o = JSON.parse(j); return o && typeof o === 'object' && !Array.isArray(o) ? Object.keys(o) : null; } catch { return null; }
+  };
+  if (sp !== -1 && (p === -1 || sp < p)) return { name: s.slice(0, sp), keys: keysOf(s.slice(sp + 1)) };
+  if (p === -1) return { name: s, keys: null };
+  return { name: s.slice(0, p), keys: s.endsWith(')') ? keysOf(s.slice(p + 1, -1)) : null };
+}
+/** The texts a captured tool_call row of this call holds, in either engine's form. */
+function callTexts(name, args) {
+  return [base.capText(`${name}(${args})`, base.TOOL_RESULT_CAP), base.capText(`${name} ${args}`, base.TOOL_RESULT_CAP)];
+}
+
+// ── rows ─────────────────────────────────────────────────────────────────────────────────
+/** The newest version of every key (a tombstone outranks an equal rev): what FINAL would answer, tombstones kept. */
+function newestPerKey(rows, keyOf) {
+  const best = new Map();
+  for (const r of rows) {
+    const k = keyOf(r);
+    const cur = best.get(k);
+    const v = BigInt(r.rev || 0);
+    if (!cur || v > cur.v || (v === cur.v && Number(r.is_deleted) && !Number(cur.r.is_deleted))) best.set(k, { v, r });
+  }
+  return [...best.values()].map((x) => x.r);
+}
+const dead = (r) => Number(r.is_deleted) === 1;
+function seqOf(id) { const m = /:(\d+)$/.exec(String(id)); return m ? Number(m[1]) : 0; }
+function cmp(a, b) {
+  for (let i = 0; i < a.length; i++) {
+    if (a[i] < b[i]) return -1;
+    if (a[i] > b[i]) return 1;
+  }
+  return 0;
+}
+
+// ── the chain ────────────────────────────────────────────────────────────────────────────
+/** Prefix maximum over legacy positions: (score, candidate index), ties to the earlier candidate. */
+class MaxTree {
+  constructor(n) { this.n = n; this.s = new Float64Array(n + 1).fill(-1); this.i = new Int32Array(n + 1).fill(-1); }
+  better(s, i, at) { return s > this.s[at] || (s === this.s[at] && i !== -1 && (this.i[at] === -1 || i < this.i[at])); }
+  update(pos, s, i) { for (let p = pos + 1; p <= this.n; p += p & -p) if (this.better(s, i, p)) { this.s[p] = s; this.i[p] = i; } }
+  query(pos) {
+    let bs = -1; let bi = -1;
+    for (let p = pos + 1; p > 0; p -= p & -p) if (this.s[p] > bs || (this.s[p] === bs && this.i[p] !== -1 && (bi === -1 || this.i[p] < bi))) { bs = this.s[p]; bi = this.i[p]; }
+    return bi === -1 ? null : { s: bs, i: bi };
+  }
+}
+/**
+ * The heaviest chain of candidates { k, s, e, w }: k (legacy position) strictly increasing, and
+ * each run [s, e] after the previous one's end. Returns the chosen candidates.
+ */
+function bestChain(cands, nLegacy) {
+  if (!cands.length) return [];
+  cands.sort((x, y) => x.s - y.s || x.k - y.k || x.e - y.e);
+  const byEnd = cands.map((_, i) => i).sort((x, y) => cands[x].e - cands[y].e || x - y);
+  const tree = new MaxTree(nLegacy);
+  const score = new Float64Array(cands.length);
+  const prev = new Int32Array(cands.length).fill(-1);
+  let rel = 0; let top = -1;
+  for (let i = 0; i < cands.length; i++) {
+    const c = cands[i];
+    while (rel < byEnd.length && cands[byEnd[rel]].e < c.s) { const j = byEnd[rel++]; tree.update(cands[j].k, score[j], j); }
+    const best = c.k > 0 ? tree.query(c.k - 1) : null;
+    score[i] = c.w + (best ? best.s : 0);
+    prev[i] = best ? best.i : -1;
+    if (top === -1 || score[i] > score[top]) top = i;
+  }
+  const out = [];
+  for (let i = top; i !== -1; i = prev[i]) out.push(cands[i]);
+  return out.reverse();
+}
+
+// ── turns ────────────────────────────────────────────────────────────────────────────────
+const TEXTISH = new Set(['text', 'context']);
+const WHOLE = new Set(['text', 'thinking', 'context']);   // kinds whose every line a legacy row would have held
+
+// The captured row in the matcher's terms, whichever engine wrote it. The harness-session-adapters package
+// (records-to-rows.js) names some content as sessions-model does where the 0.20 engines had text:
+// a compaction summary is a `summary` part, an API error or a harness notice an `event` row, and
+// reasoning arrives as `reasoning` from a profile row. The 0.18 parser rendered every one of them as
+// a plain line, so they are text here. A tool message (role `tool`, or `user` as records-to-rows
+// files it) is on the user side.
+const KIND_AS = { reasoning: 'thinking', summary: 'text', event: 'text', notice: 'text', error: 'text' };
+const capClass = (role) => (role === 'tool' ? 'user' : roleClass(role));
+
+function atomsOf(rows, harness, prof) {
+  const atoms = rows.map((r) => {
+    const text = String(r.text ?? '');
+    const kind = KIND_AS[String(r.kind)] || String(r.kind);
+    const cls = capClass(r.role);
+    const a = {
+      row: r, cls, kind, agent: String(r.agent_id || ''), ts: String(r.ts), mid: String(r.message_id),
+      part: Number(r.part_ord) || 0, ord: Number(r.ord) || 0, resp: String(r.response_id || ''),
+      text, lines: linesOf(text), flat: null, call: kind === 'tool_call' ? callOfText(text) : null,
+    };
+    a.unit = prof.unit === 'turn' && cls === 'assistant' && a.resp ? `${a.agent}\u0000turn:${a.resp}` : `${a.agent}\u0000${a.mid}`;
+    return a;
+  });
+  const key = prof.tsExact ? (a) => [a.ts, a.agent, a.ord, a.part, a.mid] : (a) => [a.agent, a.ord, a.part, a.ts, a.mid];
+  atoms.sort((x, y) => cmp(key(x), key(y)));
+  atoms.forEach((a, i) => { a.i = i; });
+  if (prof.unit === 'turn') {
+    // A tool result belongs to the turn that called it. The 0.20 Codex engine filed it under the
+    // turn's response; the package files it as a tool message of its own, outside any response. In
+    // file order it sits after its turn's call and before the next user message: it joins that turn,
+    // on the assistant side, as the 0.18 parser rendered it ('[tool-result: …]' inside the turn row).
+    let cur = null; let agent = null;
+    for (const a of atoms) {
+      if (a.agent !== agent) { cur = null; agent = a.agent; }
+      if (a.unit.includes('\u0000turn:')) cur = a.unit;
+      else if (a.kind === 'tool_result' && cur) { a.unit = cur; a.cls = 'assistant'; }
+      else if (a.cls === 'user') cur = null;
+    }
+  }
+  return atoms;
+}
+const flatOf = (a) => (a.flat === null ? (a.flat = norm(a.text)) : a.flat);
+
+/** Signatures an atom can START a run under, for the candidate index. */
+function atomSigs(a, prof) {
+  const pre = prof.tsExact ? `${a.ts}\u0001` : `${a.agent}\u0001`;
+  const out = [];
+  if (TEXTISH.has(a.kind) && a.lines.length) out.push(`L${a.cls}\u0001${a.lines[0]}`);
+  if (a.kind === 'thinking') {
+    out.push('T*');
+    if (a.lines.length) { out.push(`T\u0001${a.lines[0]}`); const u = unsubject(a.lines[0]); if (u !== a.lines[0] && u) out.push(`T\u0001${u}`); }
+  }
+  if (a.kind === 'tool_call' && a.call) out.push(`C\u0001${a.call.name}`);
+  if (a.kind === 'tool_result') { out.push('R*'); const f = flatOf(a); if (f.length >= 16) out.push(`R\u0001${f.slice(0, 16)}`); }
+  if (a.kind === 'usage') out.push('U');
+  return out.map((s) => pre + s);
+}
+function legacySig(L, prof) {
+  const pre = prof.tsExact ? `${L.ts}\u0001` : `${L.agent}\u0001`;
+  const p = L.pieces[0];
+  if (!p) return null;
+  if (p.t === 'line') return L.cut && L.pieces.length === 1 ? null : pre + `L${L.cls}\u0001${p.x}`;
+  if (p.t === 'thinking') return pre + (p.x ? `T\u0001${p.x}` : 'T*');
+  if (p.t === 'call') return pre + `C\u0001${p.name}`;
+  if (p.t === 'result') return pre + (p.x.length >= 16 ? `R\u0001${p.x.slice(0, 16)}` : 'R*');
+  return pre + 'U';
+}
+
+/**
+ * Does legacy row L match a run of atoms starting at s? → { e, used, w } or null.
+ * Greedy: a plain line continues the current atom when it is that atom's next line, else it
+ * opens the next text atom of the run; every marker opens the next atom of its kind.
+ */
+function tryMatch(L, s, atoms) {
+  const start = atoms[s];
+  if (start.cls !== L.cls) return null;
+  const pieces = L.pieces;
+  let a = -1; let j = 0;
+  let gaps = 0; let partial = 0; let soft = 0;
+  const used = [];
+  const leave = () => { if (a !== -1 && WHOLE.has(atoms[a].kind) && j < atoms[a].lines.length) partial++; };
+  // The next atom of the run (from s for the first piece) that satisfies pred.
+  const open = (pred) => {
+    const from = a === -1 ? s : a + 1;
+    for (let k = from; k < atoms.length; k++) {
+      const x = atoms[k];
+      if (a === -1 && k > s) return -1;                       // the first piece starts AT s
+      if (x.unit !== start.unit) {
+        if (x.cls === 'system' && L.cls !== 'system') continue;   // a context row inside a turn
+        return -1;
+      }
+      if (pred(x)) { leave(); a = k; j = 0; used.push(k); return k; }
+      if (x.lines.length && x.kind !== 'usage') gaps++;
+    }
+    return -1;
+  };
+  const eq = (cap, leg, last) => cap === leg || (last && L.cut && cap.startsWith(leg));
+  // The next atom with content after the current one, in the run's unit (or null).
+  const nextContent = () => {
+    for (let k = a + 1; k < atoms.length; k++) {
+      const x = atoms[k];
+      if (x.unit !== start.unit) { if (x.cls === 'system' && L.cls !== 'system') continue; return null; }
+      if (x.lines.length && x.kind !== 'usage') return x;
+    }
+    return null;
+  };
+  for (let pi = 0; pi < pieces.length; pi++) {
+    const p = pieces[pi];
+    const last = pi === pieces.length - 1;
+    let ok = true;
+    if (p.t === 'line') {
+      const cur = a === -1 ? null : atoms[a];
+      if (cur && WHOLE.has(cur.kind) && j < cur.lines.length && eq(cur.lines[j], p.x, last)) j++;
+      // A preview spanning lines continues the result, unless the line opens the text that follows
+      // it ('{' after a JSON output is also inside that output).
+      else if (cur && cur.kind === 'tool_result' && flatOf(cur).includes(p.x)
+        && !(() => { const x = nextContent(); return x && TEXTISH.has(x.kind) && eq(x.lines[0], p.x, last); })()) { /* a multi-line preview */ }
+      else if (open((x) => TEXTISH.has(x.kind) && x.lines.length > 0 && eq(x.lines[0], p.x, last)) !== -1) j = 1;
+      else ok = false;
+    } else if (p.t === 'thinking') {
+      if (open((x) => x.kind === 'thinking' && (p.x === '' || (x.lines.length > 0 && (eq(x.lines[0], p.x, last) || eq(unsubject(x.lines[0]), p.x, last))))) !== -1) j = p.x === '' ? 0 : 1;
+      else ok = false;
+    } else if (p.t === 'call') {
+      if (open((x) => x.kind === 'tool_call' && x.call && x.call.name === p.name) !== -1) {
+        const c = atoms[a].call;
+        if (p.keys && c.keys && p.keys.join(',') !== c.keys.join(',')) soft++;
+        j = atoms[a].lines.length;
+      } else ok = false;
+    } else if (p.t === 'result') {
+      if (open((x) => x.kind === 'tool_result') !== -1) {
+        if (p.x && !flatOf(atoms[a]).startsWith(p.x)) soft++;
+        j = atoms[a].lines.length;
+      } else ok = false;
+    } else if (open((x) => x.kind === 'usage') === -1) ok = false;   // activity
+    if (!ok) {
+      // A row cut at the legacy cap may end in a broken piece: what came before it decides.
+      if (last && L.cut && used.length) break;
+      return null;
+    }
+  }
+  leave();
+  if (!used.length) return null;
+  const w = MATCH + (start.agent === L.agent ? 40 : 0) + (partial ? 0 : 20) - Math.min(gaps, 10) - Math.min(5 * soft, 20);
+  return { e: used[used.length - 1], used, w };
+}
+
+// ── the re-report proof (Codex) ──────────────────────────────────────────────────────────
+/**
+ * What `rereport` needs from a Codex rollout's lines (strings or parsed, in file order): the
+ * last_token_usage of every token_count that repeats the previous total_token_usage inside a turn
+ * (task_started … task_complete | turn_aborted) where the model produced nothing — no response
+ * item but a user or developer message. In the 0.18 split: input without cached input, output.
+ * A turn still open at the end proves nothing (it may yet produce output); a rollout without
+ * task_started proves nothing.
+ */
+function codexRepeats(lines) {
+  const out = [];
+  let prevTotal = null; let turn = null;
+  const close = () => { if (turn && !turn.model) out.push(...turn.repeats); turn = null; };
+  for (const line of lines) {
+    let o = line;
+    if (typeof line === 'string') {
+      if (!line.includes('"event_msg"') && !line.includes('"response_item"')) continue;
+      try { o = JSON.parse(line); } catch { continue; }
+    }
+    const p = o && o.payload;
+    if (!p || typeof p !== 'object') continue;
+    if (o.type === 'response_item') { if (turn && !(p.type === 'message' && p.role !== 'assistant')) turn.model = true; continue; }
+    if (o.type !== 'event_msg') continue;
+    if (p.type === 'task_started') { close(); turn = { model: false, repeats: [] }; } else if (p.type === 'task_complete' || p.type === 'turn_aborted') close();
+    else if (p.type === 'token_count' && p.info && p.info.total_token_usage) {
+      const total = JSON.stringify(p.info.total_token_usage);
+      const last = p.info.last_token_usage;
+      if (turn && total === prevTotal && last) {
+        turn.repeats.push({ input: Math.max(Number(last.input_tokens || 0) - Number(last.cached_input_tokens || 0), 0), output: Number(last.output_tokens || 0) });
+      }
+      prevTotal = total;
+    }
+  }
+  return out;
+}
+const repeatKey = (input, output) => `${Number(input) || 0}/${Number(output) || 0}`;
+function repeatPool(repeats) {
+  const m = new Map();
+  for (const r of repeats) { const k = repeatKey(r.input, r.output); m.set(k, (m.get(k) || 0) + 1); }
+  return m;
+}
+
+function matchTurns(harness, prof, shipRows, capturedTurns, repeats = null) {
+  const atoms = atomsOf(capturedTurns, harness, prof);
+  const legacy = shipRows.map((r) => {
+    const cls = roleClass(r.role);
+    const text = String(r.text ?? '');
+    return {
+      row: r, cls, agent: String(r.agent_id || ''), ts: String(r.ts), ord: Number(r.ord) || seqOf(r.message_id),
+      pieces: legacyPieces(text, cls, { systemMarkers: !!prof.systemMarkers }), cut: text.length >= LEGACY_TEXT_MAX - 1, flat: norm(stripTag(text)),
+      match: null, how: null,
+    };
+  });
+  const lkey = prof.tsExact ? (L) => [L.ts, L.agent, L.ord, L.row.message_id] : (L) => [L.agent, L.ord, L.ts, L.row.message_id];
+  legacy.sort((x, y) => cmp(lkey(x), lkey(y)));
+  // Candidate index.
+  const index = new Map();
+  for (const a of atoms) for (const sig of atomSigs(a, prof)) { if (!index.has(sig)) index.set(sig, []); index.get(sig).push(a.i); }
+  const startsOf = (L) => {
+    const sig = legacySig(L, prof);
+    if (sig) return { starts: index.get(sig) || [], selective: prof.tsExact || SELECTIVE.test(sig) };
+    if (!L.pieces.length) return { starts: [], selective: true };
+    // A single line cut at the legacy cap: its own prefix is all there is to look for.
+    const x = L.pieces[0].x;
+    return {
+      starts: atoms.filter((a) => TEXTISH.has(a.kind) && a.cls === L.cls && a.lines.length && a.lines[0].startsWith(x)
+        && (prof.tsExact ? a.ts === L.ts : a.agent === L.agent)).map((a) => a.i),
+      selective: true,
+    };
+  };
+  const cands = [];
+  const tryAll = (L, k, starts) => {
+    if (starts.length > CANDIDATES_PER_ROW) starts = nearest(starts, k, legacy.length, atoms.length);
+    for (const s of starts) {
+      if (!prof.tsExact && atoms[s].agent !== L.agent) continue;
+      const m = tryMatch(L, s, atoms);
+      if (m) cands.push({ k, s, e: m.e, w: m.w, used: m.used });
+    }
+  };
+  // Two phases, so a row whose first piece is common (a call of the one shell tool, a result with
+  // no preview, a usage row) is not tried against every such atom of a long session: rows with a
+  // distinctive first piece are aligned first, and the others are tried only between the runs their
+  // aligned neighbours took.
+  const loose = [];
+  legacy.forEach((L, k) => {
+    const { starts, selective } = startsOf(L);
+    if (selective) tryAll(L, k, starts); else loose.push({ L, k, starts });
+  });
+  if (loose.length) {
+    const { lo, hi } = windows(bestChain(cands.slice(), legacy.length), legacy.length, atoms.length);
+    for (const { L, k, starts } of loose) tryAll(L, k, between(starts, lo[k], hi[k]));
+  }
+  for (const c of bestChain(cands, legacy.length)) { legacy[c.k].match = c.used.map((i) => atoms[i]); legacy[c.k].how = 'aligned'; }
+
+  if (prof.tsExact) {
+    // A verbatim copy of one record: within the session, the same role, ts (to the millisecond) and
+    // text as a matched row — or as one captured text row — is that record again, whatever agent id
+    // the copy was filed under. The agent is deliberately not in the key: the 0.18 parse filed one
+    // sub-agent conversation under eleven agent ids (copies of its sidechain file), the capture
+    // under one (2026-10-14, the dev house).
+    const seen = new Map();
+    for (const L of legacy) if (L.match) seen.set(`${L.cls}\u0000${L.ts}\u0000${L.flat}`, L.match);
+    for (const a of atoms) {
+      if (!TEXTISH.has(a.kind)) continue;
+      const k = `${a.cls}\u0000${a.ts}\u0000${flatOf(a)}`;
+      if (!seen.has(k)) seen.set(k, [a]);
+    }
+    for (const L of legacy) {
+      if (L.match) continue;
+      const m = seen.get(`${L.cls}\u0000${L.ts}\u0000${L.flat}`);
+      if (m) { L.match = m; L.how = 'echo'; }
+    }
+  }
+  if (prof.unit === 'turn') {
+    // '[assistant activity]' with no usage atom of its own: tokens the legacy parser flushed
+    // between two parts of ONE turn (at a steering message, at a repeated turn_context), which
+    // the capture bills on the turn's first row. Matched when it lies inside a captured turn:
+    // the rows around it matched atoms inside that turn's span (a steering message sits inside
+    // it), or — at the end — the row before it matched into the turn.
+    const turnAt = new Array(atoms.length).fill(null);
+    const span = new Map();
+    for (const a of atoms) {
+      if (!a.unit.includes('\u0000turn:')) continue;
+      const v = span.get(a.unit) || [a.i, a.i];
+      v[0] = Math.min(v[0], a.i); v[1] = Math.max(v[1], a.i);
+      span.set(a.unit, v);
+    }
+    for (const [u, [lo, hi]] of span) for (let i = lo; i <= hi; i++) if (!turnAt[i]) turnAt[i] = u;
+    const lastAt = (L) => (L && L.match && L.match.length ? L.match[L.match.length - 1].i : -1);
+    const firstAt = (L) => (L && L.match && L.match.length ? L.match[0].i : -1);
+    for (let k = 0; k < legacy.length; k++) {
+      const L = legacy[k];
+      if (L.match || L.pieces.length !== 1 || L.pieces[0].t !== 'activity') continue;
+      let p = k - 1; while (p >= 0 && (legacy[p].agent !== L.agent || (legacy[p].match && !legacy[p].match.length))) p--;
+      let n = k + 1; while (n < legacy.length && legacy[n].agent !== L.agent) n++;
+      const P = p >= 0 ? legacy[p] : null;
+      const Nx = n < legacy.length ? legacy[n] : null;
+      const pe = lastAt(P);
+      if (pe !== -1 && P.cls === 'user' && (!Nx || (Nx.cls === 'user' && firstAt(Nx) > pe))) {
+        // A turn the user aborted before the model produced anything. Codex opens a turn by
+        // repeating the last token_count (the same total_token_usage); in such a turn that repeat
+        // is all there is, and the 0.18 parser counted it again: '[assistant activity]' holding the
+        // previous turn's tokens. The capture adds nothing for a running total seen again (the
+        // 0.20 engine and the package alike), so between the two user messages there is no
+        // captured assistant row. Matched only on proof from the transcript (`repeats`, from
+        // codexRepeats): a repeat in an empty turn with the same input and output as the row. An
+        // aborted turn that really spent tokens has other numbers and stays live (2026-10-23).
+        const end = Nx ? firstAt(Nx) : atoms.length;
+        let empty = true;
+        for (let i = pe + 1; i < end && empty; i++) if (atoms[i].agent === L.agent && atoms[i].cls === 'assistant') empty = false;
+        if (empty) {
+          const key = repeatKey(L.row.input_tokens, L.row.output_tokens);
+          if (repeats && repeats.get(key) > 0) { repeats.set(key, repeats.get(key) - 1); L.match = []; L.how = 'rereport'; continue; }
+          L.unproven = true;
+        }
+      }
+      const u = pe === -1 ? null : turnAt[pe];
+      if (!u) continue;
+      const ns = firstAt(Nx);
+      const inside = (ns !== -1 && turnAt[ns] === u && ns > pe)
+        || (P.cls === 'assistant' && (!Nx || Nx.cls === 'user') && (ns === -1 || ns > pe));
+      if (inside) { L.match = []; L.how = 'activity'; }
+    }
+  }
+  return { legacy, atoms };
+}
+const SELECTIVE = /\u0001(L[a-z]+\u0001|T\u0001|R\u0001)/;
+/** Per legacy position: the end of the nearest chosen run before it and the start of the nearest after it. */
+function windows(chain, n, size) {
+  const lo = new Int32Array(n).fill(-1);
+  const hi = new Int32Array(n).fill(size);
+  let j = 0; let last = -1;
+  for (let k = 0; k < n; k++) { while (j < chain.length && chain[j].k < k) last = chain[j++].e; lo[k] = last; }
+  j = chain.length - 1; let next = size;
+  for (let k = n - 1; k >= 0; k--) { while (j >= 0 && chain[j].k > k) next = chain[j--].s; hi[k] = next; }
+  return { lo, hi };
+}
+/** The entries of an ascending list strictly between lo and hi. */
+function between(list, lo, hi) {
+  let a = 0; let b = list.length;
+  while (a < b) { const m = (a + b) >> 1; if (list[m] <= lo) a = m + 1; else b = m; }
+  const out = [];
+  for (let i = a; i < list.length && list[i] < hi; i++) out.push(list[i]);
+  return out;
+}
+/** Of a long candidate list, the ones nearest the legacy row's relative position. */
+function nearest(starts, k, nLegacy, nAtoms) {
+  const at = nLegacy > 1 ? (k / (nLegacy - 1)) * nAtoms : 0;
+  return starts.slice().sort((x, y) => Math.abs(x - at) - Math.abs(y - at) || x - y).slice(0, CANDIDATES_PER_ROW).sort((x, y) => x - y);
+}
+
+// ── calls ────────────────────────────────────────────────────────────────────────────────
+/**
+ * Arguments as an object. A Codex custom tool's input (apply_patch) is free text: the 0.18 parser
+ * and the 0.20 engine wrapped it as { input }, the harness-session-adapters package keeps it as the bare
+ * JSON string — read as { input } here, so the forms compare. A bare string cut at the args cap
+ * (20,000) is closed again and read as the prefix it is: the 0.18 preview is its first 300 chars.
+ */
+function parseObj(s) {
+  try {
+    const o = JSON.parse(s);
+    if (typeof o === 'string') return { input: o };
+    return o && typeof o === 'object' && !Array.isArray(o) ? o : null;
+  } catch {
+    if (!s.startsWith('"')) return null;
+    try { const o = JSON.parse(`${s.replace(/\\(u[0-9a-fA-F]{0,3})?$/, '')}"`); return typeof o === 'string' ? { input: o } : null; } catch { return null; }
+  }
+}
+function deepEq(a, b) {
+  if (a === b) return true;
+  if (typeof a !== typeof b || a === null || b === null || typeof a !== 'object') return false;
+  if (Array.isArray(a) !== Array.isArray(b)) return false;
+  const ka = Object.keys(a); const kb = Object.keys(b);
+  if (ka.length !== kb.length) return false;
+  return ka.every((k) => Object.prototype.hasOwnProperty.call(b, k) && deepEq(a[k], b[k]));
+}
+function containedIn(big, small) {
+  return Object.keys(small).every((k) => {
+    if (!Object.prototype.hasOwnProperty.call(big, k)) return false;
+    if (deepEq(big[k], small[k])) return true;
+    // A value the legacy parser shortened to one line (Codex custom tools: 300 chars + '…').
+    return typeof big[k] === 'string' && typeof small[k] === 'string' && norm(big[k]).startsWith(norm(small[k]).replace(/…$/, ''));
+  });
+}
+/** Arguments, read once: the string, the object it parses to (or null), the string without whitespace. */
+function argsOf(v) { const s = String(v ?? ''); return { s, o: parseObj(s), flat: null }; }
+/** 'exact' | 'contained' | 'weak' | 'none' — legacy arguments against captured ones (strings, or argsOf results). */
+function argsFit(legacyArgs, capturedArgs) {
+  const L = typeof legacyArgs === 'object' && legacyArgs ? legacyArgs : argsOf(legacyArgs);
+  const C = typeof capturedArgs === 'object' && capturedArgs ? capturedArgs : argsOf(capturedArgs);
+  if (L.s === C.s) return 'exact';
+  if (L.o && !Object.keys(L.o).length) return 'weak';        // '[tool-call: X]': the legacy parser saw no arguments
+  if (L.o && C.o) return deepEq(L.o, C.o) ? 'exact' : (containedIn(C.o, L.o) ? 'contained' : 'none');
+  // One side cut at the 20,000-char cap does not parse: compare as text.
+  if (L.flat === null) L.flat = L.s.replace(/\s+/g, '');
+  if (C.flat === null) C.flat = C.s.replace(/\s+/g, '');
+  return L.flat && C.flat && (L.flat.startsWith(C.flat) || C.flat.startsWith(L.flat)) ? 'contained' : 'none';
+}
+const FIT_W = { exact: 30, contained: 20, weak: 0 };
+
+function matchCalls(prof, shipCalls, capturedCalls, legacyTurns, atoms) {
+  // Where each captured call sits in its transcript: the ord of its message's atoms, and its
+  // place among that message's tool_call atoms.
+  const atomsByMid = new Map();
+  for (const a of atoms) { if (!atomsByMid.has(a.mid)) atomsByMid.set(a.mid, []); atomsByMid.get(a.mid).push(a); }
+  const cap = capturedCalls.map((r) => {
+    const mine = atomsByMid.get(String(r.message_id)) || [];
+    const texts = callTexts(r.tool_name, r.args);
+    let ord = mine.length ? Math.min(...mine.map((a) => a.ord)) : Number.MAX_SAFE_INTEGER;
+    let part = 255;
+    for (const a of mine) {
+      if (a.kind !== 'tool_call') continue;
+      let x = null;
+      try { x = JSON.parse(a.row.extra || '{}').call_id; } catch { /* no extra */ }
+      if (x === String(r.call_id) || texts.includes(a.text)) { part = a.part; ord = a.ord; break; }
+    }
+    return { row: r, ts: String(r.ts), agent: String(r.agent_id || ''), name: String(r.tool_name), mid: String(r.message_id), ord, part, id: String(r.call_id), args: argsOf(r.args) };
+  });
+  const ckey = prof.tsExact ? (c) => [c.ts, c.agent, c.ord, c.part, c.id] : (c) => [c.agent, c.ord, c.part, c.ts, c.id];
+  cap.sort((x, y) => cmp(ckey(x), ckey(y)));
+  const byMid = new Map();
+  cap.forEach((c, i) => { if (!byMid.has(c.mid)) byMid.set(c.mid, []); byMid.get(c.mid).push(i); });
+  // The run each legacy message matched: the message ids of its atoms.
+  const runOf = new Map();
+  for (const L of legacyTurns) if (L.match) runOf.set(String(L.row.message_id), [...new Set(L.match.map((a) => a.mid))]);
+  const leg = shipCalls.map((r) => ({
+    row: r, ts: String(r.ts), agent: String(r.agent_id || ''), name: String(r.tool_name), seq: seqOf(r.message_id), idx: seqOf(r.call_id), match: null, args: argsOf(r.args),
+  }));
+  const lkey = prof.tsExact ? (c) => [c.ts, c.agent, c.seq, c.idx] : (c) => [c.agent, c.seq, c.idx, c.ts];
+  leg.sort((x, y) => cmp(lkey(x), lkey(y)));
+  const index = new Map();
+  cap.forEach((c, i) => {
+    const sig = `${prof.tsExact ? c.ts : c.agent}\u0001${c.name}`;
+    if (!index.has(sig)) index.set(sig, []);
+    index.get(sig).push(i);
+  });
+  const cands = [];
+  const add = (L, k, s, linked) => {
+    const c = cap[s];
+    if (c.name !== L.name || (prof.tsExact ? c.ts !== L.ts : c.agent !== L.agent)) return false;
+    const fit = argsFit(L.args, c.args);
+    if (fit === 'none') return false;
+    cands.push({ k, s, e: s, w: MATCH + (linked ? 200 : 0) + FIT_W[fit] + (c.agent === L.agent ? 40 : 0) });
+    return true;
+  };
+  // Phase 1: a call whose message matched looks in that message's run. Phase 2: the rest look
+  // between the calls their neighbours matched — or everywhere, when nothing matched at all.
+  const rest = [];
+  leg.forEach((L, k) => {
+    let any = false;
+    for (const mid of runOf.get(String(L.row.message_id)) || []) for (const s of byMid.get(mid) || []) any = add(L, k, s, true) || any;
+    if (!any) rest.push(k);
+  });
+  if (rest.length) {
+    const { lo, hi } = windows(bestChain(cands.slice(), leg.length), leg.length, cap.length);
+    for (const k of rest) {
+      const L = leg[k];
+      let starts = between(index.get(`${prof.tsExact ? L.ts : L.agent}\u0001${L.name}`) || [], lo[k], hi[k]);
+      if (starts.length > CANDIDATES_PER_ROW) starts = nearest(starts, k, leg.length, cap.length);
+      for (const s of starts) add(L, k, s, false);
+    }
+  }
+  for (const c of bestChain(cands, leg.length)) leg[c.k].match = cap[c.s];
+  // Echoes: the 0.18 parser emitted a record re-appended to the transcript (a resume, a sub-agent
+  // replay) once per copy, the capture once per uuid. The turns matcher folds those copies into
+  // the one captured record (`echo`); a call is one-to-one above, so the extra copies of its call
+  // stayed live (2026-10-14, the dev house: 77 pairs in 16 sessions). Where the legacy ts is the
+  // record's own time, an unmatched legacy call with the same ts, tool and arguments as a captured
+  // call is that call again.
+  if (prof.tsExact) {
+    for (const L of leg) {
+      if (L.match) continue;
+      for (const s of index.get(`${L.ts}\u0001${L.name}`) || []) {
+        const fit = argsFit(L.args, cap[s].args);
+        if (fit === 'exact' || fit === 'contained') { L.match = cap[s]; L.echo = true; break; }
+      }
+    }
+  }
+  // Phantoms: the 0.18 conversion parsed '[tool-call: X]' wherever a text held it, prose included
+  // (a session describing the marker, another house's transcript quoted), into a call without
+  // arguments that the transcript never made (2026-10-23, the dev house: 14 calls in 4 Claude
+  // Code sessions). A `{}` call is retired, not matched (`phantom`), when its message matched into
+  // a run that holds no tool_call of that name and holds the marker as prose. After a tombstone
+  // cleanup the message's own legacy row is gone with its tombstone; where the legacy ts is the
+  // record's own, the run is then the captured records at the call's ts, of any agent: a 0.18 parse
+  // before per-agent numbering shipped a sub-agent's calls with no agent id, and the turns matcher
+  // keys these by ts alone too.
+  const known = new Set(legacyTurns.map((L) => String(L.row.message_id)));
+  const atTs = new Map();
+  if (prof.tsExact) for (const a of atoms) { if (!atTs.has(a.ts)) atTs.set(a.ts, new Set()); atTs.get(a.ts).add(a.mid); }
+  for (const L of leg) {
+    if (L.match || !L.args.o || Object.keys(L.args.o).length) continue;
+    const mid = String(L.row.message_id);
+    const run = runOf.get(mid) || (prof.tsExact && !known.has(mid) ? [...(atTs.get(L.ts) || [])] : []);
+    const mine = run.flatMap((m) => atomsByMid.get(m) || []);
+    if (mine.some((a) => a.kind === 'tool_call' && a.call && a.call.name === L.name)) continue;
+    const marker = `[tool-call: ${L.name}`;
+    if (mine.some((a) => a.kind !== 'tool_call' && a.text.includes(marker))) L.phantom = true;
+  }
+  return leg;
+}
+
+// ── the plan ─────────────────────────────────────────────────────────────────────────────
+function numTurn(r) {
+  return {
+    ...r, text_len: Number(r.text_len) || 0, input_tokens: Number(r.input_tokens) || 0, output_tokens: Number(r.output_tokens) || 0,
+    cache_read_tokens: Number(r.cache_read_tokens) || 0, cache_write_tokens: Number(r.cache_write_tokens) || 0,
+    reasoning_tokens: Number(r.reasoning_tokens) || 0, agent_id: String(r.agent_id || ''), model: String(r.model || ''),
+  };
+}
+const truthy = (v) => v === true || v === 1 || v === '1' || v === 'true';
+
+/** The contribution (contrib JSON, as a files row carries it) of the legacy rows that stay live. */
+function keptContrib(turns, calls) {
+  const c = base.newContrib();
+  for (const r of turns) base.applyTurn(c, numTurn(r), +1);
+  for (const r of calls) base.applyCall(c, { tool_name: String(r.tool_name), is_error: truthy(r.is_error), ts: String(r.ts) }, +1);
+  const out = base.serializeContrib(c);
+  // What the legacy rows knew about their sub-agents (convert-v3 kept it in extra.agent).
+  const byAgent = new Map();
+  for (const r of turns) {
+    if (!r.agent_id || byAgent.has(r.agent_id)) continue;
+    let ag = null;
+    try { ag = (JSON.parse(r.extra || '{}') || {}).agent || null; } catch { /* not JSON */ }
+    if (ag) byAgent.set(r.agent_id, ag);
+  }
+  for (const a of out.agents) {
+    const ag = byAgent.get(a.agent_id);
+    if (ag) { a.type = String(ag.type || ''); a.description = String(ag.description || ''); a.workflow = String(ag.workflow || ''); }
+  }
+  // A fallback the session builder uses only when the capture has none.
+  const first = turns.filter((r) => r.role === 'user' && !r.agent_id && r.kind === 'text' && String(r.text || '').trim())
+    .sort((x, y) => cmp([String(x.ts), Number(x.ord) || 0], [String(y.ts), Number(y.ord) || 0]))[0];
+  out.meta = {};
+  if (first) out.meta.first_prompt = String(first.text).slice(0, 2000);
+  const folder = turns.find((r) => r.folder);
+  if (folder) out.meta.folder = String(folder.folder);
+  out.legacy = {
+    turns: turns.length, calls: calls.length,
+    ship_turns: turns.filter((r) => isShip(r.message_id)).length, ship_calls: calls.filter((r) => isShip(r.call_id)).length,
+  };
+  return out;
+}
+
+/**
+ * Decide every legacy row of ONE session.
+ *
+ * @param o.harness, o.session_id
+ * @param o.legacyTurns   every `legacy:` turn of the session — all versions or the newest per key,
+ *                        live AND tombstoned, full rows (every insertable column)
+ * @param o.legacyCalls   the same for calls
+ * @param o.capturedTurns live captured turns (message_id not 'legacy:'), at least: message_id, part,
+ *                        ts, ord, role, kind, text, agent_id, response_id, extra
+ * @param o.capturedCalls live captured calls: call_id, message_id, ts, tool_name, args, agent_id
+ * @param o.repeats       Codex only: codexRepeats of the session's rollouts, the proof `rereport`
+ *                        needs; without it those rows stay live (report.rereport_unproven counts them)
+ * @returns {{ tombstones: {turns, calls}, kept: {turns, calls}, contrib, report, deadShip, pairs }}
+ */
+function planDedupe(o) {
+  const prof = o.profile || profileFor(o.harness);
+  const turns = newestPerKey(o.legacyTurns || [], base.turnKey).filter((r) => isLegacy(r.message_id));
+  const calls = newestPerKey(o.legacyCalls || [], base.callKey).filter((r) => isLegacy(r.call_id));
+  const capT = (o.capturedTurns || []).filter((r) => !isLegacy(r.message_id) && !dead(r));
+  const capC = (o.capturedCalls || []).filter((r) => !isLegacy(r.call_id) && !dead(r));
+  const shipT = turns.filter((r) => isShip(r.message_id));
+  const shipC = calls.filter((r) => isShip(r.call_id));
+
+  let LT; let LC;
+  if (prof.byText) {
+    ({ LT, LC } = matchByText(shipT, shipC, capT, capC));
+    const rest = LT.filter((L) => !L.match);
+    if (rest.length) {
+      const used = new Set(LT.filter((L) => L.match).flatMap((L) => L.match.map((a) => `${a.mid}\u0000${a.part}`)));
+      const free = capT.filter((r) => !used.has(`${r.message_id}\u0000${Number(r.part_ord) || 0}`));
+      const m = matchTurns(o.harness, PIECE_PROFILE, rest.map((L) => L.row), free);
+      const got = new Map(m.legacy.filter((x) => x.match && x.match.length).map((x) => [base.turnKey(x.row), x]));
+      for (const L of rest) { const x = got.get(base.turnKey(L.row)); if (x) { L.match = x.match.map((a) => ({ mid: a.mid, part: a.part })); L.how = 'pieces'; } }
+    }
+  }
+  else {
+    const m = matchTurns(o.harness, prof, shipT, capT, o.repeats ? repeatPool(o.repeats) : null);
+    LT = m.legacy;
+    LC = matchCalls(prof, shipC, capC, LT, m.atoms);
+  }
+
+  const tomb = { turns: [], calls: [] };
+  const matched = { turns: new Set(), calls: new Set() };
+  const how = { aligned: 0, echo: 0, activity: 0, rereport: 0, pieces: 0 };
+  const unproven = LT.filter((L) => !L.match && L.unproven && !dead(L.row)).length;
+  for (const L of LT) {
+    if (!L.match) continue;
+    matched.turns.add(base.turnKey(L.row));
+    if (!dead(L.row)) { tomb.turns.push(base.tombstoneOf('turns', L.row, BigInt(L.row.rev || 0) + 1n)); how[L.how]++; }
+  }
+  const phantom = new Set();
+  for (const L of LC) {
+    if (!L.match && !L.phantom) continue;
+    (L.match ? matched.calls : phantom).add(base.callKey(L.row));
+    if (!dead(L.row)) tomb.calls.push(base.tombstoneOf('calls', L.row, BigInt(L.row.rev || 0) + 1n));
+  }
+  const kept = {
+    turns: turns.filter((r) => !dead(r) && !matched.turns.has(base.turnKey(r))),
+    calls: calls.filter((r) => !dead(r) && !matched.calls.has(base.callKey(r)) && !phantom.has(base.callKey(r))),
+  };
+  const deadShip = shipT.filter(dead).length + shipC.filter(dead).length + tomb.turns.length + tomb.calls.length;
+  const count = (rows, f) => rows.filter(f).length;
+  const report = {
+    harness: o.harness, session_id: o.session_id,
+    legacy: { turns: turns.length, calls: calls.length },
+    ship: { turns: shipT.length, calls: shipC.length },
+    dead_before: { turns: count(shipT, dead), calls: count(shipC, dead) },
+    matched: { turns: matched.turns.size, calls: matched.calls.size },
+    tombstoned: { turns: tomb.turns.length, calls: tomb.calls.length },
+    kept: { turns: count(kept.turns, (r) => isShip(r.message_id)), calls: count(kept.calls, (r) => isShip(r.call_id)) },
+    imports: { turns: count(kept.turns, (r) => !isShip(r.message_id)), calls: count(kept.calls, (r) => !isShip(r.call_id)) },
+    captured: { turns: capT.length, calls: capC.length },
+    how, rereport_unproven: unproven, phantom_calls: phantom.size,
+  };
+  const pairs = LT.filter((L) => L.match).map((L) => [L.row.message_id, L.match.map((a) => `${a.mid}#${a.part}`)]);
+  return { tombstones: tomb, kept, contrib: keptContrib(kept.turns, kept.calls), report, deadShip, pairs };
+}
+
+/**
+ * The `files` row that carries the live legacy rows' share of the session totals and marks the
+ * session deduped. Not a source file: fgen 0, and `upto` a counter (1 + the legacy ship rows
+ * tombstoned) — so its ver only rises, and sessions.rev = Σ files.ver rises with it.
+ */
+function legacyFilesRow({ harness, session_id, harness_path = '', incarnation = '' }, plan, host) {
+  const c = plan.contrib;
+  return {
+    harness, harness_path, session_id, incarnation, file_key: LEGACY_FILE_KEY, agent_id: '', role: LEGACY_ROLE, agent_type: '', agent_desc: '',
+    fgen: 0, upto: plan.deadShip + 1, reparse_from: 0,
+    derive_state: JSON.stringify({ v: 1, dedupe: plan.report }),
+    adapter_ver: 0, size: 0, inode: 0, anchor_sha: ZERO_SHA, prefix_sha: ZERO_SHA, is_deleted: 0,
+    contrib: JSON.stringify(c), first_ts: c.first_ts || EPOCH_TS, last_ts: c.last_ts || EPOCH_TS, host: String(host || ''),
+  };
+}
+
+// ── doctor: captured + legacy pairs with the same text ───────────────────────────────────
+// What dedupe leaves behind, measured without the matcher: per session, pairs of live rows —
+// one captured, one legacy — of the same role class whose normalised text is identical (the
+// '[subagent] ' tag and a leading '[thinking] ' marker dropped, whitespace collapsed). Pairs
+// count multiplicity: three legacy and two captured rows of one text are two pairs. Only
+// single-piece rows can pair (a folded Codex turn or a multi-block record equals no one row),
+// so this undercounts what it cannot see and never invents a duplicate of a different text.
+//   pairs_ship      legacy ship rows — 0 after dedupe, except a text the transcript repeats
+//                   fewer times than the legacy parse did (the transcript lost one)
+//   same_ts_pairs   ship pairs with the same ts too: 0 after dedupe for harnesses whose legacy
+//                   rows carry the record's own time (Claude Code)
+//   pairs_import    legacy import rows: never tombstoned, reported so they are seen
+//   call_pairs      live legacy ship calls with a captured call of the same tool and arguments
+//                   (whitespace removed)
+const DUP_KINDS = ['text', 'thinking', 'context'];
+function dupText(text) {
+  let t = stripTag(text);
+  if (t.startsWith('[thinking] ')) t = t.slice(11);
+  return norm(t);
+}
+/** The doctor on rows already read (live rows of one or more sessions). */
+function findDuplicates(turns, calls = []) {
+  const per = new Map();
+  const S = (r) => { const k = `${r.harness}\u0000${r.session_id}`; if (!per.has(k)) per.set(k, { harness: r.harness, session_id: r.session_id, t: new Map(), ts: new Map(), c: new Map() }); return per.get(k); };
+  const bump = (m, k, f) => { const v = m.get(k) || { l: 0, i: 0, c: 0 }; v[f]++; m.set(k, v); };
+  for (const r of turns) {
+    if (dead(r) || !DUP_KINDS.includes(String(r.kind))) continue;
+    const x = dupText(r.text);
+    if (!x) continue;
+    const f = !isLegacy(r.message_id) ? 'c' : (isShip(r.message_id) ? 'l' : 'i');
+    const s = S(r);
+    bump(s.t, `${roleClass(r.role)}\u0000${x}`, f);
+    if (f !== 'i') bump(s.ts, `${roleClass(r.role)}\u0000${r.ts}\u0000${x}`, f);
+  }
+  for (const r of calls) {
+    if (dead(r) || (isLegacy(r.call_id) && !isShip(r.call_id))) continue;
+    bump(S(r).c, `${r.tool_name}\u0000${String(r.args ?? '').replace(/\s+/g, '')}`, isLegacy(r.call_id) ? 'l' : 'c');
+  }
+  const out = [];
+  for (const s of per.values()) {
+    const sum = (m, f) => [...m.values()].reduce((a, v) => a + Math.min(v[f], v.c), 0);
+    const row = { harness: s.harness, session_id: s.session_id, pairs_ship: sum(s.t, 'l'), same_ts_pairs: sum(s.ts, 'l'), pairs_import: sum(s.t, 'i'), call_pairs: sum(s.c, 'l') };
+    if (row.pairs_ship || row.same_ts_pairs || row.pairs_import || row.call_pairs) out.push(row);
+  }
+  return out.sort((a, b) => cmp([a.harness, a.session_id], [b.harness, b.session_id]));
+}
+
+function lit(v) { return `'${String(v).replace(/\\/g, '\\\\').replace(/'/g, "\\'")}'`; }
+/**
+ * The same doctor in the house: `q.rows(sql, settings)`, N = { turns, calls } (schema-v3 v3Names).
+ * `session` ({ harness, session_id }) narrows it to one. Reads with FINAL and is_deleted = 0.
+ * @returns {{ sessions: [{ harness, session_id, pairs_ship, same_ts_pairs, pairs_import, call_pairs }], total }}
+ */
+async function duplicates(q, N, { session = null } = {}) {
+  const only = session ? ` AND harness = ${lit(session.harness)} AND session_id = ${lit(session.session_id)}` : '';
+  const t = `if(startsWith(text, '[subagent] '), substring(text, 12), text)`;
+  const x = `trimBoth(replaceRegexpAll(if(startsWith(${t}, '[thinking] '), substring(${t}, 12), ${t}), '\\\\s+', ' '))`;
+  const settings = { final: 0, join_use_nulls: 0, output_format_json_quote_64bit_integers: 0 };
+  const rows = await q.rows(`
+    SELECT h AS harness, s AS session_id, sum(pl) AS pairs_ship, sum(pim) AS pairs_import, sum(pt) AS same_ts_pairs, sum(pc) AS call_pairs
+    FROM (
+      SELECT h, s, least(countIf(o = 'l'), countIf(o = 'c')) AS pl, least(countIf(o = 'i'), countIf(o = 'c')) AS pim, 0 AS pt, 0 AS pc
+      FROM (SELECT toString(harness) AS h, session_id AS s, multiIf(NOT startsWith(message_id, 'legacy:'), 'c', startsWith(message_id, 'legacy:ship:'), 'l', 'i') AS o,
+                   multiIf(role = 'user', 'user', role = 'system', 'system', 'assistant') AS rc, cityHash64(${x}) AS th
+            FROM ${N.turns} FINAL WHERE is_deleted = 0 AND removed = 0 AND kind IN ('text', 'thinking', 'context') AND ${x} != ''${only})
+      GROUP BY h, s, rc, th
+      UNION ALL
+      SELECT h, s, 0, 0, least(countIf(o = 'l'), countIf(o = 'c')), 0
+      FROM (SELECT toString(harness) AS h, session_id AS s, if(startsWith(message_id, 'legacy:'), 'l', 'c') AS o,
+                   multiIf(role = 'user', 'user', role = 'system', 'system', 'assistant') AS rc, ts, cityHash64(${x}) AS th
+            FROM ${N.turns} FINAL WHERE is_deleted = 0 AND removed = 0 AND kind IN ('text', 'thinking', 'context') AND ${x} != ''
+              AND (NOT startsWith(message_id, 'legacy:') OR startsWith(message_id, 'legacy:ship:'))${only})
+      GROUP BY h, s, rc, ts, th
+      UNION ALL
+      SELECT h, s, 0, 0, 0, least(countIf(o = 'l'), countIf(o = 'c'))
+      FROM (SELECT toString(harness) AS h, session_id AS s, if(startsWith(call_id, 'legacy:'), 'l', 'c') AS o,
+                   cityHash64(toString(tool_name), replaceRegexpAll(args, '\\\\s+', '')) AS th
+            FROM ${N.calls} FINAL WHERE is_deleted = 0 AND removed = 0 AND (NOT startsWith(call_id, 'legacy:') OR startsWith(call_id, 'legacy:ship:'))${only})
+      GROUP BY h, s, th
+    )
+    GROUP BY h, s
+    HAVING pairs_ship + pairs_import + same_ts_pairs + call_pairs > 0
+    ORDER BY h, s`, settings);
+  const sessions = rows.map((r) => ({
+    harness: r.harness, session_id: r.session_id, pairs_ship: Number(r.pairs_ship), same_ts_pairs: Number(r.same_ts_pairs),
+    pairs_import: Number(r.pairs_import), call_pairs: Number(r.call_pairs),
+  }));
+  const total = { sessions: sessions.length, pairs_ship: 0, same_ts_pairs: 0, pairs_import: 0, call_pairs: 0 };
+  for (const s of sessions) for (const k of ['pairs_ship', 'same_ts_pairs', 'pairs_import', 'call_pairs']) total[k] += s[k];
+  return { sessions, total };
+}
+
+module.exports = {
+  LEGACY_FILE_KEY, LEGACY_ROLE, PROFILES, profileFor,
+  planDedupe, legacyFilesRow, keptContrib, findDuplicates, duplicates,
+  // exported for the unit gate
+  legacyPieces, argsFit, bestChain, newestPerKey, matchTurns, dupText, codexRepeats,
+};