memhouse 0.18.12 → 0.20.1

package/memhouse/shipper/dedupe-profile.js

@@ -0,0 +1,237 @@
+// Step B's legacy-to-profile reconciliation: dedupe-v3's matcher over harness-sessions-sql profile rows.
+//
+// One matcher for both paths. On the profile, the legacy rows are the conversion's (adapter
+// 'legacy', convert-profile.js) and the captured rows are an adapter's (records-to-profile.js).
+// Two pure adapters at the matcher's boundary turn them into the shapes it already reads; the
+// matcher core does not change.
+//
+//   converted legacy part row → legacy shape
+//     text          where the conversion kept 0.18's text: `text` (text, reasoning), `output`
+//                   (tool_result), extra.legacy_text (tool_call); a usage-only header row (kind
+//                   'none') is '[assistant activity]'
+//     role          the conversion's role 'tool' comes from a tool_result marker in a 0.18
+//                   assistant row: back to 'assistant'. A 0.18 row stored with role 'tool' kept
+//                   kind 'text' and keeps its role
+//     kind          back to 0.18's names: reasoning → thinking, none → usage
+//     tokens        input and output from the row's message-anchored session_usage row, in any
+//                   version (a tombstone keeps them, so a pass stopped after the usage went still
+//                   proves; Codex's re-report proof reads them); a per_record run's usage is not split back
+//   profile part row → captured shape
+//     part_ord      part_index; ord the message's rank by order_key in the session
+//     text          a tool call is `tool_name args` (the package form the matcher reads); a tool
+//                   result is `output`; an error part its `error_message` (where the profile keeps an
+//                   API error's line); other kinds their text. Kinds and role 'tool' pass as is
+//     calls         one per tool_call part with a call_id
+//   ts              both sides in the house's wire form ('YYYY-MM-DD hh:mm:ss.mmm', UTC)
+//
+// A retired row is tombstoned as a full copy of the newest version of the profile row it came
+// from: same key, rev + 1, is_deleted 1. A retired message's session_usage rows go with it: a
+// message-anchored row by its message, a Claude Code run's row ('response:<run>#<n>') by the
+// message's rank in its run (the conversion counts n in seq order). Every dead legacy message's
+// usage is retired, not only this plan's, so a pass that stopped between the two writes is
+// finished by the next. Pure: rows in, the plan out.
+
+'use strict';
+
+const { capture: base } = require('../../editors/base');
+const dedupe = require('./dedupe-v3');
+
+const s = (v) => (v == null ? '' : String(v));
+/** A ts in the house's wire form; an ISO string (records-to-profile) is converted, null stays null. */
+function wireTs(v) {
+  if (v == null || v === '') return null;
+  const t = String(v);
+  if (/^\d{4}-\d\d-\d\d \d\d:\d\d:\d\d\.\d{3}$/.test(t)) return t;
+  const ms = Date.parse(t);
+  return Number.isFinite(ms) ? base.chTs(ms) : t;
+}
+const LEGACY_KIND = { reasoning: 'thinking', none: 'usage' };
+
+/** The 0.18 text the conversion kept for a converted legacy part row. */
+function legacyTextOf(p) {
+  const kind = s(p.kind);
+  if (kind === 'tool_result') return s(p.output);
+  if (kind === 'none') return '[assistant activity]';
+  if (kind === 'tool_call') {
+    try { const x = JSON.parse(s(p.extra) || '{}').legacy_text; if (typeof x === 'string') return x; } catch { /* no extra */ }
+    return `[tool-call: ${s(p.tool_name)}]`;
+  }
+  return s(p.text);
+}
+
+/** The schema-3 turn columns of a row, defaults where the profile has none. */
+function turnRow(o) {
+  return {
+    harness: s(o.harness), harness_version: '', harness_path: s(o.harness_path), session_id: s(o.session_id), incarnation: s(o.incarnation),
+    agent_id: s(o.agent_id), message_id: s(o.message_id), part_id: s(o.part_id), part_ord: Number(o.part_ord) || 0, parent_id: s(o.parent_id),
+    response_id: s(o.response_id), ts: o.ts, ord: Number(o.ord) || 0, rev: s(o.rev || 0), is_deleted: Number(o.is_deleted) || 0, removed: Number(o.removed) || 0,
+    state: s(o.state), role: s(o.role), kind: s(o.kind), model: s(o.model), effort: '',
+    input_tokens: Number(o.input_tokens) || 0, output_tokens: Number(o.output_tokens) || 0, cache_read_tokens: 0, cache_write_tokens: 0, reasoning_tokens: 0,
+    text: s(o.text), text_len: [...s(o.text)].length, folder: '', project: '', host: s(o.host), origin: '', extra: s(o.extra || '{}'), adapter_ver: 0,
+  };
+}
+function callRow(o) {
+  return {
+    harness: s(o.harness), harness_path: s(o.harness_path), session_id: s(o.session_id), incarnation: s(o.incarnation), agent_id: s(o.agent_id),
+    call_id: s(o.call_id), message_id: s(o.message_id), ts: o.ts, rev: s(o.rev || 0), is_deleted: Number(o.is_deleted) || 0, removed: Number(o.removed) || 0,
+    tool_name: s(o.tool_name), args: s(o.args), result_ts: null, is_error: false, result_len: 0, model: s(o.model), folder: '', project: '', host: s(o.host), adapter_ver: 0,
+  };
+}
+
+/** Converted legacy part rows (and their session_usage rows) → legacy turn rows. */
+function legacyTurnsOf(parts, usage = []) {
+  const tokens = new Map();
+  for (const u of usage) if (s(u.anchor_kind) === 'message' && !tokens.has(s(u.anchor_id))) tokens.set(s(u.anchor_id), u);
+  return parts.map((p) => {
+    const kind = s(p.kind);
+    const u = tokens.get(s(p.message_id)) || {};
+    const seq = Number(s(p.message_id).split(':').pop());
+    return turnRow({
+      ...p, ts: wireTs(p.ts), ord: Number.isFinite(seq) ? seq : 0, part_ord: 0, model: p.served_model,
+      role: kind === 'tool_result' ? 'assistant' : s(p.role), kind: LEGACY_KIND[kind] || kind, text: legacyTextOf(p),
+      input_tokens: u.input, output_tokens: u.output,
+    });
+  });
+}
+/** Converted legacy calls rows ({m}_calls, adapter 'legacy') → legacy call rows. */
+function legacyCallsOf(calls) { return calls.map((c) => callRow({ ...c, ts: wireTs(c.ts) })); }
+
+/**
+ * An adapter's session_parts rows → captured turn rows and calls. With its session_usage rows: a usage
+ * record whose anchor (a message, or a response) holds no part 0.18 would have shown (a Codex turn that
+ * spent its tokens on opaque reasoning, or kept nothing at all) is a `usage` turn as well, as the 0.20
+ * engines wrote one: what a converted '[assistant activity]' row matches. It sits at its anchor's
+ * message, or by its time between the messages around it. One with neither (an anchor that kept no
+ * message: its usage has no time) has no place in the order: it comes back in `loose`.
+ */
+function capturedOf(parts, usage = []) {
+  const order = [...new Set([...parts].sort((a, b) => (s(a.order_key) < s(b.order_key) ? -1 : s(a.order_key) > s(b.order_key) ? 1 : 0)).map((p) => s(p.message_id)))];
+  const rank = new Map(order.map((m, i) => [m, i]));
+  const turns = []; const calls = [];
+  const textOf = (p, kind) => (kind === 'tool_call' ? `${s(p.tool_name)} ${s(p.args)}` : kind === 'tool_result' ? s(p.output)
+    : kind === 'error' ? (s(p.text) || s(p.error_message)) : s(p.text));
+  for (const p of parts) {
+    const kind = s(p.kind);
+    const ts = wireTs(p.ts);
+    turns.push(turnRow({ ...p, ts, ord: rank.get(s(p.message_id)), part_ord: p.part_index, model: p.served_model || p.requested_model, text: textOf(p, kind),
+      extra: JSON.stringify(p.call_id ? { call_id: s(p.call_id) } : {}) }));
+    if (kind === 'tool_call' && s(p.call_id)) calls.push(callRow({ ...p, ts, model: p.served_model || p.requested_model }));
+  }
+  const shown = (p) => { const kind = s(p.kind); return kind === 'tool_call' || kind === 'tool_result' || textOf(p, kind) !== ''; };
+  const firstTs = new Map();
+  const byMessage = new Map(); const byResponse = new Map();
+  const add = (m, k, p) => { if (!k) return; if (!m.has(k)) m.set(k, []); m.get(k).push(p); };
+  for (const p of parts) {
+    const t = wireTs(p.ts); const m = s(p.message_id);
+    if (t && (!firstTs.has(m) || t < firstTs.get(m))) firstTs.set(m, t);
+    add(byMessage, m, p); add(byResponse, s(p.response_id), p);
+  }
+  const loose = [];
+  for (const u of usage) {
+    const ak = s(u.anchor_kind); const id = s(u.anchor_id);
+    if (ak !== 'message' && ak !== 'response') continue;
+    const at = (ak === 'message' ? byMessage : byResponse).get(id) || [];
+    if (at.some(shown)) continue;
+    const ts = wireTs(u.ts) || (at.length ? wireTs(at[0].ts) : null);
+    if (!at.length && !ts) { loose.push(u); continue; }
+    let ord;
+    if (at.length) ord = rank.get(s(at[0].message_id));
+    else {
+      // Between the messages around it: after the last one that began no later.
+      let before = -1;
+      for (const m of order) if (ts && firstTs.get(m) && firstTs.get(m) <= ts) before = Math.max(before, rank.get(m));
+      ord = before + 0.5;
+    }
+    turns.push(turnRow({ ...(at[0] || parts[0] || {}), message_id: `@usage:${s(u.usage_id)}`, part_id: '', part_ord: at.reduce((a, p) => Math.max(a, (Number(p.part_index) || 0) + 1), 0),
+      ts, ord, role: 'assistant', kind: 'usage', text: '', response_id: ak === 'response' ? id : s(at[0] && at[0].response_id), model: s(u.model), extra: '{}' }));
+  }
+  return { turns, calls, loose };
+}
+
+/** A full copy of the profile row a retired legacy row came from: rev + 1, is_deleted 1. */
+function tombstone(src) { return { ...src, rev: String(BigInt(src.rev || 0) + 1n), is_deleted: 1 }; }
+
+/** key → the newest version's source row (rows[i] was read from src[i]). */
+function newestSources(rows, src, keyOf) {
+  return new Map(dedupe.newestPerKey(rows.map((r, i) => ({ ...r, i })), keyOf).map((r) => [keyOf(r), src[r.i]]));
+}
+
+const cmp = (a, b) => (a < b ? -1 : a > b ? 1 : 0);
+/** The live session_usage rows of these legacy messages, in the stores they were converted to. */
+function usageOf(dead, parts, usage) {
+  const store = (r) => `${s(r.host)}\u0000${s(r.harness_path)}`;
+  const runs = new Map();
+  for (const p of parts) {
+    if (!s(p.response_id)) continue;
+    const k = `${store(p)}\u0000${s(p.response_id)}`;
+    if (!runs.has(k)) runs.set(k, new Map());
+    runs.get(k).set(s(p.message_id), s(p.order_key));
+  }
+  const ids = new Set();
+  for (const p of dead) {
+    const rid = s(p.response_id);
+    if (!rid) { ids.add(`${store(p)}\u0000message:${s(p.message_id)}#1`); continue; }
+    const run = [...runs.get(`${store(p)}\u0000${rid}`)].sort((a, b) => cmp(a[1], b[1]) || cmp(a[0], b[0])).map(([m]) => m);
+    ids.add(`${store(p)}\u0000response:${rid}#${run.indexOf(s(p.message_id)) + 1}`);
+  }
+  const key = (u) => `${store(u)}\u0000${s(u.usage_id)}`;
+  return dedupe.newestPerKey(usage, key).filter((u) => !Number(u.is_deleted) && ids.has(key(u)));
+}
+
+/**
+ * The dedupe plan on profile rows.
+ * @param {object} o
+ * @param {string} o.harness
+ * @param {string} o.session_id
+ * @param {object[]} o.legacyParts    converted session_parts rows (adapter 'legacy'), live and tombstoned
+ * @param {object[]} [o.legacyUsage]  their session_usage rows, live and tombstoned (tokens, and what a retired message takes with it)
+ * @param {object[]} [o.legacyCalls]  converted {m}_calls rows
+ * @param {object[]} o.capturedParts  an adapter's session_parts rows
+ * @param {object[]} [o.capturedUsage] its session_usage rows (usage_id, anchor_kind, anchor_id, ts, model, input, output): a turn's tokens with no part to show
+ * @param {Array} [o.repeats]         Codex re-report proofs, [{ input, output }] (dedupe-v3 codexRepeats)
+ * @returns {{tombstones: {session_parts: object[], calls: object[], session_usage: object[]}, kept: {session_parts: object[], calls: object[]},
+ *   report: object, pairs: Array, dead: number}}  dead: the legacy ship rows dead once the tombstones land (dedupe-v3 deadShip)
+ */
+function planProfileDedupe(o) {
+  const parts = o.legacyParts || []; const calls = o.legacyCalls || []; const usage = o.legacyUsage || [];
+  const lt = legacyTurnsOf(parts, usage);
+  const lc = legacyCallsOf(calls);
+  const cap = capturedOf(o.capturedParts || [], o.capturedUsage || []);
+  const srcT = newestSources(lt, parts, base.turnKey);
+  const srcC = newestSources(lc, calls, base.callKey);
+  const plan = dedupe.planDedupe({
+    harness: o.harness, session_id: o.session_id, profile: o.profile, repeats: o.repeats,
+    legacyTurns: lt, legacyCalls: lc, capturedTurns: cap.turns, capturedCalls: cap.calls,
+  });
+  // A kept '[assistant activity]' row (a shipped one: imports are never touched) whose turn kept no
+  // message: its captured usage has no place in the order, so it is matched by its tokens, each record
+  // once: the same non-zero output, and the same input net of the cache on each side (0.18 kept Codex
+  // input without its cached tokens; the package counts them in).
+  const net = (u) => { const i = Number(u.input) || 0; return u.input_includes_cache === true || u.input_includes_cache === 'true' || u.input_includes_cache === 1
+    ? Math.max(i - (Number(u.cache_read) || 0) - (Number(u.cache_write) || 0), 0) : i; };
+  const legacyUsageOf = new Map(usage.filter((u) => s(u.anchor_kind) === 'message').map((u) => [s(u.anchor_id), u]));
+  const loose = [...cap.loose];
+  const retiredTurns = [...plan.tombstones.turns]; const keptTurns = [];
+  for (const t of plan.kept.turns) {
+    const lu = legacyUsageOf.get(s(t.message_id));
+    const i = t.kind === 'usage' && s(t.message_id).startsWith('legacy:ship:') && lu && (Number(lu.input) || Number(lu.output))
+      ? loose.findIndex((u) => Number(u.output) === Number(lu.output) && net(u) === net(lu)) : -1;
+    if (i < 0) { keptTurns.push(t); continue; }
+    loose.splice(i, 1);
+    retiredTurns.push(t);
+    plan.report.matched.turns++; plan.report.tombstoned.turns++; plan.report.kept.turns--; plan.deadShip++;
+  }
+  const retired = retiredTurns.map((t) => srcT.get(base.turnKey(t)));
+  const deadBefore = [...srcT.values()].filter((p) => Number(p.is_deleted));
+  return {
+    tombstones: {
+      session_parts: retired.map(tombstone),
+      calls: plan.tombstones.calls.map((t) => tombstone(srcC.get(base.callKey(t)))),
+      session_usage: usageOf([...deadBefore, ...retired], parts, usage).map(tombstone),
+    },
+    kept: { session_parts: keptTurns.map((t) => srcT.get(base.turnKey(t))), calls: plan.kept.calls.map((t) => srcC.get(base.callKey(t))) },
+    report: plan.report, pairs: plan.pairs, dead: plan.deadShip,
+  };
+}
+
+module.exports = { planProfileDedupe, legacyTurnsOf, legacyCallsOf, capturedOf, legacyTextOf, wireTs };