memhouse 0.20.1 → 0.20.2

package/memhouse/shipper/capture.js

@@ -445,6 +445,18 @@ async function tombstonesFor(ctx, fc, vanished, rev, { removed = false } = {}) {
   }
   return out;
 }
+/**
+ * An adapter upgrade's rows, at a rev above every row the house holds for the file. An unchanged
+ * file re-derives at revOf(fgen, upto), the rev its last pass wrote, and at an equal rev the new
+ * version's row would win by insertion order alone. The raise is a rev or two; the next append's
+ * revOf moves on by the length of what it read.
+ */
+async function aboveHouse(ctx, ref, agentId, agentFileIds, rev, res) {
+  const floor = BigInt(await ctx.house.maxRev(ref, agentId, agentFileIds)) + 1n;
+  if (BigInt(rev) >= floor) return { rev, turns: res.turns, calls: res.calls };
+  const up = String(floor);
+  return { rev: up, turns: res.turns.map((r) => ({ ...r, rev: up })), calls: res.calls.map((r) => ({ ...r, rev: up })) };
+}
 /** Keys ({ t: [[message_id, part_id, ts]], c: [[call_id, ts]] }) the new derivation no longer produces. */
 function vanishedFrom(old, res) {
   const have = { t: new Set(res.turns.map(base.turnKey)), c: new Set(res.calls.map(base.callKey)) };
@@ -571,7 +583,10 @@ async function captureFile(ctx, f, H, C, budget) {
       const fbIds = (fb) => Object.fromEntries(Object.entries(fb || {}).filter(([k]) => k.startsWith('id:')));
       const freshState = { ...base.initialState(ad.adapter), fb: rewrite ? fbIds(state && state.fb) : ((state && state.fb) || {}) };
       const res = ad.derive({ file: fc, records, state: freshState });
-      const rev = String(base.revOf(newFgen, newUpto));
+      const at = reAdapter
+        ? await aboveHouse(ctx, refOf(f, ctx.host), f.agent_id || '', ctx.agentFileIds(f), String(base.revOf(newFgen, newUpto)), res)
+        : { rev: String(base.revOf(newFgen, newUpto)), turns: res.turns, calls: res.calls };
+      const rev = at.rev;
       const tomb = rewrite && reAdapter ? await retireBoth(ctx, f, H, state, fc, oldKeys, res, rev)
         : await tombstonesFor(ctx, fc, vanishedFrom(oldKeys, res), rev, { removed: !!rewrite });
       // A new generation is archived whole; an upgrade archives only what lies past `upto`.
@@ -580,8 +595,8 @@ async function captureFile(ctx, f, H, C, budget) {
       // an upgrade that read past the cursor appended to the one it had.
       if (rewrite) out.archive.push(manifestRow(f, newFgen, { progress: newUpto, continuity: 'reset', units: unitsOf(out.archive) }, ctx.host, base.chTs(now)));
       else if (newUpto > upto) out.archive.push(manifestRow(f, newFgen, { progress: newUpto, continuity: 'appended', units: unitsOf(out.archive) }, ctx.host, base.chTs(now)));
-      out.turns = [...res.turns, ...tomb.turns];
-      out.calls = [...res.calls, ...tomb.calls];
+      out.turns = [...at.turns, ...tomb.turns];
+      out.calls = [...at.calls, ...tomb.calls];
       const digest = prefixDigest(fd, newUpto);
       out.files = filesRow(f, ctx, {
         attr, fgen: newFgen, upto: newUpto, reparse_from: res.reparse_from, state: res.state, adapter_ver: ad.deriveVersion,
@@ -1238,9 +1253,10 @@ async function upgradeSweep(ctx, sessions, stats) {
       const res = ad.derive({ file: fc, records: recs, state: { ...base.initialState(ad.adapter), fb: (state && state.fb) || {} } });
       const S = sessions.get(skey(harness, H.session_id, H.harness_path, H.incarnation));
       const agentIds = S ? [...S.files.values()].filter((x) => x.role === 'agent').map((x) => x.agent_id) : [];
-      const tomb = await tombstonesFor(ctx, fc, vanishedFrom(await ctx.house.fileKeys(ref, f.agent_id || '', agentIds), res), String(base.revOf(H.fgen, H.upto)));
-      await ctx.house.insert('turns', [...res.turns, ...tomb.turns]);
-      await ctx.house.insert('calls', [...res.calls, ...tomb.calls]);
+      const at = await aboveHouse(ctx, ref, f.agent_id || '', agentIds, String(base.revOf(H.fgen, H.upto)), res);
+      const tomb = await tombstonesFor(ctx, fc, vanishedFrom(await ctx.house.fileKeys(ref, f.agent_id || '', agentIds), res), at.rev);
+      await ctx.house.insert('turns', [...at.turns, ...tomb.turns]);
+      await ctx.house.insert('calls', [...at.calls, ...tomb.calls]);
       const row = { ...H, reparse_from: res.reparse_from, derive_state: JSON.stringify(res.state), adapter_ver: ad.deriveVersion, contrib: JSON.stringify(res.contrib), host: ctx.host };
       delete row.ver;
       await ctx.house.insert('files', [row]);
@@ -1553,6 +1569,11 @@ function storeWhere(ref) {
   return `harness = ${sqlStr(ref.harness)} AND host = ${sqlStr(ref.host)} AND harness_path = ${sqlStr(ref.harness_path)}`
     + ` AND session_id = ${sqlStr(ref.session_id)} AND incarnation = ${sqlStr(ref.incarnation || '')}`;
 }
+/** WHERE clause for one file of a session: a sub-agent's by its id, the main file as every other id. */
+function fileWho(agentId, agentFileIds) {
+  return agentId ? `agent_id = ${sqlStr(agentId)}`
+    : (agentFileIds.length ? `agent_id NOT IN (${agentFileIds.map(sqlStr).join(', ')})` : '1');
+}
 const STORE_TUPLE = '(harness, host, harness_path, session_id, incarnation)';
 const storeTuple = (ref) => `(${[ref.harness, ref.host, ref.harness_path, ref.session_id, ref.incarnation || ''].map(sqlStr).join(', ')})`;
 /** Split a column list at its top-level commas. */
@@ -1774,17 +1795,25 @@ class HttpHouse {
   }
   /** The keys this file's shipped rows hold in the conversation layer (adapter upgrade only). */
   async fileKeys(ref, agentId, agentFileIds) {
-    const who = agentId ? `agent_id = ${sqlStr(agentId)}`
-      : (agentFileIds.length ? `agent_id NOT IN (${agentFileIds.map(sqlStr).join(', ')})` : '1');
     // Removed history (removed = 1) is never a candidate: it is what the source no longer has, not a
     // duplicate of the new derivation, and an upgrade must not erase it.
-    const where = `${storeWhere(ref)} AND ${who} AND is_deleted = 0 AND removed = 0`;
+    const where = `${storeWhere(ref)} AND ${fileWho(agentId, agentFileIds)} AND is_deleted = 0 AND removed = 0`;
     const t = await this.query(`SELECT message_id, part_id, toString(ts) AS ts FROM ${this.t.turns} FINAL
       WHERE ${where} AND origin = 'ship' AND NOT startsWith(message_id, 'legacy:')`);
     const c = await this.query(`SELECT call_id, toString(ts) AS ts FROM ${this.t.calls} FINAL
       WHERE ${where} AND NOT startsWith(call_id, 'legacy:')`);
     return { t: t.map((r) => [r.message_id, r.part_id, r.ts]), c: c.map((r) => [r.call_id, r.ts]) };
   }
+  /** The highest rev this file's shipped rows hold, tombstones and removed versions included (adapter upgrade only). */
+  async maxRev(ref, agentId, agentFileIds) {
+    const where = `${storeWhere(ref)} AND ${fileWho(agentId, agentFileIds)}`;
+    const [t] = await this.query(`SELECT toString(max(rev)) AS rev FROM ${this.t.turns}
+      WHERE ${where} AND origin = 'ship' AND NOT startsWith(message_id, 'legacy:')`);
+    const [c] = await this.query(`SELECT toString(max(rev)) AS rev FROM ${this.t.calls}
+      WHERE ${where} AND NOT startsWith(call_id, 'legacy:')`);
+    const a = BigInt((t && t.rev) || 0); const b = BigInt((c && c.rev) || 0);
+    return String(a > b ? a : b);
+  }
   /**
    * Every `legacy:` row of one session in EVERY version — no FINAL, so tombstoned rows come back
    * too (dedupe-v3 takes the newest per key: a tombstone is a full copy, it still has its text).
@@ -1957,6 +1986,7 @@ class DryHouse {
   async firstCaptured() { return new Map(); }
   async rowsByKey() { return []; }
   async fileKeys() { return { t: [], c: [] }; }
+  async maxRev() { return '0'; }
   async hasRemoved() { return false; }
   async legacyRowsAll() { return { turns: [], calls: [] }; }
   async capturedRows() { return { turns: [], calls: [] }; }

package/memhouse/shipper/log-adapter.js

@@ -14,7 +14,9 @@
 //             A line longer than the read cap is read whole: a read always makes progress
 //   derive    the log's records as sessions-model records, the last line of a key winning:
 //     session   title, cwd, version, created, updated, archived, mode, kind, engine, client; its
-//               `links` (stated on the child) as link records
+//               `links` (stated on the child) as link records; its `children` (a child no message
+//               names, Cursor's sub-agents) as links stated on the origin, one per child: a `child`
+//               record for the same child that says where it was found is the one kept
 //     message   order_key the native `order`, then the line's offset; ts the source's or null; a
 //               `tool` part as a tool_call part and, once it has an output or an error, a
 //               tool_result part; `file` as an attachment, `error` and `notice` as theirs; its
@@ -31,7 +33,7 @@ const path = require('path');
 const slog = require('../../editors/sessions-log');
 
 const ADAPTER = 'memhouse-bridge';
-const ADAPTER_VER = '1.0.1';
+const ADAPTER_VER = '1.0.2';
 const STATE_VER = 1;
 const MAX_READ_BYTES = 64 * 1024 * 1024;
 const UNFLAGGED = { 'gemini-cli': { input_includes_cache: true, output_includes_reasoning: false } };
@@ -70,6 +72,12 @@ function usageRecord(id, anchor, u, harness, model) {
     reasoning: num(u.reasoning), cost: num(u.cost), input_includes_cache: iic, output_includes_reasoning: oir, granularity: 'per_message', model: String(model || '') };
 }
 
+/** A child the session started, as a link stated on the origin: a `child` record, or one of the session record's `children`. */
+function originLink(c, harness) {
+  return { type: 'link', kind: String(c.kind || ''), stated_on: 'origin', origin: {}, child: { harness, session_id: String(c.session_id || '') },
+    at: String(c.at || ''), evidence: c.evidence === 'inferred' ? 'inferred' : 'explicit' };
+}
+
 /** One log line (its offset `at`) as sessions-model records. */
 function recordsOf(o, at, harness) {
   if (!o || typeof o !== 'object') return [];
@@ -80,16 +88,14 @@ function recordsOf(o, at, harness) {
     const links = (Array.isArray(o.links) ? o.links : []).map((l) => ({ type: 'link', kind: String(l.kind || ''), stated_on: 'child', child: {},
       origin: { harness, session_id: String(l.origin || ''), ...(l.origin_store ? { harness_path: String(l.origin_store) } : {}) }, at: String(l.at || ''),
       inherits: l.through ? { through_message_id: String(l.through) } : {}, evidence: l.evidence === 'inferred' ? 'inferred' : 'explicit' }));
-    return [s, ...links];
+    const children = (Array.isArray(o.children) ? o.children : []).map((c) => originLink(c, harness));
+    return [s, ...links, ...children];
   }
   if (o.k === 'usage') {
     if (!o.usage) return [];
     return [usageRecord(o.usage_id != null && o.usage_id !== '' ? `usage:${o.usage_id}` : `usage:@${at}`, {}, o.usage, harness, o.model)];
   }
-  if (o.k === 'child') {
-    return [{ type: 'link', kind: String(o.kind || ''), stated_on: 'origin', origin: {}, child: { harness, session_id: String(o.session_id || '') },
-      at: String(o.at || ''), evidence: o.evidence === 'inferred' ? 'inferred' : 'explicit' }];
-  }
+  if (o.k === 'child') return [originLink(o, harness)];
   if (o.k !== 'message' || o.message_id == null) return [];
   const id = String(o.message_id);
   const parts = [];
@@ -204,6 +210,11 @@ function logAdapter(home, harness, { maxReadBytes = MAX_READ_BYTES } = {}) {
       for (const at of Object.keys(st.lines).sort((a, b) => Number(a) - Number(b))) {
         for (const r of recordsOf(st.lines[at], Number(at), harness)) byKey.set(`${r.type}\u0000${keyOf(r)}`, r);
       }
+      // One link per child: a link with no `at` (a session record's child) gives way to one of the
+      // same kind and child that says where it was found (a `child` record).
+      const placed = new Set();
+      for (const r of byKey.values()) if (r.type === 'link' && r.stated_on === 'origin' && r.at) placed.add(canon([r.kind, r.child]));
+      for (const [k, r] of byKey) if (r.type === 'link' && r.stated_on === 'origin' && !r.at && placed.has(canon([r.kind, r.child]))) byKey.delete(k);
       const changed = []; const live = {}; const emitted = {};
       for (const [k, r] of byKey) {
         const j = JSON.stringify(r);

package/memhouse/shipper/normalise-run.js

@@ -56,10 +56,16 @@ const RETRY_MS = 5 * 60 * 1000;
 const BUDGET_MS = 30 * 1000;
 const keyOf = (s, inc = '') => `${s.harness}\t${s.harness_path}\t${s.session_id}\t${inc}`;
 
-/** Messages in an existing log (0 when there is none). */
+/**
+ * Messages in an existing log (0 when there is none). A log that is there but cannot be read throws:
+ * counted as 0, it would disarm the refusal to rewrite a log of messages into none.
+ */
 function messagesIn(file) {
   let s;
-  try { s = fs.readFileSync(file, 'utf-8'); } catch { return 0; }
+  try { s = fs.readFileSync(file, 'utf-8'); } catch (e) {
+    if (e && (e.code === 'ENOENT' || e.code === 'ENOTDIR')) return 0;
+    throw new Error(`cannot read the existing log (${(e && e.code) || 'error'}): ${path.basename(file)}`);
+  }
   let n = 0;
   for (const l of s.split('\n')) if (l.startsWith('{"k":"message"') || /"k"\s*:\s*"message"/.test(l.slice(0, 200))) n++;
   return n;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "memhouse",
-  "version": "0.20.1",
+  "version": "0.20.2",
   "description": "memhouse — agent conversation memory across 17 editors: parse-on-client shippers, a typed ClickHouse store with one set of rooms per member, and the agentlytics dashboard",
   "main": "bin/memhouse.js",
   "bin": {
@@ -66,7 +66,7 @@
     "email": "security@memhouse.io"
   },
   "dependencies": {
-    "@agent-realm/harness-session-adapters": "1.0.0",
+    "@agent-realm/harness-session-adapters": "1.1.0",
     "@clickhouse/client": "^1.23.1",
     "express": "^4.22.1"
   }