memhouse 0.20.1 → 0.20.3

package/memhouse/shipper/profile-capture.js

@@ -7,13 +7,16 @@
 // THE PASS (each line is a decision; the rest is adapter-capture.js's)
 //   archive     session_units: every unit verbatim (enc, sha256, redacted, orig_sha256), its body in
 //               1 MiB chunks; and a read manifest (unit_key '', enc 'manifest') for every read that
-//               opens a generation or delivers units or gone keys (harness-sessions-sql §4.1). Only a reset
-//               opens a generation past the first (sessions §3); a snapshot replaces its one unit.
+//               opens a generation or delivers units or gone keys (harness-sessions-sql §4.1). A read
+//               opens a generation when it is the source's first, a reset or a snapshot (§4.1): a
+//               changed snapshot's one unit goes into a generation of its own.
 //               An archive write carries one captured_at, set by the capture and later than the
 //               session's last one (kept in the local state as archivedAt; on a refold, the
 //               archive's latest), so a capped read's steps stay apart and a clock that steps back
 //               between passes never reorders them; never left to the server's default, which a
-//               parallel parse of a big insert may evaluate per chunk: it is the replay's pass id. A write that fits one batch is one statement, manifests last;
+//               parallel parse of a big insert may evaluate per chunk: it is the replay's pass id. The one
+//               exception is the re-run of a pass that stopped before its cursor write: it writes under
+//               that pass's captured_at, so the two replay as one pass (header: crash). A write that fits one batch is one statement, manifests last;
 //               a bigger one writes its units first (~8 MiB a statement), then all its manifests in
 //               one statement however big: a manifest never lands before its units (a stop between
 //               the two leaves units no manifest names, which the fold ignores and the re-read
@@ -35,9 +38,13 @@
 //               statement holds a pass's manifests, header: archive). Each pass reads the unit
 //               versions it needs and does not hold, at most REPLAY_MAX_BYTES (32 MiB) and 1,000
 //               units a response, and keeps only what the fold holds, so a unit edited at every pass
-//               is not read or held in all its versions at once. A pass
-//               that stopped before its cursor write cannot be told from one that did not: its reads
-//               replay as a pass of their own
+//               is not read or held in all its versions at once. A pass that stopped before its
+//               cursor write cannot be told from one that did not by the archive alone; its re-run,
+//               which sees the archive ahead of the cursor, writes under the stopped pass's
+//               captured_at (header: crash), so the stopped read and the re-run replay as one pass.
+//               Each source's input to a pass's derive is that pass's net change to its fold: with
+//               one manifest a source, as every other pass has, exactly that manifest's units and gone
+//               keys; across a stopped read and its re-run, a unit the re-run took back is no change
 //   re-derive   an upgrade's refold (harness-sessions-sql §8: a new adapter_ver replayed from session_units
 //               alone) replays the archive the same way, under the new version, over the house's rows:
 //               every row it makes, current or a removed version, is written above the house's revs at
@@ -84,13 +91,22 @@
 //   crash       the local state is dropped before the first write and saved after the last, so a
 //               pass that stops anywhere leaves none, and the next one refolds from the house. A
 //               refold reads the archive's manifests first: when a stopped pass left them ahead of the
-//               cursor, its generation, progress and delta fold continue from the archive (a reset
-//               opens the generation past the archive's), so no two manifests of a generation share a
-//               progress and the fold is what the archive says. A source re-read from scratch that
-//               delivers exactly what the generation the stopped pass opened holds (its own read: no
-//               cursor yet, or a reset; never a source_gone generation) stays in it and archives
-//               nothing: only its position moves. So does a stream re-read from the cursor that
-//               delivers only what the stopped pass appended to the cursor's generation
+//               cursor, its generation, progress and delta fold continue from the archive (a reset or
+//               a snapshot opens the generation past the archive's), so no two manifests of a
+//               generation share a progress and the fold is what the archive says. A source re-read
+//               from scratch that delivers exactly what the generation the stopped pass opened holds
+//               (its own read: no cursor yet, a reset or a snapshot; never a source_gone generation)
+//               stays in it and archives nothing: only its position moves. So does a stream re-read
+//               from the cursor that delivers only what the stopped pass appended to the cursor's
+//               generation, and a snapshot that delivers what a stopped pass archived in the cursor's
+//               own generation (an archive from before a snapshot opened a generation). The re-run
+//               writes its archive under the stopped pass's captured_at (archive), so a replay folds
+//               the stopped read and the re-run as one pass: a unit the stopped read delivered that the
+//               re-run no longer sees is never derived, as in a clean run of what was committed
+//               (harness-sessions-sql SPEC-QUESTIONS q20). A stop past the rows has already written the
+//               stopped read's records: the re-run keeps any it no longer sees as removed versions,
+//               which a replay does not make (nor does the reference engine's), and an upgrade's
+//               re-derive tombstones them
 //   edges       after the rows, before the cursor (harness-sessions-sql §5 step 4): the engine edges the session
 //               is the fork or the origin of (profile-edges.js), so the rollup bills a fork's copied
 //               usage once. A fork whose edge moved and is not this session gets its sessions row
@@ -284,6 +300,9 @@ async function captureSession(ctx, ad, d, store, listed) {
     ...archOnly.filter((sk) => ours.has(sk) && !A.get(sk).gone).map((sk) => ({ source_key: sk, mode: '', role: 'main', omitted: true })),
   ];
   const confirmed = new Set();   // omitted sources the adapter answered are not there
+  // The captured_at of a pass that stopped before its cursor write (an archive ahead of the cursor): this
+  // re-run writes under it (header: crash). null when no source is ahead.
+  let stoppedAt = null;
   for (const src of [...listed.sources, ...omitted]) {
     const sk = src.source_key;
     const h = H.get(sk);
@@ -306,6 +325,7 @@ async function captureSession(ctx, ad, d, store, listed) {
         // Only a generation the stopped pass opened (newer than the cursor's) and that holds a read
         // (not a source_gone one) can be the one a re-read from scratch repeats.
         resumed = a.gen > hGen && !a.gone ? a.fold : null;
+        if (a.at && (stoppedAt === null || String(a.at) > stoppedAt)) stoppedAt = String(a.at);
         appendedTo = a.gen === hGen && !a.gone ? { fold: a.fold, progress: String(a.progress) } : null;
       } else prior = h && !goneOf(h) ? { gen: Number(h.generation), progress: String(h.progress), position: parse(h.position, null), known: parse(h.known, {}) } : null;
     }
@@ -339,7 +359,15 @@ async function captureSession(ctx, ad, d, store, listed) {
         units = r.units; gone = []; knownNext = {}; full = true; opens = true;
         break;
       case 'snapshot':
-        units = r.units; knownNext = {};
+        // A changed snapshot opens the next generation (harness-sessions-sql §4.1), unless it is the
+        // stopped pass's own read again (header: crash): the unit a generation the stopped pass opened
+        // holds, or one it archived in the cursor's own generation (before a snapshot opened one).
+        if ((resumed && sameUnits(r.units, resumed)) || (appendedTo && sameUnits(r.units, appendedTo.fold))) {
+          reread = r.units; knownNext = Object.fromEntries(resumed || appendedTo.fold); full = true;
+          break;
+        }
+        gen = prior ? gen + 1 : gen;
+        units = r.units; knownNext = {}; opens = true;
         break;
       case 'appended': case 'delta':
         units = r.units;
@@ -368,7 +396,7 @@ async function captureSession(ctx, ad, d, store, listed) {
     }
     sources[sk] = { gen, progress, position: r.position ?? null, known: src.mode === 'stream' ? {} : knownNext, mode: src.mode || '', role: src.role || 'main' };
     if (changed) fresh.push({ source_key: sk, generation: gen, complete: false, units, gone });
-    if (!prior || r.continuity === 'reset') {
+    if (!prior || r.continuity === 'reset' || r.continuity === 'snapshot') {
       const byKey = new Map();
       for (const u of reread || units) { const k = String(u.unit_key); byKey.delete(k); byKey.set(k, { ...u, unit_key: k }); }
       opened.set(sk, { gen, units: [...byKey.values()] });
@@ -501,7 +529,9 @@ async function captureSession(ctx, ad, d, store, listed) {
       last = known.length || local == null ? Math.max(0, ...known)
         : (typeof ctx.house.archivedAt === 'function' ? await ctx.house.archivedAt(ref) : 0);
     }
-    archivedAt = Math.max(Number(ctx.now()), last + 1);
+    // The re-run of a stopped pass writes under that pass's captured_at (header: crash); every other
+    // write is later than the session's last one.
+    archivedAt = stoppedAt !== null ? Date.parse(`${stoppedAt.replace(' ', 'T')}Z`) : Math.max(Number(ctx.now()), last + 1);
     for (const r of archive) r.captured_at = base.chTs(archivedAt);
     const units = archive.filter((r) => r.enc !== 'manifest');
     const manifests = archive.filter((r) => r.enc === 'manifest');
@@ -792,8 +822,9 @@ async function replayArchive(ctx, ad, d, key, ref, { adapter, adapterVer, order,
   const byOrder = (a, b) => (rank.has(a) ? rank.get(a) : order.length) - (rank.has(b) ? rank.get(b) : order.length) || (a < b ? -1 : a > b ? 1 : 0);
   const sources = new Map();
   // The units a pass needs and does not hold yet are read for that pass (paged by pass), and only
-  // what the fold still holds is kept after it: a snapshot or delta source keeps one generation and
-  // adds a version of a unit at every edit, so holding (or reading at once) every version a
+  // what the fold still holds is kept after it: a delta source keeps one generation and adds a
+  // version of a unit at every edit (so did a snapshot source, in an archive from before a snapshot
+  // opened a generation), so holding (or reading at once) every version a
   // generation ever named grows with the session's history. A read is bounded by bytes, at most
   // `maxBytes` of the units' archived text and 1,000 units a response (one unit larger than that
   // alone); the sizes of every unit a generation's manifests name are read once, with that
@@ -863,12 +894,24 @@ async function replayArchive(ctx, ad, d, key, ref, { adapter, adapterVer, order,
   let view = new Map();
   for (const pass of passes) {
     let full = state === null;
-    const fresh = [];
+    // Each source's input is the pass's net change to its fold (header: replay): with one manifest a
+    // source, that manifest's units and gone keys; across a stopped read and its re-run (one captured_at,
+    // header: crash), a unit the re-run took back is no change, and is never derived.
+    const before = new Map();
     for (const m of pass) {
+      if (!before.has(m.source_key)) { const s = sources.get(m.source_key); before.set(m.source_key, s ? { generation: s.generation, gone: s.gone, units: new Map(s.units) } : null); }
       foldManifest(sources, m);
       if (m.source_gone ? sources.get(m.source_key).gone !== 'kept' : m.continuity === 'reset') full = true;
-      else if (!m.source_gone && ((m.units || []).length || (m.gone || []).length)) {
-        fresh.push({ source_key: m.source_key, generation: m.generation, complete: false, units: await unitsAt(m.source_key, m.generation, m.units || []), gone: m.gone || [] });
+    }
+    const fresh = [];
+    if (!full) {
+      for (const [sk, b] of before) {
+        const s = sources.get(sk);
+        if (s.gone) continue;   // a kept vanish: its units stay as they were
+        const same = !!b && !b.gone && b.generation === s.generation;
+        const units = [...s.units].filter(([uk, sha]) => !same || b.units.get(uk) !== sha);
+        const gone = same ? [...b.units.keys()].filter((uk) => !s.units.has(uk)) : [];
+        if (units.length || gone.length) fresh.push({ source_key: sk, generation: s.generation, complete: false, units: await unitsAt(sk, s.generation, units), gone });
       }
     }
     fresh.sort((a, b) => byOrder(a.source_key, b.source_key));

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "memhouse",
-  "version": "0.20.1",
+  "version": "0.20.3",
   "description": "memhouse — agent conversation memory across 17 editors: parse-on-client shippers, a typed ClickHouse store with one set of rooms per member, and the agentlytics dashboard",
   "main": "bin/memhouse.js",
   "bin": {
@@ -66,7 +66,7 @@
     "email": "security@memhouse.io"
   },
   "dependencies": {
-    "@agent-realm/harness-session-adapters": "1.0.0",
+    "@agent-realm/harness-session-adapters": "1.1.0",
     "@clickhouse/client": "^1.23.1",
     "express": "^4.22.1"
   }