memhouse 0.20.1 → 0.20.3

package/memhouse/house/schema-profile.js

@@ -8,7 +8,10 @@
 //                                       option, nothing in <db>_archive granted to anyone else
 //
 // The DDL lives in schema-profile.sql.tpl, rendered with {db} and {m}; every statement is
-// CREATE … IF NOT EXISTS. Transport-free like schema-v3.js: every entry point takes `q`.
+// CREATE … IF NOT EXISTS. The reading views (harness-sessions-sql v1.1 §6) live apart, in
+// schema-profile-views.sql.tpl, as CREATE OR REPLACE VIEW: they are not tables, so they stay out
+// of templateColumns, the column heal and the key check. Transport-free like schema-v3.js: every
+// entry point takes `q`.
 
 const fs = require('fs');
 const path = require('path');
@@ -16,6 +19,7 @@ const { assertUsableName, legacyTextIndexDialect, isTextIndexGrammarRefusal, tex
 const { stripComments, qFromClient, DDL_SETTINGS } = require('./schema-v3');
 
 const TEMPLATE = path.join(__dirname, 'schema-profile.sql.tpl');
+const VIEWS_TEMPLATE = path.join(__dirname, 'schema-profile-views.sql.tpl');
 
 // The columns retyped in harness-sessions-sql v1, which ensureProfileSchema brings an older table to:
 // adapter_ver UInt32 -> LowCardinality(String), child_stored Bool -> Nullable(Bool), orig_sha256
@@ -47,19 +51,43 @@ const SERVER_COLUMNS = ['writer', 'ingested_at', 'search_ngram', 'search_word',
 
 function archiveDb(db) { return `${db}_archive`; }
 
-/** Every fully-qualified name of one member's profile tables. */
+/** The reading views, by their name after `<m>_` (schema-profile-views.sql.tpl). */
+const VIEW_TYPES = ['session_turns_view'];
+
+/** Every fully-qualified name of one member's profile tables and views. */
 function profileNames(db, member) {
   assertUsableName(db, 'house');
   assertUsableName(member, 'member');
   const out = { db, archive: archiveDb(db), member };
   for (const t of CONVERSATION_TYPES) out[t] = `${db}.${member}_${t}`;
   for (const t of CAPTURE_TYPES) out[t] = `${archiveDb(db)}.${member}_${t}`;
+  for (const t of VIEW_TYPES) out[t] = `${db}.${member}_${t}`;
   out.meta = `${db}.${member}_meta`;
   out.events = `${db}.${member}_events`;
   return out;
 }
 
 function readTemplate() { return fs.readFileSync(TEMPLATE, 'utf-8'); }
+function readViewsTemplate() { return fs.readFileSync(VIEWS_TEMPLATE, 'utf-8'); }
+
+/** The view statements for one member, in template order: [{ type, fq, sql }]. Each reads the member's own house. */
+function renderViews(tpl, { db, member }) {
+  assertUsableName(db, 'house');
+  assertUsableName(member, 'member');
+  const body = stripComments(tpl).replaceAll('{db}', db).replaceAll('{m}', member);
+  const out = [];
+  for (const raw of body.split(';')) {
+    const sql = raw.trim();
+    if (!sql) continue;
+    const m = /^CREATE OR REPLACE VIEW ([A-Za-z0-9_]+)\.([A-Za-z0-9_]+) SQL SECURITY INVOKER AS\b/.exec(sql);
+    if (!m) throw new Error(`schema-profile views template: unexpected statement: ${sql.slice(0, 80)}`);
+    if (m[1] !== db) throw new Error(`schema-profile views template: '${m[1]}.${m[2]}' is outside the house`);
+    const type = m[2].startsWith(`${member}_`) ? m[2].slice(member.length + 1) : m[2];
+    if (!VIEW_TYPES.includes(type)) throw new Error(`schema-profile views template declares an unknown view '${m[2]}'`);
+    out.push({ type, fq: `${m[1]}.${m[2]}`, sql });
+  }
+  return out;
+}
 
 /** The CREATE statements for one member, in template order: [{ type, database, table, fq, sql }]. */
 function renderStatements(tpl, { db, member }) {
@@ -156,12 +184,12 @@ function msg(e) { return e && e.message ? e.message : String(e); }
  * Create one member's profile tables, idempotently, then verify them against PROFILE_KEYS.
  * As ensureV3Schema: a privilege denial is counted and skipped, a lost create race is success.
  */
-async function ensureProfileSchema(q, { db, member, createDatabases = false, grantCapture = false, tpl = null } = {}, ui = null) {
+async function ensureProfileSchema(q, { db, member, createDatabases = false, grantCapture = false, tpl = null, viewsTpl = null } = {}, ui = null) {
   const n = profileNames(db, member);
   const say = (m) => ui && ui.ok && ui.ok(m);
   const version = String(((await q.rows('SELECT version() AS v', { final: 0 }))[0] || {}).v || '');
   const dialect = textIndexDialectFor(version);
-  const out = { version, dialect, created: [], denied: [], unhealed: [], grants: [], grant_failures: [], tables: {}, ok: false };
+  const out = { version, dialect, created: [], denied: [], unhealed: [], grants: [], grant_failures: [], views: [], view_failures: [], tables: {}, ok: false };
   if (createDatabases) {
     for (const d of [n.db, n.archive]) {
       try { await q.sql(`CREATE DATABASE IF NOT EXISTS ${d}`, DDL_SETTINGS); } catch (e) {
@@ -257,6 +285,18 @@ async function ensureProfileSchema(q, { db, member, createDatabases = false, gra
     }
   }
   if (out.retyped.length) say(`profile tables for ${member}: retyped ${out.retyped.join(', ')}`);
+  // The reading views, over session_parts as healed above. A view is a reader's convenience, not
+  // something the capture writes: a refusal goes to `denied`, any other failure to view_failures,
+  // and neither makes the tables not ok. Not over a session_parts that is missing or lacks a column.
+  if (out.created.includes(n.session_parts) && !lacking.has('session_parts')) {
+    for (const v of renderViews(viewsTpl || readViewsTemplate(), { db, member })) {
+      try { await q.sql(v.sql, DDL_SETTINGS); out.views.push(v.fq); } catch (e) {
+        const m = msg(e);
+        if (isDenied(m)) out.denied.push(`CREATE VIEW ${v.fq}`); else out.view_failures.push(`${v.fq}: ${m.split('\n')[0]}`);
+      }
+    }
+    if (out.views.length) say(`reading views for ${member}: ${out.views.join(', ')}`);
+  }
   if (grantCapture) {
     for (const g of captureGrants(db, member)) {
       try { await q.sql(g, DDL_SETTINGS); out.grants.push(g); } catch (e) {
@@ -284,7 +324,7 @@ async function ensureProfileSchema(q, { db, member, createDatabases = false, gra
 }
 
 module.exports = {
-  TEMPLATE, CONVERSATION_TYPES, CAPTURE_TYPES, PROFILE_TYPES, PROFILE_KEYS, SERVER_COLUMNS,
-  archiveDb, profileNames, readTemplate, renderStatements, templateColumns, templateKeys, rowColumns, insertColumns,
+  TEMPLATE, VIEWS_TEMPLATE, CONVERSATION_TYPES, CAPTURE_TYPES, PROFILE_TYPES, VIEW_TYPES, PROFILE_KEYS, SERVER_COLUMNS,
+  archiveDb, profileNames, readTemplate, readViewsTemplate, renderStatements, renderViews, templateColumns, templateKeys, rowColumns, insertColumns,
   keyProblem, captureGrants, ensureProfileSchema, qFromClient,
 };

package/memhouse/house/schema-profile.sql.tpl

@@ -1,5 +1,7 @@
 -- memhouse's house tables as harness-sessions-sql's ClickHouse profile (step B1, tables and keys). The
--- tables, columns, keys and engines are harness-sessions-sql's schema/clickhouse.sql (v1.0.0);
+-- tables, columns, keys and engines are harness-sessions-sql's schema/clickhouse.sql (v1.1: v1's,
+-- with the native and units columns of §4.2 and §4.3; its reading view is
+-- schema-profile-views.sql.tpl);
 -- memhouse adds, beside them:
 --   * the member prefix {m}_ and the archive database {db}_archive (spec §3: physical names);
 --   * on session_parts: first_seen (a first-seen time, never written into ts), folder, project,
@@ -48,6 +50,8 @@ CREATE TABLE IF NOT EXISTS {db}.{m}_session_parts (
     message_aliases      Array(Tuple(id String, kind LowCardinality(String))),
     cmp_sha              FixedString(64),
     message_content_sha  FixedString(64),
+    message_native       String DEFAULT '',       -- v1.1: the message's native fields (sessions-model §2.7), canonical JSON; '' when none
+    message_units        Array(Tuple(source_key String, generation UInt32, unit_key String)),  -- v1.1: the units the message came from
     kind                 LowCardinality(String),
     text                 String DEFAULT '' CODEC(ZSTD(3)),
     call_id              String DEFAULT '',
@@ -78,6 +82,7 @@ CREATE TABLE IF NOT EXISTS {db}.{m}_session_parts (
     native_type          LowCardinality(String) DEFAULT '',
     part_source          LowCardinality(String) DEFAULT '',
     part_state           LowCardinality(String) DEFAULT '',
+    part_native          String DEFAULT '',       -- v1.1: the part's own native fields, canonical JSON; '' when none
     content_sha          FixedString(64),
     adapter              LowCardinality(String),
     adapter_ver          LowCardinality(String),
@@ -131,6 +136,8 @@ CREATE TABLE IF NOT EXISTS {db}.{m}_sessions (
     aliases          Array(Tuple(id String, kind LowCardinality(String))),
     fingerprint      Array(FixedString(64)),
     setup            Nullable(String),
+    native           String DEFAULT '',                -- v1.1: the session's native fields (sessions-model §2.7), canonical JSON; '' when none
+    units            Array(Tuple(source_key String, generation UInt32, unit_key String)),  -- v1.1: the units the session record came from
     content_sha      FixedString(64),
     adapter          LowCardinality(String),
     adapter_ver      LowCardinality(String),

package/memhouse/share.js

@@ -195,9 +195,11 @@ function memberCollisions(member, tables) {
 }
 
 // The names memhouse gives a member's tables in the house, as the part after `<m>_`: its tables,
-// the copies migrate keeps of a 0.18 room while it moves one (`<room>__migrating`) and after
-// (`<room>_pre_epoch`), and the scratch table share's probe makes and drops (`_mh_probe_<pid>`).
-const MEMBER_TABLES = [...new Set([...V3_TABLES, ...LEGACY_ROOMS, ...DEV_ROOMS, 'meta', 'events', 'session_stats', 'session_model_stats', 'session_tool_stats',
+// its reading views (schema-profile.js VIEW_TYPES: session_turns_view, named so it does not end in
+// _turns, which memberCollisions reads as a 0.18 member's room), the copies migrate keeps of a
+// 0.18 room while it moves one (`<room>__migrating`) and after (`<room>_pre_epoch`), and the
+// scratch table share's probe makes and drops (`_mh_probe_<pid>`).
+const MEMBER_TABLES = [...new Set([...V3_TABLES, ...LEGACY_ROOMS, ...DEV_ROOMS, 'meta', 'events', 'session_stats', 'session_model_stats', 'session_tool_stats', 'session_turns_view',
   ...LEGACY_ROOMS.flatMap((r) => [`${r}_pre_epoch`, `${r}__migrating`])])];
 const isMemberTable = (rest) => MEMBER_TABLES.includes(rest) || /^_mh_probe_\d+$/.test(rest);
 const startsMemberTable = (p) => [...MEMBER_TABLES, '_mh_probe_1'].some((n) => n.startsWith(`${p}_`));

package/memhouse/shipper/capture.js

@@ -445,6 +445,18 @@ async function tombstonesFor(ctx, fc, vanished, rev, { removed = false } = {}) {
   }
   return out;
 }
+/**
+ * An adapter upgrade's rows, at a rev above every row the house holds for the file. An unchanged
+ * file re-derives at revOf(fgen, upto), the rev its last pass wrote, and at an equal rev the new
+ * version's row would win by insertion order alone. The raise is a rev or two; the next append's
+ * revOf moves on by the length of what it read.
+ */
+async function aboveHouse(ctx, ref, agentId, agentFileIds, rev, res) {
+  const floor = BigInt(await ctx.house.maxRev(ref, agentId, agentFileIds)) + 1n;
+  if (BigInt(rev) >= floor) return { rev, turns: res.turns, calls: res.calls };
+  const up = String(floor);
+  return { rev: up, turns: res.turns.map((r) => ({ ...r, rev: up })), calls: res.calls.map((r) => ({ ...r, rev: up })) };
+}
 /** Keys ({ t: [[message_id, part_id, ts]], c: [[call_id, ts]] }) the new derivation no longer produces. */
 function vanishedFrom(old, res) {
   const have = { t: new Set(res.turns.map(base.turnKey)), c: new Set(res.calls.map(base.callKey)) };
@@ -571,7 +583,10 @@ async function captureFile(ctx, f, H, C, budget) {
       const fbIds = (fb) => Object.fromEntries(Object.entries(fb || {}).filter(([k]) => k.startsWith('id:')));
       const freshState = { ...base.initialState(ad.adapter), fb: rewrite ? fbIds(state && state.fb) : ((state && state.fb) || {}) };
       const res = ad.derive({ file: fc, records, state: freshState });
-      const rev = String(base.revOf(newFgen, newUpto));
+      const at = reAdapter
+        ? await aboveHouse(ctx, refOf(f, ctx.host), f.agent_id || '', ctx.agentFileIds(f), String(base.revOf(newFgen, newUpto)), res)
+        : { rev: String(base.revOf(newFgen, newUpto)), turns: res.turns, calls: res.calls };
+      const rev = at.rev;
       const tomb = rewrite && reAdapter ? await retireBoth(ctx, f, H, state, fc, oldKeys, res, rev)
         : await tombstonesFor(ctx, fc, vanishedFrom(oldKeys, res), rev, { removed: !!rewrite });
       // A new generation is archived whole; an upgrade archives only what lies past `upto`.
@@ -580,8 +595,8 @@ async function captureFile(ctx, f, H, C, budget) {
       // an upgrade that read past the cursor appended to the one it had.
       if (rewrite) out.archive.push(manifestRow(f, newFgen, { progress: newUpto, continuity: 'reset', units: unitsOf(out.archive) }, ctx.host, base.chTs(now)));
       else if (newUpto > upto) out.archive.push(manifestRow(f, newFgen, { progress: newUpto, continuity: 'appended', units: unitsOf(out.archive) }, ctx.host, base.chTs(now)));
-      out.turns = [...res.turns, ...tomb.turns];
-      out.calls = [...res.calls, ...tomb.calls];
+      out.turns = [...at.turns, ...tomb.turns];
+      out.calls = [...at.calls, ...tomb.calls];
       const digest = prefixDigest(fd, newUpto);
       out.files = filesRow(f, ctx, {
         attr, fgen: newFgen, upto: newUpto, reparse_from: res.reparse_from, state: res.state, adapter_ver: ad.deriveVersion,
@@ -1238,9 +1253,10 @@ async function upgradeSweep(ctx, sessions, stats) {
       const res = ad.derive({ file: fc, records: recs, state: { ...base.initialState(ad.adapter), fb: (state && state.fb) || {} } });
       const S = sessions.get(skey(harness, H.session_id, H.harness_path, H.incarnation));
       const agentIds = S ? [...S.files.values()].filter((x) => x.role === 'agent').map((x) => x.agent_id) : [];
-      const tomb = await tombstonesFor(ctx, fc, vanishedFrom(await ctx.house.fileKeys(ref, f.agent_id || '', agentIds), res), String(base.revOf(H.fgen, H.upto)));
-      await ctx.house.insert('turns', [...res.turns, ...tomb.turns]);
-      await ctx.house.insert('calls', [...res.calls, ...tomb.calls]);
+      const at = await aboveHouse(ctx, ref, f.agent_id || '', agentIds, String(base.revOf(H.fgen, H.upto)), res);
+      const tomb = await tombstonesFor(ctx, fc, vanishedFrom(await ctx.house.fileKeys(ref, f.agent_id || '', agentIds), res), at.rev);
+      await ctx.house.insert('turns', [...at.turns, ...tomb.turns]);
+      await ctx.house.insert('calls', [...at.calls, ...tomb.calls]);
       const row = { ...H, reparse_from: res.reparse_from, derive_state: JSON.stringify(res.state), adapter_ver: ad.deriveVersion, contrib: JSON.stringify(res.contrib), host: ctx.host };
       delete row.ver;
       await ctx.house.insert('files', [row]);
@@ -1553,6 +1569,11 @@ function storeWhere(ref) {
   return `harness = ${sqlStr(ref.harness)} AND host = ${sqlStr(ref.host)} AND harness_path = ${sqlStr(ref.harness_path)}`
     + ` AND session_id = ${sqlStr(ref.session_id)} AND incarnation = ${sqlStr(ref.incarnation || '')}`;
 }
+/** WHERE clause for one file of a session: a sub-agent's by its id, the main file as every other id. */
+function fileWho(agentId, agentFileIds) {
+  return agentId ? `agent_id = ${sqlStr(agentId)}`
+    : (agentFileIds.length ? `agent_id NOT IN (${agentFileIds.map(sqlStr).join(', ')})` : '1');
+}
 const STORE_TUPLE = '(harness, host, harness_path, session_id, incarnation)';
 const storeTuple = (ref) => `(${[ref.harness, ref.host, ref.harness_path, ref.session_id, ref.incarnation || ''].map(sqlStr).join(', ')})`;
 /** Split a column list at its top-level commas. */
@@ -1774,17 +1795,25 @@ class HttpHouse {
   }
   /** The keys this file's shipped rows hold in the conversation layer (adapter upgrade only). */
   async fileKeys(ref, agentId, agentFileIds) {
-    const who = agentId ? `agent_id = ${sqlStr(agentId)}`
-      : (agentFileIds.length ? `agent_id NOT IN (${agentFileIds.map(sqlStr).join(', ')})` : '1');
     // Removed history (removed = 1) is never a candidate: it is what the source no longer has, not a
     // duplicate of the new derivation, and an upgrade must not erase it.
-    const where = `${storeWhere(ref)} AND ${who} AND is_deleted = 0 AND removed = 0`;
+    const where = `${storeWhere(ref)} AND ${fileWho(agentId, agentFileIds)} AND is_deleted = 0 AND removed = 0`;
     const t = await this.query(`SELECT message_id, part_id, toString(ts) AS ts FROM ${this.t.turns} FINAL
       WHERE ${where} AND origin = 'ship' AND NOT startsWith(message_id, 'legacy:')`);
     const c = await this.query(`SELECT call_id, toString(ts) AS ts FROM ${this.t.calls} FINAL
       WHERE ${where} AND NOT startsWith(call_id, 'legacy:')`);
     return { t: t.map((r) => [r.message_id, r.part_id, r.ts]), c: c.map((r) => [r.call_id, r.ts]) };
   }
+  /** The highest rev this file's shipped rows hold, tombstones and removed versions included (adapter upgrade only). */
+  async maxRev(ref, agentId, agentFileIds) {
+    const where = `${storeWhere(ref)} AND ${fileWho(agentId, agentFileIds)}`;
+    const [t] = await this.query(`SELECT toString(max(rev)) AS rev FROM ${this.t.turns}
+      WHERE ${where} AND origin = 'ship' AND NOT startsWith(message_id, 'legacy:')`);
+    const [c] = await this.query(`SELECT toString(max(rev)) AS rev FROM ${this.t.calls}
+      WHERE ${where} AND NOT startsWith(call_id, 'legacy:')`);
+    const a = BigInt((t && t.rev) || 0); const b = BigInt((c && c.rev) || 0);
+    return String(a > b ? a : b);
+  }
   /**
    * Every `legacy:` row of one session in EVERY version — no FINAL, so tombstoned rows come back
    * too (dedupe-v3 takes the newest per key: a tombstone is a full copy, it still has its text).
@@ -1957,6 +1986,7 @@ class DryHouse {
   async firstCaptured() { return new Map(); }
   async rowsByKey() { return []; }
   async fileKeys() { return { t: [], c: [] }; }
+  async maxRev() { return '0'; }
   async hasRemoved() { return false; }
   async legacyRowsAll() { return { turns: [], calls: [] }; }
   async capturedRows() { return { turns: [], calls: [] }; }
@@ -2006,6 +2036,25 @@ const refusesVanish = (x) => !!(x.vanishRefused || x.stillThere || x.candidateEr
 const storesN = (n) => `${n} store${n === 1 ? '' : 's'}`;
 /** One shell word: as is when it is plain, else single-quoted (a store path may hold spaces). */
 const shellWord = (w) => (/^[A-Za-z0-9_./:@%+=,-]+$/.test(w) ? w : `'${String(w).replace(/'/g, "'\\''")}'`);
+/**
+ * The pass's normaliser errors (normalise-run.js), one line per editor and code, and the advice for
+ * a denial (normalise/fsx.js hintOf): `status` and `doctor` read them back from the log (passlog.js).
+ */
+function normaliseLines(s) {
+  const by = new Map();
+  for (const e of (s.normalise && s.normalise.errors) || []) {
+    const k = `${e.harness}\u0000${e.code || ''}`;
+    if (!by.has(k)) by.set(k, { ...e, n: 0 });
+    by.get(k).n++;
+  }
+  const out = [];
+  for (const e of by.values()) {
+    out.push(`[memhouse] normalise error: ${e.harness} (${e.code || 'error'}): ${e.error}${e.n > 1 ? ` (and ${e.n - 1} more)` : ''}`);
+    const hint = require('../../editors/normalise/fsx').hintOf(e);
+    if (hint) out.push(`[memhouse] normalise hint: ${e.harness} (${e.code || 'error'}): ${hint}`);
+  }
+  return out;
+}
 function vanishLines(s) {
   const out = [];
   for (const x of s.adapters || []) {
@@ -2531,6 +2580,7 @@ async function main(argv = process.argv.slice(2), opts = {}) {
         s = await runPass(ctx);
         console.log(`${summary(s)} in ${((Date.now() - t0) / 1000).toFixed(1)}s`);
         for (const line of vanishLines(s)) console.log(line);
+        for (const line of normaliseLines(s)) console.log(line);
         // Without the package a profile pass captures no Claude Code or Codex session at all; that
         // must not read as a quiet pass with nothing new.
         const ac = require('./adapter-capture');
@@ -2561,7 +2611,7 @@ module.exports = {
   applyAdapters, adapterHarnesses, adapterPass, routeAdapterSessions, ADAPTER_MAX_SESSION_BYTES,
   parseSourceRoots, classifyRoot, discover, Cache, cachePath, captureFile, runPass, buildSession, batches,
   archiveRows, recordsFromArchive, rebuildFromArchive, replayArchive, manifestRow, fkey, splitLines, prefixDigest, extendDigest, digestOf, digestBuffer, HttpHouse, DryHouse,
-  TABLE_COLUMNS, dedupeSession, dedupeSweep, codexRepeatsOf, profileRepeatsOf, profileAdapterPass, profileRouting, main, defaultAdapters, applyBridge, combineRoots, storeRoots, adapterOf, summary, vanishLines,
+  TABLE_COLUMNS, dedupeSession, dedupeSweep, codexRepeatsOf, profileRepeatsOf, profileAdapterPass, profileRouting, main, defaultAdapters, applyBridge, combineRoots, storeRoots, adapterOf, summary, vanishLines, normaliseLines,
   passLock, PASS_LOCK_STALE_MS, PASS_LOCK_REFRESH_MS, duplicatesRefusal, reloadCache,
   BLOCK, ANCHOR, CHUNK, BATCH_ROWS, BATCH_BYTES, FRAGMENT_MS,
 };

package/memhouse/shipper/log-adapter.js

@@ -14,7 +14,9 @@
 //             A line longer than the read cap is read whole: a read always makes progress
 //   derive    the log's records as sessions-model records, the last line of a key winning:
 //     session   title, cwd, version, created, updated, archived, mode, kind, engine, client; its
-//               `links` (stated on the child) as link records
+//               `links` (stated on the child) as link records; its `children` (a child no message
+//               names, Cursor's sub-agents) as links stated on the origin, one per child: a `child`
+//               record for the same child that says where it was found is the one kept
 //     message   order_key the native `order`, then the line's offset; ts the source's or null; a
 //               `tool` part as a tool_call part and, once it has an output or an error, a
 //               tool_result part; `file` as an attachment, `error` and `notice` as theirs; its
@@ -31,7 +33,7 @@ const path = require('path');
 const slog = require('../../editors/sessions-log');
 
 const ADAPTER = 'memhouse-bridge';
-const ADAPTER_VER = '1.0.1';
+const ADAPTER_VER = '1.0.2';
 const STATE_VER = 1;
 const MAX_READ_BYTES = 64 * 1024 * 1024;
 const UNFLAGGED = { 'gemini-cli': { input_includes_cache: true, output_includes_reasoning: false } };
@@ -70,6 +72,12 @@ function usageRecord(id, anchor, u, harness, model) {
     reasoning: num(u.reasoning), cost: num(u.cost), input_includes_cache: iic, output_includes_reasoning: oir, granularity: 'per_message', model: String(model || '') };
 }
 
+/** A child the session started, as a link stated on the origin: a `child` record, or one of the session record's `children`. */
+function originLink(c, harness) {
+  return { type: 'link', kind: String(c.kind || ''), stated_on: 'origin', origin: {}, child: { harness, session_id: String(c.session_id || '') },
+    at: String(c.at || ''), evidence: c.evidence === 'inferred' ? 'inferred' : 'explicit' };
+}
+
 /** One log line (its offset `at`) as sessions-model records. */
 function recordsOf(o, at, harness) {
   if (!o || typeof o !== 'object') return [];
@@ -80,16 +88,14 @@ function recordsOf(o, at, harness) {
     const links = (Array.isArray(o.links) ? o.links : []).map((l) => ({ type: 'link', kind: String(l.kind || ''), stated_on: 'child', child: {},
       origin: { harness, session_id: String(l.origin || ''), ...(l.origin_store ? { harness_path: String(l.origin_store) } : {}) }, at: String(l.at || ''),
       inherits: l.through ? { through_message_id: String(l.through) } : {}, evidence: l.evidence === 'inferred' ? 'inferred' : 'explicit' }));
-    return [s, ...links];
+    const children = (Array.isArray(o.children) ? o.children : []).map((c) => originLink(c, harness));
+    return [s, ...links, ...children];
   }
   if (o.k === 'usage') {
     if (!o.usage) return [];
     return [usageRecord(o.usage_id != null && o.usage_id !== '' ? `usage:${o.usage_id}` : `usage:@${at}`, {}, o.usage, harness, o.model)];
   }
-  if (o.k === 'child') {
-    return [{ type: 'link', kind: String(o.kind || ''), stated_on: 'origin', origin: {}, child: { harness, session_id: String(o.session_id || '') },
-      at: String(o.at || ''), evidence: o.evidence === 'inferred' ? 'inferred' : 'explicit' }];
-  }
+  if (o.k === 'child') return [originLink(o, harness)];
   if (o.k !== 'message' || o.message_id == null) return [];
   const id = String(o.message_id);
   const parts = [];
@@ -204,6 +210,11 @@ function logAdapter(home, harness, { maxReadBytes = MAX_READ_BYTES } = {}) {
       for (const at of Object.keys(st.lines).sort((a, b) => Number(a) - Number(b))) {
         for (const r of recordsOf(st.lines[at], Number(at), harness)) byKey.set(`${r.type}\u0000${keyOf(r)}`, r);
       }
+      // One link per child: a link with no `at` (a session record's child) gives way to one of the
+      // same kind and child that says where it was found (a `child` record).
+      const placed = new Set();
+      for (const r of byKey.values()) if (r.type === 'link' && r.stated_on === 'origin' && r.at) placed.add(canon([r.kind, r.child]));
+      for (const [k, r] of byKey) if (r.type === 'link' && r.stated_on === 'origin' && !r.at && placed.has(canon([r.kind, r.child]))) byKey.delete(k);
       const changed = []; const live = {}; const emitted = {};
       for (const [k, r] of byKey) {
         const j = JSON.stringify(r);

package/memhouse/shipper/normalise-run.js

@@ -56,15 +56,33 @@ const RETRY_MS = 5 * 60 * 1000;
 const BUDGET_MS = 30 * 1000;
 const keyOf = (s, inc = '') => `${s.harness}\t${s.harness_path}\t${s.session_id}\t${inc}`;
 
-/** Messages in an existing log (0 when there is none). */
+/**
+ * Messages in an existing log (0 when there is none). A log that is there but cannot be read throws:
+ * counted as 0, it would disarm the refusal to rewrite a log of messages into none.
+ */
 function messagesIn(file) {
   let s;
-  try { s = fs.readFileSync(file, 'utf-8'); } catch { return 0; }
+  try { s = fs.readFileSync(file, 'utf-8'); } catch (e) {
+    if (e && (e.code === 'ENOENT' || e.code === 'ENOTDIR')) return 0;
+    throw new Error(`cannot read the existing log (${(e && e.code) || 'error'}): ${path.basename(file)}`);
+  }
   let n = 0;
   for (const l of s.split('\n')) if (l.startsWith('{"k":"message"') || /"k"\s*:\s*"message"/.test(l.slice(0, 200))) n++;
   return n;
 }
 
+// An error as the pass reports it: its first line, and a code `status` and `doctor` key their advice on,
+// with the path when a read named one. The code is the errno of a file read (normalise/fsx.js: EACCES,
+// EPERM, EIO) or SQLite's, named (SQLITE_CANTOPEN on a file the probe could open, SQLITE_NOTADB).
+const SQLITE_CODES = { 5: 'BUSY', 6: 'LOCKED', 10: 'IOERR', 11: 'CORRUPT', 14: 'CANTOPEN', 23: 'AUTH', 26: 'NOTADB' };
+function errorOf(e) {
+  const out = { error: String(e && e.message).split('\n')[0] };
+  if (e && typeof e.code === 'string' && /^E[A-Z0-9]+$/.test(e.code)) out.code = e.code;
+  else if (e && Number.isInteger(e.errcode)) out.code = `SQLITE_${SQLITE_CODES[e.errcode & 0xff] || e.errcode & 0xff}`;
+  if (e && e.path) out.path = String(e.path);
+  return out;
+}
+
 function loadIndex(file) {
   try { const d = JSON.parse(fs.readFileSync(file, 'utf-8')); return d && d.v === 1 && d.sessions ? d : { v: 1, sessions: {} }; } catch { return { v: 1, sessions: {} }; }
 }
@@ -77,7 +95,7 @@ function saveIndex(file, idx) {
 
 /**
  * Bring every normalised log up to date. Returns { harnesses, sessions, normalised, written:
- * { new, append, rewrite, same }, errors: [{ harness, session_id?, error }], ms }.
+ * { new, append, rewrite, same }, errors: [{ harness, session_id?, error, code?, path? }], ms } (errorOf).
  * opts.mods: the modules (default: editors/normalise/); opts.storeRoot(harness): a store to list
  * instead of the module's default (tests).
  */
@@ -93,12 +111,19 @@ function run({ home, mods = modules(), storeRoot = null, log = () => {}, now = D
   for (const m of mods.filter((x) => scope.normaliserInScope(x.HARNESS, editors))) {
     let list;
     try { list = storeRoot && storeRoot(m.HARNESS) ? m.list(storeRoot(m.HARNESS)) : m.list(); } catch (e) {
-      st.errors.push({ harness: m.HARNESS, error: String(e && e.message).split('\n')[0] });
+      st.errors.push({ harness: m.HARNESS, ...errorOf(e) });
       // A store busy for a moment (an editor mid-save) is no reason to forget its sessions: keep
       // their index entries, or the next pass would normalise every one of them again.
       for (const [k, v] of Object.entries(idx.sessions)) if (k.startsWith(`${m.HARNESS}\t`)) seen[k] = v;
       continue;
     }
+    // An entry a readable store could not read (editors/normalise/fsx.js entry: one session's file or
+    // folder refused) is that session's error, the store's other sessions listed. Its index entries are
+    // kept, as a failed list keeps them, so it is not normalised again from scratch once it reads.
+    for (const e of (list && list.errors) || []) {
+      st.errors.push({ harness: m.HARNESS, ...e });
+      for (const [k, v] of Object.entries(idx.sessions)) { const [h, , sid] = k.split('\t'); if (h === m.HARNESS && sid === e.session_id) seen[k] = v; }
+    }
     if (!list || !list.length) continue;
     st.harnesses++;
     for (const s of list) {
@@ -132,7 +157,7 @@ function run({ home, mods = modules(), storeRoot = null, log = () => {}, now = D
         st.normalised++;
         seen[k0] = { sig, file, at: now };
       } catch (e) {
-        st.errors.push({ harness: m.HARNESS, session_id: String(s.session_id), error: String(e && e.message).split('\n')[0] });
+        st.errors.push({ harness: m.HARNESS, session_id: String(s.session_id), ...errorOf(e) });
         seen[k0] = { sig, file: '', failed: true, at: now, tries: prev && prev.failed && prev.sig === sig ? (prev.tries || 0) + 1 : 0 };
       }
     }
@@ -145,4 +170,4 @@ function run({ home, mods = modules(), storeRoot = null, log = () => {}, now = D
   return st;
 }
 
-module.exports = { run, modules, sigOf, DIR };
+module.exports = { run, modules, sigOf, errorOf, DIR };

package/memhouse/shipper/passlog.js

@@ -19,8 +19,12 @@ const FAIL = /\[memhouse\] (?:v3 )?pass failed: ([^\n]*)/;
 // A pass's refused vanish sweeps and its vanish warnings, printed after its pass line (capture.js vanishLines).
 const VANISH = /^\[memhouse\] vanish refused: ([^\n]*)$/gm;
 const WARNED = /^\[memhouse\] vanish warning: ([^\n]*)$/gm;
+// The pass's normaliser errors and their advice, printed after its pass line (capture.js normaliseLines).
+const NORM_ERR = /^\[memhouse\] normalise error: (\S+) \(([^)]*)\): ([^\n]*)$/gm;
+// A hint names its editor and code: one editor can fail on two codes that need different advice.
+const NORM_HINT = /^\[memhouse\] normalise hint: (\S+) \(([^)]*)\): ([^\n]*)$/gm;
 
-/** @returns {{outcome:'ok'|'failed'|null, sessions?:number, rows?:number, why?:string, auth?:boolean, vanishRefused?:string[]}} */
+/** @returns {{outcome:'ok'|'failed'|null, sessions?:number, rows?:number, why?:string, auth?:boolean, vanishRefused?:string[], normaliseErrors?:object[]}} */
 function lastPass(logText) {
   const text = String(logText || '');
   let lastOk = -1; let lastFail = -1; let okM = null; let failM = null;
@@ -33,12 +37,18 @@ function lastPass(logText) {
   }
   const refused = [...text.slice(lastOk).matchAll(VANISH)].map((m) => m[1].trim());
   const warned = [...text.slice(lastOk).matchAll(WARNED)].map((m) => m[1].trim());
+  const hints = new Map([...text.slice(lastOk).matchAll(NORM_HINT)].map((m) => [`${m[1]}\u0000${m[2]}`, m[3].trim()]));
+  const normalise = [...text.slice(lastOk).matchAll(NORM_ERR)].map((m) => {
+    const hint = hints.get(`${m[1]}\u0000${m[2]}`);
+    return { harness: m[1], code: m[2], error: m[3].trim(), ...(hint ? { hint } : {}) };
+  });
   return {
     ...(okM[1] !== undefined ? { outcome: 'ok', sessions: Number(okM[1]), rows: Number(okM[2]) }
       : okM[3] !== undefined ? { outcome: 'ok', sessions: Number(okM[3]), rows: Number(okM[4]), v3: true }
         : { outcome: 'ok', sessions: Number(okM[5]), rows: Number(okM[6]), v3: true, profile: true }),
     ...(refused.length ? { vanishRefused: refused } : {}),
     ...(warned.length ? { vanishWarned: warned } : {}),
+    ...(normalise.length ? { normaliseErrors: normalise } : {}),
   };
 }
 
@@ -86,6 +96,16 @@ function vanishChecks(lp) {
   return out;
 }
 
+/**
+ * doctor's checks for the last pass's normaliser errors: one per editor and code, a failure, with the
+ * advice capture.js printed (macOS: Full Disk Access) or the log to read.
+ */
+function normaliseChecks(lp) {
+  return ((lp && lp.normaliseErrors) || []).map((e) => ({
+    ok: false, label: `${e.harness}: a read failed in the last pass (${e.code}): ${e.error}`, hint: e.hint || 'read: ~/.memhouse/logs/shipper.log',
+  }));
+}
+
 /** The last pass of the shipper log in `logDir` (its last 64 KiB): what `status` and doctor read. */
 function readLastPass(logDir) {
   const fs = require('fs'); const path = require('path');
@@ -97,4 +117,4 @@ function readLastPass(logDir) {
   } catch { return { outcome: null }; }
 }
 
-module.exports = { lastPass, readLastPass, describeFailure, describeVanish, describeVanishWarning, vanishChecks };
+module.exports = { lastPass, readLastPass, describeFailure, describeVanish, describeVanishWarning, vanishChecks, normaliseChecks };