memhouse 0.18.12 → 0.20.1

package/memhouse/import/writer-profile.js

@@ -0,0 +1,192 @@
+'use strict';
+// The profile writer for imported sessions: harness-sessions-sql's profile tables (<m>_session_parts,
+// <m>_calls, <m>_sessions; memhouse/house/schema-profile.sql.tpl). The parse becomes sessions-model
+// records (records.js), records-to-profile.js turns them into rows, and the shared rollup
+// (profile-rollup.js) fills the sessions row's derived columns after the rows are in. The keys are
+// the parser's (claude-ai.js), copied; only the store differs from the layout 3.2 writer.
+//
+// The store of a claude.ai conversation is the service, not a machine (the sessions-model v1.x
+// service-host proposal): host s- + the first 32 hex of sha256("sessions-model/service-host/v0:" +
+// lowercase domain), harness_path the service origin, host_label the domain.
+
+const crypto = require('crypto');
+const py = require('./pyjson');
+const C = require('./claude-ai');
+const { recordsOf, iso } = require('./records');
+const { profileRows } = require('../house/records-to-profile');
+const sp = require('../house/schema-profile');
+const rollup = require('../house/profile-rollup');
+const V = require('./writer-v3');
+
+const LAYOUT = 'profile';
+const TABLES = ['session_parts', 'calls', 'sessions'];
+const PARTS = 'session_parts';
+const TOMBSTONE_ORDER = ['calls', 'session_parts', 'sessions'];
+const ADAPTER = 'memhouse-import';
+const { sqlStr, batchTag, tombstoneSQL, revBySession } = V;
+
+/** A service's host (sessions-model v1.x service-host proposal): s- + 32 hex. */
+function serviceHost(domain) {
+  return `s-${crypto.createHash('sha256').update(`sessions-model/service-host/v0:${String(domain).toLowerCase()}`).digest('hex').slice(0, 32)}`;
+}
+
+const SRC = {
+  harness: C.HARNESS, host: serviceHost(C.STORE), store: `https://${C.STORE}`, hostLabel: C.STORE,
+  adapter: ADAPTER, parserVersion: C.PARSER_VERSION,
+};
+
+function names(db, member) { return sp.profileNames(db, member); }
+
+function storeWhere(src) {
+  return `host = ${sqlStr(src.host)} AND harness = ${sqlStr(src.harness)} AND harness_path = ${sqlStr(src.store)} AND incarnation = ''`;
+}
+
+let COLS = null;
+const columns = (t) => {
+  if (!COLS) { const tpl = sp.readTemplate(); COLS = Object.fromEntries(TABLES.map((x) => [x, new Set(sp.insertColumns(tpl, x))])); }
+  return COLS[t];
+};
+const pick = (t, r) => Object.fromEntries(Object.entries(r).filter(([k, v]) => columns(t).has(k) && v !== undefined));
+
+/** Every row of one session, by table. The sessions row holds the spec columns; the rollup adds the rest. */
+function rows(s, { src, batch, rev }) {
+  const tag = batchTag(batch);
+  const key = { host: src.host, harness: src.harness, harness_path: src.store, session_id: s.sessionId, incarnation: '' };
+  const mapped = profileRows(recordsOf(s, { harnessVersion: tag }), key, { firstSeen: () => iso(s.created) });
+  const own = { rev, is_deleted: 0, removed: 0, adapter: src.adapter, adapter_ver: String(src.parserVersion), model_version: '' };
+  const byKey = new Map(s.parts.map((p) => [`${p.messageId}\u0000${p.partId}`, p]));
+  // A tool call's arguments as the export wrote them (compact JSON, numbers and key order kept): a
+  // JSON round trip would round an integer past 2^53 and drop a float's ".0".
+  const argsOf = new Map(s.parts.filter((p) => p.kind === 'tool_call').map((p) => [p.callId, py.compact(p.input === undefined ? {} : p.input)]));
+  const parts = mapped.parts.map((r) => {
+    const p = byKey.get(`${r.message_id}\u0000${r.part_id}`) || {};
+    return pick('session_parts', {
+      // text_len counts what was said, as the capture does: a tool result's output is in neither role's characters.
+      ...r, ...(r.kind === 'tool_call' && argsOf.has(r.call_id) ? { args: argsOf.get(r.call_id) } : {}), ...own, folder: s.folder, project: s.project, harness_version: tag, text_len: py.pyLen(String(r.text || '')),
+      origin: 'import', extra: py.dumps({ import_batch: batch, ...(p.extra || {}) }),
+    });
+  });
+  const calls = mapped.calls.map((c) => pick('calls', { ...c, ...(argsOf.has(c.call_id) ? { args: argsOf.get(c.call_id) } : {}), ...own, folder: s.folder, project: s.project }));
+  const sessions = [pick('sessions', { ...mapped.session, ...own, host_label: src.hostLabel, folder: s.folder, project: s.project, origin: 'import' })];
+  return { session_parts: parts, calls, sessions };
+}
+
+function presentSQL(N, src) {
+  return `SELECT session_id, toString(updated_at) AS updated_at, project, folder, title AS name, toString(mode) AS mode,
+      toString(created_at) AS created_at
+    FROM ${N.sessions} FINAL WHERE is_deleted = 0 AND ${storeWhere(src)}`;
+}
+
+function maxRevSQL(N, src, ids) {
+  const list = ids.map(sqlStr).join(', ');
+  return TABLES.map((t) => `SELECT session_id, max(rev) AS rev FROM ${N[t]} WHERE ${storeWhere(src)} AND session_id IN (${list}) GROUP BY session_id`);
+}
+
+/** A batch's live rows, per table: its parts and sessions carry the batch; calls go by its live parts. */
+function batchWhere(N, src, batch) {
+  const own = `${storeWhere(src)} AND origin = 'import' AND harness_version = ${sqlStr(batchTag(batch))}`;
+  return {
+    session_parts: own,
+    sessions: own,
+    calls: `${storeWhere(src)} AND session_id IN (SELECT DISTINCT session_id FROM ${N.session_parts} FINAL WHERE is_deleted = 0 AND ${own})`,
+  };
+}
+
+function sessionsWhere(src, ids) {
+  const w = `${storeWhere(src)} AND session_id IN (${ids.map(sqlStr).join(', ')})`;
+  return { session_parts: w, calls: w, sessions: w };
+}
+
+// ── content: what a conversation says, apart from where and when it was imported ──────────
+//
+// The parts' own shas (records-to-profile.js: the part as it sits in its message, and the
+// message's), which cover the text, the tool calls, their arguments and results, the roles and
+// times, and nothing of the store or the batch; plus the session's name, mode and created_at.
+// Calls are derived from the parts and add nothing. updated_at is compared on its own.
+
+function contentSQL(N, src, ids) {
+  return {
+    // extra as stored (a String, returned verbatim): JSONExtractRaw would write the JSON out again.
+    parts: `SELECT session_id, message_id, part_id, content_sha, message_content_sha, extra FROM ${N.session_parts} FINAL
+      WHERE is_deleted = 0 AND removed = 0 AND ${storeWhere(src)} AND session_id IN (${ids.map(sqlStr).join(', ')})`,
+  };
+}
+
+function contentDigest(session, byName) {
+  const h = crypto.createHash('sha256');
+  h.update(JSON.stringify([session.name, session.mode, session.created_at === undefined ? null : session.created_at]));
+  // The files attached to a message ride in its first part's extra (claude-ai.js), outside the shas:
+  // compared parsed, so how the JSON was written (separators, escapes) never counts.
+  for (const l of (byName.parts || []).map((r) => JSON.stringify([r.message_id, r.part_id, String(r.content_sha), String(r.message_content_sha), filesOfExtra(r.extra)])).sort()) h.update(`\nP${l}`);
+  return h.digest('hex');
+}
+
+/** A part's attached file names, from its stored extra: '' when it has none. */
+function filesOfExtra(extra) {
+  let o = null;
+  try { o = typeof extra === 'string' ? JSON.parse(extra || '{}') : extra; } catch { return ''; }
+  return o && Array.isArray(o.files) && o.files.length ? JSON.stringify(o.files) : '';
+}
+
+function parsedDigest(s, src) {
+  const r = rows(s, { src, batch: 'digest', rev: 0 });
+  const x = r.sessions[0];
+  return contentDigest({ name: x.title, mode: x.mode, created_at: s.created }, { parts: r.session_parts });
+}
+
+// ClickHouse refuses a statement over max_query_size (262,144 bytes by default), and the rollup
+// repeats its session filter many times: each refresh stays under this.
+const MAX_STATEMENT = 200000;
+
+/** Fill the derived columns of these sessions' rows (the shared rollup), in statements under MAX_STATEMENT. */
+async function refresh(house, N, ids, src) {
+  let size = 500;
+  for (let i = 0; i < ids.length;) {
+    const part = ids.slice(i, i + size);
+    const sql = rollup.refreshSessionsSQL(N, `${storeWhere(src)} AND session_id IN (${part.map(sqlStr).join(', ')})`);
+    if (sql.length > MAX_STATEMENT && part.length > 1) { size = Math.max(1, Math.floor(part.length / 2)); continue; }
+    await house.exec(sql);
+    i += part.length;
+  }
+}
+
+/**
+ * The sessions of this store whose row the rollup never filled: a run stopped between the sessions
+ * insert and the refresh (it left a live row with no start) while their parts are in. plan() lists
+ * them, and apply() refreshes them, whatever else it writes.
+ */
+function unrefreshedSQL(N, src) {
+  return `SELECT session_id, harness_version FROM ${N.sessions} FINAL WHERE is_deleted = 0 AND removed = 0 AND ${storeWhere(src)} AND started IS NULL
+      AND session_id IN (SELECT DISTINCT session_id FROM ${N.session_parts} FINAL WHERE is_deleted = 0 AND ${storeWhere(src)})`;
+}
+
+/** Insert the rows of a batch, then let the shared rollup fill each sessions row's derived columns. */
+async function writeRows(house, N, built, src, onProgress = () => {}, onInserted = async () => {}) {
+  const written = {};
+  for (const t of TABLES) {
+    const all = built.flatMap((b) => b[t]);
+    await house.insert(N[t], all);
+    written[t] = all.length;
+    onProgress(`inserted ${t}: ${all.length}`);
+  }
+  await onInserted();
+  const ids = built.map((b) => b.sessions[0].session_id);
+  await refresh(house, N, ids, src);
+  onProgress(`refreshed the sessions rows: ${ids.length}`);
+  return written;
+}
+
+function countsSQL(N, src, batch) {
+  const w = batchWhere(N, src, batch);
+  return {
+    sessions: `SELECT count() AS sessions, sum(messages) AS messages, sum(tool_calls) AS tool_calls FROM ${N.sessions} FINAL WHERE is_deleted = 0 AND ${w.sessions}`,
+    parts: `SELECT kind, count() AS n FROM ${N.session_parts} FINAL WHERE is_deleted = 0 AND ${w.session_parts} GROUP BY kind ORDER BY kind`,
+    calls: `SELECT count() AS n FROM ${N.calls} FINAL WHERE is_deleted = 0 AND ${w.calls}`,
+  };
+}
+
+module.exports = {
+  LAYOUT, TABLES, PARTS, TOMBSTONE_ORDER, ADAPTER, SRC, serviceHost, names, sqlStr, batchTag, storeWhere, rows, presentSQL, maxRevSQL,
+  batchWhere, sessionsWhere, tombstoneSQL, revBySession, contentSQL, contentDigest, parsedDigest, writeRows, countsSQL,
+  refresh, unrefreshedSQL, MAX_STATEMENT,
+};

package/memhouse/import/writer-v3.js

@@ -0,0 +1,240 @@
+'use strict';
+// The layout 3.2 writer for imported sessions: <m>_turns, <m>_calls and <m>_sessions
+// (memhouse/house/schema-v3.sql.tpl). It turns the parser's sessions into rows and says which
+// rows belong to the import. Keys come from the parser unchanged (claude-ai.js); a writer for
+// another layout implements the same functions over its own tables.
+
+const crypto = require('crypto');
+const py = require('./pyjson');
+
+const C = require('./claude-ai');
+const V3 = require('../house/schema-v3');
+
+const LAYOUT = '3.2';
+const TABLES = ['turns', 'calls', 'sessions'];
+const PARTS = 'turns';
+// Calls first: an undo finds them through the batch's live turns.
+const TOMBSTONE_ORDER = ['calls', 'turns', 'sessions'];
+const SRC = { harness: C.HARNESS, store: C.STORE, parserVersion: C.PARSER_VERSION };
+function names(db, member) { return V3.v3Names(db, member); }
+
+const sqlStr = (s) => `'${String(s).replace(/\\/g, '\\\\').replace(/'/g, "\\'")}'`;
+
+/** harness_version of every turn and session row an import batch writes. */
+function batchTag(batch) { return `import:${batch}`; }
+
+/** The import's store: every row of an imported session matches this, and nothing else does. */
+function storeWhere(src) {
+  return `harness = ${sqlStr(src.harness)} AND host = ${sqlStr(src.store)} AND harness_path = ${sqlStr(src.store)} AND incarnation = ''`;
+}
+
+function turnRows(s, { src, batch, rev }) {
+  const tag = batchTag(batch);
+  return s.parts.map((p, ord) => ({
+    harness: src.harness, harness_version: tag, harness_path: src.store, session_id: s.sessionId, incarnation: '',
+    agent_id: '', message_id: p.messageId, part_id: p.partId, part_ord: p.partOrd, parent_id: p.parentId,
+    response_id: p.responseId, ts: p.ts, ord, rev, is_deleted: 0, removed: 0, state: '', role: p.role, kind: p.kind,
+    model: '', effort: '', input_tokens: 0, output_tokens: 0, cache_read_tokens: 0, cache_write_tokens: 0,
+    reasoning_tokens: 0, text: p.text, text_len: py.pyLen(p.text), folder: s.folder, project: s.project,
+    host: src.store, origin: 'import', extra: py.dumps({ import_batch: batch, ...(p.extra || {}) }),
+    adapter_ver: src.parserVersion,
+  }));
+}
+
+function callRows(s, { src, rev }) {
+  return s.calls.map((c) => ({
+    harness: src.harness, harness_path: src.store, session_id: s.sessionId, incarnation: '', agent_id: '',
+    call_id: c.callId, message_id: c.messageId, ts: c.ts, rev, is_deleted: 0, removed: 0, tool_name: c.toolName,
+    args: c.args, result_ts: c.resultTs, is_error: c.isError, result_len: c.resultLen, model: '',
+    folder: s.folder, project: s.project, host: src.store, adapter_ver: src.parserVersion,
+  }));
+}
+
+const byCodePoint = (a, b) => (a < b ? -1 : a > b ? 1 : 0);
+
+function sessionRow(s, { src, batch, rev }) {
+  const turns = turnRows(s, { src, batch, rev });
+  const calls = callRows(s, { src, rev });
+  const text = turns.filter((t) => t.kind === 'text');
+  const ts = turns.map((t) => t.ts).sort(byCodePoint);
+  const hours = new Map();
+  for (const t of text) { const h = `${t.ts.slice(0, 13)}:00:00`; hours.set(h, (hours.get(h) || 0) + 1); }
+  const tools = new Map();
+  for (const c of calls) {
+    const v = tools.get(c.tool_name) || [0, 0];
+    v[0] += 1; v[1] += c.is_error ? 1 : 0;
+    tools.set(c.tool_name, v);
+  }
+  const hs = [...hours.keys()].sort(byCodePoint);
+  const tn = [...tools.keys()].sort(byCodePoint);
+  const first = text.find((t) => t.role === 'user');
+  const zeros = hs.map(() => 0);
+  return {
+    harness: src.harness, session_id: s.sessionId, incarnation: '', harness_version: batchTag(batch),
+    harness_path: src.store, host: src.store, host_label: src.store, name: s.name, mode: s.mode,
+    folder: s.folder, project: s.project, git_branch: '', created_at: s.created, updated_at: s.updated,
+    started: ts[0], ended: ts[ts.length - 1],
+    messages: text.length,
+    user_messages: text.filter((t) => t.role === 'user').length,
+    assistant_messages: text.filter((t) => t.role === 'assistant').length,
+    agent_messages: 0, tool_calls: calls.length,
+    input_tokens: 0, output_tokens: 0, cache_read_tokens: 0, cache_write_tokens: 0, reasoning_tokens: 0,
+    user_chars: text.filter((t) => t.role === 'user').reduce((a, t) => a + t.text_len, 0),
+    assistant_chars: text.filter((t) => t.role === 'assistant').reduce((a, t) => a + t.text_len, 0),
+    first_prompt: first ? first.text : '', models: [], dominant_model: '',
+    days: [...new Set(turns.map((t) => t.ts.slice(0, 10)))].sort(byCodePoint),
+    'usage.hour': hs, 'usage.model': hs.map(() => ''), 'usage.messages': hs.map((h) => hours.get(h)),
+    'usage.input': zeros, 'usage.output': zeros, 'usage.cache_read': zeros, 'usage.cache_write': zeros,
+    'usage.reasoning': zeros,
+    'tools.name': tn, 'tools.calls': tn.map((n) => tools.get(n)[0]), 'tools.errors': tn.map((n) => tools.get(n)[1]),
+    'agents.agent_id': [], 'agents.type': [], 'agents.description': [], 'agents.workflow': [],
+    'agents.messages': [], 'agents.first_ts': [],
+    'links.kind': [], 'links.origin': [], 'links.origin_store': [], 'links.evidence': [], 'links.at': [],
+    'links.through': [],
+    // The export carries no token counts: unmeasured, not free.
+    usage_basis: 'none', origin: 'import', rev, is_deleted: 0, removed: 0,
+  };
+}
+
+/** Every row of one session, by table. */
+function rows(s, opts) {
+  return { turns: turnRows(s, opts), calls: callRows(s, opts), sessions: [sessionRow(s, opts)] };
+}
+
+/** The imported sessions the house holds now: id, the export's updated_at, project, folder, and the session's content fields. */
+function presentSQL(names, src) {
+  return `SELECT session_id, toString(updated_at) AS updated_at, project, folder, name, toString(mode) AS mode,
+      toString(created_at) AS created_at
+    FROM ${names.sessions} FINAL WHERE is_deleted = 0 AND ${storeWhere(src)}`;
+}
+
+// ── content: what a conversation says, apart from where and when it was imported ──────────
+//
+// Two imports of one conversation are the same content when these match:
+//   session  name, mode, created_at
+//   turns    message_id, part_id, part_ord, parent_id, response_id, ts, ord, role, kind,
+//            md5(text), md5(extra without import_batch)
+//   calls    call_id, message_id, ts, tool_name, md5(args), result_ts, is_error, result_len
+// Never in it: project and folder (an archive gives a slug and a path, an export none),
+// harness_version and extra.import_batch (they name the batch), rev, is_deleted, removed,
+// ingested_at, adapter_ver, updated_at (compared on its own), the session's derived totals, and
+// the columns every imported row has alike. Long values go as their md5, computed by the house
+// for stored rows and here for parsed ones, so the comparison reads hashes, not texts.
+
+const md5 = (s) => crypto.createHash('md5').update(String(s), 'utf8').digest('hex');
+// extra as the writer stores it is {"import_batch": "<batch>", …the part's own…}, pyjson's text with
+// the batch first; the batch id has no quote in it (import.js checkBatchId). One pattern removes
+// exactly that key, applied as text on both sides: in the house to the stored extra, here to the
+// extra the writer would store. Parsing the JSON in ClickHouse would change its separators and
+// number forms, and every row would differ.
+const STRIP_BATCH = '^\\{"import_batch": "[^"]*"(, )?';
+const STRIP_BATCH_SQL = `replaceRegexpOne(extra, '${STRIP_BATCH.replace(/\\/g, '\\\\')}', '{')`;
+const stripBatch = (extra) => extra.replace(new RegExp(STRIP_BATCH), '{');
+
+const turnContent = (t) => JSON.stringify([t.message_id, t.part_id, Number(t.part_ord), t.parent_id, t.response_id, t.ts,
+  Number(t.ord), t.role, t.kind, t.text_md5, t.extra_md5]);
+const callContent = (c) => JSON.stringify([c.call_id, c.message_id, c.ts, c.tool_name, c.args_md5, c.result_ts === undefined ? null : c.result_ts,
+  Boolean(c.is_error), Number(c.result_len)]);
+const sessionContent = (x) => JSON.stringify([x.name, x.mode, x.created_at === undefined ? null : x.created_at]);
+
+/** One digest from a session's content lines; order-free. */
+function contentDigest(session, turns, calls) {
+  const h = crypto.createHash('sha256');
+  h.update(sessionContent(session));
+  for (const l of turns.map(turnContent).sort()) h.update(`\nT${l}`);
+  for (const l of calls.map(callContent).sort()) h.update(`\nC${l}`);
+  return h.digest('hex');
+}
+
+/** The content digest of a parsed session: of the rows this writer would store for it. */
+function parsedDigest(s, src) {
+  const r = rows(s, { src, batch: 'digest', rev: 0 });
+  const turns = r.turns.map((t) => ({ ...t, text_md5: md5(t.text), extra_md5: md5(stripBatch(t.extra)) }));
+  const calls = r.calls.map((c) => ({ ...c, args_md5: md5(c.args) }));
+  return contentDigest(r.sessions[0], turns, calls);
+}
+
+/** The stored rows' content of these sessions, per table, hashed in the house. */
+function contentSQL(names, src, ids) {
+  const w = `is_deleted = 0 AND ${storeWhere(src)} AND session_id IN (${ids.map(sqlStr).join(', ')})`;
+  return {
+    turns: `SELECT session_id, message_id, part_id, part_ord, parent_id, response_id, toString(ts) AS ts, ord, toString(role) AS role,
+        toString(kind) AS kind, lower(hex(MD5(text))) AS text_md5, lower(hex(MD5(${STRIP_BATCH_SQL}))) AS extra_md5
+      FROM ${names.turns} FINAL WHERE ${w}`,
+    calls: `SELECT session_id, call_id, message_id, toString(ts) AS ts, toString(tool_name) AS tool_name, lower(hex(MD5(args))) AS args_md5,
+        toString(result_ts) AS result_ts, is_error, result_len
+      FROM ${names.calls} FINAL WHERE ${w}`,
+  };
+}
+
+/** The highest rev any row of these sessions carries, tombstones included (no FINAL), per table. */
+function maxRevSQL(names, src, ids) {
+  const list = ids.map(sqlStr).join(', ');
+  return TABLES.map((t) => `SELECT session_id, max(rev) AS rev FROM ${names[t]}
+    WHERE ${storeWhere(src)} AND session_id IN (${list}) GROUP BY session_id`);
+}
+
+/** Which live rows belong to a batch, per table. Calls carry no batch, so they go by the batch's live turns. */
+function batchWhere(names, src, batch) {
+  const own = `${storeWhere(src)} AND origin = 'import' AND harness_version = ${sqlStr(batchTag(batch))}`;
+  return {
+    turns: own,
+    sessions: own,
+    calls: `${storeWhere(src)} AND session_id IN (SELECT DISTINCT session_id FROM ${names.turns} FINAL WHERE is_deleted = 0 AND ${own})`,
+  };
+}
+
+/** Which live rows belong to these sessions, per table. */
+function sessionsWhere(src, ids) {
+  const w = `${storeWhere(src)} AND session_id IN (${ids.map(sqlStr).join(', ')})`;
+  return { turns: w, calls: w, sessions: w };
+}
+
+/**
+ * A tombstone for every live row the predicate selects: a full copy at a higher rev with
+ * is_deleted = 1, which FINAL reads hide at once and the shipper's maintenance cleanup drops.
+ * `cols` are the table's insertable columns; ingested_at is left to its default. `revExpr` is
+ * SQL over the row (`rev + 1`, or a per-session value).
+ */
+function tombstoneSQL(fq, cols, where, revExpr) {
+  const use = cols.filter((c) => c !== 'ingested_at');
+  const q = (c) => `\`${c}\``;
+  const sel = use.map((c) => (c === 'rev' ? `toUInt64(${revExpr})` : c === 'is_deleted' ? 'toUInt8(1)' : q(c)));
+  return `INSERT INTO ${fq} (${use.map(q).join(', ')}) SELECT ${sel.join(', ')} FROM ${fq} FINAL WHERE is_deleted = 0 AND ${where}`;
+}
+
+/** `rev` for each session id, as SQL over session_id. */
+function revBySession(revs) {
+  const ids = [...revs.keys()];
+  return `transform(session_id, [${ids.map(sqlStr).join(', ')}], [${ids.map((i) => `toUInt64(${revs.get(i)})`).join(', ')}], toUInt64(0))`;
+}
+
+/** contentDigest over contentSQL's rows, by query name. */
+const contentDigestOf = (session, by) => contentDigest(session, by.turns || [], by.calls || []);
+
+/** Insert the rows of a batch: turns, calls, then the session rows. */
+async function writeRows(house, N, built, src, onProgress = () => {}, onInserted = async () => {}) {
+  const written = {};
+  for (const t of TABLES) {
+    const all = built.flatMap((b) => b[t]);
+    await house.insert(N[t], all);
+    written[t] = all.length;
+    onProgress(`inserted ${t}: ${all.length}`);
+  }
+  await onInserted();
+  return written;
+}
+
+function countsSQL(N, src, batch) {
+  const w = batchWhere(N, src, batch);
+  return {
+    sessions: `SELECT count() AS sessions, sum(messages) AS messages, sum(tool_calls) AS tool_calls FROM ${N.sessions} FINAL WHERE is_deleted = 0 AND ${w.sessions}`,
+    parts: `SELECT kind, count() AS n FROM ${N.turns} FINAL WHERE is_deleted = 0 AND ${w.turns} GROUP BY kind ORDER BY kind`,
+    calls: `SELECT count() AS n FROM ${N.calls} FINAL WHERE is_deleted = 0 AND ${w.calls}`,
+  };
+}
+
+module.exports = {
+  LAYOUT, TABLES, PARTS, TOMBSTONE_ORDER, SRC, names, writeRows, countsSQL, contentDigestOf, sqlStr, batchTag, storeWhere, turnRows, callRows, sessionRow, rows,
+  presentSQL, maxRevSQL, batchWhere, sessionsWhere, tombstoneSQL, revBySession, contentSQL, contentDigest, parsedDigest, stripBatch, STRIP_BATCH_SQL,
+};

package/memhouse/import/zip.js

@@ -0,0 +1,112 @@
+'use strict';
+// Read entries out of a .zip file: enough for a claude.ai data export, with no dependency.
+// Stored and deflated entries, ZIP64 sizes and offsets, CRC-checked. Reads only the entries
+// asked for, through the file descriptor, so a large export is never loaded whole.
+
+const fs = require('fs');
+const zlib = require('zlib');
+
+const SIG = { eocd: 0x06054b50, eocd64: 0x06064b50, locator64: 0x07064b50, central: 0x02014b50, local: 0x04034b50 };
+
+class ZipError extends Error {}
+
+function readAt(fd, pos, len) {
+  const buf = Buffer.alloc(len);
+  let got = 0;
+  while (got < len) {
+    const r = fs.readSync(fd, buf, got, len - got, pos + got);
+    if (r === 0) throw new ZipError('unexpected end of file');
+    got += r;
+  }
+  return buf;
+}
+
+function isZip(file) {
+  let fd;
+  try {
+    fd = fs.openSync(file, 'r');
+    const b = Buffer.alloc(4);
+    return fs.readSync(fd, b, 0, 4, 0) === 4 && b.readUInt32LE(0) === SIG.local;
+  } catch { return false; } finally { if (fd !== undefined) fs.closeSync(fd); }
+}
+
+/** Open a zip: { names, has(name), read(name) -> Buffer, close() }. */
+function openZip(file) {
+  const fd = fs.openSync(file, 'r');
+  try {
+    const size = fs.fstatSync(fd).size;
+    const tailLen = Math.min(size, 22 + 0xffff);
+    const tail = readAt(fd, size - tailLen, tailLen);
+    let at = -1;
+    for (let k = tailLen - 22; k >= 0; k--) if (tail.readUInt32LE(k) === SIG.eocd) { at = k; break; }
+    if (at < 0) throw new ZipError('not a zip file (no end of central directory)');
+    let count = tail.readUInt16LE(at + 10);
+    let cdSize = tail.readUInt32LE(at + 12);
+    let cdOffset = tail.readUInt32LE(at + 16);
+    if (count === 0xffff || cdSize === 0xffffffff || cdOffset === 0xffffffff) {
+      const locPos = size - tailLen + at - 20;
+      const loc = readAt(fd, locPos, 20);
+      if (loc.readUInt32LE(0) !== SIG.locator64) throw new ZipError('ZIP64 locator missing');
+      const e64 = readAt(fd, Number(loc.readBigUInt64LE(8)), 56);
+      if (e64.readUInt32LE(0) !== SIG.eocd64) throw new ZipError('ZIP64 end of central directory missing');
+      count = Number(e64.readBigUInt64LE(32));
+      cdSize = Number(e64.readBigUInt64LE(40));
+      cdOffset = Number(e64.readBigUInt64LE(48));
+    }
+    const cd = readAt(fd, cdOffset, cdSize);
+    const entries = new Map();
+    let p = 0;
+    for (let k = 0; k < count; k++) {
+      if (cd.readUInt32LE(p) !== SIG.central) throw new ZipError('bad central directory entry');
+      const flags = cd.readUInt16LE(p + 8);
+      const method = cd.readUInt16LE(p + 10);
+      const crc = cd.readUInt32LE(p + 16);
+      let comp = cd.readUInt32LE(p + 20);
+      let uncomp = cd.readUInt32LE(p + 24);
+      const nameLen = cd.readUInt16LE(p + 28);
+      const extraLen = cd.readUInt16LE(p + 30);
+      const commentLen = cd.readUInt16LE(p + 32);
+      let offset = cd.readUInt32LE(p + 42);
+      const name = cd.toString('utf-8', p + 46, p + 46 + nameLen);
+      let x = p + 46 + nameLen;
+      const xEnd = x + extraLen;
+      while (x + 4 <= xEnd) {
+        const id = cd.readUInt16LE(x);
+        const len = cd.readUInt16LE(x + 2);
+        if (id === 0x0001) {
+          let y = x + 4;
+          if (uncomp === 0xffffffff) { uncomp = Number(cd.readBigUInt64LE(y)); y += 8; }
+          if (comp === 0xffffffff) { comp = Number(cd.readBigUInt64LE(y)); y += 8; }
+          if (offset === 0xffffffff) { offset = Number(cd.readBigUInt64LE(y)); y += 8; }
+        }
+        x += 4 + len;
+      }
+      entries.set(name, { name, flags, method, crc, comp, uncomp, offset });
+      p = xEnd + commentLen;
+    }
+    const read = (name) => {
+      const e = entries.get(name);
+      if (!e) throw new ZipError(`no entry ${name}`);
+      if (e.flags & 1) throw new ZipError(`${name} is encrypted`);
+      const h = readAt(fd, e.offset, 30);
+      if (h.readUInt32LE(0) !== SIG.local) throw new ZipError(`bad local header for ${name}`);
+      const data = readAt(fd, e.offset + 30 + h.readUInt16LE(26) + h.readUInt16LE(28), e.comp);
+      let out;
+      if (e.method === 0) out = data;
+      else if (e.method === 8) {
+        // Never inflate past the declared size: an entry that claims little and expands to a lot
+        // is refused here, not after it has filled memory.
+        try { out = zlib.inflateRawSync(data, { maxOutputLength: Math.max(1, e.uncomp) }); } catch (err) {
+          throw new ZipError(`${name}: ${err && err.code === 'ERR_BUFFER_TOO_LARGE' ? 'larger than its declared size' : `cannot inflate (${err.message})`}`);
+        }
+      }
+      else throw new ZipError(`${name}: compression method ${e.method} is not supported`);
+      if (out.length !== e.uncomp) throw new ZipError(`${name}: size mismatch`);
+      if (typeof zlib.crc32 === 'function' && (zlib.crc32(out) >>> 0) !== e.crc) throw new ZipError(`${name}: CRC mismatch`);
+      return out;
+    };
+    return { names: [...entries.keys()], has: (n) => entries.has(n), read, close: () => fs.closeSync(fd) };
+  } catch (e) { fs.closeSync(fd); throw e; }
+}
+
+module.exports = { openZip, isZip, ZipError };

package/memhouse/invitation.js

@@ -0,0 +1,102 @@
+// The invitation page, MEMHOUSE-INVITATION.md: what `memhouse invite` writes beside the invite
+// file it hands over (delivery/INVITATION.md.tpl, rendered here).
+//
+// Rendered, not linked: it names THIS invite's file, this house, the tables this invite created
+// and, until the build is published, the exact tarball to install. A static page can carry none
+// of that. Nothing secret goes into it, and nothing here reads the invite file.
+//
+// Two ways the page used to contradict the invite it came with:
+//   * the tables. It listed the 0.18 rooms (`<m>_messages`, …) for every invite, so an invite
+//     into a schema-3 house described tables that do not exist. The names now come from the
+//     functions provisioning names them with: profileNames() for schema 3, roomNames() and
+//     statTableNames() for the 0.18 rooms.
+//   * the file. Step 0 was `ls invite-*.env`, which matches nothing when the invite was written
+//     with `--out <dir>/<other-name>.env`. Every command now names the file this invite wrote.
+//
+// Pure apart from reading the template: a test renders it without a server.
+
+const fs = require('fs');
+const path = require('path');
+const { roomNames, statTableNames, ROOM_TYPES, META_TYPES } = require('./house/house');
+const SP = require('./house/schema-profile');
+const { isInviteFileName } = require('./invitefile');
+const { quoteShell } = require('./envfile');
+
+const PAGE = 'MEMHOUSE-INVITATION.md';
+const TEMPLATE = path.join(__dirname, 'delivery', 'INVITATION.md.tpl');
+
+function readTemplate() { return fs.readFileSync(TEMPLATE, 'utf-8'); }
+
+const code = (s) => `\`${s}\``;
+
+/** A file name as one shell word: bare when it needs no quoting, never read as an option. */
+function shellWord(name) {
+  const s = String(name);
+  const w = /^[A-Za-z0-9._+@%=:,-]+$/.test(s) ? s : quoteShell(s);
+  return s.startsWith('-') ? `./${w}` : w;
+}
+
+/** Section 4's opening: the tables this invite created, by the layout it provisioned. */
+function ownedTables({ db, name, schema }) {
+  if (String(schema) === '3') {
+    const N = SP.profileNames(db, name);
+    return [
+      `Your memory lives in tables named for you. Your conversations, in ${code(db)}, under one grant on ${code(`${db}.${name}_*`)}:`,
+      '',
+      ...SP.CONVERSATION_TYPES.map((t) => `- ${code(N[t])}`),
+      '',
+      `Every transcript byte as captured, in the archive database ${code(N.archive)}, where you may read and add rows (${code('SELECT, INSERT')}) and nothing more:`,
+      '',
+      ...SP.CAPTURE_TYPES.map((t) => `- ${code(N[t])}`),
+    ].join('\n');
+  }
+  const R = roomNames(name).physical;
+  const fq = (t) => code(`${db}.${t}`);
+  return [
+    `Your memory lives in tables named for you inside ${code(db)}, and you hold exactly one grant, on ${code(`${db}.${name}_*`)}:`,
+    '',
+    `- ${ROOM_TYPES.map((t) => fq(R[t])).join(', ')}: your conversations`,
+    `- ${META_TYPES.map((t) => fq(R[t])).join(', ')}: bookkeeping`,
+    `- ${Object.values(statTableNames(name)).map(fq).join(', ')}: rollups the dashboard reads`,
+  ].join('\n');
+}
+
+/** Step 2's command. Bare `memhouse` offers only a file it recognises by name (invitefile.js). */
+function joinStep(file) {
+  const arg = shellWord(file);
+  if (isInviteFileName(file)) {
+    return [
+      `From the directory holding ${code(file)}, just run memhouse — it finds the invitation and asks:`,
+      '', '```', 'memhouse', '```', '',
+      `(or, without the question: ${code(`memhouse install --env ${arg}`)})`,
+    ].join('\n');
+  }
+  return [`From the directory holding ${code(file)}:`, '', '```', `memhouse install --env ${arg}`, '```'].join('\n');
+}
+
+/** `{{KEY}}` → values[KEY], in one pass, so a value is never itself rendered. */
+function render(tpl, values) {
+  return String(tpl).replace(/\{\{([A-Z_]+)\}\}/g, (m, k) => (Object.hasOwn(values, k) ? String(values[k]) : m));
+}
+
+/**
+ * The page for the invite file written at `out`: where it goes (beside that file) and what it
+ * says. `schema` is the layout the invite provisioned ('3', anything else for the 0.18 rooms);
+ * `install` is step 1's block and `version` what `memhouse --version` should print.
+ */
+/** When the invite's password stops working, or '' for an invite with no expiry. */
+function expiryLine(expires, inviter) {
+  return expires ? `**This invitation expires at ${expires} (UTC).** Its password stops working then; install before that, or ask ${inviter} for a new one.\n\n` : '';
+}
+
+function invitationPage({ out, name, url, db, schema, version, install, inviter, template, expires = null }) {
+  const file = path.basename(out);
+  const text = render(template === undefined ? readTemplate() : template, {
+    NAME: name, FILE: file, FILE_ARG: shellWord(file), URL: url, DB: db, VERSION: version,
+    INSTALL: install, INVITER: inviter, JOIN: joinStep(file), OWNED: ownedTables({ db, name, schema }),
+    EXPIRY: expiryLine(expires, inviter),
+  });
+  return { path: path.join(path.dirname(out), PAGE), file, text };
+}
+
+module.exports = { PAGE, TEMPLATE, readTemplate, shellWord, ownedTables, joinStep, expiryLine, render, invitationPage };