hippo-memory 1.69.0 → 1.71.0

package/dist/{compaction-record.js → capture/compaction-record.js}

@@ -1,23 +1,23 @@
 // One record per Claude Code compaction, and what turns its summary into kept memories.
 import * as fs from 'fs';
 import * as path from 'path';
-import { ConflictError } from './api-errors.js';
-import { isObjectLike, isStringValue } from './capture-contract.js';
+import { ConflictError } from '../core/api-errors.js';
+import { isStringValue } from '../core/capture-contract.js';
 import { COMPACTION_ITEM_MAX_CHARS, compactSummaryBody, parseCompactionItems, selectItemRows } from './compaction-items.js';
 import { importSpool, spool } from './compaction-spool.js';
-import { isSharedStore, loadConfig } from './config.js';
-import { closeHippoDb, isSqliteBusy, openHippoDb } from './db.js';
-import { gatedWrite } from './gated-write.js';
-import { COMPACTION_MEMORY_TAG, COMPACTION_SOURCE_PREFIX, Layer, createMemory, generateId } from './memory.js';
-import { fallbackOrigin, isGlobalStoreRoot, originInSql, projectId, projectNames, resolveProjectIdentity } from './project-identity.js';
-import { scopeAdmitSql } from './recall-scope.js';
-import { maskEmails, redactSecretsStrict } from './secret-detect.js';
-import { strengthenRetrievedOn, writeEntryMirrors } from './store/entry-writes.js';
-import { isRecallBoostAblated } from './ablation.js';
-import { updateStats } from './store/index-and-stats.js';
-import { resolveTenantId } from './tenant.js';
-import { errorMessage, log as logger } from './log.js';
-import { readTranscriptTail, truncateCodePointSafe } from './transcript-tail.js';
+import { isSharedStore, loadConfig } from '../core/config.js';
+import { closeHippoDb, isSqliteBusy, openHippoDb, withWriteScopeOr } from '../db/index.js';
+import { gatedWrite } from '../trust/gated-write.js';
+import { COMPACTION_MEMORY_TAG, COMPACTION_SOURCE_PREFIX, Layer, createMemory, generateId } from '../core/memory.js';
+import { fallbackOrigin, isGlobalStoreRoot, projectId, projectNames, resolveProjectIdentity } from '../core/project-identity.js';
+import { maskEmails, redactSecretsStrict } from '../util/secret-detect.js';
+import { closeStartedWithoutSummary, compactionProgress, compactionRowsByRequest, heldMemoryRows, insertStartedCompaction, insertSummarisedCompaction, latestCompactionRows, latestStartedRows, markCompactionDone, markCompactionSummarised, markSnapshotSavedRow, nextCompactionStart, openTranscriptRows, stalledSummarisedRows, } from '../store/compactions.js';
+import { strengthenRetrievedOn, writeEntryMirrors } from '../store/entry-writes.js';
+import { isRecallBoostAblated } from '../core/ablation.js';
+import { updateStats } from '../store/index-and-stats.js';
+import { resolveTenantId } from '../store/tenant.js';
+import { errorMessage, log as logger } from '../util/log.js';
+import { readTranscriptTail, truncateCodePointSafe } from '../util/transcript-tail.js';
 /** PostCompact has 10 s in all (PreCompact 30 s), so a locked store must be given up on early. */
 export const COMPACTION_DB_WAIT_MS = 2000;
 /** Tested verbatim: Claude Code hands PreCompact stdout to the summariser as instructions. */
@@ -81,25 +81,18 @@ function toRecord(row) {
         status: row.status,
     };
 }
-const COLUMNS = 'tenant_id, id, session_id, origin_project, compact_trigger, cwd, transcript_path, snapshot_saved, started_at, summarised_at, summary, items_json, items_written, status';
-function selectRecords(db, where, ...params) {
-    // SAFETY: the SELECT names exactly COLUMNS, matching CompactionRow's field set.
-    const rows = db.prepare(`SELECT ${COLUMNS} FROM compactions WHERE ${where}`).all(...params);
-    return rows.map(toRecord);
-}
 export function startCompaction(db, tenantId, start, at = new Date()) {
     const id = generateId('cmp');
-    db.prepare(`INSERT INTO compactions(tenant_id, id, session_id, origin_project, compact_trigger, cwd, transcript_path, started_at)
-     VALUES (?, ?, ?, ?, ?, ?, ?, ?)`).run(tenantId, id, start.sessionId, start.originProject, start.trigger, start.cwd, start.transcriptPath, at.toISOString());
+    insertStartedCompaction(db, tenantId, { id, ...start, startedAt: at.toISOString() });
     return id;
 }
 /** A session's newest record, or null. */
 export function latestCompaction(db, tenantId, sessionId) {
-    return selectRecords(db, 'tenant_id = ? AND session_id = ? ORDER BY started_at DESC, id DESC LIMIT 1', tenantId, sessionId)[0] ?? null;
+    return latestCompactionRows(db, tenantId, sessionId).map(toRecord)[0] ?? null;
 }
 /** The record a caller's request made, whatever state it reached, so a retry neither writes its items twice nor starts a second record; another session's id is a ConflictError. */
 export function compactionByRequest(db, tenantId, requestId, sessionId) {
-    const record = selectRecords(db, 'tenant_id = ? AND request_id = ?', tenantId, requestId)[0] ?? null;
+    const record = compactionRowsByRequest(db, tenantId, requestId).map(toRecord)[0] ?? null;
     // Sessions are owner-bound, so this also keeps one owner from reading or finishing another's record.
     if (record !== null && record.sessionId !== sessionId)
         throw new ConflictError('request id belongs to another session');
@@ -107,15 +100,11 @@ export function compactionByRequest(db, tenantId, requestId, sessionId) {
 }
 /** Pre-compact's record for the compaction that is ending: the session's newest `started` one within REPLAY_AFTER_MS before `at`, so an older one is left for the transcript fill. */
 function latestStarted(db, tenantId, sessionId, at) {
-    return selectRecords(db, `tenant_id = ? AND session_id = ? AND status = 'started' AND started_at <= ? AND started_at >= ? ORDER BY started_at DESC, id DESC LIMIT 1`, tenantId, sessionId, at.toISOString(), new Date(at.getTime() - REPLAY_AFTER_MS).toISOString())[0] ?? null;
+    return latestStartedRows(db, tenantId, sessionId, at.toISOString(), new Date(at.getTime() - REPLAY_AFTER_MS).toISOString()).map(toRecord)[0] ?? null;
 }
-/** Moves a `started` record to `summarised`; false when another process already moved it. The request id lands in the same statement, so no crash leaves the record unfindable by its retry. */
-function markSummarised(db, tenantId, id, text, summarisedAt, requestId) {
-    // Only a caller names the column, so a store from before it was added still takes local writes.
-    const stamp = requestId === undefined ? [] : [requestId];
-    const result = db.prepare(`UPDATE compactions SET summary = ?, items_json = ?, summarised_at = ?, status = 'summarised'${stamp.length === 0 ? '' : ', request_id = ?'}
-     WHERE tenant_id = ? AND id = ? AND status = 'started'`).run(text.summary, JSON.stringify(text.items), summarisedAt, ...stamp, tenantId, id);
-    return (result.changes ?? 0) > 0;
+function markSummarised(db, tenantId, id, options) {
+    const { text, summarisedAt, requestId } = options;
+    return markCompactionSummarised(db, tenantId, id, { summary: text.summary, itemsJson: JSON.stringify(text.items), summarisedAt, requestId });
 }
 /** Best effort, never throws: a compaction must not fail because its record could not be written. */
 export function recordCompactionStart(hippoRoot, start, log) {
@@ -135,7 +124,7 @@ export function recordCompactionStart(hippoRoot, start, log) {
 }
 /** On the caller's handle and tenant, so a server marks the record under the caller's tenant with its own wait. */
 export function markSnapshotSaved(db, tenantId, recordId) {
-    db.prepare(`UPDATE compactions SET snapshot_saved = 1 WHERE tenant_id = ? AND id = ?`).run(tenantId, recordId);
+    markSnapshotSavedRow(db, tenantId, recordId);
 }
 export function recordSnapshotSaved(hippoRoot, tenantId, recordId, log) {
     let db;
@@ -152,21 +141,18 @@ export function recordSnapshotSaved(hippoRoot, tenantId, recordId, log) {
     }
 }
 /** Puts the summary on the session's `started` record, or inserts a `summarised` one when pre-compact wrote none. One statement each. */
-export function recordSummary(db, hippoRoot, tenantId, meta, text, at, caller = {}) {
+export function recordSummary(db, hippoRoot, tenantId, options) {
+    const { meta, text, at, caller = {} } = options;
     const now = new Date().toISOString();
     const itemsJson = JSON.stringify(text.items);
     const { requestId } = caller;
     const started = latestStarted(db, tenantId, meta.sessionId, at);
-    if (started && markSummarised(db, tenantId, started.id, text, now, requestId)) {
+    if (started && markSummarised(db, tenantId, started.id, { text, summarisedAt: now, requestId })) {
         return { ...started, summary: text.summary, items: text.items, summarisedAt: now, status: 'summarised' };
     }
     const originProject = caller.originProject ?? compactionOrigin(hippoRoot, meta.cwd);
     const id = generateId('cmp');
-    // Only a caller names the column, so a store from before it was added still takes local writes.
-    const stamp = requestId === undefined ? [] : [requestId];
-    const [col, mark] = stamp.length === 0 ? ['', ''] : [', request_id', ', ?'];
-    db.prepare(`INSERT INTO compactions(tenant_id, id, session_id, origin_project, compact_trigger, cwd, transcript_path, started_at, summarised_at, summary, items_json, status${col})
-     VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'summarised'${mark})`).run(tenantId, id, meta.sessionId, originProject, meta.trigger, meta.cwd, meta.transcriptPath, at.toISOString(), now, text.summary, itemsJson, ...stamp);
+    insertSummarisedCompaction(db, tenantId, { ...meta, originProject, startedAt: at.toISOString() }, { id, summary: text.summary, itemsJson, summarisedAt: now, requestId });
     return {
         id, tenantId, sessionId: meta.sessionId, originProject, trigger: meta.trigger, cwd: meta.cwd, transcriptPath: meta.transcriptPath,
         snapshotSaved: false, startedAt: at.toISOString(), summarisedAt: now, summary: text.summary, items: text.items, itemsWritten: 0, status: 'summarised',
@@ -195,11 +181,7 @@ function restates(item, held) {
 }
 /** Live rows of one tenant and origin that default recall shows: a compaction often restates what an earlier one, or the user, already saved. */
 function heldRows(db, tenantId, origins) {
-    const deny = scopeAdmitSql('');
-    // SAFETY: the SELECT names the id, source_session_id and content columns.
-    const rows = db.prepare(`SELECT id, source_session_id, content FROM memories WHERE tenant_id = ? AND ${originInSql(origins)} AND superseded_by IS NULL AND kind != 'raw'
-       AND ${deny.sql}`).all(tenantId, ...origins, ...deny.params);
-    return rows.map((r) => ({ id: r.id, sessionId: r.source_session_id, words: words(r.content) }));
+    return heldMemoryRows(db, tenantId, origins).map((r) => ({ id: r.id, sessionId: r.source_session_id, words: words(r.content) }));
 }
 function compactionEntry(text, ctx, origin, baseHalfLifeDays) {
     return {
@@ -277,55 +259,79 @@ export function saveItems(db, hippoRoot, ctx, log) {
     if (capped > 0)
         log(`capped: ${capped} more kept in the record only`);
     const baseHalfLifeDays = loadConfig(hippoRoot).defaultHalfLifeDays;
-    let writes;
-    db.exec('BEGIN IMMEDIATE');
-    try {
+    let finishedItems = 0;
+    const writes = withWriteScopeOr(db, 'save_items', (rollback) => {
         if (ctx.recordId !== null) {
             // A replayer that read the record before another finished it must not write its items again.
-            const current = db.prepare(`SELECT status, items_written FROM compactions WHERE tenant_id = ? AND id = ?`)
-                .get(ctx.tenantId, ctx.recordId);
+            const current = compactionProgress(db, ctx.tenantId, ctx.recordId);
             if (current?.status !== 'summarised') {
-                db.exec('ROLLBACK');
-                log(`${ctx.recordId} was already finished by another process`);
-                return current?.items_written ?? 0;
+                finishedItems = current?.items_written ?? 0;
+                return rollback(null);
             }
         }
-        writes = writeItemRows(db, hippoRoot, ctx, rows, baseHalfLifeDays);
+        const written = writeItemRows(db, hippoRoot, ctx, rows, baseHalfLifeDays);
         // Said again by another session is the same signal as being recalled; the same session carrying it forward is not.
-        strengthenRetrievedOn(db, writes.restated, { tenantId: ctx.tenantId, recallBoostAblated: isRecallBoostAblated() });
+        strengthenRetrievedOn(db, written.restated, { tenantId: ctx.tenantId, recallBoostAblated: isRecallBoostAblated() });
         if (ctx.recordId !== null) {
-            db.prepare(`UPDATE compactions SET items_written = ?, status = 'done' WHERE tenant_id = ? AND id = ?`).run(writes.written.length, ctx.tenantId, ctx.recordId);
-        }
-        db.exec('COMMIT');
-    }
-    catch (err) {
-        try {
-            db.exec('ROLLBACK');
+            markCompactionDone(db, ctx.tenantId, ctx.recordId, written.written.length);
         }
-        catch { /* already rolled back; keep the original error */ }
-        throw err;
+        return written;
+    });
+    if (writes === null) {
+        log(`${ctx.recordId} was already finished by another process`);
+        return finishedItems;
     }
     finishItemWrites(hippoRoot, writes, log);
     return writes.written.length;
 }
-/** null when the text is not a PostCompact payload naming a session. */
-export function parsePostCompactPayload(stdinText) {
-    let raw;
+/** One failure, told once to the hook's output and once to the structured log. */
+function reportFailure(log, step, reason) {
+    log(`${step} failed: ${reason}`);
+    logger.error(`post-compact: ${step} failed: ${reason}`);
+}
+function recordStep(step) {
     try {
-        raw = JSON.parse((stdinText ?? '').trim());
-    }
-    catch {
-        return null; // non-JSON stdin is not a PostCompact payload; the caller skips it
+        const record = recordSummary(step.db, step.hippoRoot, step.tenantId, { meta: step.payload, text: step.text, at: step.at });
+        step.result.snapshotSaved = record.snapshotSaved;
+        return record;
     }
-    if (!isObjectLike(raw) || !('session_id' in raw) || !isStringValue(raw.session_id) || raw.session_id === '')
+    catch (err) {
+        if (isSqliteBusy(err))
+            throw err;
+        reportFailure(step.log, 'record step', errorMessage(err));
         return null;
-    return {
-        sessionId: raw.session_id,
-        trigger: 'trigger' in raw && isStringValue(raw.trigger) ? raw.trigger : null,
-        cwd: 'cwd' in raw && isStringValue(raw.cwd) ? raw.cwd : null,
-        transcriptPath: 'transcript_path' in raw && isStringValue(raw.transcript_path) ? raw.transcript_path : null,
-        compactSummary: 'compact_summary' in raw && isStringValue(raw.compact_summary) ? raw.compact_summary : null,
-    };
+    }
+}
+/** Busy with a saved record defers to sleep; any other failure goes to the caller. */
+function itemsStep(step, record) {
+    const { payload, hippoRoot, log } = step;
+    try {
+        step.result.written = saveItems(step.db, hippoRoot, {
+            tenantId: step.tenantId,
+            recordId: record?.id ?? null,
+            sessionId: payload.sessionId,
+            originProject: record?.originProject ?? compactionOrigin(hippoRoot, payload.cwd),
+            cwd: payload.cwd,
+            items: step.text.items,
+        }, log);
+    }
+    catch (err) {
+        if (!isSqliteBusy(err) || record === null)
+            throw err;
+        // The record already holds the items; sleep finishes them.
+        step.result.deferred = true;
+        log(`store busy, items left for the next sleep: ${errorMessage(err)}`);
+    }
+}
+function spoolSummary(hippoRoot, payload, text, at, reason, result, log) {
+    try {
+        spool(hippoRoot, resolveTenantId({}), payload, text, at);
+        result.deferred = true;
+        log(`store busy, summary spooled: ${reason}`);
+    }
+    catch (spoolErr) {
+        reportFailure(log, 'spool', errorMessage(spoolErr));
+    }
 }
 /** The PostCompact work: record the summary, then write its items. Each step is independent; a busy store spools or defers. Never throws. */
 export function saveCompaction(hippoRoot, payload, log) {
@@ -341,52 +347,14 @@ export function saveCompaction(hippoRoot, payload, log) {
     let db;
     try {
         db = openHippoDb(hippoRoot, { busyWaitMs: COMPACTION_DB_WAIT_MS });
-        const tenantId = resolveTenantId({});
-        let record = null;
-        try {
-            record = recordSummary(db, hippoRoot, tenantId, payload, text, at);
-            result.snapshotSaved = record.snapshotSaved;
-        }
-        catch (err) {
-            if (isSqliteBusy(err))
-                throw err;
-            log(`record step failed: ${errorMessage(err)}`);
-            logger.error(`post-compact: record step failed: ${errorMessage(err)}`);
-        }
-        try {
-            result.written = saveItems(db, hippoRoot, {
-                tenantId,
-                recordId: record?.id ?? null,
-                sessionId: payload.sessionId,
-                originProject: record?.originProject ?? compactionOrigin(hippoRoot, payload.cwd),
-                cwd: payload.cwd,
-                items: text.items,
-            }, log);
-        }
-        catch (err) {
-            if (!isSqliteBusy(err) || record === null)
-                throw err;
-            // The record already holds the items; sleep finishes them.
-            result.deferred = true;
-            log(`store busy, items left for the next sleep: ${errorMessage(err)}`);
-        }
+        const step = { db, hippoRoot, tenantId: resolveTenantId({}), payload, text, at, result, log };
+        itemsStep(step, recordStep(step));
     }
     catch (err) {
-        if (isSqliteBusy(err) || db === undefined) {
-            try {
-                spool(hippoRoot, resolveTenantId({}), payload, text, at);
-                result.deferred = true;
-                log(`store busy, summary spooled: ${errorMessage(err)}`);
-            }
-            catch (spoolErr) {
-                log(`spool failed: ${errorMessage(spoolErr)}`);
-                logger.error(`post-compact: spool failed: ${errorMessage(spoolErr)}`);
-            }
-        }
-        else {
-            log(`items step failed: ${errorMessage(err)}`);
-            logger.error(`post-compact: items step failed: ${errorMessage(err)}`);
-        }
+        if (isSqliteBusy(err) || db === undefined)
+            spoolSummary(hippoRoot, payload, text, at, errorMessage(err), result, log);
+        else
+            reportFailure(log, 'items step', errorMessage(err));
     }
     finally {
         if (db)
@@ -437,70 +405,80 @@ function transcriptSummary(transcriptPath, afterMs, beforeMs) {
     return null;
 }
 function closeWithoutSummary(db, tenantId, id, log) {
-    const result = db.prepare(`UPDATE compactions SET status = 'no-summary' WHERE tenant_id = ? AND id = ? AND status = 'started'`).run(tenantId, id);
-    if ((result.changes ?? 0) > 0)
+    if (closeStartedWithoutSummary(db, tenantId, id) > 0)
         log(`${id} closed as no-summary: its transcript holds no summary for it`);
 }
 function nextStartedAt(db, record) {
-    const row = db.prepare(`SELECT MIN(started_at) AS at FROM compactions WHERE tenant_id = ? AND session_id = ? AND started_at > ?`)
-        .get(record.tenantId, record.sessionId, record.startedAt);
-    return row?.at ?? null;
+    return nextCompactionStart(db, record.tenantId, record.sessionId, record.startedAt);
+}
+/** The one place a record becomes the argument saveItems takes. */
+function itemContext(tenantId, record, items = record.items) {
+    return { tenantId, recordId: record.id, sessionId: record.sessionId, originProject: record.originProject, cwd: record.cwd, items };
 }
 /** Records a spooled summary, then writes its items. */
 function spoolImporter(db, hippoRoot, log) {
     return (spooled, recorded) => {
-        const record = recordSummary(db, hippoRoot, spooled.tenantId, spooled.payload, spooled.text, spooled.at);
+        const record = recordSummary(db, hippoRoot, spooled.tenantId, { meta: spooled.payload, text: spooled.text, at: spooled.at });
         // The record holds the items now, so the file is done even if the write below fails.
         recorded();
-        saveItems(db, hippoRoot, { tenantId: spooled.tenantId, recordId: record.id, sessionId: record.sessionId, originProject: record.originProject, cwd: record.cwd, items: record.items }, log);
+        saveItems(db, hippoRoot, itemContext(spooled.tenantId, record), log);
     };
 }
-/** Finishes what a killed hook or a busy store left: `summarised` records, spool files, and `started` records the transcript can fill. Returns how many compactions it saved. */
-export function replayCompactions(db, hippoRoot, log, deadline = Number.POSITIVE_INFINITY) {
-    const tenantId = resolveTenantId({});
-    const now = Date.now();
+function replayStalled(db, hippoRoot, tenantId, now, log, deadline) {
     let finished = 0;
-    const stalled = selectRecords(db, `tenant_id = ? AND status = 'summarised' AND summarised_at < ?`, tenantId, new Date(now - REPLAY_AFTER_MS).toISOString());
+    const stalled = stalledSummarisedRows(db, tenantId, new Date(now - REPLAY_AFTER_MS).toISOString()).map(toRecord);
     for (const record of stalled) {
         if (Date.now() > deadline)
             break;
         try {
-            saveItems(db, hippoRoot, { tenantId, recordId: record.id, sessionId: record.sessionId, originProject: record.originProject, cwd: record.cwd, items: record.items }, log);
+            saveItems(db, hippoRoot, itemContext(tenantId, record), log);
             finished++;
         }
         catch (err) {
             log(`replay of ${record.id} failed: ${errorMessage(err)}`);
         }
     }
+    return finished;
+}
+function replaySpool(db, hippoRoot, tenantId, log, deadline) {
     try {
-        finished += importSpool(hippoRoot, tenantId, log, deadline, spoolImporter(db, hippoRoot, log));
+        return importSpool(hippoRoot, tenantId, log, deadline, spoolImporter(db, hippoRoot, log));
     }
     catch (err) {
         log(`spool import failed: ${errorMessage(err)}`);
+        return 0;
     }
-    const open = selectRecords(db, `tenant_id = ? AND status = 'started' AND started_at < ? AND started_at > ? AND transcript_path IS NOT NULL ORDER BY started_at`, tenantId, new Date(now - REPLAY_AFTER_MS).toISOString(), new Date(now - TRANSCRIPT_FILL_WINDOW_MS).toISOString());
+}
+/** Fills one open record from its transcript; true when it saved items. */
+function fillFromTranscript(db, hippoRoot, tenantId, record, log) {
+    if (record.transcriptPath === null || !fs.existsSync(record.transcriptPath))
+        return false;
+    const next = nextStartedAt(db, record);
+    const found = transcriptSummary(record.transcriptPath, Date.parse(record.startedAt), next === null ? Number.POSITIVE_INFINITY : Date.parse(next));
+    if (found === null) {
+        // The record is past REPLAY_AFTER_MS, so a window that is fully scanned or out of reach stays empty.
+        closeWithoutSummary(db, tenantId, record.id, log);
+        return false;
+    }
+    const { found: listed, ...text } = readCompactionText(found);
+    if (!listed)
+        log(`no memories section in the transcript summary for ${record.id}`);
+    if (!markSummarised(db, tenantId, record.id, { text, summarisedAt: new Date().toISOString() })) {
+        log(`${record.id} was filled by another process`);
+        return false;
+    }
+    saveItems(db, hippoRoot, itemContext(tenantId, record, text.items), log);
+    return true;
+}
+function replayOpen(db, hippoRoot, tenantId, now, log, deadline) {
+    let finished = 0;
+    const open = openTranscriptRows(db, tenantId, new Date(now - REPLAY_AFTER_MS).toISOString(), new Date(now - TRANSCRIPT_FILL_WINDOW_MS).toISOString()).map(toRecord);
     for (const record of open) {
         if (Date.now() > deadline)
             break;
         try {
-            if (record.transcriptPath === null || !fs.existsSync(record.transcriptPath))
-                continue;
-            const next = nextStartedAt(db, record);
-            const found = transcriptSummary(record.transcriptPath, Date.parse(record.startedAt), next === null ? Number.POSITIVE_INFINITY : Date.parse(next));
-            if (found === null) {
-                // The record is past REPLAY_AFTER_MS, so a window that is fully scanned or out of reach stays empty.
-                closeWithoutSummary(db, tenantId, record.id, log);
-                continue;
-            }
-            const { found: listed, ...text } = readCompactionText(found);
-            if (!listed)
-                log(`no memories section in the transcript summary for ${record.id}`);
-            if (!markSummarised(db, tenantId, record.id, text, new Date().toISOString())) {
-                log(`${record.id} was filled by another process`);
-                continue;
-            }
-            saveItems(db, hippoRoot, { tenantId, recordId: record.id, sessionId: record.sessionId, originProject: record.originProject, cwd: record.cwd, items: text.items }, log);
-            finished++;
+            if (fillFromTranscript(db, hippoRoot, tenantId, record, log))
+                finished++;
         }
         catch (err) {
             log(`transcript fill of ${record.id} failed: ${errorMessage(err)}`);
@@ -508,6 +486,14 @@ export function replayCompactions(db, hippoRoot, log, deadline = Number.POSITIVE
     }
     return finished;
 }
+/** Finishes what a killed hook or a busy store left: `summarised` records, spool files, and `started` records the transcript can fill. Returns how many compactions it saved. */
+export function replayCompactions(db, hippoRoot, log, deadline = Number.POSITIVE_INFINITY) {
+    const tenantId = resolveTenantId({});
+    const now = Date.now();
+    return replayStalled(db, hippoRoot, tenantId, now, log, deadline)
+        + replaySpool(db, hippoRoot, tenantId, log, deadline)
+        + replayOpen(db, hippoRoot, tenantId, now, log, deadline);
+}
 /** For `hippo sleep` and post-compact: opens the store itself and never throws. */
 export function replayCompactionsAt(hippoRoot, log, opts = {}) {
     let db;

package/dist/{compaction-spool.d.ts → capture/compaction-spool.d.ts}

@@ -21,7 +21,7 @@ export interface SpoolFs {
     }): void;
 }
 /** Test-only seam, so a test can make one fs call fail; null restores the real fs. */
-export declare function __setSpoolFs(next: SpoolFs | null): void;
+export declare function _setSpoolFsForTests(next: SpoolFs | null): void;
 export interface SpooledCompaction {
     tenantId: string;
     payload: PostCompactPayload;

package/dist/{compaction-spool.js → capture/compaction-spool.js}

@@ -2,9 +2,10 @@
 import { randomBytes } from 'crypto';
 import * as fs from 'fs';
 import * as path from 'path';
-import { isObjectLike, isStringValue } from './capture-contract.js';
-import { isSqliteBusy } from './db/busy.js';
-import { errorMessage } from './log.js';
+import { isObjectLike, isStringValue } from '../core/capture-contract.js';
+import { isSqliteBusy } from '../db/busy.js';
+import { errorMessage } from '../util/log.js';
+const STAMP_DIGITS = 13;
 export const SPOOL_DIR = 'compactions-spool';
 /** Starts the log line of a step that broke with an unexpected error, so the session-start banner counts it; a set-aside file is counted by its `.bad` name instead. */
 export const SPOOL_PROBLEM = 'spool problem: ';
@@ -28,10 +29,10 @@ const NEW_STEM = /^(\d{13})-[0-9a-f]{8}$/;
 const LEGACY_STEM_TIME = /-(\d+)$/;
 let fsx = fs;
 /** Test-only seam, so a test can make one fs call fail; null restores the real fs. */
-export function __setSpoolFs(next) {
+export function _setSpoolFsForTests(next) {
     fsx = next ?? fs;
 }
-const stamp = (ms) => String(Math.trunc(ms)).padStart(13, '0');
+const stamp = (ms) => String(Math.trunc(ms)).padStart(STAMP_DIGITS, '0');
 const isTime = (value) => typeof value === 'number' && Number.isFinite(value);
 const idle = new Int32Array(new SharedArrayBuffer(4));
 function errCode(cause) {
@@ -244,7 +245,7 @@ function recoverStaleClaims(dir, names, log) {
                 putBackOne = true;
             }
         }
-        else if (moveBad(dir, file, claim.stem, claim.attempt, 'interrupted', log)) {
+        else if (moveBad(dir, file, { stem: claim.stem, attempt: claim.attempt, cause: 'interrupted', log })) {
             log(`spool file ${claim.name} was claimed by a replayer that never finished ${MAX_ATTEMPTS} times, set aside as .bad`);
         }
     }
@@ -258,7 +259,8 @@ function putBack(dir, claim, stem, attempt, log) {
     }
 }
 /** Writes `<stem>.<cause>.bad` through a random temp file, so the rename replaces any torn `.bad` a crash left. */
-function writeBad(dir, stem, cause, text, log, what) {
+function writeBad(dir, options) {
+    const { stem, cause, text, log, what } = options;
     const tmp = path.join(dir, `${stem}.${cause}.bad.${randomBytes(4).toString('hex')}.tmp`);
     const written = settle(() => fsx.writeFileSync(tmp, text, { encoding: 'utf8', flag: 'wx' }), log, what);
     const moved = written === 'done' ? settle(() => fsx.renameSync(tmp, path.join(dir, `${stem}.${cause}.bad`)), log, what) : written;
@@ -267,12 +269,13 @@ function writeBad(dir, stem, cause, text, log, what) {
     return moved === 'done';
 }
 /** Sets a claim aside as `.bad` before the claim goes, so a crash between leaves a whole copy; a busy write puts the claim back. */
-function moveBad(dir, claim, stem, attempt, cause, log, raw) {
+function moveBad(dir, claim, options) {
+    const { stem, attempt, cause, log, raw } = options;
     const what = `spool file ${path.basename(claim)} not set aside`;
     let text = raw ?? '';
     if (raw === undefined && settle(() => { text = fsx.readFileSync(claim, 'utf8'); }, log, what) !== 'done')
         return false;
-    if (!writeBad(dir, stem, cause, text, log, what)) {
+    if (!writeBad(dir, { stem, cause, text, log, what })) {
         putBack(dir, claim, stem, attempt, log);
         return false;
     }
@@ -306,7 +309,7 @@ function promoteTmp(dir, names, tenantId, log) {
                 promoted = true;
             }
         }
-        else if (writeBad(dir, m[1], 'unreadable', text, log, what)) {
+        else if (writeBad(dir, { stem: m[1], cause: 'unreadable', text, log, what })) {
             settle(() => fsx.unlinkSync(file), log, what);
             log(`spool file ${name} was never finished, set aside as .bad`);
         }
@@ -320,7 +323,8 @@ function removeClaim(claim, name, log) {
         log(`spool file ${name} saved; its claim could not be removed (${removed.code}); it will be imported again`);
 }
 /** After an importer throw: a busy store stops the run with the count unchanged, any other error counts one try toward .bad. */
-function importFailed(dir, claim, entry, text, recorded, cause, log) {
+function importFailed(dir, claim, entry, options) {
+    const { text, recorded, cause, log } = options;
     const { name } = entry;
     const busy = isSqliteBusy(cause);
     if (recorded) {
@@ -338,7 +342,7 @@ function importFailed(dir, claim, entry, text, recorded, cause, log) {
         putBack(dir, claim, entry.stem, next, log);
         log(`spool file ${name} failed to import (try ${next} of ${MAX_ATTEMPTS}): ${errorMessage(cause)}`);
     }
-    else if (moveBad(dir, claim, entry.stem, entry.attempt, 'failed', log, text)) {
+    else if (moveBad(dir, claim, { stem: entry.stem, attempt: entry.attempt, cause: 'failed', log, raw: text })) {
         log(`spool file ${name} set aside as .bad after ${MAX_ATTEMPTS} tries: ${errorMessage(cause)}`);
     }
     return 'skipped';
@@ -362,7 +366,7 @@ function importFile(dir, entry, tenantId, log, importOne) {
         return 'skipped';
     const spooled = parseSpooled(text, tenantId);
     if (spooled === null) {
-        if (moveBad(dir, claim, entry.stem, entry.attempt, 'unreadable', log, text))
+        if (moveBad(dir, claim, { stem: entry.stem, attempt: entry.attempt, cause: 'unreadable', log, raw: text }))
             log(`spool file ${name} is not readable, set aside as .bad`);
         return 'skipped';
     }
@@ -379,7 +383,7 @@ function importFile(dir, entry, tenantId, log, importOne) {
         return 'imported';
     }
     catch (err) {
-        return importFailed(dir, claim, entry, text, recorded, err, log);
+        return importFailed(dir, claim, entry, { text, recorded, cause: err, log });
     }
 }
 function mtimeOf(file) {

package/dist/capture/copilot-transcript.js

@@ -3,7 +3,7 @@
 import * as fs from 'fs';
 import * as path from 'path';
 import { copilotHomeDir } from '../hooks/shared.js';
-import { isObjectLike, isStringValue } from '../capture-contract.js';
+import { isObjectLike, isStringValue } from '../core/capture-contract.js';
 // A session id becomes a folder name, so anything beyond word characters and hyphens could climb out of session-state.
 export const SESSION_ID_RE = /^[\w-]+$/;
 // user.message sources that mark text hippo did not get from the human: a hidden skill injection or another agent's prompt.

package/dist/capture/extract.js

@@ -1,5 +1,5 @@
-import { duplicateKey } from '../same-text.js';
-import { assessAutomaticMemory } from '../memory-quality.js';
+import { duplicateKey } from '../util/same-text.js';
+import { assessAutomaticMemory } from '../core/memory-quality.js';
 const DECISION_PATTERNS = [
     /\b(?:we(?:'ve| have)?|i(?:'ve| have)?|let's)\s+decid(?:ed|e)\b/i,
     /\b(?:let's|we(?:'ll| will| should)?)\s+(?:go with|do|use|try|build|implement|switch to)\b/i,

package/dist/capture/failure-caller.d.ts

@@ -1,6 +1,6 @@
 import type { Context } from '../api/types.js';
-import type { CallerProject } from '../prompt-hook.js';
-import { type CaptureErrorOutcome, type RoutineRule } from './failure-reading.js';
+import { type CaptureErrorOutcome, type RoutineRule } from '../store/failure-log.js';
+import type { CallerProject } from '../api/prompt-hook.js';
 type FailureSkip = Exclude<CaptureErrorOutcome, 'stored' | 'duplicate'>;
 /** failureReport's reading of the payload, read on the caller's machine, plus the id its retries carry. */
 export interface CallerFailureRequest {

package/dist/capture/failure-caller.js

@@ -1,10 +1,10 @@
 // PostToolUseFailure for a caller on another machine: the lesson and its log row carry the caller's owner and project, and a retried request id gets the first answer.
-import { BadRequestError } from '../api-errors.js';
-import { storeLesson } from '../capture-error.js';
-import { recordFailure, requestOutcome, settleFailureOutcome } from '../failure-log.js';
-import { errorMessage, log } from '../log.js';
-import { scrubForSharing } from '../share-scrub.js';
-import { truncateCodePointSafe } from '../transcript-tail.js';
+import { BadRequestError } from '../core/api-errors.js';
+import { storeLesson } from './capture-error.js';
+import { recordFailure, requestOutcome, settleFailureOutcome } from '../store/failure-log.js';
+import { errorMessage, log } from '../util/log.js';
+import { scrubForSharing } from './share-scrub.js';
+import { truncateCodePointSafe } from '../util/transcript-tail.js';
 import { assertRequestId, bindCaller, withCallerDb } from './caller-session.js';
 import { FAILURE_TEXT_MAX_CHARS, failureHash } from './failure-reading.js';
 const SKIPS = { 'skipped-interrupt': true, 'skipped-routine': true, 'skipped-invalid': true };

package/dist/capture/failure-reading.d.ts

@@ -1,8 +1,5 @@
-import type { JsonValue } from '../json.js';
-/** Why a failure was not stored, or `stored`. */
-export type CaptureErrorOutcome = 'stored' | 'duplicate' | 'skipped-interrupt' | 'skipped-routine' | 'skipped-invalid';
-/** Which routine check skipped a failure; the log keeps it so declines can be told apart from empty searches. */
-export type RoutineRule = 'declined' | 'os-permission' | 'no-match' | 'search-tool' | 'quiet-exit';
+import { type JsonValue } from '../util/json.js';
+import type { CaptureErrorOutcome, RoutineRule } from '../store/failure-log.js';
 /** What {@link lessonFromFailure} read from a payload; `detail` is the finer failure-log key (untruncated, command head). */
 export type FailureReading = {
     text: string;