hippo-memory 1.63.1 → 1.64.0

package/dist/compaction-record.js

@@ -1,13 +1,16 @@
 // One record per Claude Code compaction, and what turns its summary into kept memories.
 import * as fs from 'fs';
 import * as path from 'path';
+import { ConflictError } from './api-errors.js';
 import { isObjectLike, isStringValue } from './capture-contract.js';
-import { compactSummaryBody, parseCompactionItems, selectItemRows } from './compaction-items.js';
-import { loadConfig } from './config.js';
+import { COMPACTION_ITEM_MAX_CHARS, compactSummaryBody, parseCompactionItems, selectItemRows } from './compaction-items.js';
+import { importSpool, spool } from './compaction-spool.js';
+import { isSharedStore, loadConfig } from './config.js';
 import { closeHippoDb, isSqliteBusy, openHippoDb } from './db.js';
 import { gatedWrite } from './gated-write.js';
 import { COMPACTION_MEMORY_TAG, COMPACTION_SOURCE_PREFIX, Layer, createMemory, generateId } from './memory.js';
-import { isGlobalStoreRoot, originInSql, projectId, projectNames, resolveProjectIdentity } from './project-identity.js';
+import { fallbackOrigin, isGlobalStoreRoot, originInSql, projectId, projectNames, resolveProjectIdentity } from './project-identity.js';
+import { scopeAdmitSql } from './recall-scope.js';
 import { maskEmails, redactSecretsStrict } from './secret-detect.js';
 import { strengthenRetrievedOn, writeEntryMirrors } from './store/entry-writes.js';
 import { isRecallBoostAblated } from './ablation.js';
@@ -25,17 +28,19 @@ export const REPLAY_AFTER_MS = 10 * 60_000;
 /** Claude Code deletes transcripts after 30 days, so an older gap can never be filled. */
 export const TRANSCRIPT_FILL_WINDOW_MS = 30 * 24 * 60 * 60_000;
 const TRANSCRIPT_TAIL_CAPS = [1 << 20, 8 << 20, 64 << 20];
-const SPOOL_DIR = 'compactions-spool';
-const CLAIMED_SUFFIX = '.claimed';
 /** The marker redactSecretsStrict writes; an item holding it was a secret before it was stored. */
 const REDACTED = '[REDACTED]';
-/** Where the session ran: rows written through the global store keep the project the session was in. */
+/** Where the session ran: rows written through the global store or a shared store keep the project the session was in. */
 function compactionProject(hippoRoot, cwd) {
-    if (!isGlobalStoreRoot(hippoRoot))
+    if (!isGlobalStoreRoot(hippoRoot) && !isSharedStore(hippoRoot))
         return resolveProjectIdentity(path.dirname(hippoRoot));
     // No cwd means user-global, as stampOriginProject gives the global store; undefined would fall back to the hook's own cwd.
     return cwd === null ? '' : resolveProjectIdentity(cwd);
 }
+/** The record column cannot hold NULL, but an item from an unknown folder on a shared store must not read as user-global. */
+function itemOrigin(hippoRoot, ctx) {
+    return ctx.cwd === null && isSharedStore(hippoRoot) ? fallbackOrigin(hippoRoot) : ctx.originProject;
+}
 function compactionOrigin(hippoRoot, cwd) {
     return projectId(compactionProject(hippoRoot, cwd));
 }
@@ -53,6 +58,10 @@ export function readCompactionText(compactSummary) {
     const parsed = parseCompactionItems(body);
     return { summary: truncateCodePointSafe(scrub(body), SUMMARY_MAX_CHARS), items: parsed.items.map(scrub), found: parsed.found };
 }
+/** A caller's items scrubbed as readCompactionText scrubs a summary's, since another machine's scrub is not trusted; cut after it, as a mask can run longer than what it hides. */
+export function scrubCompactionItems(items) {
+    return items.map((item) => truncateCodePointSafe(scrub(item), COMPACTION_ITEM_MAX_CHARS));
+}
 function toRecord(row) {
     const listed = row.items_json === null ? [] : JSON.parse(row.items_json);
     return {
@@ -88,13 +97,24 @@ export function startCompaction(db, tenantId, start, at = new Date()) {
 export function latestCompaction(db, tenantId, sessionId) {
     return selectRecords(db, 'tenant_id = ? AND session_id = ? ORDER BY started_at DESC, id DESC LIMIT 1', tenantId, sessionId)[0] ?? null;
 }
+/** The record a caller's request made, whatever state it reached, so a retry neither writes its items twice nor starts a second record; another session's id is a ConflictError. */
+export function compactionByRequest(db, tenantId, requestId, sessionId) {
+    const record = selectRecords(db, 'tenant_id = ? AND request_id = ?', tenantId, requestId)[0] ?? null;
+    // Sessions are owner-bound, so this also keeps one owner from reading or finishing another's record.
+    if (record !== null && record.sessionId !== sessionId)
+        throw new ConflictError('request id belongs to another session');
+    return record;
+}
 /** Pre-compact's record for the compaction that is ending: the session's newest `started` one within REPLAY_AFTER_MS before `at`, so an older one is left for the transcript fill. */
 function latestStarted(db, tenantId, sessionId, at) {
     return selectRecords(db, `tenant_id = ? AND session_id = ? AND status = 'started' AND started_at <= ? AND started_at >= ? ORDER BY started_at DESC, id DESC LIMIT 1`, tenantId, sessionId, at.toISOString(), new Date(at.getTime() - REPLAY_AFTER_MS).toISOString())[0] ?? null;
 }
-/** Moves a `started` record to `summarised`; false when another process already moved it. */
-function markSummarised(db, tenantId, id, text, summarisedAt) {
-    const result = db.prepare(`UPDATE compactions SET summary = ?, items_json = ?, summarised_at = ?, status = 'summarised' WHERE tenant_id = ? AND id = ? AND status = 'started'`).run(text.summary, JSON.stringify(text.items), summarisedAt, tenantId, id);
+/** Moves a `started` record to `summarised`; false when another process already moved it. The request id lands in the same statement, so no crash leaves the record unfindable by its retry. */
+function markSummarised(db, tenantId, id, text, summarisedAt, requestId) {
+    // Only a caller names the column, so a store from before it was added still takes local writes.
+    const stamp = requestId === undefined ? [] : [requestId];
+    const result = db.prepare(`UPDATE compactions SET summary = ?, items_json = ?, summarised_at = ?, status = 'summarised'${stamp.length === 0 ? '' : ', request_id = ?'}
+     WHERE tenant_id = ? AND id = ? AND status = 'started'`).run(text.summary, JSON.stringify(text.items), summarisedAt, ...stamp, tenantId, id);
     return (result.changes ?? 0) > 0;
 }
 /** Best effort, never throws: a compaction must not fail because its record could not be written. */
@@ -113,11 +133,15 @@ export function recordCompactionStart(hippoRoot, start, log) {
             closeHippoDb(db);
     }
 }
-export function recordSnapshotSaved(hippoRoot, recordId, log) {
+/** On the caller's handle and tenant, so a server marks the record under the caller's tenant with its own wait. */
+export function markSnapshotSaved(db, tenantId, recordId) {
+    db.prepare(`UPDATE compactions SET snapshot_saved = 1 WHERE tenant_id = ? AND id = ?`).run(tenantId, recordId);
+}
+export function recordSnapshotSaved(hippoRoot, tenantId, recordId, log) {
     let db;
     try {
         db = openHippoDb(hippoRoot, { busyWaitMs: COMPACTION_DB_WAIT_MS });
-        db.prepare(`UPDATE compactions SET snapshot_saved = 1 WHERE tenant_id = ? AND id = ?`).run(resolveTenantId({}), recordId);
+        markSnapshotSaved(db, tenantId, recordId);
     }
     catch (err) {
         log(`compaction record not marked with its snapshot: ${errorMessage(err)}`);
@@ -128,17 +152,21 @@ export function recordSnapshotSaved(hippoRoot, recordId, log) {
     }
 }
 /** Puts the summary on the session's `started` record, or inserts a `summarised` one when pre-compact wrote none. One statement each. */
-function recordSummary(db, hippoRoot, tenantId, meta, text, at) {
+export function recordSummary(db, hippoRoot, tenantId, meta, text, at, caller = {}) {
     const now = new Date().toISOString();
     const itemsJson = JSON.stringify(text.items);
+    const { requestId } = caller;
     const started = latestStarted(db, tenantId, meta.sessionId, at);
-    if (started && markSummarised(db, tenantId, started.id, text, now)) {
+    if (started && markSummarised(db, tenantId, started.id, text, now, requestId)) {
         return { ...started, summary: text.summary, items: text.items, summarisedAt: now, status: 'summarised' };
     }
-    const originProject = compactionOrigin(hippoRoot, meta.cwd);
+    const originProject = caller.originProject ?? compactionOrigin(hippoRoot, meta.cwd);
     const id = generateId('cmp');
-    db.prepare(`INSERT INTO compactions(tenant_id, id, session_id, origin_project, compact_trigger, cwd, transcript_path, started_at, summarised_at, summary, items_json, status)
-     VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'summarised')`).run(tenantId, id, meta.sessionId, originProject, meta.trigger, meta.cwd, meta.transcriptPath, at.toISOString(), now, text.summary, itemsJson);
+    // Only a caller names the column, so a store from before it was added still takes local writes.
+    const stamp = requestId === undefined ? [] : [requestId];
+    const [col, mark] = stamp.length === 0 ? ['', ''] : [', request_id', ', ?'];
+    db.prepare(`INSERT INTO compactions(tenant_id, id, session_id, origin_project, compact_trigger, cwd, transcript_path, started_at, summarised_at, summary, items_json, status${col})
+     VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'summarised'${mark})`).run(tenantId, id, meta.sessionId, originProject, meta.trigger, meta.cwd, meta.transcriptPath, at.toISOString(), now, text.summary, itemsJson, ...stamp);
     return {
         id, tenantId, sessionId: meta.sessionId, originProject, trigger: meta.trigger, cwd: meta.cwd, transcriptPath: meta.transcriptPath,
         snapshotSaved: false, startedAt: at.toISOString(), summarisedAt: now, summary: text.summary, items: text.items, itemsWritten: 0, status: 'summarised',
@@ -167,12 +195,13 @@ function restates(item, held) {
 }
 /** Live rows of one tenant and origin that default recall shows: a compaction often restates what an earlier one, or the user, already saved. */
 function heldRows(db, tenantId, origins) {
+    const deny = scopeAdmitSql('');
     // SAFETY: the SELECT names the id, source_session_id and content columns.
     const rows = db.prepare(`SELECT id, source_session_id, content FROM memories WHERE tenant_id = ? AND ${originInSql(origins)} AND superseded_by IS NULL AND kind != 'raw'
-       AND (scope IS NULL OR (scope != 'unknown:legacy' AND scope NOT LIKE '%:private:%'))`).all(tenantId, ...origins);
+       AND ${deny.sql}`).all(tenantId, ...origins, ...deny.params);
     return rows.map((r) => ({ id: r.id, sessionId: r.source_session_id, words: words(r.content) }));
 }
-function compactionEntry(text, ctx, baseHalfLifeDays) {
+function compactionEntry(text, ctx, origin, baseHalfLifeDays) {
     return {
         ...createMemory(text, {
             layer: Layer.Episodic,
@@ -184,13 +213,14 @@ function compactionEntry(text, ctx, baseHalfLifeDays) {
             tenantId: ctx.tenantId,
             baseHalfLifeDays,
         }),
-        origin_project: ctx.originProject,
+        origin_project: origin,
     };
 }
 /** Runs inside saveItems' transaction: restatements of held rows are counted, the rest go through the write gate. */
 function writeItemRows(db, hippoRoot, ctx, rows, baseHalfLifeDays) {
     const out = { written: [], repeats: 0, refused: 0, restated: [] };
-    const held = heldRows(db, ctx.tenantId, heldOrigins(hippoRoot, ctx.cwd, ctx.originProject));
+    const held = heldRows(db, ctx.tenantId, ctx.caller?.origins ?? heldOrigins(hippoRoot, ctx.cwd, ctx.originProject));
+    const origin = ctx.caller === undefined ? itemOrigin(hippoRoot, ctx) : ctx.originProject;
     const restated = new Set();
     for (const text of rows) {
         const itemWords = words(text);
@@ -206,8 +236,8 @@ function writeItemRows(db, hippoRoot, ctx, rows, baseHalfLifeDays) {
             out.refused++;
             continue;
         }
-        const entry = compactionEntry(text, ctx, baseHalfLifeDays);
-        if (gatedWrite(db, hippoRoot, entry, { actor: 'post-compact' }) === 'written') {
+        const entry = compactionEntry(text, ctx, origin, baseHalfLifeDays);
+        if (gatedWrite(db, hippoRoot, entry, { actor: ctx.caller?.actor ?? 'post-compact' }) === 'written') {
             out.written.push(entry);
             held.push({ id: null, sessionId: ctx.sessionId, words: itemWords });
         }
@@ -297,17 +327,6 @@ export function parsePostCompactPayload(stdinText) {
         compactSummary: 'compact_summary' in raw && isStringValue(raw.compact_summary) ? raw.compact_summary : null,
     };
 }
-function spoolFile(hippoRoot, sessionId) {
-    return path.join(hippoRoot, SPOOL_DIR, `${sessionId.replace(/[^\w-]/g, '_').slice(0, 80)}-${Date.now()}.json`);
-}
-function spool(hippoRoot, tenantId, payload, text, at) {
-    const file = spoolFile(hippoRoot, payload.sessionId);
-    fs.mkdirSync(path.dirname(file), { recursive: true });
-    const body = { tenantId, sessionId: payload.sessionId, trigger: payload.trigger, cwd: payload.cwd, transcriptPath: payload.transcriptPath, at: at.toISOString(), summary: text.summary, items: text.items };
-    // Renamed into place so a replayer listing `.json` files never reads half a file.
-    fs.writeFileSync(`${file}.tmp`, JSON.stringify(body), 'utf8');
-    fs.renameSync(`${file}.tmp`, file);
-}
 /** The PostCompact work: record the summary, then write its items. Each step is independent; a busy store spools or defers. Never throws. */
 export function saveCompaction(hippoRoot, payload, log) {
     const result = { written: null, deferred: false, snapshotSaved: false };
@@ -427,105 +446,15 @@ function nextStartedAt(db, record) {
         .get(record.tenantId, record.sessionId, record.startedAt);
     return row?.at ?? null;
 }
-function readSpooled(file, fallbackTenantId) {
-    let raw;
-    try {
-        raw = JSON.parse(fs.readFileSync(file, 'utf8'));
-    }
-    catch {
-        return null; // a half-written or vanished spool file is skipped; the caller treats null as unreadable
-    }
-    if (!isObjectLike(raw) || !('sessionId' in raw) || !isStringValue(raw.sessionId) || !('summary' in raw) || !isStringValue(raw.summary))
-        return null;
-    if (!('at' in raw) || !isStringValue(raw.at) || Number.isNaN(Date.parse(raw.at)))
-        return null;
-    const items = 'items' in raw && Array.isArray(raw.items) ? raw.items.filter(isStringValue) : [];
-    return {
-        tenantId: 'tenantId' in raw && isStringValue(raw.tenantId) && raw.tenantId !== '' ? raw.tenantId : fallbackTenantId,
-        payload: {
-            sessionId: raw.sessionId,
-            trigger: 'trigger' in raw && isStringValue(raw.trigger) ? raw.trigger : null,
-            cwd: 'cwd' in raw && isStringValue(raw.cwd) ? raw.cwd : null,
-            transcriptPath: 'transcriptPath' in raw && isStringValue(raw.transcriptPath) ? raw.transcriptPath : null,
-            compactSummary: null,
-        },
-        text: { summary: raw.summary, items },
-        at: new Date(raw.at),
+/** Records a spooled summary, then writes its items. */
+function spoolImporter(db, hippoRoot, log) {
+    return (spooled, recorded) => {
+        const record = recordSummary(db, hippoRoot, spooled.tenantId, spooled.payload, spooled.text, spooled.at);
+        // The record holds the items now, so the file is done even if the write below fails.
+        recorded();
+        saveItems(db, hippoRoot, { tenantId: spooled.tenantId, recordId: record.id, sessionId: record.sessionId, originProject: record.originProject, cwd: record.cwd, items: record.items }, log);
     };
 }
-function isMissingFile(cause) {
-    return cause instanceof Error && 'code' in cause && cause.code === 'ENOENT';
-}
-/** A live replayer refreshes its claim's mtime when it takes it, so an old claim means the replayer died. */
-function recoverStaleClaims(dir, log) {
-    const staleBefore = Date.now() - REPLAY_AFTER_MS;
-    for (const name of fs.readdirSync(dir).filter((n) => n.endsWith(`.json${CLAIMED_SUFFIX}`))) {
-        const claimed = path.join(dir, name);
-        try {
-            if (fs.statSync(claimed).mtimeMs >= staleBefore)
-                continue;
-            fs.renameSync(claimed, claimed.slice(0, -CLAIMED_SUFFIX.length));
-            log(`spool file ${name} was claimed by a replayer that never finished, put back`);
-        }
-        catch (err) {
-            if (!isMissingFile(err))
-                log(`spool file ${name} not recovered: ${errorMessage(err)}`);
-        }
-    }
-}
-function releaseClaim(claimed, file, log) {
-    try {
-        fs.renameSync(claimed, file);
-    }
-    catch (err) {
-        log(`spool file ${path.basename(file)} could not be put back: ${errorMessage(err)}`);
-    }
-}
-/** Each file is claimed by rename before it is read, so two replayers never import the same one. */
-function importSpool(db, hippoRoot, tenantId, log, deadline) {
-    const dir = path.join(hippoRoot, SPOOL_DIR);
-    if (!fs.existsSync(dir))
-        return 0;
-    recoverStaleClaims(dir, log);
-    let finished = 0;
-    for (const name of fs.readdirSync(dir).filter((n) => n.endsWith('.json')).sort()) {
-        if (Date.now() > deadline)
-            break;
-        const file = path.join(dir, name);
-        const claimed = `${file}${CLAIMED_SUFFIX}`;
-        try {
-            fs.renameSync(file, claimed);
-            const now = new Date();
-            fs.utimesSync(claimed, now, now);
-        }
-        catch (err) {
-            if (!isMissingFile(err))
-                log(`spool file ${name} not claimed: ${errorMessage(err)}`);
-            continue;
-        }
-        const spooled = readSpooled(claimed, tenantId);
-        if (!spooled) {
-            log(`spool file ${name} is not readable, set aside`);
-            fs.renameSync(claimed, `${file}.bad`);
-            continue;
-        }
-        let removed = false;
-        try {
-            const record = recordSummary(db, hippoRoot, spooled.tenantId, spooled.payload, spooled.text, spooled.at);
-            // The record holds the items now, so the file is done even if the write below fails.
-            fs.rmSync(claimed, { force: true });
-            removed = true;
-            saveItems(db, hippoRoot, { tenantId: spooled.tenantId, recordId: record.id, sessionId: record.sessionId, originProject: record.originProject, cwd: record.cwd, items: record.items }, log);
-            finished++;
-        }
-        catch (err) {
-            log(`spool file ${name} not imported: ${errorMessage(err)}`);
-            if (!removed)
-                releaseClaim(claimed, file, log);
-        }
-    }
-    return finished;
-}
 /** Finishes what a killed hook or a busy store left: `summarised` records, spool files, and `started` records the transcript can fill. Returns how many compactions it saved. */
 export function replayCompactions(db, hippoRoot, log, deadline = Number.POSITIVE_INFINITY) {
     const tenantId = resolveTenantId({});
@@ -544,7 +473,7 @@ export function replayCompactions(db, hippoRoot, log, deadline = Number.POSITIVE
         }
     }
     try {
-        finished += importSpool(db, hippoRoot, tenantId, log, deadline);
+        finished += importSpool(hippoRoot, tenantId, log, deadline, spoolImporter(db, hippoRoot, log));
     }
     catch (err) {
         log(`spool import failed: ${errorMessage(err)}`);

package/dist/compaction-spool.d.ts

@@ -0,0 +1,45 @@
+import type { CompactionText, Log, PostCompactPayload } from './compaction-record.js';
+export declare const SPOOL_DIR = "compactions-spool";
+/** Starts the log line of a step that broke with an unexpected error, so the session-start banner counts it; a set-aside file is counted by its `.bad` name instead. */
+export declare const SPOOL_PROBLEM = "spool problem: ";
+/** Only the fs calls the spool makes, so a test can stand in for one of them. */
+export interface SpoolFs {
+    existsSync(file: string): boolean;
+    mkdirSync(dir: string, options: {
+        recursive: true;
+    }): void;
+    readdirSync(dir: string): string[];
+    readFileSync(file: string, encoding: 'utf8'): string;
+    renameSync(from: string, to: string): void;
+    statSync(file: string): {
+        mtimeMs: number;
+    };
+    unlinkSync(file: string): void;
+    writeFileSync(file: string, data: string, options: {
+        encoding: 'utf8';
+        flag?: 'wx';
+    }): void;
+}
+/** Test-only seam, so a test can make one fs call fail; null restores the real fs. */
+export declare function __setSpoolFs(next: SpoolFs | null): void;
+export interface SpooledCompaction {
+    tenantId: string;
+    payload: PostCompactPayload;
+    text: CompactionText;
+    at: Date;
+}
+/** Records one spooled compaction; it calls `recorded` once the store holds the summary, so the file is done even if a later step throws. */
+export type SpoolImporter = (spooled: SpooledCompaction, recorded: () => void) => void;
+export declare function spool(hippoRoot: string, tenantId: string, payload: PostCompactPayload, text: CompactionText, at: Date): void;
+/** null when the text is not a spooled compaction. */
+export declare function parseSpooled(text: string, fallbackTenantId: string): SpooledCompaction | null;
+export interface SpoolCounts {
+    waiting: number;
+    stale: number;
+    bad: number;
+}
+/** What doctor reports, read only: files a replay should have taken by now, claims and tmp files a dead process left, and `.bad` files. */
+export declare function spoolCounts(hippoRoot: string, now: Date, waitingAfterMs: number): SpoolCounts;
+/** One replayer at a time holds `replay.lock`, so only one process claims a file even where Windows lets two renames of it both land. */
+export declare function importSpool(hippoRoot: string, tenantId: string, log: Log, deadline: number, importOne: SpoolImporter): number;
+//# sourceMappingURL=compaction-spool.d.ts.map