hippo-memory 1.71.0 → 2.0.0

package/dist/capture/command.js

@@ -2,21 +2,16 @@ import * as fs from 'fs';
 import * as path from 'path';
 import { createMemory, Layer } from '../core/memory.js';
 import { duplicateKey, longestWord, storedTextKeys } from '../util/same-text.js';
-import { stampOriginProject } from '../store/entry-row.js';
-import { isInitialized, openStore } from '../store/open.js';
-import { writeEntryMirrors } from '../store/entry-writes.js';
+import { writeCapturedItems } from '../store/capture-write.js';
+import { embedMemory } from '../search/embed-memories.js';
 import { EVERY_SCOPE, loadTextsHoldingWords } from '../store/candidates.js';
-import { updateStatsOn } from '../store/index-and-stats.js';
-import { gatedWrite } from '../trust/gated-write.js';
 import { getGlobalRoot, initGlobal } from '../sharing/global-store.js';
-import { embedMemory } from '../store/embeddings/index.js';
 import { maskEmails, redactSecretsStrict } from '../util/secret-detect.js';
-import { RejectedValueError, checkRejectionGuard } from '../store/rejection.js';
-import { openHippoDb, closeHippoDb } from '../db/index.js';
+import { withRejectionProbe } from '../store/rejection-probe.js';
 import { loadConfig } from '../core/config.js';
-import { classifyOriginProject, projectId } from '../core/project-identity.js';
-import { isStringValue } from '../core/capture-contract.js';
+import { classifyOriginProject, projectId, isInitialized } from '../core/project-identity.js';
 import { errorMessage, log } from '../util/log.js';
+import { teeStdStreams } from '../util/stream-tee.js';
 import { extractFromTexts } from './extract.js';
 import { collectSessionTurns, sessionTail, resolveLastSessionTranscript } from './transcript.js';
 export function cmdCapture(hippoRoot, options) {
@@ -37,28 +32,11 @@ export function cmdCapture(hippoRoot, options) {
             restoreStdio();
     }
 }
-/** Append-mode tee: mirror every stdout/stderr chunk to `logFile` until the returned restore function is called; a log-write failure is non-fatal. */
+/** Append-mode tee: mirror every stdout/stderr chunk to `logFile` until the returned restore function is called. */
 function beginLogTee(logFile) {
     if (!writeLogBanner(logFile))
         return () => { };
-    const origStdoutWrite = process.stdout.write.bind(process.stdout);
-    const origStderrWrite = process.stderr.write.bind(process.stderr);
-    const tee = (chunk) => appendToLog(logFile, chunk);
-    const wrapWrite = (origWrite) => {
-        const wrapped = (...args) => {
-            tee(args[0]);
-            // SAFETY: forwarding the exact arguments the real overloaded `write` received is safe; `StreamWriteArgs` is the union of both overloads' parameter lists.
-            return origWrite(...args);
-        };
-        // SAFETY: `wrapped` matches both real `write` overload shapes; TS cannot verify one implementation covers an overloaded type.
-        return wrapped;
-    };
-    process.stdout.write = wrapWrite(origStdoutWrite);
-    process.stderr.write = wrapWrite(origStderrWrite);
-    return () => {
-        process.stdout.write = origStdoutWrite;
-        process.stderr.write = origStderrWrite;
-    };
+    return teeStdStreams(logFile);
 }
 /** Creates the log's folder and appends the banner line; false, after a warning, when the log cannot be written. */
 function writeLogBanner(logFile) {
@@ -72,15 +50,6 @@ function writeLogBanner(logFile) {
     }
     return true;
 }
-function appendToLog(logFile, chunk) {
-    try {
-        const buf = isStringValue(chunk) ? chunk : Buffer.from(chunk).toString('utf8');
-        fs.appendFileSync(logFile, buf, 'utf8');
-    }
-    catch {
-        // log failures are non-fatal
-    }
-}
 // Console lines here are the `hippo capture` command's printed result, so they stay off the logger.
 function cmdCaptureCore(hippoRoot, options) {
     const useGlobal = options.global;
@@ -94,15 +63,16 @@ function cmdCaptureCore(hippoRoot, options) {
             process.exit(1);
         }
     }
-    const texts = readCaptureTexts(options);
-    if (texts === null)
+    const read = readCaptureTexts(options);
+    if (read === null)
         return;
+    const { texts, userTexts } = read;
     if (texts.every((text) => text.trim().length === 0)) {
         console.log('No text to capture from.');
         return;
     }
     // Scrub once here, as the snapshot fields are: every source can carry a pasted token (AGENTS.md: no secrets in memories).
-    const extracted = extractFromTexts(texts.map((text) => maskEmails(redactSecretsStrict(text))));
+    const extracted = extractFromTexts(texts.map((text) => maskEmails(redactSecretsStrict(text))), userTexts);
     if (extracted.length === 0) {
         console.log('No actionable items found in the input.');
         return;
@@ -132,12 +102,13 @@ function printCaptureTally(options, { captured, skipped, rejected }) {
         (rejected > 0 ? `, ${rejected} rejected` : '') +
         ')');
 }
-/** The raw texts for the chosen source, one per session turn; null after printing why a last-session capture has nothing. */
+/** The raw texts for the chosen source, one per session turn, the person's turns first; null after printing why a last-session capture has nothing.
+ *  Only a session says who spoke, so a piped or file capture counts no user texts. */
 function readCaptureTexts(options) {
     switch (options.source) {
         case 'stdin': {
             try {
-                return [fs.readFileSync(0, 'utf8')];
+                return { texts: [fs.readFileSync(0, 'utf8')], userTexts: 0 };
             }
             catch {
                 console.error('No input on stdin. Pipe text in or use --file <path>.');
@@ -153,7 +124,7 @@ function readCaptureTexts(options) {
                 console.error(`File not found: ${options.filePath}`);
                 process.exit(1);
             }
-            return [fs.readFileSync(options.filePath, 'utf8')];
+            return { texts: [fs.readFileSync(options.filePath, 'utf8')], userTexts: 0 };
         }
         case 'last-session': {
             let turns = options.sessionTurns;
@@ -170,85 +141,55 @@ function readCaptureTexts(options) {
                 console.log('Transcript had no user/assistant messages to summarise.');
                 return null;
             }
-            return [...users, ...assistants];
+            return { texts: [...users, ...assistants], userTexts: users.length };
         }
     }
 }
 export function captureExtractedItems(targetRoot, options, extracted, keys) {
     const tally = { captured: 0, skipped: 0, rejected: 0 };
     const baseHalfLifeDays = loadConfig(targetRoot).defaultHalfLifeDays;
+    if (!options.dryRun) {
+        const outcomes = writeCapturedItems(targetRoot, extracted.map((item) => ({ content: item.content, makeEntry: () => captureEntry(item, options, baseHalfLifeDays) })), keys, { actor: options.actor, lean: options.lean, onCaptured: options.lean ? undefined : (entry) => void embedMemory(targetRoot, entry) });
+        for (const outcome of outcomes)
+            tally[outcome]++;
+        return tally;
+    }
     // Dry run probes the same checkRejectionGuard the real write uses, on a read-only handle,
     // so a tombstoned item previews as rejected rather than captured.
-    const dryRunDb = options.dryRun ? openHippoDb(targetRoot) : null;
-    const writeDb = options.dryRun ? null : openStore(targetRoot);
-    try {
+    return withRejectionProbe(targetRoot, (wouldReject) => {
         for (const item of extracted) {
             if (keys.has(duplicateKey(item.content))) {
                 tally.skipped++;
-                if (options.dryRun) {
-                    console.log(`  [skip] (${item.category}) ${item.content.slice(0, 80)}`);
-                }
+                console.log(`  [skip] (${item.category}) ${item.content.slice(0, 80)}`);
                 continue;
             }
             const entry = captureEntry(item, options, baseHalfLifeDays);
-            tally[captureOne({ targetRoot, options, dryRunDb, writeDb, keys }, item, entry)]++;
+            if (wouldReject(entry.tenant_id ?? 'default', entry.id, entry.content)) {
+                console.log(`  [reject] (${item.category}) ${item.content.slice(0, 80)} - matches a rejected value`);
+                tally.rejected++;
+                continue;
+            }
+            console.log(`  [capture] (${item.category}) ${item.content}`);
+            tally.captured++;
         }
-    }
-    finally {
-        if (dryRunDb)
-            closeHippoDb(dryRunDb);
-        if (writeDb)
-            closeHippoDb(writeDb);
-    }
-    return tally;
+        return tally;
+    });
 }
 function captureEntry(item, options, baseHalfLifeDays) {
     // kind stays 'distilled': these are curated items, not raw transcript (see MEMORY_ENVELOPE.md).
     // The write tenant must match the dedup read's, or scoped dedup passes and the row lands in 'default'.
+    // A rule the person stated is pinned, so it injects even when a later prompt shares none of its words.
+    // SHORTCUT: every stated rule pins; cap or age these pins if a store's pins outgrow the hook budget.
     const created = createMemory(item.content, {
         layer: Layer.Episodic,
         tags: item.tags,
         source: 'capture',
         confidence: 'observed',
+        pinned: item.fromUser === true && item.category === 'rule',
         tenantId: options.tenantId,
         source_session_id: options.sessionId,
         baseHalfLifeDays,
     });
     return options.originProject === undefined ? created : { ...created, origin_project: projectId(options.originProject) };
 }
-/** Previews or writes one non-duplicate item and names the tally it counts toward. */
-function captureOne(ctx, item, entry) {
-    const { targetRoot, options, dryRunDb, writeDb, keys } = ctx;
-    if (options.dryRun) {
-        if (dryRunDb) {
-            try {
-                checkRejectionGuard(dryRunDb, entry.tenantId ?? 'default', entry.id, entry.content);
-            }
-            catch (err) {
-                if (err instanceof RejectedValueError) {
-                    console.log(`  [reject] (${item.category}) ${item.content.slice(0, 80)} - matches a rejected value`);
-                    return 'rejected';
-                }
-                throw err;
-            }
-        }
-        console.log(`  [capture] (${item.category}) ${item.content}`);
-    }
-    else if (writeDb !== null) {
-        // One rejected item must not abort the rest of this capture's items.
-        const stamped = stampOriginProject(targetRoot, entry);
-        const outcome = gatedWrite(writeDb, targetRoot, stamped, { actor: options.actor });
-        if (outcome === 'skipped:rejected')
-            return 'rejected';
-        if (outcome !== 'written')
-            return 'skipped';
-        writeEntryMirrors(targetRoot, stamped);
-        if (!options.lean)
-            updateStatsOn(writeDb, targetRoot, { remembered: 1 });
-        keys.add(duplicateKey(item.content)); // within-batch dedup
-        if (!options.lean)
-            void embedMemory(targetRoot, entry);
-    }
-    return 'captured';
-}
 //# sourceMappingURL=command.js.map

package/dist/capture/compact.js

@@ -1,8 +1,10 @@
 import * as fs from 'fs';
 import * as path from 'path';
-import { isInitialized } from '../store/open.js';
+import { isInitialized } from '../core/project-identity.js';
 import { saveActiveTaskSnapshot, loadActiveTaskSnapshot } from '../store/sessions.js';
-import { PRE_COMPACT_INSTRUCTION, postCompactLine, recordCompactionStart, recordSnapshotSaved, saveCompaction, replayCompactionsAt, COMPACTION_DB_WAIT_MS, } from './compaction-record.js';
+import { PRE_COMPACT_INSTRUCTION, postCompactLine, saveCompaction, replayCompactionsAt } from './compaction-record.js';
+import { recordCompactionStart, recordSnapshotSaved } from '../store/compactions-record.js';
+import { COMPACTION_DB_WAIT_MS } from '../core/compaction-timing.js';
 import { resolveTenantId } from '../store/tenant.js';
 import { defaultPreCompactLogPath, vscodeUserHooksFile } from '../hooks/shared.js';
 import { readClaudeCodePostCompact, readClaudeCodePreCompact } from '../core/capture-contract.js';
@@ -49,7 +51,8 @@ function isReadableFile(filePath) {
         return false; // missing and unreadable both mean "no file" to the caller
     }
 }
-/** PreCompact stdout is the summariser's instructions; sent before the snapshot work because a locked store can run the hook past its 30 s limit, and via writeSync because process.exit drops buffered pipe output. */
+/** PreCompact stdout is the summariser's instructions; sent before the snapshot work because a locked store can run the hook past its 30
+ * s limit, and via writeSync because process.exit drops buffered pipe output. */
 function printPreCompactInstruction(logFile) {
     const text = `${PRE_COMPACT_INSTRUCTION}\n`;
     try {
@@ -69,7 +72,8 @@ function snapshotJustSaved(hippoRoot, sessionId) {
     const active = loadActiveTaskSnapshot(hippoRoot, resolveTenantId({}));
     return active?.session_id === sessionId && active.source === 'pre-compact' && Date.now() - Date.parse(active.updated_at) < TWIN_FIRE_MS;
 }
-/** Runs the PreCompact producer: records the compaction, asks the summariser for memories, saves a working-state snapshot. Never extracts memories itself; SessionEnd capture owns that. */
+/** Runs the PreCompact producer: records the compaction, asks the summariser for memories, saves a working-state snapshot. Never extracts
+ * memories itself; SessionEnd capture owns that. */
 function runPreCompact(hippoRoot, options, logFile) {
     const { stdinText, stdinTimedOut = false, runtime = 'claude-code' } = options;
     // PreCompact fires in every Claude Code project, including ones that never ran `hippo init`; gate before any call that would create a store.
@@ -103,7 +107,8 @@ function runPreCompact(hippoRoot, options, logFile) {
         return;
     saveDerivedSnapshot(hippoRoot, logFile, sessionId, recordId, derived);
 }
-/** Starts the compaction record when the host will close one (`closable`), then tells `onBoundary` what was printed; the record id, or null without a record. */
+/** Starts the compaction record when the host will close one (`closable`), then tells `onBoundary` what was printed; the record id, or
+ * null without a record. */
 function markCompactionBoundary(hippoRoot, options, logFile, input, closable) {
     const { sessionId } = input;
     // The record is the "something saved before every compaction", so it lands even when no snapshot is derivable later.

package/dist/capture/compaction-items-caller.js

@@ -1,34 +1,25 @@
 // PostCompact for a caller on another machine: the items it parsed become rows under its owner and project, and a retried request id writes nothing twice.
 import { BadRequestError } from '../core/api-errors.js';
-import { COMPACTION_ITEM_MAX_CHARS } from './compaction-items.js';
-import { compactionByRequest, recordSummary, saveItems, scrubCompactionItems } from './compaction-record.js';
+import { COMPACTION_ITEM_MAX_CHARS } from '../util/compaction-items.js';
+import { saveCallerItems } from '../store/compaction-caller.js';
+import { scrubCompactionItems } from './compaction-record.js';
 import { log } from '../util/log.js';
-import { assertRequestId, assertTrigger, bindCaller, callerInHoldout, withCallerDb } from './caller-session.js';
+import { assertRequestId, assertTrigger, bindCaller, callerInHoldout } from './caller-session.js';
 /** Busy throws, for the server's 503: the retry carries the same request id, so nothing is spooled or deferred here. */
 export function saveCompactionItemsForCaller(ctx, req) {
     assertTrigger(req.trigger);
     assertRequestId(req.requestId);
     // Refused before the scrub, as failure text is, so the cut after it only ever takes back a mask's growth.
-    if (req.items.some((item) => item.length > COMPACTION_ITEM_MAX_CHARS))
+    if (req.items.some((item) => item.length > COMPACTION_ITEM_MAX_CHARS)) {
         throw new BadRequestError(`items: each at most ${COMPACTION_ITEM_MAX_CHARS} characters`);
+    }
     const key = bindCaller(ctx, req.sessionId, req.project);
     if (callerInHoldout(ctx, req.sessionId))
         return { written: 0 };
-    return withCallerDb(ctx, (db) => {
-        const earlier = compactionByRequest(db, ctx.tenantId, req.requestId, req.sessionId);
-        // Past `summarised` the first try finished, so its count is the answer; a `summarised` one failed at the items and is reused.
-        if (earlier !== null && earlier.status !== 'summarised')
-            return { written: earlier.itemsWritten };
-        const meta = { sessionId: req.sessionId, trigger: req.trigger, cwd: null, transcriptPath: null };
-        const text = { summary: '', items: scrubCompactionItems(req.items) };
-        const record = earlier ?? recordSummary(db, ctx.hippoRoot, ctx.tenantId, {
-            meta, text, at: new Date(), caller: { originProject: req.project.name, requestId: req.requestId },
-        });
-        const written = saveItems(db, ctx.hippoRoot, {
-            tenantId: ctx.tenantId, recordId: record.id, sessionId: req.sessionId, originProject: record.originProject, cwd: null,
-            items: record.items, caller: { actor: key.owner, origins: key.project },
-        }, (message) => log.info(`post-compact: ${message}`));
-        return { written };
-    });
+    const written = saveCallerItems(ctx.hippoRoot, ctx.tenantId, {
+        sessionId: req.sessionId, trigger: req.trigger, requestId: req.requestId, project: req.project.name, items: scrubCompactionItems(req.items),
+        owner: key.owner, origins: key.project,
+    }, (message) => log.info(`post-compact: ${message}`));
+    return { written };
 }
 //# sourceMappingURL=compaction-items-caller.js.map

package/dist/capture/compaction-record.d.ts

@@ -1,31 +1,6 @@
-import { type DatabaseSyncLike } from '../db/index.js';
-import { type CompactionStatus } from '../store/compactions.js';
-/** PostCompact has 10 s in all (PreCompact 30 s), so a locked store must be given up on early. */
-export declare const COMPACTION_DB_WAIT_MS = 2000;
+import { type CompactionSaveResult, type Log, type ReplayOptions } from '../store/compactions-record.js';
 /** Tested verbatim: Claude Code hands PreCompact stdout to the summariser as instructions. */
-export declare const PRE_COMPACT_INSTRUCTION = "In your summary, add a last section titled 'Memories for hippo'. List, one per line starting with '- ', each lesson learned, decision made (with its reason) and correction the user gave in this session that should outlive it. Write each as a standalone sentence that names its subject. Leave out anything an earlier summary already listed under 'Memories for hippo', and anything this session already saved with `hippo remember`. Write '- none' if nothing new remains.";
-/** Long enough that a live hook has finished with its own record. */
-export declare const REPLAY_AFTER_MS: number;
-/** Claude Code deletes transcripts after 30 days, so an older gap can never be filled. */
-export declare const TRANSCRIPT_FILL_WINDOW_MS: number;
-export type { CompactionStatus };
-export interface CompactionRecord {
-    id: string;
-    tenantId: string;
-    sessionId: string;
-    originProject: string;
-    trigger: string | null;
-    cwd: string | null;
-    transcriptPath: string | null;
-    snapshotSaved: boolean;
-    startedAt: string;
-    summarisedAt: string | null;
-    summary: string | null;
-    items: string[];
-    itemsWritten: number;
-    status: CompactionStatus;
-}
-export type Log = (message: string) => void;
+export declare const PRE_COMPACT_INSTRUCTION: string;
 interface ScrubbedSummary {
     summary: string;
     items: string[];
@@ -34,59 +9,9 @@ interface ScrubbedSummary {
 }
 /** What the record keeps of a compact_summary: the scrubbed body, and every parsed item scrubbed. */
 export declare function readCompactionText(compactSummary: string): ScrubbedSummary;
-/** A caller's items scrubbed as readCompactionText scrubs a summary's, since another machine's scrub is not trusted; cut after it, as a mask can run longer than what it hides. */
+/** A caller's items scrubbed as readCompactionText scrubs a summary's, since another machine's scrub is not trusted; cut after it, as a
+ * mask can run longer than what it hides. */
 export declare function scrubCompactionItems(items: readonly string[]): string[];
-export interface CompactionStart {
-    sessionId: string;
-    originProject: string;
-    trigger: string | null;
-    cwd: string | null;
-    transcriptPath: string | null;
-}
-export declare function startCompaction(db: DatabaseSyncLike, tenantId: string, start: CompactionStart, at?: Date): string;
-/** A session's newest record, or null. */
-export declare function latestCompaction(db: DatabaseSyncLike, tenantId: string, sessionId: string): CompactionRecord | null;
-/** The record a caller's request made, whatever state it reached, so a retry neither writes its items twice nor starts a second record; another session's id is a ConflictError. */
-export declare function compactionByRequest(db: DatabaseSyncLike, tenantId: string, requestId: string, sessionId: string): CompactionRecord | null;
-/** Best effort, never throws: a compaction must not fail because its record could not be written. */
-export declare function recordCompactionStart(hippoRoot: string, start: Omit<CompactionStart, 'originProject'>, log: Log): string | null;
-/** On the caller's handle and tenant, so a server marks the record under the caller's tenant with its own wait. */
-export declare function markSnapshotSaved(db: DatabaseSyncLike, tenantId: string, recordId: string): void;
-export declare function recordSnapshotSaved(hippoRoot: string, tenantId: string, recordId: string, log: Log): void;
-export interface CompactionText {
-    summary: string;
-    items: string[];
-}
-/** A caller's checked project, and the id its retries carry so a retry finds this record. */
-export interface SummaryCaller {
-    originProject?: string;
-    requestId?: string;
-}
-export interface RecordSummaryOptions {
-    readonly meta: Omit<CompactionStart, 'originProject'>;
-    readonly text: CompactionText;
-    readonly at: Date;
-    readonly caller?: SummaryCaller;
-}
-/** Puts the summary on the session's `started` record, or inserts a `summarised` one when pre-compact wrote none. One statement each. */
-export declare function recordSummary(db: DatabaseSyncLike, hippoRoot: string, tenantId: string, options: RecordSummaryOptions): CompactionRecord;
-export interface ItemContext {
-    tenantId: string;
-    /** null when the record step failed: the items still go in, with no record to close. */
-    recordId: string | null;
-    sessionId: string;
-    originProject: string;
-    /** Where the session ran, so rows held under the project's older folder name count too. */
-    cwd: string | null;
-    items: string[];
-    /** Set when another machine sent the items: its audit actor and its project's names, since the server's folder is neither. */
-    caller?: {
-        actor: string;
-        origins: readonly string[];
-    };
-}
-/** Items that become rows, then the record `done`, in one transaction so two sessions cannot both insert one text. Mirrors after commit. */
-export declare function saveItems(db: DatabaseSyncLike, hippoRoot: string, ctx: ItemContext, log: Log): number;
 export interface PostCompactPayload {
     sessionId: string;
     trigger: string | null;
@@ -95,22 +20,11 @@ export interface PostCompactPayload {
     /** null when Claude Code sent none; the transcript fills the record later. */
     compactSummary: string | null;
 }
-export interface CompactionSaveResult {
-    /** Memories written; null when neither the record nor the items could be saved. */
-    written: number | null;
-    /** True when the rest waits for the next sleep or post-compact. */
-    deferred: boolean;
-    snapshotSaved: boolean;
-}
 /** The PostCompact work: record the summary, then write its items. Each step is independent; a busy store spools or defers. Never throws. */
 export declare function saveCompaction(hippoRoot: string, payload: PostCompactPayload, log: Log): CompactionSaveResult;
 /** The one line PostCompact prints, or null when nothing was saved. */
 export declare function postCompactLine(result: CompactionSaveResult): string | null;
-/** Finishes what a killed hook or a busy store left: `summarised` records, spool files, and `started` records the transcript can fill. Returns how many compactions it saved. */
-export declare function replayCompactions(db: DatabaseSyncLike, hippoRoot: string, log: Log, deadline?: number): number;
-/** For `hippo sleep` and post-compact: opens the store itself and never throws. */
-export declare function replayCompactionsAt(hippoRoot: string, log: Log, opts?: {
-    busyWaitMs?: number;
-    deadline?: number;
-}): number;
+/** For `hippo sleep` and post-compact: finishes what a killed hook or a busy store left, never throws. */
+export declare function replayCompactionsAt(hippoRoot: string, log: Log, opts?: ReplayOptions): number;
+export {};
 //# sourceMappingURL=compaction-record.d.ts.map