hippo-memory 1.52.8 → 1.52.9

package/dist/capture.d.ts

@@ -14,6 +14,7 @@ export interface ExtractedItem {
     category: string;
     tags: string[];
 }
+export declare function splitSentences(text: string): string[];
 /**
  * Main extraction function. Scans text for actionable items using heuristics.
  */
@@ -28,6 +29,7 @@ export interface CaptureOptions {
      * `stdinTimedOut` marks an empty read "unknown", not "no payload". */
     stdinText?: string;
     stdinTimedOut?: boolean;
+    sessionTurns?: readonly SessionTurn[];
     /**
      * Tee stdout/stderr to this log file while capture runs. Mirrors the
      * pattern used by `hippo sleep --log-file` so the SessionEnd hook output
@@ -45,7 +47,41 @@ export interface CaptureOptions {
      * behaviour. Ignored when `global: true` (global captures are host-wide).
      */
     tenantId?: string;
+    originProject?: string;
 }
+/**
+ * Runtime shape guards used throughout this file wherever a value arrives
+ * unparsed (JSONL transcript records, stdout/stderr write() chunks). Generic
+ * over the input so the parameter is never annotated `unknown` directly — TS
+ * infers it from the call site — while the check itself avoids `typeof` by
+ * testing identity against the coercion (`String(x) === x`) /
+ * prototype-chain (`instanceof Object`) instead. Behaviourally equivalent to
+ * `typeof x === 'string'` / a truthy `typeof x === 'object'` check for
+ * anything `JSON.parse` can produce (the only divergence is boxed
+ * primitives, which JSON.parse never yields).
+ */
+export declare function isStringValue<T>(value: T): value is T & string;
+export declare function isObjectLike<T>(value: T): value is T & object;
+/** Message for a caught value of unknown shape. `cause` names the sanctioned unknown-input case (error-cause enrichment). */
+export declare function errorMessage(cause: unknown): string;
+/** Transcript-line flags isNonHumanUserLine reads (any may be absent); `promptSource: 'system'` marks Claude Code's own notices. */
+interface TranscriptLineFlags {
+    type?: unknown;
+    isMeta?: unknown;
+    isSidechain?: unknown;
+    isCompactSummary?: unknown;
+    promptSource?: unknown;
+}
+export interface SessionTurn {
+    role: 'user' | 'assistant';
+    text: string;
+}
+export interface TranscriptRecord extends TranscriptLineFlags {
+    message?: unknown;
+    payload?: unknown;
+    cwd?: unknown;
+}
+export declare function collectSessionTurns(jsonl: string, visit?: (record: TranscriptRecord) => void): SessionTurn[];
 export declare function summariseTranscript(jsonl: string): string;
 /**
  * Resolve a transcript path for `--last-session`.
@@ -139,4 +175,5 @@ export declare function cmdPreCompact(hippoRoot: string, options: PreCompactOpti
  * Never throws.
  */
 export declare function postCompactMessage(stdinText: string | undefined, logFile?: string, now?: Date): string | null;
+export {};
 //# sourceMappingURL=capture.d.ts.map

package/dist/capture.js

@@ -11,17 +11,18 @@
 import * as fs from 'fs';
 import * as path from 'path';
 import { createMemory, Layer } from './memory.js';
+import { duplicateKey, storedTextKeys } from './same-text.js';
 import { isContentWorthStoring } from './audit.js';
 import { isInitialized, writeEntry, loadAllEntries, updateStats, saveActiveTaskSnapshot, loadActiveTaskSnapshot, } from './store.js';
 import { getGlobalRoot, initGlobal } from './shared.js';
 import { embedMemory } from './embeddings.js';
-import { isEmbeddingConfigured } from './embedding-provider.js';
 import { resolveTenantId } from './tenant.js';
 import { defaultPreCompactLogPath } from './hooks.js';
-import { redactSecretsStrict } from './secret-detect.js';
+import { maskEmails, redactSecretsStrict } from './secret-detect.js';
 import { RejectedValueError, checkRejectionGuard } from './rejection.js';
 import { openHippoDb, closeHippoDb } from './db.js';
 import { loadConfig } from './config.js';
+import { classifyOriginProject } from './project-identity.js';
 // Sentence-level patterns
 //
 // T1 (DF2): each pattern now carries TWO capture groups — group 1 is the
@@ -50,7 +51,7 @@ const DECISION_PATTERNS = [
     /(decision:\s*)(.{1,500})/i,
 ];
 const RULE_PATTERNS = [
-    /((?:never|always|must(?:\s+not)?|do(?:n't| not)\s+ever)\s+)(.{1,500})/i,
+    /\b((?:never|always|must(?:\s+not)?|do(?:n't| not)\s+ever)\s+)(.{1,500})/i,
     /((?:the rule is|rule:)\s*)(.{1,500})/i,
     /((?:important|critical|remember):\s*)(.{1,500})/i,
     /((?:make sure|ensure)\s+(?:to\s+)?)(.{1,500})/i,
@@ -77,7 +78,7 @@ const SPEC_HEADING_PATTERNS = [
 // ---------------------------------------------------------------------------
 // Extraction engine
 // ---------------------------------------------------------------------------
-function splitSentences(text) {
+export function splitSentences(text) {
     // Split on sentence boundaries, keeping reasonable chunks
     return text
         .split(/(?<=[.!?])\s+|\n/)
@@ -300,6 +301,21 @@ function cleanExtract(raw) {
     }
     return content;
 }
+/** The 500-char write gate minus the 200-char bound counted from the keyword. */
+const RULE_LEAD_CAP = 300;
+/** A modal rule keeps its subject ("We must never ..."); null keeps the keyword-start form when the lead is long or the keyword sits inside a bracket. */
+function ruleLead(sentence, keywordStart, contentStart) {
+    if (keywordStart < 0 || contentStart < 0 || contentStart > RULE_LEAD_CAP)
+        return null;
+    let depth = 0;
+    for (const ch of sentence.slice(0, keywordStart)) {
+        if (ch === '(' || ch === '[' || ch === '{')
+            depth++;
+        else if ((ch === ')' || ch === ']' || ch === '}') && depth > 0)
+            depth--;
+    }
+    return depth > 0 ? null : sentence.slice(0, contentStart);
+}
 function extractFromPatterns(sentence, patterns, category, tag) {
     for (const pat of patterns) {
         // `d` gives per-group offsets; see the contentStart comment below.
@@ -355,7 +371,11 @@ function extractFromPatterns(sentence, patterns, category, tag) {
                 // capture, shipped in the previous commit. Codex P1, r6.
                 const contentStart = match.indices?.[2]?.[0] ?? -1;
                 const afterKeyword = contentStart >= 0 ? sentence.slice(contentStart) : (match[2] ?? match[0]);
-                bounded = boundToClause(keywordPrefix + afterKeyword, keywordPrefix.length);
+                const keywordStart = match.indices?.[1]?.[0] ?? -1;
+                const lead = pat === RULE_PATTERNS[0] ? ruleLead(sentence, keywordStart, contentStart) : null;
+                bounded = lead !== null
+                    ? boundToClause(lead + afterKeyword, lead.length, keywordStart + 200)
+                    : boundToClause(keywordPrefix + afterKeyword, keywordPrefix.length);
             }
             const content = cleanExtract(bounded);
             if (content.length >= 8 && content.length <= 500) {
@@ -410,7 +430,7 @@ export function extractFromText(text) {
     const items = [];
     const seen = new Set();
     const addIfNew = (item) => {
-        const norm = item.content.toLowerCase().replace(/\s+/g, ' ').trim();
+        const norm = duplicateKey(item.content);
         if (seen.has(norm))
             return;
         if (!isContentWorthStoring(item.content))
@@ -449,26 +469,6 @@ export function extractFromText(text) {
     }
     return items;
 }
-// ---------------------------------------------------------------------------
-// Normalisation for deduplication (mirrors import.ts)
-// ---------------------------------------------------------------------------
-function normalise(text) {
-    return text
-        .toLowerCase()
-        .replace(/[^\w\s]/g, '')
-        .replace(/\s+/g, ' ')
-        .trim();
-}
-function isDuplicate(content, existing) {
-    const norm = normalise(content);
-    if (!norm)
-        return true;
-    for (const e of existing) {
-        if (normalise(e.content) === norm)
-            return true;
-    }
-    return false;
-}
 /**
  * Runtime shape guards used throughout this file wherever a value arrives
  * unparsed (JSONL transcript records, stdout/stderr write() chunks). Generic
@@ -480,14 +480,14 @@ function isDuplicate(content, existing) {
  * anything `JSON.parse` can produce (the only divergence is boxed
  * primitives, which JSON.parse never yields).
  */
-function isStringValue(value) {
+export function isStringValue(value) {
     return String(value) === value;
 }
-function isObjectLike(value) {
+export function isObjectLike(value) {
     return value !== null && value instanceof Object;
 }
 /** Message for a caught value of unknown shape. `cause` names the sanctioned unknown-input case (error-cause enrichment). */
-function errorMessage(cause) {
+export function errorMessage(cause) {
     return cause instanceof Error ? cause.message : String(cause);
 }
 /**
@@ -506,10 +506,29 @@ function isNonHumanUserLine(entry, content) {
     const head = content.trimStart();
     return CLAUDE_CODE_COMMAND_PREFIXES.some((p) => head.startsWith(p));
 }
-export function summariseTranscript(jsonl) {
+const NON_HUMAN_BLOCK_PREFIXES = ['<ide_', '[Request interrupted by user'];
+/** The human's words on a Claude Code user line, '' when none; VS Code stores prompts as text blocks, beside its own open-file, selection and interrupt blocks. */
+function humanUserText(entry, message) {
+    const content = message.content;
+    let text = '';
+    if (isStringValue(content)) {
+        text = content;
+    }
+    else if (Array.isArray(content) && !content.some((b) => isObjectLike(b) && 'type' in b && b.type === 'tool_result')) {
+        const parts = [];
+        for (const block of content) {
+            const blockText = isObjectLike(block) && 'type' in block && block.type === 'text' && 'text' in block ? block.text : undefined;
+            if (isStringValue(blockText) && !NON_HUMAN_BLOCK_PREFIXES.some((p) => blockText.trimStart().startsWith(p)))
+                parts.push(blockText);
+        }
+        text = parts.join('\n');
+    }
+    text = text.trim();
+    return text && !isNonHumanUserLine(entry, text) ? text : '';
+}
+export function collectSessionTurns(jsonl, visit) {
     const lines = jsonl.split('\n').filter((l) => l.trim());
-    const userMessages = [];
-    const assistantTexts = [];
+    const turns = [];
     for (const line of lines) {
         let entry;
         try {
@@ -518,19 +537,18 @@ export function summariseTranscript(jsonl) {
         catch {
             continue;
         }
-        if (!isObjectLike(entry))
+        if (!isObjectLike(entry) || !('type' in entry))
             continue;
-        if (('type' in entry) && (entry.type === 'user' || entry.type === 'assistant')) {
+        visit?.(entry);
+        if (entry.type === 'user' || entry.type === 'assistant') {
             const message = 'message' in entry && isObjectLike(entry.message) ? entry.message : undefined;
             if (!message)
                 continue;
             const content = 'content' in message ? message.content : undefined;
             if (entry.type === 'user') {
-                // Plain text user messages only (skip tool_result arrays), and only
-                // ones the human wrote (see isNonHumanUserLine).
-                if (isStringValue(content) && content.trim() && !isNonHumanUserLine(entry, content)) {
-                    userMessages.push(content.trim());
-                }
+                const text = humanUserText(entry, message);
+                if (text)
+                    turns.push({ role: 'user', text });
             }
             else if (Array.isArray(content)) {
                 // Keep assistant text blocks; drop thinking + tool_use
@@ -544,13 +562,13 @@ export function summariseTranscript(jsonl) {
                     }
                 }
                 if (chunks.length > 0) {
-                    assistantTexts.push(chunks.join('\n'));
+                    turns.push({ role: 'assistant', text: chunks.join('\n') });
                 }
             }
             continue;
         }
         // Codex rollout transcript shape: response_item -> payload.message
-        if ('type' in entry && entry.type === 'response_item') {
+        if (entry.type === 'response_item') {
             const payload = 'payload' in entry && isObjectLike(entry.payload) ? entry.payload : undefined;
             if (!payload || !('type' in payload) || payload.type !== 'message')
                 continue;
@@ -574,11 +592,19 @@ export function summariseTranscript(jsonl) {
             if (chunks.length === 0)
                 continue;
             if (role === 'user')
-                userMessages.push(chunks.join('\n'));
+                turns.push({ role: 'user', text: chunks.join('\n') });
             if (role === 'assistant')
-                assistantTexts.push(chunks.join('\n'));
+                turns.push({ role: 'assistant', text: chunks.join('\n') });
         }
     }
+    return turns;
+}
+export function summariseTranscript(jsonl) {
+    return summariseSessionTurns(collectSessionTurns(jsonl));
+}
+function summariseSessionTurns(turns) {
+    const userMessages = turns.filter((t) => t.role === 'user').map((t) => t.text);
+    const assistantTexts = turns.filter((t) => t.role === 'assistant').map((t) => t.text);
     if (userMessages.length === 0 && assistantTexts.length === 0)
         return '';
     // Keep the tail: last ~20 user turns and last ~10 assistant replies.
@@ -760,13 +786,16 @@ function cmdCaptureCore(hippoRoot, options) {
             break;
         }
         case 'last-session': {
-            const resolved = resolveLastSessionTranscript(options.transcriptPath, options.stdinText, options.stdinTimedOut);
-            if (!resolved) {
-                console.log('No transcript found. Pass --transcript <path> or run from a SessionEnd hook.');
-                return;
+            let turns = options.sessionTurns;
+            if (!turns) {
+                const resolved = resolveLastSessionTranscript(options.transcriptPath, options.stdinText, options.stdinTimedOut);
+                if (!resolved) {
+                    console.log('No transcript found. Pass --transcript <path> or run from a SessionEnd hook.');
+                    return;
+                }
+                turns = collectSessionTurns(fs.readFileSync(resolved, 'utf8'));
             }
-            const jsonl = fs.readFileSync(resolved, 'utf8');
-            text = summariseTranscript(jsonl);
+            text = summariseSessionTurns(turns);
             if (!text) {
                 console.log('Transcript had no user/assistant messages to summarise.');
                 return;
@@ -778,16 +807,19 @@ function cmdCaptureCore(hippoRoot, options) {
         console.log('No text to capture from.');
         return;
     }
-    // Extract items
-    const extracted = extractFromText(text);
+    // Scrub once here, as the snapshot fields are: every source can carry a pasted token (AGENTS.md: no secrets in memories).
+    const extracted = extractFromText(maskEmails(redactSecretsStrict(text)));
     if (extracted.length === 0) {
         console.log('No actionable items found in the input.');
         return;
     }
-    // Load existing for dedup. L9: when options.tenantId is set on a non-global
-    // capture, scope the dedup read so tenant A's captures don't get suppressed
-    // by tenant B's existing content. Undefined preserves host-wide behaviour.
-    const existing = loadAllEntries(targetRoot, useGlobal ? undefined : options.tenantId);
+    // Dedup only against rows this capture's reader sees: another tenant's rows (L9), or another
+    // project's, are hidden from it, so they must not stop its own copy.
+    const stored = loadAllEntries(targetRoot, useGlobal ? undefined : options.tenantId);
+    const origin = options.originProject;
+    const keys = storedTextKeys(origin === undefined
+        ? stored
+        : stored.filter((e) => classifyOriginProject(e.origin_project, origin) !== 'cross-project'));
     let captured = 0;
     let skipped = 0;
     let rejected = 0;
@@ -802,7 +834,7 @@ function cmdCaptureCore(hippoRoot, options) {
     const dryRunDb = options.dryRun ? openHippoDb(targetRoot) : null;
     try {
         for (const item of extracted) {
-            if (isDuplicate(item.content, existing)) {
+            if (keys.has(duplicateKey(item.content))) {
                 skipped++;
                 if (options.dryRun) {
                     console.log(`  [skip] (${item.category}) ${item.content.slice(0, 80)}`);
@@ -819,7 +851,7 @@ function cmdCaptureCore(hippoRoot, options) {
             // global: true, the global store is host-wide and tenant is irrelevant
             // (createMemory's default 'default' applies). When global: false,
             // options.tenantId scopes the write to the same tenant as the dedup.
-            const entry = createMemory(item.content, {
+            const created = createMemory(item.content, {
                 layer: Layer.Episodic,
                 tags: item.tags,
                 source: 'capture',
@@ -827,6 +859,7 @@ function cmdCaptureCore(hippoRoot, options) {
                 tenantId: useGlobal ? undefined : options.tenantId,
                 baseHalfLifeDays,
             });
+            const entry = options.originProject === undefined ? created : { ...created, origin_project: options.originProject };
             if (options.dryRun) {
                 if (dryRunDb) {
                     try {
@@ -857,10 +890,8 @@ function cmdCaptureCore(hippoRoot, options) {
                     throw err;
                 }
                 updateStats(targetRoot, { remembered: 1 });
-                existing.push(entry); // within-batch dedup
-                if (isEmbeddingConfigured(targetRoot)) {
-                    embedMemory(targetRoot, entry).catch(() => { });
-                }
+                keys.add(duplicateKey(item.content)); // within-batch dedup
+                void embedMemory(targetRoot, entry);
             }
             captured++;
         }
@@ -987,9 +1018,9 @@ function lastPlainUserMessage(jsonl) {
         const message = 'message' in entry && isObjectLike(entry.message) ? entry.message : undefined;
         if (!message)
             continue;
-        const content = 'content' in message ? message.content : undefined;
-        if (isStringValue(content) && content.trim() && !isNonHumanUserLine(entry, content))
-            return content.trim();
+        const text = humanUserText(entry, message);
+        if (text)
+            return text;
     }
     return '';
 }
@@ -1082,20 +1113,21 @@ export function preCompactReportPath(logFile) {
 }
 /** A session's task, summary and next step from its transcript tail, secrets scrubbed and capped, '' where none; null with a logged reason when nothing is derivable. */
 export function transcriptWorkingState(transcriptPath, log) {
-    let tail;
+    let tail = '';
+    let rawTask = '';
+    let rawNextStep = '';
     try {
-        // CX7 (codex round 2): a final record bigger than the window leaves an empty tail, as when a huge tool_result
-        // triggered compaction, so grow the window a bounded number of times until one complete line survives.
-        tail = readTranscriptTail(transcriptPath, PRE_COMPACT_TAIL_BYTES);
-        let prevCap = PRE_COMPACT_TAIL_BYTES;
-        for (const grownCap of [PRE_COMPACT_TAIL_BYTES * 4, PRE_COMPACT_TAIL_BYTES * 16]) {
-            if (tail.trim() !== '')
+        // Compaction fires when a big tool_result lands (CX7), so the last human and assistant turns can sit
+        // megabytes back: grow the window a bounded number of times until both turns are in it.
+        const size = fs.statSync(transcriptPath).size;
+        for (const cap of [PRE_COMPACT_TAIL_BYTES, PRE_COMPACT_TAIL_BYTES * 4, PRE_COMPACT_TAIL_BYTES * 16]) {
+            if (cap > PRE_COMPACT_TAIL_BYTES)
+                log(`tail window grown to ${cap} bytes (last ${rawTask ? 'assistant' : 'user'} turn is further back)`);
+            tail = readTranscriptTail(transcriptPath, cap);
+            rawTask = lastPlainUserMessage(tail);
+            rawNextStep = lastAssistantTextBlock(tail);
+            if ((rawTask && rawNextStep) || size <= cap)
                 break;
-            if (fs.statSync(transcriptPath).size <= prevCap)
-                break; // already read the whole file
-            log(`tail window grown to ${grownCap} bytes (oversized final record)`);
-            tail = readTranscriptTail(transcriptPath, grownCap);
-            prevCap = grownCap;
         }
     }
     catch (err) {
@@ -1103,17 +1135,15 @@ export function transcriptWorkingState(transcriptPath, log) {
         return null;
     }
     const rawSummary = summariseTranscript(tail);
-    const rawTask = lastPlainUserMessage(tail);
-    const rawNextStep = lastAssistantTextBlock(tail);
     if (!rawTask.trim() && !rawSummary.trim() && !rawNextStep.trim()) {
         log('skip: empty summary');
         return null;
     }
     // X9: these fields skip the capture content gate and reach a prompt, so the strict scrub runs. The caps protect the
     // re-injection token budget and never split a surrogate pair (X2); `hippo snapshot save` stays uncapped.
-    const task = redactSecretsStrict(rawTask);
-    const summary = redactSecretsStrict(rawSummary);
-    const nextStep = redactSecretsStrict(rawNextStep);
+    const task = maskEmails(redactSecretsStrict(rawTask));
+    const summary = maskEmails(redactSecretsStrict(rawSummary));
+    const nextStep = maskEmails(redactSecretsStrict(rawNextStep));
     return {
         task: task.trim() ? truncateCodePointSafe(task, PRE_COMPACT_TASK_CAP) : '',
         summary: summary.trim() ? truncateKeepNewest(summary, PRE_COMPACT_SUMMARY_CAP) : '',