hippo-memory 1.57.0 → 1.59.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (109) hide show
  1. package/README.md +24 -1
  2. package/dist/agent-memories/apply.d.ts +1 -1
  3. package/dist/agent-memories/claude-code.js +1 -1
  4. package/dist/agent-memories/gemini.js +1 -1
  5. package/dist/agent-memories/legacy.js +4 -1
  6. package/dist/api-errors.d.ts +27 -0
  7. package/dist/api-errors.js +37 -0
  8. package/dist/api.d.ts +5 -5
  9. package/dist/api.js +40 -47
  10. package/dist/audit.d.ts +5 -1
  11. package/dist/audit.js +13 -0
  12. package/dist/autolearn.d.ts +1 -1
  13. package/dist/autolearn.js +7 -5
  14. package/dist/capture-contract.d.ts +47 -0
  15. package/dist/capture-contract.js +49 -0
  16. package/dist/capture-error.js +2 -1
  17. package/dist/capture.d.ts +0 -13
  18. package/dist/capture.js +5 -66
  19. package/dist/cli/output.d.ts +3 -0
  20. package/dist/cli/output.js +7 -0
  21. package/dist/cli/projects.d.ts +4 -0
  22. package/dist/cli/projects.js +90 -0
  23. package/dist/cli/shared.js +23 -13
  24. package/dist/cli/sleep.js +5 -3
  25. package/dist/cli.d.ts +1 -0
  26. package/dist/cli.js +486 -397
  27. package/dist/client.js +9 -0
  28. package/dist/codex-patch.js +1 -1
  29. package/dist/compaction-record.d.ts +1 -1
  30. package/dist/compaction-record.js +3 -2
  31. package/dist/config.d.ts +5 -0
  32. package/dist/config.js +17 -0
  33. package/dist/connectors/github/dlq.js +5 -2
  34. package/dist/connectors/github/octokit-client.js +4 -2
  35. package/dist/connectors/slack/dlq.js +6 -2
  36. package/dist/connectors/slack/web-client.js +7 -5
  37. package/dist/consolidate.d.ts +10 -0
  38. package/dist/consolidate.js +48 -35
  39. package/dist/customer-notes.js +14 -13
  40. package/dist/dag.js +7 -4
  41. package/dist/dashboard.js +1 -1
  42. package/dist/db.d.ts +12 -0
  43. package/dist/db.js +62 -1
  44. package/dist/decisions.js +9 -8
  45. package/dist/dedupe.js +1 -1
  46. package/dist/doctor.js +28 -0
  47. package/dist/dormant.d.ts +2 -2
  48. package/dist/embedding-provider.js +3 -3
  49. package/dist/embeddings.d.ts +4 -4
  50. package/dist/embeddings.js +72 -16
  51. package/dist/extract.js +19 -18
  52. package/dist/http-retry.d.ts +21 -0
  53. package/dist/http-retry.js +50 -0
  54. package/dist/http-util.d.ts +8 -0
  55. package/dist/http-util.js +10 -0
  56. package/dist/importers.d.ts +2 -0
  57. package/dist/importers.js +16 -5
  58. package/dist/incidents.js +11 -10
  59. package/dist/judgment.js +10 -17
  60. package/dist/log.d.ts +25 -0
  61. package/dist/log.js +48 -0
  62. package/dist/mcp/server.js +52 -24
  63. package/dist/mcp/tool-args.d.ts +21 -0
  64. package/dist/mcp/tool-args.js +80 -0
  65. package/dist/memory.js +3 -2
  66. package/dist/overlap-index.d.ts +7 -0
  67. package/dist/overlap-index.js +38 -0
  68. package/dist/pilot-arm.d.ts +9 -0
  69. package/dist/pilot-arm.js +47 -0
  70. package/dist/policies.js +12 -11
  71. package/dist/predictions.js +9 -8
  72. package/dist/processes.js +14 -13
  73. package/dist/project-briefs.js +16 -15
  74. package/dist/project-identity.d.ts +1 -1
  75. package/dist/project-identity.js +25 -1
  76. package/dist/project-merge.d.ts +52 -0
  77. package/dist/project-merge.js +168 -0
  78. package/dist/raw-archive.js +7 -6
  79. package/dist/recall-scope.d.ts +5 -4
  80. package/dist/recall-scope.js +7 -5
  81. package/dist/refine-llm.js +3 -2
  82. package/dist/reject-flow.js +6 -9
  83. package/dist/rejection.d.ts +2 -1
  84. package/dist/rejection.js +2 -1
  85. package/dist/rerankers/clef.d.ts +29 -0
  86. package/dist/rerankers/clef.js +182 -0
  87. package/dist/rerankers/index.js +3 -0
  88. package/dist/rerankers/jev.d.ts +11 -0
  89. package/dist/rerankers/jev.js +10 -5
  90. package/dist/rerankers/types.d.ts +16 -0
  91. package/dist/search.js +14 -2
  92. package/dist/secret-detect.d.ts +13 -1
  93. package/dist/secret-detect.js +33 -1
  94. package/dist/server.d.ts +9 -2
  95. package/dist/server.js +188 -411
  96. package/dist/session-digest.js +2 -1
  97. package/dist/shared.js +7 -6
  98. package/dist/skills.js +15 -14
  99. package/dist/store.js +10 -10
  100. package/dist/token-ledger.d.ts +4 -2
  101. package/dist/token-ledger.js +2 -2
  102. package/dist/version.d.ts +1 -1
  103. package/dist/version.js +1 -1
  104. package/extensions/openclaw-plugin/openclaw.plugin.json +1 -1
  105. package/extensions/openclaw-plugin/package.json +1 -1
  106. package/openclaw.plugin.json +1 -1
  107. package/package.json +5 -2
  108. package/dist/connectors/slack/ratelimit.d.ts +0 -9
  109. package/dist/connectors/slack/ratelimit.js +0 -18
package/dist/db.d.ts CHANGED
@@ -10,6 +10,8 @@ export interface DatabaseSyncLike {
10
10
  exec(sql: string): void;
11
11
  prepare(sql: string): StatementSyncLike;
12
12
  close(): void;
13
+ readonly isOpen?: boolean;
14
+ readonly isTransaction?: boolean;
13
15
  }
14
16
  export declare function getHippoDbPath(hippoRoot: string): string;
15
17
  export declare function getCurrentSchemaVersion(): number;
@@ -17,6 +19,16 @@ export declare function getCurrentSchemaVersion(): number;
17
19
  export declare class IncompatibleBinaryError extends Error {
18
20
  }
19
21
  export declare function isSqliteBusy(error: unknown): boolean;
22
+ export declare function execWithBusyRetry(db: DatabaseSyncLike, sql: string, timeoutMs?: number): void;
23
+ /** Lock wait for hook commands: under the 5 s prompt-hook budget even after a few skipped writes, and far above a normal write's hold. */
24
+ export declare const HOOK_DB_WAIT_MS = 1000;
25
+ /** A busy store made a command skip work: warn once per process (the holder is usually `hippo sleep`). */
26
+ export declare function noteStoreBusy(skipped: string): void;
27
+ /** Runs `fn` with one handle per store: openHippoDb reuses it and closeHippoDb leaves it open until `fn` settles or the process exits.
28
+ * `busyWaitMs` is the lock wait of every open inside `fn` that does not pass its own. */
29
+ export declare function withSharedStoreHandles<T>(fn: () => T | Promise<T>, opts?: {
30
+ busyWaitMs?: number;
31
+ }): Promise<T>;
20
32
  /** `busyWaitMs` shortens every lock wait of this open, for a hook that must finish inside its own timeout. */
21
33
  export declare function openHippoDb(hippoRoot: string, opts?: {
22
34
  busyWaitMs?: number;
package/dist/db.js CHANGED
@@ -6,6 +6,7 @@ import { createPhysicsTable } from './physics-state.js';
6
6
  import { cleanupArchivedMirrors } from './raw-archive-mirror-cleanup.js';
7
7
  import { PACKAGE_VERSION, compareSemver } from './version.js';
8
8
  import { deriveOriginProject, originFromSource, isGlobalStoreRoot } from './project-identity.js';
9
+ import { log } from './log.js';
9
10
  const require = createRequire(import.meta.url);
10
11
  // SAFETY: node:sqlite's DatabaseSync constructor genuinely has this shape at
11
12
  // runtime (Node's built-in synchronous SQLite module); there are no bundled
@@ -2607,7 +2608,7 @@ export function isSqliteBusy(error) {
2607
2608
  // busy_timeout covers neither of this file's two contended statements: SQLite
2608
2609
  // skips the busy handler for `PRAGMA journal_mode` and for a write that upgrades
2609
2610
  // a deferred read snapshot. Both need an explicit wait instead.
2610
- function execWithBusyRetry(db, sql, timeoutMs = 30000) {
2611
+ export function execWithBusyRetry(db, sql, timeoutMs = 30000) {
2611
2612
  const deadline = Date.now() + timeoutMs;
2612
2613
  const idle = new Int32Array(new SharedArrayBuffer(4));
2613
2614
  for (;;) {
@@ -2622,8 +2623,66 @@ function execWithBusyRetry(db, sql, timeoutMs = 30000) {
2622
2623
  }
2623
2624
  }
2624
2625
  }
2626
+ // Hook commands run on every prompt, so inside withSharedStoreHandles each store pays its pragmas, migration check and mirror cleanup once.
2627
+ const sharedHandles = new Map();
2628
+ const sharedSet = new WeakSet();
2629
+ let shareDepth = 0;
2630
+ let shareBusyWaitMs;
2631
+ /** Lock wait for hook commands: under the 5 s prompt-hook budget even after a few skipped writes, and far above a normal write's hold. */
2632
+ export const HOOK_DB_WAIT_MS = 1000;
2633
+ /** A busy store made a command skip work: warn once per process (the holder is usually `hippo sleep`). */
2634
+ export function noteStoreBusy(skipped) {
2635
+ log.once('store-busy', 'warn', `store busy (another hippo process holds the write lock); ${skipped}`);
2636
+ // A lock held past one full wait belongs to a long transaction, so the hook's later writes skip at once.
2637
+ for (const db of sharedHandles.values()) {
2638
+ if (db.isOpen !== false)
2639
+ db.exec('PRAGMA busy_timeout = 0');
2640
+ }
2641
+ }
2642
+ function closeSharedStoreHandles() {
2643
+ for (const db of sharedHandles.values()) {
2644
+ sharedSet.delete(db);
2645
+ if (db.isOpen !== false)
2646
+ db.close();
2647
+ }
2648
+ sharedHandles.clear();
2649
+ }
2650
+ /** Runs `fn` with one handle per store: openHippoDb reuses it and closeHippoDb leaves it open until `fn` settles or the process exits.
2651
+ * `busyWaitMs` is the lock wait of every open inside `fn` that does not pass its own. */
2652
+ export async function withSharedStoreHandles(fn, opts) {
2653
+ if (shareDepth++ === 0) {
2654
+ process.once('exit', closeSharedStoreHandles);
2655
+ shareBusyWaitMs = opts?.busyWaitMs;
2656
+ }
2657
+ try {
2658
+ return await fn();
2659
+ }
2660
+ finally {
2661
+ if (--shareDepth === 0) {
2662
+ process.off('exit', closeSharedStoreHandles);
2663
+ closeSharedStoreHandles();
2664
+ shareBusyWaitMs = undefined;
2665
+ }
2666
+ }
2667
+ }
2625
2668
  /** `busyWaitMs` shortens every lock wait of this open, for a hook that must finish inside its own timeout. */
2626
2669
  export function openHippoDb(hippoRoot, opts) {
2670
+ if (shareDepth === 0)
2671
+ return openOwnHippoDb(hippoRoot, opts);
2672
+ const busyWaitMs = opts?.busyWaitMs ?? shareBusyWaitMs;
2673
+ const key = `${path.resolve(getHippoDbPath(hippoRoot))}\0${busyWaitMs ?? ''}`;
2674
+ const shared = sharedHandles.get(key);
2675
+ if (shared?.isOpen && !shared.isTransaction)
2676
+ return shared;
2677
+ const db = openOwnHippoDb(hippoRoot, { busyWaitMs });
2678
+ // An open nested inside a transaction gets its own connection, as it did before sharing.
2679
+ if (!shared?.isOpen) {
2680
+ sharedHandles.set(key, db);
2681
+ sharedSet.add(db);
2682
+ }
2683
+ return db;
2684
+ }
2685
+ function openOwnHippoDb(hippoRoot, opts) {
2627
2686
  fs.mkdirSync(hippoRoot, { recursive: true });
2628
2687
  const db = new DatabaseSync(getHippoDbPath(hippoRoot));
2629
2688
  const busyWaitMs = opts?.busyWaitMs;
@@ -2937,6 +2996,8 @@ function backfillFtsIndex(db) {
2937
2996
  `);
2938
2997
  }
2939
2998
  export function closeHippoDb(db) {
2999
+ if (sharedSet.has(db))
3000
+ return;
2940
3001
  db.close();
2941
3002
  }
2942
3003
  export function getMeta(db, key, fallback = '') {
package/dist/decisions.js CHANGED
@@ -23,6 +23,7 @@
23
23
  * 'write_entry' (store.ts:1196) via the afterWrite hook, so a failure in any
24
24
  * step rolls all of them back. Pattern matches savePrediction (predictions.ts).
25
25
  */
26
+ import { BadRequestError, ConflictError, NotFoundError } from './api-errors.js';
26
27
  import { openHippoDb, closeHippoDb } from './db.js';
27
28
  import { writeEntry } from './store.js';
28
29
  import { assertTenantId } from './tenant.js';
@@ -73,7 +74,7 @@ const DECISION_COLS = `
73
74
  export function saveDecision(hippoRoot, tenantId, opts, actor = 'cli') {
74
75
  assertTenantId('saveDecision', tenantId);
75
76
  if (!opts.decisionText)
76
- throw new Error('saveDecision: decisionText is required');
77
+ throw new BadRequestError('saveDecision: decisionText is required');
77
78
  const now = new Date().toISOString();
78
79
  const content = opts.context
79
80
  ? `${opts.decisionText}\n\nContext: ${opts.context}`
@@ -103,10 +104,10 @@ export function saveDecision(hippoRoot, tenantId, opts, actor = 'cli') {
103
104
  // SAFETY: row shape matches the single `status` column named in the SELECT below.
104
105
  const pred = db.prepare(`SELECT status FROM decisions WHERE id = ? AND tenant_id = ?`).get(opts.supersedesDecisionId, tenantId);
105
106
  if (!pred) {
106
- throw new Error(`saveDecision: decision ${opts.supersedesDecisionId} to supersede not found for tenant ${tenantId}`);
107
+ throw new NotFoundError(`saveDecision: decision ${opts.supersedesDecisionId} to supersede not found for tenant ${tenantId}`);
107
108
  }
108
109
  if (pred.status !== 'active') {
109
- throw new Error(`saveDecision: decision ${opts.supersedesDecisionId} is not active (status='${pred.status}'); only active decisions can be superseded.`);
110
+ throw new ConflictError(`saveDecision: decision ${opts.supersedesDecisionId} is not active (status='${pred.status}'); only active decisions can be superseded.`);
110
111
  }
111
112
  }
112
113
  const result = db.prepare(`
@@ -127,7 +128,7 @@ export function saveDecision(hippoRoot, tenantId, opts, actor = 'cli') {
127
128
  WHERE id = ? AND tenant_id = ? AND status = 'active' AND id != ?
128
129
  `).run(decisionId, now, opts.supersedesDecisionId, tenantId, decisionId);
129
130
  if (sup.changes === 0) {
130
- throw new Error(`saveDecision: decision ${opts.supersedesDecisionId} could not be superseded (no longer active or self-reference).`);
131
+ throw new BadRequestError(`saveDecision: decision ${opts.supersedesDecisionId} could not be superseded (no longer active or self-reference).`);
131
132
  }
132
133
  appendAuditEvent(db, {
133
134
  tenantId,
@@ -191,15 +192,15 @@ export function closeDecision(hippoRoot, tenantId, id, actor = 'cli') {
191
192
  // SAFETY: row shape matches the single `status` column named in the SELECT above.
192
193
  const existing = db.prepare(`SELECT status FROM decisions WHERE id = ? AND tenant_id = ?`).get(id, tenantId);
193
194
  if (!existing) {
194
- throw new Error(`closeDecision: decision ${id} not found for tenant ${tenantId}`);
195
+ throw new NotFoundError(`closeDecision: decision ${id} not found for tenant ${tenantId}`);
195
196
  }
196
- throw new Error(`closeDecision: decision ${id} is not active (status='${existing.status}'); only active decisions can be closed.`);
197
+ throw new ConflictError(`closeDecision: decision ${id} is not active (status='${existing.status}'); only active decisions can be closed.`);
197
198
  }
198
199
  // SAFETY: row's shape matches the columns named in DECISION_COLS above.
199
200
  const row = db.prepare(`SELECT ${DECISION_COLS} FROM decisions WHERE id = ? AND tenant_id = ?`)
200
201
  .get(id, tenantId);
201
202
  if (!row)
202
- throw new Error(`closeDecision: decision ${id} not found after UPDATE`);
203
+ throw new NotFoundError(`closeDecision: decision ${id} not found after UPDATE`);
203
204
  appendAuditEvent(db, {
204
205
  tenantId,
205
206
  actor,
@@ -256,7 +257,7 @@ export function loadDecisions(hippoRoot, tenantId, opts = {}) {
256
257
  let rows;
257
258
  if (opts.status) {
258
259
  if (!VALID_DECISION_STATES.has(opts.status)) {
259
- throw new Error(`loadDecisions: status must be one of ${Array.from(VALID_DECISION_STATES).join('|')}; got ${opts.status}`);
260
+ throw new BadRequestError(`loadDecisions: status must be one of ${Array.from(VALID_DECISION_STATES).join('|')}; got ${opts.status}`);
260
261
  }
261
262
  // SAFETY: rows' shape matches the columns named in DECISION_COLS above.
262
263
  rows = db.prepare(`
package/dist/dedupe.js CHANGED
@@ -78,7 +78,7 @@ export function deduplicateStore(hippoRoot, options = {}) {
78
78
  const entriesByTenant = new Map();
79
79
  for (const entry of entries) {
80
80
  // EI2: also split by restricted scope, else a private copy can delete the readable one.
81
- const key = derivationPartitionKey(entry.tenantId, entry.scope);
81
+ const key = derivationPartitionKey(entry.tenantId, entry.scope, entry.origin_project);
82
82
  const bucket = entriesByTenant.get(key);
83
83
  if (bucket)
84
84
  bucket.push(entry);
package/dist/doctor.js CHANGED
@@ -10,10 +10,13 @@ import * as path from 'node:path';
10
10
  import { findHippoStoreDir } from './project-identity.js';
11
11
  import { getGlobalRoot } from './shared.js';
12
12
  import { isInitialized } from './store.js';
13
+ import { loadConfig } from './config.js';
13
14
  import { openHippoDbReadOnly, closeHippoDb, getSchemaVersion, getCurrentSchemaVersion, countTableRows, IncompatibleBinaryError } from './db.js';
14
15
  import { REPLAY_AFTER_MS, TRANSCRIPT_FILL_WINDOW_MS } from './compaction-record.js';
15
16
  import { isEmbeddingAvailable } from './embeddings.js';
16
17
  import { CODEX_TRUST_LINE, codexHomeDir, isCodexPresent, isJsonObject } from './hooks.js';
18
+ import { planUserGlobalRepair } from './project-merge.js';
19
+ import { resolveTenantId } from './tenant.js';
17
20
  /** Minimum Node.js version hippo supports (package.json engines). */
18
21
  export const MIN_NODE = '22.16.0';
19
22
  function versionAtLeast(actual, min) {
@@ -127,6 +130,25 @@ function compactionsCheck(db, now) {
127
130
  : { id: 'compactions', status: 'warn', detail: `cannot read the compaction records: ${message}` };
128
131
  }
129
132
  }
133
+ /** Merged rows older sleep saved as user-global, counted by the repair's own dry run so a truly user-global merge never warns. */
134
+ function projectsCheck(globalRoot) {
135
+ let db = null;
136
+ try {
137
+ db = openHippoDbReadOnly(globalRoot);
138
+ const r = planUserGlobalRepair(db, resolveTenantId({}));
139
+ const n = r.toProject.length + r.setAside.length;
140
+ return n === 0
141
+ ? { id: 'projects', status: 'pass', detail: 'no merged memories in the global store are tagged user-global by mistake' }
142
+ : { id: 'projects', status: 'warn', detail: `${n} merged memories in the global store are tagged user-global, so every project's context can show them`, fix: 'hippo projects repair --global (dry run; add --apply to write)' };
143
+ }
144
+ catch (err) {
145
+ return { id: 'projects', status: 'info', detail: `project tags not checked (${err instanceof Error ? err.message : String(err)})` };
146
+ }
147
+ finally {
148
+ if (db !== null)
149
+ closeHippoDb(db);
150
+ }
151
+ }
130
152
  /** Run every check. Never throws for a broken install; broken parts become failed checks. */
131
153
  export function runDoctor(opts) {
132
154
  const cwd = opts.cwd ?? process.cwd();
@@ -208,6 +230,12 @@ export function runDoctor(opts) {
208
230
  closeHippoDb(db);
209
231
  }
210
232
  }
233
+ const holdoutRateBp = store === null ? 0 : loadConfig(store).pilot.holdoutRateBp;
234
+ if (holdoutRateBp > 0) {
235
+ checks.push({ id: 'pilot', status: 'info', detail: `pilot holdout on: about ${holdoutRateBp / 100}% of sessions get no memories pushed by hippo (pilot.holdoutRateBp=${holdoutRateBp})` });
236
+ }
237
+ if (hasGlobal)
238
+ checks.push(projectsCheck(globalRoot));
211
239
  const claudeDir = path.join(home, '.claude');
212
240
  if (fs.existsSync(claudeDir)) {
213
241
  const settings = readJson(path.join(claudeDir, 'settings.json'));
package/dist/dormant.d.ts CHANGED
@@ -19,8 +19,8 @@
19
19
  */
20
20
  import type { DatabaseSyncLike } from './db.js';
21
21
  import type { MemoryEntry } from './memory.js';
22
- /** Why a memory went dormant: sleep's decay pass, or an imported agent memory whose note was deleted. */
23
- export type DormantReason = 'decay' | 'source-deleted';
22
+ /** Why a memory went dormant: sleep's decay pass, an imported agent memory whose note was deleted, or `hippo projects repair` splitting a two-project merge. */
23
+ export type DormantReason = 'decay' | 'source-deleted' | 'project-repair';
24
24
  /** One memory that sleep is moving out of active memory into the dormant store. */
25
25
  export interface DormantMove {
26
26
  /** The memory as it stood when it faded; restored verbatim apart from its recall clock. */
@@ -33,6 +33,7 @@
33
33
  import { getEmbedding, isEmbeddingAvailable, resolveEmbeddingModel, DEFAULT_EMBEDDING_MODEL, } from './embeddings.js';
34
34
  import { loadConfig } from './config.js';
35
35
  import { redactSecrets } from './secret-detect.js';
36
+ import { fetchWithRetry } from './http-retry.js';
36
37
  export const API_PROVIDER_KINDS = ['openai', 'voyage', 'cohere'];
37
38
  function isApiProviderKind(x) {
38
39
  return x === 'openai' || x === 'voyage' || x === 'cohere';
@@ -184,15 +185,14 @@ class ApiEmbeddingProvider {
184
185
  const url = `${this.baseUrl.replace(/\/$/, '')}/${spec.path}`;
185
186
  let resp;
186
187
  try {
187
- resp = await fetch(url, {
188
+ resp = await fetchWithRetry(url, {
188
189
  method: 'POST',
189
190
  headers: {
190
191
  'content-type': 'application/json',
191
192
  authorization: `Bearer ${key}`,
192
193
  },
193
194
  body: JSON.stringify(spec.buildBody(this.model, chunk.map(redactSecrets), role)),
194
- signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS),
195
- });
195
+ }, { timeoutMs: REQUEST_TIMEOUT_MS });
196
196
  }
197
197
  catch (err) {
198
198
  const msg = err instanceof Error ? err.message : String(err);
@@ -4,6 +4,7 @@
4
4
  * Falls back silently if the library is not installed.
5
5
  */
6
6
  import { MemoryEntry } from './memory.js';
7
+ import { type EmbeddingProvider } from './embedding-provider.js';
7
8
  export declare const DEFAULT_EMBEDDING_MODEL = "Xenova/all-MiniLM-L6-v2";
8
9
  export declare const EMBEDDING_MODEL_META_KEY = "embedding_model";
9
10
  /**
@@ -114,8 +115,7 @@ export declare function getEmbedding(text: string, model?: string, role?: Embedd
114
115
  */
115
116
  export declare function cosineSimilarity(a: number[], b: number[]): number;
116
117
  /**
117
- * Load the cached embedding index from disk.
118
- * Returns an empty object if the file doesn't exist or is corrupt.
118
+ * Load the cached embedding index; `{}` when the file is missing. A corrupt file is moved aside and rebuilt on the next embed; any other read error throws, so nothing saves over an index it could not read.
119
119
  */
120
120
  export declare function loadEmbeddingIndex(hippoRoot: string): Record<string, number[]>;
121
121
  /**
@@ -129,7 +129,7 @@ export declare function embedMemory(hippoRoot: string, entry: MemoryEntry, model
129
129
  /**
130
130
  * Embed all entries in hippoRoot that don't already have cached vectors.
131
131
  * Prunes orphaned embeddings for memories that no longer exist.
132
- * Returns the count of newly embedded entries.
132
+ * Returns the count of newly embedded entries. `provider` defaults to the store's configured one.
133
133
  */
134
- export declare function embedAll(hippoRoot: string, model?: string): Promise<number>;
134
+ export declare function embedAll(hippoRoot: string, model?: string, provider?: EmbeddingProvider): Promise<number>;
135
135
  //# sourceMappingURL=embeddings.d.ts.map
@@ -13,6 +13,7 @@ import { initializeParticle, savePhysicsState, loadPhysicsState, resetAllPhysics
13
13
  import { loadConfig } from './config.js';
14
14
  import { resolveEmbeddingProvider } from './embedding-provider.js';
15
15
  import { redactSecretsStrict } from './secret-detect.js';
16
+ import { log } from './log.js';
16
17
  // Use createRequire for synchronous module resolution check in ESM
17
18
  const _require = createRequire(import.meta.url);
18
19
  // Cached availability check
@@ -270,6 +271,9 @@ async function rebuildEmbeddingIndex(entries, provider) {
270
271
  if (vec && vec.length > 0) {
271
272
  rebuilt[entries[i].id] = vec;
272
273
  }
274
+ else {
275
+ noteSkippedEmbedding(entries[i].id);
276
+ }
273
277
  }
274
278
  return rebuilt;
275
279
  }
@@ -283,10 +287,15 @@ function resetPhysicsFromIndex(hippoRoot, entries, index) {
283
287
  closeHippoDb(db);
284
288
  }
285
289
  }
286
- catch {
287
- // Physics reset is best-effort; retrieval will still fall back gracefully.
290
+ catch (err) {
291
+ // Best effort: retrieval still falls back without physics state.
292
+ log.warn(`physics reset after reindex failed: ${err instanceof Error ? err.message : String(err)}`);
288
293
  }
289
294
  }
295
+ /** A provider's `[]` row is a swallowed per-item failure; name the memory so the gap can be traced. */
296
+ function noteSkippedEmbedding(id) {
297
+ log.warn('memory not embedded; the next embed run retries it', { id });
298
+ }
290
299
  /**
291
300
  * Get an embedding vector for a piece of text.
292
301
  * Returns an empty array if transformers is not available or fails.
@@ -314,7 +323,9 @@ export async function getEmbedding(text, model = DEFAULT_EMBEDDING_MODEL, role)
314
323
  // pipeline's documented tensor output shape.
315
324
  return Array.from(output.data);
316
325
  }
317
- catch {
326
+ catch (err) {
327
+ // The caller sees `[]` and names the memory; the reason only shows at debug.
328
+ log.debug(`local embedding failed: ${err instanceof Error ? err.message : String(err)}`);
318
329
  return [];
319
330
  }
320
331
  }
@@ -340,23 +351,66 @@ export function cosineSimilarity(a, b) {
340
351
  return Math.min(1, Math.max(-1, dot / denom));
341
352
  }
342
353
  const EMBEDDINGS_FILE = 'embeddings.json';
354
+ // Never equal to a real index identity, so the next embed run treats it as a model change and rebuilds every vector.
355
+ const QUARANTINED_INDEX_IDENTITY = 'quarantined-corrupt-index';
356
+ function isErrnoCode(err, code) {
357
+ return err instanceof Error && 'code' in err && err.code === code;
358
+ }
359
+ function parseEmbeddingIndex(raw) {
360
+ try {
361
+ const parsed = JSON.parse(raw);
362
+ // SAFETY: saveEmbeddingIndex is the only writer and always writes this shape; anything that is not an object is corrupt.
363
+ return parsed instanceof Object && !Array.isArray(parsed) ? parsed : null;
364
+ }
365
+ catch {
366
+ return null;
367
+ }
368
+ }
369
+ /** Move a corrupt index aside and flag a full rebuild, so no later save can write over the only copy of those bytes. */
370
+ function quarantineCorruptIndex(hippoRoot, fp) {
371
+ const aside = `${fp}.corrupt-${new Date().toISOString().replace(/[:.]/g, '-')}-${process.pid}`;
372
+ try {
373
+ fs.renameSync(fp, aside);
374
+ }
375
+ catch (err) {
376
+ if (isErrnoCode(err, 'ENOENT'))
377
+ return;
378
+ // A rename blocked by an open handle (Windows) still gets a copy kept; if the copy fails too, the throw stops the save.
379
+ fs.copyFileSync(fp, aside, fs.constants.COPYFILE_EXCL);
380
+ }
381
+ log.error(`${EMBEDDINGS_FILE} could not be parsed; kept it as ${path.basename(aside)} and the next embed rebuilds the index`, { hippoRoot });
382
+ try {
383
+ const db = openHippoDb(hippoRoot);
384
+ try {
385
+ setMeta(db, EMBEDDING_MODEL_META_KEY, QUARANTINED_INDEX_IDENTITY);
386
+ }
387
+ finally {
388
+ closeHippoDb(db);
389
+ }
390
+ }
391
+ catch (err) {
392
+ log.warn(`could not flag the embedding index for rebuild; run 'hippo embed' to restore vectors (${err instanceof Error ? err.message : String(err)})`, { hippoRoot });
393
+ }
394
+ }
343
395
  /**
344
- * Load the cached embedding index from disk.
345
- * Returns an empty object if the file doesn't exist or is corrupt.
396
+ * Load the cached embedding index; `{}` when the file is missing. A corrupt file is moved aside and rebuilt on the next embed; any other read error throws, so nothing saves over an index it could not read.
346
397
  */
347
398
  export function loadEmbeddingIndex(hippoRoot) {
348
399
  const fp = path.join(hippoRoot, EMBEDDINGS_FILE);
349
- if (!fs.existsSync(fp))
350
- return {};
400
+ let raw;
351
401
  try {
352
- // SAFETY: embeddings.json is written exclusively by saveEmbeddingIndex with
353
- // this exact shape; a corrupt or foreign file is caught by the try/catch
354
- // below and treated as an empty index.
355
- return JSON.parse(fs.readFileSync(fp, 'utf8'));
402
+ raw = fs.readFileSync(fp, 'utf8');
356
403
  }
357
- catch {
358
- return {};
404
+ catch (err) {
405
+ if (isErrnoCode(err, 'ENOENT'))
406
+ return {};
407
+ throw err;
359
408
  }
409
+ const index = parseEmbeddingIndex(raw);
410
+ if (index)
411
+ return index;
412
+ quarantineCorruptIndex(hippoRoot, fp);
413
+ return {};
360
414
  }
361
415
  /**
362
416
  * Save the embedding index to disk.
@@ -543,10 +597,9 @@ export async function embedMemory(hippoRoot, entry, model) {
543
597
  /**
544
598
  * Embed all entries in hippoRoot that don't already have cached vectors.
545
599
  * Prunes orphaned embeddings for memories that no longer exist.
546
- * Returns the count of newly embedded entries.
600
+ * Returns the count of newly embedded entries. `provider` defaults to the store's configured one.
547
601
  */
548
- export async function embedAll(hippoRoot, model) {
549
- const provider = resolveEmbeddingProvider(hippoRoot, { model });
602
+ export async function embedAll(hippoRoot, model, provider = resolveEmbeddingProvider(hippoRoot, { model })) {
550
603
  if (!provider.isAvailable()) {
551
604
  // A configured (non-disabled) API provider with a missing key is a
552
605
  // misconfiguration, not a no-op: surface it so programmatic callers of the
@@ -618,6 +671,9 @@ export async function embedAll(hippoRoot, model) {
618
671
  dirty = true;
619
672
  chunkDirty = true;
620
673
  }
674
+ else {
675
+ noteSkippedEmbedding(chunk[j].id);
676
+ }
621
677
  }
622
678
  if (chunkDirty)
623
679
  saveEmbeddingIndex(hippoRoot, index);
package/dist/extract.js CHANGED
@@ -3,6 +3,7 @@ import { writeEntry } from './store.js';
3
3
  import { loadConfig } from './config.js';
4
4
  import { RejectedValueError } from './rejection.js';
5
5
  import { redactSecrets } from './secret-detect.js';
6
+ import { fetchWithRetry, llmTimeoutMs } from './http-retry.js';
6
7
  import { neverAutoShareTags } from './shared.js';
7
8
  function isJsonString(value) {
8
9
  return typeof value === 'string';
@@ -24,7 +25,7 @@ export async function extractFacts(text, opts) {
24
25
  const fetchFn = opts.fetcher ?? fetch;
25
26
  let res;
26
27
  try {
27
- res = await fetchFn('https://api.anthropic.com/v1/messages', {
28
+ res = await fetchWithRetry('https://api.anthropic.com/v1/messages', {
28
29
  method: 'POST',
29
30
  headers: {
30
31
  'content-type': 'application/json',
@@ -36,7 +37,7 @@ export async function extractFacts(text, opts) {
36
37
  max_tokens: 1200,
37
38
  messages: [{ role: 'user', content: EXTRACTION_PROMPT + redactSecrets(text) }],
38
39
  }),
39
- });
40
+ }, { timeoutMs: llmTimeoutMs(), fetchFn });
40
41
  }
41
42
  catch (err) {
42
43
  opts.onError?.(`request failed: ${err instanceof Error ? err.message : String(err)}`);
@@ -91,22 +92,22 @@ export function storeExtractedFacts(hippoRoot, source, facts) {
91
92
  const baseHalfLifeDays = loadConfig(hippoRoot).defaultHalfLifeDays;
92
93
  for (const fact of facts) {
93
94
  const tags = ['extracted', ...inheritedTags, ...fact.tags];
94
- const entry = createMemory(fact.content, {
95
- layer: Layer.Semantic,
96
- tags,
97
- emotional_valence: fact.valence,
98
- confidence: 'inferred',
99
- source: source.source,
100
- extracted_from: source.id,
101
- scope: source.scope,
102
- // T1 executor check (2026-08-15 hardening pass): same defect as the
103
- // consolidate.ts merge/trace passes — createMemory with no tenantId
104
- // option stamps 'default' (memory.ts:535) regardless of the source
105
- // entry's own tenant. Thread it through so extracted facts land in
106
- // the same tenant as the episodic memory they were extracted from.
107
- tenantId: source.tenantId,
108
- baseHalfLifeDays,
109
- });
95
+ const entry = { ...createMemory(fact.content, {
96
+ layer: Layer.Semantic,
97
+ tags,
98
+ emotional_valence: fact.valence,
99
+ confidence: 'inferred',
100
+ source: source.source,
101
+ extracted_from: source.id,
102
+ scope: source.scope,
103
+ // T1 executor check (2026-08-15 hardening pass): same defect as the
104
+ // consolidate.ts merge/trace passes — createMemory with no tenantId
105
+ // option stamps 'default' (memory.ts:535) regardless of the source
106
+ // entry's own tenant. Thread it through so extracted facts land in
107
+ // the same tenant as the episodic memory they were extracted from.
108
+ tenantId: source.tenantId,
109
+ baseHalfLifeDays,
110
+ }), origin_project: source.origin_project };
110
111
  // AT1 containment: a refusal is per-VALUE — one rejected fact must not
111
112
  // drop the rest of this batch. writeEntry has already audited the
112
113
  // refusal (reject_refusal) before rethrowing, so skip-and-count here.
@@ -0,0 +1,21 @@
1
+ /** One retry policy for outbound HTTP: a timeout on every attempt, and backoff on 429 and 5xx only. */
2
+ export interface RetryPolicy {
3
+ /** Per-attempt limit; a stalled peer ends as a thrown `TimeoutError`, never a hang. */
4
+ timeoutMs: number;
5
+ /** Total attempts including the first. */
6
+ attempts?: number;
7
+ baseDelayMs?: number;
8
+ /** A Retry-After longer than this hands the response back, so a caller with its own long pause keeps it. */
9
+ maxDelayMs?: number;
10
+ fetchFn?: typeof fetch;
11
+ sleep?: (ms: number) => Promise<void>;
12
+ random?: () => number;
13
+ }
14
+ /** LLM calls (consolidation refine, DAG summaries, fact extraction) share one budget; `HIPPO_LLM_TIMEOUT_MS` overrides it. */
15
+ export declare function llmTimeoutMs(): number;
16
+ export declare function isRetryableStatus(status: number): boolean;
17
+ /** Retry-After as milliseconds: delta-seconds or an HTTP date; null when absent or unreadable. */
18
+ export declare function parseRetryAfterMs(header: string | null, now?: number): number | null;
19
+ /** `fetch` with a per-attempt timeout and up to `attempts` tries on 429 and 5xx; the last response comes back as is and transport errors throw at once. */
20
+ export declare function fetchWithRetry(url: string | URL, init: RequestInit, policy: RetryPolicy): Promise<Response>;
21
+ //# sourceMappingURL=http-retry.d.ts.map
@@ -0,0 +1,50 @@
1
+ /** One retry policy for outbound HTTP: a timeout on every attempt, and backoff on 429 and 5xx only. */
2
+ const DEFAULT_ATTEMPTS = 3;
3
+ const DEFAULT_BASE_DELAY_MS = 250;
4
+ const DEFAULT_MAX_DELAY_MS = 8_000;
5
+ const DEFAULT_LLM_TIMEOUT_MS = 60_000;
6
+ /** LLM calls (consolidation refine, DAG summaries, fact extraction) share one budget; `HIPPO_LLM_TIMEOUT_MS` overrides it. */
7
+ export function llmTimeoutMs() {
8
+ const parsed = Number.parseInt(process.env.HIPPO_LLM_TIMEOUT_MS ?? '', 10);
9
+ return parsed > 0 ? parsed : DEFAULT_LLM_TIMEOUT_MS;
10
+ }
11
+ export function isRetryableStatus(status) {
12
+ return status === 429 || (status >= 500 && status <= 599);
13
+ }
14
+ /** Retry-After as milliseconds: delta-seconds or an HTTP date; null when absent or unreadable. */
15
+ export function parseRetryAfterMs(header, now = Date.now()) {
16
+ if (header === null || header.trim() === '')
17
+ return null;
18
+ const seconds = Number(header);
19
+ if (Number.isFinite(seconds))
20
+ return Math.max(0, seconds * 1000);
21
+ const at = Date.parse(header);
22
+ return Number.isNaN(at) ? null : Math.max(0, at - now);
23
+ }
24
+ const realSleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
25
+ /** `fetch` with a per-attempt timeout and up to `attempts` tries on 429 and 5xx; the last response comes back as is and transport errors throw at once. */
26
+ export async function fetchWithRetry(url, init, policy) {
27
+ const fetchFn = policy.fetchFn ?? fetch;
28
+ const attempts = Math.max(1, policy.attempts ?? DEFAULT_ATTEMPTS);
29
+ const baseDelayMs = policy.baseDelayMs ?? DEFAULT_BASE_DELAY_MS;
30
+ const maxDelayMs = policy.maxDelayMs ?? DEFAULT_MAX_DELAY_MS;
31
+ const sleep = policy.sleep ?? realSleep;
32
+ const random = policy.random ?? Math.random;
33
+ for (let attempt = 1;; attempt++) {
34
+ const timeout = AbortSignal.timeout(policy.timeoutMs);
35
+ const signal = init.signal ? AbortSignal.any([init.signal, timeout]) : timeout;
36
+ const res = await fetchFn(url, { ...init, signal });
37
+ if (!isRetryableStatus(res.status) || attempt >= attempts)
38
+ return res;
39
+ const retryAfter = parseRetryAfterMs(res.headers.get('retry-after'));
40
+ if (retryAfter !== null && retryAfter > maxDelayMs)
41
+ return res;
42
+ // Jitter over the upper half keeps parallel callers from retrying in lockstep.
43
+ const ceiling = Math.min(maxDelayMs, baseDelayMs * 2 ** (attempt - 1));
44
+ const delay = retryAfter ?? ceiling / 2 + random() * (ceiling / 2);
45
+ // Frees the pooled socket before the next attempt.
46
+ await res.body?.cancel();
47
+ await sleep(delay);
48
+ }
49
+ }
50
+ //# sourceMappingURL=http-retry.js.map
@@ -13,6 +13,14 @@ export declare class HttpError extends Error {
13
13
  }
14
14
  export declare class BodyTooLargeError extends Error {
15
15
  }
16
+ /** The status and client-facing message for one failed request. */
17
+ export interface ApiErrorReply {
18
+ status: number;
19
+ message: string;
20
+ }
21
+ export declare const INTERNAL_ERROR_MESSAGE = "internal server error";
22
+ /** Maps by class so rewording a message never moves a status; an untyped error is a 500 whose text stays in the server log. */
23
+ export declare function mapApiError<E>(err: E): ApiErrorReply;
16
24
  export declare function sendJson<T>(res: ServerResponse, status: number, body: T): void;
17
25
  /**
18
26
  * Read the entire request body into a Buffer. Caps at MAX_BODY_BYTES to keep
package/dist/http-util.js CHANGED
@@ -1,3 +1,4 @@
1
+ import { ApiError } from './api-errors.js';
1
2
  export function isJsonObjectRecord(value) {
2
3
  return value !== undefined && value !== null && typeof value === 'object' && !Array.isArray(value);
3
4
  }
@@ -19,6 +20,15 @@ export class HttpError extends Error {
19
20
  }
20
21
  export class BodyTooLargeError extends Error {
21
22
  }
23
+ export const INTERNAL_ERROR_MESSAGE = 'internal server error';
24
+ /** Maps by class so rewording a message never moves a status; an untyped error is a 500 whose text stays in the server log. */
25
+ export function mapApiError(err) {
26
+ if (err instanceof HttpError || err instanceof ApiError)
27
+ return { status: err.status, message: err.message };
28
+ if (err instanceof BodyTooLargeError)
29
+ return { status: 413, message: err.message };
30
+ return { status: 500, message: INTERNAL_ERROR_MESSAGE };
31
+ }
22
32
  export function sendJson(res, status, body) {
23
33
  res.writeHead(status, JSON_HEADERS);
24
34
  res.end(JSON.stringify(body));
@@ -20,6 +20,8 @@ export interface ImportResult {
20
20
  /** K1 vault import: rows archived this run (changed + source-deleted). In a
21
21
  * dryRun this is the would-be count (a true deletion-sync preview). */
22
22
  archived?: number;
23
+ /** Entries stored with secret-shaped text redacted; optional for the same published-surface reason as `rejected`. */
24
+ redacted?: number;
23
25
  entries: MemoryEntry[];
24
26
  }
25
27
  export interface ImportOptions {