hippo-memory 1.59.0 → 1.60.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. package/dist/api.d.ts +2 -0
  2. package/dist/api.js +25 -9
  3. package/dist/audit.js +1 -0
  4. package/dist/autolearn.js +5 -2
  5. package/dist/capture.js +8 -6
  6. package/dist/cli.d.ts +355 -0
  7. package/dist/cli.js +1236 -1081
  8. package/dist/compaction-record.js +7 -6
  9. package/dist/config.js +12 -11
  10. package/dist/connectors/github/cli-impl.js +1 -0
  11. package/dist/consolidate.js +4 -3
  12. package/dist/dag.js +7 -7
  13. package/dist/dashboard.js +4 -2
  14. package/dist/db.js +16 -3
  15. package/dist/delivery-recorder.js +1 -0
  16. package/dist/doctor.js +1 -0
  17. package/dist/dormant.js +1 -0
  18. package/dist/embedding-provider.js +3 -2
  19. package/dist/embeddings.js +10 -7
  20. package/dist/extract.js +4 -3
  21. package/dist/graph.js +3 -8
  22. package/dist/handoff.js +3 -0
  23. package/dist/hooks.js +3 -0
  24. package/dist/importers.js +6 -3
  25. package/dist/incidents.js +1 -0
  26. package/dist/judgment.js +5 -2
  27. package/dist/mcp/server.d.ts +5 -0
  28. package/dist/mcp/server.js +32 -13
  29. package/dist/memory.d.ts +5 -3
  30. package/dist/processes.js +1 -0
  31. package/dist/project-identity.d.ts +1 -1
  32. package/dist/project-identity.js +9 -4
  33. package/dist/raw-archive-mirror-cleanup.js +2 -1
  34. package/dist/recall-trace.d.ts +2 -2
  35. package/dist/recall-trace.js +10 -14
  36. package/dist/refine-llm.js +18 -10
  37. package/dist/rerankers/clef.d.ts +2 -2
  38. package/dist/rerankers/clef.js +68 -27
  39. package/dist/rerankers/cross-encoder.js +6 -4
  40. package/dist/rerankers/jev.d.ts +3 -1
  41. package/dist/rerankers/jev.js +20 -15
  42. package/dist/rerankers/llm.js +3 -3
  43. package/dist/same-text.d.ts +2 -0
  44. package/dist/same-text.js +4 -0
  45. package/dist/scheduler.js +1 -0
  46. package/dist/search.js +4 -2
  47. package/dist/secret-detect.js +1 -0
  48. package/dist/server.js +29 -12
  49. package/dist/shared.js +21 -19
  50. package/dist/stdin.js +1 -0
  51. package/dist/store.d.ts +33 -1
  52. package/dist/store.js +117 -16
  53. package/dist/token-ledger.js +1 -0
  54. package/dist/version.d.ts +1 -1
  55. package/dist/version.js +1 -1
  56. package/extensions/openclaw-plugin/openclaw.plugin.json +1 -1
  57. package/extensions/openclaw-plugin/package.json +1 -1
  58. package/openclaw.plugin.json +1 -1
  59. package/package.json +2 -1
@@ -25,8 +25,13 @@ interface McpResponse {
25
25
  error?: {
26
26
  code: number;
27
27
  message: string;
28
+ data?: {
29
+ requestId: string;
30
+ };
28
31
  };
29
32
  }
33
+ /** JSON-RPC reply for a request that threw: typed API errors keep their text; anything else is logged and answered generically. */
34
+ export declare function mcpErrorResponse<E>(id: McpResponse['id'], err: E, requestId?: string): McpResponse;
30
35
  export type { McpRequest, McpResponse };
31
36
  /**
32
37
  * Optional execution context threaded from a non-stdio transport. When the
@@ -9,14 +9,17 @@
9
9
  */
10
10
  import * as fs from 'fs';
11
11
  import * as path from 'path';
12
+ import { randomUUID } from 'node:crypto';
13
+ import { INTERNAL_ERROR_MESSAGE, mapApiError } from '../http-util.js';
14
+ import { log } from '../log.js';
12
15
  import { createMemory, Layer, calculateStrength, } from '../memory.js';
13
16
  import { fitBudget, estimateTokens } from '../search.js';
14
17
  import { evalNow } from '../ablation.js';
15
- import { loadAllEntries, writeEntry, readEntry, listMemoryConflicts, resolveConflict, countCreatedSinceLastSleep } from '../store.js';
18
+ import { loadStrengthRows, loadTextsHoldingWords, writeEntry, readEntry, listMemoryConflicts, resolveConflict, countCreatedSinceLastSleep } from '../store.js';
16
19
  import { shareMemory, listPeers, getGlobalRoot, initGlobal } from '../shared.js';
17
20
  import { consolidate } from '../consolidate.js';
18
21
  import { fetchGitLog, extractLessons, partitionLessons, isGitRepo } from '../autolearn.js';
19
- import { dropHeldCopies, duplicateKey, storedTextKeys } from '../same-text.js';
22
+ import { dropHeldCopies, duplicateKey, longestWord, storedTextKeys } from '../same-text.js';
20
23
  import { loadConfig } from '../config.js';
21
24
  import { confidenceLabel } from '../memory.js';
22
25
  import { resolveTenantId } from '../tenant.js';
@@ -50,6 +53,18 @@ export function findHippoRoot(cwd = process.cwd(), opts) {
50
53
  const global = getGlobalRoot();
51
54
  return fs.existsSync(global) ? global : null;
52
55
  }
56
+ /** JSON-RPC reply for a request that threw: typed API errors keep their text; anything else is logged and answered generically. */
57
+ export function mcpErrorResponse(id, err, requestId = randomUUID()) {
58
+ const { status, message } = mapApiError(err);
59
+ if (status !== 500)
60
+ return { jsonrpc: '2.0', id, error: { code: -32603, message } };
61
+ log.error(`mcp request failed: ${err instanceof Error ? err.message : String(err)}`, { requestId });
62
+ return {
63
+ jsonrpc: '2.0',
64
+ id,
65
+ error: { code: -32603, message: `${INTERNAL_ERROR_MESSAGE} (request id ${requestId})`, data: { requestId } },
66
+ };
67
+ }
53
68
  /**
54
69
  * The api-layer actor for a tool call. Stdio (no ctx) is the local operator
55
70
  * and runs as admin; over HTTP the transport's authenticated role is used, so
@@ -477,7 +492,7 @@ function resolveClientKey(ctx) {
477
492
  function createGlobalStoreOnFirstRun() {
478
493
  initGlobal();
479
494
  const root = getGlobalRoot();
480
- console.error(`hippo: no memory store found; created the global store at ${root}. Run \`hippo init\` in a project for a project store.`);
495
+ log.warn(`no memory store found; created the global store at ${root}. Run \`hippo init\` in a project for a project store.`);
481
496
  return root;
482
497
  }
483
498
  // ── Token ledger (ROADMAP TE0) ──
@@ -524,7 +539,7 @@ async function executeTool(name, args, ctx) {
524
539
  // resolve tenant from HIPPO_TENANT.
525
540
  const hippoRoot = ctx?.hippoRoot ?? findHippoRoot() ?? createGlobalStoreOnFirstRun();
526
541
  const config = loadConfig(hippoRoot);
527
- // A5: every loadAllEntries() in this server returns to the caller and is
542
+ // A5: every store read in this server returns to the caller and is
528
543
  // tenant-isolated. Resolved once per tool call: prefer the transport's
529
544
  // ctx.tenantId so an HTTP Bearer for tenant B doesn't drop to HIPPO_TENANT.
530
545
  const tenantId = ctx?.tenantId ?? resolveTenantId({});
@@ -866,7 +881,7 @@ async function executeTool(name, args, ctx) {
866
881
  // Fire-and-forget (never block the response); an unhandled rejection would kill the server, so log it.
867
882
  consolidate(hippoRoot)
868
883
  .catch((err) => {
869
- console.error(`auto-sleep consolidate failed (tenant ${tenantId}): ${err instanceof Error ? err.message : err}`);
884
+ log.error(`auto-sleep consolidate failed (tenant ${tenantId}): ${err instanceof Error ? err.message : String(err)}`);
870
885
  })
871
886
  .finally(() => autoSleepInFlight.delete(hippoRoot));
872
887
  }
@@ -922,7 +937,8 @@ async function executeTool(name, args, ctx) {
922
937
  + formatMemories(result.entries);
923
938
  }
924
939
  case 'hippo_status': {
925
- const entries = loadAllEntries(hippoRoot, tenantId);
940
+ // Every row counts toward the averages, so this scans the store, but without its text.
941
+ const entries = loadStrengthRows(hippoRoot, tenantId);
926
942
  const now = evalNow(); // honors HIPPO_FAKE_NOW (eval-only; see ablation.ts)
927
943
  let atRisk = 0;
928
944
  let totalStrength = 0;
@@ -965,7 +981,7 @@ async function executeTool(name, args, ctx) {
965
981
  let added = 0;
966
982
  let skipped = 0;
967
983
  let rejected = 0;
968
- const keys = storedTextKeys(loadAllEntries(hippoRoot, tenantId));
984
+ const keys = storedTextKeys(loadTextsHoldingWords(hippoRoot, tenantId, lessons.map(longestWord)));
969
985
  for (const lesson of lessons) {
970
986
  if (keys.has(duplicateKey(lesson))) {
971
987
  skipped++;
@@ -1138,17 +1154,20 @@ function dispatch(body) {
1138
1154
  req = JSON.parse(body);
1139
1155
  }
1140
1156
  catch {
1141
- return; // skip malformed
1157
+ log.debug('mcp: skipped a frame that is not valid JSON');
1158
+ return;
1142
1159
  }
1143
1160
  if (!req.method)
1144
1161
  return;
1145
1162
  if (req.method.startsWith('notifications/')) {
1146
- handleMcpRequest(req).catch(() => { });
1163
+ handleMcpRequest(req).catch((err) => {
1164
+ log.error(`mcp notification ${req.method} failed: ${err instanceof Error ? err.message : String(err)}`);
1165
+ });
1147
1166
  return;
1148
1167
  }
1149
1168
  handleMcpRequest(req).then((resp) => { if (resp)
1150
1169
  send(resp); }).catch((err) => {
1151
- send({ jsonrpc: '2.0', id: req.id, error: { code: -32603, message: err?.message ?? 'Internal error' } });
1170
+ send(mcpErrorResponse(req.id, err));
1152
1171
  });
1153
1172
  }
1154
1173
  /**
@@ -1171,10 +1190,10 @@ export function startStdioLoop() {
1171
1190
  });
1172
1191
  process.stdin.on('end', () => process.exit(0));
1173
1192
  process.on('uncaughtException', (err) => {
1174
- process.stderr.write(`hippo-mcp uncaught: ${err?.message ?? err}\n`);
1193
+ log.error(`mcp uncaught: ${err instanceof Error ? err.message : String(err)}`);
1175
1194
  });
1176
1195
  process.on('unhandledRejection', (err) => {
1177
- process.stderr.write(`hippo-mcp unhandled: ${err instanceof Error ? err.message : String(err)}\n`);
1196
+ log.error(`mcp unhandled: ${err instanceof Error ? err.message : String(err)}`);
1178
1197
  });
1179
1198
  }
1180
1199
  // Auto-start when invoked as the main module (node dist/mcp/server.js or via
@@ -1196,7 +1215,7 @@ const isMainModule = (() => {
1196
1215
  return import.meta.url === mainUrl || import.meta.url === `file:///${argv1.replace(/\\/g, '/')}`;
1197
1216
  }
1198
1217
  catch {
1199
- return false;
1218
+ return false; // an unreadable argv means this file was imported, not run; never start the stdio loop then
1200
1219
  }
1201
1220
  })();
1202
1221
  if (isMainModule) {
package/dist/memory.d.ts CHANGED
@@ -135,19 +135,21 @@ export declare function _resetLossAversionRatioCacheForTests(): void;
135
135
  * Modulates effective half-life: memories with consistent positive outcomes
136
136
  * decay slower; consistent negative outcomes decay faster.
137
137
  */
138
- export declare function calculateRewardFactor(entry: MemoryEntry): number;
138
+ export declare function calculateRewardFactor(entry: Pick<MemoryEntry, 'outcome_positive' | 'outcome_negative'>): number;
139
139
  /**
140
140
  * Net wrongness: bad outcome marks past good ones, never below zero.
141
141
  * Strength halves per unit (capped at 3) and recall stops strengthening
142
142
  * the memory, so a correction outranks pinning, error tags and heavy recall.
143
143
  */
144
- export declare function netWrong(entry: MemoryEntry): number;
144
+ export declare function netWrong(entry: Pick<MemoryEntry, 'outcome_positive' | 'outcome_negative'>): number;
145
145
  /**
146
146
  * Options for decay basis.
147
147
  * - clock: wall-clock time (default pre-v0.15)
148
148
  * - session: decay by sleep cycle count (for intermittent agents)
149
149
  * - adaptive: auto-scale half-life by session frequency (default v0.15+)
150
150
  */
151
+ /** What calculateStrength reads, so a caller can score a row without loading its text. */
152
+ export type StrengthInputs = Pick<MemoryEntry, 'pinned' | 'created' | 'last_retrieved' | 'half_life_days' | 'retrieval_count' | 'emotional_valence' | 'outcome_positive' | 'outcome_negative'>;
151
153
  export interface DecayOptions {
152
154
  decayBasis?: 'clock' | 'session' | 'adaptive';
153
155
  /** Average interval between sleep cycles, in days. Used by 'adaptive' and 'session' modes. */
@@ -166,7 +168,7 @@ export interface DecayOptions {
166
168
  *
167
169
  * Pinned memories skip time decay; being marked wrong still fades them (netWrong).
168
170
  */
169
- export declare function calculateStrength(entry: MemoryEntry, now?: Date, options?: DecayOptions): number;
171
+ export declare function calculateStrength(entry: StrengthInputs, now?: Date, options?: DecayOptions): number;
170
172
  /**
171
173
  * Derive half-life based on signals, as per PLAN.md table.
172
174
  */
package/dist/processes.js CHANGED
@@ -91,6 +91,7 @@ function parseSteps(raw) {
91
91
  return [];
92
92
  }
93
93
  catch {
94
+ // Legacy garbage reads back as no steps, per the docblock.
94
95
  return [];
95
96
  }
96
97
  }
@@ -7,7 +7,7 @@
7
7
  * directory is the project root; if none exists, the nearest ancestor
8
8
  * containing `.git` (directory or worktree file).
9
9
  * - The user home directory is NEVER a project, even though it contains the
10
- * global store at `~/.hippo`. Reaching home ends the walk.
10
+ * global store at `~/.hippo`. Reaching home or the temp root (inside home on Windows) ends the walk.
11
11
  * - A directory with no marker anywhere up the walk is NOT a project: it
12
12
  * resolves to the user-global identity (empty name), so memories written
13
13
  * there stay injectable everywhere (matches pre-isolation behavior).
@@ -1,6 +1,7 @@
1
1
  import * as fs from 'fs';
2
2
  import * as os from 'os';
3
3
  import * as path from 'path';
4
+ import { log } from './log.js';
4
5
  const MAX_WALK_DEPTH = 64;
5
6
  const identityCache = new Map();
6
7
  /** Clear the per-process identity cache (test seam). */
@@ -15,7 +16,8 @@ export function realpathOrResolve(p) {
15
16
  try {
16
17
  return fs.realpathSync.native(p);
17
18
  }
18
- catch {
19
+ catch (err) {
20
+ log.debug(`project identity: realpath fell back to resolve for ${p}: ${err instanceof Error ? err.message : String(err)}`);
19
21
  return path.resolve(p);
20
22
  }
21
23
  }
@@ -43,7 +45,8 @@ function isDirectoryAt(p) {
43
45
  try {
44
46
  return fs.statSync(p).isDirectory();
45
47
  }
46
- catch {
48
+ catch (err) {
49
+ log.debug(`project identity: no marker at ${p}: ${err instanceof Error ? err.message : String(err)}`);
47
50
  return false;
48
51
  }
49
52
  }
@@ -60,9 +63,11 @@ export function resolveProjectIdentity(cwd, opts) {
60
63
  return cached;
61
64
  }
62
65
  const home = realpathOrResolve(opts?.homeDir ?? os.homedir());
63
- const stopDir = opts?.stopDir ? realpathOrResolve(opts.stopDir) : null;
66
+ const stops = [realpathOrResolve(os.tmpdir())];
67
+ if (opts?.stopDir)
68
+ stops.push(realpathOrResolve(opts.stopDir));
64
69
  const start = realpathOrResolve(startInput);
65
- const { hippoRoot, gitRoot, reachedHome } = walkProjectMarkers(start, home, stopDir === null ? [] : [stopDir]);
70
+ const { hippoRoot, gitRoot, reachedHome } = walkProjectMarkers(start, home, stops);
66
71
  let identity;
67
72
  const root = hippoRoot ?? gitRoot;
68
73
  if (root !== null) {
@@ -1,5 +1,6 @@
1
1
  import * as fs from 'fs';
2
2
  import * as path from 'path';
3
+ import { log } from './log.js';
3
4
  const LAYERS = ['episodic', 'buffer', 'semantic'];
4
5
  const MAX_WARN_LOGS = 5;
5
6
  /**
@@ -42,7 +43,7 @@ export function cleanupArchivedMirrors(hippoRoot, db) {
42
43
  catch (err) {
43
44
  allOk = false;
44
45
  if (warnCount < MAX_WARN_LOGS) {
45
- console.warn(`cleanupArchivedMirrors: unlink failed for ${filePath} (will retry on next DB open):`, err);
46
+ log.warn(`cleanupArchivedMirrors: unlink failed for ${filePath} (will retry on next DB open): ${err instanceof Error ? err.message : String(err)}`);
46
47
  warnCount += 1;
47
48
  }
48
49
  }
@@ -102,8 +102,8 @@ export interface RecordTraceOutcomeInput {
102
102
  * this function from caller-side state (`last_trace_id` / applied outcome
103
103
  * ids) that can go stale relative to the trace it names — a forgotten
104
104
  * memory, a tenant switch mid-session, or a race between two callers. Two
105
- * checks run before the insert, both skip silently (console.error one
106
- * line) rather than throw:
105
+ * checks run before the insert, both skip with one log.warn line
106
+ * rather than throw:
107
107
  * 1. The named trace must exist and belong to `input.tenantId` — a
108
108
  * tenant mismatch or a dangling id (deleted trace) skips.
109
109
  * 2. `input.memoryIds` is intersected against the trace's OWN
@@ -18,6 +18,7 @@
18
18
  import { createHash } from 'node:crypto';
19
19
  import { openHippoDb, closeHippoDb } from './db.js';
20
20
  import { DELIVERY_LEDGER_VERSION } from './delivery-recorder.js';
21
+ import { log } from './log.js';
21
22
  /**
22
23
  * Strip a RerankStep down to {stage, multiplier, scoreBefore, scoreAfter}
23
24
  * before persisting (F3 privacy fix, codex cross-model finding). `note` is
@@ -76,8 +77,7 @@ export function writeRecallTrace(db, input) {
76
77
  }
77
78
  }
78
79
  catch (error) {
79
- // eslint-disable-next-line no-console
80
- console.error(`[hippo] recall trace write failed: ${error instanceof Error ? error.message : String(error)}`);
80
+ log.error(`recall trace write failed: ${error instanceof Error ? error.message : String(error)}`);
81
81
  return null;
82
82
  }
83
83
  }
@@ -113,8 +113,7 @@ export function writeRecallTraceAtRoot(root, input) {
113
113
  db = openHippoDb(root);
114
114
  }
115
115
  catch (error) {
116
- // eslint-disable-next-line no-console
117
- console.error(`[hippo] recall trace connection failed: ${error instanceof Error ? error.message : String(error)}`);
116
+ log.error(`recall trace connection failed: ${error instanceof Error ? error.message : String(error)}`);
118
117
  return null;
119
118
  }
120
119
  try {
@@ -140,8 +139,8 @@ export function writeRecallTraceAtRoot(root, input) {
140
139
  * this function from caller-side state (`last_trace_id` / applied outcome
141
140
  * ids) that can go stale relative to the trace it names — a forgotten
142
141
  * memory, a tenant switch mid-session, or a race between two callers. Two
143
- * checks run before the insert, both skip silently (console.error one
144
- * line) rather than throw:
142
+ * checks run before the insert, both skip with one log.warn line
143
+ * rather than throw:
145
144
  * 1. The named trace must exist and belong to `input.tenantId` — a
146
145
  * tenant mismatch or a dangling id (deleted trace) skips.
147
146
  * 2. `input.memoryIds` is intersected against the trace's OWN
@@ -158,8 +157,7 @@ export function recordTraceOutcome(db, input) {
158
157
  // SELECT above; sqlite returns undefined when no row matches.
159
158
  const trace = db.prepare(`SELECT tenant_id FROM recall_traces WHERE id = ?`).get(input.traceId);
160
159
  if (!trace || trace.tenant_id !== input.tenantId) {
161
- // eslint-disable-next-line no-console
162
- console.error(`[hippo] recall trace outcome skipped: trace ${input.traceId} missing or tenant mismatch`);
160
+ log.warn(`recall trace outcome skipped: trace ${input.traceId} missing or tenant mismatch`);
163
161
  return;
164
162
  }
165
163
  // SAFETY: row shape matches the single `memory_id` column named in the
@@ -170,8 +168,7 @@ export function recordTraceOutcome(db, input) {
170
168
  const members = new Set(memberRows.map((r) => r.memory_id));
171
169
  const credited = input.memoryIds.filter((id) => members.has(id));
172
170
  if (credited.length === 0) {
173
- // eslint-disable-next-line no-console
174
- console.error(`[hippo] recall trace outcome skipped: no credited ids intersect trace ${input.traceId}'s results`);
171
+ log.warn(`recall trace outcome skipped: no credited ids intersect trace ${input.traceId}'s results`);
175
172
  return;
176
173
  }
177
174
  db.prepare(`
@@ -180,8 +177,7 @@ export function recordTraceOutcome(db, input) {
180
177
  `).run(input.traceId, new Date().toISOString(), input.tenantId, input.outcome, JSON.stringify(credited));
181
178
  }
182
179
  catch (error) {
183
- // eslint-disable-next-line no-console
184
- console.error(`[hippo] recall trace outcome write failed: ${error instanceof Error ? error.message : String(error)}`);
180
+ log.error(`recall trace outcome write failed: ${error instanceof Error ? error.message : String(error)}`);
185
181
  }
186
182
  }
187
183
  /** Pruned on write, counted back from the event's ts capped at the real clock, so a far-future fake time spares real rows. */
@@ -270,7 +266,7 @@ export function writeDeliveryEvent(db, input) {
270
266
  }
271
267
  }
272
268
  catch (error) {
273
- // eslint-disable-next-line no-console
269
+ // The prompt hook's stderr shows this exact `[hippo] delivery ledger` line, so it stays off the logger's format.
274
270
  console.error(`[hippo] delivery ledger write failed: ${error instanceof Error ? error.message : String(error)}`);
275
271
  return null;
276
272
  }
@@ -282,7 +278,7 @@ export function writeDeliveryEventAtRoot(root, input) {
282
278
  db = openHippoDb(root, { busyWaitMs: DELIVERY_LEDGER_WAIT_MS });
283
279
  }
284
280
  catch (error) {
285
- // eslint-disable-next-line no-console
281
+ // Same hook stderr line as writeDeliveryEvent above.
286
282
  console.error(`[hippo] delivery ledger write failed: ${error instanceof Error ? error.message : String(error)}`);
287
283
  return null;
288
284
  }
@@ -14,8 +14,9 @@
14
14
  */
15
15
  import { Layer } from './memory.js';
16
16
  import { loadAllEntries, readEntry, writeEntry } from './store.js';
17
- import { redactSecrets } from './secret-detect.js';
17
+ import { redactSecretsStrict } from './secret-detect.js';
18
18
  import { fetchWithRetry, llmTimeoutMs } from './http-retry.js';
19
+ import { log } from './log.js';
19
20
  const REFINED_TAG = 'llm-refined';
20
21
  const CONSOLIDATED_MARKERS = [
21
22
  '[Consolidated from',
@@ -31,7 +32,7 @@ export async function refineSemanticMemory(merged, sources, opts) {
31
32
  const fetchFn = opts.fetcher ?? fetch;
32
33
  const sourceBlock = sources
33
34
  .slice(0, 8)
34
- .map((s, i) => `[source ${i + 1}] ${redactSecrets(s.content).slice(0, 400)}`)
35
+ .map((s, i) => `[source ${i + 1}] ${redactSecretsStrict(s.content).slice(0, 400)}`)
35
36
  .join('\n\n');
36
37
  const prompt = `You are refining a semantic memory in an agent's memory store. The rule-based consolidator merged several related episodic memories into one, but the output is clumsy. Produce a single coherent semantic memory that captures the underlying principle.
37
38
 
@@ -43,7 +44,7 @@ Rules:
43
44
  - Do NOT include the "[Consolidated from N ...]" marker.
44
45
 
45
46
  Current merged content:
46
- ${redactSecrets(merged)}
47
+ ${redactSecretsStrict(merged)}
47
48
 
48
49
  Source memories (up to 8 shown):
49
50
  ${sourceBlock}`;
@@ -63,24 +64,31 @@ ${sourceBlock}`;
63
64
  }),
64
65
  }, { timeoutMs: llmTimeoutMs(), fetchFn });
65
66
  }
66
- catch {
67
+ catch (err) {
68
+ log.warn(`refine: request failed: ${err instanceof Error ? err.message : String(err)}`);
67
69
  return null;
68
70
  }
69
- if (!res.ok)
71
+ if (!res.ok) {
72
+ log.warn(`refine: API answered HTTP ${res.status}`);
70
73
  return null;
74
+ }
75
+ let text;
71
76
  try {
72
77
  // SAFETY: data is the Anthropic Messages API response body; the
73
78
  // documented response shape is `{ content: [{ type, text, ... }] }`
74
79
  // for a text-generating request like this one.
75
80
  const data = await res.json();
76
- const text = data.content?.[0]?.text?.trim() ?? '';
77
- if (text.length < 10)
78
- return null;
79
- return text;
81
+ text = data.content?.[0]?.text?.trim() ?? '';
82
+ }
83
+ catch (err) {
84
+ log.warn(`refine: unreadable response: ${err instanceof Error ? err.message : String(err)}`);
85
+ return null;
80
86
  }
81
- catch {
87
+ if (text.length < 10) {
88
+ log.warn('refine: response was empty or too short to use');
82
89
  return null;
83
90
  }
91
+ return text;
84
92
  }
85
93
  function isConsolidated(entry) {
86
94
  if (entry.layer !== Layer.Semantic)
@@ -1,4 +1,4 @@
1
- import type { RerankerFn } from './types.js';
1
+ import type { RerankerFn, RerankProvenance } from './types.js';
2
2
  import { type JsonValue } from '../http-util.js';
3
3
  /** The two pretrained CLEF decision models served by Cloudflare Workers AI. */
4
4
  export type ClefModel = 'clef-flash' | 'clef';
@@ -7,7 +7,7 @@ export declare function isClefModel(name: string): name is ClefModel;
7
7
  interface ClefRoute {
8
8
  url: string;
9
9
  token: string | undefined;
10
- backend: 'cloudflare' | 'private-endpoint';
10
+ backend: Exclude<RerankProvenance['backend'], 'native'>;
11
11
  }
12
12
  interface ClefScores {
13
13
  scores: number[];
@@ -1,7 +1,11 @@
1
- import { buildRelevanceRequest, JEV_DEFAULT_TOP_K } from './jev.js';
1
+ import { buildRelevanceRequest, JEV_DEFAULT_TOP_K, rankByScores } from './jev.js';
2
2
  import { isJsonObjectRecord } from '../http-util.js';
3
+ import { log } from '../log.js';
3
4
  const CLEF_MODELS = ['clef-flash', 'clef'];
4
5
  const DEFAULT_TIMEOUT_MS = 15_000;
6
+ const MAX_TIMEOUT_MS = 120_000;
7
+ // 64 answers fit in a few KB; 1 MiB leaves room for a verbose envelope.
8
+ const MAX_REPLY_BYTES = 1024 * 1024;
5
9
  // Workers AI rejects a request with more than 64 questions, one per candidate here.
6
10
  const MAX_CANDIDATES = 64;
7
11
  const ACCOUNT_ID = /^[0-9a-f]{32}$/i;
@@ -9,6 +13,10 @@ const ACCOUNT_ID = /^[0-9a-f]{32}$/i;
9
13
  export function isClefModel(name) {
10
14
  return CLEF_MODELS.some((m) => m === name);
11
15
  }
16
+ // Plain http would put the token and the memory text on the wire, so only the local machine may use it.
17
+ function isLoopback(hostname) {
18
+ return hostname === 'localhost' || hostname === '[::1]' || /^127(\.\d{1,3}){3}$/.test(hostname);
19
+ }
12
20
  function isNumber(v) {
13
21
  return Number.isFinite(v);
14
22
  }
@@ -16,7 +24,12 @@ function isString(v) {
16
24
  return v !== undefined && v !== null && v.constructor === String;
17
25
  }
18
26
  function isRejection(v) {
19
- return typeof v === 'string';
27
+ return v.constructor === String;
28
+ }
29
+ // fetch quotes a rejected header value in its error, so a token it would reject must never reach it.
30
+ function checkHeaderSafe(name, token) {
31
+ if (!/^[\x21-\x7e]+$/.test(token))
32
+ throw new Error(`${name} has characters a header cannot carry`);
20
33
  }
21
34
  /** Transport from trusted local env, never call arguments: HIPPO_CLEF_ENDPOINT wins over hosted Workers AI. */
22
35
  export function resolveClefRoute(model) {
@@ -32,7 +45,17 @@ export function resolveClefRoute(model) {
32
45
  if (parsed.protocol !== 'https:' && parsed.protocol !== 'http:') {
33
46
  throw new Error('HIPPO_CLEF_ENDPOINT must be an http or https URL');
34
47
  }
35
- return { url: parsed.href, token: process.env.HIPPO_CLEF_ENDPOINT_TOKEN?.trim() || undefined, backend: 'private-endpoint' };
48
+ // fetch echoes a URL with credentials in its error text, which reaches stderr.
49
+ if (parsed.username || parsed.password) {
50
+ throw new Error('HIPPO_CLEF_ENDPOINT must not embed credentials; set HIPPO_CLEF_ENDPOINT_TOKEN');
51
+ }
52
+ if (parsed.protocol === 'http:' && !isLoopback(parsed.hostname)) {
53
+ throw new Error('HIPPO_CLEF_ENDPOINT must use https unless it is on this machine');
54
+ }
55
+ const endpointToken = process.env.HIPPO_CLEF_ENDPOINT_TOKEN?.trim() || undefined;
56
+ if (endpointToken)
57
+ checkHeaderSafe('HIPPO_CLEF_ENDPOINT_TOKEN', endpointToken);
58
+ return { url: parsed.href, token: endpointToken, backend: 'private-endpoint' };
36
59
  }
37
60
  const account = process.env.CLOUDFLARE_ACCOUNT_ID?.trim() ?? '';
38
61
  const token = process.env.CLOUDFLARE_API_TOKEN?.trim() ?? '';
@@ -40,6 +63,7 @@ export function resolveClefRoute(model) {
40
63
  throw new Error('CLOUDFLARE_ACCOUNT_ID and CLOUDFLARE_API_TOKEN not set');
41
64
  if (!ACCOUNT_ID.test(account))
42
65
  throw new Error('CLOUDFLARE_ACCOUNT_ID is not a 32-character hex id');
66
+ checkHeaderSafe('CLOUDFLARE_API_TOKEN', token);
43
67
  return {
44
68
  url: `https://api.cloudflare.com/client/v4/accounts/${account}/ai/run/@cf/cloudflare/${model}`,
45
69
  token,
@@ -83,15 +107,43 @@ export function parseClefReply(body, n, model, requireModel) {
83
107
  outputTokens: isNumber(usage.output_tokens) ? usage.output_tokens : undefined,
84
108
  };
85
109
  }
110
+ // The timeout bounds time, not bytes: a hostile endpoint could stream a huge 2xx body into memory.
111
+ async function readCappedJson(resp) {
112
+ if (!resp.body)
113
+ throw new Error('reply has no body');
114
+ const reader = resp.body.getReader();
115
+ const decoder = new TextDecoder();
116
+ let raw = '';
117
+ let received = 0;
118
+ for (;;) {
119
+ const { done, value } = await reader.read();
120
+ if (done)
121
+ break;
122
+ received += value.byteLength;
123
+ if (received > MAX_REPLY_BYTES) {
124
+ await reader.cancel();
125
+ throw new Error(`reply over ${MAX_REPLY_BYTES} bytes`);
126
+ }
127
+ raw += decoder.decode(value, { stream: true });
128
+ }
129
+ raw += decoder.decode();
130
+ try {
131
+ return JSON.parse(raw);
132
+ }
133
+ catch {
134
+ throw new Error('reply is not JSON');
135
+ }
136
+ }
86
137
  async function requestScores(model, query, head, route) {
87
138
  const { state, questions } = buildRelevanceRequest(query, head);
88
- const parsedTimeout = Number.parseInt(process.env.HIPPO_CLEF_TIMEOUT_MS ?? '', 10);
89
- const timeoutMs = parsedTimeout > 0 ? parsedTimeout : DEFAULT_TIMEOUT_MS;
90
- const controller = new AbortController();
91
- const timer = setTimeout(() => controller.abort(), timeoutMs);
139
+ // Strict parse: parseInt would read "15s" as 15 ms, and Node clamps a delay past 2^31-1 to 1 ms.
140
+ const requested = Number(process.env.HIPPO_CLEF_TIMEOUT_MS);
141
+ const timeoutMs = Number.isInteger(requested) && requested > 0 && requested <= MAX_TIMEOUT_MS ? requested : DEFAULT_TIMEOUT_MS;
92
142
  const headers = new Headers({ 'content-type': 'application/json' });
93
143
  if (route.token)
94
144
  headers.set('authorization', `Bearer ${route.token}`);
145
+ const controller = new AbortController();
146
+ const timer = setTimeout(() => controller.abort(), timeoutMs);
95
147
  try {
96
148
  const resp = await fetch(route.url, {
97
149
  method: 'POST',
@@ -105,7 +157,7 @@ async function requestScores(model, query, head, route) {
105
157
  await resp.body?.cancel();
106
158
  throw new Error(`HTTP ${resp.status}${ray ? `, ray ${ray}` : ''}`);
107
159
  }
108
- const body = await resp.json();
160
+ const body = await readCappedJson(resp);
109
161
  const parsed = parseClefReply(body, head.length, model, route.backend === 'cloudflare');
110
162
  if (isRejection(parsed))
111
163
  throw new Error(parsed);
@@ -127,7 +179,8 @@ function nativeOrder(head, provenance) {
127
179
  rerankScore: r.score,
128
180
  preRerankRank: r.preRerankRank ?? i + 1,
129
181
  postRerankRank: i + 1,
130
- rerankProvenance: provenance,
182
+ // A copy per row, so a caller editing one result cannot change another's provenance.
183
+ rerankProvenance: { ...provenance },
131
184
  }));
132
185
  }
133
186
  /** A CLEF reranker for one model: Jev's request shape and pool; any failure keeps the native order (never paid Jev), warning once. */
@@ -137,42 +190,30 @@ export function createClefReranker(model) {
137
190
  const head = results.slice(0, options?.topK ?? JEV_DEFAULT_TOP_K);
138
191
  if (head.length === 0)
139
192
  return [];
140
- let backend = 'native';
193
+ let route;
141
194
  let got;
142
195
  try {
143
196
  if (head.length > MAX_CANDIDATES)
144
197
  throw new Error(`more than ${MAX_CANDIDATES} candidates`);
145
- const route = resolveClefRoute(model);
146
- backend = route.backend;
198
+ route = resolveClefRoute(model);
147
199
  got = await requestScores(model, query, head, route);
148
200
  }
149
201
  catch (err) {
150
202
  const reason = err instanceof Error ? err.message : 'unknown error';
151
203
  if (!warned) {
152
204
  warned = true;
153
- // eslint-disable-next-line no-console
154
- console.warn(`[hippo] ${model} reranker unavailable (${reason}); keeping the native order. Subsequent calls will not repeat this warning.`);
205
+ log.warn(`${model} reranker unavailable (${reason}); keeping the native order. Subsequent calls will not repeat this warning.`);
155
206
  }
156
207
  return nativeOrder(head, { backend: 'native', requestedModel: model, fallbackReason: reason });
157
208
  }
158
- const provenance = {
159
- backend,
209
+ const rerankProvenance = {
210
+ backend: route.backend,
160
211
  requestedModel: model,
161
212
  actualModel: got.actualModel,
162
213
  inputTokens: got.inputTokens,
163
214
  outputTokens: got.outputTokens,
164
215
  };
165
- const scored = head.map((r, i) => ({
166
- ...r,
167
- rerankScore: got.scores[i],
168
- preRerankRank: r.preRerankRank ?? i + 1,
169
- postRerankRank: 0,
170
- rerankProvenance: provenance,
171
- }));
172
- // Stable sort: ties fall back to the prior relevance order.
173
- scored.sort((a, b) => b.rerankScore - a.rerankScore);
174
- scored.forEach((r, i) => (r.postRerankRank = i + 1));
175
- return scored;
216
+ return rankByScores(head, got.scores).map((r) => ({ ...r, rerankProvenance: { ...rerankProvenance } }));
176
217
  };
177
218
  }
178
219
  /** Opt-in CLEF-flash reranker (Cloudflare Workers AI or HIPPO_CLEF_ENDPOINT); off unless named, so defaults stay native. */
@@ -1,5 +1,6 @@
1
1
  import { createRequire } from 'node:module';
2
2
  import { pathToFileURL } from 'node:url';
3
+ import { log } from '../log.js';
3
4
  const MODEL_NAME = 'Xenova/ms-marco-MiniLM-L-6-v2';
4
5
  const _require = createRequire(import.meta.url);
5
6
  const TRANSFORMERS_PACKAGES = ['@huggingface/transformers', '@xenova/transformers'];
@@ -36,7 +37,8 @@ async function loadTransformersModule() {
36
37
  const seq = mod.AutoModelForSequenceClassification ?? mod.default?.AutoModelForSequenceClassification;
37
38
  return tok && seq ? { AutoTokenizer: tok, AutoModelForSequenceClassification: seq } : null;
38
39
  }
39
- catch {
40
+ catch (err) {
41
+ log.debug(`cross-encoder: transformers import failed: ${err instanceof Error ? err.message : String(err)}`);
40
42
  return null;
41
43
  }
42
44
  }
@@ -78,7 +80,8 @@ async function buildPipeline() {
78
80
  return score;
79
81
  };
80
82
  }
81
- catch {
83
+ catch (err) {
84
+ log.debug(`cross-encoder: model load failed: ${err instanceof Error ? err.message : String(err)}`);
82
85
  return null;
83
86
  }
84
87
  }
@@ -102,8 +105,7 @@ export const crossEncoderReranker = async (query, results, options) => {
102
105
  // working reranker.
103
106
  if (!warnedOnFallback) {
104
107
  warnedOnFallback = true;
105
- // eslint-disable-next-line no-console
106
- console.warn('[hippo] cross-encoder reranker unavailable (no Transformers.js backend, or model fetch blocked); falling back to identity ordering. Subsequent calls will not repeat this warning.');
108
+ log.warn('cross-encoder reranker unavailable (no Transformers.js backend, or model fetch blocked); falling back to identity ordering. Subsequent calls will not repeat this warning.');
107
109
  }
108
110
  return head.map((r, i) => ({
109
111
  ...r,