hippo-memory 1.59.0 → 1.60.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api.d.ts +2 -0
- package/dist/api.js +25 -9
- package/dist/audit.js +1 -0
- package/dist/autolearn.js +5 -2
- package/dist/capture.js +8 -6
- package/dist/cli.d.ts +355 -0
- package/dist/cli.js +1236 -1081
- package/dist/compaction-record.js +7 -6
- package/dist/config.js +12 -11
- package/dist/connectors/github/cli-impl.js +1 -0
- package/dist/consolidate.js +4 -3
- package/dist/dag.js +7 -7
- package/dist/dashboard.js +4 -2
- package/dist/db.js +16 -3
- package/dist/delivery-recorder.js +1 -0
- package/dist/doctor.js +1 -0
- package/dist/dormant.js +1 -0
- package/dist/embedding-provider.js +3 -2
- package/dist/embeddings.js +10 -7
- package/dist/extract.js +4 -3
- package/dist/graph.js +3 -8
- package/dist/handoff.js +3 -0
- package/dist/hooks.js +3 -0
- package/dist/importers.js +6 -3
- package/dist/incidents.js +1 -0
- package/dist/judgment.js +5 -2
- package/dist/mcp/server.d.ts +5 -0
- package/dist/mcp/server.js +32 -13
- package/dist/memory.d.ts +5 -3
- package/dist/processes.js +1 -0
- package/dist/project-identity.d.ts +1 -1
- package/dist/project-identity.js +9 -4
- package/dist/raw-archive-mirror-cleanup.js +2 -1
- package/dist/recall-trace.d.ts +2 -2
- package/dist/recall-trace.js +10 -14
- package/dist/refine-llm.js +18 -10
- package/dist/rerankers/clef.d.ts +2 -2
- package/dist/rerankers/clef.js +68 -27
- package/dist/rerankers/cross-encoder.js +6 -4
- package/dist/rerankers/jev.d.ts +3 -1
- package/dist/rerankers/jev.js +20 -15
- package/dist/rerankers/llm.js +3 -3
- package/dist/same-text.d.ts +2 -0
- package/dist/same-text.js +4 -0
- package/dist/scheduler.js +1 -0
- package/dist/search.js +4 -2
- package/dist/secret-detect.js +1 -0
- package/dist/server.js +29 -12
- package/dist/shared.js +21 -19
- package/dist/stdin.js +1 -0
- package/dist/store.d.ts +33 -1
- package/dist/store.js +117 -16
- package/dist/token-ledger.js +1 -0
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/extensions/openclaw-plugin/openclaw.plugin.json +1 -1
- package/extensions/openclaw-plugin/package.json +1 -1
- package/openclaw.plugin.json +1 -1
- package/package.json +2 -1
package/dist/mcp/server.d.ts
CHANGED
|
@@ -25,8 +25,13 @@ interface McpResponse {
|
|
|
25
25
|
error?: {
|
|
26
26
|
code: number;
|
|
27
27
|
message: string;
|
|
28
|
+
data?: {
|
|
29
|
+
requestId: string;
|
|
30
|
+
};
|
|
28
31
|
};
|
|
29
32
|
}
|
|
33
|
+
/** JSON-RPC reply for a request that threw: typed API errors keep their text; anything else is logged and answered generically. */
|
|
34
|
+
export declare function mcpErrorResponse<E>(id: McpResponse['id'], err: E, requestId?: string): McpResponse;
|
|
30
35
|
export type { McpRequest, McpResponse };
|
|
31
36
|
/**
|
|
32
37
|
* Optional execution context threaded from a non-stdio transport. When the
|
package/dist/mcp/server.js
CHANGED
|
@@ -9,14 +9,17 @@
|
|
|
9
9
|
*/
|
|
10
10
|
import * as fs from 'fs';
|
|
11
11
|
import * as path from 'path';
|
|
12
|
+
import { randomUUID } from 'node:crypto';
|
|
13
|
+
import { INTERNAL_ERROR_MESSAGE, mapApiError } from '../http-util.js';
|
|
14
|
+
import { log } from '../log.js';
|
|
12
15
|
import { createMemory, Layer, calculateStrength, } from '../memory.js';
|
|
13
16
|
import { fitBudget, estimateTokens } from '../search.js';
|
|
14
17
|
import { evalNow } from '../ablation.js';
|
|
15
|
-
import {
|
|
18
|
+
import { loadStrengthRows, loadTextsHoldingWords, writeEntry, readEntry, listMemoryConflicts, resolveConflict, countCreatedSinceLastSleep } from '../store.js';
|
|
16
19
|
import { shareMemory, listPeers, getGlobalRoot, initGlobal } from '../shared.js';
|
|
17
20
|
import { consolidate } from '../consolidate.js';
|
|
18
21
|
import { fetchGitLog, extractLessons, partitionLessons, isGitRepo } from '../autolearn.js';
|
|
19
|
-
import { dropHeldCopies, duplicateKey, storedTextKeys } from '../same-text.js';
|
|
22
|
+
import { dropHeldCopies, duplicateKey, longestWord, storedTextKeys } from '../same-text.js';
|
|
20
23
|
import { loadConfig } from '../config.js';
|
|
21
24
|
import { confidenceLabel } from '../memory.js';
|
|
22
25
|
import { resolveTenantId } from '../tenant.js';
|
|
@@ -50,6 +53,18 @@ export function findHippoRoot(cwd = process.cwd(), opts) {
|
|
|
50
53
|
const global = getGlobalRoot();
|
|
51
54
|
return fs.existsSync(global) ? global : null;
|
|
52
55
|
}
|
|
56
|
+
/** JSON-RPC reply for a request that threw: typed API errors keep their text; anything else is logged and answered generically. */
|
|
57
|
+
export function mcpErrorResponse(id, err, requestId = randomUUID()) {
|
|
58
|
+
const { status, message } = mapApiError(err);
|
|
59
|
+
if (status !== 500)
|
|
60
|
+
return { jsonrpc: '2.0', id, error: { code: -32603, message } };
|
|
61
|
+
log.error(`mcp request failed: ${err instanceof Error ? err.message : String(err)}`, { requestId });
|
|
62
|
+
return {
|
|
63
|
+
jsonrpc: '2.0',
|
|
64
|
+
id,
|
|
65
|
+
error: { code: -32603, message: `${INTERNAL_ERROR_MESSAGE} (request id ${requestId})`, data: { requestId } },
|
|
66
|
+
};
|
|
67
|
+
}
|
|
53
68
|
/**
|
|
54
69
|
* The api-layer actor for a tool call. Stdio (no ctx) is the local operator
|
|
55
70
|
* and runs as admin; over HTTP the transport's authenticated role is used, so
|
|
@@ -477,7 +492,7 @@ function resolveClientKey(ctx) {
|
|
|
477
492
|
function createGlobalStoreOnFirstRun() {
|
|
478
493
|
initGlobal();
|
|
479
494
|
const root = getGlobalRoot();
|
|
480
|
-
|
|
495
|
+
log.warn(`no memory store found; created the global store at ${root}. Run \`hippo init\` in a project for a project store.`);
|
|
481
496
|
return root;
|
|
482
497
|
}
|
|
483
498
|
// ── Token ledger (ROADMAP TE0) ──
|
|
@@ -524,7 +539,7 @@ async function executeTool(name, args, ctx) {
|
|
|
524
539
|
// resolve tenant from HIPPO_TENANT.
|
|
525
540
|
const hippoRoot = ctx?.hippoRoot ?? findHippoRoot() ?? createGlobalStoreOnFirstRun();
|
|
526
541
|
const config = loadConfig(hippoRoot);
|
|
527
|
-
// A5: every
|
|
542
|
+
// A5: every store read in this server returns to the caller and is
|
|
528
543
|
// tenant-isolated. Resolved once per tool call: prefer the transport's
|
|
529
544
|
// ctx.tenantId so an HTTP Bearer for tenant B doesn't drop to HIPPO_TENANT.
|
|
530
545
|
const tenantId = ctx?.tenantId ?? resolveTenantId({});
|
|
@@ -866,7 +881,7 @@ async function executeTool(name, args, ctx) {
|
|
|
866
881
|
// Fire-and-forget (never block the response); an unhandled rejection would kill the server, so log it.
|
|
867
882
|
consolidate(hippoRoot)
|
|
868
883
|
.catch((err) => {
|
|
869
|
-
|
|
884
|
+
log.error(`auto-sleep consolidate failed (tenant ${tenantId}): ${err instanceof Error ? err.message : String(err)}`);
|
|
870
885
|
})
|
|
871
886
|
.finally(() => autoSleepInFlight.delete(hippoRoot));
|
|
872
887
|
}
|
|
@@ -922,7 +937,8 @@ async function executeTool(name, args, ctx) {
|
|
|
922
937
|
+ formatMemories(result.entries);
|
|
923
938
|
}
|
|
924
939
|
case 'hippo_status': {
|
|
925
|
-
|
|
940
|
+
// Every row counts toward the averages, so this scans the store, but without its text.
|
|
941
|
+
const entries = loadStrengthRows(hippoRoot, tenantId);
|
|
926
942
|
const now = evalNow(); // honors HIPPO_FAKE_NOW (eval-only; see ablation.ts)
|
|
927
943
|
let atRisk = 0;
|
|
928
944
|
let totalStrength = 0;
|
|
@@ -965,7 +981,7 @@ async function executeTool(name, args, ctx) {
|
|
|
965
981
|
let added = 0;
|
|
966
982
|
let skipped = 0;
|
|
967
983
|
let rejected = 0;
|
|
968
|
-
const keys = storedTextKeys(
|
|
984
|
+
const keys = storedTextKeys(loadTextsHoldingWords(hippoRoot, tenantId, lessons.map(longestWord)));
|
|
969
985
|
for (const lesson of lessons) {
|
|
970
986
|
if (keys.has(duplicateKey(lesson))) {
|
|
971
987
|
skipped++;
|
|
@@ -1138,17 +1154,20 @@ function dispatch(body) {
|
|
|
1138
1154
|
req = JSON.parse(body);
|
|
1139
1155
|
}
|
|
1140
1156
|
catch {
|
|
1141
|
-
|
|
1157
|
+
log.debug('mcp: skipped a frame that is not valid JSON');
|
|
1158
|
+
return;
|
|
1142
1159
|
}
|
|
1143
1160
|
if (!req.method)
|
|
1144
1161
|
return;
|
|
1145
1162
|
if (req.method.startsWith('notifications/')) {
|
|
1146
|
-
handleMcpRequest(req).catch(() => {
|
|
1163
|
+
handleMcpRequest(req).catch((err) => {
|
|
1164
|
+
log.error(`mcp notification ${req.method} failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
1165
|
+
});
|
|
1147
1166
|
return;
|
|
1148
1167
|
}
|
|
1149
1168
|
handleMcpRequest(req).then((resp) => { if (resp)
|
|
1150
1169
|
send(resp); }).catch((err) => {
|
|
1151
|
-
send(
|
|
1170
|
+
send(mcpErrorResponse(req.id, err));
|
|
1152
1171
|
});
|
|
1153
1172
|
}
|
|
1154
1173
|
/**
|
|
@@ -1171,10 +1190,10 @@ export function startStdioLoop() {
|
|
|
1171
1190
|
});
|
|
1172
1191
|
process.stdin.on('end', () => process.exit(0));
|
|
1173
1192
|
process.on('uncaughtException', (err) => {
|
|
1174
|
-
|
|
1193
|
+
log.error(`mcp uncaught: ${err instanceof Error ? err.message : String(err)}`);
|
|
1175
1194
|
});
|
|
1176
1195
|
process.on('unhandledRejection', (err) => {
|
|
1177
|
-
|
|
1196
|
+
log.error(`mcp unhandled: ${err instanceof Error ? err.message : String(err)}`);
|
|
1178
1197
|
});
|
|
1179
1198
|
}
|
|
1180
1199
|
// Auto-start when invoked as the main module (node dist/mcp/server.js or via
|
|
@@ -1196,7 +1215,7 @@ const isMainModule = (() => {
|
|
|
1196
1215
|
return import.meta.url === mainUrl || import.meta.url === `file:///${argv1.replace(/\\/g, '/')}`;
|
|
1197
1216
|
}
|
|
1198
1217
|
catch {
|
|
1199
|
-
return false;
|
|
1218
|
+
return false; // an unreadable argv means this file was imported, not run; never start the stdio loop then
|
|
1200
1219
|
}
|
|
1201
1220
|
})();
|
|
1202
1221
|
if (isMainModule) {
|
package/dist/memory.d.ts
CHANGED
|
@@ -135,19 +135,21 @@ export declare function _resetLossAversionRatioCacheForTests(): void;
|
|
|
135
135
|
* Modulates effective half-life: memories with consistent positive outcomes
|
|
136
136
|
* decay slower; consistent negative outcomes decay faster.
|
|
137
137
|
*/
|
|
138
|
-
export declare function calculateRewardFactor(entry: MemoryEntry): number;
|
|
138
|
+
export declare function calculateRewardFactor(entry: Pick<MemoryEntry, 'outcome_positive' | 'outcome_negative'>): number;
|
|
139
139
|
/**
|
|
140
140
|
* Net wrongness: bad outcome marks past good ones, never below zero.
|
|
141
141
|
* Strength halves per unit (capped at 3) and recall stops strengthening
|
|
142
142
|
* the memory, so a correction outranks pinning, error tags and heavy recall.
|
|
143
143
|
*/
|
|
144
|
-
export declare function netWrong(entry: MemoryEntry): number;
|
|
144
|
+
export declare function netWrong(entry: Pick<MemoryEntry, 'outcome_positive' | 'outcome_negative'>): number;
|
|
145
145
|
/**
|
|
146
146
|
* Options for decay basis.
|
|
147
147
|
* - clock: wall-clock time (default pre-v0.15)
|
|
148
148
|
* - session: decay by sleep cycle count (for intermittent agents)
|
|
149
149
|
* - adaptive: auto-scale half-life by session frequency (default v0.15+)
|
|
150
150
|
*/
|
|
151
|
+
/** What calculateStrength reads, so a caller can score a row without loading its text. */
|
|
152
|
+
export type StrengthInputs = Pick<MemoryEntry, 'pinned' | 'created' | 'last_retrieved' | 'half_life_days' | 'retrieval_count' | 'emotional_valence' | 'outcome_positive' | 'outcome_negative'>;
|
|
151
153
|
export interface DecayOptions {
|
|
152
154
|
decayBasis?: 'clock' | 'session' | 'adaptive';
|
|
153
155
|
/** Average interval between sleep cycles, in days. Used by 'adaptive' and 'session' modes. */
|
|
@@ -166,7 +168,7 @@ export interface DecayOptions {
|
|
|
166
168
|
*
|
|
167
169
|
* Pinned memories skip time decay; being marked wrong still fades them (netWrong).
|
|
168
170
|
*/
|
|
169
|
-
export declare function calculateStrength(entry:
|
|
171
|
+
export declare function calculateStrength(entry: StrengthInputs, now?: Date, options?: DecayOptions): number;
|
|
170
172
|
/**
|
|
171
173
|
* Derive half-life based on signals, as per PLAN.md table.
|
|
172
174
|
*/
|
package/dist/processes.js
CHANGED
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
* directory is the project root; if none exists, the nearest ancestor
|
|
8
8
|
* containing `.git` (directory or worktree file).
|
|
9
9
|
* - The user home directory is NEVER a project, even though it contains the
|
|
10
|
-
* global store at `~/.hippo`. Reaching home ends the walk.
|
|
10
|
+
* global store at `~/.hippo`. Reaching home or the temp root (inside home on Windows) ends the walk.
|
|
11
11
|
* - A directory with no marker anywhere up the walk is NOT a project: it
|
|
12
12
|
* resolves to the user-global identity (empty name), so memories written
|
|
13
13
|
* there stay injectable everywhere (matches pre-isolation behavior).
|
package/dist/project-identity.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import * as fs from 'fs';
|
|
2
2
|
import * as os from 'os';
|
|
3
3
|
import * as path from 'path';
|
|
4
|
+
import { log } from './log.js';
|
|
4
5
|
const MAX_WALK_DEPTH = 64;
|
|
5
6
|
const identityCache = new Map();
|
|
6
7
|
/** Clear the per-process identity cache (test seam). */
|
|
@@ -15,7 +16,8 @@ export function realpathOrResolve(p) {
|
|
|
15
16
|
try {
|
|
16
17
|
return fs.realpathSync.native(p);
|
|
17
18
|
}
|
|
18
|
-
catch {
|
|
19
|
+
catch (err) {
|
|
20
|
+
log.debug(`project identity: realpath fell back to resolve for ${p}: ${err instanceof Error ? err.message : String(err)}`);
|
|
19
21
|
return path.resolve(p);
|
|
20
22
|
}
|
|
21
23
|
}
|
|
@@ -43,7 +45,8 @@ function isDirectoryAt(p) {
|
|
|
43
45
|
try {
|
|
44
46
|
return fs.statSync(p).isDirectory();
|
|
45
47
|
}
|
|
46
|
-
catch {
|
|
48
|
+
catch (err) {
|
|
49
|
+
log.debug(`project identity: no marker at ${p}: ${err instanceof Error ? err.message : String(err)}`);
|
|
47
50
|
return false;
|
|
48
51
|
}
|
|
49
52
|
}
|
|
@@ -60,9 +63,11 @@ export function resolveProjectIdentity(cwd, opts) {
|
|
|
60
63
|
return cached;
|
|
61
64
|
}
|
|
62
65
|
const home = realpathOrResolve(opts?.homeDir ?? os.homedir());
|
|
63
|
-
const
|
|
66
|
+
const stops = [realpathOrResolve(os.tmpdir())];
|
|
67
|
+
if (opts?.stopDir)
|
|
68
|
+
stops.push(realpathOrResolve(opts.stopDir));
|
|
64
69
|
const start = realpathOrResolve(startInput);
|
|
65
|
-
const { hippoRoot, gitRoot, reachedHome } = walkProjectMarkers(start, home,
|
|
70
|
+
const { hippoRoot, gitRoot, reachedHome } = walkProjectMarkers(start, home, stops);
|
|
66
71
|
let identity;
|
|
67
72
|
const root = hippoRoot ?? gitRoot;
|
|
68
73
|
if (root !== null) {
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import * as fs from 'fs';
|
|
2
2
|
import * as path from 'path';
|
|
3
|
+
import { log } from './log.js';
|
|
3
4
|
const LAYERS = ['episodic', 'buffer', 'semantic'];
|
|
4
5
|
const MAX_WARN_LOGS = 5;
|
|
5
6
|
/**
|
|
@@ -42,7 +43,7 @@ export function cleanupArchivedMirrors(hippoRoot, db) {
|
|
|
42
43
|
catch (err) {
|
|
43
44
|
allOk = false;
|
|
44
45
|
if (warnCount < MAX_WARN_LOGS) {
|
|
45
|
-
|
|
46
|
+
log.warn(`cleanupArchivedMirrors: unlink failed for ${filePath} (will retry on next DB open): ${err instanceof Error ? err.message : String(err)}`);
|
|
46
47
|
warnCount += 1;
|
|
47
48
|
}
|
|
48
49
|
}
|
package/dist/recall-trace.d.ts
CHANGED
|
@@ -102,8 +102,8 @@ export interface RecordTraceOutcomeInput {
|
|
|
102
102
|
* this function from caller-side state (`last_trace_id` / applied outcome
|
|
103
103
|
* ids) that can go stale relative to the trace it names — a forgotten
|
|
104
104
|
* memory, a tenant switch mid-session, or a race between two callers. Two
|
|
105
|
-
* checks run before the insert, both skip
|
|
106
|
-
*
|
|
105
|
+
* checks run before the insert, both skip with one log.warn line
|
|
106
|
+
* rather than throw:
|
|
107
107
|
* 1. The named trace must exist and belong to `input.tenantId` — a
|
|
108
108
|
* tenant mismatch or a dangling id (deleted trace) skips.
|
|
109
109
|
* 2. `input.memoryIds` is intersected against the trace's OWN
|
package/dist/recall-trace.js
CHANGED
|
@@ -18,6 +18,7 @@
|
|
|
18
18
|
import { createHash } from 'node:crypto';
|
|
19
19
|
import { openHippoDb, closeHippoDb } from './db.js';
|
|
20
20
|
import { DELIVERY_LEDGER_VERSION } from './delivery-recorder.js';
|
|
21
|
+
import { log } from './log.js';
|
|
21
22
|
/**
|
|
22
23
|
* Strip a RerankStep down to {stage, multiplier, scoreBefore, scoreAfter}
|
|
23
24
|
* before persisting (F3 privacy fix, codex cross-model finding). `note` is
|
|
@@ -76,8 +77,7 @@ export function writeRecallTrace(db, input) {
|
|
|
76
77
|
}
|
|
77
78
|
}
|
|
78
79
|
catch (error) {
|
|
79
|
-
|
|
80
|
-
console.error(`[hippo] recall trace write failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
80
|
+
log.error(`recall trace write failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
81
81
|
return null;
|
|
82
82
|
}
|
|
83
83
|
}
|
|
@@ -113,8 +113,7 @@ export function writeRecallTraceAtRoot(root, input) {
|
|
|
113
113
|
db = openHippoDb(root);
|
|
114
114
|
}
|
|
115
115
|
catch (error) {
|
|
116
|
-
|
|
117
|
-
console.error(`[hippo] recall trace connection failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
116
|
+
log.error(`recall trace connection failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
118
117
|
return null;
|
|
119
118
|
}
|
|
120
119
|
try {
|
|
@@ -140,8 +139,8 @@ export function writeRecallTraceAtRoot(root, input) {
|
|
|
140
139
|
* this function from caller-side state (`last_trace_id` / applied outcome
|
|
141
140
|
* ids) that can go stale relative to the trace it names — a forgotten
|
|
142
141
|
* memory, a tenant switch mid-session, or a race between two callers. Two
|
|
143
|
-
* checks run before the insert, both skip
|
|
144
|
-
*
|
|
142
|
+
* checks run before the insert, both skip with one log.warn line
|
|
143
|
+
* rather than throw:
|
|
145
144
|
* 1. The named trace must exist and belong to `input.tenantId` — a
|
|
146
145
|
* tenant mismatch or a dangling id (deleted trace) skips.
|
|
147
146
|
* 2. `input.memoryIds` is intersected against the trace's OWN
|
|
@@ -158,8 +157,7 @@ export function recordTraceOutcome(db, input) {
|
|
|
158
157
|
// SELECT above; sqlite returns undefined when no row matches.
|
|
159
158
|
const trace = db.prepare(`SELECT tenant_id FROM recall_traces WHERE id = ?`).get(input.traceId);
|
|
160
159
|
if (!trace || trace.tenant_id !== input.tenantId) {
|
|
161
|
-
|
|
162
|
-
console.error(`[hippo] recall trace outcome skipped: trace ${input.traceId} missing or tenant mismatch`);
|
|
160
|
+
log.warn(`recall trace outcome skipped: trace ${input.traceId} missing or tenant mismatch`);
|
|
163
161
|
return;
|
|
164
162
|
}
|
|
165
163
|
// SAFETY: row shape matches the single `memory_id` column named in the
|
|
@@ -170,8 +168,7 @@ export function recordTraceOutcome(db, input) {
|
|
|
170
168
|
const members = new Set(memberRows.map((r) => r.memory_id));
|
|
171
169
|
const credited = input.memoryIds.filter((id) => members.has(id));
|
|
172
170
|
if (credited.length === 0) {
|
|
173
|
-
|
|
174
|
-
console.error(`[hippo] recall trace outcome skipped: no credited ids intersect trace ${input.traceId}'s results`);
|
|
171
|
+
log.warn(`recall trace outcome skipped: no credited ids intersect trace ${input.traceId}'s results`);
|
|
175
172
|
return;
|
|
176
173
|
}
|
|
177
174
|
db.prepare(`
|
|
@@ -180,8 +177,7 @@ export function recordTraceOutcome(db, input) {
|
|
|
180
177
|
`).run(input.traceId, new Date().toISOString(), input.tenantId, input.outcome, JSON.stringify(credited));
|
|
181
178
|
}
|
|
182
179
|
catch (error) {
|
|
183
|
-
|
|
184
|
-
console.error(`[hippo] recall trace outcome write failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
180
|
+
log.error(`recall trace outcome write failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
185
181
|
}
|
|
186
182
|
}
|
|
187
183
|
/** Pruned on write, counted back from the event's ts capped at the real clock, so a far-future fake time spares real rows. */
|
|
@@ -270,7 +266,7 @@ export function writeDeliveryEvent(db, input) {
|
|
|
270
266
|
}
|
|
271
267
|
}
|
|
272
268
|
catch (error) {
|
|
273
|
-
//
|
|
269
|
+
// The prompt hook's stderr shows this exact `[hippo] delivery ledger` line, so it stays off the logger's format.
|
|
274
270
|
console.error(`[hippo] delivery ledger write failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
275
271
|
return null;
|
|
276
272
|
}
|
|
@@ -282,7 +278,7 @@ export function writeDeliveryEventAtRoot(root, input) {
|
|
|
282
278
|
db = openHippoDb(root, { busyWaitMs: DELIVERY_LEDGER_WAIT_MS });
|
|
283
279
|
}
|
|
284
280
|
catch (error) {
|
|
285
|
-
//
|
|
281
|
+
// Same hook stderr line as writeDeliveryEvent above.
|
|
286
282
|
console.error(`[hippo] delivery ledger write failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
287
283
|
return null;
|
|
288
284
|
}
|
package/dist/refine-llm.js
CHANGED
|
@@ -14,8 +14,9 @@
|
|
|
14
14
|
*/
|
|
15
15
|
import { Layer } from './memory.js';
|
|
16
16
|
import { loadAllEntries, readEntry, writeEntry } from './store.js';
|
|
17
|
-
import {
|
|
17
|
+
import { redactSecretsStrict } from './secret-detect.js';
|
|
18
18
|
import { fetchWithRetry, llmTimeoutMs } from './http-retry.js';
|
|
19
|
+
import { log } from './log.js';
|
|
19
20
|
const REFINED_TAG = 'llm-refined';
|
|
20
21
|
const CONSOLIDATED_MARKERS = [
|
|
21
22
|
'[Consolidated from',
|
|
@@ -31,7 +32,7 @@ export async function refineSemanticMemory(merged, sources, opts) {
|
|
|
31
32
|
const fetchFn = opts.fetcher ?? fetch;
|
|
32
33
|
const sourceBlock = sources
|
|
33
34
|
.slice(0, 8)
|
|
34
|
-
.map((s, i) => `[source ${i + 1}] ${
|
|
35
|
+
.map((s, i) => `[source ${i + 1}] ${redactSecretsStrict(s.content).slice(0, 400)}`)
|
|
35
36
|
.join('\n\n');
|
|
36
37
|
const prompt = `You are refining a semantic memory in an agent's memory store. The rule-based consolidator merged several related episodic memories into one, but the output is clumsy. Produce a single coherent semantic memory that captures the underlying principle.
|
|
37
38
|
|
|
@@ -43,7 +44,7 @@ Rules:
|
|
|
43
44
|
- Do NOT include the "[Consolidated from N ...]" marker.
|
|
44
45
|
|
|
45
46
|
Current merged content:
|
|
46
|
-
${
|
|
47
|
+
${redactSecretsStrict(merged)}
|
|
47
48
|
|
|
48
49
|
Source memories (up to 8 shown):
|
|
49
50
|
${sourceBlock}`;
|
|
@@ -63,24 +64,31 @@ ${sourceBlock}`;
|
|
|
63
64
|
}),
|
|
64
65
|
}, { timeoutMs: llmTimeoutMs(), fetchFn });
|
|
65
66
|
}
|
|
66
|
-
catch {
|
|
67
|
+
catch (err) {
|
|
68
|
+
log.warn(`refine: request failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
67
69
|
return null;
|
|
68
70
|
}
|
|
69
|
-
if (!res.ok)
|
|
71
|
+
if (!res.ok) {
|
|
72
|
+
log.warn(`refine: API answered HTTP ${res.status}`);
|
|
70
73
|
return null;
|
|
74
|
+
}
|
|
75
|
+
let text;
|
|
71
76
|
try {
|
|
72
77
|
// SAFETY: data is the Anthropic Messages API response body; the
|
|
73
78
|
// documented response shape is `{ content: [{ type, text, ... }] }`
|
|
74
79
|
// for a text-generating request like this one.
|
|
75
80
|
const data = await res.json();
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
81
|
+
text = data.content?.[0]?.text?.trim() ?? '';
|
|
82
|
+
}
|
|
83
|
+
catch (err) {
|
|
84
|
+
log.warn(`refine: unreadable response: ${err instanceof Error ? err.message : String(err)}`);
|
|
85
|
+
return null;
|
|
80
86
|
}
|
|
81
|
-
|
|
87
|
+
if (text.length < 10) {
|
|
88
|
+
log.warn('refine: response was empty or too short to use');
|
|
82
89
|
return null;
|
|
83
90
|
}
|
|
91
|
+
return text;
|
|
84
92
|
}
|
|
85
93
|
function isConsolidated(entry) {
|
|
86
94
|
if (entry.layer !== Layer.Semantic)
|
package/dist/rerankers/clef.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { RerankerFn } from './types.js';
|
|
1
|
+
import type { RerankerFn, RerankProvenance } from './types.js';
|
|
2
2
|
import { type JsonValue } from '../http-util.js';
|
|
3
3
|
/** The two pretrained CLEF decision models served by Cloudflare Workers AI. */
|
|
4
4
|
export type ClefModel = 'clef-flash' | 'clef';
|
|
@@ -7,7 +7,7 @@ export declare function isClefModel(name: string): name is ClefModel;
|
|
|
7
7
|
interface ClefRoute {
|
|
8
8
|
url: string;
|
|
9
9
|
token: string | undefined;
|
|
10
|
-
backend: '
|
|
10
|
+
backend: Exclude<RerankProvenance['backend'], 'native'>;
|
|
11
11
|
}
|
|
12
12
|
interface ClefScores {
|
|
13
13
|
scores: number[];
|
package/dist/rerankers/clef.js
CHANGED
|
@@ -1,7 +1,11 @@
|
|
|
1
|
-
import { buildRelevanceRequest, JEV_DEFAULT_TOP_K } from './jev.js';
|
|
1
|
+
import { buildRelevanceRequest, JEV_DEFAULT_TOP_K, rankByScores } from './jev.js';
|
|
2
2
|
import { isJsonObjectRecord } from '../http-util.js';
|
|
3
|
+
import { log } from '../log.js';
|
|
3
4
|
const CLEF_MODELS = ['clef-flash', 'clef'];
|
|
4
5
|
const DEFAULT_TIMEOUT_MS = 15_000;
|
|
6
|
+
const MAX_TIMEOUT_MS = 120_000;
|
|
7
|
+
// 64 answers fit in a few KB; 1 MiB leaves room for a verbose envelope.
|
|
8
|
+
const MAX_REPLY_BYTES = 1024 * 1024;
|
|
5
9
|
// Workers AI rejects a request with more than 64 questions, one per candidate here.
|
|
6
10
|
const MAX_CANDIDATES = 64;
|
|
7
11
|
const ACCOUNT_ID = /^[0-9a-f]{32}$/i;
|
|
@@ -9,6 +13,10 @@ const ACCOUNT_ID = /^[0-9a-f]{32}$/i;
|
|
|
9
13
|
export function isClefModel(name) {
|
|
10
14
|
return CLEF_MODELS.some((m) => m === name);
|
|
11
15
|
}
|
|
16
|
+
// Plain http would put the token and the memory text on the wire, so only the local machine may use it.
|
|
17
|
+
function isLoopback(hostname) {
|
|
18
|
+
return hostname === 'localhost' || hostname === '[::1]' || /^127(\.\d{1,3}){3}$/.test(hostname);
|
|
19
|
+
}
|
|
12
20
|
function isNumber(v) {
|
|
13
21
|
return Number.isFinite(v);
|
|
14
22
|
}
|
|
@@ -16,7 +24,12 @@ function isString(v) {
|
|
|
16
24
|
return v !== undefined && v !== null && v.constructor === String;
|
|
17
25
|
}
|
|
18
26
|
function isRejection(v) {
|
|
19
|
-
return
|
|
27
|
+
return v.constructor === String;
|
|
28
|
+
}
|
|
29
|
+
// fetch quotes a rejected header value in its error, so a token it would reject must never reach it.
|
|
30
|
+
function checkHeaderSafe(name, token) {
|
|
31
|
+
if (!/^[\x21-\x7e]+$/.test(token))
|
|
32
|
+
throw new Error(`${name} has characters a header cannot carry`);
|
|
20
33
|
}
|
|
21
34
|
/** Transport from trusted local env, never call arguments: HIPPO_CLEF_ENDPOINT wins over hosted Workers AI. */
|
|
22
35
|
export function resolveClefRoute(model) {
|
|
@@ -32,7 +45,17 @@ export function resolveClefRoute(model) {
|
|
|
32
45
|
if (parsed.protocol !== 'https:' && parsed.protocol !== 'http:') {
|
|
33
46
|
throw new Error('HIPPO_CLEF_ENDPOINT must be an http or https URL');
|
|
34
47
|
}
|
|
35
|
-
|
|
48
|
+
// fetch echoes a URL with credentials in its error text, which reaches stderr.
|
|
49
|
+
if (parsed.username || parsed.password) {
|
|
50
|
+
throw new Error('HIPPO_CLEF_ENDPOINT must not embed credentials; set HIPPO_CLEF_ENDPOINT_TOKEN');
|
|
51
|
+
}
|
|
52
|
+
if (parsed.protocol === 'http:' && !isLoopback(parsed.hostname)) {
|
|
53
|
+
throw new Error('HIPPO_CLEF_ENDPOINT must use https unless it is on this machine');
|
|
54
|
+
}
|
|
55
|
+
const endpointToken = process.env.HIPPO_CLEF_ENDPOINT_TOKEN?.trim() || undefined;
|
|
56
|
+
if (endpointToken)
|
|
57
|
+
checkHeaderSafe('HIPPO_CLEF_ENDPOINT_TOKEN', endpointToken);
|
|
58
|
+
return { url: parsed.href, token: endpointToken, backend: 'private-endpoint' };
|
|
36
59
|
}
|
|
37
60
|
const account = process.env.CLOUDFLARE_ACCOUNT_ID?.trim() ?? '';
|
|
38
61
|
const token = process.env.CLOUDFLARE_API_TOKEN?.trim() ?? '';
|
|
@@ -40,6 +63,7 @@ export function resolveClefRoute(model) {
|
|
|
40
63
|
throw new Error('CLOUDFLARE_ACCOUNT_ID and CLOUDFLARE_API_TOKEN not set');
|
|
41
64
|
if (!ACCOUNT_ID.test(account))
|
|
42
65
|
throw new Error('CLOUDFLARE_ACCOUNT_ID is not a 32-character hex id');
|
|
66
|
+
checkHeaderSafe('CLOUDFLARE_API_TOKEN', token);
|
|
43
67
|
return {
|
|
44
68
|
url: `https://api.cloudflare.com/client/v4/accounts/${account}/ai/run/@cf/cloudflare/${model}`,
|
|
45
69
|
token,
|
|
@@ -83,15 +107,43 @@ export function parseClefReply(body, n, model, requireModel) {
|
|
|
83
107
|
outputTokens: isNumber(usage.output_tokens) ? usage.output_tokens : undefined,
|
|
84
108
|
};
|
|
85
109
|
}
|
|
110
|
+
// The timeout bounds time, not bytes: a hostile endpoint could stream a huge 2xx body into memory.
|
|
111
|
+
async function readCappedJson(resp) {
|
|
112
|
+
if (!resp.body)
|
|
113
|
+
throw new Error('reply has no body');
|
|
114
|
+
const reader = resp.body.getReader();
|
|
115
|
+
const decoder = new TextDecoder();
|
|
116
|
+
let raw = '';
|
|
117
|
+
let received = 0;
|
|
118
|
+
for (;;) {
|
|
119
|
+
const { done, value } = await reader.read();
|
|
120
|
+
if (done)
|
|
121
|
+
break;
|
|
122
|
+
received += value.byteLength;
|
|
123
|
+
if (received > MAX_REPLY_BYTES) {
|
|
124
|
+
await reader.cancel();
|
|
125
|
+
throw new Error(`reply over ${MAX_REPLY_BYTES} bytes`);
|
|
126
|
+
}
|
|
127
|
+
raw += decoder.decode(value, { stream: true });
|
|
128
|
+
}
|
|
129
|
+
raw += decoder.decode();
|
|
130
|
+
try {
|
|
131
|
+
return JSON.parse(raw);
|
|
132
|
+
}
|
|
133
|
+
catch {
|
|
134
|
+
throw new Error('reply is not JSON');
|
|
135
|
+
}
|
|
136
|
+
}
|
|
86
137
|
async function requestScores(model, query, head, route) {
|
|
87
138
|
const { state, questions } = buildRelevanceRequest(query, head);
|
|
88
|
-
|
|
89
|
-
const
|
|
90
|
-
const
|
|
91
|
-
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
|
139
|
+
// Strict parse: parseInt would read "15s" as 15 ms, and Node clamps a delay past 2^31-1 to 1 ms.
|
|
140
|
+
const requested = Number(process.env.HIPPO_CLEF_TIMEOUT_MS);
|
|
141
|
+
const timeoutMs = Number.isInteger(requested) && requested > 0 && requested <= MAX_TIMEOUT_MS ? requested : DEFAULT_TIMEOUT_MS;
|
|
92
142
|
const headers = new Headers({ 'content-type': 'application/json' });
|
|
93
143
|
if (route.token)
|
|
94
144
|
headers.set('authorization', `Bearer ${route.token}`);
|
|
145
|
+
const controller = new AbortController();
|
|
146
|
+
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
|
95
147
|
try {
|
|
96
148
|
const resp = await fetch(route.url, {
|
|
97
149
|
method: 'POST',
|
|
@@ -105,7 +157,7 @@ async function requestScores(model, query, head, route) {
|
|
|
105
157
|
await resp.body?.cancel();
|
|
106
158
|
throw new Error(`HTTP ${resp.status}${ray ? `, ray ${ray}` : ''}`);
|
|
107
159
|
}
|
|
108
|
-
const body = await resp
|
|
160
|
+
const body = await readCappedJson(resp);
|
|
109
161
|
const parsed = parseClefReply(body, head.length, model, route.backend === 'cloudflare');
|
|
110
162
|
if (isRejection(parsed))
|
|
111
163
|
throw new Error(parsed);
|
|
@@ -127,7 +179,8 @@ function nativeOrder(head, provenance) {
|
|
|
127
179
|
rerankScore: r.score,
|
|
128
180
|
preRerankRank: r.preRerankRank ?? i + 1,
|
|
129
181
|
postRerankRank: i + 1,
|
|
130
|
-
|
|
182
|
+
// A copy per row, so a caller editing one result cannot change another's provenance.
|
|
183
|
+
rerankProvenance: { ...provenance },
|
|
131
184
|
}));
|
|
132
185
|
}
|
|
133
186
|
/** A CLEF reranker for one model: Jev's request shape and pool; any failure keeps the native order (never paid Jev), warning once. */
|
|
@@ -137,42 +190,30 @@ export function createClefReranker(model) {
|
|
|
137
190
|
const head = results.slice(0, options?.topK ?? JEV_DEFAULT_TOP_K);
|
|
138
191
|
if (head.length === 0)
|
|
139
192
|
return [];
|
|
140
|
-
let
|
|
193
|
+
let route;
|
|
141
194
|
let got;
|
|
142
195
|
try {
|
|
143
196
|
if (head.length > MAX_CANDIDATES)
|
|
144
197
|
throw new Error(`more than ${MAX_CANDIDATES} candidates`);
|
|
145
|
-
|
|
146
|
-
backend = route.backend;
|
|
198
|
+
route = resolveClefRoute(model);
|
|
147
199
|
got = await requestScores(model, query, head, route);
|
|
148
200
|
}
|
|
149
201
|
catch (err) {
|
|
150
202
|
const reason = err instanceof Error ? err.message : 'unknown error';
|
|
151
203
|
if (!warned) {
|
|
152
204
|
warned = true;
|
|
153
|
-
|
|
154
|
-
console.warn(`[hippo] ${model} reranker unavailable (${reason}); keeping the native order. Subsequent calls will not repeat this warning.`);
|
|
205
|
+
log.warn(`${model} reranker unavailable (${reason}); keeping the native order. Subsequent calls will not repeat this warning.`);
|
|
155
206
|
}
|
|
156
207
|
return nativeOrder(head, { backend: 'native', requestedModel: model, fallbackReason: reason });
|
|
157
208
|
}
|
|
158
|
-
const
|
|
159
|
-
backend,
|
|
209
|
+
const rerankProvenance = {
|
|
210
|
+
backend: route.backend,
|
|
160
211
|
requestedModel: model,
|
|
161
212
|
actualModel: got.actualModel,
|
|
162
213
|
inputTokens: got.inputTokens,
|
|
163
214
|
outputTokens: got.outputTokens,
|
|
164
215
|
};
|
|
165
|
-
|
|
166
|
-
...r,
|
|
167
|
-
rerankScore: got.scores[i],
|
|
168
|
-
preRerankRank: r.preRerankRank ?? i + 1,
|
|
169
|
-
postRerankRank: 0,
|
|
170
|
-
rerankProvenance: provenance,
|
|
171
|
-
}));
|
|
172
|
-
// Stable sort: ties fall back to the prior relevance order.
|
|
173
|
-
scored.sort((a, b) => b.rerankScore - a.rerankScore);
|
|
174
|
-
scored.forEach((r, i) => (r.postRerankRank = i + 1));
|
|
175
|
-
return scored;
|
|
216
|
+
return rankByScores(head, got.scores).map((r) => ({ ...r, rerankProvenance: { ...rerankProvenance } }));
|
|
176
217
|
};
|
|
177
218
|
}
|
|
178
219
|
/** Opt-in CLEF-flash reranker (Cloudflare Workers AI or HIPPO_CLEF_ENDPOINT); off unless named, so defaults stay native. */
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { createRequire } from 'node:module';
|
|
2
2
|
import { pathToFileURL } from 'node:url';
|
|
3
|
+
import { log } from '../log.js';
|
|
3
4
|
const MODEL_NAME = 'Xenova/ms-marco-MiniLM-L-6-v2';
|
|
4
5
|
const _require = createRequire(import.meta.url);
|
|
5
6
|
const TRANSFORMERS_PACKAGES = ['@huggingface/transformers', '@xenova/transformers'];
|
|
@@ -36,7 +37,8 @@ async function loadTransformersModule() {
|
|
|
36
37
|
const seq = mod.AutoModelForSequenceClassification ?? mod.default?.AutoModelForSequenceClassification;
|
|
37
38
|
return tok && seq ? { AutoTokenizer: tok, AutoModelForSequenceClassification: seq } : null;
|
|
38
39
|
}
|
|
39
|
-
catch {
|
|
40
|
+
catch (err) {
|
|
41
|
+
log.debug(`cross-encoder: transformers import failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
40
42
|
return null;
|
|
41
43
|
}
|
|
42
44
|
}
|
|
@@ -78,7 +80,8 @@ async function buildPipeline() {
|
|
|
78
80
|
return score;
|
|
79
81
|
};
|
|
80
82
|
}
|
|
81
|
-
catch {
|
|
83
|
+
catch (err) {
|
|
84
|
+
log.debug(`cross-encoder: model load failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
82
85
|
return null;
|
|
83
86
|
}
|
|
84
87
|
}
|
|
@@ -102,8 +105,7 @@ export const crossEncoderReranker = async (query, results, options) => {
|
|
|
102
105
|
// working reranker.
|
|
103
106
|
if (!warnedOnFallback) {
|
|
104
107
|
warnedOnFallback = true;
|
|
105
|
-
|
|
106
|
-
console.warn('[hippo] cross-encoder reranker unavailable (no Transformers.js backend, or model fetch blocked); falling back to identity ordering. Subsequent calls will not repeat this warning.');
|
|
108
|
+
log.warn('cross-encoder reranker unavailable (no Transformers.js backend, or model fetch blocked); falling back to identity ordering. Subsequent calls will not repeat this warning.');
|
|
107
109
|
}
|
|
108
110
|
return head.map((r, i) => ({
|
|
109
111
|
...r,
|