@gamaze/hicortex 0.19.2 → 0.19.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -5
- package/dist/backup.d.ts +50 -0
- package/dist/backup.js +118 -2
- package/dist/capture.d.ts +27 -2
- package/dist/capture.js +103 -6
- package/dist/embedder.d.ts +23 -0
- package/dist/embedder.js +29 -0
- package/dist/index.d.ts +19 -0
- package/dist/index.js +286 -5
- package/dist/mcp-server.d.ts +25 -4
- package/dist/mcp-server.js +101 -38
- package/dist/nightly.d.ts +25 -0
- package/dist/nightly.js +125 -20
- package/dist/prompts.d.ts +15 -0
- package/dist/prompts.js +21 -3
- package/dist/recall-index.d.ts +105 -4
- package/dist/recall-index.js +217 -2
- package/dist/retrieval.d.ts +37 -0
- package/dist/retrieval.js +17 -2
- package/dist/storage.d.ts +30 -0
- package/dist/storage.js +46 -1
- package/dist/type-classify.js +3 -1
- package/dist/types.d.ts +17 -0
- package/dist/uninstall.d.ts +33 -0
- package/dist/uninstall.js +78 -26
- package/dist/viz.d.ts +1 -1
- package/dist/viz.js +28 -1
- package/openclaw.plugin.json +10 -1
- package/package.json +1 -1
package/dist/recall-index.js
CHANGED
|
@@ -19,6 +19,33 @@
|
|
|
19
19
|
* per-session TURN-based dedup (SessionRecallRegistry), short-prompt skip,
|
|
20
20
|
* and a hard item cap. On a prompt with no relevant memories the block is
|
|
21
21
|
* null and the hook prints nothing.
|
|
22
|
+
*
|
|
23
|
+
* Novelty floor (#324): the session-intent blend (#192 session-intent keying)
|
|
24
|
+
* can dilute a topic-switching prompt below the relevance floor — the live
|
|
25
|
+
* failure was a technically-primed session asking about "my Sargo" and getting
|
|
26
|
+
* ZERO relevant memories while a fresh session with the identical prompt got
|
|
27
|
+
* the perfect top hit. So a second, PURE-prompt search (no centroid blend,
|
|
28
|
+
* SAME candidate window as the blended search) runs alongside the blended
|
|
29
|
+
* one, and its top hit(s) that pass the floor are GUARANTEED slots in the
|
|
30
|
+
* index (dedup by id against the blended picks, capped by
|
|
31
|
+
* `noveltyFloorSlots`; rendered first). When the pure top hits are already
|
|
32
|
+
* among the blended picks — the common continuing-intent case — the output is
|
|
33
|
+
* unchanged. Turn suppression still wins: a recently shown novelty pick is
|
|
34
|
+
* suppressed like any other (the guarantee is about candidate inclusion, not
|
|
35
|
+
* forcing re-shows).
|
|
36
|
+
*
|
|
37
|
+
* #329 item 3 — the pure search is SKIPPED when it would be byte-identical
|
|
38
|
+
* to the blended one: turn 1 (no centroid yet — nothing to blend) or
|
|
39
|
+
* sessionIntentWeight 0 (blend disabled). The blended result IS the pure
|
|
40
|
+
* result there, so the floor is trivially satisfied by the blended picks and
|
|
41
|
+
* the second search (embeds aside, its whole DB + FTS half) is pure waste.
|
|
42
|
+
*
|
|
43
|
+
* #329 item 4 — novelty backfill: when the blended picks are empty/short,
|
|
44
|
+
* unclaimed maxItems slots are filled from the remaining filtered
|
|
45
|
+
* pure-prompt tail (gate + suppression already applied). Without it the
|
|
46
|
+
* topic-switch turn — the one the floor exists for — got the MOST truncated
|
|
47
|
+
* menu: novelty slots + a diluted remainder, while further pure candidates
|
|
48
|
+
* that had already passed every gate sat unused.
|
|
22
49
|
*/
|
|
23
50
|
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
24
51
|
if (k2 === undefined) k2 = k;
|
|
@@ -54,15 +81,19 @@ var __importStar = (this && this.__importStar) || (function () {
|
|
|
54
81
|
};
|
|
55
82
|
})();
|
|
56
83
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
84
|
+
exports.MAX_SESSION_ID_CHARS = exports.DEFAULT_NOVELTY_FLOOR_SLOTS = void 0;
|
|
85
|
+
exports.resolveNoveltyFloorSlots = resolveNoveltyFloorSlots;
|
|
57
86
|
exports.memoryTitle = memoryTitle;
|
|
58
87
|
exports.formatIndexLine = formatIndexLine;
|
|
59
88
|
exports.passesRelevanceGate = passesRelevanceGate;
|
|
89
|
+
exports.createRecallRetrieveFn = createRecallRetrieveFn;
|
|
60
90
|
exports.parseStringListParam = parseStringListParam;
|
|
61
91
|
exports.handleRecallIndex = handleRecallIndex;
|
|
62
92
|
exports.handleMemoryGet = handleMemoryGet;
|
|
63
93
|
exports.formatMemoryGetText = formatMemoryGetText;
|
|
64
94
|
const storage = __importStar(require("./storage.js"));
|
|
65
95
|
const type_labels_js_1 = require("./type-labels.js");
|
|
96
|
+
const retrieval_js_1 = require("./retrieval.js");
|
|
66
97
|
/** Relevance-gate floor for vector-only candidates (config `recallMinSimilarity`).
|
|
67
98
|
* 0.62 (was 0.55; raised 2026-08-03 on the fine-grain floor sweep — see the
|
|
68
99
|
* minSimilarity doc above). */
|
|
@@ -74,12 +105,40 @@ const DEFAULT_MIN_PROMPT_LENGTH = 20;
|
|
|
74
105
|
/** Default index-line title length. 100 (reverted from 150 on 2026-08-03:
|
|
75
106
|
* eval #3 §5 showed 100 vs 150 statistically identical; 100 saves ~13% tokens). */
|
|
76
107
|
const DEFAULT_TITLE_CHARS = 100;
|
|
108
|
+
/** Default #324 novelty-floor slots (config `noveltyFloorSlots`). 2 mirrors
|
|
109
|
+
* coldExposureSlots sizing — enough to guarantee the pure-prompt top hit
|
|
110
|
+
* plus a runner-up, never a takeover of the index. The floor only SPENDS
|
|
111
|
+
* slots when a pure-prompt hit differs from the blended picks (topic
|
|
112
|
+
* switch); continuing-intent sessions pay nothing. Exported for the boot
|
|
113
|
+
* log's knob line (mcp-server resolves config-vs-default here, once). */
|
|
114
|
+
exports.DEFAULT_NOVELTY_FLOOR_SLOTS = 2;
|
|
115
|
+
/** Resolve the EFFECTIVE novelty floor (raw ?? default, clamped to
|
|
116
|
+
* [0, maxItems]) — one definition shared by the handler and the boot knob
|
|
117
|
+
* line so the logged value is what handleRecallIndex actually uses.
|
|
118
|
+
* maxItems may be the handler's already-resolved number OR raw config
|
|
119
|
+
* (boot-log site) — raw is resolved with the handler's exact constants. */
|
|
120
|
+
function resolveNoveltyFloorSlots(rawSlots, rawMaxItems) {
|
|
121
|
+
const maxItems = typeof rawMaxItems === "number"
|
|
122
|
+
? rawMaxItems
|
|
123
|
+
: clampInt(rawMaxItems, DEFAULT_MAX_ITEMS, 1, 20);
|
|
124
|
+
return clampInt(rawSlots, exports.DEFAULT_NOVELTY_FLOOR_SLOTS, 0, maxItems);
|
|
125
|
+
}
|
|
77
126
|
/** Over-fetch multiplier: retrieve `maxItems × 3` candidates so gating + dedup
|
|
78
127
|
* still leave a full menu. Kept at 3 after maxItems 6→5 and the higher floor —
|
|
79
128
|
* permit-short is intended (returning fewer than maxItems when fewer clear the
|
|
80
129
|
* gate is correct, not a defect); raise only if blocks are persistently
|
|
81
130
|
* under-filled in production. */
|
|
82
131
|
const CANDIDATE_MULTIPLIER = 3;
|
|
132
|
+
/**
|
|
133
|
+
* Hard cap on `session_id` length (#328 item 2a). The id is retained as a Map
|
|
134
|
+
* key by SessionRecallRegistry for the process lifetime (maxSessions=500 LRU
|
|
135
|
+
* + a per-session shown-set + intent centroid), so an unbounded id is an OOM
|
|
136
|
+
* vector: ~4.9MB ids × 500 sessions ≈ 2.4GB of retained keys from an
|
|
137
|
+
* authenticated-but-hostile tenant. Real session ids (CC UUIDs, plugin
|
|
138
|
+
* session keys) are ≤64 chars — 128 is generous headroom. Longer → 400 with
|
|
139
|
+
* a clear error; the client treats it like any bad request.
|
|
140
|
+
*/
|
|
141
|
+
exports.MAX_SESSION_ID_CHARS = 128;
|
|
83
142
|
/** First content line, de-markdowned and truncated — the index line title. */
|
|
84
143
|
function memoryTitle(content, maxLen = DEFAULT_TITLE_CHARS) {
|
|
85
144
|
const firstLine = content
|
|
@@ -139,6 +198,78 @@ function passesRelevanceGate(r, minSimilarity) {
|
|
|
139
198
|
return true;
|
|
140
199
|
return typeof r.similarity === "number" && r.similarity >= minSimilarity;
|
|
141
200
|
}
|
|
201
|
+
/**
|
|
202
|
+
* The PRODUCTION /recall-index retrieveFn (what mcp-server wires into
|
|
203
|
+
* handleRecallIndex), extracted from the route handler so the #324 path is
|
|
204
|
+
* testable without HTTP — same precedent as blendQueryVector/recallQueryVector
|
|
205
|
+
* ("extracted from the /recall-index closure so the exact decision is
|
|
206
|
+
* unit-testable").
|
|
207
|
+
*
|
|
208
|
+
* Per call:
|
|
209
|
+
* - embed the prompt ONCE per request — a single-entry promise memo keyed
|
|
210
|
+
* on the query text. The blended and pure-prompt searches of one request
|
|
211
|
+
* carry the same prompt, so they share one embed; the factory is built
|
|
212
|
+
* per request, so the memo never outlives it.
|
|
213
|
+
* - resolve the search vector via retrieval.recallQueryVector (blend + EMA
|
|
214
|
+
* fold, or the pure prompt with NO centroid state for #324);
|
|
215
|
+
* - retrieve() with noStrengthen (exposure is recorded by
|
|
216
|
+
* handleRecallIndex via touchMemoriesShown, never here).
|
|
217
|
+
*
|
|
218
|
+
* #329 CR finding 1b: the FTS candidate list is ALSO computed once per
|
|
219
|
+
* request (ftsOnce, keyed on query + candidate window) and threaded into both
|
|
220
|
+
* retrieve() calls via the ftsCandidates provider — the blended and pure
|
|
221
|
+
* searches of one request carry identical query text and window, so their FTS
|
|
222
|
+
* halves were byte-identical SQL executed twice. `ftsFn` is the DI seam for
|
|
223
|
+
* tests (production: storage.searchFts); a throwing FTS computation memoizes
|
|
224
|
+
* to an empty shared list — the same vector-only degradation retrieve()'s
|
|
225
|
+
* catch always produced, never an error.
|
|
226
|
+
*/
|
|
227
|
+
function createRecallRetrieveFn(deps) {
|
|
228
|
+
let embMemo = null;
|
|
229
|
+
const embedOnce = (query) => {
|
|
230
|
+
if (!embMemo || embMemo.query !== query) {
|
|
231
|
+
embMemo = { query, p: deps.embedFn(query) };
|
|
232
|
+
}
|
|
233
|
+
return embMemo.p;
|
|
234
|
+
};
|
|
235
|
+
const ftsResolve = deps.ftsFn ?? storage.searchFts;
|
|
236
|
+
let ftsMemo = null;
|
|
237
|
+
const ftsOnce = (query, limit) => {
|
|
238
|
+
if (!ftsMemo || ftsMemo.query !== query || ftsMemo.limit !== limit) {
|
|
239
|
+
try {
|
|
240
|
+
ftsMemo = { query, limit, rows: ftsResolve(deps.db, query, limit) };
|
|
241
|
+
}
|
|
242
|
+
catch {
|
|
243
|
+
// Same degradation retrieve()'s own catch always produced — the FTS
|
|
244
|
+
// list is dropped and the search proceeds vector-only.
|
|
245
|
+
ftsMemo = { query, limit, rows: [] };
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
return ftsMemo.rows;
|
|
249
|
+
};
|
|
250
|
+
return async (query, limit, filters, sessionId, purePrompt) => {
|
|
251
|
+
const { weight, alpha } = (0, retrieval_js_1.getSessionIntent)();
|
|
252
|
+
const promptEmb = await embedOnce(query);
|
|
253
|
+
const queryVec = (0, retrieval_js_1.recallQueryVector)(deps.registry, sessionId, promptEmb, {
|
|
254
|
+
weight,
|
|
255
|
+
alpha,
|
|
256
|
+
purePrompt,
|
|
257
|
+
});
|
|
258
|
+
return (0, retrieval_js_1.retrieve)(deps.db, deps.embedFn, query, {
|
|
259
|
+
limit,
|
|
260
|
+
noStrengthen: true,
|
|
261
|
+
// #203: project + mission_domains are SOFT affinity (zero-boost
|
|
262
|
+
// neutral), threaded into computeScore.
|
|
263
|
+
project: filters?.project,
|
|
264
|
+
missionDomains: filters?.mission_domains,
|
|
265
|
+
queryEmbedding: queryVec,
|
|
266
|
+
// #329: shared per-request FTS list. The recall path never passes
|
|
267
|
+
// sourceAgent, so the memo is keyed on (query, fetchLimit) only —
|
|
268
|
+
// exactly the two things retrieve() would pass to searchFts.
|
|
269
|
+
ftsCandidates: (fetchLimit) => ftsOnce(query, fetchLimit),
|
|
270
|
+
});
|
|
271
|
+
};
|
|
272
|
+
}
|
|
142
273
|
/** Normalize a request-supplied string-list param: array of strings or a CSV
|
|
143
274
|
* string → string[] | undefined. Anything else (or an empty result) means
|
|
144
275
|
* "absent" — never a partial guess. Used by `mission_domains` (#203) so it
|
|
@@ -162,6 +293,16 @@ async function handleRecallIndex(deps, body) {
|
|
|
162
293
|
if (!sessionId) {
|
|
163
294
|
return { status: 400, body: { error: "Missing 'session_id'" } };
|
|
164
295
|
}
|
|
296
|
+
// Length cap (#328 item 2a) — BEFORE the reset branch so an oversized id
|
|
297
|
+
// never reaches ANY registry call (reset() itself only deletes, but the
|
|
298
|
+
// next non-reset call with the same id would beginTurn it into a retained
|
|
299
|
+
// Map key). Clear error so a misbehaving client can self-diagnose.
|
|
300
|
+
if (sessionId.length > exports.MAX_SESSION_ID_CHARS) {
|
|
301
|
+
return {
|
|
302
|
+
status: 400,
|
|
303
|
+
body: { error: `'session_id' too long (max ${exports.MAX_SESSION_ID_CHARS} chars, got ${sessionId.length})` },
|
|
304
|
+
};
|
|
305
|
+
}
|
|
165
306
|
// Reset: SessionStart (startup/resume/clear/compact) — fresh context, so the
|
|
166
307
|
// shown-set is stale by definition.
|
|
167
308
|
if (req.reset === true) {
|
|
@@ -176,6 +317,9 @@ async function handleRecallIndex(deps, body) {
|
|
|
176
317
|
const maxItems = clampInt(deps.options?.maxItems, DEFAULT_MAX_ITEMS, 1, 20);
|
|
177
318
|
const titleChars = clampInt(deps.options?.titleChars, DEFAULT_TITLE_CHARS, 40, 400);
|
|
178
319
|
const minSimilarity = clampNumber(deps.options?.minSimilarity, DEFAULT_MIN_SIMILARITY, 0, 1);
|
|
320
|
+
// #324: clamped to [0, maxItems] — the floor is a reservation inside the
|
|
321
|
+
// item cap, never an expansion of it.
|
|
322
|
+
const noveltySlots = resolveNoveltyFloorSlots(deps.options?.noveltyFloorSlots, maxItems);
|
|
179
323
|
const turn = deps.registry.beginTurn(sessionId);
|
|
180
324
|
// Optional client-side scoping (F1 + #203): project + mission_domains (soft
|
|
181
325
|
// affinity) ride the body and are pushed into retrieval. project is cwd-
|
|
@@ -187,9 +331,33 @@ async function handleRecallIndex(deps, body) {
|
|
|
187
331
|
project: typeof req.project === "string" && req.project ? req.project : undefined,
|
|
188
332
|
mission_domains: parseStringListParam(req.mission_domains),
|
|
189
333
|
};
|
|
334
|
+
// #324 + #329 item 3: when the floor is armed AND would differ from the
|
|
335
|
+
// blended search, TWO searches run per recall — the blended (session-intent)
|
|
336
|
+
// query that has always run, and a PURE-prompt query with no centroid blend.
|
|
337
|
+
// Issued together so the second adds no wall-clock latency beyond its own DB
|
|
338
|
+
// work (the prompt is embedded once — the closure memoizes). Same failure
|
|
339
|
+
// domain (same db + embedder): either failing fails the request explicitly;
|
|
340
|
+
// no silent blended-only degradation.
|
|
341
|
+
//
|
|
342
|
+
// The SKIP (#329 item 3): on turn 1 the registry has no centroid yet (the
|
|
343
|
+
// blended call reads-before-fold — recallQueryVector), and at
|
|
344
|
+
// sessionIntentWeight 0 the centroid is never read at all. In both cases
|
|
345
|
+
// the blended query vector IS the pure prompt vector, so the second search
|
|
346
|
+
// would return byte-identical candidates — skip it (the floor is trivially
|
|
347
|
+
// satisfied: every pure hit is by construction among the blended picks).
|
|
348
|
+
// The decision is made BEFORE any retrieveFn call, i.e. on the centroid
|
|
349
|
+
// state of the PREVIOUS turns — exactly the turn-1/turn-2 distinction.
|
|
350
|
+
const runPureSearch = noveltySlots > 0 &&
|
|
351
|
+
(0, retrieval_js_1.getSessionIntent)().weight > 0 &&
|
|
352
|
+
deps.registry.getCentroid(sessionId) !== undefined;
|
|
190
353
|
let results;
|
|
354
|
+
let pureResults;
|
|
191
355
|
try {
|
|
192
|
-
|
|
356
|
+
const blended = deps.retrieveFn(prompt, maxItems * CANDIDATE_MULTIPLIER, filters, sessionId);
|
|
357
|
+
const pure = runPureSearch
|
|
358
|
+
? deps.retrieveFn(prompt, maxItems * CANDIDATE_MULTIPLIER, filters, sessionId, true)
|
|
359
|
+
: Promise.resolve([]);
|
|
360
|
+
[results, pureResults] = await Promise.all([blended, pure]);
|
|
193
361
|
}
|
|
194
362
|
catch (err) {
|
|
195
363
|
return {
|
|
@@ -197,10 +365,57 @@ async function handleRecallIndex(deps, body) {
|
|
|
197
365
|
body: { error: err instanceof Error ? err.message : String(err) },
|
|
198
366
|
};
|
|
199
367
|
}
|
|
200
|
-
|
|
368
|
+
// Blended (session-intent) picks: relevance gate + turn-based suppression,
|
|
369
|
+
// top maxItems — exactly what the index would show with no novelty floor.
|
|
370
|
+
const blendedPicks = results
|
|
201
371
|
.filter((r) => passesRelevanceGate(r, minSimilarity))
|
|
202
372
|
.filter((r) => deps.registry.isShowable(sessionId, r.id))
|
|
203
373
|
.slice(0, maxItems);
|
|
374
|
+
// #324 novelty floor: the best match(es) for the CURRENT PROMPT ALONE are
|
|
375
|
+
// guaranteed a place in the index. The blend exists to follow session
|
|
376
|
+
// intent, not to veto the prompt — so pure-prompt hits that pass the floor
|
|
377
|
+
// enter even when the blended query diluted them out of `results`
|
|
378
|
+
// entirely. Dedup is against the blended PICKS (the no-floor outcome): when
|
|
379
|
+
// the pure top hit is already shown by the blended path — the common
|
|
380
|
+
// continuing-intent case — the floor costs nothing and the output is
|
|
381
|
+
// unchanged. Suppression applies BEFORE the guarantee (suppression wins:
|
|
382
|
+
// the floor is about candidate inclusion, not forcing re-shows). FTS-sourced
|
|
383
|
+
// pure hits pass the gate unconditionally, same as the blended path.
|
|
384
|
+
//
|
|
385
|
+
// The gate + suppression are applied ONCE to the pure list: the head feeds
|
|
386
|
+
// the novelty floor, the tail feeds the #329 backfill below.
|
|
387
|
+
const pureFiltered = pureResults
|
|
388
|
+
.filter((r) => passesRelevanceGate(r, minSimilarity))
|
|
389
|
+
.filter((r) => deps.registry.isShowable(sessionId, r.id));
|
|
390
|
+
const blendedIds = new Set(blendedPicks.map((r) => r.id));
|
|
391
|
+
const noveltyPicks = pureFiltered
|
|
392
|
+
.filter((r) => !blendedIds.has(r.id))
|
|
393
|
+
.slice(0, noveltySlots);
|
|
394
|
+
// The floor takes precedence (#324 vs #192 cold slots): novelty picks hold
|
|
395
|
+
// their slots; blended picks keep the remainder, evicted from the TAIL
|
|
396
|
+
// (lowest rank first) so the session-intent head survives. Cold-exposure
|
|
397
|
+
// slots continue to apply inside each retrieve()'s own top-k. Total never
|
|
398
|
+
// exceeds maxItems. Render order: novelty picks FIRST — on a topic switch
|
|
399
|
+
// they are the most relevant lines to the CURRENT turn, and the head of the
|
|
400
|
+
// block carries the most weight for a reader scanning the menu.
|
|
401
|
+
let picked = [
|
|
402
|
+
...noveltyPicks,
|
|
403
|
+
...blendedPicks.slice(0, Math.max(0, maxItems - noveltyPicks.length)),
|
|
404
|
+
];
|
|
405
|
+
// #329 item 4 — backfill: a topic-switch turn dilutes the blended picks, so
|
|
406
|
+
// picked can land below maxItems even though FURTHER pure candidates have
|
|
407
|
+
// already passed the gate + suppression + dedup (they sit in the pure tail
|
|
408
|
+
// beyond the first noveltyFloorSlots). Fill the unclaimed slots from that
|
|
409
|
+
// tail — without it, the turn the floor exists for got the most truncated
|
|
410
|
+
// menu. Continuing-intent sessions are untouched: blended picks full →
|
|
411
|
+
// nothing to backfill (zero-delta output preserved).
|
|
412
|
+
if (picked.length < maxItems) {
|
|
413
|
+
const pickedIds = new Set(picked.map((r) => r.id));
|
|
414
|
+
const backfill = pureFiltered
|
|
415
|
+
.filter((r) => !pickedIds.has(r.id))
|
|
416
|
+
.slice(0, maxItems - picked.length);
|
|
417
|
+
picked = [...picked, ...backfill];
|
|
418
|
+
}
|
|
204
419
|
if (picked.length === 0) {
|
|
205
420
|
return { status: 200, body: { block: null, shown: [], turn } };
|
|
206
421
|
}
|
package/dist/retrieval.d.ts
CHANGED
|
@@ -115,6 +115,32 @@ export declare function getSessionIntent(): {
|
|
|
115
115
|
* closure-integration harness.
|
|
116
116
|
*/
|
|
117
117
|
export declare function blendQueryVector(promptEmb: Float32Array, centroid: Float32Array | undefined, weight: number): Float32Array;
|
|
118
|
+
/**
|
|
119
|
+
* The /recall-index closure's PER-CALL search-vector decision (#199 + #324),
|
|
120
|
+
* extracted next to blendQueryVector (same precedent: the exact decision must
|
|
121
|
+
* be unit-testable without a closure-integration harness).
|
|
122
|
+
*
|
|
123
|
+
* - purePrompt (#324 novelty floor): return the prompt embedding UNBLENDED
|
|
124
|
+
* and touch NO centroid state — neither read nor the EMA fold. The folded
|
|
125
|
+
* turn is owned by the blended call; a second fold here would double-count
|
|
126
|
+
* the prompt and skew every later turn's blend (the single worst
|
|
127
|
+
* regression this extraction exists to lock out).
|
|
128
|
+
* - blended (default): read the prior centroid (weight>0 only), blend, then
|
|
129
|
+
* fold this turn's prompt ONCE (weight>0 only) — read-before-update so
|
|
130
|
+
* turn 1 searches pure and seeds the centroid for turn 2.
|
|
131
|
+
*
|
|
132
|
+
* `registry` is the structural surface needed (SessionRecallRegistry
|
|
133
|
+
* satisfies it) — keeps this module decoupled from the registry class.
|
|
134
|
+
*/
|
|
135
|
+
export interface CentroidStore {
|
|
136
|
+
getCentroid(sessionId: string): Float32Array | undefined;
|
|
137
|
+
updateCentroid(sessionId: string, promptEmbedding: Float32Array, alpha: number): Float32Array;
|
|
138
|
+
}
|
|
139
|
+
export declare function recallQueryVector(registry: CentroidStore, sessionId: string, promptEmb: Float32Array, opts: {
|
|
140
|
+
weight: number;
|
|
141
|
+
alpha: number;
|
|
142
|
+
purePrompt?: boolean;
|
|
143
|
+
}): Float32Array;
|
|
118
144
|
/**
|
|
119
145
|
* Ids among `candidateIds` that have been superseded by a later memory — i.e.
|
|
120
146
|
* they are the SOURCE of a `superseded_by` link (stageSupersession links
|
|
@@ -205,6 +231,17 @@ export declare function retrieve(db: Database.Database, embedFn: EmbedFn, query:
|
|
|
205
231
|
* and get pure-prompt behavior (the query string is embedded here). The
|
|
206
232
|
* FTS path still uses the raw `query` text regardless. */
|
|
207
233
|
queryEmbedding?: Float32Array;
|
|
234
|
+
/** #329 CR finding 1b: caller-provided FTS candidate resolution, called
|
|
235
|
+
* INSTEAD of running storage.searchFts here. The /recall-index closure
|
|
236
|
+
* passes a per-request memoized provider so the blended and pure
|
|
237
|
+
* searches of ONE request — same query text, same candidate window —
|
|
238
|
+
* execute the FTS half exactly once and share the list. The provider
|
|
239
|
+
* receives the fetchLimit/sourceAgent THIS call would have used, so the
|
|
240
|
+
* shared list is always computed with the right window. Callers that
|
|
241
|
+
* omit it get the previous behavior (retrieve runs searchFts itself). */
|
|
242
|
+
ftsCandidates?: (fetchLimit: number, sourceAgent?: string) => Array<Memory & {
|
|
243
|
+
rank: number;
|
|
244
|
+
}>;
|
|
208
245
|
}): Promise<MemorySearchResult[]>;
|
|
209
246
|
/**
|
|
210
247
|
* Get recent context, optionally filtered by project.
|
package/dist/retrieval.js
CHANGED
|
@@ -61,6 +61,7 @@ exports.getScoringWeights = getScoringWeights;
|
|
|
61
61
|
exports.configureSessionIntent = configureSessionIntent;
|
|
62
62
|
exports.getSessionIntent = getSessionIntent;
|
|
63
63
|
exports.blendQueryVector = blendQueryVector;
|
|
64
|
+
exports.recallQueryVector = recallQueryVector;
|
|
64
65
|
exports.findSupersededIds = findSupersededIds;
|
|
65
66
|
exports.l2ToCosine = l2ToCosine;
|
|
66
67
|
exports.effectiveStrength = effectiveStrength;
|
|
@@ -250,6 +251,16 @@ function blendQueryVector(promptEmb, centroid, weight) {
|
|
|
250
251
|
? (0, schema_prototypes_js_1.l2Normalize)((0, schema_prototypes_js_1.weightedAdd)(promptEmb, 1 - weight, centroid, weight))
|
|
251
252
|
: promptEmb;
|
|
252
253
|
}
|
|
254
|
+
function recallQueryVector(registry, sessionId, promptEmb, opts) {
|
|
255
|
+
if (opts.purePrompt)
|
|
256
|
+
return promptEmb;
|
|
257
|
+
// weight=0 (kill-switch): the centroid is neither read nor written.
|
|
258
|
+
const centroid = opts.weight > 0 ? registry.getCentroid(sessionId) : undefined;
|
|
259
|
+
const queryVec = blendQueryVector(promptEmb, centroid, opts.weight);
|
|
260
|
+
if (opts.weight > 0)
|
|
261
|
+
registry.updateCentroid(sessionId, promptEmb, opts.alpha);
|
|
262
|
+
return queryVec;
|
|
263
|
+
}
|
|
253
264
|
/**
|
|
254
265
|
* Ids among `candidateIds` that have been superseded by a later memory — i.e.
|
|
255
266
|
* they are the SOURCE of a `superseded_by` link (stageSupersession links
|
|
@@ -545,8 +556,12 @@ async function retrieve(db, embedFn, query, options) {
|
|
|
545
556
|
try {
|
|
546
557
|
// sourceAgent is pushed into the FTS SQL (hard filter). project is NOT (it
|
|
547
558
|
// is a soft affinity boost in computeScore as of #203). privacy is NOT
|
|
548
|
-
// (0.16.x: vestigial column, never filtered).
|
|
549
|
-
|
|
559
|
+
// (0.16.x: vestigial column, never filtered). With a caller-provided
|
|
560
|
+
// provider (#329 request-level memo) the same list is shared across the
|
|
561
|
+
// retrieves of one recall request instead of re-executed.
|
|
562
|
+
ftsCandidates = options?.ftsCandidates
|
|
563
|
+
? options.ftsCandidates(fetchLimit, sourceAgent)
|
|
564
|
+
: storage.searchFts(db, query, fetchLimit, sourceAgent);
|
|
550
565
|
}
|
|
551
566
|
catch {
|
|
552
567
|
// FTS5 search can fail on special characters; fall back to vector-only
|
package/dist/storage.d.ts
CHANGED
|
@@ -137,6 +137,36 @@ export interface Bm25Weights {
|
|
|
137
137
|
export declare function configureBm25Fts(config?: Record<string, unknown> | null): Bm25Weights;
|
|
138
138
|
/** Current resolved weights (tests + status output). */
|
|
139
139
|
export declare function getBm25Weights(): Bm25Weights;
|
|
140
|
+
/**
|
|
141
|
+
* Cap on tokens fed to an FTS5 MATCH expression (#329 CR finding 1a).
|
|
142
|
+
* Quoting made pasted term lists LEGAL queries — and an all-common-tokens AND
|
|
143
|
+
* is expensive: measured at 100K rows, a 50-token AND runs ~518ms and a
|
|
144
|
+
* 200-token one 6.3s, and the /recall-index hot path would pay it twice per
|
|
145
|
+
* prompt. Beyond ~24 tokens the implicit AND is semantic noise anyway (a
|
|
146
|
+
* memory matching 24+ ANDed prompt tokens is either the exact text or
|
|
147
|
+
* nothing), so the FIRST 24 tokens are used. 24 is a shipped bound, not a
|
|
148
|
+
* config knob — change it deliberately, with a perf measurement.
|
|
149
|
+
*/
|
|
150
|
+
export declare const FTS_MATCH_MAX_TOKENS = 24;
|
|
151
|
+
/**
|
|
152
|
+
* FTS5 MATCH-safety quoting (#329 item 1). The raw prompt is NOT valid FTS5
|
|
153
|
+
* query syntax: ordinary prompt punctuation (?, -, (, :, URLs, apostrophes, a
|
|
154
|
+
* leading AND/OR) crashes the FTS5 parser, and retrieval.retrieve's catch then
|
|
155
|
+
* silently drops the ENTIRE FTS candidate list — the perf sweep measured 8/12
|
|
156
|
+
* realistic prompts affected, and it is why relevance eval #3 saw 0 FTS rows
|
|
157
|
+
* in 2,208 candidates. Fix: tokenize on whitespace, strip embedded double
|
|
158
|
+
* quotes (a raw `"` would terminate our own quoting), and wrap each token in
|
|
159
|
+
* double quotes — a quoted token is a phrase of LITERAL strings, immune to
|
|
160
|
+
* FTS5 query syntax (`"what" "is" "the" "deployment" "status"`). Punctuation
|
|
161
|
+
* INSIDE a token is kept: the tokenizer strips it identically on both sides,
|
|
162
|
+
* so `"status?"` still matches content containing "status". Joined with spaces
|
|
163
|
+
* (implicit AND — the same semantics clean prompts always had; a PROSE prompt
|
|
164
|
+
* whose content holds only most of the tokens matches nothing, which is why
|
|
165
|
+
* FTS fires on short keyword prompts, not prose recall). Capped at the first
|
|
166
|
+
* FTS_MATCH_MAX_TOKENS tokens. A query that quotes away to nothing yields ""
|
|
167
|
+
* and the caller skips the SQL entirely.
|
|
168
|
+
*/
|
|
169
|
+
export declare function buildFtsMatchExpression(query: string): string;
|
|
140
170
|
/**
|
|
141
171
|
* Full-text search using FTS5 fielded BM25 (BM25F) ranking.
|
|
142
172
|
* Returns memories with a rank field (lower is better — see sign note below).
|
package/dist/storage.js
CHANGED
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
* Ported from hicortex/storage.py. All functions are synchronous (better-sqlite3).
|
|
5
5
|
*/
|
|
6
6
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
7
|
+
exports.FTS_MATCH_MAX_TOKENS = void 0;
|
|
7
8
|
exports.embedToBlob = embedToBlob;
|
|
8
9
|
exports.insertMemory = insertMemory;
|
|
9
10
|
exports.resolveMemoryId = resolveMemoryId;
|
|
@@ -20,6 +21,7 @@ exports.getStoredEmbedding = getStoredEmbedding;
|
|
|
20
21
|
exports.vectorSearch = vectorSearch;
|
|
21
22
|
exports.configureBm25Fts = configureBm25Fts;
|
|
22
23
|
exports.getBm25Weights = getBm25Weights;
|
|
24
|
+
exports.buildFtsMatchExpression = buildFtsMatchExpression;
|
|
23
25
|
exports.searchFts = searchFts;
|
|
24
26
|
exports.addLink = addLink;
|
|
25
27
|
exports.getLinks = getLinks;
|
|
@@ -345,6 +347,44 @@ function configureBm25Fts(config) {
|
|
|
345
347
|
function getBm25Weights() {
|
|
346
348
|
return { ...bm25Weights };
|
|
347
349
|
}
|
|
350
|
+
/**
|
|
351
|
+
* Cap on tokens fed to an FTS5 MATCH expression (#329 CR finding 1a).
|
|
352
|
+
* Quoting made pasted term lists LEGAL queries — and an all-common-tokens AND
|
|
353
|
+
* is expensive: measured at 100K rows, a 50-token AND runs ~518ms and a
|
|
354
|
+
* 200-token one 6.3s, and the /recall-index hot path would pay it twice per
|
|
355
|
+
* prompt. Beyond ~24 tokens the implicit AND is semantic noise anyway (a
|
|
356
|
+
* memory matching 24+ ANDed prompt tokens is either the exact text or
|
|
357
|
+
* nothing), so the FIRST 24 tokens are used. 24 is a shipped bound, not a
|
|
358
|
+
* config knob — change it deliberately, with a perf measurement.
|
|
359
|
+
*/
|
|
360
|
+
exports.FTS_MATCH_MAX_TOKENS = 24;
|
|
361
|
+
/**
|
|
362
|
+
* FTS5 MATCH-safety quoting (#329 item 1). The raw prompt is NOT valid FTS5
|
|
363
|
+
* query syntax: ordinary prompt punctuation (?, -, (, :, URLs, apostrophes, a
|
|
364
|
+
* leading AND/OR) crashes the FTS5 parser, and retrieval.retrieve's catch then
|
|
365
|
+
* silently drops the ENTIRE FTS candidate list — the perf sweep measured 8/12
|
|
366
|
+
* realistic prompts affected, and it is why relevance eval #3 saw 0 FTS rows
|
|
367
|
+
* in 2,208 candidates. Fix: tokenize on whitespace, strip embedded double
|
|
368
|
+
* quotes (a raw `"` would terminate our own quoting), and wrap each token in
|
|
369
|
+
* double quotes — a quoted token is a phrase of LITERAL strings, immune to
|
|
370
|
+
* FTS5 query syntax (`"what" "is" "the" "deployment" "status"`). Punctuation
|
|
371
|
+
* INSIDE a token is kept: the tokenizer strips it identically on both sides,
|
|
372
|
+
* so `"status?"` still matches content containing "status". Joined with spaces
|
|
373
|
+
* (implicit AND — the same semantics clean prompts always had; a PROSE prompt
|
|
374
|
+
* whose content holds only most of the tokens matches nothing, which is why
|
|
375
|
+
* FTS fires on short keyword prompts, not prose recall). Capped at the first
|
|
376
|
+
* FTS_MATCH_MAX_TOKENS tokens. A query that quotes away to nothing yields ""
|
|
377
|
+
* and the caller skips the SQL entirely.
|
|
378
|
+
*/
|
|
379
|
+
function buildFtsMatchExpression(query) {
|
|
380
|
+
return query
|
|
381
|
+
.split(/\s+/)
|
|
382
|
+
.map((token) => token.replace(/"/g, ""))
|
|
383
|
+
.filter((token) => token.length > 0)
|
|
384
|
+
.slice(0, exports.FTS_MATCH_MAX_TOKENS)
|
|
385
|
+
.map((token) => `"${token}"`)
|
|
386
|
+
.join(" ");
|
|
387
|
+
}
|
|
348
388
|
/**
|
|
349
389
|
* Full-text search using FTS5 fielded BM25 (BM25F) ranking.
|
|
350
390
|
* Returns memories with a rank field (lower is better — see sign note below).
|
|
@@ -366,8 +406,13 @@ function getBm25Weights() {
|
|
|
366
406
|
* caching is unaffected and the config path is the only editor.
|
|
367
407
|
*/
|
|
368
408
|
function searchFts(db, query, limit = 10, sourceAgent) {
|
|
409
|
+
// #329: quote the query into literal phrases — a raw prompt crashes the
|
|
410
|
+
// FTS5 parser on punctuation and the caller's catch drops the whole list.
|
|
411
|
+
const matchExpr = buildFtsMatchExpression(query);
|
|
412
|
+
if (!matchExpr)
|
|
413
|
+
return [];
|
|
369
414
|
const conditions = ["memories_fts MATCH ?"];
|
|
370
|
-
const params = [
|
|
415
|
+
const params = [matchExpr];
|
|
371
416
|
if (sourceAgent) {
|
|
372
417
|
conditions.push("m.source_agent = ?");
|
|
373
418
|
params.push(sourceAgent);
|
package/dist/type-classify.js
CHANGED
|
@@ -84,7 +84,9 @@ function buildTypeClassifyPrompt(content) {
|
|
|
84
84
|
`"adopted the graded-schema tag model"). Not knowledge (it can change) and ` +
|
|
85
85
|
`not an experience (it persists). A bare AI recommendation or proposal is ` +
|
|
86
86
|
`NEVER a decision — "AI proposed X → user declined/held" is experience ` +
|
|
87
|
-
`(#290)
|
|
87
|
+
`(#290). Even if carried out by the user, a version bump, merge, or count ` +
|
|
88
|
+
`is never a decision — only the durable user-confirmed standardization it ` +
|
|
89
|
+
`embodies is (#329).\n\n` +
|
|
88
90
|
`IMPORTANCE (0.0–1.0):\n` +
|
|
89
91
|
`- 0.8–1.0: load-bearing — a core piece of knowledge or a decision the ` +
|
|
90
92
|
`agent must know.\n` +
|
package/dist/types.d.ts
CHANGED
|
@@ -413,6 +413,15 @@ export interface HicortexConfig {
|
|
|
413
413
|
* accepts no client limit.
|
|
414
414
|
*/
|
|
415
415
|
recallLimit?: number;
|
|
416
|
+
/**
|
|
417
|
+
* OC plugin (#326): auto-scaffold the dead-man guard line into the agent
|
|
418
|
+
* workspace bootstrap file (BOOTSTRAP.md) at service start — the #313
|
|
419
|
+
* SECONDARY layer under the injected IDENTITY UNAVAILABLE banner (which is
|
|
420
|
+
* the primary, plugin-side mechanism). Idempotent: a bootstrap already
|
|
421
|
+
* carrying the line is never rewritten. Default true; `false` disables both
|
|
422
|
+
* the write and any file creation entirely.
|
|
423
|
+
*/
|
|
424
|
+
scaffoldDeadMan?: boolean;
|
|
416
425
|
/**
|
|
417
426
|
* Soft cap on the memory corpus (default 10000). When the corpus exceeds this,
|
|
418
427
|
* the nightly's capacity-eviction stage (#245) removes the lowest-
|
|
@@ -465,6 +474,14 @@ export interface HicortexConfig {
|
|
|
465
474
|
* Operator-owned: point at a mounted backup volume, a tmpfs, etc.
|
|
466
475
|
*/
|
|
467
476
|
backupDir?: string;
|
|
477
|
+
/**
|
|
478
|
+
* Backup retention (#327): how many of the newest `hicortex-*.tar.gz`
|
|
479
|
+
* artifacts the backup dir keeps after each successful write. Default 7;
|
|
480
|
+
* 0 keeps everything. Without it every full nightly (and `hicortex backup`)
|
|
481
|
+
* adds an artifact forever — unbounded growth, per hosted tenant too. Only
|
|
482
|
+
* artifacts matching the product's own name pattern are ever pruned.
|
|
483
|
+
*/
|
|
484
|
+
backupRetention?: number;
|
|
468
485
|
/**
|
|
469
486
|
* Post-backup offsite hook (#6). When set, `hicortex backup` and the nightly
|
|
470
487
|
* backup stage invoke this command with the artifact path appended as the LAST
|
package/dist/uninstall.d.ts
CHANGED
|
@@ -14,4 +14,37 @@
|
|
|
14
14
|
export declare const SESSION_START_HOOK_COMMAND_RE: RegExp;
|
|
15
15
|
/** True when a CC hook `command` string runs the Hicortex SessionStart hook. */
|
|
16
16
|
export declare function isHicortexSessionStartHook(command: string): boolean;
|
|
17
|
+
/**
|
|
18
|
+
* Matches a CC hook `command` that runs the Hicortex recall hook — the
|
|
19
|
+
* `recall-hook` subcommand (#192). Same word-boundary discipline as the
|
|
20
|
+
* learnings matcher: an unrelated command that merely CONTAINS the substring
|
|
21
|
+
* ("my-recall-hook", "recall-hooks-old") is never swept up. Used for BOTH
|
|
22
|
+
* event arrays the installer writes (UserPromptSubmit + SessionStart).
|
|
23
|
+
*/
|
|
24
|
+
export declare const RECALL_HOOK_COMMAND_RE: RegExp;
|
|
25
|
+
/** True when a CC hook `command` string runs the Hicortex recall hook. */
|
|
26
|
+
export declare function isHicortexRecallHook(command: string): boolean;
|
|
27
|
+
/** One hook group removed from settings.json (for per-group logging). */
|
|
28
|
+
export interface RemovedHookGroup {
|
|
29
|
+
/** CC event array the entries were removed from ("SessionStart", "UserPromptSubmit"). */
|
|
30
|
+
event: string;
|
|
31
|
+
/** Which Hicortex hook set: "learnings" (learnings-identity/lessons-context) or "recall". */
|
|
32
|
+
kind: "learnings" | "recall";
|
|
33
|
+
/** Number of matcher entries removed. */
|
|
34
|
+
count: number;
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* Remove every Hicortex hook entry from a PARSED ~/.claude/settings.json
|
|
38
|
+
* (#327): the SessionStart learnings hook (canonical + legacy alias) AND the
|
|
39
|
+
* recall-hook pair (UserPromptSubmit + SessionStart, installed together by
|
|
40
|
+
* installRecallHooks — leaving either behind is a silent npx spawn per prompt
|
|
41
|
+
* forever). Mutates `settings` in place; returns what was removed (empty when
|
|
42
|
+
* nothing matched — a clean no-op). Exact-match discipline throughout: only
|
|
43
|
+
* entries whose `hooks[].command` matches a Hicortex subcommand are removed;
|
|
44
|
+
* foreign hooks (and prefix-colliding names) stay untouched.
|
|
45
|
+
*
|
|
46
|
+
* Pure on the parsed object so the uninstall behavior is unit-testable
|
|
47
|
+
* without spinning up CC; runUninstall owns the file I/O.
|
|
48
|
+
*/
|
|
49
|
+
export declare function removeHicortexCcHooks(settings: Record<string, unknown>): RemovedHookGroup[];
|
|
17
50
|
export declare function runUninstall(): Promise<void>;
|