@gamaze/hicortex 0.22.0 → 0.22.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/dashboard.html +209 -76
- package/dist/calibration.d.ts +92 -12
- package/dist/calibration.js +102 -15
- package/dist/classify-domains.js +6 -1
- package/dist/consolidate.js +87 -19
- package/dist/dashboard.d.ts +11 -0
- package/dist/dashboard.js +9 -0
- package/dist/db.js +74 -0
- package/dist/eval/decay-eval.d.ts +4 -2
- package/dist/eval/decay-eval.js +4 -4
- package/dist/eval/eval-clock.d.ts +32 -0
- package/dist/eval/eval-clock.js +47 -0
- package/dist/eval/graph-eval.d.ts +15 -2
- package/dist/eval/graph-eval.js +51 -5
- package/dist/eval/planted-eval.d.ts +4 -0
- package/dist/eval/planted-eval.js +27 -2
- package/dist/eval/planted-harness.d.ts +7 -0
- package/dist/eval/planted-harness.js +2 -0
- package/dist/eval/ranking-battery.d.ts +49 -2
- package/dist/eval/ranking-battery.js +110 -2
- package/dist/eval/ranking-eval.d.ts +26 -6
- package/dist/eval/ranking-eval.js +197 -34
- package/dist/eval/ranking-fixtures.d.ts +41 -1
- package/dist/eval/ranking-fixtures.js +261 -2
- package/dist/eval/recall-sweep.d.ts +7 -2
- package/dist/eval/recall-sweep.js +42 -13
- package/dist/eval/relevance-eval.d.ts +115 -1
- package/dist/eval/relevance-eval.js +318 -32
- package/dist/eval/run-eval.d.ts +7 -4
- package/dist/eval/run-eval.js +36 -9
- package/dist/mcp-server.js +37 -9
- package/dist/nightly.js +14 -0
- package/dist/recall-index.d.ts +46 -3
- package/dist/recall-index.js +83 -26
- package/dist/recall-precision.d.ts +212 -0
- package/dist/recall-precision.js +381 -0
- package/dist/retrieval.d.ts +34 -15
- package/dist/retrieval.js +132 -59
- package/dist/types.d.ts +7 -0
- package/package.json +1 -1
- package/server.json +3 -3
package/dist/mcp-server.js
CHANGED
|
@@ -90,6 +90,7 @@ const dedup_js_1 = require("./dedup.js");
|
|
|
90
90
|
const reconsolidation_js_1 = require("./reconsolidation.js");
|
|
91
91
|
const redact_js_1 = require("./redact.js");
|
|
92
92
|
const capture_health_js_1 = require("./capture-health.js");
|
|
93
|
+
const recall_precision_js_1 = require("./recall-precision.js");
|
|
93
94
|
const capture_pause_js_1 = require("./capture-pause.js");
|
|
94
95
|
const init_js_1 = require("./init.js");
|
|
95
96
|
// ---------------------------------------------------------------------------
|
|
@@ -207,8 +208,10 @@ function createMcpServer() {
|
|
|
207
208
|
// (incl. the #204 FETCHED marker) is built in ONE place shared with the
|
|
208
209
|
// REST GET /memory path. CC reaches Hicortex through THIS MCP tool;
|
|
209
210
|
// before #207's fix it got a marker-less citation built inline here.
|
|
211
|
+
// #476: the fetch recorder forwards through the same funnel — exactly
|
|
212
|
+
// one precision event per fetch, whichever path served it.
|
|
210
213
|
try {
|
|
211
|
-
const r = (0, recall_index_js_1.formatMemoryGetText)(db, { id });
|
|
214
|
+
const r = (0, recall_index_js_1.formatMemoryGetText)(db, { id }, { recordFetch: recall_precision_js_1.recordRecallFetch });
|
|
212
215
|
return { content: [{ type: "text", text: r.text }], isError: r.status !== 200 };
|
|
213
216
|
}
|
|
214
217
|
catch (err) {
|
|
@@ -787,7 +790,8 @@ async function startServer(options = {}) {
|
|
|
787
790
|
`/novelty=${CALIBRATION.NOVELTY_FLOOR_SLOTS}` +
|
|
788
791
|
` · ` +
|
|
789
792
|
`score sim=${scoringCfg.similarity}/str=${scoringCfg.strength}/conn=${scoringCfg.connections}` +
|
|
790
|
-
`/
|
|
793
|
+
`/K=${scoringCfg.connectionsSaturation}` +
|
|
794
|
+
`/rec=${scoringCfg.recency}, head=${scoringCfg.recencyHead}@${scoringCfg.recencyHeadDays}d, ` +
|
|
791
795
|
`superseded×${scoringCfg.supersededDemotion}` +
|
|
792
796
|
`, intent w=${sessionIntentCfg.weight}` +
|
|
793
797
|
(sessionIntentCfg.weight === 0 ? " (disabled)" : ""));
|
|
@@ -1052,6 +1056,22 @@ async function startServer(options = {}) {
|
|
|
1052
1056
|
// last_accessed, NOT access_count (that stays reserved for hicortex_get /
|
|
1053
1057
|
// GET /memory — real use). {reset: true} clears the session's dedup state
|
|
1054
1058
|
// (SessionStart/compaction).
|
|
1059
|
+
//
|
|
1060
|
+
// #476 Memory Precision: the route also wires the precision seams — the
|
|
1061
|
+
// factory's exposed embedPrompt (the per-request memo: the recorder
|
|
1062
|
+
// measures per-line similarity with ZERO extra embeds), the 24h-TTL
|
|
1063
|
+
// standing-context basis (lessons in /learnings order + identity only when
|
|
1064
|
+
// every known client is served — the redundancy reference), and the
|
|
1065
|
+
// fail-soft recorder. handleRecallIndex owns the fail-soft law: a
|
|
1066
|
+
// recording failure never touches the response.
|
|
1067
|
+
const standingContextBasis = (0, recall_precision_js_1.createStandingContextBasis)({
|
|
1068
|
+
// Lazy getters: identityClients is resolved at boot and stateDir is
|
|
1069
|
+
// assigned before the routes serve, but both land AFTER this provider is
|
|
1070
|
+
// created — read per cache rebuild (a config restart applies at the next
|
|
1071
|
+
// TTL), never captured once.
|
|
1072
|
+
clients: () => identityClients,
|
|
1073
|
+
identityDir: () => (0, node_path_1.join)(stateDir, "identity"),
|
|
1074
|
+
});
|
|
1055
1075
|
app.post("/recall-index", async (req, res) => {
|
|
1056
1076
|
if (!db) {
|
|
1057
1077
|
res.status(503).json({ error: "Server not initialized" });
|
|
@@ -1066,22 +1086,30 @@ async function startServer(options = {}) {
|
|
|
1066
1086
|
// that searches unblended and touches no centroid state. Extracted so the
|
|
1067
1087
|
// exact behavior is unit-testable without HTTP (blendQueryVector
|
|
1068
1088
|
// precedent); this adapter stays thin.
|
|
1089
|
+
const factory = (0, recall_index_js_1.createRecallRetrieveFn)({
|
|
1090
|
+
db,
|
|
1091
|
+
registry: recallRegistry,
|
|
1092
|
+
embedFn: embedder_js_1.embed,
|
|
1093
|
+
});
|
|
1069
1094
|
const r = await (0, recall_index_js_1.handleRecallIndex)({
|
|
1070
1095
|
db,
|
|
1071
1096
|
registry: recallRegistry,
|
|
1072
|
-
retrieveFn:
|
|
1073
|
-
db,
|
|
1074
|
-
registry: recallRegistry,
|
|
1075
|
-
embedFn: embedder_js_1.embed,
|
|
1076
|
-
}),
|
|
1097
|
+
retrieveFn: factory.retrieveFn,
|
|
1077
1098
|
options: recallIndexOptions,
|
|
1099
|
+
precision: {
|
|
1100
|
+
promptEmbed: factory.embedPrompt,
|
|
1101
|
+
basis: standingContextBasis,
|
|
1102
|
+
recorder: recall_precision_js_1.recordRecallPush,
|
|
1103
|
+
},
|
|
1078
1104
|
}, req.body);
|
|
1079
1105
|
res.status(r.status).json(r.body);
|
|
1080
1106
|
});
|
|
1081
1107
|
// REST /memory?id= — fetch one memory's full content (lazy-load counterpart
|
|
1082
1108
|
// of /recall-index for REST clients: Hermes/OC plugins). Marks it as used.
|
|
1083
1109
|
// Prefix ids resolve. 0.16.x: the `privacy` query param is accepted but
|
|
1084
|
-
// ignored (column is vestigial, never filtered). Logic in handleMemoryGet
|
|
1110
|
+
// ignored (column is vestigial, never filtered). Logic in handleMemoryGet;
|
|
1111
|
+
// the #476 fetch recorder rides the same funnel (exactly one event per
|
|
1112
|
+
// fetch, shared with the MCP hicortex_get path).
|
|
1085
1113
|
app.get("/memory", (req, res) => {
|
|
1086
1114
|
if (!db) {
|
|
1087
1115
|
res.status(503).json({ error: "Server not initialized" });
|
|
@@ -1089,7 +1117,7 @@ async function startServer(options = {}) {
|
|
|
1089
1117
|
}
|
|
1090
1118
|
warnDeprecatedPrivacyParamIfPresent(req.query, "memory");
|
|
1091
1119
|
try {
|
|
1092
|
-
const r = (0, recall_index_js_1.handleMemoryGet)(db, { id: req.query.id });
|
|
1120
|
+
const r = (0, recall_index_js_1.handleMemoryGet)(db, { id: req.query.id }, { recordFetch: recall_precision_js_1.recordRecallFetch });
|
|
1093
1121
|
res.status(r.status).json(r.body);
|
|
1094
1122
|
}
|
|
1095
1123
|
catch (err) {
|
package/dist/nightly.js
CHANGED
|
@@ -79,6 +79,7 @@ const capture_cursors_js_1 = require("./capture-cursors.js");
|
|
|
79
79
|
const capture_js_1 = require("./capture.js");
|
|
80
80
|
const run_deadline_js_1 = require("./run-deadline.js");
|
|
81
81
|
const dashboard_js_1 = require("./dashboard.js");
|
|
82
|
+
const recall_precision_js_1 = require("./recall-precision.js");
|
|
82
83
|
const telemetry_js_1 = require("./telemetry.js");
|
|
83
84
|
const init_js_1 = require("./init.js");
|
|
84
85
|
const backup_js_1 = require("./backup.js");
|
|
@@ -1071,6 +1072,19 @@ async function runNightly(options = {}) {
|
|
|
1071
1072
|
console.warn(`[hicortex] Dashboard snapshot write failed: ` +
|
|
1072
1073
|
`${snapErr instanceof Error ? snapErr.message : String(snapErr)}`);
|
|
1073
1074
|
}
|
|
1075
|
+
// #476 — recall-precision retention prune (zero-LLM, full nightly only):
|
|
1076
|
+
// both event tables drop rows outside the rolling window whose length IS
|
|
1077
|
+
// the retention constant (the capture-health single-constant law — the
|
|
1078
|
+
// Memory Precision card can never claim a window the store no longer
|
|
1079
|
+
// holds rows for). Own try/catch like the snapshot writer: telemetry
|
|
1080
|
+
// housekeeping must never fail the run.
|
|
1081
|
+
try {
|
|
1082
|
+
(0, recall_precision_js_1.pruneRecallPrecision)(db);
|
|
1083
|
+
}
|
|
1084
|
+
catch (pruneErr) {
|
|
1085
|
+
console.warn(`[hicortex] Recall-precision retention prune failed: ` +
|
|
1086
|
+
`${pruneErr instanceof Error ? pruneErr.message : String(pruneErr)}`);
|
|
1087
|
+
}
|
|
1074
1088
|
}
|
|
1075
1089
|
// Anonymous telemetry (fire-and-forget, full nightly only).
|
|
1076
1090
|
// Capture-only runs are excluded to avoid inflating install pings.
|
package/dist/recall-index.d.ts
CHANGED
|
@@ -51,6 +51,7 @@ import type Database from "better-sqlite3";
|
|
|
51
51
|
import type { MemorySearchResult } from "./types.js";
|
|
52
52
|
import * as storage from "./storage.js";
|
|
53
53
|
import { SessionRecallRegistry } from "./recall-registry.js";
|
|
54
|
+
import type { RecallPushEntry, RecallFetchRecorder } from "./recall-precision.js";
|
|
54
55
|
export interface RecallIndexOptions {
|
|
55
56
|
/** Minimum measured cosine for vector-only candidates (release-managed
|
|
56
57
|
* since #408 — calibration.ts RECALL_MIN_SIMILARITY; this field is the
|
|
@@ -161,6 +162,16 @@ export interface RecallFilters {
|
|
|
161
162
|
* RecallIndexDeps.retrieveFn). Named so the production factory
|
|
162
163
|
* (createRecallRetrieveFn) and test doubles share one type. */
|
|
163
164
|
export type RecallRetrieveFn = (query: string, limit: number, filters: RecallFilters | undefined, sessionId: string, purePrompt?: boolean) => Promise<MemorySearchResult[]>;
|
|
165
|
+
/** What createRecallRetrieveFn returns (#476): the search closure PLUS the
|
|
166
|
+
* per-request prompt-embed memo it already maintained — exposed so the
|
|
167
|
+
* precision recorder reuses the SAME embedding (zero extra embeds) instead
|
|
168
|
+
* of re-embedding the prompt to measure per-line similarity. */
|
|
169
|
+
export interface RecallRetrieveFactory {
|
|
170
|
+
retrieveFn: RecallRetrieveFn;
|
|
171
|
+
/** The single-entry embed memo (keyed on the query text; the factory is
|
|
172
|
+
* built per request, so the memo never outlives it). */
|
|
173
|
+
embedPrompt: (query: string) => Promise<Float32Array>;
|
|
174
|
+
}
|
|
164
175
|
export interface RecallIndexDeps {
|
|
165
176
|
db: Database.Database;
|
|
166
177
|
registry: SessionRecallRegistry;
|
|
@@ -176,6 +187,30 @@ export interface RecallIndexDeps {
|
|
|
176
187
|
* no breakage. */
|
|
177
188
|
retrieveFn: RecallRetrieveFn;
|
|
178
189
|
options?: RecallIndexOptions;
|
|
190
|
+
/** #476 Memory Precision seams — OPTIONAL so existing callers (tests,
|
|
191
|
+
* library use) keep their exact no-recording behavior. Absent → no event
|
|
192
|
+
* rows, the plain exposure write runs as before. mcp-server wires the
|
|
193
|
+
* production set (the real recorder, the request-memoized prompt embed,
|
|
194
|
+
* the 24h-TTL standing-context basis). Recording is FAIL-SOFT: any error
|
|
195
|
+
* is caught here and the recall response (200 + block) is never affected;
|
|
196
|
+
* on failure the exposure touch re-runs standalone so shown_count never
|
|
197
|
+
* depends on telemetry. */
|
|
198
|
+
precision?: RecallPrecisionDeps;
|
|
199
|
+
}
|
|
200
|
+
/** The #476 precision-recording seams (see RecallIndexDeps.precision). All
|
|
201
|
+
* three are injectable; tests force failures through them. */
|
|
202
|
+
export interface RecallPrecisionDeps {
|
|
203
|
+
/** The request's memoized pure-prompt embed (createRecallRetrieveFn's
|
|
204
|
+
* embedPrompt) — the recorder computes per-line cosines with ZERO extra
|
|
205
|
+
* embeds (the perf law). */
|
|
206
|
+
promptEmbed: (prompt: string) => Promise<Float32Array>;
|
|
207
|
+
/** Standing-context basis vectors (recall-precision.ts's 24h-TTL cached
|
|
208
|
+
* provider; empty array → redundancy NULL, unmeasured). */
|
|
209
|
+
basis: (db: Database.Database) => Promise<Float32Array[]>;
|
|
210
|
+
/** The recorder (recall-precision.ts recordRecallPush). Throws on failure —
|
|
211
|
+
* this handler catches (fail-soft) and falls back to the plain exposure
|
|
212
|
+
* write. */
|
|
213
|
+
recorder: (db: Database.Database, entry: RecallPushEntry, basis?: Float32Array[]) => void;
|
|
179
214
|
}
|
|
180
215
|
/**
|
|
181
216
|
* The PRODUCTION /recall-index retrieveFn (what mcp-server wires into
|
|
@@ -209,7 +244,7 @@ export declare function createRecallRetrieveFn(deps: {
|
|
|
209
244
|
embedFn: (text: string) => Promise<Float32Array>;
|
|
210
245
|
/** FTS resolution override (tests). Defaults to storage.searchFts. */
|
|
211
246
|
ftsFn?: typeof storage.searchFts;
|
|
212
|
-
}):
|
|
247
|
+
}): RecallRetrieveFactory;
|
|
213
248
|
/** Normalize a request-supplied string-list param: array of strings or a CSV
|
|
214
249
|
* string → string[] | undefined. Anything else (or an empty result) means
|
|
215
250
|
* "absent" — never a partial guess. Used by `mission_domains` (#203) so it
|
|
@@ -220,6 +255,14 @@ export declare function parseStringListParam(v: unknown): string[] | undefined;
|
|
|
220
255
|
* all behavior lives here so tests exercise it directly.
|
|
221
256
|
*/
|
|
222
257
|
export declare function handleRecallIndex(deps: RecallIndexDeps, body: unknown): Promise<RecallIndexResult>;
|
|
258
|
+
/** Optional #476 deps for handleMemoryGet: the fetch-event recorder. Absent
|
|
259
|
+
* (old callers, tests) → no recording, byte-identical behavior. mcp-server
|
|
260
|
+
* wires recordRecallFetch so BOTH fetch paths (REST GET /memory, MCP
|
|
261
|
+
* hicortex_get via formatMemoryGetText) record exactly once — this is the
|
|
262
|
+
* ONE funnel. Fail-soft: a recorder error is caught here, never surfaced. */
|
|
263
|
+
export interface MemoryGetDeps {
|
|
264
|
+
recordFetch: RecallFetchRecorder;
|
|
265
|
+
}
|
|
223
266
|
/**
|
|
224
267
|
* Handle a GET /memory request (lazy-load counterpart of the recall index for
|
|
225
268
|
* REST clients). Thin Express adapter in mcp-server.ts; behavior lives here so
|
|
@@ -235,7 +278,7 @@ export declare function handleRecallIndex(deps: RecallIndexDeps, body: unknown):
|
|
|
235
278
|
*/
|
|
236
279
|
export declare function handleMemoryGet(db: Database.Database, query: {
|
|
237
280
|
id?: unknown;
|
|
238
|
-
}): RecallIndexResult;
|
|
281
|
+
}, deps?: MemoryGetDeps): RecallIndexResult;
|
|
239
282
|
/**
|
|
240
283
|
* MCP `hicortex_get` presentation: handleMemoryGet's result framed as the
|
|
241
284
|
* text block the MCP tool returns (provenance header + the SHARED citation +
|
|
@@ -249,7 +292,7 @@ export declare function handleMemoryGet(db: Database.Database, query: {
|
|
|
249
292
|
*/
|
|
250
293
|
export declare function formatMemoryGetText(db: Database.Database, query: {
|
|
251
294
|
id?: unknown;
|
|
252
|
-
}): {
|
|
295
|
+
}, deps?: MemoryGetDeps): {
|
|
253
296
|
status: number;
|
|
254
297
|
text: string;
|
|
255
298
|
};
|
package/dist/recall-index.js
CHANGED
|
@@ -251,27 +251,34 @@ function createRecallRetrieveFn(deps) {
|
|
|
251
251
|
}
|
|
252
252
|
return ftsMemo.rows;
|
|
253
253
|
};
|
|
254
|
-
return
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
254
|
+
return {
|
|
255
|
+
retrieveFn: async (query, limit, filters, sessionId, purePrompt) => {
|
|
256
|
+
const { weight, alpha } = (0, retrieval_js_1.getSessionIntent)();
|
|
257
|
+
const promptEmb = await embedOnce(query);
|
|
258
|
+
const queryVec = (0, retrieval_js_1.recallQueryVector)(deps.registry, sessionId, promptEmb, {
|
|
259
|
+
weight,
|
|
260
|
+
alpha,
|
|
261
|
+
purePrompt,
|
|
262
|
+
});
|
|
263
|
+
return (0, retrieval_js_1.retrieve)(deps.db, deps.embedFn, query, {
|
|
264
|
+
limit,
|
|
265
|
+
noStrengthen: true,
|
|
266
|
+
// #203: project + mission_domains are SOFT affinity (zero-boost
|
|
267
|
+
// neutral), threaded into computeScore.
|
|
268
|
+
project: filters?.project,
|
|
269
|
+
missionDomains: filters?.mission_domains,
|
|
270
|
+
queryEmbedding: queryVec,
|
|
271
|
+
// #329: shared per-request FTS list. The recall path never passes
|
|
272
|
+
// sourceAgent, so the memo is keyed on (query, fetchLimit) only —
|
|
273
|
+
// exactly the two things retrieve() would pass to searchFts.
|
|
274
|
+
ftsCandidates: (fetchLimit) => ftsOnce(query, fetchLimit),
|
|
275
|
+
});
|
|
276
|
+
},
|
|
277
|
+
// #476: the memoized prompt embed, EXPOSED so the precision recorder
|
|
278
|
+
// computes per-line cosines with ZERO additional embeds (the perf law —
|
|
279
|
+
// the memo semantics are unchanged: one embed per distinct query string
|
|
280
|
+
// per factory instance).
|
|
281
|
+
embedPrompt: embedOnce,
|
|
275
282
|
};
|
|
276
283
|
}
|
|
277
284
|
/** Normalize a request-supplied string-list param: array of strings or a CSV
|
|
@@ -421,12 +428,48 @@ async function handleRecallIndex(deps, body) {
|
|
|
421
428
|
picked = [...picked, ...backfill];
|
|
422
429
|
}
|
|
423
430
|
if (picked.length === 0) {
|
|
431
|
+
// #476: a silent turn (searches ran, no line passed the gates) still
|
|
432
|
+
// records its PUSH row — the silence rate is computable later and the
|
|
433
|
+
// judge's population stays complete. Zero event rows, no exposure touch.
|
|
434
|
+
// Fail-soft like every recording site.
|
|
435
|
+
if (deps.precision) {
|
|
436
|
+
try {
|
|
437
|
+
deps.precision.recorder(deps.db, { sessionId, prompt, ids: [] });
|
|
438
|
+
}
|
|
439
|
+
catch (err) {
|
|
440
|
+
console.warn(`[hicortex] recall-precision push recording failed (fail-soft): ` +
|
|
441
|
+
`${err instanceof Error ? err.message : String(err)}`);
|
|
442
|
+
}
|
|
443
|
+
}
|
|
424
444
|
return { status: 200, body: { block: null, shown: [], turn } };
|
|
425
445
|
}
|
|
426
446
|
const ids = picked.map((r) => r.id);
|
|
427
447
|
deps.registry.markShown(sessionId, ids);
|
|
428
|
-
|
|
429
|
-
|
|
448
|
+
const nowIso = new Date().toISOString();
|
|
449
|
+
// #476 Memory Precision: record the push + per-line events IN THE SAME
|
|
450
|
+
// TRANSACTION as the exposure write (the recorder owns the unit — it calls
|
|
451
|
+
// touchMemoriesShown inside its own transaction). FAIL-SOFT: on any error
|
|
452
|
+
// (or when the seams are absent — library callers, test doubles) the plain
|
|
453
|
+
// exposure write runs instead, so shown_count NEVER depends on telemetry.
|
|
454
|
+
let exposureWritten = false;
|
|
455
|
+
if (deps.precision) {
|
|
456
|
+
try {
|
|
457
|
+
const [promptEmb, basis] = await Promise.all([
|
|
458
|
+
deps.precision.promptEmbed(prompt),
|
|
459
|
+
deps.precision.basis(deps.db),
|
|
460
|
+
]);
|
|
461
|
+
deps.precision.recorder(deps.db, { ts: nowIso, sessionId, prompt, ids, promptEmbedding: promptEmb }, basis);
|
|
462
|
+
exposureWritten = true;
|
|
463
|
+
}
|
|
464
|
+
catch (err) {
|
|
465
|
+
console.warn(`[hicortex] recall-precision recording failed (fail-soft): ` +
|
|
466
|
+
`${err instanceof Error ? err.message : String(err)}`);
|
|
467
|
+
}
|
|
468
|
+
}
|
|
469
|
+
if (!exposureWritten) {
|
|
470
|
+
// Exposure signal: shown_count + last_accessed refresh, NOT access_count.
|
|
471
|
+
storage.touchMemoriesShown(deps.db, ids, nowIso);
|
|
472
|
+
}
|
|
430
473
|
const lines = picked.map((r) => formatIndexLine(r, titleChars));
|
|
431
474
|
const block = [
|
|
432
475
|
"## Memory recall (auto)",
|
|
@@ -456,7 +499,7 @@ async function handleRecallIndex(deps, body) {
|
|
|
456
499
|
* never filtered. Callers may still send a `privacy` field (backward compat)
|
|
457
500
|
* but it is ignored.
|
|
458
501
|
*/
|
|
459
|
-
function handleMemoryGet(db, query) {
|
|
502
|
+
function handleMemoryGet(db, query, deps) {
|
|
460
503
|
const id = typeof query.id === "string" ? query.id : "";
|
|
461
504
|
if (!id)
|
|
462
505
|
return { status: 400, body: { error: "Missing 'id'" } };
|
|
@@ -471,6 +514,18 @@ function handleMemoryGet(db, query) {
|
|
|
471
514
|
if (!mem)
|
|
472
515
|
return notFound;
|
|
473
516
|
storage.strengthenMemory(db, fullId, new Date().toISOString());
|
|
517
|
+
// #476: one kind='fetch' precision event per successful fetch (the use
|
|
518
|
+
// signal for Level 2 / divergence). Fail-soft — the fetch result is never
|
|
519
|
+
// affected by a recording failure.
|
|
520
|
+
if (deps?.recordFetch) {
|
|
521
|
+
try {
|
|
522
|
+
deps.recordFetch(db, fullId, new Date().toISOString());
|
|
523
|
+
}
|
|
524
|
+
catch (err) {
|
|
525
|
+
console.warn(`[hicortex] recall-precision fetch recording failed (fail-soft): ` +
|
|
526
|
+
`${err instanceof Error ? err.message : String(err)}`);
|
|
527
|
+
}
|
|
528
|
+
}
|
|
474
529
|
// `citation` is server-rendered so every plugin surfaces the same built-in
|
|
475
530
|
// provenance norm (owner directive 27.07) — see #193.
|
|
476
531
|
const date = (mem.created_at ?? "").slice(0, 10);
|
|
@@ -498,8 +553,10 @@ function handleMemoryGet(db, query) {
|
|
|
498
553
|
* built inline, while the REST path used handleMemoryGet — same contract, two
|
|
499
554
|
* implementations, one updated).
|
|
500
555
|
*/
|
|
501
|
-
function formatMemoryGetText(db, query) {
|
|
502
|
-
|
|
556
|
+
function formatMemoryGetText(db, query, deps) {
|
|
557
|
+
// deps (the #476 fetch recorder) forwards so the MCP path records exactly
|
|
558
|
+
// like the REST path — one funnel, one event per fetch.
|
|
559
|
+
const r = handleMemoryGet(db, query, deps);
|
|
503
560
|
if (r.status !== 200) {
|
|
504
561
|
return { status: r.status, text: String(r.body.error ?? `No memory with id ${query.id ?? ""}`) };
|
|
505
562
|
}
|
|
@@ -0,0 +1,212 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Memory Precision (#476) — the event store behind the console's Memory
|
|
3
|
+
* Precision card, and the two-level context model made visible:
|
|
4
|
+
*
|
|
5
|
+
* - Level 1 (pushed index): what every prompt receives unasked. One
|
|
6
|
+
* `recall_pushes` row per NON-skipped /recall-index call + one
|
|
7
|
+
* `recall_events` kind='shown' row per pushed line, carrying the RAW
|
|
8
|
+
* cosines (similarity to the prompt, redundancy vs standing context) —
|
|
9
|
+
* stored threshold-free, verdicts computed at render so recalibration
|
|
10
|
+
* never rewrites history.
|
|
11
|
+
* - Level 2 (recall depth): what the agent fetches when needed. One
|
|
12
|
+
* `recall_events` kind='fetch' row per handleMemoryGet — the ONE funnel
|
|
13
|
+
* the REST GET /memory and MCP hicortex_get paths share.
|
|
14
|
+
*
|
|
15
|
+
* Layering (the recall-index.ts / capture-health.ts convention): all write +
|
|
16
|
+
* aggregation logic lives here as pure functions over the db handle so tests
|
|
17
|
+
* exercise them without HTTP; mcp-server.ts only wires the record calls into
|
|
18
|
+
* the /recall-index and fetch funnels, and dashboard.ts spreads the window
|
|
19
|
+
* aggregation into /dashboard/data.
|
|
20
|
+
*
|
|
21
|
+
* FAIL-SOFT LAW: recording is telemetry. Any error here is caught by the
|
|
22
|
+
* CALLER (handleRecallIndex / handleMemoryGet) and swallowed — the recall
|
|
23
|
+
* response (status 200 + block) is never affected, and the exposure signal
|
|
24
|
+
* (touchMemoriesShown) never depends on it (the handler falls back to the
|
|
25
|
+
* plain exposure write when recording is absent or fails).
|
|
26
|
+
*
|
|
27
|
+
* The perf law (bounded hot-path cost): the recorder computes cosines with
|
|
28
|
+
* the request's ALREADY-MEMOIZED prompt embedding (createRecallRetrieveFn's
|
|
29
|
+
* embedPrompt — zero extra embeds) against STORED memory vectors, and writes
|
|
30
|
+
* one transaction of ≤ maxItems+1 INSERTs — microseconds inside an endpoint
|
|
31
|
+
* that already writes and spends ~100-300 ms embedding and searching.
|
|
32
|
+
*/
|
|
33
|
+
import type Database from "better-sqlite3";
|
|
34
|
+
/** One non-skipped /recall-index call to record (recorder seam contract —
|
|
35
|
+
* recall-index.ts types its precision deps against this). */
|
|
36
|
+
export interface RecallPushEntry {
|
|
37
|
+
/** ISO timestamp of the push; defaults to now. */
|
|
38
|
+
ts?: string;
|
|
39
|
+
/** The request's session_id (nullable — the wire field is optional). */
|
|
40
|
+
sessionId: string | null;
|
|
41
|
+
/** The prompt text (only the ≤256-char excerpt is persisted). */
|
|
42
|
+
prompt: string;
|
|
43
|
+
/** The shown memory ids, in index order. EMPTY = silent turn (the push row
|
|
44
|
+
* is still recorded — the silence rate is computable and the judge's
|
|
45
|
+
* population complete — but no event rows, no exposure touch). */
|
|
46
|
+
ids: string[];
|
|
47
|
+
/** The PURE-prompt embedding — REQUIRED when ids.length > 0 (the request's
|
|
48
|
+
* memoized embed; unused on a silent turn, so no embed is ever spent on
|
|
49
|
+
* recording). */
|
|
50
|
+
promptEmbedding?: Float32Array;
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* Record one /recall-index push: the push row + (when lines were shown) the
|
|
54
|
+
* per-line kind='shown' event rows, in ONE transaction WITH the existing
|
|
55
|
+
* exposure write (storage.touchMemoriesShown — the spec's "same transaction";
|
|
56
|
+
* better-sqlite3 nests the inner transaction as a savepoint, so a recording
|
|
57
|
+
* failure rolls back the whole unit and the CALLER's fallback re-runs the
|
|
58
|
+
* exposure touch alone — shown_count never depends on telemetry).
|
|
59
|
+
*
|
|
60
|
+
* similarity = cosine(memory embedding, PURE prompt embedding), computed
|
|
61
|
+
* UNIFORMLY via cosineBetweenVectors against the stored vector — FTS-sourced
|
|
62
|
+
* picks (which bypass the cosine floor and carry similarity:null in
|
|
63
|
+
* MemorySearchResult) get a measured value here too. redundancy = MAX cosine
|
|
64
|
+
* against the standing-context basis vectors (NULL when the basis is empty —
|
|
65
|
+
* embed failure degrades to "unmeasured", never a guessed 0). Both stored
|
|
66
|
+
* raw; no thresholds touch this row.
|
|
67
|
+
*/
|
|
68
|
+
export declare function recordRecallPush(db: Database.Database, entry: RecallPushEntry, basis?: Float32Array[]): void;
|
|
69
|
+
/** Fetch-event recorder contract (handleMemoryGet's optional #476 dep —
|
|
70
|
+
* mcp-server wires recordRecallFetch; tests can force a failure). */
|
|
71
|
+
export type RecallFetchRecorder = (db: Database.Database, memoryId: string, ts?: string) => void;
|
|
72
|
+
/**
|
|
73
|
+
* Record one explicit fetch (handleMemoryGet — the funnel both REST GET
|
|
74
|
+
* /memory and MCP hicortex_get route through, so the row is written exactly
|
|
75
|
+
* once per fetch). similarity/redundancy are NULL on fetch rows: Level 2 is
|
|
76
|
+
* the use signal, not a relevance measure. push_id NULL — a fetch stands
|
|
77
|
+
* alone. Fail-soft at the CALLER (handleMemoryGet wraps this).
|
|
78
|
+
*/
|
|
79
|
+
export declare function recordRecallFetch(db: Database.Database, memoryId: string, ts?: string): void;
|
|
80
|
+
/**
|
|
81
|
+
* Nightly retention prune (zero-LLM): drop both event tables' rows outside
|
|
82
|
+
* the rolling window whose length IS the retention constant (the
|
|
83
|
+
* CAPTURE_HEALTH_WINDOW_DAYS single-constant law — the card can never claim a
|
|
84
|
+
* window the store no longer has rows for). Inclusive cutoff: the last N
|
|
85
|
+
* calendar days with today counted stay (day >= today−(N−1)), the complement
|
|
86
|
+
* is deleted — retained rows == window rows.
|
|
87
|
+
*/
|
|
88
|
+
export declare function pruneRecallPrecision(db: Database.Database): void;
|
|
89
|
+
/**
|
|
90
|
+
* True when the identity scope is served to EVERY known client — the only
|
|
91
|
+
* case where identity sections belong in the corpus-level standing-context
|
|
92
|
+
* basis. /recall-index carries no client type, so a SCOPED identity config
|
|
93
|
+
* (any subset) excludes identity rather than guessing which sections the
|
|
94
|
+
* calling session sees. `clients` is the boot-resolved
|
|
95
|
+
* resolveIdentityClientsConfig output; the absent-config default ["cc"] is
|
|
96
|
+
* NOT all → identity excluded.
|
|
97
|
+
*/
|
|
98
|
+
export declare function identityServedToAllClients(clients: string[]): boolean;
|
|
99
|
+
export interface StandingContextBasisConfig {
|
|
100
|
+
/** The daemon's boot-resolved identityClients (read lazily — assigned at
|
|
101
|
+
* boot, after the routes that consume the basis are registered). */
|
|
102
|
+
clients: () => string[];
|
|
103
|
+
/** The daemon's identity dir (sections read only when all-clients). */
|
|
104
|
+
identityDir: () => string;
|
|
105
|
+
/** Embedding seam — tests inject a deterministic embedder. Default embedBatch. */
|
|
106
|
+
embedFn?: (texts: string[]) => Promise<Float32Array[]>;
|
|
107
|
+
/** Clock seam (tests). Default Date.now. */
|
|
108
|
+
now?: () => number;
|
|
109
|
+
/** Cache TTL seam (tests). Default 24h. */
|
|
110
|
+
ttlMs?: number;
|
|
111
|
+
}
|
|
112
|
+
/** The basis provider: db → basis vectors (may be empty). Callable, plus a
|
|
113
|
+
* test-only cache reset (the cache is deliberately per-provider so suites
|
|
114
|
+
* with different configs never share it). */
|
|
115
|
+
export interface StandingContextBasis {
|
|
116
|
+
(db: Database.Database): Promise<Float32Array[]>;
|
|
117
|
+
resetForTests(): void;
|
|
118
|
+
}
|
|
119
|
+
/**
|
|
120
|
+
* The standing-context basis: the top lessons in /learnings order
|
|
121
|
+
* (storage.getLessons(db, 30) — exactly what the /learnings handler serves
|
|
122
|
+
* and every client's session-start hook injects) PLUS identity sections ONLY
|
|
123
|
+
* when identityServedToAllClients (see above). Embedded with the LOCAL
|
|
124
|
+
* embedder — no LLM anywhere in this feature. Cached for 24h (single-flight:
|
|
125
|
+
* concurrent callers share the embedding pass); an embed failure degrades to
|
|
126
|
+
* an EMPTY cached basis (redundancy NULL — "unmeasured", never guessed) and
|
|
127
|
+
* is retried after the TTL.
|
|
128
|
+
*/
|
|
129
|
+
export declare function createStandingContextBasis(config: StandingContextBasisConfig): StandingContextBasis;
|
|
130
|
+
/** The Level-1 half: pushed-index precision proxies over the window's stored
|
|
131
|
+
* events. Thresholds are ECHOED (the page never hardcodes an edge);
|
|
132
|
+
* mean/share are computed at read time from raw values. */
|
|
133
|
+
export interface MemoryPrecisionLevel1 {
|
|
134
|
+
/** Non-skipped /recall-index calls in the window (silent turns included). */
|
|
135
|
+
pushes: number;
|
|
136
|
+
/** Pushed index lines (kind='shown' event rows) in the window. */
|
|
137
|
+
lines: number;
|
|
138
|
+
/** Mean cosine(line, its prompt) over lines with a measured similarity;
|
|
139
|
+
* null when nothing was measured. */
|
|
140
|
+
mean_similarity: number | null;
|
|
141
|
+
/** Share of measured lines at/above the redundancy threshold; null when
|
|
142
|
+
* nothing was measured (empty/unmeasured — undefined, not zero). */
|
|
143
|
+
redundant_share: number | null;
|
|
144
|
+
thresholds: {
|
|
145
|
+
/** The redundancy render edge (calibration MEMORY_PRECISION_REDUNDANT_ABOVE). */
|
|
146
|
+
redundant_above: number;
|
|
147
|
+
/** The divergence bar (calibration MEMORY_PRECISION_DIVERGENCE_MIN_SHOWN). */
|
|
148
|
+
divergence_min_shown: number;
|
|
149
|
+
};
|
|
150
|
+
}
|
|
151
|
+
/** The Level-2 half: recall depth (fetches per showing) over the window,
|
|
152
|
+
* derived from corpus-wide cumulative counter DELTAS — live adoption minus
|
|
153
|
+
* the newest pre-window snapshot. No second source of truth: the events
|
|
154
|
+
* above are per-line; this is the corpus aggregate the nightly already
|
|
155
|
+
* snapshots. Null = no honest value (no pre-window baseline), never 0. */
|
|
156
|
+
export interface MemoryPrecisionLevel2 {
|
|
157
|
+
shown: number | null;
|
|
158
|
+
used: number | null;
|
|
159
|
+
uses_per_showing: number | null;
|
|
160
|
+
}
|
|
161
|
+
/** The whole /dashboard/data memory_precision block (one card's payload,
|
|
162
|
+
* budget ≤ 2 KB — the top-3 divergence lines are the only content). */
|
|
163
|
+
export interface MemoryPrecision {
|
|
164
|
+
/** The EFFECTIVE window (min(range, retention)); the page renders
|
|
165
|
+
* "last N days" from this echo, never a hardcoded literal. */
|
|
166
|
+
window_days: number;
|
|
167
|
+
level1: MemoryPrecisionLevel1;
|
|
168
|
+
level2: MemoryPrecisionLevel2;
|
|
169
|
+
divergence: {
|
|
170
|
+
/** Live memories with ≥ thresholds.divergence_min_shown window showings
|
|
171
|
+
* and ZERO window fetches — the index keeps pushing, nothing reads. */
|
|
172
|
+
count: number;
|
|
173
|
+
/** Top-3 by window showings DESC, lines rendered through the SHARED
|
|
174
|
+
* formatIndexLine (≤ RECALL_TITLE_CHARS — the card cannot drift from
|
|
175
|
+
* what agents see). Hover-only on the card face (no memory lists on
|
|
176
|
+
* the main page — owner ruling 2026-09-12). */
|
|
177
|
+
top: Array<{
|
|
178
|
+
id: string;
|
|
179
|
+
line: string;
|
|
180
|
+
shown: number;
|
|
181
|
+
}>;
|
|
182
|
+
};
|
|
183
|
+
}
|
|
184
|
+
/** Read-time seams (the #408 experiment pattern — the eval/tests sweep
|
|
185
|
+
* values through them; production never passes anything). */
|
|
186
|
+
export interface MemoryPrecisionReadOptions {
|
|
187
|
+
/** Fixed clock (tests); default now. */
|
|
188
|
+
now?: Date;
|
|
189
|
+
/** Redundancy render threshold override (tests — changing it must move
|
|
190
|
+
* only the share, never a stored row). */
|
|
191
|
+
redundantAbove?: number;
|
|
192
|
+
}
|
|
193
|
+
/**
|
|
194
|
+
* The window aggregation behind /dashboard/data's memory_precision block
|
|
195
|
+
* (the readCaptureHealthWindow pattern). Effective window = min(range,
|
|
196
|
+
* MEMORY_PRECISION_WINDOW_DAYS) for every range incl. 180d/all (the #452
|
|
197
|
+
* selector's wide options clamp to what the retention honestly holds).
|
|
198
|
+
*
|
|
199
|
+
* Level 1 groups the stored event rows over the window (day >= today−(N−1),
|
|
200
|
+
* the inclusive capture-health convention); Level 2 is the corpus-wide
|
|
201
|
+
* adoption delta — live sums minus the NEWEST non-null `adoption` in
|
|
202
|
+
* dashboard_snapshots at/before now−N days (cumulative counters telescope:
|
|
203
|
+
* a missing nightly widens the effective window rather than corrupting it;
|
|
204
|
+
* two snapshots on one day are harmless — the newest wins; no pre-window
|
|
205
|
+
* baseline → null, undefined rather than zero). Divergence joins the
|
|
206
|
+
* window's per-memory shown/fetch counts against LIVE (non-absorbed)
|
|
207
|
+
* memories and renders the top-3 through the shared index-line path.
|
|
208
|
+
*/
|
|
209
|
+
export declare function readMemoryPrecision(db: Database.Database, range: string, liveAdoption: {
|
|
210
|
+
shown_sum: number;
|
|
211
|
+
used_sum: number;
|
|
212
|
+
}, opts?: MemoryPrecisionReadOptions): MemoryPrecision;
|