@gamaze/hicortex 0.15.2 → 0.15.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/dist/eval/recall-sweep.d.ts +59 -0
- package/dist/eval/recall-sweep.js +715 -0
- package/dist/index.js +1 -1
- package/dist/mcp-server.js +28 -8
- package/dist/memory-instructions.js +1 -1
- package/dist/recall-index.d.ts +4 -1
- package/dist/recall-index.js +2 -2
- package/dist/recall-registry.d.ts +39 -1
- package/dist/recall-registry.js +52 -1
- package/dist/retrieval.d.ts +36 -0
- package/dist/retrieval.js +60 -3
- package/dist/schema-prototypes.d.ts +15 -0
- package/dist/schema-prototypes.js +24 -0
- package/package.json +2 -1
package/README.md
CHANGED
|
@@ -225,6 +225,7 @@ Config at `~/.hicortex/config.json`. Created by `init`. Key options:
|
|
|
225
225
|
| `recallMinSimilarity` | Relevance floor for index entries (default: 0.55; text-search matches always pass) |
|
|
226
226
|
| `recallReshowTurns` | Turns before an already-shown memory may reappear in the same session (default: 30) |
|
|
227
227
|
| `recallMinPromptChars` | Prompts shorter than this skip the recall index (default: 20) |
|
|
228
|
+
| `sessionIntentWeight` | Blend weight of the session-intent rolling centroid in the recall search vector: `query = (1-w)·prompt + w·centroid` (default: 0.33; set 0 to disable — pure-prompt recall, the kill-switch). The first turn of a session searches with pure prompt and seeds the centroid; subsequent turns blend so recall follows the session's intent instead of being query-literal. The EMA rate (0.4) is a shipped constant, not configurable |
|
|
228
229
|
| `dedupMergeThreshold` | Minimum cosine similarity for `hicortex dedup` to cluster memories as near-duplicates (default: 0.92) |
|
|
229
230
|
| `supersessionMinSimilarity` | Minimum cosine similarity for a nightly supersession candidate pair (default: 0.80) |
|
|
230
231
|
| `supersessionMaxCalls` | Max classify-tier LLM calls the nightly's supersession stage spends per run (default: 30) |
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* Recall@k sweep eval for session-intent keying (#192, commit 0867d6c).
|
|
4
|
+
*
|
|
5
|
+
* SHIP GATE: does blending the prompt with the per-session EMA centroid
|
|
6
|
+
* (retrieval.blendQueryVector) improve recall across rephrased turns, AND
|
|
7
|
+
* what does it COST when a session shifts topics mid-stream? Picks the
|
|
8
|
+
* default `sessionIntentWeight`.
|
|
9
|
+
*
|
|
10
|
+
* Two scenario families, same corpus:
|
|
11
|
+
*
|
|
12
|
+
* FOCUSED — 6 sessions (one per topic), 4 turns each of GENUINE REPHRASES
|
|
13
|
+
* of one intent. Tests the UPSIDE: does the centroid pull drifting
|
|
14
|
+
* rephrases back on-topic? (recall@5 / p@5 should rise with w.)
|
|
15
|
+
*
|
|
16
|
+
* SHIFT — 5 sessions, each 2 turns on topic A then 2 turns on topic B (a
|
|
17
|
+
* clean mid-session topic change). Tests the DOWNSIDE that focused-only
|
|
18
|
+
* can't see: a higher weight leaves the centroid partly A right after the
|
|
19
|
+
* shift, so the first turn(s) on B may LAG (lower recall@5 for B). The
|
|
20
|
+
* load-bearing read-out is post-shift recovery at turn-3 (first B turn)
|
|
21
|
+
* and turn-4, plus turns-to-recover.
|
|
22
|
+
*
|
|
23
|
+
* Method (both families):
|
|
24
|
+
* - Synthetic corpus of 30 memories across 6 well-separated topics (5 each),
|
|
25
|
+
* embedded with the REAL bge-small-en-v1.5 model (the production embedder).
|
|
26
|
+
* - For each weight w in {0.0, 0.2, 0.4, 0.5, 0.6, 0.8}: fresh
|
|
27
|
+
* SessionRecallRegistry per session; per turn, embed the prompt ONCE,
|
|
28
|
+
* blend with the live centroid, pass the blended vector to retrieve() via
|
|
29
|
+
* queryEmbedding, record recall@5 + p@5 for the turn's CURRENT topic,
|
|
30
|
+
* then fold the prompt into the centroid (EMA α=0.4).
|
|
31
|
+
*
|
|
32
|
+
* Fairness controls (load-bearing — the eval is useless if these slip):
|
|
33
|
+
* - `noStrengthen: true` on every retrieve(): the DB stays STATIC across all
|
|
34
|
+
* retrieve calls. Strengthening would mutate effective_strength/access_count
|
|
35
|
+
* between runs and contaminate cross-weight comparisons.
|
|
36
|
+
* - Uniform memory metadata: every memory has base_strength=0.5, created_at
|
|
37
|
+
* ≈ now, access_count=0, and NO links. So effective_strength, recency,
|
|
38
|
+
* connections, and the freshness boost are all uniform → the ONLY
|
|
39
|
+
* discriminator is vector cosine + RRF rank. That isolates the blend.
|
|
40
|
+
* - Cold-exposure slots are a no-op here (all candidates equally cold),
|
|
41
|
+
* so the top-k is the plain score order.
|
|
42
|
+
* - The centroid update is weight-INDEPENDENT (EMA of prompts at α=0.4), so
|
|
43
|
+
* the per-turn centroid is identical across all weights for a given
|
|
44
|
+
* session; only the blend differs. Embeddings are precomputed once.
|
|
45
|
+
*
|
|
46
|
+
* Metrics (per turn, for the turn's CURRENT topic):
|
|
47
|
+
* - recall@5 — did >=1 same-topic memory surface in the top-5? (coarse; the
|
|
48
|
+
* requested ship-gate metric — saturates on a small corpus.)
|
|
49
|
+
* - p@5 — on-topic count in the top-5 (0..5, finer).
|
|
50
|
+
* - recall@1 — was the SINGLE top result on-topic? (finest; focused only.)
|
|
51
|
+
*
|
|
52
|
+
* Honest by construction: it prints whatever the numbers are, including
|
|
53
|
+
* w=0 winning, the blend hurting a focused scenario, or high weights lagging
|
|
54
|
+
* badly post-shift. Report written to data/eval-recall-sweep/report.md and
|
|
55
|
+
* printed to stdout.
|
|
56
|
+
*
|
|
57
|
+
* Run: npm run eval:recall-sweep (== node dist/eval/recall-sweep.js)
|
|
58
|
+
*/
|
|
59
|
+
export {};
|
|
@@ -0,0 +1,715 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
"use strict";
|
|
3
|
+
/**
|
|
4
|
+
* Recall@k sweep eval for session-intent keying (#192, commit 0867d6c).
|
|
5
|
+
*
|
|
6
|
+
* SHIP GATE: does blending the prompt with the per-session EMA centroid
|
|
7
|
+
* (retrieval.blendQueryVector) improve recall across rephrased turns, AND
|
|
8
|
+
* what does it COST when a session shifts topics mid-stream? Picks the
|
|
9
|
+
* default `sessionIntentWeight`.
|
|
10
|
+
*
|
|
11
|
+
* Two scenario families, same corpus:
|
|
12
|
+
*
|
|
13
|
+
* FOCUSED — 6 sessions (one per topic), 4 turns each of GENUINE REPHRASES
|
|
14
|
+
* of one intent. Tests the UPSIDE: does the centroid pull drifting
|
|
15
|
+
* rephrases back on-topic? (recall@5 / p@5 should rise with w.)
|
|
16
|
+
*
|
|
17
|
+
* SHIFT — 5 sessions, each 2 turns on topic A then 2 turns on topic B (a
|
|
18
|
+
* clean mid-session topic change). Tests the DOWNSIDE that focused-only
|
|
19
|
+
* can't see: a higher weight leaves the centroid partly A right after the
|
|
20
|
+
* shift, so the first turn(s) on B may LAG (lower recall@5 for B). The
|
|
21
|
+
* load-bearing read-out is post-shift recovery at turn-3 (first B turn)
|
|
22
|
+
* and turn-4, plus turns-to-recover.
|
|
23
|
+
*
|
|
24
|
+
* Method (both families):
|
|
25
|
+
* - Synthetic corpus of 30 memories across 6 well-separated topics (5 each),
|
|
26
|
+
* embedded with the REAL bge-small-en-v1.5 model (the production embedder).
|
|
27
|
+
* - For each weight w in {0.0, 0.2, 0.4, 0.5, 0.6, 0.8}: fresh
|
|
28
|
+
* SessionRecallRegistry per session; per turn, embed the prompt ONCE,
|
|
29
|
+
* blend with the live centroid, pass the blended vector to retrieve() via
|
|
30
|
+
* queryEmbedding, record recall@5 + p@5 for the turn's CURRENT topic,
|
|
31
|
+
* then fold the prompt into the centroid (EMA α=0.4).
|
|
32
|
+
*
|
|
33
|
+
* Fairness controls (load-bearing — the eval is useless if these slip):
|
|
34
|
+
* - `noStrengthen: true` on every retrieve(): the DB stays STATIC across all
|
|
35
|
+
* retrieve calls. Strengthening would mutate effective_strength/access_count
|
|
36
|
+
* between runs and contaminate cross-weight comparisons.
|
|
37
|
+
* - Uniform memory metadata: every memory has base_strength=0.5, created_at
|
|
38
|
+
* ≈ now, access_count=0, and NO links. So effective_strength, recency,
|
|
39
|
+
* connections, and the freshness boost are all uniform → the ONLY
|
|
40
|
+
* discriminator is vector cosine + RRF rank. That isolates the blend.
|
|
41
|
+
* - Cold-exposure slots are a no-op here (all candidates equally cold),
|
|
42
|
+
* so the top-k is the plain score order.
|
|
43
|
+
* - The centroid update is weight-INDEPENDENT (EMA of prompts at α=0.4), so
|
|
44
|
+
* the per-turn centroid is identical across all weights for a given
|
|
45
|
+
* session; only the blend differs. Embeddings are precomputed once.
|
|
46
|
+
*
|
|
47
|
+
* Metrics (per turn, for the turn's CURRENT topic):
|
|
48
|
+
* - recall@5 — did >=1 same-topic memory surface in the top-5? (coarse; the
|
|
49
|
+
* requested ship-gate metric — saturates on a small corpus.)
|
|
50
|
+
* - p@5 — on-topic count in the top-5 (0..5, finer).
|
|
51
|
+
* - recall@1 — was the SINGLE top result on-topic? (finest; focused only.)
|
|
52
|
+
*
|
|
53
|
+
* Honest by construction: it prints whatever the numbers are, including
|
|
54
|
+
* w=0 winning, the blend hurting a focused scenario, or high weights lagging
|
|
55
|
+
* badly post-shift. Report written to data/eval-recall-sweep/report.md and
|
|
56
|
+
* printed to stdout.
|
|
57
|
+
*
|
|
58
|
+
* Run: npm run eval:recall-sweep (== node dist/eval/recall-sweep.js)
|
|
59
|
+
*/
|
|
60
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
61
|
+
if (k2 === undefined) k2 = k;
|
|
62
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
63
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
64
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
65
|
+
}
|
|
66
|
+
Object.defineProperty(o, k2, desc);
|
|
67
|
+
}) : (function(o, m, k, k2) {
|
|
68
|
+
if (k2 === undefined) k2 = k;
|
|
69
|
+
o[k2] = m[k];
|
|
70
|
+
}));
|
|
71
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
72
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
73
|
+
}) : function(o, v) {
|
|
74
|
+
o["default"] = v;
|
|
75
|
+
});
|
|
76
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
77
|
+
var ownKeys = function(o) {
|
|
78
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
79
|
+
var ar = [];
|
|
80
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
81
|
+
return ar;
|
|
82
|
+
};
|
|
83
|
+
return ownKeys(o);
|
|
84
|
+
};
|
|
85
|
+
return function (mod) {
|
|
86
|
+
if (mod && mod.__esModule) return mod;
|
|
87
|
+
var result = {};
|
|
88
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
89
|
+
__setModuleDefault(result, mod);
|
|
90
|
+
return result;
|
|
91
|
+
};
|
|
92
|
+
})();
|
|
93
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
94
|
+
const node_os_1 = require("node:os");
|
|
95
|
+
const node_path_1 = require("node:path");
|
|
96
|
+
const node_fs_1 = require("node:fs");
|
|
97
|
+
const node_crypto_1 = require("node:crypto");
|
|
98
|
+
const db_js_1 = require("../db.js");
|
|
99
|
+
const storage = __importStar(require("../storage.js"));
|
|
100
|
+
const embedder_js_1 = require("../embedder.js");
|
|
101
|
+
const retrieval_js_1 = require("../retrieval.js");
|
|
102
|
+
const recall_registry_js_1 = require("../recall-registry.js");
|
|
103
|
+
const TOPICS = ["auth", "database", "ui", "deploy", "testing", "networking"];
|
|
104
|
+
/**
|
|
105
|
+
* 30 memories, 5 per topic. Each 2-4 sentences of concrete on-topic prose.
|
|
106
|
+
* Generic verbs/adjectives (fix, broken, failed, wrong, timeout, test) recur
|
|
107
|
+
* ACROSS topics to create realistic cross-topic competition for the top-5 —
|
|
108
|
+
* without that, recall@5 trivially saturates at 1.0 and the eval can't
|
|
109
|
+
* discriminate.
|
|
110
|
+
*/
|
|
111
|
+
const CORPUS = [
|
|
112
|
+
// --- auth (login / authentication / OAuth / sessions) ---
|
|
113
|
+
{
|
|
114
|
+
topic: "auth",
|
|
115
|
+
text: "Debugging the login form: the password field validation rejected valid credentials because the regex was too strict. Users with special characters in their passwords couldn't authenticate. Relaxed the character class and re-ran the auth test suite.",
|
|
116
|
+
},
|
|
117
|
+
{
|
|
118
|
+
topic: "auth",
|
|
119
|
+
text: "OAuth 2.0 token refresh flow: the refresh token wasn't persisted to the session store, so users got logged out after an hour. Moved token persistence into the auth middleware so it survives across requests.",
|
|
120
|
+
},
|
|
121
|
+
{
|
|
122
|
+
topic: "auth",
|
|
123
|
+
text: "Session management bug: the JWT expiry claim was set in seconds but the library expected milliseconds, causing immediate token invalidation. Every authenticated request returned 401 right after login.",
|
|
124
|
+
},
|
|
125
|
+
{
|
|
126
|
+
topic: "auth",
|
|
127
|
+
text: "Password reset endpoint: the email token was signed with the wrong secret, so reset links 404'd on confirmation. Regenerated the secret and invalidated all outstanding reset tokens.",
|
|
128
|
+
},
|
|
129
|
+
{
|
|
130
|
+
topic: "auth",
|
|
131
|
+
text: "Rate limiting on the login route: brute-force attempts locked out real users because the limiter counted by IP and the office NAT shares one address. Switched to per-account limiting with a CAPTCHA fallback.",
|
|
132
|
+
},
|
|
133
|
+
// --- database (SQL / Postgres / ORM / queries) ---
|
|
134
|
+
{
|
|
135
|
+
topic: "database",
|
|
136
|
+
text: "Slow query on the orders table: a missing index on created_at made the dashboard date-range scan do a full table scan. Added a composite index on (user_id, created_at) and the query dropped from 4s to 12ms.",
|
|
137
|
+
},
|
|
138
|
+
{
|
|
139
|
+
topic: "database",
|
|
140
|
+
text: "Postgres connection pool exhaustion: the ORM wasn't releasing connections after read transactions, so the pool hit its ceiling under load and requests queued. Added explicit transaction boundaries and a leak detector.",
|
|
141
|
+
},
|
|
142
|
+
{
|
|
143
|
+
topic: "database",
|
|
144
|
+
text: "N+1 query in the catalog serializer: each product fetched its categories separately, generating hundreds of round trips. Rewrote it as a single JOIN and used the ORM's eager-load hint.",
|
|
145
|
+
},
|
|
146
|
+
{
|
|
147
|
+
topic: "database",
|
|
148
|
+
text: "SQL migration deadlock: two deployments ran ALTER TABLE concurrently and Postgres serialized them, timing out the second. Switched to advisory locks in the migration runner.",
|
|
149
|
+
},
|
|
150
|
+
{
|
|
151
|
+
topic: "database",
|
|
152
|
+
text: "Wrong aggregate results: the revenue report double-counted refunded orders because the JOIN to the refunds table didn't exclude cancelled refunds. Added a status filter and a regression test.",
|
|
153
|
+
},
|
|
154
|
+
// --- ui (frontend / CSS / layout / rendering) ---
|
|
155
|
+
{
|
|
156
|
+
topic: "ui",
|
|
157
|
+
text: "Broken grid layout on Safari: CSS grid's auto-fit collapsed to a single column because the minmax lower bound was zero. Set a sensible minimum and added a flexbox fallback for older browsers.",
|
|
158
|
+
},
|
|
159
|
+
{
|
|
160
|
+
topic: "ui",
|
|
161
|
+
text: "Button not rendering: the component's conditional render checked a prop that was undefined rather than null, so the truthy branch never fired. Standardized on nullish coalescing across the component tree.",
|
|
162
|
+
},
|
|
163
|
+
{
|
|
164
|
+
topic: "ui",
|
|
165
|
+
text: "Responsive nav overflow: the hamburger menu pushed content off-screen on phones in landscape because the viewport meta tag was missing. Added the meta tag and a max-width on the header.",
|
|
166
|
+
},
|
|
167
|
+
{
|
|
168
|
+
topic: "ui",
|
|
169
|
+
text: "Theme flash on load: the CSS variables weren't set before the first paint, causing a white-then-dark flicker. Inlined the theme bootstrap script in the document head.",
|
|
170
|
+
},
|
|
171
|
+
{
|
|
172
|
+
topic: "ui",
|
|
173
|
+
text: "Form validation UX: error messages appeared on every keystroke, which felt jarring. Debounced the validation to fire on blur, or after the user paused typing.",
|
|
174
|
+
},
|
|
175
|
+
// --- deploy (CI / release / rollback) ---
|
|
176
|
+
{
|
|
177
|
+
topic: "deploy",
|
|
178
|
+
text: "Deploy failed at the asset compilation step: webpack ran out of memory on the CI runner. Raised the Node heap to 4GB and split the vendor bundle so the build stays under the limit.",
|
|
179
|
+
},
|
|
180
|
+
{
|
|
181
|
+
topic: "deploy",
|
|
182
|
+
text: "GitHub Actions workflow broke after the runner image updated: the preinstalled Python changed minor versions and our pinned dependency wouldn't install. Pinned the toolchain explicitly in the workflow.",
|
|
183
|
+
},
|
|
184
|
+
{
|
|
185
|
+
topic: "deploy",
|
|
186
|
+
text: "Blue-green deploy rollback: the new version had a migration that wasn't backward-compatible, so rolling back left half the schema on the new version. Added a deploy-time migration compatibility check.",
|
|
187
|
+
},
|
|
188
|
+
{
|
|
189
|
+
topic: "deploy",
|
|
190
|
+
text: "Container registry rate limit: CI pulled the base image on every run and hit Docker Hub's anonymous pull cap. Set up a pull-through cache and authenticated the registry.",
|
|
191
|
+
},
|
|
192
|
+
{
|
|
193
|
+
topic: "deploy",
|
|
194
|
+
text: "Canary deploy caught a memory leak: the new build's steady-state RSS crept up under synthetic traffic. The canary's OOM-kills metric failed the promotion gate and the deploy auto-rolled back.",
|
|
195
|
+
},
|
|
196
|
+
// --- testing (unit / integration / TDD) ---
|
|
197
|
+
{
|
|
198
|
+
topic: "testing",
|
|
199
|
+
text: "Flaky integration test: the test assumed a clean database but ran in parallel with the orders suite, so it saw phantom rows. Gave each test its own schema and a transactional rollback fixture.",
|
|
200
|
+
},
|
|
201
|
+
{
|
|
202
|
+
topic: "testing",
|
|
203
|
+
text: "Unit test for the pricing module: the pure-function test didn't cover the bulk-discount branch. Added cases for the boundary quantities and a property-based test for rounding.",
|
|
204
|
+
},
|
|
205
|
+
{
|
|
206
|
+
topic: "testing",
|
|
207
|
+
text: "Mocking the database in tests: the suite hit a real Postgres and timed out in CI. Introduced a repository seam and swapped in an in-memory fake for the fast path.",
|
|
208
|
+
},
|
|
209
|
+
{
|
|
210
|
+
topic: "testing",
|
|
211
|
+
text: "Test suite runtime: the full suite took nine minutes, mostly from synchronous waits. Parallelized with a sharded runner and dropped the wall clock to under two minutes.",
|
|
212
|
+
},
|
|
213
|
+
{
|
|
214
|
+
topic: "testing",
|
|
215
|
+
text: "Snapshot test churn: every cosmetic change updated fifty snapshots and reviewers stopped reading them. Replaced the brittle snapshots with explicit assertions on the relevant fields.",
|
|
216
|
+
},
|
|
217
|
+
// --- networking (HTTP / API / CORS / webhooks) ---
|
|
218
|
+
{
|
|
219
|
+
topic: "networking",
|
|
220
|
+
text: "API returning 500s: the upstream service started sending a new error envelope and our deserializer threw on the unknown field. Switched to lenient parsing and added a regression test for the new shape.",
|
|
221
|
+
},
|
|
222
|
+
{
|
|
223
|
+
topic: "networking",
|
|
224
|
+
text: "CORS preflight failing: the OPTIONS handler didn't echo back the request's Origin header, so browsers blocked the response. Fixed the header reflection and tightened the allowed-methods list.",
|
|
225
|
+
},
|
|
226
|
+
{
|
|
227
|
+
topic: "networking",
|
|
228
|
+
text: "HTTP request timeout: the client's default timeout was 30s but the upstream's P99 was 45s under load. Tuned the timeout per endpoint and added a circuit breaker for sustained failures.",
|
|
229
|
+
},
|
|
230
|
+
{
|
|
231
|
+
topic: "networking",
|
|
232
|
+
text: "Webhook delivery failures: retries used a fixed delay and the receiving endpoint rate-limited us after the first burst. Moved to exponential backoff with jitter and a dead-letter queue.",
|
|
233
|
+
},
|
|
234
|
+
{
|
|
235
|
+
topic: "networking",
|
|
236
|
+
text: "TLS handshake errors: the server's certificate chain was missing the intermediate, so mobile clients failed the trust check. Bundled the full chain in the PEM and renewed via ACME.",
|
|
237
|
+
},
|
|
238
|
+
];
|
|
239
|
+
/**
|
|
240
|
+
* 4-turn rephrase set per topic — GENUINE REPHRASES of one intent: same
|
|
241
|
+
* meaning, different surface wording, deliberately spanning terse/oblique to
|
|
242
|
+
* descriptive. The vocabulary shift is the focused-scenario fairness lever
|
|
243
|
+
* (pure-prompt ranking should plausibly move turn-to-turn). Turn order is a
|
|
244
|
+
* realistic session flow; the EMA centroid (α=0.4) weights the most recent
|
|
245
|
+
* turn most and retains geometrically-decaying signal from earlier ones, so
|
|
246
|
+
* ordering a strong matcher early keeps it in the centroid as turns accumulate.
|
|
247
|
+
* (The shipped default 0.33 sits in the eval-supported 0.3–0.4 band but is
|
|
248
|
+
* not swept directly — the sweep validates the band, not the exact default.)
|
|
249
|
+
*
|
|
250
|
+
* Shift sessions reuse turns[0..1] of A then turns[0..1] of B — so the same
|
|
251
|
+
* rephrases drive both families (no separate, easier shift corpus).
|
|
252
|
+
*/
|
|
253
|
+
const FOCUSED_TURNS = {
|
|
254
|
+
auth: [
|
|
255
|
+
"fix the login bug", // terse; "login" lexical, "bug" generic
|
|
256
|
+
"why is authentication failing", // "authenticate" vs "login"; "failing" generic
|
|
257
|
+
"the signin page is broken", // "signin"/"page" loose; "broken" generic
|
|
258
|
+
"users can't log in", // terse; "log in" vs "login"/"authenticate"
|
|
259
|
+
],
|
|
260
|
+
database: [
|
|
261
|
+
"the database query is slow", // direct: "query"/"slow"
|
|
262
|
+
"optimize this SQL statement", // "SQL"/"optimize" vs "slow"
|
|
263
|
+
"postgres is returning wrong results", // "postgres"/"wrong" generic
|
|
264
|
+
"the ORM is generating bad queries", // "ORM"/"queries"
|
|
265
|
+
],
|
|
266
|
+
ui: [
|
|
267
|
+
"the layout is broken on mobile", // "layout"/"mobile"; "broken" generic
|
|
268
|
+
"fix the CSS grid", // "CSS"/"grid"
|
|
269
|
+
"the button doesn't render", // "button"/"render"
|
|
270
|
+
"responsive design is off", // "responsive"; "off" terse/generic
|
|
271
|
+
],
|
|
272
|
+
deploy: [
|
|
273
|
+
"the deploy failed", // terse; "deploy"/"failed" generic
|
|
274
|
+
"CI pipeline is red", // "CI"; "red" = failing
|
|
275
|
+
"the GitHub Actions workflow broke", // "GitHub Actions"/"workflow"/"broke"
|
|
276
|
+
"rollback the production release", // "rollback"/"release"
|
|
277
|
+
],
|
|
278
|
+
testing: [
|
|
279
|
+
"write a unit test for this", // "unit test"
|
|
280
|
+
"the test suite is flaky", // "test suite"/"flaky"
|
|
281
|
+
"fix the failing assertion", // "assertion"; "failing" generic
|
|
282
|
+
"mock the database in tests", // "mock"/"tests"
|
|
283
|
+
],
|
|
284
|
+
networking: [
|
|
285
|
+
"the API returns 500 errors", // "API"/"500"
|
|
286
|
+
"fix the CORS issue", // "CORS"; "issue" generic
|
|
287
|
+
"the HTTP request times out", // "HTTP"/"times out"
|
|
288
|
+
"webhook delivery is failing", // "webhook"; "failing" generic
|
|
289
|
+
],
|
|
290
|
+
};
|
|
291
|
+
/**
|
|
292
|
+
* Shift pairs: 2 turns on A → 2 turns on B. A and B are deliberately different
|
|
293
|
+
* so the A-seeded centroid actively opposes the first B turn(s) at high w.
|
|
294
|
+
* Five pairs cover each topic once or twice on each side of a shift.
|
|
295
|
+
*/
|
|
296
|
+
const SHIFT_PAIRS = [
|
|
297
|
+
["auth", "database"],
|
|
298
|
+
["ui", "deploy"],
|
|
299
|
+
["testing", "networking"],
|
|
300
|
+
["database", "ui"],
|
|
301
|
+
["deploy", "testing"],
|
|
302
|
+
];
|
|
303
|
+
const WEIGHTS = [0.0, 0.2, 0.4, 0.5, 0.6, 0.8];
|
|
304
|
+
function buildSessions() {
|
|
305
|
+
const focused = TOPICS.map((t) => ({
|
|
306
|
+
kind: "focused",
|
|
307
|
+
id: `focused-${t}`,
|
|
308
|
+
label: t,
|
|
309
|
+
turns: FOCUSED_TURNS[t].map((p) => ({ prompt: p, topic: t })),
|
|
310
|
+
}));
|
|
311
|
+
const shifts = SHIFT_PAIRS.map(([a, b], i) => ({
|
|
312
|
+
kind: "shift",
|
|
313
|
+
id: `shift-${i}-${a}-${b}`,
|
|
314
|
+
label: `${a}->${b}`,
|
|
315
|
+
topicA: a,
|
|
316
|
+
topicB: b,
|
|
317
|
+
turns: [
|
|
318
|
+
{ prompt: FOCUSED_TURNS[a][0], topic: a },
|
|
319
|
+
{ prompt: FOCUSED_TURNS[a][1], topic: a },
|
|
320
|
+
{ prompt: FOCUSED_TURNS[b][0], topic: b },
|
|
321
|
+
{ prompt: FOCUSED_TURNS[b][1], topic: b },
|
|
322
|
+
],
|
|
323
|
+
}));
|
|
324
|
+
return [...focused, ...shifts];
|
|
325
|
+
}
|
|
326
|
+
// ---------------------------------------------------------------------------
|
|
327
|
+
// DB construction
|
|
328
|
+
// ---------------------------------------------------------------------------
|
|
329
|
+
async function buildCorpusDb(dbPath) {
|
|
330
|
+
const db = (0, db_js_1.initDb)(dbPath);
|
|
331
|
+
const idToTopic = new Map();
|
|
332
|
+
for (const mem of CORPUS) {
|
|
333
|
+
const vec = await (0, embedder_js_1.embed)(mem.text);
|
|
334
|
+
const id = storage.insertMemory(db, mem.text, vec, {
|
|
335
|
+
sourceAgent: "eval-corpus",
|
|
336
|
+
memoryType: "episode",
|
|
337
|
+
baseStrength: 0.5, // uniform — strength is NOT a discriminator here
|
|
338
|
+
});
|
|
339
|
+
idToTopic.set(id, mem.topic);
|
|
340
|
+
}
|
|
341
|
+
return { db, idToTopic };
|
|
342
|
+
}
|
|
343
|
+
/** Embed every distinct prompt once; return a cache keyed by prompt text. */
|
|
344
|
+
async function embedAllPrompts(sessions) {
|
|
345
|
+
const cache = new Map();
|
|
346
|
+
for (const s of sessions) {
|
|
347
|
+
for (const t of s.turns) {
|
|
348
|
+
if (!cache.has(t.prompt))
|
|
349
|
+
cache.set(t.prompt, await (0, embedder_js_1.embed)(t.prompt));
|
|
350
|
+
}
|
|
351
|
+
}
|
|
352
|
+
return cache;
|
|
353
|
+
}
|
|
354
|
+
/**
|
|
355
|
+
* Run the full sweep. Prompts are pre-embedded once (the centroid update is
|
|
356
|
+
* weight-independent, so the per-turn centroid is identical across weights;
|
|
357
|
+
* only the blend differs). A fresh SessionRecallRegistry per (weight,session)
|
|
358
|
+
* keeps each centroid seeded only by that session's own turns.
|
|
359
|
+
*/
|
|
360
|
+
async function runSweep(db, idToTopic, sessions) {
|
|
361
|
+
const promptEmb = await embedAllPrompts(sessions);
|
|
362
|
+
// Fail-explicit embedFn: we ALWAYS supply queryEmbedding, so retrieve()
|
|
363
|
+
// must never call this. If it does, the sweep is measuring the wrong thing.
|
|
364
|
+
const neverCalledEmbed = async () => {
|
|
365
|
+
throw new Error("recall-sweep: retrieve() called the embedFn — queryEmbedding was not honored. Eval aborted (results would be invalid).");
|
|
366
|
+
};
|
|
367
|
+
const records = [];
|
|
368
|
+
for (const w of WEIGHTS) {
|
|
369
|
+
for (const s of sessions) {
|
|
370
|
+
const registry = new recall_registry_js_1.SessionRecallRegistry();
|
|
371
|
+
const sessionId = `w${w}-${s.id}`;
|
|
372
|
+
for (let i = 0; i < s.turns.length; i++) {
|
|
373
|
+
const turnDef = s.turns[i];
|
|
374
|
+
const promptText = turnDef.prompt;
|
|
375
|
+
const promptVec = promptEmb.get(promptText);
|
|
376
|
+
const centroid = registry.getCentroid(sessionId);
|
|
377
|
+
const queryVec = (0, retrieval_js_1.blendQueryVector)(promptVec, centroid, w);
|
|
378
|
+
const results = await (0, retrieval_js_1.retrieve)(db, neverCalledEmbed, promptText, {
|
|
379
|
+
limit: 5,
|
|
380
|
+
queryEmbedding: queryVec,
|
|
381
|
+
noStrengthen: true,
|
|
382
|
+
});
|
|
383
|
+
const topIds = results.map((r) => r.id);
|
|
384
|
+
const onTopic = topIds.filter((id) => idToTopic.get(id) === turnDef.topic);
|
|
385
|
+
const topTopic = topIds.length > 0 ? idToTopic.get(topIds[0]) : undefined;
|
|
386
|
+
records.push({
|
|
387
|
+
weight: w,
|
|
388
|
+
sessionId: s.id,
|
|
389
|
+
kind: s.kind,
|
|
390
|
+
label: s.label,
|
|
391
|
+
topicA: s.topicA,
|
|
392
|
+
topicB: s.topicB,
|
|
393
|
+
turn: i + 1,
|
|
394
|
+
currentTopic: turnDef.topic,
|
|
395
|
+
recallAt5: onTopic.length > 0 ? 1 : 0,
|
|
396
|
+
recallAt1: topTopic === turnDef.topic ? 1 : 0,
|
|
397
|
+
precisionAt5: onTopic.length,
|
|
398
|
+
topIds,
|
|
399
|
+
});
|
|
400
|
+
// Fold this prompt into the centroid AFTER the search (seeds on turn 1,
|
|
401
|
+
// blends after). α is the shipped constant.
|
|
402
|
+
registry.updateCentroid(sessionId, promptVec, retrieval_js_1.SESSION_INTENT_ALPHA);
|
|
403
|
+
}
|
|
404
|
+
}
|
|
405
|
+
}
|
|
406
|
+
return records;
|
|
407
|
+
}
|
|
408
|
+
// ---------------------------------------------------------------------------
|
|
409
|
+
// Aggregation helpers
|
|
410
|
+
// ---------------------------------------------------------------------------
|
|
411
|
+
function avg(rs, field) {
|
|
412
|
+
if (rs.length === 0)
|
|
413
|
+
return 0;
|
|
414
|
+
return rs.reduce((s, r) => s + r[field], 0) / rs.length;
|
|
415
|
+
}
|
|
416
|
+
function pct(x) {
|
|
417
|
+
return `${(x * 100).toFixed(1)}%`;
|
|
418
|
+
}
|
|
419
|
+
function fmtW(w) {
|
|
420
|
+
return w.toFixed(1);
|
|
421
|
+
}
|
|
422
|
+
function summarizeFocused(records, sessions) {
|
|
423
|
+
const focusedSessions = sessions.filter((s) => s.kind === "focused");
|
|
424
|
+
return WEIGHTS.map((w) => {
|
|
425
|
+
const wr = records.filter((r) => r.weight === w && r.kind === "focused");
|
|
426
|
+
const turnN = (n) => wr.filter((r) => r.turn === n);
|
|
427
|
+
const byScenario = focusedSessions.map((s) => avg(wr.filter((r) => r.sessionId === s.id), "recallAt5"));
|
|
428
|
+
return {
|
|
429
|
+
weight: w,
|
|
430
|
+
recallAt5All: avg(wr, "recallAt5"),
|
|
431
|
+
recallAt5Turn1: avg(turnN(1), "recallAt5"),
|
|
432
|
+
recallAt5Turn4: avg(turnN(4), "recallAt5"),
|
|
433
|
+
recallAt1All: avg(wr, "recallAt1"),
|
|
434
|
+
precisionAt5All: avg(wr, "precisionAt5"),
|
|
435
|
+
byScenario,
|
|
436
|
+
};
|
|
437
|
+
});
|
|
438
|
+
}
|
|
439
|
+
function summarizeShift(records) {
|
|
440
|
+
return WEIGHTS.map((w) => {
|
|
441
|
+
const wr = records.filter((r) => r.weight === w && r.kind === "shift");
|
|
442
|
+
const pre = wr.filter((r) => r.turn <= 2);
|
|
443
|
+
const t3 = wr.filter((r) => r.turn === 3);
|
|
444
|
+
const t4 = wr.filter((r) => r.turn === 4);
|
|
445
|
+
// Recovery per session: recall@5 for B at turn 3 (and turn 4).
|
|
446
|
+
const sessionLabels = Array.from(new Set(wr.map((r) => r.sessionId)));
|
|
447
|
+
let recByT3 = 0;
|
|
448
|
+
let recByT4 = 0;
|
|
449
|
+
for (const sid of sessionLabels) {
|
|
450
|
+
const t3r = wr.find((r) => r.sessionId === sid && r.turn === 3);
|
|
451
|
+
const t4r = wr.find((r) => r.sessionId === sid && r.turn === 4);
|
|
452
|
+
if (t3r && t3r.recallAt5 === 1)
|
|
453
|
+
recByT3++;
|
|
454
|
+
if ((t3r && t3r.recallAt5 === 1) || (t4r && t4r.recallAt5 === 1))
|
|
455
|
+
recByT4++;
|
|
456
|
+
}
|
|
457
|
+
const n = sessionLabels.length || 1;
|
|
458
|
+
return {
|
|
459
|
+
weight: w,
|
|
460
|
+
preRecallAt5: avg(pre, "recallAt5"),
|
|
461
|
+
prePrecisionAt5: avg(pre, "precisionAt5"),
|
|
462
|
+
postT3RecallAt5: avg(t3, "recallAt5"),
|
|
463
|
+
postT3PrecisionAt5: avg(t3, "precisionAt5"),
|
|
464
|
+
postT4RecallAt5: avg(t4, "recallAt5"),
|
|
465
|
+
postT4PrecisionAt5: avg(t4, "precisionAt5"),
|
|
466
|
+
recoveredByT3: recByT3 / n,
|
|
467
|
+
recoveredByT4: recByT4 / n,
|
|
468
|
+
};
|
|
469
|
+
});
|
|
470
|
+
}
|
|
471
|
+
function renderReport(focused, shifts, records, sessions, meta) {
|
|
472
|
+
const L = [];
|
|
473
|
+
const nFocused = sessions.filter((s) => s.kind === "focused").length;
|
|
474
|
+
const nShift = sessions.filter((s) => s.kind === "shift").length;
|
|
475
|
+
L.push("# Recall@k Sweep — session-intent keying (#192, 0867d6c)\n");
|
|
476
|
+
L.push(`Corpus: ${meta.memoryCount} memories across ${TOPICS.length} topics (5 each), embedded with the ` +
|
|
477
|
+
`real bge-small-en-v1.5 model. ${nFocused} focused scenarios x 4 turns + ${nShift} shift scenarios ` +
|
|
478
|
+
`x 4 turns (2 on A -> 2 on B).\n`);
|
|
479
|
+
L.push("_Static DB across all retrieve() calls (`noStrengthen: true`); uniform memory metadata " +
|
|
480
|
+
"(base_strength=0.5, created_at~now, no links) so vector cosine + RRF is the only discriminator; " +
|
|
481
|
+
"cold-exposure slots are a no-op (all candidates equally cold). Centroid EMA at α=0.4. " +
|
|
482
|
+
"Per-turn metrics are vs the turn's CURRENT topic (A then B for shifts)._\n");
|
|
483
|
+
// ---- 1. Focused ----
|
|
484
|
+
L.push("## 1. Focused recall@5 / p@5 by weight (upside)\n");
|
|
485
|
+
L.push("Does the centroid pull same-intent rephrases back on-topic? recall@5 = >=1 on-topic in top-5; " +
|
|
486
|
+
"recall@1 = TOP result on-topic; p@5 = mean on-topic count in top-5 (0..5).\n");
|
|
487
|
+
L.push("| weight | recall@5 (all) | recall@5 turn-1 | recall@5 turn-4 | recall@1 (all) | p@5 (all) |");
|
|
488
|
+
L.push("|---|---|---|---|---|---|");
|
|
489
|
+
for (const s of focused) {
|
|
490
|
+
L.push(`| ${fmtW(s.weight)} | ${pct(s.recallAt5All)} | ${pct(s.recallAt5Turn1)} | ${pct(s.recallAt5Turn4)} | ${pct(s.recallAt1All)} | ${s.precisionAt5All.toFixed(2)} |`);
|
|
491
|
+
}
|
|
492
|
+
L.push("");
|
|
493
|
+
const f0 = focused.find((s) => s.weight === 0);
|
|
494
|
+
const fOthers = focused.filter((s) => s.weight > 0);
|
|
495
|
+
const fBestOverall = focused.reduce((a, b) => (b.recallAt5All > a.recallAt5All + 1e-9 ? b : a));
|
|
496
|
+
const fBeatR5 = fOthers.some((s) => s.recallAt5All > f0.recallAt5All + 1e-9);
|
|
497
|
+
const fBeatR1 = fOthers.some((s) => s.recallAt1All > f0.recallAt1All + 1e-9);
|
|
498
|
+
const fBeatP5 = fOthers.some((s) => s.precisionAt5All > f0.precisionAt5All + 1e-9);
|
|
499
|
+
L.push("### Focused verdict\n");
|
|
500
|
+
L.push(`- Best overall recall@5: w=${fmtW(fBestOverall.weight)} (${pct(fBestOverall.recallAt5All)})`);
|
|
501
|
+
L.push(`- Any w>0 beat w=0 on recall@5: **${fBeatR5 ? "YES" : "NO"}**`);
|
|
502
|
+
L.push(`- Any w>0 beat w=0 on recall@1: **${fBeatR1 ? "YES" : "NO"}**`);
|
|
503
|
+
L.push(`- Any w>0 beat w=0 on p@5: **${fBeatP5 ? "YES" : "NO"}**`);
|
|
504
|
+
// Focused regressions (blend hurt a focused scenario vs w=0)
|
|
505
|
+
const fRegs = [];
|
|
506
|
+
for (let si = 0; si < focused[0].byScenario.length; si++) {
|
|
507
|
+
const v0 = focused.find((x) => x.weight === 0).byScenario[si];
|
|
508
|
+
for (const s of focused) {
|
|
509
|
+
if (s.weight === 0)
|
|
510
|
+
continue;
|
|
511
|
+
const v = s.byScenario[si];
|
|
512
|
+
if (v < v0 - 1e-9) {
|
|
513
|
+
const lbl = sessions.filter((x) => x.kind === "focused")[si].label;
|
|
514
|
+
fRegs.push(`- ${lbl} at w=${fmtW(s.weight)}: ${pct(v)} vs w=0 ${pct(v0)}`);
|
|
515
|
+
}
|
|
516
|
+
}
|
|
517
|
+
}
|
|
518
|
+
L.push(`- Focused regressions (blend HURT vs w=0): ${fRegs.length > 0 ? fRegs.join("; ") : "_(none)_"}\n`);
|
|
519
|
+
// ---- 2. Shift recovery ----
|
|
520
|
+
L.push("## 2. Shift recovery by weight (downside)\n");
|
|
521
|
+
L.push("2 turns on topic A, then 2 turns on topic B (clean mid-session shift). Metrics are vs the turn's " +
|
|
522
|
+
"CURRENT topic. **Turn-3 = first turn on B, right after the shift** — the load-bearing read-out: " +
|
|
523
|
+
"a higher weight leaves the centroid partly A, so B recall@5 should DROP at high w. Turn-4 = second " +
|
|
524
|
+
"B turn — has the EMA recovered toward B?\n");
|
|
525
|
+
L.push("| weight | pre recall@5 (A, t1-2) | pre p@5 | **post recall@5 (B, t3)** | post p@5 (t3) | post recall@5 (B, t4) | post p@5 (t4) | recovered by t3 | recovered by t4 |");
|
|
526
|
+
L.push("|---|---|---|---|---|---|---|---|---|");
|
|
527
|
+
for (const s of shifts) {
|
|
528
|
+
L.push(`| ${fmtW(s.weight)} | ${pct(s.preRecallAt5)} | ${s.prePrecisionAt5.toFixed(2)} | ` +
|
|
529
|
+
`**${pct(s.postT3RecallAt5)}** | ${s.postT3PrecisionAt5.toFixed(2)} | ` +
|
|
530
|
+
`${pct(s.postT4RecallAt5)} | ${s.postT4PrecisionAt5.toFixed(2)} | ` +
|
|
531
|
+
`${pct(s.recoveredByT3)} | ${pct(s.recoveredByT4)} |`);
|
|
532
|
+
}
|
|
533
|
+
L.push("");
|
|
534
|
+
// Per-shift-scenario post-shift recall@5 (turn-3) by weight — where does lag show up?
|
|
535
|
+
L.push("### Per-shift-scenario post-shift recall@5 (first turn on B = turn-3)\n");
|
|
536
|
+
L.push("Each cell = recall@5 for the NEW topic at the first post-shift turn. w=0 column = pure-prompt baseline (no centroid lag). Drops below the w=0 column = the centroid LAGGING the shift.\n");
|
|
537
|
+
const shiftSessions = sessions.filter((s) => s.kind === "shift");
|
|
538
|
+
const head = ["shift (A->B)", ...WEIGHTS.map((w) => `w=${fmtW(w)}`)];
|
|
539
|
+
L.push("| " + head.join(" | ") + " |");
|
|
540
|
+
L.push("|" + head.map(() => "---").join("|") + "|");
|
|
541
|
+
for (const s of shiftSessions) {
|
|
542
|
+
const cells = WEIGHTS.map((w) => {
|
|
543
|
+
const rec = records.find((r) => r.weight === w && r.kind === "shift" && r.sessionId === s.id && r.turn === 3);
|
|
544
|
+
return rec && rec.recallAt5 === 1 ? "1" : "0";
|
|
545
|
+
});
|
|
546
|
+
L.push(`| ${s.label} | ${cells.join(" | ")} |`);
|
|
547
|
+
}
|
|
548
|
+
L.push("");
|
|
549
|
+
// Shift verdict — does any weight lag vs w=0?
|
|
550
|
+
const sh0 = shifts.find((s) => s.weight === 0);
|
|
551
|
+
const laggersT3 = [];
|
|
552
|
+
const laggersT4 = [];
|
|
553
|
+
for (const s of shifts) {
|
|
554
|
+
if (s.weight === 0)
|
|
555
|
+
continue;
|
|
556
|
+
if (s.postT3RecallAt5 < sh0.postT3RecallAt5 - 1e-9) {
|
|
557
|
+
laggersT3.push(`w=${fmtW(s.weight)}: ${pct(s.postT3RecallAt5)} (vs w=0 ${pct(sh0.postT3RecallAt5)}, −${(sh0.postT3RecallAt5 - s.postT3RecallAt5).toFixed(2)})`);
|
|
558
|
+
}
|
|
559
|
+
if (s.postT4RecallAt5 < sh0.postT4RecallAt5 - 1e-9) {
|
|
560
|
+
laggersT4.push(`w=${fmtW(s.weight)}: ${pct(s.postT4RecallAt5)} (vs w=0 ${pct(sh0.postT4RecallAt5)}, −${(sh0.postT4RecallAt5 - s.postT4RecallAt5).toFixed(2)})`);
|
|
561
|
+
}
|
|
562
|
+
}
|
|
563
|
+
L.push("### Shift verdict\n");
|
|
564
|
+
L.push(`- Baseline (w=0) post-shift recall@5: turn-3 ${pct(sh0.postT3RecallAt5)}, turn-4 ${pct(sh0.postT4RecallAt5)}`);
|
|
565
|
+
L.push(`- Weights that LAG at turn-3 (B recall@5 < w=0): ${laggersT3.length > 0 ? laggersT3.join("; ") : "_(none — no shift lag at any weight)_"}`);
|
|
566
|
+
L.push(`- Weights still lagging at turn-4: ${laggersT4.length > 0 ? laggersT4.join("; ") : "_(none — all recovered by turn-4)_"}\n`);
|
|
567
|
+
// ---- 3. Combined verdict ----
|
|
568
|
+
L.push("## 3. Combined verdict (focused upside vs shift cost)\n");
|
|
569
|
+
L.push("Two recommendations, deliberately side by side. recall@5 is BINARY (>=1 on-topic) so it hides the " +
|
|
570
|
+
"steady p@5 erosion the shift table shows — read both. The p@5-aware pick is the finer signal.\n");
|
|
571
|
+
// (a) recall@5-based: highest weight with NO turn-3 recall@5 lag AND maximal
|
|
572
|
+
// focused recall@5. Coarse — a weight can pass this while halving the
|
|
573
|
+
// on-topic count in the top-5 (p@5), because one survivor still flips
|
|
574
|
+
// recall@5 to 1.
|
|
575
|
+
const noLagT3 = shifts.filter((s) => s.postT3RecallAt5 >= sh0.postT3RecallAt5 - 1e-9);
|
|
576
|
+
const noLagWeights = noLagT3.map((s) => s.weight).filter((w) => w > 0);
|
|
577
|
+
const focusedCeiling = Math.max(...focused.map((s) => s.recallAt5All));
|
|
578
|
+
const r5Cands = noLagWeights.filter((w) => {
|
|
579
|
+
const f = focused.find((x) => x.weight === w);
|
|
580
|
+
return f.recallAt5All >= focusedCeiling - 1e-9;
|
|
581
|
+
});
|
|
582
|
+
const recR5 = r5Cands.length > 0 ? Math.max(...r5Cands) : null;
|
|
583
|
+
// (b) p@5-aware: among weights that capture the focused p@5 upside (within
|
|
584
|
+
// 5% of the plateau), pick the one with the HIGHEST turn-3 shift p@5 (least
|
|
585
|
+
// shift cost). p@5 is non-saturating on this corpus, so this respects BOTH
|
|
586
|
+
// the focused upside and the shift cost — the recommendation to actually ship.
|
|
587
|
+
const p5Plateau = Math.max(...focused.map((s) => s.precisionAt5All));
|
|
588
|
+
const p5Cands = WEIGHTS.filter((w) => w > 0 &&
|
|
589
|
+
focused.find((x) => x.weight === w).precisionAt5All >= 0.95 * p5Plateau - 1e-9);
|
|
590
|
+
let recP5 = null;
|
|
591
|
+
let bestT3p5 = -Infinity;
|
|
592
|
+
for (const w of p5Cands) {
|
|
593
|
+
const t3p5 = shifts.find((x) => x.weight === w).postT3PrecisionAt5;
|
|
594
|
+
if (t3p5 > bestT3p5 + 1e-9) {
|
|
595
|
+
bestT3p5 = t3p5;
|
|
596
|
+
recP5 = w;
|
|
597
|
+
}
|
|
598
|
+
}
|
|
599
|
+
L.push(`- **recall@5-based recommendation: w=${recR5 !== null ? fmtW(recR5) : "n/a"}** — highest weight with ` +
|
|
600
|
+
`no turn-3 recall@5 lag and maximal focused recall@5 (${pct(focusedCeiling)}). Coarse: it does NOT ` +
|
|
601
|
+
`see the p@5 erosion.`);
|
|
602
|
+
L.push(`- **p@5-aware recommendation: w=${recP5 !== null ? fmtW(recP5) : "n/a"}** — among weights capturing ` +
|
|
603
|
+
`>=95% of the focused p@5 plateau (${p5Plateau.toFixed(2)}), the one with the highest turn-3 shift ` +
|
|
604
|
+
`p@5 (${bestT3p5.toFixed(2)} on-topic in the top-5 at the first post-shift turn). This is the pick ` +
|
|
605
|
+
`that respects both upside and shift cost.\n`);
|
|
606
|
+
// Plain-English summary line — honest about the divergence.
|
|
607
|
+
const maxLagT3r5 = Math.max(0, ...shifts.filter((s) => s.weight > 0).map((s) => sh0.postT3RecallAt5 - s.postT3RecallAt5));
|
|
608
|
+
// Worst p@5 erosion at turn-3 vs the w=0 baseline (the finer, earlier signal).
|
|
609
|
+
const maxP5drop = Math.max(0, ...shifts
|
|
610
|
+
.filter((s) => s.weight > 0)
|
|
611
|
+
.map((s) => sh0.postT3PrecisionAt5 - s.postT3PrecisionAt5));
|
|
612
|
+
const diverge = recR5 !== null && recP5 !== null && Math.abs(recR5 - recP5) > 1e-9;
|
|
613
|
+
let plain;
|
|
614
|
+
if (!fBeatR5 && !fBeatR1 && !fBeatP5 && maxLagT3r5 <= 0 && maxP5drop <= 0) {
|
|
615
|
+
plain =
|
|
616
|
+
"The blend shows no focused benefit and no shift cost — on this corpus it is a no-op. w=0 is the honest default.";
|
|
617
|
+
}
|
|
618
|
+
else if (diverge) {
|
|
619
|
+
plain =
|
|
620
|
+
`The recall@5-based pick (w=${fmtW(recR5)}) and the p@5-aware pick (w=${fmtW(recP5)}) DIVERGE — ` +
|
|
621
|
+
`recall@5 stays at 100% through w=${fmtW(recR5)} but p@5 at the first post-shift turn erodes steadily ` +
|
|
622
|
+
`(down up to ${maxP5drop.toFixed(2)} on-topic memories in the top-5 vs w=0), and collapses at w=0.8 ` +
|
|
623
|
+
`(recall@5 ${pct(shifts.find((s) => s.weight === 0.8).postT3RecallAt5)} — the new topic misses the ` +
|
|
624
|
+
`top-5 entirely in 3/5 shift scenarios). The finer metric argues for the lower default w=${fmtW(recP5)}: ` +
|
|
625
|
+
`it captures the focused upside (p@5 within 5% of plateau) with the least shift cost. Ship w=${fmtW(recP5)}.`;
|
|
626
|
+
}
|
|
627
|
+
else if (maxLagT3r5 > 0) {
|
|
628
|
+
plain =
|
|
629
|
+
`The blend buys focused recall but costs recall@5 lag at turn-3 at the highest weight ` +
|
|
630
|
+
`(up to −${maxLagT3r5.toFixed(2)}). Recommended w=${recP5 !== null ? fmtW(recP5) : recR5 !== null ? fmtW(recR5) : "?"} keeps the upside without the lag.`;
|
|
631
|
+
}
|
|
632
|
+
else {
|
|
633
|
+
plain =
|
|
634
|
+
`The blend helps focused recall with NO recall@5 shift lag at any tested weight — ship a non-zero ` +
|
|
635
|
+
`default (recommended w=${recP5 !== null ? fmtW(recP5) : recR5 !== null ? fmtW(recR5) : "0.4"}).`;
|
|
636
|
+
}
|
|
637
|
+
L.push(`**${plain}**\n`);
|
|
638
|
+
// Sanity: w=0 per-turn recall@5 for focused (did pure-prompt actually diverge?)
|
|
639
|
+
L.push("## Appendix: w=0 per-turn recall@5 — did pure-prompt diverge across rephrases?\n");
|
|
640
|
+
L.push("| scenario | turn-1 | turn-2 | turn-3 | turn-4 |");
|
|
641
|
+
L.push("|---|---|---|---|---|");
|
|
642
|
+
for (const s of sessions.filter((x) => x.kind === "focused")) {
|
|
643
|
+
const cells = [1, 2, 3, 4].map((n) => {
|
|
644
|
+
const r = records.find((r) => r.weight === 0 && r.sessionId === s.id && r.turn === n);
|
|
645
|
+
return r.recallAt5 === 1 ? "1" : "0";
|
|
646
|
+
});
|
|
647
|
+
L.push(`| ${s.label} | ${cells.join(" | ")} |`);
|
|
648
|
+
}
|
|
649
|
+
L.push("");
|
|
650
|
+
return L.join("\n");
|
|
651
|
+
}
|
|
652
|
+
// ---------------------------------------------------------------------------
|
|
653
|
+
// main
|
|
654
|
+
// ---------------------------------------------------------------------------
|
|
655
|
+
async function main() {
|
|
656
|
+
// Pin all module-level knobs to shipped defaults so a host config can't
|
|
657
|
+
// leak into the sweep. The blend weight is varied per-run via blendQueryVector
|
|
658
|
+
// directly (NOT via configureSessionIntent), so the global sessionIntentWeight
|
|
659
|
+
// is irrelevant here.
|
|
660
|
+
(0, retrieval_js_1.configureScoring)();
|
|
661
|
+
(0, retrieval_js_1.configureDecay)();
|
|
662
|
+
(0, retrieval_js_1.configureRecall)();
|
|
663
|
+
(0, retrieval_js_1.configureSessionIntent)();
|
|
664
|
+
const sessions = buildSessions();
|
|
665
|
+
const tmpDir = (0, node_path_1.join)((0, node_os_1.tmpdir)(), `hicortex-recall-sweep-${(0, node_crypto_1.randomUUID)().slice(0, 8)}`);
|
|
666
|
+
(0, node_fs_1.mkdirSync)(tmpDir, { recursive: true });
|
|
667
|
+
const dbPath = (0, node_path_1.join)(tmpDir, "sweep.db");
|
|
668
|
+
const reportDir = (0, node_path_1.join)(process.cwd(), "data", "eval-recall-sweep");
|
|
669
|
+
console.log(`[recall-sweep] temp DB: ${dbPath}`);
|
|
670
|
+
console.log(`[recall-sweep] report dir: ${reportDir}`);
|
|
671
|
+
console.log(`[recall-sweep] ${sessions.filter((s) => s.kind === "focused").length} focused + ` +
|
|
672
|
+
`${sessions.filter((s) => s.kind === "shift").length} shift sessions; ` +
|
|
673
|
+
`${WEIGHTS.length} weights = ${sessions.length * WEIGHTS.length} sweeps x 4 turns = ` +
|
|
674
|
+
`${sessions.length * WEIGHTS.length * 4} retrieves`);
|
|
675
|
+
let db = null;
|
|
676
|
+
try {
|
|
677
|
+
console.log("[recall-sweep] building corpus (embedding 30 memories)...");
|
|
678
|
+
const t0 = Date.now();
|
|
679
|
+
const { db: opened, idToTopic } = await buildCorpusDb(dbPath);
|
|
680
|
+
db = opened;
|
|
681
|
+
console.log(`[recall-sweep] corpus ready in ${Date.now() - t0}ms (${idToTopic.size} memories)`);
|
|
682
|
+
console.log("[recall-sweep] running sweep...");
|
|
683
|
+
const t1 = Date.now();
|
|
684
|
+
const records = await runSweep(db, idToTopic, sessions);
|
|
685
|
+
console.log(`[recall-sweep] sweep done in ${Date.now() - t1}ms (${records.length} turn records)`);
|
|
686
|
+
const focused = summarizeFocused(records, sessions);
|
|
687
|
+
const shifts = summarizeShift(records);
|
|
688
|
+
const report = renderReport(focused, shifts, records, sessions, {
|
|
689
|
+
memoryCount: CORPUS.length,
|
|
690
|
+
});
|
|
691
|
+
(0, node_fs_1.mkdirSync)(reportDir, { recursive: true });
|
|
692
|
+
const reportPath = (0, node_path_1.join)(reportDir, "report.md");
|
|
693
|
+
(0, node_fs_1.writeFileSync)(reportPath, report, "utf-8");
|
|
694
|
+
console.log("\n" + report);
|
|
695
|
+
console.log(`[recall-sweep] report written to ${reportPath}`);
|
|
696
|
+
}
|
|
697
|
+
finally {
|
|
698
|
+
if (db) {
|
|
699
|
+
try {
|
|
700
|
+
db.close();
|
|
701
|
+
}
|
|
702
|
+
catch { /* already closed */ }
|
|
703
|
+
}
|
|
704
|
+
if ((0, node_fs_1.existsSync)(tmpDir)) {
|
|
705
|
+
try {
|
|
706
|
+
(0, node_fs_1.rmSync)(tmpDir, { recursive: true, force: true });
|
|
707
|
+
}
|
|
708
|
+
catch { /* non-fatal */ }
|
|
709
|
+
}
|
|
710
|
+
}
|
|
711
|
+
}
|
|
712
|
+
main().catch((err) => {
|
|
713
|
+
console.error("[recall-sweep] FAILED:", err instanceof Error ? err.stack : String(err));
|
|
714
|
+
process.exitCode = 1;
|
|
715
|
+
});
|
package/dist/index.js
CHANGED
|
@@ -378,7 +378,7 @@ exports.default = {
|
|
|
378
378
|
}), { name: "hicortex_search" });
|
|
379
379
|
api.registerTool((_ctx) => ({
|
|
380
380
|
name: "hicortex_get",
|
|
381
|
-
description: "Fetch ONE memory's full content by id — use this to lazy-load entries from the '## Memory recall (auto)' index or from search results whose snippet was not enough. Fetching a memory marks it as used (strengthens it), so
|
|
381
|
+
description: "Fetch ONE memory's full content by id — use this to lazy-load entries from the '## Memory recall (auto)' index or from search results whose snippet was not enough. Fetching a memory marks it as used (strengthens it), so fetch entries that could change your action — not every shown one. When the memory shapes your answer, cite it as given in the response.",
|
|
382
382
|
parameters: {
|
|
383
383
|
type: "object",
|
|
384
384
|
properties: {
|
package/dist/mcp-server.js
CHANGED
|
@@ -128,7 +128,7 @@ function createMcpServer() {
|
|
|
128
128
|
}
|
|
129
129
|
});
|
|
130
130
|
// -- hicortex_get --
|
|
131
|
-
server.tool("hicortex_get", "Fetch ONE memory's full content by id — use this to lazy-load entries from the '## Memory recall (auto)' index or from search results whose snippet was not enough. Fetching a memory marks it as used (strengthens it), so
|
|
131
|
+
server.tool("hicortex_get", "Fetch ONE memory's full content by id — use this to lazy-load entries from the '## Memory recall (auto)' index or from search results whose snippet was not enough. Fetching a memory marks it as used (strengthens it), so fetch entries that could change your action — not every shown one. When the memory shapes your answer, cite it to the user (id + date + origin agent).", {
|
|
132
132
|
id: zod_1.z.string().describe("Memory id (as shown in recall index/search results)"),
|
|
133
133
|
}, async ({ id }) => {
|
|
134
134
|
if (!db)
|
|
@@ -488,11 +488,14 @@ async function startServer(options = {}) {
|
|
|
488
488
|
retrieval.configureDecay({ halfLifeDays: savedConfig?.decayHalfLifeDays });
|
|
489
489
|
const recallCfg = retrieval.configureRecall(savedConfig);
|
|
490
490
|
const scoringCfg = retrieval.configureScoring(savedConfig);
|
|
491
|
+
const sessionIntentCfg = retrieval.configureSessionIntent(savedConfig);
|
|
491
492
|
console.log(`[hicortex] Recall: k=${recallCfg.searchLimit}/recent=${recallCfg.recentLimit}` +
|
|
492
493
|
`/window=${recallCfg.recentWindowDays}d/cold=${recallCfg.coldExposureSlots} · ` +
|
|
493
494
|
`score sim=${scoringCfg.similarity}/str=${scoringCfg.strength}/conn=${scoringCfg.connections}` +
|
|
494
495
|
`/rec=${scoringCfg.recency}, fresh=${scoringCfg.freshnessBoostWeight}@${scoringCfg.freshnessBoostDays}d, ` +
|
|
495
|
-
`superseded×${scoringCfg.supersededDemotion}`
|
|
496
|
+
`superseded×${scoringCfg.supersededDemotion}` +
|
|
497
|
+
`, intent w=${sessionIntentCfg.weight}` +
|
|
498
|
+
(sessionIntentCfg.weight === 0 ? " (disabled)" : ""));
|
|
496
499
|
recallRegistry = new recall_registry_js_1.SessionRecallRegistry({
|
|
497
500
|
reshowTurns: savedConfig?.recallReshowTurns,
|
|
498
501
|
});
|
|
@@ -668,12 +671,29 @@ async function startServer(options = {}) {
|
|
|
668
671
|
registry: recallRegistry,
|
|
669
672
|
// Client-pushed project/privacy scoping (F1) rides through to
|
|
670
673
|
// retrieval, which handles the filtered over-fetch itself.
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
674
|
+
// #192 session-intent keying (0.15.3): embed the prompt ONCE here,
|
|
675
|
+
// blend with the session's rolling centroid, and pass the blended
|
|
676
|
+
// vector to retrieve() via queryEmbedding so retrieve() does NOT
|
|
677
|
+
// re-embed. Turn 1 (no centroid yet) and weight=0 both reduce to a
|
|
678
|
+
// pure-prompt search (the kill-switch). The centroid is updated AFTER
|
|
679
|
+
// reading the prior one — so turn 1 searches with pure prompt, then
|
|
680
|
+
// seeds the centroid for turn 2+ to blend against.
|
|
681
|
+
retrieveFn: async (query, limit, filters, sessionId) => {
|
|
682
|
+
const { weight, alpha } = retrieval.getSessionIntent();
|
|
683
|
+
const promptEmb = await (0, embedder_js_1.embed)(query);
|
|
684
|
+
// weight=0 (kill-switch): the centroid is neither read nor written.
|
|
685
|
+
const centroid = weight > 0 ? recallRegistry.getCentroid(sessionId) : undefined;
|
|
686
|
+
const queryVec = retrieval.blendQueryVector(promptEmb, centroid, weight);
|
|
687
|
+
if (weight > 0)
|
|
688
|
+
recallRegistry.updateCentroid(sessionId, promptEmb, alpha);
|
|
689
|
+
return retrieval.retrieve(db, embedder_js_1.embed, query, {
|
|
690
|
+
limit,
|
|
691
|
+
noStrengthen: true,
|
|
692
|
+
project: filters?.project,
|
|
693
|
+
privacy: filters?.privacy,
|
|
694
|
+
queryEmbedding: queryVec,
|
|
695
|
+
});
|
|
696
|
+
},
|
|
677
697
|
options: recallIndexOptions,
|
|
678
698
|
}, req.body);
|
|
679
699
|
res.status(r.status).json(r.body);
|
|
@@ -31,7 +31,7 @@ exports.MEMORY_SECTION_NAME = "memory";
|
|
|
31
31
|
function renderMemoryInstructions() {
|
|
32
32
|
return [
|
|
33
33
|
"Your long-term memory is Hicortex — shared across all agents and sessions.",
|
|
34
|
-
"- A `## Memory recall (auto)` index may arrive with prompts: it is a MENU, not content. Fetch a full memory with `hicortex_get(id)`
|
|
34
|
+
"- A `## Memory recall (auto)` index may arrive with prompts: it is a MENU, not content. Fetch a full memory with `hicortex_get(id)` when the entry could change how you handle the current task.",
|
|
35
35
|
"- Recall before assuming: `hicortex_search` for prior decisions/facts/preferences, `hicortex_recent` to catch up on a project.",
|
|
36
36
|
"- Cite any memory you rely on (id, date); on conflicts, newer memories supersede older.",
|
|
37
37
|
"- Capture is automatic (nightly). Do not manually ingest routine content — `hicortex_ingest` is for explicitly requested learnings only.",
|
package/dist/recall-index.d.ts
CHANGED
|
@@ -52,7 +52,10 @@ export interface RecallFilters {
|
|
|
52
52
|
export interface RecallIndexDeps {
|
|
53
53
|
db: Database.Database;
|
|
54
54
|
registry: SessionRecallRegistry;
|
|
55
|
-
|
|
55
|
+
/** Search closure. `sessionId` is forwarded so the closure (in mcp-server)
|
|
56
|
+
* can resolve/update the session-intent centroid and pass a blended query
|
|
57
|
+
* vector into retrieve() — see #192 session-intent keying (0.15.3). */
|
|
58
|
+
retrieveFn: (query: string, limit: number, filters: RecallFilters | undefined, sessionId: string) => Promise<MemorySearchResult[]>;
|
|
56
59
|
options?: RecallIndexOptions;
|
|
57
60
|
}
|
|
58
61
|
/** Normalize a request-supplied privacy filter: array of strings or a CSV
|
package/dist/recall-index.js
CHANGED
|
@@ -142,7 +142,7 @@ async function handleRecallIndex(deps, body) {
|
|
|
142
142
|
};
|
|
143
143
|
let results;
|
|
144
144
|
try {
|
|
145
|
-
results = await deps.retrieveFn(prompt, maxItems * CANDIDATE_MULTIPLIER, filters);
|
|
145
|
+
results = await deps.retrieveFn(prompt, maxItems * CANDIDATE_MULTIPLIER, filters, sessionId);
|
|
146
146
|
}
|
|
147
147
|
catch (err) {
|
|
148
148
|
return {
|
|
@@ -170,7 +170,7 @@ async function handleRecallIndex(deps, body) {
|
|
|
170
170
|
// and cite-what-you-rely-on (covers snippet-only use, the common case per
|
|
171
171
|
// the 0.14.0 field test). The full citation format + origin agent ride on
|
|
172
172
|
// the hicortex_get response / GET /memory `citation` field (use-time).
|
|
173
|
-
"Possibly relevant memories — dates matter, newer supersedes older. Fetch full content with `hicortex_get(id)`
|
|
173
|
+
"Possibly relevant memories — dates matter, newer supersedes older. Fetch full content with `hicortex_get(id)` when an entry could change your action; cite any memory you rely on (id, date):",
|
|
174
174
|
...lines,
|
|
175
175
|
].join("\n");
|
|
176
176
|
return { status: 200, body: { block, shown: ids, turn } };
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* SessionRecallRegistry — per-session, TURN-based dedup for pushed recall
|
|
3
|
-
* (#192, POST /recall-index)
|
|
3
|
+
* (#192, POST /recall-index), and owner of the session-intent rolling
|
|
4
|
+
* centroid (#192 session-intent keying, 0.15.3).
|
|
4
5
|
*
|
|
5
6
|
* Why turn-based, not time-based: suppression must track the session's
|
|
6
7
|
* CONTEXT, not the wall clock. A multi-day CC session with a 1M window can
|
|
@@ -17,6 +18,19 @@
|
|
|
17
18
|
* SessionStart hook — which includes source=compact, i.e. after
|
|
18
19
|
* compaction the fresh context may legitimately re-receive everything).
|
|
19
20
|
*
|
|
21
|
+
* Session-intent centroid (0.15.3): a rolling EMA of the session's prompt
|
|
22
|
+
* embeddings lives on SessionState. The recall path blends the current prompt
|
|
23
|
+
* with this centroid before the vector search so recall follows the session's
|
|
24
|
+
* intent instead of being query-literal. reset() deletes the whole session
|
|
25
|
+
* entry, so the centroid is cleared for free on SessionStart/compact — the
|
|
26
|
+
* next recall re-seeds.
|
|
27
|
+
*
|
|
28
|
+
* Concurrency: the registry assumes ONE in-flight recall per session at a time
|
|
29
|
+
* (CC's UserPromptSubmit fires once per turn; Hermes/OC plugins call per-turn
|
|
30
|
+
* too). Two concurrent same-session calls could race updateCentroid and drop
|
|
31
|
+
* one EMA step — harmless (self-correcting on the next turn) and not worth a
|
|
32
|
+
* lock for a path that does not fire concurrently in any current harness.
|
|
33
|
+
*
|
|
20
34
|
* Purely in-memory: a server restart forgets shown-state, worst case a few
|
|
21
35
|
* early re-shows (~15 tokens each) — harmless by design. Sessions are pruned
|
|
22
36
|
* LRU beyond maxSessions so long-running servers don't accumulate state.
|
|
@@ -41,6 +55,30 @@ export declare class SessionRecallRegistry {
|
|
|
41
55
|
markShown(sessionId: string, memoryIds: string[]): void;
|
|
42
56
|
/** Forget a session's shown-set (SessionStart / compaction). */
|
|
43
57
|
reset(sessionId: string): void;
|
|
58
|
+
/**
|
|
59
|
+
* Current session-intent centroid, or undefined when no prompt has seeded it
|
|
60
|
+
* yet (first turn / after a reset). The recall path reads this BEFORE
|
|
61
|
+
* updateCentroid to decide whether to blend — a missing centroid means
|
|
62
|
+
* "first turn, pure prompt, no behavior change".
|
|
63
|
+
*/
|
|
64
|
+
getCentroid(sessionId: string): Float32Array | undefined;
|
|
65
|
+
/**
|
|
66
|
+
* Fold this turn's prompt embedding into the session-intent centroid via
|
|
67
|
+
* EMA: `centroid_new = l2Normalize((1-α)·centroid_old + α·prompt)`.
|
|
68
|
+
*
|
|
69
|
+
* First call (no centroid yet) SEEDS the centroid = l2Normalize(prompt) —
|
|
70
|
+
* this is the "after that first recall" step in the design: turn 1's search
|
|
71
|
+
* runs with pure prompt, then the centroid is seeded so turn 2+ can blend.
|
|
72
|
+
*
|
|
73
|
+
* `alpha` is the EMA rate (a shipped constant — retrieval.SESSION_INTENT_ALPHA,
|
|
74
|
+
* 0.4; NOT a config knob per the 0.15.3 scope). Callers (the recall closure)
|
|
75
|
+
* read it from retrieval.getSessionIntent(). We do not re-clamp here — the
|
|
76
|
+
* registry is a pure data owner, not a config interpreter.
|
|
77
|
+
*
|
|
78
|
+
* Returns the new centroid. The centroid lives on SessionState, so reset()
|
|
79
|
+
* (which deletes the session entry) clears it for free.
|
|
80
|
+
*/
|
|
81
|
+
updateCentroid(sessionId: string, promptEmbedding: Float32Array, alpha: number): Float32Array;
|
|
44
82
|
/** Number of tracked sessions (for /recall-index introspection + tests). */
|
|
45
83
|
size(): number;
|
|
46
84
|
private getOrCreate;
|
package/dist/recall-registry.js
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
/**
|
|
3
3
|
* SessionRecallRegistry — per-session, TURN-based dedup for pushed recall
|
|
4
|
-
* (#192, POST /recall-index)
|
|
4
|
+
* (#192, POST /recall-index), and owner of the session-intent rolling
|
|
5
|
+
* centroid (#192 session-intent keying, 0.15.3).
|
|
5
6
|
*
|
|
6
7
|
* Why turn-based, not time-based: suppression must track the session's
|
|
7
8
|
* CONTEXT, not the wall clock. A multi-day CC session with a 1M window can
|
|
@@ -18,12 +19,26 @@
|
|
|
18
19
|
* SessionStart hook — which includes source=compact, i.e. after
|
|
19
20
|
* compaction the fresh context may legitimately re-receive everything).
|
|
20
21
|
*
|
|
22
|
+
* Session-intent centroid (0.15.3): a rolling EMA of the session's prompt
|
|
23
|
+
* embeddings lives on SessionState. The recall path blends the current prompt
|
|
24
|
+
* with this centroid before the vector search so recall follows the session's
|
|
25
|
+
* intent instead of being query-literal. reset() deletes the whole session
|
|
26
|
+
* entry, so the centroid is cleared for free on SessionStart/compact — the
|
|
27
|
+
* next recall re-seeds.
|
|
28
|
+
*
|
|
29
|
+
* Concurrency: the registry assumes ONE in-flight recall per session at a time
|
|
30
|
+
* (CC's UserPromptSubmit fires once per turn; Hermes/OC plugins call per-turn
|
|
31
|
+
* too). Two concurrent same-session calls could race updateCentroid and drop
|
|
32
|
+
* one EMA step — harmless (self-correcting on the next turn) and not worth a
|
|
33
|
+
* lock for a path that does not fire concurrently in any current harness.
|
|
34
|
+
*
|
|
21
35
|
* Purely in-memory: a server restart forgets shown-state, worst case a few
|
|
22
36
|
* early re-shows (~15 tokens each) — harmless by design. Sessions are pruned
|
|
23
37
|
* LRU beyond maxSessions so long-running servers don't accumulate state.
|
|
24
38
|
*/
|
|
25
39
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
26
40
|
exports.SessionRecallRegistry = exports.DEFAULT_RESHOW_TURNS = void 0;
|
|
41
|
+
const schema_prototypes_js_1 = require("./schema-prototypes.js");
|
|
27
42
|
exports.DEFAULT_RESHOW_TURNS = 30;
|
|
28
43
|
const DEFAULT_MAX_SESSIONS = 500;
|
|
29
44
|
class SessionRecallRegistry {
|
|
@@ -68,6 +83,42 @@ class SessionRecallRegistry {
|
|
|
68
83
|
reset(sessionId) {
|
|
69
84
|
this.sessions.delete(sessionId);
|
|
70
85
|
}
|
|
86
|
+
/**
|
|
87
|
+
* Current session-intent centroid, or undefined when no prompt has seeded it
|
|
88
|
+
* yet (first turn / after a reset). The recall path reads this BEFORE
|
|
89
|
+
* updateCentroid to decide whether to blend — a missing centroid means
|
|
90
|
+
* "first turn, pure prompt, no behavior change".
|
|
91
|
+
*/
|
|
92
|
+
getCentroid(sessionId) {
|
|
93
|
+
return this.sessions.get(sessionId)?.centroid;
|
|
94
|
+
}
|
|
95
|
+
/**
|
|
96
|
+
* Fold this turn's prompt embedding into the session-intent centroid via
|
|
97
|
+
* EMA: `centroid_new = l2Normalize((1-α)·centroid_old + α·prompt)`.
|
|
98
|
+
*
|
|
99
|
+
* First call (no centroid yet) SEEDS the centroid = l2Normalize(prompt) —
|
|
100
|
+
* this is the "after that first recall" step in the design: turn 1's search
|
|
101
|
+
* runs with pure prompt, then the centroid is seeded so turn 2+ can blend.
|
|
102
|
+
*
|
|
103
|
+
* `alpha` is the EMA rate (a shipped constant — retrieval.SESSION_INTENT_ALPHA,
|
|
104
|
+
* 0.4; NOT a config knob per the 0.15.3 scope). Callers (the recall closure)
|
|
105
|
+
* read it from retrieval.getSessionIntent(). We do not re-clamp here — the
|
|
106
|
+
* registry is a pure data owner, not a config interpreter.
|
|
107
|
+
*
|
|
108
|
+
* Returns the new centroid. The centroid lives on SessionState, so reset()
|
|
109
|
+
* (which deletes the session entry) clears it for free.
|
|
110
|
+
*/
|
|
111
|
+
updateCentroid(sessionId, promptEmbedding, alpha) {
|
|
112
|
+
const s = this.getOrCreate(sessionId);
|
|
113
|
+
if (!s.centroid) {
|
|
114
|
+
s.centroid = (0, schema_prototypes_js_1.l2Normalize)(promptEmbedding);
|
|
115
|
+
}
|
|
116
|
+
else {
|
|
117
|
+
s.centroid = (0, schema_prototypes_js_1.l2Normalize)((0, schema_prototypes_js_1.weightedAdd)(s.centroid, 1 - alpha, promptEmbedding, alpha));
|
|
118
|
+
}
|
|
119
|
+
s.lastUsedAt = Date.now();
|
|
120
|
+
return s.centroid;
|
|
121
|
+
}
|
|
71
122
|
/** Number of tracked sessions (for /recall-index introspection + tests). */
|
|
72
123
|
size() {
|
|
73
124
|
return this.sessions.size;
|
package/dist/retrieval.d.ts
CHANGED
|
@@ -70,6 +70,35 @@ interface ScoringWeights {
|
|
|
70
70
|
export declare function configureScoring(config?: Record<string, unknown> | null): ScoringWeights;
|
|
71
71
|
/** Current resolved weights (tests + status output). */
|
|
72
72
|
export declare function getScoringWeights(): ScoringWeights;
|
|
73
|
+
/** EMA rate for the session-intent centroid: centroid_new = (1-α)·old + α·prompt. */
|
|
74
|
+
export declare const SESSION_INTENT_ALPHA = 0.4;
|
|
75
|
+
/**
|
|
76
|
+
* Configure session-intent keying from config. Called at server boot next to
|
|
77
|
+
* configureScoring (the nightly does no recall, so it does not need this).
|
|
78
|
+
* Reads only `sessionIntentWeight` ([0,1]; 0 = disabled). Invalid/out-of-range
|
|
79
|
+
* values keep the shipped default. Returns `{ weight, alpha }` — alpha is the
|
|
80
|
+
* fixed constant, surfaced so the recall closure passes it to the registry in
|
|
81
|
+
* one call.
|
|
82
|
+
*/
|
|
83
|
+
export declare function configureSessionIntent(config?: Record<string, unknown> | null): {
|
|
84
|
+
weight: number;
|
|
85
|
+
alpha: number;
|
|
86
|
+
};
|
|
87
|
+
/** Current resolved session-intent weight + the shipped alpha (closure + tests). */
|
|
88
|
+
export declare function getSessionIntent(): {
|
|
89
|
+
weight: number;
|
|
90
|
+
alpha: number;
|
|
91
|
+
};
|
|
92
|
+
/**
|
|
93
|
+
* Blend the prompt embedding with the session-intent centroid for the vector
|
|
94
|
+
* search: `query = l2Normalize((1-w)·prompt + w·centroid)`. Returns the prompt
|
|
95
|
+
* UNCHANGED when `centroid` is undefined (first turn — no behavior change) or
|
|
96
|
+
* `weight` is 0 (the kill-switch — pure prompt). Extracted from the
|
|
97
|
+
* /recall-index closure (mcp-server.ts) so the exact blend decision is
|
|
98
|
+
* unit-testable directly, locking the ternary against a refactor without a
|
|
99
|
+
* closure-integration harness.
|
|
100
|
+
*/
|
|
101
|
+
export declare function blendQueryVector(promptEmb: Float32Array, centroid: Float32Array | undefined, weight: number): Float32Array;
|
|
73
102
|
/**
|
|
74
103
|
* Ids among `candidateIds` that have been superseded by a later memory — i.e.
|
|
75
104
|
* they are the SOURCE of a `superseded_by` link (stageSupersession links
|
|
@@ -117,6 +146,13 @@ export declare function retrieve(db: Database.Database, embedFn: EmbedFn, query:
|
|
|
117
146
|
/** #192: skip access strengthening — for pushed recall (/recall-index),
|
|
118
147
|
* where appearing in results must not count as use. */
|
|
119
148
|
noStrengthen?: boolean;
|
|
149
|
+
/** #192 session-intent keying (0.15.3): a pre-computed query embedding
|
|
150
|
+
* (e.g. the session-centroid blend from the /recall-index closure). When
|
|
151
|
+
* provided, the internal embed() call is SKIPPED — the caller owns the
|
|
152
|
+
* one embed per recall. /search and other unblended callers omit this
|
|
153
|
+
* and get pure-prompt behavior (the query string is embedded here). The
|
|
154
|
+
* FTS path still uses the raw `query` text regardless. */
|
|
155
|
+
queryEmbedding?: Float32Array;
|
|
120
156
|
}): Promise<MemorySearchResult[]>;
|
|
121
157
|
/**
|
|
122
158
|
* Get recent context, optionally filtered by project and privacy.
|
package/dist/retrieval.js
CHANGED
|
@@ -52,12 +52,15 @@ var __importStar = (this && this.__importStar) || (function () {
|
|
|
52
52
|
};
|
|
53
53
|
})();
|
|
54
54
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
55
|
-
exports.DEFAULT_DECAY_HALF_LIFE_DAYS = void 0;
|
|
55
|
+
exports.SESSION_INTENT_ALPHA = exports.DEFAULT_DECAY_HALF_LIFE_DAYS = void 0;
|
|
56
56
|
exports.decayConstantForHalfLife = decayConstantForHalfLife;
|
|
57
57
|
exports.configureDecay = configureDecay;
|
|
58
58
|
exports.configureRecall = configureRecall;
|
|
59
59
|
exports.configureScoring = configureScoring;
|
|
60
60
|
exports.getScoringWeights = getScoringWeights;
|
|
61
|
+
exports.configureSessionIntent = configureSessionIntent;
|
|
62
|
+
exports.getSessionIntent = getSessionIntent;
|
|
63
|
+
exports.blendQueryVector = blendQueryVector;
|
|
61
64
|
exports.findSupersededIds = findSupersededIds;
|
|
62
65
|
exports.l2ToCosine = l2ToCosine;
|
|
63
66
|
exports.effectiveStrength = effectiveStrength;
|
|
@@ -65,6 +68,7 @@ exports.computeScore = computeScore;
|
|
|
65
68
|
exports.retrieve = retrieve;
|
|
66
69
|
exports.searchRecent = searchRecent;
|
|
67
70
|
const storage = __importStar(require("./storage.js"));
|
|
71
|
+
const schema_prototypes_js_1 = require("./schema-prototypes.js");
|
|
68
72
|
/** Default decay half-life (days) at importance 0.5. #192: was 0.0005/h
|
|
69
73
|
* (~115-day half-life at base 0.5) — aggressive enough to bury the long tail
|
|
70
74
|
* in ranking. Long-term remembering is the product; time preference stays,
|
|
@@ -152,6 +156,59 @@ function configureScoring(config) {
|
|
|
152
156
|
function getScoringWeights() {
|
|
153
157
|
return { ...scoringWeights };
|
|
154
158
|
}
|
|
159
|
+
// ---------------------------------------------------------------------------
|
|
160
|
+
// Session-intent keying (#192, 0.15.3). ONE config knob:
|
|
161
|
+
// sessionIntentWeight 0.33 blend weight of the rolling centroid in the
|
|
162
|
+
// search vector: query = (1-w)·prompt + w·centroid.
|
|
163
|
+
// 0 = DISABLED (pure prompt, the kill-switch —
|
|
164
|
+
// current behavior). Range [0, 1].
|
|
165
|
+
//
|
|
166
|
+
// The EMA rate α is a shipped constant (SESSION_INTENT_ALPHA, 0.4), not a
|
|
167
|
+
// second knob — owner directive 0.15.3: one knob is enough to tune/disable;
|
|
168
|
+
// exposing α was speculative generality.
|
|
169
|
+
//
|
|
170
|
+
// The centroid itself lives on SessionRecallRegistry; retrieval only needs to
|
|
171
|
+
// ACCEPT a pre-blended query vector (options.queryEmbedding) so the recall
|
|
172
|
+
// closure can do the one-embed-per-recall + blend without retrieve()
|
|
173
|
+
// re-embedding. /search and other unblended callers omit queryEmbedding and
|
|
174
|
+
// get pure-prompt behavior unchanged.
|
|
175
|
+
// ---------------------------------------------------------------------------
|
|
176
|
+
/** EMA rate for the session-intent centroid: centroid_new = (1-α)·old + α·prompt. */
|
|
177
|
+
exports.SESSION_INTENT_ALPHA = 0.4;
|
|
178
|
+
const SESSION_INTENT_DEFAULT_WEIGHT = 0.33;
|
|
179
|
+
let sessionIntentWeight = SESSION_INTENT_DEFAULT_WEIGHT;
|
|
180
|
+
/**
|
|
181
|
+
* Configure session-intent keying from config. Called at server boot next to
|
|
182
|
+
* configureScoring (the nightly does no recall, so it does not need this).
|
|
183
|
+
* Reads only `sessionIntentWeight` ([0,1]; 0 = disabled). Invalid/out-of-range
|
|
184
|
+
* values keep the shipped default. Returns `{ weight, alpha }` — alpha is the
|
|
185
|
+
* fixed constant, surfaced so the recall closure passes it to the registry in
|
|
186
|
+
* one call.
|
|
187
|
+
*/
|
|
188
|
+
function configureSessionIntent(config) {
|
|
189
|
+
const v = Number(config?.sessionIntentWeight);
|
|
190
|
+
sessionIntentWeight =
|
|
191
|
+
Number.isFinite(v) && v >= 0 && v <= 1 ? v : SESSION_INTENT_DEFAULT_WEIGHT;
|
|
192
|
+
return { weight: sessionIntentWeight, alpha: exports.SESSION_INTENT_ALPHA };
|
|
193
|
+
}
|
|
194
|
+
/** Current resolved session-intent weight + the shipped alpha (closure + tests). */
|
|
195
|
+
function getSessionIntent() {
|
|
196
|
+
return { weight: sessionIntentWeight, alpha: exports.SESSION_INTENT_ALPHA };
|
|
197
|
+
}
|
|
198
|
+
/**
|
|
199
|
+
* Blend the prompt embedding with the session-intent centroid for the vector
|
|
200
|
+
* search: `query = l2Normalize((1-w)·prompt + w·centroid)`. Returns the prompt
|
|
201
|
+
* UNCHANGED when `centroid` is undefined (first turn — no behavior change) or
|
|
202
|
+
* `weight` is 0 (the kill-switch — pure prompt). Extracted from the
|
|
203
|
+
* /recall-index closure (mcp-server.ts) so the exact blend decision is
|
|
204
|
+
* unit-testable directly, locking the ternary against a refactor without a
|
|
205
|
+
* closure-integration harness.
|
|
206
|
+
*/
|
|
207
|
+
function blendQueryVector(promptEmb, centroid, weight) {
|
|
208
|
+
return centroid && weight > 0
|
|
209
|
+
? (0, schema_prototypes_js_1.l2Normalize)((0, schema_prototypes_js_1.weightedAdd)(promptEmb, 1 - weight, centroid, weight))
|
|
210
|
+
: promptEmb;
|
|
211
|
+
}
|
|
155
212
|
/**
|
|
156
213
|
* Ids among `candidateIds` that have been superseded by a later memory — i.e.
|
|
157
214
|
* they are the SOURCE of a `superseded_by` link (stageSupersession links
|
|
@@ -370,8 +427,8 @@ async function retrieve(db, embedFn, query, options) {
|
|
|
370
427
|
const privacy = options?.privacy;
|
|
371
428
|
const sourceAgent = options?.sourceAgent;
|
|
372
429
|
const now = new Date();
|
|
373
|
-
// 1. Embed
|
|
374
|
-
const queryEmbedding = await embedFn(query);
|
|
430
|
+
// 1. Embed — or reuse the caller-provided vector (session-intent blend).
|
|
431
|
+
const queryEmbedding = options?.queryEmbedding ?? (await embedFn(query));
|
|
375
432
|
// 2. Dual retrieval — vector + BM25.
|
|
376
433
|
// #192: sqlite-vec can't push filters into the KNN, so filtered queries must
|
|
377
434
|
// over-fetch — the old flat limit*3 intersected a global top-15 with (for the
|
|
@@ -38,6 +38,21 @@ export declare function blobToVec(buf: Buffer): Float32Array;
|
|
|
38
38
|
* is returned as an all-zero copy rather than dividing by zero.
|
|
39
39
|
*/
|
|
40
40
|
export declare function l2Normalize(vec: Float32Array): Float32Array;
|
|
41
|
+
/**
|
|
42
|
+
* Weighted sum of two vectors into a NEW Float32Array (inputs untouched). The
|
|
43
|
+
* result is NOT renormalized — callers normalize explicitly via `l2Normalize`
|
|
44
|
+
* when they need a unit vector (both call sites below do, because embeddings
|
|
45
|
+
* are L2-normalized and the blend must stay on the unit sphere to keep cosine
|
|
46
|
+
* meaningful). Throws on a dimension mismatch rather than silently truncating:
|
|
47
|
+
* the embedding dim is fixed at 384 in practice, so a mismatch signals a
|
|
48
|
+
* mid-process model swap or a bug, which must surface (CLAUDE.md: fail
|
|
49
|
+
* explicitly), not get quietly papered over.
|
|
50
|
+
*
|
|
51
|
+
* Used by the session-intent centroid EMA and the recall query blend (#192
|
|
52
|
+
* session-intent keying): `weightedAdd(a, 1-α, b, α)` is the EMA step,
|
|
53
|
+
* `weightedAdd(prompt, 1-w, centroid, w)` is the blended search vector.
|
|
54
|
+
*/
|
|
55
|
+
export declare function weightedAdd(a: Float32Array, wA: number, b: Float32Array, wB: number): Float32Array;
|
|
41
56
|
/**
|
|
42
57
|
* Association weight of a memory for a tag = cosine(memory embedding, domain
|
|
43
58
|
* prototype). Both inputs are L2-normalized (embedder.ts normalizes memory
|
|
@@ -28,6 +28,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
28
28
|
exports.PROTOTYPE_MIN_MEMBERS = void 0;
|
|
29
29
|
exports.blobToVec = blobToVec;
|
|
30
30
|
exports.l2Normalize = l2Normalize;
|
|
31
|
+
exports.weightedAdd = weightedAdd;
|
|
31
32
|
exports.tagWeight = tagWeight;
|
|
32
33
|
exports.compartmentSet = compartmentSet;
|
|
33
34
|
exports.derivePrimary = derivePrimary;
|
|
@@ -70,6 +71,29 @@ function l2Normalize(vec) {
|
|
|
70
71
|
out[i] = vec[i] / norm;
|
|
71
72
|
return out;
|
|
72
73
|
}
|
|
74
|
+
/**
|
|
75
|
+
* Weighted sum of two vectors into a NEW Float32Array (inputs untouched). The
|
|
76
|
+
* result is NOT renormalized — callers normalize explicitly via `l2Normalize`
|
|
77
|
+
* when they need a unit vector (both call sites below do, because embeddings
|
|
78
|
+
* are L2-normalized and the blend must stay on the unit sphere to keep cosine
|
|
79
|
+
* meaningful). Throws on a dimension mismatch rather than silently truncating:
|
|
80
|
+
* the embedding dim is fixed at 384 in practice, so a mismatch signals a
|
|
81
|
+
* mid-process model swap or a bug, which must surface (CLAUDE.md: fail
|
|
82
|
+
* explicitly), not get quietly papered over.
|
|
83
|
+
*
|
|
84
|
+
* Used by the session-intent centroid EMA and the recall query blend (#192
|
|
85
|
+
* session-intent keying): `weightedAdd(a, 1-α, b, α)` is the EMA step,
|
|
86
|
+
* `weightedAdd(prompt, 1-w, centroid, w)` is the blended search vector.
|
|
87
|
+
*/
|
|
88
|
+
function weightedAdd(a, wA, b, wB) {
|
|
89
|
+
if (a.length !== b.length) {
|
|
90
|
+
throw new Error(`weightedAdd: dimension mismatch (${a.length} vs ${b.length}) — expected equal-length L2-normalized embeddings`);
|
|
91
|
+
}
|
|
92
|
+
const out = new Float32Array(a.length);
|
|
93
|
+
for (let i = 0; i < a.length; i++)
|
|
94
|
+
out[i] = a[i] * wA + b[i] * wB;
|
|
95
|
+
return out;
|
|
96
|
+
}
|
|
73
97
|
/**
|
|
74
98
|
* Association weight of a memory for a tag = cosine(memory embedding, domain
|
|
75
99
|
* prototype). Both inputs are L2-normalized (embedder.ts normalizes memory
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@gamaze/hicortex",
|
|
3
|
-
"version": "0.15.
|
|
3
|
+
"version": "0.15.3",
|
|
4
4
|
"description": "Self-learning memory for AI agents — experience captured automatically, distilled into lessons overnight, shared across your whole fleet. Works with Hermes, OpenClaw, Claude Code, and Pi.",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"bin": {
|
|
@@ -38,6 +38,7 @@
|
|
|
38
38
|
"test": "vitest run",
|
|
39
39
|
"test:watch": "vitest",
|
|
40
40
|
"eval": "node dist/eval/run-eval.js",
|
|
41
|
+
"eval:recall-sweep": "node dist/eval/recall-sweep.js",
|
|
41
42
|
"prepack": "npm run build && rm -rf ./hermes-plugin && mkdir -p ./hermes-plugin && cp -r ../../hermes-plugin/hicortex ./hermes-plugin/ && find ./hermes-plugin -name __pycache__ -type d -exec rm -rf {} + 2>/dev/null || true",
|
|
42
43
|
"prepublishOnly": "npm run build && rm -rf ./hermes-plugin && mkdir -p ./hermes-plugin && cp -r ../../hermes-plugin/hicortex ./hermes-plugin/ && find ./hermes-plugin -name __pycache__ -type d -exec rm -rf {} + 2>/dev/null || true"
|
|
43
44
|
},
|