@gamaze/hicortex 0.23.1 → 0.23.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/dashboard.html +25 -13
- package/dist/dashboard.d.ts +18 -7
- package/dist/dashboard.js +42 -10
- package/dist/dedup.js +2 -2
- package/dist/index.js +17 -2
- package/dist/learnings-identity.js +20 -1
- package/dist/mcp-server.js +6 -2
- package/dist/nightly.js +1 -1
- package/hermes-plugin/hicortex/provider.py +23 -0
- package/opencode-plugin/hicortex/index.ts +24 -1
- package/package.json +2 -1
- package/pi-extension/hicortex/index.ts +24 -1
- package/server.json +2 -2
- package/dist/eval/decay-eval.d.ts +0 -111
- package/dist/eval/decay-eval.js +0 -214
- package/dist/eval/dups.d.ts +0 -100
- package/dist/eval/dups.js +0 -174
- package/dist/eval/eval-clock.d.ts +0 -32
- package/dist/eval/eval-clock.js +0 -47
- package/dist/eval/eval-db.d.ts +0 -25
- package/dist/eval/eval-db.js +0 -67
- package/dist/eval/graph-eval.d.ts +0 -89
- package/dist/eval/graph-eval.js +0 -246
- package/dist/eval/importance-eval.d.ts +0 -85
- package/dist/eval/importance-eval.js +0 -286
- package/dist/eval/planted-eval.d.ts +0 -30
- package/dist/eval/planted-eval.js +0 -122
- package/dist/eval/planted-fixtures.d.ts +0 -107
- package/dist/eval/planted-fixtures.js +0 -283
- package/dist/eval/planted-harness.d.ts +0 -183
- package/dist/eval/planted-harness.js +0 -651
- package/dist/eval/ranking-battery.d.ts +0 -125
- package/dist/eval/ranking-battery.js +0 -289
- package/dist/eval/ranking-eval.d.ts +0 -61
- package/dist/eval/ranking-eval.js +0 -554
- package/dist/eval/ranking-fixtures.d.ts +0 -117
- package/dist/eval/ranking-fixtures.js +0 -485
- package/dist/eval/recall-sweep.d.ts +0 -87
- package/dist/eval/recall-sweep.js +0 -1030
- package/dist/eval/reflection-census.d.ts +0 -19
- package/dist/eval/reflection-census.js +0 -25
- package/dist/eval/relevance-eval.d.ts +0 -178
- package/dist/eval/relevance-eval.js +0 -2240
- package/dist/eval/run-eval.d.ts +0 -20
- package/dist/eval/run-eval.js +0 -299
|
@@ -1,485 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
/**
|
|
3
|
-
* Planted ranking fixtures for the search-trust calibration gate (#425).
|
|
4
|
-
*
|
|
5
|
-
* Two fixture cases, both shaped like the field failure that opened the
|
|
6
|
-
* issue (the owner's live console test, 2026-09-13): a user searches a
|
|
7
|
-
* DISTINCTIVE PROPER NOUN they know exists; the one memory carrying that
|
|
8
|
-
* token has the best similarity of all candidates AND the literal FTS hit,
|
|
9
|
-
* yet ranks behind high-strength but less-relevant memories that win on
|
|
10
|
-
* effective strength + graph connections.
|
|
11
|
-
*
|
|
12
|
-
* (a) sirnas_exact_match — the AC1 fixture: a low-strength, unlinked,
|
|
13
|
-
* never-accessed memory carrying an invented proper noun against
|
|
14
|
-
* base-≥0.9 rivals with links among themselves, access history, and
|
|
15
|
-
* older creation dates. The rivals share the query's DOMAIN
|
|
16
|
-
* vocabulary (harbours, docks, boats) but never the proper noun.
|
|
17
|
-
* Declared expectation: the proper-noun row ranks #1.
|
|
18
|
-
* (b) no_match_control — the AC2 fixture: a "mamma"-class query with no
|
|
19
|
-
* relevant match and NO FTS hits anywhere in the corpus. Whatever
|
|
20
|
-
* nearest-neighbor junk vector search returns must be byte-stable
|
|
21
|
-
* under the ranking work (no both-channel candidates exist, so the
|
|
22
|
-
* boost can never fire — the case pins that the no-regression
|
|
23
|
-
* guarantee is structural, not lucky).
|
|
24
|
-
* (c) linked_fallback — the #449 (items 1+3) connections fixture: a
|
|
25
|
-
* 16-link vs 20-link identical-content hub pair (the saturation tie)
|
|
26
|
-
* plus an unlinked higher-similarity row vs a 4-link lower-similarity
|
|
27
|
-
* row (the direction pin). Planted in its OWN throwaway DB — the 40
|
|
28
|
-
* link-neighbor rows would pollute the case-1/2 corpus.
|
|
29
|
-
*
|
|
30
|
-
* Honesty rules (the planted-pairs #393 A discipline):
|
|
31
|
-
* - The texts are synthetic and generic (this package publishes to npm —
|
|
32
|
-
* no real infrastructure, people, or fleet detail), but SHAPED like the
|
|
33
|
-
* field failure: the proper noun is invented (it must appear nowhere
|
|
34
|
-
* else — verified by an invariant test), the rivals carry the metadata
|
|
35
|
-
* the production pipeline itself writes (base strength, access history,
|
|
36
|
-
* link counts).
|
|
37
|
-
* - Similarities are MEASURED at run time with the real bge-small-en-v1.5
|
|
38
|
-
* embedder (ranking-eval.ts) — never asserted into existence. The
|
|
39
|
-
* declared expectation is about ORDER under the shipped ranking, and
|
|
40
|
-
* the report prints the measured similarity/score of every top row.
|
|
41
|
-
* - Strength metadata is written through the production helpers
|
|
42
|
-
* (insertMemory + updateMemory + addLink), not hand-rolled SQL, so the
|
|
43
|
-
* fixture exercises the same rows retrieval() will see.
|
|
44
|
-
*/
|
|
45
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
46
|
-
exports.LINKED_FALLBACK_FIXTURE = exports.REAL_SIRNAS_MEMORY_ID = exports.NO_MATCH_FIXTURE = exports.SIRNAS_FIXTURE = exports.RANKING_SOURCE_AGENT = exports.RANKING_PROJECT = void 0;
|
|
47
|
-
exports.checkFixtureInvariants = checkFixtureInvariants;
|
|
48
|
-
/** All planted rows share project + source_agent (metadata never refuses). */
|
|
49
|
-
exports.RANKING_PROJECT = "ranking-eval";
|
|
50
|
-
exports.RANKING_SOURCE_AGENT = "ranking-eval";
|
|
51
|
-
// ---------------------------------------------------------------------------
|
|
52
|
-
// Case 1 — sirnas_exact_match (AC1)
|
|
53
|
-
// ---------------------------------------------------------------------------
|
|
54
|
-
//
|
|
55
|
-
// The target mirrors the field case's SHAPE with invented names: a home
|
|
56
|
-
// harbour memory carrying one distinctive proper noun, born weak (base 0.35,
|
|
57
|
-
// the "routine-but-curated" band), never accessed, unlinked, a few months
|
|
58
|
-
// old. The rivals are what the decay model produces for early high-strength
|
|
59
|
-
// technical rows: base 0.90-0.95, mutual links, access history, created
|
|
60
|
-
// BEFORE the target. They share the harbour/boating domain vocabulary — so
|
|
61
|
-
// a bare proper-noun query lands in their neighborhood — but none carries
|
|
62
|
-
// the proper noun.
|
|
63
|
-
const SIRNAS_ROWS = [
|
|
64
|
-
// --- rivals: high-strength, connected, older, domain-adjacent ---
|
|
65
|
-
{
|
|
66
|
-
key: "rival_fees",
|
|
67
|
-
content: "Harbour fees and guest dock rules: pay at the machine by the fuel quay, electricity on the guest dock is metered, " +
|
|
68
|
-
"and visiting boats must clear their berth by ten in the morning during the summer season.",
|
|
69
|
-
createdAt: "2026-03-02T09:00:00.000Z",
|
|
70
|
-
memoryType: "knowledge",
|
|
71
|
-
baseStrength: 0.9,
|
|
72
|
-
accessCount: 4,
|
|
73
|
-
lastAccessedDaysAgo: 6,
|
|
74
|
-
linksTo: ["rival_channel", "rival_maintenance"],
|
|
75
|
-
},
|
|
76
|
-
{
|
|
77
|
-
key: "rival_channel",
|
|
78
|
-
content: "Sailing notes for the home bay: the channel is marked with cardinal buoys, the shallow spit near the outer dock " +
|
|
79
|
-
"dries at low water, and the ferry has right of way inside the harbour basin.",
|
|
80
|
-
createdAt: "2026-03-16T09:00:00.000Z",
|
|
81
|
-
memoryType: "knowledge",
|
|
82
|
-
baseStrength: 0.95,
|
|
83
|
-
accessCount: 6,
|
|
84
|
-
lastAccessedDaysAgo: 2,
|
|
85
|
-
linksTo: ["rival_fees", "rival_maintenance"],
|
|
86
|
-
},
|
|
87
|
-
{
|
|
88
|
-
key: "rival_maintenance",
|
|
89
|
-
content: "Boat maintenance log: scrub the hull and check the anodes before launch, service the winch every spring, and " +
|
|
90
|
-
"book the boatyard crane early — the lifting slot list fills up in May.",
|
|
91
|
-
createdAt: "2026-03-30T09:00:00.000Z",
|
|
92
|
-
memoryType: "experience",
|
|
93
|
-
baseStrength: 0.9,
|
|
94
|
-
accessCount: 3,
|
|
95
|
-
lastAccessedDaysAgo: 11,
|
|
96
|
-
linksTo: ["rival_fees"],
|
|
97
|
-
},
|
|
98
|
-
{
|
|
99
|
-
key: "rival_passage",
|
|
100
|
-
content: "Passage plan habits: file the float plan before leaving the dock, plot the tidal gate times, and always take the " +
|
|
101
|
-
"safe water mark side of the reef when the light is obscured.",
|
|
102
|
-
createdAt: "2026-04-13T09:00:00.000Z",
|
|
103
|
-
memoryType: "knowledge",
|
|
104
|
-
baseStrength: 0.9,
|
|
105
|
-
accessCount: 2,
|
|
106
|
-
lastAccessedDaysAgo: 20,
|
|
107
|
-
linksTo: ["rival_channel"],
|
|
108
|
-
},
|
|
109
|
-
// --- the target: the exact proper-noun match, born humble ---
|
|
110
|
-
{
|
|
111
|
-
key: "target_harbour",
|
|
112
|
-
content: "Home harbour: Vindarsvik (near Stensundet). The guest berth is east of the ferry pier; hold slack when the " +
|
|
113
|
-
"fishing boats unload.",
|
|
114
|
-
createdAt: "2026-05-25T09:00:00.000Z",
|
|
115
|
-
memoryType: "knowledge",
|
|
116
|
-
baseStrength: 0.35,
|
|
117
|
-
accessCount: 0,
|
|
118
|
-
lastAccessedDaysAgo: 118,
|
|
119
|
-
linksTo: [],
|
|
120
|
-
},
|
|
121
|
-
// --- neutral filler: keeps neighborhoods non-trivial, off-domain ---
|
|
122
|
-
{
|
|
123
|
-
key: "fill_recipe",
|
|
124
|
-
content: "Batch-cooked a tomato and butter sauce from the Rome cookbook: San Marzano tomatoes, one whole onion halved, and far more butter than seems reasonable.",
|
|
125
|
-
createdAt: "2026-04-01T09:00:00.000Z",
|
|
126
|
-
memoryType: "experience",
|
|
127
|
-
baseStrength: 0.5,
|
|
128
|
-
accessCount: 0,
|
|
129
|
-
lastAccessedDaysAgo: 160,
|
|
130
|
-
linksTo: [],
|
|
131
|
-
},
|
|
132
|
-
{
|
|
133
|
-
key: "fill_books",
|
|
134
|
-
content: "Finished the third novel in the frontier trilogy. The middle book dragged through the mining-town chapters, but the finale pays off when the surveyor reads her grandmother's letters.",
|
|
135
|
-
createdAt: "2026-04-20T09:00:00.000Z",
|
|
136
|
-
memoryType: "experience",
|
|
137
|
-
baseStrength: 0.5,
|
|
138
|
-
accessCount: 0,
|
|
139
|
-
lastAccessedDaysAgo: 140,
|
|
140
|
-
linksTo: [],
|
|
141
|
-
},
|
|
142
|
-
{
|
|
143
|
-
key: "fill_lang",
|
|
144
|
-
content: "Language study notes: the dative prepositions finally clicked after drilling them as a sung list; irregular verbs still need spaced repetition.",
|
|
145
|
-
createdAt: "2026-05-10T09:00:00.000Z",
|
|
146
|
-
memoryType: "experience",
|
|
147
|
-
baseStrength: 0.5,
|
|
148
|
-
accessCount: 0,
|
|
149
|
-
lastAccessedDaysAgo: 120,
|
|
150
|
-
linksTo: [],
|
|
151
|
-
},
|
|
152
|
-
{
|
|
153
|
-
key: "fill_garden",
|
|
154
|
-
content: "The raised beds need refreshing before spring: compost the spent tomato vines, rotate the legume row, and prune the apple espalier before the buds swell.",
|
|
155
|
-
createdAt: "2026-06-02T09:00:00.000Z",
|
|
156
|
-
memoryType: "experience",
|
|
157
|
-
baseStrength: 0.5,
|
|
158
|
-
accessCount: 0,
|
|
159
|
-
lastAccessedDaysAgo: 100,
|
|
160
|
-
linksTo: [],
|
|
161
|
-
},
|
|
162
|
-
];
|
|
163
|
-
exports.SIRNAS_FIXTURE = {
|
|
164
|
-
key: "sirnas_exact_match",
|
|
165
|
-
description: "AC1: a distinctive proper-noun query must rank the exact-match (both-channel) memory first, " +
|
|
166
|
-
"ahead of high-strength domain-adjacent rivals that win on strength + connections.",
|
|
167
|
-
rows: SIRNAS_ROWS,
|
|
168
|
-
query: "Vindarsvik",
|
|
169
|
-
distinctiveToken: "vindarsvik",
|
|
170
|
-
expectedTop1: "target_harbour",
|
|
171
|
-
// Undirected degrees (#449 invariant extension): the rivals' mutual-link
|
|
172
|
-
// cluster — fees/channel 4 each, maintenance 3, passage 1 (each linksTo
|
|
173
|
-
// entry is one link row; both directions of a mutual pair count).
|
|
174
|
-
expectedUndirectedDegree: {
|
|
175
|
-
rival_fees: 4,
|
|
176
|
-
rival_channel: 4,
|
|
177
|
-
rival_maintenance: 3,
|
|
178
|
-
rival_passage: 1,
|
|
179
|
-
target_harbour: 0,
|
|
180
|
-
fill_recipe: 0,
|
|
181
|
-
fill_books: 0,
|
|
182
|
-
fill_lang: 0,
|
|
183
|
-
fill_garden: 0,
|
|
184
|
-
},
|
|
185
|
-
};
|
|
186
|
-
// ---------------------------------------------------------------------------
|
|
187
|
-
// Case 2 — no_match_control (AC2)
|
|
188
|
-
// ---------------------------------------------------------------------------
|
|
189
|
-
//
|
|
190
|
-
// A "mamma"-class query: a common household word in the user's language that
|
|
191
|
-
// appears in NO corpus row — no FTS token, no semantic neighbor. Vector
|
|
192
|
-
// search still returns nearest-neighbor junk (the field's 0.53-similarity
|
|
193
|
-
// technical rows); the case pins the FULL returned list so any drift in
|
|
194
|
-
// how unrelated junk is ordered is caught, and asserts structurally that
|
|
195
|
-
// the ranking work cannot touch it (no both-channel candidates exist).
|
|
196
|
-
exports.NO_MATCH_FIXTURE = {
|
|
197
|
-
key: "no_match_control",
|
|
198
|
-
description: "AC2: a no-match query (no FTS hits anywhere) returns a byte-stable list — no both-channel " +
|
|
199
|
-
"candidates exist, so the boost is structurally inert.",
|
|
200
|
-
// Same corpus as case 1 (minus nothing): the query matches none of it.
|
|
201
|
-
rows: SIRNAS_ROWS,
|
|
202
|
-
query: "mamma",
|
|
203
|
-
distinctiveToken: "",
|
|
204
|
-
// No declared top-1 — the assertion is stability, not order (the photo
|
|
205
|
-
// records the list; the gate compares runs).
|
|
206
|
-
expectedTop1: "",
|
|
207
|
-
};
|
|
208
|
-
/** The battery runs against a real snapshot copy (ranking-eval.ts case 4). */
|
|
209
|
-
exports.REAL_SIRNAS_MEMORY_ID = "12980c9d-0eba-4963-95eb-0548bc2aa93e";
|
|
210
|
-
// ---------------------------------------------------------------------------
|
|
211
|
-
// Case 3 — linked_fallback (#449 items 1+3, PR E)
|
|
212
|
-
// ---------------------------------------------------------------------------
|
|
213
|
-
//
|
|
214
|
-
// The connections-term fixture for the log-saturation reshape. Two probes in
|
|
215
|
-
// ONE corpus (planted in its own throwaway DB by ranking-eval.ts — the 40
|
|
216
|
-
// link-neighbor rows stay out of the case-1/2 corpus):
|
|
217
|
-
//
|
|
218
|
-
// saturation pair — two hubs with IDENTICAL content strings and identical
|
|
219
|
-
// metadata (same baseStrength, accessCount, lastAccessedDaysAgo,
|
|
220
|
-
// createdAt, memoryType); hub A carries exactly 16 undirected links, hub B
|
|
221
|
-
// exactly 20, through disjoint off-domain neighbor rows. Declared
|
|
222
|
-
// expectation POST-change (Option A, K = 16): the two hubs' scores are
|
|
223
|
-
// byte-EQUAL — the term min(1, log1p(k)/log1p(16)) saturates at 1 for both.
|
|
224
|
-
// Under the pre-#449 linear term (connectionCount / maxConnections = k/20)
|
|
225
|
-
// they differ by 0.2 × 0.15 ≈ 0.03 composite — that red is the recorded
|
|
226
|
-
// before-picture; the harness does NOT make it pass.
|
|
227
|
-
//
|
|
228
|
-
// direction pair — an unlinked row carrying the query's distinctive proper
|
|
229
|
-
// noun (the highest measured similarity) vs a linked row (exactly 4 links)
|
|
230
|
-
// whose MEASURED similarity is engineered ~0.20 LOWER, with equal metadata
|
|
231
|
-
// otherwise (equal strength/dates). Declared expectation (Option A, owner
|
|
232
|
-
// decision 2026-09-17): the unlinked row ranks ABOVE the linked one — a
|
|
233
|
-
// 0.20 gap is worth 0.10 composite at the 0.50 similarity weight, more
|
|
234
|
-
// than the k=4 connections credit (0.0852 post-change). The gap must
|
|
235
|
-
// exceed 0.171 for the pin to be sound at k = 4; the wording is tuned
|
|
236
|
-
// until the real-embedder measurement (recorded in the photo/report)
|
|
237
|
-
// lands at ~0.20.
|
|
238
|
-
//
|
|
239
|
-
// Neighbors are DISTINCT filler rows, disjoint between the two hubs and from
|
|
240
|
-
// the direction pair's cluster, all off-domain from the query — they exist
|
|
241
|
-
// to carry the hub/dir-linked degrees and nothing else.
|
|
242
|
-
const HUB_CONTENT = "Community noticeboard archive: the autumn fair needs tombola volunteers, choir practice moves to " +
|
|
243
|
-
"Thursday evenings, and the book-swap shelf now carries the walking-map folder.";
|
|
244
|
-
/** Off-domain one-line filler rows (deterministic keys, uniform metadata). */
|
|
245
|
-
function fillerRows(prefix, contents) {
|
|
246
|
-
return contents.map((content, i) => ({
|
|
247
|
-
key: `${prefix}_n${i}`,
|
|
248
|
-
content,
|
|
249
|
-
createdAt: "2026-04-15T09:00:00.000Z",
|
|
250
|
-
memoryType: "experience",
|
|
251
|
-
baseStrength: 0.5,
|
|
252
|
-
accessCount: 0,
|
|
253
|
-
lastAccessedDaysAgo: 150,
|
|
254
|
-
linksTo: [],
|
|
255
|
-
}));
|
|
256
|
-
}
|
|
257
|
-
const HUB_A_NEIGHBOR_CONTENTS = [
|
|
258
|
-
"Slow-cooker beans: soak overnight, then four hours on low with a bay leaf and a piece of smoked pork.",
|
|
259
|
-
"The second detective novel in the series drags until the lighthouse keeper's ledger turns up.",
|
|
260
|
-
"German grammar drill: subordinate clauses shove the verb to the end, so read the whole sentence before translating.",
|
|
261
|
-
"Raised-bed rotation: legumes follow brassicas, and lime the bed the cabbage family just emptied.",
|
|
262
|
-
"October weather pattern: the first night frosts usually arrive right after the harvest moon.",
|
|
263
|
-
"Bike maintenance: true the wheel by tightening spokes a quarter turn at a time, listening for the ping.",
|
|
264
|
-
"Chess opening note: against the locked center, flank with pawns only after the pieces are developed.",
|
|
265
|
-
"The documentary about the alpine tunnels spent too long on the geologists' cafeteria.",
|
|
266
|
-
"Coffee grinding: a burr grind for the pour-over; the blade grinder is fine enough only for the French press.",
|
|
267
|
-
"Hiking the ridge trail: start before the thermals build, and carry an extra liter once the pass is in sight.",
|
|
268
|
-
"Piano practice: separate hands at half tempo until the fingering is automatic, then join them slowly.",
|
|
269
|
-
"Sewing patch pockets: interface the fabric, and topstitch the same distance from every edge.",
|
|
270
|
-
"Bird notes: the chiffchaff repeats its own name all day; the willow warbler slides down the scale.",
|
|
271
|
-
"Sourdough rhythm: feed the starter after work, shape in the morning, bake when the poke test springs back halfway.",
|
|
272
|
-
"Replacing the washing-machine hose: shut the valve, expect a cup of water, and check the jubilee clip.",
|
|
273
|
-
"Audiobook note: the narrator's accent makes every ship's name sound like a Hebridean island.",
|
|
274
|
-
];
|
|
275
|
-
const HUB_B_NEIGHBOR_CONTENTS = [
|
|
276
|
-
"Winter tyres: swap before the first ice, and store the summer set flat in the dark basement.",
|
|
277
|
-
"The boardgame group settled on tiles-and-traders for the long evenings; the card game stays in the drawer.",
|
|
278
|
-
"Compost thermometer: the pile is cooking properly when it reads hotter than a warm bath for a week.",
|
|
279
|
-
"Frost-proof terracotta: empty the pots, stack them dry, and the succulents overwinter on the south sill.",
|
|
280
|
-
"The photography club theme this month is doors; shoot in morning shade and bracket the exposures.",
|
|
281
|
-
"Robot vacuum note: keep the charging dock's floor clear, or it nudges the rug and gives up mid-run.",
|
|
282
|
-
"Knife sharpening: the ceramic rod for touch-ups, the whetstone when the tomato skin resists.",
|
|
283
|
-
"The lecture on Roman concrete held the room until the slide about seawater pozzolana.",
|
|
284
|
-
"Necklace repair: jump rings open sideways, never apart, or the circle never closes again.",
|
|
285
|
-
"Ferry timetable change: the winter crossing departs an hour earlier and stops running on Sunday evenings.",
|
|
286
|
-
"Ski waxing: glide wax for the thaw, grip wax for the freeze, and never both in the same zone.",
|
|
287
|
-
"The archive boxes are labelled by decade now; the negatives live in the fire cupboard.",
|
|
288
|
-
"Beekeeping note: heft the hive — if one side lifts easily, the winter stores are short.",
|
|
289
|
-
"Jigsaw strategy: edges first, then the sky, and give the lamp a clean bulb for the blue hours.",
|
|
290
|
-
"The museum's clock gallery is closed for restoration until the pendulum clock returns.",
|
|
291
|
-
"Preserving basil: pesto freezes flat in bags; the leaves kept in water go black within a week.",
|
|
292
|
-
"Draft excluder: the brush strip on the letterbox cut the hallway draught more than the door seal did.",
|
|
293
|
-
"The organ recital opened with a toccata nobody knew and closed with everyone humming it.",
|
|
294
|
-
"Train-set wiring: one feeder per rail section, or the far loop slows to a crawl.",
|
|
295
|
-
"The pottery class moved on to wheel work; my first cylinder collapsed into a bowl with charm.",
|
|
296
|
-
];
|
|
297
|
-
const DIR_NEIGHBOR_CONTENTS = [
|
|
298
|
-
"Vaccine appointment bookkeeping: the clinic sends both a text and a letter, and only the letter carries the time.",
|
|
299
|
-
"Reading glasses: the cheap pair from the station shop outlasted the optician's by two years.",
|
|
300
|
-
"The new debit card needs the reader for anything over the limit; the shop keeps the old machine as backup.",
|
|
301
|
-
"Laundering the down jacket: tennis balls in the dryer, low heat, and patience.",
|
|
302
|
-
];
|
|
303
|
-
const hubNeighbors = (prefix, contents) => contents.map((_, i) => `${prefix}_n${i}`);
|
|
304
|
-
const LINKED_FALLBACK_ROWS = [
|
|
305
|
-
// --- saturation pair: identical content + metadata, degrees 16 vs 20 ---
|
|
306
|
-
{
|
|
307
|
-
key: "hub_sat_16",
|
|
308
|
-
content: HUB_CONTENT,
|
|
309
|
-
createdAt: "2026-05-10T09:00:00.000Z",
|
|
310
|
-
memoryType: "knowledge",
|
|
311
|
-
baseStrength: 0.6,
|
|
312
|
-
accessCount: 0,
|
|
313
|
-
lastAccessedDaysAgo: 120,
|
|
314
|
-
linksTo: hubNeighbors("huba", HUB_A_NEIGHBOR_CONTENTS),
|
|
315
|
-
},
|
|
316
|
-
{
|
|
317
|
-
key: "hub_sat_20",
|
|
318
|
-
content: HUB_CONTENT,
|
|
319
|
-
createdAt: "2026-05-10T09:00:00.000Z",
|
|
320
|
-
memoryType: "knowledge",
|
|
321
|
-
baseStrength: 0.6,
|
|
322
|
-
accessCount: 0,
|
|
323
|
-
lastAccessedDaysAgo: 120,
|
|
324
|
-
linksTo: hubNeighbors("hubb", HUB_B_NEIGHBOR_CONTENTS),
|
|
325
|
-
},
|
|
326
|
-
// --- direction pair: equal metadata; similarity gap engineered ~0.20 ---
|
|
327
|
-
// MEASURED (bge-small-en-v1.5, 2026-09-17): unlinked 0.8233 vs linked
|
|
328
|
-
// 0.6022 — gap 0.2212. The linked row deliberately carries the plural
|
|
329
|
-
// "guests" (an FTS-DISTINCT token from the query's "guest" under the
|
|
330
|
-
// unicode61 tokenizer) so it stays single-channel: the pin isolates
|
|
331
|
-
// similarity vs connections credit, not the both-channel boost.
|
|
332
|
-
{
|
|
333
|
-
key: "dir_unlinked",
|
|
334
|
-
content: "Home anchorage: Anchorholme, east of the skerries. The guest mooring lies by the old timber jetty — " +
|
|
335
|
-
"arrive at slack water and mind the ferry wake inside the bay.",
|
|
336
|
-
createdAt: "2026-07-01T09:00:00.000Z",
|
|
337
|
-
memoryType: "knowledge",
|
|
338
|
-
baseStrength: 0.6,
|
|
339
|
-
accessCount: 0,
|
|
340
|
-
lastAccessedDaysAgo: 90,
|
|
341
|
-
linksTo: [],
|
|
342
|
-
},
|
|
343
|
-
{
|
|
344
|
-
key: "dir_linked",
|
|
345
|
-
content: "Anchoring for the night behind the skerries: lay out chain at least four times the depth in the crowded " +
|
|
346
|
-
"bay, take one of the buoys kept for guests if the harbour has room, and check the swing radius around " +
|
|
347
|
-
"the mooring field before dark.",
|
|
348
|
-
createdAt: "2026-07-01T09:00:00.000Z",
|
|
349
|
-
memoryType: "knowledge",
|
|
350
|
-
baseStrength: 0.6,
|
|
351
|
-
accessCount: 0,
|
|
352
|
-
lastAccessedDaysAgo: 90,
|
|
353
|
-
linksTo: ["dirn_n0", "dirn_n1", "dirn_n2", "dirn_n3"],
|
|
354
|
-
},
|
|
355
|
-
// --- link carriers: 40 disjoint off-domain filler rows (degrees all 1) ---
|
|
356
|
-
...fillerRows("huba", HUB_A_NEIGHBOR_CONTENTS),
|
|
357
|
-
...fillerRows("hubb", HUB_B_NEIGHBOR_CONTENTS),
|
|
358
|
-
...fillerRows("dirn", DIR_NEIGHBOR_CONTENTS),
|
|
359
|
-
];
|
|
360
|
-
/** Declared degrees: hubs 16/20, direction-linked 4, every neighbor 1. */
|
|
361
|
-
function linkedFallbackDegrees() {
|
|
362
|
-
const degrees = {
|
|
363
|
-
hub_sat_16: 16,
|
|
364
|
-
hub_sat_20: 20,
|
|
365
|
-
dir_linked: 4,
|
|
366
|
-
dir_unlinked: 0,
|
|
367
|
-
};
|
|
368
|
-
for (const prefix of ["huba", "hubb", "dirn"]) {
|
|
369
|
-
const count = prefix === "huba"
|
|
370
|
-
? HUB_A_NEIGHBOR_CONTENTS.length
|
|
371
|
-
: prefix === "hubb"
|
|
372
|
-
? HUB_B_NEIGHBOR_CONTENTS.length
|
|
373
|
-
: DIR_NEIGHBOR_CONTENTS.length;
|
|
374
|
-
for (let i = 0; i < count; i++)
|
|
375
|
-
degrees[`${prefix}_n${i}`] = 1;
|
|
376
|
-
}
|
|
377
|
-
return degrees;
|
|
378
|
-
}
|
|
379
|
-
exports.LINKED_FALLBACK_FIXTURE = {
|
|
380
|
-
key: "linked_fallback",
|
|
381
|
-
description: "#449 (items 1+3): the connections reshape — a 16-link and a 20-link identical-content hub " +
|
|
382
|
-
"must score byte-equal post-change (both saturate at K=16), and an unlinked row ~0.20 more " +
|
|
383
|
-
"similar must outrank a 4-link rival (similarity dominates the saturated connections credit).",
|
|
384
|
-
rows: LINKED_FALLBACK_ROWS,
|
|
385
|
-
query: "Anchorholme guest anchorage",
|
|
386
|
-
distinctiveToken: "anchorholme",
|
|
387
|
-
expectedTop1: "dir_unlinked",
|
|
388
|
-
expectedUndirectedDegree: linkedFallbackDegrees(),
|
|
389
|
-
linkedExpectations: {
|
|
390
|
-
saturationPair: { hubA: "hub_sat_16", hubB: "hub_sat_20" },
|
|
391
|
-
directionPair: { unlinked: "dir_unlinked", linked: "dir_linked" },
|
|
392
|
-
},
|
|
393
|
-
};
|
|
394
|
-
// ---------------------------------------------------------------------------
|
|
395
|
-
// Fixture invariants (checked by ranking-eval at run time AND pinned in
|
|
396
|
-
// tests/ranking-eval-tooling.test.ts — a corpus that silently drifts from
|
|
397
|
-
// its declared shape would fake a pass/fail)
|
|
398
|
-
// ---------------------------------------------------------------------------
|
|
399
|
-
/**
|
|
400
|
-
* Undirected degree per row key from the declared linksTo arrays (#449):
|
|
401
|
-
* a row's own linksTo entries PLUS the times other rows reference it — each
|
|
402
|
-
* linksTo entry is one link row, so a mutual pair counts twice, mirroring
|
|
403
|
-
* storage.getLinks(id, "both").length.
|
|
404
|
-
*/
|
|
405
|
-
function undirectedDegrees(rows) {
|
|
406
|
-
const degree = new Map(rows.map((r) => [r.key, 0]));
|
|
407
|
-
for (const r of rows) {
|
|
408
|
-
for (const to of r.linksTo) {
|
|
409
|
-
degree.set(r.key, (degree.get(r.key) ?? 0) + 1);
|
|
410
|
-
degree.set(to, (degree.get(to) ?? 0) + 1);
|
|
411
|
-
}
|
|
412
|
-
}
|
|
413
|
-
return degree;
|
|
414
|
-
}
|
|
415
|
-
function checkFixtureInvariants(f) {
|
|
416
|
-
const problems = [];
|
|
417
|
-
const keys = new Set(f.rows.map((r) => r.key));
|
|
418
|
-
if (keys.size !== f.rows.length)
|
|
419
|
-
problems.push("duplicate row keys");
|
|
420
|
-
for (const r of f.rows) {
|
|
421
|
-
for (const to of r.linksTo) {
|
|
422
|
-
if (!keys.has(to))
|
|
423
|
-
problems.push(`${r.key} links to unknown key ${to}`);
|
|
424
|
-
if (to === r.key)
|
|
425
|
-
problems.push(`${r.key} links to itself`);
|
|
426
|
-
}
|
|
427
|
-
}
|
|
428
|
-
if (f.expectedTop1 && !keys.has(f.expectedTop1)) {
|
|
429
|
-
problems.push(`expectedTop1 ${f.expectedTop1} is not a row key`);
|
|
430
|
-
}
|
|
431
|
-
if (f.linkedExpectations) {
|
|
432
|
-
const declared = [
|
|
433
|
-
["hubA", f.linkedExpectations.saturationPair.hubA],
|
|
434
|
-
["hubB", f.linkedExpectations.saturationPair.hubB],
|
|
435
|
-
["unlinked", f.linkedExpectations.directionPair.unlinked],
|
|
436
|
-
["linked", f.linkedExpectations.directionPair.linked],
|
|
437
|
-
];
|
|
438
|
-
for (const [role, key] of declared) {
|
|
439
|
-
if (!keys.has(key)) {
|
|
440
|
-
problems.push(`linkedExpectations ${role} ${key} is not a row key`);
|
|
441
|
-
}
|
|
442
|
-
}
|
|
443
|
-
const hubs = f.rows.filter((r) => r.key === f.linkedExpectations.saturationPair.hubA ||
|
|
444
|
-
r.key === f.linkedExpectations.saturationPair.hubB);
|
|
445
|
-
if (hubs.length === 2) {
|
|
446
|
-
const [a, b] = hubs;
|
|
447
|
-
const metaEqual = a.content === b.content &&
|
|
448
|
-
a.createdAt === b.createdAt &&
|
|
449
|
-
a.memoryType === b.memoryType &&
|
|
450
|
-
a.baseStrength === b.baseStrength &&
|
|
451
|
-
a.accessCount === b.accessCount &&
|
|
452
|
-
a.lastAccessedDaysAgo === b.lastAccessedDaysAgo;
|
|
453
|
-
if (!metaEqual) {
|
|
454
|
-
problems.push("saturation-pair hubs differ beyond their links (content/metadata must be identical)");
|
|
455
|
-
}
|
|
456
|
-
}
|
|
457
|
-
}
|
|
458
|
-
if (f.expectedUndirectedDegree) {
|
|
459
|
-
const degree = undirectedDegrees(f.rows);
|
|
460
|
-
for (const [key, expected] of Object.entries(f.expectedUndirectedDegree)) {
|
|
461
|
-
if (!keys.has(key)) {
|
|
462
|
-
problems.push(`expectedUndirectedDegree names unknown key ${key}`);
|
|
463
|
-
continue;
|
|
464
|
-
}
|
|
465
|
-
const actual = degree.get(key) ?? 0;
|
|
466
|
-
if (actual !== expected) {
|
|
467
|
-
problems.push(`${key} undirected degree is ${actual}, declared ${expected}`);
|
|
468
|
-
}
|
|
469
|
-
}
|
|
470
|
-
}
|
|
471
|
-
if (f.distinctiveToken) {
|
|
472
|
-
const token = f.distinctiveToken.toLowerCase();
|
|
473
|
-
const carriers = f.rows.filter((r) => r.content.toLowerCase().includes(token));
|
|
474
|
-
if (carriers.length !== 1) {
|
|
475
|
-
problems.push(`distinctive token "${f.distinctiveToken}" carried by ${carriers.length} rows (expected exactly 1)`);
|
|
476
|
-
}
|
|
477
|
-
else if (f.expectedTop1 && carriers[0].key !== f.expectedTop1) {
|
|
478
|
-
problems.push(`distinctive token carried by ${carriers[0].key}, not expectedTop1 ${f.expectedTop1}`);
|
|
479
|
-
}
|
|
480
|
-
if (!f.query.toLowerCase().includes(token)) {
|
|
481
|
-
problems.push(`query does not contain the distinctive token`);
|
|
482
|
-
}
|
|
483
|
-
}
|
|
484
|
-
return problems;
|
|
485
|
-
}
|
|
@@ -1,87 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env node
|
|
2
|
-
/**
|
|
3
|
-
* Recall@k sweep eval for session-intent keying (#192, commit 0867d6c).
|
|
4
|
-
*
|
|
5
|
-
* SHIP GATE: does blending the prompt with the per-session EMA centroid
|
|
6
|
-
* (retrieval.blendQueryVector) improve recall across rephrased turns, AND
|
|
7
|
-
* what does it COST when a session shifts topics mid-stream? Picks the
|
|
8
|
-
* default `sessionIntentWeight`.
|
|
9
|
-
*
|
|
10
|
-
* Two scenario families, same corpus:
|
|
11
|
-
*
|
|
12
|
-
* FOCUSED — 6 sessions (one per topic), 4 turns each of GENUINE REPHRASES
|
|
13
|
-
* of one intent. Tests the UPSIDE: does the centroid pull drifting
|
|
14
|
-
* rephrases back on-topic? (recall@5 / p@5 should rise with w.)
|
|
15
|
-
*
|
|
16
|
-
* SHIFT — 5 sessions, each 2 turns on topic A then 2 turns on topic B (a
|
|
17
|
-
* clean mid-session topic change). Tests the DOWNSIDE that focused-only
|
|
18
|
-
* can't see: a higher weight leaves the centroid partly A right after the
|
|
19
|
-
* shift, so the first turn(s) on B may LAG (lower recall@5 for B). The
|
|
20
|
-
* load-bearing read-out is post-shift recovery at turn-3 (first B turn)
|
|
21
|
-
* and turn-4, plus turns-to-recover.
|
|
22
|
-
*
|
|
23
|
-
* Method (both families):
|
|
24
|
-
* - Synthetic corpus of 30 memories across 6 well-separated topics (5 each),
|
|
25
|
-
* embedded with the REAL bge-small-en-v1.5 model (the production embedder).
|
|
26
|
-
* - For each weight w in {0.0, 0.2, 0.4, 0.5, 0.6, 0.8}: fresh
|
|
27
|
-
* SessionRecallRegistry per session; per turn, embed the prompt ONCE,
|
|
28
|
-
* blend with the live centroid, pass the blended vector to retrieve() via
|
|
29
|
-
* queryEmbedding, record recall@5 + p@5 for the turn's CURRENT topic,
|
|
30
|
-
* then fold the prompt into the centroid (EMA α=0.4).
|
|
31
|
-
*
|
|
32
|
-
* Fairness controls (load-bearing — the eval is useless if these slip):
|
|
33
|
-
* - `noStrengthen: true` on every retrieve(): the DB stays STATIC across all
|
|
34
|
-
* retrieve calls. Strengthening would mutate effective_strength/access_count
|
|
35
|
-
* between runs and contaminate cross-weight comparisons.
|
|
36
|
-
* - Uniform memory metadata: every memory has base_strength=0.5, created_at
|
|
37
|
-
* ≈ now, access_count=0, and NO links. So effective_strength, recency,
|
|
38
|
-
* connections, and the freshness boost are all uniform → the ONLY
|
|
39
|
-
* discriminator is vector cosine + RRF rank. That isolates the blend.
|
|
40
|
-
* - Cold-exposure slots are a no-op here (all candidates equally cold),
|
|
41
|
-
* so the top-k is the plain score order.
|
|
42
|
-
* - The centroid update is weight-INDEPENDENT (EMA of prompts at α=0.4), so
|
|
43
|
-
* the per-turn centroid is identical across all weights for a given
|
|
44
|
-
* session; only the blend differs. Embeddings are precomputed once.
|
|
45
|
-
*
|
|
46
|
-
* Metrics (per turn, for the turn's CURRENT topic):
|
|
47
|
-
* - recall@5 — did >=1 same-topic memory surface in the top-5? (coarse; the
|
|
48
|
-
* requested ship-gate metric — saturates on a small corpus.)
|
|
49
|
-
* - p@5 — on-topic count in the top-5 (0..5, finer).
|
|
50
|
-
* - recall@1 — was the SINGLE top result on-topic? (finest; focused only.)
|
|
51
|
-
*
|
|
52
|
-
* Honest by construction: it prints whatever the numbers are, including
|
|
53
|
-
* w=0 winning, the blend hurting a focused scenario, or high weights lagging
|
|
54
|
-
* badly post-shift. Report written to data/eval-recall-sweep/report.md and
|
|
55
|
-
* printed to stdout.
|
|
56
|
-
*
|
|
57
|
-
* ## Section 4 — SCOPE dimension (#203 soft project affinity)
|
|
58
|
-
*
|
|
59
|
-
* A THIRD family on a SEPARATE project-labeled corpus (the blend-sweep corpus
|
|
60
|
-
* above has no project labels). Goal: prove #203's soft project affinity drops
|
|
61
|
-
* cross-scope noise (the boat/battery case — a "hardware" query surfacing
|
|
62
|
-
* "marine" memories on a shared token like "battery") below the cap WITHOUT a
|
|
63
|
-
* hard filter, and WITHOUT losing same-scope recall.
|
|
64
|
-
*
|
|
65
|
-
* Corpus: 6 hardware + 6 marine memories. 3 marine memories are
|
|
66
|
-
* CONTAMINATION SEEDS (share a token with the hardware queries); 3 are
|
|
67
|
-
* marine-only fillers (control).
|
|
68
|
-
*
|
|
69
|
-
* Each hardware query runs TWICE on the same static DB:
|
|
70
|
-
* scope OFF — no `project` sent (byte-identical to pre-#203).
|
|
71
|
-
* scope ON — `project: "hardware"` (computeScore adds +scopeAffinity 0.15
|
|
72
|
-
* to hardware memories; marine gets 0; no hard filter).
|
|
73
|
-
*
|
|
74
|
-
* Metrics: contamination@5 (marine in top-5 / 5 — LOWER is better),
|
|
75
|
-
* recall@5 (gold hardware memory surfaced), sameScope@5 (hardware in top-5).
|
|
76
|
-
* Scope is orthogonal to the blend, so the blend weight is held at 0 here —
|
|
77
|
-
* `project` is the ONLY variable. Same invariants (noStrengthen, real
|
|
78
|
-
* embedder, uniform metadata, neverCalledEmbed self-check).
|
|
79
|
-
*
|
|
80
|
-
* Run: npm run eval:recall-sweep [--now <ISO>] (== node dist/eval/recall-sweep.js)
|
|
81
|
-
*
|
|
82
|
-
* `--now <ISO>` (#458) pins the eval clock: the planted corpora's created_at
|
|
83
|
-
* AND every retrieve() call score against the SAME instant, so before/after
|
|
84
|
-
* runs are wall-clock-independent. Default: the live clock (the pre-#458
|
|
85
|
-
* behavior); the report header records which clock produced it.
|
|
86
|
-
*/
|
|
87
|
-
export {};
|