agent-recall-core 3.4.48 → 3.4.49
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +25 -0
- package/dist/digest/store.d.ts +15 -6
- package/dist/digest/store.d.ts.map +1 -1
- package/dist/digest/store.js +33 -20
- package/dist/digest/store.js.map +1 -1
- package/dist/embeddings/chunker.d.ts +102 -0
- package/dist/embeddings/chunker.d.ts.map +1 -0
- package/dist/embeddings/chunker.js +260 -0
- package/dist/embeddings/chunker.js.map +1 -0
- package/dist/embeddings/config.d.ts +93 -0
- package/dist/embeddings/config.d.ts.map +1 -0
- package/dist/embeddings/config.js +174 -0
- package/dist/embeddings/config.js.map +1 -0
- package/dist/embeddings/index-store.d.ts +61 -0
- package/dist/embeddings/index-store.d.ts.map +1 -0
- package/dist/embeddings/index-store.js +158 -0
- package/dist/embeddings/index-store.js.map +1 -0
- package/dist/embeddings/indexer.d.ts +75 -0
- package/dist/embeddings/indexer.d.ts.map +1 -0
- package/dist/embeddings/indexer.js +196 -0
- package/dist/embeddings/indexer.js.map +1 -0
- package/dist/embeddings/runtime.d.ts +66 -0
- package/dist/embeddings/runtime.d.ts.map +1 -0
- package/dist/embeddings/runtime.js +197 -0
- package/dist/embeddings/runtime.js.map +1 -0
- package/dist/helpers/associative-link.d.ts.map +1 -1
- package/dist/helpers/associative-link.js +26 -1
- package/dist/helpers/associative-link.js.map +1 -1
- package/dist/helpers/tokenize.d.ts +1 -0
- package/dist/helpers/tokenize.d.ts.map +1 -1
- package/dist/helpers/tokenize.js +18 -0
- package/dist/helpers/tokenize.js.map +1 -1
- package/dist/index.d.ts +21 -5
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +38 -4
- package/dist/index.js.map +1 -1
- package/dist/palace/awareness.d.ts +34 -10
- package/dist/palace/awareness.d.ts.map +1 -1
- package/dist/palace/awareness.js +249 -180
- package/dist/palace/awareness.js.map +1 -1
- package/dist/palace/consolidate.d.ts +1 -1
- package/dist/palace/consolidate.d.ts.map +1 -1
- package/dist/palace/consolidate.js +9 -9
- package/dist/palace/consolidate.js.map +1 -1
- package/dist/palace/decay-pass.d.ts +1 -1
- package/dist/palace/decay-pass.d.ts.map +1 -1
- package/dist/palace/decay-pass.js +2 -2
- package/dist/palace/decay-pass.js.map +1 -1
- package/dist/palace/fan-out.d.ts +1 -1
- package/dist/palace/fan-out.d.ts.map +1 -1
- package/dist/palace/fan-out.js +5 -5
- package/dist/palace/fan-out.js.map +1 -1
- package/dist/palace/index-manager.d.ts +1 -1
- package/dist/palace/index-manager.d.ts.map +1 -1
- package/dist/palace/index-manager.js +1 -1
- package/dist/palace/index-manager.js.map +1 -1
- package/dist/palace/insights-index.d.ts +19 -6
- package/dist/palace/insights-index.d.ts.map +1 -1
- package/dist/palace/insights-index.js +36 -13
- package/dist/palace/insights-index.js.map +1 -1
- package/dist/palace/keystone.d.ts +1 -1
- package/dist/palace/keystone.d.ts.map +1 -1
- package/dist/palace/keystone.js +2 -2
- package/dist/palace/keystone.js.map +1 -1
- package/dist/palace/rooms.d.ts +3 -3
- package/dist/palace/rooms.d.ts.map +1 -1
- package/dist/palace/rooms.js +34 -6
- package/dist/palace/rooms.js.map +1 -1
- package/dist/retrieval/candidates.d.ts.map +1 -1
- package/dist/retrieval/candidates.js +17 -2
- package/dist/retrieval/candidates.js.map +1 -1
- package/dist/retrieval/query-memory.d.ts +65 -7
- package/dist/retrieval/query-memory.d.ts.map +1 -1
- package/dist/retrieval/query-memory.js +518 -78
- package/dist/retrieval/query-memory.js.map +1 -1
- package/dist/retrieval/semantic-leg.d.ts +74 -0
- package/dist/retrieval/semantic-leg.d.ts.map +1 -0
- package/dist/retrieval/semantic-leg.js +180 -0
- package/dist/retrieval/semantic-leg.js.map +1 -0
- package/dist/storage/archive-write.d.ts.map +1 -1
- package/dist/storage/archive-write.js +29 -8
- package/dist/storage/archive-write.js.map +1 -1
- package/dist/storage/capture-router.d.ts +1 -1
- package/dist/storage/capture-router.d.ts.map +1 -1
- package/dist/storage/capture-router.js +26 -9
- package/dist/storage/capture-router.js.map +1 -1
- package/dist/storage/consolidation-queue.d.ts +11 -2
- package/dist/storage/consolidation-queue.d.ts.map +1 -1
- package/dist/storage/consolidation-queue.js +45 -16
- package/dist/storage/consolidation-queue.js.map +1 -1
- package/dist/storage/corrections.d.ts +53 -5
- package/dist/storage/corrections.d.ts.map +1 -1
- package/dist/storage/corrections.js +412 -124
- package/dist/storage/corrections.js.map +1 -1
- package/dist/storage/durable-intent.d.ts.map +1 -1
- package/dist/storage/durable-intent.js +24 -1
- package/dist/storage/durable-intent.js.map +1 -1
- package/dist/storage/filelock.d.ts +84 -12
- package/dist/storage/filelock.d.ts.map +1 -1
- package/dist/storage/filelock.js +309 -47
- package/dist/storage/filelock.js.map +1 -1
- package/dist/storage/fs-utils.d.ts +20 -0
- package/dist/storage/fs-utils.d.ts.map +1 -1
- package/dist/storage/fs-utils.js +27 -0
- package/dist/storage/fs-utils.js.map +1 -1
- package/dist/storage/paths.d.ts +34 -0
- package/dist/storage/paths.d.ts.map +1 -1
- package/dist/storage/paths.js +71 -0
- package/dist/storage/paths.js.map +1 -1
- package/dist/storage/pending.d.ts +154 -0
- package/dist/storage/pending.d.ts.map +1 -0
- package/dist/storage/pending.js +454 -0
- package/dist/storage/pending.js.map +1 -0
- package/dist/storage/project.d.ts.map +1 -1
- package/dist/storage/project.js +68 -16
- package/dist/storage/project.js.map +1 -1
- package/dist/storage/session-card.d.ts +13 -0
- package/dist/storage/session-card.d.ts.map +1 -1
- package/dist/storage/session-card.js +34 -3
- package/dist/storage/session-card.js.map +1 -1
- package/dist/storage/unclaimed.d.ts +165 -0
- package/dist/storage/unclaimed.d.ts.map +1 -0
- package/dist/storage/unclaimed.js +442 -0
- package/dist/storage/unclaimed.js.map +1 -0
- package/dist/storage/working-memory.d.ts.map +1 -1
- package/dist/storage/working-memory.js +19 -1
- package/dist/storage/working-memory.js.map +1 -1
- package/dist/supabase/recall-backend.d.ts +28 -2
- package/dist/supabase/recall-backend.d.ts.map +1 -1
- package/dist/supabase/recall-backend.js +80 -23
- package/dist/supabase/recall-backend.js.map +1 -1
- package/dist/supabase/sync.d.ts.map +1 -1
- package/dist/supabase/sync.js +13 -0
- package/dist/supabase/sync.js.map +1 -1
- package/dist/tools-logic/alignment-check.js +1 -1
- package/dist/tools-logic/alignment-check.js.map +1 -1
- package/dist/tools-logic/awareness-update.d.ts.map +1 -1
- package/dist/tools-logic/awareness-update.js +25 -16
- package/dist/tools-logic/awareness-update.js.map +1 -1
- package/dist/tools-logic/check-action.js +1 -1
- package/dist/tools-logic/check-action.js.map +1 -1
- package/dist/tools-logic/check.d.ts +55 -1
- package/dist/tools-logic/check.d.ts.map +1 -1
- package/dist/tools-logic/check.js +230 -57
- package/dist/tools-logic/check.js.map +1 -1
- package/dist/tools-logic/confidence.d.ts +11 -6
- package/dist/tools-logic/confidence.d.ts.map +1 -1
- package/dist/tools-logic/confidence.js +11 -6
- package/dist/tools-logic/confidence.js.map +1 -1
- package/dist/tools-logic/context-synthesize.js +2 -2
- package/dist/tools-logic/context-synthesize.js.map +1 -1
- package/dist/tools-logic/digest-read.js +2 -2
- package/dist/tools-logic/digest-read.js.map +1 -1
- package/dist/tools-logic/digest-recall.js +1 -1
- package/dist/tools-logic/digest-recall.js.map +1 -1
- package/dist/tools-logic/insight-promotion.d.ts +1 -1
- package/dist/tools-logic/insight-promotion.d.ts.map +1 -1
- package/dist/tools-logic/insight-promotion.js +17 -6
- package/dist/tools-logic/insight-promotion.js.map +1 -1
- package/dist/tools-logic/journal-capture.js +3 -3
- package/dist/tools-logic/journal-capture.js.map +1 -1
- package/dist/tools-logic/journal-write.js +3 -3
- package/dist/tools-logic/journal-write.js.map +1 -1
- package/dist/tools-logic/knowledge-write.js +2 -2
- package/dist/tools-logic/knowledge-write.js.map +1 -1
- package/dist/tools-logic/palace-lint.js +1 -1
- package/dist/tools-logic/palace-lint.js.map +1 -1
- package/dist/tools-logic/palace-read.js +1 -1
- package/dist/tools-logic/palace-read.js.map +1 -1
- package/dist/tools-logic/palace-search.js +1 -1
- package/dist/tools-logic/palace-search.js.map +1 -1
- package/dist/tools-logic/palace-walk.js +3 -3
- package/dist/tools-logic/palace-walk.js.map +1 -1
- package/dist/tools-logic/palace-write.js +4 -4
- package/dist/tools-logic/palace-write.js.map +1 -1
- package/dist/tools-logic/predict-correction.js +1 -1
- package/dist/tools-logic/predict-correction.js.map +1 -1
- package/dist/tools-logic/recall-backend.d.ts +11 -2
- package/dist/tools-logic/recall-backend.d.ts.map +1 -1
- package/dist/tools-logic/recall-backend.js +2 -2
- package/dist/tools-logic/recall-backend.js.map +1 -1
- package/dist/tools-logic/resurrect.d.ts.map +1 -1
- package/dist/tools-logic/resurrect.js +9 -1
- package/dist/tools-logic/resurrect.js.map +1 -1
- package/dist/tools-logic/safety-consolidation.d.ts.map +1 -1
- package/dist/tools-logic/safety-consolidation.js +32 -30
- package/dist/tools-logic/safety-consolidation.js.map +1 -1
- package/dist/tools-logic/session-end.d.ts +8 -0
- package/dist/tools-logic/session-end.d.ts.map +1 -1
- package/dist/tools-logic/session-end.js +51 -13
- package/dist/tools-logic/session-end.js.map +1 -1
- package/dist/tools-logic/session-start.d.ts +23 -0
- package/dist/tools-logic/session-start.d.ts.map +1 -1
- package/dist/tools-logic/session-start.js +44 -1
- package/dist/tools-logic/session-start.js.map +1 -1
- package/dist/tools-logic/smart-recall.d.ts +185 -10
- package/dist/tools-logic/smart-recall.d.ts.map +1 -1
- package/dist/tools-logic/smart-recall.js +363 -85
- package/dist/tools-logic/smart-recall.js.map +1 -1
- package/dist/tools-logic/store-doctor.d.ts.map +1 -1
- package/dist/tools-logic/store-doctor.js +7 -4
- package/dist/tools-logic/store-doctor.js.map +1 -1
- package/dist/tools-logic/store-repair.js +1 -1
- package/dist/tools-logic/store-repair.js.map +1 -1
- package/dist/tools-logic/supersession.d.ts +1 -1
- package/dist/tools-logic/supersession.d.ts.map +1 -1
- package/dist/tools-logic/supersession.js +2 -2
- package/dist/tools-logic/supersession.js.map +1 -1
- package/dist/types.d.ts +1 -1
- package/dist/types.js +1 -1
- package/dist/vector/embedding.d.ts +8 -0
- package/dist/vector/embedding.d.ts.map +1 -1
- package/dist/vector/embedding.js +8 -0
- package/dist/vector/embedding.js.map +1 -1
- package/package.json +1 -1
|
@@ -34,9 +34,12 @@
|
|
|
34
34
|
* change on day one." The five formulas smart-recall.ts documents (RRF k=60,
|
|
35
35
|
* Ebbinghaus, hot-window recency, betaUtility feedback, two-stage fusion) are
|
|
36
36
|
* split by WHERE they belong in a genuinely reusable pipeline, not merged:
|
|
37
|
-
* - RRF (two-stage: applyRRF + fuseCanonical)
|
|
38
|
-
*
|
|
39
|
-
*
|
|
37
|
+
* - RRF (two-stage: applyRRF + fuseCanonical): pipeline-owned (RANK/FUSE
|
|
38
|
+
* stage below) — generic to any multi-tier candidate set, not
|
|
39
|
+
* smart_recall-specific. The hot-window recency boost that used to run
|
|
40
|
+
* here was REMOVED from default ranking by fix4b (2026-09-12) — it
|
|
41
|
+
* survives only behind the explicit `freshnessBias` opt-in; see
|
|
42
|
+
* applyLegacyHotWindowBoost's doc block for the measurements.
|
|
40
43
|
* - Ebbinghaus decay, palace salience/IDF blend, insight confirmation blend:
|
|
41
44
|
* pipeline-owned (TOKENIZE+SCORE stage, per-tier) — these ARE the
|
|
42
45
|
* per-tier strategies the review names.
|
|
@@ -171,7 +174,10 @@
|
|
|
171
174
|
* grammar to find, within one tier's own already-scored candidate list,
|
|
172
175
|
* pairs that assert the same fact-key with a different semver value. A
|
|
173
176
|
* candidate detected as the STALE side of such a pair is DOWN-RANKED
|
|
174
|
-
* (multiplicative score penalty
|
|
177
|
+
* (multiplicative score penalty; historically mirrored `applyHotWindowBoost`'s
|
|
178
|
+
* pattern — fix4b, 2026-09-12, removed that boost from default ranking
|
|
179
|
+
* [legacy opt-in only, see applyLegacyHotWindowBoost]; the contradiction
|
|
180
|
+
* penalty here is unchanged)
|
|
175
181
|
* and ANNOTATED (`supersededBy`) — it is NEVER removed from the set. This
|
|
176
182
|
* closes part of the reconciliation gap `reports/2026-08-18-eval-redteam.md`'s
|
|
177
183
|
* HIGH-2 finding named (0% of grammar-detectable contradictions were
|
|
@@ -239,6 +245,7 @@ import { CONFIDENCE_FLOOR } from "../tools-logic/confidence.js";
|
|
|
239
245
|
import { scrubForCloud, fenceMemory } from "../storage/content-guard.js";
|
|
240
246
|
import { isRescueSourcedContent, extractFrontmatterSource } from "../helpers/journal-filter.js";
|
|
241
247
|
import { getLegacyRoot } from "../types.js";
|
|
248
|
+
import { embeddingsEnabled } from "../embeddings/config.js";
|
|
242
249
|
import * as fs from "node:fs";
|
|
243
250
|
import * as path from "node:path";
|
|
244
251
|
// ---------------------------------------------------------------------------
|
|
@@ -272,6 +279,112 @@ function keywordExactness(query, text) {
|
|
|
272
279
|
const matches = expandedQuery.filter((w) => textSet.has(w) || textLower.includes(w));
|
|
273
280
|
return Math.min(1.0, matches.length / rawWords.length);
|
|
274
281
|
}
|
|
282
|
+
// ---------------------------------------------------------------------------
|
|
283
|
+
// fix4 S5 (2026-09-11, reports/agentrecall-fix4-retrieval-2026-09-11.md):
|
|
284
|
+
// BM25-lite IDF, pure local, computed IN-PASS over the already-scanned
|
|
285
|
+
// candidate corpus. Before this, `keywordExactness` weighted every query
|
|
286
|
+
// token equally, so ubiquitous tokens ("version", "push") matched everything
|
|
287
|
+
// and drowned the rare discriminative ones ("semver") — the S2-standard
|
|
288
|
+
// eval's no-IDF finding. No persistence, no new dependency: DF is counted
|
|
289
|
+
// over the texts the tier scorer has ALREADY read for this query.
|
|
290
|
+
// ---------------------------------------------------------------------------
|
|
291
|
+
/** The BM25 IDF curve — the one shared formula every tier's IDF pass uses.
|
|
292
|
+
* Always > 0 (the +0.5 smoothing keeps df == n above zero), monotonically
|
|
293
|
+
* decreasing in df. */
|
|
294
|
+
function bm25Idf(df, n) {
|
|
295
|
+
return Math.log(1 + (n - df + 0.5) / (df + 0.5));
|
|
296
|
+
}
|
|
297
|
+
/**
|
|
298
|
+
* Build the per-query IDF model from the scanned corpus. Weights are per
|
|
299
|
+
* RAW query word (a word matches if ANY of its stem/synonym variants
|
|
300
|
+
* matches — same containment semantics as `keywordExactness`), BM25-curved
|
|
301
|
+
* over document frequency, then max-normalized to [0, 1] so the weighted
|
|
302
|
+
* exactness stays on the same scale the unweighted ratio lived on.
|
|
303
|
+
*
|
|
304
|
+
* TWO deliberate semantic differences from `keywordExactness`, both
|
|
305
|
+
* per-word SATURATION (independent review M3, 2026-09-11 — this comment's
|
|
306
|
+
* first cut overclaimed "degenerate cases reproduce pre-IDF behavior
|
|
307
|
+
* exactly", which is only true when additionally no word matches via more
|
|
308
|
+
* than one variant and no raw word repeats):
|
|
309
|
+
* 1. A raw word contributes its weight AT MOST ONCE, however many of its
|
|
310
|
+
* stem/synonym variants match ("push" matching both "deploy" and
|
|
311
|
+
* "release" in one text counted twice pre-IDF, capped at min(1, ·)).
|
|
312
|
+
* 2. Duplicate raw query words collapse to one weighted entry
|
|
313
|
+
* (`variantsByRaw` is keyed by word) instead of inflating the
|
|
314
|
+
* denominator.
|
|
315
|
+
* Both are the BM25 saturation stance — evidence for the same query word
|
|
316
|
+
* repeats, it doesn't multiply — pinned by this tranche's mechanism tests.
|
|
317
|
+
*
|
|
318
|
+
* DF and the match predicate use the SAME containment semantics
|
|
319
|
+
* (independent review M4, same day): each corpus doc is tokenized+stemmed
|
|
320
|
+
* once into a Set, and a word counts toward df if any variant hits the
|
|
321
|
+
* doc's stemmed-token set OR its lowercase text — identical to
|
|
322
|
+
* `weightedExactness`'s own predicate below. (The first cut counted df by
|
|
323
|
+
* substring only, so a word matching every doc via stemming — "queries" vs
|
|
324
|
+
* stem "query" — could keep df≈0 and a falsely-maximal weight.)
|
|
325
|
+
*
|
|
326
|
+
* CJK: `tokenizeWords` already segments Han runs, and DF/matching include
|
|
327
|
+
* substring containment (script-agnostic), so a rare Han token earns
|
|
328
|
+
* exactly the same discriminative weight as a rare ASCII one.
|
|
329
|
+
*/
|
|
330
|
+
function buildIdfModel(query, corpusTexts) {
|
|
331
|
+
const rawWords = tokenizeWords(query);
|
|
332
|
+
const variantsByRaw = new Map();
|
|
333
|
+
for (const w of rawWords) {
|
|
334
|
+
if (variantsByRaw.has(w))
|
|
335
|
+
continue;
|
|
336
|
+
const variants = new Set([w]);
|
|
337
|
+
for (const v of expandQuery([w]))
|
|
338
|
+
variants.add(v);
|
|
339
|
+
variantsByRaw.set(w, [...variants]);
|
|
340
|
+
}
|
|
341
|
+
const n = corpusTexts.length;
|
|
342
|
+
const docs = corpusTexts.map((t) => {
|
|
343
|
+
const lower = t.toLowerCase();
|
|
344
|
+
return { lower, tokenSet: new Set(tokenizeWords(t).map((tok) => stem(tok))) };
|
|
345
|
+
});
|
|
346
|
+
const idf = new Map();
|
|
347
|
+
let maxIdf = 0;
|
|
348
|
+
for (const [w, variants] of variantsByRaw) {
|
|
349
|
+
let df = 0;
|
|
350
|
+
for (const d of docs) {
|
|
351
|
+
if (variants.some((v) => d.tokenSet.has(v) || d.lower.includes(v)))
|
|
352
|
+
df++;
|
|
353
|
+
}
|
|
354
|
+
const weight = n > 0 ? bm25Idf(df, n) : 1;
|
|
355
|
+
idf.set(w, weight);
|
|
356
|
+
if (weight > maxIdf)
|
|
357
|
+
maxIdf = weight;
|
|
358
|
+
}
|
|
359
|
+
if (maxIdf > 0) {
|
|
360
|
+
for (const [w, v] of idf)
|
|
361
|
+
idf.set(w, v / maxIdf);
|
|
362
|
+
}
|
|
363
|
+
else {
|
|
364
|
+
for (const w of idf.keys())
|
|
365
|
+
idf.set(w, 1);
|
|
366
|
+
}
|
|
367
|
+
const totalMass = [...idf.values()].reduce((a, b) => a + b, 0);
|
|
368
|
+
return {
|
|
369
|
+
weightedExactness(text) {
|
|
370
|
+
if (rawWords.length === 0)
|
|
371
|
+
return 0;
|
|
372
|
+
const textSet = new Set(tokenizeWords(text).map((t) => stem(t)));
|
|
373
|
+
const textLower = text.toLowerCase();
|
|
374
|
+
let matchedMass = 0;
|
|
375
|
+
let matchedCount = 0;
|
|
376
|
+
for (const [w, variants] of variantsByRaw) {
|
|
377
|
+
if (variants.some((v) => textSet.has(v) || textLower.includes(v))) {
|
|
378
|
+
matchedMass += idf.get(w) ?? 0;
|
|
379
|
+
matchedCount++;
|
|
380
|
+
}
|
|
381
|
+
}
|
|
382
|
+
if (totalMass <= 0)
|
|
383
|
+
return Math.min(1, matchedCount / rawWords.length);
|
|
384
|
+
return Math.min(1, matchedMass / totalMass);
|
|
385
|
+
},
|
|
386
|
+
};
|
|
387
|
+
}
|
|
275
388
|
/**
|
|
276
389
|
* Simple stable hash for result IDs — ported verbatim from smart-recall.ts.
|
|
277
390
|
*
|
|
@@ -560,7 +673,9 @@ function readLegacyJournalCandidates(project) {
|
|
|
560
673
|
continue;
|
|
561
674
|
let files;
|
|
562
675
|
try {
|
|
563
|
-
|
|
676
|
+
// fix4 (2026-09-11): same generated-index / reserved-namespace
|
|
677
|
+
// exclusion as candidates.ts's journal halves (see that file).
|
|
678
|
+
files = fs.readdirSync(legacyJournal).filter((f) => f.endsWith(".md") && f !== "index.md" && !f.startsWith("_"));
|
|
564
679
|
}
|
|
565
680
|
catch {
|
|
566
681
|
continue;
|
|
@@ -609,9 +724,22 @@ function scoreJournalTier(project, query, opts) {
|
|
|
609
724
|
const sinceCutoff = opts.since ? parseSinceDate(opts.since) : null;
|
|
610
725
|
const limit = opts.perTierLimit ?? 25;
|
|
611
726
|
const hits = [];
|
|
727
|
+
// fix4 S3 (2026-09-11, reports/agentrecall-fix4-retrieval-2026-09-11.md):
|
|
728
|
+
// SCORE-THEN-TRUNCATE. The pre-fix4 loop broke out at `hits.length >=
|
|
729
|
+
// limit` while iterating candidates in date-DESCENDING order — i.e. it
|
|
730
|
+
// truncated by RECENCY before any hit was ever scored, so an old entry
|
|
731
|
+
// matching the query strongly was dropped UNSCORED whenever `limit`
|
|
732
|
+
// weaker-but-newer line matches existed (the S2-standard eval's latent
|
|
733
|
+
// "journal recency pre-truncation" defect; the W2 sort-before-truncate
|
|
734
|
+
// work made the pre-truncation deterministic, not correct). Now every
|
|
735
|
+
// matching line across every candidate is collected and SCORED
|
|
736
|
+
// (recency×0.5 + exactness×0.5, unchanged formula), and `limit` is
|
|
737
|
+
// applied to the score-sorted list at the end — one truncation criterion,
|
|
738
|
+
// the same one RRF ranking uses. Cost: the full line scan always runs
|
|
739
|
+
// (previously it stopped early once `limit` hits were found) — bounded by
|
|
740
|
+
// the same already-read candidate set, and measured within the S11
|
|
741
|
+
// latency budget by this tranche's eval runs.
|
|
612
742
|
for (const candidate of candidates) {
|
|
613
|
-
if (hits.length >= limit)
|
|
614
|
-
break;
|
|
615
743
|
if (sinceCutoff && candidate.date) {
|
|
616
744
|
const fileDate = new Date(candidate.date);
|
|
617
745
|
if (!isNaN(fileDate.getTime()) && fileDate < sinceCutoff)
|
|
@@ -620,8 +748,6 @@ function scoreJournalTier(project, query, opts) {
|
|
|
620
748
|
const lines = candidate.content.split("\n");
|
|
621
749
|
let currentSection = "top";
|
|
622
750
|
for (let i = 0; i < lines.length; i++) {
|
|
623
|
-
if (hits.length >= limit)
|
|
624
|
-
break;
|
|
625
751
|
const line = lines[i];
|
|
626
752
|
if (line.startsWith("## ")) {
|
|
627
753
|
currentSection = line.slice(3).trim().toLowerCase().replace(/\s+/g, "_");
|
|
@@ -640,6 +766,14 @@ function scoreJournalTier(project, query, opts) {
|
|
|
640
766
|
hits.push({ title: `${date} / ${currentSection}`, excerpt, date, line: i + 1 });
|
|
641
767
|
}
|
|
642
768
|
}
|
|
769
|
+
// fix4 S5: IDF over the scanned journal corpus (one doc = one journal
|
|
770
|
+
// file's content) — weights each query word by rarity before the
|
|
771
|
+
// unchanged recency*0.5 + exactness*0.5 blend below. Review L2
|
|
772
|
+
// (2026-09-11): skipped entirely when nothing matched — building the
|
|
773
|
+
// model tokenizes/lowercases the whole corpus, pure waste at zero hits.
|
|
774
|
+
if (hits.length === 0)
|
|
775
|
+
return [];
|
|
776
|
+
const idfModel = buildIdfModel(query, candidates.map((c) => c.content));
|
|
643
777
|
const items = hits.map((h) => {
|
|
644
778
|
// Independent review fix (W3b, 2026-08-30): `id` must be unique PER HIT,
|
|
645
779
|
// not per (date,section) — `applyRRF` (below) groups same-tier items by
|
|
@@ -660,12 +794,26 @@ function scoreJournalTier(project, query, opts) {
|
|
|
660
794
|
const id = stableId("journal", `${h.title}::${h.line}::${h.excerpt}`);
|
|
661
795
|
const days = daysSince(h.date);
|
|
662
796
|
const recency = ebbinghaus(days, EBBINGHAUS_S.journal);
|
|
663
|
-
const exactness =
|
|
797
|
+
const exactness = idfModel.weightedExactness(h.excerpt);
|
|
664
798
|
const internalScore = recency * 0.5 + exactness * 0.5;
|
|
665
799
|
return { id, source: "journal", title: h.title, excerpt: h.excerpt, score: internalScore, date: h.date, line: h.line };
|
|
666
800
|
});
|
|
667
801
|
items.sort((a, b) => b.score - a.score);
|
|
668
|
-
|
|
802
|
+
// fix4 S4-completion: one (date, section) -> one competitive slot, best
|
|
803
|
+
// line wins (items are already score-sorted, so first-seen per title is
|
|
804
|
+
// the best) — opt-in, see QueryMemoryInput.journal.perSectionDedupe.
|
|
805
|
+
let deduped = items;
|
|
806
|
+
if (opts.perSectionDedupe) {
|
|
807
|
+
const seenSections = new Set();
|
|
808
|
+
deduped = items.filter((it) => {
|
|
809
|
+
if (seenSections.has(it.title))
|
|
810
|
+
return false;
|
|
811
|
+
seenSections.add(it.title);
|
|
812
|
+
return true;
|
|
813
|
+
});
|
|
814
|
+
}
|
|
815
|
+
// fix4 S3: truncate AFTER scoring+sorting — see the loop comment above.
|
|
816
|
+
return deduped.slice(0, limit);
|
|
669
817
|
}
|
|
670
818
|
/** Palace tier: ported from palace-search.ts's tagBonus + per-line matching +
|
|
671
819
|
* IDF re-scoring, operating on already-fetched room-file MemoryCandidates
|
|
@@ -673,7 +821,7 @@ function scoreJournalTier(project, query, opts) {
|
|
|
673
821
|
* smart-recall.ts's original `internalScore = keyScore*0.65 + salience*0.35`.
|
|
674
822
|
* TRUST-FILTERED before any of this runs — CLOSES the known gap
|
|
675
823
|
* documented in this file's header, CHALLENGE (b). */
|
|
676
|
-
function scorePalaceTier(project, query, opts) {
|
|
824
|
+
async function scorePalaceTier(project, query, opts) {
|
|
677
825
|
// Matches palace-search.ts's own first executable step — scaffolds the
|
|
678
826
|
// palace dir + default rooms (writing each default room's `_room.json`) on
|
|
679
827
|
// first touch. Without this, `listRooms()` (both here and inside
|
|
@@ -749,30 +897,32 @@ function scorePalaceTier(project, query, opts) {
|
|
|
749
897
|
roomsWithHits.add(room);
|
|
750
898
|
}
|
|
751
899
|
}
|
|
752
|
-
// IDF re-scoring — ported
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
}
|
|
766
|
-
}
|
|
900
|
+
// IDF re-scoring — structure ported from palace-search.ts's Fix RC3;
|
|
901
|
+
// fix4 S5 (2026-09-11): DF is now counted over the SCANNED CANDIDATE
|
|
902
|
+
// CORPUS (every room file this call already read: full content + room +
|
|
903
|
+
// filename), not over the hit set. Hits-only DF was self-referential —
|
|
904
|
+
// any doc containing a query word IS a hit for that word, so a word's
|
|
905
|
+
// rarity was measured against only the documents that matched it (e.g. a
|
|
906
|
+
// word appearing solely in skipped heading lines contributed nothing),
|
|
907
|
+
// systematically flattening the very signal IDF exists to provide. The
|
|
908
|
+
// idf curve itself moves to the shared `bm25Idf` (the one formula every
|
|
909
|
+
// fix4 tier uses); the 0.70/0.30 idf/raw blend and per-hit matched-word
|
|
910
|
+
// semantics are unchanged.
|
|
911
|
+
if (hits.length > 0 && candidates.length > 0) {
|
|
912
|
+
const corpusDocs = candidates.map((c) => (c.content + " " + (c.room ?? "") + " " + c.file).toLowerCase());
|
|
767
913
|
const idfRaw = new Map();
|
|
768
914
|
for (const w of queryWords) {
|
|
769
|
-
|
|
770
|
-
|
|
915
|
+
let df = 0;
|
|
916
|
+
for (const doc of corpusDocs) {
|
|
917
|
+
if (doc.includes(w))
|
|
918
|
+
df++;
|
|
919
|
+
}
|
|
920
|
+
idfRaw.set(w, bm25Idf(df, corpusDocs.length));
|
|
771
921
|
}
|
|
772
|
-
const maxIdf = Math.max(
|
|
922
|
+
const maxIdf = Math.max(...Array.from(idfRaw.values()), 0);
|
|
773
923
|
const idf = new Map();
|
|
774
924
|
for (const [w, v] of idfRaw)
|
|
775
|
-
idf.set(w, v / maxIdf);
|
|
925
|
+
idf.set(w, maxIdf > 0 ? v / maxIdf : 1);
|
|
776
926
|
for (const h of hits) {
|
|
777
927
|
const combined = (h.excerpt + " " + h.room + " " + h.file).toLowerCase();
|
|
778
928
|
const matchedIdfWords = queryWords.filter((w) => combined.includes(w));
|
|
@@ -786,10 +936,38 @@ function scorePalaceTier(project, query, opts) {
|
|
|
786
936
|
// (once per room that contributed >=1 hit).
|
|
787
937
|
for (const room of roomsWithHits) {
|
|
788
938
|
try {
|
|
789
|
-
recordAccess(project, room);
|
|
939
|
+
await recordAccess(project, room);
|
|
790
940
|
}
|
|
791
941
|
catch { /* best-effort, never block scoring on a bookkeeping write */ }
|
|
792
942
|
}
|
|
943
|
+
// fix4 S4-completion (2026-09-11, diagnosed in-tranche, reports/
|
|
944
|
+
// agentrecall-fix4-retrieval-2026-09-11.md): ONE DOCUMENT, ONE VOTE.
|
|
945
|
+
// Palace item ids are stableId("palace", `${room}/${file}`) — identical
|
|
946
|
+
// for every matching LINE of the same file — so applyRRF's per-id
|
|
947
|
+
// accumulation summed one 1/(60+rank) contribution PER LINE into a single
|
|
948
|
+
// entry. A long, frequently-touched catch-all file (goals/evolution)
|
|
949
|
+
// matching a couple of ubiquitous query words on 30-40 lines reached
|
|
950
|
+
// fused scores of 0.44-0.71 against 0.0164 for a genuinely-relevant
|
|
951
|
+
// single-line match — rank 1 in 15/20 golden queries measured at
|
|
952
|
+
// baseline. Line COUNT tracks document length, not relevance (the signal
|
|
953
|
+
// class BM25's TF-saturation caps); S4's weight rebalance alone could not
|
|
954
|
+
// fix this because the design's stated mechanism (salience) was only the
|
|
955
|
+
// minor leg. Fix: reduce hits to the BEST-scoring line per document
|
|
956
|
+
// before materializing items, so RRF sees exactly one vote per document
|
|
957
|
+
// and the surfaced excerpt is always the document's strongest evidence.
|
|
958
|
+
// This also structurally retires the W3b-documented palace id-collision
|
|
959
|
+
// (two same-file hits can no longer meet applyRRF) and the accidental
|
|
960
|
+
// score inflation helpers/associative-link.ts depended on — its
|
|
961
|
+
// similarity threshold is recalibrated in the same change (see
|
|
962
|
+
// linkToSimilar's own comment).
|
|
963
|
+
const bestByDoc = new Map();
|
|
964
|
+
for (const h of hits) {
|
|
965
|
+
const key = `${h.room}/${h.file}`;
|
|
966
|
+
const cur = bestByDoc.get(key);
|
|
967
|
+
if (!cur || h.keywordScore > cur.keywordScore)
|
|
968
|
+
bestByDoc.set(key, h);
|
|
969
|
+
}
|
|
970
|
+
const docHits = [...bestByDoc.values()];
|
|
793
971
|
const limit = opts.perTierLimit ?? 40; // matches smart-recall's `limit*2` request to palaceSearch
|
|
794
972
|
// Score ALL hits first, sort by the SAME internal score used for RRF
|
|
795
973
|
// ranking, THEN truncate — a single consistent truncation criterion.
|
|
@@ -798,34 +976,35 @@ function scorePalaceTier(project, query, opts) {
|
|
|
798
976
|
// smart-recall.ts re-scores by keyScore*0.65+salienceFloor*0.35 for its own
|
|
799
977
|
// ranking; this collapses that two-stage, two-criteria truncation into one,
|
|
800
978
|
// a characterized simplification, not a regression — see this file's header.)
|
|
801
|
-
const items =
|
|
979
|
+
const items = docHits.map((h) => {
|
|
802
980
|
const title = `${h.room}/${h.file}`;
|
|
803
|
-
//
|
|
804
|
-
//
|
|
805
|
-
//
|
|
806
|
-
//
|
|
807
|
-
//
|
|
808
|
-
//
|
|
809
|
-
//
|
|
810
|
-
//
|
|
811
|
-
//
|
|
812
|
-
//
|
|
813
|
-
//
|
|
814
|
-
//
|
|
815
|
-
// never-intended) score-inflation side effect of this same bug that
|
|
816
|
-
// helpers/associative-link.ts's `linkToSimilar` — an entirely different
|
|
817
|
-
// subsystem this wave does not touch — happens to depend on for its
|
|
818
|
-
// hardcoded `score > 0.03` similarity threshold
|
|
819
|
-
// (associative-link.test.mjs's "linkToSimilar creates bidirectional
|
|
820
|
-
// edges..." fails if this is fixed in isolation, because each hit's own
|
|
821
|
-
// un-inflated RRF contribution, ~0.016, is genuinely below that
|
|
822
|
-
// threshold). Fixing this properly needs a decision about
|
|
823
|
-
// linkToSimilar's OWN threshold calibration, not a query-memory.ts-only
|
|
824
|
-
// change — flagged in the Wave 3b report for the orchestrator, not
|
|
825
|
-
// silently fixed or silently left undocumented.
|
|
981
|
+
// W3b HISTORY (2026-08-30) + fix4 RESOLUTION (2026-09-11): `title`
|
|
982
|
+
// (`${room}/${file}`) was not unique per HIT, so two matching lines of
|
|
983
|
+
// the same room file collided in applyRRF's per-tier id-Map —
|
|
984
|
+
// accumulating both hits' RRF contributions into one entry (the
|
|
985
|
+
// mega-file dominance mechanism the S4-completion comment above
|
|
986
|
+
// documents) while discarding the loser's excerpt. W3b flagged the fix
|
|
987
|
+
// as blocked on a linkToSimilar threshold-calibration decision; fix4
|
|
988
|
+
// resolves it the OTHER way around — items are now one-per-DOCUMENT
|
|
989
|
+
// (`docHits`, best line as evidence), for which this id is genuinely
|
|
990
|
+
// unique, and linkToSimilar's threshold is recalibrated to the
|
|
991
|
+
// un-inflated score scale in the same change (see that function's own
|
|
992
|
+
// comment).
|
|
826
993
|
const id = stableId("palace", title);
|
|
827
994
|
const salience = Math.max(0.4, salienceByRoom.get(h.room) ?? 0.5);
|
|
828
|
-
|
|
995
|
+
// fix4 S4 (2026-09-11, reports/agentrecall-fix4-retrieval-2026-09-11.md):
|
|
996
|
+
// relevance-over-salience rebalance, 0.65/0.35 -> 0.90/0.10. Under the
|
|
997
|
+
// old weights the maximum salience differential (0.35 × (1.0 − 0.4
|
|
998
|
+
// floor) = 0.21) exceeded typical keyword-score deltas between a
|
|
999
|
+
// topically-right room and a high-traffic catch-all room — the
|
|
1000
|
+
// S2-standard eval measured the goals/evolution mega-room taking rank 1
|
|
1001
|
+
// in 15/20 golden queries across 8 projects on salience alone. At
|
|
1002
|
+
// 0.10 the salience spread caps at 0.06: enough to break genuine
|
|
1003
|
+
// relevance ties (its documented role — see the S4b mechanism test),
|
|
1004
|
+
// never enough to overturn a >=0.07 keyword advantage. Query relevance
|
|
1005
|
+
// now dominates the palace tier's internal ordering the same way it
|
|
1006
|
+
// already dominates every other tier's.
|
|
1007
|
+
const internalScore = h.keywordScore * 0.90 + salience * 0.10;
|
|
829
1008
|
const datePattern = h.excerpt.match(/(\d{4}-\d{2}-\d{2})/);
|
|
830
1009
|
return {
|
|
831
1010
|
id,
|
|
@@ -916,17 +1095,37 @@ function scoreInsightTier(project, query, opts) {
|
|
|
916
1095
|
* no-op today; matching every sibling tier's own call shape keeps that
|
|
917
1096
|
* true structurally, not by omission.
|
|
918
1097
|
*
|
|
919
|
-
* SCORING
|
|
920
|
-
*
|
|
921
|
-
*
|
|
922
|
-
*
|
|
923
|
-
*
|
|
924
|
-
*
|
|
925
|
-
*
|
|
926
|
-
*
|
|
927
|
-
*
|
|
928
|
-
*
|
|
929
|
-
*
|
|
1098
|
+
* SCORING (fix4 S1, 2026-09-11, reports/agentrecall-fix4-retrieval-2026-09-11.md):
|
|
1099
|
+
* matches on rule+context text via the shared tokenizer (`keywordExactness`
|
|
1100
|
+
* — the SAME helper `scorePalaceTier`/`scoreInsightTier` already use above),
|
|
1101
|
+
* blended with the record's own IMPORTANCE/EVIDENCE signals, mirroring the
|
|
1102
|
+
* insight tier's shape (`relevance*0.40 + exactness*0.35 + confirmation*0.25`
|
|
1103
|
+
* — see `scoreInsightTier` above):
|
|
1104
|
+
*
|
|
1105
|
+
* internalScore = exactness*0.70 + severityBoost*0.15 + proofBoost*0.15
|
|
1106
|
+
* severityBoost: p0 -> 1.0, p1 -> 0.5 (P0-ness reuses the EXISTING
|
|
1107
|
+
* `severity` field, not a parallel signal)
|
|
1108
|
+
* proofBoost: min(1, log2(proof_count+1)/3) — byte-identical to the
|
|
1109
|
+
* insight tier's `confirmed_count` curve; proof_count is the
|
|
1110
|
+
* corrections ledger's own on-write consolidation counter.
|
|
1111
|
+
*
|
|
1112
|
+
* Relevance DOMINATES (0.70): severity/proof can reorder near-equal matches
|
|
1113
|
+
* but can never surface a non-matching record (zero-keyword candidates are
|
|
1114
|
+
* skipped before scoring) nor lift a weak match past a strong one by more
|
|
1115
|
+
* than 0.30. NON-TIME-DECAYING like the insight tier: no recency/decay term
|
|
1116
|
+
* — an April rule and a September rule with equal relevance score equally
|
|
1117
|
+
* (a correction is a standing rule, not an episodic memory).
|
|
1118
|
+
*
|
|
1119
|
+
* `confidence`/`decayClass` remain ANNOTATION ONLY on the returned item —
|
|
1120
|
+
* per the design memo's own repeated finding, rejected TWICE independently
|
|
1121
|
+
* (the July 2 proposal's Option 2, and W5fix's removal of
|
|
1122
|
+
* `CONTRADICTION_PENALTY`, both for the identical stated reason: blending a
|
|
1123
|
+
* confidence/STALENESS signal into a ranking score hides WHY an item
|
|
1124
|
+
* dropped and risks a silent rank inversion) — `internalScore` below never
|
|
1125
|
+
* reads either field. severity/proof_count are a DIFFERENT class of signal
|
|
1126
|
+
* (importance/evidence, same class as the insight tier's already-scored
|
|
1127
|
+
* `confirmed_count`), not belief-confidence or staleness — the invariant
|
|
1128
|
+
* those two rejections protect is preserved, not relaxed.
|
|
930
1129
|
*
|
|
931
1130
|
* `id` is the correction's OWN real, stable `CorrectionRecord.id`
|
|
932
1131
|
* (`meta.correction_id`) — deliberately NOT a `stableId()` hash like every
|
|
@@ -935,21 +1134,48 @@ function scoreInsightTier(project, query, opts) {
|
|
|
935
1134
|
* `recordOutcome`) keys off directly; reusing it here means a caller acting
|
|
936
1135
|
* on a surfaced correction never needs a secondary id lookup.
|
|
937
1136
|
*/
|
|
1137
|
+
/** Review L3 (2026-09-11): enumerated severity ladder for the corrections
|
|
1138
|
+
* scorer — extending coverage is a new ROW here, never a new branch. */
|
|
1139
|
+
const SEVERITY_BOOST = {
|
|
1140
|
+
p0: 1.0,
|
|
1141
|
+
p1: 0.5,
|
|
1142
|
+
p2: 0.25,
|
|
1143
|
+
p3: 0.1,
|
|
1144
|
+
};
|
|
938
1145
|
function scoreCorrectionsTier(project, query, opts) {
|
|
939
1146
|
const candidates = filterTrusted(readTierCandidates("corrections", project, { includeUntrusted: true }));
|
|
940
1147
|
const keywords = tokenizeWords(query);
|
|
941
1148
|
if (keywords.length === 0)
|
|
942
1149
|
return [];
|
|
943
1150
|
const limit = opts.perTierLimit ?? 25;
|
|
1151
|
+
// fix4 S5: IDF over the scanned corrections corpus (one doc = one
|
|
1152
|
+
// record's rule+context match text) — materialized once, before the
|
|
1153
|
+
// match/score loop, so DF covers ALL scanned records, not just matches.
|
|
1154
|
+
const matchTexts = candidates.map((candidate) => {
|
|
1155
|
+
const context = candidate.meta?.context ?? "";
|
|
1156
|
+
return context ? `${candidate.content}\n${context}` : candidate.content;
|
|
1157
|
+
});
|
|
1158
|
+
const idfModel = buildIdfModel(query, matchTexts);
|
|
944
1159
|
const items = [];
|
|
945
|
-
for (
|
|
1160
|
+
for (let ci = 0; ci < candidates.length; ci++) {
|
|
1161
|
+
const candidate = candidates[ci];
|
|
946
1162
|
const meta = candidate.meta ?? {};
|
|
947
|
-
const
|
|
948
|
-
const matchText = context ? `${candidate.content}\n${context}` : candidate.content;
|
|
1163
|
+
const matchText = matchTexts[ci];
|
|
949
1164
|
const matchTextLower = matchText.toLowerCase();
|
|
950
1165
|
if (!keywords.some((kw) => matchTextLower.includes(kw)))
|
|
951
1166
|
continue;
|
|
952
|
-
const exactness =
|
|
1167
|
+
const exactness = idfModel.weightedExactness(matchText);
|
|
1168
|
+
// fix4 S1: severity/proof_count-aware blend — see this function's doc
|
|
1169
|
+
// comment for the formula and the annotation-only invariant it preserves.
|
|
1170
|
+
// Review L3 (2026-09-11): the ladder is an enumerated CLASS table —
|
|
1171
|
+
// CorrectionRecord.severity types only p0|p1, but legacy on-disk records
|
|
1172
|
+
// carry p2/p3 (corrections.ts severityRank handles them); a binary
|
|
1173
|
+
// p0-else check silently flattened those onto p1. Unknown values score
|
|
1174
|
+
// as p1 (the type's own default tier).
|
|
1175
|
+
const severityBoost = SEVERITY_BOOST[meta.severity ?? "p1"] ?? SEVERITY_BOOST.p1;
|
|
1176
|
+
const proofCount = Math.max(1, parseInt(meta.proof_count ?? "1", 10) || 1);
|
|
1177
|
+
const proofBoost = Math.min(1.0, Math.log2(proofCount + 1) / 3);
|
|
1178
|
+
const internalScore = exactness * 0.70 + severityBoost * 0.15 + proofBoost * 0.15;
|
|
953
1179
|
const excerpt = matchText.length > 300 ? matchText.slice(0, 300) + "..." : matchText;
|
|
954
1180
|
const title = candidate.content.length > 300 ? candidate.content.slice(0, 300) + "..." : candidate.content;
|
|
955
1181
|
// Fallback (candidate.file sans ".json") only covers the defensive case
|
|
@@ -977,7 +1203,7 @@ function scoreCorrectionsTier(project, query, opts) {
|
|
|
977
1203
|
source: "corrections",
|
|
978
1204
|
title,
|
|
979
1205
|
excerpt,
|
|
980
|
-
score:
|
|
1206
|
+
score: internalScore,
|
|
981
1207
|
date: candidate.date || undefined,
|
|
982
1208
|
severity: meta.severity,
|
|
983
1209
|
confidence: meta.confidence || undefined,
|
|
@@ -987,19 +1213,76 @@ function scoreCorrectionsTier(project, query, opts) {
|
|
|
987
1213
|
items.sort((a, b) => b.score - a.score);
|
|
988
1214
|
return items.slice(0, limit);
|
|
989
1215
|
}
|
|
990
|
-
|
|
1216
|
+
/**
|
|
1217
|
+
* fix7 (2026-09-12): `weight` (default 1 — every lexical tier) scales one
|
|
1218
|
+
* LEG's rank contributions: `weight / (RRF_K + rank)`. This is weighted
|
|
1219
|
+
* Reciprocal Rank Fusion (the standard multi-leg RRF extension) — still
|
|
1220
|
+
* strictly RANK-based, never raw-score-based, so the incompatible-scale
|
|
1221
|
+
* class (smart-recall.ts Fix 1) cannot recur. Used ONLY by the opt-in
|
|
1222
|
+
* semantic leg (see queryMemory below); its value was chosen by golden-eval
|
|
1223
|
+
* measurement, not tuned per-query (fix7 report's placement/weight matrix).
|
|
1224
|
+
*
|
|
1225
|
+
* fix7 review H2 (2026-09-12): on an id-level fold, LEXICAL FIELDS WIN
|
|
1226
|
+
* regardless of insertion order. The semantic leg fuses BEFORE
|
|
1227
|
+
* palace/journal/insight (its measured tie-band placement), so without this
|
|
1228
|
+
* rule a dual-evidence item would keep the semantic chunk's excerpt/line
|
|
1229
|
+
* (unanchored, chunk-start line → wrong verbatim drill-down) and a
|
|
1230
|
+
* `semantic: true` marker whose documented contract is "semantic-ONLY
|
|
1231
|
+
* discoveries". When a LEXICAL item folds into an entry currently held by a
|
|
1232
|
+
* SEMANTIC-originated item, the entry's `item` is REPLACED by the lexical
|
|
1233
|
+
* one (match-anchored excerpt, real line, no `semantic` marker) while the
|
|
1234
|
+
* accumulated score, the sources set, and — critically — the entry's MAP
|
|
1235
|
+
* POSITION (the tie-break authority) are preserved. Flag-off items never
|
|
1236
|
+
* carry `semantic`, so this branch is structurally unreachable flag-off.
|
|
1237
|
+
*/
|
|
1238
|
+
function applyRRF(rankedItems, rrfMap, weight = 1) {
|
|
991
1239
|
rankedItems.forEach((item, idx) => {
|
|
992
1240
|
const rank = idx + 1;
|
|
993
|
-
const contribution =
|
|
1241
|
+
const contribution = weight / (RRF_K + rank);
|
|
994
1242
|
const existing = rrfMap.get(item.id);
|
|
995
1243
|
if (existing) {
|
|
996
1244
|
existing.score += contribution;
|
|
1245
|
+
// fix7 review H2: lexical fields displace semantic-originated fields
|
|
1246
|
+
// on a fold (Map value replacement keeps the entry's insertion
|
|
1247
|
+
// position, so tie-break placement is unaffected). No sources.add
|
|
1248
|
+
// here: semantic items carry NATIVE tier source labels, so a fold is
|
|
1249
|
+
// always same-source — and the flag-off path never folds cross-tier
|
|
1250
|
+
// (id spaces disjoint by construction, see scoreCorrectionsTier).
|
|
1251
|
+
if (existing.item.semantic && !item.semantic) {
|
|
1252
|
+
existing.item = item;
|
|
1253
|
+
}
|
|
997
1254
|
}
|
|
998
1255
|
else {
|
|
999
1256
|
rrfMap.set(item.id, { score: contribution, item, sources: new Set([item.source]) });
|
|
1000
1257
|
}
|
|
1001
1258
|
});
|
|
1002
1259
|
}
|
|
1260
|
+
/**
|
|
1261
|
+
* fix7 semantic-leg fusion parameters — MEASURED on the golden eval
|
|
1262
|
+
* (agentrecall-fix7-embeddings-2026-09-12.md, placement/weight matrix),
|
|
1263
|
+
* not designed a priori. Both only ever apply under the embeddings opt-in.
|
|
1264
|
+
*
|
|
1265
|
+
* - SEMANTIC_RRF_WEIGHT: weighted-RRF multiplier for the semantic leg's
|
|
1266
|
+
* contributions (see applyRRF).
|
|
1267
|
+
* - SEMANTIC_AFTER_CORRECTIONS: when true, the semantic leg's items enter
|
|
1268
|
+
* the fusion map immediately AFTER the corrections tier — at EXACT
|
|
1269
|
+
* fused-score ties they then outrank palace/journal/insight lexical
|
|
1270
|
+
* singles (semantic evidence over incidental keyword overlap) while the
|
|
1271
|
+
* owner's own captured rules keep top tie authority. When false, the
|
|
1272
|
+
* leg inserts last (loses all ties).
|
|
1273
|
+
*
|
|
1274
|
+
* MEASURED (fix7 report, twin-clone golden eval; certified post-review-fix
|
|
1275
|
+
* with the default MiniLM-L12-v2 model):
|
|
1276
|
+
* weight 1.0 + after-corrections → 90.0% hit-rate, 3/5 paraphrase
|
|
1277
|
+
* recovered, ZERO regressions, all six protected hits hold. weight 1.0 +
|
|
1278
|
+
* last placement → a semantic-only golden loses every 1/61 tie (measured
|
|
1279
|
+
* −1 hit on the pre-fix matrix). weight 1.3 / 1.1 (any placement) →
|
|
1280
|
+
* semantic items vault the lexical-single band wholesale and FLOOD top-5
|
|
1281
|
+
* (three regressions, one protected) — weights > 1 are measured-toxic on
|
|
1282
|
+
* this store; do not raise without re-running the full protected battery.
|
|
1283
|
+
*/
|
|
1284
|
+
const SEMANTIC_RRF_WEIGHT = 1.0;
|
|
1285
|
+
const SEMANTIC_AFTER_CORRECTIONS = true;
|
|
1003
1286
|
function fuseCanonical(rrfMap) {
|
|
1004
1287
|
const canonical = new Map();
|
|
1005
1288
|
for (const entry of rrfMap.values()) {
|
|
@@ -1016,7 +1299,49 @@ function fuseCanonical(rrfMap) {
|
|
|
1016
1299
|
}
|
|
1017
1300
|
return canonical;
|
|
1018
1301
|
}
|
|
1019
|
-
|
|
1302
|
+
// fix4b — hot-window redesign (2026-09-12, reports/agentrecall-fix4b-
|
|
1303
|
+
// hotwindow-2026-09-12.md; diagnosed in reports/agentrecall-fix4-retrieval-
|
|
1304
|
+
// 2026-09-11.md Escalation §1). PRODUCT-BEHAVIOR CHANGE, stated prominently:
|
|
1305
|
+
// no POST-FUSION freshness signal exists on default paths — fused scores are
|
|
1306
|
+
// never freshness-multiplied and ties are freshness-blind. (The journal
|
|
1307
|
+
// tier's INTERNAL Ebbinghaus recency×0.5 blend — a per-tier scoring input,
|
|
1308
|
+
// see scoreJournalTier — is a different mechanism and is unchanged by fix4b.)
|
|
1309
|
+
//
|
|
1310
|
+
// The old `applyHotWindowBoost` multiplied FUSED RRF scores ×3.0/×2.0/×1.3
|
|
1311
|
+
// for items dated <6h/<24h/<72h. But RRF compresses adjacent-rank score
|
|
1312
|
+
// differences to ~1.6% (1/(60+r) vs 1/(61+r)), so ANY multiplier > ~1.02
|
|
1313
|
+
// vaulted an off-topic-but-recent item over every un-boosted on-topic
|
|
1314
|
+
// result — in a store written to daily, every query's top-5 filled with the
|
|
1315
|
+
// last 72h of diary sections regardless of relevance. Measured on the golden
|
|
1316
|
+
// eval as the single largest hit-rate loss (55.0% with the boost, 75.0%
|
|
1317
|
+
// with it neutralized).
|
|
1318
|
+
//
|
|
1319
|
+
// The design brief's PRIMARY redesign (D1) kept freshness as a within-
|
|
1320
|
+
// tie-band reorder: fused score decides the band, hotness (<6h → <24h →
|
|
1321
|
+
// <72h → none) reorders inside a band of exactly-tied scores, then the
|
|
1322
|
+
// authority/insertion order. Measured on the same twin-clone protocol, D1
|
|
1323
|
+
// scored 65.0% — below the 70% exit gate — because post one-doc-one-vote,
|
|
1324
|
+
// cross-TIER exact ties at 1/(60+r) are the common case, and hotness-over-
|
|
1325
|
+
// authority handed those bands to <72h diary lines over the owner's own
|
|
1326
|
+
// captured rules (gq05/gq06 measured casualties). Per the brief's explicit
|
|
1327
|
+
// fallback clause, what ships is FULL NEUTRALIZATION: the default pipeline
|
|
1328
|
+
// applies no post-fusion freshness signal; exact fused-score ties resolve
|
|
1329
|
+
// by the fix4 authority/insertion order alone (corrections first — see
|
|
1330
|
+
// localRecallSearch's `tiers` comment). The legacy multiplicative boost
|
|
1331
|
+
// survives SOLELY behind the explicit `freshnessBias` opt-in below, for the
|
|
1332
|
+
// one audited caller whose downstream score floor requires it. A measured
|
|
1333
|
+
// middle design (authority pinned above hotness inside bands, hotness
|
|
1334
|
+
// ordering the rest) scored per-query identical to neutralization (75.0%)
|
|
1335
|
+
// and is documented in the fix4b report as a future option — deliberately
|
|
1336
|
+
// NOT shipped: the brief sanctioned exactly two designs.
|
|
1337
|
+
/** LEGACY multiplicative boost — the exact pre-fix4b `applyHotWindowBoost`
|
|
1338
|
+
* body (all defects included: date-only strings parse to 00:00 UTC so the
|
|
1339
|
+
* bucket depends on wall-clock time-of-day — audit Finding 2; palace
|
|
1340
|
+
* regex-scraped excerpt dates are trusted as timestamps; recent items vault
|
|
1341
|
+
* entire relevance bands). Kept ONLY behind `QueryMemoryInput.freshnessBias`
|
|
1342
|
+
* (see that field's doc comment for the single audited caller class that
|
|
1343
|
+
* needs it). Never runs on a default-configured query. */
|
|
1344
|
+
function applyLegacyHotWindowBoost(fusedMap) {
|
|
1020
1345
|
for (const entry of fusedMap.values()) {
|
|
1021
1346
|
if (entry.item.date) {
|
|
1022
1347
|
const hoursAgo = (Date.now() - new Date(entry.item.date).getTime()) / (1000 * 60 * 60);
|
|
@@ -1038,6 +1363,7 @@ const TIER_SCORERS = {
|
|
|
1038
1363
|
// Matches smart-recall.ts's original `journalSearch({..., limit: Math.ceil(limit*1.5)})`.
|
|
1039
1364
|
perTierLimit: input.journal?.perTierLimit ?? Math.ceil((input.limit ?? 10) * 1.5),
|
|
1040
1365
|
since: input.since,
|
|
1366
|
+
perSectionDedupe: input.journal?.perSectionDedupe,
|
|
1041
1367
|
}),
|
|
1042
1368
|
palace: async (input) => scorePalaceTier(input.project, input.query, {
|
|
1043
1369
|
room: input.palace?.room,
|
|
@@ -1088,15 +1414,122 @@ export async function queryMemory(input) {
|
|
|
1088
1414
|
// try/catch isolation.
|
|
1089
1415
|
}
|
|
1090
1416
|
}
|
|
1091
|
-
//
|
|
1417
|
+
// SEMANTIC LEG (fix7, 2026-09-12) — OPT-IN ONLY (AGENT_RECALL_EMBEDDINGS
|
|
1418
|
+
// / config.json embeddings_enabled; read per-call, the RECALL_FUSION
|
|
1419
|
+
// precedent). Flag OFF: this block costs one env check (plus, when the
|
|
1420
|
+
// env var is unset, embeddingsEnabled()'s small mtime-cached config.json
|
|
1421
|
+
// probe) — no index read, no model touch, no semantic module executes,
|
|
1422
|
+
// and the RESULT is byte-identical to fix4b (pinned by
|
|
1423
|
+
// fix7-embeddings.test.mjs's equivalence test). Flag ON: one ADDITIONAL
|
|
1424
|
+
// ranked list joins the same RRF fusion below. The leg's items are
|
|
1425
|
+
// trust-filtered/scoped through the same chokepoints as every lexical
|
|
1426
|
+
// tier (see retrieval/semantic-leg.ts's header — the security property,
|
|
1427
|
+
// adversarially tested) and carry native tier source labels. The dynamic
|
|
1428
|
+
// import keeps this FILE decoupled from the embeddings modules (review
|
|
1429
|
+
// note: the package barrel index.ts re-exports them statically for
|
|
1430
|
+
// CLI/test consumers, so package-level import already parses them — they
|
|
1431
|
+
// are side-effect-free at top level by design; the honest claim is
|
|
1432
|
+
// "never EXECUTES flag-off", not "never loads").
|
|
1433
|
+
let semanticItems;
|
|
1434
|
+
let semanticNote;
|
|
1435
|
+
if (embeddingsEnabled()) {
|
|
1436
|
+
try {
|
|
1437
|
+
const { runSemanticLeg } = await import("./semantic-leg.js");
|
|
1438
|
+
const sem = await runSemanticLeg({
|
|
1439
|
+
query: input.query,
|
|
1440
|
+
project: input.project,
|
|
1441
|
+
// Tier scoping — the leg may only contribute tiers this caller
|
|
1442
|
+
// requested (journalSearch passes tiers:["journal"]; a palace item
|
|
1443
|
+
// leaking into its adapter would corrupt its external contract).
|
|
1444
|
+
tiers: input.tiers,
|
|
1445
|
+
scope: input.scope,
|
|
1446
|
+
since: input.since,
|
|
1447
|
+
room: input.palace?.room,
|
|
1448
|
+
// fix7 review L: no silent param discard — the leg honors the
|
|
1449
|
+
// caller's rollup-archive exclusion the same way the lexical
|
|
1450
|
+
// journal scorer does.
|
|
1451
|
+
includeRollupArchive: input.journal?.includeRollupArchive,
|
|
1452
|
+
});
|
|
1453
|
+
semanticItems = sem.items;
|
|
1454
|
+
semanticNote = sem.note;
|
|
1455
|
+
// fix7 review H1 (2026-09-12): journal id-space JOIN. Palace/insight/
|
|
1456
|
+
// corrections semantic items mint the SAME ids as their lexical twins
|
|
1457
|
+
// (chunker.ts id conventions), so applyRRF folds dual-evidence items
|
|
1458
|
+
// into one entry with summed votes. Journal could not: lexical journal
|
|
1459
|
+
// ids embed line+excerpt (per-hit uniqueness, W3b) and are
|
|
1460
|
+
// unreproducible from a chunk — so a section found by BOTH legs
|
|
1461
|
+
// surfaced as TWO near-duplicate slots with SPLIT votes (defeating
|
|
1462
|
+
// perSectionDedupe cross-leg and under-ranking multi-evidence docs).
|
|
1463
|
+
// Join here, where both sides are in hand: a semantic journal item
|
|
1464
|
+
// adopts the id of the BEST lexical item sharing its exact
|
|
1465
|
+
// "${date} / ${section}" title (both sides construct it identically;
|
|
1466
|
+
// scoreJournalTier returns score-sorted items, so first-seen per
|
|
1467
|
+
// title is the best). The semantic leg emits at most one journal item
|
|
1468
|
+
// per title (docKey = journal/title), so no intra-leg id collision
|
|
1469
|
+
// can result. With applyRRF's lexical-fields-win fold rule (H2), the
|
|
1470
|
+
// merged entry carries the lexical line-anchored excerpt.
|
|
1471
|
+
if (semanticItems.length > 0) {
|
|
1472
|
+
const lexicalJournal = byTier.journal;
|
|
1473
|
+
if (lexicalJournal && lexicalJournal.length > 0) {
|
|
1474
|
+
const bestByTitle = new Map();
|
|
1475
|
+
for (const it of lexicalJournal) {
|
|
1476
|
+
if (!bestByTitle.has(it.title))
|
|
1477
|
+
bestByTitle.set(it.title, it);
|
|
1478
|
+
}
|
|
1479
|
+
semanticItems = semanticItems.map((it) => {
|
|
1480
|
+
if (it.source !== "journal")
|
|
1481
|
+
return it;
|
|
1482
|
+
const lex = bestByTitle.get(it.title);
|
|
1483
|
+
return lex ? { ...it, id: lex.id } : it;
|
|
1484
|
+
});
|
|
1485
|
+
}
|
|
1486
|
+
}
|
|
1487
|
+
}
|
|
1488
|
+
catch (err) {
|
|
1489
|
+
// Defense in depth — runSemanticLeg already never throws; this guard
|
|
1490
|
+
// covers the dynamic import itself. The recall must not fail.
|
|
1491
|
+
semanticNote = {
|
|
1492
|
+
status: "error",
|
|
1493
|
+
model: "unknown",
|
|
1494
|
+
message: err instanceof Error ? err.message : String(err),
|
|
1495
|
+
};
|
|
1496
|
+
}
|
|
1497
|
+
}
|
|
1498
|
+
// RANK/FUSE stage. fix7: the semantic leg joins as one additional
|
|
1499
|
+
// weighted-RRF leg — a same-id item found both ways accumulates
|
|
1500
|
+
// contributions into its lexical entry (multi-evidence, the applyRRF
|
|
1501
|
+
// id-level fold), a semantic-only item enters at the MEASURED map
|
|
1502
|
+
// position (SEMANTIC_AFTER_CORRECTIONS): map insertion order is the
|
|
1503
|
+
// tie-break authority order, so "after corrections" gives semantic items
|
|
1504
|
+
// tie priority over palace/journal/insight lexical singles while the
|
|
1505
|
+
// owner's captured rules keep the top; "last" cedes all ties. Weight and
|
|
1506
|
+
// placement were selected on the golden eval — see the fix7 report's
|
|
1507
|
+
// placement/weight matrix, and applyRRF's doc comment for why weighted
|
|
1508
|
+
// RRF stays rank-based.
|
|
1092
1509
|
const rrfMap = new Map();
|
|
1510
|
+
let semanticFused = false; // idempotence guard — the leg fuses exactly once
|
|
1511
|
+
const fuseSemantic = () => {
|
|
1512
|
+
if (!semanticFused && semanticItems && semanticItems.length > 0) {
|
|
1513
|
+
semanticFused = true;
|
|
1514
|
+
applyRRF(semanticItems, rrfMap, SEMANTIC_RRF_WEIGHT);
|
|
1515
|
+
}
|
|
1516
|
+
};
|
|
1093
1517
|
for (const tier of input.tiers) {
|
|
1094
1518
|
const items = byTier[tier];
|
|
1095
1519
|
if (items)
|
|
1096
1520
|
applyRRF(items, rrfMap);
|
|
1521
|
+
if (SEMANTIC_AFTER_CORRECTIONS && tier === "corrections")
|
|
1522
|
+
fuseSemantic();
|
|
1097
1523
|
}
|
|
1524
|
+
// Last-position placement, and the fallback for tier sets that don't
|
|
1525
|
+
// include corrections at all (e.g. journalSearch's ["journal"]).
|
|
1526
|
+
if (!SEMANTIC_AFTER_CORRECTIONS || !input.tiers.includes("corrections"))
|
|
1527
|
+
fuseSemantic();
|
|
1098
1528
|
const fusedMap = fuseCanonical(rrfMap);
|
|
1099
|
-
|
|
1529
|
+
// fix4b (2026-09-12): the legacy multiplicative boost runs ONLY behind the
|
|
1530
|
+
// explicit freshnessBias opt-in — see QueryMemoryInput.freshnessBias.
|
|
1531
|
+
if (input.freshnessBias)
|
|
1532
|
+
applyLegacyHotWindowBoost(fusedMap);
|
|
1100
1533
|
const seen = new Set();
|
|
1101
1534
|
const fused = [];
|
|
1102
1535
|
for (const { score, item, sources } of fusedMap.values()) {
|
|
@@ -1107,11 +1540,18 @@ export async function queryMemory(input) {
|
|
|
1107
1540
|
const alsoFoundIn = [...sources].filter((s) => s !== item.source);
|
|
1108
1541
|
fused.push({ ...item, score, ...(alsoFoundIn.length > 0 ? { alsoFoundIn } : {}) });
|
|
1109
1542
|
}
|
|
1543
|
+
// fix4b (2026-09-12): the sort is stable, so exact fused-score ties
|
|
1544
|
+
// resolve purely by insertion order — the fix4 authority tie-break
|
|
1545
|
+
// (corrections first). No post-fusion freshness signal touches default
|
|
1546
|
+
// ranking; see the fallback doc block above applyLegacyHotWindowBoost.
|
|
1110
1547
|
fused.sort((a, b) => b.score - a.score);
|
|
1111
1548
|
return {
|
|
1112
1549
|
items: fused,
|
|
1113
1550
|
candidatesBySource,
|
|
1114
1551
|
sourcesQueried,
|
|
1552
|
+
// fix7: present ONLY under the opt-in — flag-off objects are shape-
|
|
1553
|
+
// identical to fix4b's (spread of undefined adds nothing).
|
|
1554
|
+
...(semanticNote ? { semanticLeg: semanticNote } : {}),
|
|
1115
1555
|
renderFenced(limit) {
|
|
1116
1556
|
const capped = typeof limit === "number" ? fused.slice(0, limit) : fused;
|
|
1117
1557
|
const lines = capped.map((it, i) => `[${i + 1}][${it.source}${it.room ? `/${it.room}` : ""}] ${it.title} — ${it.excerpt}`);
|