agent-recall-core 3.4.46 → 3.4.47

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (95) hide show
  1. package/dist/helpers/handoff.d.ts.map +1 -1
  2. package/dist/helpers/handoff.js +17 -9
  3. package/dist/helpers/handoff.js.map +1 -1
  4. package/dist/helpers/journal-files.d.ts +24 -1
  5. package/dist/helpers/journal-files.d.ts.map +1 -1
  6. package/dist/helpers/journal-files.js +29 -10
  7. package/dist/helpers/journal-files.js.map +1 -1
  8. package/dist/helpers/journal-filter.d.ts +25 -0
  9. package/dist/helpers/journal-filter.d.ts.map +1 -1
  10. package/dist/helpers/journal-filter.js +12 -1
  11. package/dist/helpers/journal-filter.js.map +1 -1
  12. package/dist/index.d.ts +12 -4
  13. package/dist/index.d.ts.map +1 -1
  14. package/dist/index.js +33 -4
  15. package/dist/index.js.map +1 -1
  16. package/dist/retrieval/candidates.d.ts +314 -0
  17. package/dist/retrieval/candidates.d.ts.map +1 -0
  18. package/dist/retrieval/candidates.js +321 -0
  19. package/dist/retrieval/candidates.js.map +1 -0
  20. package/dist/retrieval/contradiction.d.ts +232 -0
  21. package/dist/retrieval/contradiction.d.ts.map +1 -0
  22. package/dist/retrieval/contradiction.js +320 -0
  23. package/dist/retrieval/contradiction.js.map +1 -0
  24. package/dist/retrieval/query-memory.d.ts +419 -0
  25. package/dist/retrieval/query-memory.d.ts.map +1 -0
  26. package/dist/retrieval/query-memory.js +1068 -0
  27. package/dist/retrieval/query-memory.js.map +1 -0
  28. package/dist/retrieval/scope.d.ts +75 -0
  29. package/dist/retrieval/scope.d.ts.map +1 -0
  30. package/dist/retrieval/scope.js +91 -0
  31. package/dist/retrieval/scope.js.map +1 -0
  32. package/dist/storage/corrections.d.ts +31 -1
  33. package/dist/storage/corrections.d.ts.map +1 -1
  34. package/dist/storage/corrections.js +34 -1
  35. package/dist/storage/corrections.js.map +1 -1
  36. package/dist/storage/lifecycle-telemetry.d.ts +20 -5
  37. package/dist/storage/lifecycle-telemetry.d.ts.map +1 -1
  38. package/dist/storage/lifecycle-telemetry.js +16 -5
  39. package/dist/storage/lifecycle-telemetry.js.map +1 -1
  40. package/dist/storage/session.d.ts +28 -1
  41. package/dist/storage/session.d.ts.map +1 -1
  42. package/dist/storage/session.js +41 -8
  43. package/dist/storage/session.js.map +1 -1
  44. package/dist/supabase/recall-backend.d.ts +25 -2
  45. package/dist/supabase/recall-backend.d.ts.map +1 -1
  46. package/dist/supabase/recall-backend.js +82 -21
  47. package/dist/supabase/recall-backend.js.map +1 -1
  48. package/dist/supabase/sync.d.ts +41 -0
  49. package/dist/supabase/sync.d.ts.map +1 -1
  50. package/dist/supabase/sync.js +46 -0
  51. package/dist/supabase/sync.js.map +1 -1
  52. package/dist/tools-logic/check.d.ts.map +1 -1
  53. package/dist/tools-logic/check.js +62 -36
  54. package/dist/tools-logic/check.js.map +1 -1
  55. package/dist/tools-logic/drill-down.d.ts +24 -0
  56. package/dist/tools-logic/drill-down.d.ts.map +1 -1
  57. package/dist/tools-logic/drill-down.js +36 -0
  58. package/dist/tools-logic/drill-down.js.map +1 -1
  59. package/dist/tools-logic/journal-cold-start.d.ts.map +1 -1
  60. package/dist/tools-logic/journal-cold-start.js +48 -9
  61. package/dist/tools-logic/journal-cold-start.js.map +1 -1
  62. package/dist/tools-logic/journal-list.d.ts +42 -0
  63. package/dist/tools-logic/journal-list.d.ts.map +1 -1
  64. package/dist/tools-logic/journal-list.js +60 -10
  65. package/dist/tools-logic/journal-list.js.map +1 -1
  66. package/dist/tools-logic/journal-read.d.ts.map +1 -1
  67. package/dist/tools-logic/journal-read.js +22 -13
  68. package/dist/tools-logic/journal-read.js.map +1 -1
  69. package/dist/tools-logic/journal-search.d.ts +58 -0
  70. package/dist/tools-logic/journal-search.d.ts.map +1 -1
  71. package/dist/tools-logic/journal-search.js +123 -92
  72. package/dist/tools-logic/journal-search.js.map +1 -1
  73. package/dist/tools-logic/palace-search.d.ts.map +1 -1
  74. package/dist/tools-logic/palace-search.js +30 -11
  75. package/dist/tools-logic/palace-search.js.map +1 -1
  76. package/dist/tools-logic/palace-walk.d.ts +10 -0
  77. package/dist/tools-logic/palace-walk.d.ts.map +1 -1
  78. package/dist/tools-logic/palace-walk.js +25 -16
  79. package/dist/tools-logic/palace-walk.js.map +1 -1
  80. package/dist/tools-logic/recall-insight.d.ts +20 -0
  81. package/dist/tools-logic/recall-insight.d.ts.map +1 -1
  82. package/dist/tools-logic/recall-insight.js +12 -1
  83. package/dist/tools-logic/recall-insight.js.map +1 -1
  84. package/dist/tools-logic/session-end.d.ts.map +1 -1
  85. package/dist/tools-logic/session-end.js +23 -7
  86. package/dist/tools-logic/session-end.js.map +1 -1
  87. package/dist/tools-logic/session-start.d.ts +85 -0
  88. package/dist/tools-logic/session-start.d.ts.map +1 -1
  89. package/dist/tools-logic/session-start.js +245 -108
  90. package/dist/tools-logic/session-start.js.map +1 -1
  91. package/dist/tools-logic/smart-recall.d.ts +87 -62
  92. package/dist/tools-logic/smart-recall.d.ts.map +1 -1
  93. package/dist/tools-logic/smart-recall.js +159 -510
  94. package/dist/tools-logic/smart-recall.js.map +1 -1
  95. package/package.json +1 -1
@@ -1,6 +1,34 @@
1
1
  /**
2
2
  * smart_recall — unified cross-store search. v3.3.14
3
3
  *
4
+ * ── WAVE 2 (2026-08-29/30, plywood SOP ecbd4351) ────────────────────────────
5
+ * Internals migrated onto the shared retrieval pipeline
6
+ * (`retrieval/query-memory.ts`'s `queryMemory()`). EXTERNAL CONTRACT
7
+ * UNCHANGED: same `SmartRecallInput`/`SmartRecallResult` shape, same
8
+ * feedback-rating footer, same archive-fallback/bridge/graph-walk behavior.
9
+ * The composition that used to live directly in this file — journalSearch +
10
+ * palaceSearch + recallInsight + archiveSearch, plus the 5 scattered scoring
11
+ * formulas below and the two-stage RRF fusion — now lives in
12
+ * `retrieval/query-memory.ts` as MANDATORY PIPELINE STAGES (fetch ->
13
+ * trust-filter -> tokenize+score -> scope -> rank/fuse -> fence). See that
14
+ * file's header comment for the full CHALLENGE-by-CHALLENGE design reasoning
15
+ * (why scoring stays per-tier/pluggable rather than unified into one formula;
16
+ * how the Wave-1 candidate superset is handled per source; why FENCE is not
17
+ * applied to this surface's per-item fields). This file now owns exactly the
18
+ * smart_recall-SPECIFIC post-pipeline concerns that do not generalize to
19
+ * other future `queryMemory()` consumers: confidence calibration
20
+ * (`calibratedConfidence`), the Beta-feedback multiplier + feedback log, the
21
+ * remote/local backend race + degraded-timeout handling, the Bridge
22
+ * drilldown, and the F4 archive-fallback's confidence GATE (the policy
23
+ * decision of *when* to call `queryArchiveFallback` — its fetch/score logic
24
+ * itself moved to query-memory.ts).
25
+ *
26
+ * The formulas described below (Fix 1-5b) are UNCHANGED in substance — they
27
+ * are simply now implemented in query-memory.ts's per-tier scoring functions
28
+ * and RANK/FUSE stage instead of inline here. Kept as historical/design
29
+ * documentation because the RATIONALE is still exactly why those formulas
30
+ * are what they are.
31
+ *
4
32
  * ## Scoring Architecture (why it works this way)
5
33
  *
6
34
  * ### Problem with the old approach (< v3.3.14): Linear Score Fusion
@@ -25,7 +53,6 @@
25
53
  * Formula: R(t) = e^(-t/S), where S = memory strength (days).
26
54
  * Different memory types have different S values based on psychological research:
27
55
  * - Journal (episodic, low meaning): S = 2 → 60% retained after 1 day
28
- * - Knowledge/bug-fix (procedural): S = 180 → 99.4% retained after 1 day
29
56
  * - Palace/decisions (semantic): S = 9999 → barely decays
30
57
  * - Insight (conceptual): not time-based; uses confirmation count instead
31
58
  * This replaces the uniform 0.95^days that treated all memory equally.
@@ -59,62 +86,35 @@
59
86
  * duplicates (same id) could. A later "dedup by excerpt" pass then silently
60
87
  * collapsed same-excerpt entries by first-inserted-wins, DISCARDING the other
61
88
  * source's score entirely instead of summing it in.
62
- * Fix: fusion is now TWO stages. Stage 1 (applyRRF, unchanged key: `item.id`)
63
- * still consolidates multiple hits from the SAME source document — e.g.
64
- * palaceSearch legitimately returns one hit per matching LINE within a file,
65
- * all sharing that file's id, and those must accumulate into one per-document
66
- * entry, not fragment (an excerpt-only key at this stage was tried and
67
- * REJECTED — it broke associative-link.test.mjs by starving genuinely-
68
- * matched files of their combined per-file weight). Stage 2 (fuseCanonical)
69
- * then re-keys those already-consolidated per-document entries by NORMALIZED
70
- * EXCERPT CONTENT — the same identity notion the old post-hoc dedup pass
71
- * already used to decide "same memory". Two per-document entries from
72
- * different sources whose representative excerpt matches now land in the
73
- * SAME canonical entry, so cross-source accumulation happens. Provenance
74
- * from every contributing source is preserved via `alsoFoundIn` on the fused
75
- * item (primary/display source is whichever source ran first — unchanged),
76
- * rather than being dropped.
89
+ * Fix: fusion is now TWO stages. Stage 1 (applyRRF, keyed by `item.id`) still
90
+ * consolidates multiple hits from the SAME source document. Stage 2
91
+ * (fuseCanonical) then re-keys those already-consolidated per-document
92
+ * entries by NORMALIZED EXCERPT CONTENT. Provenance from every contributing
93
+ * source is preserved via `alsoFoundIn` on the fused item, rather than being
94
+ * dropped. (Both stages live in query-memory.ts now — see that file's
95
+ * `applyRRF`/`fuseCanonical`.)
77
96
  *
78
97
  * ### Fix 5b: insight excerpt is too low-entropy to be a fusion identity
79
98
  * Stage 2's "normalized excerpt content" identity assumption (Fix 5) is
80
99
  * sound for palace/journal, whose `excerpt` is a real matched text snippet.
81
100
  * It was broken for the insight source: its excerpt was synthesized from
82
- * ONLY `severity` + `applies_when` (`[important] deployment, database`),
83
- * omitting the insight's own distinguishing `title` entirely. Two UNRELATED
84
- * insights sharing a severity + tag set produced byte-identical excerpts and
85
- * silently fused into one — one insight vanished from results, and the
86
- * survivor's score absorbed the other's RRF contribution with zero trace
87
- * (same-source collisions don't show up in `alsoFoundIn`, which only records
88
- * source NAMES). Fix: insight items now carry a separate `fusionKey`
101
+ * ONLY `severity` + `applies_when`, omitting the insight's own distinguishing
102
+ * `title` entirely. Fix: insight items now carry a separate `fusionKey`
89
103
  * (`${title} [severity] tags`) that fuseCanonical() and the defensive dedup
90
- * pass key on INSTEAD of `excerpt` when present (see `fusionKey`'s doc
91
- * comment on `SmartRecallResultItem`). The displayed `excerpt` stays terse —
92
- * embedding `title` into it would duplicate title text in every consumer
93
- * that reads `excerpt` directly (recall.ts's display line, the CLI's
94
- * ambient-injection word-overlap scorer, consistency/conflict-scan's
95
- * `title + excerpt` concatenation), which for the word-overlap scorer would
96
- * also double-count title tokens and inflate insight relevance versus other
97
- * sources. Palace/journal items leave `fusionKey` unset (falls back to
98
- * `excerpt`, unchanged from Fix 5) — insights are a distinct "confirmed
99
- * pattern" memory type that isn't expected to share literal content with a
100
- * journal/palace excerpt, so this doesn't reopen the cross-source fusion
101
- * Fix 5 was built for; it only stops insights colliding WITH EACH OTHER.
104
+ * pass key on INSTEAD of `excerpt` when present.
102
105
  */
103
106
  import * as fs from "node:fs";
104
107
  import * as path from "node:path";
105
- import { palaceSearch } from "./palace-search.js";
106
- import { journalSearch } from "./journal-search.js";
107
- import { recallInsight } from "./recall-insight.js";
108
108
  import { getRoot } from "../types.js";
109
109
  import { ensureDir } from "../storage/fs-utils.js";
110
- import { stem, expandQuery } from "../helpers/normalize.js";
110
+ import { expandQuery } from "../helpers/normalize.js";
111
111
  import { tokenizeWords } from "../helpers/tokenize.js";
112
112
  import { getConnectedRooms } from "../palace/graph.js";
113
- import { palaceDir, archiveRawDir } from "../storage/paths.js";
113
+ import { palaceDir } from "../storage/paths.js";
114
114
  import { calibratedConfidence, CONFIDENCE_FLOOR } from "./confidence.js";
115
115
  import { fetchVerbatim } from "./drill-down.js";
116
116
  import { resolveProject } from "../storage/project.js";
117
- import { scrubForCloud } from "../storage/content-guard.js";
117
+ import { queryMemory, queryArchiveFallback } from "../retrieval/query-memory.js";
118
118
  /** Compute both the human label and the stored calibrated value for a score. */
119
119
  function label(score, scale) {
120
120
  const c = calibratedConfidence(score, scale);
@@ -123,105 +123,15 @@ function label(score, scale) {
123
123
  // ---------------------------------------------------------------------------
124
124
  // Constants
125
125
  // ---------------------------------------------------------------------------
126
- /** RRF constant. k=60 is the empirically validated default (Cormack et al. 2009). */
127
- const RRF_K = 60;
128
- /**
129
- * Ebbinghaus memory strength S (days) per source type.
130
- * R(t) = e^(-t/S): higher S = slower decay.
131
- * Journal decays fast (low-meaning episodic); palace barely decays (semantic).
132
- */
133
- const EBBINGHAUS_S = {
134
- journal: 2, // ~60% retained after 1 day, ~7% after 1 week
135
- knowledge: 180, // ~99.4% after 1 day, ~84.6% after 1 month
136
- palace: 9999, // effectively no decay
137
- };
138
126
  /**
139
- * Max items the explicit archive-fallback source (F4, see archiveSearch
140
- * below) may append to a single smartRecall() call. Kept small — this is a
141
- * confidence-gated last resort, not a competing ranked source.
127
+ * Max items the explicit archive-fallback source (F4, see the gate inside
128
+ * smartRecall() below) may append to a single smartRecall() call. Kept small
129
+ * — this is a confidence-gated last resort, not a competing ranked source.
142
130
  */
143
131
  const ARCHIVE_SOURCE_CAP = 3;
144
132
  // ---------------------------------------------------------------------------
145
- // Math helpers
133
+ // Feedback store
146
134
  // ---------------------------------------------------------------------------
147
- /**
148
- * Normalize excerpt text into a canonical cross-source identity key.
149
- * Two items from different sources that describe the same conceptual memory
150
- * typically carry byte-identical (or near-identical) excerpt text — this is
151
- * the same identity notion the old post-hoc dedup pass (Step 5) already used
152
- * to decide "same memory"; it's now applied at RRF fusion time instead
153
- * (Fix 5), so cross-source accumulation happens naturally.
154
- */
155
- function normalizeExcerpt(excerpt) {
156
- return excerpt.toLowerCase().replace(/\s+/g, " ").trim();
157
- }
158
- /**
159
- * Cross-source fusion identity for an item (Fix 5b). Prefers the item's
160
- * `fusionKey` override (set by the insight source, whose displayed `excerpt`
161
- * is too low-entropy — see `fusionKey`'s doc comment on
162
- * `SmartRecallResultItem`) and falls back to the displayed `excerpt` for
163
- * every other source, unchanged from Fix 5's original design.
164
- */
165
- function fusionIdentity(item) {
166
- return normalizeExcerpt(item.fusionKey ?? item.excerpt);
167
- }
168
- /**
169
- * Internal side channel: raw per-source candidate counts (Fix 4/5), attached
170
- * to the array localRecallSearch returns so smartRecall() can report a
171
- * genuine pre-fusion total_searched without changing localRecallSearch's
172
- * public return type (still a plain SmartRecallResultItem[] — several
173
- * existing tests and recall-backend.ts depend on that exact shape).
174
- */
175
- const RAW_CANDIDATE_COUNTS = Symbol("rawCandidateCounts");
176
- /** Simple stable hash for result IDs. */
177
- function stableId(source, title) {
178
- let hash = 0;
179
- const str = `${source}:${title}`;
180
- for (let i = 0; i < str.length; i++) {
181
- const char = str.charCodeAt(i);
182
- hash = ((hash << 5) - hash) + char;
183
- hash |= 0;
184
- }
185
- return Math.abs(hash).toString(36).slice(0, 8);
186
- }
187
- /** Days elapsed since a date string. */
188
- function daysSince(dateStr) {
189
- const d = new Date(dateStr);
190
- if (isNaN(d.getTime()))
191
- return 365;
192
- return Math.max(0, (Date.now() - d.getTime()) / (1000 * 60 * 60 * 24));
193
- }
194
- /**
195
- * Ebbinghaus forgetting curve: R(t) = e^(-t/S).
196
- * Returns retention [0,1] after `days` with strength S.
197
- */
198
- function ebbinghaus(days, S) {
199
- return Math.exp(-days / S);
200
- }
201
- /**
202
- * Keyword overlap ratio between query and text, with stemming + synonym expansion.
203
- * CJK-aware (P0-b, 2026-08-18): both sides go through the shared tokenizer
204
- * so an unspaced Chinese/Japanese query/text segments into real words
205
- * instead of one giant token that must match byte-for-byte (2026-08-18 L1
206
- * eval: CJK hit@5 = 0/6). Default minLength=3 preserves the original
207
- * `length > 2` floor for ASCII.
208
- */
209
- function keywordExactness(query, text) {
210
- const rawWords = tokenizeWords(query);
211
- if (rawWords.length === 0)
212
- return 0;
213
- // Expand query with stems + synonyms
214
- const expandedQuery = expandQuery(rawWords);
215
- // Stem the text words for matching
216
- const textWords = tokenizeWords(text).map(w => stem(w));
217
- const textSet = new Set(textWords);
218
- // Also check raw text for direct substring matches (preserves old behavior)
219
- const textLower = text.toLowerCase();
220
- // Count matches: expanded query word found in stemmed text OR as substring
221
- const matches = expandedQuery.filter(w => textSet.has(w) || textLower.includes(w));
222
- // Score relative to ORIGINAL query length (not expanded), capped at 1.0
223
- return Math.min(1.0, matches.length / rawWords.length);
224
- }
225
135
  /**
226
136
  * Beta distribution expected value for binary feedback.
227
137
  * E[Beta(α,β)] = α/(α+β) where α=pos+1, β=neg+1 (Laplace smoothing).
@@ -279,287 +189,98 @@ function getFeedbackCounts(id, title, queryWords, log) {
279
189
  negatives: relevant.filter((f) => match(f) && !f.useful).length,
280
190
  };
281
191
  }
282
- /**
283
- * Apply Reciprocal Rank Fusion scores from a ranked list of items.
284
- * Mutates the provided rrfMap in place.
285
- * RRF_score += 1 / (k + rank)
286
- *
287
- * STAGE 1 of fusion — keyed by `item.id` (the per-source occurrence id,
288
- * `stableId(source, title)`), exactly as before Fix 5. This intentionally
289
- * stays id-keyed: a single source document can legitimately produce SEVERAL
290
- * distinct-excerpt hits that share one `item.id` (e.g. palaceSearch returns
291
- * one hit per matching LINE within the same room/file, all sharing that
292
- * file's id) — those must accumulate into ONE per-document entry, not
293
- * fragment into many. Keying this stage by excerpt instead (as an earlier
294
- * version of this fix mistakenly did) breaks exactly that: it turned a
295
- * single matched file into N separate low-score fragments and starved
296
- * genuinely-different documents of their combined per-file RRF weight
297
- * (caught by associative-link.test.mjs). Cross-SOURCE fusion (the actual
298
- * bug this file's Fix 5 addresses) happens in a SEPARATE stage afterward —
299
- * see fuseCanonical() below — once each source's own per-document hits are
300
- * already consolidated here.
301
- */
302
- function applyRRF(rankedItems, rrfMap) {
303
- rankedItems.forEach((item, idx) => {
304
- const rank = idx + 1;
305
- const contribution = 1 / (RRF_K + rank);
306
- const existing = rrfMap.get(item.id);
307
- if (existing) {
308
- existing.score += contribution;
309
- }
310
- else {
311
- rrfMap.set(item.id, { score: contribution, item, sources: new Set([item.source]) });
312
- }
313
- });
314
- }
315
- /**
316
- * STAGE 2 of fusion (Fix 5) — cross-source canonical merge, run AFTER
317
- * applyRRF() has consolidated each source's own per-document hits (Stage 1).
318
- * Re-keys the (already within-source-deduped) entries by `fusionIdentity()`
319
- * (Fix 5b: `item.fusionKey` when set, else `normalizeExcerpt(item.excerpt)`)
320
- * — the same identity notion Step 5's old post-hoc dedup pass used to decide
321
- * "same memory". Two per-document entries from DIFFERENT sources (or,
322
- * rarely, the same source) whose identity is the same after normalization
323
- * are now genuinely the SAME conceptual memory, so their scores are summed
324
- * and their sources merged — instead of the old behavior where a later dedup
325
- * pass silently discarded one side's score entirely.
326
- * Iteration order of `rrfMap` is insertion order (palace's entries first,
327
- * then journal, then insight — see localRecallSearch's call order), so ties
328
- * keep the same source-precedence the rest of this file already assumes.
329
- */
330
- function fuseCanonical(rrfMap) {
331
- const canonical = new Map();
332
- for (const entry of rrfMap.values()) {
333
- const key = fusionIdentity(entry.item);
334
- const existing = canonical.get(key);
335
- if (existing) {
336
- existing.score += entry.score;
337
- for (const s of entry.sources)
338
- existing.sources.add(s);
339
- }
340
- else {
341
- canonical.set(key, { score: entry.score, item: entry.item, sources: new Set(entry.sources) });
342
- }
343
- }
344
- return canonical;
345
- }
346
192
  // ---------------------------------------------------------------------------
347
193
  // Main
348
194
  // ---------------------------------------------------------------------------
195
+ /** Reconstruct a `VerbatimKey` for the Bridge from a pipeline item's generic
196
+ * `room`/`file`/`date` fields — the pipeline itself stays decoupled from
197
+ * drill-down.ts's `VerbatimKey` shape (see query-memory.ts's own comment on
198
+ * `QueryMemoryItem.file`). */
199
+ function verbatimKeyFor(item) {
200
+ if (item.source === "journal" && item.date)
201
+ return { kind: "journal", date: item.date };
202
+ if (item.source === "palace" && item.room && item.file)
203
+ return { kind: "palace", room: item.room, file: item.file };
204
+ if (item.source === "archive" && item.file)
205
+ return { kind: "archive", date: item.date, file: item.file };
206
+ return undefined;
207
+ }
349
208
  /**
350
209
  * localRecallSearch — the core local search logic (palace + journal + insight).
351
- * Performs three parallel searches, scores internally, merges via RRF,
352
- * applies hot-window recency boost, and deduplicates.
353
210
  *
354
- * Called by LocalRecallBackend.search() in recall-backend.ts.
355
- * Feedback multiplier is NOT applied here — that is done in smartRecall()
356
- * so it applies uniformly across all backends.
211
+ * WAVE 2: delegates FETCH/TRUST-FILTER/TOKENIZE+SCORE/RANK-FUSE entirely to
212
+ * `retrieval/query-memory.ts`'s `queryMemory()` — see this file's header.
213
+ * This function's own remaining job: resolve `project` (queryMemory()
214
+ * requires an already-resolved slug, matching Wave 1's `readTierCandidates`
215
+ * convention — previously this resolution happened independently 3x, inside
216
+ * journalSearch/palaceSearch/nowhere-for-insight; now once, here — a
217
+ * characterized simplification, not a behavior change for any caller passing
218
+ * an already-valid explicit slug, which is every existing test and the
219
+ * common real-world case), label each fused item with a calibrated
220
+ * confidence (rrf-local scale, matching the original dedup loop exactly),
221
+ * reconstruct `verbatimKey`, run the graph-walk 1-hop related-room surfacing
222
+ * (smart_recall-specific; no other Wave-3 migration target has this), and
223
+ * attach the raw-candidate-count side channel.
224
+ *
225
+ * Called by LocalRecallBackend.search() in recall-backend.ts, and directly
226
+ * by helpers/associative-link.ts and (in tests) by
227
+ * audit-retrieval-accounting.test.mjs — its signature and the RAW_CANDIDATE_COUNTS
228
+ * side-channel contract are UNCHANGED, since those are real, direct external
229
+ * callers, not just smartRecall()'s own internals.
357
230
  */
358
231
  export async function localRecallSearch(query, project, limit, since) {
359
- const sourcesQueried = [];
360
- // Candidate buckets — each source scores its items internally, then RRF merges
361
- const palaceItems = [];
362
- const journalItems = [];
363
- const insightItems = [];
364
- // ── 1. Palace ────────────────────────────────────────────────────────────
365
- // Internal score: keyword match quality × salience (structural importance).
366
- // Ebbinghaus decay is minimal for palace (S=9999); salience already
367
- // incorporates access recency via recordAccess().
368
- try {
369
- const palaceResults = await palaceSearch({ query, project, limit: limit * 2 });
370
- sourcesQueried.push("palace");
371
- for (const r of palaceResults.results) {
372
- const title = `${r.room}/${r.file}`;
373
- const id = stableId("palace", title);
374
- // keyword_score comes from updated palace-search (keyword overlap, not substring).
375
- // salience floor of 0.4 prevents new rooms (salience=0.5) from being unfairly
376
- // penalized against rooms with years of access history.
377
- const keyScore = r.keyword_score ?? keywordExactness(query, r.excerpt);
378
- const salience = Math.max(0.4, r.salience);
379
- const internalScore = keyScore * 0.65 + salience * 0.35;
380
- // Try to extract a date from the excerpt (many palace entries have ### YYYY-MM-DD headers)
381
- let palaceDate;
382
- const datePattern = r.excerpt.match(/(\d{4}-\d{2}-\d{2})/);
383
- if (datePattern) {
384
- palaceDate = datePattern[1];
385
- }
386
- palaceItems.push({
387
- id,
388
- source: "palace",
389
- title,
390
- excerpt: r.excerpt,
391
- score: internalScore,
392
- // internalScore is already 0..1 → cosine scale (NOT rrf-local).
393
- ...label(internalScore, "cosine"),
394
- verbatimKey: { kind: "palace", room: r.room, file: r.file },
395
- room: r.room,
396
- date: palaceDate,
397
- });
398
- }
399
- }
400
- catch { /* palace may not be initialized */ }
401
- // ── 2. Journal ───────────────────────────────────────────────────────────
402
- // Internal score: Ebbinghaus decay (S=2 days, fast) × keyword match.
403
- // Journal is ephemeral — recent entries are useful; old ones rarely are.
404
- try {
405
- const journalResults = await journalSearch({
406
- query,
407
- project,
408
- include_palace: false,
409
- limit: Math.ceil(limit * 1.5),
410
- since,
411
- });
412
- sourcesQueried.push("journal");
413
- for (const r of journalResults.results) {
414
- const title = `${r.date} / ${r.section}`;
415
- const id = stableId("journal", title);
416
- const days = daysSince(r.date);
417
- const recency = ebbinghaus(days, EBBINGHAUS_S.journal);
418
- const exactness = keywordExactness(query, r.excerpt);
419
- // Equal weight: if the entry is recent AND relevant it scores well.
420
- // Old journal entries drop fast due to S=2.
421
- const internalScore = recency * 0.50 + exactness * 0.50;
422
- journalItems.push({
423
- id,
424
- source: "journal",
425
- title,
426
- excerpt: r.excerpt,
427
- score: internalScore,
428
- // internalScore is already 0..1 → cosine scale.
429
- ...label(internalScore, "cosine"),
430
- verbatimKey: { kind: "journal", date: r.date },
431
- date: r.date,
432
- });
433
- }
434
- }
435
- catch { /* journal may not exist */ }
436
- // ── 3. Insights ──────────────────────────────────────────────────────────
437
- // Internal score: keyword relevance × confirmation signal (log-scaled).
438
- // Insights are timeless learned patterns — confirmation count is the signal,
439
- // not recency. More confirmations = more reliable.
232
+ let resolvedProject;
440
233
  try {
441
- const insightResults = await recallInsight({
442
- context: query,
443
- limit: limit * 2,
444
- include_awareness: false,
445
- });
446
- sourcesQueried.push("insight");
447
- const maxRelevance = Math.max(1, ...insightResults.matching_insights.map((i) => i.relevance));
448
- for (const i of insightResults.matching_insights) {
449
- const id = stableId("insight", i.title);
450
- const relevance = i.relevance / maxRelevance;
451
- const exactness = keywordExactness(query, i.title);
452
- // log2(confirmed+1)/3 gives: 0→0, 1→0.33, 3→0.67, 7→1.0
453
- const confirmation = Math.min(1.0, Math.log2(i.confirmed + 1) / 3);
454
- const internalScore = relevance * 0.40 + exactness * 0.35 + confirmation * 0.25;
455
- // Terse, display-friendly excerpt — deliberately just metadata (kept
456
- // this way for recall.ts / CLI ambient-injection compactness; see
457
- // `fusionKey`'s doc comment on SmartRecallResultItem for why the
458
- // insight's `title` is NOT folded in here).
459
- const rawExcerpt = `[${i.severity}] ${i.applies_when.join(", ")}`;
460
- // Fix 5b: `rawExcerpt` alone (severity+tags only) is too low-entropy to
461
- // serve as a cross-source fusion identity — two UNRELATED insights that
462
- // happen to share a severity + applies_when tag set would otherwise
463
- // collide in fuseCanonical() and silently absorb one another's score.
464
- // Leading with the insight's own `title` (its real distinguishing
465
- // content) makes that structurally impossible.
466
- const fusionSeed = `${i.title} ${rawExcerpt}`;
467
- insightItems.push({
468
- id,
469
- source: "insight",
470
- title: i.title,
471
- excerpt: rawExcerpt.length > 300 ? rawExcerpt.slice(0, 300) + "..." : rawExcerpt,
472
- fusionKey: fusionSeed.length > 300 ? fusionSeed.slice(0, 300) + "..." : fusionSeed,
473
- score: internalScore,
474
- // internalScore is already 0..1 → cosine scale.
475
- ...label(internalScore, "cosine"),
476
- severity: i.severity,
477
- });
478
- }
234
+ resolvedProject = await resolveProject(project);
479
235
  }
480
- catch { /* insights may be empty */ }
481
- // ── 4. Rank within each source, then merge via RRF ───────────────────────
482
- // Each source ranks by its own internal score (apples vs apples).
483
- // RRF then combines by rank position — no cross-source score comparison.
484
- palaceItems.sort((a, b) => b.score - a.score);
485
- journalItems.sort((a, b) => b.score - a.score);
486
- insightItems.sort((a, b) => b.score - a.score);
487
- // Fix 4/5: raw per-source candidate counts, captured BEFORE fusion
488
- // collapses same-excerpt cross-source duplicates into one canonical entry.
489
- // This is the number smartRecall()'s total_searched should report.
490
- const rawCandidateCounts = {
491
- palace: palaceItems.length,
492
- journal: journalItems.length,
493
- insight: insightItems.length,
494
- };
495
- // Stage 1: within-source, per-document accumulation (unchanged from
496
- // pre-Fix-5 behavior — see applyRRF's doc comment).
497
- const rrfMap = new Map();
498
- applyRRF(palaceItems, rrfMap);
499
- applyRRF(journalItems, rrfMap);
500
- applyRRF(insightItems, rrfMap);
501
- // Stage 2 (Fix 5): cross-source canonical fusion by normalized excerpt.
502
- const fusedMap = fuseCanonical(rrfMap);
503
- // ── 4.5. Hot-window recency boost ──────────────────────────────────────
504
- // Very recent items get a score multiplier. In active project work,
505
- // the most recent context is almost always the most relevant.
506
- // This supplements Ebbinghaus decay (which handles medium-term) with
507
- // an ultra-short-term boost.
508
- // Palace items have date: undefined — they are timeless and unaffected.
509
- for (const entry of fusedMap.values()) {
510
- if (entry.item.date) {
511
- const hoursAgo = (Date.now() - new Date(entry.item.date).getTime()) / (1000 * 60 * 60);
512
- if (hoursAgo < 6) {
513
- entry.score *= 3.0;
514
- }
515
- else if (hoursAgo < 24) {
516
- entry.score *= 2.0;
517
- }
518
- else if (hoursAgo < 72) {
519
- entry.score *= 1.3;
520
- }
521
- // > 72 hours: no boost (normal decay handles it)
522
- }
523
- }
524
- // ── 5. Deduplicate by fusion identity ────────────────────────────────────
525
- // Defensive pass only (Fix 5): fusedMap is already keyed by
526
- // fusionIdentity() (Stage 2 above, Fix 5b), so it is structurally
527
- // impossible for two entries already in fusedMap to share an identity —
528
- // the real cross-source fusion job now happens in fuseCanonical(), not
529
- // here. This loop is kept as a cheap safety net for any future code path
530
- // that might append near-duplicate entries, and to keep score/label
531
- // materialization + `alsoFoundIn` derivation in one place.
532
- const seen = new Set();
533
- const deduped = [];
534
- for (const { score, item, sources } of fusedMap.values()) {
535
- const key = fusionIdentity(item);
536
- if (seen.has(key))
537
- continue;
538
- seen.add(key);
539
- // Provenance: every OTHER source (besides the primary/display `item.source`)
540
- // that contributed a hit fusing into this canonical entry (Fix 5).
541
- const alsoFoundIn = [...sources].filter((s) => s !== item.source);
542
- // fusionKey is an internal-only identity override (Fix 5b) — strip it
543
- // before materializing the public result item so it never leaks into
544
- // smart_recall's JSON output (the smart_recall MCP tool serializes
545
- // SmartRecallResultItem[] verbatim via JSON.stringify), matching this
546
- // file's existing discipline of keeping internal-only signals out of
547
- // consumer-facing objects (see the RAW_CANDIDATE_COUNTS side channel).
548
- const { fusionKey: _fusionKey, ...displayItem } = item;
549
- // Post-RRF (+ hot-window boost) score → rrf-local scale. This SET of
550
- // `calibrated` is what the bridge gate reads (NOT the later boosted score).
551
- deduped.push({
552
- ...displayItem,
553
- score,
554
- ...label(score, "rrf-local"),
555
- ...(alsoFoundIn.length > 0 ? { alsoFoundIn } : {}),
556
- });
236
+ catch {
237
+ resolvedProject = project ?? "auto";
557
238
  }
558
- // ── 6. Final sort ─────────────────────────────────────────────────────────
559
- deduped.sort((a, b) => b.score - a.score);
560
- // Graph walk — surface 1-hop linked memories not already in results
561
- if (deduped.length > 0 && project) {
562
- const pd = palaceDir(project);
239
+ const result = await queryMemory({
240
+ query,
241
+ project: resolvedProject,
242
+ // Order matters: RRF/fuseCanonical's "primary/display source" is
243
+ // whichever source's items were inserted into the fusion map FIRST (Map
244
+ // iteration = insertion order). The ORIGINAL localRecallSearch queried
245
+ // palace, then journal, then insight — this order must be preserved
246
+ // exactly (audit-retrieval-accounting.test.mjs asserts on it directly).
247
+ tiers: ["palace", "journal", "insight"],
248
+ limit,
249
+ since,
250
+ });
251
+ // Final materialization: rrf-local confidence label (matches the ORIGINAL
252
+ // dedup loop's `...label(score, "rrf-local")` — the only labeling that
253
+ // ever survived to the final result; a pre-fusion "cosine" label was
254
+ // computed by the old code too but was always overwritten here, so
255
+ // query-memory.ts's pipeline items never compute it at all — dead weight
256
+ // correctly dropped, not a behavior change).
257
+ const deduped = result.items.map((item) => ({
258
+ id: item.id,
259
+ source: item.source,
260
+ ...(item.alsoFoundIn && item.alsoFoundIn.length > 0 ? { alsoFoundIn: item.alsoFoundIn } : {}),
261
+ title: item.title,
262
+ excerpt: item.excerpt,
263
+ score: item.score,
264
+ ...label(item.score, "rrf-local"),
265
+ verbatimKey: verbatimKeyFor(item),
266
+ ...(item.room ? { room: item.room } : {}),
267
+ ...(item.date ? { date: item.date } : {}),
268
+ ...(item.severity ? { severity: item.severity } : {}),
269
+ // W5a salvage (HIGH-3, 2026-08-31): thread the CONTRADICTION stage's
270
+ // annotation through — this field-list map was previously the exact
271
+ // place `supersededBy`/`conflictsWith` were silently dropped, even
272
+ // though they had already, invisibly, changed this item's `score`/rank.
273
+ ...(item.supersededBy ? { supersededBy: item.supersededBy } : {}),
274
+ ...(item.conflictsWith && item.conflictsWith.length > 0 ? { conflictsWith: item.conflictsWith } : {}),
275
+ }));
276
+ // Graph walk — surface 1-hop linked memories not already in results.
277
+ // Uses the RESOLVED project (a characterized fix over the original, which
278
+ // used the raw, possibly-unresolved `project` parameter here — a latent
279
+ // H1-class inconsistency for the "auto"-literal edge case; every existing
280
+ // caller passes an already-resolved explicit slug, so this is a no-op
281
+ // difference for the common case and a strict improvement otherwise).
282
+ if (deduped.length > 0 && resolvedProject) {
283
+ const pd = palaceDir(resolvedProject);
563
284
  const resultIds = new Set(deduped.map((r) => r.id));
564
285
  const topRoom = deduped[0].room;
565
286
  if (topRoom) {
@@ -585,107 +306,22 @@ export async function localRecallSearch(query, project, limit, since) {
585
306
  // Attach the raw pre-fusion candidate counts as a hidden side channel
586
307
  // (Fix 4/5) — invisible to JSON.stringify/Object.keys/for-in and to every
587
308
  // existing consumer that treats this as a plain SmartRecallResultItem[].
309
+ const rawCandidateCounts = {
310
+ palace: result.candidatesBySource.palace ?? 0,
311
+ journal: result.candidatesBySource.journal ?? 0,
312
+ insight: result.candidatesBySource.insight ?? 0,
313
+ };
588
314
  deduped[RAW_CANDIDATE_COUNTS] = rawCandidateCounts;
589
315
  return deduped;
590
316
  }
591
317
  /**
592
- * Explicit 4th source (F4, continuity wave 2026-07-31): journal/archive/raw/
593
- * mechanical transcript dumps, line-grepped for the query. NEVER part of the
594
- * palace/journal/insight RRF fusion above — this only runs from smartRecall()
595
- * itself, gated on the fused top result being below medium confidence (see
596
- * the call site). Every returned item's score is hard-capped just under
597
- * CONFIDENCE_FLOOR.medium on the "cosine" scale, so it can NEVER be labeled
598
- * "medium"/"high" — the label + provenance path are the load-bearing
599
- * contract here, not the raw keyword-overlap number. This is the gated,
600
- * labeled replacement for the old accidental `journalDirs(..., true)`
601
- * descent into archive/raw (see journalDirs' doc comment in storage/paths.ts
602
- * for the incident this fixes).
603
- *
604
- * Deliberately unranked beyond "newest file first, top-to-bottom within a
605
- * file" (same line-grep flavor as journalSearch) — this is a last-resort
606
- * fallback, not a competing ranked source, so it stops as soon as `limit`
607
- * items are collected rather than scanning every archive file to find the
608
- * globally-best matches.
318
+ * Internal side channel: raw per-source candidate counts (Fix 4/5), attached
319
+ * to the array localRecallSearch returns so smartRecall() can report a
320
+ * genuine pre-fusion total_searched without changing localRecallSearch's
321
+ * public return type (still a plain SmartRecallResultItem[] — several
322
+ * existing tests and recall-backend.ts depend on that exact shape).
609
323
  */
610
- function archiveSearch(project, query, limit) {
611
- const dir = archiveRawDir(project);
612
- if (!fs.existsSync(dir))
613
- return [];
614
- // CJK-aware (P0-b): shared tokenizer — see keywordExactness above for the
615
- // same fix applied to this fallback source's query-side matching.
616
- const keywords = tokenizeWords(query);
617
- if (keywords.length === 0 || limit <= 0)
618
- return [];
619
- let files;
620
- try {
621
- files = fs.readdirSync(dir).filter((f) => f.endsWith(".md"));
622
- }
623
- catch {
624
- return [];
625
- }
626
- // Newest raw dump first — a low-confidence fallback should prefer recent work.
627
- files.sort((a, b) => b.localeCompare(a));
628
- const items = [];
629
- for (const file of files) {
630
- if (items.length >= limit)
631
- break;
632
- let content;
633
- try {
634
- content = fs.readFileSync(path.join(dir, file), "utf-8");
635
- }
636
- catch {
637
- continue;
638
- }
639
- const dateMatch = file.match(/^(\d{4}-\d{2}-\d{2})/);
640
- const date = dateMatch ? dateMatch[1] : undefined;
641
- const lines = content.split("\n");
642
- for (let i = 0; i < lines.length; i++) {
643
- if (items.length >= limit)
644
- break;
645
- const line = lines[i];
646
- const lineLower = line.toLowerCase();
647
- if (!keywords.some((kw) => lineLower.includes(kw)))
648
- continue;
649
- let matchIdx = line.length;
650
- for (const kw of keywords) {
651
- const idx = lineLower.indexOf(kw);
652
- if (idx !== -1 && idx < matchIdx)
653
- matchIdx = idx;
654
- }
655
- const start = Math.max(0, matchIdx - 100);
656
- const end = Math.min(line.length, matchIdx + 150);
657
- let snippet = line.slice(start, end).trim();
658
- if (start > 0)
659
- snippet = "..." + snippet;
660
- if (end < line.length)
661
- snippet = snippet + "...";
662
- if (!snippet)
663
- continue; // an all-whitespace "match" line is not a useful excerpt
664
- // P0-a (2026-08-18): archive/raw stays byte-identical ON DISK (lossless
665
- // tier), but this function greps that raw content directly into a
666
- // recall() result's `excerpt` field — the SURFACING BOUNDARY. Scrub
667
- // the snippet here, at the read/return edge, never the file itself.
668
- snippet = scrubForCloud(snippet);
669
- const provenance = path.join("journal", "archive", "raw", file);
670
- // Keyword overlap reflects match quality among archive hits, but is
671
- // hard-capped below CONFIDENCE_FLOOR.medium so this source structurally
672
- // can never present itself as anything but low/weak confidence.
673
- const rawScore = Math.min(keywordExactness(query, line), CONFIDENCE_FLOOR.medium - 0.01);
674
- items.push({
675
- id: stableId("archive", `${file}:${i}`),
676
- source: "archive",
677
- title: `archive/${file}`,
678
- excerpt: `[raw-archive · low-confidence · ${provenance}] ${snippet}`,
679
- score: rawScore,
680
- // internalScore is 0..1 → cosine scale, same convention as the other 3 sources.
681
- ...label(rawScore, "cosine"),
682
- verbatimKey: { kind: "archive", date, file },
683
- date,
684
- });
685
- }
686
- }
687
- return items;
688
- }
324
+ const RAW_CANDIDATE_COUNTS = Symbol("rawCandidateCounts");
689
325
  /**
690
326
  * Budget for the semantic (remote) backend in ms.
691
327
  * Overridable via AGENT_RECALL_RECALL_BUDGET_MS for tuning / tests.
@@ -815,17 +451,19 @@ export async function smartRecall(input) {
815
451
  bridged = collected;
816
452
  }
817
453
  // ── EXPLICIT 4TH SOURCE: archive fallback (F4, continuity wave 2026-07-31) ──
454
+ // WAVE 2: fetch/score logic moved to retrieval/query-memory.ts's
455
+ // `queryArchiveFallback` (see this file's header + that function's own doc
456
+ // comment for why the GATING policy below stays here, unchanged).
818
457
  // Gated on the SAME CONFIDENCE_FLOOR.medium constant as the Bridge gate
819
458
  // above, but on the fused TOP result only (not every low item) — "the
820
459
  // fused top-confidence of palace/journal/insight". This adds brand-new
821
460
  // result items sourced from journal/archive/raw/, so it must never compete
822
- // for rank inside localRecallSearch's palace/journal/insight RRF fusion —
823
- // it only steps in once those 3 sources have already failed to produce a
824
- // confident #1 answer. Placed AFTER the Bridge above so the Bridge's own
825
- // `low` filter (which also matches any verbatimKey-bearing item) only ever
826
- // considers genuine palace/journal/insight items — an archive item is
827
- // already a raw excerpt and would gain nothing from being drilled into
828
- // itself.
461
+ // for rank inside the palace/journal/insight RRF fusion — it only steps in
462
+ // once those 3 sources have already failed to produce a confident #1
463
+ // answer. Placed AFTER the Bridge above so the Bridge's own `low` filter
464
+ // (which also matches any verbatimKey-bearing item) only ever considers
465
+ // genuine palace/journal/insight items — an archive item is already a raw
466
+ // excerpt and would gain nothing from being drilled into itself.
829
467
  let archiveSourceRan = false;
830
468
  // L3 (review, 2026-07-31; documented only — no behavior change this wave):
831
469
  // `finalResults[0]` is rank-0 by the POST-FEEDBACK boosted `score` (see the
@@ -851,8 +489,19 @@ export async function smartRecall(input) {
851
489
  // the gate fired without the item COUNT ever violating `limit`.
852
490
  const remainingBudget = Math.max(0, limit - finalResults.length);
853
491
  if (remainingBudget > 0) {
854
- const archiveItems = archiveSearch(resolvedProject, input.query, Math.min(ARCHIVE_SOURCE_CAP, remainingBudget));
855
- finalResults.push(...archiveItems);
492
+ const archiveItems = queryArchiveFallback(resolvedProject, input.query, Math.min(ARCHIVE_SOURCE_CAP, remainingBudget));
493
+ for (const item of archiveItems) {
494
+ finalResults.push({
495
+ id: item.id,
496
+ source: "archive",
497
+ title: item.title,
498
+ excerpt: item.excerpt,
499
+ score: item.score,
500
+ ...label(item.score, "cosine"),
501
+ verbatimKey: verbatimKeyFor(item),
502
+ ...(item.date ? { date: item.date } : {}),
503
+ });
504
+ }
856
505
  }
857
506
  }
858
507
  // Fix 4/5: total_searched should be the true distinct-candidate count from