agent-recall-core 3.4.45 → 3.4.47
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/helpers/activity-feed.d.ts.map +1 -1
- package/dist/helpers/activity-feed.js +8 -0
- package/dist/helpers/activity-feed.js.map +1 -1
- package/dist/helpers/handoff.d.ts.map +1 -1
- package/dist/helpers/handoff.js +17 -9
- package/dist/helpers/handoff.js.map +1 -1
- package/dist/helpers/journal-files.d.ts +24 -1
- package/dist/helpers/journal-files.d.ts.map +1 -1
- package/dist/helpers/journal-files.js +29 -10
- package/dist/helpers/journal-files.js.map +1 -1
- package/dist/helpers/journal-filter.d.ts +53 -0
- package/dist/helpers/journal-filter.d.ts.map +1 -1
- package/dist/helpers/journal-filter.js +101 -0
- package/dist/helpers/journal-filter.js.map +1 -1
- package/dist/index.d.ts +12 -4
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +33 -4
- package/dist/index.js.map +1 -1
- package/dist/palace/consolidate.d.ts.map +1 -1
- package/dist/palace/consolidate.js +10 -0
- package/dist/palace/consolidate.js.map +1 -1
- package/dist/retrieval/candidates.d.ts +314 -0
- package/dist/retrieval/candidates.d.ts.map +1 -0
- package/dist/retrieval/candidates.js +321 -0
- package/dist/retrieval/candidates.js.map +1 -0
- package/dist/retrieval/contradiction.d.ts +232 -0
- package/dist/retrieval/contradiction.d.ts.map +1 -0
- package/dist/retrieval/contradiction.js +320 -0
- package/dist/retrieval/contradiction.js.map +1 -0
- package/dist/retrieval/query-memory.d.ts +419 -0
- package/dist/retrieval/query-memory.d.ts.map +1 -0
- package/dist/retrieval/query-memory.js +1068 -0
- package/dist/retrieval/query-memory.js.map +1 -0
- package/dist/retrieval/scope.d.ts +75 -0
- package/dist/retrieval/scope.d.ts.map +1 -0
- package/dist/retrieval/scope.js +91 -0
- package/dist/retrieval/scope.js.map +1 -0
- package/dist/storage/corrections.d.ts +31 -1
- package/dist/storage/corrections.d.ts.map +1 -1
- package/dist/storage/corrections.js +34 -1
- package/dist/storage/corrections.js.map +1 -1
- package/dist/storage/cwd-allowlist.d.ts +58 -0
- package/dist/storage/cwd-allowlist.d.ts.map +1 -1
- package/dist/storage/cwd-allowlist.js +46 -7
- package/dist/storage/cwd-allowlist.js.map +1 -1
- package/dist/storage/lifecycle-telemetry.d.ts +20 -5
- package/dist/storage/lifecycle-telemetry.d.ts.map +1 -1
- package/dist/storage/lifecycle-telemetry.js +16 -5
- package/dist/storage/lifecycle-telemetry.js.map +1 -1
- package/dist/storage/project.d.ts.map +1 -1
- package/dist/storage/project.js +162 -35
- package/dist/storage/project.js.map +1 -1
- package/dist/storage/recency-index.d.ts +14 -0
- package/dist/storage/recency-index.d.ts.map +1 -1
- package/dist/storage/recency-index.js.map +1 -1
- package/dist/storage/session.d.ts +28 -1
- package/dist/storage/session.d.ts.map +1 -1
- package/dist/storage/session.js +41 -8
- package/dist/storage/session.js.map +1 -1
- package/dist/storage/working-memory.d.ts.map +1 -1
- package/dist/storage/working-memory.js +10 -1
- package/dist/storage/working-memory.js.map +1 -1
- package/dist/supabase/recall-backend.d.ts +25 -2
- package/dist/supabase/recall-backend.d.ts.map +1 -1
- package/dist/supabase/recall-backend.js +82 -21
- package/dist/supabase/recall-backend.js.map +1 -1
- package/dist/supabase/sync.d.ts +41 -0
- package/dist/supabase/sync.d.ts.map +1 -1
- package/dist/supabase/sync.js +46 -0
- package/dist/supabase/sync.js.map +1 -1
- package/dist/tools-logic/check.d.ts.map +1 -1
- package/dist/tools-logic/check.js +62 -36
- package/dist/tools-logic/check.js.map +1 -1
- package/dist/tools-logic/context-synthesize.d.ts.map +1 -1
- package/dist/tools-logic/context-synthesize.js +24 -3
- package/dist/tools-logic/context-synthesize.js.map +1 -1
- package/dist/tools-logic/drill-down.d.ts +24 -0
- package/dist/tools-logic/drill-down.d.ts.map +1 -1
- package/dist/tools-logic/drill-down.js +36 -0
- package/dist/tools-logic/drill-down.js.map +1 -1
- package/dist/tools-logic/journal-archive.d.ts.map +1 -1
- package/dist/tools-logic/journal-archive.js +12 -2
- package/dist/tools-logic/journal-archive.js.map +1 -1
- package/dist/tools-logic/journal-cold-start.d.ts.map +1 -1
- package/dist/tools-logic/journal-cold-start.js +48 -9
- package/dist/tools-logic/journal-cold-start.js.map +1 -1
- package/dist/tools-logic/journal-list.d.ts +42 -0
- package/dist/tools-logic/journal-list.d.ts.map +1 -1
- package/dist/tools-logic/journal-list.js +60 -10
- package/dist/tools-logic/journal-list.js.map +1 -1
- package/dist/tools-logic/journal-read.d.ts.map +1 -1
- package/dist/tools-logic/journal-read.js +22 -13
- package/dist/tools-logic/journal-read.js.map +1 -1
- package/dist/tools-logic/journal-search.d.ts +58 -0
- package/dist/tools-logic/journal-search.d.ts.map +1 -1
- package/dist/tools-logic/journal-search.js +123 -80
- package/dist/tools-logic/journal-search.js.map +1 -1
- package/dist/tools-logic/palace-search.d.ts.map +1 -1
- package/dist/tools-logic/palace-search.js +30 -11
- package/dist/tools-logic/palace-search.js.map +1 -1
- package/dist/tools-logic/palace-walk.d.ts +10 -0
- package/dist/tools-logic/palace-walk.d.ts.map +1 -1
- package/dist/tools-logic/palace-walk.js +25 -16
- package/dist/tools-logic/palace-walk.js.map +1 -1
- package/dist/tools-logic/recall-insight.d.ts +20 -0
- package/dist/tools-logic/recall-insight.d.ts.map +1 -1
- package/dist/tools-logic/recall-insight.js +12 -1
- package/dist/tools-logic/recall-insight.js.map +1 -1
- package/dist/tools-logic/recognition-builder.d.ts.map +1 -1
- package/dist/tools-logic/recognition-builder.js +33 -19
- package/dist/tools-logic/recognition-builder.js.map +1 -1
- package/dist/tools-logic/resurrect.d.ts +11 -0
- package/dist/tools-logic/resurrect.d.ts.map +1 -1
- package/dist/tools-logic/resurrect.js +64 -2
- package/dist/tools-logic/resurrect.js.map +1 -1
- package/dist/tools-logic/session-end-reflect.d.ts.map +1 -1
- package/dist/tools-logic/session-end-reflect.js +11 -0
- package/dist/tools-logic/session-end-reflect.js.map +1 -1
- package/dist/tools-logic/session-end.d.ts.map +1 -1
- package/dist/tools-logic/session-end.js +23 -7
- package/dist/tools-logic/session-end.js.map +1 -1
- package/dist/tools-logic/session-start-lite.d.ts.map +1 -1
- package/dist/tools-logic/session-start-lite.js +13 -2
- package/dist/tools-logic/session-start-lite.js.map +1 -1
- package/dist/tools-logic/session-start.d.ts +101 -0
- package/dist/tools-logic/session-start.d.ts.map +1 -1
- package/dist/tools-logic/session-start.js +301 -78
- package/dist/tools-logic/session-start.js.map +1 -1
- package/dist/tools-logic/smart-recall.d.ts +87 -62
- package/dist/tools-logic/smart-recall.d.ts.map +1 -1
- package/dist/tools-logic/smart-recall.js +159 -510
- package/dist/tools-logic/smart-recall.js.map +1 -1
- package/dist/types.d.ts +1 -1
- package/dist/types.js +1 -1
- package/package.json +1 -1
|
@@ -1,6 +1,34 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* smart_recall — unified cross-store search. v3.3.14
|
|
3
3
|
*
|
|
4
|
+
* ── WAVE 2 (2026-08-29/30, plywood SOP ecbd4351) ────────────────────────────
|
|
5
|
+
* Internals migrated onto the shared retrieval pipeline
|
|
6
|
+
* (`retrieval/query-memory.ts`'s `queryMemory()`). EXTERNAL CONTRACT
|
|
7
|
+
* UNCHANGED: same `SmartRecallInput`/`SmartRecallResult` shape, same
|
|
8
|
+
* feedback-rating footer, same archive-fallback/bridge/graph-walk behavior.
|
|
9
|
+
* The composition that used to live directly in this file — journalSearch +
|
|
10
|
+
* palaceSearch + recallInsight + archiveSearch, plus the 5 scattered scoring
|
|
11
|
+
* formulas below and the two-stage RRF fusion — now lives in
|
|
12
|
+
* `retrieval/query-memory.ts` as MANDATORY PIPELINE STAGES (fetch ->
|
|
13
|
+
* trust-filter -> tokenize+score -> scope -> rank/fuse -> fence). See that
|
|
14
|
+
* file's header comment for the full CHALLENGE-by-CHALLENGE design reasoning
|
|
15
|
+
* (why scoring stays per-tier/pluggable rather than unified into one formula;
|
|
16
|
+
* how the Wave-1 candidate superset is handled per source; why FENCE is not
|
|
17
|
+
* applied to this surface's per-item fields). This file now owns exactly the
|
|
18
|
+
* smart_recall-SPECIFIC post-pipeline concerns that do not generalize to
|
|
19
|
+
* other future `queryMemory()` consumers: confidence calibration
|
|
20
|
+
* (`calibratedConfidence`), the Beta-feedback multiplier + feedback log, the
|
|
21
|
+
* remote/local backend race + degraded-timeout handling, the Bridge
|
|
22
|
+
* drilldown, and the F4 archive-fallback's confidence GATE (the policy
|
|
23
|
+
* decision of *when* to call `queryArchiveFallback` — its fetch/score logic
|
|
24
|
+
* itself moved to query-memory.ts).
|
|
25
|
+
*
|
|
26
|
+
* The formulas described below (Fix 1-5b) are UNCHANGED in substance — they
|
|
27
|
+
* are simply now implemented in query-memory.ts's per-tier scoring functions
|
|
28
|
+
* and RANK/FUSE stage instead of inline here. Kept as historical/design
|
|
29
|
+
* documentation because the RATIONALE is still exactly why those formulas
|
|
30
|
+
* are what they are.
|
|
31
|
+
*
|
|
4
32
|
* ## Scoring Architecture (why it works this way)
|
|
5
33
|
*
|
|
6
34
|
* ### Problem with the old approach (< v3.3.14): Linear Score Fusion
|
|
@@ -25,7 +53,6 @@
|
|
|
25
53
|
* Formula: R(t) = e^(-t/S), where S = memory strength (days).
|
|
26
54
|
* Different memory types have different S values based on psychological research:
|
|
27
55
|
* - Journal (episodic, low meaning): S = 2 → 60% retained after 1 day
|
|
28
|
-
* - Knowledge/bug-fix (procedural): S = 180 → 99.4% retained after 1 day
|
|
29
56
|
* - Palace/decisions (semantic): S = 9999 → barely decays
|
|
30
57
|
* - Insight (conceptual): not time-based; uses confirmation count instead
|
|
31
58
|
* This replaces the uniform 0.95^days that treated all memory equally.
|
|
@@ -59,62 +86,35 @@
|
|
|
59
86
|
* duplicates (same id) could. A later "dedup by excerpt" pass then silently
|
|
60
87
|
* collapsed same-excerpt entries by first-inserted-wins, DISCARDING the other
|
|
61
88
|
* source's score entirely instead of summing it in.
|
|
62
|
-
* Fix: fusion is now TWO stages. Stage 1 (applyRRF,
|
|
63
|
-
*
|
|
64
|
-
*
|
|
65
|
-
*
|
|
66
|
-
*
|
|
67
|
-
*
|
|
68
|
-
*
|
|
69
|
-
* then re-keys those already-consolidated per-document entries by NORMALIZED
|
|
70
|
-
* EXCERPT CONTENT — the same identity notion the old post-hoc dedup pass
|
|
71
|
-
* already used to decide "same memory". Two per-document entries from
|
|
72
|
-
* different sources whose representative excerpt matches now land in the
|
|
73
|
-
* SAME canonical entry, so cross-source accumulation happens. Provenance
|
|
74
|
-
* from every contributing source is preserved via `alsoFoundIn` on the fused
|
|
75
|
-
* item (primary/display source is whichever source ran first — unchanged),
|
|
76
|
-
* rather than being dropped.
|
|
89
|
+
* Fix: fusion is now TWO stages. Stage 1 (applyRRF, keyed by `item.id`) still
|
|
90
|
+
* consolidates multiple hits from the SAME source document. Stage 2
|
|
91
|
+
* (fuseCanonical) then re-keys those already-consolidated per-document
|
|
92
|
+
* entries by NORMALIZED EXCERPT CONTENT. Provenance from every contributing
|
|
93
|
+
* source is preserved via `alsoFoundIn` on the fused item, rather than being
|
|
94
|
+
* dropped. (Both stages live in query-memory.ts now — see that file's
|
|
95
|
+
* `applyRRF`/`fuseCanonical`.)
|
|
77
96
|
*
|
|
78
97
|
* ### Fix 5b: insight excerpt is too low-entropy to be a fusion identity
|
|
79
98
|
* Stage 2's "normalized excerpt content" identity assumption (Fix 5) is
|
|
80
99
|
* sound for palace/journal, whose `excerpt` is a real matched text snippet.
|
|
81
100
|
* It was broken for the insight source: its excerpt was synthesized from
|
|
82
|
-
* ONLY `severity` + `applies_when
|
|
83
|
-
*
|
|
84
|
-
* insights sharing a severity + tag set produced byte-identical excerpts and
|
|
85
|
-
* silently fused into one — one insight vanished from results, and the
|
|
86
|
-
* survivor's score absorbed the other's RRF contribution with zero trace
|
|
87
|
-
* (same-source collisions don't show up in `alsoFoundIn`, which only records
|
|
88
|
-
* source NAMES). Fix: insight items now carry a separate `fusionKey`
|
|
101
|
+
* ONLY `severity` + `applies_when`, omitting the insight's own distinguishing
|
|
102
|
+
* `title` entirely. Fix: insight items now carry a separate `fusionKey`
|
|
89
103
|
* (`${title} [severity] tags`) that fuseCanonical() and the defensive dedup
|
|
90
|
-
* pass key on INSTEAD of `excerpt` when present
|
|
91
|
-
* comment on `SmartRecallResultItem`). The displayed `excerpt` stays terse —
|
|
92
|
-
* embedding `title` into it would duplicate title text in every consumer
|
|
93
|
-
* that reads `excerpt` directly (recall.ts's display line, the CLI's
|
|
94
|
-
* ambient-injection word-overlap scorer, consistency/conflict-scan's
|
|
95
|
-
* `title + excerpt` concatenation), which for the word-overlap scorer would
|
|
96
|
-
* also double-count title tokens and inflate insight relevance versus other
|
|
97
|
-
* sources. Palace/journal items leave `fusionKey` unset (falls back to
|
|
98
|
-
* `excerpt`, unchanged from Fix 5) — insights are a distinct "confirmed
|
|
99
|
-
* pattern" memory type that isn't expected to share literal content with a
|
|
100
|
-
* journal/palace excerpt, so this doesn't reopen the cross-source fusion
|
|
101
|
-
* Fix 5 was built for; it only stops insights colliding WITH EACH OTHER.
|
|
104
|
+
* pass key on INSTEAD of `excerpt` when present.
|
|
102
105
|
*/
|
|
103
106
|
import * as fs from "node:fs";
|
|
104
107
|
import * as path from "node:path";
|
|
105
|
-
import { palaceSearch } from "./palace-search.js";
|
|
106
|
-
import { journalSearch } from "./journal-search.js";
|
|
107
|
-
import { recallInsight } from "./recall-insight.js";
|
|
108
108
|
import { getRoot } from "../types.js";
|
|
109
109
|
import { ensureDir } from "../storage/fs-utils.js";
|
|
110
|
-
import {
|
|
110
|
+
import { expandQuery } from "../helpers/normalize.js";
|
|
111
111
|
import { tokenizeWords } from "../helpers/tokenize.js";
|
|
112
112
|
import { getConnectedRooms } from "../palace/graph.js";
|
|
113
|
-
import { palaceDir
|
|
113
|
+
import { palaceDir } from "../storage/paths.js";
|
|
114
114
|
import { calibratedConfidence, CONFIDENCE_FLOOR } from "./confidence.js";
|
|
115
115
|
import { fetchVerbatim } from "./drill-down.js";
|
|
116
116
|
import { resolveProject } from "../storage/project.js";
|
|
117
|
-
import {
|
|
117
|
+
import { queryMemory, queryArchiveFallback } from "../retrieval/query-memory.js";
|
|
118
118
|
/** Compute both the human label and the stored calibrated value for a score. */
|
|
119
119
|
function label(score, scale) {
|
|
120
120
|
const c = calibratedConfidence(score, scale);
|
|
@@ -123,105 +123,15 @@ function label(score, scale) {
|
|
|
123
123
|
// ---------------------------------------------------------------------------
|
|
124
124
|
// Constants
|
|
125
125
|
// ---------------------------------------------------------------------------
|
|
126
|
-
/** RRF constant. k=60 is the empirically validated default (Cormack et al. 2009). */
|
|
127
|
-
const RRF_K = 60;
|
|
128
|
-
/**
|
|
129
|
-
* Ebbinghaus memory strength S (days) per source type.
|
|
130
|
-
* R(t) = e^(-t/S): higher S = slower decay.
|
|
131
|
-
* Journal decays fast (low-meaning episodic); palace barely decays (semantic).
|
|
132
|
-
*/
|
|
133
|
-
const EBBINGHAUS_S = {
|
|
134
|
-
journal: 2, // ~60% retained after 1 day, ~7% after 1 week
|
|
135
|
-
knowledge: 180, // ~99.4% after 1 day, ~84.6% after 1 month
|
|
136
|
-
palace: 9999, // effectively no decay
|
|
137
|
-
};
|
|
138
126
|
/**
|
|
139
|
-
* Max items the explicit archive-fallback source (F4, see
|
|
140
|
-
* below) may append to a single smartRecall() call. Kept small
|
|
141
|
-
* confidence-gated last resort, not a competing ranked source.
|
|
127
|
+
* Max items the explicit archive-fallback source (F4, see the gate inside
|
|
128
|
+
* smartRecall() below) may append to a single smartRecall() call. Kept small
|
|
129
|
+
* — this is a confidence-gated last resort, not a competing ranked source.
|
|
142
130
|
*/
|
|
143
131
|
const ARCHIVE_SOURCE_CAP = 3;
|
|
144
132
|
// ---------------------------------------------------------------------------
|
|
145
|
-
//
|
|
133
|
+
// Feedback store
|
|
146
134
|
// ---------------------------------------------------------------------------
|
|
147
|
-
/**
|
|
148
|
-
* Normalize excerpt text into a canonical cross-source identity key.
|
|
149
|
-
* Two items from different sources that describe the same conceptual memory
|
|
150
|
-
* typically carry byte-identical (or near-identical) excerpt text — this is
|
|
151
|
-
* the same identity notion the old post-hoc dedup pass (Step 5) already used
|
|
152
|
-
* to decide "same memory"; it's now applied at RRF fusion time instead
|
|
153
|
-
* (Fix 5), so cross-source accumulation happens naturally.
|
|
154
|
-
*/
|
|
155
|
-
function normalizeExcerpt(excerpt) {
|
|
156
|
-
return excerpt.toLowerCase().replace(/\s+/g, " ").trim();
|
|
157
|
-
}
|
|
158
|
-
/**
|
|
159
|
-
* Cross-source fusion identity for an item (Fix 5b). Prefers the item's
|
|
160
|
-
* `fusionKey` override (set by the insight source, whose displayed `excerpt`
|
|
161
|
-
* is too low-entropy — see `fusionKey`'s doc comment on
|
|
162
|
-
* `SmartRecallResultItem`) and falls back to the displayed `excerpt` for
|
|
163
|
-
* every other source, unchanged from Fix 5's original design.
|
|
164
|
-
*/
|
|
165
|
-
function fusionIdentity(item) {
|
|
166
|
-
return normalizeExcerpt(item.fusionKey ?? item.excerpt);
|
|
167
|
-
}
|
|
168
|
-
/**
|
|
169
|
-
* Internal side channel: raw per-source candidate counts (Fix 4/5), attached
|
|
170
|
-
* to the array localRecallSearch returns so smartRecall() can report a
|
|
171
|
-
* genuine pre-fusion total_searched without changing localRecallSearch's
|
|
172
|
-
* public return type (still a plain SmartRecallResultItem[] — several
|
|
173
|
-
* existing tests and recall-backend.ts depend on that exact shape).
|
|
174
|
-
*/
|
|
175
|
-
const RAW_CANDIDATE_COUNTS = Symbol("rawCandidateCounts");
|
|
176
|
-
/** Simple stable hash for result IDs. */
|
|
177
|
-
function stableId(source, title) {
|
|
178
|
-
let hash = 0;
|
|
179
|
-
const str = `${source}:${title}`;
|
|
180
|
-
for (let i = 0; i < str.length; i++) {
|
|
181
|
-
const char = str.charCodeAt(i);
|
|
182
|
-
hash = ((hash << 5) - hash) + char;
|
|
183
|
-
hash |= 0;
|
|
184
|
-
}
|
|
185
|
-
return Math.abs(hash).toString(36).slice(0, 8);
|
|
186
|
-
}
|
|
187
|
-
/** Days elapsed since a date string. */
|
|
188
|
-
function daysSince(dateStr) {
|
|
189
|
-
const d = new Date(dateStr);
|
|
190
|
-
if (isNaN(d.getTime()))
|
|
191
|
-
return 365;
|
|
192
|
-
return Math.max(0, (Date.now() - d.getTime()) / (1000 * 60 * 60 * 24));
|
|
193
|
-
}
|
|
194
|
-
/**
|
|
195
|
-
* Ebbinghaus forgetting curve: R(t) = e^(-t/S).
|
|
196
|
-
* Returns retention [0,1] after `days` with strength S.
|
|
197
|
-
*/
|
|
198
|
-
function ebbinghaus(days, S) {
|
|
199
|
-
return Math.exp(-days / S);
|
|
200
|
-
}
|
|
201
|
-
/**
|
|
202
|
-
* Keyword overlap ratio between query and text, with stemming + synonym expansion.
|
|
203
|
-
* CJK-aware (P0-b, 2026-08-18): both sides go through the shared tokenizer
|
|
204
|
-
* so an unspaced Chinese/Japanese query/text segments into real words
|
|
205
|
-
* instead of one giant token that must match byte-for-byte (2026-08-18 L1
|
|
206
|
-
* eval: CJK hit@5 = 0/6). Default minLength=3 preserves the original
|
|
207
|
-
* `length > 2` floor for ASCII.
|
|
208
|
-
*/
|
|
209
|
-
function keywordExactness(query, text) {
|
|
210
|
-
const rawWords = tokenizeWords(query);
|
|
211
|
-
if (rawWords.length === 0)
|
|
212
|
-
return 0;
|
|
213
|
-
// Expand query with stems + synonyms
|
|
214
|
-
const expandedQuery = expandQuery(rawWords);
|
|
215
|
-
// Stem the text words for matching
|
|
216
|
-
const textWords = tokenizeWords(text).map(w => stem(w));
|
|
217
|
-
const textSet = new Set(textWords);
|
|
218
|
-
// Also check raw text for direct substring matches (preserves old behavior)
|
|
219
|
-
const textLower = text.toLowerCase();
|
|
220
|
-
// Count matches: expanded query word found in stemmed text OR as substring
|
|
221
|
-
const matches = expandedQuery.filter(w => textSet.has(w) || textLower.includes(w));
|
|
222
|
-
// Score relative to ORIGINAL query length (not expanded), capped at 1.0
|
|
223
|
-
return Math.min(1.0, matches.length / rawWords.length);
|
|
224
|
-
}
|
|
225
135
|
/**
|
|
226
136
|
* Beta distribution expected value for binary feedback.
|
|
227
137
|
* E[Beta(α,β)] = α/(α+β) where α=pos+1, β=neg+1 (Laplace smoothing).
|
|
@@ -279,287 +189,98 @@ function getFeedbackCounts(id, title, queryWords, log) {
|
|
|
279
189
|
negatives: relevant.filter((f) => match(f) && !f.useful).length,
|
|
280
190
|
};
|
|
281
191
|
}
|
|
282
|
-
/**
|
|
283
|
-
* Apply Reciprocal Rank Fusion scores from a ranked list of items.
|
|
284
|
-
* Mutates the provided rrfMap in place.
|
|
285
|
-
* RRF_score += 1 / (k + rank)
|
|
286
|
-
*
|
|
287
|
-
* STAGE 1 of fusion — keyed by `item.id` (the per-source occurrence id,
|
|
288
|
-
* `stableId(source, title)`), exactly as before Fix 5. This intentionally
|
|
289
|
-
* stays id-keyed: a single source document can legitimately produce SEVERAL
|
|
290
|
-
* distinct-excerpt hits that share one `item.id` (e.g. palaceSearch returns
|
|
291
|
-
* one hit per matching LINE within the same room/file, all sharing that
|
|
292
|
-
* file's id) — those must accumulate into ONE per-document entry, not
|
|
293
|
-
* fragment into many. Keying this stage by excerpt instead (as an earlier
|
|
294
|
-
* version of this fix mistakenly did) breaks exactly that: it turned a
|
|
295
|
-
* single matched file into N separate low-score fragments and starved
|
|
296
|
-
* genuinely-different documents of their combined per-file RRF weight
|
|
297
|
-
* (caught by associative-link.test.mjs). Cross-SOURCE fusion (the actual
|
|
298
|
-
* bug this file's Fix 5 addresses) happens in a SEPARATE stage afterward —
|
|
299
|
-
* see fuseCanonical() below — once each source's own per-document hits are
|
|
300
|
-
* already consolidated here.
|
|
301
|
-
*/
|
|
302
|
-
function applyRRF(rankedItems, rrfMap) {
|
|
303
|
-
rankedItems.forEach((item, idx) => {
|
|
304
|
-
const rank = idx + 1;
|
|
305
|
-
const contribution = 1 / (RRF_K + rank);
|
|
306
|
-
const existing = rrfMap.get(item.id);
|
|
307
|
-
if (existing) {
|
|
308
|
-
existing.score += contribution;
|
|
309
|
-
}
|
|
310
|
-
else {
|
|
311
|
-
rrfMap.set(item.id, { score: contribution, item, sources: new Set([item.source]) });
|
|
312
|
-
}
|
|
313
|
-
});
|
|
314
|
-
}
|
|
315
|
-
/**
|
|
316
|
-
* STAGE 2 of fusion (Fix 5) — cross-source canonical merge, run AFTER
|
|
317
|
-
* applyRRF() has consolidated each source's own per-document hits (Stage 1).
|
|
318
|
-
* Re-keys the (already within-source-deduped) entries by `fusionIdentity()`
|
|
319
|
-
* (Fix 5b: `item.fusionKey` when set, else `normalizeExcerpt(item.excerpt)`)
|
|
320
|
-
* — the same identity notion Step 5's old post-hoc dedup pass used to decide
|
|
321
|
-
* "same memory". Two per-document entries from DIFFERENT sources (or,
|
|
322
|
-
* rarely, the same source) whose identity is the same after normalization
|
|
323
|
-
* are now genuinely the SAME conceptual memory, so their scores are summed
|
|
324
|
-
* and their sources merged — instead of the old behavior where a later dedup
|
|
325
|
-
* pass silently discarded one side's score entirely.
|
|
326
|
-
* Iteration order of `rrfMap` is insertion order (palace's entries first,
|
|
327
|
-
* then journal, then insight — see localRecallSearch's call order), so ties
|
|
328
|
-
* keep the same source-precedence the rest of this file already assumes.
|
|
329
|
-
*/
|
|
330
|
-
function fuseCanonical(rrfMap) {
|
|
331
|
-
const canonical = new Map();
|
|
332
|
-
for (const entry of rrfMap.values()) {
|
|
333
|
-
const key = fusionIdentity(entry.item);
|
|
334
|
-
const existing = canonical.get(key);
|
|
335
|
-
if (existing) {
|
|
336
|
-
existing.score += entry.score;
|
|
337
|
-
for (const s of entry.sources)
|
|
338
|
-
existing.sources.add(s);
|
|
339
|
-
}
|
|
340
|
-
else {
|
|
341
|
-
canonical.set(key, { score: entry.score, item: entry.item, sources: new Set(entry.sources) });
|
|
342
|
-
}
|
|
343
|
-
}
|
|
344
|
-
return canonical;
|
|
345
|
-
}
|
|
346
192
|
// ---------------------------------------------------------------------------
|
|
347
193
|
// Main
|
|
348
194
|
// ---------------------------------------------------------------------------
|
|
195
|
+
/** Reconstruct a `VerbatimKey` for the Bridge from a pipeline item's generic
|
|
196
|
+
* `room`/`file`/`date` fields — the pipeline itself stays decoupled from
|
|
197
|
+
* drill-down.ts's `VerbatimKey` shape (see query-memory.ts's own comment on
|
|
198
|
+
* `QueryMemoryItem.file`). */
|
|
199
|
+
function verbatimKeyFor(item) {
|
|
200
|
+
if (item.source === "journal" && item.date)
|
|
201
|
+
return { kind: "journal", date: item.date };
|
|
202
|
+
if (item.source === "palace" && item.room && item.file)
|
|
203
|
+
return { kind: "palace", room: item.room, file: item.file };
|
|
204
|
+
if (item.source === "archive" && item.file)
|
|
205
|
+
return { kind: "archive", date: item.date, file: item.file };
|
|
206
|
+
return undefined;
|
|
207
|
+
}
|
|
349
208
|
/**
|
|
350
209
|
* localRecallSearch — the core local search logic (palace + journal + insight).
|
|
351
|
-
* Performs three parallel searches, scores internally, merges via RRF,
|
|
352
|
-
* applies hot-window recency boost, and deduplicates.
|
|
353
210
|
*
|
|
354
|
-
*
|
|
355
|
-
*
|
|
356
|
-
*
|
|
211
|
+
* WAVE 2: delegates FETCH/TRUST-FILTER/TOKENIZE+SCORE/RANK-FUSE entirely to
|
|
212
|
+
* `retrieval/query-memory.ts`'s `queryMemory()` — see this file's header.
|
|
213
|
+
* This function's own remaining job: resolve `project` (queryMemory()
|
|
214
|
+
* requires an already-resolved slug, matching Wave 1's `readTierCandidates`
|
|
215
|
+
* convention — previously this resolution happened independently 3x, inside
|
|
216
|
+
* journalSearch/palaceSearch/nowhere-for-insight; now once, here — a
|
|
217
|
+
* characterized simplification, not a behavior change for any caller passing
|
|
218
|
+
* an already-valid explicit slug, which is every existing test and the
|
|
219
|
+
* common real-world case), label each fused item with a calibrated
|
|
220
|
+
* confidence (rrf-local scale, matching the original dedup loop exactly),
|
|
221
|
+
* reconstruct `verbatimKey`, run the graph-walk 1-hop related-room surfacing
|
|
222
|
+
* (smart_recall-specific; no other Wave-3 migration target has this), and
|
|
223
|
+
* attach the raw-candidate-count side channel.
|
|
224
|
+
*
|
|
225
|
+
* Called by LocalRecallBackend.search() in recall-backend.ts, and directly
|
|
226
|
+
* by helpers/associative-link.ts and (in tests) by
|
|
227
|
+
* audit-retrieval-accounting.test.mjs — its signature and the RAW_CANDIDATE_COUNTS
|
|
228
|
+
* side-channel contract are UNCHANGED, since those are real, direct external
|
|
229
|
+
* callers, not just smartRecall()'s own internals.
|
|
357
230
|
*/
|
|
358
231
|
export async function localRecallSearch(query, project, limit, since) {
|
|
359
|
-
|
|
360
|
-
// Candidate buckets — each source scores its items internally, then RRF merges
|
|
361
|
-
const palaceItems = [];
|
|
362
|
-
const journalItems = [];
|
|
363
|
-
const insightItems = [];
|
|
364
|
-
// ── 1. Palace ────────────────────────────────────────────────────────────
|
|
365
|
-
// Internal score: keyword match quality × salience (structural importance).
|
|
366
|
-
// Ebbinghaus decay is minimal for palace (S=9999); salience already
|
|
367
|
-
// incorporates access recency via recordAccess().
|
|
368
|
-
try {
|
|
369
|
-
const palaceResults = await palaceSearch({ query, project, limit: limit * 2 });
|
|
370
|
-
sourcesQueried.push("palace");
|
|
371
|
-
for (const r of palaceResults.results) {
|
|
372
|
-
const title = `${r.room}/${r.file}`;
|
|
373
|
-
const id = stableId("palace", title);
|
|
374
|
-
// keyword_score comes from updated palace-search (keyword overlap, not substring).
|
|
375
|
-
// salience floor of 0.4 prevents new rooms (salience=0.5) from being unfairly
|
|
376
|
-
// penalized against rooms with years of access history.
|
|
377
|
-
const keyScore = r.keyword_score ?? keywordExactness(query, r.excerpt);
|
|
378
|
-
const salience = Math.max(0.4, r.salience);
|
|
379
|
-
const internalScore = keyScore * 0.65 + salience * 0.35;
|
|
380
|
-
// Try to extract a date from the excerpt (many palace entries have ### YYYY-MM-DD headers)
|
|
381
|
-
let palaceDate;
|
|
382
|
-
const datePattern = r.excerpt.match(/(\d{4}-\d{2}-\d{2})/);
|
|
383
|
-
if (datePattern) {
|
|
384
|
-
palaceDate = datePattern[1];
|
|
385
|
-
}
|
|
386
|
-
palaceItems.push({
|
|
387
|
-
id,
|
|
388
|
-
source: "palace",
|
|
389
|
-
title,
|
|
390
|
-
excerpt: r.excerpt,
|
|
391
|
-
score: internalScore,
|
|
392
|
-
// internalScore is already 0..1 → cosine scale (NOT rrf-local).
|
|
393
|
-
...label(internalScore, "cosine"),
|
|
394
|
-
verbatimKey: { kind: "palace", room: r.room, file: r.file },
|
|
395
|
-
room: r.room,
|
|
396
|
-
date: palaceDate,
|
|
397
|
-
});
|
|
398
|
-
}
|
|
399
|
-
}
|
|
400
|
-
catch { /* palace may not be initialized */ }
|
|
401
|
-
// ── 2. Journal ───────────────────────────────────────────────────────────
|
|
402
|
-
// Internal score: Ebbinghaus decay (S=2 days, fast) × keyword match.
|
|
403
|
-
// Journal is ephemeral — recent entries are useful; old ones rarely are.
|
|
404
|
-
try {
|
|
405
|
-
const journalResults = await journalSearch({
|
|
406
|
-
query,
|
|
407
|
-
project,
|
|
408
|
-
include_palace: false,
|
|
409
|
-
limit: Math.ceil(limit * 1.5),
|
|
410
|
-
since,
|
|
411
|
-
});
|
|
412
|
-
sourcesQueried.push("journal");
|
|
413
|
-
for (const r of journalResults.results) {
|
|
414
|
-
const title = `${r.date} / ${r.section}`;
|
|
415
|
-
const id = stableId("journal", title);
|
|
416
|
-
const days = daysSince(r.date);
|
|
417
|
-
const recency = ebbinghaus(days, EBBINGHAUS_S.journal);
|
|
418
|
-
const exactness = keywordExactness(query, r.excerpt);
|
|
419
|
-
// Equal weight: if the entry is recent AND relevant it scores well.
|
|
420
|
-
// Old journal entries drop fast due to S=2.
|
|
421
|
-
const internalScore = recency * 0.50 + exactness * 0.50;
|
|
422
|
-
journalItems.push({
|
|
423
|
-
id,
|
|
424
|
-
source: "journal",
|
|
425
|
-
title,
|
|
426
|
-
excerpt: r.excerpt,
|
|
427
|
-
score: internalScore,
|
|
428
|
-
// internalScore is already 0..1 → cosine scale.
|
|
429
|
-
...label(internalScore, "cosine"),
|
|
430
|
-
verbatimKey: { kind: "journal", date: r.date },
|
|
431
|
-
date: r.date,
|
|
432
|
-
});
|
|
433
|
-
}
|
|
434
|
-
}
|
|
435
|
-
catch { /* journal may not exist */ }
|
|
436
|
-
// ── 3. Insights ──────────────────────────────────────────────────────────
|
|
437
|
-
// Internal score: keyword relevance × confirmation signal (log-scaled).
|
|
438
|
-
// Insights are timeless learned patterns — confirmation count is the signal,
|
|
439
|
-
// not recency. More confirmations = more reliable.
|
|
232
|
+
let resolvedProject;
|
|
440
233
|
try {
|
|
441
|
-
|
|
442
|
-
context: query,
|
|
443
|
-
limit: limit * 2,
|
|
444
|
-
include_awareness: false,
|
|
445
|
-
});
|
|
446
|
-
sourcesQueried.push("insight");
|
|
447
|
-
const maxRelevance = Math.max(1, ...insightResults.matching_insights.map((i) => i.relevance));
|
|
448
|
-
for (const i of insightResults.matching_insights) {
|
|
449
|
-
const id = stableId("insight", i.title);
|
|
450
|
-
const relevance = i.relevance / maxRelevance;
|
|
451
|
-
const exactness = keywordExactness(query, i.title);
|
|
452
|
-
// log2(confirmed+1)/3 gives: 0→0, 1→0.33, 3→0.67, 7→1.0
|
|
453
|
-
const confirmation = Math.min(1.0, Math.log2(i.confirmed + 1) / 3);
|
|
454
|
-
const internalScore = relevance * 0.40 + exactness * 0.35 + confirmation * 0.25;
|
|
455
|
-
// Terse, display-friendly excerpt — deliberately just metadata (kept
|
|
456
|
-
// this way for recall.ts / CLI ambient-injection compactness; see
|
|
457
|
-
// `fusionKey`'s doc comment on SmartRecallResultItem for why the
|
|
458
|
-
// insight's `title` is NOT folded in here).
|
|
459
|
-
const rawExcerpt = `[${i.severity}] ${i.applies_when.join(", ")}`;
|
|
460
|
-
// Fix 5b: `rawExcerpt` alone (severity+tags only) is too low-entropy to
|
|
461
|
-
// serve as a cross-source fusion identity — two UNRELATED insights that
|
|
462
|
-
// happen to share a severity + applies_when tag set would otherwise
|
|
463
|
-
// collide in fuseCanonical() and silently absorb one another's score.
|
|
464
|
-
// Leading with the insight's own `title` (its real distinguishing
|
|
465
|
-
// content) makes that structurally impossible.
|
|
466
|
-
const fusionSeed = `${i.title} ${rawExcerpt}`;
|
|
467
|
-
insightItems.push({
|
|
468
|
-
id,
|
|
469
|
-
source: "insight",
|
|
470
|
-
title: i.title,
|
|
471
|
-
excerpt: rawExcerpt.length > 300 ? rawExcerpt.slice(0, 300) + "..." : rawExcerpt,
|
|
472
|
-
fusionKey: fusionSeed.length > 300 ? fusionSeed.slice(0, 300) + "..." : fusionSeed,
|
|
473
|
-
score: internalScore,
|
|
474
|
-
// internalScore is already 0..1 → cosine scale.
|
|
475
|
-
...label(internalScore, "cosine"),
|
|
476
|
-
severity: i.severity,
|
|
477
|
-
});
|
|
478
|
-
}
|
|
234
|
+
resolvedProject = await resolveProject(project);
|
|
479
235
|
}
|
|
480
|
-
catch {
|
|
481
|
-
|
|
482
|
-
// Each source ranks by its own internal score (apples vs apples).
|
|
483
|
-
// RRF then combines by rank position — no cross-source score comparison.
|
|
484
|
-
palaceItems.sort((a, b) => b.score - a.score);
|
|
485
|
-
journalItems.sort((a, b) => b.score - a.score);
|
|
486
|
-
insightItems.sort((a, b) => b.score - a.score);
|
|
487
|
-
// Fix 4/5: raw per-source candidate counts, captured BEFORE fusion
|
|
488
|
-
// collapses same-excerpt cross-source duplicates into one canonical entry.
|
|
489
|
-
// This is the number smartRecall()'s total_searched should report.
|
|
490
|
-
const rawCandidateCounts = {
|
|
491
|
-
palace: palaceItems.length,
|
|
492
|
-
journal: journalItems.length,
|
|
493
|
-
insight: insightItems.length,
|
|
494
|
-
};
|
|
495
|
-
// Stage 1: within-source, per-document accumulation (unchanged from
|
|
496
|
-
// pre-Fix-5 behavior — see applyRRF's doc comment).
|
|
497
|
-
const rrfMap = new Map();
|
|
498
|
-
applyRRF(palaceItems, rrfMap);
|
|
499
|
-
applyRRF(journalItems, rrfMap);
|
|
500
|
-
applyRRF(insightItems, rrfMap);
|
|
501
|
-
// Stage 2 (Fix 5): cross-source canonical fusion by normalized excerpt.
|
|
502
|
-
const fusedMap = fuseCanonical(rrfMap);
|
|
503
|
-
// ── 4.5. Hot-window recency boost ──────────────────────────────────────
|
|
504
|
-
// Very recent items get a score multiplier. In active project work,
|
|
505
|
-
// the most recent context is almost always the most relevant.
|
|
506
|
-
// This supplements Ebbinghaus decay (which handles medium-term) with
|
|
507
|
-
// an ultra-short-term boost.
|
|
508
|
-
// Palace items have date: undefined — they are timeless and unaffected.
|
|
509
|
-
for (const entry of fusedMap.values()) {
|
|
510
|
-
if (entry.item.date) {
|
|
511
|
-
const hoursAgo = (Date.now() - new Date(entry.item.date).getTime()) / (1000 * 60 * 60);
|
|
512
|
-
if (hoursAgo < 6) {
|
|
513
|
-
entry.score *= 3.0;
|
|
514
|
-
}
|
|
515
|
-
else if (hoursAgo < 24) {
|
|
516
|
-
entry.score *= 2.0;
|
|
517
|
-
}
|
|
518
|
-
else if (hoursAgo < 72) {
|
|
519
|
-
entry.score *= 1.3;
|
|
520
|
-
}
|
|
521
|
-
// > 72 hours: no boost (normal decay handles it)
|
|
522
|
-
}
|
|
523
|
-
}
|
|
524
|
-
// ── 5. Deduplicate by fusion identity ────────────────────────────────────
|
|
525
|
-
// Defensive pass only (Fix 5): fusedMap is already keyed by
|
|
526
|
-
// fusionIdentity() (Stage 2 above, Fix 5b), so it is structurally
|
|
527
|
-
// impossible for two entries already in fusedMap to share an identity —
|
|
528
|
-
// the real cross-source fusion job now happens in fuseCanonical(), not
|
|
529
|
-
// here. This loop is kept as a cheap safety net for any future code path
|
|
530
|
-
// that might append near-duplicate entries, and to keep score/label
|
|
531
|
-
// materialization + `alsoFoundIn` derivation in one place.
|
|
532
|
-
const seen = new Set();
|
|
533
|
-
const deduped = [];
|
|
534
|
-
for (const { score, item, sources } of fusedMap.values()) {
|
|
535
|
-
const key = fusionIdentity(item);
|
|
536
|
-
if (seen.has(key))
|
|
537
|
-
continue;
|
|
538
|
-
seen.add(key);
|
|
539
|
-
// Provenance: every OTHER source (besides the primary/display `item.source`)
|
|
540
|
-
// that contributed a hit fusing into this canonical entry (Fix 5).
|
|
541
|
-
const alsoFoundIn = [...sources].filter((s) => s !== item.source);
|
|
542
|
-
// fusionKey is an internal-only identity override (Fix 5b) — strip it
|
|
543
|
-
// before materializing the public result item so it never leaks into
|
|
544
|
-
// smart_recall's JSON output (the smart_recall MCP tool serializes
|
|
545
|
-
// SmartRecallResultItem[] verbatim via JSON.stringify), matching this
|
|
546
|
-
// file's existing discipline of keeping internal-only signals out of
|
|
547
|
-
// consumer-facing objects (see the RAW_CANDIDATE_COUNTS side channel).
|
|
548
|
-
const { fusionKey: _fusionKey, ...displayItem } = item;
|
|
549
|
-
// Post-RRF (+ hot-window boost) score → rrf-local scale. This SET of
|
|
550
|
-
// `calibrated` is what the bridge gate reads (NOT the later boosted score).
|
|
551
|
-
deduped.push({
|
|
552
|
-
...displayItem,
|
|
553
|
-
score,
|
|
554
|
-
...label(score, "rrf-local"),
|
|
555
|
-
...(alsoFoundIn.length > 0 ? { alsoFoundIn } : {}),
|
|
556
|
-
});
|
|
236
|
+
catch {
|
|
237
|
+
resolvedProject = project ?? "auto";
|
|
557
238
|
}
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
239
|
+
const result = await queryMemory({
|
|
240
|
+
query,
|
|
241
|
+
project: resolvedProject,
|
|
242
|
+
// Order matters: RRF/fuseCanonical's "primary/display source" is
|
|
243
|
+
// whichever source's items were inserted into the fusion map FIRST (Map
|
|
244
|
+
// iteration = insertion order). The ORIGINAL localRecallSearch queried
|
|
245
|
+
// palace, then journal, then insight — this order must be preserved
|
|
246
|
+
// exactly (audit-retrieval-accounting.test.mjs asserts on it directly).
|
|
247
|
+
tiers: ["palace", "journal", "insight"],
|
|
248
|
+
limit,
|
|
249
|
+
since,
|
|
250
|
+
});
|
|
251
|
+
// Final materialization: rrf-local confidence label (matches the ORIGINAL
|
|
252
|
+
// dedup loop's `...label(score, "rrf-local")` — the only labeling that
|
|
253
|
+
// ever survived to the final result; a pre-fusion "cosine" label was
|
|
254
|
+
// computed by the old code too but was always overwritten here, so
|
|
255
|
+
// query-memory.ts's pipeline items never compute it at all — dead weight
|
|
256
|
+
// correctly dropped, not a behavior change).
|
|
257
|
+
const deduped = result.items.map((item) => ({
|
|
258
|
+
id: item.id,
|
|
259
|
+
source: item.source,
|
|
260
|
+
...(item.alsoFoundIn && item.alsoFoundIn.length > 0 ? { alsoFoundIn: item.alsoFoundIn } : {}),
|
|
261
|
+
title: item.title,
|
|
262
|
+
excerpt: item.excerpt,
|
|
263
|
+
score: item.score,
|
|
264
|
+
...label(item.score, "rrf-local"),
|
|
265
|
+
verbatimKey: verbatimKeyFor(item),
|
|
266
|
+
...(item.room ? { room: item.room } : {}),
|
|
267
|
+
...(item.date ? { date: item.date } : {}),
|
|
268
|
+
...(item.severity ? { severity: item.severity } : {}),
|
|
269
|
+
// W5a salvage (HIGH-3, 2026-08-31): thread the CONTRADICTION stage's
|
|
270
|
+
// annotation through — this field-list map was previously the exact
|
|
271
|
+
// place `supersededBy`/`conflictsWith` were silently dropped, even
|
|
272
|
+
// though they had already, invisibly, changed this item's `score`/rank.
|
|
273
|
+
...(item.supersededBy ? { supersededBy: item.supersededBy } : {}),
|
|
274
|
+
...(item.conflictsWith && item.conflictsWith.length > 0 ? { conflictsWith: item.conflictsWith } : {}),
|
|
275
|
+
}));
|
|
276
|
+
// Graph walk — surface 1-hop linked memories not already in results.
|
|
277
|
+
// Uses the RESOLVED project (a characterized fix over the original, which
|
|
278
|
+
// used the raw, possibly-unresolved `project` parameter here — a latent
|
|
279
|
+
// H1-class inconsistency for the "auto"-literal edge case; every existing
|
|
280
|
+
// caller passes an already-resolved explicit slug, so this is a no-op
|
|
281
|
+
// difference for the common case and a strict improvement otherwise).
|
|
282
|
+
if (deduped.length > 0 && resolvedProject) {
|
|
283
|
+
const pd = palaceDir(resolvedProject);
|
|
563
284
|
const resultIds = new Set(deduped.map((r) => r.id));
|
|
564
285
|
const topRoom = deduped[0].room;
|
|
565
286
|
if (topRoom) {
|
|
@@ -585,107 +306,22 @@ export async function localRecallSearch(query, project, limit, since) {
|
|
|
585
306
|
// Attach the raw pre-fusion candidate counts as a hidden side channel
|
|
586
307
|
// (Fix 4/5) — invisible to JSON.stringify/Object.keys/for-in and to every
|
|
587
308
|
// existing consumer that treats this as a plain SmartRecallResultItem[].
|
|
309
|
+
const rawCandidateCounts = {
|
|
310
|
+
palace: result.candidatesBySource.palace ?? 0,
|
|
311
|
+
journal: result.candidatesBySource.journal ?? 0,
|
|
312
|
+
insight: result.candidatesBySource.insight ?? 0,
|
|
313
|
+
};
|
|
588
314
|
deduped[RAW_CANDIDATE_COUNTS] = rawCandidateCounts;
|
|
589
315
|
return deduped;
|
|
590
316
|
}
|
|
591
317
|
/**
|
|
592
|
-
*
|
|
593
|
-
*
|
|
594
|
-
*
|
|
595
|
-
*
|
|
596
|
-
*
|
|
597
|
-
* CONFIDENCE_FLOOR.medium on the "cosine" scale, so it can NEVER be labeled
|
|
598
|
-
* "medium"/"high" — the label + provenance path are the load-bearing
|
|
599
|
-
* contract here, not the raw keyword-overlap number. This is the gated,
|
|
600
|
-
* labeled replacement for the old accidental `journalDirs(..., true)`
|
|
601
|
-
* descent into archive/raw (see journalDirs' doc comment in storage/paths.ts
|
|
602
|
-
* for the incident this fixes).
|
|
603
|
-
*
|
|
604
|
-
* Deliberately unranked beyond "newest file first, top-to-bottom within a
|
|
605
|
-
* file" (same line-grep flavor as journalSearch) — this is a last-resort
|
|
606
|
-
* fallback, not a competing ranked source, so it stops as soon as `limit`
|
|
607
|
-
* items are collected rather than scanning every archive file to find the
|
|
608
|
-
* globally-best matches.
|
|
318
|
+
* Internal side channel: raw per-source candidate counts (Fix 4/5), attached
|
|
319
|
+
* to the array localRecallSearch returns so smartRecall() can report a
|
|
320
|
+
* genuine pre-fusion total_searched without changing localRecallSearch's
|
|
321
|
+
* public return type (still a plain SmartRecallResultItem[] — several
|
|
322
|
+
* existing tests and recall-backend.ts depend on that exact shape).
|
|
609
323
|
*/
|
|
610
|
-
|
|
611
|
-
const dir = archiveRawDir(project);
|
|
612
|
-
if (!fs.existsSync(dir))
|
|
613
|
-
return [];
|
|
614
|
-
// CJK-aware (P0-b): shared tokenizer — see keywordExactness above for the
|
|
615
|
-
// same fix applied to this fallback source's query-side matching.
|
|
616
|
-
const keywords = tokenizeWords(query);
|
|
617
|
-
if (keywords.length === 0 || limit <= 0)
|
|
618
|
-
return [];
|
|
619
|
-
let files;
|
|
620
|
-
try {
|
|
621
|
-
files = fs.readdirSync(dir).filter((f) => f.endsWith(".md"));
|
|
622
|
-
}
|
|
623
|
-
catch {
|
|
624
|
-
return [];
|
|
625
|
-
}
|
|
626
|
-
// Newest raw dump first — a low-confidence fallback should prefer recent work.
|
|
627
|
-
files.sort((a, b) => b.localeCompare(a));
|
|
628
|
-
const items = [];
|
|
629
|
-
for (const file of files) {
|
|
630
|
-
if (items.length >= limit)
|
|
631
|
-
break;
|
|
632
|
-
let content;
|
|
633
|
-
try {
|
|
634
|
-
content = fs.readFileSync(path.join(dir, file), "utf-8");
|
|
635
|
-
}
|
|
636
|
-
catch {
|
|
637
|
-
continue;
|
|
638
|
-
}
|
|
639
|
-
const dateMatch = file.match(/^(\d{4}-\d{2}-\d{2})/);
|
|
640
|
-
const date = dateMatch ? dateMatch[1] : undefined;
|
|
641
|
-
const lines = content.split("\n");
|
|
642
|
-
for (let i = 0; i < lines.length; i++) {
|
|
643
|
-
if (items.length >= limit)
|
|
644
|
-
break;
|
|
645
|
-
const line = lines[i];
|
|
646
|
-
const lineLower = line.toLowerCase();
|
|
647
|
-
if (!keywords.some((kw) => lineLower.includes(kw)))
|
|
648
|
-
continue;
|
|
649
|
-
let matchIdx = line.length;
|
|
650
|
-
for (const kw of keywords) {
|
|
651
|
-
const idx = lineLower.indexOf(kw);
|
|
652
|
-
if (idx !== -1 && idx < matchIdx)
|
|
653
|
-
matchIdx = idx;
|
|
654
|
-
}
|
|
655
|
-
const start = Math.max(0, matchIdx - 100);
|
|
656
|
-
const end = Math.min(line.length, matchIdx + 150);
|
|
657
|
-
let snippet = line.slice(start, end).trim();
|
|
658
|
-
if (start > 0)
|
|
659
|
-
snippet = "..." + snippet;
|
|
660
|
-
if (end < line.length)
|
|
661
|
-
snippet = snippet + "...";
|
|
662
|
-
if (!snippet)
|
|
663
|
-
continue; // an all-whitespace "match" line is not a useful excerpt
|
|
664
|
-
// P0-a (2026-08-18): archive/raw stays byte-identical ON DISK (lossless
|
|
665
|
-
// tier), but this function greps that raw content directly into a
|
|
666
|
-
// recall() result's `excerpt` field — the SURFACING BOUNDARY. Scrub
|
|
667
|
-
// the snippet here, at the read/return edge, never the file itself.
|
|
668
|
-
snippet = scrubForCloud(snippet);
|
|
669
|
-
const provenance = path.join("journal", "archive", "raw", file);
|
|
670
|
-
// Keyword overlap reflects match quality among archive hits, but is
|
|
671
|
-
// hard-capped below CONFIDENCE_FLOOR.medium so this source structurally
|
|
672
|
-
// can never present itself as anything but low/weak confidence.
|
|
673
|
-
const rawScore = Math.min(keywordExactness(query, line), CONFIDENCE_FLOOR.medium - 0.01);
|
|
674
|
-
items.push({
|
|
675
|
-
id: stableId("archive", `${file}:${i}`),
|
|
676
|
-
source: "archive",
|
|
677
|
-
title: `archive/${file}`,
|
|
678
|
-
excerpt: `[raw-archive · low-confidence · ${provenance}] ${snippet}`,
|
|
679
|
-
score: rawScore,
|
|
680
|
-
// internalScore is 0..1 → cosine scale, same convention as the other 3 sources.
|
|
681
|
-
...label(rawScore, "cosine"),
|
|
682
|
-
verbatimKey: { kind: "archive", date, file },
|
|
683
|
-
date,
|
|
684
|
-
});
|
|
685
|
-
}
|
|
686
|
-
}
|
|
687
|
-
return items;
|
|
688
|
-
}
|
|
324
|
+
const RAW_CANDIDATE_COUNTS = Symbol("rawCandidateCounts");
|
|
689
325
|
/**
|
|
690
326
|
* Budget for the semantic (remote) backend in ms.
|
|
691
327
|
* Overridable via AGENT_RECALL_RECALL_BUDGET_MS for tuning / tests.
|
|
@@ -815,17 +451,19 @@ export async function smartRecall(input) {
|
|
|
815
451
|
bridged = collected;
|
|
816
452
|
}
|
|
817
453
|
// ── EXPLICIT 4TH SOURCE: archive fallback (F4, continuity wave 2026-07-31) ──
|
|
454
|
+
// WAVE 2: fetch/score logic moved to retrieval/query-memory.ts's
|
|
455
|
+
// `queryArchiveFallback` (see this file's header + that function's own doc
|
|
456
|
+
// comment for why the GATING policy below stays here, unchanged).
|
|
818
457
|
// Gated on the SAME CONFIDENCE_FLOOR.medium constant as the Bridge gate
|
|
819
458
|
// above, but on the fused TOP result only (not every low item) — "the
|
|
820
459
|
// fused top-confidence of palace/journal/insight". This adds brand-new
|
|
821
460
|
// result items sourced from journal/archive/raw/, so it must never compete
|
|
822
|
-
// for rank inside
|
|
823
|
-
//
|
|
824
|
-
//
|
|
825
|
-
//
|
|
826
|
-
//
|
|
827
|
-
//
|
|
828
|
-
// itself.
|
|
461
|
+
// for rank inside the palace/journal/insight RRF fusion — it only steps in
|
|
462
|
+
// once those 3 sources have already failed to produce a confident #1
|
|
463
|
+
// answer. Placed AFTER the Bridge above so the Bridge's own `low` filter
|
|
464
|
+
// (which also matches any verbatimKey-bearing item) only ever considers
|
|
465
|
+
// genuine palace/journal/insight items — an archive item is already a raw
|
|
466
|
+
// excerpt and would gain nothing from being drilled into itself.
|
|
829
467
|
let archiveSourceRan = false;
|
|
830
468
|
// L3 (review, 2026-07-31; documented only — no behavior change this wave):
|
|
831
469
|
// `finalResults[0]` is rank-0 by the POST-FEEDBACK boosted `score` (see the
|
|
@@ -851,8 +489,19 @@ export async function smartRecall(input) {
|
|
|
851
489
|
// the gate fired without the item COUNT ever violating `limit`.
|
|
852
490
|
const remainingBudget = Math.max(0, limit - finalResults.length);
|
|
853
491
|
if (remainingBudget > 0) {
|
|
854
|
-
const archiveItems =
|
|
855
|
-
|
|
492
|
+
const archiveItems = queryArchiveFallback(resolvedProject, input.query, Math.min(ARCHIVE_SOURCE_CAP, remainingBudget));
|
|
493
|
+
for (const item of archiveItems) {
|
|
494
|
+
finalResults.push({
|
|
495
|
+
id: item.id,
|
|
496
|
+
source: "archive",
|
|
497
|
+
title: item.title,
|
|
498
|
+
excerpt: item.excerpt,
|
|
499
|
+
score: item.score,
|
|
500
|
+
...label(item.score, "cosine"),
|
|
501
|
+
verbatimKey: verbatimKeyFor(item),
|
|
502
|
+
...(item.date ? { date: item.date } : {}),
|
|
503
|
+
});
|
|
504
|
+
}
|
|
856
505
|
}
|
|
857
506
|
}
|
|
858
507
|
// Fix 4/5: total_searched should be the true distinct-candidate count from
|