agent-recall-core 3.4.46 → 3.4.47
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/helpers/handoff.d.ts.map +1 -1
- package/dist/helpers/handoff.js +17 -9
- package/dist/helpers/handoff.js.map +1 -1
- package/dist/helpers/journal-files.d.ts +24 -1
- package/dist/helpers/journal-files.d.ts.map +1 -1
- package/dist/helpers/journal-files.js +29 -10
- package/dist/helpers/journal-files.js.map +1 -1
- package/dist/helpers/journal-filter.d.ts +25 -0
- package/dist/helpers/journal-filter.d.ts.map +1 -1
- package/dist/helpers/journal-filter.js +12 -1
- package/dist/helpers/journal-filter.js.map +1 -1
- package/dist/index.d.ts +12 -4
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +33 -4
- package/dist/index.js.map +1 -1
- package/dist/retrieval/candidates.d.ts +314 -0
- package/dist/retrieval/candidates.d.ts.map +1 -0
- package/dist/retrieval/candidates.js +321 -0
- package/dist/retrieval/candidates.js.map +1 -0
- package/dist/retrieval/contradiction.d.ts +232 -0
- package/dist/retrieval/contradiction.d.ts.map +1 -0
- package/dist/retrieval/contradiction.js +320 -0
- package/dist/retrieval/contradiction.js.map +1 -0
- package/dist/retrieval/query-memory.d.ts +419 -0
- package/dist/retrieval/query-memory.d.ts.map +1 -0
- package/dist/retrieval/query-memory.js +1068 -0
- package/dist/retrieval/query-memory.js.map +1 -0
- package/dist/retrieval/scope.d.ts +75 -0
- package/dist/retrieval/scope.d.ts.map +1 -0
- package/dist/retrieval/scope.js +91 -0
- package/dist/retrieval/scope.js.map +1 -0
- package/dist/storage/corrections.d.ts +31 -1
- package/dist/storage/corrections.d.ts.map +1 -1
- package/dist/storage/corrections.js +34 -1
- package/dist/storage/corrections.js.map +1 -1
- package/dist/storage/lifecycle-telemetry.d.ts +20 -5
- package/dist/storage/lifecycle-telemetry.d.ts.map +1 -1
- package/dist/storage/lifecycle-telemetry.js +16 -5
- package/dist/storage/lifecycle-telemetry.js.map +1 -1
- package/dist/storage/session.d.ts +28 -1
- package/dist/storage/session.d.ts.map +1 -1
- package/dist/storage/session.js +41 -8
- package/dist/storage/session.js.map +1 -1
- package/dist/supabase/recall-backend.d.ts +25 -2
- package/dist/supabase/recall-backend.d.ts.map +1 -1
- package/dist/supabase/recall-backend.js +82 -21
- package/dist/supabase/recall-backend.js.map +1 -1
- package/dist/supabase/sync.d.ts +41 -0
- package/dist/supabase/sync.d.ts.map +1 -1
- package/dist/supabase/sync.js +46 -0
- package/dist/supabase/sync.js.map +1 -1
- package/dist/tools-logic/check.d.ts.map +1 -1
- package/dist/tools-logic/check.js +62 -36
- package/dist/tools-logic/check.js.map +1 -1
- package/dist/tools-logic/drill-down.d.ts +24 -0
- package/dist/tools-logic/drill-down.d.ts.map +1 -1
- package/dist/tools-logic/drill-down.js +36 -0
- package/dist/tools-logic/drill-down.js.map +1 -1
- package/dist/tools-logic/journal-cold-start.d.ts.map +1 -1
- package/dist/tools-logic/journal-cold-start.js +48 -9
- package/dist/tools-logic/journal-cold-start.js.map +1 -1
- package/dist/tools-logic/journal-list.d.ts +42 -0
- package/dist/tools-logic/journal-list.d.ts.map +1 -1
- package/dist/tools-logic/journal-list.js +60 -10
- package/dist/tools-logic/journal-list.js.map +1 -1
- package/dist/tools-logic/journal-read.d.ts.map +1 -1
- package/dist/tools-logic/journal-read.js +22 -13
- package/dist/tools-logic/journal-read.js.map +1 -1
- package/dist/tools-logic/journal-search.d.ts +58 -0
- package/dist/tools-logic/journal-search.d.ts.map +1 -1
- package/dist/tools-logic/journal-search.js +123 -92
- package/dist/tools-logic/journal-search.js.map +1 -1
- package/dist/tools-logic/palace-search.d.ts.map +1 -1
- package/dist/tools-logic/palace-search.js +30 -11
- package/dist/tools-logic/palace-search.js.map +1 -1
- package/dist/tools-logic/palace-walk.d.ts +10 -0
- package/dist/tools-logic/palace-walk.d.ts.map +1 -1
- package/dist/tools-logic/palace-walk.js +25 -16
- package/dist/tools-logic/palace-walk.js.map +1 -1
- package/dist/tools-logic/recall-insight.d.ts +20 -0
- package/dist/tools-logic/recall-insight.d.ts.map +1 -1
- package/dist/tools-logic/recall-insight.js +12 -1
- package/dist/tools-logic/recall-insight.js.map +1 -1
- package/dist/tools-logic/session-end.d.ts.map +1 -1
- package/dist/tools-logic/session-end.js +23 -7
- package/dist/tools-logic/session-end.js.map +1 -1
- package/dist/tools-logic/session-start.d.ts +85 -0
- package/dist/tools-logic/session-start.d.ts.map +1 -1
- package/dist/tools-logic/session-start.js +245 -108
- package/dist/tools-logic/session-start.js.map +1 -1
- package/dist/tools-logic/smart-recall.d.ts +87 -62
- package/dist/tools-logic/smart-recall.d.ts.map +1 -1
- package/dist/tools-logic/smart-recall.js +159 -510
- package/dist/tools-logic/smart-recall.js.map +1 -1
- package/package.json +1 -1
|
@@ -0,0 +1,1068 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* retrieval/query-memory.ts — Wave 2 of the shared retrieval pipeline
|
|
3
|
+
* (reports/2026-08-21-architecture-review.md §4, reports/2026-08-29-kickoff-plan.md
|
|
4
|
+
* Wave 2, plywood SOP ecbd4351).
|
|
5
|
+
*
|
|
6
|
+
* THE PROBLEM THIS FILE FIXES: Wave 1 (retrieval/candidates.ts) built a shared
|
|
7
|
+
* TYPED reader with trust-tagging baked in, but shipped it as a LEAF UTILITY —
|
|
8
|
+
* nothing was forced to call it. The architecture review's whole point is that
|
|
9
|
+
* a leaf utility gets forgotten by exactly the surfaces that matter most. This
|
|
10
|
+
* file is the fix: a MANDATORY PIPELINE every candidate is forced through
|
|
11
|
+
* before it can be ranked or surfaced — `queryMemory()` composes six stages
|
|
12
|
+
* (fetch -> trust-filter -> tokenize+score -> scope -> rank/fuse -> fence) and
|
|
13
|
+
* NO stage is individually callable/skippable by a consumer; a caller gets
|
|
14
|
+
* `queryMemory()`'s return value or nothing.
|
|
15
|
+
*
|
|
16
|
+
* WAVE 2 SCOPE: build the pipeline, then migrate exactly ONE caller onto it —
|
|
17
|
+
* `smartRecall()` (tools-logic/smart-recall.ts). journalSearch, palaceSearch,
|
|
18
|
+
* recallInsight, resurrect, session_start are UNCHANGED this wave (Wave 3).
|
|
19
|
+
*
|
|
20
|
+
* ── CHALLENGE (a): one canonical scoring formula, or pluggable per tier? ──
|
|
21
|
+
* PLUGGABLE PER TIER — confirmed, not unified. smart-recall.ts's own file
|
|
22
|
+
* header (still true, read in full before writing this file) spends 100 lines
|
|
23
|
+
* justifying why journal (Ebbinghaus S=2, fast episodic decay), palace
|
|
24
|
+
* (S=9999, salience-weighted, near-zero decay) and insight (confirmation-count
|
|
25
|
+
* driven, not time-based at all) encode three genuinely different memory-type
|
|
26
|
+
* semantics — collapsing them into one formula would not be "reconciliation",
|
|
27
|
+
* it would be the same cross-source-raw-score-averaging bug smart-recall.ts's
|
|
28
|
+
* own Fix 1 (RRF) already fixed once, reintroduced one level up. What DOES
|
|
29
|
+
* unify into "ONE score stage" is the STAGE itself (TOKENIZE+SCORE as one named
|
|
30
|
+
* step in the pipeline, always run, never skippable) with an internal per-tier
|
|
31
|
+
* strategy table (`scoreJournalTier`/`scorePalaceTier`/`scoreInsightTier`
|
|
32
|
+
* below) — this is exactly the architecture review's own §4.2 step 3 wording:
|
|
33
|
+
* "applies one scoring strategy per tier ... no scoring behavior needs to
|
|
34
|
+
* change on day one." The five formulas smart-recall.ts documents (RRF k=60,
|
|
35
|
+
* Ebbinghaus, hot-window recency, betaUtility feedback, two-stage fusion) are
|
|
36
|
+
* split by WHERE they belong in a genuinely reusable pipeline, not merged:
|
|
37
|
+
* - RRF (two-stage: applyRRF + fuseCanonical) + hot-window recency:
|
|
38
|
+
* pipeline-owned (RANK/FUSE stage below) — generic to any date-bearing,
|
|
39
|
+
* multi-tier candidate set, not smart_recall-specific.
|
|
40
|
+
* - Ebbinghaus decay, palace salience/IDF blend, insight confirmation blend:
|
|
41
|
+
* pipeline-owned (TOKENIZE+SCORE stage, per-tier) — these ARE the
|
|
42
|
+
* per-tier strategies the review names.
|
|
43
|
+
* - betaUtility feedback multiplier + calibratedConfidence labeling:
|
|
44
|
+
* LEFT IN smart-recall.ts, NOT pulled into the pipeline. Both are
|
|
45
|
+
* smart_recall-SPECIFIC concepts (a user feedback log keyed by
|
|
46
|
+
* result id/title; a confidence scale built to compare smart_recall's
|
|
47
|
+
* OWN local-vs-remote backends) that no other Wave-3 migration target
|
|
48
|
+
* (journalSearch, palaceSearch, resurrect, session_start) has any
|
|
49
|
+
* equivalent of today. Baking them into queryMemory() would force every
|
|
50
|
+
* future consumer to inherit smart_recall's own feedback-log semantics
|
|
51
|
+
* whether or not they make sense for that surface — exactly the
|
|
52
|
+
* "leaf utility everyone must remember NOT to blindly trust" anti-pattern
|
|
53
|
+
* this effort exists to close, just relocated one level deeper. They stay
|
|
54
|
+
* as smart-recall.ts's OWN post-pipeline processing, applied uniformly to
|
|
55
|
+
* whatever queryMemory() returns.
|
|
56
|
+
*
|
|
57
|
+
* ── CHALLENGE (b): the Wave-1 superset — what does smart_recall now SEE? ──
|
|
58
|
+
* readTierCandidates("palace-room", ...) surfaces every `.md` file in a room,
|
|
59
|
+
* whereas the OLD smart-recall.ts called `palaceSearch()` directly, which has
|
|
60
|
+
* a KNOWN, DOCUMENTED gap (packages/core/test/identity-trust-completeness.test.mjs's
|
|
61
|
+
* `ALLOWLIST_PALACE["tools-logic/palace-search.ts"]`, marked "(KNOWN GAP)"):
|
|
62
|
+
* palaceSearch() never calls the rescue-quarantine choke, so a rescue-tagged
|
|
63
|
+
* file planted in a room would have surfaced through smart_recall's OLD palace
|
|
64
|
+
* source completely unfiltered. Wave 2's migration CLOSES this gap for the
|
|
65
|
+
* smart_recall path specifically (see `scorePalaceTier` below: trust-filter
|
|
66
|
+
* runs before scoring, unconditionally) — a genuine, characterized SECURITY
|
|
67
|
+
* IMPROVEMENT, not a neutral refactor; see this file's own destination-proof
|
|
68
|
+
* test (retrieval/query-memory.test.mjs) for the planted-rescue-content proof.
|
|
69
|
+
* `palaceSearch()` itself (the standalone MCP tool / `ar palace search`
|
|
70
|
+
* surface) is UNCHANGED and still carries the gap — only smart_recall's
|
|
71
|
+
* INTERNAL route to palace-room content no longer does. This wave's report
|
|
72
|
+
* updates identity-trust-completeness.test.mjs's ALLOWLIST_PALACE entry for
|
|
73
|
+
* palace-search.ts to say so precisely (the function is still a known gap;
|
|
74
|
+
* smart-recall.ts no longer calls it).
|
|
75
|
+
*
|
|
76
|
+
* Week-summaries / rollup-archive: smart-recall.ts's journal source already
|
|
77
|
+
* reads rollup-archive today (journalSearch(...) internally calls
|
|
78
|
+
* `journalDirs(slug, true)`) — `scoreJournalTier` below defaults
|
|
79
|
+
* `includeRollupArchive: true` to match, so this is NOT new surface area, it
|
|
80
|
+
* is preserved surface area.
|
|
81
|
+
*
|
|
82
|
+
* Raw-archive: NOT part of the competing tier set here (see the "archive
|
|
83
|
+
* never enters RRF" note on `queryArchiveFallback` below) — unchanged
|
|
84
|
+
* structurally, just relocated into this module.
|
|
85
|
+
*
|
|
86
|
+
* ── CHALLENGE (c): where "exact equivalence" genuinely cannot hold ──
|
|
87
|
+
* Documented per-decision below, not glossed over:
|
|
88
|
+
* 1. FENCE (stage 6) is NOT applied inside this pipeline to `smart_recall`'s
|
|
89
|
+
* per-item fields. See `renderFenced()`'s doc comment — forcing
|
|
90
|
+
* `fenceMemory()` onto every candidate excerpt would corrupt
|
|
91
|
+
* `SmartRecallResultItem.excerpt` (breaking the very "preserve external
|
|
92
|
+
* contract exactly" requirement this wave is graded on) AND double-fence
|
|
93
|
+
* the payload, since `packages/mcp-server/src/tools/{smart-recall,recall}.ts`
|
|
94
|
+
* ALREADY correctly call `fenceMemory()` ONCE around the whole rendered
|
|
95
|
+
* payload at the true MCP-tool surfacing boundary (P1 fence, TOW2-388,
|
|
96
|
+
* predates this wave). This is a genuine architectural conflict between
|
|
97
|
+
* the SOP's literal "fence as the final stage" wording and the
|
|
98
|
+
* already-shipped, already-tested, more-correct fencing architecture —
|
|
99
|
+
* resolved in favor of the shipped architecture; see `renderFenced()`.
|
|
100
|
+
* 2. Journal-tier truncation order: the OLD `journalSearch()` applies its
|
|
101
|
+
* `limit` cutoff DURING an unsorted (raw `readdirSync` order)
|
|
102
|
+
* directory/file/line traversal, THEN sorts by date descending — so
|
|
103
|
+
* when total matches exceed `limit`, WHICH matches survive depends on
|
|
104
|
+
* filesystem enumeration order, not recency. `scoreJournalTier` below
|
|
105
|
+
* iterates `readTierCandidates`'s already-date-sorted-descending live
|
|
106
|
+
* half first — a characterized IMPROVEMENT (more predictable, favors
|
|
107
|
+
* recency, matches the tier's own Ebbinghaus-decay intent), not a
|
|
108
|
+
* silent behavior change. CORRECTION (independent review, W2,
|
|
109
|
+
* 2026-08-30): this file's initial cut only sorted the live half — the
|
|
110
|
+
* rollup-archive half (candidates.ts) and the legacy-journal half
|
|
111
|
+
* (`readLegacyJournalCandidates` below) were still built in raw,
|
|
112
|
+
* filesystem-enumeration order before being concatenated on, so a
|
|
113
|
+
* truncation that spilled into either half would still keep an
|
|
114
|
+
* arbitrary subset rather than the newest. Both halves are now ALSO
|
|
115
|
+
* sorted date-descending before concatenation — the sort-before-
|
|
116
|
+
* truncate class is closed across live AND archive (and legacy).
|
|
117
|
+
* SCOPE OF EQUIVALENCE (independent review, W2, 2026-08-30): this
|
|
118
|
+
* archive/legacy date-desc sort is an INTENTIONAL ranking change, and
|
|
119
|
+
* it is NOT order-preserving even below the truncation limit. When two
|
|
120
|
+
* candidates carry TIED per-tier scores (realistic for similarly-aged
|
|
121
|
+
* archive entries whose Ebbinghaus-decay scores coincide), `applyRRF`'s
|
|
122
|
+
* rank/`score` is decided by array position, so re-sorting the archive
|
|
123
|
+
* half changes the tie-break order — and the resulting numeric RRF
|
|
124
|
+
* `score` — of the tied items, even when nothing is truncated. This is
|
|
125
|
+
* a deterministic recency-favoring improvement over the prior
|
|
126
|
+
* arbitrary-filesystem-order tie-break, NOT a byte-identical no-op.
|
|
127
|
+
* Only STEP 1 (the `includeUntrusted` reconciliation below) is
|
|
128
|
+
* byte-identical; this sort is not, and the report's equivalence
|
|
129
|
+
* section is scoped accordingly.
|
|
130
|
+
* 3. Legacy journal directory (`~/.claude/projects/<entry>/memory/journal/` —
|
|
131
|
+
* `storage/paths.ts`'s `journalDirs()` legacy-fallback branch): Wave 1's
|
|
132
|
+
* `readTierCandidates` does NOT cover this (confirmed by reading
|
|
133
|
+
* candidates.ts — it never imports `getLegacyRoot`). `journalSearch()`
|
|
134
|
+
* today DOES scan it (via `journalDirs(slug, true)`). Rather than
|
|
135
|
+
* silently dropping this surface for smart_recall (a real, if likely
|
|
136
|
+
* rare, regression) or re-opening Wave 1's already-merged, already-tested
|
|
137
|
+
* `candidates.ts` to add it there, this file adds a small, self-contained
|
|
138
|
+
* `readLegacyJournalCandidates()` below that mirrors `journalDirs()`'s own
|
|
139
|
+
* legacy-root traversal — closing the gap without touching Wave 1's file.
|
|
140
|
+
* Also now sorted date-descending before returning (see item 2's
|
|
141
|
+
* correction above) — it is a third half of the same journal-tier
|
|
142
|
+
* concatenation, not exempt from the same truncation hazard.
|
|
143
|
+
*
|
|
144
|
+
* ── SCOPE NOTE (independent review, W2, 2026-08-30) ──
|
|
145
|
+
* CRITICAL-2 (rescue-quarantine injection) is closed for `smart_recall` this
|
|
146
|
+
* wave. `palaceSearch()` / `resurrect()` / `session_start()` remain on their
|
|
147
|
+
* own pre-pipeline paths and are UNCHANGED — they carry whatever
|
|
148
|
+
* trust-filtering (or lack of it) they had before this wave. Do not read
|
|
149
|
+
* this file's fixes as closing the gap workspace-wide.
|
|
150
|
+
*
|
|
151
|
+
* ── WAVE 3b UPDATE (2026-08-30, reports/2026-08-30-pipe-w3b-migrate-report.md) ──
|
|
152
|
+
* `journalSearch()` and `recallInsight()` are now migrated onto this
|
|
153
|
+
* pipeline (journalSearch's primary journal scan via `queryMemory({tiers:
|
|
154
|
+
* ["journal"]})`; recallInsight() shares this file's `applyScope` — see
|
|
155
|
+
* `./scope.ts` — directly on its own `recallInsights()` call, without
|
|
156
|
+
* routing through `queryMemory()` itself, to avoid corrupting its external
|
|
157
|
+
* contract's `relevance`/`applies_when`/`confirmed`/`file` fields, which
|
|
158
|
+
* `QueryMemoryItem` does not carry). `palaceSearch()` / `resurrect()` /
|
|
159
|
+
* `session_start()` remain unmigrated — see the Wave 3b report's resurrect
|
|
160
|
+
* assessment for why `resurrect()` in particular is a RECOMMENDATION, not a
|
|
161
|
+
* migration, this wave.
|
|
162
|
+
*
|
|
163
|
+
* ── WAVE 5a UPDATE (2026-08-31, reports/2026-08-31-pipe-w5a-contradiction-
|
|
164
|
+
* report.md) ──
|
|
165
|
+
* A new CONTRADICTION stage (`./contradiction.ts`'s `detectContradictions`)
|
|
166
|
+
* now runs per-tier, immediately after TOKENIZE+SCORE and before RANK/FUSE
|
|
167
|
+
* (see `applyContradictionStage` below) — for `journal` and `palace` tiers
|
|
168
|
+
* ONLY (the `insight` tier has no per-item authored-date signal to resolve a
|
|
169
|
+
* direction with, so it is deliberately skipped this wave, not silently
|
|
170
|
+
* inherited). It reuses `helpers/conflict-scan.ts`'s existing version token
|
|
171
|
+
* grammar to find, within one tier's own already-scored candidate list,
|
|
172
|
+
* pairs that assert the same fact-key with a different semver value. A
|
|
173
|
+
* candidate detected as the STALE side of such a pair is DOWN-RANKED
|
|
174
|
+
* (multiplicative score penalty, mirroring `applyHotWindowBoost`'s pattern)
|
|
175
|
+
* and ANNOTATED (`supersededBy`) — it is NEVER removed from the set. This
|
|
176
|
+
* closes part of the reconciliation gap `reports/2026-08-18-eval-redteam.md`'s
|
|
177
|
+
* HIGH-2 finding named (0% of grammar-detectable contradictions were
|
|
178
|
+
* previously surfaced with any corrective signal) for the subset of that gap
|
|
179
|
+
* the existing grammar can actually see — semantic-prose contradictions
|
|
180
|
+
* (that same finding's own PostgreSQL→CockroachDB example) remain out of
|
|
181
|
+
* reach; see `./contradiction.ts`'s header for why, and this wave's report
|
|
182
|
+
* for the follow-up.
|
|
183
|
+
*
|
|
184
|
+
* ── W5a SALVAGE (2026-08-31, "## W5a salvage" section of the same report) ──
|
|
185
|
+
* The original W5a cut also ran `extractStatusTokens`/`extractKVTokens`
|
|
186
|
+
* through this stage. An independent review found those two branches
|
|
187
|
+
* false-positive-prone in a way that DEFEATS this stage's own safety intent
|
|
188
|
+
* (HIGH-1: common phrasing like "status: blocked" vs "status: stuck" — the
|
|
189
|
+
* SAME status category — got flagged conflicting via the KV branch's literal
|
|
190
|
+
* value comparison; HIGH-2: generic single-word keys like "priority" gave no
|
|
191
|
+
* topical protection, flagging topically-unrelated candidates) and found the
|
|
192
|
+
* `supersededBy`/`conflictsWith` annotation never reached `smart_recall`'s
|
|
193
|
+
* or `journal_search`'s external contract (HIGH-3, invisible to agents).
|
|
194
|
+
* FIXED: `./contradiction.ts`'s `grammarConflict` now checks version tokens
|
|
195
|
+
* ONLY (see that file's header for the full reasoning); the annotation now
|
|
196
|
+
* threads into `SmartRecallResultItem`/`JournalSearchResult.results` (see
|
|
197
|
+
* `smart-recall.ts`'s `localRecallSearch` and `journal-search.ts`'s
|
|
198
|
+
* `journalSearch`). The down-rank+re-sort+never-drop mechanism itself,
|
|
199
|
+
* proven correct by the original W5a review, is UNCHANGED as of this note —
|
|
200
|
+
* see the PRE-SHIP UPDATE immediately below for the wave that DOES change it.
|
|
201
|
+
*
|
|
202
|
+
* ── PRE-SHIP RED-TEAM FIXES (2026-09-01, two adversarial red-teams, both
|
|
203
|
+
* NO-SHIP; wave/pipe-w5fix) ──
|
|
204
|
+
* Two independent, unrelated fixes landed together in this pass (sequenced to
|
|
205
|
+
* avoid touching the same function):
|
|
206
|
+
* 1. SECURITY (CRITICAL): `readLegacyJournalCandidates` below no longer
|
|
207
|
+
* hardcodes `untrusted: false` — see that function's own updated doc
|
|
208
|
+
* comment for the full CRITICAL-1-class rescue-leak this closed.
|
|
209
|
+
* 2. CORRECTNESS (W5a contradiction, MEDIUM->fixed): the DOWN-RANK half of
|
|
210
|
+
* `applyContradictionStage` (the `CONTRADICTION_PENALTY` multiplier +
|
|
211
|
+
* its re-sort) is REMOVED — the stage is now ANNOTATE-ONLY. A second,
|
|
212
|
+
* independent (correctness, not security) red-team proved the down-rank
|
|
213
|
+
* was net-negative: the version-token grammar false-positives on
|
|
214
|
+
* common number-shaped prose that was never a genuine version mention
|
|
215
|
+
* (IP addresses, dot-formatted dates, dotted step/section numbers), and
|
|
216
|
+
* when it does, the multiplicative penalty INVERTS ranking — demoting a
|
|
217
|
+
* correct, unrelated fact below a weaker match purely because both
|
|
218
|
+
* happened to contain three dot-separated digit groups. Annotate-only
|
|
219
|
+
* makes a false-positive detection harmless (an extra `conflictsWith`
|
|
220
|
+
* flag an agent can ignore) instead of rank-corrupting. Paired with a
|
|
221
|
+
* HIGH-PRECISION tightening of the grammar itself (`./contradiction.ts`'s
|
|
222
|
+
* `grammarConflict` — now requires an explicit version marker adjacent
|
|
223
|
+
* to the digits and rejects a 4th trailing dot-group, i.e. an IPv4
|
|
224
|
+
* shape) and a RESOLVABLE-annotation fix (`JournalSearchResult.results`
|
|
225
|
+
* now carries an additive `id`, mirroring `SmartRecallResultItem`, so
|
|
226
|
+
* `supersededBy`/`conflictsWith` are cross-referenceable there too, not
|
|
227
|
+
* just for `smart_recall`). See `./contradiction.ts`'s own header for
|
|
228
|
+
* the grammar fix's full false-positive-by-false-positive proof.
|
|
229
|
+
*/
|
|
230
|
+
import { readTierCandidates, filterTrusted } from "./candidates.js";
|
|
231
|
+
import { applyScope } from "./scope.js";
|
|
232
|
+
import { detectContradictions } from "./contradiction.js";
|
|
233
|
+
import { listRooms, recordAccess, ensurePalaceInitialized } from "../palace/rooms.js";
|
|
234
|
+
import { stem, expandQuery } from "../helpers/normalize.js";
|
|
235
|
+
import { tokenizeWords } from "../helpers/tokenize.js";
|
|
236
|
+
import { recallInsights } from "../palace/insights-index.js";
|
|
237
|
+
import { parseSinceDate } from "../tools-logic/journal-search.js";
|
|
238
|
+
import { CONFIDENCE_FLOOR } from "../tools-logic/confidence.js";
|
|
239
|
+
import { scrubForCloud, fenceMemory } from "../storage/content-guard.js";
|
|
240
|
+
import { isRescueSourcedContent, extractFrontmatterSource } from "../helpers/journal-filter.js";
|
|
241
|
+
import { getLegacyRoot } from "../types.js";
|
|
242
|
+
import * as fs from "node:fs";
|
|
243
|
+
import * as path from "node:path";
|
|
244
|
+
// ---------------------------------------------------------------------------
|
|
245
|
+
// Math helpers — ported from smart-recall.ts, unchanged formulas (see
|
|
246
|
+
// CHALLENGE (a) above for why these stay per-tier, not unified).
|
|
247
|
+
// ---------------------------------------------------------------------------
|
|
248
|
+
const RRF_K = 60;
|
|
249
|
+
const EBBINGHAUS_S = {
|
|
250
|
+
journal: 2,
|
|
251
|
+
palace: 9999,
|
|
252
|
+
};
|
|
253
|
+
function daysSince(dateStr) {
|
|
254
|
+
const d = new Date(dateStr);
|
|
255
|
+
if (isNaN(d.getTime()))
|
|
256
|
+
return 365;
|
|
257
|
+
return Math.max(0, (Date.now() - d.getTime()) / (1000 * 60 * 60 * 24));
|
|
258
|
+
}
|
|
259
|
+
function ebbinghaus(days, S) {
|
|
260
|
+
return Math.exp(-days / S);
|
|
261
|
+
}
|
|
262
|
+
/** Keyword overlap ratio between query and text — ported verbatim from
|
|
263
|
+
* smart-recall.ts's `keywordExactness` (CJK-aware via the shared tokenizer). */
|
|
264
|
+
function keywordExactness(query, text) {
|
|
265
|
+
const rawWords = tokenizeWords(query);
|
|
266
|
+
if (rawWords.length === 0)
|
|
267
|
+
return 0;
|
|
268
|
+
const expandedQuery = expandQuery(rawWords);
|
|
269
|
+
const textWords = tokenizeWords(text).map((w) => stem(w));
|
|
270
|
+
const textSet = new Set(textWords);
|
|
271
|
+
const textLower = text.toLowerCase();
|
|
272
|
+
const matches = expandedQuery.filter((w) => textSet.has(w) || textLower.includes(w));
|
|
273
|
+
return Math.min(1.0, matches.length / rawWords.length);
|
|
274
|
+
}
|
|
275
|
+
/**
|
|
276
|
+
* Simple stable hash for result IDs — ported verbatim from smart-recall.ts.
|
|
277
|
+
*
|
|
278
|
+
* Exported (STEP 4c, pre-ship red-team fix, 2026-09-01) so
|
|
279
|
+
* `tools-logic/journal-search.ts`'s `journalSearch()` can mint the SAME
|
|
280
|
+
* kind of stable id for its own `include_palace` branch's locally-scanned
|
|
281
|
+
* rows (which never become `QueryMemoryItem`s — that branch bypasses
|
|
282
|
+
* queryMemory() entirely, see `journalSearch`'s own comment) — giving
|
|
283
|
+
* `JournalSearchResult.results` a real `id` on EVERY row, not just the
|
|
284
|
+
* primary journal-tier ones, without forking a second hash implementation.
|
|
285
|
+
* journal-search.ts already imports `queryMemory`/`parseSinceDate` in a
|
|
286
|
+
* pre-existing circular pair with this module (this file imports
|
|
287
|
+
* `parseSinceDate` FROM journal-search.ts); adding this export extends that
|
|
288
|
+
* SAME already-tolerated cycle, not a new one.
|
|
289
|
+
*/
|
|
290
|
+
export function stableId(source, title) {
|
|
291
|
+
let hash = 0;
|
|
292
|
+
const str = `${source}:${title}`;
|
|
293
|
+
for (let i = 0; i < str.length; i++) {
|
|
294
|
+
const char = str.charCodeAt(i);
|
|
295
|
+
hash = (hash << 5) - hash + char;
|
|
296
|
+
hash |= 0;
|
|
297
|
+
}
|
|
298
|
+
return Math.abs(hash).toString(36).slice(0, 8);
|
|
299
|
+
}
|
|
300
|
+
function normalizeExcerpt(excerpt) {
|
|
301
|
+
return excerpt.toLowerCase().replace(/\s+/g, " ").trim();
|
|
302
|
+
}
|
|
303
|
+
function fusionIdentity(item) {
|
|
304
|
+
return normalizeExcerpt(item.fusionKey ?? item.excerpt);
|
|
305
|
+
}
|
|
306
|
+
/** Find first keyword match position in a line, for excerpt anchoring
|
|
307
|
+
* (ported from journal-search.ts's `firstMatchIndex`). */
|
|
308
|
+
function firstMatchIndex(line, keywords) {
|
|
309
|
+
const lineLower = line.toLowerCase();
|
|
310
|
+
let first = line.length;
|
|
311
|
+
for (const kw of keywords) {
|
|
312
|
+
const idx = lineLower.indexOf(kw);
|
|
313
|
+
if (idx !== -1 && idx < first)
|
|
314
|
+
first = idx;
|
|
315
|
+
}
|
|
316
|
+
return first;
|
|
317
|
+
}
|
|
318
|
+
function lineMatchesQuery(line, keywords) {
|
|
319
|
+
if (keywords.length === 0)
|
|
320
|
+
return false;
|
|
321
|
+
const lineLower = line.toLowerCase();
|
|
322
|
+
return keywords.some((kw) => lineLower.includes(kw));
|
|
323
|
+
}
|
|
324
|
+
// ---------------------------------------------------------------------------
|
|
325
|
+
// TRUST-FILTER stage — mandatory, applied to every candidate before scoring.
|
|
326
|
+
// Not individually callable/skippable: every tier-scoring function below
|
|
327
|
+
// filters through `filterTrusted` as its FIRST step, before any
|
|
328
|
+
// tokenize/score work.
|
|
329
|
+
//
|
|
330
|
+
// Independent review fix (W2, 2026-08-30): this stage now delegates to
|
|
331
|
+
// `filterTrusted` (retrieval/candidates.ts) — the SAME canonical
|
|
332
|
+
// implementation `readTierCandidates`'s own safe-by-default filtering uses —
|
|
333
|
+
// rather than a local, forked `!c.untrusted` predicate. Every `readTierCandidates`
|
|
334
|
+
// call site below passes `includeUntrusted: true` so this stage still sees
|
|
335
|
+
// (and still filters) the full candidate set — the reader's own new
|
|
336
|
+
// safe-by-default drop is bypassed here on purpose, because this pipeline
|
|
337
|
+
// stage is the mandatory, non-bypassable place that decision belongs;
|
|
338
|
+
// keeping BOTH the reader's own default-safe boundary AND this stage intact
|
|
339
|
+
// preserves queryMemory()'s exact pre-existing ranking, byte-identical, for
|
|
340
|
+
// THIS reconciliation (the `includeUntrusted` relocation — STEP 1; see this
|
|
341
|
+
// wave's report for the smartRecall() equivalence proof) while closing the
|
|
342
|
+
// direct-caller escape hatch a naive `readTierCandidates()` caller previously
|
|
343
|
+
// had. (The separate archive/legacy date-desc sort — MEDIUM-1 — is NOT
|
|
344
|
+
// byte-identical under tied per-tier scores; see CHALLENGE (c)-2 above.)
|
|
345
|
+
// ---------------------------------------------------------------------------
|
|
346
|
+
// ---------------------------------------------------------------------------
|
|
347
|
+
// SCOPE stage — REAL as of Wave 3b (2026-08-30, reports/2026-08-30-pipe-w3b-
|
|
348
|
+
// migrate-report.md). `applyScope` itself now lives in ./scope.ts (see that
|
|
349
|
+
// file's header for why — recall-insight.ts shares it, and it must not
|
|
350
|
+
// create an import cycle with this file). What lives HERE is the tier-level
|
|
351
|
+
// decision of WHICH tiers get run through it at all.
|
|
352
|
+
//
|
|
353
|
+
// SCOPE_ATTRIBUTED_TIERS: only these tiers' items carry a genuine, populated
|
|
354
|
+
// `projects` attribution — every OTHER tier is short-circuited (scope never
|
|
355
|
+
// applied), NOT because `applyScope` would silently no-op for them (it would
|
|
356
|
+
// NOT — an absent `projects` field reads as "unattributed" and would be
|
|
357
|
+
// wrongly EXCLUDED under scope:"project"), but because journal/palace-room
|
|
358
|
+
// candidates are inherently per-slug (`readTierCandidates(tier, project,
|
|
359
|
+
// ...)` only ever reads `project`'s own tree) and therefore have nothing
|
|
360
|
+
// cross-project to filter — see scope.ts's own doc comment for the full
|
|
361
|
+
// reasoning. A future genuinely cross-project journal-like tier (the W3
|
|
362
|
+
// plan's recency-ledger / working-memory-live / session-card rows) joins
|
|
363
|
+
// this set WHEN its scorer is built to populate `projects`/an equivalent
|
|
364
|
+
// attribution field on its QueryMemoryItems — not before.
|
|
365
|
+
const SCOPE_ATTRIBUTED_TIERS = new Set(["insight"]);
|
|
366
|
+
// ---------------------------------------------------------------------------
|
|
367
|
+
// CONTRADICTION stage (Wave 5a, 2026-08-31) — runs per-tier, AFTER
|
|
368
|
+
// TOKENIZE+SCORE and (for tiers that have one) after SCOPE, BEFORE RANK/FUSE.
|
|
369
|
+
// `detectContradictions` itself (`./contradiction.ts`) is a pure, tier-
|
|
370
|
+
// agnostic comparator; what lives HERE, mirroring SCOPE_ATTRIBUTED_TIERS
|
|
371
|
+
// immediately above, is the tier-level decision of which tiers run through
|
|
372
|
+
// it and how each tier's `order` tie-break signal is supplied.
|
|
373
|
+
//
|
|
374
|
+
// CONTRADICTION_TIERS: journal and palace only — matches this wave's
|
|
375
|
+
// ASSERT_INVARIANTS exactly ("DO NOT apply the stage to the insight tier
|
|
376
|
+
// this sub-wave (no date signal)"). `insight` items have no per-item
|
|
377
|
+
// authored-date at all (their score is confirmation-count/relevance driven,
|
|
378
|
+
// not time-based — see scoreInsightTier's own doc comment above), so there
|
|
379
|
+
// is no DIRECTION signal `detectContradictions` could ever resolve for that
|
|
380
|
+
// tier; short-circuiting it here (a new tier joins this Set, never a new
|
|
381
|
+
// branch inside `applyContradictionStage`) keeps that decision explicit and
|
|
382
|
+
// enumerable rather than an implicit side effect of insight items lacking a
|
|
383
|
+
// `date` field.
|
|
384
|
+
const CONTRADICTION_TIERS = new Set(["journal", "palace"]);
|
|
385
|
+
/**
|
|
386
|
+
* ANNOTATE-ONLY (pre-ship red-team fix, STEP 4a, 2026-09-01, wave/pipe-w5fix
|
|
387
|
+
* — correctness red-team, reports/2026-09-01-pipe-w5fix-report.md).
|
|
388
|
+
*
|
|
389
|
+
* The ORIGINAL W5a design multiplicatively down-ranked a confidently-
|
|
390
|
+
* resolved-stale candidate (a flat `CONTRADICTION_PENALTY = 0.5`, re-sorted
|
|
391
|
+
* immediately after so the halving actually moved the item's ARRAY
|
|
392
|
+
* POSITION — `applyRRF` reads position, not raw `.score`, for its
|
|
393
|
+
* `1/(RRF_K+rank)` contribution). A second, independent (correctness, not
|
|
394
|
+
* security) red-team proved this is NET-NEGATIVE, not merely imperfect:
|
|
395
|
+
* the version-token grammar (see `./contradiction.ts`'s `grammarConflict`)
|
|
396
|
+
* false-positives on common number-shaped prose that is never a genuine
|
|
397
|
+
* version mention at all — IP addresses ("at 10.0.0.1" / "at 10.0.1.5"),
|
|
398
|
+
* dot-formatted dates ("08.15.2026" vs "08.16.2026"), dotted step/section
|
|
399
|
+
* numbers ("step 1.2.3" vs "step 5.6.7"). When a false positive like this
|
|
400
|
+
* fires, the down-rank+re-sort mechanism doesn't just add noise — it
|
|
401
|
+
* INVERTS ranking, demoting a correct, unrelated fact below a weaker match
|
|
402
|
+
* purely because both happen to contain three dot-separated digit groups.
|
|
403
|
+
* A missed real contradiction degrades gracefully (decay/recency still
|
|
404
|
+
* favor the newer entry over time); a false-positive-driven rank inversion
|
|
405
|
+
* actively returns the WRONG answer with high confidence.
|
|
406
|
+
*
|
|
407
|
+
* FIX: `applyContradictionStage` below still detects and ANNOTATES
|
|
408
|
+
* (`supersededBy`/`conflictsWith`) exactly as before — the detection +
|
|
409
|
+
* annotation mechanism was never the problem — but it no longer touches
|
|
410
|
+
* `score` or reorders `items` at all. A false-positive grammar match is now
|
|
411
|
+
* harmless (an extra, ignorable annotation field) instead of rank-
|
|
412
|
+
* corrupting. This is paired with a HIGH-PRECISION tightening of the
|
|
413
|
+
* grammar itself (`./contradiction.ts`'s own header) so the annotation
|
|
414
|
+
* itself also fires far less often on non-version prose — belt AND
|
|
415
|
+
* suspenders, not a substitute for each other: even a genuine version
|
|
416
|
+
* conflict now only annotates, never reorders.
|
|
417
|
+
*
|
|
418
|
+
* Runs `detectContradictions` over one tier's already-scored item list and
|
|
419
|
+
* returns a NEW array (same length, same membership, SAME ORDER — annotation
|
|
420
|
+
* never reorders) with conflicting items annotated (`conflictsWith`) and
|
|
421
|
+
* confidently-resolved-stale items additionally annotated (`supersededBy`).
|
|
422
|
+
*
|
|
423
|
+
* No-ops (returns `items` UNCHANGED, same reference) for: any tier not in
|
|
424
|
+
* `CONTRADICTION_TIERS`, and any tier with fewer than 2 items (nothing to
|
|
425
|
+
* compare — also the O(n²) stage's own trivial-input guard, so a 0- or
|
|
426
|
+
* 1-candidate tier can never pay a quadratic cost or crash on an empty
|
|
427
|
+
* pairwise loop).
|
|
428
|
+
*/
|
|
429
|
+
function applyContradictionStage(tier, items) {
|
|
430
|
+
if (!CONTRADICTION_TIERS.has(tier) || items.length < 2)
|
|
431
|
+
return items;
|
|
432
|
+
const view = items.map((it) => ({
|
|
433
|
+
text: `${it.title} ${it.excerpt}`,
|
|
434
|
+
date: it.date,
|
|
435
|
+
// Journal deliberately does NOT supply `order` — this wave's brief:
|
|
436
|
+
// "journal → older authored date" only; a same-date journal tie must
|
|
437
|
+
// fall through to the ambiguous (annotate-both, penalize-neither)
|
|
438
|
+
// branch inside detectContradictions, not guess off line position.
|
|
439
|
+
// Palace DOES supply it (the room-file append-order proxy — see
|
|
440
|
+
// QueryMemoryItem.line's doc comment) as its same-day tie-break.
|
|
441
|
+
order: tier === "palace" ? it.line : undefined,
|
|
442
|
+
}));
|
|
443
|
+
const { supersededBy, conflictsWith } = detectContradictions(view);
|
|
444
|
+
if (supersededBy.size === 0 && conflictsWith.size === 0)
|
|
445
|
+
return items;
|
|
446
|
+
// ANNOTATE-ONLY: map to a new array (so callers still get fresh objects
|
|
447
|
+
// for the items that changed) but never touch `.score` and never reorder
|
|
448
|
+
// — `items`' own array order (already RANK/FUSE-input order from the
|
|
449
|
+
// TOKENIZE+SCORE stage immediately before this one) is preserved exactly.
|
|
450
|
+
return items.map((it, idx) => {
|
|
451
|
+
const conflicts = conflictsWith.get(idx);
|
|
452
|
+
const staleOf = supersededBy.get(idx);
|
|
453
|
+
if (!conflicts && staleOf === undefined)
|
|
454
|
+
return it;
|
|
455
|
+
const next = { ...it };
|
|
456
|
+
if (conflicts && conflicts.length > 0) {
|
|
457
|
+
next.conflictsWith = conflicts.map((i) => items[i].id);
|
|
458
|
+
}
|
|
459
|
+
if (staleOf !== undefined) {
|
|
460
|
+
next.supersededBy = items[staleOf].id;
|
|
461
|
+
}
|
|
462
|
+
return next;
|
|
463
|
+
});
|
|
464
|
+
}
|
|
465
|
+
// ---------------------------------------------------------------------------
|
|
466
|
+
// CHALLENGE (c)-3: legacy journal directory. Wave 1's readTierCandidates does
|
|
467
|
+
// not cover `~/.claude/projects/<entry>/memory/journal/` (storage/paths.ts's
|
|
468
|
+
// journalDirs() legacy-fallback branch) — journalSearch()/smart_recall today
|
|
469
|
+
// DO scan it. Mirrors journalDirs()'s own traversal exactly, read-only, never
|
|
470
|
+
// throws.
|
|
471
|
+
//
|
|
472
|
+
// SECURITY FIX (pre-ship red-team Break #1, 2026-09-01): this reader used to
|
|
473
|
+
// HARDCODE `untrusted: false` on every candidate, reasoning (now proven
|
|
474
|
+
// WRONG) that "this content predates the working-memory-rescue mechanism's
|
|
475
|
+
// existence entirely, so it structurally cannot carry a `source:
|
|
476
|
+
// working-memory-rescue` tag." That was an ASSERTION about the CONTENT
|
|
477
|
+
// living at this path historically, not an ENFORCED property of the path
|
|
478
|
+
// itself — `getLegacyRoot()` (`~/.claude/projects`) is scanned unconditionally
|
|
479
|
+
// on every install, and nothing stops a rescue-tagged card from being placed
|
|
480
|
+
// there today (a stale AgentRecall install, a manually-copied file, or any
|
|
481
|
+
// future writer that reuses this legacy convention). Because `untrusted` was
|
|
482
|
+
// hardcoded rather than derived, `filterTrusted(readLegacyJournalCandidates(
|
|
483
|
+
// project))` (this file's TRUST-FILTER stage, see `scoreJournalTier` below)
|
|
484
|
+
// was a no-op for this entire source — a planted rescue file here surfaced
|
|
485
|
+
// UNLABELED through the default-on `recall`/`smart_recall` MCP tools via
|
|
486
|
+
// this tier. FIXED: `untrusted`/`sourceTag` are now DERIVED per-candidate
|
|
487
|
+
// from the file's own frontmatter, via the SAME `isRescueSourcedContent`/
|
|
488
|
+
// `extractFrontmatterSource` choke every sibling reader in
|
|
489
|
+
// `retrieval/candidates.ts` already uses (`readJournalCandidates`,
|
|
490
|
+
// `readPalaceRoomCandidates`) — not a second, forked trust decision. See
|
|
491
|
+
// `packages/core/test/query-memory-pipeline.test.mjs`'s legacy-journal
|
|
492
|
+
// destination-proof for the planted-rescue-content regression test, and
|
|
493
|
+
// `packages/core/test/identity-trust-completeness.test.mjs`'s new
|
|
494
|
+
// hardcoded-untrusted-false completeness check (STEP 3, same date) for the
|
|
495
|
+
// harness-level class fix that would have caught this before ship.
|
|
496
|
+
//
|
|
497
|
+
// Independent review fix (MEDIUM-1, W2, 2026-08-30): returns date-descending
|
|
498
|
+
// (sorted below, before returning) — same sort-before-truncate closure this
|
|
499
|
+
// wave applies to candidates.ts's rollup-archive half. Raw traversal order
|
|
500
|
+
// here is nested-readdirSync (project-dir enumeration outer, file-within-dir
|
|
501
|
+
// enumeration inner) — filesystem-dependent, not date order — so a caller
|
|
502
|
+
// truncating this list (scoreJournalTier's perTierLimit break, after
|
|
503
|
+
// concatenating this onto the already-sorted live+rollup-archive half) must
|
|
504
|
+
// not see an arbitrary enumeration-order subset when total legacy entries
|
|
505
|
+
// exceed the limit.
|
|
506
|
+
// ---------------------------------------------------------------------------
|
|
507
|
+
function readLegacyJournalCandidates(project) {
|
|
508
|
+
const out = [];
|
|
509
|
+
let legacyRoot;
|
|
510
|
+
try {
|
|
511
|
+
legacyRoot = getLegacyRoot();
|
|
512
|
+
}
|
|
513
|
+
catch {
|
|
514
|
+
return out;
|
|
515
|
+
}
|
|
516
|
+
if (!fs.existsSync(legacyRoot))
|
|
517
|
+
return out;
|
|
518
|
+
let entries;
|
|
519
|
+
try {
|
|
520
|
+
entries = fs.readdirSync(legacyRoot);
|
|
521
|
+
}
|
|
522
|
+
catch {
|
|
523
|
+
return out;
|
|
524
|
+
}
|
|
525
|
+
for (const entry of entries) {
|
|
526
|
+
if (!entry.includes(project))
|
|
527
|
+
continue;
|
|
528
|
+
const legacyJournal = path.join(legacyRoot, entry, "memory", "journal");
|
|
529
|
+
if (!fs.existsSync(legacyJournal))
|
|
530
|
+
continue;
|
|
531
|
+
let files;
|
|
532
|
+
try {
|
|
533
|
+
files = fs.readdirSync(legacyJournal).filter((f) => f.endsWith(".md"));
|
|
534
|
+
}
|
|
535
|
+
catch {
|
|
536
|
+
continue;
|
|
537
|
+
}
|
|
538
|
+
for (const file of files) {
|
|
539
|
+
const sourcePath = path.join(legacyJournal, file);
|
|
540
|
+
let content;
|
|
541
|
+
try {
|
|
542
|
+
content = fs.readFileSync(sourcePath, "utf-8");
|
|
543
|
+
}
|
|
544
|
+
catch {
|
|
545
|
+
continue;
|
|
546
|
+
}
|
|
547
|
+
const dateMatch = file.match(/^(\d{4}-\d{2}-\d{2})/);
|
|
548
|
+
out.push({
|
|
549
|
+
content,
|
|
550
|
+
tier: "journal",
|
|
551
|
+
project,
|
|
552
|
+
date: dateMatch ? dateMatch[1] : "",
|
|
553
|
+
sourcePath,
|
|
554
|
+
file,
|
|
555
|
+
sourceKind: "journal-live",
|
|
556
|
+
untrusted: isRescueSourcedContent(content),
|
|
557
|
+
sourceTag: extractFrontmatterSource(content),
|
|
558
|
+
});
|
|
559
|
+
}
|
|
560
|
+
}
|
|
561
|
+
out.sort((a, b) => b.date.localeCompare(a.date));
|
|
562
|
+
return out;
|
|
563
|
+
}
|
|
564
|
+
// ---------------------------------------------------------------------------
|
|
565
|
+
// TOKENIZE+SCORE stage — one strategy per tier.
|
|
566
|
+
// ---------------------------------------------------------------------------
|
|
567
|
+
/** Journal tier: ported from journal-search.ts's per-line matching (section
|
|
568
|
+
* tracking, excerpt window) + smart-recall.ts's Ebbinghaus×exactness scoring —
|
|
569
|
+
* now operating on already-fetched MemoryCandidate content instead of a
|
|
570
|
+
* fresh fs read. */
|
|
571
|
+
function scoreJournalTier(project, query, opts) {
|
|
572
|
+
const candidates = filterTrusted(readTierCandidates("journal", project, {
|
|
573
|
+
includeRollupArchive: opts.includeRollupArchive ?? true,
|
|
574
|
+
includeUntrusted: true,
|
|
575
|
+
})).concat(filterTrusted(readLegacyJournalCandidates(project)));
|
|
576
|
+
const keywords = tokenizeWords(query);
|
|
577
|
+
if (keywords.length === 0)
|
|
578
|
+
return [];
|
|
579
|
+
const sinceCutoff = opts.since ? parseSinceDate(opts.since) : null;
|
|
580
|
+
const limit = opts.perTierLimit ?? 25;
|
|
581
|
+
const hits = [];
|
|
582
|
+
for (const candidate of candidates) {
|
|
583
|
+
if (hits.length >= limit)
|
|
584
|
+
break;
|
|
585
|
+
if (sinceCutoff && candidate.date) {
|
|
586
|
+
const fileDate = new Date(candidate.date);
|
|
587
|
+
if (!isNaN(fileDate.getTime()) && fileDate < sinceCutoff)
|
|
588
|
+
continue;
|
|
589
|
+
}
|
|
590
|
+
const lines = candidate.content.split("\n");
|
|
591
|
+
let currentSection = "top";
|
|
592
|
+
for (let i = 0; i < lines.length; i++) {
|
|
593
|
+
if (hits.length >= limit)
|
|
594
|
+
break;
|
|
595
|
+
const line = lines[i];
|
|
596
|
+
if (line.startsWith("## ")) {
|
|
597
|
+
currentSection = line.slice(3).trim().toLowerCase().replace(/\s+/g, "_");
|
|
598
|
+
}
|
|
599
|
+
if (!lineMatchesQuery(line, keywords))
|
|
600
|
+
continue;
|
|
601
|
+
const date = candidate.date || candidate.file;
|
|
602
|
+
const matchIdx = firstMatchIndex(line, keywords);
|
|
603
|
+
const start = Math.max(0, matchIdx - 100);
|
|
604
|
+
const end = Math.min(line.length, matchIdx + 150);
|
|
605
|
+
let excerpt = line.slice(start, end).trim();
|
|
606
|
+
if (start > 0)
|
|
607
|
+
excerpt = "..." + excerpt;
|
|
608
|
+
if (end < line.length)
|
|
609
|
+
excerpt = excerpt + "...";
|
|
610
|
+
hits.push({ title: `${date} / ${currentSection}`, excerpt, date, line: i + 1 });
|
|
611
|
+
}
|
|
612
|
+
}
|
|
613
|
+
const items = hits.map((h) => {
|
|
614
|
+
// Independent review fix (W3b, 2026-08-30): `id` must be unique PER HIT,
|
|
615
|
+
// not per (date,section) — `applyRRF` (below) groups same-tier items by
|
|
616
|
+
// `item.id` in its own accumulation Map, and on a collision it keeps
|
|
617
|
+
// ONLY the first-encountered item, silently discarding every subsequent
|
|
618
|
+
// hit's own excerpt/line (its score contribution is merged in, but the
|
|
619
|
+
// item itself vanishes). `h.title` alone (`"${date} / ${section}"`) is
|
|
620
|
+
// NOT unique per hit — TWO DIFFERENT matching lines in the SAME file's
|
|
621
|
+
// SAME section (a very common real case: a verbose journal entry
|
|
622
|
+
// mentioning a term twice) previously collapsed into ONE result, a
|
|
623
|
+
// silent, untested data-loss regression newly exposed on journalSearch's
|
|
624
|
+
// DEFAULT path by this wave's migration (journalSearch's PRE-migration
|
|
625
|
+
// implementation had no id/RRF grouping at all — every line match was
|
|
626
|
+
// pushed independently). Incorporating `line` + `excerpt` makes the id
|
|
627
|
+
// genuinely unique per hit; `fusionIdentity` (the CROSS-tier/cross-hit
|
|
628
|
+
// dedup stage, unaffected by this change) still correctly collapses
|
|
629
|
+
// hits whose excerpt content is a genuine duplicate.
|
|
630
|
+
const id = stableId("journal", `${h.title}::${h.line}::${h.excerpt}`);
|
|
631
|
+
const days = daysSince(h.date);
|
|
632
|
+
const recency = ebbinghaus(days, EBBINGHAUS_S.journal);
|
|
633
|
+
const exactness = keywordExactness(query, h.excerpt);
|
|
634
|
+
const internalScore = recency * 0.5 + exactness * 0.5;
|
|
635
|
+
return { id, source: "journal", title: h.title, excerpt: h.excerpt, score: internalScore, date: h.date, line: h.line };
|
|
636
|
+
});
|
|
637
|
+
items.sort((a, b) => b.score - a.score);
|
|
638
|
+
return items;
|
|
639
|
+
}
|
|
640
|
+
/** Palace tier: ported from palace-search.ts's tagBonus + per-line matching +
|
|
641
|
+
* IDF re-scoring, operating on already-fetched room-file MemoryCandidates
|
|
642
|
+
* instead of a fresh fs read, then blended with salience exactly as
|
|
643
|
+
* smart-recall.ts's original `internalScore = keyScore*0.65 + salience*0.35`.
|
|
644
|
+
* TRUST-FILTERED before any of this runs — CLOSES the known gap
|
|
645
|
+
* documented in this file's header, CHALLENGE (b). */
|
|
646
|
+
function scorePalaceTier(project, query, opts) {
|
|
647
|
+
// Matches palace-search.ts's own first executable step — scaffolds the
|
|
648
|
+
// palace dir + default rooms (writing each default room's `_room.json`) on
|
|
649
|
+
// first touch. Without this, `listRooms()` (both here and inside
|
|
650
|
+
// `readTierCandidates("palace-room", ...)`) silently sees zero rooms for a
|
|
651
|
+
// project whose palace/ has never been initialized, even when `rooms/*/`.md
|
|
652
|
+
// content already exists on disk — `listRooms()` requires a `_room.json`
|
|
653
|
+
// per room dir, which only `ensurePalaceInitialized`/`createRoom` write.
|
|
654
|
+
ensurePalaceInitialized(project);
|
|
655
|
+
const candidates = filterTrusted(readTierCandidates("palace-room", project, { room: opts.room, includeUntrusted: true }));
|
|
656
|
+
const rooms = listRooms(project);
|
|
657
|
+
const salienceByRoom = new Map(rooms.map((r) => [r.slug, r.salience]));
|
|
658
|
+
const rawQueryWords = tokenizeWords(query);
|
|
659
|
+
const projectVariants = new Set();
|
|
660
|
+
{
|
|
661
|
+
const base = project.toLowerCase();
|
|
662
|
+
projectVariants.add(base);
|
|
663
|
+
projectVariants.add(base.replace(/[-_\s]+/g, ""));
|
|
664
|
+
projectVariants.add(stem(base));
|
|
665
|
+
for (const part of base.split(/[-_\s]+/)) {
|
|
666
|
+
if (part.length > 2) {
|
|
667
|
+
projectVariants.add(part);
|
|
668
|
+
projectVariants.add(stem(part));
|
|
669
|
+
}
|
|
670
|
+
}
|
|
671
|
+
}
|
|
672
|
+
const filteredRawQueryWords = rawQueryWords.filter((w) => !projectVariants.has(w) && !projectVariants.has(stem(w)));
|
|
673
|
+
const queryWords = expandQuery(filteredRawQueryWords);
|
|
674
|
+
if (queryWords.length === 0)
|
|
675
|
+
return [];
|
|
676
|
+
const hits = [];
|
|
677
|
+
const roomsWithHits = new Set();
|
|
678
|
+
function parseFrontmatterTags(content) {
|
|
679
|
+
const match = content.match(/^---\s*\n([\s\S]*?)\n---/);
|
|
680
|
+
if (!match)
|
|
681
|
+
return [];
|
|
682
|
+
const tagsLine = match[1].split("\n").find((l) => l.trim().startsWith("tags:"));
|
|
683
|
+
if (!tagsLine)
|
|
684
|
+
return [];
|
|
685
|
+
const arrayMatch = tagsLine.match(/tags:\s*\[([^\]]*)\]/);
|
|
686
|
+
if (!arrayMatch)
|
|
687
|
+
return [];
|
|
688
|
+
return arrayMatch[1].split(",").map((t) => t.trim().replace(/^["']|["']$/g, "")).filter((t) => t.length > 0);
|
|
689
|
+
}
|
|
690
|
+
for (const candidate of candidates) {
|
|
691
|
+
const room = candidate.room ?? "";
|
|
692
|
+
const file = candidate.file.replace(".md", "");
|
|
693
|
+
const lines = candidate.content.split("\n");
|
|
694
|
+
const fileTags = parseFrontmatterTags(candidate.content);
|
|
695
|
+
const tagBonus = queryWords.some((w) => fileTags.some((t) => t.toLowerCase().includes(w) || w.includes(t.toLowerCase())))
|
|
696
|
+
? 0.10
|
|
697
|
+
: 0;
|
|
698
|
+
for (let i = 0; i < lines.length; i++) {
|
|
699
|
+
const lineLower = lines[i].toLowerCase();
|
|
700
|
+
if (/^#{1,6}\s/.test(lines[i]))
|
|
701
|
+
continue; // skip structural headings
|
|
702
|
+
const lineWords = tokenizeWords(lineLower).map((w) => stem(w));
|
|
703
|
+
const lineWordSet = new Set(lineWords);
|
|
704
|
+
const matchedWords = queryWords.filter((w) => lineWordSet.has(w) || lineLower.includes(w));
|
|
705
|
+
if (matchedWords.length === 0)
|
|
706
|
+
continue;
|
|
707
|
+
const rawKeywordScore = matchedWords.length / queryWords.length;
|
|
708
|
+
const keywordScore = Math.min(1.0, rawKeywordScore + tagBonus);
|
|
709
|
+
const firstKw = matchedWords[0];
|
|
710
|
+
const matchIdx = lineLower.indexOf(firstKw);
|
|
711
|
+
const start = Math.max(0, matchIdx - 40);
|
|
712
|
+
const end = Math.min(lines[i].length, matchIdx + firstKw.length + 80);
|
|
713
|
+
let excerpt = lines[i].slice(start, end).trim();
|
|
714
|
+
if (start > 0)
|
|
715
|
+
excerpt = "..." + excerpt;
|
|
716
|
+
if (end < lines[i].length)
|
|
717
|
+
excerpt = excerpt + "...";
|
|
718
|
+
hits.push({ room, file, excerpt, line: i + 1, keywordScore });
|
|
719
|
+
roomsWithHits.add(room);
|
|
720
|
+
}
|
|
721
|
+
}
|
|
722
|
+
// IDF re-scoring — ported verbatim from palace-search.ts's Fix RC3.
|
|
723
|
+
if (hits.length > 0) {
|
|
724
|
+
const totalDocs = new Set(hits.map((h) => `${h.room}/${h.file}`)).size;
|
|
725
|
+
const docFreq = new Map();
|
|
726
|
+
for (const h of hits) {
|
|
727
|
+
const docId = `${h.room}/${h.file}`;
|
|
728
|
+
const combined = (h.excerpt + " " + h.room + " " + h.file).toLowerCase();
|
|
729
|
+
for (const w of queryWords) {
|
|
730
|
+
if (combined.includes(w)) {
|
|
731
|
+
if (!docFreq.has(w))
|
|
732
|
+
docFreq.set(w, new Set());
|
|
733
|
+
docFreq.get(w).add(docId);
|
|
734
|
+
}
|
|
735
|
+
}
|
|
736
|
+
}
|
|
737
|
+
const idfRaw = new Map();
|
|
738
|
+
for (const w of queryWords) {
|
|
739
|
+
const df = docFreq.get(w)?.size ?? 0;
|
|
740
|
+
idfRaw.set(w, Math.log(1 + totalDocs / (df + 1)));
|
|
741
|
+
}
|
|
742
|
+
const maxIdf = Math.max(1, ...Array.from(idfRaw.values()));
|
|
743
|
+
const idf = new Map();
|
|
744
|
+
for (const [w, v] of idfRaw)
|
|
745
|
+
idf.set(w, v / maxIdf);
|
|
746
|
+
for (const h of hits) {
|
|
747
|
+
const combined = (h.excerpt + " " + h.room + " " + h.file).toLowerCase();
|
|
748
|
+
const matchedIdfWords = queryWords.filter((w) => combined.includes(w));
|
|
749
|
+
if (matchedIdfWords.length === 0)
|
|
750
|
+
continue;
|
|
751
|
+
const idfWeightedScore = matchedIdfWords.reduce((sum, w) => sum + (idf.get(w) ?? 0), 0) / queryWords.length;
|
|
752
|
+
h.keywordScore = Math.min(1.0, idfWeightedScore * 0.70 + h.keywordScore * 0.30);
|
|
753
|
+
}
|
|
754
|
+
}
|
|
755
|
+
// recordAccess side effect — matches palaceSearch()'s original behavior
|
|
756
|
+
// (once per room that contributed >=1 hit).
|
|
757
|
+
for (const room of roomsWithHits) {
|
|
758
|
+
try {
|
|
759
|
+
recordAccess(project, room);
|
|
760
|
+
}
|
|
761
|
+
catch { /* best-effort, never block scoring on a bookkeeping write */ }
|
|
762
|
+
}
|
|
763
|
+
const limit = opts.perTierLimit ?? 40; // matches smart-recall's `limit*2` request to palaceSearch
|
|
764
|
+
// Score ALL hits first, sort by the SAME internal score used for RRF
|
|
765
|
+
// ranking, THEN truncate — a single consistent truncation criterion.
|
|
766
|
+
// (The original palaceSearch() truncates internally by a DIFFERENT
|
|
767
|
+
// criterion — keyword_score * raw roomMeta.salience, no floor — before
|
|
768
|
+
// smart-recall.ts re-scores by keyScore*0.65+salienceFloor*0.35 for its own
|
|
769
|
+
// ranking; this collapses that two-stage, two-criteria truncation into one,
|
|
770
|
+
// a characterized simplification, not a regression — see this file's header.)
|
|
771
|
+
const items = hits.map((h) => {
|
|
772
|
+
const title = `${h.room}/${h.file}`;
|
|
773
|
+
// NOTE (W3b, 2026-08-30 — deliberately NOT fixed this wave, see the
|
|
774
|
+
// report's resurrect/harness-scope section): `title` alone
|
|
775
|
+
// (`${room}/${file}`) is not unique per hit, and shares the EXACT SAME
|
|
776
|
+
// applyRRF-collision class scoreJournalTier's own fix just above closes
|
|
777
|
+
// (see that comment for the mechanism) — two distinct matching lines in
|
|
778
|
+
// the SAME room file collide in applyRRF's per-tier id-Map today,
|
|
779
|
+
// silently discarding one's excerpt while accumulating both hits' RRF
|
|
780
|
+
// contribution into whichever one survives. Unlike the journal case,
|
|
781
|
+
// this defect is PRE-EXISTING (live in smart_recall's palace tier since
|
|
782
|
+
// Wave 2) and OUT OF W3b's scope (journalSearch/recallInsight only) —
|
|
783
|
+
// and, verified while investigating this wave's own journal fix, giving
|
|
784
|
+
// palace items the same per-hit-unique id REMOVES an (accidental,
|
|
785
|
+
// never-intended) score-inflation side effect of this same bug that
|
|
786
|
+
// helpers/associative-link.ts's `linkToSimilar` — an entirely different
|
|
787
|
+
// subsystem this wave does not touch — happens to depend on for its
|
|
788
|
+
// hardcoded `score > 0.03` similarity threshold
|
|
789
|
+
// (associative-link.test.mjs's "linkToSimilar creates bidirectional
|
|
790
|
+
// edges..." fails if this is fixed in isolation, because each hit's own
|
|
791
|
+
// un-inflated RRF contribution, ~0.016, is genuinely below that
|
|
792
|
+
// threshold). Fixing this properly needs a decision about
|
|
793
|
+
// linkToSimilar's OWN threshold calibration, not a query-memory.ts-only
|
|
794
|
+
// change — flagged in the Wave 3b report for the orchestrator, not
|
|
795
|
+
// silently fixed or silently left undocumented.
|
|
796
|
+
const id = stableId("palace", title);
|
|
797
|
+
const salience = Math.max(0.4, salienceByRoom.get(h.room) ?? 0.5);
|
|
798
|
+
const internalScore = h.keywordScore * 0.65 + salience * 0.35;
|
|
799
|
+
const datePattern = h.excerpt.match(/(\d{4}-\d{2}-\d{2})/);
|
|
800
|
+
return {
|
|
801
|
+
id,
|
|
802
|
+
source: "palace",
|
|
803
|
+
title,
|
|
804
|
+
excerpt: h.excerpt,
|
|
805
|
+
score: internalScore,
|
|
806
|
+
room: h.room,
|
|
807
|
+
file: h.file,
|
|
808
|
+
date: datePattern ? datePattern[1] : undefined,
|
|
809
|
+
keywordScore: h.keywordScore,
|
|
810
|
+
// Wave 5a: append-order signal for the CONTRADICTION stage's
|
|
811
|
+
// same-day/no-date tie-break — see QueryMemoryItem.line's doc comment.
|
|
812
|
+
line: h.line,
|
|
813
|
+
};
|
|
814
|
+
});
|
|
815
|
+
items.sort((a, b) => b.score - a.score);
|
|
816
|
+
return items.slice(0, limit);
|
|
817
|
+
}
|
|
818
|
+
/**
|
|
819
|
+
* Insight tier: fetches directly from `recallInsights()`
|
|
820
|
+
* (`palace/insights-index.ts`, its own single-owner tier per the
|
|
821
|
+
* architecture review §4.1 — insights are a curated confirmed-pattern
|
|
822
|
+
* store, not raw retrievable file content, so there is no
|
|
823
|
+
* MemoryCandidate/trust-tag notion for this tier; every insight item is
|
|
824
|
+
* `untrusted: false` by construction, a deliberate, documented decision —
|
|
825
|
+
* not an oversight). Scoring formula ported verbatim from smart-recall.ts.
|
|
826
|
+
*
|
|
827
|
+
* Wave 3b (2026-08-30) change: previously fetched via the tools-logic
|
|
828
|
+
* `recallInsight()` wrapper, which (a) never threaded `project` through to
|
|
829
|
+
* `recallInsights()`'s own project-correlation boost — the SAME missing-
|
|
830
|
+
* project-boost gap this wave's report names for `recallInsight()` itself,
|
|
831
|
+
* fixed here too since it is the same root cause (`recallInsights()` never
|
|
832
|
+
* receiving a `currentProject` argument), not two separate bugs — and (b)
|
|
833
|
+
* stripped `IndexedInsight.projects` from its returned shape entirely,
|
|
834
|
+
* making the SCOPE stage impossible to wire for this tier (there would be
|
|
835
|
+
* nothing to filter by). Calling `recallInsights()` directly here also
|
|
836
|
+
* removes this file's former dependency on tools-logic/recall-insight.ts —
|
|
837
|
+
* recall-insight.ts now depends the OTHER direction (on `./scope.ts`, which
|
|
838
|
+
* this file also uses), avoiding a circular import that routing THROUGH
|
|
839
|
+
* `recallInsight()` in both directions would have created.
|
|
840
|
+
*
|
|
841
|
+
* CHARACTERIZED, non-substantive precision note: `recallInsight()`'s own
|
|
842
|
+
* output rounds `relevance` to 2 decimal places before this tier used to
|
|
843
|
+
* re-normalize it (`i.relevance / maxRelevance`). Reading `recallInsights()`
|
|
844
|
+
* directly uses the UNROUNDED relevance number instead — a strictly more
|
|
845
|
+
* precise input to the same ratio, not a behavior change any caller could
|
|
846
|
+
* observe (`internalScore` differences are sub-0.005, far below anything
|
|
847
|
+
* that could flip an RRF rank order in practice, and no test in this
|
|
848
|
+
* package asserts on this tier's exact internal `score` value).
|
|
849
|
+
*/
|
|
850
|
+
function scoreInsightTier(project, query, opts) {
|
|
851
|
+
const limit = opts.perTierLimit ?? 20;
|
|
852
|
+
const rawInsights = recallInsights(query, limit, project);
|
|
853
|
+
const maxRelevance = Math.max(1, ...rawInsights.map((i) => i.relevance));
|
|
854
|
+
const items = rawInsights.map((i) => {
|
|
855
|
+
const id = stableId("insight", i.title);
|
|
856
|
+
const relevance = i.relevance / maxRelevance;
|
|
857
|
+
const exactness = keywordExactness(query, i.title);
|
|
858
|
+
const confirmation = Math.min(1.0, Math.log2(i.confirmed_count + 1) / 3);
|
|
859
|
+
const internalScore = relevance * 0.40 + exactness * 0.35 + confirmation * 0.25;
|
|
860
|
+
const rawExcerpt = `[${i.severity}] ${i.applies_when.join(", ")}`;
|
|
861
|
+
const fusionSeed = `${i.title} ${rawExcerpt}`;
|
|
862
|
+
return {
|
|
863
|
+
id,
|
|
864
|
+
source: "insight",
|
|
865
|
+
title: i.title,
|
|
866
|
+
excerpt: rawExcerpt.length > 300 ? rawExcerpt.slice(0, 300) + "..." : rawExcerpt,
|
|
867
|
+
fusionKey: fusionSeed.length > 300 ? fusionSeed.slice(0, 300) + "..." : fusionSeed,
|
|
868
|
+
score: internalScore,
|
|
869
|
+
severity: i.severity,
|
|
870
|
+
projects: i.projects,
|
|
871
|
+
};
|
|
872
|
+
});
|
|
873
|
+
items.sort((a, b) => b.score - a.score);
|
|
874
|
+
return items;
|
|
875
|
+
}
|
|
876
|
+
function applyRRF(rankedItems, rrfMap) {
|
|
877
|
+
rankedItems.forEach((item, idx) => {
|
|
878
|
+
const rank = idx + 1;
|
|
879
|
+
const contribution = 1 / (RRF_K + rank);
|
|
880
|
+
const existing = rrfMap.get(item.id);
|
|
881
|
+
if (existing) {
|
|
882
|
+
existing.score += contribution;
|
|
883
|
+
}
|
|
884
|
+
else {
|
|
885
|
+
rrfMap.set(item.id, { score: contribution, item, sources: new Set([item.source]) });
|
|
886
|
+
}
|
|
887
|
+
});
|
|
888
|
+
}
|
|
889
|
+
function fuseCanonical(rrfMap) {
|
|
890
|
+
const canonical = new Map();
|
|
891
|
+
for (const entry of rrfMap.values()) {
|
|
892
|
+
const key = fusionIdentity(entry.item);
|
|
893
|
+
const existing = canonical.get(key);
|
|
894
|
+
if (existing) {
|
|
895
|
+
existing.score += entry.score;
|
|
896
|
+
for (const s of entry.sources)
|
|
897
|
+
existing.sources.add(s);
|
|
898
|
+
}
|
|
899
|
+
else {
|
|
900
|
+
canonical.set(key, { score: entry.score, item: entry.item, sources: new Set(entry.sources) });
|
|
901
|
+
}
|
|
902
|
+
}
|
|
903
|
+
return canonical;
|
|
904
|
+
}
|
|
905
|
+
function applyHotWindowBoost(fusedMap) {
|
|
906
|
+
for (const entry of fusedMap.values()) {
|
|
907
|
+
if (entry.item.date) {
|
|
908
|
+
const hoursAgo = (Date.now() - new Date(entry.item.date).getTime()) / (1000 * 60 * 60);
|
|
909
|
+
if (hoursAgo < 6)
|
|
910
|
+
entry.score *= 3.0;
|
|
911
|
+
else if (hoursAgo < 24)
|
|
912
|
+
entry.score *= 2.0;
|
|
913
|
+
else if (hoursAgo < 72)
|
|
914
|
+
entry.score *= 1.3;
|
|
915
|
+
}
|
|
916
|
+
}
|
|
917
|
+
}
|
|
918
|
+
// ---------------------------------------------------------------------------
|
|
919
|
+
// queryMemory() — the mandatory pipeline entry point.
|
|
920
|
+
// ---------------------------------------------------------------------------
|
|
921
|
+
const TIER_SCORERS = {
|
|
922
|
+
journal: async (input) => scoreJournalTier(input.project, input.query, {
|
|
923
|
+
includeRollupArchive: input.journal?.includeRollupArchive,
|
|
924
|
+
// Matches smart-recall.ts's original `journalSearch({..., limit: Math.ceil(limit*1.5)})`.
|
|
925
|
+
perTierLimit: input.journal?.perTierLimit ?? Math.ceil((input.limit ?? 10) * 1.5),
|
|
926
|
+
since: input.since,
|
|
927
|
+
}),
|
|
928
|
+
palace: async (input) => scorePalaceTier(input.project, input.query, {
|
|
929
|
+
room: input.palace?.room,
|
|
930
|
+
// Matches smart-recall.ts's original `palaceSearch({..., limit: limit*2})`.
|
|
931
|
+
perTierLimit: input.palace?.perTierLimit ?? (input.limit ?? 10) * 2,
|
|
932
|
+
}),
|
|
933
|
+
insight: async (input) => scoreInsightTier(input.project, input.query, {
|
|
934
|
+
// Matches smart-recall.ts's original `recallInsight({..., limit: limit*2})`.
|
|
935
|
+
perTierLimit: input.insight?.perTierLimit ?? (input.limit ?? 10) * 2,
|
|
936
|
+
}),
|
|
937
|
+
};
|
|
938
|
+
export async function queryMemory(input) {
|
|
939
|
+
const sourcesQueried = [];
|
|
940
|
+
const candidatesBySource = {};
|
|
941
|
+
const byTier = {};
|
|
942
|
+
for (const tier of input.tiers) {
|
|
943
|
+
try {
|
|
944
|
+
// FETCH + TRUST-FILTER + TOKENIZE+SCORE (per-tier strategy table above).
|
|
945
|
+
let items = await TIER_SCORERS[tier](input);
|
|
946
|
+
// SCOPE stage (Wave 3b) — real per-candidate project-attribution
|
|
947
|
+
// filter, but only for tiers whose items carry genuine cross-project
|
|
948
|
+
// attribution (see SCOPE_ATTRIBUTED_TIERS above for why journal/
|
|
949
|
+
// palace-room must short-circuit instead of relying on applyScope's
|
|
950
|
+
// own undefined-`projects` handling).
|
|
951
|
+
if (SCOPE_ATTRIBUTED_TIERS.has(tier)) {
|
|
952
|
+
items = applyScope(items, input.project, input.scope);
|
|
953
|
+
}
|
|
954
|
+
// CONTRADICTION stage (Wave 5a; ANNOTATE-ONLY as of the STEP 4a
|
|
955
|
+
// pre-ship fix, 2026-09-01 — see applyContradictionStage's own doc
|
|
956
|
+
// comment) — annotates superseded candidates within this tier; never
|
|
957
|
+
// drops, never re-scores, never reorders (see CONTRADICTION_TIERS
|
|
958
|
+
// above for why insight is short-circuited this wave).
|
|
959
|
+
items = applyContradictionStage(tier, items);
|
|
960
|
+
byTier[tier] = items;
|
|
961
|
+
candidatesBySource[tier] = items.length;
|
|
962
|
+
sourcesQueried.push(tier);
|
|
963
|
+
}
|
|
964
|
+
catch {
|
|
965
|
+
// A tier failing to initialize (e.g. palace not yet created) must not
|
|
966
|
+
// fail the whole query — matches smart-recall.ts's original per-source
|
|
967
|
+
// try/catch isolation.
|
|
968
|
+
}
|
|
969
|
+
}
|
|
970
|
+
// RANK/FUSE stage.
|
|
971
|
+
const rrfMap = new Map();
|
|
972
|
+
for (const tier of input.tiers) {
|
|
973
|
+
const items = byTier[tier];
|
|
974
|
+
if (items)
|
|
975
|
+
applyRRF(items, rrfMap);
|
|
976
|
+
}
|
|
977
|
+
const fusedMap = fuseCanonical(rrfMap);
|
|
978
|
+
applyHotWindowBoost(fusedMap);
|
|
979
|
+
const seen = new Set();
|
|
980
|
+
const fused = [];
|
|
981
|
+
for (const { score, item, sources } of fusedMap.values()) {
|
|
982
|
+
const key = fusionIdentity(item);
|
|
983
|
+
if (seen.has(key))
|
|
984
|
+
continue;
|
|
985
|
+
seen.add(key);
|
|
986
|
+
const alsoFoundIn = [...sources].filter((s) => s !== item.source);
|
|
987
|
+
fused.push({ ...item, score, ...(alsoFoundIn.length > 0 ? { alsoFoundIn } : {}) });
|
|
988
|
+
}
|
|
989
|
+
fused.sort((a, b) => b.score - a.score);
|
|
990
|
+
return {
|
|
991
|
+
items: fused,
|
|
992
|
+
candidatesBySource,
|
|
993
|
+
sourcesQueried,
|
|
994
|
+
renderFenced(limit) {
|
|
995
|
+
const capped = typeof limit === "number" ? fused.slice(0, limit) : fused;
|
|
996
|
+
const lines = capped.map((it, i) => `[${i + 1}][${it.source}${it.room ? `/${it.room}` : ""}] ${it.title} — ${it.excerpt}`);
|
|
997
|
+
return fenceMemory(lines.join("\n"));
|
|
998
|
+
},
|
|
999
|
+
};
|
|
1000
|
+
}
|
|
1001
|
+
// ---------------------------------------------------------------------------
|
|
1002
|
+
// Archive fallback — F4 (continuity wave, 2026-07-31), relocated here from
|
|
1003
|
+
// smart-recall.ts per this wave's brief ("archiveSearch composition ... move
|
|
1004
|
+
// INTO the pipeline"). Deliberately NOT part of `queryMemory()`'s `tiers`
|
|
1005
|
+
// array / RRF fusion: the ORIGINAL design's own doc comment says this can
|
|
1006
|
+
// "NEVER be part of the palace/journal/insight RRF fusion" — it is a
|
|
1007
|
+
// confidence-GATED last resort, and the gate (fused top confidence vs
|
|
1008
|
+
// CONFIDENCE_FLOOR.medium) is a smart_recall-SPECIFIC policy decision, not a
|
|
1009
|
+
// generic pipeline stage. Baking that gate into queryMemory() itself would
|
|
1010
|
+
// leak surface-specific policy into the shared pipeline — the same
|
|
1011
|
+
// "leaf utility everyone must remember not to (mis)trust" anti-pattern this
|
|
1012
|
+
// whole effort exists to close, just at a different layer. So: the FETCH +
|
|
1013
|
+
// TRUST-FILTER + capped-SCORE logic moves here (this file); the GATING
|
|
1014
|
+
// decision (when to call it) stays exactly where it always was —
|
|
1015
|
+
// smart-recall.ts's own post-fusion policy, unchanged.
|
|
1016
|
+
// ---------------------------------------------------------------------------
|
|
1017
|
+
export function queryArchiveFallback(project, query, limit) {
|
|
1018
|
+
const keywords = tokenizeWords(query);
|
|
1019
|
+
if (keywords.length === 0 || limit <= 0)
|
|
1020
|
+
return [];
|
|
1021
|
+
const candidates = filterTrusted(readTierCandidates("journal", project, { includeRawArchive: true, includeUntrusted: true }).filter((c) => c.sourceKind === "journal-archive-raw"));
|
|
1022
|
+
// Newest raw dump first — matches the original's `files.sort((a,b) =>
|
|
1023
|
+
// b.localeCompare(a))` file-order preference for a last-resort fallback.
|
|
1024
|
+
candidates.sort((a, b) => b.file.localeCompare(a.file));
|
|
1025
|
+
const items = [];
|
|
1026
|
+
for (const candidate of candidates) {
|
|
1027
|
+
if (items.length >= limit)
|
|
1028
|
+
break;
|
|
1029
|
+
const lines = candidate.content.split("\n");
|
|
1030
|
+
for (let i = 0; i < lines.length; i++) {
|
|
1031
|
+
if (items.length >= limit)
|
|
1032
|
+
break;
|
|
1033
|
+
const line = lines[i];
|
|
1034
|
+
const lineLower = line.toLowerCase();
|
|
1035
|
+
if (!keywords.some((kw) => lineLower.includes(kw)))
|
|
1036
|
+
continue;
|
|
1037
|
+
let matchIdx = line.length;
|
|
1038
|
+
for (const kw of keywords) {
|
|
1039
|
+
const idx = lineLower.indexOf(kw);
|
|
1040
|
+
if (idx !== -1 && idx < matchIdx)
|
|
1041
|
+
matchIdx = idx;
|
|
1042
|
+
}
|
|
1043
|
+
const start = Math.max(0, matchIdx - 100);
|
|
1044
|
+
const end = Math.min(line.length, matchIdx + 150);
|
|
1045
|
+
let snippet = line.slice(start, end).trim();
|
|
1046
|
+
if (start > 0)
|
|
1047
|
+
snippet = "..." + snippet;
|
|
1048
|
+
if (end < line.length)
|
|
1049
|
+
snippet = snippet + "...";
|
|
1050
|
+
if (!snippet)
|
|
1051
|
+
continue;
|
|
1052
|
+
snippet = scrubForCloud(snippet);
|
|
1053
|
+
const provenance = path.join("journal", "archive", "raw", candidate.file);
|
|
1054
|
+
const rawScore = Math.min(keywordExactness(query, line), CONFIDENCE_FLOOR.medium - 0.01);
|
|
1055
|
+
items.push({
|
|
1056
|
+
id: stableId("archive", `${candidate.file}:${i}`),
|
|
1057
|
+
source: "archive",
|
|
1058
|
+
title: `archive/${candidate.file}`,
|
|
1059
|
+
excerpt: `[raw-archive · low-confidence · ${provenance}] ${snippet}`,
|
|
1060
|
+
score: rawScore,
|
|
1061
|
+
file: candidate.file,
|
|
1062
|
+
date: candidate.date || undefined,
|
|
1063
|
+
});
|
|
1064
|
+
}
|
|
1065
|
+
}
|
|
1066
|
+
return items;
|
|
1067
|
+
}
|
|
1068
|
+
//# sourceMappingURL=query-memory.js.map
|