agent-recall-core 3.4.51 → 3.5.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +670 -330
- package/dist/digest/match.d.ts +2 -0
- package/dist/digest/match.d.ts.map +1 -1
- package/dist/digest/match.js +5 -0
- package/dist/digest/match.js.map +1 -1
- package/dist/digest/store.d.ts.map +1 -1
- package/dist/digest/store.js +2 -3
- package/dist/digest/store.js.map +1 -1
- package/dist/embeddings/config.d.ts.map +1 -1
- package/dist/embeddings/config.js +17 -4
- package/dist/embeddings/config.js.map +1 -1
- package/dist/helpers/alignment-patterns.d.ts +46 -0
- package/dist/helpers/alignment-patterns.d.ts.map +1 -1
- package/dist/helpers/alignment-patterns.js +103 -2
- package/dist/helpers/alignment-patterns.js.map +1 -1
- package/dist/helpers/journal-files.d.ts +12 -6
- package/dist/helpers/journal-files.d.ts.map +1 -1
- package/dist/helpers/journal-files.js +13 -7
- package/dist/helpers/journal-files.js.map +1 -1
- package/dist/index.d.ts +23 -6
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +23 -5
- package/dist/index.js.map +1 -1
- package/dist/palace/rooms.d.ts +7 -0
- package/dist/palace/rooms.d.ts.map +1 -1
- package/dist/palace/rooms.js +13 -0
- package/dist/palace/rooms.js.map +1 -1
- package/dist/prompts/consolidation-prompt.js +14 -14
- package/dist/retrieval/candidates.d.ts +17 -0
- package/dist/retrieval/candidates.d.ts.map +1 -1
- package/dist/retrieval/candidates.js +4 -2
- package/dist/retrieval/candidates.js.map +1 -1
- package/dist/retrieval/contradiction.d.ts +42 -0
- package/dist/retrieval/contradiction.d.ts.map +1 -1
- package/dist/retrieval/contradiction.js +126 -7
- package/dist/retrieval/contradiction.js.map +1 -1
- package/dist/retrieval/lexical-index.d.ts +150 -0
- package/dist/retrieval/lexical-index.d.ts.map +1 -0
- package/dist/retrieval/lexical-index.js +516 -0
- package/dist/retrieval/lexical-index.js.map +1 -0
- package/dist/retrieval/query-memory.d.ts +17 -7
- package/dist/retrieval/query-memory.d.ts.map +1 -1
- package/dist/retrieval/query-memory.js +279 -71
- package/dist/retrieval/query-memory.js.map +1 -1
- package/dist/storage/classification.d.ts.map +1 -1
- package/dist/storage/classification.js +6 -1
- package/dist/storage/classification.js.map +1 -1
- package/dist/storage/content-guard.d.ts +32 -0
- package/dist/storage/content-guard.d.ts.map +1 -1
- package/dist/storage/content-guard.js +467 -48
- package/dist/storage/content-guard.js.map +1 -1
- package/dist/storage/corrections.d.ts +3 -1
- package/dist/storage/corrections.d.ts.map +1 -1
- package/dist/storage/corrections.js +52 -67
- package/dist/storage/corrections.js.map +1 -1
- package/dist/storage/cwd-allowlist.d.ts +30 -2
- package/dist/storage/cwd-allowlist.d.ts.map +1 -1
- package/dist/storage/cwd-allowlist.js +105 -22
- package/dist/storage/cwd-allowlist.js.map +1 -1
- package/dist/storage/deferred-bookkeeping.d.ts +34 -0
- package/dist/storage/deferred-bookkeeping.d.ts.map +1 -0
- package/dist/storage/deferred-bookkeeping.js +201 -0
- package/dist/storage/deferred-bookkeeping.js.map +1 -0
- package/dist/storage/filelock.d.ts +26 -8
- package/dist/storage/filelock.d.ts.map +1 -1
- package/dist/storage/filelock.js +202 -26
- package/dist/storage/filelock.js.map +1 -1
- package/dist/storage/frontmatter.d.ts +14 -0
- package/dist/storage/frontmatter.d.ts.map +1 -0
- package/dist/storage/frontmatter.js +22 -0
- package/dist/storage/frontmatter.js.map +1 -0
- package/dist/storage/fs-utils.d.ts +53 -0
- package/dist/storage/fs-utils.d.ts.map +1 -1
- package/dist/storage/fs-utils.js +140 -2
- package/dist/storage/fs-utils.js.map +1 -1
- package/dist/storage/hook-session.d.ts +9 -0
- package/dist/storage/hook-session.d.ts.map +1 -0
- package/dist/storage/hook-session.js +49 -0
- package/dist/storage/hook-session.js.map +1 -0
- package/dist/storage/identity-card.d.ts +27 -0
- package/dist/storage/identity-card.d.ts.map +1 -0
- package/dist/storage/identity-card.js +86 -0
- package/dist/storage/identity-card.js.map +1 -0
- package/dist/storage/lifecycle-telemetry.d.ts +1 -1
- package/dist/storage/lifecycle-telemetry.d.ts.map +1 -1
- package/dist/storage/lifecycle-telemetry.js +2 -2
- package/dist/storage/lifecycle-telemetry.js.map +1 -1
- package/dist/storage/memory-protocol.js +55 -55
- package/dist/storage/paths.d.ts.map +1 -1
- package/dist/storage/paths.js +7 -2
- package/dist/storage/paths.js.map +1 -1
- package/dist/storage/pending.d.ts +39 -3
- package/dist/storage/pending.d.ts.map +1 -1
- package/dist/storage/pending.js +126 -20
- package/dist/storage/pending.js.map +1 -1
- package/dist/storage/project.d.ts.map +1 -1
- package/dist/storage/project.js +34 -9
- package/dist/storage/project.js.map +1 -1
- package/dist/storage/session.d.ts.map +1 -1
- package/dist/storage/session.js +43 -16
- package/dist/storage/session.js.map +1 -1
- package/dist/storage/state-card.d.ts +59 -0
- package/dist/storage/state-card.d.ts.map +1 -0
- package/dist/storage/state-card.js +162 -0
- package/dist/storage/state-card.js.map +1 -0
- package/dist/storage/store-manifest.js +57 -57
- package/dist/storage/unclaimed.js +2 -2
- package/dist/storage/unclaimed.js.map +1 -1
- package/dist/supabase/config.d.ts.map +1 -1
- package/dist/supabase/config.js +7 -1
- package/dist/supabase/config.js.map +1 -1
- package/dist/supabase/embedding.d.ts.map +1 -1
- package/dist/supabase/embedding.js +7 -0
- package/dist/supabase/embedding.js.map +1 -1
- package/dist/supabase/sync.d.ts.map +1 -1
- package/dist/supabase/sync.js +42 -18
- package/dist/supabase/sync.js.map +1 -1
- package/dist/tools-logic/bootstrap.d.ts.map +1 -1
- package/dist/tools-logic/bootstrap.js +4 -2
- package/dist/tools-logic/bootstrap.js.map +1 -1
- package/dist/tools-logic/check.d.ts +69 -2
- package/dist/tools-logic/check.d.ts.map +1 -1
- package/dist/tools-logic/check.js +280 -57
- package/dist/tools-logic/check.js.map +1 -1
- package/dist/tools-logic/dream-admission.js +29 -29
- package/dist/tools-logic/recall-backend.d.ts.map +1 -1
- package/dist/tools-logic/recall-backend.js +11 -1
- package/dist/tools-logic/recall-backend.js.map +1 -1
- package/dist/tools-logic/session-end.d.ts +18 -1
- package/dist/tools-logic/session-end.d.ts.map +1 -1
- package/dist/tools-logic/session-end.js +36 -5
- package/dist/tools-logic/session-end.js.map +1 -1
- package/dist/tools-logic/session-start-render.d.ts +35 -0
- package/dist/tools-logic/session-start-render.d.ts.map +1 -0
- package/dist/tools-logic/session-start-render.js +240 -0
- package/dist/tools-logic/session-start-render.js.map +1 -0
- package/dist/tools-logic/session-start.d.ts +21 -0
- package/dist/tools-logic/session-start.d.ts.map +1 -1
- package/dist/tools-logic/session-start.js +279 -225
- package/dist/tools-logic/session-start.js.map +1 -1
- package/dist/tools-logic/smart-recall.d.ts +60 -0
- package/dist/tools-logic/smart-recall.d.ts.map +1 -1
- package/dist/tools-logic/smart-recall.js +221 -31
- package/dist/tools-logic/smart-recall.js.map +1 -1
- package/dist/tools-logic/smart-remember-vector.d.ts +3 -1
- package/dist/tools-logic/smart-remember-vector.d.ts.map +1 -1
- package/dist/tools-logic/smart-remember-vector.js +3 -1
- package/dist/tools-logic/smart-remember-vector.js.map +1 -1
- package/dist/tools-logic/smart-remember.d.ts.map +1 -1
- package/dist/tools-logic/smart-remember.js +8 -5
- package/dist/tools-logic/smart-remember.js.map +1 -1
- package/dist/types.d.ts +98 -2
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js +164 -3
- package/dist/types.js.map +1 -1
- package/dist/vector/embedding.d.ts +20 -1
- package/dist/vector/embedding.d.ts.map +1 -1
- package/dist/vector/embedding.js +30 -1
- package/dist/vector/embedding.js.map +1 -1
- package/dist/vector/local-vector-backend.d.ts +4 -1
- package/dist/vector/local-vector-backend.d.ts.map +1 -1
- package/dist/vector/local-vector-backend.js +7 -3
- package/dist/vector/local-vector-backend.js.map +1 -1
- package/package.json +59 -59
|
@@ -234,9 +234,11 @@
|
|
|
234
234
|
* the grammar fix's full false-positive-by-false-positive proof.
|
|
235
235
|
*/
|
|
236
236
|
import { readTierCandidates, filterTrusted } from "./candidates.js";
|
|
237
|
+
import { loadJournalLexicalSnapshot } from "./lexical-index.js";
|
|
237
238
|
import { applyScope } from "./scope.js";
|
|
238
239
|
import { detectContradictions } from "./contradiction.js";
|
|
239
|
-
import { listRooms, recordAccess,
|
|
240
|
+
import { listRooms, recordAccess, ensurePalaceInitializedForRead } from "../palace/rooms.js";
|
|
241
|
+
import { bookkeep } from "../storage/deferred-bookkeeping.js";
|
|
240
242
|
import { stem, expandQuery } from "../helpers/normalize.js";
|
|
241
243
|
import { tokenizeWords } from "../helpers/tokenize.js";
|
|
242
244
|
import { recallInsights } from "../palace/insights-index.js";
|
|
@@ -327,7 +329,7 @@ function bm25Idf(df, n) {
|
|
|
327
329
|
* substring containment (script-agnostic), so a rare Han token earns
|
|
328
330
|
* exactly the same discriminative weight as a rare ASCII one.
|
|
329
331
|
*/
|
|
330
|
-
function
|
|
332
|
+
function buildQueryVariants(query) {
|
|
331
333
|
const rawWords = tokenizeWords(query);
|
|
332
334
|
const variantsByRaw = new Map();
|
|
333
335
|
for (const w of rawWords) {
|
|
@@ -338,19 +340,22 @@ function buildIdfModel(query, corpusTexts) {
|
|
|
338
340
|
variants.add(v);
|
|
339
341
|
variantsByRaw.set(w, [...variants]);
|
|
340
342
|
}
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
343
|
+
return { rawWords, variantsByRaw };
|
|
344
|
+
}
|
|
345
|
+
/**
|
|
346
|
+
* Assemble the IDF model from a per-word DF oracle. This is buildIdfModel's
|
|
347
|
+
* exact math (bm25Idf → max-normalize → totalMass → weightedExactness),
|
|
348
|
+
* extracted verbatim so the brute-force corpus scan and the lexical-index
|
|
349
|
+
* postings path (scoreJournalTier) produce byte-identical weights whenever
|
|
350
|
+
* their DF counts agree — the only thing that varies between them is HOW df
|
|
351
|
+
* is counted, pinned equal by test/lexical-index.test.mjs.
|
|
352
|
+
*/
|
|
353
|
+
function assembleIdfModel(qv, n, dfOf) {
|
|
354
|
+
const { rawWords, variantsByRaw } = qv;
|
|
346
355
|
const idf = new Map();
|
|
347
356
|
let maxIdf = 0;
|
|
348
357
|
for (const [w, variants] of variantsByRaw) {
|
|
349
|
-
|
|
350
|
-
for (const d of docs) {
|
|
351
|
-
if (variants.some((v) => d.tokenSet.has(v) || d.lower.includes(v)))
|
|
352
|
-
df++;
|
|
353
|
-
}
|
|
358
|
+
const df = dfOf(variants);
|
|
354
359
|
const weight = n > 0 ? bm25Idf(df, n) : 1;
|
|
355
360
|
idf.set(w, weight);
|
|
356
361
|
if (weight > maxIdf)
|
|
@@ -385,6 +390,22 @@ function buildIdfModel(query, corpusTexts) {
|
|
|
385
390
|
},
|
|
386
391
|
};
|
|
387
392
|
}
|
|
393
|
+
function buildIdfModel(query, corpusTexts) {
|
|
394
|
+
const qv = buildQueryVariants(query);
|
|
395
|
+
const n = corpusTexts.length;
|
|
396
|
+
const docs = corpusTexts.map((t) => {
|
|
397
|
+
const lower = t.toLowerCase();
|
|
398
|
+
return { lower, tokenSet: new Set(tokenizeWords(t).map((tok) => stem(tok))) };
|
|
399
|
+
});
|
|
400
|
+
return assembleIdfModel(qv, n, (variants) => {
|
|
401
|
+
let df = 0;
|
|
402
|
+
for (const d of docs) {
|
|
403
|
+
if (variants.some((v) => d.tokenSet.has(v) || d.lower.includes(v)))
|
|
404
|
+
df++;
|
|
405
|
+
}
|
|
406
|
+
return df;
|
|
407
|
+
});
|
|
408
|
+
}
|
|
388
409
|
/**
|
|
389
410
|
* Simple stable hash for result IDs — ported verbatim from smart-recall.ts.
|
|
390
411
|
*
|
|
@@ -706,74 +727,175 @@ function readLegacyJournalCandidates(project) {
|
|
|
706
727
|
out.sort((a, b) => b.date.localeCompare(a.date));
|
|
707
728
|
return out;
|
|
708
729
|
}
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
730
|
+
/** The journal tier's per-line matcher — the old inline loop body, extracted
|
|
731
|
+
* VERBATIM (perf/lexical-index, 2026-10-16) so the brute-force path and the
|
|
732
|
+
* lexical-index fast path run the IDENTICAL scan (section tracking, excerpt
|
|
733
|
+
* window, per-hit line numbers) over identical content. `dateLabel` is the
|
|
734
|
+
* old loop's `candidate.date || candidate.file` — computed once per
|
|
735
|
+
* candidate by both callers. */
|
|
736
|
+
function collectJournalLineHits(content, dateLabel, keywords, hits) {
|
|
737
|
+
const lines = content.split("\n");
|
|
738
|
+
let currentSection = "top";
|
|
739
|
+
for (let i = 0; i < lines.length; i++) {
|
|
740
|
+
const line = lines[i];
|
|
741
|
+
if (line.startsWith("## ")) {
|
|
742
|
+
currentSection = line.slice(3).trim().toLowerCase().replace(/\s+/g, "_");
|
|
743
|
+
}
|
|
744
|
+
if (!lineMatchesQuery(line, keywords))
|
|
745
|
+
continue;
|
|
746
|
+
const matchIdx = firstMatchIndex(line, keywords);
|
|
747
|
+
const start = Math.max(0, matchIdx - 100);
|
|
748
|
+
const end = Math.min(line.length, matchIdx + 150);
|
|
749
|
+
let excerpt = line.slice(start, end).trim();
|
|
750
|
+
if (start > 0)
|
|
751
|
+
excerpt = "..." + excerpt;
|
|
752
|
+
if (end < line.length)
|
|
753
|
+
excerpt = excerpt + "...";
|
|
754
|
+
hits.push({ title: `${dateLabel} / ${currentSection}`, excerpt, date: dateLabel, line: i + 1 });
|
|
755
|
+
}
|
|
756
|
+
}
|
|
712
757
|
/** Journal tier: ported from journal-search.ts's per-line matching (section
|
|
713
758
|
* tracking, excerpt window) + smart-recall.ts's Ebbinghaus×exactness scoring —
|
|
714
759
|
* now operating on already-fetched MemoryCandidate content instead of a
|
|
715
|
-
* fresh fs read.
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
760
|
+
* fresh fs read.
|
|
761
|
+
*
|
|
762
|
+
* perf/lexical-index (2026-10-16): async, with a persistent-index FAST PATH.
|
|
763
|
+
* When the lexical index is usable (default; AGENT_RECALL_LEXICAL_INDEX=0
|
|
764
|
+
* disables, and ANY index failure falls back), the fetch stage no longer
|
|
765
|
+
* reads/tokenizes the whole corpus per query: trust bits, the line-match
|
|
766
|
+
* prefilter, and IDF document frequencies come from retrieval/lexical-index.ts,
|
|
767
|
+
* and file content is read ONLY for docs that can actually produce a hit.
|
|
768
|
+
* RANKING-IDENTICAL to the brute path by construction (same enumeration
|
|
769
|
+
* order, same trust filter, the same collectJournalLineHits scan, and
|
|
770
|
+
* assembleIdfModel fed DF counts that match buildIdfModel's predicate
|
|
771
|
+
* exactly) — pinned by test/lexical-index.test.mjs. The legacy-journal half
|
|
772
|
+
* (readLegacyJournalCandidates) stays brute-force on both paths: it scans
|
|
773
|
+
* an out-of-root directory tree that is empty on virtually every install. */
|
|
774
|
+
async function scoreJournalTier(project, query, opts) {
|
|
721
775
|
const keywords = tokenizeWords(query);
|
|
722
776
|
if (keywords.length === 0)
|
|
723
777
|
return [];
|
|
724
778
|
const sinceCutoff = opts.since ? parseSinceDate(opts.since) : null;
|
|
725
779
|
const limit = opts.perTierLimit ?? 25;
|
|
780
|
+
const includeRollupArchive = opts.includeRollupArchive ?? true;
|
|
726
781
|
const hits = [];
|
|
727
|
-
|
|
728
|
-
//
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
if (!isNaN(fileDate.getTime()) && fileDate < sinceCutoff)
|
|
746
|
-
continue;
|
|
782
|
+
let idfModel;
|
|
783
|
+
// ---- FAST PATH: persistent lexical index (null → brute fallback) ----
|
|
784
|
+
let snapshotOrNull = null;
|
|
785
|
+
try {
|
|
786
|
+
snapshotOrNull = await loadJournalLexicalSnapshot(project, { includeRollupArchive });
|
|
787
|
+
}
|
|
788
|
+
catch {
|
|
789
|
+
snapshotOrNull = null; // defense in depth — the loader already never throws
|
|
790
|
+
}
|
|
791
|
+
const snapshot = snapshotOrNull;
|
|
792
|
+
if (snapshot) {
|
|
793
|
+
// TRUST-FILTER stage, same predicate as filterTrusted (drop only
|
|
794
|
+
// untrusted === true) — applied to index docs by flag, with content
|
|
795
|
+
// reads below re-verifying through the rescue choke regardless.
|
|
796
|
+
const trustedIdx = [];
|
|
797
|
+
for (let i = 0; i < snapshot.docs.length; i++) {
|
|
798
|
+
if (snapshot.docs[i].untrusted !== true)
|
|
799
|
+
trustedIdx.push(i);
|
|
747
800
|
}
|
|
748
|
-
const
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
801
|
+
const legacy = filterTrusted(readLegacyJournalCandidates(project));
|
|
802
|
+
// fix4 S3's SCORE-THEN-TRUNCATE loop (see the brute path below for the
|
|
803
|
+
// full history note) — identical scan, but content is only read for
|
|
804
|
+
// docs the substring prefilter says CAN contain a keyword
|
|
805
|
+
// (lineMatchesQuery is pure `includes` over whitespace-free keywords,
|
|
806
|
+
// so docsContaining has no false negatives — lexical-index.ts header).
|
|
807
|
+
for (const i of trustedIdx) {
|
|
808
|
+
const doc = snapshot.docs[i];
|
|
809
|
+
if (sinceCutoff && doc.date) {
|
|
810
|
+
const fileDate = new Date(doc.date);
|
|
811
|
+
if (!isNaN(fileDate.getTime()) && fileDate < sinceCutoff)
|
|
812
|
+
continue;
|
|
754
813
|
}
|
|
755
|
-
if (!
|
|
814
|
+
if (!keywords.some((kw) => snapshot.docsContaining(kw).has(i)))
|
|
756
815
|
continue;
|
|
757
|
-
const
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
816
|
+
const content = snapshot.trustedContent(i);
|
|
817
|
+
if (content === null)
|
|
818
|
+
continue;
|
|
819
|
+
collectJournalLineHits(content, doc.date || doc.file, keywords, hits);
|
|
820
|
+
}
|
|
821
|
+
for (const candidate of legacy) {
|
|
822
|
+
if (sinceCutoff && candidate.date) {
|
|
823
|
+
const fileDate = new Date(candidate.date);
|
|
824
|
+
if (!isNaN(fileDate.getTime()) && fileDate < sinceCutoff)
|
|
825
|
+
continue;
|
|
826
|
+
}
|
|
827
|
+
collectJournalLineHits(candidate.content, candidate.date || candidate.file, keywords, hits);
|
|
828
|
+
}
|
|
829
|
+
if (hits.length === 0)
|
|
830
|
+
return [];
|
|
831
|
+
// fix4 S5's IDF, df counted from the index postings instead of a fresh
|
|
832
|
+
// full-corpus tokenization — same corpus (every trusted candidate,
|
|
833
|
+
// since-filter deliberately NOT applied, matching buildIdfModel's brute
|
|
834
|
+
// call below), same per-word predicate (stemmed-token-set hit OR
|
|
835
|
+
// lowercase substring), same math (assembleIdfModel).
|
|
836
|
+
const qv = buildQueryVariants(query);
|
|
837
|
+
const legacyDocs = legacy.map((c) => ({
|
|
838
|
+
lower: c.content.toLowerCase(),
|
|
839
|
+
tokenSet: new Set(tokenizeWords(c.content).map((tok) => stem(tok))),
|
|
840
|
+
}));
|
|
841
|
+
const trustedSet = new Set(trustedIdx);
|
|
842
|
+
const n = trustedIdx.length + legacyDocs.length;
|
|
843
|
+
idfModel = assembleIdfModel(qv, n, (variants) => {
|
|
844
|
+
const matched = new Set();
|
|
845
|
+
for (const v of variants) {
|
|
846
|
+
for (const d of snapshot.docsWithStemToken(v))
|
|
847
|
+
if (trustedSet.has(d))
|
|
848
|
+
matched.add(d);
|
|
849
|
+
for (const d of snapshot.docsContaining(v))
|
|
850
|
+
if (trustedSet.has(d))
|
|
851
|
+
matched.add(d);
|
|
852
|
+
}
|
|
853
|
+
let df = matched.size;
|
|
854
|
+
for (const d of legacyDocs) {
|
|
855
|
+
if (variants.some((v) => d.tokenSet.has(v) || d.lower.includes(v)))
|
|
856
|
+
df++;
|
|
857
|
+
}
|
|
858
|
+
return df;
|
|
859
|
+
});
|
|
860
|
+
}
|
|
861
|
+
else {
|
|
862
|
+
// ---- BRUTE PATH (pre-index behavior, also the fallback + test oracle) ----
|
|
863
|
+
const candidates = filterTrusted(readTierCandidates("journal", project, {
|
|
864
|
+
includeRollupArchive,
|
|
865
|
+
includeUntrusted: true,
|
|
866
|
+
})).concat(filterTrusted(readLegacyJournalCandidates(project)));
|
|
867
|
+
// fix4 S3 (2026-09-11, reports/agentrecall-fix4-retrieval-2026-09-11.md):
|
|
868
|
+
// SCORE-THEN-TRUNCATE. The pre-fix4 loop broke out at `hits.length >=
|
|
869
|
+
// limit` while iterating candidates in date-DESCENDING order — i.e. it
|
|
870
|
+
// truncated by RECENCY before any hit was ever scored, so an old entry
|
|
871
|
+
// matching the query strongly was dropped UNSCORED whenever `limit`
|
|
872
|
+
// weaker-but-newer line matches existed (the S2-standard eval's latent
|
|
873
|
+
// "journal recency pre-truncation" defect; the W2 sort-before-truncate
|
|
874
|
+
// work made the pre-truncation deterministic, not correct). Now every
|
|
875
|
+
// matching line across every candidate is collected and SCORED
|
|
876
|
+
// (recency×0.5 + exactness×0.5, unchanged formula), and `limit` is
|
|
877
|
+
// applied to the score-sorted list at the end — one truncation criterion,
|
|
878
|
+
// the same one RRF ranking uses. Cost: the full line scan always runs
|
|
879
|
+
// (previously it stopped early once `limit` hits were found) — bounded by
|
|
880
|
+
// the same already-read candidate set, and measured within the S11
|
|
881
|
+
// latency budget by this tranche's eval runs.
|
|
882
|
+
for (const candidate of candidates) {
|
|
883
|
+
if (sinceCutoff && candidate.date) {
|
|
884
|
+
const fileDate = new Date(candidate.date);
|
|
885
|
+
if (!isNaN(fileDate.getTime()) && fileDate < sinceCutoff)
|
|
886
|
+
continue;
|
|
887
|
+
}
|
|
888
|
+
collectJournalLineHits(candidate.content, candidate.date || candidate.file, keywords, hits);
|
|
767
889
|
}
|
|
890
|
+
// fix4 S5: IDF over the scanned journal corpus (one doc = one journal
|
|
891
|
+
// file's content) — weights each query word by rarity before the
|
|
892
|
+
// unchanged recency*0.5 + exactness*0.5 blend below. Review L2
|
|
893
|
+
// (2026-09-11): skipped entirely when nothing matched — building the
|
|
894
|
+
// model tokenizes/lowercases the whole corpus, pure waste at zero hits.
|
|
895
|
+
if (hits.length === 0)
|
|
896
|
+
return [];
|
|
897
|
+
idfModel = buildIdfModel(query, candidates.map((c) => c.content));
|
|
768
898
|
}
|
|
769
|
-
// fix4 S5: IDF over the scanned journal corpus (one doc = one journal
|
|
770
|
-
// file's content) — weights each query word by rarity before the
|
|
771
|
-
// unchanged recency*0.5 + exactness*0.5 blend below. Review L2
|
|
772
|
-
// (2026-09-11): skipped entirely when nothing matched — building the
|
|
773
|
-
// model tokenizes/lowercases the whole corpus, pure waste at zero hits.
|
|
774
|
-
if (hits.length === 0)
|
|
775
|
-
return [];
|
|
776
|
-
const idfModel = buildIdfModel(query, candidates.map((c) => c.content));
|
|
777
899
|
const items = hits.map((h) => {
|
|
778
900
|
// Independent review fix (W3b, 2026-08-30): `id` must be unique PER HIT,
|
|
779
901
|
// not per (date,section) — `applyRRF` (below) groups same-tier items by
|
|
@@ -829,7 +951,7 @@ async function scorePalaceTier(project, query, opts) {
|
|
|
829
951
|
// project whose palace/ has never been initialized, even when `rooms/*/`.md
|
|
830
952
|
// content already exists on disk — `listRooms()` requires a `_room.json`
|
|
831
953
|
// per room dir, which only `ensurePalaceInitialized`/`createRoom` write.
|
|
832
|
-
|
|
954
|
+
ensurePalaceInitializedForRead(project);
|
|
833
955
|
const candidates = filterTrusted(readTierCandidates("palace-room", project, { room: opts.room, includeUntrusted: true }));
|
|
834
956
|
const rooms = listRooms(project);
|
|
835
957
|
const salienceByRoom = new Map(rooms.map((r) => [r.slug, r.salience]));
|
|
@@ -936,7 +1058,8 @@ async function scorePalaceTier(project, query, opts) {
|
|
|
936
1058
|
// (once per room that contributed >=1 hit).
|
|
937
1059
|
for (const room of roomsWithHits) {
|
|
938
1060
|
try {
|
|
939
|
-
|
|
1061
|
+
// Ranking input (access_count, salience): queued inside a read; N accesses to one room are N increments.
|
|
1062
|
+
await bookkeep({ key: `room-access:${project}:${room}`, repeat: true, apply: () => recordAccess(project, room) });
|
|
940
1063
|
}
|
|
941
1064
|
catch { /* best-effort, never block scoring on a bookkeeping write */ }
|
|
942
1065
|
}
|
|
@@ -1334,6 +1457,77 @@ function fuseCanonical(rrfMap) {
|
|
|
1334
1457
|
// ordering the rest) scored per-query identical to neutralization (75.0%)
|
|
1335
1458
|
// and is documented in the fix4b report as a future option — deliberately
|
|
1336
1459
|
// NOT shipped: the brief sanctioned exactly two designs.
|
|
1460
|
+
/**
|
|
1461
|
+
* 3.5.2 BOUNDED SUPERSESSION CAP (2026-10-16, adversarial T5a,
|
|
1462
|
+
* reports/2026-10-15-rigorous-eval.md Finding 2 / §7 item 2).
|
|
1463
|
+
*
|
|
1464
|
+
* THE GAP: `applyContradictionStage` is per-tier and annotate-only, and the
|
|
1465
|
+
* semantic leg's votes carry no recency signal — so under the hybrid opt-in
|
|
1466
|
+
* a confidently-superseded fact ("rate limit 100 req/min", 2026-08-01)
|
|
1467
|
+
* deterministically outranked its own replacement ("500 req/min",
|
|
1468
|
+
* 2026-10-12). A stale-first answer is the most dangerous retrieval failure
|
|
1469
|
+
* mode: confidently wrong, invisible to the caller.
|
|
1470
|
+
*
|
|
1471
|
+
* THE FIX: after fusion, an item carrying `supersededBy` whose superseder is
|
|
1472
|
+
* ALSO present in the fused list has its fused score capped at
|
|
1473
|
+
* `superseder.score × SUPERSEDED_RANK_CAP` — i.e. a superseded fact can
|
|
1474
|
+
* never rank above the sibling that superseded it. Chains (A←B←C) converge
|
|
1475
|
+
* via the fixed-point loop: supersession edges always point stale→newer
|
|
1476
|
+
* (detectContradictions resolves direction from DATES, which totally order),
|
|
1477
|
+
* so the edge graph is acyclic and each pass only ever lowers scores —
|
|
1478
|
+
* convergence in ≤ items.length passes, deterministic.
|
|
1479
|
+
*
|
|
1480
|
+
* WHY THIS IS NOT fix4b's REMOVED HOT-WINDOW (nor STEP 4a's removed
|
|
1481
|
+
* CONTRADICTION_PENALTY) COMING BACK — the three defects that killed those
|
|
1482
|
+
* are each structurally absent here:
|
|
1483
|
+
* 1. NOT a global freshness signal: only items with a grammar-detected,
|
|
1484
|
+
* date-unambiguous `supersededBy` annotation are touched (the
|
|
1485
|
+
* high-precision version grammar + the 3.5.2 rate-fact grammar — see
|
|
1486
|
+
* ./contradiction.ts). An undated, un-conflicted, or ambiguously-tied
|
|
1487
|
+
* item is never moved; there is no time-window bucketing at all.
|
|
1488
|
+
* 2. BOUNDED AND RELATIVE: the cap is relative to the ONE superseding
|
|
1489
|
+
* sibling (≈ one adjacent RRF rank below it — 1/62 ÷ 1/61 ≈ 0.984),
|
|
1490
|
+
* never a flat multiplier that can vault items across relevance bands.
|
|
1491
|
+
* `Math.min` only ever LOWERS the stale side; the superseder and every
|
|
1492
|
+
* unrelated item keep their exact fused scores.
|
|
1493
|
+
* 3. FAIL-SAFE ON FALSE POSITIVES: a wrong grammar match at worst swaps a
|
|
1494
|
+
* near-duplicate pair's relative order (both still returned, both
|
|
1495
|
+
* annotated) — not the old ×0.5 wholesale inversion. And on a missing/
|
|
1496
|
+
* unparseable/equal date, `supersededBy` is never set in the first
|
|
1497
|
+
* place (detectContradictions' never-guess invariant), so this cap
|
|
1498
|
+
* cannot fire from no signal.
|
|
1499
|
+
*
|
|
1500
|
+
* Mutates `fused` items in place (they are already this function's own
|
|
1501
|
+
* spread copies — see the fused-array construction in queryMemory below);
|
|
1502
|
+
* the caller re-sorts immediately after. Skips silently when the superseder
|
|
1503
|
+
* is not in the fused list (e.g. filtered by `since`): the cap is relative
|
|
1504
|
+
* by design, and with no reference score there is nothing bounded to do.
|
|
1505
|
+
*/
|
|
1506
|
+
const SUPERSEDED_RANK_CAP = 0.98;
|
|
1507
|
+
function applySupersessionCap(fused) {
|
|
1508
|
+
if (fused.length < 2)
|
|
1509
|
+
return;
|
|
1510
|
+
const byId = new Map();
|
|
1511
|
+
for (const it of fused)
|
|
1512
|
+
byId.set(it.id, it);
|
|
1513
|
+
for (let pass = 0; pass < fused.length; pass++) {
|
|
1514
|
+
let changed = false;
|
|
1515
|
+
for (const it of fused) {
|
|
1516
|
+
if (!it.supersededBy)
|
|
1517
|
+
continue;
|
|
1518
|
+
const superseder = byId.get(it.supersededBy);
|
|
1519
|
+
if (!superseder)
|
|
1520
|
+
continue;
|
|
1521
|
+
const capped = superseder.score * SUPERSEDED_RANK_CAP;
|
|
1522
|
+
if (it.score > capped) {
|
|
1523
|
+
it.score = capped;
|
|
1524
|
+
changed = true;
|
|
1525
|
+
}
|
|
1526
|
+
}
|
|
1527
|
+
if (!changed)
|
|
1528
|
+
break;
|
|
1529
|
+
}
|
|
1530
|
+
}
|
|
1337
1531
|
/** LEGACY multiplicative boost — the exact pre-fix4b `applyHotWindowBoost`
|
|
1338
1532
|
* body (all defects included: date-only strings parse to 00:00 UTC so the
|
|
1339
1533
|
* bucket depends on wall-clock time-of-day — audit Finding 2; palace
|
|
@@ -1540,10 +1734,19 @@ export async function queryMemory(input) {
|
|
|
1540
1734
|
const alsoFoundIn = [...sources].filter((s) => s !== item.source);
|
|
1541
1735
|
fused.push({ ...item, score, ...(alsoFoundIn.length > 0 ? { alsoFoundIn } : {}) });
|
|
1542
1736
|
}
|
|
1737
|
+
// 3.5.2 (2026-10-16): bounded supersession cap — a grammar-detected,
|
|
1738
|
+
// date-unambiguous superseded fact never outranks its own superseder.
|
|
1739
|
+
// Runs BEFORE the sort (order-independent: pure score caps + fixed point).
|
|
1740
|
+
// This is the one deliberate, narrow exception to fix4b's "no post-fusion
|
|
1741
|
+
// freshness signal" — see applySupersessionCap's doc block for why it is
|
|
1742
|
+
// not the removed hot-window/CONTRADICTION_PENALTY class.
|
|
1743
|
+
applySupersessionCap(fused);
|
|
1543
1744
|
// fix4b (2026-09-12): the sort is stable, so exact fused-score ties
|
|
1544
1745
|
// resolve purely by insertion order — the fix4 authority tie-break
|
|
1545
|
-
// (corrections first). No post-fusion freshness signal touches
|
|
1546
|
-
// ranking
|
|
1746
|
+
// (corrections first). No GENERAL post-fusion freshness signal touches
|
|
1747
|
+
// default ranking (see the fallback doc block above
|
|
1748
|
+
// applyLegacyHotWindowBoost); the 3.5.2 supersession cap above is
|
|
1749
|
+
// supersession-gated, not time-window-gated.
|
|
1547
1750
|
fused.sort((a, b) => b.score - a.score);
|
|
1548
1751
|
return {
|
|
1549
1752
|
items: fused,
|
|
@@ -1579,7 +1782,12 @@ export function queryArchiveFallback(project, query, limit) {
|
|
|
1579
1782
|
const keywords = tokenizeWords(query);
|
|
1580
1783
|
if (keywords.length === 0 || limit <= 0)
|
|
1581
1784
|
return [];
|
|
1582
|
-
|
|
1785
|
+
// rawArchiveOnly (perf/lexical-index, 2026-10-16): this function keeps only
|
|
1786
|
+
// raw-archive candidates, so the reader is told not to read/trust-tag the
|
|
1787
|
+
// live + rollup halves at all — identical output (the `.filter` is now a
|
|
1788
|
+
// belt-and-braces no-op), minus a full live-journal read per low-confidence
|
|
1789
|
+
// query (~0.8 s at 10k entries — the 2026-10-15 scale eval's p95 driver).
|
|
1790
|
+
const candidates = filterTrusted(readTierCandidates("journal", project, { includeRawArchive: true, rawArchiveOnly: true, includeUntrusted: true }).filter((c) => c.sourceKind === "journal-archive-raw"));
|
|
1583
1791
|
// Newest raw dump first — matches the original's `files.sort((a,b) =>
|
|
1584
1792
|
// b.localeCompare(a))` file-order preference for a last-resort fallback.
|
|
1585
1793
|
candidates.sort((a, b) => b.file.localeCompare(a.file));
|