agent-recall-core 3.4.51 → 3.5.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +559 -330
- package/dist/embeddings/config.d.ts.map +1 -1
- package/dist/embeddings/config.js +17 -4
- package/dist/embeddings/config.js.map +1 -1
- package/dist/helpers/alignment-patterns.d.ts +46 -0
- package/dist/helpers/alignment-patterns.d.ts.map +1 -1
- package/dist/helpers/alignment-patterns.js +103 -2
- package/dist/helpers/alignment-patterns.js.map +1 -1
- package/dist/helpers/journal-files.d.ts +12 -6
- package/dist/helpers/journal-files.d.ts.map +1 -1
- package/dist/helpers/journal-files.js +13 -7
- package/dist/helpers/journal-files.js.map +1 -1
- package/dist/index.d.ts +8 -4
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +15 -4
- package/dist/index.js.map +1 -1
- package/dist/prompts/consolidation-prompt.js +14 -14
- package/dist/retrieval/candidates.d.ts +17 -0
- package/dist/retrieval/candidates.d.ts.map +1 -1
- package/dist/retrieval/candidates.js +4 -2
- package/dist/retrieval/candidates.js.map +1 -1
- package/dist/retrieval/contradiction.d.ts +42 -0
- package/dist/retrieval/contradiction.d.ts.map +1 -1
- package/dist/retrieval/contradiction.js +126 -7
- package/dist/retrieval/contradiction.js.map +1 -1
- package/dist/retrieval/lexical-index.d.ts +150 -0
- package/dist/retrieval/lexical-index.d.ts.map +1 -0
- package/dist/retrieval/lexical-index.js +514 -0
- package/dist/retrieval/lexical-index.js.map +1 -0
- package/dist/retrieval/query-memory.d.ts +17 -7
- package/dist/retrieval/query-memory.d.ts.map +1 -1
- package/dist/retrieval/query-memory.js +274 -68
- package/dist/retrieval/query-memory.js.map +1 -1
- package/dist/storage/classification.d.ts.map +1 -1
- package/dist/storage/classification.js +4 -1
- package/dist/storage/classification.js.map +1 -1
- package/dist/storage/content-guard.d.ts +26 -0
- package/dist/storage/content-guard.d.ts.map +1 -1
- package/dist/storage/content-guard.js +450 -48
- package/dist/storage/content-guard.js.map +1 -1
- package/dist/storage/corrections.d.ts.map +1 -1
- package/dist/storage/corrections.js +47 -66
- package/dist/storage/corrections.js.map +1 -1
- package/dist/storage/cwd-allowlist.d.ts +28 -2
- package/dist/storage/cwd-allowlist.d.ts.map +1 -1
- package/dist/storage/cwd-allowlist.js +61 -13
- package/dist/storage/cwd-allowlist.js.map +1 -1
- package/dist/storage/filelock.d.ts +26 -8
- package/dist/storage/filelock.d.ts.map +1 -1
- package/dist/storage/filelock.js +202 -26
- package/dist/storage/filelock.js.map +1 -1
- package/dist/storage/fs-utils.d.ts +51 -0
- package/dist/storage/fs-utils.d.ts.map +1 -1
- package/dist/storage/fs-utils.js +123 -2
- package/dist/storage/fs-utils.js.map +1 -1
- package/dist/storage/memory-protocol.js +55 -55
- package/dist/storage/paths.d.ts.map +1 -1
- package/dist/storage/paths.js +7 -2
- package/dist/storage/paths.js.map +1 -1
- package/dist/storage/project.d.ts.map +1 -1
- package/dist/storage/project.js +9 -2
- package/dist/storage/project.js.map +1 -1
- package/dist/storage/store-manifest.js +57 -57
- package/dist/supabase/embedding.d.ts.map +1 -1
- package/dist/supabase/embedding.js +7 -0
- package/dist/supabase/embedding.js.map +1 -1
- package/dist/supabase/sync.d.ts.map +1 -1
- package/dist/supabase/sync.js +42 -18
- package/dist/supabase/sync.js.map +1 -1
- package/dist/tools-logic/bootstrap.d.ts.map +1 -1
- package/dist/tools-logic/bootstrap.js +4 -2
- package/dist/tools-logic/bootstrap.js.map +1 -1
- package/dist/tools-logic/check.d.ts +7 -0
- package/dist/tools-logic/check.d.ts.map +1 -1
- package/dist/tools-logic/check.js +54 -44
- package/dist/tools-logic/check.js.map +1 -1
- package/dist/tools-logic/dream-admission.js +29 -29
- package/dist/tools-logic/recall-backend.d.ts.map +1 -1
- package/dist/tools-logic/recall-backend.js +3 -1
- package/dist/tools-logic/recall-backend.js.map +1 -1
- package/dist/tools-logic/smart-recall.d.ts +52 -0
- package/dist/tools-logic/smart-recall.d.ts.map +1 -1
- package/dist/tools-logic/smart-recall.js +202 -31
- package/dist/tools-logic/smart-recall.js.map +1 -1
- package/dist/tools-logic/smart-remember-vector.d.ts +3 -1
- package/dist/tools-logic/smart-remember-vector.d.ts.map +1 -1
- package/dist/tools-logic/smart-remember-vector.js +3 -1
- package/dist/tools-logic/smart-remember-vector.js.map +1 -1
- package/dist/tools-logic/smart-remember.d.ts.map +1 -1
- package/dist/tools-logic/smart-remember.js +6 -2
- package/dist/tools-logic/smart-remember.js.map +1 -1
- package/dist/types.d.ts +17 -2
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js +31 -3
- package/dist/types.js.map +1 -1
- package/dist/vector/embedding.d.ts +20 -1
- package/dist/vector/embedding.d.ts.map +1 -1
- package/dist/vector/embedding.js +27 -1
- package/dist/vector/embedding.js.map +1 -1
- package/dist/vector/local-vector-backend.d.ts +4 -1
- package/dist/vector/local-vector-backend.d.ts.map +1 -1
- package/dist/vector/local-vector-backend.js +7 -3
- package/dist/vector/local-vector-backend.js.map +1 -1
- package/package.json +59 -59
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* retrieval/lexical-index.ts — persistent, incremental lexical index for the
|
|
3
|
+
* journal tier (perf/lexical-index, 2026-10-16; reports/2026-10-15-rigorous-
|
|
4
|
+
* eval.md scale finding: 10k journal entries → recall p50 ~1.0-1.3 s, because
|
|
5
|
+
* `scoreJournalTier` re-reads and re-tokenizes the WHOLE corpus on every
|
|
6
|
+
* query — 10k `readFileSync`s + trust-tagging in the fetch stage alone
|
|
7
|
+
* measured ~1.2 s warm on win32; the full-corpus IDF tokenization adds
|
|
8
|
+
* another ~190 ms).
|
|
9
|
+
*
|
|
10
|
+
* WHAT IT STORES — per journal file, keyed by absolute path and validated by
|
|
11
|
+
* (size, mtimeNs): the DERIVED, query-independent data the journal scorer
|
|
12
|
+
* needs before it has to touch file content:
|
|
13
|
+
* - `u` — the rescue-quarantine trust bit (isRescueSourcedContent), so the
|
|
14
|
+
* trust FILTER no longer requires reading every file per query;
|
|
15
|
+
* - `t` — the file's unique stemmed token set (tokenizeWords + stem, the
|
|
16
|
+
* exact inputs `buildIdfModel`'s per-doc `tokenSet` is built from);
|
|
17
|
+
* - `r` — the file's unique lowercase whitespace-run set. Because every
|
|
18
|
+
* query keyword/variant is whitespace-free (tokenizeWords output),
|
|
19
|
+
* `content.toLowerCase().includes(term)` ⟺ some run contains
|
|
20
|
+
* `term` as a substring — this makes BOTH the per-line
|
|
21
|
+
* `lineMatchesQuery` prefilter AND `buildIdfModel`'s
|
|
22
|
+
* `d.lower.includes(v)` DF predicate answerable from the index
|
|
23
|
+
* with zero false negatives (a false POSITIVE only costs one
|
|
24
|
+
* file read that then produces no hits).
|
|
25
|
+
* Terms are stored once in a shared `vocab` table; docs reference vocab ids.
|
|
26
|
+
*
|
|
27
|
+
* WHAT IT NEVER STORES — file content, excerpts, scores, or anything
|
|
28
|
+
* query-dependent. Line-level hit generation still reads the real file (only
|
|
29
|
+
* for docs the prefilter says CAN match), so a stale/poisoned index can skew
|
|
30
|
+
* ranking weights but cannot fabricate content. Contents ARE cached
|
|
31
|
+
* IN-PROCESS (per (size, mtimeNs), pruned to the enumerated file set on each
|
|
32
|
+
* drift) — a long-lived MCP server holds roughly one copy of the hot
|
|
33
|
+
* project's journal text in RAM, the same order of memory the brute-force
|
|
34
|
+
* path already allocated transiently on EVERY query.
|
|
35
|
+
*
|
|
36
|
+
* BOUNDED CACHE (review fix, 2026-10-16, reports/2026-10-16-review-
|
|
37
|
+
* correctness.md MEDIUM): the in-process per-project states are an LRU —
|
|
38
|
+
* at most AGENT_RECALL_LEXICAL_MAX_PROJECTS project states (default 4) and
|
|
39
|
+
* at most AGENT_RECALL_LEXICAL_MAX_CONTENT_MB of cached content text
|
|
40
|
+
* (default 256 MB, UTF-16 accounting) are retained, least-recently-queried
|
|
41
|
+
* project evicted first; the project being queried is never evicted by its
|
|
42
|
+
* own call, so a single over-budget hot project keeps working (documented
|
|
43
|
+
* exemption, not a leak — it is one corpus, the brute path's own transient
|
|
44
|
+
* working set). Eviction only drops the IN-PROCESS cache: the next query for
|
|
45
|
+
* an evicted project reloads the persisted index from disk and lazily
|
|
46
|
+
* re-reads contents — results identical (pinned by the LRU/byte-budget
|
|
47
|
+
* tests). Bounds are enforced per loadJournalLexicalSnapshot call;
|
|
48
|
+
* trustedContent reads between calls can transiently exceed the byte budget
|
|
49
|
+
* by at most the active project's own corpus until the next query.
|
|
50
|
+
*
|
|
51
|
+
* TRUST (security): the cached `u` bit is an OPTIMIZATION, not the trust
|
|
52
|
+
* boundary. Every content read performed through this module re-runs
|
|
53
|
+
* `isRescueSourcedContent` on the bytes actually read (`trustedContent`
|
|
54
|
+
* below) and refuses to return rescue-tagged content — so a hand-poisoned
|
|
55
|
+
* index file (or the accepted same-(size,mtime)-different-content staleness
|
|
56
|
+
* window, same class the embedding index-store documents) can never SURFACE
|
|
57
|
+
* quarantined content; at worst it perturbs IDF document-frequency counts
|
|
58
|
+
* for files that are never read. An attacker who can write this index file
|
|
59
|
+
* lives inside the AgentRecall root and could already edit the journal
|
|
60
|
+
* files/frontmatter themselves — no new privilege is created (same residual
|
|
61
|
+
* class as storage/corrections.ts's planted-file note).
|
|
62
|
+
*
|
|
63
|
+
* INVALIDATION — every snapshot load re-enumerates the journal directories
|
|
64
|
+
* (listJournalFiles + the same archive readdir/filter the Wave-1 reader
|
|
65
|
+
* uses) and re-stats every enumerated file (bigint mtimeNs + size). A file
|
|
66
|
+
* created, deleted, or edited OUTSIDE the tool is therefore picked up on the
|
|
67
|
+
* very next query — there is no write-path hook to forget, and no TTL
|
|
68
|
+
* window. Stale/missing entries are re-read and re-derived inline (the
|
|
69
|
+
* incremental update), and the refreshed index is persisted.
|
|
70
|
+
*
|
|
71
|
+
* WRITES follow the embeddings/index-store.ts conventions: full payload
|
|
72
|
+
* built first, then written under `withLock("lexical-index-<project>")` via
|
|
73
|
+
* tmp+rename (writeJsonAtomic) — lock-free readers never observe a torn
|
|
74
|
+
* file; concurrent refreshes serialize. READS are lock-free and
|
|
75
|
+
* corruption-TOLERANT: any structural problem (truncated JSON from a killed
|
|
76
|
+
* writer's half-flushed tmp, garbage bytes, wrong version, malformed doc
|
|
77
|
+
* rows) degrades to "entry missing" → re-derive from the real files →
|
|
78
|
+
* rewrite. The recall path never throws from here (`loadJournalLexicalSnapshot`
|
|
79
|
+
* returns null on any unexpected failure, and scoreJournalTier falls back to
|
|
80
|
+
* the brute-force path).
|
|
81
|
+
*
|
|
82
|
+
* EQUIVALENCE — the consumer (query-memory.ts's scoreJournalTier) is built
|
|
83
|
+
* so the indexed path produces IDENTICAL rankings to the brute-force path:
|
|
84
|
+
* same candidate enumeration order (listJournalFiles live half; the same
|
|
85
|
+
* archive filter + date-desc stable sort as candidates.ts's rollup half),
|
|
86
|
+
* same trust filter, same line-scan code on the same content, and an IDF
|
|
87
|
+
* model assembled from index-derived DF counts that match buildIdfModel's
|
|
88
|
+
* predicate exactly (see the `r`/`t` notes above). Pinned by
|
|
89
|
+
* test/lexical-index.test.mjs's brute-vs-indexed deep-equality suite. The
|
|
90
|
+
* ONE documented divergence: a file deleted between this module's stat sweep
|
|
91
|
+
* and a (failed) content read stays counted in the IDF corpus size for that
|
|
92
|
+
* query, where brute force would have dropped it at fetch — a transient,
|
|
93
|
+
* mid-query-mutation-only window.
|
|
94
|
+
*/
|
|
95
|
+
/** Kill switch / test oracle toggle — default ON; "0"/"false"/"off" disables. */
|
|
96
|
+
export declare function lexicalIndexEnabled(): boolean;
|
|
97
|
+
/** The index file's on-disk location — built via projectSubPath so the
|
|
98
|
+
* sanitize + assertInsideRoot guards apply exactly once, like every other
|
|
99
|
+
* project-scoped path in this package. */
|
|
100
|
+
export declare function lexicalJournalIndexPath(project: string): string;
|
|
101
|
+
/** Reset all in-process caches (tests — simulates a fresh process). */
|
|
102
|
+
export declare function resetLexicalIndexCaches(): void;
|
|
103
|
+
/** Introspection for tests / a future `ar` doctor surface: how many project
|
|
104
|
+
* states are resident and how much content text they cache (UTF-16 bytes). */
|
|
105
|
+
export declare function lexicalIndexCacheStats(): {
|
|
106
|
+
states: number;
|
|
107
|
+
contentBytes: number;
|
|
108
|
+
};
|
|
109
|
+
export interface LexicalSnapshotDoc {
|
|
110
|
+
/** Filename date prefix for live/archive docs; "" when undeterminable (same semantics as MemoryCandidate.date for the journal tier). */
|
|
111
|
+
date: string;
|
|
112
|
+
/** Basename. */
|
|
113
|
+
file: string;
|
|
114
|
+
/** Absolute path. */
|
|
115
|
+
path: string;
|
|
116
|
+
/** Cached rescue-quarantine bit (content reads re-verify; see module header). */
|
|
117
|
+
untrusted: boolean;
|
|
118
|
+
}
|
|
119
|
+
export interface LexicalJournalSnapshot {
|
|
120
|
+
/** Docs in the EXACT order the brute-force journal fetch would produce
|
|
121
|
+
* (live half via listJournalFiles' own date-desc order, then the rollup-
|
|
122
|
+
* archive half date-desc) — hit-generation order, and therefore stable-
|
|
123
|
+
* sort tie-breaks, match brute force by construction. */
|
|
124
|
+
docs: ReadonlyArray<LexicalSnapshotDoc>;
|
|
125
|
+
/** Indices of docs whose lowercase content contains `term` as a substring
|
|
126
|
+
* (term must be whitespace-free — every tokenizeWords/expandQuery output
|
|
127
|
+
* is). Includes untrusted docs; callers filter. Memoized per snapshot. */
|
|
128
|
+
docsContaining(term: string): ReadonlySet<number>;
|
|
129
|
+
/** Indices of docs whose stemmed token set contains `term` exactly. */
|
|
130
|
+
docsWithStemToken(term: string): ReadonlySet<number>;
|
|
131
|
+
/** The doc's content, re-verified through the rescue-quarantine choke at
|
|
132
|
+
* read time — returns null for unreadable files AND for content that is
|
|
133
|
+
* rescue-tagged on the actual bytes read (whatever the cached bit says). */
|
|
134
|
+
trustedContent(i: number): string | null;
|
|
135
|
+
}
|
|
136
|
+
/**
|
|
137
|
+
* Load (building/refreshing as needed) the lexical snapshot for a project's
|
|
138
|
+
* journal tier. Returns null when the index is disabled or anything
|
|
139
|
+
* unexpected fails — the caller falls back to the brute-force path.
|
|
140
|
+
*
|
|
141
|
+
* Per-call work when nothing changed: one readdir pass per journal dir +
|
|
142
|
+
* one statSync per enumerated file (the external-edit detection the module
|
|
143
|
+
* header documents — ~115 ms for 10k files on win32, measured) + an
|
|
144
|
+
* elementwise fingerprint compare. Derivation/persist run only for drifted
|
|
145
|
+
* files.
|
|
146
|
+
*/
|
|
147
|
+
export declare function loadJournalLexicalSnapshot(project: string, opts: {
|
|
148
|
+
includeRollupArchive: boolean;
|
|
149
|
+
}): Promise<LexicalJournalSnapshot | null>;
|
|
150
|
+
//# sourceMappingURL=lexical-index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"lexical-index.d.ts","sourceRoot":"","sources":["../../src/retrieval/lexical-index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6FG;AAeH,iFAAiF;AACjF,wBAAgB,mBAAmB,IAAI,OAAO,CAG7C;AAED;;2CAE2C;AAC3C,wBAAgB,uBAAuB,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAE/D;AA8HD,uEAAuE;AACvE,wBAAgB,uBAAuB,IAAI,IAAI,CAE9C;AAED;+EAC+E;AAC/E,wBAAgB,sBAAsB,IAAI;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,YAAY,EAAE,MAAM,CAAA;CAAE,CAIjF;AAMD,MAAM,WAAW,kBAAkB;IACjC,wIAAwI;IACxI,IAAI,EAAE,MAAM,CAAC;IACb,gBAAgB;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,qBAAqB;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,iFAAiF;IACjF,SAAS,EAAE,OAAO,CAAC;CACpB;AAED,MAAM,WAAW,sBAAsB;IACrC;;;8DAG0D;IAC1D,IAAI,EAAE,aAAa,CAAC,kBAAkB,CAAC,CAAC;IACxC;;+EAE2E;IAC3E,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC;IAClD,uEAAuE;IACvE,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC;IACrD;;iFAE6E;IAC7E,cAAc,CAAC,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAAC;CAC1C;AAgHD;;;;;;;;;;GAUG;AACH,wBAAsB,0BAA0B,CAC9C,OAAO,EAAE,MAAM,EACf,IAAI,EAAE;IAAE,oBAAoB,EAAE,OAAO,CAAA;CAAE,GACtC,OAAO,CAAC,sBAAsB,GAAG,IAAI,CAAC,CA2KxC"}
|
|
@@ -0,0 +1,514 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* retrieval/lexical-index.ts — persistent, incremental lexical index for the
|
|
3
|
+
* journal tier (perf/lexical-index, 2026-10-16; reports/2026-10-15-rigorous-
|
|
4
|
+
* eval.md scale finding: 10k journal entries → recall p50 ~1.0-1.3 s, because
|
|
5
|
+
* `scoreJournalTier` re-reads and re-tokenizes the WHOLE corpus on every
|
|
6
|
+
* query — 10k `readFileSync`s + trust-tagging in the fetch stage alone
|
|
7
|
+
* measured ~1.2 s warm on win32; the full-corpus IDF tokenization adds
|
|
8
|
+
* another ~190 ms).
|
|
9
|
+
*
|
|
10
|
+
* WHAT IT STORES — per journal file, keyed by absolute path and validated by
|
|
11
|
+
* (size, mtimeNs): the DERIVED, query-independent data the journal scorer
|
|
12
|
+
* needs before it has to touch file content:
|
|
13
|
+
* - `u` — the rescue-quarantine trust bit (isRescueSourcedContent), so the
|
|
14
|
+
* trust FILTER no longer requires reading every file per query;
|
|
15
|
+
* - `t` — the file's unique stemmed token set (tokenizeWords + stem, the
|
|
16
|
+
* exact inputs `buildIdfModel`'s per-doc `tokenSet` is built from);
|
|
17
|
+
* - `r` — the file's unique lowercase whitespace-run set. Because every
|
|
18
|
+
* query keyword/variant is whitespace-free (tokenizeWords output),
|
|
19
|
+
* `content.toLowerCase().includes(term)` ⟺ some run contains
|
|
20
|
+
* `term` as a substring — this makes BOTH the per-line
|
|
21
|
+
* `lineMatchesQuery` prefilter AND `buildIdfModel`'s
|
|
22
|
+
* `d.lower.includes(v)` DF predicate answerable from the index
|
|
23
|
+
* with zero false negatives (a false POSITIVE only costs one
|
|
24
|
+
* file read that then produces no hits).
|
|
25
|
+
* Terms are stored once in a shared `vocab` table; docs reference vocab ids.
|
|
26
|
+
*
|
|
27
|
+
* WHAT IT NEVER STORES — file content, excerpts, scores, or anything
|
|
28
|
+
* query-dependent. Line-level hit generation still reads the real file (only
|
|
29
|
+
* for docs the prefilter says CAN match), so a stale/poisoned index can skew
|
|
30
|
+
* ranking weights but cannot fabricate content. Contents ARE cached
|
|
31
|
+
* IN-PROCESS (per (size, mtimeNs), pruned to the enumerated file set on each
|
|
32
|
+
* drift) — a long-lived MCP server holds roughly one copy of the hot
|
|
33
|
+
* project's journal text in RAM, the same order of memory the brute-force
|
|
34
|
+
* path already allocated transiently on EVERY query.
|
|
35
|
+
*
|
|
36
|
+
* BOUNDED CACHE (review fix, 2026-10-16, reports/2026-10-16-review-
|
|
37
|
+
* correctness.md MEDIUM): the in-process per-project states are an LRU —
|
|
38
|
+
* at most AGENT_RECALL_LEXICAL_MAX_PROJECTS project states (default 4) and
|
|
39
|
+
* at most AGENT_RECALL_LEXICAL_MAX_CONTENT_MB of cached content text
|
|
40
|
+
* (default 256 MB, UTF-16 accounting) are retained, least-recently-queried
|
|
41
|
+
* project evicted first; the project being queried is never evicted by its
|
|
42
|
+
* own call, so a single over-budget hot project keeps working (documented
|
|
43
|
+
* exemption, not a leak — it is one corpus, the brute path's own transient
|
|
44
|
+
* working set). Eviction only drops the IN-PROCESS cache: the next query for
|
|
45
|
+
* an evicted project reloads the persisted index from disk and lazily
|
|
46
|
+
* re-reads contents — results identical (pinned by the LRU/byte-budget
|
|
47
|
+
* tests). Bounds are enforced per loadJournalLexicalSnapshot call;
|
|
48
|
+
* trustedContent reads between calls can transiently exceed the byte budget
|
|
49
|
+
* by at most the active project's own corpus until the next query.
|
|
50
|
+
*
|
|
51
|
+
* TRUST (security): the cached `u` bit is an OPTIMIZATION, not the trust
|
|
52
|
+
* boundary. Every content read performed through this module re-runs
|
|
53
|
+
* `isRescueSourcedContent` on the bytes actually read (`trustedContent`
|
|
54
|
+
* below) and refuses to return rescue-tagged content — so a hand-poisoned
|
|
55
|
+
* index file (or the accepted same-(size,mtime)-different-content staleness
|
|
56
|
+
* window, same class the embedding index-store documents) can never SURFACE
|
|
57
|
+
* quarantined content; at worst it perturbs IDF document-frequency counts
|
|
58
|
+
* for files that are never read. An attacker who can write this index file
|
|
59
|
+
* lives inside the AgentRecall root and could already edit the journal
|
|
60
|
+
* files/frontmatter themselves — no new privilege is created (same residual
|
|
61
|
+
* class as storage/corrections.ts's planted-file note).
|
|
62
|
+
*
|
|
63
|
+
* INVALIDATION — every snapshot load re-enumerates the journal directories
|
|
64
|
+
* (listJournalFiles + the same archive readdir/filter the Wave-1 reader
|
|
65
|
+
* uses) and re-stats every enumerated file (bigint mtimeNs + size). A file
|
|
66
|
+
* created, deleted, or edited OUTSIDE the tool is therefore picked up on the
|
|
67
|
+
* very next query — there is no write-path hook to forget, and no TTL
|
|
68
|
+
* window. Stale/missing entries are re-read and re-derived inline (the
|
|
69
|
+
* incremental update), and the refreshed index is persisted.
|
|
70
|
+
*
|
|
71
|
+
* WRITES follow the embeddings/index-store.ts conventions: full payload
|
|
72
|
+
* built first, then written under `withLock("lexical-index-<project>")` via
|
|
73
|
+
* tmp+rename (writeJsonAtomic) — lock-free readers never observe a torn
|
|
74
|
+
* file; concurrent refreshes serialize. READS are lock-free and
|
|
75
|
+
* corruption-TOLERANT: any structural problem (truncated JSON from a killed
|
|
76
|
+
* writer's half-flushed tmp, garbage bytes, wrong version, malformed doc
|
|
77
|
+
* rows) degrades to "entry missing" → re-derive from the real files →
|
|
78
|
+
* rewrite. The recall path never throws from here (`loadJournalLexicalSnapshot`
|
|
79
|
+
* returns null on any unexpected failure, and scoreJournalTier falls back to
|
|
80
|
+
* the brute-force path).
|
|
81
|
+
*
|
|
82
|
+
* EQUIVALENCE — the consumer (query-memory.ts's scoreJournalTier) is built
|
|
83
|
+
* so the indexed path produces IDENTICAL rankings to the brute-force path:
|
|
84
|
+
* same candidate enumeration order (listJournalFiles live half; the same
|
|
85
|
+
* archive filter + date-desc stable sort as candidates.ts's rollup half),
|
|
86
|
+
* same trust filter, same line-scan code on the same content, and an IDF
|
|
87
|
+
* model assembled from index-derived DF counts that match buildIdfModel's
|
|
88
|
+
* predicate exactly (see the `r`/`t` notes above). Pinned by
|
|
89
|
+
* test/lexical-index.test.mjs's brute-vs-indexed deep-equality suite. The
|
|
90
|
+
* ONE documented divergence: a file deleted between this module's stat sweep
|
|
91
|
+
* and a (failed) content read stays counted in the IDF corpus size for that
|
|
92
|
+
* query, where brute force would have dropped it at fetch — a transient,
|
|
93
|
+
* mid-query-mutation-only window.
|
|
94
|
+
*/
|
|
95
|
+
import * as fs from "node:fs";
|
|
96
|
+
import * as path from "node:path";
|
|
97
|
+
import { projectSubPath, journalDir } from "../storage/paths.js";
|
|
98
|
+
import { listJournalFiles } from "../helpers/journal-files.js";
|
|
99
|
+
import { isRescueSourcedContent } from "../helpers/journal-filter.js";
|
|
100
|
+
import { tokenizeWords } from "../helpers/tokenize.js";
|
|
101
|
+
import { stem } from "../helpers/normalize.js";
|
|
102
|
+
import { withLock } from "../storage/filelock.js";
|
|
103
|
+
import { writeJsonAtomic } from "../storage/fs-utils.js";
|
|
104
|
+
const INDEX_VERSION = 1;
|
|
105
|
+
const INDEX_FILENAME = "lexical-journal-v1.json";
|
|
106
|
+
/** Kill switch / test oracle toggle — default ON; "0"/"false"/"off" disables. */
|
|
107
|
+
export function lexicalIndexEnabled() {
|
|
108
|
+
const v = (process.env.AGENT_RECALL_LEXICAL_INDEX ?? "").toLowerCase();
|
|
109
|
+
return !(v === "0" || v === "false" || v === "off");
|
|
110
|
+
}
|
|
111
|
+
/** The index file's on-disk location — built via projectSubPath so the
|
|
112
|
+
* sanitize + assertInsideRoot guards apply exactly once, like every other
|
|
113
|
+
* project-scoped path in this package. */
|
|
114
|
+
export function lexicalJournalIndexPath(project) {
|
|
115
|
+
return projectSubPath(project, "index", INDEX_FILENAME);
|
|
116
|
+
}
|
|
117
|
+
/** Insertion order == LRU order: every loadJournalLexicalSnapshot call
|
|
118
|
+
* delete+set-refreshes its key, so the FIRST key is always the
|
|
119
|
+
* least-recently-queried project (see enforceCacheBounds). */
|
|
120
|
+
const states = new Map();
|
|
121
|
+
const DEFAULT_MAX_PROJECT_STATES = 4;
|
|
122
|
+
const DEFAULT_MAX_CONTENT_MB = 256;
|
|
123
|
+
/** Positive-number env override with a safe fallback — config parsing only,
|
|
124
|
+
* never fed to a path/RegExp/fs call. */
|
|
125
|
+
function envPositive(name, fallback) {
|
|
126
|
+
const raw = process.env[name];
|
|
127
|
+
if (raw === undefined || raw === "")
|
|
128
|
+
return fallback;
|
|
129
|
+
const n = Number(raw);
|
|
130
|
+
return Number.isFinite(n) && n > 0 ? n : fallback;
|
|
131
|
+
}
|
|
132
|
+
function maxProjectStates() {
|
|
133
|
+
return envPositive("AGENT_RECALL_LEXICAL_MAX_PROJECTS", DEFAULT_MAX_PROJECT_STATES);
|
|
134
|
+
}
|
|
135
|
+
function maxContentBytes() {
|
|
136
|
+
return envPositive("AGENT_RECALL_LEXICAL_MAX_CONTENT_MB", DEFAULT_MAX_CONTENT_MB) * 1024 * 1024;
|
|
137
|
+
}
|
|
138
|
+
/** The ONLY mutators of a state's contents map — keep contentBytes exact. */
|
|
139
|
+
function cacheContent(state, p, rec) {
|
|
140
|
+
const prev = state.contents.get(p);
|
|
141
|
+
if (prev)
|
|
142
|
+
state.contentBytes -= prev.c.length * 2;
|
|
143
|
+
state.contents.set(p, rec);
|
|
144
|
+
state.contentBytes += rec.c.length * 2;
|
|
145
|
+
}
|
|
146
|
+
function dropContent(state, p) {
|
|
147
|
+
const prev = state.contents.get(p);
|
|
148
|
+
if (!prev)
|
|
149
|
+
return;
|
|
150
|
+
state.contentBytes -= prev.c.length * 2;
|
|
151
|
+
state.contents.delete(p);
|
|
152
|
+
}
|
|
153
|
+
/**
|
|
154
|
+
* Evict least-recently-queried project states beyond the project-count cap,
|
|
155
|
+
* then beyond the content-byte budget — never the project currently being
|
|
156
|
+
* queried (`activeKey`). Eviction is cache-only and loss-free: dirty entries
|
|
157
|
+
* are always persisted inside the same load call that derived them (before
|
|
158
|
+
* bounds run), so an evicted project's next query reloads the on-disk index
|
|
159
|
+
* and produces identical results (pinned by the bounded-cache tests).
|
|
160
|
+
*/
|
|
161
|
+
function enforceCacheBounds(activeKey) {
|
|
162
|
+
const cap = maxProjectStates();
|
|
163
|
+
for (const key of states.keys()) {
|
|
164
|
+
if (states.size <= cap)
|
|
165
|
+
break;
|
|
166
|
+
if (key === activeKey)
|
|
167
|
+
continue;
|
|
168
|
+
states.delete(key);
|
|
169
|
+
}
|
|
170
|
+
const budget = maxContentBytes();
|
|
171
|
+
let total = 0;
|
|
172
|
+
for (const s of states.values())
|
|
173
|
+
total += s.contentBytes;
|
|
174
|
+
for (const [key, s] of states) {
|
|
175
|
+
if (total <= budget)
|
|
176
|
+
break;
|
|
177
|
+
if (key === activeKey)
|
|
178
|
+
continue;
|
|
179
|
+
total -= s.contentBytes;
|
|
180
|
+
states.delete(key);
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
/** Reset all in-process caches (tests — simulates a fresh process). */
|
|
184
|
+
export function resetLexicalIndexCaches() {
|
|
185
|
+
states.clear();
|
|
186
|
+
}
|
|
187
|
+
/** Introspection for tests / a future `ar` doctor surface: how many project
|
|
188
|
+
* states are resident and how much content text they cache (UTF-16 bytes). */
|
|
189
|
+
export function lexicalIndexCacheStats() {
|
|
190
|
+
let contentBytes = 0;
|
|
191
|
+
for (const s of states.values())
|
|
192
|
+
contentBytes += s.contentBytes;
|
|
193
|
+
return { states: states.size, contentBytes };
|
|
194
|
+
}
|
|
195
|
+
/** Derive the query-independent term data for one file's content.
|
|
196
|
+
* MUST stay byte-aligned with query-memory.ts's brute-force predicates:
|
|
197
|
+
* runs ⇒ `content.toLowerCase().includes(term)` for whitespace-free terms;
|
|
198
|
+
* stems ⇒ `new Set(tokenizeWords(content).map(stem))`. */
|
|
199
|
+
function deriveTerms(content) {
|
|
200
|
+
const runSet = new Set();
|
|
201
|
+
for (const r of content.toLowerCase().split(/\s+/)) {
|
|
202
|
+
if (r.length > 0)
|
|
203
|
+
runSet.add(r);
|
|
204
|
+
}
|
|
205
|
+
const stemSet = new Set();
|
|
206
|
+
for (const t of tokenizeWords(content))
|
|
207
|
+
stemSet.add(stem(t));
|
|
208
|
+
return { runs: [...runSet], stems: [...stemSet] };
|
|
209
|
+
}
|
|
210
|
+
/** Parse + validate a persisted index into an entries map. Any structural
|
|
211
|
+
* problem (garbage, truncation, wrong version, malformed row) degrades to
|
|
212
|
+
* "fewer cached entries" — never throws, never trusts shape. */
|
|
213
|
+
function loadPersistedEntries(indexPath) {
|
|
214
|
+
const entries = new Map();
|
|
215
|
+
let raw;
|
|
216
|
+
try {
|
|
217
|
+
raw = fs.readFileSync(indexPath, "utf-8");
|
|
218
|
+
}
|
|
219
|
+
catch {
|
|
220
|
+
return entries; // missing — first run
|
|
221
|
+
}
|
|
222
|
+
let parsed;
|
|
223
|
+
try {
|
|
224
|
+
parsed = JSON.parse(raw);
|
|
225
|
+
}
|
|
226
|
+
catch {
|
|
227
|
+
return entries; // corrupt/truncated — rebuild from files
|
|
228
|
+
}
|
|
229
|
+
const idx = parsed;
|
|
230
|
+
if (!idx || idx.version !== INDEX_VERSION || !Array.isArray(idx.vocab) || !Array.isArray(idx.docs)) {
|
|
231
|
+
return entries;
|
|
232
|
+
}
|
|
233
|
+
const vocab = idx.vocab;
|
|
234
|
+
const term = (id) => typeof id === "number" && Number.isInteger(id) && id >= 0 && id < vocab.length && typeof vocab[id] === "string"
|
|
235
|
+
? vocab[id]
|
|
236
|
+
: null;
|
|
237
|
+
for (const d of idx.docs) {
|
|
238
|
+
if (!d || typeof d.k !== "string" || typeof d.s !== "number" || !Number.isFinite(d.s))
|
|
239
|
+
continue;
|
|
240
|
+
if (typeof d.m !== "string" || !/^\d+$/.test(d.m))
|
|
241
|
+
continue;
|
|
242
|
+
if ((d.u !== 0 && d.u !== 1) || !Array.isArray(d.r) || !Array.isArray(d.t))
|
|
243
|
+
continue;
|
|
244
|
+
const runs = [];
|
|
245
|
+
const stems = [];
|
|
246
|
+
let ok = true;
|
|
247
|
+
for (const id of d.r) {
|
|
248
|
+
const t = term(id);
|
|
249
|
+
if (t === null) {
|
|
250
|
+
ok = false;
|
|
251
|
+
break;
|
|
252
|
+
}
|
|
253
|
+
runs.push(t);
|
|
254
|
+
}
|
|
255
|
+
if (!ok)
|
|
256
|
+
continue;
|
|
257
|
+
for (const id of d.t) {
|
|
258
|
+
const t = term(id);
|
|
259
|
+
if (t === null) {
|
|
260
|
+
ok = false;
|
|
261
|
+
break;
|
|
262
|
+
}
|
|
263
|
+
stems.push(t);
|
|
264
|
+
}
|
|
265
|
+
if (!ok)
|
|
266
|
+
continue;
|
|
267
|
+
entries.set(d.k, { s: d.s, m: BigInt(d.m), u: d.u === 1, runs, stems });
|
|
268
|
+
}
|
|
269
|
+
return entries;
|
|
270
|
+
}
|
|
271
|
+
/** Serialize entries (vocab-table form) and persist atomically under the
|
|
272
|
+
* project's own lock. Best-effort: the index is a cache — a failed persist
|
|
273
|
+
* must never fail the query that triggered it. */
|
|
274
|
+
async function persistEntries(project, indexPath, entries) {
|
|
275
|
+
const vocab = [];
|
|
276
|
+
const idByTerm = new Map();
|
|
277
|
+
const idOf = (t) => {
|
|
278
|
+
let id = idByTerm.get(t);
|
|
279
|
+
if (id === undefined) {
|
|
280
|
+
id = vocab.length;
|
|
281
|
+
vocab.push(t);
|
|
282
|
+
idByTerm.set(t, id);
|
|
283
|
+
}
|
|
284
|
+
return id;
|
|
285
|
+
};
|
|
286
|
+
const docs = [];
|
|
287
|
+
for (const [k, e] of entries) {
|
|
288
|
+
docs.push({
|
|
289
|
+
k,
|
|
290
|
+
s: e.s,
|
|
291
|
+
m: e.m.toString(),
|
|
292
|
+
u: e.u ? 1 : 0,
|
|
293
|
+
r: e.runs.map(idOf),
|
|
294
|
+
t: e.stems.map(idOf),
|
|
295
|
+
});
|
|
296
|
+
}
|
|
297
|
+
const payload = { version: INDEX_VERSION, builtAt: new Date().toISOString(), vocab, docs };
|
|
298
|
+
try {
|
|
299
|
+
await withLock(`lexical-index-${project}`, () => {
|
|
300
|
+
writeJsonAtomic(indexPath, payload);
|
|
301
|
+
});
|
|
302
|
+
}
|
|
303
|
+
catch {
|
|
304
|
+
// Lock contention / fs failure — skip; next drifted query retries.
|
|
305
|
+
}
|
|
306
|
+
}
|
|
307
|
+
/** Same-enumeration check against the cached snapshot (cheap elementwise
|
|
308
|
+
* compare of the freshly-statted (path, size, mtimeNs) triples). */
|
|
309
|
+
function sameScan(a, b) {
|
|
310
|
+
if (a.length !== b.length)
|
|
311
|
+
return false;
|
|
312
|
+
for (let i = 0; i < a.length; i++) {
|
|
313
|
+
if (a[i].path !== b[i].path || a[i].s !== b[i].s || a[i].m !== b[i].m)
|
|
314
|
+
return false;
|
|
315
|
+
}
|
|
316
|
+
return true;
|
|
317
|
+
}
|
|
318
|
+
/**
|
|
319
|
+
* Load (building/refreshing as needed) the lexical snapshot for a project's
|
|
320
|
+
* journal tier. Returns null when the index is disabled or anything
|
|
321
|
+
* unexpected fails — the caller falls back to the brute-force path.
|
|
322
|
+
*
|
|
323
|
+
* Per-call work when nothing changed: one readdir pass per journal dir +
|
|
324
|
+
* one statSync per enumerated file (the external-edit detection the module
|
|
325
|
+
* header documents — ~115 ms for 10k files on win32, measured) + an
|
|
326
|
+
* elementwise fingerprint compare. Derivation/persist run only for drifted
|
|
327
|
+
* files.
|
|
328
|
+
*/
|
|
329
|
+
export async function loadJournalLexicalSnapshot(project, opts) {
|
|
330
|
+
if (!lexicalIndexEnabled())
|
|
331
|
+
return null;
|
|
332
|
+
try {
|
|
333
|
+
const indexPath = lexicalJournalIndexPath(project);
|
|
334
|
+
let state = states.get(indexPath);
|
|
335
|
+
if (!state) {
|
|
336
|
+
state = { entries: loadPersistedEntries(indexPath), contents: new Map(), contentBytes: 0 };
|
|
337
|
+
}
|
|
338
|
+
else {
|
|
339
|
+
states.delete(indexPath); // LRU refresh — re-inserted below as most recent
|
|
340
|
+
}
|
|
341
|
+
states.set(indexPath, state);
|
|
342
|
+
// ---- enumerate (must mirror candidates.ts's readJournalCandidates) ----
|
|
343
|
+
// Live half: listJournalFiles(project, false) — the SAME shared reader the
|
|
344
|
+
// brute-force fetch's live half delegates to, so ordering and inclusion
|
|
345
|
+
// rules are inherited, not re-implemented.
|
|
346
|
+
const live = listJournalFiles(project, false);
|
|
347
|
+
const archiveDir = path.join(journalDir(project), "archive");
|
|
348
|
+
let archiveNames = [];
|
|
349
|
+
if (opts.includeRollupArchive) {
|
|
350
|
+
try {
|
|
351
|
+
// Same filter as candidates.ts's rollup-archive half (index.md + the
|
|
352
|
+
// `_`-reserved namespace excluded by name).
|
|
353
|
+
archiveNames = fs.readdirSync(archiveDir).filter((f) => f.endsWith(".md") && f !== "index.md" && !f.startsWith("_"));
|
|
354
|
+
}
|
|
355
|
+
catch {
|
|
356
|
+
archiveNames = [];
|
|
357
|
+
}
|
|
358
|
+
}
|
|
359
|
+
// ---- stat sweep (bigint mtimeNs — external-edit detection) ----
|
|
360
|
+
const scanned = [];
|
|
361
|
+
for (const entry of live) {
|
|
362
|
+
const p = path.join(entry.dir, entry.file);
|
|
363
|
+
const st = fs.statSync(p, { bigint: true, throwIfNoEntry: false });
|
|
364
|
+
if (!st)
|
|
365
|
+
continue;
|
|
366
|
+
scanned.push({ path: p, file: entry.file, date: entry.date, s: Number(st.size), m: st.mtimeNs });
|
|
367
|
+
}
|
|
368
|
+
const archiveScanned = [];
|
|
369
|
+
for (const file of archiveNames) {
|
|
370
|
+
const p = path.join(archiveDir, file);
|
|
371
|
+
const st = fs.statSync(p, { bigint: true, throwIfNoEntry: false });
|
|
372
|
+
if (!st)
|
|
373
|
+
continue;
|
|
374
|
+
const dateMatch = file.match(/^(\d{4}-\d{2}-\d{2})/);
|
|
375
|
+
archiveScanned.push({ path: p, file, date: dateMatch ? dateMatch[1] : "", s: Number(st.size), m: st.mtimeNs });
|
|
376
|
+
}
|
|
377
|
+
// Stable date-desc sort — same comparator (and same tie behavior: stable
|
|
378
|
+
// sort over readdir enumeration order) as the brute half's
|
|
379
|
+
// archiveCandidates.sort((a, b) => b.date.localeCompare(a.date)).
|
|
380
|
+
archiveScanned.sort((a, b) => b.date.localeCompare(a.date));
|
|
381
|
+
scanned.push(...archiveScanned);
|
|
382
|
+
// ---- unchanged? serve the cached snapshot (postings + memos intact) ----
|
|
383
|
+
if (state.snap && state.snap.includeRollupArchive === opts.includeRollupArchive && sameScan(state.snap.scanned, scanned)) {
|
|
384
|
+
enforceCacheBounds(indexPath);
|
|
385
|
+
return state.snap.snapshot;
|
|
386
|
+
}
|
|
387
|
+
// ---- incremental refresh: (re-)derive stale/missing entries ----
|
|
388
|
+
let dirty = false;
|
|
389
|
+
const docs = [];
|
|
390
|
+
const docEntries = [];
|
|
391
|
+
for (const sc of scanned) {
|
|
392
|
+
let e = state.entries.get(sc.path);
|
|
393
|
+
if (!e || e.s !== sc.s || e.m !== sc.m) {
|
|
394
|
+
let content;
|
|
395
|
+
try {
|
|
396
|
+
content = fs.readFileSync(sc.path, "utf-8");
|
|
397
|
+
}
|
|
398
|
+
catch {
|
|
399
|
+
state.entries.delete(sc.path);
|
|
400
|
+
dirty = true;
|
|
401
|
+
continue; // vanished/unreadable between stat and read — skip, like brute force's safeReadFile
|
|
402
|
+
}
|
|
403
|
+
const terms = deriveTerms(content);
|
|
404
|
+
// THE trust choke, applied at derive time on the bytes just read —
|
|
405
|
+
// the same isRescueSourcedContent every sibling journal reader calls.
|
|
406
|
+
e = { s: sc.s, m: sc.m, u: isRescueSourcedContent(content), runs: terms.runs, stems: terms.stems };
|
|
407
|
+
state.entries.set(sc.path, e);
|
|
408
|
+
cacheContent(state, sc.path, { s: sc.s, m: sc.m, c: content });
|
|
409
|
+
dirty = true;
|
|
410
|
+
}
|
|
411
|
+
docs.push({ date: sc.date, file: sc.file, path: sc.path, untrusted: e.u });
|
|
412
|
+
docEntries.push(e);
|
|
413
|
+
}
|
|
414
|
+
// ---- prune deleted files + persist when anything drifted ----
|
|
415
|
+
// Kept set = everything enumerated this scan, PLUS (when the archive was
|
|
416
|
+
// NOT enumerated this call) previously-known archive-scope entries — so a
|
|
417
|
+
// rare includeRollupArchive:false query never clobbers the archive half.
|
|
418
|
+
const keep = new Map();
|
|
419
|
+
for (let i = 0; i < docs.length; i++)
|
|
420
|
+
keep.set(docs[i].path, docEntries[i]);
|
|
421
|
+
if (!opts.includeRollupArchive) {
|
|
422
|
+
const archivePrefix = archiveDir + path.sep;
|
|
423
|
+
for (const [k, e] of state.entries) {
|
|
424
|
+
if (k.startsWith(archivePrefix) && !keep.has(k))
|
|
425
|
+
keep.set(k, e);
|
|
426
|
+
}
|
|
427
|
+
}
|
|
428
|
+
if (keep.size !== state.entries.size)
|
|
429
|
+
dirty = true;
|
|
430
|
+
state.entries = keep;
|
|
431
|
+
for (const k of [...state.contents.keys()]) {
|
|
432
|
+
if (!keep.has(k))
|
|
433
|
+
dropContent(state, k);
|
|
434
|
+
}
|
|
435
|
+
if (dirty)
|
|
436
|
+
await persistEntries(project, indexPath, keep);
|
|
437
|
+
// ---- build snapshot structures (postings + memoized lookups) ----
|
|
438
|
+
const runPostings = new Map();
|
|
439
|
+
const stemPostings = new Map();
|
|
440
|
+
for (let i = 0; i < docEntries.length; i++) {
|
|
441
|
+
for (const t of docEntries[i].runs) {
|
|
442
|
+
let arr = runPostings.get(t);
|
|
443
|
+
if (!arr) {
|
|
444
|
+
arr = [];
|
|
445
|
+
runPostings.set(t, arr);
|
|
446
|
+
}
|
|
447
|
+
arr.push(i);
|
|
448
|
+
}
|
|
449
|
+
for (const t of docEntries[i].stems) {
|
|
450
|
+
let arr = stemPostings.get(t);
|
|
451
|
+
if (!arr) {
|
|
452
|
+
arr = [];
|
|
453
|
+
stemPostings.set(t, arr);
|
|
454
|
+
}
|
|
455
|
+
arr.push(i);
|
|
456
|
+
}
|
|
457
|
+
}
|
|
458
|
+
const substringMemo = new Map();
|
|
459
|
+
const stemMemo = new Map();
|
|
460
|
+
const st = state; // narrow for the closures below
|
|
461
|
+
const snapshot = {
|
|
462
|
+
docs,
|
|
463
|
+
docsContaining(term) {
|
|
464
|
+
let got = substringMemo.get(term);
|
|
465
|
+
if (!got) {
|
|
466
|
+
got = new Set();
|
|
467
|
+
for (const [t, posting] of runPostings) {
|
|
468
|
+
if (t.includes(term)) {
|
|
469
|
+
for (const i of posting)
|
|
470
|
+
got.add(i);
|
|
471
|
+
}
|
|
472
|
+
}
|
|
473
|
+
substringMemo.set(term, got);
|
|
474
|
+
}
|
|
475
|
+
return got;
|
|
476
|
+
},
|
|
477
|
+
docsWithStemToken(term) {
|
|
478
|
+
let got = stemMemo.get(term);
|
|
479
|
+
if (!got) {
|
|
480
|
+
const posting = stemPostings.get(term);
|
|
481
|
+
got = posting ? new Set(posting) : new Set();
|
|
482
|
+
stemMemo.set(term, got);
|
|
483
|
+
}
|
|
484
|
+
return got;
|
|
485
|
+
},
|
|
486
|
+
trustedContent(i) {
|
|
487
|
+
const d = docs[i];
|
|
488
|
+
let rec = st.contents.get(d.path);
|
|
489
|
+
if (!rec || rec.s !== docEntries[i].s || rec.m !== docEntries[i].m) {
|
|
490
|
+
try {
|
|
491
|
+
rec = { s: docEntries[i].s, m: docEntries[i].m, c: fs.readFileSync(d.path, "utf-8") };
|
|
492
|
+
}
|
|
493
|
+
catch {
|
|
494
|
+
return null;
|
|
495
|
+
}
|
|
496
|
+
cacheContent(st, d.path, rec);
|
|
497
|
+
}
|
|
498
|
+
// Read-time re-verification through the SAME choke — a poisoned/stale
|
|
499
|
+
// cached trust bit can never surface quarantined content (see TRUST
|
|
500
|
+
// in the module header).
|
|
501
|
+
if (isRescueSourcedContent(rec.c))
|
|
502
|
+
return null;
|
|
503
|
+
return rec.c;
|
|
504
|
+
},
|
|
505
|
+
};
|
|
506
|
+
state.snap = { includeRollupArchive: opts.includeRollupArchive, scanned, snapshot };
|
|
507
|
+
enforceCacheBounds(indexPath);
|
|
508
|
+
return snapshot;
|
|
509
|
+
}
|
|
510
|
+
catch {
|
|
511
|
+
return null; // any unexpected failure → caller takes the brute-force path
|
|
512
|
+
}
|
|
513
|
+
}
|
|
514
|
+
//# sourceMappingURL=lexical-index.js.map
|