agent-recall-core 3.4.50 → 3.5.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +559 -330
- package/dist/embeddings/config.d.ts.map +1 -1
- package/dist/embeddings/config.js +17 -4
- package/dist/embeddings/config.js.map +1 -1
- package/dist/helpers/alignment-patterns.d.ts +46 -0
- package/dist/helpers/alignment-patterns.d.ts.map +1 -1
- package/dist/helpers/alignment-patterns.js +103 -2
- package/dist/helpers/alignment-patterns.js.map +1 -1
- package/dist/helpers/journal-files.d.ts +12 -6
- package/dist/helpers/journal-files.d.ts.map +1 -1
- package/dist/helpers/journal-files.js +13 -7
- package/dist/helpers/journal-files.js.map +1 -1
- package/dist/helpers/transcript-project.d.ts +69 -0
- package/dist/helpers/transcript-project.d.ts.map +1 -0
- package/dist/helpers/transcript-project.js +212 -0
- package/dist/helpers/transcript-project.js.map +1 -0
- package/dist/index.d.ts +28 -7
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +62 -6
- package/dist/index.js.map +1 -1
- package/dist/palace/awareness.d.ts +6 -0
- package/dist/palace/awareness.d.ts.map +1 -1
- package/dist/palace/awareness.js +10 -4
- package/dist/palace/awareness.js.map +1 -1
- package/dist/prompts/consolidation-prompt.js +14 -14
- package/dist/retrieval/activation.d.ts +202 -0
- package/dist/retrieval/activation.d.ts.map +1 -0
- package/dist/retrieval/activation.js +313 -0
- package/dist/retrieval/activation.js.map +1 -0
- package/dist/retrieval/candidates.d.ts +17 -0
- package/dist/retrieval/candidates.d.ts.map +1 -1
- package/dist/retrieval/candidates.js +4 -2
- package/dist/retrieval/candidates.js.map +1 -1
- package/dist/retrieval/contradiction.d.ts +42 -0
- package/dist/retrieval/contradiction.d.ts.map +1 -1
- package/dist/retrieval/contradiction.js +126 -7
- package/dist/retrieval/contradiction.js.map +1 -1
- package/dist/retrieval/lexical-index.d.ts +150 -0
- package/dist/retrieval/lexical-index.d.ts.map +1 -0
- package/dist/retrieval/lexical-index.js +514 -0
- package/dist/retrieval/lexical-index.js.map +1 -0
- package/dist/retrieval/query-memory.d.ts +17 -7
- package/dist/retrieval/query-memory.d.ts.map +1 -1
- package/dist/retrieval/query-memory.js +274 -68
- package/dist/retrieval/query-memory.js.map +1 -1
- package/dist/storage/classification.d.ts.map +1 -1
- package/dist/storage/classification.js +4 -1
- package/dist/storage/classification.js.map +1 -1
- package/dist/storage/content-guard.d.ts +26 -0
- package/dist/storage/content-guard.d.ts.map +1 -1
- package/dist/storage/content-guard.js +450 -48
- package/dist/storage/content-guard.js.map +1 -1
- package/dist/storage/corrections.d.ts +140 -2
- package/dist/storage/corrections.d.ts.map +1 -1
- package/dist/storage/corrections.js +144 -70
- package/dist/storage/corrections.js.map +1 -1
- package/dist/storage/cwd-allowlist.d.ts +28 -2
- package/dist/storage/cwd-allowlist.d.ts.map +1 -1
- package/dist/storage/cwd-allowlist.js +61 -13
- package/dist/storage/cwd-allowlist.js.map +1 -1
- package/dist/storage/filelock.d.ts +26 -8
- package/dist/storage/filelock.d.ts.map +1 -1
- package/dist/storage/filelock.js +202 -26
- package/dist/storage/filelock.js.map +1 -1
- package/dist/storage/fs-utils.d.ts +51 -0
- package/dist/storage/fs-utils.d.ts.map +1 -1
- package/dist/storage/fs-utils.js +123 -2
- package/dist/storage/fs-utils.js.map +1 -1
- package/dist/storage/heed-tiers.d.ts +206 -0
- package/dist/storage/heed-tiers.d.ts.map +1 -0
- package/dist/storage/heed-tiers.js +318 -0
- package/dist/storage/heed-tiers.js.map +1 -0
- package/dist/storage/memory-protocol.js +55 -55
- package/dist/storage/paths.d.ts.map +1 -1
- package/dist/storage/paths.js +7 -2
- package/dist/storage/paths.js.map +1 -1
- package/dist/storage/project.d.ts.map +1 -1
- package/dist/storage/project.js +9 -2
- package/dist/storage/project.js.map +1 -1
- package/dist/storage/store-manifest.js +57 -57
- package/dist/supabase/embedding.d.ts.map +1 -1
- package/dist/supabase/embedding.js +7 -0
- package/dist/supabase/embedding.js.map +1 -1
- package/dist/supabase/sync.d.ts +1 -1
- package/dist/supabase/sync.d.ts.map +1 -1
- package/dist/supabase/sync.js +88 -28
- package/dist/supabase/sync.js.map +1 -1
- package/dist/tools-logic/association.d.ts +260 -0
- package/dist/tools-logic/association.d.ts.map +1 -0
- package/dist/tools-logic/association.js +502 -0
- package/dist/tools-logic/association.js.map +1 -0
- package/dist/tools-logic/bootstrap.d.ts.map +1 -1
- package/dist/tools-logic/bootstrap.js +4 -2
- package/dist/tools-logic/bootstrap.js.map +1 -1
- package/dist/tools-logic/check-action.d.ts.map +1 -1
- package/dist/tools-logic/check-action.js +14 -0
- package/dist/tools-logic/check-action.js.map +1 -1
- package/dist/tools-logic/check.d.ts +7 -0
- package/dist/tools-logic/check.d.ts.map +1 -1
- package/dist/tools-logic/check.js +54 -44
- package/dist/tools-logic/check.js.map +1 -1
- package/dist/tools-logic/dream-admission.js +29 -29
- package/dist/tools-logic/implicit-harvest.d.ts +196 -0
- package/dist/tools-logic/implicit-harvest.d.ts.map +1 -0
- package/dist/tools-logic/implicit-harvest.js +675 -0
- package/dist/tools-logic/implicit-harvest.js.map +1 -0
- package/dist/tools-logic/insight-promotion.d.ts.map +1 -1
- package/dist/tools-logic/insight-promotion.js +53 -1
- package/dist/tools-logic/insight-promotion.js.map +1 -1
- package/dist/tools-logic/recall-backend.d.ts.map +1 -1
- package/dist/tools-logic/recall-backend.js +3 -1
- package/dist/tools-logic/recall-backend.js.map +1 -1
- package/dist/tools-logic/retier.d.ts +274 -0
- package/dist/tools-logic/retier.d.ts.map +1 -0
- package/dist/tools-logic/retier.js +511 -0
- package/dist/tools-logic/retier.js.map +1 -0
- package/dist/tools-logic/session-end.d.ts +35 -0
- package/dist/tools-logic/session-end.d.ts.map +1 -1
- package/dist/tools-logic/session-end.js +26 -7
- package/dist/tools-logic/session-end.js.map +1 -1
- package/dist/tools-logic/session-start.d.ts +11 -0
- package/dist/tools-logic/session-start.d.ts.map +1 -1
- package/dist/tools-logic/session-start.js +34 -4
- package/dist/tools-logic/session-start.js.map +1 -1
- package/dist/tools-logic/smart-recall.d.ts +63 -0
- package/dist/tools-logic/smart-recall.d.ts.map +1 -1
- package/dist/tools-logic/smart-recall.js +213 -31
- package/dist/tools-logic/smart-recall.js.map +1 -1
- package/dist/tools-logic/smart-remember-vector.d.ts +3 -1
- package/dist/tools-logic/smart-remember-vector.d.ts.map +1 -1
- package/dist/tools-logic/smart-remember-vector.js +3 -1
- package/dist/tools-logic/smart-remember-vector.js.map +1 -1
- package/dist/tools-logic/smart-remember.d.ts.map +1 -1
- package/dist/tools-logic/smart-remember.js +6 -2
- package/dist/tools-logic/smart-remember.js.map +1 -1
- package/dist/tools-logic/transcript-audit.d.ts +128 -0
- package/dist/tools-logic/transcript-audit.d.ts.map +1 -0
- package/dist/tools-logic/transcript-audit.js +428 -0
- package/dist/tools-logic/transcript-audit.js.map +1 -0
- package/dist/types.d.ts +17 -2
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js +31 -3
- package/dist/types.js.map +1 -1
- package/dist/vector/embedding.d.ts +20 -1
- package/dist/vector/embedding.d.ts.map +1 -1
- package/dist/vector/embedding.js +27 -1
- package/dist/vector/embedding.js.map +1 -1
- package/dist/vector/local-vector-backend.d.ts +4 -1
- package/dist/vector/local-vector-backend.d.ts.map +1 -1
- package/dist/vector/local-vector-backend.js +7 -3
- package/dist/vector/local-vector-backend.js.map +1 -1
- package/package.json +59 -59
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"contradiction.js","sourceRoot":"","sources":["../../src/retrieval/contradiction.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiMG;AA4DH,8EAA8E;AAC9E,4EAA4E;AAC5E,gEAAgE;AAChE,8EAA8E;AAE9E;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,SAAS,eAAe,CAAC,CAAoB,EAAE,CAAoB;IACjE,MAAM,EAAE,GAAG,iCAAiC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IACrD,IAAI,EAAE,CAAC,IAAI,KAAK,CAAC;
|
|
1
|
+
{"version":3,"file":"contradiction.js","sourceRoot":"","sources":["../../src/retrieval/contradiction.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiMG;AA4DH,8EAA8E;AAC9E,4EAA4E;AAC5E,gEAAgE;AAChE,8EAA8E;AAE9E;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,SAAS,eAAe,CAAC,CAAoB,EAAE,CAAoB;IACjE,MAAM,EAAE,GAAG,iCAAiC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IACrD,IAAI,EAAE,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;QAChB,MAAM,EAAE,GAAG,iCAAiC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QACrD,KAAK,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC;YAC5B,MAAM,IAAI,GAAG,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACzB,IAAI,IAAI,IAAI,IAAI,KAAK,GAAG;gBAAE,OAAO,IAAI,CAAC;QACxC,CAAC;IACH,CAAC;IACD,yEAAyE;IACzE,sEAAsE;IACtE,2EAA2E;IAC3E,gEAAgE;IAChE,MAAM,EAAE,GAAG,qBAAqB,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IACzC,IAAI,EAAE,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;QAChB,MAAM,EAAE,GAAG,qBAAqB,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QACzC,KAAK,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC;YAC5B,MAAM,IAAI,GAAG,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACzB,IAAI,IAAI,IAAI,IAAI,KAAK,GAAG;gBAAE,OAAO,IAAI,CAAC;QACxC,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AACH,MAAM,UAAU,iCAAiC,CAAC,IAAY;IAC5D,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;IACzC,MAAM,OAAO,GAAG,oGAAoG,CAAC;IACrH,IAAI,CAAyB,CAAC;IAC9B,OAAO,CAAC,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QACzC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC;QAC3D,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACnB,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACxB,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,8EAA8E;AAC9E,wDAAwD;AACxD,6DAA6D;AAC7D,8EAA8E;AAE9E;;;;;;;;GAQG;AACH,MAAM,iBAAiB,GAA2B;IAChD,GAAG,EAAE,QAAQ;IACb,IAAI,EAAE,QAAQ;IACd,GAAG,EAAE,QAAQ;IACb,IAAI,EAAE,QAAQ;IACd,EAAE,EAAE,MAAM;IACV,GAAG,EAAE,MAAM;IACX,GAAG,EAAE,SAAS;IACd,IAAI,EAAE,SAAS;CAChB,CAAC;AAEF,SAAS,iBAAiB,CAAC,IAAY;IACrC,MAAM,CAAC,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;IAC7B,MAAM,OAAO,GAAG,iBAAiB,CAAC,CAAC,CAAC,CAAC;IACrC,IAAI,OAAO;QAAE,OAAO,OAAO,CAAC;IAC5B,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC;QAAE,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACjF,OAAO,CAAC,CAAC;AACX,CAAC;AAED;;;;;;;;;;;;;;;GAeG;AACH,MAAM,kBAAkB,GAAG,IAAI,GAAG,CAAC,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC,CAAC;AAE3D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAwCG;AACH,MAAM,UAAU,qBAAqB,CAAC,IAAY;IAChD,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;IACzC,MAAM,OAAO,GAAG,iFAAiF,CAAC;IAClG,IAAI,CAAyB,CAAC;IAC9B,OAAO,CAAC,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QACzC,MAAM,IAAI,GAAG,iBAAiB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACrC,gEAAgE;QAChE,wEAAwE;QACxE,IAAI,kBAAkB,CAAC,GAAG,CAAC,IAAI,CAAC;YAAE,SAAS;QAC3C,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACnC,MAAM,GAAG,GAAG,GAAG,IAAI,QAAQ,iBAAiB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACrD,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IACzB,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAQD,SAAS,gBAAgB,CAAC,CAAoB,EAAE,CAAoB;IAClE,IAAI,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;QACrB,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAC9B,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAC9B,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;YAC1C,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC;QACzC,CAAC;QACD,iEAAiE;IACnE,CAAC;IACD,IAAI,CAAC,CAAC,KAAK,KAAK,SAAS,IAAI,CAAC,CAAC,KAAK,KAAK,SAAS,IAAI,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,KAAK,EAAE,CAAC;QAC1E,OAAO,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC;IACnD,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,8EAA8E;AAC9E,OAAO;AACP,8EAA8E;AAE9E;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,oBAAoB,CAAC,KAA0B;IAC7D,MAAM,aAAa,GAAG,IAAI,GAAG,EAAoB,CAAC;IAClD,MAAM,YAAY,GAAG,IAAI,GAAG,EAAkB,CAAC;IAE/C,MAAM,WAAW,GAAG,CAAC,CAAS,EAAE,CAAS,EAAE,EAAE;QAC3C,MAAM,QAAQ,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QACtC,IAAI,QAAQ;YAAE,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;;YAC1B,aAAa,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IACjC,CAAC,CAAC;IAEF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,KAAK,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC1C,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC;gBAAE,SAAS;YACnD,WAAW,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YAClB,WAAW,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YAElB,MAAM,GAAG,GAAG,gBAAgB,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;YACjD,IAAI,GAAG,KAAK,KAAK;gBAAE,SAAS,CAAC,yCAAyC;YAEtE,MAAM,QAAQ,GAAG,GAAG,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAC3C,MAAM,UAAU,GAAG,GAAG,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAE7C,MAAM,kBAAkB,GAAG,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YACtD,IAAI,kBAAkB,KAAK,SAAS,EAAE,CAAC;gBACrC,YAAY,CAAC,GAAG,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;YACzC,CAAC;iBAAM,CAAC;gBACN,oEAAoE;gBACpE,sEAAsE;gBACtE,MAAM,MAAM,GAAG,gBAAgB,CAAC,KAAK,CAAC,UAAU,CAAC,EAAE,KAAK,CAAC,kBAAkB,CAAC,CAAC,CAAC;gBAC9E,IAAI,MAAM,KAAK,SAAS;oBAAE,YAAY,CAAC,GAAG,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;YACnE,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,EAAE,aAAa,EAAE,YAAY,EAAE,CAAC;AACzC,CAAC"}
|
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* retrieval/lexical-index.ts — persistent, incremental lexical index for the
|
|
3
|
+
* journal tier (perf/lexical-index, 2026-10-16; reports/2026-10-15-rigorous-
|
|
4
|
+
* eval.md scale finding: 10k journal entries → recall p50 ~1.0-1.3 s, because
|
|
5
|
+
* `scoreJournalTier` re-reads and re-tokenizes the WHOLE corpus on every
|
|
6
|
+
* query — 10k `readFileSync`s + trust-tagging in the fetch stage alone
|
|
7
|
+
* measured ~1.2 s warm on win32; the full-corpus IDF tokenization adds
|
|
8
|
+
* another ~190 ms).
|
|
9
|
+
*
|
|
10
|
+
* WHAT IT STORES — per journal file, keyed by absolute path and validated by
|
|
11
|
+
* (size, mtimeNs): the DERIVED, query-independent data the journal scorer
|
|
12
|
+
* needs before it has to touch file content:
|
|
13
|
+
* - `u` — the rescue-quarantine trust bit (isRescueSourcedContent), so the
|
|
14
|
+
* trust FILTER no longer requires reading every file per query;
|
|
15
|
+
* - `t` — the file's unique stemmed token set (tokenizeWords + stem, the
|
|
16
|
+
* exact inputs `buildIdfModel`'s per-doc `tokenSet` is built from);
|
|
17
|
+
* - `r` — the file's unique lowercase whitespace-run set. Because every
|
|
18
|
+
* query keyword/variant is whitespace-free (tokenizeWords output),
|
|
19
|
+
* `content.toLowerCase().includes(term)` ⟺ some run contains
|
|
20
|
+
* `term` as a substring — this makes BOTH the per-line
|
|
21
|
+
* `lineMatchesQuery` prefilter AND `buildIdfModel`'s
|
|
22
|
+
* `d.lower.includes(v)` DF predicate answerable from the index
|
|
23
|
+
* with zero false negatives (a false POSITIVE only costs one
|
|
24
|
+
* file read that then produces no hits).
|
|
25
|
+
* Terms are stored once in a shared `vocab` table; docs reference vocab ids.
|
|
26
|
+
*
|
|
27
|
+
* WHAT IT NEVER STORES — file content, excerpts, scores, or anything
|
|
28
|
+
* query-dependent. Line-level hit generation still reads the real file (only
|
|
29
|
+
* for docs the prefilter says CAN match), so a stale/poisoned index can skew
|
|
30
|
+
* ranking weights but cannot fabricate content. Contents ARE cached
|
|
31
|
+
* IN-PROCESS (per (size, mtimeNs), pruned to the enumerated file set on each
|
|
32
|
+
* drift) — a long-lived MCP server holds roughly one copy of the hot
|
|
33
|
+
* project's journal text in RAM, the same order of memory the brute-force
|
|
34
|
+
* path already allocated transiently on EVERY query.
|
|
35
|
+
*
|
|
36
|
+
* BOUNDED CACHE (review fix, 2026-10-16, reports/2026-10-16-review-
|
|
37
|
+
* correctness.md MEDIUM): the in-process per-project states are an LRU —
|
|
38
|
+
* at most AGENT_RECALL_LEXICAL_MAX_PROJECTS project states (default 4) and
|
|
39
|
+
* at most AGENT_RECALL_LEXICAL_MAX_CONTENT_MB of cached content text
|
|
40
|
+
* (default 256 MB, UTF-16 accounting) are retained, least-recently-queried
|
|
41
|
+
* project evicted first; the project being queried is never evicted by its
|
|
42
|
+
* own call, so a single over-budget hot project keeps working (documented
|
|
43
|
+
* exemption, not a leak — it is one corpus, the brute path's own transient
|
|
44
|
+
* working set). Eviction only drops the IN-PROCESS cache: the next query for
|
|
45
|
+
* an evicted project reloads the persisted index from disk and lazily
|
|
46
|
+
* re-reads contents — results identical (pinned by the LRU/byte-budget
|
|
47
|
+
* tests). Bounds are enforced per loadJournalLexicalSnapshot call;
|
|
48
|
+
* trustedContent reads between calls can transiently exceed the byte budget
|
|
49
|
+
* by at most the active project's own corpus until the next query.
|
|
50
|
+
*
|
|
51
|
+
* TRUST (security): the cached `u` bit is an OPTIMIZATION, not the trust
|
|
52
|
+
* boundary. Every content read performed through this module re-runs
|
|
53
|
+
* `isRescueSourcedContent` on the bytes actually read (`trustedContent`
|
|
54
|
+
* below) and refuses to return rescue-tagged content — so a hand-poisoned
|
|
55
|
+
* index file (or the accepted same-(size,mtime)-different-content staleness
|
|
56
|
+
* window, same class the embedding index-store documents) can never SURFACE
|
|
57
|
+
* quarantined content; at worst it perturbs IDF document-frequency counts
|
|
58
|
+
* for files that are never read. An attacker who can write this index file
|
|
59
|
+
* lives inside the AgentRecall root and could already edit the journal
|
|
60
|
+
* files/frontmatter themselves — no new privilege is created (same residual
|
|
61
|
+
* class as storage/corrections.ts's planted-file note).
|
|
62
|
+
*
|
|
63
|
+
* INVALIDATION — every snapshot load re-enumerates the journal directories
|
|
64
|
+
* (listJournalFiles + the same archive readdir/filter the Wave-1 reader
|
|
65
|
+
* uses) and re-stats every enumerated file (bigint mtimeNs + size). A file
|
|
66
|
+
* created, deleted, or edited OUTSIDE the tool is therefore picked up on the
|
|
67
|
+
* very next query — there is no write-path hook to forget, and no TTL
|
|
68
|
+
* window. Stale/missing entries are re-read and re-derived inline (the
|
|
69
|
+
* incremental update), and the refreshed index is persisted.
|
|
70
|
+
*
|
|
71
|
+
* WRITES follow the embeddings/index-store.ts conventions: full payload
|
|
72
|
+
* built first, then written under `withLock("lexical-index-<project>")` via
|
|
73
|
+
* tmp+rename (writeJsonAtomic) — lock-free readers never observe a torn
|
|
74
|
+
* file; concurrent refreshes serialize. READS are lock-free and
|
|
75
|
+
* corruption-TOLERANT: any structural problem (truncated JSON from a killed
|
|
76
|
+
* writer's half-flushed tmp, garbage bytes, wrong version, malformed doc
|
|
77
|
+
* rows) degrades to "entry missing" → re-derive from the real files →
|
|
78
|
+
* rewrite. The recall path never throws from here (`loadJournalLexicalSnapshot`
|
|
79
|
+
* returns null on any unexpected failure, and scoreJournalTier falls back to
|
|
80
|
+
* the brute-force path).
|
|
81
|
+
*
|
|
82
|
+
* EQUIVALENCE — the consumer (query-memory.ts's scoreJournalTier) is built
|
|
83
|
+
* so the indexed path produces IDENTICAL rankings to the brute-force path:
|
|
84
|
+
* same candidate enumeration order (listJournalFiles live half; the same
|
|
85
|
+
* archive filter + date-desc stable sort as candidates.ts's rollup half),
|
|
86
|
+
* same trust filter, same line-scan code on the same content, and an IDF
|
|
87
|
+
* model assembled from index-derived DF counts that match buildIdfModel's
|
|
88
|
+
* predicate exactly (see the `r`/`t` notes above). Pinned by
|
|
89
|
+
* test/lexical-index.test.mjs's brute-vs-indexed deep-equality suite. The
|
|
90
|
+
* ONE documented divergence: a file deleted between this module's stat sweep
|
|
91
|
+
* and a (failed) content read stays counted in the IDF corpus size for that
|
|
92
|
+
* query, where brute force would have dropped it at fetch — a transient,
|
|
93
|
+
* mid-query-mutation-only window.
|
|
94
|
+
*/
|
|
95
|
+
/** Kill switch / test oracle toggle — default ON; "0"/"false"/"off" disables. */
|
|
96
|
+
export declare function lexicalIndexEnabled(): boolean;
|
|
97
|
+
/** The index file's on-disk location — built via projectSubPath so the
|
|
98
|
+
* sanitize + assertInsideRoot guards apply exactly once, like every other
|
|
99
|
+
* project-scoped path in this package. */
|
|
100
|
+
export declare function lexicalJournalIndexPath(project: string): string;
|
|
101
|
+
/** Reset all in-process caches (tests — simulates a fresh process). */
|
|
102
|
+
export declare function resetLexicalIndexCaches(): void;
|
|
103
|
+
/** Introspection for tests / a future `ar` doctor surface: how many project
|
|
104
|
+
* states are resident and how much content text they cache (UTF-16 bytes). */
|
|
105
|
+
export declare function lexicalIndexCacheStats(): {
|
|
106
|
+
states: number;
|
|
107
|
+
contentBytes: number;
|
|
108
|
+
};
|
|
109
|
+
export interface LexicalSnapshotDoc {
|
|
110
|
+
/** Filename date prefix for live/archive docs; "" when undeterminable (same semantics as MemoryCandidate.date for the journal tier). */
|
|
111
|
+
date: string;
|
|
112
|
+
/** Basename. */
|
|
113
|
+
file: string;
|
|
114
|
+
/** Absolute path. */
|
|
115
|
+
path: string;
|
|
116
|
+
/** Cached rescue-quarantine bit (content reads re-verify; see module header). */
|
|
117
|
+
untrusted: boolean;
|
|
118
|
+
}
|
|
119
|
+
export interface LexicalJournalSnapshot {
|
|
120
|
+
/** Docs in the EXACT order the brute-force journal fetch would produce
|
|
121
|
+
* (live half via listJournalFiles' own date-desc order, then the rollup-
|
|
122
|
+
* archive half date-desc) — hit-generation order, and therefore stable-
|
|
123
|
+
* sort tie-breaks, match brute force by construction. */
|
|
124
|
+
docs: ReadonlyArray<LexicalSnapshotDoc>;
|
|
125
|
+
/** Indices of docs whose lowercase content contains `term` as a substring
|
|
126
|
+
* (term must be whitespace-free — every tokenizeWords/expandQuery output
|
|
127
|
+
* is). Includes untrusted docs; callers filter. Memoized per snapshot. */
|
|
128
|
+
docsContaining(term: string): ReadonlySet<number>;
|
|
129
|
+
/** Indices of docs whose stemmed token set contains `term` exactly. */
|
|
130
|
+
docsWithStemToken(term: string): ReadonlySet<number>;
|
|
131
|
+
/** The doc's content, re-verified through the rescue-quarantine choke at
|
|
132
|
+
* read time — returns null for unreadable files AND for content that is
|
|
133
|
+
* rescue-tagged on the actual bytes read (whatever the cached bit says). */
|
|
134
|
+
trustedContent(i: number): string | null;
|
|
135
|
+
}
|
|
136
|
+
/**
|
|
137
|
+
* Load (building/refreshing as needed) the lexical snapshot for a project's
|
|
138
|
+
* journal tier. Returns null when the index is disabled or anything
|
|
139
|
+
* unexpected fails — the caller falls back to the brute-force path.
|
|
140
|
+
*
|
|
141
|
+
* Per-call work when nothing changed: one readdir pass per journal dir +
|
|
142
|
+
* one statSync per enumerated file (the external-edit detection the module
|
|
143
|
+
* header documents — ~115 ms for 10k files on win32, measured) + an
|
|
144
|
+
* elementwise fingerprint compare. Derivation/persist run only for drifted
|
|
145
|
+
* files.
|
|
146
|
+
*/
|
|
147
|
+
export declare function loadJournalLexicalSnapshot(project: string, opts: {
|
|
148
|
+
includeRollupArchive: boolean;
|
|
149
|
+
}): Promise<LexicalJournalSnapshot | null>;
|
|
150
|
+
//# sourceMappingURL=lexical-index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"lexical-index.d.ts","sourceRoot":"","sources":["../../src/retrieval/lexical-index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6FG;AAeH,iFAAiF;AACjF,wBAAgB,mBAAmB,IAAI,OAAO,CAG7C;AAED;;2CAE2C;AAC3C,wBAAgB,uBAAuB,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAE/D;AA8HD,uEAAuE;AACvE,wBAAgB,uBAAuB,IAAI,IAAI,CAE9C;AAED;+EAC+E;AAC/E,wBAAgB,sBAAsB,IAAI;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,YAAY,EAAE,MAAM,CAAA;CAAE,CAIjF;AAMD,MAAM,WAAW,kBAAkB;IACjC,wIAAwI;IACxI,IAAI,EAAE,MAAM,CAAC;IACb,gBAAgB;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,qBAAqB;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,iFAAiF;IACjF,SAAS,EAAE,OAAO,CAAC;CACpB;AAED,MAAM,WAAW,sBAAsB;IACrC;;;8DAG0D;IAC1D,IAAI,EAAE,aAAa,CAAC,kBAAkB,CAAC,CAAC;IACxC;;+EAE2E;IAC3E,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC;IAClD,uEAAuE;IACvE,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC;IACrD;;iFAE6E;IAC7E,cAAc,CAAC,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAAC;CAC1C;AAgHD;;;;;;;;;;GAUG;AACH,wBAAsB,0BAA0B,CAC9C,OAAO,EAAE,MAAM,EACf,IAAI,EAAE;IAAE,oBAAoB,EAAE,OAAO,CAAA;CAAE,GACtC,OAAO,CAAC,sBAAsB,GAAG,IAAI,CAAC,CA2KxC"}
|