agent-recall-core 3.4.48 → 3.4.49

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (215) hide show
  1. package/README.md +25 -0
  2. package/dist/digest/store.d.ts +15 -6
  3. package/dist/digest/store.d.ts.map +1 -1
  4. package/dist/digest/store.js +33 -20
  5. package/dist/digest/store.js.map +1 -1
  6. package/dist/embeddings/chunker.d.ts +102 -0
  7. package/dist/embeddings/chunker.d.ts.map +1 -0
  8. package/dist/embeddings/chunker.js +260 -0
  9. package/dist/embeddings/chunker.js.map +1 -0
  10. package/dist/embeddings/config.d.ts +93 -0
  11. package/dist/embeddings/config.d.ts.map +1 -0
  12. package/dist/embeddings/config.js +174 -0
  13. package/dist/embeddings/config.js.map +1 -0
  14. package/dist/embeddings/index-store.d.ts +61 -0
  15. package/dist/embeddings/index-store.d.ts.map +1 -0
  16. package/dist/embeddings/index-store.js +158 -0
  17. package/dist/embeddings/index-store.js.map +1 -0
  18. package/dist/embeddings/indexer.d.ts +75 -0
  19. package/dist/embeddings/indexer.d.ts.map +1 -0
  20. package/dist/embeddings/indexer.js +196 -0
  21. package/dist/embeddings/indexer.js.map +1 -0
  22. package/dist/embeddings/runtime.d.ts +66 -0
  23. package/dist/embeddings/runtime.d.ts.map +1 -0
  24. package/dist/embeddings/runtime.js +197 -0
  25. package/dist/embeddings/runtime.js.map +1 -0
  26. package/dist/helpers/associative-link.d.ts.map +1 -1
  27. package/dist/helpers/associative-link.js +26 -1
  28. package/dist/helpers/associative-link.js.map +1 -1
  29. package/dist/helpers/tokenize.d.ts +1 -0
  30. package/dist/helpers/tokenize.d.ts.map +1 -1
  31. package/dist/helpers/tokenize.js +18 -0
  32. package/dist/helpers/tokenize.js.map +1 -1
  33. package/dist/index.d.ts +21 -5
  34. package/dist/index.d.ts.map +1 -1
  35. package/dist/index.js +38 -4
  36. package/dist/index.js.map +1 -1
  37. package/dist/palace/awareness.d.ts +34 -10
  38. package/dist/palace/awareness.d.ts.map +1 -1
  39. package/dist/palace/awareness.js +249 -180
  40. package/dist/palace/awareness.js.map +1 -1
  41. package/dist/palace/consolidate.d.ts +1 -1
  42. package/dist/palace/consolidate.d.ts.map +1 -1
  43. package/dist/palace/consolidate.js +9 -9
  44. package/dist/palace/consolidate.js.map +1 -1
  45. package/dist/palace/decay-pass.d.ts +1 -1
  46. package/dist/palace/decay-pass.d.ts.map +1 -1
  47. package/dist/palace/decay-pass.js +2 -2
  48. package/dist/palace/decay-pass.js.map +1 -1
  49. package/dist/palace/fan-out.d.ts +1 -1
  50. package/dist/palace/fan-out.d.ts.map +1 -1
  51. package/dist/palace/fan-out.js +5 -5
  52. package/dist/palace/fan-out.js.map +1 -1
  53. package/dist/palace/index-manager.d.ts +1 -1
  54. package/dist/palace/index-manager.d.ts.map +1 -1
  55. package/dist/palace/index-manager.js +1 -1
  56. package/dist/palace/index-manager.js.map +1 -1
  57. package/dist/palace/insights-index.d.ts +19 -6
  58. package/dist/palace/insights-index.d.ts.map +1 -1
  59. package/dist/palace/insights-index.js +36 -13
  60. package/dist/palace/insights-index.js.map +1 -1
  61. package/dist/palace/keystone.d.ts +1 -1
  62. package/dist/palace/keystone.d.ts.map +1 -1
  63. package/dist/palace/keystone.js +2 -2
  64. package/dist/palace/keystone.js.map +1 -1
  65. package/dist/palace/rooms.d.ts +3 -3
  66. package/dist/palace/rooms.d.ts.map +1 -1
  67. package/dist/palace/rooms.js +34 -6
  68. package/dist/palace/rooms.js.map +1 -1
  69. package/dist/retrieval/candidates.d.ts.map +1 -1
  70. package/dist/retrieval/candidates.js +17 -2
  71. package/dist/retrieval/candidates.js.map +1 -1
  72. package/dist/retrieval/query-memory.d.ts +65 -7
  73. package/dist/retrieval/query-memory.d.ts.map +1 -1
  74. package/dist/retrieval/query-memory.js +518 -78
  75. package/dist/retrieval/query-memory.js.map +1 -1
  76. package/dist/retrieval/semantic-leg.d.ts +74 -0
  77. package/dist/retrieval/semantic-leg.d.ts.map +1 -0
  78. package/dist/retrieval/semantic-leg.js +180 -0
  79. package/dist/retrieval/semantic-leg.js.map +1 -0
  80. package/dist/storage/archive-write.d.ts.map +1 -1
  81. package/dist/storage/archive-write.js +29 -8
  82. package/dist/storage/archive-write.js.map +1 -1
  83. package/dist/storage/capture-router.d.ts +1 -1
  84. package/dist/storage/capture-router.d.ts.map +1 -1
  85. package/dist/storage/capture-router.js +26 -9
  86. package/dist/storage/capture-router.js.map +1 -1
  87. package/dist/storage/consolidation-queue.d.ts +11 -2
  88. package/dist/storage/consolidation-queue.d.ts.map +1 -1
  89. package/dist/storage/consolidation-queue.js +45 -16
  90. package/dist/storage/consolidation-queue.js.map +1 -1
  91. package/dist/storage/corrections.d.ts +53 -5
  92. package/dist/storage/corrections.d.ts.map +1 -1
  93. package/dist/storage/corrections.js +412 -124
  94. package/dist/storage/corrections.js.map +1 -1
  95. package/dist/storage/durable-intent.d.ts.map +1 -1
  96. package/dist/storage/durable-intent.js +24 -1
  97. package/dist/storage/durable-intent.js.map +1 -1
  98. package/dist/storage/filelock.d.ts +84 -12
  99. package/dist/storage/filelock.d.ts.map +1 -1
  100. package/dist/storage/filelock.js +309 -47
  101. package/dist/storage/filelock.js.map +1 -1
  102. package/dist/storage/fs-utils.d.ts +20 -0
  103. package/dist/storage/fs-utils.d.ts.map +1 -1
  104. package/dist/storage/fs-utils.js +27 -0
  105. package/dist/storage/fs-utils.js.map +1 -1
  106. package/dist/storage/paths.d.ts +34 -0
  107. package/dist/storage/paths.d.ts.map +1 -1
  108. package/dist/storage/paths.js +71 -0
  109. package/dist/storage/paths.js.map +1 -1
  110. package/dist/storage/pending.d.ts +154 -0
  111. package/dist/storage/pending.d.ts.map +1 -0
  112. package/dist/storage/pending.js +454 -0
  113. package/dist/storage/pending.js.map +1 -0
  114. package/dist/storage/project.d.ts.map +1 -1
  115. package/dist/storage/project.js +68 -16
  116. package/dist/storage/project.js.map +1 -1
  117. package/dist/storage/session-card.d.ts +13 -0
  118. package/dist/storage/session-card.d.ts.map +1 -1
  119. package/dist/storage/session-card.js +34 -3
  120. package/dist/storage/session-card.js.map +1 -1
  121. package/dist/storage/unclaimed.d.ts +165 -0
  122. package/dist/storage/unclaimed.d.ts.map +1 -0
  123. package/dist/storage/unclaimed.js +442 -0
  124. package/dist/storage/unclaimed.js.map +1 -0
  125. package/dist/storage/working-memory.d.ts.map +1 -1
  126. package/dist/storage/working-memory.js +19 -1
  127. package/dist/storage/working-memory.js.map +1 -1
  128. package/dist/supabase/recall-backend.d.ts +28 -2
  129. package/dist/supabase/recall-backend.d.ts.map +1 -1
  130. package/dist/supabase/recall-backend.js +80 -23
  131. package/dist/supabase/recall-backend.js.map +1 -1
  132. package/dist/supabase/sync.d.ts.map +1 -1
  133. package/dist/supabase/sync.js +13 -0
  134. package/dist/supabase/sync.js.map +1 -1
  135. package/dist/tools-logic/alignment-check.js +1 -1
  136. package/dist/tools-logic/alignment-check.js.map +1 -1
  137. package/dist/tools-logic/awareness-update.d.ts.map +1 -1
  138. package/dist/tools-logic/awareness-update.js +25 -16
  139. package/dist/tools-logic/awareness-update.js.map +1 -1
  140. package/dist/tools-logic/check-action.js +1 -1
  141. package/dist/tools-logic/check-action.js.map +1 -1
  142. package/dist/tools-logic/check.d.ts +55 -1
  143. package/dist/tools-logic/check.d.ts.map +1 -1
  144. package/dist/tools-logic/check.js +230 -57
  145. package/dist/tools-logic/check.js.map +1 -1
  146. package/dist/tools-logic/confidence.d.ts +11 -6
  147. package/dist/tools-logic/confidence.d.ts.map +1 -1
  148. package/dist/tools-logic/confidence.js +11 -6
  149. package/dist/tools-logic/confidence.js.map +1 -1
  150. package/dist/tools-logic/context-synthesize.js +2 -2
  151. package/dist/tools-logic/context-synthesize.js.map +1 -1
  152. package/dist/tools-logic/digest-read.js +2 -2
  153. package/dist/tools-logic/digest-read.js.map +1 -1
  154. package/dist/tools-logic/digest-recall.js +1 -1
  155. package/dist/tools-logic/digest-recall.js.map +1 -1
  156. package/dist/tools-logic/insight-promotion.d.ts +1 -1
  157. package/dist/tools-logic/insight-promotion.d.ts.map +1 -1
  158. package/dist/tools-logic/insight-promotion.js +17 -6
  159. package/dist/tools-logic/insight-promotion.js.map +1 -1
  160. package/dist/tools-logic/journal-capture.js +3 -3
  161. package/dist/tools-logic/journal-capture.js.map +1 -1
  162. package/dist/tools-logic/journal-write.js +3 -3
  163. package/dist/tools-logic/journal-write.js.map +1 -1
  164. package/dist/tools-logic/knowledge-write.js +2 -2
  165. package/dist/tools-logic/knowledge-write.js.map +1 -1
  166. package/dist/tools-logic/palace-lint.js +1 -1
  167. package/dist/tools-logic/palace-lint.js.map +1 -1
  168. package/dist/tools-logic/palace-read.js +1 -1
  169. package/dist/tools-logic/palace-read.js.map +1 -1
  170. package/dist/tools-logic/palace-search.js +1 -1
  171. package/dist/tools-logic/palace-search.js.map +1 -1
  172. package/dist/tools-logic/palace-walk.js +3 -3
  173. package/dist/tools-logic/palace-walk.js.map +1 -1
  174. package/dist/tools-logic/palace-write.js +4 -4
  175. package/dist/tools-logic/palace-write.js.map +1 -1
  176. package/dist/tools-logic/predict-correction.js +1 -1
  177. package/dist/tools-logic/predict-correction.js.map +1 -1
  178. package/dist/tools-logic/recall-backend.d.ts +11 -2
  179. package/dist/tools-logic/recall-backend.d.ts.map +1 -1
  180. package/dist/tools-logic/recall-backend.js +2 -2
  181. package/dist/tools-logic/recall-backend.js.map +1 -1
  182. package/dist/tools-logic/resurrect.d.ts.map +1 -1
  183. package/dist/tools-logic/resurrect.js +9 -1
  184. package/dist/tools-logic/resurrect.js.map +1 -1
  185. package/dist/tools-logic/safety-consolidation.d.ts.map +1 -1
  186. package/dist/tools-logic/safety-consolidation.js +32 -30
  187. package/dist/tools-logic/safety-consolidation.js.map +1 -1
  188. package/dist/tools-logic/session-end.d.ts +8 -0
  189. package/dist/tools-logic/session-end.d.ts.map +1 -1
  190. package/dist/tools-logic/session-end.js +51 -13
  191. package/dist/tools-logic/session-end.js.map +1 -1
  192. package/dist/tools-logic/session-start.d.ts +23 -0
  193. package/dist/tools-logic/session-start.d.ts.map +1 -1
  194. package/dist/tools-logic/session-start.js +44 -1
  195. package/dist/tools-logic/session-start.js.map +1 -1
  196. package/dist/tools-logic/smart-recall.d.ts +185 -10
  197. package/dist/tools-logic/smart-recall.d.ts.map +1 -1
  198. package/dist/tools-logic/smart-recall.js +363 -85
  199. package/dist/tools-logic/smart-recall.js.map +1 -1
  200. package/dist/tools-logic/store-doctor.d.ts.map +1 -1
  201. package/dist/tools-logic/store-doctor.js +7 -4
  202. package/dist/tools-logic/store-doctor.js.map +1 -1
  203. package/dist/tools-logic/store-repair.js +1 -1
  204. package/dist/tools-logic/store-repair.js.map +1 -1
  205. package/dist/tools-logic/supersession.d.ts +1 -1
  206. package/dist/tools-logic/supersession.d.ts.map +1 -1
  207. package/dist/tools-logic/supersession.js +2 -2
  208. package/dist/tools-logic/supersession.js.map +1 -1
  209. package/dist/types.d.ts +1 -1
  210. package/dist/types.js +1 -1
  211. package/dist/vector/embedding.d.ts +8 -0
  212. package/dist/vector/embedding.d.ts.map +1 -1
  213. package/dist/vector/embedding.js +8 -0
  214. package/dist/vector/embedding.js.map +1 -1
  215. package/package.json +1 -1
@@ -0,0 +1,93 @@
1
+ /**
2
+ * embeddings/config.ts — fix7 (plan-v2 #7, 2026-09-12): OPT-IN local
3
+ * semantic embeddings for the paraphrase retrieval gap.
4
+ *
5
+ * WHY THIS EXISTS: after fix4/fix4b the golden eval sits at 75% top-5
6
+ * hit-rate and every remaining miss is the SEMANTIC-PARAPHRASE class
7
+ * (fix4 report Escalation §2): the query shares almost no lexical tokens
8
+ * with its golden ("semver increment" vs "one version bump per release";
9
+ * a zh query against an English rule). No lexical mechanism reaches them —
10
+ * the goldens never enter the lexical top-k at all, so re-ranking cannot
11
+ * help; the semantic leg contributes CANDIDATES (an additional RRF leg,
12
+ * see retrieval/semantic-leg.ts).
13
+ *
14
+ * OPT-IN ONLY (adversarial plan review, the grounds on which default-on
15
+ * embeddings were REJECTED): flag OFF (the default) is byte-identical to
16
+ * fix4b — no index read, no model load, no new code on the recall path
17
+ * beyond one boolean check. Follows the `AGENT_RECALL_RECALL_FUSION` house
18
+ * precedent exactly: env read PER-CALL (never cached at module load, so a
19
+ * test can toggle it within one process), plus a config-file equivalent
20
+ * (`<root>/config.json`, the same file supabase/config.ts already owns a
21
+ * few keys of).
22
+ *
23
+ * ZERO-CLOUD: inference is a LOCAL ONNX model via transformers.js. The
24
+ * one-time model download happens ONLY inside `ar embeddings setup`
25
+ * (embeddings/runtime.ts sets `allowRemoteModels=false` everywhere else,
26
+ * including the whole recall path — a recall can NEVER touch the network
27
+ * through this feature). No telemetry, no remote inference.
28
+ */
29
+ export interface EmbeddingModelSpec {
30
+ /** Registry key (also the index filename tag — keep filename-safe). */
31
+ id: string;
32
+ /** HuggingFace repo transformers.js loads (ONNX weights). */
33
+ hfRepo: string;
34
+ /** Embedding dimensionality. */
35
+ dim: number;
36
+ /** Prefix prepended to QUERY texts at embed time (e5 family requires
37
+ * "query: " / "passage: "; models that don't use prefixes leave these
38
+ * empty). Part of the model's contract, NOT part of the content hash —
39
+ * the index hashes raw chunk text only, so switching models never
40
+ * poisons another model's index (each model has its own index file). */
41
+ queryPrefix: string;
42
+ /** Prefix prepended to PASSAGE (indexed chunk) texts at embed time. */
43
+ passagePrefix: string;
44
+ /** transformers.js dtype (quantization) to load. */
45
+ dtype: string;
46
+ /** Internal/test-only rows are hidden from user-facing surfaces
47
+ * (`ar embeddings setup` model list, docs). */
48
+ internal?: boolean;
49
+ }
50
+ export declare const EMBEDDING_MODELS: Record<string, EmbeddingModelSpec>;
51
+ /**
52
+ * Default model — chosen by MEASUREMENT on the golden eval (fix7 report,
53
+ * 2026-09-12, tradeoff table; numbers are the POST-review-fix certified
54
+ * runs — the H1 journal-fold fix changed the fusion landscape and the
55
+ * matrix was fully re-measured after it):
56
+ * MiniLM-L12-v2 90.0% hit-rate · 3/5 paraphrase recovered ·
57
+ * 0 regressions · ~145MB cache · ~3ms/query warm
58
+ * multilingual-e5-base 85.0% · 2/5 · 0 regressions · ~289MB · ~5-10ms
59
+ * multilingual-e5-small 80.0% · 2/5 · 1 regression (gq08) · ~144MB
60
+ * MiniLM is the only candidate that clears the fix7 exit gate (≥85% AND
61
+ * ≥3/5 paraphrase AND no regression) after the fold fix: its tighter
62
+ * cosine space (unrelated ≈0.1-0.35) admits fewer journal chunks past the
63
+ * MIN_COSINE floor, so fewer dual-evidence folds crowd out semantic-only
64
+ * goldens — pre-fix it looked worst (its "regressions" were the split-slot
65
+ * bug), post-fix it wins. Re-run the full matrix before changing this.
66
+ */
67
+ export declare const DEFAULT_EMBEDDING_MODEL = "paraphrase-multilingual-minilm-l12-v2";
68
+ /** Resolve the active model spec (env override → default). Unknown ids fall
69
+ * back to the default LOUDLY at the call sites that surface status (the
70
+ * semantic leg reports the resolved model in its note). */
71
+ export declare function resolveEmbeddingModel(): EmbeddingModelSpec;
72
+ export declare function embeddingsEnabled(): boolean;
73
+ /**
74
+ * Machine-level home for the embedding RUNTIME (a self-contained
75
+ * `npm --prefix` install of transformers.js, ~380MB with onnxruntime — the
76
+ * reason it is NOT a dependency of this package) and the MODEL cache
77
+ * (~140-290MB per model). Overridable via AGENT_RECALL_EMBEDDINGS_HOME so
78
+ * multiple stores / eval clones can share one runtime+model download.
79
+ *
80
+ * The INDEX deliberately does NOT live here — it is derived from store
81
+ * content and lives under the store root (see embeddingsIndexPath), so a
82
+ * store clone carries its own index and never shares another store's.
83
+ */
84
+ export declare function embeddingsHome(): string;
85
+ /** `<home>/runtime` — self-contained transformers.js install (setup step). */
86
+ export declare function embeddingsRuntimeDir(): string;
87
+ /** `<home>/models` — transformers.js cacheDir (one-time model downloads). */
88
+ export declare function embeddingsModelsDir(): string;
89
+ /** `<root>/embeddings/index-v1-<model>.bin` — the persistent, content-hash-
90
+ * keyed vector index (embeddings/index-store.ts). Per-model filename: two
91
+ * models never share vectors (dims/spaces are incompatible). */
92
+ export declare function embeddingsIndexPath(spec?: EmbeddingModelSpec): string;
93
+ //# sourceMappingURL=config.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../src/embeddings/config.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AAYH,MAAM,WAAW,kBAAkB;IACjC,uEAAuE;IACvE,EAAE,EAAE,MAAM,CAAC;IACX,6DAA6D;IAC7D,MAAM,EAAE,MAAM,CAAC;IACf,gCAAgC;IAChC,GAAG,EAAE,MAAM,CAAC;IACZ;;;;6EAIyE;IACzE,WAAW,EAAE,MAAM,CAAC;IACpB,uEAAuE;IACvE,aAAa,EAAE,MAAM,CAAC;IACtB,oDAAoD;IACpD,KAAK,EAAE,MAAM,CAAC;IACd;oDACgD;IAChD,QAAQ,CAAC,EAAE,OAAO,CAAC;CACpB;AAED,eAAO,MAAM,gBAAgB,EAAE,MAAM,CAAC,MAAM,EAAE,kBAAkB,CA0C/D,CAAC;AAEF;;;;;;;;;;;;;;;GAeG;AACH,eAAO,MAAM,uBAAuB,0CAA0C,CAAC;AAE/E;;4DAE4D;AAC5D,wBAAgB,qBAAqB,IAAI,kBAAkB,CAI1D;AAwBD,wBAAgB,iBAAiB,IAAI,OAAO,CAiB3C;AAMD;;;;;;;;;;GAUG;AACH,wBAAgB,cAAc,IAAI,MAAM,CAEvC;AAED,8EAA8E;AAC9E,wBAAgB,oBAAoB,IAAI,MAAM,CAE7C;AAED,6EAA6E;AAC7E,wBAAgB,mBAAmB,IAAI,MAAM,CAE5C;AAED;;iEAEiE;AACjE,wBAAgB,mBAAmB,CAAC,IAAI,CAAC,EAAE,kBAAkB,GAAG,MAAM,CAGrE"}
@@ -0,0 +1,174 @@
1
+ /**
2
+ * embeddings/config.ts — fix7 (plan-v2 #7, 2026-09-12): OPT-IN local
3
+ * semantic embeddings for the paraphrase retrieval gap.
4
+ *
5
+ * WHY THIS EXISTS: after fix4/fix4b the golden eval sits at 75% top-5
6
+ * hit-rate and every remaining miss is the SEMANTIC-PARAPHRASE class
7
+ * (fix4 report Escalation §2): the query shares almost no lexical tokens
8
+ * with its golden ("semver increment" vs "one version bump per release";
9
+ * a zh query against an English rule). No lexical mechanism reaches them —
10
+ * the goldens never enter the lexical top-k at all, so re-ranking cannot
11
+ * help; the semantic leg contributes CANDIDATES (an additional RRF leg,
12
+ * see retrieval/semantic-leg.ts).
13
+ *
14
+ * OPT-IN ONLY (adversarial plan review, the grounds on which default-on
15
+ * embeddings were REJECTED): flag OFF (the default) is byte-identical to
16
+ * fix4b — no index read, no model load, no new code on the recall path
17
+ * beyond one boolean check. Follows the `AGENT_RECALL_RECALL_FUSION` house
18
+ * precedent exactly: env read PER-CALL (never cached at module load, so a
19
+ * test can toggle it within one process), plus a config-file equivalent
20
+ * (`<root>/config.json`, the same file supabase/config.ts already owns a
21
+ * few keys of).
22
+ *
23
+ * ZERO-CLOUD: inference is a LOCAL ONNX model via transformers.js. The
24
+ * one-time model download happens ONLY inside `ar embeddings setup`
25
+ * (embeddings/runtime.ts sets `allowRemoteModels=false` everywhere else,
26
+ * including the whole recall path — a recall can NEVER touch the network
27
+ * through this feature). No telemetry, no remote inference.
28
+ */
29
+ import * as fs from "node:fs";
30
+ import * as path from "node:path";
31
+ import { getRoot } from "../types.js";
32
+ export const EMBEDDING_MODELS = {
33
+ "multilingual-e5-small": {
34
+ id: "multilingual-e5-small",
35
+ hfRepo: "Xenova/multilingual-e5-small",
36
+ dim: 384,
37
+ queryPrefix: "query: ",
38
+ passagePrefix: "passage: ",
39
+ dtype: "q8",
40
+ },
41
+ "paraphrase-multilingual-minilm-l12-v2": {
42
+ id: "paraphrase-multilingual-minilm-l12-v2",
43
+ hfRepo: "Xenova/paraphrase-multilingual-MiniLM-L12-v2",
44
+ dim: 384,
45
+ queryPrefix: "",
46
+ passagePrefix: "",
47
+ dtype: "q8",
48
+ },
49
+ "multilingual-e5-base": {
50
+ id: "multilingual-e5-base",
51
+ hfRepo: "Xenova/multilingual-e5-base",
52
+ dim: 768,
53
+ queryPrefix: "query: ",
54
+ passagePrefix: "passage: ",
55
+ dtype: "q8",
56
+ },
57
+ /**
58
+ * TEST SEAM — a deterministic, dependency-free, network-free pseudo-
59
+ * embedder (hashed bag-of-tokens, see runtime.ts). Exists so the whole
60
+ * pipeline (index build → semantic leg → RRF fusion → fence) is testable
61
+ * end-to-end in CI without the 380MB runtime or a model download. It is
62
+ * NOT a semantic model (cosine ≈ token overlap) and is never selected
63
+ * unless AGENT_RECALL_EMBEDDINGS_MODEL names it explicitly.
64
+ */
65
+ "_fake-hash-bow": {
66
+ id: "_fake-hash-bow",
67
+ hfRepo: "(builtin test embedder — no download)",
68
+ dim: 64,
69
+ queryPrefix: "",
70
+ passagePrefix: "",
71
+ dtype: "none",
72
+ internal: true,
73
+ },
74
+ };
75
+ /**
76
+ * Default model — chosen by MEASUREMENT on the golden eval (fix7 report,
77
+ * 2026-09-12, tradeoff table; numbers are the POST-review-fix certified
78
+ * runs — the H1 journal-fold fix changed the fusion landscape and the
79
+ * matrix was fully re-measured after it):
80
+ * MiniLM-L12-v2 90.0% hit-rate · 3/5 paraphrase recovered ·
81
+ * 0 regressions · ~145MB cache · ~3ms/query warm
82
+ * multilingual-e5-base 85.0% · 2/5 · 0 regressions · ~289MB · ~5-10ms
83
+ * multilingual-e5-small 80.0% · 2/5 · 1 regression (gq08) · ~144MB
84
+ * MiniLM is the only candidate that clears the fix7 exit gate (≥85% AND
85
+ * ≥3/5 paraphrase AND no regression) after the fold fix: its tighter
86
+ * cosine space (unrelated ≈0.1-0.35) admits fewer journal chunks past the
87
+ * MIN_COSINE floor, so fewer dual-evidence folds crowd out semantic-only
88
+ * goldens — pre-fix it looked worst (its "regressions" were the split-slot
89
+ * bug), post-fix it wins. Re-run the full matrix before changing this.
90
+ */
91
+ export const DEFAULT_EMBEDDING_MODEL = "paraphrase-multilingual-minilm-l12-v2";
92
+ /** Resolve the active model spec (env override → default). Unknown ids fall
93
+ * back to the default LOUDLY at the call sites that surface status (the
94
+ * semantic leg reports the resolved model in its note). */
95
+ export function resolveEmbeddingModel() {
96
+ const requested = process.env.AGENT_RECALL_EMBEDDINGS_MODEL;
97
+ if (requested && EMBEDDING_MODELS[requested])
98
+ return EMBEDDING_MODELS[requested];
99
+ return EMBEDDING_MODELS[DEFAULT_EMBEDDING_MODEL];
100
+ }
101
+ // ---------------------------------------------------------------------------
102
+ // Opt-in flag — read PER-CALL (AGENT_RECALL_RECALL_FUSION precedent).
103
+ // ---------------------------------------------------------------------------
104
+ /**
105
+ * Is the semantic leg enabled?
106
+ *
107
+ * Precedence (explicit env always wins, so a caller/test/eval-harness can
108
+ * force either state regardless of the store's config file):
109
+ * 1. AGENT_RECALL_EMBEDDINGS === "1" → on
110
+ * 2. AGENT_RECALL_EMBEDDINGS === any other non-empty value → off
111
+ * 3. <root>/config.json `"embeddings_enabled": true` → on
112
+ * 4. default → off
113
+ *
114
+ * The env check is per-call (RECALL_FUSION precedent). The config-file half
115
+ * is best-effort and mtime-CACHED (fix7 review L: one statSync per call
116
+ * instead of exists+read+parse — config.json edits are picked up on the
117
+ * next call via the mtime key); any read/parse failure means OFF — the
118
+ * flag must never make a recall throw.
119
+ */
120
+ let _cfgCache = null;
121
+ export function embeddingsEnabled() {
122
+ const env = process.env.AGENT_RECALL_EMBEDDINGS;
123
+ if (env !== undefined && env !== "")
124
+ return env === "1";
125
+ try {
126
+ const p = path.join(getRoot(), "config.json");
127
+ const stat = fs.statSync(p, { throwIfNoEntry: false });
128
+ if (!stat)
129
+ return false;
130
+ if (_cfgCache && _cfgCache.path === p && _cfgCache.mtimeMs === stat.mtimeMs) {
131
+ return _cfgCache.enabled;
132
+ }
133
+ const cfg = JSON.parse(fs.readFileSync(p, "utf-8"));
134
+ const enabled = cfg?.embeddings_enabled === true;
135
+ _cfgCache = { path: p, mtimeMs: stat.mtimeMs, enabled };
136
+ return enabled;
137
+ }
138
+ catch {
139
+ return false;
140
+ }
141
+ }
142
+ // ---------------------------------------------------------------------------
143
+ // Paths
144
+ // ---------------------------------------------------------------------------
145
+ /**
146
+ * Machine-level home for the embedding RUNTIME (a self-contained
147
+ * `npm --prefix` install of transformers.js, ~380MB with onnxruntime — the
148
+ * reason it is NOT a dependency of this package) and the MODEL cache
149
+ * (~140-290MB per model). Overridable via AGENT_RECALL_EMBEDDINGS_HOME so
150
+ * multiple stores / eval clones can share one runtime+model download.
151
+ *
152
+ * The INDEX deliberately does NOT live here — it is derived from store
153
+ * content and lives under the store root (see embeddingsIndexPath), so a
154
+ * store clone carries its own index and never shares another store's.
155
+ */
156
+ export function embeddingsHome() {
157
+ return process.env.AGENT_RECALL_EMBEDDINGS_HOME ?? path.join(getRoot(), "embeddings");
158
+ }
159
+ /** `<home>/runtime` — self-contained transformers.js install (setup step). */
160
+ export function embeddingsRuntimeDir() {
161
+ return path.join(embeddingsHome(), "runtime");
162
+ }
163
+ /** `<home>/models` — transformers.js cacheDir (one-time model downloads). */
164
+ export function embeddingsModelsDir() {
165
+ return path.join(embeddingsHome(), "models");
166
+ }
167
+ /** `<root>/embeddings/index-v1-<model>.bin` — the persistent, content-hash-
168
+ * keyed vector index (embeddings/index-store.ts). Per-model filename: two
169
+ * models never share vectors (dims/spaces are incompatible). */
170
+ export function embeddingsIndexPath(spec) {
171
+ const model = spec ?? resolveEmbeddingModel();
172
+ return path.join(getRoot(), "embeddings", `index-v1-${model.id}.bin`);
173
+ }
174
+ //# sourceMappingURL=config.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"config.js","sourceRoot":"","sources":["../../src/embeddings/config.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AAEH,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AA8BtC,MAAM,CAAC,MAAM,gBAAgB,GAAuC;IAClE,uBAAuB,EAAE;QACvB,EAAE,EAAE,uBAAuB;QAC3B,MAAM,EAAE,8BAA8B;QACtC,GAAG,EAAE,GAAG;QACR,WAAW,EAAE,SAAS;QACtB,aAAa,EAAE,WAAW;QAC1B,KAAK,EAAE,IAAI;KACZ;IACD,uCAAuC,EAAE;QACvC,EAAE,EAAE,uCAAuC;QAC3C,MAAM,EAAE,8CAA8C;QACtD,GAAG,EAAE,GAAG;QACR,WAAW,EAAE,EAAE;QACf,aAAa,EAAE,EAAE;QACjB,KAAK,EAAE,IAAI;KACZ;IACD,sBAAsB,EAAE;QACtB,EAAE,EAAE,sBAAsB;QAC1B,MAAM,EAAE,6BAA6B;QACrC,GAAG,EAAE,GAAG;QACR,WAAW,EAAE,SAAS;QACtB,aAAa,EAAE,WAAW;QAC1B,KAAK,EAAE,IAAI;KACZ;IACD;;;;;;;OAOG;IACH,gBAAgB,EAAE;QAChB,EAAE,EAAE,gBAAgB;QACpB,MAAM,EAAE,uCAAuC;QAC/C,GAAG,EAAE,EAAE;QACP,WAAW,EAAE,EAAE;QACf,aAAa,EAAE,EAAE;QACjB,KAAK,EAAE,MAAM;QACb,QAAQ,EAAE,IAAI;KACf;CACF,CAAC;AAEF;;;;;;;;;;;;;;;GAeG;AACH,MAAM,CAAC,MAAM,uBAAuB,GAAG,uCAAuC,CAAC;AAE/E;;4DAE4D;AAC5D,MAAM,UAAU,qBAAqB;IACnC,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,6BAA6B,CAAC;IAC5D,IAAI,SAAS,IAAI,gBAAgB,CAAC,SAAS,CAAC;QAAE,OAAO,gBAAgB,CAAC,SAAS,CAAC,CAAC;IACjF,OAAO,gBAAgB,CAAC,uBAAuB,CAAC,CAAC;AACnD,CAAC;AAED,8EAA8E;AAC9E,sEAAsE;AACtE,8EAA8E;AAE9E;;;;;;;;;;;;;;;GAeG;AACH,IAAI,SAAS,GAA+D,IAAI,CAAC;AAEjF,MAAM,UAAU,iBAAiB;IAC/B,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,uBAAuB,CAAC;IAChD,IAAI,GAAG,KAAK,SAAS,IAAI,GAAG,KAAK,EAAE;QAAE,OAAO,GAAG,KAAK,GAAG,CAAC;IACxD,IAAI,CAAC;QACH,MAAM,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,aAAa,CAAC,CAAC;QAC9C,MAAM,IAAI,GAAG,EAAE,CAAC,QAAQ,CAAC,CAAC,EAAE,EAAE,cAAc,EAAE,KAAK,EAAE,CAAC,CAAC;QACvD,IAAI,CAAC,IAAI;YAAE,OAAO,KAAK,CAAC;QACxB,IAAI,SAAS,IAAI,SAAS,CAAC,IAAI,KAAK,CAAC,IAAI,SAAS,CAAC,OAAO,KAAK,IAAI,CAAC,OAAO,EAAE,CAAC;YAC5E,OAAO,SAAS,CAAC,OAAO,CAAC;QAC3B,CAAC;QACD,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC;QACpD,MAAM,OAAO,GAAG,GAAG,EAAE,kBAAkB,KAAK,IAAI,CAAC;QACjD,SAAS,GAAG,EAAE,IAAI,EAAE,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,CAAC;QACxD,OAAO,OAAO,CAAC;IACjB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,8EAA8E;AAC9E,QAAQ;AACR,8EAA8E;AAE9E;;;;;;;;;;GAUG;AACH,MAAM,UAAU,cAAc;IAC5B,OAAO,OAAO,CAAC,GAAG,CAAC,4BAA4B,IAAI,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,YAAY,CAAC,CAAC;AACxF,CAAC;AAED,8EAA8E;AAC9E,MAAM,UAAU,oBAAoB;IAClC,OAAO,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,EAAE,SAAS,CAAC,CAAC;AAChD,CAAC;AAED,6EAA6E;AAC7E,MAAM,UAAU,mBAAmB;IACjC,OAAO,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,EAAE,QAAQ,CAAC,CAAC;AAC/C,CAAC;AAED;;iEAEiE;AACjE,MAAM,UAAU,mBAAmB,CAAC,IAAyB;IAC3D,MAAM,KAAK,GAAG,IAAI,IAAI,qBAAqB,EAAE,CAAC;IAC9C,OAAO,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,YAAY,EAAE,YAAY,KAAK,CAAC,EAAE,MAAM,CAAC,CAAC;AACxE,CAAC"}
@@ -0,0 +1,61 @@
1
+ /**
2
+ * embeddings/index-store.ts — fix7: the persistent, content-hash-keyed
3
+ * vector index.
4
+ *
5
+ * WHAT IT STORES — hash→vector ONLY. No chunk text, no provenance, no trust
6
+ * bits ever live in the index: at query time the semantic leg re-reads the
7
+ * store through the same trust-filtered readers as every lexical tier and
8
+ * only LOOKS UP vectors by content hash (see embeddings/chunker.ts's
9
+ * header). A corrupt/poisoned/stale index can therefore degrade recall
10
+ * quality but can never inject content or bypass a trust/scope/fence stage.
11
+ *
12
+ * FORMAT (v1, single file per model — `index-v1-<model>.bin`):
13
+ * bytes 0..8 magic "AREMBIX1"
14
+ * bytes 8..12 uint32 LE — header JSON byte length
15
+ * bytes 12..N header JSON (utf-8): { version, model, dim, count,
16
+ * built_at, hashes: string[count] }
17
+ * bytes N..EOF count × dim float32 LE (row i = hashes[i]'s vector,
18
+ * L2-normalized at embed time so cosine = dot product)
19
+ * Binary because a JSON-of-number-arrays index at realistic store sizes
20
+ * (tens of thousands of 384-dim vectors) costs hundreds of ms of parse on
21
+ * EVERY recall; this format is one read + one small JSON parse + one memcpy.
22
+ *
23
+ * WRITES follow the fix6 conventions: the full payload is built first, then
24
+ * written inside `withLock("embeddings-index")` via tmp+rename (atomic on
25
+ * POSIX) — lock-free READERS can never observe a torn file, and two
26
+ * concurrent rebuilds serialize instead of interleaving.
27
+ *
28
+ * READS are lock-free and corruption-TOLERANT: any structural problem
29
+ * (short file, bad magic, unparsable header, dim/model mismatch, size
30
+ * mismatch) returns `{ error }` — the semantic leg degrades to lexical-only
31
+ * with a diagnosable note; the recall path never throws.
32
+ */
33
+ import { type EmbeddingModelSpec } from "./config.js";
34
+ export interface EmbeddingIndex {
35
+ model: string;
36
+ dim: number;
37
+ builtAt: string;
38
+ /** Row order matches `vectors` rows. */
39
+ hashes: string[];
40
+ /** count × dim, L2-normalized rows. */
41
+ vectors: Float32Array;
42
+ /** hash → row number (built at load). */
43
+ rowByHash: Map<string, number>;
44
+ }
45
+ export interface IndexReadError {
46
+ reason: "missing" | "corrupt" | "model-mismatch";
47
+ message: string;
48
+ }
49
+ /** Serialize + atomically write the index under the shared store lock. */
50
+ export declare function writeEmbeddingIndex(spec: EmbeddingModelSpec, hashes: string[], vectors: Float32Array): Promise<string>;
51
+ /**
52
+ * Load the index for a model. Lock-free (writes are atomic-rename).
53
+ * NEVER throws — every failure is a typed `{ error }` for the semantic
54
+ * leg's degrade note.
55
+ */
56
+ export declare function readEmbeddingIndex(spec: EmbeddingModelSpec): EmbeddingIndex | {
57
+ error: IndexReadError;
58
+ };
59
+ /** Reset the in-process read cache (tests). */
60
+ export declare function resetEmbeddingIndexCache(): void;
61
+ //# sourceMappingURL=index-store.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index-store.d.ts","sourceRoot":"","sources":["../../src/embeddings/index-store.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AAMH,OAAO,EAAuB,KAAK,kBAAkB,EAAE,MAAM,aAAa,CAAC;AAI3E,MAAM,WAAW,cAAc;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC;IAChB,wCAAwC;IACxC,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,uCAAuC;IACvC,OAAO,EAAE,YAAY,CAAC;IACtB,yCAAyC;IACzC,SAAS,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAChC;AAED,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,SAAS,GAAG,SAAS,GAAG,gBAAgB,CAAC;IACjD,OAAO,EAAE,MAAM,CAAC;CACjB;AAgBD,0EAA0E;AAC1E,wBAAsB,mBAAmB,CACvC,IAAI,EAAE,kBAAkB,EACxB,MAAM,EAAE,MAAM,EAAE,EAChB,OAAO,EAAE,YAAY,GACpB,OAAO,CAAC,MAAM,CAAC,CAqBjB;AAWD;;;;GAIG;AACH,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,kBAAkB,GAAG,cAAc,GAAG;IAAE,KAAK,EAAE,cAAc,CAAA;CAAE,CAuEvG;AAED,+CAA+C;AAC/C,wBAAgB,wBAAwB,IAAI,IAAI,CAE/C"}
@@ -0,0 +1,158 @@
1
+ /**
2
+ * embeddings/index-store.ts — fix7: the persistent, content-hash-keyed
3
+ * vector index.
4
+ *
5
+ * WHAT IT STORES — hash→vector ONLY. No chunk text, no provenance, no trust
6
+ * bits ever live in the index: at query time the semantic leg re-reads the
7
+ * store through the same trust-filtered readers as every lexical tier and
8
+ * only LOOKS UP vectors by content hash (see embeddings/chunker.ts's
9
+ * header). A corrupt/poisoned/stale index can therefore degrade recall
10
+ * quality but can never inject content or bypass a trust/scope/fence stage.
11
+ *
12
+ * FORMAT (v1, single file per model — `index-v1-<model>.bin`):
13
+ * bytes 0..8 magic "AREMBIX1"
14
+ * bytes 8..12 uint32 LE — header JSON byte length
15
+ * bytes 12..N header JSON (utf-8): { version, model, dim, count,
16
+ * built_at, hashes: string[count] }
17
+ * bytes N..EOF count × dim float32 LE (row i = hashes[i]'s vector,
18
+ * L2-normalized at embed time so cosine = dot product)
19
+ * Binary because a JSON-of-number-arrays index at realistic store sizes
20
+ * (tens of thousands of 384-dim vectors) costs hundreds of ms of parse on
21
+ * EVERY recall; this format is one read + one small JSON parse + one memcpy.
22
+ *
23
+ * WRITES follow the fix6 conventions: the full payload is built first, then
24
+ * written inside `withLock("embeddings-index")` via tmp+rename (atomic on
25
+ * POSIX) — lock-free READERS can never observe a torn file, and two
26
+ * concurrent rebuilds serialize instead of interleaving.
27
+ *
28
+ * READS are lock-free and corruption-TOLERANT: any structural problem
29
+ * (short file, bad magic, unparsable header, dim/model mismatch, size
30
+ * mismatch) returns `{ error }` — the semantic leg degrades to lexical-only
31
+ * with a diagnosable note; the recall path never throws.
32
+ */
33
+ import * as fs from "node:fs";
34
+ import * as path from "node:path";
35
+ import { ensureDir } from "../storage/fs-utils.js";
36
+ import { withLock } from "../storage/filelock.js";
37
+ import { embeddingsIndexPath } from "./config.js";
38
+ const MAGIC = "AREMBIX1";
39
+ function headerFor(index) {
40
+ return Buffer.from(JSON.stringify({
41
+ version: 1,
42
+ model: index.model,
43
+ dim: index.dim,
44
+ count: index.hashes.length,
45
+ built_at: index.builtAt,
46
+ hashes: index.hashes,
47
+ }), "utf-8");
48
+ }
49
+ /** Serialize + atomically write the index under the shared store lock. */
50
+ export async function writeEmbeddingIndex(spec, hashes, vectors) {
51
+ if (vectors.length !== hashes.length * spec.dim) {
52
+ throw new Error(`embedding index shape mismatch: ${hashes.length} hashes × dim ${spec.dim} != ${vectors.length} floats`);
53
+ }
54
+ const indexPath = embeddingsIndexPath(spec);
55
+ const header = headerFor({ model: spec.id, dim: spec.dim, builtAt: new Date().toISOString(), hashes });
56
+ const headBuf = Buffer.alloc(12);
57
+ headBuf.write(MAGIC, 0, "ascii");
58
+ headBuf.writeUInt32LE(header.length, 8);
59
+ const vecBuf = Buffer.from(vectors.buffer, vectors.byteOffset, vectors.byteLength);
60
+ const payload = Buffer.concat([headBuf, header, vecBuf]);
61
+ await withLock("embeddings-index", () => {
62
+ ensureDir(path.dirname(indexPath));
63
+ const tmp = indexPath + ".tmp." + process.pid;
64
+ fs.writeFileSync(tmp, payload);
65
+ fs.renameSync(tmp, indexPath); // atomic on POSIX — readers never see a torn file
66
+ });
67
+ return indexPath;
68
+ }
69
+ /** In-process read cache keyed by (path, mtimeMs, size) — repeated recalls
70
+ * in one MCP-server process skip re-reading a multi-MB file. KNOWN
71
+ * NARROW STALENESS WINDOW (fix7 review L, accepted): two rebuilds landing
72
+ * within the same mtime tick AND producing byte-equal sizes would serve
73
+ * the older copy until the next tick — content-hash keys make same-size
74
+ * different-content rebuilds vanishingly rare, and a rebuild is a manual
75
+ * CLI action; `resetEmbeddingIndexCache()` is the test/debug escape. */
76
+ const _readCache = new Map();
77
+ /**
78
+ * Load the index for a model. Lock-free (writes are atomic-rename).
79
+ * NEVER throws — every failure is a typed `{ error }` for the semantic
80
+ * leg's degrade note.
81
+ */
82
+ export function readEmbeddingIndex(spec) {
83
+ const indexPath = embeddingsIndexPath(spec);
84
+ let stat;
85
+ try {
86
+ stat = fs.statSync(indexPath);
87
+ }
88
+ catch {
89
+ return {
90
+ error: {
91
+ reason: "missing",
92
+ message: `no embedding index at ${indexPath} — run \`ar embeddings rebuild\` once`,
93
+ },
94
+ };
95
+ }
96
+ const cached = _readCache.get(indexPath);
97
+ if (cached && cached.mtimeMs === stat.mtimeMs && cached.size === stat.size) {
98
+ return cached.index;
99
+ }
100
+ try {
101
+ const buf = fs.readFileSync(indexPath);
102
+ if (buf.length < 12 || buf.toString("ascii", 0, 8) !== MAGIC) {
103
+ return { error: { reason: "corrupt", message: `embedding index has bad magic/size (${indexPath}) — re-run \`ar embeddings rebuild\`` } };
104
+ }
105
+ const headerLen = buf.readUInt32LE(8);
106
+ if (12 + headerLen > buf.length) {
107
+ return { error: { reason: "corrupt", message: `embedding index header truncated (${indexPath}) — re-run \`ar embeddings rebuild\`` } };
108
+ }
109
+ const header = JSON.parse(buf.toString("utf-8", 12, 12 + headerLen));
110
+ if (header.version !== 1 || !Array.isArray(header.hashes) || header.hashes.length !== header.count) {
111
+ return { error: { reason: "corrupt", message: `embedding index header invalid (${indexPath}) — re-run \`ar embeddings rebuild\`` } };
112
+ }
113
+ if (header.model !== spec.id || header.dim !== spec.dim) {
114
+ return {
115
+ error: {
116
+ reason: "model-mismatch",
117
+ message: `embedding index was built for model "${header.model}" (dim ${header.dim}), active model is "${spec.id}" (dim ${spec.dim}) — run \`ar embeddings rebuild\``,
118
+ },
119
+ };
120
+ }
121
+ const vecBytes = header.count * header.dim * 4;
122
+ const vecStart = 12 + headerLen;
123
+ if (vecStart + vecBytes !== buf.length) {
124
+ return { error: { reason: "corrupt", message: `embedding index vector block size mismatch (${indexPath}) — re-run \`ar embeddings rebuild\`` } };
125
+ }
126
+ // Copy into an aligned ArrayBuffer (Buffer pool offsets are not
127
+ // guaranteed 4-byte aligned for a Float32Array view).
128
+ const aligned = new ArrayBuffer(vecBytes);
129
+ buf.copy(Buffer.from(aligned), 0, vecStart, vecStart + vecBytes);
130
+ const vectors = new Float32Array(aligned);
131
+ const rowByHash = new Map();
132
+ for (let i = 0; i < header.hashes.length; i++)
133
+ rowByHash.set(header.hashes[i], i);
134
+ const index = {
135
+ model: header.model,
136
+ dim: header.dim,
137
+ builtAt: header.built_at,
138
+ hashes: header.hashes,
139
+ vectors,
140
+ rowByHash,
141
+ };
142
+ _readCache.set(indexPath, { mtimeMs: stat.mtimeMs, size: stat.size, index });
143
+ return index;
144
+ }
145
+ catch (err) {
146
+ return {
147
+ error: {
148
+ reason: "corrupt",
149
+ message: `embedding index unreadable (${indexPath}): ${err instanceof Error ? err.message : String(err)} — re-run \`ar embeddings rebuild\``,
150
+ },
151
+ };
152
+ }
153
+ }
154
+ /** Reset the in-process read cache (tests). */
155
+ export function resetEmbeddingIndexCache() {
156
+ _readCache.clear();
157
+ }
158
+ //# sourceMappingURL=index-store.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index-store.js","sourceRoot":"","sources":["../../src/embeddings/index-store.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AAEH,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAC;AACnD,OAAO,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAC;AAClD,OAAO,EAAE,mBAAmB,EAA2B,MAAM,aAAa,CAAC;AAE3E,MAAM,KAAK,GAAG,UAAU,CAAC;AAmBzB,SAAS,SAAS,CAAC,KAAwE;IACzF,OAAO,MAAM,CAAC,IAAI,CAChB,IAAI,CAAC,SAAS,CAAC;QACb,OAAO,EAAE,CAAC;QACV,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,GAAG,EAAE,KAAK,CAAC,GAAG;QACd,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM;QAC1B,QAAQ,EAAE,KAAK,CAAC,OAAO;QACvB,MAAM,EAAE,KAAK,CAAC,MAAM;KACrB,CAAC,EACF,OAAO,CACR,CAAC;AACJ,CAAC;AAED,0EAA0E;AAC1E,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,IAAwB,EACxB,MAAgB,EAChB,OAAqB;IAErB,IAAI,OAAO,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAChD,MAAM,IAAI,KAAK,CACb,mCAAmC,MAAM,CAAC,MAAM,iBAAiB,IAAI,CAAC,GAAG,OAAO,OAAO,CAAC,MAAM,SAAS,CACxG,CAAC;IACJ,CAAC;IACD,MAAM,SAAS,GAAG,mBAAmB,CAAC,IAAI,CAAC,CAAC;IAC5C,MAAM,MAAM,GAAG,SAAS,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,EAAE,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,OAAO,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;IACvG,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IACjC,OAAO,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,EAAE,OAAO,CAAC,CAAC;IACjC,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IACxC,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,UAAU,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;IACnF,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IAEzD,MAAM,QAAQ,CAAC,kBAAkB,EAAE,GAAG,EAAE;QACtC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC;QACnC,MAAM,GAAG,GAAG,SAAS,GAAG,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC;QAC9C,EAAE,CAAC,aAAa,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QAC/B,EAAE,CAAC,UAAU,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC,CAAC,kDAAkD;IACnF,CAAC,CAAC,CAAC;IACH,OAAO,SAAS,CAAC;AACnB,CAAC;AAED;;;;;;yEAMyE;AACzE,MAAM,UAAU,GAAG,IAAI,GAAG,EAAoE,CAAC;AAE/F;;;;GAIG;AACH,MAAM,UAAU,kBAAkB,CAAC,IAAwB;IACzD,MAAM,SAAS,GAAG,mBAAmB,CAAC,IAAI,CAAC,CAAC;IAC5C,IAAI,IAAc,CAAC;IACnB,IAAI,CAAC;QACH,IAAI,GAAG,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;IAChC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO;YACL,KAAK,EAAE;gBACL,MAAM,EAAE,SAAS;gBACjB,OAAO,EAAE,yBAAyB,SAAS,uCAAuC;aACnF;SACF,CAAC;IACJ,CAAC;IACD,MAAM,MAAM,GAAG,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;IACzC,IAAI,MAAM,IAAI,MAAM,CAAC,OAAO,KAAK,IAAI,CAAC,OAAO,IAAI,MAAM,CAAC,IAAI,KAAK,IAAI,CAAC,IAAI,EAAE,CAAC;QAC3E,OAAO,MAAM,CAAC,KAAK,CAAC;IACtB,CAAC;IAED,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,EAAE,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC;QACvC,IAAI,GAAG,CAAC,MAAM,GAAG,EAAE,IAAI,GAAG,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC,KAAK,KAAK,EAAE,CAAC;YAC7D,OAAO,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,uCAAuC,SAAS,sCAAsC,EAAE,EAAE,CAAC;QAC3I,CAAC;QACD,MAAM,SAAS,GAAG,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACtC,IAAI,EAAE,GAAG,SAAS,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC;YAChC,OAAO,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,qCAAqC,SAAS,sCAAsC,EAAE,EAAE,CAAC;QACzI,CAAC;QACD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,OAAO,EAAE,EAAE,EAAE,EAAE,GAAG,SAAS,CAAC,CAElE,CAAC;QACF,IAAI,MAAM,CAAC,OAAO,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,KAAK,MAAM,CAAC,KAAK,EAAE,CAAC;YACnG,OAAO,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,mCAAmC,SAAS,sCAAsC,EAAE,EAAE,CAAC;QACvI,CAAC;QACD,IAAI,MAAM,CAAC,KAAK,KAAK,IAAI,CAAC,EAAE,IAAI,MAAM,CAAC,GAAG,KAAK,IAAI,CAAC,GAAG,EAAE,CAAC;YACxD,OAAO;gBACL,KAAK,EAAE;oBACL,MAAM,EAAE,gBAAgB;oBACxB,OAAO,EAAE,wCAAwC,MAAM,CAAC,KAAK,UAAU,MAAM,CAAC,GAAG,uBAAuB,IAAI,CAAC,EAAE,UAAU,IAAI,CAAC,GAAG,mCAAmC;iBACrK;aACF,CAAC;QACJ,CAAC;QACD,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,GAAG,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC;QAC/C,MAAM,QAAQ,GAAG,EAAE,GAAG,SAAS,CAAC;QAChC,IAAI,QAAQ,GAAG,QAAQ,KAAK,GAAG,CAAC,MAAM,EAAE,CAAC;YACvC,OAAO,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,+CAA+C,SAAS,sCAAsC,EAAE,EAAE,CAAC;QACnJ,CAAC;QACD,gEAAgE;QAChE,sDAAsD;QACtD,MAAM,OAAO,GAAG,IAAI,WAAW,CAAC,QAAQ,CAAC,CAAC;QAC1C,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE,QAAQ,EAAE,QAAQ,GAAG,QAAQ,CAAC,CAAC;QACjE,MAAM,OAAO,GAAG,IAAI,YAAY,CAAC,OAAO,CAAC,CAAC;QAC1C,MAAM,SAAS,GAAG,IAAI,GAAG,EAAkB,CAAC;QAC5C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE;YAAE,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAClF,MAAM,KAAK,GAAmB;YAC5B,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,GAAG,EAAE,MAAM,CAAC,GAAG;YACf,OAAO,EAAE,MAAM,CAAC,QAAQ;YACxB,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,OAAO;YACP,SAAS;SACV,CAAC;QACF,UAAU,CAAC,GAAG,CAAC,SAAS,EAAE,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;QAC7E,OAAO,KAAK,CAAC;IACf,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO;YACL,KAAK,EAAE;gBACL,MAAM,EAAE,SAAS;gBACjB,OAAO,EAAE,+BAA+B,SAAS,MAAM,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,qCAAqC;aAC7I;SACF,CAAC;IACJ,CAAC;AACH,CAAC;AAED,+CAA+C;AAC/C,MAAM,UAAU,wBAAwB;IACtC,UAAU,CAAC,KAAK,EAAE,CAAC;AACrB,CAAC"}
@@ -0,0 +1,75 @@
1
+ /**
2
+ * embeddings/indexer.ts — fix7: incremental, content-hash-keyed index build.
3
+ *
4
+ * INCREMENTAL BY CONSTRUCTION: chunks are keyed by sha256(content) (see
5
+ * embeddings/chunker.ts), so a rebuild embeds ONLY hashes not already in
6
+ * the on-disk index — unchanged content is never re-embedded, and renaming/
7
+ * moving a file costs nothing (path-independent keys; the repo's
8
+ * content-hash cache pattern). `force: true` re-embeds everything.
9
+ *
10
+ * PRUNING: a FULL build (no `projects` filter) enumerates the complete
11
+ * retrievable corpus, so hashes absent from the enumeration are dropped
12
+ * (stale vectors from deleted/edited content). A project-SCOPED build only
13
+ * adds/updates — it cannot know whether an unrecognized hash belongs to an
14
+ * unenumerated project, so it never prunes (documented CLI behavior).
15
+ *
16
+ * NEVER runs on the recall path — index building is a CLI/explicit-API
17
+ * operation only (`ar embeddings rebuild`); recall reads the index and
18
+ * degrades if it is missing/stale (retrieval/semantic-leg.ts).
19
+ */
20
+ import { type EmbeddingModelSpec } from "./config.js";
21
+ export interface BuildEmbeddingsOptions {
22
+ /** Restrict to these project slugs (default: every dir under projects/,
23
+ * `_`-namespace excluded — the same BY-NAME reserved-namespace rule the
24
+ * rest of the store uses). */
25
+ projects?: string[];
26
+ /** Re-embed every chunk even if its hash is already indexed. */
27
+ force?: boolean;
28
+ /** Model override (default: resolveEmbeddingModel()). */
29
+ model?: EmbeddingModelSpec;
30
+ /** Progress callback (CLI renders it; embedding a large store takes
31
+ * minutes on first build). */
32
+ onProgress?: (done: number, total: number) => void;
33
+ /** Embed batch size (texts per model call). */
34
+ batchSize?: number;
35
+ }
36
+ export interface BuildEmbeddingsReport {
37
+ ok: boolean;
38
+ error?: {
39
+ reason: string;
40
+ message: string;
41
+ };
42
+ model: string;
43
+ dim: number;
44
+ indexPath?: string;
45
+ projects: string[];
46
+ totalChunks: number;
47
+ embeddedNew: number;
48
+ reused: number;
49
+ pruned: number;
50
+ durationMs: number;
51
+ }
52
+ /**
53
+ * Build (or incrementally update) the embedding index.
54
+ *
55
+ * The write is one atomic, locked replacement of the whole index file
56
+ * (embeddings/index-store.ts) — a crash mid-build leaves the previous index
57
+ * intact, and concurrent recalls keep reading the old file until rename.
58
+ */
59
+ export declare function buildEmbeddingsIndex(opts?: BuildEmbeddingsOptions): Promise<BuildEmbeddingsReport>;
60
+ export interface EmbeddingsStatus {
61
+ enabled: boolean;
62
+ model: string;
63
+ dim: number;
64
+ runtimeInstalled: boolean;
65
+ modelCached: boolean;
66
+ indexPath: string;
67
+ indexExists: boolean;
68
+ indexCount?: number;
69
+ indexBuiltAt?: string;
70
+ indexBytes?: number;
71
+ indexError?: string;
72
+ }
73
+ /** Diagnostic snapshot for `ar embeddings status`. Read-only. */
74
+ export declare function embeddingsStatus(): Promise<EmbeddingsStatus>;
75
+ //# sourceMappingURL=indexer.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"indexer.d.ts","sourceRoot":"","sources":["../../src/embeddings/indexer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAMH,OAAO,EAAyB,KAAK,kBAAkB,EAAE,MAAM,aAAa,CAAC;AAG7E,MAAM,WAAW,sBAAsB;IACrC;;mCAE+B;IAC/B,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;IACpB,gEAAgE;IAChE,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,yDAAyD;IACzD,KAAK,CAAC,EAAE,kBAAkB,CAAC;IAC3B;mCAC+B;IAC/B,UAAU,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;IACnD,+CAA+C;IAC/C,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,qBAAqB;IACpC,EAAE,EAAE,OAAO,CAAC;IACZ,KAAK,CAAC,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC;IAC5C,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;CACpB;AAgBD;;;;;;GAMG;AACH,wBAAsB,oBAAoB,CAAC,IAAI,GAAE,sBAA2B,GAAG,OAAO,CAAC,qBAAqB,CAAC,CAsH5G;AAED,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,gBAAgB,EAAE,OAAO,CAAC;IAC1B,WAAW,EAAE,OAAO,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,OAAO,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,iEAAiE;AACjE,wBAAsB,gBAAgB,IAAI,OAAO,CAAC,gBAAgB,CAAC,CA2BlE"}