agent-recall-core 3.4.48 → 3.4.49

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (215) hide show
  1. package/README.md +25 -0
  2. package/dist/digest/store.d.ts +15 -6
  3. package/dist/digest/store.d.ts.map +1 -1
  4. package/dist/digest/store.js +33 -20
  5. package/dist/digest/store.js.map +1 -1
  6. package/dist/embeddings/chunker.d.ts +102 -0
  7. package/dist/embeddings/chunker.d.ts.map +1 -0
  8. package/dist/embeddings/chunker.js +260 -0
  9. package/dist/embeddings/chunker.js.map +1 -0
  10. package/dist/embeddings/config.d.ts +93 -0
  11. package/dist/embeddings/config.d.ts.map +1 -0
  12. package/dist/embeddings/config.js +174 -0
  13. package/dist/embeddings/config.js.map +1 -0
  14. package/dist/embeddings/index-store.d.ts +61 -0
  15. package/dist/embeddings/index-store.d.ts.map +1 -0
  16. package/dist/embeddings/index-store.js +158 -0
  17. package/dist/embeddings/index-store.js.map +1 -0
  18. package/dist/embeddings/indexer.d.ts +75 -0
  19. package/dist/embeddings/indexer.d.ts.map +1 -0
  20. package/dist/embeddings/indexer.js +196 -0
  21. package/dist/embeddings/indexer.js.map +1 -0
  22. package/dist/embeddings/runtime.d.ts +66 -0
  23. package/dist/embeddings/runtime.d.ts.map +1 -0
  24. package/dist/embeddings/runtime.js +197 -0
  25. package/dist/embeddings/runtime.js.map +1 -0
  26. package/dist/helpers/associative-link.d.ts.map +1 -1
  27. package/dist/helpers/associative-link.js +26 -1
  28. package/dist/helpers/associative-link.js.map +1 -1
  29. package/dist/helpers/tokenize.d.ts +1 -0
  30. package/dist/helpers/tokenize.d.ts.map +1 -1
  31. package/dist/helpers/tokenize.js +18 -0
  32. package/dist/helpers/tokenize.js.map +1 -1
  33. package/dist/index.d.ts +21 -5
  34. package/dist/index.d.ts.map +1 -1
  35. package/dist/index.js +38 -4
  36. package/dist/index.js.map +1 -1
  37. package/dist/palace/awareness.d.ts +34 -10
  38. package/dist/palace/awareness.d.ts.map +1 -1
  39. package/dist/palace/awareness.js +249 -180
  40. package/dist/palace/awareness.js.map +1 -1
  41. package/dist/palace/consolidate.d.ts +1 -1
  42. package/dist/palace/consolidate.d.ts.map +1 -1
  43. package/dist/palace/consolidate.js +9 -9
  44. package/dist/palace/consolidate.js.map +1 -1
  45. package/dist/palace/decay-pass.d.ts +1 -1
  46. package/dist/palace/decay-pass.d.ts.map +1 -1
  47. package/dist/palace/decay-pass.js +2 -2
  48. package/dist/palace/decay-pass.js.map +1 -1
  49. package/dist/palace/fan-out.d.ts +1 -1
  50. package/dist/palace/fan-out.d.ts.map +1 -1
  51. package/dist/palace/fan-out.js +5 -5
  52. package/dist/palace/fan-out.js.map +1 -1
  53. package/dist/palace/index-manager.d.ts +1 -1
  54. package/dist/palace/index-manager.d.ts.map +1 -1
  55. package/dist/palace/index-manager.js +1 -1
  56. package/dist/palace/index-manager.js.map +1 -1
  57. package/dist/palace/insights-index.d.ts +19 -6
  58. package/dist/palace/insights-index.d.ts.map +1 -1
  59. package/dist/palace/insights-index.js +36 -13
  60. package/dist/palace/insights-index.js.map +1 -1
  61. package/dist/palace/keystone.d.ts +1 -1
  62. package/dist/palace/keystone.d.ts.map +1 -1
  63. package/dist/palace/keystone.js +2 -2
  64. package/dist/palace/keystone.js.map +1 -1
  65. package/dist/palace/rooms.d.ts +3 -3
  66. package/dist/palace/rooms.d.ts.map +1 -1
  67. package/dist/palace/rooms.js +34 -6
  68. package/dist/palace/rooms.js.map +1 -1
  69. package/dist/retrieval/candidates.d.ts.map +1 -1
  70. package/dist/retrieval/candidates.js +17 -2
  71. package/dist/retrieval/candidates.js.map +1 -1
  72. package/dist/retrieval/query-memory.d.ts +65 -7
  73. package/dist/retrieval/query-memory.d.ts.map +1 -1
  74. package/dist/retrieval/query-memory.js +518 -78
  75. package/dist/retrieval/query-memory.js.map +1 -1
  76. package/dist/retrieval/semantic-leg.d.ts +74 -0
  77. package/dist/retrieval/semantic-leg.d.ts.map +1 -0
  78. package/dist/retrieval/semantic-leg.js +180 -0
  79. package/dist/retrieval/semantic-leg.js.map +1 -0
  80. package/dist/storage/archive-write.d.ts.map +1 -1
  81. package/dist/storage/archive-write.js +29 -8
  82. package/dist/storage/archive-write.js.map +1 -1
  83. package/dist/storage/capture-router.d.ts +1 -1
  84. package/dist/storage/capture-router.d.ts.map +1 -1
  85. package/dist/storage/capture-router.js +26 -9
  86. package/dist/storage/capture-router.js.map +1 -1
  87. package/dist/storage/consolidation-queue.d.ts +11 -2
  88. package/dist/storage/consolidation-queue.d.ts.map +1 -1
  89. package/dist/storage/consolidation-queue.js +45 -16
  90. package/dist/storage/consolidation-queue.js.map +1 -1
  91. package/dist/storage/corrections.d.ts +53 -5
  92. package/dist/storage/corrections.d.ts.map +1 -1
  93. package/dist/storage/corrections.js +412 -124
  94. package/dist/storage/corrections.js.map +1 -1
  95. package/dist/storage/durable-intent.d.ts.map +1 -1
  96. package/dist/storage/durable-intent.js +24 -1
  97. package/dist/storage/durable-intent.js.map +1 -1
  98. package/dist/storage/filelock.d.ts +84 -12
  99. package/dist/storage/filelock.d.ts.map +1 -1
  100. package/dist/storage/filelock.js +309 -47
  101. package/dist/storage/filelock.js.map +1 -1
  102. package/dist/storage/fs-utils.d.ts +20 -0
  103. package/dist/storage/fs-utils.d.ts.map +1 -1
  104. package/dist/storage/fs-utils.js +27 -0
  105. package/dist/storage/fs-utils.js.map +1 -1
  106. package/dist/storage/paths.d.ts +34 -0
  107. package/dist/storage/paths.d.ts.map +1 -1
  108. package/dist/storage/paths.js +71 -0
  109. package/dist/storage/paths.js.map +1 -1
  110. package/dist/storage/pending.d.ts +154 -0
  111. package/dist/storage/pending.d.ts.map +1 -0
  112. package/dist/storage/pending.js +454 -0
  113. package/dist/storage/pending.js.map +1 -0
  114. package/dist/storage/project.d.ts.map +1 -1
  115. package/dist/storage/project.js +68 -16
  116. package/dist/storage/project.js.map +1 -1
  117. package/dist/storage/session-card.d.ts +13 -0
  118. package/dist/storage/session-card.d.ts.map +1 -1
  119. package/dist/storage/session-card.js +34 -3
  120. package/dist/storage/session-card.js.map +1 -1
  121. package/dist/storage/unclaimed.d.ts +165 -0
  122. package/dist/storage/unclaimed.d.ts.map +1 -0
  123. package/dist/storage/unclaimed.js +442 -0
  124. package/dist/storage/unclaimed.js.map +1 -0
  125. package/dist/storage/working-memory.d.ts.map +1 -1
  126. package/dist/storage/working-memory.js +19 -1
  127. package/dist/storage/working-memory.js.map +1 -1
  128. package/dist/supabase/recall-backend.d.ts +28 -2
  129. package/dist/supabase/recall-backend.d.ts.map +1 -1
  130. package/dist/supabase/recall-backend.js +80 -23
  131. package/dist/supabase/recall-backend.js.map +1 -1
  132. package/dist/supabase/sync.d.ts.map +1 -1
  133. package/dist/supabase/sync.js +13 -0
  134. package/dist/supabase/sync.js.map +1 -1
  135. package/dist/tools-logic/alignment-check.js +1 -1
  136. package/dist/tools-logic/alignment-check.js.map +1 -1
  137. package/dist/tools-logic/awareness-update.d.ts.map +1 -1
  138. package/dist/tools-logic/awareness-update.js +25 -16
  139. package/dist/tools-logic/awareness-update.js.map +1 -1
  140. package/dist/tools-logic/check-action.js +1 -1
  141. package/dist/tools-logic/check-action.js.map +1 -1
  142. package/dist/tools-logic/check.d.ts +55 -1
  143. package/dist/tools-logic/check.d.ts.map +1 -1
  144. package/dist/tools-logic/check.js +230 -57
  145. package/dist/tools-logic/check.js.map +1 -1
  146. package/dist/tools-logic/confidence.d.ts +11 -6
  147. package/dist/tools-logic/confidence.d.ts.map +1 -1
  148. package/dist/tools-logic/confidence.js +11 -6
  149. package/dist/tools-logic/confidence.js.map +1 -1
  150. package/dist/tools-logic/context-synthesize.js +2 -2
  151. package/dist/tools-logic/context-synthesize.js.map +1 -1
  152. package/dist/tools-logic/digest-read.js +2 -2
  153. package/dist/tools-logic/digest-read.js.map +1 -1
  154. package/dist/tools-logic/digest-recall.js +1 -1
  155. package/dist/tools-logic/digest-recall.js.map +1 -1
  156. package/dist/tools-logic/insight-promotion.d.ts +1 -1
  157. package/dist/tools-logic/insight-promotion.d.ts.map +1 -1
  158. package/dist/tools-logic/insight-promotion.js +17 -6
  159. package/dist/tools-logic/insight-promotion.js.map +1 -1
  160. package/dist/tools-logic/journal-capture.js +3 -3
  161. package/dist/tools-logic/journal-capture.js.map +1 -1
  162. package/dist/tools-logic/journal-write.js +3 -3
  163. package/dist/tools-logic/journal-write.js.map +1 -1
  164. package/dist/tools-logic/knowledge-write.js +2 -2
  165. package/dist/tools-logic/knowledge-write.js.map +1 -1
  166. package/dist/tools-logic/palace-lint.js +1 -1
  167. package/dist/tools-logic/palace-lint.js.map +1 -1
  168. package/dist/tools-logic/palace-read.js +1 -1
  169. package/dist/tools-logic/palace-read.js.map +1 -1
  170. package/dist/tools-logic/palace-search.js +1 -1
  171. package/dist/tools-logic/palace-search.js.map +1 -1
  172. package/dist/tools-logic/palace-walk.js +3 -3
  173. package/dist/tools-logic/palace-walk.js.map +1 -1
  174. package/dist/tools-logic/palace-write.js +4 -4
  175. package/dist/tools-logic/palace-write.js.map +1 -1
  176. package/dist/tools-logic/predict-correction.js +1 -1
  177. package/dist/tools-logic/predict-correction.js.map +1 -1
  178. package/dist/tools-logic/recall-backend.d.ts +11 -2
  179. package/dist/tools-logic/recall-backend.d.ts.map +1 -1
  180. package/dist/tools-logic/recall-backend.js +2 -2
  181. package/dist/tools-logic/recall-backend.js.map +1 -1
  182. package/dist/tools-logic/resurrect.d.ts.map +1 -1
  183. package/dist/tools-logic/resurrect.js +9 -1
  184. package/dist/tools-logic/resurrect.js.map +1 -1
  185. package/dist/tools-logic/safety-consolidation.d.ts.map +1 -1
  186. package/dist/tools-logic/safety-consolidation.js +32 -30
  187. package/dist/tools-logic/safety-consolidation.js.map +1 -1
  188. package/dist/tools-logic/session-end.d.ts +8 -0
  189. package/dist/tools-logic/session-end.d.ts.map +1 -1
  190. package/dist/tools-logic/session-end.js +51 -13
  191. package/dist/tools-logic/session-end.js.map +1 -1
  192. package/dist/tools-logic/session-start.d.ts +23 -0
  193. package/dist/tools-logic/session-start.d.ts.map +1 -1
  194. package/dist/tools-logic/session-start.js +44 -1
  195. package/dist/tools-logic/session-start.js.map +1 -1
  196. package/dist/tools-logic/smart-recall.d.ts +185 -10
  197. package/dist/tools-logic/smart-recall.d.ts.map +1 -1
  198. package/dist/tools-logic/smart-recall.js +363 -85
  199. package/dist/tools-logic/smart-recall.js.map +1 -1
  200. package/dist/tools-logic/store-doctor.d.ts.map +1 -1
  201. package/dist/tools-logic/store-doctor.js +7 -4
  202. package/dist/tools-logic/store-doctor.js.map +1 -1
  203. package/dist/tools-logic/store-repair.js +1 -1
  204. package/dist/tools-logic/store-repair.js.map +1 -1
  205. package/dist/tools-logic/supersession.d.ts +1 -1
  206. package/dist/tools-logic/supersession.d.ts.map +1 -1
  207. package/dist/tools-logic/supersession.js +2 -2
  208. package/dist/tools-logic/supersession.js.map +1 -1
  209. package/dist/types.d.ts +1 -1
  210. package/dist/types.js +1 -1
  211. package/dist/vector/embedding.d.ts +8 -0
  212. package/dist/vector/embedding.d.ts.map +1 -1
  213. package/dist/vector/embedding.js +8 -0
  214. package/dist/vector/embedding.js.map +1 -1
  215. package/package.json +1 -1
@@ -0,0 +1,196 @@
1
+ /**
2
+ * embeddings/indexer.ts — fix7: incremental, content-hash-keyed index build.
3
+ *
4
+ * INCREMENTAL BY CONSTRUCTION: chunks are keyed by sha256(content) (see
5
+ * embeddings/chunker.ts), so a rebuild embeds ONLY hashes not already in
6
+ * the on-disk index — unchanged content is never re-embedded, and renaming/
7
+ * moving a file costs nothing (path-independent keys; the repo's
8
+ * content-hash cache pattern). `force: true` re-embeds everything.
9
+ *
10
+ * PRUNING: a FULL build (no `projects` filter) enumerates the complete
11
+ * retrievable corpus, so hashes absent from the enumeration are dropped
12
+ * (stale vectors from deleted/edited content). A project-SCOPED build only
13
+ * adds/updates — it cannot know whether an unrecognized hash belongs to an
14
+ * unenumerated project, so it never prunes (documented CLI behavior).
15
+ *
16
+ * NEVER runs on the recall path — index building is a CLI/explicit-API
17
+ * operation only (`ar embeddings rebuild`); recall reads the index and
18
+ * degrades if it is missing/stale (retrieval/semantic-leg.ts).
19
+ */
20
+ import * as fs from "node:fs";
21
+ import { projectsRootDir } from "../storage/paths.js";
22
+ import { chunkProject, chunkGlobalInsights } from "./chunker.js";
23
+ import { getEmbedder } from "./runtime.js";
24
+ import { resolveEmbeddingModel } from "./config.js";
25
+ import { readEmbeddingIndex, writeEmbeddingIndex } from "./index-store.js";
26
+ function listProjectDirs() {
27
+ try {
28
+ const root = projectsRootDir();
29
+ if (!fs.existsSync(root))
30
+ return [];
31
+ return fs
32
+ .readdirSync(root, { withFileTypes: true })
33
+ .filter((e) => e.isDirectory() && !e.name.startsWith("_"))
34
+ .map((e) => e.name)
35
+ .sort();
36
+ }
37
+ catch {
38
+ return [];
39
+ }
40
+ }
41
+ /**
42
+ * Build (or incrementally update) the embedding index.
43
+ *
44
+ * The write is one atomic, locked replacement of the whole index file
45
+ * (embeddings/index-store.ts) — a crash mid-build leaves the previous index
46
+ * intact, and concurrent recalls keep reading the old file until rename.
47
+ */
48
+ export async function buildEmbeddingsIndex(opts = {}) {
49
+ const t0 = performance.now();
50
+ const spec = opts.model ?? resolveEmbeddingModel();
51
+ const scoped = Array.isArray(opts.projects) && opts.projects.length > 0;
52
+ const projects = scoped ? opts.projects : listProjectDirs();
53
+ const report = {
54
+ ok: false,
55
+ model: spec.id,
56
+ dim: spec.dim,
57
+ projects,
58
+ totalChunks: 0,
59
+ embeddedNew: 0,
60
+ reused: 0,
61
+ pruned: 0,
62
+ durationMs: 0,
63
+ };
64
+ // Rebuild is offline-strict: the ONLY sanctioned downloader is
65
+ // `ar embeddings setup` (embeddings/runtime.ts's network contract).
66
+ const embedder = await getEmbedder(spec, { allowRemote: false });
67
+ if ("error" in embedder) {
68
+ report.error = embedder.error;
69
+ report.durationMs = performance.now() - t0;
70
+ return report;
71
+ }
72
+ // ---- enumerate + dedupe chunks (hash-keyed; identical content once) ----
73
+ const chunksByHash = new Map();
74
+ for (const project of projects) {
75
+ for (const chunk of chunkProject(project, { skipInsights: true })) {
76
+ if (!chunksByHash.has(chunk.hash))
77
+ chunksByHash.set(chunk.hash, chunk);
78
+ }
79
+ }
80
+ for (const chunk of chunkGlobalInsights()) {
81
+ if (!chunksByHash.has(chunk.hash))
82
+ chunksByHash.set(chunk.hash, chunk);
83
+ }
84
+ report.totalChunks = chunksByHash.size;
85
+ // ---- reuse existing vectors (content-hash cache hit) ----
86
+ // fix7 review M1 (2026-09-12): the existing index is ALWAYS read first.
87
+ // - scoped build over a CORRUPT/mismatched index → REFUSE: a scoped
88
+ // build cannot preserve other projects' vectors it cannot read, and
89
+ // proceeding would silently narrow the index to the scoped project
90
+ // while reporting pruned: 0 (the reviewer's reproduced data-loss bug).
91
+ // A MISSING index is fine (fresh partial build, coverage diagnosable).
92
+ // - scoped + force → re-embed the scoped chunk set but PRESERVE every
93
+ // foreign hash (previously `force` dropped the whole existing index).
94
+ // - full + force → rebuild from scratch (unchanged).
95
+ const existing = readEmbeddingIndex(spec);
96
+ const existingErr = "error" in existing ? existing.error : null;
97
+ const existingOk = existingErr ? null : existing;
98
+ if (scoped && existingErr && existingErr.reason !== "missing") {
99
+ report.error = {
100
+ reason: "existing-index-unreadable",
101
+ message: `existing index unreadable (${existingErr.message}) — a project-scoped build cannot ` +
102
+ `preserve other projects' vectors from an unreadable index; run a full \`ar embeddings rebuild\``,
103
+ };
104
+ report.durationMs = performance.now() - t0;
105
+ return report;
106
+ }
107
+ // Which existing index (if any) backs the scoped-preserve block below,
108
+ // and whether CURRENT-set hashes may reuse cached vectors.
109
+ const reusable = existingOk && (scoped || !opts.force) ? existingOk : null;
110
+ const reuseCurrent = !opts.force;
111
+ const outHashes = [];
112
+ const rows = [];
113
+ const toEmbed = [];
114
+ for (const [hash, chunk] of chunksByHash) {
115
+ const row = reuseCurrent ? reusable?.rowByHash.get(hash) : undefined;
116
+ if (row !== undefined) {
117
+ outHashes.push(hash);
118
+ rows.push(reusable.vectors.subarray(row * spec.dim, (row + 1) * spec.dim));
119
+ report.reused++;
120
+ }
121
+ else {
122
+ toEmbed.push(chunk);
123
+ }
124
+ }
125
+ // Scoped builds keep every unrecognized existing hash (cannot prove it
126
+ // stale — see file header); full builds drop them (that IS the prune).
127
+ if (reusable && scoped) {
128
+ const current = new Set(outHashes);
129
+ for (let i = 0; i < reusable.hashes.length; i++) {
130
+ const hash = reusable.hashes[i];
131
+ if (current.has(hash) || chunksByHash.has(hash))
132
+ continue;
133
+ outHashes.push(hash);
134
+ rows.push(reusable.vectors.subarray(i * spec.dim, (i + 1) * spec.dim));
135
+ }
136
+ }
137
+ else if (reusable) {
138
+ report.pruned = reusable.hashes.length - report.reused;
139
+ }
140
+ // ---- embed the new chunks in batches ----
141
+ const batchSize = opts.batchSize ?? 16;
142
+ let done = 0;
143
+ for (let i = 0; i < toEmbed.length; i += batchSize) {
144
+ const batch = toEmbed.slice(i, i + batchSize);
145
+ const vecs = await embedder.embedPassages(batch.map((c) => c.text));
146
+ for (let j = 0; j < batch.length; j++) {
147
+ if (!vecs[j] || vecs[j].length !== spec.dim)
148
+ continue; // defensive — skip malformed rows
149
+ outHashes.push(batch[j].hash);
150
+ rows.push(vecs[j]);
151
+ report.embeddedNew++;
152
+ }
153
+ done += batch.length;
154
+ opts.onProgress?.(done, toEmbed.length);
155
+ }
156
+ // ---- write (locked, atomic) ----
157
+ const flat = new Float32Array(outHashes.length * spec.dim);
158
+ for (let i = 0; i < rows.length; i++)
159
+ flat.set(rows[i], i * spec.dim);
160
+ report.indexPath = await writeEmbeddingIndex(spec, outHashes, flat);
161
+ report.ok = true;
162
+ report.durationMs = performance.now() - t0;
163
+ return report;
164
+ }
165
+ /** Diagnostic snapshot for `ar embeddings status`. Read-only. */
166
+ export async function embeddingsStatus() {
167
+ const { embeddingsEnabled, embeddingsIndexPath } = await import("./config.js");
168
+ const { runtimeInstalled, modelCached } = await import("./runtime.js");
169
+ const spec = resolveEmbeddingModel();
170
+ const indexPath = embeddingsIndexPath(spec);
171
+ const status = {
172
+ enabled: embeddingsEnabled(),
173
+ model: spec.id,
174
+ dim: spec.dim,
175
+ runtimeInstalled: runtimeInstalled(),
176
+ modelCached: modelCached(spec),
177
+ indexPath,
178
+ indexExists: fs.existsSync(indexPath),
179
+ };
180
+ if (status.indexExists) {
181
+ try {
182
+ status.indexBytes = fs.statSync(indexPath).size;
183
+ }
184
+ catch { /* best-effort */ }
185
+ const index = readEmbeddingIndex(spec);
186
+ if ("error" in index) {
187
+ status.indexError = index.error.message;
188
+ }
189
+ else {
190
+ status.indexCount = index.hashes.length;
191
+ status.indexBuiltAt = index.builtAt;
192
+ }
193
+ }
194
+ return status;
195
+ }
196
+ //# sourceMappingURL=indexer.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"indexer.js","sourceRoot":"","sources":["../../src/embeddings/indexer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAEH,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AACtD,OAAO,EAAE,YAAY,EAAE,mBAAmB,EAAuB,MAAM,cAAc,CAAC;AACtF,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAC3C,OAAO,EAAE,qBAAqB,EAA2B,MAAM,aAAa,CAAC;AAC7E,OAAO,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,MAAM,kBAAkB,CAAC;AAgC3E,SAAS,eAAe;IACtB,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,eAAe,EAAE,CAAC;QAC/B,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC;YAAE,OAAO,EAAE,CAAC;QACpC,OAAO,EAAE;aACN,WAAW,CAAC,IAAI,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC;aAC1C,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;aACzD,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;aAClB,IAAI,EAAE,CAAC;IACZ,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,OAA+B,EAAE;IAC1E,MAAM,EAAE,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;IAC7B,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,IAAI,qBAAqB,EAAE,CAAC;IACnD,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC;IACxE,MAAM,QAAQ,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,QAAS,CAAC,CAAC,CAAC,eAAe,EAAE,CAAC;IAE7D,MAAM,MAAM,GAA0B;QACpC,EAAE,EAAE,KAAK;QACT,KAAK,EAAE,IAAI,CAAC,EAAE;QACd,GAAG,EAAE,IAAI,CAAC,GAAG;QACb,QAAQ;QACR,WAAW,EAAE,CAAC;QACd,WAAW,EAAE,CAAC;QACd,MAAM,EAAE,CAAC;QACT,MAAM,EAAE,CAAC;QACT,UAAU,EAAE,CAAC;KACd,CAAC;IAEF,+DAA+D;IAC/D,oEAAoE;IACpE,MAAM,QAAQ,GAAG,MAAM,WAAW,CAAC,IAAI,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC;IACjE,IAAI,OAAO,IAAI,QAAQ,EAAE,CAAC;QACxB,MAAM,CAAC,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC;QAC9B,MAAM,CAAC,UAAU,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC;QAC3C,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,2EAA2E;IAC3E,MAAM,YAAY,GAAG,IAAI,GAAG,EAA0B,CAAC;IACvD,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,KAAK,MAAM,KAAK,IAAI,YAAY,CAAC,OAAO,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;YAClE,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC;gBAAE,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QACzE,CAAC;IACH,CAAC;IACD,KAAK,MAAM,KAAK,IAAI,mBAAmB,EAAE,EAAE,CAAC;QAC1C,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC;YAAE,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IACzE,CAAC;IACD,MAAM,CAAC,WAAW,GAAG,YAAY,CAAC,IAAI,CAAC;IAEvC,4DAA4D;IAC5D,wEAAwE;IACxE,qEAAqE;IACrE,uEAAuE;IACvE,sEAAsE;IACtE,0EAA0E;IAC1E,0EAA0E;IAC1E,uEAAuE;IACvE,yEAAyE;IACzE,sDAAsD;IACtD,MAAM,QAAQ,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAC;IAC1C,MAAM,WAAW,GAAG,OAAO,IAAI,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC;IAChE,MAAM,UAAU,GAAG,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAE,QAAyD,CAAC;IACnG,IAAI,MAAM,IAAI,WAAW,IAAI,WAAW,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;QAC9D,MAAM,CAAC,KAAK,GAAG;YACb,MAAM,EAAE,2BAA2B;YACnC,OAAO,EACL,8BAA8B,WAAW,CAAC,OAAO,oCAAoC;gBACrF,iGAAiG;SACpG,CAAC;QACF,MAAM,CAAC,UAAU,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC;QAC3C,OAAO,MAAM,CAAC;IAChB,CAAC;IACD,uEAAuE;IACvE,2DAA2D;IAC3D,MAAM,QAAQ,GAAG,UAAU,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC;IAC3E,MAAM,YAAY,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC;IAEjC,MAAM,SAAS,GAAa,EAAE,CAAC;IAC/B,MAAM,IAAI,GAAmB,EAAE,CAAC;IAChC,MAAM,OAAO,GAAqB,EAAE,CAAC;IACrC,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,YAAY,EAAE,CAAC;QACzC,MAAM,GAAG,GAAG,YAAY,CAAC,CAAC,CAAC,QAAQ,EAAE,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QACrE,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;YACtB,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACrB,IAAI,CAAC,IAAI,CAAC,QAAS,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;YAC5E,MAAM,CAAC,MAAM,EAAE,CAAC;QAClB,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACtB,CAAC;IACH,CAAC;IAED,uEAAuE;IACvE,uEAAuE;IACvE,IAAI,QAAQ,IAAI,MAAM,EAAE,CAAC;QACvB,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;QACnC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAChD,MAAM,IAAI,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;YAChC,IAAI,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC;gBAAE,SAAS;YAC1D,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACrB,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;QACzE,CAAC;IACH,CAAC;SAAM,IAAI,QAAQ,EAAE,CAAC;QACpB,MAAM,CAAC,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;IACzD,CAAC;IAED,4CAA4C;IAC5C,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,EAAE,CAAC;IACvC,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;QACnD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC;QAC9C,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QACpE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACtC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,KAAK,IAAI,CAAC,GAAG;gBAAE,SAAS,CAAC,kCAAkC;YACzF,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YAC9B,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;YACnB,MAAM,CAAC,WAAW,EAAE,CAAC;QACvB,CAAC;QACD,IAAI,IAAI,KAAK,CAAC,MAAM,CAAC;QACrB,IAAI,CAAC,UAAU,EAAE,CAAC,IAAI,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;IAC1C,CAAC;IAED,mCAAmC;IACnC,MAAM,IAAI,GAAG,IAAI,YAAY,CAAC,SAAS,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC;IAC3D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE;QAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC;IACtE,MAAM,CAAC,SAAS,GAAG,MAAM,mBAAmB,CAAC,IAAI,EAAE,SAAS,EAAE,IAAI,CAAC,CAAC;IACpE,MAAM,CAAC,EAAE,GAAG,IAAI,CAAC;IACjB,MAAM,CAAC,UAAU,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC;IAC3C,OAAO,MAAM,CAAC;AAChB,CAAC;AAgBD,iEAAiE;AACjE,MAAM,CAAC,KAAK,UAAU,gBAAgB;IACpC,MAAM,EAAE,iBAAiB,EAAE,mBAAmB,EAAE,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,CAAC;IAC/E,MAAM,EAAE,gBAAgB,EAAE,WAAW,EAAE,GAAG,MAAM,MAAM,CAAC,cAAc,CAAC,CAAC;IACvE,MAAM,IAAI,GAAG,qBAAqB,EAAE,CAAC;IACrC,MAAM,SAAS,GAAG,mBAAmB,CAAC,IAAI,CAAC,CAAC;IAC5C,MAAM,MAAM,GAAqB;QAC/B,OAAO,EAAE,iBAAiB,EAAE;QAC5B,KAAK,EAAE,IAAI,CAAC,EAAE;QACd,GAAG,EAAE,IAAI,CAAC,GAAG;QACb,gBAAgB,EAAE,gBAAgB,EAAE;QACpC,WAAW,EAAE,WAAW,CAAC,IAAI,CAAC;QAC9B,SAAS;QACT,WAAW,EAAE,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC;KACtC,CAAC;IACF,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC;QACvB,IAAI,CAAC;YACH,MAAM,CAAC,UAAU,GAAG,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC;QAClD,CAAC;QAAC,MAAM,CAAC,CAAC,iBAAiB,CAAC,CAAC;QAC7B,MAAM,KAAK,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAC;QACvC,IAAI,OAAO,IAAI,KAAK,EAAE,CAAC;YACrB,MAAM,CAAC,UAAU,GAAG,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC;QAC1C,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC;YACxC,MAAM,CAAC,YAAY,GAAG,KAAK,CAAC,OAAO,CAAC;QACtC,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC"}
@@ -0,0 +1,66 @@
1
+ /**
2
+ * embeddings/runtime.ts — fix7: loading the LOCAL ONNX embedder.
3
+ *
4
+ * The transformers.js runtime is NOT a dependency of this package (it pulls
5
+ * ~380MB of onnxruntime binaries — see embeddings/config.ts's header). It is
6
+ * installed once, self-contained, under `<embeddings-home>/runtime` by
7
+ * `ar embeddings setup` (the CLI owns the `npm --prefix` spawn; this module
8
+ * only LOADS from that location) and models are cached once under
9
+ * `<embeddings-home>/models`.
10
+ *
11
+ * NETWORK CONTRACT (zero-cloud, adversarial-review requirement):
12
+ * - `allowRemote: false` (the default, used by the ENTIRE recall path and
13
+ * by `ar embeddings rebuild`): transformers.js is configured with
14
+ * `env.allowRemoteModels = false` — a cached model loads from disk; a
15
+ * missing model produces a CLEAN, actionable error (never a network
16
+ * fetch). A recall can never touch the network through this feature.
17
+ * - `allowRemote: true` is passed ONLY by `ar embeddings setup`, the one
18
+ * sanctioned downloader (one-time, cached, never re-fetched when cached
19
+ * — transformers.js's cacheDir handles the skip-when-present logic).
20
+ *
21
+ * Every failure path returns a typed { error } — this module never throws
22
+ * into the recall path.
23
+ */
24
+ import { type EmbeddingModelSpec } from "./config.js";
25
+ /** Embeds a batch of texts → one normalized Float32Array per text. */
26
+ export type EmbedFn = (texts: string[]) => Promise<Float32Array[]>;
27
+ export interface Embedder {
28
+ spec: EmbeddingModelSpec;
29
+ /** Embed QUERY texts (model's query prefix applied internally). */
30
+ embedQueries: EmbedFn;
31
+ /** Embed PASSAGE/document texts (model's passage prefix applied internally). */
32
+ embedPassages: EmbedFn;
33
+ }
34
+ export interface EmbedderError {
35
+ /** Enumerated failure class — surfaced verbatim in the semantic leg's
36
+ * result note so a degraded recall is diagnosable, never silent-silent. */
37
+ reason: "runtime-missing" | "model-missing" | "load-failed";
38
+ message: string;
39
+ }
40
+ /** The npm package the setup step installs into the runtime dir. Version
41
+ * pinned to a major verified against this module's load path (v4 ships
42
+ * `dist/transformers.node.cjs` via the `require` export condition). */
43
+ export declare const RUNTIME_PACKAGE = "@huggingface/transformers";
44
+ export declare const RUNTIME_PACKAGE_RANGE = "^4.2.0";
45
+ /** Is the self-contained runtime install present? (Cheap existence check —
46
+ * `ar embeddings status` and the semantic leg's degrade note both use it.) */
47
+ export declare function runtimeInstalled(): boolean;
48
+ /** Best-effort "is this model's cache present" check (used by status/setup
49
+ * messaging only — the authoritative check is the actual load, which
50
+ * transformers.js performs against its own cache layout). */
51
+ export declare function modelCached(spec: EmbeddingModelSpec): boolean;
52
+ /**
53
+ * Get (and cache, per process) the embedder for a model spec.
54
+ *
55
+ * `allowRemote` MUST stay false everywhere except `ar embeddings setup` —
56
+ * see this file's header. Never throws: every failure is a typed
57
+ * `{ error }` the caller degrades on.
58
+ */
59
+ export declare function getEmbedder(spec: EmbeddingModelSpec, opts?: {
60
+ allowRemote?: boolean;
61
+ }): Promise<Embedder | {
62
+ error: EmbedderError;
63
+ }>;
64
+ /** Reset the per-process embedder cache (tests). */
65
+ export declare function resetEmbedderCache(): void;
66
+ //# sourceMappingURL=runtime.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"runtime.d.ts","sourceRoot":"","sources":["../../src/embeddings/runtime.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAMH,OAAO,EAGL,KAAK,kBAAkB,EACxB,MAAM,aAAa,CAAC;AAIrB,sEAAsE;AACtE,MAAM,MAAM,OAAO,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,OAAO,CAAC,YAAY,EAAE,CAAC,CAAC;AAEnE,MAAM,WAAW,QAAQ;IACvB,IAAI,EAAE,kBAAkB,CAAC;IACzB,mEAAmE;IACnE,YAAY,EAAE,OAAO,CAAC;IACtB,gFAAgF;IAChF,aAAa,EAAE,OAAO,CAAC;CACxB;AAED,MAAM,WAAW,aAAa;IAC5B;gFAC4E;IAC5E,MAAM,EAAE,iBAAiB,GAAG,eAAe,GAAG,aAAa,CAAC;IAC5D,OAAO,EAAE,MAAM,CAAC;CACjB;AAED;;wEAEwE;AACxE,eAAO,MAAM,eAAe,8BAA8B,CAAC;AAC3D,eAAO,MAAM,qBAAqB,WAAW,CAAC;AAE9C;+EAC+E;AAC/E,wBAAgB,gBAAgB,IAAI,OAAO,CAQ1C;AAED;;8DAE8D;AAC9D,wBAAgB,WAAW,CAAC,IAAI,EAAE,kBAAkB,GAAG,OAAO,CAQ7D;AAuHD;;;;;;GAMG;AACH,wBAAsB,WAAW,CAC/B,IAAI,EAAE,kBAAkB,EACxB,IAAI,GAAE;IAAE,WAAW,CAAC,EAAE,OAAO,CAAA;CAAO,GACnC,OAAO,CAAC,QAAQ,GAAG;IAAE,KAAK,EAAE,aAAa,CAAA;CAAE,CAAC,CAmB9C;AAED,oDAAoD;AACpD,wBAAgB,kBAAkB,IAAI,IAAI,CAEzC"}
@@ -0,0 +1,197 @@
1
+ /**
2
+ * embeddings/runtime.ts — fix7: loading the LOCAL ONNX embedder.
3
+ *
4
+ * The transformers.js runtime is NOT a dependency of this package (it pulls
5
+ * ~380MB of onnxruntime binaries — see embeddings/config.ts's header). It is
6
+ * installed once, self-contained, under `<embeddings-home>/runtime` by
7
+ * `ar embeddings setup` (the CLI owns the `npm --prefix` spawn; this module
8
+ * only LOADS from that location) and models are cached once under
9
+ * `<embeddings-home>/models`.
10
+ *
11
+ * NETWORK CONTRACT (zero-cloud, adversarial-review requirement):
12
+ * - `allowRemote: false` (the default, used by the ENTIRE recall path and
13
+ * by `ar embeddings rebuild`): transformers.js is configured with
14
+ * `env.allowRemoteModels = false` — a cached model loads from disk; a
15
+ * missing model produces a CLEAN, actionable error (never a network
16
+ * fetch). A recall can never touch the network through this feature.
17
+ * - `allowRemote: true` is passed ONLY by `ar embeddings setup`, the one
18
+ * sanctioned downloader (one-time, cached, never re-fetched when cached
19
+ * — transformers.js's cacheDir handles the skip-when-present logic).
20
+ *
21
+ * Every failure path returns a typed { error } — this module never throws
22
+ * into the recall path.
23
+ */
24
+ import * as fs from "node:fs";
25
+ import * as path from "node:path";
26
+ import { createRequire } from "node:module";
27
+ import { pathToFileURL } from "node:url";
28
+ import { embeddingsRuntimeDir, embeddingsModelsDir, } from "./config.js";
29
+ import { stem } from "../helpers/normalize.js";
30
+ import { tokenizeWords } from "../helpers/tokenize.js";
31
+ /** The npm package the setup step installs into the runtime dir. Version
32
+ * pinned to a major verified against this module's load path (v4 ships
33
+ * `dist/transformers.node.cjs` via the `require` export condition). */
34
+ export const RUNTIME_PACKAGE = "@huggingface/transformers";
35
+ export const RUNTIME_PACKAGE_RANGE = "^4.2.0";
36
+ /** Is the self-contained runtime install present? (Cheap existence check —
37
+ * `ar embeddings status` and the semantic leg's degrade note both use it.) */
38
+ export function runtimeInstalled() {
39
+ try {
40
+ return fs.existsSync(path.join(embeddingsRuntimeDir(), "node_modules", RUNTIME_PACKAGE, "package.json"));
41
+ }
42
+ catch {
43
+ return false;
44
+ }
45
+ }
46
+ /** Best-effort "is this model's cache present" check (used by status/setup
47
+ * messaging only — the authoritative check is the actual load, which
48
+ * transformers.js performs against its own cache layout). */
49
+ export function modelCached(spec) {
50
+ if (spec.id === "_fake-hash-bow")
51
+ return true;
52
+ try {
53
+ const dir = path.join(embeddingsModelsDir(), ...spec.hfRepo.split("/"));
54
+ return fs.existsSync(dir) && fs.readdirSync(dir).length > 0;
55
+ }
56
+ catch {
57
+ return false;
58
+ }
59
+ }
60
+ // ---------------------------------------------------------------------------
61
+ // Deterministic test embedder (`_fake-hash-bow` registry row)
62
+ // ---------------------------------------------------------------------------
63
+ /**
64
+ * Hashed bag-of-tokens pseudo-embedding: each stemmed token hashes to a
65
+ * (dimension, sign) pair; token vectors accumulate; result is L2-normalized.
66
+ * Cosine similarity ≈ token overlap — deterministic, dependency-free, and
67
+ * good enough for tests to construct "semantically close" pairs on purpose.
68
+ * NEVER a real semantic model; see the registry row's doc comment.
69
+ */
70
+ function fakeHashBowEmbed(texts, dim) {
71
+ return texts.map((text) => {
72
+ const vec = new Float32Array(dim);
73
+ for (const raw of tokenizeWords(text)) {
74
+ const tok = stem(raw);
75
+ let h = 2166136261; // FNV-1a
76
+ for (let i = 0; i < tok.length; i++) {
77
+ h ^= tok.charCodeAt(i);
78
+ h = Math.imul(h, 16777619);
79
+ }
80
+ const idx = Math.abs(h) % dim;
81
+ const sign = (h & 1) === 0 ? 1 : -1;
82
+ vec[idx] += sign;
83
+ }
84
+ let norm = 0;
85
+ for (let i = 0; i < dim; i++)
86
+ norm += vec[i] * vec[i];
87
+ norm = Math.sqrt(norm);
88
+ if (norm > 0)
89
+ for (let i = 0; i < dim; i++)
90
+ vec[i] /= norm;
91
+ return vec;
92
+ });
93
+ }
94
+ // ---------------------------------------------------------------------------
95
+ // Real embedder (transformers.js from the self-contained runtime install)
96
+ // ---------------------------------------------------------------------------
97
+ /** One embedder per (model id, allowRemote) per process — model load is the
98
+ * expensive part (~0.3-3s); recalls after the first reuse the pipeline. */
99
+ const _embedderCache = new Map();
100
+ async function loadRealEmbedder(spec, allowRemote) {
101
+ if (!runtimeInstalled()) {
102
+ return {
103
+ error: {
104
+ reason: "runtime-missing",
105
+ message: `embedding runtime not installed at ${embeddingsRuntimeDir()} — ` +
106
+ `run \`ar embeddings setup\` once (installs ${RUNTIME_PACKAGE} locally + downloads the model; nothing ships in this package)`,
107
+ },
108
+ };
109
+ }
110
+ let mod;
111
+ try {
112
+ const req = createRequire(path.join(embeddingsRuntimeDir(), "package.json"));
113
+ const resolved = req.resolve(RUNTIME_PACKAGE);
114
+ mod = await import(pathToFileURL(resolved).href);
115
+ }
116
+ catch (err) {
117
+ return {
118
+ error: {
119
+ reason: "load-failed",
120
+ message: `embedding runtime failed to load: ${err instanceof Error ? err.message : String(err)} — re-run \`ar embeddings setup\``,
121
+ },
122
+ };
123
+ }
124
+ // Configure BEFORE the model load. cacheDir keeps model files under the
125
+ // embeddings home; allowRemoteModels=false is the zero-network guarantee
126
+ // for every caller except the sanctioned setup downloader. The env
127
+ // assignments live INSIDE the try (fix7 review L): this function's
128
+ // never-throws contract must hold even against a runtime whose `env`
129
+ // export is missing/frozen.
130
+ let extractor;
131
+ try {
132
+ mod.env.cacheDir = embeddingsModelsDir();
133
+ mod.env.allowRemoteModels = allowRemote;
134
+ mod.env.allowLocalModels = true;
135
+ extractor = await mod.pipeline("feature-extraction", spec.hfRepo, { dtype: spec.dtype });
136
+ }
137
+ catch (err) {
138
+ const msg = err instanceof Error ? err.message : String(err);
139
+ const offlineMiss = /allowRemoteModels|local_files_only|not.*found.*locally/i.test(msg);
140
+ return {
141
+ error: {
142
+ reason: offlineMiss ? "model-missing" : "load-failed",
143
+ message: offlineMiss
144
+ ? `embedding model ${spec.hfRepo} is not cached locally (and remote fetch is disabled outside setup) — run \`ar embeddings setup\` once while online`
145
+ : `embedding model ${spec.hfRepo} failed to load: ${msg}`,
146
+ },
147
+ };
148
+ }
149
+ const embedWithPrefix = (prefix) => async (texts) => {
150
+ const inputs = prefix ? texts.map((t) => prefix + t) : texts;
151
+ const out = await extractor(inputs, { pooling: "mean", normalize: true });
152
+ const [n, dim] = out.dims;
153
+ const result = [];
154
+ for (let i = 0; i < n; i++) {
155
+ result.push(out.data.slice(i * dim, (i + 1) * dim));
156
+ }
157
+ return result;
158
+ };
159
+ return {
160
+ spec,
161
+ embedQueries: embedWithPrefix(spec.queryPrefix),
162
+ embedPassages: embedWithPrefix(spec.passagePrefix),
163
+ };
164
+ }
165
+ /**
166
+ * Get (and cache, per process) the embedder for a model spec.
167
+ *
168
+ * `allowRemote` MUST stay false everywhere except `ar embeddings setup` —
169
+ * see this file's header. Never throws: every failure is a typed
170
+ * `{ error }` the caller degrades on.
171
+ */
172
+ export async function getEmbedder(spec, opts = {}) {
173
+ if (spec.id === "_fake-hash-bow") {
174
+ const embed = async (texts) => fakeHashBowEmbed(texts, spec.dim);
175
+ return { spec, embedQueries: embed, embedPassages: embed };
176
+ }
177
+ const allowRemote = opts.allowRemote === true;
178
+ const key = `${spec.id}::${allowRemote ? "remote-ok" : "local-only"}`;
179
+ let pending = _embedderCache.get(key);
180
+ if (!pending) {
181
+ pending = loadRealEmbedder(spec, allowRemote);
182
+ _embedderCache.set(key, pending);
183
+ // A FAILED load must not poison the cache forever (e.g. setup runs in
184
+ // another terminal, user retries recall) — evict error results so the
185
+ // next call re-probes. Successful embedders stay cached.
186
+ pending.then((r) => {
187
+ if ("error" in r)
188
+ _embedderCache.delete(key);
189
+ }).catch(() => _embedderCache.delete(key));
190
+ }
191
+ return pending;
192
+ }
193
+ /** Reset the per-process embedder cache (tests). */
194
+ export function resetEmbedderCache() {
195
+ _embedderCache.clear();
196
+ }
197
+ //# sourceMappingURL=runtime.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"runtime.js","sourceRoot":"","sources":["../../src/embeddings/runtime.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAEH,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,EACL,oBAAoB,EACpB,mBAAmB,GAEpB,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,IAAI,EAAE,MAAM,yBAAyB,CAAC;AAC/C,OAAO,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AAoBvD;;wEAEwE;AACxE,MAAM,CAAC,MAAM,eAAe,GAAG,2BAA2B,CAAC;AAC3D,MAAM,CAAC,MAAM,qBAAqB,GAAG,QAAQ,CAAC;AAE9C;+EAC+E;AAC/E,MAAM,UAAU,gBAAgB;IAC9B,IAAI,CAAC;QACH,OAAO,EAAE,CAAC,UAAU,CAClB,IAAI,CAAC,IAAI,CAAC,oBAAoB,EAAE,EAAE,cAAc,EAAE,eAAe,EAAE,cAAc,CAAC,CACnF,CAAC;IACJ,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED;;8DAE8D;AAC9D,MAAM,UAAU,WAAW,CAAC,IAAwB;IAClD,IAAI,IAAI,CAAC,EAAE,KAAK,gBAAgB;QAAE,OAAO,IAAI,CAAC;IAC9C,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,mBAAmB,EAAE,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;QACxE,OAAO,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IAC9D,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,8EAA8E;AAC9E,8DAA8D;AAC9D,8EAA8E;AAE9E;;;;;;GAMG;AACH,SAAS,gBAAgB,CAAC,KAAe,EAAE,GAAW;IACpD,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACxB,MAAM,GAAG,GAAG,IAAI,YAAY,CAAC,GAAG,CAAC,CAAC;QAClC,KAAK,MAAM,GAAG,IAAI,aAAa,CAAC,IAAI,CAAC,EAAE,CAAC;YACtC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC;YACtB,IAAI,CAAC,GAAG,UAAU,CAAC,CAAC,SAAS;YAC7B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACpC,CAAC,IAAI,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;gBACvB,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;YAC7B,CAAC;YACD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC;YAC9B,MAAM,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACpC,GAAG,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC;QACnB,CAAC;QACD,IAAI,IAAI,GAAG,CAAC,CAAC;QACb,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE;YAAE,IAAI,IAAI,GAAG,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;QACtD,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACvB,IAAI,IAAI,GAAG,CAAC;YAAE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE;gBAAE,GAAG,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC;QAC3D,OAAO,GAAG,CAAC;IACb,CAAC,CAAC,CAAC;AACL,CAAC;AAED,8EAA8E;AAC9E,0EAA0E;AAC1E,8EAA8E;AAE9E;4EAC4E;AAC5E,MAAM,cAAc,GAAG,IAAI,GAAG,EAAwD,CAAC;AAEvF,KAAK,UAAU,gBAAgB,CAC7B,IAAwB,EACxB,WAAoB;IAEpB,IAAI,CAAC,gBAAgB,EAAE,EAAE,CAAC;QACxB,OAAO;YACL,KAAK,EAAE;gBACL,MAAM,EAAE,iBAAiB;gBACzB,OAAO,EACL,sCAAsC,oBAAoB,EAAE,KAAK;oBACjE,8CAA8C,eAAe,gEAAgE;aAChI;SACF,CAAC;IACJ,CAAC;IAED,IAAI,GAIH,CAAC;IACF,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,oBAAoB,EAAE,EAAE,cAAc,CAAC,CAAC,CAAC;QAC7E,MAAM,QAAQ,GAAG,GAAG,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC;QAC9C,GAAG,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,CAAC;IACnD,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO;YACL,KAAK,EAAE;gBACL,MAAM,EAAE,aAAa;gBACrB,OAAO,EAAE,qCAAqC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,mCAAmC;aAClI;SACF,CAAC;IACJ,CAAC;IAED,wEAAwE;IACxE,yEAAyE;IACzE,mEAAmE;IACnE,mEAAmE;IACnE,qEAAqE;IACrE,4BAA4B;IAC5B,IAAI,SAA8G,CAAC;IACnH,IAAI,CAAC;QACH,GAAG,CAAC,GAAG,CAAC,QAAQ,GAAG,mBAAmB,EAAE,CAAC;QACzC,GAAG,CAAC,GAAG,CAAC,iBAAiB,GAAG,WAAW,CAAC;QACxC,GAAG,CAAC,GAAG,CAAC,gBAAgB,GAAG,IAAI,CAAC;QAChC,SAAS,GAAG,MAAM,GAAG,CAAC,QAAQ,CAAC,oBAAoB,EAAE,IAAI,CAAC,MAAM,EAAE,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC;IAC3F,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAC7D,MAAM,WAAW,GAAG,yDAAyD,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACxF,OAAO;YACL,KAAK,EAAE;gBACL,MAAM,EAAE,WAAW,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,aAAa;gBACrD,OAAO,EAAE,WAAW;oBAClB,CAAC,CAAC,mBAAmB,IAAI,CAAC,MAAM,qHAAqH;oBACrJ,CAAC,CAAC,mBAAmB,IAAI,CAAC,MAAM,oBAAoB,GAAG,EAAE;aAC5D;SACF,CAAC;IACJ,CAAC;IAED,MAAM,eAAe,GAAG,CAAC,MAAc,EAAW,EAAE,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE;QACnE,MAAM,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;QAC7D,MAAM,GAAG,GAAG,MAAM,SAAS,CAAC,MAAM,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC1E,MAAM,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,GAAG,CAAC,IAAI,CAAC;QAC1B,MAAM,MAAM,GAAmB,EAAE,CAAC;QAClC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3B,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC;QACtD,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC,CAAC;IAEF,OAAO;QACL,IAAI;QACJ,YAAY,EAAE,eAAe,CAAC,IAAI,CAAC,WAAW,CAAC;QAC/C,aAAa,EAAE,eAAe,CAAC,IAAI,CAAC,aAAa,CAAC;KACnD,CAAC;AACJ,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,IAAwB,EACxB,OAAkC,EAAE;IAEpC,IAAI,IAAI,CAAC,EAAE,KAAK,gBAAgB,EAAE,CAAC;QACjC,MAAM,KAAK,GAAY,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,gBAAgB,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;QAC1E,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,KAAK,EAAE,aAAa,EAAE,KAAK,EAAE,CAAC;IAC7D,CAAC;IACD,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,KAAK,IAAI,CAAC;IAC9C,MAAM,GAAG,GAAG,GAAG,IAAI,CAAC,EAAE,KAAK,WAAW,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,YAAY,EAAE,CAAC;IACtE,IAAI,OAAO,GAAG,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACtC,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,OAAO,GAAG,gBAAgB,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;QAC9C,cAAc,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QACjC,sEAAsE;QACtE,sEAAsE;QACtE,yDAAyD;QACzD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE;YACjB,IAAI,OAAO,IAAI,CAAC;gBAAE,cAAc,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAC/C,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,cAAc,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;IAC7C,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,oDAAoD;AACpD,MAAM,UAAU,kBAAkB;IAChC,cAAc,CAAC,KAAK,EAAE,CAAC;AACzB,CAAC"}
@@ -1 +1 @@
1
- {"version":3,"file":"associative-link.d.ts","sourceRoot":"","sources":["../../src/helpers/associative-link.ts"],"names":[],"mappings":"AAIA;;;GAGG;AACH,wBAAsB,aAAa,CACjC,OAAO,EAAE,MAAM,EACf,OAAO,EAAE,MAAM,EACf,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC,IAAI,CAAC,CAoBf"}
1
+ {"version":3,"file":"associative-link.d.ts","sourceRoot":"","sources":["../../src/helpers/associative-link.ts"],"names":[],"mappings":"AAIA;;;GAGG;AACH,wBAAsB,aAAa,CACjC,OAAO,EAAE,MAAM,EACf,OAAO,EAAE,MAAM,EACf,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC,IAAI,CAAC,CA6Cf"}
@@ -10,8 +10,33 @@ export async function linkToSimilar(project, content, savedSlug) {
10
10
  const pd = palaceDir(project);
11
11
  const snippet = content.slice(0, 300).replace(/\n+/g, " ");
12
12
  const results = await localRecallSearch(snippet, project, 6);
13
+ // Threshold recalibrated (fix4 S4-completion, 2026-09-11; corrected per
14
+ // independent review M2 same day): the old 0.03 was calibrated against
15
+ // palace scores INFLATED by the per-line applyRRF accumulation bug (a
16
+ // single file with >=2 matching lines summed 1/61 + 1/62 ≈ 0.033), so
17
+ // its EFFECTIVE semantics were "link only multi-evidence memories":
18
+ // >=2 same-file lines, a cross-source fusion (≈0.032), or a hot-window-
19
+ // boosted recent item (>=×1.3 ⇒ >=0.021). With one-doc-one-vote scoring
20
+ // (query-memory.ts scorePalaceTier) every single-source match scores
21
+ // 1/(60+rank) <= 0.0164 — the review caught that this fix's first cut
22
+ // (0.015) sat BELOW the whole single-source band (1/(60+6) ≈ 0.01515 at
23
+ // limit 6), turning the gate into a no-op that linked the top-3 of ANY
24
+ // keyword match on every save. 0.02 restores the original multi-
25
+ // evidence class on the un-inflated scale.
26
+ //
27
+ // fix4b (2026-09-12): the hot-window boost no longer multiplies fused
28
+ // scores on default paths (removed outright; legacy opt-in only — see
29
+ // query-memory.ts's applyLegacyHotWindowBoost, and this caller does NOT
30
+ // opt in), so the "boosted-recent single-source" leg of the old pass
31
+ // class is retired: under the multiplicative boost a LONE keyword match
32
+ // from the last 72h (1/61×1.3 = 0.0213, ×2 = 0.0328, ×3 = 0.0492 —
33
+ // every window clears 0.02) auto-linked on every save, exactly the
34
+ // single-evidence class review M2's gate exists to block. The gate is
35
+ // now strictly multi-evidence: a cross-source fusion (≥ 2/62 ≈ 0.0323)
36
+ // passes, any single-source match (≤ 0.0164, regardless of age) does
37
+ // not.
13
38
  const candidates = results
14
- .filter((r) => r.id !== savedSlug && r.score > 0.03)
39
+ .filter((r) => r.id !== savedSlug && r.score > 0.02)
15
40
  .slice(0, 3);
16
41
  for (const candidate of candidates) {
17
42
  const targetSlug = candidate.room
@@ -1 +1 @@
1
- {"version":3,"file":"associative-link.js","sourceRoot":"","sources":["../../src/helpers/associative-link.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAC7C,OAAO,EAAE,iBAAiB,EAAE,MAAM,gCAAgC,CAAC;AACnE,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAEhD;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,OAAe,EACf,OAAe,EACf,SAAiB;IAEjB,IAAI,CAAC;QACH,MAAM,EAAE,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC;QAC9B,MAAM,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QAC3D,MAAM,OAAO,GAAG,MAAM,iBAAiB,CAAC,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC;QAE7D,MAAM,UAAU,GAAG,OAAO;aACvB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,SAAS,IAAI,CAAC,CAAC,KAAK,GAAG,IAAI,CAAC;aACnD,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAEf,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;YACnC,MAAM,UAAU,GAAG,SAAS,CAAC,IAAI;gBAC/B,CAAC,CAAC,GAAG,SAAS,CAAC,IAAI,IAAI,SAAS,CAAC,EAAE,EAAE;gBACrC,CAAC,CAAC,SAAS,CAAC,EAAE,CAAC;YACjB,OAAO,CAAC,EAAE,EAAE,SAAS,EAAE,UAAU,EAAE,kBAAkB,EAAE,SAAS,CAAC,KAAK,CAAC,CAAC;YACxE,OAAO,CAAC,EAAE,EAAE,UAAU,EAAE,SAAS,EAAE,kBAAkB,EAAE,SAAS,CAAC,KAAK,CAAC,CAAC;QAC1E,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,qEAAqE;IACvE,CAAC;AACH,CAAC"}
1
+ {"version":3,"file":"associative-link.js","sourceRoot":"","sources":["../../src/helpers/associative-link.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAC7C,OAAO,EAAE,iBAAiB,EAAE,MAAM,gCAAgC,CAAC;AACnE,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAEhD;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,OAAe,EACf,OAAe,EACf,SAAiB;IAEjB,IAAI,CAAC;QACH,MAAM,EAAE,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC;QAC9B,MAAM,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QAC3D,MAAM,OAAO,GAAG,MAAM,iBAAiB,CAAC,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC;QAE7D,wEAAwE;QACxE,uEAAuE;QACvE,sEAAsE;QACtE,sEAAsE;QACtE,oEAAoE;QACpE,wEAAwE;QACxE,wEAAwE;QACxE,qEAAqE;QACrE,sEAAsE;QACtE,wEAAwE;QACxE,uEAAuE;QACvE,iEAAiE;QACjE,2CAA2C;QAC3C,EAAE;QACF,sEAAsE;QACtE,sEAAsE;QACtE,wEAAwE;QACxE,qEAAqE;QACrE,wEAAwE;QACxE,mEAAmE;QACnE,mEAAmE;QACnE,sEAAsE;QACtE,uEAAuE;QACvE,qEAAqE;QACrE,OAAO;QACP,MAAM,UAAU,GAAG,OAAO;aACvB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,SAAS,IAAI,CAAC,CAAC,KAAK,GAAG,IAAI,CAAC;aACnD,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAEf,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;YACnC,MAAM,UAAU,GAAG,SAAS,CAAC,IAAI;gBAC/B,CAAC,CAAC,GAAG,SAAS,CAAC,IAAI,IAAI,SAAS,CAAC,EAAE,EAAE;gBACrC,CAAC,CAAC,SAAS,CAAC,EAAE,CAAC;YACjB,OAAO,CAAC,EAAE,EAAE,SAAS,EAAE,UAAU,EAAE,kBAAkB,EAAE,SAAS,CAAC,KAAK,CAAC,CAAC;YACxE,OAAO,CAAC,EAAE,EAAE,UAAU,EAAE,SAAS,EAAE,kBAAkB,EAAE,SAAS,CAAC,KAAK,CAAC,CAAC;QAC1E,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,qEAAqE;IACvE,CAAC;AACH,CAAC"}
@@ -23,6 +23,7 @@
23
23
  */
24
24
  export declare const HAN_CHAR_RE: RegExp;
25
25
  export declare const HAN_RUN_RE: RegExp;
26
+ export declare const NON_ASCII_RE: RegExp;
26
27
  export interface TokenizeOptions {
27
28
  /**
28
29
  * Minimum length for non-Han (ASCII/Latin) tokens. Han (CJK) tokens are
@@ -1 +1 @@
1
- {"version":3,"file":"tokenize.d.ts","sourceRoot":"","sources":["../../src/helpers/tokenize.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAKH,eAAO,MAAM,WAAW,QAAoB,CAAC;AAC7C,eAAO,MAAM,UAAU,QAAsB,CAAC;AAgD9C,MAAM,WAAW,eAAe;IAC9B;;;;;;;;OAQG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;;;OAKG;IACH,SAAS,CAAC,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;IAChC;;;;;;;;;OASG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAgB,aAAa,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,GAAE,eAAoB,GAAG,MAAM,EAAE,CA+C7E;AAED,gFAAgF;AAChF,wBAAgB,QAAQ,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,eAAe,GAAG,GAAG,CAAC,MAAM,CAAC,CAEvE"}
1
+ {"version":3,"file":"tokenize.d.ts","sourceRoot":"","sources":["../../src/helpers/tokenize.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAKH,eAAO,MAAM,WAAW,QAAoB,CAAC;AAC7C,eAAO,MAAM,UAAU,QAAsB,CAAC;AAmB9C,eAAO,MAAM,YAAY,QAAiB,CAAC;AAgD3C,MAAM,WAAW,eAAe;IAC9B;;;;;;;;OAQG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;;;OAKG;IACH,SAAS,CAAC,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;IAChC;;;;;;;;;OASG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAgB,aAAa,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,GAAE,eAAoB,GAAG,MAAM,EAAE,CA+C7E;AAED,gFAAgF;AAChF,wBAAgB,QAAQ,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,eAAe,GAAG,GAAG,CAAC,MAAM,CAAC,CAEvE"}
@@ -26,6 +26,24 @@
26
26
  // this package's floor per root package.json "engines").
27
27
  export const HAN_CHAR_RE = /\p{Script=Han}/u;
28
28
  export const HAN_RUN_RE = /\p{Script=Han}+/gu;
29
+ // Any non-ASCII character. CLASS rule (fix #3, 2026-09-11): English-tuned
30
+ // token-length floors (`w.length >= 3`, `w.length > 3`, …) exist to drop
31
+ // low-signal short ENGLISH words — applied to non-Latin scripts they silently
32
+ // drop ordinary content words (most meaningful Chinese/Japanese/Korean words
33
+ // are 1-3 CHARACTERS). Every call site that keeps a length floor on its own
34
+ // token stream must exempt tokens matching this pattern, the same way
35
+ // `tokenizeWords` itself never length-filters Han tokens. Enumerating the
36
+ // class here (rather than testing one script per site) is what keeps
37
+ // hangul/kana/Han all covered by a single row.
38
+ //
39
+ // SCOPE (code-review 2026-09-11): this row covers length FLOORS only. Word-
40
+ // COUNT gates (e.g. awareness addInsight's `< 3 words` title gate) count
41
+ // TOKENS, and `tokenizeWords` segments only Han runs — an unspaced pure-kana
42
+ // or unspaced-hangul run stays ONE token and still fails a >=3-token count.
43
+ // KNOWN LIMITATION, pinned by cjk-insights-promotion.test.mjs; closing it
44
+ // means segmenting kana runs too (a `ja` Intl.Segmenter row here), not
45
+ // another per-site exemption.
46
+ export const NON_ASCII_RE = /[^\x00-\x7F]/;
29
47
  // Feature-detected once at module load. Node >=18 (this project's engines
30
48
  // floor) ships Intl.Segmenter unconditionally, but we still feature-detect
31
49
  // defensively rather than assume every runtime that imports this module is
@@ -1 +1 @@
1
- {"version":3,"file":"tokenize.js","sourceRoot":"","sources":["../../src/helpers/tokenize.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAEH,gFAAgF;AAChF,6EAA6E;AAC7E,yDAAyD;AACzD,MAAM,CAAC,MAAM,WAAW,GAAG,iBAAiB,CAAC;AAC7C,MAAM,CAAC,MAAM,UAAU,GAAG,mBAAmB,CAAC;AAE9C,0EAA0E;AAC1E,2EAA2E;AAC3E,2EAA2E;AAC3E,2EAA2E;AAC3E,IAAI,YAAwC,CAAC;AAC7C,IAAI,CAAC;IACH,IAAI,OAAO,IAAI,KAAK,WAAW,IAAI,OAAO,IAAI,CAAC,SAAS,KAAK,UAAU,EAAE,CAAC;QACxE,YAAY,GAAG,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC,CAAC;IACnE,CAAC;AACH,CAAC;AAAC,MAAM,CAAC;IACP,YAAY,GAAG,SAAS,CAAC;AAC3B,CAAC;AAED;;;;;;GAMG;AACH,SAAS,cAAc,CAAC,GAAW;IACjC,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,oDAAoD;IACnF,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IACpC,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE;QAAE,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IAC7E,OAAO,GAAG,CAAC;AACb,CAAC;AAED,+EAA+E;AAC/E,SAAS,aAAa,CAAC,GAAW;IAChC,IAAI,YAAY,EAAE,CAAC;QACjB,MAAM,GAAG,GAAa,EAAE,CAAC;QACzB,KAAK,MAAM,GAAG,IAAI,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;YAC5C,uEAAuE;YACvE,sEAAsE;YACtE,wEAAwE;YACxE,mDAAmD;YACnD,IAAI,GAAG,CAAC,UAAU,KAAK,KAAK,IAAI,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;gBAC9D,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YACxB,CAAC;QACH,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC;IACD,OAAO,cAAc,CAAC,GAAG,CAAC,CAAC;AAC7B,CAAC;AAiCD;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,UAAU,aAAa,CAAC,CAAS,EAAE,OAAwB,EAAE;IACjE,MAAM,EAAE,SAAS,GAAG,CAAC,EAAE,SAAS,EAAE,eAAe,EAAE,GAAG,IAAI,CAAC;IAE3D,sEAAsE;IACtE,yEAAyE;IACzE,6CAA6C;IAC7C,MAAM,UAAU,GAAG,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IAEvC,MAAM,GAAG,GAAa,EAAE,CAAC;IAEzB,0EAA0E;IAC1E,wEAAwE;IACxE,0EAA0E;IAC1E,yDAAyD;IACzD,MAAM,OAAO,GAAG,UAAU,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;IAC7C,IAAI,OAAO,EAAE,CAAC;QACZ,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;YAC1B,KAAK,MAAM,GAAG,IAAI,aAAa,CAAC,GAAG,CAAC;gBAAE,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACtD,CAAC;IACH,CAAC;IAED,2BAA2B;IAC3B,uEAAuE;IACvE,0EAA0E;IAC1E,sEAAsE;IACtE,IAAI,IAAI,GAAG,UAAU,CAAC,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC;IAC7D,IAAI,eAAe,EAAE,CAAC;QACpB,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,eAAe,EAAE,GAAG,CAAC,CAAC;IAC9D,CAAC;IAED,MAAM,WAAW,GAAG,IAAI;SACrB,KAAK,CAAC,KAAK,CAAC;QACb,wEAAwE;QACxE,sEAAsE;QACtE,qEAAqE;QACrE,yEAAyE;QACzE,uEAAuE;QACvE,oEAAoE;QACpE,uEAAuE;QACvE,sEAAsE;QACtE,kEAAkE;QAClE,iEAAiE;QACjE,mEAAmE;SAClE,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,CAAC,MAAM,IAAI,SAAS,IAAI,CAAC,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAChF,GAAG,CAAC,IAAI,CAAC,GAAG,WAAW,CAAC,CAAC;IAEzB,OAAO,GAAG,CAAC;AACb,CAAC;AAED,gFAAgF;AAChF,MAAM,UAAU,QAAQ,CAAC,CAAS,EAAE,IAAsB;IACxD,OAAO,IAAI,GAAG,CAAC,aAAa,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC;AACzC,CAAC"}
1
+ {"version":3,"file":"tokenize.js","sourceRoot":"","sources":["../../src/helpers/tokenize.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAEH,gFAAgF;AAChF,6EAA6E;AAC7E,yDAAyD;AACzD,MAAM,CAAC,MAAM,WAAW,GAAG,iBAAiB,CAAC;AAC7C,MAAM,CAAC,MAAM,UAAU,GAAG,mBAAmB,CAAC;AAE9C,0EAA0E;AAC1E,yEAAyE;AACzE,8EAA8E;AAC9E,6EAA6E;AAC7E,4EAA4E;AAC5E,sEAAsE;AACtE,0EAA0E;AAC1E,qEAAqE;AACrE,+CAA+C;AAC/C,EAAE;AACF,4EAA4E;AAC5E,yEAAyE;AACzE,6EAA6E;AAC7E,4EAA4E;AAC5E,0EAA0E;AAC1E,uEAAuE;AACvE,8BAA8B;AAC9B,MAAM,CAAC,MAAM,YAAY,GAAG,cAAc,CAAC;AAE3C,0EAA0E;AAC1E,2EAA2E;AAC3E,2EAA2E;AAC3E,2EAA2E;AAC3E,IAAI,YAAwC,CAAC;AAC7C,IAAI,CAAC;IACH,IAAI,OAAO,IAAI,KAAK,WAAW,IAAI,OAAO,IAAI,CAAC,SAAS,KAAK,UAAU,EAAE,CAAC;QACxE,YAAY,GAAG,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC,CAAC;IACnE,CAAC;AACH,CAAC;AAAC,MAAM,CAAC;IACP,YAAY,GAAG,SAAS,CAAC;AAC3B,CAAC;AAED;;;;;;GAMG;AACH,SAAS,cAAc,CAAC,GAAW;IACjC,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,oDAAoD;IACnF,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IACpC,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE;QAAE,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IAC7E,OAAO,GAAG,CAAC;AACb,CAAC;AAED,+EAA+E;AAC/E,SAAS,aAAa,CAAC,GAAW;IAChC,IAAI,YAAY,EAAE,CAAC;QACjB,MAAM,GAAG,GAAa,EAAE,CAAC;QACzB,KAAK,MAAM,GAAG,IAAI,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;YAC5C,uEAAuE;YACvE,sEAAsE;YACtE,wEAAwE;YACxE,mDAAmD;YACnD,IAAI,GAAG,CAAC,UAAU,KAAK,KAAK,IAAI,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;gBAC9D,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YACxB,CAAC;QACH,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC;IACD,OAAO,cAAc,CAAC,GAAG,CAAC,CAAC;AAC7B,CAAC;AAiCD;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,UAAU,aAAa,CAAC,CAAS,EAAE,OAAwB,EAAE;IACjE,MAAM,EAAE,SAAS,GAAG,CAAC,EAAE,SAAS,EAAE,eAAe,EAAE,GAAG,IAAI,CAAC;IAE3D,sEAAsE;IACtE,yEAAyE;IACzE,6CAA6C;IAC7C,MAAM,UAAU,GAAG,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IAEvC,MAAM,GAAG,GAAa,EAAE,CAAC;IAEzB,0EAA0E;IAC1E,wEAAwE;IACxE,0EAA0E;IAC1E,yDAAyD;IACzD,MAAM,OAAO,GAAG,UAAU,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;IAC7C,IAAI,OAAO,EAAE,CAAC;QACZ,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;YAC1B,KAAK,MAAM,GAAG,IAAI,aAAa,CAAC,GAAG,CAAC;gBAAE,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACtD,CAAC;IACH,CAAC;IAED,2BAA2B;IAC3B,uEAAuE;IACvE,0EAA0E;IAC1E,sEAAsE;IACtE,IAAI,IAAI,GAAG,UAAU,CAAC,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC;IAC7D,IAAI,eAAe,EAAE,CAAC;QACpB,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,eAAe,EAAE,GAAG,CAAC,CAAC;IAC9D,CAAC;IAED,MAAM,WAAW,GAAG,IAAI;SACrB,KAAK,CAAC,KAAK,CAAC;QACb,wEAAwE;QACxE,sEAAsE;QACtE,qEAAqE;QACrE,yEAAyE;QACzE,uEAAuE;QACvE,oEAAoE;QACpE,uEAAuE;QACvE,sEAAsE;QACtE,kEAAkE;QAClE,iEAAiE;QACjE,mEAAmE;SAClE,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,CAAC,MAAM,IAAI,SAAS,IAAI,CAAC,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAChF,GAAG,CAAC,IAAI,CAAC,GAAG,WAAW,CAAC,CAAC;IAEzB,OAAO,GAAG,CAAC;AACb,CAAC;AAED,gFAAgF;AAChF,MAAM,UAAU,QAAQ,CAAC,CAAS,EAAE,IAAsB;IACxD,OAAO,IAAI,GAAG,CAAC,aAAa,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC;AACzC,CAAC"}