@omgbase/core 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (214) hide show
  1. package/LICENSE +21 -0
  2. package/dist/src/cli/eval-matcher.d.ts +13 -0
  3. package/dist/src/cli/eval-matcher.js +36 -0
  4. package/dist/src/core/attach.d.ts +5 -0
  5. package/dist/src/core/attach.js +38 -0
  6. package/dist/src/core/cursor.d.ts +11 -0
  7. package/dist/src/core/cursor.js +35 -0
  8. package/dist/src/core/hash.d.ts +23 -0
  9. package/dist/src/core/hash.js +82 -0
  10. package/dist/src/core/ids.d.ts +6 -0
  11. package/dist/src/core/ids.js +26 -0
  12. package/dist/src/core/index.d.ts +1 -0
  13. package/dist/src/core/index.js +2 -0
  14. package/dist/src/core/ingest.d.ts +43 -0
  15. package/dist/src/core/ingest.js +287 -0
  16. package/dist/src/core/order-key.d.ts +8 -0
  17. package/dist/src/core/order-key.js +72 -0
  18. package/dist/src/core/parse/parse.d.ts +3 -0
  19. package/dist/src/core/parse/parse.js +104 -0
  20. package/dist/src/core/parse/render.d.ts +3 -0
  21. package/dist/src/core/parse/render.js +20 -0
  22. package/dist/src/core/parse/tree.d.ts +8 -0
  23. package/dist/src/core/parse/tree.js +43 -0
  24. package/dist/src/core/parse/types.d.ts +52 -0
  25. package/dist/src/core/parse/types.js +9 -0
  26. package/dist/src/core/read/document.d.ts +107 -0
  27. package/dist/src/core/read/document.js +191 -0
  28. package/dist/src/core/read/nodes.d.ts +47 -0
  29. package/dist/src/core/read/nodes.js +102 -0
  30. package/dist/src/core/read/outline.d.ts +12 -0
  31. package/dist/src/core/read/outline.js +76 -0
  32. package/dist/src/core/read/reader.d.ts +125 -0
  33. package/dist/src/core/read/reader.js +206 -0
  34. package/dist/src/core/read/refs.d.ts +15 -0
  35. package/dist/src/core/read/refs.js +51 -0
  36. package/dist/src/core/store/edges.d.ts +26 -0
  37. package/dist/src/core/store/edges.js +101 -0
  38. package/dist/src/core/store/fts.d.ts +10 -0
  39. package/dist/src/core/store/fts.js +22 -0
  40. package/dist/src/core/store/gc.d.ts +18 -0
  41. package/dist/src/core/store/gc.js +69 -0
  42. package/dist/src/core/store/nodes.d.ts +5 -0
  43. package/dist/src/core/store/nodes.js +57 -0
  44. package/dist/src/core/store/properties.d.ts +38 -0
  45. package/dist/src/core/store/properties.js +177 -0
  46. package/dist/src/core/store/rebuild.d.ts +3 -0
  47. package/dist/src/core/store/rebuild.js +29 -0
  48. package/dist/src/core/store/schema.d.ts +9 -0
  49. package/dist/src/core/store/schema.js +386 -0
  50. package/dist/src/core/store/sections.d.ts +2 -0
  51. package/dist/src/core/store/sections.js +29 -0
  52. package/dist/src/core/store/store.d.ts +24 -0
  53. package/dist/src/core/store/store.js +164 -0
  54. package/dist/src/core/store/writers.d.ts +52 -0
  55. package/dist/src/core/store/writers.js +89 -0
  56. package/dist/src/core/vec.d.ts +10 -0
  57. package/dist/src/core/vec.js +33 -0
  58. package/dist/src/format/adapter.d.ts +97 -0
  59. package/dist/src/format/adapter.js +14 -0
  60. package/dist/src/format/index.d.ts +6 -0
  61. package/dist/src/format/index.js +15 -0
  62. package/dist/src/format/json.d.ts +3 -0
  63. package/dist/src/format/json.js +324 -0
  64. package/dist/src/format/markdown.d.ts +5 -0
  65. package/dist/src/format/markdown.js +217 -0
  66. package/dist/src/format/registry.d.ts +6 -0
  67. package/dist/src/format/registry.js +24 -0
  68. package/dist/src/format/yaml.d.ts +5 -0
  69. package/dist/src/format/yaml.js +413 -0
  70. package/dist/src/graph/extract.d.ts +38 -0
  71. package/dist/src/graph/extract.js +242 -0
  72. package/dist/src/graph/history.d.ts +103 -0
  73. package/dist/src/graph/history.js +210 -0
  74. package/dist/src/graph/inbound-links.d.ts +33 -0
  75. package/dist/src/graph/inbound-links.js +130 -0
  76. package/dist/src/graph/link-destinations.d.ts +22 -0
  77. package/dist/src/graph/link-destinations.js +84 -0
  78. package/dist/src/graph/link-health.d.ts +79 -0
  79. package/dist/src/graph/link-health.js +146 -0
  80. package/dist/src/graph/links.d.ts +23 -0
  81. package/dist/src/graph/links.js +42 -0
  82. package/dist/src/index.d.ts +60 -0
  83. package/dist/src/index.js +70 -0
  84. package/dist/src/mcp/errors.d.ts +17 -0
  85. package/dist/src/mcp/errors.js +18 -0
  86. package/dist/src/mcp/graph.d.ts +63 -0
  87. package/dist/src/mcp/graph.js +212 -0
  88. package/dist/src/mcp/reference.d.ts +1 -0
  89. package/dist/src/mcp/reference.js +227 -0
  90. package/dist/src/mcp/server.d.ts +30 -0
  91. package/dist/src/mcp/server.js +1076 -0
  92. package/dist/src/mcp/stdio.d.ts +13 -0
  93. package/dist/src/mcp/stdio.js +24 -0
  94. package/dist/src/migrate/mrplex.d.ts +27 -0
  95. package/dist/src/migrate/mrplex.js +29 -0
  96. package/dist/src/mutate/apply.d.ts +83 -0
  97. package/dist/src/mutate/apply.js +302 -0
  98. package/dist/src/mutate/doc-store.d.ts +49 -0
  99. package/dist/src/mutate/doc-store.js +71 -0
  100. package/dist/src/mutate/docs.d.ts +60 -0
  101. package/dist/src/mutate/docs.js +232 -0
  102. package/dist/src/mutate/known-ids.d.ts +12 -0
  103. package/dist/src/mutate/known-ids.js +154 -0
  104. package/dist/src/mutate/load.d.ts +3 -0
  105. package/dist/src/mutate/load.js +66 -0
  106. package/dist/src/mutate/lower.d.ts +32 -0
  107. package/dist/src/mutate/lower.js +232 -0
  108. package/dist/src/mutate/macros.d.ts +91 -0
  109. package/dist/src/mutate/macros.js +230 -0
  110. package/dist/src/mutate/ops.d.ts +40 -0
  111. package/dist/src/mutate/ops.js +513 -0
  112. package/dist/src/mutate/opset.d.ts +92 -0
  113. package/dist/src/mutate/opset.js +79 -0
  114. package/dist/src/mutate/plan-update.d.ts +51 -0
  115. package/dist/src/mutate/plan-update.js +187 -0
  116. package/dist/src/mutate/tree.d.ts +46 -0
  117. package/dist/src/mutate/tree.js +167 -0
  118. package/dist/src/oqx/run.d.ts +1 -0
  119. package/dist/src/oqx/run.js +6 -0
  120. package/dist/src/oqx-js/context.d.ts +23 -0
  121. package/dist/src/oqx-js/context.js +564 -0
  122. package/dist/src/oqx-js/planner.d.ts +10 -0
  123. package/dist/src/oqx-js/planner.js +91 -0
  124. package/dist/src/oqx-js/run.d.ts +41 -0
  125. package/dist/src/oqx-js/run.js +332 -0
  126. package/dist/src/oqx-js/sql/translate.d.ts +27 -0
  127. package/dist/src/oqx-js/sql/translate.js +290 -0
  128. package/dist/src/reconcile/crossdoc.d.ts +34 -0
  129. package/dist/src/reconcile/crossdoc.js +76 -0
  130. package/dist/src/reconcile/eval/generator.d.ts +21 -0
  131. package/dist/src/reconcile/eval/generator.js +129 -0
  132. package/dist/src/reconcile/eval/metrics.d.ts +32 -0
  133. package/dist/src/reconcile/eval/metrics.js +74 -0
  134. package/dist/src/reconcile/flatten.d.ts +12 -0
  135. package/dist/src/reconcile/flatten.js +37 -0
  136. package/dist/src/reconcile/phase5.d.ts +8 -0
  137. package/dist/src/reconcile/phase5.js +167 -0
  138. package/dist/src/reconcile/phase6a.d.ts +2 -0
  139. package/dist/src/reconcile/phase6a.js +159 -0
  140. package/dist/src/reconcile/phases.d.ts +28 -0
  141. package/dist/src/reconcile/phases.js +0 -0
  142. package/dist/src/reconcile/reconcile.d.ts +21 -0
  143. package/dist/src/reconcile/reconcile.js +115 -0
  144. package/dist/src/reconcile/similarity.d.ts +7 -0
  145. package/dist/src/reconcile/similarity.js +41 -0
  146. package/dist/src/reconcile/types.d.ts +46 -0
  147. package/dist/src/reconcile/types.js +19 -0
  148. package/dist/src/search/cel/ast.d.ts +74 -0
  149. package/dist/src/search/cel/ast.js +3 -0
  150. package/dist/src/search/cel/compile.d.ts +60 -0
  151. package/dist/src/search/cel/compile.js +836 -0
  152. package/dist/src/search/cel/lexer.d.ts +9 -0
  153. package/dist/src/search/cel/lexer.js +139 -0
  154. package/dist/src/search/cel/parser.d.ts +7 -0
  155. package/dist/src/search/cel/parser.js +219 -0
  156. package/dist/src/search/drain.d.ts +39 -0
  157. package/dist/src/search/drain.js +97 -0
  158. package/dist/src/search/embeddings.d.ts +131 -0
  159. package/dist/src/search/embeddings.js +263 -0
  160. package/dist/src/search/external.d.ts +13 -0
  161. package/dist/src/search/external.js +177 -0
  162. package/dist/src/search/fts-query.d.ts +1 -0
  163. package/dist/src/search/fts-query.js +64 -0
  164. package/dist/src/search/provider.d.ts +17 -0
  165. package/dist/src/search/provider.js +13 -0
  166. package/dist/src/search/resolve.d.ts +18 -0
  167. package/dist/src/search/resolve.js +23 -0
  168. package/dist/src/search/rrf.d.ts +36 -0
  169. package/dist/src/search/rrf.js +84 -0
  170. package/dist/src/search/tasks.d.ts +12 -0
  171. package/dist/src/search/tasks.js +131 -0
  172. package/dist/src/search/text.d.ts +16 -0
  173. package/dist/src/search/text.js +21 -0
  174. package/dist/src/search/vector.d.ts +18 -0
  175. package/dist/src/search/vector.js +49 -0
  176. package/dist/src/sync/admin.d.ts +59 -0
  177. package/dist/src/sync/admin.js +53 -0
  178. package/dist/src/sync/attach.d.ts +13 -0
  179. package/dist/src/sync/attach.js +29 -0
  180. package/dist/src/sync/checkpoint.d.ts +21 -0
  181. package/dist/src/sync/checkpoint.js +52 -0
  182. package/dist/src/sync/driver.d.ts +25 -0
  183. package/dist/src/sync/driver.js +80 -0
  184. package/dist/src/sync/external-source.d.ts +14 -0
  185. package/dist/src/sync/external-source.js +137 -0
  186. package/dist/src/sync/freshness.d.ts +42 -0
  187. package/dist/src/sync/freshness.js +152 -0
  188. package/dist/src/sync/fs-util.d.ts +12 -0
  189. package/dist/src/sync/fs-util.js +55 -0
  190. package/dist/src/sync/git-heuristics.d.ts +6 -0
  191. package/dist/src/sync/git-heuristics.js +16 -0
  192. package/dist/src/sync/observe.d.ts +63 -0
  193. package/dist/src/sync/observe.js +84 -0
  194. package/dist/src/sync/plugin.d.ts +54 -0
  195. package/dist/src/sync/plugin.js +12 -0
  196. package/dist/src/sync/reconciling-ingest.d.ts +12 -0
  197. package/dist/src/sync/reconciling-ingest.js +209 -0
  198. package/dist/src/sync/recovery.d.ts +6 -0
  199. package/dist/src/sync/recovery.js +34 -0
  200. package/dist/src/sync/settings.d.ts +18 -0
  201. package/dist/src/sync/settings.js +58 -0
  202. package/dist/src/sync/sources.d.ts +37 -0
  203. package/dist/src/sync/sources.js +75 -0
  204. package/dist/src/sync/tombstone.d.ts +9 -0
  205. package/dist/src/sync/tombstone.js +55 -0
  206. package/dist/src/sync/watch-lease.d.ts +14 -0
  207. package/dist/src/sync/watch-lease.js +91 -0
  208. package/dist/src/sync/watcher.d.ts +19 -0
  209. package/dist/src/sync/watcher.js +34 -0
  210. package/dist/src/sync/workspace.d.ts +34 -0
  211. package/dist/src/sync/workspace.js +120 -0
  212. package/dist/src/sync/writer-lock.d.ts +16 -0
  213. package/dist/src/sync/writer-lock.js +121 -0
  214. package/package.json +80 -0
@@ -0,0 +1,263 @@
1
+ import { sha256 } from "../core/hash.js";
2
+ export class SemanticUnavailable extends Error {
3
+ code = "semantic_unavailable";
4
+ }
5
+ // Context prefix (05 §6): "{doc title} · {path} · {heading chain} · {type}\n{text}".
6
+ export function contextPrefix(input) {
7
+ const chain = input.headingChain.join(" › ");
8
+ return `${input.docTitle} · ${input.path} · ${chain} · ${input.blockType}`;
9
+ }
10
+ export function embedInput(ctx, blockText) {
11
+ return `${ctx}\n${blockText}`;
12
+ }
13
+ export function ctxHashHex(ctx) {
14
+ return sha256(ctx).toString("hex");
15
+ }
16
+ // Default cache-miss chunk size for process(). Embedding is done in bounded
17
+ // batches so a single request never grows unbounded — a large all-at-once
18
+ // request can exceed the provider's per-request limits (stdio line size, HTTP
19
+ // body cap) and fail silently. Callers may override via opts.batchSize; the
20
+ // value only affects request chunking, never whether work happens.
21
+ const DEFAULT_EMBED_BATCH = 32;
22
+ // Rows < minTokens roll into their section aggregate rather than embedding
23
+ // alone (05 §6). We surface this as a filter the caller applies.
24
+ export function shouldEmbed(text, minTokens = 24) {
25
+ return text.trim().split(/\s+/).filter(Boolean).length >= minTokens;
26
+ }
27
+ // Coarse token estimate for budget decisions and pool weighting. We don't have
28
+ // the provider's tokenizer (it's out-of-process), so approximate: word-piece
29
+ // tokenizers emit slightly MORE tokens than whitespace words (subword splits,
30
+ // punctuation), so scale words up by ~1.3 to stay on the safe side of the
31
+ // budget rather than under-counting and letting a doc silently truncate.
32
+ export function estimateTokens(text) {
33
+ const words = text.trim().split(/\s+/).filter(Boolean).length;
34
+ return Math.ceil(words * 1.3);
35
+ }
36
+ // Default whole-doc token budget when the provider doesn't report its own
37
+ // maxInputTokens and config doesn't override. Deliberately conservative: beyond
38
+ // a model's real input limit the input is truncated, at which point pooling the
39
+ // block vectors represents the whole document better than an embedding of only
40
+ // its first N tokens. Callers should prefer the provider's actual limit.
41
+ export const DEFAULT_DOC_TOKEN_BUDGET = 512;
42
+ // Tokens reserved for the doc-embedding header line so the budget check leaves
43
+ // room for it (the header is prepended to the body — see buildDocEmbedTasks).
44
+ const DOC_HEADER_MARGIN_TOKENS = 16;
45
+ export class EmbeddingWorker {
46
+ store;
47
+ provider;
48
+ constructor(store, provider) {
49
+ this.store = store;
50
+ this.provider = provider;
51
+ }
52
+ get available() {
53
+ return this.provider !== null;
54
+ }
55
+ cacheKey(contentHashHex, ctxHex) {
56
+ return [Buffer.from(contentHashHex, "hex"), Buffer.from(ctxHex, "hex"), this.provider.model];
57
+ }
58
+ /** Look up a cached vector; null if absent (stale/missing). */
59
+ getCached(contentHashHex, ctx) {
60
+ if (!this.provider)
61
+ return null;
62
+ const ctxHex = ctxHashHex(ctx);
63
+ const row = this.store.db
64
+ .prepare("SELECT vec FROM embeddings WHERE content_hash = ? AND ctx_hash = ? AND model = ?")
65
+ .get(...this.cacheKey(contentHashHex, ctxHex));
66
+ if (!row)
67
+ return null;
68
+ return new Float32Array(row.vec.buffer, row.vec.byteOffset, row.vec.byteLength / 4);
69
+ }
70
+ /**
71
+ * Embed a batch (skipping cache hits) and persist. Throws if no provider.
72
+ * Cache misses are embedded in chunks of `batchSize` (default:
73
+ * DEFAULT_EMBED_BATCH); `onProgress` fires after each chunk is persisted so
74
+ * callers can show live progress on a long drain.
75
+ */
76
+ async process(tasks, opts = {}) {
77
+ if (!this.provider)
78
+ throw new SemanticUnavailable("no embedding provider configured");
79
+ const misses = [];
80
+ let cached = 0;
81
+ for (const t of tasks) {
82
+ if (this.getCached(t.contentHashHex, t.ctx))
83
+ cached++;
84
+ else
85
+ misses.push(t);
86
+ }
87
+ const insert = this.store.db.prepare("INSERT OR REPLACE INTO embeddings (content_hash, ctx_hash, model, dim, vec) VALUES (?, ?, ?, ?, ?)");
88
+ const batchSize = opts.batchSize && opts.batchSize > 0 ? opts.batchSize : DEFAULT_EMBED_BATCH;
89
+ let embedded = 0;
90
+ for (let start = 0; start < misses.length; start += batchSize) {
91
+ const chunk = misses.slice(start, start + batchSize);
92
+ const vectors = await this.provider.embed(chunk.map((t) => embedInput(t.ctx, t.text)));
93
+ this.store.write(() => {
94
+ chunk.forEach((t, i) => {
95
+ const v = Float32Array.from(vectors[i]);
96
+ insert.run(Buffer.from(t.contentHashHex, "hex"), Buffer.from(ctxHashHex(t.ctx), "hex"), this.provider.model, this.provider.dim, Buffer.from(v.buffer));
97
+ });
98
+ });
99
+ embedded += chunk.length;
100
+ opts.onProgress?.({ embedded, total: misses.length });
101
+ }
102
+ return { embedded, cached };
103
+ }
104
+ /** Embed a bare query string (no context prefix, 05 §6). */
105
+ async embedQuery(query) {
106
+ if (!this.provider)
107
+ throw new SemanticUnavailable("no embedding provider configured");
108
+ const [v] = await this.provider.embed([query]);
109
+ return Float32Array.from(v);
110
+ }
111
+ // ---- doc-grain embeddings (doc semantic retrieval) ------------------------
112
+ docTokenBudget() {
113
+ const provider = this.provider;
114
+ const budget = provider.maxInputTokens ?? DEFAULT_DOC_TOKEN_BUDGET;
115
+ return Math.max(1, budget - DOC_HEADER_MARGIN_TOKENS);
116
+ }
117
+ /** The freshness key for a doc task: sha256 of the whole-doc embed input. */
118
+ docInputHash(input) {
119
+ return sha256(input);
120
+ }
121
+ /** Look up a cached doc vector; null if absent or stale (input changed). */
122
+ getCachedDoc(docId, input) {
123
+ if (!this.provider)
124
+ return null;
125
+ const row = this.store.db
126
+ .prepare("SELECT input_hash, method, vec FROM doc_embeddings WHERE doc_id = ? AND model = ?")
127
+ .get(docId, this.provider.model);
128
+ if (!row)
129
+ return null;
130
+ if (!row.input_hash.equals(this.docInputHash(input)))
131
+ return null; // stale
132
+ return {
133
+ vec: new Float32Array(row.vec.buffer, row.vec.byteOffset, row.vec.byteLength / 4),
134
+ method: row.method,
135
+ };
136
+ }
137
+ /** Doc ids whose current input has no fresh cached vector — the doc queue. */
138
+ staleDocs(tasks) {
139
+ return tasks.filter((t) => !this.getCachedDoc(t.docId, t.input)).map((t) => t.docId);
140
+ }
141
+ /**
142
+ * Compute one vector per document and persist. Whole-doc path (primary):
143
+ * embed the doc's input directly. Pooled fallback: only when the input
144
+ * exceeds the provider's token budget, pool the doc's already-cached block
145
+ * vectors (token-weighted mean, L2-normalized) — zero embedding calls. Skips
146
+ * docs whose fresh vector is already cached. Throws if no provider.
147
+ */
148
+ async processDocs(tasks, opts = {}) {
149
+ if (!this.provider)
150
+ throw new SemanticUnavailable("no embedding provider configured");
151
+ const budget = this.docTokenBudget();
152
+ const misses = [];
153
+ let cached = 0;
154
+ for (const t of tasks) {
155
+ if (this.getCachedDoc(t.docId, t.input))
156
+ cached++;
157
+ else
158
+ misses.push(t);
159
+ }
160
+ // Split misses by strategy: within budget ⇒ whole-doc embed; over budget ⇒
161
+ // pooled fallback (no provider call).
162
+ const whole = misses.filter((t) => estimateTokens(t.input) <= budget);
163
+ const pooledTasks = misses.filter((t) => estimateTokens(t.input) > budget);
164
+ const insert = this.store.db.prepare("INSERT OR REPLACE INTO doc_embeddings (doc_id, model, input_hash, method, dim, vec) VALUES (?, ?, ?, ?, ?, ?)");
165
+ let embedded = 0;
166
+ let pooled = 0;
167
+ const batchSize = opts.batchSize && opts.batchSize > 0 ? opts.batchSize : DEFAULT_EMBED_BATCH;
168
+ for (let start = 0; start < whole.length; start += batchSize) {
169
+ const chunk = whole.slice(start, start + batchSize);
170
+ const vectors = await this.provider.embed(chunk.map((t) => t.input));
171
+ this.store.write(() => {
172
+ chunk.forEach((t, i) => {
173
+ const v = Float32Array.from(vectors[i]);
174
+ insert.run(t.docId, this.provider.model, this.docInputHash(t.input), "whole", this.provider.dim, Buffer.from(v.buffer));
175
+ });
176
+ });
177
+ embedded += chunk.length;
178
+ opts.onProgress?.({ embedded, total: whole.length });
179
+ }
180
+ // Pooled fallback: reuse block vectors already in the embeddings cache.
181
+ if (pooledTasks.length > 0) {
182
+ this.store.write(() => {
183
+ for (const t of pooledTasks) {
184
+ const v = this.poolBlockVectors(t.blocks);
185
+ if (!v)
186
+ continue; // no cached block vectors yet — leave it queued
187
+ insert.run(t.docId, this.provider.model, this.docInputHash(t.input), "pooled", this.provider.dim, Buffer.from(v.buffer));
188
+ pooled++;
189
+ }
190
+ });
191
+ }
192
+ return { embedded, cached, pooled };
193
+ }
194
+ /**
195
+ * Token-weighted mean of a document's cached block vectors, L2-normalized.
196
+ * Pool = Σ(wᵢ·vᵢ)/Σ(wᵢ) with wᵢ = block token count, then normalized (pooling
197
+ * breaks unit norm; downstream cosine assumes unit vectors). Returns null when
198
+ * no referenced block has a cached vector yet.
199
+ */
200
+ poolBlockVectors(refs) {
201
+ const dim = this.provider.dim;
202
+ const acc = new Float64Array(dim);
203
+ let weightSum = 0;
204
+ for (const ref of refs) {
205
+ const v = this.getCached(ref.contentHashHex, ref.ctx);
206
+ if (!v)
207
+ continue;
208
+ const w = ref.tokens > 0 ? ref.tokens : 1;
209
+ const n = Math.min(dim, v.length);
210
+ for (let i = 0; i < n; i++)
211
+ acc[i] += w * v[i];
212
+ weightSum += w;
213
+ }
214
+ if (weightSum === 0)
215
+ return null;
216
+ let norm = 0;
217
+ for (let i = 0; i < dim; i++) {
218
+ acc[i] /= weightSum;
219
+ norm += acc[i] * acc[i];
220
+ }
221
+ norm = Math.sqrt(norm);
222
+ const out = new Float32Array(dim);
223
+ if (norm === 0)
224
+ return out; // all-zero pool (degenerate) — return zeros
225
+ for (let i = 0; i < dim; i++)
226
+ out[i] = acc[i] / norm;
227
+ return out;
228
+ }
229
+ /**
230
+ * Staleness: a heading rename changes descendants' context prefixes, so their
231
+ * cached vectors (keyed by the OLD ctx_hash) no longer match the current
232
+ * context. This returns the block ids in a doc whose current ctx has no cached
233
+ * vector — i.e. stale/pending — for the worker queue.
234
+ */
235
+ staleBlocks(tasks) {
236
+ return tasks.filter((t) => shouldEmbed(t.text) && !this.getCached(t.contentHashHex, t.ctx)).map((t) => t.blockId);
237
+ }
238
+ /**
239
+ * Vectors left behind by a previous model. The cache and both search paths key
240
+ * on model name, so after switching models the old model's rows are dead
241
+ * weight: never read, never overwritten (the new model writes under its own
242
+ * key). Reports how many block + doc vectors belong to a model OTHER than the
243
+ * current provider's, so callers can offer to reclaim the space.
244
+ */
245
+ foreignVectorCount() {
246
+ if (!this.provider)
247
+ return { blocks: 0, docs: 0 };
248
+ const blocks = this.store.db.prepare("SELECT count(*) c FROM embeddings WHERE model != ?").get(this.provider.model).c;
249
+ const docs = this.store.db.prepare("SELECT count(*) c FROM doc_embeddings WHERE model != ?").get(this.provider.model).c;
250
+ return { blocks, docs };
251
+ }
252
+ /** Delete every cached vector not produced by the current model. */
253
+ pruneForeignVectors() {
254
+ if (!this.provider)
255
+ return { blocks: 0, docs: 0 };
256
+ const model = this.provider.model;
257
+ return this.store.write((db) => ({
258
+ blocks: db.prepare("DELETE FROM embeddings WHERE model != ?").run(model).changes,
259
+ docs: db.prepare("DELETE FROM doc_embeddings WHERE model != ?").run(model).changes,
260
+ }));
261
+ }
262
+ }
263
+ //# sourceMappingURL=embeddings.js.map
@@ -0,0 +1,13 @@
1
+ import type { EmbeddingProvider } from "./embeddings.js";
2
+ import type { EmbeddingSettings } from "./provider.js";
3
+ export interface ExternalProvider {
4
+ provider: EmbeddingProvider;
5
+ /** Release resources (kill a spawned process; no-op for HTTP). */
6
+ close(): Promise<void>;
7
+ }
8
+ /**
9
+ * Build an embedding provider from repo settings by connecting to an external
10
+ * command (stdio) or endpoint (HTTP). Throws on misconfiguration or handshake
11
+ * failure. Callers must call close() when done.
12
+ */
13
+ export declare function createExternalProvider(settings: EmbeddingSettings): Promise<ExternalProvider | null>;
@@ -0,0 +1,177 @@
1
+ import { spawn } from "node:child_process";
2
+ import { createInterface } from "node:readline";
3
+ const isUrl = (s) => /^https?:\/\//i.test(s.trim());
4
+ /**
5
+ * Build an embedding provider from repo settings by connecting to an external
6
+ * command (stdio) or endpoint (HTTP). Throws on misconfiguration or handshake
7
+ * failure. Callers must call close() when done.
8
+ */
9
+ export async function createExternalProvider(settings) {
10
+ if (!settings.provider)
11
+ return null;
12
+ return isUrl(settings.provider) ? connectHttp(settings) : connectStdio(settings);
13
+ }
14
+ // ---- HTTP -------------------------------------------------------------------
15
+ async function connectHttp(settings) {
16
+ const url = settings.provider;
17
+ let model = settings.model ?? "http";
18
+ let dim = settings.dim ?? 0;
19
+ let maxInputTokens = settings.maxInputTokens;
20
+ try {
21
+ const res = await fetch(url, { method: "GET" });
22
+ if (res.ok) {
23
+ const meta = (await res.json());
24
+ if (meta.model)
25
+ model = meta.model;
26
+ if (typeof meta.dim === "number")
27
+ dim = meta.dim;
28
+ if (typeof meta.maxInputTokens === "number")
29
+ maxInputTokens = meta.maxInputTokens;
30
+ }
31
+ }
32
+ catch {
33
+ // Metadata is best-effort; embed() failures below are the real signal.
34
+ }
35
+ const provider = {
36
+ model,
37
+ dim,
38
+ ...(maxInputTokens !== undefined ? { maxInputTokens } : {}),
39
+ async embed(texts) {
40
+ if (texts.length === 0)
41
+ return [];
42
+ const res = await fetch(url, {
43
+ method: "POST",
44
+ headers: { "content-type": "application/json" },
45
+ body: JSON.stringify({ texts, model }),
46
+ });
47
+ if (!res.ok)
48
+ throw new Error(`embedding endpoint ${url} returned ${res.status}`);
49
+ const body = (await res.json());
50
+ if (!body.vectors)
51
+ throw new Error(`embedding endpoint ${url} returned no "vectors"`);
52
+ return body.vectors;
53
+ },
54
+ };
55
+ return { provider, close: async () => { } };
56
+ }
57
+ // ---- stdio ------------------------------------------------------------------
58
+ async function connectStdio(settings) {
59
+ const parts = settings.provider.trim().split(/\s+/);
60
+ const cmd = parts[0];
61
+ const args = parts.slice(1);
62
+ // stderr is PIPED (not inherited) and forwarded manually. Inheriting would give
63
+ // the child a duplicate of our own stderr fd; when our process is itself the
64
+ // child of a synchronous spawn (execFileSync/spawnSync, as the CLI tests use),
65
+ // that captured pipe only reaches EOF once every holder closes it — so a
66
+ // lingering embedder would keep the grandparent's execFileSync blocked forever
67
+ // even after we exit. A private pipe we own severs that coupling.
68
+ let child;
69
+ try {
70
+ child = spawn(cmd, args, { stdio: ["pipe", "pipe", "pipe"] });
71
+ }
72
+ catch (err) {
73
+ throw new Error(`embedding command '${settings.provider}' failed to spawn: ${err.message}`);
74
+ }
75
+ child.stderr.pipe(process.stderr);
76
+ // Don't let a still-running embedder keep our own event loop alive on exit.
77
+ child.unref();
78
+ const rl = createInterface({ input: child.stdout });
79
+ const lines = lineQueue(rl);
80
+ const spawnFailed = new Promise((_, reject) => {
81
+ child.once("error", (err) => reject(new Error(`embedding command '${cmd}' failed to spawn: ${err.message}`)));
82
+ child.once("exit", (code) => reject(new Error(`embedding command '${cmd}' exited early (code ${code ?? "?"})`)));
83
+ });
84
+ // The race consumers below observe this rejection; guard the reference itself
85
+ // so that a benign post-close `exit` doesn't surface as an unhandled rejection.
86
+ spawnFailed.catch(() => { });
87
+ // Handshake: first stdout line carries model + dim (+ optional maxInputTokens).
88
+ const handshakeRaw = await Promise.race([lines.next(), spawnFailed]);
89
+ let model = settings.model ?? "stdio";
90
+ let dim = settings.dim ?? 0;
91
+ let maxInputTokens = settings.maxInputTokens;
92
+ try {
93
+ const meta = JSON.parse(handshakeRaw);
94
+ if (meta.model)
95
+ model = meta.model;
96
+ if (typeof meta.dim === "number")
97
+ dim = meta.dim;
98
+ if (typeof meta.maxInputTokens === "number")
99
+ maxInputTokens = meta.maxInputTokens;
100
+ }
101
+ catch {
102
+ throw new Error(`embedding command '${cmd}' sent an invalid handshake line: ${handshakeRaw.slice(0, 120)}`);
103
+ }
104
+ let nextId = 1;
105
+ const provider = {
106
+ model,
107
+ dim,
108
+ ...(maxInputTokens !== undefined ? { maxInputTokens } : {}),
109
+ async embed(texts) {
110
+ if (texts.length === 0)
111
+ return [];
112
+ const id = nextId++;
113
+ child.stdin.write(JSON.stringify({ id, texts }) + "\n");
114
+ // Read lines until the matching id arrives (tolerate interleaved ids).
115
+ for (;;) {
116
+ const raw = await Promise.race([lines.next(), spawnFailed]);
117
+ const resp = JSON.parse(raw);
118
+ if (resp.error)
119
+ throw new Error(`embedder error: ${resp.error}`);
120
+ if (resp.id === id) {
121
+ if (!resp.vectors)
122
+ throw new Error("embedder response missing vectors");
123
+ return resp.vectors;
124
+ }
125
+ }
126
+ },
127
+ };
128
+ const close = async () => {
129
+ rl.close();
130
+ child.stderr.unpipe(process.stderr);
131
+ try {
132
+ child.stdin.end();
133
+ }
134
+ catch { /* already closed */ }
135
+ if (child.exitCode !== null || child.signalCode !== null)
136
+ return; // already gone
137
+ // Reap the child before returning: SIGTERM, wait briefly, then SIGKILL. This
138
+ // keeps a wedged provider from outliving us and holding pipes open.
139
+ await new Promise((resolve) => {
140
+ const t = setTimeout(() => { try {
141
+ child.kill("SIGKILL");
142
+ }
143
+ catch { /* gone */ } resolve(); }, 2000);
144
+ child.once("exit", () => { clearTimeout(t); resolve(); });
145
+ try {
146
+ child.kill("SIGTERM");
147
+ }
148
+ catch {
149
+ clearTimeout(t);
150
+ resolve();
151
+ }
152
+ });
153
+ };
154
+ return { provider, close };
155
+ }
156
+ // A pull-based async line reader over a readline Interface: next() resolves with
157
+ // the next line, buffering lines that arrive before they're requested.
158
+ function lineQueue(rl) {
159
+ const buffered = [];
160
+ const waiters = [];
161
+ rl.on("line", (line) => {
162
+ const w = waiters.shift();
163
+ if (w)
164
+ w(line);
165
+ else
166
+ buffered.push(line);
167
+ });
168
+ return {
169
+ next() {
170
+ const b = buffered.shift();
171
+ if (b !== undefined)
172
+ return Promise.resolve(b);
173
+ return new Promise((resolve) => waiters.push(resolve));
174
+ },
175
+ };
176
+ }
177
+ //# sourceMappingURL=external.js.map
@@ -0,0 +1 @@
1
+ export declare function sanitizeFtsQuery(input: string): string;
@@ -0,0 +1,64 @@
1
+ // Sanitize a user-supplied search string into a safe FTS5 MATCH expression.
2
+ //
3
+ // FTS5 MATCH has its own query grammar: bareword operators (AND/OR/NOT/NEAR),
4
+ // prefix `*`, column filters `:`, grouping `()`, and assorted punctuation. A
5
+ // raw string like "guides/onboarding" is parsed as syntax and errors with
6
+ // `fts5: syntax error near "/"`. Agents and humans expect a search box, not a
7
+ // query DSL, so we compile their input down to a grammar-free form:
8
+ //
9
+ // - text inside double quotes stays a phrase (adjacent-token match);
10
+ // - every other run of characters becomes one double-quoted token.
11
+ //
12
+ // Quoting each token makes FTS5 treat inner punctuation as token separators
13
+ // (the unicode61 tokenizer already strips non-alphanumerics), so `/`, `-`, `:`
14
+ // and friends can never reach the query parser. Space-separated tokens keep
15
+ // FTS5's implicit-AND semantics; a trailing `*` on a bareword is preserved as a
16
+ // prefix match. Returns "" when the input carries no searchable token, so
17
+ // callers can skip the MATCH entirely rather than emit an empty-query error.
18
+ function hasWordChar(s) {
19
+ return /[\p{L}\p{N}]/u.test(s);
20
+ }
21
+ // Wrap one raw token as a quoted FTS5 string, doubling embedded quotes.
22
+ function quoteToken(token) {
23
+ return `"${token.replace(/"/g, '""')}"`;
24
+ }
25
+ export function sanitizeFtsQuery(input) {
26
+ const out = [];
27
+ let i = 0;
28
+ const n = input.length;
29
+ while (i < n) {
30
+ const c = input[i];
31
+ // Skip whitespace between tokens.
32
+ if (/\s/.test(c)) {
33
+ i++;
34
+ continue;
35
+ }
36
+ // A user-supplied phrase: capture until the closing quote (or end).
37
+ if (c === '"') {
38
+ i++;
39
+ let phrase = "";
40
+ while (i < n && input[i] !== '"') {
41
+ phrase += input[i];
42
+ i++;
43
+ }
44
+ if (i < n)
45
+ i++; // consume closing quote
46
+ if (hasWordChar(phrase))
47
+ out.push(quoteToken(phrase));
48
+ continue;
49
+ }
50
+ // A bareword: read until whitespace or a quote.
51
+ let word = "";
52
+ while (i < n && !/\s/.test(input[i]) && input[i] !== '"') {
53
+ word += input[i];
54
+ i++;
55
+ }
56
+ // Preserve a trailing `*` as an FTS5 prefix match on the quoted token.
57
+ const prefix = word.endsWith("*");
58
+ const core = prefix ? word.slice(0, -1) : word;
59
+ if (hasWordChar(core))
60
+ out.push(quoteToken(core) + (prefix ? "*" : ""));
61
+ }
62
+ return out.join(" ");
63
+ }
64
+ //# sourceMappingURL=fts-query.js.map
@@ -0,0 +1,17 @@
1
+ export interface EmbeddingSettings {
2
+ /** Package name to import, e.g. "@omgbase/embedder". Absent ⇒ no provider. */
3
+ provider?: string;
4
+ /** Optional model id passed through to the provider factory. */
5
+ model?: string;
6
+ /** Optional dimension override passed through to the provider factory. */
7
+ dim?: number;
8
+ /**
9
+ * Optional max input length (tokens) the provider's model accepts before it
10
+ * truncates. Sets the doc-embedding whole-doc vs pooled-fallback threshold; a
11
+ * provider that reports it in its handshake overrides this. Absent everywhere
12
+ * ⇒ the worker's conservative default budget applies.
13
+ */
14
+ maxInputTokens?: number;
15
+ }
16
+ /** Read the embedding.* block out of a repo's parsed settings JSON. */
17
+ export declare function embeddingSettings(settings: Record<string, unknown>): EmbeddingSettings;
@@ -0,0 +1,13 @@
1
+ // Provider configuration (05 §6). The embedding provider is a plugin named in
2
+ // repo settings (embedding.provider = a package that exports
3
+ // `createProvider(opts?) => EmbeddingProvider`). The dynamic import itself lives
4
+ // in the application (the CLI), not here, so it resolves against the app's
5
+ // dependency tree and core carries no ML dependency. @omgbase/embedder is the
6
+ // default local implementation; a remote HTTP provider is the same contract
7
+ // behind a different package name — nothing privileges local vs. remote.
8
+ /** Read the embedding.* block out of a repo's parsed settings JSON. */
9
+ export function embeddingSettings(settings) {
10
+ const e = settings.embedding;
11
+ return e && typeof e === "object" ? e : {};
12
+ }
13
+ //# sourceMappingURL=provider.js.map
@@ -0,0 +1,18 @@
1
+ import type { Store } from "../core/store/store.js";
2
+ export interface ResolveHit {
3
+ id: string;
4
+ locator: string;
5
+ preview: string;
6
+ evidence: unknown;
7
+ }
8
+ export interface ResolveInput {
9
+ repoId: string;
10
+ query: string;
11
+ vector?: {
12
+ model: string;
13
+ vec: Float32Array;
14
+ };
15
+ limit?: number;
16
+ }
17
+ /** resolve: ranked candidates for the thing the caller means. */
18
+ export declare function resolve(store: Store, input: ResolveInput): ResolveHit[];
@@ -0,0 +1,23 @@
1
+ import { hybridSearch } from "./rrf.js";
2
+ function locatorFor(store, blockId) {
3
+ const row = store.db.prepare("SELECT d.path AS path, b.ordinal AS ordinal, b.type AS type FROM blocks b JOIN docs d ON d.doc_id = b.doc_id WHERE b.block_id = ?").get(blockId);
4
+ if (!row)
5
+ return blockId;
6
+ return `${row.path}#${row.type}[${row.ordinal}]`;
7
+ }
8
+ function previewFor(store, blockId, words = 12) {
9
+ const row = store.db.prepare("SELECT text FROM blocks WHERE block_id = ?").get(blockId);
10
+ if (!row)
11
+ return "";
12
+ const w = row.text.split(/\s+/).filter(Boolean);
13
+ return w.length <= words ? w.join(" ") : w.slice(0, words).join(" ") + "…";
14
+ }
15
+ /** resolve: ranked candidates for the thing the caller means. */
16
+ export function resolve(store, input) {
17
+ const hi = { repoId: input.repoId, text: input.query, limit: input.limit ?? 10 };
18
+ if (input.vector)
19
+ hi.vector = input.vector;
20
+ const hits = hybridSearch(store, hi);
21
+ return hits.map((h) => ({ id: h.blockId, locator: locatorFor(store, h.blockId), preview: previewFor(store, h.blockId), evidence: h.evidence }));
22
+ }
23
+ //# sourceMappingURL=resolve.js.map
@@ -0,0 +1,36 @@
1
+ import type { Store } from "../core/store/store.js";
2
+ export interface Boosts {
3
+ title?: number;
4
+ heading?: number;
5
+ path?: number;
6
+ layer?: number;
7
+ recency?: number;
8
+ }
9
+ export interface HybridHit {
10
+ blockId: string;
11
+ docId: string;
12
+ path: string;
13
+ score: number;
14
+ evidence: {
15
+ ftsRank?: number;
16
+ vectorRank?: number;
17
+ cosine?: number;
18
+ rrf: number;
19
+ boosts: Boosts;
20
+ };
21
+ }
22
+ export interface HybridInput {
23
+ repoId: string;
24
+ /** FTS query string (optional). */
25
+ text?: string;
26
+ /** query vector + model for the vector ranker (optional). */
27
+ vector?: {
28
+ model: string;
29
+ vec: Float32Array;
30
+ };
31
+ /** raw terms for boost matching (title/heading/path). */
32
+ terms?: string[];
33
+ limit?: number;
34
+ }
35
+ /** Hybrid search: RRF-fuse FTS + vector rankings, then multiplicative boosts. */
36
+ export declare function hybridSearch(store: Store, input: HybridInput): HybridHit[];