@gmickel/gno 1.46.0 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (230) hide show
  1. package/README.md +1 -1
  2. package/THIRD_PARTY_NOTICES.md +46 -0
  3. package/assets/skill/SKILL.md +7 -6
  4. package/assets/spa-production.json.gz +0 -0
  5. package/browser-extension/artifacts/gno-browser-clipper-v2.0.0.zip +0 -0
  6. package/browser-extension/artifacts/gno-browser-clipper-v2.0.0.zip.sha256 +1 -0
  7. package/browser-extension/dist/chunk-4tc9v0ja.js +74 -0
  8. package/browser-extension/dist/manifest.json +1 -1
  9. package/browser-extension/dist/preview.html +1 -1
  10. package/browser-extension/dist/service-worker.js +32 -33
  11. package/bunfig.toml +2 -0
  12. package/package.json +40 -26
  13. package/spec/cli.md +21 -4
  14. package/spec/db/schema.sql +146 -1
  15. package/spec/mcp.md +26 -0
  16. package/src/app/context-runtime-types.ts +3 -0
  17. package/src/app/context-runtime.ts +2 -0
  18. package/src/cli/commands/ask.ts +6 -1
  19. package/src/cli/commands/daemon.ts +21 -8
  20. package/src/cli/commands/embed.ts +77 -41
  21. package/src/cli/detach.ts +3 -2
  22. package/src/config/types.ts +3 -3
  23. package/src/converters/adapters/markitdownTs/adapter.ts +1 -2
  24. package/src/converters/adapters/officeparser/adapter.ts +1 -2
  25. package/src/converters/versions.ts +6 -8
  26. package/src/core/context-evidence.ts +8 -4
  27. package/src/core/job-manager.ts +95 -13
  28. package/src/core/network-boundary-inventory.ts +10 -0
  29. package/src/core/shutdown-budget.ts +45 -0
  30. package/src/embed/backlog.ts +107 -4
  31. package/src/embed/batch.ts +42 -2
  32. package/src/embed/fingerprint.ts +16 -0
  33. package/src/embed/retry.ts +113 -5
  34. package/src/embed/variant-backlog.ts +105 -0
  35. package/src/embed/variant-plan.ts +62 -0
  36. package/src/embed/variant-retry.ts +113 -0
  37. package/src/ingestion/graph-reconciliation.ts +327 -0
  38. package/src/ingestion/sync.ts +9 -272
  39. package/src/llm/http-inference.ts +6 -0
  40. package/src/llm/httpEmbedding.ts +37 -6
  41. package/src/llm/httpGeneration.ts +18 -3
  42. package/src/llm/httpRerank.ts +23 -5
  43. package/src/llm/inference-cancellation.ts +168 -0
  44. package/src/llm/inference-scope.ts +202 -0
  45. package/src/llm/lazy-ports.ts +115 -0
  46. package/src/llm/native-worker/client.ts +541 -0
  47. package/src/llm/native-worker/dispatcher.ts +228 -0
  48. package/src/llm/native-worker/embedding-identity.ts +33 -0
  49. package/src/llm/native-worker/entry.ts +173 -0
  50. package/src/llm/native-worker/errors.ts +32 -0
  51. package/src/llm/native-worker/evaluation.ts +16 -0
  52. package/src/llm/native-worker/owned-exit.ts +108 -0
  53. package/src/llm/native-worker/owner.ts +141 -0
  54. package/src/llm/native-worker/ports.ts +317 -0
  55. package/src/llm/native-worker/protocol.ts +442 -0
  56. package/src/llm/native-worker/runtime-config.ts +92 -0
  57. package/src/llm/nodeLlamaCpp/adapter.ts +77 -20
  58. package/src/llm/nodeLlamaCpp/embedding.ts +130 -46
  59. package/src/llm/nodeLlamaCpp/generation.ts +34 -5
  60. package/src/llm/nodeLlamaCpp/lifecycle-options.ts +99 -0
  61. package/src/llm/nodeLlamaCpp/lifecycle.ts +209 -204
  62. package/src/llm/nodeLlamaCpp/rerank-capacity.ts +111 -0
  63. package/src/llm/nodeLlamaCpp/rerank.ts +118 -27
  64. package/src/llm/nodeLlamaCpp/simulator-handle.ts +73 -0
  65. package/src/llm/nodeLlamaCpp/simulator-install.ts +124 -0
  66. package/src/llm/nodeLlamaCpp/simulator-session.ts +240 -0
  67. package/src/llm/nodeLlamaCpp/simulator-types.ts +80 -0
  68. package/src/llm/types.ts +35 -5
  69. package/src/mcp/context.ts +27 -0
  70. package/src/mcp/http-transport.ts +12 -10
  71. package/src/mcp/server.ts +3 -0
  72. package/src/mcp/tool-profile.ts +30 -8
  73. package/src/mcp/tools/context.ts +8 -11
  74. package/src/mcp/tools/embed.ts +1 -1
  75. package/src/mcp/tools/index-cmd.ts +1 -1
  76. package/src/mcp/tools/index.ts +10 -8
  77. package/src/mcp/tools/query.ts +14 -30
  78. package/src/mcp/tools/vsearch.ts +1 -1
  79. package/src/pipeline/answer.ts +23 -3
  80. package/src/pipeline/claim-verifier.ts +6 -0
  81. package/src/pipeline/expansion.ts +43 -40
  82. package/src/pipeline/explain.ts +6 -2
  83. package/src/pipeline/filters.ts +63 -0
  84. package/src/pipeline/fusion.ts +29 -9
  85. package/src/pipeline/graph-retrieval.ts +29 -9
  86. package/src/pipeline/hybrid.ts +198 -55
  87. package/src/pipeline/hydration.ts +161 -0
  88. package/src/pipeline/owner-fusion.ts +87 -0
  89. package/src/pipeline/rerank.ts +35 -11
  90. package/src/pipeline/search.ts +13 -2
  91. package/src/pipeline/types.ts +5 -3
  92. package/src/pipeline/vsearch.ts +87 -7
  93. package/src/sdk/client.ts +47 -3
  94. package/src/sdk/embed.ts +63 -39
  95. package/src/serve/background-runtime.ts +1 -1
  96. package/src/serve/context.ts +41 -56
  97. package/src/serve/embed-scheduler.ts +58 -35
  98. package/src/serve/public/components/IndexingProgress.tsx +46 -60
  99. package/src/serve/public/globals.built.css +1 -1
  100. package/src/serve/public/lib/shiki-language-ids.ts +14 -0
  101. package/src/serve/resident-admission.ts +36 -36
  102. package/src/serve/resident-background-work.ts +20 -2
  103. package/src/serve/resident-request.ts +11 -5
  104. package/src/serve/resident-runtime.ts +97 -61
  105. package/src/serve/resident-shutdown.ts +153 -0
  106. package/src/serve/routes/api.ts +3 -1
  107. package/src/serve/server.ts +47 -26
  108. package/src/store/migrations/028-vector-variants.ts +54 -0
  109. package/src/store/migrations/029-graph-reference-state.ts +77 -0
  110. package/src/store/migrations/index.ts +4 -0
  111. package/src/store/sqlite/adapter.ts +251 -183
  112. package/src/store/sqlite/eligibility.ts +174 -0
  113. package/src/store/sqlite/graph-edge-application.ts +66 -0
  114. package/src/store/sqlite/graph-reference-state.ts +194 -0
  115. package/src/store/sqlite/legacy-vector-ownership.ts +79 -0
  116. package/src/store/types.ts +80 -12
  117. package/src/store/vector/eligibility.ts +36 -0
  118. package/src/store/vector/freshness.ts +33 -6
  119. package/src/store/vector/lazy.ts +81 -0
  120. package/src/store/vector/sqlite-vec.ts +106 -54
  121. package/src/store/vector/stats.ts +14 -3
  122. package/src/store/vector/types.ts +35 -2
  123. package/src/store/vector/variant-search.ts +192 -0
  124. package/src/store/vector/variants.ts +451 -0
  125. package/vendor/converters/markitdown-ts/LICENSE +21 -0
  126. package/vendor/converters/markitdown-ts/dist/index.cjs +1180 -0
  127. package/vendor/converters/markitdown-ts/dist/index.d.cts +46 -0
  128. package/vendor/converters/markitdown-ts/dist/index.d.mts +46 -0
  129. package/vendor/converters/markitdown-ts/dist/index.d.ts +46 -0
  130. package/vendor/converters/markitdown-ts/dist/index.mjs +1152 -0
  131. package/vendor/converters/markitdown-ts/package.json +77 -0
  132. package/vendor/converters/officeparser/LICENSE +21 -0
  133. package/vendor/converters/officeparser/dist/OfficeConverter.d.ts +47 -0
  134. package/vendor/converters/officeparser/dist/OfficeConverter.js +76 -0
  135. package/vendor/converters/officeparser/dist/OfficeGenerator.d.ts +23 -0
  136. package/vendor/converters/officeparser/dist/OfficeGenerator.js +73 -0
  137. package/vendor/converters/officeparser/dist/OfficeParser.d.ts +106 -0
  138. package/vendor/converters/officeparser/dist/OfficeParser.js +332 -0
  139. package/vendor/converters/officeparser/dist/cli.d.ts +28 -0
  140. package/vendor/converters/officeparser/dist/cli.js +381 -0
  141. package/vendor/converters/officeparser/dist/defaults.d.ts +41 -0
  142. package/vendor/converters/officeparser/dist/defaults.js +218 -0
  143. package/vendor/converters/officeparser/dist/generators/BaseGenerator.d.ts +107 -0
  144. package/vendor/converters/officeparser/dist/generators/BaseGenerator.js +248 -0
  145. package/vendor/converters/officeparser/dist/generators/ChunkingGenerator.d.ts +82 -0
  146. package/vendor/converters/officeparser/dist/generators/ChunkingGenerator.js +797 -0
  147. package/vendor/converters/officeparser/dist/generators/CsvGenerator.d.ts +38 -0
  148. package/vendor/converters/officeparser/dist/generators/CsvGenerator.js +245 -0
  149. package/vendor/converters/officeparser/dist/generators/EpubGenerator.d.ts +43 -0
  150. package/vendor/converters/officeparser/dist/generators/EpubGenerator.js +315 -0
  151. package/vendor/converters/officeparser/dist/generators/HtmlGenerator.d.ts +59 -0
  152. package/vendor/converters/officeparser/dist/generators/HtmlGenerator.js +1942 -0
  153. package/vendor/converters/officeparser/dist/generators/MarkdownGenerator.d.ts +96 -0
  154. package/vendor/converters/officeparser/dist/generators/MarkdownGenerator.js +1175 -0
  155. package/vendor/converters/officeparser/dist/generators/PdfGenerator.d.ts +22 -0
  156. package/vendor/converters/officeparser/dist/generators/PdfGenerator.js +194 -0
  157. package/vendor/converters/officeparser/dist/generators/RtfGenerator.d.ts +29 -0
  158. package/vendor/converters/officeparser/dist/generators/RtfGenerator.js +316 -0
  159. package/vendor/converters/officeparser/dist/generators/TextGenerator.d.ts +13 -0
  160. package/vendor/converters/officeparser/dist/generators/TextGenerator.js +201 -0
  161. package/vendor/converters/officeparser/dist/index.d.ts +60 -0
  162. package/vendor/converters/officeparser/dist/index.js +72 -0
  163. package/vendor/converters/officeparser/dist/index.mjs +18 -0
  164. package/vendor/converters/officeparser/dist/officeparser.browser.d.ts +2621 -0
  165. package/vendor/converters/officeparser/dist/officeparser.browser.iife.js +1336 -0
  166. package/vendor/converters/officeparser/dist/officeparser.browser.mjs +1335 -0
  167. package/vendor/converters/officeparser/dist/officeparser.browser.slim.d.ts +2621 -0
  168. package/vendor/converters/officeparser/dist/officeparser.browser.slim.iife.js +1336 -0
  169. package/vendor/converters/officeparser/dist/officeparser.browser.slim.mjs +1335 -0
  170. package/vendor/converters/officeparser/dist/parsers/CsvParser.d.ts +9 -0
  171. package/vendor/converters/officeparser/dist/parsers/CsvParser.js +115 -0
  172. package/vendor/converters/officeparser/dist/parsers/EpubParser.d.ts +8 -0
  173. package/vendor/converters/officeparser/dist/parsers/EpubParser.js +217 -0
  174. package/vendor/converters/officeparser/dist/parsers/ExcelParser.d.ts +32 -0
  175. package/vendor/converters/officeparser/dist/parsers/ExcelParser.js +736 -0
  176. package/vendor/converters/officeparser/dist/parsers/HtmlParser.d.ts +2 -0
  177. package/vendor/converters/officeparser/dist/parsers/HtmlParser.js +1287 -0
  178. package/vendor/converters/officeparser/dist/parsers/MarkdownParser.d.ts +2 -0
  179. package/vendor/converters/officeparser/dist/parsers/MarkdownParser.js +1272 -0
  180. package/vendor/converters/officeparser/dist/parsers/OpenOfficeParser.d.ts +31 -0
  181. package/vendor/converters/officeparser/dist/parsers/OpenOfficeParser.js +1819 -0
  182. package/vendor/converters/officeparser/dist/parsers/PdfParser.d.ts +67 -0
  183. package/vendor/converters/officeparser/dist/parsers/PdfParser.js +848 -0
  184. package/vendor/converters/officeparser/dist/parsers/PowerPointParser.d.ts +32 -0
  185. package/vendor/converters/officeparser/dist/parsers/PowerPointParser.js +950 -0
  186. package/vendor/converters/officeparser/dist/parsers/RtfParser.d.ts +187 -0
  187. package/vendor/converters/officeparser/dist/parsers/RtfParser.js +1801 -0
  188. package/vendor/converters/officeparser/dist/parsers/WordParser.d.ts +79 -0
  189. package/vendor/converters/officeparser/dist/parsers/WordParser.js +1177 -0
  190. package/vendor/converters/officeparser/dist/sbom.cdx.json +1763 -0
  191. package/vendor/converters/officeparser/dist/types.d.ts +2507 -0
  192. package/vendor/converters/officeparser/dist/types.js +107 -0
  193. package/vendor/converters/officeparser/dist/utils/astUtils.d.ts +16 -0
  194. package/vendor/converters/officeparser/dist/utils/astUtils.js +33 -0
  195. package/vendor/converters/officeparser/dist/utils/chartUtils.d.ts +6 -0
  196. package/vendor/converters/officeparser/dist/utils/chartUtils.js +257 -0
  197. package/vendor/converters/officeparser/dist/utils/configUtils.d.ts +44 -0
  198. package/vendor/converters/officeparser/dist/utils/configUtils.js +315 -0
  199. package/vendor/converters/officeparser/dist/utils/dateUtils.d.ts +17 -0
  200. package/vendor/converters/officeparser/dist/utils/dateUtils.js +69 -0
  201. package/vendor/converters/officeparser/dist/utils/envUtils.d.ts +29 -0
  202. package/vendor/converters/officeparser/dist/utils/envUtils.js +152 -0
  203. package/vendor/converters/officeparser/dist/utils/errorUtils.d.ts +72 -0
  204. package/vendor/converters/officeparser/dist/utils/errorUtils.js +245 -0
  205. package/vendor/converters/officeparser/dist/utils/imageUtils.d.ts +66 -0
  206. package/vendor/converters/officeparser/dist/utils/imageUtils.js +133 -0
  207. package/vendor/converters/officeparser/dist/utils/mathUtils.d.ts +42 -0
  208. package/vendor/converters/officeparser/dist/utils/mathUtils.js +385 -0
  209. package/vendor/converters/officeparser/dist/utils/moduleLoader.d.ts +18 -0
  210. package/vendor/converters/officeparser/dist/utils/moduleLoader.js +106 -0
  211. package/vendor/converters/officeparser/dist/utils/ocrUtils.d.ts +42 -0
  212. package/vendor/converters/officeparser/dist/utils/ocrUtils.js +428 -0
  213. package/vendor/converters/officeparser/dist/utils/sanitize.d.ts +148 -0
  214. package/vendor/converters/officeparser/dist/utils/sanitize.js +344 -0
  215. package/vendor/converters/officeparser/dist/utils/sheetUtils.d.ts +7 -0
  216. package/vendor/converters/officeparser/dist/utils/sheetUtils.js +35 -0
  217. package/vendor/converters/officeparser/dist/utils/styleMapper.d.ts +36 -0
  218. package/vendor/converters/officeparser/dist/utils/styleMapper.js +224 -0
  219. package/vendor/converters/officeparser/dist/utils/xmlUtils.d.ts +163 -0
  220. package/vendor/converters/officeparser/dist/utils/xmlUtils.js +461 -0
  221. package/vendor/converters/officeparser/dist/utils/zipUtils.d.ts +134 -0
  222. package/vendor/converters/officeparser/dist/utils/zipUtils.js +337 -0
  223. package/vendor/converters/officeparser/package.json +147 -0
  224. package/vendor/converters/upstream-manifest.json +124 -0
  225. package/vendor/dependency-fixes/README.md +77 -0
  226. package/vendor/dependency-fixes/vendor-converters.py +83 -0
  227. package/browser-extension/artifacts/gno-browser-clipper-v1.46.0.zip +0 -0
  228. package/browser-extension/artifacts/gno-browser-clipper-v1.46.0.zip.sha256 +0 -1
  229. package/browser-extension/dist/chunk-627emwpj.js +0 -75
  230. /package/browser-extension/dist/{chunk-ydfx5d7p.css → chunk-z74y8n8c.css} +0 -0
@@ -1,25 +1,35 @@
1
+ import type { EmbeddingPort } from "../llm/types";
1
2
  /**
2
3
  * Shared embedding backlog processor.
3
4
  * Used by CLI embed, Web scheduler, and MCP tools.
4
5
  *
5
6
  * @module src/embed/backlog
6
7
  */
7
-
8
- import type { EmbeddingPort } from "../llm/types";
9
8
  import type { StoreResult } from "../store/types";
10
9
  import type {
11
10
  BacklogItem,
12
11
  VectorIndexPort,
13
12
  VectorStatsPort,
14
13
  } from "../store/vector";
14
+ import type { VectorVariantStore } from "../store/vector/variants";
15
15
 
16
+ import {
17
+ assertInferenceActive,
18
+ isBackgroundInference,
19
+ } from "../llm/inference-scope";
16
20
  import { err, ok } from "../store/types";
17
- import { getEmbeddingFingerprint } from "./fingerprint";
21
+ import { getVectorStatsDatabase } from "../store/vector/stats";
22
+ import { createVectorVariantStore } from "../store/vector/variants";
23
+ import {
24
+ getEmbeddingFingerprint,
25
+ getVariantModelFingerprint,
26
+ } from "./fingerprint";
18
27
  import {
19
28
  chunkRetryKey,
20
29
  embedAndStoreBatch,
21
30
  MAX_EMBED_CHUNK_ATTEMPTS,
22
31
  } from "./retry";
32
+ import { embedVariantBacklog } from "./variant-backlog";
23
33
 
24
34
  // ─────────────────────────────────────────────────────────────────────────────
25
35
  // Types
@@ -32,6 +42,11 @@ export interface EmbedBacklogDeps {
32
42
  collection?: string;
33
43
  modelUri: string;
34
44
  batchSize?: number;
45
+ force?: boolean;
46
+ onProgress?: (embedded: number, errors: number) => void;
47
+ variantStore?: VectorVariantStore;
48
+ /** Recheck the effective runtime identity after asynchronous inference. */
49
+ identityStillCurrent?: () => boolean;
35
50
  }
36
51
 
37
52
  export interface EmbedBacklogResult {
@@ -62,8 +77,16 @@ interface Cursor {
62
77
  export async function embedBacklog(
63
78
  deps: EmbedBacklogDeps
64
79
  ): Promise<StoreResult<EmbedBacklogResult>> {
80
+ assertInferenceActive();
81
+ const prepared = await prepareEmbeddingBacklog(deps);
82
+ if (!prepared.ok) return prepared;
83
+ deps = prepared.value;
84
+ if (deps.variantStore) return embedVariantBacklog(deps, deps.variantStore);
65
85
  const { statsPort, embedPort, vectorIndex, modelUri, collection } = deps;
66
- const batchSize = deps.batchSize ?? 32;
86
+ const background = isBackgroundInference();
87
+ const batchSize = background
88
+ ? Math.min(deps.batchSize ?? 32, 32)
89
+ : (deps.batchSize ?? 32);
67
90
  const embedFingerprint = getEmbeddingFingerprint({
68
91
  modelUri,
69
92
  dimensions: vectorIndex.dimensions,
@@ -77,6 +100,7 @@ export async function embedBacklog(
77
100
 
78
101
  const enqueueRetryItems = (items: BacklogItem[], attempts: number): void => {
79
102
  for (const item of items) {
103
+ assertInferenceActive();
80
104
  const key = chunkRetryKey(item);
81
105
  const existing = retryQueue.get(key);
82
106
  retryQueue.set(key, {
@@ -97,8 +121,10 @@ export async function embedBacklog(
97
121
  );
98
122
 
99
123
  for (let idx = 0; idx < entries.length; idx += batchSize) {
124
+ assertInferenceActive();
100
125
  const slice = entries.slice(idx, idx + batchSize);
101
126
  for (const entry of slice) {
127
+ assertInferenceActive();
102
128
  retryQueue.delete(chunkRetryKey(entry.item));
103
129
  entry.attempts += 1;
104
130
  }
@@ -109,6 +135,8 @@ export async function embedBacklog(
109
135
  items: slice.map((entry) => entry.item),
110
136
  modelUri,
111
137
  embedFingerprint,
138
+ identityStillCurrent: deps.identityStillCurrent,
139
+ statsPort,
112
140
  });
113
141
 
114
142
  embedded += retryResult.embedded;
@@ -120,6 +148,7 @@ export async function embedBacklog(
120
148
  retryResult.retryItems.map((item) => chunkRetryKey(item))
121
149
  );
122
150
  for (const entry of slice) {
151
+ assertInferenceActive();
123
152
  if (!retryByKey.has(chunkRetryKey(entry.item))) {
124
153
  continue;
125
154
  }
@@ -136,6 +165,7 @@ export async function embedBacklog(
136
165
 
137
166
  try {
138
167
  while (true) {
168
+ assertInferenceActive();
139
169
  // Get next batch using seek pagination
140
170
  const batchResult = await statsPort.getBacklog(
141
171
  modelUri,
@@ -169,10 +199,20 @@ export async function embedBacklog(
169
199
  items: batch,
170
200
  modelUri,
171
201
  embedFingerprint,
202
+ identityStillCurrent: deps.identityStillCurrent,
203
+ statsPort,
172
204
  });
173
205
  embedded += batchStoreResult.embedded;
174
206
  errors += batchStoreResult.errors;
175
207
  contentionErrors += batchStoreResult.contentionErrors;
208
+ if (background) {
209
+ errors += batchStoreResult.retryItems.length;
210
+ deps.onProgress?.(embedded, errors);
211
+ // Each cursor page is a turn. Failed early pages cannot starve later work.
212
+ await Bun.sleep(0);
213
+ if (deps.identityStillCurrent && !deps.identityStillCurrent()) break;
214
+ continue;
215
+ }
176
216
  enqueueRetryItems(batchStoreResult.retryItems, 1);
177
217
 
178
218
  if (embedded > beforeEmbedded) {
@@ -198,6 +238,7 @@ export async function embedBacklog(
198
238
  }
199
239
  }
200
240
 
241
+ assertInferenceActive();
201
242
  return ok({ embedded, errors, contentionErrors, syncError });
202
243
  } catch (e) {
203
244
  return err(
@@ -206,3 +247,65 @@ export async function embedBacklog(
206
247
  );
207
248
  }
208
249
  }
250
+
251
+ /** Resolve authority before counts, dry runs, forced work, or early returns. */
252
+ export async function prepareEmbeddingBacklog(
253
+ deps: EmbedBacklogDeps
254
+ ): Promise<StoreResult<EmbedBacklogDeps>> {
255
+ if (deps.variantStore) return ok(deps);
256
+ const db = getVectorStatsDatabase(deps.statsPort);
257
+ if (db) {
258
+ try {
259
+ const initialized = await deps.embedPort.init();
260
+ if (!initialized.ok) return err("INTERNAL", initialized.error.message);
261
+ const identity = deps.embedPort.getIdentity?.();
262
+ if (identity) {
263
+ const identitySnapshot = JSON.stringify(identity);
264
+ const dimensions = deps.embedPort.dimensions();
265
+ const variantStore = await createVectorVariantStore(db, {
266
+ model: deps.modelUri,
267
+ modelFingerprint: getVariantModelFingerprint(
268
+ { modelUri: deps.modelUri, dimensions },
269
+ identity
270
+ ),
271
+ contextSize: identity.contextSize,
272
+ truncationPolicy: identity.truncationPolicy,
273
+ dimensions,
274
+ });
275
+ return ok({
276
+ ...deps,
277
+ variantStore,
278
+ identityStillCurrent: () =>
279
+ (deps.identityStillCurrent?.() ?? true) &&
280
+ deps.embedPort.modelUri === deps.modelUri &&
281
+ deps.embedPort.dimensions() === dimensions &&
282
+ JSON.stringify(deps.embedPort.getIdentity?.()) === identitySnapshot,
283
+ });
284
+ }
285
+ // Unverified/HTTP ports retain legacy behavior until variant authority exists.
286
+ if (
287
+ db
288
+ .query(
289
+ "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'vector_partitions'"
290
+ )
291
+ .get() &&
292
+ db
293
+ .query(
294
+ "SELECT 1 FROM vector_partitions WHERE model = ? AND state = ? AND activated_epoch IS NOT NULL LIMIT 1"
295
+ )
296
+ .get(deps.modelUri, "active")
297
+ ) {
298
+ return err(
299
+ "INVALID_INPUT",
300
+ "Effective embedding identity unavailable after variant activation"
301
+ );
302
+ }
303
+ } catch (cause) {
304
+ return err(
305
+ "QUERY_FAILED",
306
+ cause instanceof Error ? cause.message : String(cause)
307
+ );
308
+ }
309
+ }
310
+ return ok(deps);
311
+ }
@@ -1,13 +1,17 @@
1
+ import type { EmbeddingPort, LlmResult } from "../llm/types";
1
2
  /**
2
3
  * Shared embedding batch helpers.
3
4
  *
4
5
  * @module src/embed/batch
5
6
  */
6
7
 
7
- import type { EmbeddingPort, LlmResult } from "../llm/types";
8
-
9
8
  import { getEmbeddingCompatibilityProfile } from "../llm/embedding-compatibility";
10
9
  import { inferenceFailedError } from "../llm/errors";
10
+ import {
11
+ assertInferenceActive,
12
+ assertInferenceResult,
13
+ isBackgroundInference,
14
+ } from "../llm/inference-scope";
11
15
 
12
16
  export interface EmbedBatchRecoveryResult {
13
17
  vectors: Array<number[] | null>;
@@ -70,8 +74,26 @@ export async function embedTextsWithRecovery(
70
74
  }
71
75
 
72
76
  const profile = getEmbeddingCompatibilityProfile(embedPort.modelUri);
77
+ // A background page gets one provider attempt. Recovery remains durable for
78
+ // the next pass instead of expanding the current native scheduling turn.
79
+ if (isBackgroundInference() && profile.batchEmbeddingTrusted) {
80
+ const result = await embedPort.embedBatch(texts);
81
+ assertInferenceResult(result);
82
+ if (!result.ok) return result;
83
+ const complete = result.value.length === texts.length;
84
+ return {
85
+ ok: true,
86
+ value: {
87
+ vectors: complete ? result.value : texts.map(() => null),
88
+ batchFailed: !complete,
89
+ fallbackErrors: complete ? 0 : texts.length,
90
+ failureSamples: complete ? [] : ["Embedding count mismatch"],
91
+ },
92
+ };
93
+ }
73
94
  if (profile.batchEmbeddingTrusted) {
74
95
  let batchResult = await embedPort.embedBatch(texts);
96
+ assertInferenceResult(batchResult);
75
97
  if (!batchResult.ok) {
76
98
  const formattedBatchError = formatFailureMessage(batchResult.error);
77
99
  if (isDisposedFailure(formattedBatchError)) {
@@ -80,6 +102,7 @@ export async function embedTextsWithRecovery(
80
102
  return reset;
81
103
  }
82
104
  batchResult = await embedPort.embedBatch(texts);
105
+ assertInferenceResult(batchResult);
83
106
  }
84
107
  }
85
108
  if (batchResult.ok && batchResult.value.length === texts.length) {
@@ -166,6 +189,7 @@ async function recoverWithAdaptiveBatches(
166
189
 
167
190
  if (rangeTexts.length === 1) {
168
191
  const result = await embedPort.embed(rangeTexts[0] ?? "");
192
+ assertInferenceResult(result);
169
193
  if (result.ok) {
170
194
  vectors[offset] = result.value;
171
195
  return;
@@ -179,6 +203,7 @@ async function recoverWithAdaptiveBatches(
179
203
  null;
180
204
  if (!batchAlreadyFailed) {
181
205
  batchResult = await embedPort.embedBatch(rangeTexts);
206
+ assertInferenceResult(batchResult);
182
207
  }
183
208
  if (
184
209
  batchResult &&
@@ -186,6 +211,7 @@ async function recoverWithAdaptiveBatches(
186
211
  batchResult.value.length === rangeTexts.length
187
212
  ) {
188
213
  for (const [index, vector] of batchResult.value.entries()) {
214
+ assertInferenceActive();
189
215
  vectors[offset + index] = vector;
190
216
  }
191
217
  return;
@@ -223,6 +249,12 @@ async function recoverWithAdaptiveBatches(
223
249
  },
224
250
  };
225
251
  } catch (error) {
252
+ assertInferenceActive();
253
+ if (
254
+ error instanceof Error &&
255
+ ["AbortError", "TimeoutError"].includes(error.name)
256
+ )
257
+ throw error;
226
258
  return {
227
259
  ok: false,
228
260
  error: inferenceFailedError(
@@ -245,7 +277,9 @@ async function recoverIndividually(
245
277
  let fallbackErrors = 0;
246
278
 
247
279
  for (const text of texts) {
280
+ assertInferenceActive();
248
281
  const result = await embedPort.embed(text);
282
+ assertInferenceResult(result);
249
283
  if (result.ok) {
250
284
  vectors.push(result.value);
251
285
  } else {
@@ -266,6 +300,12 @@ async function recoverIndividually(
266
300
  },
267
301
  };
268
302
  } catch (error) {
303
+ assertInferenceActive();
304
+ if (
305
+ error instanceof Error &&
306
+ ["AbortError", "TimeoutError"].includes(error.name)
307
+ )
308
+ throw error;
269
309
  return {
270
310
  ok: false,
271
311
  error: inferenceFailedError(
@@ -35,3 +35,19 @@ export function getEmbeddingFingerprint(
35
35
  .update(JSON.stringify(payload))
36
36
  .digest("hex");
37
37
  }
38
+
39
+ /** Partition provenance combines actual weights/runtime with the unchanged formatter policy. */
40
+ export function getVariantModelFingerprint(
41
+ input: EmbeddingFingerprintInput,
42
+ identity: { modelFingerprint: string; runtimeFingerprint: string }
43
+ ): string {
44
+ return new Bun.CryptoHasher("sha256")
45
+ .update(
46
+ JSON.stringify([
47
+ identity.modelFingerprint,
48
+ identity.runtimeFingerprint,
49
+ getEmbeddingFingerprint(input),
50
+ ])
51
+ )
52
+ .digest("hex");
53
+ }
@@ -1,9 +1,17 @@
1
1
  import type { EmbeddingPort } from "../llm/types";
2
2
  import type { StoreResult } from "../store/types";
3
- import type { BacklogItem, VectorIndexPort, VectorRow } from "../store/vector";
3
+ import type {
4
+ BacklogItem,
5
+ VectorIndexPort,
6
+ VectorRow,
7
+ VectorStatsPort,
8
+ } from "../store/vector";
4
9
 
5
10
  import { isSqliteLockContention } from "../core/file-lock";
11
+ import { assertInferenceActive } from "../llm/inference-scope";
6
12
  import { formatDocForEmbedding } from "../pipeline/contextual";
13
+ import { err, ok } from "../store/types";
14
+ import { getVectorStatsDatabase } from "../store/vector/stats";
7
15
  import { embedTextsWithRecovery } from "./batch";
8
16
 
9
17
  export const MAX_EMBED_CHUNK_ATTEMPTS = 2;
@@ -42,12 +50,21 @@ export interface EmbedStoreBatchResult {
42
50
  // src/cli/commands/index-cmd.ts) must surface contentionErrors in their
43
51
  // summary and exit non-zero — the integrator wires that.
44
52
 
45
- export function chunkRetryKey(item: Pick<BacklogItem, "mirrorHash" | "seq">) {
53
+ export function chunkRetryKey(
54
+ item: Pick<BacklogItem, "mirrorHash" | "seq"> & {
55
+ documentId?: number;
56
+ inputHash?: string;
57
+ }
58
+ ) {
59
+ if (item.documentId !== undefined && item.inputHash !== undefined) {
60
+ return `${item.documentId}\0${item.mirrorHash}\0${item.seq}\0${item.inputHash}`;
61
+ }
46
62
  return `${item.mirrorHash}\0${item.seq}`;
47
63
  }
48
64
 
49
65
  export function addUniqueSamples(target: string[], samples: string[]): void {
50
66
  for (const sample of samples) {
67
+ assertInferenceActive();
51
68
  if (target.length >= MAX_EMBED_FAILURE_SAMPLES) {
52
69
  break;
53
70
  }
@@ -127,6 +144,7 @@ export async function upsertVectorsWithContentionRetry(
127
144
  vectors: VectorRow[],
128
145
  delays?: number[]
129
146
  ): Promise<StoreResult<void>> {
147
+ assertInferenceActive();
130
148
  let storeResult = await vectorIndex.upsertVectors(vectors);
131
149
  let attempts = 1;
132
150
  while (
@@ -138,6 +156,7 @@ export async function upsertVectorsWithContentionRetry(
138
156
  if (delayMs > 0) {
139
157
  await Bun.sleep(delayMs);
140
158
  }
159
+ assertInferenceActive();
141
160
  storeResult = await vectorIndex.upsertVectors(vectors);
142
161
  attempts += 1;
143
162
  }
@@ -150,10 +169,22 @@ export async function embedAndStoreBatch(params: {
150
169
  items: BacklogItem[];
151
170
  modelUri: string;
152
171
  embedFingerprint: string;
172
+ statsPort?: VectorStatsPort;
173
+ identityStillCurrent?: () => boolean;
153
174
  /** Test seam: override contention-retry delays in milliseconds. */
154
175
  delays?: number[];
155
176
  }): Promise<EmbedStoreBatchResult> {
156
177
  const { embedPort, vectorIndex, items, modelUri, embedFingerprint } = params;
178
+ const db = params.statsPort && getVectorStatsDatabase(params.statsPort);
179
+ if (db && !vectorIndex.upsertVectorsChecked)
180
+ return {
181
+ embedded: 0,
182
+ errors: items.length,
183
+ contentionErrors: 0,
184
+ retryItems: [],
185
+ errorSamples: ["Atomic vector checkpoint unavailable"],
186
+ batchFailed: false,
187
+ };
157
188
  const embedResult = await embedTextsWithRecovery(
158
189
  embedPort,
159
190
  items.map((item) =>
@@ -180,6 +211,7 @@ export async function embedAndStoreBatch(params: {
180
211
  const vectors: VectorRow[] = [];
181
212
  const retryItems: BacklogItem[] = [];
182
213
  for (const [idx, item] of items.entries()) {
214
+ assertInferenceActive();
183
215
  const embedding = embedResult.value.vectors[idx];
184
216
  if (!embedding) {
185
217
  retryItems.push(item);
@@ -207,8 +239,75 @@ export async function embedAndStoreBatch(params: {
207
239
  };
208
240
  }
209
241
 
242
+ let written = 0;
243
+ let committedRows: VectorRow[] = [];
244
+ const itemsByKey = new Map(items.map((item) => [chunkRetryKey(item), item]));
245
+ const checkpoint = (rows: VectorRow[]): VectorRow[] => {
246
+ assertInferenceActive();
247
+ if (params.identityStillCurrent && !params.identityStillCurrent())
248
+ return [];
249
+ if (!db) return rows;
250
+ // A concurrently activated verified partition forbids a late legacy write.
251
+ if (
252
+ db
253
+ .query(
254
+ "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'vector_partitions'"
255
+ )
256
+ .get() &&
257
+ db
258
+ .query(
259
+ "SELECT 1 FROM vector_partitions WHERE model = ? AND state = 'active' AND activated_epoch IS NOT NULL LIMIT 1"
260
+ )
261
+ .get(modelUri)
262
+ )
263
+ return [];
264
+ return rows.filter((row) => {
265
+ const item = itemsByKey.get(chunkRetryKey(row));
266
+ const current = db
267
+ .query<{ text: string; title: string | null }, [string, number]>(`
268
+ SELECT c.text, d.title FROM content_chunks c
269
+ JOIN documents d ON d.mirror_hash = c.mirror_hash AND d.active = 1
270
+ WHERE c.mirror_hash = ? AND c.seq = ? ORDER BY d.id LIMIT 1
271
+ `)
272
+ .get(row.mirrorHash, row.seq);
273
+ return (
274
+ !!item &&
275
+ !!current &&
276
+ formatDocForEmbedding(
277
+ current.text,
278
+ current.title ?? undefined,
279
+ modelUri
280
+ ) ===
281
+ formatDocForEmbedding(item.text, item.title ?? undefined, modelUri)
282
+ );
283
+ });
284
+ };
210
285
  const storeResult = await upsertVectorsWithContentionRetry(
211
- vectorIndex,
286
+ {
287
+ upsertVectors: async (rows) => {
288
+ if (vectorIndex.upsertVectorsChecked) {
289
+ const result = await vectorIndex.upsertVectorsChecked(
290
+ rows,
291
+ (candidates) => {
292
+ committedRows = checkpoint(candidates);
293
+ return committedRows;
294
+ }
295
+ );
296
+ if (!result.ok) return result;
297
+ written = result.value;
298
+ return ok(undefined);
299
+ }
300
+ if (db)
301
+ return err("INVALID_INPUT", "Atomic vector checkpoint unavailable");
302
+ const valid = checkpoint(rows);
303
+ const result = await vectorIndex.upsertVectors(valid);
304
+ if (result.ok) {
305
+ written = valid.length;
306
+ committedRows = valid;
307
+ }
308
+ return result;
309
+ },
310
+ },
212
311
  vectors,
213
312
  params.delays
214
313
  );
@@ -237,11 +336,20 @@ export async function embedAndStoreBatch(params: {
237
336
  };
238
337
  }
239
338
 
339
+ const vectorKeys = new Set(vectors.map((row) => chunkRetryKey(row)));
340
+ const committedKeys = new Set(committedRows.map((row) => chunkRetryKey(row)));
240
341
  return {
241
- embedded: vectors.length,
342
+ embedded: written,
242
343
  errors: 0,
243
344
  contentionErrors: 0,
244
- retryItems,
345
+ retryItems: [
346
+ ...retryItems,
347
+ ...items.filter(
348
+ (item) =>
349
+ vectorKeys.has(chunkRetryKey(item)) &&
350
+ !committedKeys.has(chunkRetryKey(item))
351
+ ),
352
+ ],
245
353
  errorSamples: embedResult.value.failureSamples,
246
354
  suggestion: embedResult.value.retrySuggestion,
247
355
  batchFailed: embedResult.value.batchFailed,
@@ -0,0 +1,105 @@
1
+ import type { StoreResult } from "../store/types";
2
+ import type { VectorVariantStore } from "../store/vector/variants";
3
+ import type { EmbedBacklogDeps, EmbedBacklogResult } from "./backlog";
4
+
5
+ import {
6
+ assertInferenceActive,
7
+ isBackgroundInference,
8
+ } from "../llm/inference-scope";
9
+ import { err, ok } from "../store/types";
10
+ import { getVectorStatsDatabase } from "../store/vector/stats";
11
+ import { variantBacklogPage } from "./variant-plan";
12
+ import { embedVariantBatch } from "./variant-retry";
13
+
14
+ /** One bounded owner pass plus one retry; mutations stay durably pending for the next pass. */
15
+ export async function embedVariantBacklog(
16
+ deps: EmbedBacklogDeps,
17
+ store: VectorVariantStore
18
+ ): Promise<StoreResult<EmbedBacklogResult>> {
19
+ const identityStillCurrent = deps.identityStillCurrent;
20
+ if (!identityStillCurrent)
21
+ return err(
22
+ "INVALID_INPUT",
23
+ "Variant embedding requires an effective runtime identity check"
24
+ );
25
+ const db = getVectorStatsDatabase(deps.statsPort);
26
+ if (deps.collection && !db)
27
+ return err(
28
+ "INVALID_INPUT",
29
+ "Collection-scoped variant backlog requires document ownership storage"
30
+ );
31
+ const background = isBackgroundInference();
32
+ const batchSize = background
33
+ ? Math.min(deps.batchSize ?? 32, 32)
34
+ : (deps.batchSize ?? 32);
35
+ if (!Number.isSafeInteger(batchSize) || batchSize < 1)
36
+ return err("INVALID_INPUT", "Invalid embedding batch size");
37
+ const total = { embedded: 0, errors: 0, contentionErrors: 0 };
38
+ let after: { documentId: number; seq: number } | undefined;
39
+ try {
40
+ while (identityStillCurrent()) {
41
+ assertInferenceActive();
42
+ const pending = variantBacklogPage(deps, store, batchSize, after);
43
+ if (!pending.length) break;
44
+ const last = pending.at(-1)!;
45
+ after = { documentId: last.documentId, seq: last.seq };
46
+ const owners = deps.collection
47
+ ? pending.filter(
48
+ (owner) =>
49
+ !!db!
50
+ .query(
51
+ "SELECT 1 FROM documents WHERE id = ? AND collection = ? AND active = 1"
52
+ )
53
+ .get(owner.documentId, deps.collection!)
54
+ )
55
+ : pending;
56
+ let result = await embedVariantBatch({
57
+ store,
58
+ embedPort: deps.embedPort,
59
+ owners,
60
+ identityStillCurrent,
61
+ force: deps.force,
62
+ });
63
+ total.embedded += result.embedded;
64
+ total.errors += result.errors;
65
+ total.contentionErrors += result.contentionErrors;
66
+ if (background) total.errors += result.retryOwners.length;
67
+ if (!background && result.retryOwners.length) {
68
+ result = await embedVariantBatch({
69
+ store,
70
+ embedPort: deps.embedPort,
71
+ owners: result.retryOwners,
72
+ identityStillCurrent,
73
+ force: deps.force,
74
+ });
75
+ total.embedded += result.embedded;
76
+ total.errors += result.errors + result.retryOwners.length;
77
+ total.contentionErrors += result.contentionErrors;
78
+ }
79
+ deps.onProgress?.(total.embedded, total.errors);
80
+ if (background) await Bun.sleep(0);
81
+ }
82
+ assertInferenceActive();
83
+ // Capture the epoch before checking completeness; activate rechecks under write lock.
84
+ const epoch = store.epoch();
85
+ if (identityStillCurrent() && !store.pending({ limit: 1 }).length) {
86
+ try {
87
+ if (!store.isActive()) store.syncIndex();
88
+ if (!identityStillCurrent()) return ok(total);
89
+ store.activate(epoch);
90
+ } catch (cause) {
91
+ return ok({
92
+ ...total,
93
+ syncError: cause instanceof Error ? cause.message : String(cause),
94
+ });
95
+ }
96
+ }
97
+ assertInferenceActive();
98
+ return ok(total);
99
+ } catch (cause) {
100
+ return err(
101
+ "QUERY_FAILED",
102
+ cause instanceof Error ? cause.message : String(cause)
103
+ );
104
+ }
105
+ }
@@ -0,0 +1,62 @@
1
+ import type { VectorVariantStore } from "../store/vector/variants";
2
+ import type { EmbedBacklogDeps } from "./backlog";
3
+
4
+ import { getVectorStatsDatabase } from "../store/vector/stats";
5
+
6
+ /** Counts and execution enumerate the same exact current owner partition. */
7
+ export function variantBacklogPage(
8
+ deps: EmbedBacklogDeps,
9
+ store: VectorVariantStore,
10
+ limit: number,
11
+ after?: { documentId: number; seq: number }
12
+ ) {
13
+ if (!deps.force) return store.pending({ limit, after });
14
+ const db = getVectorStatsDatabase(deps.statsPort);
15
+ if (!db)
16
+ throw new Error("Forced variants require document ownership storage");
17
+ const rows = db
18
+ .query<
19
+ { documentId: number; seq: number },
20
+ [number, number, number, number]
21
+ >(`
22
+ SELECT d.id AS documentId, c.seq FROM documents d
23
+ JOIN content_chunks c ON c.mirror_hash = d.mirror_hash
24
+ WHERE d.active = 1 AND (d.id > ? OR (d.id = ? AND c.seq > ?))
25
+ ORDER BY d.id, c.seq LIMIT ?
26
+ `)
27
+ .all(
28
+ after?.documentId ?? -1,
29
+ after?.documentId ?? -1,
30
+ after?.seq ?? -1,
31
+ limit
32
+ );
33
+ return rows.flatMap((row) => {
34
+ const owner = store.current(row.documentId, row.seq);
35
+ return owner ? [owner] : [];
36
+ });
37
+ }
38
+
39
+ export function countVariantBacklog(deps: EmbedBacklogDeps): number {
40
+ const store = deps.variantStore;
41
+ if (!store) throw new Error("Verified variant identity required");
42
+ const db = getVectorStatsDatabase(deps.statsPort);
43
+ let after: { documentId: number; seq: number } | undefined;
44
+ let count = 0;
45
+ while (true) {
46
+ const owners = variantBacklogPage(deps, store, 256, after);
47
+ if (!owners.length) return count;
48
+ for (const owner of owners) {
49
+ if (
50
+ !deps.collection ||
51
+ db
52
+ ?.query(
53
+ "SELECT 1 FROM documents WHERE id = ? AND collection = ? AND active = 1"
54
+ )
55
+ .get(owner.documentId, deps.collection)
56
+ )
57
+ count += 1;
58
+ }
59
+ const last = owners.at(-1)!;
60
+ after = { documentId: last.documentId, seq: last.seq };
61
+ }
62
+ }