@gmickel/gno 1.46.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/THIRD_PARTY_NOTICES.md +46 -0
- package/assets/skill/SKILL.md +7 -6
- package/assets/spa-production.json.gz +0 -0
- package/browser-extension/artifacts/gno-browser-clipper-v2.0.0.zip +0 -0
- package/browser-extension/artifacts/gno-browser-clipper-v2.0.0.zip.sha256 +1 -0
- package/browser-extension/dist/chunk-4tc9v0ja.js +74 -0
- package/browser-extension/dist/manifest.json +1 -1
- package/browser-extension/dist/preview.html +1 -1
- package/browser-extension/dist/service-worker.js +32 -33
- package/bunfig.toml +2 -0
- package/package.json +40 -26
- package/spec/cli.md +21 -4
- package/spec/db/schema.sql +146 -1
- package/spec/mcp.md +26 -0
- package/src/app/context-runtime-types.ts +3 -0
- package/src/app/context-runtime.ts +2 -0
- package/src/cli/commands/ask.ts +6 -1
- package/src/cli/commands/daemon.ts +21 -8
- package/src/cli/commands/embed.ts +77 -41
- package/src/cli/detach.ts +3 -2
- package/src/config/types.ts +3 -3
- package/src/converters/adapters/markitdownTs/adapter.ts +1 -2
- package/src/converters/adapters/officeparser/adapter.ts +1 -2
- package/src/converters/versions.ts +6 -8
- package/src/core/context-evidence.ts +8 -4
- package/src/core/job-manager.ts +95 -13
- package/src/core/network-boundary-inventory.ts +10 -0
- package/src/core/shutdown-budget.ts +45 -0
- package/src/embed/backlog.ts +107 -4
- package/src/embed/batch.ts +42 -2
- package/src/embed/fingerprint.ts +16 -0
- package/src/embed/retry.ts +113 -5
- package/src/embed/variant-backlog.ts +105 -0
- package/src/embed/variant-plan.ts +62 -0
- package/src/embed/variant-retry.ts +113 -0
- package/src/ingestion/graph-reconciliation.ts +327 -0
- package/src/ingestion/sync.ts +9 -272
- package/src/llm/http-inference.ts +6 -0
- package/src/llm/httpEmbedding.ts +37 -6
- package/src/llm/httpGeneration.ts +18 -3
- package/src/llm/httpRerank.ts +23 -5
- package/src/llm/inference-cancellation.ts +168 -0
- package/src/llm/inference-scope.ts +202 -0
- package/src/llm/lazy-ports.ts +115 -0
- package/src/llm/native-worker/client.ts +541 -0
- package/src/llm/native-worker/dispatcher.ts +228 -0
- package/src/llm/native-worker/embedding-identity.ts +33 -0
- package/src/llm/native-worker/entry.ts +173 -0
- package/src/llm/native-worker/errors.ts +32 -0
- package/src/llm/native-worker/evaluation.ts +16 -0
- package/src/llm/native-worker/owned-exit.ts +108 -0
- package/src/llm/native-worker/owner.ts +141 -0
- package/src/llm/native-worker/ports.ts +317 -0
- package/src/llm/native-worker/protocol.ts +442 -0
- package/src/llm/native-worker/runtime-config.ts +92 -0
- package/src/llm/nodeLlamaCpp/adapter.ts +77 -20
- package/src/llm/nodeLlamaCpp/embedding.ts +130 -46
- package/src/llm/nodeLlamaCpp/generation.ts +34 -5
- package/src/llm/nodeLlamaCpp/lifecycle-options.ts +99 -0
- package/src/llm/nodeLlamaCpp/lifecycle.ts +209 -204
- package/src/llm/nodeLlamaCpp/rerank-capacity.ts +111 -0
- package/src/llm/nodeLlamaCpp/rerank.ts +118 -27
- package/src/llm/nodeLlamaCpp/simulator-handle.ts +73 -0
- package/src/llm/nodeLlamaCpp/simulator-install.ts +124 -0
- package/src/llm/nodeLlamaCpp/simulator-session.ts +240 -0
- package/src/llm/nodeLlamaCpp/simulator-types.ts +80 -0
- package/src/llm/types.ts +35 -5
- package/src/mcp/context.ts +27 -0
- package/src/mcp/http-transport.ts +12 -10
- package/src/mcp/server.ts +3 -0
- package/src/mcp/tool-profile.ts +30 -8
- package/src/mcp/tools/context.ts +8 -11
- package/src/mcp/tools/embed.ts +1 -1
- package/src/mcp/tools/index-cmd.ts +1 -1
- package/src/mcp/tools/index.ts +10 -8
- package/src/mcp/tools/query.ts +14 -30
- package/src/mcp/tools/vsearch.ts +1 -1
- package/src/pipeline/answer.ts +23 -3
- package/src/pipeline/claim-verifier.ts +6 -0
- package/src/pipeline/expansion.ts +43 -40
- package/src/pipeline/explain.ts +6 -2
- package/src/pipeline/filters.ts +63 -0
- package/src/pipeline/fusion.ts +29 -9
- package/src/pipeline/graph-retrieval.ts +29 -9
- package/src/pipeline/hybrid.ts +198 -55
- package/src/pipeline/hydration.ts +161 -0
- package/src/pipeline/owner-fusion.ts +87 -0
- package/src/pipeline/rerank.ts +35 -11
- package/src/pipeline/search.ts +13 -2
- package/src/pipeline/types.ts +5 -3
- package/src/pipeline/vsearch.ts +87 -7
- package/src/sdk/client.ts +47 -3
- package/src/sdk/embed.ts +63 -39
- package/src/serve/background-runtime.ts +1 -1
- package/src/serve/context.ts +41 -56
- package/src/serve/embed-scheduler.ts +58 -35
- package/src/serve/public/components/IndexingProgress.tsx +46 -60
- package/src/serve/public/globals.built.css +1 -1
- package/src/serve/public/lib/shiki-language-ids.ts +14 -0
- package/src/serve/resident-admission.ts +36 -36
- package/src/serve/resident-background-work.ts +20 -2
- package/src/serve/resident-request.ts +11 -5
- package/src/serve/resident-runtime.ts +97 -61
- package/src/serve/resident-shutdown.ts +153 -0
- package/src/serve/routes/api.ts +3 -1
- package/src/serve/server.ts +47 -26
- package/src/store/migrations/028-vector-variants.ts +54 -0
- package/src/store/migrations/029-graph-reference-state.ts +77 -0
- package/src/store/migrations/index.ts +4 -0
- package/src/store/sqlite/adapter.ts +251 -183
- package/src/store/sqlite/eligibility.ts +174 -0
- package/src/store/sqlite/graph-edge-application.ts +66 -0
- package/src/store/sqlite/graph-reference-state.ts +194 -0
- package/src/store/sqlite/legacy-vector-ownership.ts +79 -0
- package/src/store/types.ts +80 -12
- package/src/store/vector/eligibility.ts +36 -0
- package/src/store/vector/freshness.ts +33 -6
- package/src/store/vector/lazy.ts +81 -0
- package/src/store/vector/sqlite-vec.ts +106 -54
- package/src/store/vector/stats.ts +14 -3
- package/src/store/vector/types.ts +35 -2
- package/src/store/vector/variant-search.ts +192 -0
- package/src/store/vector/variants.ts +451 -0
- package/vendor/converters/markitdown-ts/LICENSE +21 -0
- package/vendor/converters/markitdown-ts/dist/index.cjs +1180 -0
- package/vendor/converters/markitdown-ts/dist/index.d.cts +46 -0
- package/vendor/converters/markitdown-ts/dist/index.d.mts +46 -0
- package/vendor/converters/markitdown-ts/dist/index.d.ts +46 -0
- package/vendor/converters/markitdown-ts/dist/index.mjs +1152 -0
- package/vendor/converters/markitdown-ts/package.json +77 -0
- package/vendor/converters/officeparser/LICENSE +21 -0
- package/vendor/converters/officeparser/dist/OfficeConverter.d.ts +47 -0
- package/vendor/converters/officeparser/dist/OfficeConverter.js +76 -0
- package/vendor/converters/officeparser/dist/OfficeGenerator.d.ts +23 -0
- package/vendor/converters/officeparser/dist/OfficeGenerator.js +73 -0
- package/vendor/converters/officeparser/dist/OfficeParser.d.ts +106 -0
- package/vendor/converters/officeparser/dist/OfficeParser.js +332 -0
- package/vendor/converters/officeparser/dist/cli.d.ts +28 -0
- package/vendor/converters/officeparser/dist/cli.js +381 -0
- package/vendor/converters/officeparser/dist/defaults.d.ts +41 -0
- package/vendor/converters/officeparser/dist/defaults.js +218 -0
- package/vendor/converters/officeparser/dist/generators/BaseGenerator.d.ts +107 -0
- package/vendor/converters/officeparser/dist/generators/BaseGenerator.js +248 -0
- package/vendor/converters/officeparser/dist/generators/ChunkingGenerator.d.ts +82 -0
- package/vendor/converters/officeparser/dist/generators/ChunkingGenerator.js +797 -0
- package/vendor/converters/officeparser/dist/generators/CsvGenerator.d.ts +38 -0
- package/vendor/converters/officeparser/dist/generators/CsvGenerator.js +245 -0
- package/vendor/converters/officeparser/dist/generators/EpubGenerator.d.ts +43 -0
- package/vendor/converters/officeparser/dist/generators/EpubGenerator.js +315 -0
- package/vendor/converters/officeparser/dist/generators/HtmlGenerator.d.ts +59 -0
- package/vendor/converters/officeparser/dist/generators/HtmlGenerator.js +1942 -0
- package/vendor/converters/officeparser/dist/generators/MarkdownGenerator.d.ts +96 -0
- package/vendor/converters/officeparser/dist/generators/MarkdownGenerator.js +1175 -0
- package/vendor/converters/officeparser/dist/generators/PdfGenerator.d.ts +22 -0
- package/vendor/converters/officeparser/dist/generators/PdfGenerator.js +194 -0
- package/vendor/converters/officeparser/dist/generators/RtfGenerator.d.ts +29 -0
- package/vendor/converters/officeparser/dist/generators/RtfGenerator.js +316 -0
- package/vendor/converters/officeparser/dist/generators/TextGenerator.d.ts +13 -0
- package/vendor/converters/officeparser/dist/generators/TextGenerator.js +201 -0
- package/vendor/converters/officeparser/dist/index.d.ts +60 -0
- package/vendor/converters/officeparser/dist/index.js +72 -0
- package/vendor/converters/officeparser/dist/index.mjs +18 -0
- package/vendor/converters/officeparser/dist/officeparser.browser.d.ts +2621 -0
- package/vendor/converters/officeparser/dist/officeparser.browser.iife.js +1336 -0
- package/vendor/converters/officeparser/dist/officeparser.browser.mjs +1335 -0
- package/vendor/converters/officeparser/dist/officeparser.browser.slim.d.ts +2621 -0
- package/vendor/converters/officeparser/dist/officeparser.browser.slim.iife.js +1336 -0
- package/vendor/converters/officeparser/dist/officeparser.browser.slim.mjs +1335 -0
- package/vendor/converters/officeparser/dist/parsers/CsvParser.d.ts +9 -0
- package/vendor/converters/officeparser/dist/parsers/CsvParser.js +115 -0
- package/vendor/converters/officeparser/dist/parsers/EpubParser.d.ts +8 -0
- package/vendor/converters/officeparser/dist/parsers/EpubParser.js +217 -0
- package/vendor/converters/officeparser/dist/parsers/ExcelParser.d.ts +32 -0
- package/vendor/converters/officeparser/dist/parsers/ExcelParser.js +736 -0
- package/vendor/converters/officeparser/dist/parsers/HtmlParser.d.ts +2 -0
- package/vendor/converters/officeparser/dist/parsers/HtmlParser.js +1287 -0
- package/vendor/converters/officeparser/dist/parsers/MarkdownParser.d.ts +2 -0
- package/vendor/converters/officeparser/dist/parsers/MarkdownParser.js +1272 -0
- package/vendor/converters/officeparser/dist/parsers/OpenOfficeParser.d.ts +31 -0
- package/vendor/converters/officeparser/dist/parsers/OpenOfficeParser.js +1819 -0
- package/vendor/converters/officeparser/dist/parsers/PdfParser.d.ts +67 -0
- package/vendor/converters/officeparser/dist/parsers/PdfParser.js +848 -0
- package/vendor/converters/officeparser/dist/parsers/PowerPointParser.d.ts +32 -0
- package/vendor/converters/officeparser/dist/parsers/PowerPointParser.js +950 -0
- package/vendor/converters/officeparser/dist/parsers/RtfParser.d.ts +187 -0
- package/vendor/converters/officeparser/dist/parsers/RtfParser.js +1801 -0
- package/vendor/converters/officeparser/dist/parsers/WordParser.d.ts +79 -0
- package/vendor/converters/officeparser/dist/parsers/WordParser.js +1177 -0
- package/vendor/converters/officeparser/dist/sbom.cdx.json +1763 -0
- package/vendor/converters/officeparser/dist/types.d.ts +2507 -0
- package/vendor/converters/officeparser/dist/types.js +107 -0
- package/vendor/converters/officeparser/dist/utils/astUtils.d.ts +16 -0
- package/vendor/converters/officeparser/dist/utils/astUtils.js +33 -0
- package/vendor/converters/officeparser/dist/utils/chartUtils.d.ts +6 -0
- package/vendor/converters/officeparser/dist/utils/chartUtils.js +257 -0
- package/vendor/converters/officeparser/dist/utils/configUtils.d.ts +44 -0
- package/vendor/converters/officeparser/dist/utils/configUtils.js +315 -0
- package/vendor/converters/officeparser/dist/utils/dateUtils.d.ts +17 -0
- package/vendor/converters/officeparser/dist/utils/dateUtils.js +69 -0
- package/vendor/converters/officeparser/dist/utils/envUtils.d.ts +29 -0
- package/vendor/converters/officeparser/dist/utils/envUtils.js +152 -0
- package/vendor/converters/officeparser/dist/utils/errorUtils.d.ts +72 -0
- package/vendor/converters/officeparser/dist/utils/errorUtils.js +245 -0
- package/vendor/converters/officeparser/dist/utils/imageUtils.d.ts +66 -0
- package/vendor/converters/officeparser/dist/utils/imageUtils.js +133 -0
- package/vendor/converters/officeparser/dist/utils/mathUtils.d.ts +42 -0
- package/vendor/converters/officeparser/dist/utils/mathUtils.js +385 -0
- package/vendor/converters/officeparser/dist/utils/moduleLoader.d.ts +18 -0
- package/vendor/converters/officeparser/dist/utils/moduleLoader.js +106 -0
- package/vendor/converters/officeparser/dist/utils/ocrUtils.d.ts +42 -0
- package/vendor/converters/officeparser/dist/utils/ocrUtils.js +428 -0
- package/vendor/converters/officeparser/dist/utils/sanitize.d.ts +148 -0
- package/vendor/converters/officeparser/dist/utils/sanitize.js +344 -0
- package/vendor/converters/officeparser/dist/utils/sheetUtils.d.ts +7 -0
- package/vendor/converters/officeparser/dist/utils/sheetUtils.js +35 -0
- package/vendor/converters/officeparser/dist/utils/styleMapper.d.ts +36 -0
- package/vendor/converters/officeparser/dist/utils/styleMapper.js +224 -0
- package/vendor/converters/officeparser/dist/utils/xmlUtils.d.ts +163 -0
- package/vendor/converters/officeparser/dist/utils/xmlUtils.js +461 -0
- package/vendor/converters/officeparser/dist/utils/zipUtils.d.ts +134 -0
- package/vendor/converters/officeparser/dist/utils/zipUtils.js +337 -0
- package/vendor/converters/officeparser/package.json +147 -0
- package/vendor/converters/upstream-manifest.json +124 -0
- package/vendor/dependency-fixes/README.md +77 -0
- package/vendor/dependency-fixes/vendor-converters.py +83 -0
- package/browser-extension/artifacts/gno-browser-clipper-v1.46.0.zip +0 -0
- package/browser-extension/artifacts/gno-browser-clipper-v1.46.0.zip.sha256 +0 -1
- package/browser-extension/dist/chunk-627emwpj.js +0 -75
- /package/browser-extension/dist/{chunk-ydfx5d7p.css → chunk-z74y8n8c.css} +0 -0
package/src/embed/backlog.ts
CHANGED
|
@@ -1,25 +1,35 @@
|
|
|
1
|
+
import type { EmbeddingPort } from "../llm/types";
|
|
1
2
|
/**
|
|
2
3
|
* Shared embedding backlog processor.
|
|
3
4
|
* Used by CLI embed, Web scheduler, and MCP tools.
|
|
4
5
|
*
|
|
5
6
|
* @module src/embed/backlog
|
|
6
7
|
*/
|
|
7
|
-
|
|
8
|
-
import type { EmbeddingPort } from "../llm/types";
|
|
9
8
|
import type { StoreResult } from "../store/types";
|
|
10
9
|
import type {
|
|
11
10
|
BacklogItem,
|
|
12
11
|
VectorIndexPort,
|
|
13
12
|
VectorStatsPort,
|
|
14
13
|
} from "../store/vector";
|
|
14
|
+
import type { VectorVariantStore } from "../store/vector/variants";
|
|
15
15
|
|
|
16
|
+
import {
|
|
17
|
+
assertInferenceActive,
|
|
18
|
+
isBackgroundInference,
|
|
19
|
+
} from "../llm/inference-scope";
|
|
16
20
|
import { err, ok } from "../store/types";
|
|
17
|
-
import {
|
|
21
|
+
import { getVectorStatsDatabase } from "../store/vector/stats";
|
|
22
|
+
import { createVectorVariantStore } from "../store/vector/variants";
|
|
23
|
+
import {
|
|
24
|
+
getEmbeddingFingerprint,
|
|
25
|
+
getVariantModelFingerprint,
|
|
26
|
+
} from "./fingerprint";
|
|
18
27
|
import {
|
|
19
28
|
chunkRetryKey,
|
|
20
29
|
embedAndStoreBatch,
|
|
21
30
|
MAX_EMBED_CHUNK_ATTEMPTS,
|
|
22
31
|
} from "./retry";
|
|
32
|
+
import { embedVariantBacklog } from "./variant-backlog";
|
|
23
33
|
|
|
24
34
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
25
35
|
// Types
|
|
@@ -32,6 +42,11 @@ export interface EmbedBacklogDeps {
|
|
|
32
42
|
collection?: string;
|
|
33
43
|
modelUri: string;
|
|
34
44
|
batchSize?: number;
|
|
45
|
+
force?: boolean;
|
|
46
|
+
onProgress?: (embedded: number, errors: number) => void;
|
|
47
|
+
variantStore?: VectorVariantStore;
|
|
48
|
+
/** Recheck the effective runtime identity after asynchronous inference. */
|
|
49
|
+
identityStillCurrent?: () => boolean;
|
|
35
50
|
}
|
|
36
51
|
|
|
37
52
|
export interface EmbedBacklogResult {
|
|
@@ -62,8 +77,16 @@ interface Cursor {
|
|
|
62
77
|
export async function embedBacklog(
|
|
63
78
|
deps: EmbedBacklogDeps
|
|
64
79
|
): Promise<StoreResult<EmbedBacklogResult>> {
|
|
80
|
+
assertInferenceActive();
|
|
81
|
+
const prepared = await prepareEmbeddingBacklog(deps);
|
|
82
|
+
if (!prepared.ok) return prepared;
|
|
83
|
+
deps = prepared.value;
|
|
84
|
+
if (deps.variantStore) return embedVariantBacklog(deps, deps.variantStore);
|
|
65
85
|
const { statsPort, embedPort, vectorIndex, modelUri, collection } = deps;
|
|
66
|
-
const
|
|
86
|
+
const background = isBackgroundInference();
|
|
87
|
+
const batchSize = background
|
|
88
|
+
? Math.min(deps.batchSize ?? 32, 32)
|
|
89
|
+
: (deps.batchSize ?? 32);
|
|
67
90
|
const embedFingerprint = getEmbeddingFingerprint({
|
|
68
91
|
modelUri,
|
|
69
92
|
dimensions: vectorIndex.dimensions,
|
|
@@ -77,6 +100,7 @@ export async function embedBacklog(
|
|
|
77
100
|
|
|
78
101
|
const enqueueRetryItems = (items: BacklogItem[], attempts: number): void => {
|
|
79
102
|
for (const item of items) {
|
|
103
|
+
assertInferenceActive();
|
|
80
104
|
const key = chunkRetryKey(item);
|
|
81
105
|
const existing = retryQueue.get(key);
|
|
82
106
|
retryQueue.set(key, {
|
|
@@ -97,8 +121,10 @@ export async function embedBacklog(
|
|
|
97
121
|
);
|
|
98
122
|
|
|
99
123
|
for (let idx = 0; idx < entries.length; idx += batchSize) {
|
|
124
|
+
assertInferenceActive();
|
|
100
125
|
const slice = entries.slice(idx, idx + batchSize);
|
|
101
126
|
for (const entry of slice) {
|
|
127
|
+
assertInferenceActive();
|
|
102
128
|
retryQueue.delete(chunkRetryKey(entry.item));
|
|
103
129
|
entry.attempts += 1;
|
|
104
130
|
}
|
|
@@ -109,6 +135,8 @@ export async function embedBacklog(
|
|
|
109
135
|
items: slice.map((entry) => entry.item),
|
|
110
136
|
modelUri,
|
|
111
137
|
embedFingerprint,
|
|
138
|
+
identityStillCurrent: deps.identityStillCurrent,
|
|
139
|
+
statsPort,
|
|
112
140
|
});
|
|
113
141
|
|
|
114
142
|
embedded += retryResult.embedded;
|
|
@@ -120,6 +148,7 @@ export async function embedBacklog(
|
|
|
120
148
|
retryResult.retryItems.map((item) => chunkRetryKey(item))
|
|
121
149
|
);
|
|
122
150
|
for (const entry of slice) {
|
|
151
|
+
assertInferenceActive();
|
|
123
152
|
if (!retryByKey.has(chunkRetryKey(entry.item))) {
|
|
124
153
|
continue;
|
|
125
154
|
}
|
|
@@ -136,6 +165,7 @@ export async function embedBacklog(
|
|
|
136
165
|
|
|
137
166
|
try {
|
|
138
167
|
while (true) {
|
|
168
|
+
assertInferenceActive();
|
|
139
169
|
// Get next batch using seek pagination
|
|
140
170
|
const batchResult = await statsPort.getBacklog(
|
|
141
171
|
modelUri,
|
|
@@ -169,10 +199,20 @@ export async function embedBacklog(
|
|
|
169
199
|
items: batch,
|
|
170
200
|
modelUri,
|
|
171
201
|
embedFingerprint,
|
|
202
|
+
identityStillCurrent: deps.identityStillCurrent,
|
|
203
|
+
statsPort,
|
|
172
204
|
});
|
|
173
205
|
embedded += batchStoreResult.embedded;
|
|
174
206
|
errors += batchStoreResult.errors;
|
|
175
207
|
contentionErrors += batchStoreResult.contentionErrors;
|
|
208
|
+
if (background) {
|
|
209
|
+
errors += batchStoreResult.retryItems.length;
|
|
210
|
+
deps.onProgress?.(embedded, errors);
|
|
211
|
+
// Each cursor page is a turn. Failed early pages cannot starve later work.
|
|
212
|
+
await Bun.sleep(0);
|
|
213
|
+
if (deps.identityStillCurrent && !deps.identityStillCurrent()) break;
|
|
214
|
+
continue;
|
|
215
|
+
}
|
|
176
216
|
enqueueRetryItems(batchStoreResult.retryItems, 1);
|
|
177
217
|
|
|
178
218
|
if (embedded > beforeEmbedded) {
|
|
@@ -198,6 +238,7 @@ export async function embedBacklog(
|
|
|
198
238
|
}
|
|
199
239
|
}
|
|
200
240
|
|
|
241
|
+
assertInferenceActive();
|
|
201
242
|
return ok({ embedded, errors, contentionErrors, syncError });
|
|
202
243
|
} catch (e) {
|
|
203
244
|
return err(
|
|
@@ -206,3 +247,65 @@ export async function embedBacklog(
|
|
|
206
247
|
);
|
|
207
248
|
}
|
|
208
249
|
}
|
|
250
|
+
|
|
251
|
+
/** Resolve authority before counts, dry runs, forced work, or early returns. */
|
|
252
|
+
export async function prepareEmbeddingBacklog(
|
|
253
|
+
deps: EmbedBacklogDeps
|
|
254
|
+
): Promise<StoreResult<EmbedBacklogDeps>> {
|
|
255
|
+
if (deps.variantStore) return ok(deps);
|
|
256
|
+
const db = getVectorStatsDatabase(deps.statsPort);
|
|
257
|
+
if (db) {
|
|
258
|
+
try {
|
|
259
|
+
const initialized = await deps.embedPort.init();
|
|
260
|
+
if (!initialized.ok) return err("INTERNAL", initialized.error.message);
|
|
261
|
+
const identity = deps.embedPort.getIdentity?.();
|
|
262
|
+
if (identity) {
|
|
263
|
+
const identitySnapshot = JSON.stringify(identity);
|
|
264
|
+
const dimensions = deps.embedPort.dimensions();
|
|
265
|
+
const variantStore = await createVectorVariantStore(db, {
|
|
266
|
+
model: deps.modelUri,
|
|
267
|
+
modelFingerprint: getVariantModelFingerprint(
|
|
268
|
+
{ modelUri: deps.modelUri, dimensions },
|
|
269
|
+
identity
|
|
270
|
+
),
|
|
271
|
+
contextSize: identity.contextSize,
|
|
272
|
+
truncationPolicy: identity.truncationPolicy,
|
|
273
|
+
dimensions,
|
|
274
|
+
});
|
|
275
|
+
return ok({
|
|
276
|
+
...deps,
|
|
277
|
+
variantStore,
|
|
278
|
+
identityStillCurrent: () =>
|
|
279
|
+
(deps.identityStillCurrent?.() ?? true) &&
|
|
280
|
+
deps.embedPort.modelUri === deps.modelUri &&
|
|
281
|
+
deps.embedPort.dimensions() === dimensions &&
|
|
282
|
+
JSON.stringify(deps.embedPort.getIdentity?.()) === identitySnapshot,
|
|
283
|
+
});
|
|
284
|
+
}
|
|
285
|
+
// Unverified/HTTP ports retain legacy behavior until variant authority exists.
|
|
286
|
+
if (
|
|
287
|
+
db
|
|
288
|
+
.query(
|
|
289
|
+
"SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'vector_partitions'"
|
|
290
|
+
)
|
|
291
|
+
.get() &&
|
|
292
|
+
db
|
|
293
|
+
.query(
|
|
294
|
+
"SELECT 1 FROM vector_partitions WHERE model = ? AND state = ? AND activated_epoch IS NOT NULL LIMIT 1"
|
|
295
|
+
)
|
|
296
|
+
.get(deps.modelUri, "active")
|
|
297
|
+
) {
|
|
298
|
+
return err(
|
|
299
|
+
"INVALID_INPUT",
|
|
300
|
+
"Effective embedding identity unavailable after variant activation"
|
|
301
|
+
);
|
|
302
|
+
}
|
|
303
|
+
} catch (cause) {
|
|
304
|
+
return err(
|
|
305
|
+
"QUERY_FAILED",
|
|
306
|
+
cause instanceof Error ? cause.message : String(cause)
|
|
307
|
+
);
|
|
308
|
+
}
|
|
309
|
+
}
|
|
310
|
+
return ok(deps);
|
|
311
|
+
}
|
package/src/embed/batch.ts
CHANGED
|
@@ -1,13 +1,17 @@
|
|
|
1
|
+
import type { EmbeddingPort, LlmResult } from "../llm/types";
|
|
1
2
|
/**
|
|
2
3
|
* Shared embedding batch helpers.
|
|
3
4
|
*
|
|
4
5
|
* @module src/embed/batch
|
|
5
6
|
*/
|
|
6
7
|
|
|
7
|
-
import type { EmbeddingPort, LlmResult } from "../llm/types";
|
|
8
|
-
|
|
9
8
|
import { getEmbeddingCompatibilityProfile } from "../llm/embedding-compatibility";
|
|
10
9
|
import { inferenceFailedError } from "../llm/errors";
|
|
10
|
+
import {
|
|
11
|
+
assertInferenceActive,
|
|
12
|
+
assertInferenceResult,
|
|
13
|
+
isBackgroundInference,
|
|
14
|
+
} from "../llm/inference-scope";
|
|
11
15
|
|
|
12
16
|
export interface EmbedBatchRecoveryResult {
|
|
13
17
|
vectors: Array<number[] | null>;
|
|
@@ -70,8 +74,26 @@ export async function embedTextsWithRecovery(
|
|
|
70
74
|
}
|
|
71
75
|
|
|
72
76
|
const profile = getEmbeddingCompatibilityProfile(embedPort.modelUri);
|
|
77
|
+
// A background page gets one provider attempt. Recovery remains durable for
|
|
78
|
+
// the next pass instead of expanding the current native scheduling turn.
|
|
79
|
+
if (isBackgroundInference() && profile.batchEmbeddingTrusted) {
|
|
80
|
+
const result = await embedPort.embedBatch(texts);
|
|
81
|
+
assertInferenceResult(result);
|
|
82
|
+
if (!result.ok) return result;
|
|
83
|
+
const complete = result.value.length === texts.length;
|
|
84
|
+
return {
|
|
85
|
+
ok: true,
|
|
86
|
+
value: {
|
|
87
|
+
vectors: complete ? result.value : texts.map(() => null),
|
|
88
|
+
batchFailed: !complete,
|
|
89
|
+
fallbackErrors: complete ? 0 : texts.length,
|
|
90
|
+
failureSamples: complete ? [] : ["Embedding count mismatch"],
|
|
91
|
+
},
|
|
92
|
+
};
|
|
93
|
+
}
|
|
73
94
|
if (profile.batchEmbeddingTrusted) {
|
|
74
95
|
let batchResult = await embedPort.embedBatch(texts);
|
|
96
|
+
assertInferenceResult(batchResult);
|
|
75
97
|
if (!batchResult.ok) {
|
|
76
98
|
const formattedBatchError = formatFailureMessage(batchResult.error);
|
|
77
99
|
if (isDisposedFailure(formattedBatchError)) {
|
|
@@ -80,6 +102,7 @@ export async function embedTextsWithRecovery(
|
|
|
80
102
|
return reset;
|
|
81
103
|
}
|
|
82
104
|
batchResult = await embedPort.embedBatch(texts);
|
|
105
|
+
assertInferenceResult(batchResult);
|
|
83
106
|
}
|
|
84
107
|
}
|
|
85
108
|
if (batchResult.ok && batchResult.value.length === texts.length) {
|
|
@@ -166,6 +189,7 @@ async function recoverWithAdaptiveBatches(
|
|
|
166
189
|
|
|
167
190
|
if (rangeTexts.length === 1) {
|
|
168
191
|
const result = await embedPort.embed(rangeTexts[0] ?? "");
|
|
192
|
+
assertInferenceResult(result);
|
|
169
193
|
if (result.ok) {
|
|
170
194
|
vectors[offset] = result.value;
|
|
171
195
|
return;
|
|
@@ -179,6 +203,7 @@ async function recoverWithAdaptiveBatches(
|
|
|
179
203
|
null;
|
|
180
204
|
if (!batchAlreadyFailed) {
|
|
181
205
|
batchResult = await embedPort.embedBatch(rangeTexts);
|
|
206
|
+
assertInferenceResult(batchResult);
|
|
182
207
|
}
|
|
183
208
|
if (
|
|
184
209
|
batchResult &&
|
|
@@ -186,6 +211,7 @@ async function recoverWithAdaptiveBatches(
|
|
|
186
211
|
batchResult.value.length === rangeTexts.length
|
|
187
212
|
) {
|
|
188
213
|
for (const [index, vector] of batchResult.value.entries()) {
|
|
214
|
+
assertInferenceActive();
|
|
189
215
|
vectors[offset + index] = vector;
|
|
190
216
|
}
|
|
191
217
|
return;
|
|
@@ -223,6 +249,12 @@ async function recoverWithAdaptiveBatches(
|
|
|
223
249
|
},
|
|
224
250
|
};
|
|
225
251
|
} catch (error) {
|
|
252
|
+
assertInferenceActive();
|
|
253
|
+
if (
|
|
254
|
+
error instanceof Error &&
|
|
255
|
+
["AbortError", "TimeoutError"].includes(error.name)
|
|
256
|
+
)
|
|
257
|
+
throw error;
|
|
226
258
|
return {
|
|
227
259
|
ok: false,
|
|
228
260
|
error: inferenceFailedError(
|
|
@@ -245,7 +277,9 @@ async function recoverIndividually(
|
|
|
245
277
|
let fallbackErrors = 0;
|
|
246
278
|
|
|
247
279
|
for (const text of texts) {
|
|
280
|
+
assertInferenceActive();
|
|
248
281
|
const result = await embedPort.embed(text);
|
|
282
|
+
assertInferenceResult(result);
|
|
249
283
|
if (result.ok) {
|
|
250
284
|
vectors.push(result.value);
|
|
251
285
|
} else {
|
|
@@ -266,6 +300,12 @@ async function recoverIndividually(
|
|
|
266
300
|
},
|
|
267
301
|
};
|
|
268
302
|
} catch (error) {
|
|
303
|
+
assertInferenceActive();
|
|
304
|
+
if (
|
|
305
|
+
error instanceof Error &&
|
|
306
|
+
["AbortError", "TimeoutError"].includes(error.name)
|
|
307
|
+
)
|
|
308
|
+
throw error;
|
|
269
309
|
return {
|
|
270
310
|
ok: false,
|
|
271
311
|
error: inferenceFailedError(
|
package/src/embed/fingerprint.ts
CHANGED
|
@@ -35,3 +35,19 @@ export function getEmbeddingFingerprint(
|
|
|
35
35
|
.update(JSON.stringify(payload))
|
|
36
36
|
.digest("hex");
|
|
37
37
|
}
|
|
38
|
+
|
|
39
|
+
/** Partition provenance combines actual weights/runtime with the unchanged formatter policy. */
|
|
40
|
+
export function getVariantModelFingerprint(
|
|
41
|
+
input: EmbeddingFingerprintInput,
|
|
42
|
+
identity: { modelFingerprint: string; runtimeFingerprint: string }
|
|
43
|
+
): string {
|
|
44
|
+
return new Bun.CryptoHasher("sha256")
|
|
45
|
+
.update(
|
|
46
|
+
JSON.stringify([
|
|
47
|
+
identity.modelFingerprint,
|
|
48
|
+
identity.runtimeFingerprint,
|
|
49
|
+
getEmbeddingFingerprint(input),
|
|
50
|
+
])
|
|
51
|
+
)
|
|
52
|
+
.digest("hex");
|
|
53
|
+
}
|
package/src/embed/retry.ts
CHANGED
|
@@ -1,9 +1,17 @@
|
|
|
1
1
|
import type { EmbeddingPort } from "../llm/types";
|
|
2
2
|
import type { StoreResult } from "../store/types";
|
|
3
|
-
import type {
|
|
3
|
+
import type {
|
|
4
|
+
BacklogItem,
|
|
5
|
+
VectorIndexPort,
|
|
6
|
+
VectorRow,
|
|
7
|
+
VectorStatsPort,
|
|
8
|
+
} from "../store/vector";
|
|
4
9
|
|
|
5
10
|
import { isSqliteLockContention } from "../core/file-lock";
|
|
11
|
+
import { assertInferenceActive } from "../llm/inference-scope";
|
|
6
12
|
import { formatDocForEmbedding } from "../pipeline/contextual";
|
|
13
|
+
import { err, ok } from "../store/types";
|
|
14
|
+
import { getVectorStatsDatabase } from "../store/vector/stats";
|
|
7
15
|
import { embedTextsWithRecovery } from "./batch";
|
|
8
16
|
|
|
9
17
|
export const MAX_EMBED_CHUNK_ATTEMPTS = 2;
|
|
@@ -42,12 +50,21 @@ export interface EmbedStoreBatchResult {
|
|
|
42
50
|
// src/cli/commands/index-cmd.ts) must surface contentionErrors in their
|
|
43
51
|
// summary and exit non-zero — the integrator wires that.
|
|
44
52
|
|
|
45
|
-
export function chunkRetryKey(
|
|
53
|
+
export function chunkRetryKey(
|
|
54
|
+
item: Pick<BacklogItem, "mirrorHash" | "seq"> & {
|
|
55
|
+
documentId?: number;
|
|
56
|
+
inputHash?: string;
|
|
57
|
+
}
|
|
58
|
+
) {
|
|
59
|
+
if (item.documentId !== undefined && item.inputHash !== undefined) {
|
|
60
|
+
return `${item.documentId}\0${item.mirrorHash}\0${item.seq}\0${item.inputHash}`;
|
|
61
|
+
}
|
|
46
62
|
return `${item.mirrorHash}\0${item.seq}`;
|
|
47
63
|
}
|
|
48
64
|
|
|
49
65
|
export function addUniqueSamples(target: string[], samples: string[]): void {
|
|
50
66
|
for (const sample of samples) {
|
|
67
|
+
assertInferenceActive();
|
|
51
68
|
if (target.length >= MAX_EMBED_FAILURE_SAMPLES) {
|
|
52
69
|
break;
|
|
53
70
|
}
|
|
@@ -127,6 +144,7 @@ export async function upsertVectorsWithContentionRetry(
|
|
|
127
144
|
vectors: VectorRow[],
|
|
128
145
|
delays?: number[]
|
|
129
146
|
): Promise<StoreResult<void>> {
|
|
147
|
+
assertInferenceActive();
|
|
130
148
|
let storeResult = await vectorIndex.upsertVectors(vectors);
|
|
131
149
|
let attempts = 1;
|
|
132
150
|
while (
|
|
@@ -138,6 +156,7 @@ export async function upsertVectorsWithContentionRetry(
|
|
|
138
156
|
if (delayMs > 0) {
|
|
139
157
|
await Bun.sleep(delayMs);
|
|
140
158
|
}
|
|
159
|
+
assertInferenceActive();
|
|
141
160
|
storeResult = await vectorIndex.upsertVectors(vectors);
|
|
142
161
|
attempts += 1;
|
|
143
162
|
}
|
|
@@ -150,10 +169,22 @@ export async function embedAndStoreBatch(params: {
|
|
|
150
169
|
items: BacklogItem[];
|
|
151
170
|
modelUri: string;
|
|
152
171
|
embedFingerprint: string;
|
|
172
|
+
statsPort?: VectorStatsPort;
|
|
173
|
+
identityStillCurrent?: () => boolean;
|
|
153
174
|
/** Test seam: override contention-retry delays in milliseconds. */
|
|
154
175
|
delays?: number[];
|
|
155
176
|
}): Promise<EmbedStoreBatchResult> {
|
|
156
177
|
const { embedPort, vectorIndex, items, modelUri, embedFingerprint } = params;
|
|
178
|
+
const db = params.statsPort && getVectorStatsDatabase(params.statsPort);
|
|
179
|
+
if (db && !vectorIndex.upsertVectorsChecked)
|
|
180
|
+
return {
|
|
181
|
+
embedded: 0,
|
|
182
|
+
errors: items.length,
|
|
183
|
+
contentionErrors: 0,
|
|
184
|
+
retryItems: [],
|
|
185
|
+
errorSamples: ["Atomic vector checkpoint unavailable"],
|
|
186
|
+
batchFailed: false,
|
|
187
|
+
};
|
|
157
188
|
const embedResult = await embedTextsWithRecovery(
|
|
158
189
|
embedPort,
|
|
159
190
|
items.map((item) =>
|
|
@@ -180,6 +211,7 @@ export async function embedAndStoreBatch(params: {
|
|
|
180
211
|
const vectors: VectorRow[] = [];
|
|
181
212
|
const retryItems: BacklogItem[] = [];
|
|
182
213
|
for (const [idx, item] of items.entries()) {
|
|
214
|
+
assertInferenceActive();
|
|
183
215
|
const embedding = embedResult.value.vectors[idx];
|
|
184
216
|
if (!embedding) {
|
|
185
217
|
retryItems.push(item);
|
|
@@ -207,8 +239,75 @@ export async function embedAndStoreBatch(params: {
|
|
|
207
239
|
};
|
|
208
240
|
}
|
|
209
241
|
|
|
242
|
+
let written = 0;
|
|
243
|
+
let committedRows: VectorRow[] = [];
|
|
244
|
+
const itemsByKey = new Map(items.map((item) => [chunkRetryKey(item), item]));
|
|
245
|
+
const checkpoint = (rows: VectorRow[]): VectorRow[] => {
|
|
246
|
+
assertInferenceActive();
|
|
247
|
+
if (params.identityStillCurrent && !params.identityStillCurrent())
|
|
248
|
+
return [];
|
|
249
|
+
if (!db) return rows;
|
|
250
|
+
// A concurrently activated verified partition forbids a late legacy write.
|
|
251
|
+
if (
|
|
252
|
+
db
|
|
253
|
+
.query(
|
|
254
|
+
"SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'vector_partitions'"
|
|
255
|
+
)
|
|
256
|
+
.get() &&
|
|
257
|
+
db
|
|
258
|
+
.query(
|
|
259
|
+
"SELECT 1 FROM vector_partitions WHERE model = ? AND state = 'active' AND activated_epoch IS NOT NULL LIMIT 1"
|
|
260
|
+
)
|
|
261
|
+
.get(modelUri)
|
|
262
|
+
)
|
|
263
|
+
return [];
|
|
264
|
+
return rows.filter((row) => {
|
|
265
|
+
const item = itemsByKey.get(chunkRetryKey(row));
|
|
266
|
+
const current = db
|
|
267
|
+
.query<{ text: string; title: string | null }, [string, number]>(`
|
|
268
|
+
SELECT c.text, d.title FROM content_chunks c
|
|
269
|
+
JOIN documents d ON d.mirror_hash = c.mirror_hash AND d.active = 1
|
|
270
|
+
WHERE c.mirror_hash = ? AND c.seq = ? ORDER BY d.id LIMIT 1
|
|
271
|
+
`)
|
|
272
|
+
.get(row.mirrorHash, row.seq);
|
|
273
|
+
return (
|
|
274
|
+
!!item &&
|
|
275
|
+
!!current &&
|
|
276
|
+
formatDocForEmbedding(
|
|
277
|
+
current.text,
|
|
278
|
+
current.title ?? undefined,
|
|
279
|
+
modelUri
|
|
280
|
+
) ===
|
|
281
|
+
formatDocForEmbedding(item.text, item.title ?? undefined, modelUri)
|
|
282
|
+
);
|
|
283
|
+
});
|
|
284
|
+
};
|
|
210
285
|
const storeResult = await upsertVectorsWithContentionRetry(
|
|
211
|
-
|
|
286
|
+
{
|
|
287
|
+
upsertVectors: async (rows) => {
|
|
288
|
+
if (vectorIndex.upsertVectorsChecked) {
|
|
289
|
+
const result = await vectorIndex.upsertVectorsChecked(
|
|
290
|
+
rows,
|
|
291
|
+
(candidates) => {
|
|
292
|
+
committedRows = checkpoint(candidates);
|
|
293
|
+
return committedRows;
|
|
294
|
+
}
|
|
295
|
+
);
|
|
296
|
+
if (!result.ok) return result;
|
|
297
|
+
written = result.value;
|
|
298
|
+
return ok(undefined);
|
|
299
|
+
}
|
|
300
|
+
if (db)
|
|
301
|
+
return err("INVALID_INPUT", "Atomic vector checkpoint unavailable");
|
|
302
|
+
const valid = checkpoint(rows);
|
|
303
|
+
const result = await vectorIndex.upsertVectors(valid);
|
|
304
|
+
if (result.ok) {
|
|
305
|
+
written = valid.length;
|
|
306
|
+
committedRows = valid;
|
|
307
|
+
}
|
|
308
|
+
return result;
|
|
309
|
+
},
|
|
310
|
+
},
|
|
212
311
|
vectors,
|
|
213
312
|
params.delays
|
|
214
313
|
);
|
|
@@ -237,11 +336,20 @@ export async function embedAndStoreBatch(params: {
|
|
|
237
336
|
};
|
|
238
337
|
}
|
|
239
338
|
|
|
339
|
+
const vectorKeys = new Set(vectors.map((row) => chunkRetryKey(row)));
|
|
340
|
+
const committedKeys = new Set(committedRows.map((row) => chunkRetryKey(row)));
|
|
240
341
|
return {
|
|
241
|
-
embedded:
|
|
342
|
+
embedded: written,
|
|
242
343
|
errors: 0,
|
|
243
344
|
contentionErrors: 0,
|
|
244
|
-
retryItems
|
|
345
|
+
retryItems: [
|
|
346
|
+
...retryItems,
|
|
347
|
+
...items.filter(
|
|
348
|
+
(item) =>
|
|
349
|
+
vectorKeys.has(chunkRetryKey(item)) &&
|
|
350
|
+
!committedKeys.has(chunkRetryKey(item))
|
|
351
|
+
),
|
|
352
|
+
],
|
|
245
353
|
errorSamples: embedResult.value.failureSamples,
|
|
246
354
|
suggestion: embedResult.value.retrySuggestion,
|
|
247
355
|
batchFailed: embedResult.value.batchFailed,
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
import type { StoreResult } from "../store/types";
|
|
2
|
+
import type { VectorVariantStore } from "../store/vector/variants";
|
|
3
|
+
import type { EmbedBacklogDeps, EmbedBacklogResult } from "./backlog";
|
|
4
|
+
|
|
5
|
+
import {
|
|
6
|
+
assertInferenceActive,
|
|
7
|
+
isBackgroundInference,
|
|
8
|
+
} from "../llm/inference-scope";
|
|
9
|
+
import { err, ok } from "../store/types";
|
|
10
|
+
import { getVectorStatsDatabase } from "../store/vector/stats";
|
|
11
|
+
import { variantBacklogPage } from "./variant-plan";
|
|
12
|
+
import { embedVariantBatch } from "./variant-retry";
|
|
13
|
+
|
|
14
|
+
/** One bounded owner pass plus one retry; mutations stay durably pending for the next pass. */
|
|
15
|
+
export async function embedVariantBacklog(
|
|
16
|
+
deps: EmbedBacklogDeps,
|
|
17
|
+
store: VectorVariantStore
|
|
18
|
+
): Promise<StoreResult<EmbedBacklogResult>> {
|
|
19
|
+
const identityStillCurrent = deps.identityStillCurrent;
|
|
20
|
+
if (!identityStillCurrent)
|
|
21
|
+
return err(
|
|
22
|
+
"INVALID_INPUT",
|
|
23
|
+
"Variant embedding requires an effective runtime identity check"
|
|
24
|
+
);
|
|
25
|
+
const db = getVectorStatsDatabase(deps.statsPort);
|
|
26
|
+
if (deps.collection && !db)
|
|
27
|
+
return err(
|
|
28
|
+
"INVALID_INPUT",
|
|
29
|
+
"Collection-scoped variant backlog requires document ownership storage"
|
|
30
|
+
);
|
|
31
|
+
const background = isBackgroundInference();
|
|
32
|
+
const batchSize = background
|
|
33
|
+
? Math.min(deps.batchSize ?? 32, 32)
|
|
34
|
+
: (deps.batchSize ?? 32);
|
|
35
|
+
if (!Number.isSafeInteger(batchSize) || batchSize < 1)
|
|
36
|
+
return err("INVALID_INPUT", "Invalid embedding batch size");
|
|
37
|
+
const total = { embedded: 0, errors: 0, contentionErrors: 0 };
|
|
38
|
+
let after: { documentId: number; seq: number } | undefined;
|
|
39
|
+
try {
|
|
40
|
+
while (identityStillCurrent()) {
|
|
41
|
+
assertInferenceActive();
|
|
42
|
+
const pending = variantBacklogPage(deps, store, batchSize, after);
|
|
43
|
+
if (!pending.length) break;
|
|
44
|
+
const last = pending.at(-1)!;
|
|
45
|
+
after = { documentId: last.documentId, seq: last.seq };
|
|
46
|
+
const owners = deps.collection
|
|
47
|
+
? pending.filter(
|
|
48
|
+
(owner) =>
|
|
49
|
+
!!db!
|
|
50
|
+
.query(
|
|
51
|
+
"SELECT 1 FROM documents WHERE id = ? AND collection = ? AND active = 1"
|
|
52
|
+
)
|
|
53
|
+
.get(owner.documentId, deps.collection!)
|
|
54
|
+
)
|
|
55
|
+
: pending;
|
|
56
|
+
let result = await embedVariantBatch({
|
|
57
|
+
store,
|
|
58
|
+
embedPort: deps.embedPort,
|
|
59
|
+
owners,
|
|
60
|
+
identityStillCurrent,
|
|
61
|
+
force: deps.force,
|
|
62
|
+
});
|
|
63
|
+
total.embedded += result.embedded;
|
|
64
|
+
total.errors += result.errors;
|
|
65
|
+
total.contentionErrors += result.contentionErrors;
|
|
66
|
+
if (background) total.errors += result.retryOwners.length;
|
|
67
|
+
if (!background && result.retryOwners.length) {
|
|
68
|
+
result = await embedVariantBatch({
|
|
69
|
+
store,
|
|
70
|
+
embedPort: deps.embedPort,
|
|
71
|
+
owners: result.retryOwners,
|
|
72
|
+
identityStillCurrent,
|
|
73
|
+
force: deps.force,
|
|
74
|
+
});
|
|
75
|
+
total.embedded += result.embedded;
|
|
76
|
+
total.errors += result.errors + result.retryOwners.length;
|
|
77
|
+
total.contentionErrors += result.contentionErrors;
|
|
78
|
+
}
|
|
79
|
+
deps.onProgress?.(total.embedded, total.errors);
|
|
80
|
+
if (background) await Bun.sleep(0);
|
|
81
|
+
}
|
|
82
|
+
assertInferenceActive();
|
|
83
|
+
// Capture the epoch before checking completeness; activate rechecks under write lock.
|
|
84
|
+
const epoch = store.epoch();
|
|
85
|
+
if (identityStillCurrent() && !store.pending({ limit: 1 }).length) {
|
|
86
|
+
try {
|
|
87
|
+
if (!store.isActive()) store.syncIndex();
|
|
88
|
+
if (!identityStillCurrent()) return ok(total);
|
|
89
|
+
store.activate(epoch);
|
|
90
|
+
} catch (cause) {
|
|
91
|
+
return ok({
|
|
92
|
+
...total,
|
|
93
|
+
syncError: cause instanceof Error ? cause.message : String(cause),
|
|
94
|
+
});
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
assertInferenceActive();
|
|
98
|
+
return ok(total);
|
|
99
|
+
} catch (cause) {
|
|
100
|
+
return err(
|
|
101
|
+
"QUERY_FAILED",
|
|
102
|
+
cause instanceof Error ? cause.message : String(cause)
|
|
103
|
+
);
|
|
104
|
+
}
|
|
105
|
+
}
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
import type { VectorVariantStore } from "../store/vector/variants";
|
|
2
|
+
import type { EmbedBacklogDeps } from "./backlog";
|
|
3
|
+
|
|
4
|
+
import { getVectorStatsDatabase } from "../store/vector/stats";
|
|
5
|
+
|
|
6
|
+
/** Counts and execution enumerate the same exact current owner partition. */
|
|
7
|
+
export function variantBacklogPage(
|
|
8
|
+
deps: EmbedBacklogDeps,
|
|
9
|
+
store: VectorVariantStore,
|
|
10
|
+
limit: number,
|
|
11
|
+
after?: { documentId: number; seq: number }
|
|
12
|
+
) {
|
|
13
|
+
if (!deps.force) return store.pending({ limit, after });
|
|
14
|
+
const db = getVectorStatsDatabase(deps.statsPort);
|
|
15
|
+
if (!db)
|
|
16
|
+
throw new Error("Forced variants require document ownership storage");
|
|
17
|
+
const rows = db
|
|
18
|
+
.query<
|
|
19
|
+
{ documentId: number; seq: number },
|
|
20
|
+
[number, number, number, number]
|
|
21
|
+
>(`
|
|
22
|
+
SELECT d.id AS documentId, c.seq FROM documents d
|
|
23
|
+
JOIN content_chunks c ON c.mirror_hash = d.mirror_hash
|
|
24
|
+
WHERE d.active = 1 AND (d.id > ? OR (d.id = ? AND c.seq > ?))
|
|
25
|
+
ORDER BY d.id, c.seq LIMIT ?
|
|
26
|
+
`)
|
|
27
|
+
.all(
|
|
28
|
+
after?.documentId ?? -1,
|
|
29
|
+
after?.documentId ?? -1,
|
|
30
|
+
after?.seq ?? -1,
|
|
31
|
+
limit
|
|
32
|
+
);
|
|
33
|
+
return rows.flatMap((row) => {
|
|
34
|
+
const owner = store.current(row.documentId, row.seq);
|
|
35
|
+
return owner ? [owner] : [];
|
|
36
|
+
});
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
export function countVariantBacklog(deps: EmbedBacklogDeps): number {
|
|
40
|
+
const store = deps.variantStore;
|
|
41
|
+
if (!store) throw new Error("Verified variant identity required");
|
|
42
|
+
const db = getVectorStatsDatabase(deps.statsPort);
|
|
43
|
+
let after: { documentId: number; seq: number } | undefined;
|
|
44
|
+
let count = 0;
|
|
45
|
+
while (true) {
|
|
46
|
+
const owners = variantBacklogPage(deps, store, 256, after);
|
|
47
|
+
if (!owners.length) return count;
|
|
48
|
+
for (const owner of owners) {
|
|
49
|
+
if (
|
|
50
|
+
!deps.collection ||
|
|
51
|
+
db
|
|
52
|
+
?.query(
|
|
53
|
+
"SELECT 1 FROM documents WHERE id = ? AND collection = ? AND active = 1"
|
|
54
|
+
)
|
|
55
|
+
.get(owner.documentId, deps.collection)
|
|
56
|
+
)
|
|
57
|
+
count += 1;
|
|
58
|
+
}
|
|
59
|
+
const last = owners.at(-1)!;
|
|
60
|
+
after = { documentId: last.documentId, seq: last.seq };
|
|
61
|
+
}
|
|
62
|
+
}
|