@gmickel/gno 2.5.1 → 2.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +60 -5
- package/assets/skill/README.md +5 -1
- package/assets/skill/SKILL.md +80 -4
- package/assets/skill/cli-reference.md +132 -2
- package/assets/skill/examples.md +30 -0
- package/assets/skill/mcp-reference.md +54 -1
- package/assets/skill/recipes/capture-and-file.md +6 -0
- package/assets/skill/recipes/memory-file-decision.md +6 -0
- package/assets/skill/recipes/memory-supersede-fact.md +5 -0
- package/assets/skill/recipes/session-evidence-lookup.md +98 -0
- package/assets/spa-production.json.gz +0 -0
- package/browser-extension/artifacts/{gno-browser-clipper-v2.5.1.zip → gno-browser-clipper-v2.7.0.zip} +0 -0
- package/browser-extension/artifacts/gno-browser-clipper-v2.7.0.zip.sha256 +1 -0
- package/browser-extension/dist/manifest.json +1 -1
- package/package.json +2 -1
- package/spec/cli.md +449 -35
- package/spec/mcp.md +234 -4
- package/spec/output-schemas/ask.schema.json +1 -1
- package/spec/output-schemas/capture-receipt.schema.json +4 -1
- package/spec/output-schemas/doctor.schema.json +88 -0
- package/spec/output-schemas/error.schema.json +11 -2
- package/spec/output-schemas/get.schema.json +1 -1
- package/spec/output-schemas/mcp-capture-result.schema.json +4 -2
- package/spec/output-schemas/memory-remember.schema.json +10 -4
- package/spec/output-schemas/multi-get.schema.json +4 -1
- package/spec/output-schemas/peek.schema.json +2 -9
- package/spec/output-schemas/request-status.schema.json +113 -0
- package/spec/output-schemas/resident-status.schema.json +22 -0
- package/spec/output-schemas/search-result.schema.json +1 -1
- package/spec/output-schemas/search-results.schema.json +1 -1
- package/spec/output-schemas/sessions-automation-run.schema.json +46 -0
- package/spec/output-schemas/sessions-discovery.schema.json +38 -0
- package/spec/output-schemas/sessions-import-receipt.schema.json +156 -0
- package/spec/output-schemas/sessions-status.schema.json +432 -0
- package/spec/output-schemas/status.schema.json +98 -0
- package/src/cli/commands/ask.ts +14 -2
- package/src/cli/commands/capture.ts +55 -96
- package/src/cli/commands/daemon.ts +41 -0
- package/src/cli/commands/doctor.ts +54 -20
- package/src/cli/commands/embed.ts +41 -3
- package/src/cli/commands/ls.ts +3 -0
- package/src/cli/commands/memory.ts +12 -3
- package/src/cli/commands/query.ts +5 -0
- package/src/cli/commands/request-status.ts +59 -0
- package/src/cli/commands/reset.ts +39 -5
- package/src/cli/commands/sessions.ts +713 -0
- package/src/cli/commands/shared.ts +14 -1
- package/src/cli/commands/status.ts +63 -5
- package/src/cli/commands/vec.ts +54 -0
- package/src/cli/detach.ts +29 -1
- package/src/cli/errors.ts +13 -9
- package/src/cli/program.ts +441 -2
- package/src/cli/session-binding.ts +49 -0
- package/src/config/types.ts +8 -0
- package/src/core/capture-publish.ts +239 -0
- package/src/core/capture-sync.ts +12 -2
- package/src/core/host-paths.ts +31 -0
- package/src/core/memory-remember.ts +234 -122
- package/src/core/memory-types.ts +11 -0
- package/src/core/network-boundary-inventory.ts +8 -0
- package/src/core/request-receipts.ts +671 -0
- package/src/core/shutdown-budget.ts +6 -0
- package/src/core/vector-partition-status.ts +52 -0
- package/src/embed/backlog.ts +124 -18
- package/src/embed/fingerprint.ts +6 -3
- package/src/embed/retry.ts +66 -27
- package/src/embed/variant-backlog.ts +15 -10
- package/src/embed/variant-retry.ts +31 -22
- package/src/index.ts +30 -2
- package/src/llm/native-worker/dispatcher.ts +2 -0
- package/src/llm/native-worker/embedding-identity.ts +42 -0
- package/src/llm/native-worker/protocol.ts +1 -0
- package/src/llm/types.ts +3 -0
- package/src/mcp/context.ts +17 -0
- package/src/mcp/http-egress.ts +4 -0
- package/src/mcp/http-transport.ts +2 -0
- package/src/mcp/resources/index.ts +6 -5
- package/src/mcp/tool-descriptions-core.ts +1 -1
- package/src/mcp/tools/capture.ts +87 -85
- package/src/mcp/tools/index.ts +77 -4
- package/src/mcp/tools/memory-remember.ts +8 -1
- package/src/mcp/tools/memory-shared.ts +7 -1
- package/src/mcp/tools/request-status.ts +73 -0
- package/src/mcp/tools/sessions.ts +208 -0
- package/src/mcp/tools/status.ts +4 -0
- package/src/pipeline/hybrid.ts +37 -7
- package/src/pipeline/vsearch.ts +14 -2
- package/src/sdk/client.ts +180 -84
- package/src/sdk/index.ts +6 -0
- package/src/sdk/types.ts +54 -2
- package/src/serve/capture-service.ts +98 -32
- package/src/serve/config-sync.ts +3 -2
- package/src/serve/embed-scheduler.ts +133 -19
- package/src/serve/host-path-redaction.ts +79 -0
- package/src/serve/public/app.tsx +4 -1
- package/src/serve/public/components/CaptureModal.tsx +26 -8
- package/src/serve/public/components/sessions/AutomationPanel.tsx +800 -0
- package/src/serve/public/components/sessions/ImportReceipt.tsx +238 -0
- package/src/serve/public/components/sessions/SessionSearch.tsx +286 -0
- package/src/serve/public/components/sessions/SourcesPanel.tsx +541 -0
- package/src/serve/public/components/sessions/api.ts +40 -0
- package/src/serve/public/globals.built.css +1 -1
- package/src/serve/public/hooks/use-api.ts +26 -3
- package/src/serve/public/lib/request-intent.ts +77 -0
- package/src/serve/public/lib/snippet.tsx +52 -0
- package/src/serve/public/lib/workspace-actions.ts +12 -1
- package/src/serve/public/lib/workspace-tabs.ts +2 -0
- package/src/serve/public/pages/Dashboard.tsx +22 -9
- package/src/serve/public/pages/DocView.tsx +25 -6
- package/src/serve/public/pages/DocumentEditor.tsx +224 -104
- package/src/serve/public/pages/Search.tsx +1 -41
- package/src/serve/public/pages/Sessions.tsx +350 -0
- package/src/serve/resident-runtime.ts +69 -4
- package/src/serve/resident-status.ts +13 -1
- package/src/serve/routes/api.ts +476 -147
- package/src/serve/routes/sessions.ts +766 -0
- package/src/serve/security.ts +9 -0
- package/src/serve/server.ts +215 -10
- package/src/serve/session-automation.ts +146 -0
- package/src/serve/status-model.ts +16 -0
- package/src/serve/status.ts +2 -0
- package/src/serve/watch-reconciliation-shared.ts +3 -0
- package/src/serve/watch-service-events.ts +3 -2
- package/src/serve/watch-service-run-flush.ts +35 -2
- package/src/serve/watch-service.ts +5 -0
- package/src/sessions/archive.ts +348 -0
- package/src/sessions/automation-state.ts +444 -0
- package/src/sessions/automation-status.ts +239 -0
- package/src/sessions/automation.ts +1169 -0
- package/src/sessions/binding.ts +105 -0
- package/src/sessions/claude-hook.ts +240 -0
- package/src/sessions/config.ts +176 -0
- package/src/sessions/format.ts +191 -0
- package/src/sessions/import-child-env.ts +8 -0
- package/src/sessions/import-child.ts +152 -0
- package/src/sessions/parsers/claude-code.ts +259 -0
- package/src/sessions/parsers/codex.ts +303 -0
- package/src/sessions/parsers/hermes.ts +248 -0
- package/src/sessions/parsers/openclaw.ts +496 -0
- package/src/sessions/parsers/shared.ts +184 -0
- package/src/sessions/sanitize.ts +222 -0
- package/src/sessions/service.ts +1533 -0
- package/src/sessions/setup.ts +477 -0
- package/src/sessions/sources.ts +518 -0
- package/src/sessions/state.ts +118 -0
- package/src/sessions/types.ts +457 -0
- package/src/store/migrations/031-runtime-independent-vectors.ts +29 -0
- package/src/store/migrations/032-vector-runtime-callers.ts +17 -0
- package/src/store/migrations/index.ts +4 -0
- package/src/store/sqlite/adapter.ts +76 -16
- package/src/store/sqlite/scoped-index.ts +9 -0
- package/src/store/types.ts +11 -1
- package/src/store/vector/lazy.ts +46 -43
- package/src/store/vector/runtime-compat.ts +651 -0
- package/src/store/vector/sqlite-vec.ts +20 -2
- package/src/store/vector/status.ts +276 -35
- package/src/store/vector/types.ts +2 -0
- package/src/store/vector/variant-search.ts +71 -23
- package/src/store/vector/variants.ts +49 -14
- package/browser-extension/artifacts/gno-browser-clipper-v2.5.1.zip.sha256 +0 -1
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import type {
|
|
2
|
+
VectorPartitionStatus,
|
|
3
|
+
VectorRuntimeStatus,
|
|
4
|
+
} from "../store/vector/status";
|
|
5
|
+
|
|
6
|
+
const shortId = (id: string): string => id.slice(0, 12);
|
|
7
|
+
|
|
8
|
+
function runtimeLine(runtime: VectorRuntimeStatus): string {
|
|
9
|
+
const label = runtime.label ? ` (${runtime.label})` : "";
|
|
10
|
+
if (runtime.state === "vectors")
|
|
11
|
+
return ` This runtime${label} reads ${shortId(runtime.partition ?? "")}`;
|
|
12
|
+
if (runtime.state === "unavailable")
|
|
13
|
+
return ` This runtime${label} uses lexical retrieval only: ${runtime.reason}`;
|
|
14
|
+
return " This runtime has not resolved a partition yet; run a query or `gno embed`";
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* The caller's runtime, then one line per partition. The healthy case (one
|
|
19
|
+
* current partition this runtime reads) prints nothing, so default status
|
|
20
|
+
* output is unchanged.
|
|
21
|
+
*/
|
|
22
|
+
export function formatVectorPartitionLines(
|
|
23
|
+
partitions?: VectorPartitionStatus[],
|
|
24
|
+
runtime?: VectorRuntimeStatus
|
|
25
|
+
): string[] {
|
|
26
|
+
if (!partitions?.length) return [];
|
|
27
|
+
const [only] = partitions;
|
|
28
|
+
if (
|
|
29
|
+
partitions.length === 1 &&
|
|
30
|
+
only?.retrieval &&
|
|
31
|
+
!only.legacy &&
|
|
32
|
+
!only.incompatibleRuntimes.length
|
|
33
|
+
)
|
|
34
|
+
return [];
|
|
35
|
+
const lines = ["Vector partitions:"];
|
|
36
|
+
if (runtime) lines.push(runtimeLine(runtime));
|
|
37
|
+
for (const p of partitions) {
|
|
38
|
+
const role = p.retrieval
|
|
39
|
+
? " (used by this runtime's retrieval)"
|
|
40
|
+
: p.droppable
|
|
41
|
+
? ` (drop with: gno vec drop ${shortId(p.id)})`
|
|
42
|
+
: "";
|
|
43
|
+
lines.push(
|
|
44
|
+
` ${p.retrieval ? "*" : " "} ${shortId(p.id)} ${p.state}${p.legacy ? " legacy" : ""}, ${p.owners} chunks, ${p.provenance}${role}`
|
|
45
|
+
);
|
|
46
|
+
if (p.compatibleRuntimes.length)
|
|
47
|
+
lines.push(` read by: ${p.compatibleRuntimes.join("; ")}`);
|
|
48
|
+
if (p.incompatibleRuntimes.length)
|
|
49
|
+
lines.push(` incompatible: ${p.incompatibleRuntimes.join("; ")}`);
|
|
50
|
+
}
|
|
51
|
+
return lines;
|
|
52
|
+
}
|
package/src/embed/backlog.ts
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import type { Database } from "bun:sqlite";
|
|
2
|
+
|
|
1
3
|
import type { EmbeddingPort } from "../llm/types";
|
|
2
4
|
/**
|
|
3
5
|
* Shared embedding backlog processor.
|
|
@@ -12,21 +14,26 @@ import type {
|
|
|
12
14
|
VectorStatsPort,
|
|
13
15
|
} from "../store/vector";
|
|
14
16
|
import type { VectorVariantStore } from "../store/vector/variants";
|
|
17
|
+
import type { AcquireWriteTurn } from "./retry";
|
|
15
18
|
|
|
16
19
|
import {
|
|
17
20
|
assertInferenceActive,
|
|
18
21
|
isBackgroundInference,
|
|
19
22
|
} from "../llm/inference-scope";
|
|
23
|
+
import { formatDocForEmbedding } from "../pipeline/contextual";
|
|
20
24
|
import { err, ok } from "../store/types";
|
|
25
|
+
import {
|
|
26
|
+
embeddingPartitionIdentity,
|
|
27
|
+
recordReferenceRuntime,
|
|
28
|
+
resolveRuntimePartition,
|
|
29
|
+
} from "../store/vector/runtime-compat";
|
|
21
30
|
import { getVectorStatsDatabase } from "../store/vector/stats";
|
|
22
31
|
import { createVectorVariantStore } from "../store/vector/variants";
|
|
23
|
-
import {
|
|
24
|
-
getEmbeddingFingerprint,
|
|
25
|
-
getVariantModelFingerprint,
|
|
26
|
-
} from "./fingerprint";
|
|
32
|
+
import { getEmbeddingFingerprint } from "./fingerprint";
|
|
27
33
|
import {
|
|
28
34
|
chunkRetryKey,
|
|
29
35
|
embedAndStoreBatch,
|
|
36
|
+
inWriteTurn,
|
|
30
37
|
MAX_EMBED_CHUNK_ATTEMPTS,
|
|
31
38
|
} from "./retry";
|
|
32
39
|
import { embedVariantBacklog } from "./variant-backlog";
|
|
@@ -47,6 +54,14 @@ export interface EmbedBacklogDeps {
|
|
|
47
54
|
variantStore?: VectorVariantStore;
|
|
48
55
|
/** Recheck the effective runtime identity after asynchronous inference. */
|
|
49
56
|
identityStillCurrent?: () => boolean;
|
|
57
|
+
/**
|
|
58
|
+
* Write gate for callers that do not already hold the shared writer lease
|
|
59
|
+
* (the resident scheduler): taken around each page's writes and released
|
|
60
|
+
* after. null means another writer holds it; the pass stops as deferred.
|
|
61
|
+
*/
|
|
62
|
+
acquireWriteTurn?: AcquireWriteTurn;
|
|
63
|
+
/** Explicit confirmation to build a separate vector partition (never implied by --yes). */
|
|
64
|
+
allowNewPartition?: boolean;
|
|
50
65
|
}
|
|
51
66
|
|
|
52
67
|
export interface EmbedBacklogResult {
|
|
@@ -59,6 +74,8 @@ export interface EmbedBacklogResult {
|
|
|
59
74
|
contentionErrors?: number;
|
|
60
75
|
/** Error message if vec index sync failed (embeddings stored, but search may be stale) */
|
|
61
76
|
syncError?: string;
|
|
77
|
+
/** The pass stopped early because another writer held the write gate. */
|
|
78
|
+
deferred?: boolean;
|
|
62
79
|
}
|
|
63
80
|
|
|
64
81
|
interface Cursor {
|
|
@@ -78,7 +95,18 @@ export async function embedBacklog(
|
|
|
78
95
|
deps: EmbedBacklogDeps
|
|
79
96
|
): Promise<StoreResult<EmbedBacklogResult>> {
|
|
80
97
|
assertInferenceActive();
|
|
81
|
-
|
|
98
|
+
if (deps.acquireWriteTurn && !deps.variantStore) {
|
|
99
|
+
// Model loading stays outside the write turn; preparation's partition
|
|
100
|
+
// writes below take one like every other background write.
|
|
101
|
+
const initialized = await deps.embedPort.init();
|
|
102
|
+
if (!initialized.ok) return err("INTERNAL", initialized.error.message);
|
|
103
|
+
}
|
|
104
|
+
const turn = await inWriteTurn(deps.acquireWriteTurn, () =>
|
|
105
|
+
prepareEmbeddingBacklog(deps)
|
|
106
|
+
);
|
|
107
|
+
if (turn.deferred)
|
|
108
|
+
return ok({ embedded: 0, errors: 0, contentionErrors: 0, deferred: true });
|
|
109
|
+
const prepared = turn.value;
|
|
82
110
|
if (!prepared.ok) return prepared;
|
|
83
111
|
deps = prepared.value;
|
|
84
112
|
if (deps.variantStore) return embedVariantBacklog(deps, deps.variantStore);
|
|
@@ -201,7 +229,10 @@ export async function embedBacklog(
|
|
|
201
229
|
embedFingerprint,
|
|
202
230
|
identityStillCurrent: deps.identityStillCurrent,
|
|
203
231
|
statsPort,
|
|
232
|
+
acquireWriteTurn: deps.acquireWriteTurn,
|
|
204
233
|
});
|
|
234
|
+
if (batchStoreResult.deferred)
|
|
235
|
+
return ok({ embedded, errors, contentionErrors, deferred: true });
|
|
205
236
|
embedded += batchStoreResult.embedded;
|
|
206
237
|
errors += batchStoreResult.errors;
|
|
207
238
|
contentionErrors += batchStoreResult.contentionErrors;
|
|
@@ -225,7 +256,12 @@ export async function embedBacklog(
|
|
|
225
256
|
// Sync vec index once at end if any vec0 writes failed
|
|
226
257
|
let syncError: string | undefined;
|
|
227
258
|
if (vectorIndex.vecDirty) {
|
|
228
|
-
const
|
|
259
|
+
const turn = await inWriteTurn(deps.acquireWriteTurn, () =>
|
|
260
|
+
vectorIndex.syncVecIndex()
|
|
261
|
+
);
|
|
262
|
+
if (turn.deferred)
|
|
263
|
+
return ok({ embedded, errors, contentionErrors, deferred: true });
|
|
264
|
+
const syncResult = turn.value;
|
|
229
265
|
if (syncResult.ok) {
|
|
230
266
|
const { added, removed } = syncResult.value;
|
|
231
267
|
if (added > 0 || removed > 0) {
|
|
@@ -248,6 +284,59 @@ export async function embedBacklog(
|
|
|
248
284
|
}
|
|
249
285
|
}
|
|
250
286
|
|
|
287
|
+
function formatEstimate(ms: number): string {
|
|
288
|
+
const seconds = Math.max(1, Math.round(ms / 1000));
|
|
289
|
+
if (seconds < 90) return `about ${seconds} s`;
|
|
290
|
+
const minutes = Math.round(seconds / 60);
|
|
291
|
+
return minutes < 90
|
|
292
|
+
? `about ${minutes} min`
|
|
293
|
+
: `about ${(minutes / 60).toFixed(1)} h`;
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
/** Time this port on a few current chunks when no compatibility sample ran. */
|
|
297
|
+
async function measureEmbedRate(
|
|
298
|
+
db: Database,
|
|
299
|
+
port: EmbeddingPort
|
|
300
|
+
): Promise<number | undefined> {
|
|
301
|
+
const inputs = db
|
|
302
|
+
.query<{ text: string; title: string | null }, []>(`
|
|
303
|
+
SELECT c.text, d.title FROM documents d
|
|
304
|
+
JOIN content_chunks c ON c.mirror_hash = d.mirror_hash
|
|
305
|
+
WHERE d.active = 1 ORDER BY d.id, c.seq LIMIT 8
|
|
306
|
+
`)
|
|
307
|
+
.all()
|
|
308
|
+
.map((row) =>
|
|
309
|
+
formatDocForEmbedding(row.text, row.title ?? undefined, port.modelUri)
|
|
310
|
+
);
|
|
311
|
+
if (!inputs.length) return undefined;
|
|
312
|
+
const startedAt = performance.now();
|
|
313
|
+
const result = await port.embedBatch(inputs);
|
|
314
|
+
return result.ok
|
|
315
|
+
? (performance.now() - startedAt) / inputs.length
|
|
316
|
+
: undefined;
|
|
317
|
+
}
|
|
318
|
+
|
|
319
|
+
/** R3: a fork names itself, its full size and a measured estimate before it runs. */
|
|
320
|
+
async function separatePartitionMessage(
|
|
321
|
+
db: Database,
|
|
322
|
+
port: EmbeddingPort,
|
|
323
|
+
reason: string,
|
|
324
|
+
msPerChunk: number | undefined
|
|
325
|
+
): Promise<string> {
|
|
326
|
+
const chunks = db
|
|
327
|
+
.query<{ count: number }, []>(`
|
|
328
|
+
SELECT count(*) AS count FROM documents d
|
|
329
|
+
JOIN content_chunks c ON c.mirror_hash = d.mirror_hash WHERE d.active = 1
|
|
330
|
+
`)
|
|
331
|
+
.get()!.count;
|
|
332
|
+
const rate = msPerChunk ?? (await measureEmbedRate(db, port));
|
|
333
|
+
const estimate =
|
|
334
|
+
rate === undefined
|
|
335
|
+
? "no estimate: embedding could not be timed"
|
|
336
|
+
: `estimated ${formatEstimate(rate * chunks)} at the measured ${Math.round(rate)} ms per chunk`;
|
|
337
|
+
return `Embedding would build a separate vector partition (${reason}). It re-embeds all ${chunks} chunks (${estimate}). Confirm with \`gno embed --new-partition\`; --yes alone does not confirm.`;
|
|
338
|
+
}
|
|
339
|
+
|
|
251
340
|
/** Resolve authority before counts, dry runs, forced work, or early returns. */
|
|
252
341
|
export async function prepareEmbeddingBacklog(
|
|
253
342
|
deps: EmbedBacklogDeps
|
|
@@ -259,19 +348,36 @@ export async function prepareEmbeddingBacklog(
|
|
|
259
348
|
const initialized = await deps.embedPort.init();
|
|
260
349
|
if (!initialized.ok) return err("INTERNAL", initialized.error.message);
|
|
261
350
|
const identity = deps.embedPort.getIdentity?.();
|
|
262
|
-
|
|
351
|
+
const primary = embeddingPartitionIdentity(deps.embedPort);
|
|
352
|
+
if (identity && primary) {
|
|
263
353
|
const identitySnapshot = JSON.stringify(identity);
|
|
264
|
-
const dimensions =
|
|
265
|
-
const
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
354
|
+
const dimensions = primary.dimensions;
|
|
355
|
+
const resolved = await resolveRuntimePartition(
|
|
356
|
+
db,
|
|
357
|
+
deps.embedPort,
|
|
358
|
+
primary
|
|
359
|
+
);
|
|
360
|
+
if (resolved.blocked && !deps.allowNewPartition)
|
|
361
|
+
return err(
|
|
362
|
+
"VECTOR_PARTITION_FORK",
|
|
363
|
+
await separatePartitionMessage(
|
|
364
|
+
db,
|
|
365
|
+
deps.embedPort,
|
|
366
|
+
resolved.blocked.reason,
|
|
367
|
+
resolved.msPerChunk
|
|
368
|
+
)
|
|
369
|
+
);
|
|
370
|
+
const variantStore = await createVectorVariantStore(
|
|
371
|
+
db,
|
|
372
|
+
resolved.blocked?.separate ?? resolved.identity,
|
|
373
|
+
identity.runtimeLabel
|
|
374
|
+
);
|
|
375
|
+
if (resolved.blocked || resolved.verdict === "unverified") {
|
|
376
|
+
// Vectors without a current owner cannot be measured; they must not
|
|
377
|
+
// survive to be reused by the runtime that becomes the reference.
|
|
378
|
+
variantStore.collectGarbage();
|
|
379
|
+
recordReferenceRuntime(db, variantStore.partitionId, identity);
|
|
380
|
+
}
|
|
275
381
|
variantStore.selectForEmbedding();
|
|
276
382
|
return ok({
|
|
277
383
|
...deps,
|
package/src/embed/fingerprint.ts
CHANGED
|
@@ -36,16 +36,19 @@ export function getEmbeddingFingerprint(
|
|
|
36
36
|
.digest("hex");
|
|
37
37
|
}
|
|
38
38
|
|
|
39
|
-
/**
|
|
39
|
+
/**
|
|
40
|
+
* Partition identity: actual weights plus the formatter policy. Runtime details
|
|
41
|
+
* (Bun, native binding, backend, threads) are provenance, never identity; the
|
|
42
|
+
* measured compatibility check decides whether a runtime may share vectors.
|
|
43
|
+
*/
|
|
40
44
|
export function getVariantModelFingerprint(
|
|
41
45
|
input: EmbeddingFingerprintInput,
|
|
42
|
-
identity: { modelFingerprint: string
|
|
46
|
+
identity: { modelFingerprint: string }
|
|
43
47
|
): string {
|
|
44
48
|
return new Bun.CryptoHasher("sha256")
|
|
45
49
|
.update(
|
|
46
50
|
JSON.stringify([
|
|
47
51
|
identity.modelFingerprint,
|
|
48
|
-
identity.runtimeFingerprint,
|
|
49
52
|
getEmbeddingFingerprint(input),
|
|
50
53
|
])
|
|
51
54
|
)
|
package/src/embed/retry.ts
CHANGED
|
@@ -15,6 +15,28 @@ import { getVectorStatsDatabase } from "../store/vector/stats";
|
|
|
15
15
|
import { embedTextsWithRecovery } from "./batch";
|
|
16
16
|
|
|
17
17
|
export const MAX_EMBED_CHUNK_ATTEMPTS = 2;
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* Write gate for callers that do not already hold the shared writer lease
|
|
21
|
+
* (the resident scheduler). Returns a release, or null when another writer
|
|
22
|
+
* holds the lease.
|
|
23
|
+
*/
|
|
24
|
+
export type AcquireWriteTurn = () => Promise<(() => Promise<void>) | null>;
|
|
25
|
+
|
|
26
|
+
/** Run `write` inside one write turn, or report that the gate is held elsewhere. */
|
|
27
|
+
export async function inWriteTurn<T>(
|
|
28
|
+
acquire: AcquireWriteTurn | undefined,
|
|
29
|
+
write: () => Promise<T>
|
|
30
|
+
): Promise<{ deferred: true } | { deferred: false; value: T }> {
|
|
31
|
+
if (!acquire) return { deferred: false, value: await write() };
|
|
32
|
+
const release = await acquire();
|
|
33
|
+
if (!release) return { deferred: true };
|
|
34
|
+
try {
|
|
35
|
+
return { deferred: false, value: await write() };
|
|
36
|
+
} finally {
|
|
37
|
+
await release();
|
|
38
|
+
}
|
|
39
|
+
}
|
|
18
40
|
export const MAX_EMBED_FAILURE_SAMPLES = 5;
|
|
19
41
|
|
|
20
42
|
/** Total upsert attempts (initial + retries) when persistence hits SQLITE_BUSY/LOCKED. */
|
|
@@ -44,6 +66,8 @@ export interface EmbedStoreBatchResult {
|
|
|
44
66
|
suggestion?: string;
|
|
45
67
|
batchFailed: boolean;
|
|
46
68
|
batchError?: string;
|
|
69
|
+
/** The write gate was held elsewhere; nothing was persisted. */
|
|
70
|
+
deferred?: boolean;
|
|
47
71
|
}
|
|
48
72
|
|
|
49
73
|
// fn-127 integration: CLI consumers (src/cli/commands/embed.ts,
|
|
@@ -173,6 +197,8 @@ export async function embedAndStoreBatch(params: {
|
|
|
173
197
|
identityStillCurrent?: () => boolean;
|
|
174
198
|
/** Test seam: override contention-retry delays in milliseconds. */
|
|
175
199
|
delays?: number[];
|
|
200
|
+
/** Gate held only around persistence, never around inference. */
|
|
201
|
+
acquireWriteTurn?: AcquireWriteTurn;
|
|
176
202
|
}): Promise<EmbedStoreBatchResult> {
|
|
177
203
|
const { embedPort, vectorIndex, items, modelUri, embedFingerprint } = params;
|
|
178
204
|
const db = params.statsPort && getVectorStatsDatabase(params.statsPort);
|
|
@@ -282,35 +308,48 @@ export async function embedAndStoreBatch(params: {
|
|
|
282
308
|
);
|
|
283
309
|
});
|
|
284
310
|
};
|
|
285
|
-
const
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
311
|
+
const turn = await inWriteTurn(params.acquireWriteTurn, () =>
|
|
312
|
+
upsertVectorsWithContentionRetry(
|
|
313
|
+
{
|
|
314
|
+
upsertVectors: async (rows) => {
|
|
315
|
+
if (vectorIndex.upsertVectorsChecked) {
|
|
316
|
+
const result = await vectorIndex.upsertVectorsChecked(
|
|
317
|
+
rows,
|
|
318
|
+
(candidates) => {
|
|
319
|
+
committedRows = checkpoint(candidates);
|
|
320
|
+
return committedRows;
|
|
321
|
+
}
|
|
322
|
+
);
|
|
323
|
+
if (!result.ok) return result;
|
|
324
|
+
written = result.value;
|
|
325
|
+
return ok(undefined);
|
|
326
|
+
}
|
|
327
|
+
if (db)
|
|
328
|
+
return err("INVALID_INPUT", "Atomic vector checkpoint unavailable");
|
|
329
|
+
const valid = checkpoint(rows);
|
|
330
|
+
const result = await vectorIndex.upsertVectors(valid);
|
|
331
|
+
if (result.ok) {
|
|
332
|
+
written = valid.length;
|
|
333
|
+
committedRows = valid;
|
|
334
|
+
}
|
|
335
|
+
return result;
|
|
336
|
+
},
|
|
309
337
|
},
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
338
|
+
vectors,
|
|
339
|
+
params.delays
|
|
340
|
+
)
|
|
313
341
|
);
|
|
342
|
+
if (turn.deferred)
|
|
343
|
+
return {
|
|
344
|
+
embedded: 0,
|
|
345
|
+
errors: 0,
|
|
346
|
+
contentionErrors: 0,
|
|
347
|
+
retryItems: [],
|
|
348
|
+
errorSamples: [],
|
|
349
|
+
batchFailed: false,
|
|
350
|
+
deferred: true,
|
|
351
|
+
};
|
|
352
|
+
const storeResult = turn.value;
|
|
314
353
|
if (!storeResult.ok) {
|
|
315
354
|
if (isUpsertLockContention(storeResult.error)) {
|
|
316
355
|
return {
|
|
@@ -8,6 +8,7 @@ import {
|
|
|
8
8
|
} from "../llm/inference-scope";
|
|
9
9
|
import { err, ok } from "../store/types";
|
|
10
10
|
import { getVectorStatsDatabase } from "../store/vector/stats";
|
|
11
|
+
import { inWriteTurn } from "./retry";
|
|
11
12
|
import { variantBacklogPage } from "./variant-plan";
|
|
12
13
|
import { embedVariantBatch } from "./variant-retry";
|
|
13
14
|
|
|
@@ -59,7 +60,9 @@ export async function embedVariantBacklog(
|
|
|
59
60
|
owners,
|
|
60
61
|
identityStillCurrent,
|
|
61
62
|
force: deps.force,
|
|
63
|
+
acquireWriteTurn: deps.acquireWriteTurn,
|
|
62
64
|
});
|
|
65
|
+
if (result.deferred) return ok({ ...total, deferred: true });
|
|
63
66
|
total.embedded += result.embedded;
|
|
64
67
|
total.errors += result.errors;
|
|
65
68
|
total.contentionErrors += result.contentionErrors;
|
|
@@ -83,16 +86,18 @@ export async function embedVariantBacklog(
|
|
|
83
86
|
// Capture the epoch before checking completeness; activate rechecks under write lock.
|
|
84
87
|
const epoch = store.epoch();
|
|
85
88
|
if (identityStillCurrent() && !store.pending({ limit: 1 }).length) {
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
}
|
|
89
|
+
const turn = await inWriteTurn(deps.acquireWriteTurn, async () => {
|
|
90
|
+
try {
|
|
91
|
+
if (!store.isActive()) store.syncIndex();
|
|
92
|
+
if (identityStillCurrent()) store.activate(epoch);
|
|
93
|
+
return undefined;
|
|
94
|
+
} catch (cause) {
|
|
95
|
+
return cause instanceof Error ? cause.message : String(cause);
|
|
96
|
+
}
|
|
97
|
+
});
|
|
98
|
+
if (turn.deferred) return ok({ ...total, deferred: true });
|
|
99
|
+
if (turn.value !== undefined)
|
|
100
|
+
return ok({ ...total, syncError: turn.value });
|
|
96
101
|
}
|
|
97
102
|
assertInferenceActive();
|
|
98
103
|
return ok(total);
|
|
@@ -5,7 +5,9 @@ import type { VectorVariantStore } from "../store/vector/variants";
|
|
|
5
5
|
import { err, ok } from "../store/types";
|
|
6
6
|
import { embedTextsWithRecovery } from "./batch";
|
|
7
7
|
import {
|
|
8
|
+
type AcquireWriteTurn,
|
|
8
9
|
chunkRetryKey,
|
|
10
|
+
inWriteTurn,
|
|
9
11
|
isUpsertLockContention,
|
|
10
12
|
upsertVectorsWithContentionRetry,
|
|
11
13
|
} from "./retry";
|
|
@@ -30,11 +32,14 @@ export async function embedVariantBatch(params: {
|
|
|
30
32
|
identityStillCurrent: () => boolean;
|
|
31
33
|
delays?: number[];
|
|
32
34
|
force?: boolean;
|
|
35
|
+
/** Gate held only around the checkpoint write, never around inference. */
|
|
36
|
+
acquireWriteTurn?: AcquireWriteTurn;
|
|
33
37
|
}): Promise<{
|
|
34
38
|
embedded: number;
|
|
35
39
|
errors: number;
|
|
36
40
|
contentionErrors: number;
|
|
37
41
|
retryOwners: VectorOwnerInput[];
|
|
42
|
+
deferred?: boolean;
|
|
38
43
|
}> {
|
|
39
44
|
const { store, embedPort, identityStillCurrent } = params;
|
|
40
45
|
const empty = {
|
|
@@ -76,30 +81,34 @@ export async function embedVariantBatch(params: {
|
|
|
76
81
|
.map((owner) => ({ owner, embedding: vectors.get(owner.inputHash) }));
|
|
77
82
|
if (!rows.length) return { ...empty, retryOwners };
|
|
78
83
|
let written = 0;
|
|
79
|
-
const
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
84
|
+
const turn = await inWriteTurn(params.acquireWriteTurn, () =>
|
|
85
|
+
upsertVectorsWithContentionRetry(
|
|
86
|
+
{
|
|
87
|
+
upsertVectors: () => {
|
|
88
|
+
try {
|
|
89
|
+
// A contention wait may allow document or runtime mutations. Revalidate each attempt.
|
|
90
|
+
if (!identityStillCurrent()) return Promise.resolve(ok(undefined));
|
|
91
|
+
const valid = rows.filter(({ owner }) => isCurrent(store, owner));
|
|
92
|
+
store.write(valid);
|
|
93
|
+
written = valid.length;
|
|
94
|
+
return Promise.resolve(ok(undefined));
|
|
95
|
+
} catch (cause) {
|
|
96
|
+
return Promise.resolve(
|
|
97
|
+
err(
|
|
98
|
+
"QUERY_FAILED",
|
|
99
|
+
cause instanceof Error ? cause.message : String(cause),
|
|
100
|
+
cause
|
|
101
|
+
)
|
|
102
|
+
);
|
|
103
|
+
}
|
|
104
|
+
},
|
|
98
105
|
},
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
106
|
+
[],
|
|
107
|
+
params.delays
|
|
108
|
+
)
|
|
102
109
|
);
|
|
110
|
+
if (turn.deferred) return { ...empty, deferred: true };
|
|
111
|
+
const persisted = turn.value;
|
|
103
112
|
if (!persisted.ok)
|
|
104
113
|
return {
|
|
105
114
|
...empty,
|
package/src/index.ts
CHANGED
|
@@ -8,15 +8,35 @@
|
|
|
8
8
|
|
|
9
9
|
import { runCli } from "./cli/run";
|
|
10
10
|
import { resetModelManager } from "./llm/nodeLlamaCpp/lifecycle";
|
|
11
|
+
import { IMPORT_CHILD_ENV } from "./sessions/import-child-env";
|
|
11
12
|
|
|
12
13
|
/**
|
|
13
|
-
*
|
|
14
|
-
*
|
|
14
|
+
* End `stream` and resolve once its queued writes have reached the OS.
|
|
15
|
+
* process.exit() drops pending asynchronous pipe writes, so a pipe consumer
|
|
16
|
+
* would otherwise see output cut at the pipe buffer size. Bun reports no
|
|
17
|
+
* writableLength and fires an empty write's callback immediately, so end()
|
|
18
|
+
* is the flush that actually waits. A closed consumer (EPIPE) settles
|
|
19
|
+
* through the callback or the 'error' event.
|
|
20
|
+
*/
|
|
21
|
+
function flushStream(stream: NodeJS.WriteStream): Promise<void> {
|
|
22
|
+
if (stream.destroyed || stream.writableEnded) {
|
|
23
|
+
return Promise.resolve();
|
|
24
|
+
}
|
|
25
|
+
return new Promise((resolve) => {
|
|
26
|
+
stream.once("error", () => resolve());
|
|
27
|
+
stream.end(() => resolve());
|
|
28
|
+
});
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* Cleanup models, flush output, and exit.
|
|
33
|
+
* Without the explicit exit, llama.cpp native threads can keep the process alive.
|
|
15
34
|
*/
|
|
16
35
|
async function cleanupAndExit(code: number): Promise<never> {
|
|
17
36
|
await resetModelManager().catch(() => {
|
|
18
37
|
// Ignore cleanup errors on exit
|
|
19
38
|
});
|
|
39
|
+
await Promise.all([flushStream(process.stdout), flushStream(process.stderr)]);
|
|
20
40
|
process.exit(code);
|
|
21
41
|
}
|
|
22
42
|
|
|
@@ -37,6 +57,14 @@ process.on("SIGINT", () => {
|
|
|
37
57
|
});
|
|
38
58
|
});
|
|
39
59
|
|
|
60
|
+
// A compiled executable re-run as the session import child (see
|
|
61
|
+
// src/sessions/import-child.ts) serves that one request instead of the CLI.
|
|
62
|
+
if (process.env[IMPORT_CHILD_ENV] === "1") {
|
|
63
|
+
const { runImportChild } = await import("./sessions/import-child");
|
|
64
|
+
await runImportChild();
|
|
65
|
+
await cleanupAndExit(0);
|
|
66
|
+
}
|
|
67
|
+
|
|
40
68
|
// Await module completion so pending piped stdin keeps Windows Bun alive.
|
|
41
69
|
await runCli(process.argv)
|
|
42
70
|
.then((code) => cleanupAndExit(interruptExitCode || code))
|
|
@@ -15,6 +15,7 @@ import {
|
|
|
15
15
|
fileIdentity,
|
|
16
16
|
fingerprintModel,
|
|
17
17
|
fingerprintRuntime,
|
|
18
|
+
runtimeLabel,
|
|
18
19
|
} from "./embedding-identity";
|
|
19
20
|
import { NativeWorkerError } from "./errors";
|
|
20
21
|
import { checkEvaluation } from "./evaluation";
|
|
@@ -174,6 +175,7 @@ export class NativeDispatcher {
|
|
|
174
175
|
gpu: llama.gpu,
|
|
175
176
|
cpuMathCores: llama.cpuMathCores,
|
|
176
177
|
}),
|
|
178
|
+
runtimeLabel: runtimeLabel(llama.gpu),
|
|
177
179
|
},
|
|
178
180
|
},
|
|
179
181
|
};
|
|
@@ -31,3 +31,45 @@ export function fingerprintRuntime(settings: Record<string, unknown>): string {
|
|
|
31
31
|
)
|
|
32
32
|
.digest("hex");
|
|
33
33
|
}
|
|
34
|
+
|
|
35
|
+
/** Environment that shapes the native runtime identity of an embedding. */
|
|
36
|
+
const RUNTIME_ENV = [
|
|
37
|
+
"GNO_LLAMA_GPU",
|
|
38
|
+
"NODE_LLAMA_CPP_GPU",
|
|
39
|
+
"GNO_EMBED_CONTEXT_SIZE",
|
|
40
|
+
"GNO_EMBED_CONTEXTS",
|
|
41
|
+
"GNO_EMBED_THREADS",
|
|
42
|
+
] as const;
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* What a process can know about its embedding runtime without loading the
|
|
46
|
+
* model: status looks up the identity a caller with this key last resolved.
|
|
47
|
+
*/
|
|
48
|
+
export function runtimeCallerKey(
|
|
49
|
+
model: string,
|
|
50
|
+
env: NodeJS.ProcessEnv = process.env
|
|
51
|
+
): string {
|
|
52
|
+
return new Bun.CryptoHasher("sha256")
|
|
53
|
+
.update(
|
|
54
|
+
JSON.stringify([
|
|
55
|
+
model,
|
|
56
|
+
Bun.version,
|
|
57
|
+
nativePackage.version,
|
|
58
|
+
process.platform,
|
|
59
|
+
process.arch,
|
|
60
|
+
RUNTIME_ENV.map((name) => env[name] ?? null),
|
|
61
|
+
])
|
|
62
|
+
)
|
|
63
|
+
.digest("hex");
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
const BACKEND_LABELS: Record<string, string> = {
|
|
67
|
+
cuda: "CUDA",
|
|
68
|
+
metal: "Metal",
|
|
69
|
+
vulkan: "Vulkan",
|
|
70
|
+
};
|
|
71
|
+
|
|
72
|
+
/** Readable provenance for status output; identity stays the fingerprint. */
|
|
73
|
+
export function runtimeLabel(gpu: string | false): string {
|
|
74
|
+
return `${gpu ? (BACKEND_LABELS[gpu] ?? gpu) : "CPU"}, Bun ${Bun.version}`;
|
|
75
|
+
}
|
|
@@ -112,6 +112,7 @@ export const EmbeddingIdentitySchema = z.strictObject({
|
|
|
112
112
|
truncationPolicy: z.string().min(1),
|
|
113
113
|
modelFingerprint: z.string().regex(/^[a-f0-9]{64}$/),
|
|
114
114
|
runtimeFingerprint: z.string().regex(/^[a-f0-9]{64}$/),
|
|
115
|
+
runtimeLabel: z.string().min(1),
|
|
115
116
|
});
|
|
116
117
|
const metadata = z.strictObject({
|
|
117
118
|
dimensions: id.optional(),
|
package/src/llm/types.ts
CHANGED
|
@@ -95,7 +95,10 @@ export interface EmbeddingIdentity {
|
|
|
95
95
|
/** Versioned policy including the effective token limit. */
|
|
96
96
|
truncationPolicy: string;
|
|
97
97
|
modelFingerprint: string;
|
|
98
|
+
/** Runtime provenance (Bun, binding, backend, threads); not vector identity. */
|
|
98
99
|
runtimeFingerprint: string;
|
|
100
|
+
/** Readable provenance such as "CUDA, Bun 1.4.2". */
|
|
101
|
+
runtimeLabel?: string;
|
|
99
102
|
}
|
|
100
103
|
|
|
101
104
|
export interface EmbeddingPort {
|