@gmickel/gno 2.5.1 → 2.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (160) hide show
  1. package/README.md +60 -5
  2. package/assets/skill/README.md +5 -1
  3. package/assets/skill/SKILL.md +80 -4
  4. package/assets/skill/cli-reference.md +132 -2
  5. package/assets/skill/examples.md +30 -0
  6. package/assets/skill/mcp-reference.md +54 -1
  7. package/assets/skill/recipes/capture-and-file.md +6 -0
  8. package/assets/skill/recipes/memory-file-decision.md +6 -0
  9. package/assets/skill/recipes/memory-supersede-fact.md +5 -0
  10. package/assets/skill/recipes/session-evidence-lookup.md +98 -0
  11. package/assets/spa-production.json.gz +0 -0
  12. package/browser-extension/artifacts/{gno-browser-clipper-v2.5.1.zip → gno-browser-clipper-v2.7.0.zip} +0 -0
  13. package/browser-extension/artifacts/gno-browser-clipper-v2.7.0.zip.sha256 +1 -0
  14. package/browser-extension/dist/manifest.json +1 -1
  15. package/package.json +2 -1
  16. package/spec/cli.md +449 -35
  17. package/spec/mcp.md +234 -4
  18. package/spec/output-schemas/ask.schema.json +1 -1
  19. package/spec/output-schemas/capture-receipt.schema.json +4 -1
  20. package/spec/output-schemas/doctor.schema.json +88 -0
  21. package/spec/output-schemas/error.schema.json +11 -2
  22. package/spec/output-schemas/get.schema.json +1 -1
  23. package/spec/output-schemas/mcp-capture-result.schema.json +4 -2
  24. package/spec/output-schemas/memory-remember.schema.json +10 -4
  25. package/spec/output-schemas/multi-get.schema.json +4 -1
  26. package/spec/output-schemas/peek.schema.json +2 -9
  27. package/spec/output-schemas/request-status.schema.json +113 -0
  28. package/spec/output-schemas/resident-status.schema.json +22 -0
  29. package/spec/output-schemas/search-result.schema.json +1 -1
  30. package/spec/output-schemas/search-results.schema.json +1 -1
  31. package/spec/output-schemas/sessions-automation-run.schema.json +46 -0
  32. package/spec/output-schemas/sessions-discovery.schema.json +38 -0
  33. package/spec/output-schemas/sessions-import-receipt.schema.json +156 -0
  34. package/spec/output-schemas/sessions-status.schema.json +432 -0
  35. package/spec/output-schemas/status.schema.json +98 -0
  36. package/src/cli/commands/ask.ts +14 -2
  37. package/src/cli/commands/capture.ts +55 -96
  38. package/src/cli/commands/daemon.ts +41 -0
  39. package/src/cli/commands/doctor.ts +54 -20
  40. package/src/cli/commands/embed.ts +41 -3
  41. package/src/cli/commands/ls.ts +3 -0
  42. package/src/cli/commands/memory.ts +12 -3
  43. package/src/cli/commands/query.ts +5 -0
  44. package/src/cli/commands/request-status.ts +59 -0
  45. package/src/cli/commands/reset.ts +39 -5
  46. package/src/cli/commands/sessions.ts +713 -0
  47. package/src/cli/commands/shared.ts +14 -1
  48. package/src/cli/commands/status.ts +63 -5
  49. package/src/cli/commands/vec.ts +54 -0
  50. package/src/cli/detach.ts +29 -1
  51. package/src/cli/errors.ts +13 -9
  52. package/src/cli/program.ts +441 -2
  53. package/src/cli/session-binding.ts +49 -0
  54. package/src/config/types.ts +8 -0
  55. package/src/core/capture-publish.ts +239 -0
  56. package/src/core/capture-sync.ts +12 -2
  57. package/src/core/host-paths.ts +31 -0
  58. package/src/core/memory-remember.ts +234 -122
  59. package/src/core/memory-types.ts +11 -0
  60. package/src/core/network-boundary-inventory.ts +8 -0
  61. package/src/core/request-receipts.ts +671 -0
  62. package/src/core/shutdown-budget.ts +6 -0
  63. package/src/core/vector-partition-status.ts +52 -0
  64. package/src/embed/backlog.ts +124 -18
  65. package/src/embed/fingerprint.ts +6 -3
  66. package/src/embed/retry.ts +66 -27
  67. package/src/embed/variant-backlog.ts +15 -10
  68. package/src/embed/variant-retry.ts +31 -22
  69. package/src/index.ts +30 -2
  70. package/src/llm/native-worker/dispatcher.ts +2 -0
  71. package/src/llm/native-worker/embedding-identity.ts +42 -0
  72. package/src/llm/native-worker/protocol.ts +1 -0
  73. package/src/llm/types.ts +3 -0
  74. package/src/mcp/context.ts +17 -0
  75. package/src/mcp/http-egress.ts +4 -0
  76. package/src/mcp/http-transport.ts +2 -0
  77. package/src/mcp/resources/index.ts +6 -5
  78. package/src/mcp/tool-descriptions-core.ts +1 -1
  79. package/src/mcp/tools/capture.ts +87 -85
  80. package/src/mcp/tools/index.ts +77 -4
  81. package/src/mcp/tools/memory-remember.ts +8 -1
  82. package/src/mcp/tools/memory-shared.ts +7 -1
  83. package/src/mcp/tools/request-status.ts +73 -0
  84. package/src/mcp/tools/sessions.ts +208 -0
  85. package/src/mcp/tools/status.ts +4 -0
  86. package/src/pipeline/hybrid.ts +37 -7
  87. package/src/pipeline/vsearch.ts +14 -2
  88. package/src/sdk/client.ts +180 -84
  89. package/src/sdk/index.ts +6 -0
  90. package/src/sdk/types.ts +54 -2
  91. package/src/serve/capture-service.ts +98 -32
  92. package/src/serve/config-sync.ts +3 -2
  93. package/src/serve/embed-scheduler.ts +133 -19
  94. package/src/serve/host-path-redaction.ts +79 -0
  95. package/src/serve/public/app.tsx +4 -1
  96. package/src/serve/public/components/CaptureModal.tsx +26 -8
  97. package/src/serve/public/components/sessions/AutomationPanel.tsx +800 -0
  98. package/src/serve/public/components/sessions/ImportReceipt.tsx +238 -0
  99. package/src/serve/public/components/sessions/SessionSearch.tsx +286 -0
  100. package/src/serve/public/components/sessions/SourcesPanel.tsx +541 -0
  101. package/src/serve/public/components/sessions/api.ts +40 -0
  102. package/src/serve/public/globals.built.css +1 -1
  103. package/src/serve/public/hooks/use-api.ts +26 -3
  104. package/src/serve/public/lib/request-intent.ts +77 -0
  105. package/src/serve/public/lib/snippet.tsx +52 -0
  106. package/src/serve/public/lib/workspace-actions.ts +12 -1
  107. package/src/serve/public/lib/workspace-tabs.ts +2 -0
  108. package/src/serve/public/pages/Dashboard.tsx +22 -9
  109. package/src/serve/public/pages/DocView.tsx +25 -6
  110. package/src/serve/public/pages/DocumentEditor.tsx +224 -104
  111. package/src/serve/public/pages/Search.tsx +1 -41
  112. package/src/serve/public/pages/Sessions.tsx +350 -0
  113. package/src/serve/resident-runtime.ts +69 -4
  114. package/src/serve/resident-status.ts +13 -1
  115. package/src/serve/routes/api.ts +476 -147
  116. package/src/serve/routes/sessions.ts +766 -0
  117. package/src/serve/security.ts +9 -0
  118. package/src/serve/server.ts +215 -10
  119. package/src/serve/session-automation.ts +146 -0
  120. package/src/serve/status-model.ts +16 -0
  121. package/src/serve/status.ts +2 -0
  122. package/src/serve/watch-reconciliation-shared.ts +3 -0
  123. package/src/serve/watch-service-events.ts +3 -2
  124. package/src/serve/watch-service-run-flush.ts +35 -2
  125. package/src/serve/watch-service.ts +5 -0
  126. package/src/sessions/archive.ts +348 -0
  127. package/src/sessions/automation-state.ts +444 -0
  128. package/src/sessions/automation-status.ts +239 -0
  129. package/src/sessions/automation.ts +1169 -0
  130. package/src/sessions/binding.ts +105 -0
  131. package/src/sessions/claude-hook.ts +240 -0
  132. package/src/sessions/config.ts +176 -0
  133. package/src/sessions/format.ts +191 -0
  134. package/src/sessions/import-child-env.ts +8 -0
  135. package/src/sessions/import-child.ts +152 -0
  136. package/src/sessions/parsers/claude-code.ts +259 -0
  137. package/src/sessions/parsers/codex.ts +303 -0
  138. package/src/sessions/parsers/hermes.ts +248 -0
  139. package/src/sessions/parsers/openclaw.ts +496 -0
  140. package/src/sessions/parsers/shared.ts +184 -0
  141. package/src/sessions/sanitize.ts +222 -0
  142. package/src/sessions/service.ts +1533 -0
  143. package/src/sessions/setup.ts +477 -0
  144. package/src/sessions/sources.ts +518 -0
  145. package/src/sessions/state.ts +118 -0
  146. package/src/sessions/types.ts +457 -0
  147. package/src/store/migrations/031-runtime-independent-vectors.ts +29 -0
  148. package/src/store/migrations/032-vector-runtime-callers.ts +17 -0
  149. package/src/store/migrations/index.ts +4 -0
  150. package/src/store/sqlite/adapter.ts +76 -16
  151. package/src/store/sqlite/scoped-index.ts +9 -0
  152. package/src/store/types.ts +11 -1
  153. package/src/store/vector/lazy.ts +46 -43
  154. package/src/store/vector/runtime-compat.ts +651 -0
  155. package/src/store/vector/sqlite-vec.ts +20 -2
  156. package/src/store/vector/status.ts +276 -35
  157. package/src/store/vector/types.ts +2 -0
  158. package/src/store/vector/variant-search.ts +71 -23
  159. package/src/store/vector/variants.ts +49 -14
  160. package/browser-extension/artifacts/gno-browser-clipper-v2.5.1.zip.sha256 +0 -1
@@ -0,0 +1,52 @@
1
+ import type {
2
+ VectorPartitionStatus,
3
+ VectorRuntimeStatus,
4
+ } from "../store/vector/status";
5
+
6
+ const shortId = (id: string): string => id.slice(0, 12);
7
+
8
+ function runtimeLine(runtime: VectorRuntimeStatus): string {
9
+ const label = runtime.label ? ` (${runtime.label})` : "";
10
+ if (runtime.state === "vectors")
11
+ return ` This runtime${label} reads ${shortId(runtime.partition ?? "")}`;
12
+ if (runtime.state === "unavailable")
13
+ return ` This runtime${label} uses lexical retrieval only: ${runtime.reason}`;
14
+ return " This runtime has not resolved a partition yet; run a query or `gno embed`";
15
+ }
16
+
17
+ /**
18
+ * The caller's runtime, then one line per partition. The healthy case (one
19
+ * current partition this runtime reads) prints nothing, so default status
20
+ * output is unchanged.
21
+ */
22
+ export function formatVectorPartitionLines(
23
+ partitions?: VectorPartitionStatus[],
24
+ runtime?: VectorRuntimeStatus
25
+ ): string[] {
26
+ if (!partitions?.length) return [];
27
+ const [only] = partitions;
28
+ if (
29
+ partitions.length === 1 &&
30
+ only?.retrieval &&
31
+ !only.legacy &&
32
+ !only.incompatibleRuntimes.length
33
+ )
34
+ return [];
35
+ const lines = ["Vector partitions:"];
36
+ if (runtime) lines.push(runtimeLine(runtime));
37
+ for (const p of partitions) {
38
+ const role = p.retrieval
39
+ ? " (used by this runtime's retrieval)"
40
+ : p.droppable
41
+ ? ` (drop with: gno vec drop ${shortId(p.id)})`
42
+ : "";
43
+ lines.push(
44
+ ` ${p.retrieval ? "*" : " "} ${shortId(p.id)} ${p.state}${p.legacy ? " legacy" : ""}, ${p.owners} chunks, ${p.provenance}${role}`
45
+ );
46
+ if (p.compatibleRuntimes.length)
47
+ lines.push(` read by: ${p.compatibleRuntimes.join("; ")}`);
48
+ if (p.incompatibleRuntimes.length)
49
+ lines.push(` incompatible: ${p.incompatibleRuntimes.join("; ")}`);
50
+ }
51
+ return lines;
52
+ }
@@ -1,3 +1,5 @@
1
+ import type { Database } from "bun:sqlite";
2
+
1
3
  import type { EmbeddingPort } from "../llm/types";
2
4
  /**
3
5
  * Shared embedding backlog processor.
@@ -12,21 +14,26 @@ import type {
12
14
  VectorStatsPort,
13
15
  } from "../store/vector";
14
16
  import type { VectorVariantStore } from "../store/vector/variants";
17
+ import type { AcquireWriteTurn } from "./retry";
15
18
 
16
19
  import {
17
20
  assertInferenceActive,
18
21
  isBackgroundInference,
19
22
  } from "../llm/inference-scope";
23
+ import { formatDocForEmbedding } from "../pipeline/contextual";
20
24
  import { err, ok } from "../store/types";
25
+ import {
26
+ embeddingPartitionIdentity,
27
+ recordReferenceRuntime,
28
+ resolveRuntimePartition,
29
+ } from "../store/vector/runtime-compat";
21
30
  import { getVectorStatsDatabase } from "../store/vector/stats";
22
31
  import { createVectorVariantStore } from "../store/vector/variants";
23
- import {
24
- getEmbeddingFingerprint,
25
- getVariantModelFingerprint,
26
- } from "./fingerprint";
32
+ import { getEmbeddingFingerprint } from "./fingerprint";
27
33
  import {
28
34
  chunkRetryKey,
29
35
  embedAndStoreBatch,
36
+ inWriteTurn,
30
37
  MAX_EMBED_CHUNK_ATTEMPTS,
31
38
  } from "./retry";
32
39
  import { embedVariantBacklog } from "./variant-backlog";
@@ -47,6 +54,14 @@ export interface EmbedBacklogDeps {
47
54
  variantStore?: VectorVariantStore;
48
55
  /** Recheck the effective runtime identity after asynchronous inference. */
49
56
  identityStillCurrent?: () => boolean;
57
+ /**
58
+ * Write gate for callers that do not already hold the shared writer lease
59
+ * (the resident scheduler): taken around each page's writes and released
60
+ * after. null means another writer holds it; the pass stops as deferred.
61
+ */
62
+ acquireWriteTurn?: AcquireWriteTurn;
63
+ /** Explicit confirmation to build a separate vector partition (never implied by --yes). */
64
+ allowNewPartition?: boolean;
50
65
  }
51
66
 
52
67
  export interface EmbedBacklogResult {
@@ -59,6 +74,8 @@ export interface EmbedBacklogResult {
59
74
  contentionErrors?: number;
60
75
  /** Error message if vec index sync failed (embeddings stored, but search may be stale) */
61
76
  syncError?: string;
77
+ /** The pass stopped early because another writer held the write gate. */
78
+ deferred?: boolean;
62
79
  }
63
80
 
64
81
  interface Cursor {
@@ -78,7 +95,18 @@ export async function embedBacklog(
78
95
  deps: EmbedBacklogDeps
79
96
  ): Promise<StoreResult<EmbedBacklogResult>> {
80
97
  assertInferenceActive();
81
- const prepared = await prepareEmbeddingBacklog(deps);
98
+ if (deps.acquireWriteTurn && !deps.variantStore) {
99
+ // Model loading stays outside the write turn; preparation's partition
100
+ // writes below take one like every other background write.
101
+ const initialized = await deps.embedPort.init();
102
+ if (!initialized.ok) return err("INTERNAL", initialized.error.message);
103
+ }
104
+ const turn = await inWriteTurn(deps.acquireWriteTurn, () =>
105
+ prepareEmbeddingBacklog(deps)
106
+ );
107
+ if (turn.deferred)
108
+ return ok({ embedded: 0, errors: 0, contentionErrors: 0, deferred: true });
109
+ const prepared = turn.value;
82
110
  if (!prepared.ok) return prepared;
83
111
  deps = prepared.value;
84
112
  if (deps.variantStore) return embedVariantBacklog(deps, deps.variantStore);
@@ -201,7 +229,10 @@ export async function embedBacklog(
201
229
  embedFingerprint,
202
230
  identityStillCurrent: deps.identityStillCurrent,
203
231
  statsPort,
232
+ acquireWriteTurn: deps.acquireWriteTurn,
204
233
  });
234
+ if (batchStoreResult.deferred)
235
+ return ok({ embedded, errors, contentionErrors, deferred: true });
205
236
  embedded += batchStoreResult.embedded;
206
237
  errors += batchStoreResult.errors;
207
238
  contentionErrors += batchStoreResult.contentionErrors;
@@ -225,7 +256,12 @@ export async function embedBacklog(
225
256
  // Sync vec index once at end if any vec0 writes failed
226
257
  let syncError: string | undefined;
227
258
  if (vectorIndex.vecDirty) {
228
- const syncResult = await vectorIndex.syncVecIndex();
259
+ const turn = await inWriteTurn(deps.acquireWriteTurn, () =>
260
+ vectorIndex.syncVecIndex()
261
+ );
262
+ if (turn.deferred)
263
+ return ok({ embedded, errors, contentionErrors, deferred: true });
264
+ const syncResult = turn.value;
229
265
  if (syncResult.ok) {
230
266
  const { added, removed } = syncResult.value;
231
267
  if (added > 0 || removed > 0) {
@@ -248,6 +284,59 @@ export async function embedBacklog(
248
284
  }
249
285
  }
250
286
 
287
+ function formatEstimate(ms: number): string {
288
+ const seconds = Math.max(1, Math.round(ms / 1000));
289
+ if (seconds < 90) return `about ${seconds} s`;
290
+ const minutes = Math.round(seconds / 60);
291
+ return minutes < 90
292
+ ? `about ${minutes} min`
293
+ : `about ${(minutes / 60).toFixed(1)} h`;
294
+ }
295
+
296
+ /** Time this port on a few current chunks when no compatibility sample ran. */
297
+ async function measureEmbedRate(
298
+ db: Database,
299
+ port: EmbeddingPort
300
+ ): Promise<number | undefined> {
301
+ const inputs = db
302
+ .query<{ text: string; title: string | null }, []>(`
303
+ SELECT c.text, d.title FROM documents d
304
+ JOIN content_chunks c ON c.mirror_hash = d.mirror_hash
305
+ WHERE d.active = 1 ORDER BY d.id, c.seq LIMIT 8
306
+ `)
307
+ .all()
308
+ .map((row) =>
309
+ formatDocForEmbedding(row.text, row.title ?? undefined, port.modelUri)
310
+ );
311
+ if (!inputs.length) return undefined;
312
+ const startedAt = performance.now();
313
+ const result = await port.embedBatch(inputs);
314
+ return result.ok
315
+ ? (performance.now() - startedAt) / inputs.length
316
+ : undefined;
317
+ }
318
+
319
+ /** R3: a fork names itself, its full size and a measured estimate before it runs. */
320
+ async function separatePartitionMessage(
321
+ db: Database,
322
+ port: EmbeddingPort,
323
+ reason: string,
324
+ msPerChunk: number | undefined
325
+ ): Promise<string> {
326
+ const chunks = db
327
+ .query<{ count: number }, []>(`
328
+ SELECT count(*) AS count FROM documents d
329
+ JOIN content_chunks c ON c.mirror_hash = d.mirror_hash WHERE d.active = 1
330
+ `)
331
+ .get()!.count;
332
+ const rate = msPerChunk ?? (await measureEmbedRate(db, port));
333
+ const estimate =
334
+ rate === undefined
335
+ ? "no estimate: embedding could not be timed"
336
+ : `estimated ${formatEstimate(rate * chunks)} at the measured ${Math.round(rate)} ms per chunk`;
337
+ return `Embedding would build a separate vector partition (${reason}). It re-embeds all ${chunks} chunks (${estimate}). Confirm with \`gno embed --new-partition\`; --yes alone does not confirm.`;
338
+ }
339
+
251
340
  /** Resolve authority before counts, dry runs, forced work, or early returns. */
252
341
  export async function prepareEmbeddingBacklog(
253
342
  deps: EmbedBacklogDeps
@@ -259,19 +348,36 @@ export async function prepareEmbeddingBacklog(
259
348
  const initialized = await deps.embedPort.init();
260
349
  if (!initialized.ok) return err("INTERNAL", initialized.error.message);
261
350
  const identity = deps.embedPort.getIdentity?.();
262
- if (identity) {
351
+ const primary = embeddingPartitionIdentity(deps.embedPort);
352
+ if (identity && primary) {
263
353
  const identitySnapshot = JSON.stringify(identity);
264
- const dimensions = deps.embedPort.dimensions();
265
- const variantStore = await createVectorVariantStore(db, {
266
- model: deps.modelUri,
267
- modelFingerprint: getVariantModelFingerprint(
268
- { modelUri: deps.modelUri, dimensions },
269
- identity
270
- ),
271
- contextSize: identity.contextSize,
272
- truncationPolicy: identity.truncationPolicy,
273
- dimensions,
274
- });
354
+ const dimensions = primary.dimensions;
355
+ const resolved = await resolveRuntimePartition(
356
+ db,
357
+ deps.embedPort,
358
+ primary
359
+ );
360
+ if (resolved.blocked && !deps.allowNewPartition)
361
+ return err(
362
+ "VECTOR_PARTITION_FORK",
363
+ await separatePartitionMessage(
364
+ db,
365
+ deps.embedPort,
366
+ resolved.blocked.reason,
367
+ resolved.msPerChunk
368
+ )
369
+ );
370
+ const variantStore = await createVectorVariantStore(
371
+ db,
372
+ resolved.blocked?.separate ?? resolved.identity,
373
+ identity.runtimeLabel
374
+ );
375
+ if (resolved.blocked || resolved.verdict === "unverified") {
376
+ // Vectors without a current owner cannot be measured; they must not
377
+ // survive to be reused by the runtime that becomes the reference.
378
+ variantStore.collectGarbage();
379
+ recordReferenceRuntime(db, variantStore.partitionId, identity);
380
+ }
275
381
  variantStore.selectForEmbedding();
276
382
  return ok({
277
383
  ...deps,
@@ -36,16 +36,19 @@ export function getEmbeddingFingerprint(
36
36
  .digest("hex");
37
37
  }
38
38
 
39
- /** Partition provenance combines actual weights/runtime with the unchanged formatter policy. */
39
+ /**
40
+ * Partition identity: actual weights plus the formatter policy. Runtime details
41
+ * (Bun, native binding, backend, threads) are provenance, never identity; the
42
+ * measured compatibility check decides whether a runtime may share vectors.
43
+ */
40
44
  export function getVariantModelFingerprint(
41
45
  input: EmbeddingFingerprintInput,
42
- identity: { modelFingerprint: string; runtimeFingerprint: string }
46
+ identity: { modelFingerprint: string }
43
47
  ): string {
44
48
  return new Bun.CryptoHasher("sha256")
45
49
  .update(
46
50
  JSON.stringify([
47
51
  identity.modelFingerprint,
48
- identity.runtimeFingerprint,
49
52
  getEmbeddingFingerprint(input),
50
53
  ])
51
54
  )
@@ -15,6 +15,28 @@ import { getVectorStatsDatabase } from "../store/vector/stats";
15
15
  import { embedTextsWithRecovery } from "./batch";
16
16
 
17
17
  export const MAX_EMBED_CHUNK_ATTEMPTS = 2;
18
+
19
+ /**
20
+ * Write gate for callers that do not already hold the shared writer lease
21
+ * (the resident scheduler). Returns a release, or null when another writer
22
+ * holds the lease.
23
+ */
24
+ export type AcquireWriteTurn = () => Promise<(() => Promise<void>) | null>;
25
+
26
+ /** Run `write` inside one write turn, or report that the gate is held elsewhere. */
27
+ export async function inWriteTurn<T>(
28
+ acquire: AcquireWriteTurn | undefined,
29
+ write: () => Promise<T>
30
+ ): Promise<{ deferred: true } | { deferred: false; value: T }> {
31
+ if (!acquire) return { deferred: false, value: await write() };
32
+ const release = await acquire();
33
+ if (!release) return { deferred: true };
34
+ try {
35
+ return { deferred: false, value: await write() };
36
+ } finally {
37
+ await release();
38
+ }
39
+ }
18
40
  export const MAX_EMBED_FAILURE_SAMPLES = 5;
19
41
 
20
42
  /** Total upsert attempts (initial + retries) when persistence hits SQLITE_BUSY/LOCKED. */
@@ -44,6 +66,8 @@ export interface EmbedStoreBatchResult {
44
66
  suggestion?: string;
45
67
  batchFailed: boolean;
46
68
  batchError?: string;
69
+ /** The write gate was held elsewhere; nothing was persisted. */
70
+ deferred?: boolean;
47
71
  }
48
72
 
49
73
  // fn-127 integration: CLI consumers (src/cli/commands/embed.ts,
@@ -173,6 +197,8 @@ export async function embedAndStoreBatch(params: {
173
197
  identityStillCurrent?: () => boolean;
174
198
  /** Test seam: override contention-retry delays in milliseconds. */
175
199
  delays?: number[];
200
+ /** Gate held only around persistence, never around inference. */
201
+ acquireWriteTurn?: AcquireWriteTurn;
176
202
  }): Promise<EmbedStoreBatchResult> {
177
203
  const { embedPort, vectorIndex, items, modelUri, embedFingerprint } = params;
178
204
  const db = params.statsPort && getVectorStatsDatabase(params.statsPort);
@@ -282,35 +308,48 @@ export async function embedAndStoreBatch(params: {
282
308
  );
283
309
  });
284
310
  };
285
- const storeResult = await upsertVectorsWithContentionRetry(
286
- {
287
- upsertVectors: async (rows) => {
288
- if (vectorIndex.upsertVectorsChecked) {
289
- const result = await vectorIndex.upsertVectorsChecked(
290
- rows,
291
- (candidates) => {
292
- committedRows = checkpoint(candidates);
293
- return committedRows;
294
- }
295
- );
296
- if (!result.ok) return result;
297
- written = result.value;
298
- return ok(undefined);
299
- }
300
- if (db)
301
- return err("INVALID_INPUT", "Atomic vector checkpoint unavailable");
302
- const valid = checkpoint(rows);
303
- const result = await vectorIndex.upsertVectors(valid);
304
- if (result.ok) {
305
- written = valid.length;
306
- committedRows = valid;
307
- }
308
- return result;
311
+ const turn = await inWriteTurn(params.acquireWriteTurn, () =>
312
+ upsertVectorsWithContentionRetry(
313
+ {
314
+ upsertVectors: async (rows) => {
315
+ if (vectorIndex.upsertVectorsChecked) {
316
+ const result = await vectorIndex.upsertVectorsChecked(
317
+ rows,
318
+ (candidates) => {
319
+ committedRows = checkpoint(candidates);
320
+ return committedRows;
321
+ }
322
+ );
323
+ if (!result.ok) return result;
324
+ written = result.value;
325
+ return ok(undefined);
326
+ }
327
+ if (db)
328
+ return err("INVALID_INPUT", "Atomic vector checkpoint unavailable");
329
+ const valid = checkpoint(rows);
330
+ const result = await vectorIndex.upsertVectors(valid);
331
+ if (result.ok) {
332
+ written = valid.length;
333
+ committedRows = valid;
334
+ }
335
+ return result;
336
+ },
309
337
  },
310
- },
311
- vectors,
312
- params.delays
338
+ vectors,
339
+ params.delays
340
+ )
313
341
  );
342
+ if (turn.deferred)
343
+ return {
344
+ embedded: 0,
345
+ errors: 0,
346
+ contentionErrors: 0,
347
+ retryItems: [],
348
+ errorSamples: [],
349
+ batchFailed: false,
350
+ deferred: true,
351
+ };
352
+ const storeResult = turn.value;
314
353
  if (!storeResult.ok) {
315
354
  if (isUpsertLockContention(storeResult.error)) {
316
355
  return {
@@ -8,6 +8,7 @@ import {
8
8
  } from "../llm/inference-scope";
9
9
  import { err, ok } from "../store/types";
10
10
  import { getVectorStatsDatabase } from "../store/vector/stats";
11
+ import { inWriteTurn } from "./retry";
11
12
  import { variantBacklogPage } from "./variant-plan";
12
13
  import { embedVariantBatch } from "./variant-retry";
13
14
 
@@ -59,7 +60,9 @@ export async function embedVariantBacklog(
59
60
  owners,
60
61
  identityStillCurrent,
61
62
  force: deps.force,
63
+ acquireWriteTurn: deps.acquireWriteTurn,
62
64
  });
65
+ if (result.deferred) return ok({ ...total, deferred: true });
63
66
  total.embedded += result.embedded;
64
67
  total.errors += result.errors;
65
68
  total.contentionErrors += result.contentionErrors;
@@ -83,16 +86,18 @@ export async function embedVariantBacklog(
83
86
  // Capture the epoch before checking completeness; activate rechecks under write lock.
84
87
  const epoch = store.epoch();
85
88
  if (identityStillCurrent() && !store.pending({ limit: 1 }).length) {
86
- try {
87
- if (!store.isActive()) store.syncIndex();
88
- if (!identityStillCurrent()) return ok(total);
89
- store.activate(epoch);
90
- } catch (cause) {
91
- return ok({
92
- ...total,
93
- syncError: cause instanceof Error ? cause.message : String(cause),
94
- });
95
- }
89
+ const turn = await inWriteTurn(deps.acquireWriteTurn, async () => {
90
+ try {
91
+ if (!store.isActive()) store.syncIndex();
92
+ if (identityStillCurrent()) store.activate(epoch);
93
+ return undefined;
94
+ } catch (cause) {
95
+ return cause instanceof Error ? cause.message : String(cause);
96
+ }
97
+ });
98
+ if (turn.deferred) return ok({ ...total, deferred: true });
99
+ if (turn.value !== undefined)
100
+ return ok({ ...total, syncError: turn.value });
96
101
  }
97
102
  assertInferenceActive();
98
103
  return ok(total);
@@ -5,7 +5,9 @@ import type { VectorVariantStore } from "../store/vector/variants";
5
5
  import { err, ok } from "../store/types";
6
6
  import { embedTextsWithRecovery } from "./batch";
7
7
  import {
8
+ type AcquireWriteTurn,
8
9
  chunkRetryKey,
10
+ inWriteTurn,
9
11
  isUpsertLockContention,
10
12
  upsertVectorsWithContentionRetry,
11
13
  } from "./retry";
@@ -30,11 +32,14 @@ export async function embedVariantBatch(params: {
30
32
  identityStillCurrent: () => boolean;
31
33
  delays?: number[];
32
34
  force?: boolean;
35
+ /** Gate held only around the checkpoint write, never around inference. */
36
+ acquireWriteTurn?: AcquireWriteTurn;
33
37
  }): Promise<{
34
38
  embedded: number;
35
39
  errors: number;
36
40
  contentionErrors: number;
37
41
  retryOwners: VectorOwnerInput[];
42
+ deferred?: boolean;
38
43
  }> {
39
44
  const { store, embedPort, identityStillCurrent } = params;
40
45
  const empty = {
@@ -76,30 +81,34 @@ export async function embedVariantBatch(params: {
76
81
  .map((owner) => ({ owner, embedding: vectors.get(owner.inputHash) }));
77
82
  if (!rows.length) return { ...empty, retryOwners };
78
83
  let written = 0;
79
- const persisted = await upsertVectorsWithContentionRetry(
80
- {
81
- upsertVectors: () => {
82
- try {
83
- // A contention wait may allow document or runtime mutations. Revalidate each attempt.
84
- if (!identityStillCurrent()) return Promise.resolve(ok(undefined));
85
- const valid = rows.filter(({ owner }) => isCurrent(store, owner));
86
- store.write(valid);
87
- written = valid.length;
88
- return Promise.resolve(ok(undefined));
89
- } catch (cause) {
90
- return Promise.resolve(
91
- err(
92
- "QUERY_FAILED",
93
- cause instanceof Error ? cause.message : String(cause),
94
- cause
95
- )
96
- );
97
- }
84
+ const turn = await inWriteTurn(params.acquireWriteTurn, () =>
85
+ upsertVectorsWithContentionRetry(
86
+ {
87
+ upsertVectors: () => {
88
+ try {
89
+ // A contention wait may allow document or runtime mutations. Revalidate each attempt.
90
+ if (!identityStillCurrent()) return Promise.resolve(ok(undefined));
91
+ const valid = rows.filter(({ owner }) => isCurrent(store, owner));
92
+ store.write(valid);
93
+ written = valid.length;
94
+ return Promise.resolve(ok(undefined));
95
+ } catch (cause) {
96
+ return Promise.resolve(
97
+ err(
98
+ "QUERY_FAILED",
99
+ cause instanceof Error ? cause.message : String(cause),
100
+ cause
101
+ )
102
+ );
103
+ }
104
+ },
98
105
  },
99
- },
100
- [],
101
- params.delays
106
+ [],
107
+ params.delays
108
+ )
102
109
  );
110
+ if (turn.deferred) return { ...empty, deferred: true };
111
+ const persisted = turn.value;
103
112
  if (!persisted.ok)
104
113
  return {
105
114
  ...empty,
package/src/index.ts CHANGED
@@ -8,15 +8,35 @@
8
8
 
9
9
  import { runCli } from "./cli/run";
10
10
  import { resetModelManager } from "./llm/nodeLlamaCpp/lifecycle";
11
+ import { IMPORT_CHILD_ENV } from "./sessions/import-child-env";
11
12
 
12
13
  /**
13
- * Cleanup models and exit.
14
- * Without this, llama.cpp native threads can keep the process alive.
14
+ * End `stream` and resolve once its queued writes have reached the OS.
15
+ * process.exit() drops pending asynchronous pipe writes, so a pipe consumer
16
+ * would otherwise see output cut at the pipe buffer size. Bun reports no
17
+ * writableLength and fires an empty write's callback immediately, so end()
18
+ * is the flush that actually waits. A closed consumer (EPIPE) settles
19
+ * through the callback or the 'error' event.
20
+ */
21
+ function flushStream(stream: NodeJS.WriteStream): Promise<void> {
22
+ if (stream.destroyed || stream.writableEnded) {
23
+ return Promise.resolve();
24
+ }
25
+ return new Promise((resolve) => {
26
+ stream.once("error", () => resolve());
27
+ stream.end(() => resolve());
28
+ });
29
+ }
30
+
31
+ /**
32
+ * Cleanup models, flush output, and exit.
33
+ * Without the explicit exit, llama.cpp native threads can keep the process alive.
15
34
  */
16
35
  async function cleanupAndExit(code: number): Promise<never> {
17
36
  await resetModelManager().catch(() => {
18
37
  // Ignore cleanup errors on exit
19
38
  });
39
+ await Promise.all([flushStream(process.stdout), flushStream(process.stderr)]);
20
40
  process.exit(code);
21
41
  }
22
42
 
@@ -37,6 +57,14 @@ process.on("SIGINT", () => {
37
57
  });
38
58
  });
39
59
 
60
+ // A compiled executable re-run as the session import child (see
61
+ // src/sessions/import-child.ts) serves that one request instead of the CLI.
62
+ if (process.env[IMPORT_CHILD_ENV] === "1") {
63
+ const { runImportChild } = await import("./sessions/import-child");
64
+ await runImportChild();
65
+ await cleanupAndExit(0);
66
+ }
67
+
40
68
  // Await module completion so pending piped stdin keeps Windows Bun alive.
41
69
  await runCli(process.argv)
42
70
  .then((code) => cleanupAndExit(interruptExitCode || code))
@@ -15,6 +15,7 @@ import {
15
15
  fileIdentity,
16
16
  fingerprintModel,
17
17
  fingerprintRuntime,
18
+ runtimeLabel,
18
19
  } from "./embedding-identity";
19
20
  import { NativeWorkerError } from "./errors";
20
21
  import { checkEvaluation } from "./evaluation";
@@ -174,6 +175,7 @@ export class NativeDispatcher {
174
175
  gpu: llama.gpu,
175
176
  cpuMathCores: llama.cpuMathCores,
176
177
  }),
178
+ runtimeLabel: runtimeLabel(llama.gpu),
177
179
  },
178
180
  },
179
181
  };
@@ -31,3 +31,45 @@ export function fingerprintRuntime(settings: Record<string, unknown>): string {
31
31
  )
32
32
  .digest("hex");
33
33
  }
34
+
35
+ /** Environment that shapes the native runtime identity of an embedding. */
36
+ const RUNTIME_ENV = [
37
+ "GNO_LLAMA_GPU",
38
+ "NODE_LLAMA_CPP_GPU",
39
+ "GNO_EMBED_CONTEXT_SIZE",
40
+ "GNO_EMBED_CONTEXTS",
41
+ "GNO_EMBED_THREADS",
42
+ ] as const;
43
+
44
+ /**
45
+ * What a process can know about its embedding runtime without loading the
46
+ * model: status looks up the identity a caller with this key last resolved.
47
+ */
48
+ export function runtimeCallerKey(
49
+ model: string,
50
+ env: NodeJS.ProcessEnv = process.env
51
+ ): string {
52
+ return new Bun.CryptoHasher("sha256")
53
+ .update(
54
+ JSON.stringify([
55
+ model,
56
+ Bun.version,
57
+ nativePackage.version,
58
+ process.platform,
59
+ process.arch,
60
+ RUNTIME_ENV.map((name) => env[name] ?? null),
61
+ ])
62
+ )
63
+ .digest("hex");
64
+ }
65
+
66
+ const BACKEND_LABELS: Record<string, string> = {
67
+ cuda: "CUDA",
68
+ metal: "Metal",
69
+ vulkan: "Vulkan",
70
+ };
71
+
72
+ /** Readable provenance for status output; identity stays the fingerprint. */
73
+ export function runtimeLabel(gpu: string | false): string {
74
+ return `${gpu ? (BACKEND_LABELS[gpu] ?? gpu) : "CPU"}, Bun ${Bun.version}`;
75
+ }
@@ -112,6 +112,7 @@ export const EmbeddingIdentitySchema = z.strictObject({
112
112
  truncationPolicy: z.string().min(1),
113
113
  modelFingerprint: z.string().regex(/^[a-f0-9]{64}$/),
114
114
  runtimeFingerprint: z.string().regex(/^[a-f0-9]{64}$/),
115
+ runtimeLabel: z.string().min(1),
115
116
  });
116
117
  const metadata = z.strictObject({
117
118
  dimensions: id.optional(),
package/src/llm/types.ts CHANGED
@@ -95,7 +95,10 @@ export interface EmbeddingIdentity {
95
95
  /** Versioned policy including the effective token limit. */
96
96
  truncationPolicy: string;
97
97
  modelFingerprint: string;
98
+ /** Runtime provenance (Bun, binding, backend, threads); not vector identity. */
98
99
  runtimeFingerprint: string;
100
+ /** Readable provenance such as "CUDA, Bun 1.4.2". */
101
+ runtimeLabel?: string;
99
102
  }
100
103
 
101
104
  export interface EmbeddingPort {