@gmickel/gno 2.6.0 → 2.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (89) hide show
  1. package/README.md +1 -1
  2. package/assets/skill/SKILL.md +5 -3
  3. package/assets/skill/cli-reference.md +9 -2
  4. package/assets/skill/mcp-reference.md +2 -1
  5. package/assets/spa-production.json.gz +0 -0
  6. package/browser-extension/artifacts/{gno-browser-clipper-v2.6.0.zip → gno-browser-clipper-v2.7.0.zip} +0 -0
  7. package/browser-extension/artifacts/gno-browser-clipper-v2.7.0.zip.sha256 +1 -0
  8. package/browser-extension/dist/manifest.json +1 -1
  9. package/package.json +1 -1
  10. package/spec/cli.md +109 -18
  11. package/spec/mcp.md +36 -2
  12. package/spec/output-schemas/ask.schema.json +1 -1
  13. package/spec/output-schemas/capture-receipt.schema.json +1 -1
  14. package/spec/output-schemas/doctor.schema.json +88 -0
  15. package/spec/output-schemas/error.schema.json +11 -2
  16. package/spec/output-schemas/get.schema.json +1 -1
  17. package/spec/output-schemas/mcp-capture-result.schema.json +1 -2
  18. package/spec/output-schemas/memory-remember.schema.json +2 -2
  19. package/spec/output-schemas/multi-get.schema.json +4 -1
  20. package/spec/output-schemas/peek.schema.json +2 -9
  21. package/spec/output-schemas/resident-status.schema.json +22 -0
  22. package/spec/output-schemas/search-result.schema.json +1 -1
  23. package/spec/output-schemas/search-results.schema.json +1 -1
  24. package/spec/output-schemas/status.schema.json +98 -0
  25. package/src/cli/commands/doctor.ts +54 -20
  26. package/src/cli/commands/embed.ts +41 -3
  27. package/src/cli/commands/query.ts +5 -0
  28. package/src/cli/commands/status.ts +63 -5
  29. package/src/cli/commands/vec.ts +54 -0
  30. package/src/cli/detach.ts +29 -1
  31. package/src/cli/errors.ts +13 -9
  32. package/src/cli/program.ts +53 -1
  33. package/src/core/capture-sync.ts +9 -2
  34. package/src/core/host-paths.ts +31 -0
  35. package/src/core/memory-remember.ts +4 -3
  36. package/src/core/shutdown-budget.ts +6 -0
  37. package/src/core/vector-partition-status.ts +52 -0
  38. package/src/embed/backlog.ts +124 -18
  39. package/src/embed/fingerprint.ts +6 -3
  40. package/src/embed/retry.ts +66 -27
  41. package/src/embed/variant-backlog.ts +15 -10
  42. package/src/embed/variant-retry.ts +31 -22
  43. package/src/index.ts +21 -2
  44. package/src/llm/native-worker/dispatcher.ts +2 -0
  45. package/src/llm/native-worker/embedding-identity.ts +42 -0
  46. package/src/llm/native-worker/protocol.ts +1 -0
  47. package/src/llm/types.ts +3 -0
  48. package/src/mcp/context.ts +9 -0
  49. package/src/mcp/resources/index.ts +6 -5
  50. package/src/mcp/tool-descriptions-core.ts +1 -1
  51. package/src/mcp/tools/capture.ts +1 -3
  52. package/src/mcp/tools/index.ts +11 -4
  53. package/src/mcp/tools/memory-remember.ts +1 -1
  54. package/src/mcp/tools/status.ts +4 -0
  55. package/src/pipeline/hybrid.ts +37 -7
  56. package/src/pipeline/vsearch.ts +14 -2
  57. package/src/serve/embed-scheduler.ts +133 -19
  58. package/src/serve/host-path-redaction.ts +79 -0
  59. package/src/serve/public/components/sessions/SessionSearch.tsx +2 -2
  60. package/src/serve/public/globals.built.css +1 -1
  61. package/src/serve/public/hooks/use-api.ts +17 -2
  62. package/src/serve/public/lib/request-intent.ts +8 -0
  63. package/src/serve/public/{components/sessions → lib}/snippet.tsx +2 -3
  64. package/src/serve/public/pages/Dashboard.tsx +12 -9
  65. package/src/serve/public/pages/DocView.tsx +10 -5
  66. package/src/serve/public/pages/DocumentEditor.tsx +91 -14
  67. package/src/serve/public/pages/Search.tsx +1 -41
  68. package/src/serve/resident-runtime.ts +26 -1
  69. package/src/serve/resident-status.ts +13 -1
  70. package/src/serve/server.ts +10 -9
  71. package/src/serve/status-model.ts +16 -0
  72. package/src/serve/status.ts +2 -0
  73. package/src/serve/watch-reconciliation-shared.ts +3 -0
  74. package/src/serve/watch-service-events.ts +3 -2
  75. package/src/serve/watch-service-run-flush.ts +35 -2
  76. package/src/serve/watch-service.ts +5 -0
  77. package/src/store/migrations/031-runtime-independent-vectors.ts +29 -0
  78. package/src/store/migrations/032-vector-runtime-callers.ts +17 -0
  79. package/src/store/migrations/index.ts +4 -0
  80. package/src/store/sqlite/adapter.ts +22 -1
  81. package/src/store/types.ts +11 -1
  82. package/src/store/vector/lazy.ts +46 -43
  83. package/src/store/vector/runtime-compat.ts +651 -0
  84. package/src/store/vector/sqlite-vec.ts +20 -2
  85. package/src/store/vector/status.ts +276 -35
  86. package/src/store/vector/types.ts +2 -0
  87. package/src/store/vector/variant-search.ts +71 -23
  88. package/src/store/vector/variants.ts +49 -14
  89. package/browser-extension/artifacts/gno-browser-clipper-v2.6.0.zip.sha256 +0 -1
@@ -41,7 +41,11 @@ import {
41
41
  type GlobalOptions,
42
42
  parseGlobalOptions,
43
43
  } from "./context";
44
- import { DETACHED_CHILD_FLAG } from "./detach";
44
+ import {
45
+ DETACHED_CHILD_FLAG,
46
+ formatBackgroundIssue,
47
+ residentIssues,
48
+ } from "./detach";
45
49
  import { CliError } from "./errors";
46
50
  import {
47
51
  assertFormatSupported,
@@ -3544,6 +3548,10 @@ function wireManagementCommands(program: Command): void {
3544
3548
  .option("--model <uri>", "embedding model URI")
3545
3549
  .option("--batch-size <num>", "batch size", "32")
3546
3550
  .option("--force", "regenerate all embeddings")
3551
+ .option(
3552
+ "--new-partition",
3553
+ "confirm building a separate vector partition for an incompatible runtime"
3554
+ )
3547
3555
  .option("--dry-run", "show what would be done")
3548
3556
  .option("--json", "JSON output")
3549
3557
  ).action(
@@ -3567,6 +3575,7 @@ function wireManagementCommands(program: Command): void {
3567
3575
  force: Boolean(cmdOpts.force),
3568
3576
  dryRun: Boolean(cmdOpts.dryRun),
3569
3577
  yes: globals.yes,
3578
+ newPartition: Boolean(cmdOpts.newPartition),
3570
3579
  json: format === "json",
3571
3580
  verbose: globals.verbose,
3572
3581
  offline: globals.offline,
@@ -3675,6 +3684,43 @@ function wireVecCommands(program: Command): void {
3675
3684
  );
3676
3685
  });
3677
3686
 
3687
+ // vec drop
3688
+ addWriteLeaseFlags(
3689
+ vecCmd
3690
+ .command("drop <partition>")
3691
+ .description(
3692
+ "Drop an abandoned shadow vector partition (id prefix from gno status)"
3693
+ )
3694
+ .option("--json", "JSON output")
3695
+ ).action(async (partition: string, cmdOpts: Record<string, unknown>) => {
3696
+ const format = getFormat(cmdOpts);
3697
+ const globals = getGlobals();
3698
+
3699
+ const { vecDrop, formatVecDrop } = await import("./commands/vec");
3700
+ const lease = parseWriteLeaseFlags(cmdOpts);
3701
+ const result = await withCliWriteLease(
3702
+ {
3703
+ indexName: globals.index,
3704
+ lockWaitMs: lease.lockWaitMs,
3705
+ noWait: lease.noWait,
3706
+ },
3707
+ () =>
3708
+ vecDrop(partition, {
3709
+ configPath: globals.config,
3710
+ indexName: globals.index,
3711
+ })
3712
+ );
3713
+ throwIfWriteLeaseBusy(result, format === "json");
3714
+
3715
+ if (!result.success) {
3716
+ throw new CliError("VALIDATION", result.error);
3717
+ }
3718
+
3719
+ process.stdout.write(
3720
+ `${formatVecDrop(result, { json: format === "json" })}\n`
3721
+ );
3722
+ });
3723
+
3678
3724
  // vec rebuild
3679
3725
  addWriteLeaseFlags(
3680
3726
  vecCmd
@@ -5017,6 +5063,9 @@ async function runDaemonStatus(deps: DaemonStatusDeps): Promise<void> {
5017
5063
  } else {
5018
5064
  process.stdout.write(` (${status.log_size_bytes} bytes)\n`);
5019
5065
  }
5066
+ for (const issue of residentIssues(status)) {
5067
+ process.stdout.write(` issue ${formatBackgroundIssue(issue)}\n`);
5068
+ }
5020
5069
  if (findings) {
5021
5070
  process.stdout.write(
5022
5071
  ` findings ${formatFindingsRunStatusLine(findings)}\n`
@@ -5372,6 +5421,9 @@ async function runServeStatus(deps: ServeStatusDeps): Promise<void> {
5372
5421
  } else {
5373
5422
  process.stdout.write(` (${status.log_size_bytes} bytes)\n`);
5374
5423
  }
5424
+ for (const issue of residentIssues(status)) {
5425
+ process.stdout.write(` issue ${formatBackgroundIssue(issue)}\n`);
5426
+ }
5375
5427
 
5376
5428
  if (foreign) {
5377
5429
  // Terminal mode: emit the operator-facing warning on stderr. JSON
@@ -10,6 +10,7 @@ import type { Collection, Config } from "../config/types";
10
10
  import type { StorePort } from "../store/types";
11
11
  import type { CaptureIndexStatus } from "./capture";
12
12
 
13
+ import { buildUri } from "../app/constants";
13
14
  import {
14
15
  type CollectionSyncResult,
15
16
  defaultSyncService,
@@ -30,9 +31,14 @@ export class CaptureSyncError extends Error {
30
31
  /** The sync failure itself, without the write half of the message. */
31
32
  readonly syncError: string;
32
33
 
33
- constructor(input: { absPath: string; relPath: string; cause: string }) {
34
+ constructor(input: {
35
+ absPath: string;
36
+ relPath: string;
37
+ uri: string;
38
+ cause: string;
39
+ }) {
34
40
  super(
35
- `Capture written to ${input.absPath} but lexical sync failed: ${input.cause}. Run gno update to retry indexing.`
41
+ `Capture written to ${input.uri} but lexical sync failed: ${input.cause}. Run gno update to retry indexing.`
36
42
  );
37
43
  this.name = "CaptureSyncError";
38
44
  this.absPath = input.absPath;
@@ -82,6 +88,7 @@ export async function syncCapturedFile(
82
88
  throw new CaptureSyncError({
83
89
  absPath: input.absPath,
84
90
  relPath: input.relPath,
91
+ uri: buildUri(input.collection.name, input.relPath),
85
92
  cause,
86
93
  });
87
94
  };
@@ -0,0 +1,31 @@
1
+ /**
2
+ * Host path redaction for callers that are not on the owner's machine.
3
+ *
4
+ * Result payloads name a source file's host location in `absPath`
5
+ * (`source.absPath` on search/get results, top-level on capture, memory and
6
+ * peek receipts). A remote caller identifies documents by `uri` and
7
+ * collection-relative `relPath` instead, so every `absPath` key is removed
8
+ * before a payload leaves a remote-reachable surface.
9
+ */
10
+
11
+ const HOST_PATH_FIELD = "absPath";
12
+
13
+ const isPlainObject = (value: object): boolean => {
14
+ const proto = Object.getPrototypeOf(value) as object | null;
15
+ return proto === Object.prototype || proto === null;
16
+ };
17
+
18
+ /** Deep copy of a JSON-shaped value with every `absPath` key removed. */
19
+ export function withoutHostPaths<T>(value: T): T {
20
+ if (Array.isArray(value)) {
21
+ return value.map((item: unknown) => withoutHostPaths(item)) as T;
22
+ }
23
+ if (value === null || typeof value !== "object" || !isPlainObject(value)) {
24
+ return value;
25
+ }
26
+ const copy: Record<string, unknown> = {};
27
+ for (const [key, entry] of Object.entries(value)) {
28
+ if (key !== HOST_PATH_FIELD) copy[key] = withoutHostPaths(entry);
29
+ }
30
+ return copy as T;
31
+ }
@@ -24,6 +24,7 @@ import type {
24
24
  RememberResult,
25
25
  } from "./memory-types";
26
26
 
27
+ import { buildUri } from "../app/constants";
27
28
  import { defaultSyncService, withContentTypeRules } from "../ingestion";
28
29
  import { withWriteLock } from "./file-lock";
29
30
  import { atomicCreate } from "./file-ops";
@@ -399,7 +400,7 @@ export async function rememberFact(
399
400
  if (sync.status === "failed") {
400
401
  throw new MemoryError(
401
402
  "MEMORY_SYNC_FAILED",
402
- `Memory record written to ${absPath} but lexical sync failed: ${sync.error}. Run gno update to retry indexing.`
403
+ `Memory record written to ${buildUri(collection.name, relPath)} but lexical sync failed: ${sync.error}. Run gno update to retry indexing.`
403
404
  );
404
405
  }
405
406
  const written = (doc as { value: DocumentRow }).value;
@@ -415,13 +416,13 @@ export async function rememberFact(
415
416
  if (!projected) {
416
417
  throw new MemoryError(
417
418
  "MEMORY_SUPERSEDE_PROJECTION_FAILED",
418
- `Successor written to ${absPath} but its supersedes edge did not project${projectionErrors ? ` (${projectionErrors})` : ""}; the predecessor still reads as current. Run gno update to retry the projection.`
419
+ `Successor written to ${buildUri(collection.name, relPath)} but its supersedes edge did not project${projectionErrors ? ` (${projectionErrors})` : ""}; the predecessor still reads as current. Run gno update to retry the projection.`
419
420
  );
420
421
  }
421
422
  } else if (projectionErrors.length > 0) {
422
423
  throw new MemoryError(
423
424
  "MEMORY_SYNC_FAILED",
424
- `Memory record written to ${absPath} but typed-edge projection failed: ${projectionErrors}. Run gno update to retry indexing.`
425
+ `Memory record written to ${buildUri(collection.name, relPath)} but typed-edge projection failed: ${projectionErrors}. Run gno update to retry indexing.`
425
426
  );
426
427
  }
427
428
  const record: MemoryFact = {
@@ -4,6 +4,12 @@ export const SHUTDOWN_ABORT_MS = 5_000;
4
4
  export const SHUTDOWN_EXIT_MS = 1_000;
5
5
  // The detached parent must not be killed before it can reap its native child.
6
6
  export const RESIDENT_STOP_GRACE_MS = 12_000;
7
+ /**
8
+ * Longest synchronous SQLite busy wait a resident allows itself. A signal is
9
+ * handled only after the wait in progress ends, so this plus the shutdown
10
+ * clock must fit inside the stop grace (asserted in shutdown-budget tests).
11
+ */
12
+ export const RESIDENT_BUSY_TIMEOUT_MS = 500;
7
13
 
8
14
  /** Observe settlement without abandoning rejection handling or retaining a timer. */
9
15
  export async function settlesBy(
@@ -0,0 +1,52 @@
1
+ import type {
2
+ VectorPartitionStatus,
3
+ VectorRuntimeStatus,
4
+ } from "../store/vector/status";
5
+
6
+ const shortId = (id: string): string => id.slice(0, 12);
7
+
8
+ function runtimeLine(runtime: VectorRuntimeStatus): string {
9
+ const label = runtime.label ? ` (${runtime.label})` : "";
10
+ if (runtime.state === "vectors")
11
+ return ` This runtime${label} reads ${shortId(runtime.partition ?? "")}`;
12
+ if (runtime.state === "unavailable")
13
+ return ` This runtime${label} uses lexical retrieval only: ${runtime.reason}`;
14
+ return " This runtime has not resolved a partition yet; run a query or `gno embed`";
15
+ }
16
+
17
+ /**
18
+ * The caller's runtime, then one line per partition. The healthy case (one
19
+ * current partition this runtime reads) prints nothing, so default status
20
+ * output is unchanged.
21
+ */
22
+ export function formatVectorPartitionLines(
23
+ partitions?: VectorPartitionStatus[],
24
+ runtime?: VectorRuntimeStatus
25
+ ): string[] {
26
+ if (!partitions?.length) return [];
27
+ const [only] = partitions;
28
+ if (
29
+ partitions.length === 1 &&
30
+ only?.retrieval &&
31
+ !only.legacy &&
32
+ !only.incompatibleRuntimes.length
33
+ )
34
+ return [];
35
+ const lines = ["Vector partitions:"];
36
+ if (runtime) lines.push(runtimeLine(runtime));
37
+ for (const p of partitions) {
38
+ const role = p.retrieval
39
+ ? " (used by this runtime's retrieval)"
40
+ : p.droppable
41
+ ? ` (drop with: gno vec drop ${shortId(p.id)})`
42
+ : "";
43
+ lines.push(
44
+ ` ${p.retrieval ? "*" : " "} ${shortId(p.id)} ${p.state}${p.legacy ? " legacy" : ""}, ${p.owners} chunks, ${p.provenance}${role}`
45
+ );
46
+ if (p.compatibleRuntimes.length)
47
+ lines.push(` read by: ${p.compatibleRuntimes.join("; ")}`);
48
+ if (p.incompatibleRuntimes.length)
49
+ lines.push(` incompatible: ${p.incompatibleRuntimes.join("; ")}`);
50
+ }
51
+ return lines;
52
+ }
@@ -1,3 +1,5 @@
1
+ import type { Database } from "bun:sqlite";
2
+
1
3
  import type { EmbeddingPort } from "../llm/types";
2
4
  /**
3
5
  * Shared embedding backlog processor.
@@ -12,21 +14,26 @@ import type {
12
14
  VectorStatsPort,
13
15
  } from "../store/vector";
14
16
  import type { VectorVariantStore } from "../store/vector/variants";
17
+ import type { AcquireWriteTurn } from "./retry";
15
18
 
16
19
  import {
17
20
  assertInferenceActive,
18
21
  isBackgroundInference,
19
22
  } from "../llm/inference-scope";
23
+ import { formatDocForEmbedding } from "../pipeline/contextual";
20
24
  import { err, ok } from "../store/types";
25
+ import {
26
+ embeddingPartitionIdentity,
27
+ recordReferenceRuntime,
28
+ resolveRuntimePartition,
29
+ } from "../store/vector/runtime-compat";
21
30
  import { getVectorStatsDatabase } from "../store/vector/stats";
22
31
  import { createVectorVariantStore } from "../store/vector/variants";
23
- import {
24
- getEmbeddingFingerprint,
25
- getVariantModelFingerprint,
26
- } from "./fingerprint";
32
+ import { getEmbeddingFingerprint } from "./fingerprint";
27
33
  import {
28
34
  chunkRetryKey,
29
35
  embedAndStoreBatch,
36
+ inWriteTurn,
30
37
  MAX_EMBED_CHUNK_ATTEMPTS,
31
38
  } from "./retry";
32
39
  import { embedVariantBacklog } from "./variant-backlog";
@@ -47,6 +54,14 @@ export interface EmbedBacklogDeps {
47
54
  variantStore?: VectorVariantStore;
48
55
  /** Recheck the effective runtime identity after asynchronous inference. */
49
56
  identityStillCurrent?: () => boolean;
57
+ /**
58
+ * Write gate for callers that do not already hold the shared writer lease
59
+ * (the resident scheduler): taken around each page's writes and released
60
+ * after. null means another writer holds it; the pass stops as deferred.
61
+ */
62
+ acquireWriteTurn?: AcquireWriteTurn;
63
+ /** Explicit confirmation to build a separate vector partition (never implied by --yes). */
64
+ allowNewPartition?: boolean;
50
65
  }
51
66
 
52
67
  export interface EmbedBacklogResult {
@@ -59,6 +74,8 @@ export interface EmbedBacklogResult {
59
74
  contentionErrors?: number;
60
75
  /** Error message if vec index sync failed (embeddings stored, but search may be stale) */
61
76
  syncError?: string;
77
+ /** The pass stopped early because another writer held the write gate. */
78
+ deferred?: boolean;
62
79
  }
63
80
 
64
81
  interface Cursor {
@@ -78,7 +95,18 @@ export async function embedBacklog(
78
95
  deps: EmbedBacklogDeps
79
96
  ): Promise<StoreResult<EmbedBacklogResult>> {
80
97
  assertInferenceActive();
81
- const prepared = await prepareEmbeddingBacklog(deps);
98
+ if (deps.acquireWriteTurn && !deps.variantStore) {
99
+ // Model loading stays outside the write turn; preparation's partition
100
+ // writes below take one like every other background write.
101
+ const initialized = await deps.embedPort.init();
102
+ if (!initialized.ok) return err("INTERNAL", initialized.error.message);
103
+ }
104
+ const turn = await inWriteTurn(deps.acquireWriteTurn, () =>
105
+ prepareEmbeddingBacklog(deps)
106
+ );
107
+ if (turn.deferred)
108
+ return ok({ embedded: 0, errors: 0, contentionErrors: 0, deferred: true });
109
+ const prepared = turn.value;
82
110
  if (!prepared.ok) return prepared;
83
111
  deps = prepared.value;
84
112
  if (deps.variantStore) return embedVariantBacklog(deps, deps.variantStore);
@@ -201,7 +229,10 @@ export async function embedBacklog(
201
229
  embedFingerprint,
202
230
  identityStillCurrent: deps.identityStillCurrent,
203
231
  statsPort,
232
+ acquireWriteTurn: deps.acquireWriteTurn,
204
233
  });
234
+ if (batchStoreResult.deferred)
235
+ return ok({ embedded, errors, contentionErrors, deferred: true });
205
236
  embedded += batchStoreResult.embedded;
206
237
  errors += batchStoreResult.errors;
207
238
  contentionErrors += batchStoreResult.contentionErrors;
@@ -225,7 +256,12 @@ export async function embedBacklog(
225
256
  // Sync vec index once at end if any vec0 writes failed
226
257
  let syncError: string | undefined;
227
258
  if (vectorIndex.vecDirty) {
228
- const syncResult = await vectorIndex.syncVecIndex();
259
+ const turn = await inWriteTurn(deps.acquireWriteTurn, () =>
260
+ vectorIndex.syncVecIndex()
261
+ );
262
+ if (turn.deferred)
263
+ return ok({ embedded, errors, contentionErrors, deferred: true });
264
+ const syncResult = turn.value;
229
265
  if (syncResult.ok) {
230
266
  const { added, removed } = syncResult.value;
231
267
  if (added > 0 || removed > 0) {
@@ -248,6 +284,59 @@ export async function embedBacklog(
248
284
  }
249
285
  }
250
286
 
287
+ function formatEstimate(ms: number): string {
288
+ const seconds = Math.max(1, Math.round(ms / 1000));
289
+ if (seconds < 90) return `about ${seconds} s`;
290
+ const minutes = Math.round(seconds / 60);
291
+ return minutes < 90
292
+ ? `about ${minutes} min`
293
+ : `about ${(minutes / 60).toFixed(1)} h`;
294
+ }
295
+
296
+ /** Time this port on a few current chunks when no compatibility sample ran. */
297
+ async function measureEmbedRate(
298
+ db: Database,
299
+ port: EmbeddingPort
300
+ ): Promise<number | undefined> {
301
+ const inputs = db
302
+ .query<{ text: string; title: string | null }, []>(`
303
+ SELECT c.text, d.title FROM documents d
304
+ JOIN content_chunks c ON c.mirror_hash = d.mirror_hash
305
+ WHERE d.active = 1 ORDER BY d.id, c.seq LIMIT 8
306
+ `)
307
+ .all()
308
+ .map((row) =>
309
+ formatDocForEmbedding(row.text, row.title ?? undefined, port.modelUri)
310
+ );
311
+ if (!inputs.length) return undefined;
312
+ const startedAt = performance.now();
313
+ const result = await port.embedBatch(inputs);
314
+ return result.ok
315
+ ? (performance.now() - startedAt) / inputs.length
316
+ : undefined;
317
+ }
318
+
319
+ /** R3: a fork names itself, its full size and a measured estimate before it runs. */
320
+ async function separatePartitionMessage(
321
+ db: Database,
322
+ port: EmbeddingPort,
323
+ reason: string,
324
+ msPerChunk: number | undefined
325
+ ): Promise<string> {
326
+ const chunks = db
327
+ .query<{ count: number }, []>(`
328
+ SELECT count(*) AS count FROM documents d
329
+ JOIN content_chunks c ON c.mirror_hash = d.mirror_hash WHERE d.active = 1
330
+ `)
331
+ .get()!.count;
332
+ const rate = msPerChunk ?? (await measureEmbedRate(db, port));
333
+ const estimate =
334
+ rate === undefined
335
+ ? "no estimate: embedding could not be timed"
336
+ : `estimated ${formatEstimate(rate * chunks)} at the measured ${Math.round(rate)} ms per chunk`;
337
+ return `Embedding would build a separate vector partition (${reason}). It re-embeds all ${chunks} chunks (${estimate}). Confirm with \`gno embed --new-partition\`; --yes alone does not confirm.`;
338
+ }
339
+
251
340
  /** Resolve authority before counts, dry runs, forced work, or early returns. */
252
341
  export async function prepareEmbeddingBacklog(
253
342
  deps: EmbedBacklogDeps
@@ -259,19 +348,36 @@ export async function prepareEmbeddingBacklog(
259
348
  const initialized = await deps.embedPort.init();
260
349
  if (!initialized.ok) return err("INTERNAL", initialized.error.message);
261
350
  const identity = deps.embedPort.getIdentity?.();
262
- if (identity) {
351
+ const primary = embeddingPartitionIdentity(deps.embedPort);
352
+ if (identity && primary) {
263
353
  const identitySnapshot = JSON.stringify(identity);
264
- const dimensions = deps.embedPort.dimensions();
265
- const variantStore = await createVectorVariantStore(db, {
266
- model: deps.modelUri,
267
- modelFingerprint: getVariantModelFingerprint(
268
- { modelUri: deps.modelUri, dimensions },
269
- identity
270
- ),
271
- contextSize: identity.contextSize,
272
- truncationPolicy: identity.truncationPolicy,
273
- dimensions,
274
- });
354
+ const dimensions = primary.dimensions;
355
+ const resolved = await resolveRuntimePartition(
356
+ db,
357
+ deps.embedPort,
358
+ primary
359
+ );
360
+ if (resolved.blocked && !deps.allowNewPartition)
361
+ return err(
362
+ "VECTOR_PARTITION_FORK",
363
+ await separatePartitionMessage(
364
+ db,
365
+ deps.embedPort,
366
+ resolved.blocked.reason,
367
+ resolved.msPerChunk
368
+ )
369
+ );
370
+ const variantStore = await createVectorVariantStore(
371
+ db,
372
+ resolved.blocked?.separate ?? resolved.identity,
373
+ identity.runtimeLabel
374
+ );
375
+ if (resolved.blocked || resolved.verdict === "unverified") {
376
+ // Vectors without a current owner cannot be measured; they must not
377
+ // survive to be reused by the runtime that becomes the reference.
378
+ variantStore.collectGarbage();
379
+ recordReferenceRuntime(db, variantStore.partitionId, identity);
380
+ }
275
381
  variantStore.selectForEmbedding();
276
382
  return ok({
277
383
  ...deps,
@@ -36,16 +36,19 @@ export function getEmbeddingFingerprint(
36
36
  .digest("hex");
37
37
  }
38
38
 
39
- /** Partition provenance combines actual weights/runtime with the unchanged formatter policy. */
39
+ /**
40
+ * Partition identity: actual weights plus the formatter policy. Runtime details
41
+ * (Bun, native binding, backend, threads) are provenance, never identity; the
42
+ * measured compatibility check decides whether a runtime may share vectors.
43
+ */
40
44
  export function getVariantModelFingerprint(
41
45
  input: EmbeddingFingerprintInput,
42
- identity: { modelFingerprint: string; runtimeFingerprint: string }
46
+ identity: { modelFingerprint: string }
43
47
  ): string {
44
48
  return new Bun.CryptoHasher("sha256")
45
49
  .update(
46
50
  JSON.stringify([
47
51
  identity.modelFingerprint,
48
- identity.runtimeFingerprint,
49
52
  getEmbeddingFingerprint(input),
50
53
  ])
51
54
  )
@@ -15,6 +15,28 @@ import { getVectorStatsDatabase } from "../store/vector/stats";
15
15
  import { embedTextsWithRecovery } from "./batch";
16
16
 
17
17
  export const MAX_EMBED_CHUNK_ATTEMPTS = 2;
18
+
19
+ /**
20
+ * Write gate for callers that do not already hold the shared writer lease
21
+ * (the resident scheduler). Returns a release, or null when another writer
22
+ * holds the lease.
23
+ */
24
+ export type AcquireWriteTurn = () => Promise<(() => Promise<void>) | null>;
25
+
26
+ /** Run `write` inside one write turn, or report that the gate is held elsewhere. */
27
+ export async function inWriteTurn<T>(
28
+ acquire: AcquireWriteTurn | undefined,
29
+ write: () => Promise<T>
30
+ ): Promise<{ deferred: true } | { deferred: false; value: T }> {
31
+ if (!acquire) return { deferred: false, value: await write() };
32
+ const release = await acquire();
33
+ if (!release) return { deferred: true };
34
+ try {
35
+ return { deferred: false, value: await write() };
36
+ } finally {
37
+ await release();
38
+ }
39
+ }
18
40
  export const MAX_EMBED_FAILURE_SAMPLES = 5;
19
41
 
20
42
  /** Total upsert attempts (initial + retries) when persistence hits SQLITE_BUSY/LOCKED. */
@@ -44,6 +66,8 @@ export interface EmbedStoreBatchResult {
44
66
  suggestion?: string;
45
67
  batchFailed: boolean;
46
68
  batchError?: string;
69
+ /** The write gate was held elsewhere; nothing was persisted. */
70
+ deferred?: boolean;
47
71
  }
48
72
 
49
73
  // fn-127 integration: CLI consumers (src/cli/commands/embed.ts,
@@ -173,6 +197,8 @@ export async function embedAndStoreBatch(params: {
173
197
  identityStillCurrent?: () => boolean;
174
198
  /** Test seam: override contention-retry delays in milliseconds. */
175
199
  delays?: number[];
200
+ /** Gate held only around persistence, never around inference. */
201
+ acquireWriteTurn?: AcquireWriteTurn;
176
202
  }): Promise<EmbedStoreBatchResult> {
177
203
  const { embedPort, vectorIndex, items, modelUri, embedFingerprint } = params;
178
204
  const db = params.statsPort && getVectorStatsDatabase(params.statsPort);
@@ -282,35 +308,48 @@ export async function embedAndStoreBatch(params: {
282
308
  );
283
309
  });
284
310
  };
285
- const storeResult = await upsertVectorsWithContentionRetry(
286
- {
287
- upsertVectors: async (rows) => {
288
- if (vectorIndex.upsertVectorsChecked) {
289
- const result = await vectorIndex.upsertVectorsChecked(
290
- rows,
291
- (candidates) => {
292
- committedRows = checkpoint(candidates);
293
- return committedRows;
294
- }
295
- );
296
- if (!result.ok) return result;
297
- written = result.value;
298
- return ok(undefined);
299
- }
300
- if (db)
301
- return err("INVALID_INPUT", "Atomic vector checkpoint unavailable");
302
- const valid = checkpoint(rows);
303
- const result = await vectorIndex.upsertVectors(valid);
304
- if (result.ok) {
305
- written = valid.length;
306
- committedRows = valid;
307
- }
308
- return result;
311
+ const turn = await inWriteTurn(params.acquireWriteTurn, () =>
312
+ upsertVectorsWithContentionRetry(
313
+ {
314
+ upsertVectors: async (rows) => {
315
+ if (vectorIndex.upsertVectorsChecked) {
316
+ const result = await vectorIndex.upsertVectorsChecked(
317
+ rows,
318
+ (candidates) => {
319
+ committedRows = checkpoint(candidates);
320
+ return committedRows;
321
+ }
322
+ );
323
+ if (!result.ok) return result;
324
+ written = result.value;
325
+ return ok(undefined);
326
+ }
327
+ if (db)
328
+ return err("INVALID_INPUT", "Atomic vector checkpoint unavailable");
329
+ const valid = checkpoint(rows);
330
+ const result = await vectorIndex.upsertVectors(valid);
331
+ if (result.ok) {
332
+ written = valid.length;
333
+ committedRows = valid;
334
+ }
335
+ return result;
336
+ },
309
337
  },
310
- },
311
- vectors,
312
- params.delays
338
+ vectors,
339
+ params.delays
340
+ )
313
341
  );
342
+ if (turn.deferred)
343
+ return {
344
+ embedded: 0,
345
+ errors: 0,
346
+ contentionErrors: 0,
347
+ retryItems: [],
348
+ errorSamples: [],
349
+ batchFailed: false,
350
+ deferred: true,
351
+ };
352
+ const storeResult = turn.value;
314
353
  if (!storeResult.ok) {
315
354
  if (isUpsertLockContention(storeResult.error)) {
316
355
  return {
@@ -8,6 +8,7 @@ import {
8
8
  } from "../llm/inference-scope";
9
9
  import { err, ok } from "../store/types";
10
10
  import { getVectorStatsDatabase } from "../store/vector/stats";
11
+ import { inWriteTurn } from "./retry";
11
12
  import { variantBacklogPage } from "./variant-plan";
12
13
  import { embedVariantBatch } from "./variant-retry";
13
14
 
@@ -59,7 +60,9 @@ export async function embedVariantBacklog(
59
60
  owners,
60
61
  identityStillCurrent,
61
62
  force: deps.force,
63
+ acquireWriteTurn: deps.acquireWriteTurn,
62
64
  });
65
+ if (result.deferred) return ok({ ...total, deferred: true });
63
66
  total.embedded += result.embedded;
64
67
  total.errors += result.errors;
65
68
  total.contentionErrors += result.contentionErrors;
@@ -83,16 +86,18 @@ export async function embedVariantBacklog(
83
86
  // Capture the epoch before checking completeness; activate rechecks under write lock.
84
87
  const epoch = store.epoch();
85
88
  if (identityStillCurrent() && !store.pending({ limit: 1 }).length) {
86
- try {
87
- if (!store.isActive()) store.syncIndex();
88
- if (!identityStillCurrent()) return ok(total);
89
- store.activate(epoch);
90
- } catch (cause) {
91
- return ok({
92
- ...total,
93
- syncError: cause instanceof Error ? cause.message : String(cause),
94
- });
95
- }
89
+ const turn = await inWriteTurn(deps.acquireWriteTurn, async () => {
90
+ try {
91
+ if (!store.isActive()) store.syncIndex();
92
+ if (identityStillCurrent()) store.activate(epoch);
93
+ return undefined;
94
+ } catch (cause) {
95
+ return cause instanceof Error ? cause.message : String(cause);
96
+ }
97
+ });
98
+ if (turn.deferred) return ok({ ...total, deferred: true });
99
+ if (turn.value !== undefined)
100
+ return ok({ ...total, syncError: turn.value });
96
101
  }
97
102
  assertInferenceActive();
98
103
  return ok(total);