@gmickel/gno 2.8.2 → 2.8.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. package/README.md +1 -1
  2. package/browser-extension/artifacts/{gno-browser-clipper-v2.8.2.zip → gno-browser-clipper-v2.8.3.zip} +0 -0
  3. package/browser-extension/artifacts/gno-browser-clipper-v2.8.3.zip.sha256 +1 -0
  4. package/browser-extension/dist/manifest.json +1 -1
  5. package/package.json +9 -1
  6. package/spec/cli.md +66 -4
  7. package/spec/mcp.md +5 -6
  8. package/src/cli/commands/daemon.ts +7 -2
  9. package/src/cli/commands/doctor.ts +42 -24
  10. package/src/cli/commands/index-cmd.ts +10 -2
  11. package/src/cli/commands/links.ts +5 -22
  12. package/src/cli/commands/shared.ts +26 -1
  13. package/src/cli/commands/status.ts +60 -0
  14. package/src/cli/commands/update.ts +6 -1
  15. package/src/cli/commands/vec.ts +42 -2
  16. package/src/cli/program.ts +2 -2
  17. package/src/config/types.ts +19 -0
  18. package/src/converters/adapters/markitdownTs/adapter.ts +1 -52
  19. package/src/converters/adapters/shared/ooxml-protection.ts +58 -0
  20. package/src/converters/adapters/xlsx/adapter.ts +132 -0
  21. package/src/converters/adapters/xlsx/sheet-markdown.ts +312 -0
  22. package/src/converters/registry.ts +62 -6
  23. package/src/converters/versions.ts +2 -0
  24. package/src/core/audit-freshness.ts +1 -1
  25. package/src/core/audit-outside-index.ts +48 -3
  26. package/src/core/change-diff.ts +16 -4
  27. package/src/core/link-destination-parse.ts +21 -4
  28. package/src/core/link-inventory-markdown.ts +23 -3
  29. package/src/core/links.ts +74 -16
  30. package/src/core/network-boundary-inventory.ts +18 -0
  31. package/src/index.ts +10 -0
  32. package/src/ingestion/document-metadata.ts +256 -0
  33. package/src/ingestion/file-child-env.ts +9 -0
  34. package/src/ingestion/file-child.ts +106 -0
  35. package/src/ingestion/file-processor.ts +546 -0
  36. package/src/ingestion/frontmatter.ts +6 -2
  37. package/src/ingestion/index.ts +1 -0
  38. package/src/ingestion/prepare-convert.ts +32 -0
  39. package/src/ingestion/prepare-file.ts +272 -0
  40. package/src/ingestion/prepare-worker.ts +43 -0
  41. package/src/ingestion/record-container.ts +7 -2
  42. package/src/ingestion/strip.ts +71 -8
  43. package/src/ingestion/sync-options.ts +7 -1
  44. package/src/ingestion/sync.ts +207 -400
  45. package/src/ingestion/types.ts +18 -0
  46. package/src/llm/native-worker/embedding-identity.ts +15 -3
  47. package/src/mcp/http-transport.ts +11 -0
  48. package/src/mcp/tools/links.ts +10 -38
  49. package/src/serve/resident-request.ts +19 -0
  50. package/src/serve/resident-runtime.ts +84 -8
  51. package/src/serve/routes/links.ts +6 -24
  52. package/src/serve/server.ts +3 -2
  53. package/src/sessions/config-refresh.ts +3 -2
  54. package/src/store/sqlite/graph-similarity.ts +5 -13
  55. package/src/store/vector/partition-index.ts +100 -0
  56. package/src/store/vector/stored-vectors.ts +20 -15
  57. package/browser-extension/artifacts/gno-browser-clipper-v2.8.2.zip.sha256 +0 -1
@@ -185,9 +185,18 @@ export interface SyncOptions {
185
185
  /** Conversion limits override */
186
186
  limits?: {
187
187
  maxBytes?: number;
188
+ /** Per-file conversion wall-clock budget (config `conversion.timeoutMs`). */
188
189
  timeoutMs?: number;
189
190
  maxOutputChars?: number;
191
+ /** Process memory ceiling while a file converts (`conversion.maxMemoryMb`). */
192
+ maxMemoryMb?: number;
190
193
  };
194
+ /**
195
+ * Internal: called once when one file's conversion has run longer than
196
+ * `SLOW_CONVERSION_NOTICE_MS` (or half its budget, if sooner), naming the
197
+ * file while it is still running.
198
+ */
199
+ onSlowConversion?: (event: SlowConversionEvent) => void;
191
200
  /**
192
201
  * Max concurrent file processing (default: 1).
193
202
  * Higher values improve throughput but increase memory pressure.
@@ -207,6 +216,15 @@ export interface SyncOptions {
207
216
  sourceAvailability?: "any" | "local";
208
217
  }
209
218
 
219
+ /** A file whose conversion is taking longer than the notice threshold. */
220
+ export interface SlowConversionEvent {
221
+ collection: string;
222
+ /** Collection-relative path of the file being converted. */
223
+ relPath: string;
224
+ elapsedMs: number;
225
+ budgetMs: number;
226
+ }
227
+
210
228
  export type ContentTypeSource =
211
229
  | "frontmatter-type"
212
230
  | "prefix"
@@ -1,9 +1,21 @@
1
1
  // Bun has no inode/ctime identity API. Check replacement during hashing/loading.
2
2
  import { stat } from "node:fs/promises";
3
3
 
4
- const nativePackage: { version: string } = await Bun.file(
5
- new URL("../package.json", import.meta.resolve("node-llama-cpp"))
6
- ).json();
4
+ /**
5
+ * node-llama-cpp's version, part of the embedding runtime identity. A
6
+ * standalone compiled executable ships no node-llama-cpp package (it cannot
7
+ * run local inference), so the lookup must not stop it from loading.
8
+ */
9
+ const readNativePackage = async (): Promise<{ version: string }> => {
10
+ try {
11
+ return (await Bun.file(
12
+ new URL("../package.json", import.meta.resolve("node-llama-cpp"))
13
+ ).json()) as { version: string };
14
+ } catch {
15
+ return { version: "unavailable" };
16
+ }
17
+ };
18
+ const nativePackage = await readNativePackage();
7
19
 
8
20
  export async function fileIdentity(path: string): Promise<string> {
9
21
  const info = await stat(path, { bigint: true });
@@ -17,6 +17,7 @@ import { MCP_SERVER_NAME, VERSION } from "../app/constants";
17
17
  import { EgressDeniedError } from "../core/egress-enforcement";
18
18
  import { httpMcpRequestNamespace } from "../core/request-receipts";
19
19
  import { withInferenceScope } from "../llm/inference-scope";
20
+ import { remoteSafeSessionsError } from "../sessions/types";
20
21
  import { createMcpServerSurface, type ToolContext } from "./context";
21
22
  import {
22
23
  enforceHttpMcpEgress,
@@ -44,6 +45,8 @@ const POLICY_CHANGED_SSE = new TextEncoder().encode(
44
45
  export interface HttpMcpTransportRuntime extends HttpMcpSessionRuntime {
45
46
  readonly authorizationEpoch?: string;
46
47
  readonly isShuttingDown: boolean;
48
+ /** Adopt config-file collection edits; see ResidentRuntime.refreshConfig. */
49
+ refreshConfig?(): Promise<void>;
47
50
  admitRequest(signal?: AbortSignal): ResidentRequestHandle | null;
48
51
  }
49
52
 
@@ -281,6 +284,14 @@ export class HttpMcpTransport {
281
284
  if (!MCP_HTTP_METHODS.has(request.method)) return methodNotAllowed();
282
285
  if (this.#closed || this.#runtime.isShuttingDown)
283
286
  return jsonRpcError(503, -32_000, "Resident runtime is unavailable");
287
+ // Before admission: adopting config-file collection edits moves the
288
+ // policy epoch, which would void a request already admitted.
289
+ try {
290
+ await this.#runtime.refreshConfig?.();
291
+ } catch (error) {
292
+ const typed = remoteSafeSessionsError(error);
293
+ return jsonRpcError(500, -32_000, `${typed.code}: ${typed.message}`);
294
+ }
284
295
  if (!(await this.#acquireCapacity(request.signal))) {
285
296
  if (this.#closed || this.#runtime.isShuttingDown)
286
297
  return jsonRpcError(503, -32_000, "Resident runtime is unavailable");
@@ -26,7 +26,7 @@ import { normalizeCollectionName } from "../../core/validation";
26
26
  import { getActivePreset } from "../../llm/registry";
27
27
  import { createVectorIndexPort } from "../../store/vector";
28
28
  import {
29
- readStoredDocumentVectors,
29
+ readSimilaritySourceVectors,
30
30
  resolveStoredVectorSource,
31
31
  similarityHitDocuments,
32
32
  storedVectorSearchOptions,
@@ -415,55 +415,27 @@ export function handleSimilar(
415
415
  const preset = getActivePreset(ctx.config);
416
416
  const modelUri = preset.embed;
417
417
 
418
- // Stored chunk vectors from the active partition (NO model loading required)
418
+ // Similarity source vector (first chunk, unit length) from the active
419
+ // partition (NO model loading required)
419
420
  const db = ctx.store.getRawDb();
420
421
  const source = resolveStoredVectorSource(db, modelUri);
421
422
 
422
- let vectors: Float32Array[];
423
+ let embedding: Float32Array | undefined;
423
424
  try {
424
- vectors =
425
- readStoredDocumentVectors(db, source, [
426
- { id: doc.id, mirrorHash: doc.mirrorHash },
427
- ]).get(doc.id) ?? [];
425
+ embedding = readSimilaritySourceVectors(db, source, [
426
+ { id: doc.id, mirrorHash: doc.mirrorHash },
427
+ ]).get(doc.id);
428
428
  } catch (e) {
429
429
  throw new Error(
430
430
  `Invalid stored embedding data: ${e instanceof Error ? e.message : String(e)}`
431
431
  );
432
432
  }
433
- const first = vectors[0];
434
- if (!first) {
433
+ if (!embedding) {
435
434
  throw new Error(
436
435
  `${MCP_ERRORS.NOT_FOUND.code}: Document has no embeddings. Run: gno embed`
437
436
  );
438
437
  }
439
-
440
- // Compute average embedding from all chunks
441
- const dimensions = first.length;
442
- const avgEmbedding = new Float32Array(dimensions);
443
- for (const embedding of vectors) {
444
- if (embedding.length !== dimensions) {
445
- throw new Error(
446
- `Invalid stored embedding data: Inconsistent embedding dimensions: expected ${dimensions}, got ${embedding.length}`
447
- );
448
- }
449
- for (let i = 0; i < dimensions; i++) {
450
- const current = avgEmbedding[i] ?? 0;
451
- avgEmbedding[i] = current + (embedding[i] ?? 0) / vectors.length;
452
- }
453
- }
454
-
455
- // Normalize the average embedding for cosine similarity
456
- let norm = 0;
457
- for (let i = 0; i < dimensions; i++) {
458
- const val = avgEmbedding[i] ?? 0;
459
- norm += val * val;
460
- }
461
- norm = Math.sqrt(norm);
462
- if (norm > 0) {
463
- for (let i = 0; i < dimensions; i++) {
464
- avgEmbedding[i] = (avgEmbedding[i] ?? 0) / norm;
465
- }
466
- }
438
+ const dimensions = embedding.length;
467
439
 
468
440
  // Create vector index for search
469
441
  const vectorResult = await createVectorIndexPort(db, {
@@ -496,7 +468,7 @@ export function handleSimilar(
496
468
  // Search for similar documents (larger pool for filtering)
497
469
  const candidateLimit = Math.min(limit * 20, 200);
498
470
  const searchResult = await vectorIndex.searchNearest(
499
- avgEmbedding,
471
+ embedding,
500
472
  candidateLimit,
501
473
  storedVectorSearchOptions(source)
502
474
  );
@@ -2,6 +2,23 @@ import type { ResidentRuntime } from "./resident-runtime";
2
2
  /** Admission and bounded-reader boundary for resident REST reads. */
3
3
 
4
4
  import { withInferenceScope } from "../llm/inference-scope";
5
+ import { sessionsErrorResponse } from "./routes/sessions";
6
+
7
+ /**
8
+ * Adopt config-file collection edits before a request is admitted (adoption
9
+ * moves the policy epoch, which would void an admitted request). Returns the
10
+ * error response when the config cannot be read or is bound elsewhere.
11
+ */
12
+ export async function refreshResidentConfig(
13
+ runtime: Pick<ResidentRuntime, "refreshConfig">
14
+ ): Promise<Response | null> {
15
+ try {
16
+ await runtime.refreshConfig?.();
17
+ return null;
18
+ } catch (error) {
19
+ return sessionsErrorResponse(error);
20
+ }
21
+ }
5
22
 
6
23
  function unavailableResponse(): Response {
7
24
  return Response.json(
@@ -87,6 +104,8 @@ export async function handleResidentRead(
87
104
  request: Request | undefined,
88
105
  operation: (signal: AbortSignal) => Promise<Response> | Response
89
106
  ): Promise<Response> {
107
+ const refreshFailure = await refreshResidentConfig(runtime);
108
+ if (refreshFailure) return refreshFailure;
90
109
  const admitted = runtime.admitRequest(request?.signal);
91
110
  if (!admitted) return unavailableResponse();
92
111
 
@@ -51,10 +51,16 @@ import {
51
51
  } from "../core/shutdown-budget";
52
52
  import { acquireCliWriteLease } from "../core/write-lease";
53
53
  import { defaultSyncService, withContentTypeRules } from "../ingestion";
54
+ import { disposeFileProcessor } from "../ingestion/file-processor";
54
55
  import { withOwnedInferenceScope } from "../llm/inference-scope";
55
56
  import { getActivePreset } from "../llm/registry";
56
57
  import { createToolContext, Mutex } from "../mcp/context";
57
58
  import { watchedCollections } from "../sessions/config";
59
+ import {
60
+ adoptServedConfig,
61
+ readInstanceConfig,
62
+ } from "../sessions/config-refresh";
63
+ import { remoteSafeSessionsError } from "../sessions/types";
58
64
  import { SqliteAdapter } from "../store/sqlite/adapter";
59
65
  import {
60
66
  createServerContext,
@@ -78,6 +84,8 @@ import {
78
84
  import { CollectionWatchService as DefaultCollectionWatchService } from "./watch-service";
79
85
 
80
86
  const OWNER_LOCK_TIMEOUT_MS = 0;
87
+ /** How often an idle resident re-reads its config file for collection edits. */
88
+ const CONFIG_REFRESH_INTERVAL_MS = 2_000;
81
89
 
82
90
  export type ResidentMode = "serve" | "daemon";
83
91
 
@@ -146,6 +154,15 @@ export interface ResidentRuntime {
146
154
  provider: (() => HttpMcpTransportStatus) | null
147
155
  ): void;
148
156
  setPolicySessionInvalidator(invalidator: (() => Promise<void>) | null): void;
157
+ /**
158
+ * Adopt collection (and context) edits made to the config file while the
159
+ * resident runs, with the sessions refresh's binding checks: rejects when
160
+ * the file is unreadable or bound to another index, leaving the served
161
+ * config as it was. Other settings keep their restart behaviour. Callers
162
+ * run it before admitting a request (an adoption moves the policy epoch).
163
+ * Optional only for test doubles; the real runtime always provides it.
164
+ */
165
+ refreshConfig?(): Promise<void>;
149
166
  admitRequest(signal?: AbortSignal): ResidentRequestHandle | null;
150
167
  withModelLease<T>(operation: () => Promise<T>): Promise<T>;
151
168
  markContentMutation(): void;
@@ -472,17 +489,56 @@ export async function startResidentRuntime(
472
489
  startBackgroundWork: (operation) => backgroundWork.start(operation),
473
490
  });
474
491
 
492
+ const setServedConfig = (config: Config): void => {
493
+ ctxHolder.config = config;
494
+ ctxHolder.current = { ...ctxHolder.current, config };
495
+ ctxHolder.watchService?.updateCollections(
496
+ watchedCollections(config),
497
+ withContentTypeRules({}, config)
498
+ );
499
+ };
500
+ const instance = {
501
+ configPath: actualConfigPath,
502
+ indexName: canonicalizeIndexName(options.index ?? DEFAULT_INDEX_NAME),
503
+ };
504
+ let configRefresh: Promise<void> | undefined;
505
+ const refreshConfig = (): Promise<void> => {
506
+ configRefresh ??= (async () => {
507
+ const file = await readInstanceConfig(
508
+ instance,
509
+ deps.loadConfig ?? loadConfig
510
+ );
511
+ const served = ctxHolder.config;
512
+ const next: Config = {
513
+ ...served,
514
+ collections: file.collections,
515
+ contexts: file.contexts,
516
+ };
517
+ if (Bun.deepEquals(next, served)) return;
518
+ await adoptServedConfig(
519
+ {
520
+ ...instance,
521
+ store,
522
+ config: served,
523
+ setConfig: setServedConfig,
524
+ invalidateEgressPolicy: async () => {
525
+ await ctxHolder.invalidateEgressPolicy?.();
526
+ },
527
+ markContentMutation: () => ctxHolder.markContentMutation?.(),
528
+ markIndexMutation: () => ctxHolder.markIndexMutation?.(),
529
+ },
530
+ next
531
+ );
532
+ })().finally(() => {
533
+ configRefresh = undefined;
534
+ });
535
+ return configRefresh;
536
+ };
537
+
475
538
  const mcpContext = createToolContext({
476
539
  store,
477
540
  getConfig: () => ctxHolder.config,
478
- setConfig: (config) => {
479
- ctxHolder.config = config;
480
- ctxHolder.current = { ...ctxHolder.current, config };
481
- ctxHolder.watchService?.updateCollections(
482
- watchedCollections(config),
483
- withContentTypeRules({}, config)
484
- );
485
- },
541
+ setConfig: setServedConfig,
486
542
  actualConfigPath,
487
543
  indexName: canonicalizeIndexName(options.index ?? DEFAULT_INDEX_NAME),
488
544
  toolMutex,
@@ -633,6 +689,7 @@ export async function startResidentRuntime(
633
689
  setPolicySessionInvalidator(invalidator) {
634
690
  policySessionInvalidator = invalidator;
635
691
  },
692
+ refreshConfig,
636
693
  async syncAll(syncOptions = {}) {
637
694
  const request = admission.admit();
638
695
  if (!request) throw new Error("Resident runtime is shutting down");
@@ -674,8 +731,11 @@ export async function startResidentRuntime(
674
731
  dispose(closeSurface) {
675
732
  if (disposal) return disposal;
676
733
  disposed = true;
734
+ clearInterval(configRefreshTimer);
677
735
  admission.stop();
678
736
  jobManager.stop();
737
+ // A file mid-preparation fails now instead of holding shutdown.
738
+ disposeFileProcessor();
679
739
  admissionState = "draining";
680
740
  shutdownState = "graceful";
681
741
  disposal = disposeResidentResources({
@@ -725,5 +785,21 @@ export async function startResidentRuntime(
725
785
  };
726
786
  };
727
787
  mcpContext.getResidentStatus = () => runtime.getStatus();
788
+ // Without requests (a headless daemon) the watcher still follows the file.
789
+ let lastRefreshError: string | undefined;
790
+ const configRefreshTimer = setInterval(() => {
791
+ backgroundWork.start(async () => {
792
+ try {
793
+ await refreshConfig();
794
+ lastRefreshError = undefined;
795
+ } catch (error) {
796
+ const message = remoteSafeSessionsError(error).message;
797
+ if (message !== lastRefreshError)
798
+ console.warn(`gno ${mode}: config not reloaded: ${message}`);
799
+ lastRefreshError = message;
800
+ }
801
+ });
802
+ }, CONFIG_REFRESH_INTERVAL_MS);
803
+ configRefreshTimer.unref?.();
728
804
  return { success: true, runtime };
729
805
  }
@@ -8,7 +8,7 @@ import type { SqliteAdapter } from "../../store/sqlite/adapter";
8
8
  import type { ServerContext } from "../context";
9
9
 
10
10
  import {
11
- readStoredDocumentVectors,
11
+ readSimilaritySourceVectors,
12
12
  resolveStoredVectorSource,
13
13
  similarityHitDocuments,
14
14
  storedVectorSearchOptions,
@@ -313,7 +313,7 @@ export async function handleDocBacklinks(
313
313
  * ?threshold=0.5 (min similarity score 0-1, default 0.5)
314
314
  * ?crossCollection=true (search across all collections, default false)
315
315
  *
316
- * Algorithm: seq=0 embedding (fallback to first available) -> vector search -> exclude self
316
+ * Algorithm: first-chunk source vector (readSimilaritySourceVectors) -> vector search -> exclude self
317
317
  */
318
318
  export async function handleDocSimilar(
319
319
  ctx: ServerContext,
@@ -378,18 +378,14 @@ export async function handleDocSimilar(
378
378
  } satisfies SimilarDocResponse);
379
379
  }
380
380
 
381
- // Stored vector of the document's first chunk, from the active partition
381
+ // Similarity source vector (first chunk, unit length), active partition
382
382
  const db = store.getRawDb();
383
383
  const source = resolveStoredVectorSource(db, ctx.vectorIndex.model);
384
384
  let embedding: Float32Array | undefined;
385
385
  try {
386
- [embedding] =
387
- readStoredDocumentVectors(
388
- db,
389
- source,
390
- [{ id: doc.id, mirrorHash: doc.mirrorHash }],
391
- { firstChunkOnly: true }
392
- ).get(doc.id) ?? [];
386
+ embedding = readSimilaritySourceVectors(db, source, [
387
+ { id: doc.id, mirrorHash: doc.mirrorHash },
388
+ ]).get(doc.id);
393
389
  } catch (e) {
394
390
  return errorResponse(
395
391
  "RUNTIME",
@@ -410,20 +406,6 @@ export async function handleDocSimilar(
410
406
  },
411
407
  } satisfies SimilarDocResponse);
412
408
  }
413
- const dimensions = embedding.length;
414
-
415
- // Normalize embedding for cosine similarity
416
- let norm = 0;
417
- for (let i = 0; i < dimensions; i++) {
418
- const val = embedding[i] ?? 0;
419
- norm += val * val;
420
- }
421
- norm = Math.sqrt(norm);
422
- if (norm > 0) {
423
- for (let i = 0; i < dimensions; i++) {
424
- embedding[i] = (embedding[i] ?? 0) / norm;
425
- }
426
- }
427
409
 
428
410
  // Search for similar docs (request extra to account for self-exclusion, filtering)
429
411
  const candidateLimit = Math.min(limit * 20, 200);
@@ -28,7 +28,7 @@ import { withRemoteHostPathRedaction } from "./host-path-redaction";
28
28
  import { PDFJS_ASSET_CACHE_CONTROL } from "./pdfjs-assets";
29
29
  // HTML import - Bun handles bundling TSX/CSS automatically via routes
30
30
  import homepage from "./public/index.html";
31
- import { handleResidentRead } from "./resident-request";
31
+ import { handleResidentRead, refreshResidentConfig } from "./resident-request";
32
32
  import {
33
33
  handleActiveJob,
34
34
  handleAsk,
@@ -702,7 +702,8 @@ export async function startServer(
702
702
  "/api/collections": {
703
703
  GET: async () =>
704
704
  withSecurityHeaders(
705
- await handleCollections(ctxHolder.config),
705
+ (await refreshResidentConfig(runtime as ResidentRuntime)) ??
706
+ (await handleCollections(ctxHolder.config)),
706
707
  isDev
707
708
  ),
708
709
  POST: async (req: Request) => {
@@ -53,9 +53,10 @@ export async function assertInstanceBinding(
53
53
  * before it can touch this one.
54
54
  */
55
55
  export async function readInstanceConfig(
56
- instance: SessionsInstance
56
+ instance: SessionsInstance,
57
+ load: typeof loadConfig = loadConfig
57
58
  ): Promise<Config> {
58
- const loaded = await loadConfig(instance.configPath);
59
+ const loaded = await load(instance.configPath);
59
60
  if (!loaded.ok) {
60
61
  throw new SessionsError(
61
62
  "SESSIONS_RUNTIME_FAILURE",
@@ -6,7 +6,7 @@
6
6
  import type { Database } from "bun:sqlite";
7
7
 
8
8
  import {
9
- readStoredDocumentVectors,
9
+ readSimilaritySourceVectors,
10
10
  resolveStoredVectorSource,
11
11
  } from "../vector/stored-vectors";
12
12
 
@@ -27,13 +27,6 @@ export interface StoredSimilarityEdge {
27
27
  score: number;
28
28
  }
29
29
 
30
- function unit(vector: Float32Array): Float32Array {
31
- let norm = 0;
32
- for (const value of vector) norm += value * value;
33
- norm = Math.sqrt(norm);
34
- return norm > 0 ? vector.map((value) => value / norm) : vector;
35
- }
36
-
37
30
  function dot(a: Float32Array, b: Float32Array): number {
38
31
  let sum = 0;
39
32
  for (let i = 0; i < a.length; i++) sum += (a[i] ?? 0) * (b[i] ?? 0);
@@ -61,15 +54,14 @@ export function storedSimilarityEdges(
61
54
  ORDER BY docid
62
55
  `)
63
56
  .all(JSON.stringify(docids));
64
- const vectors = readStoredDocumentVectors(
57
+ const vectors = readSimilaritySourceVectors(
65
58
  db,
66
59
  resolveStoredVectorSource(db, model),
67
- documents,
68
- { firstChunkOnly: true }
60
+ documents
69
61
  );
70
62
  const nodes = documents.flatMap((document) => {
71
- const vector = vectors.get(document.id)?.[0];
72
- return vector ? [{ ...document, vector: unit(vector) }] : [];
63
+ const vector = vectors.get(document.id);
64
+ return vector ? [{ ...document, vector }] : [];
73
65
  });
74
66
  const edges: StoredSimilarityEdge[] = [];
75
67
  for (const node of nodes) {
@@ -0,0 +1,100 @@
1
+ /**
2
+ * Maintenance of an activated vector partition's vec0 index
3
+ * (`vec_v1_<partition>`), which mirrors the partition's `vector_variants`.
4
+ * `gno vec sync` and `gno vec rebuild` use it on indexes embedded since 2.7.
5
+ * Neither changes authority, owners or variants, so the variant epoch stays.
6
+ *
7
+ * @module src/store/vector/partition-index
8
+ */
9
+ import type { Database } from "bun:sqlite";
10
+
11
+ export interface StoredPartition {
12
+ partitionId: string;
13
+ dimensions: number;
14
+ }
15
+
16
+ const tableName = (partition: StoredPartition): string =>
17
+ `vec_v1_${partition.partitionId}`;
18
+
19
+ const createTable = (db: Database, partition: StoredPartition): void => {
20
+ db.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS ${tableName(partition)} USING vec0(
21
+ variant_id INTEGER PRIMARY KEY,
22
+ embedding FLOAT[${partition.dimensions}] distance_metric=cosine
23
+ )`);
24
+ };
25
+
26
+ const insertVariants = (
27
+ db: Database,
28
+ partition: StoredPartition,
29
+ rows: Iterable<{ variantId: number; embedding: Uint8Array }>
30
+ ): number => {
31
+ let inserted = 0;
32
+ for (const row of rows) {
33
+ db.run(
34
+ `INSERT INTO ${tableName(partition)}(variant_id, embedding) VALUES (?, ?)`,
35
+ [row.variantId, row.embedding]
36
+ );
37
+ inserted += 1;
38
+ }
39
+ return inserted;
40
+ };
41
+
42
+ /**
43
+ * Remove index rows without a matching variant (orphans and stale
44
+ * embeddings), then add the variants the index lacks. Requires sqlite-vec.
45
+ */
46
+ export function syncPartitionIndex(
47
+ db: Database,
48
+ partition: StoredPartition
49
+ ): { added: number; removed: number } {
50
+ const table = tableName(partition);
51
+ return db
52
+ .transaction(() => {
53
+ createTable(db, partition);
54
+ const stale = db
55
+ .query<{ variantId: number }, [string]>(`
56
+ SELECT x.variant_id AS variantId FROM ${table} x
57
+ LEFT JOIN vector_variants v ON v.variant_id = x.variant_id
58
+ AND v.partition_id = ?
59
+ WHERE v.variant_id IS NULL OR x.embedding != v.embedding
60
+ `)
61
+ .all(partition.partitionId);
62
+ for (const { variantId } of stale) {
63
+ db.run(`DELETE FROM ${table} WHERE variant_id = ?`, [variantId]);
64
+ }
65
+ const missing = db
66
+ .query<{ variantId: number; embedding: Uint8Array }, [string]>(`
67
+ SELECT v.variant_id AS variantId, v.embedding FROM vector_variants v
68
+ LEFT JOIN ${table} x ON x.variant_id = v.variant_id
69
+ WHERE v.partition_id = ? AND x.variant_id IS NULL
70
+ `)
71
+ .all(partition.partitionId);
72
+ return {
73
+ added: insertVariants(db, partition, missing),
74
+ removed: stale.length,
75
+ };
76
+ })
77
+ .immediate();
78
+ }
79
+
80
+ /** Drop and repopulate the index from the partition's variants. */
81
+ export function rebuildPartitionIndex(
82
+ db: Database,
83
+ partition: StoredPartition
84
+ ): number {
85
+ return db
86
+ .transaction(() => {
87
+ db.exec(`DROP TABLE IF EXISTS ${tableName(partition)}`);
88
+ createTable(db, partition);
89
+ return insertVariants(
90
+ db,
91
+ partition,
92
+ db
93
+ .query<{ variantId: number; embedding: Uint8Array }, [string]>(
94
+ "SELECT variant_id AS variantId, embedding FROM vector_variants WHERE partition_id = ?"
95
+ )
96
+ .iterate(partition.partitionId)
97
+ );
98
+ })
99
+ .immediate();
100
+ }
@@ -53,25 +53,30 @@ interface PartitionVectorRow {
53
53
  embedding: Uint8Array;
54
54
  }
55
55
 
56
+ const unitVector = (vector: Float32Array): Float32Array => {
57
+ let norm = 0;
58
+ for (const value of vector) norm += value * value;
59
+ norm = Math.sqrt(norm);
60
+ return norm > 0 ? vector.map((value) => value / norm) : vector;
61
+ };
62
+
56
63
  /**
57
- * Chunk vectors of current active documents, ordered by chunk seq, keyed by
58
- * document id. Partition vectors count only while their owner still yields
59
- * the stored input (the rule vector search applies). `firstChunkOnly` keeps
60
- * the lowest-seq vector per document.
64
+ * The similarity source vector of each current active document, keyed by
65
+ * document id: the stored vector of its first chunk (lowest seq whose vector
66
+ * is current), unit-normalized. Every similarity surface (CLI, MCP, REST and
67
+ * graph edges) uses this one rule, so scores agree. Partition vectors count
68
+ * only while their owner still yields the stored input (the rule vector
69
+ * search applies).
61
70
  */
62
- export function readStoredDocumentVectors(
71
+ export function readSimilaritySourceVectors(
63
72
  db: Database,
64
73
  source: StoredVectorSource,
65
- documents: StoredVectorDocument[],
66
- options: { firstChunkOnly?: boolean } = {}
67
- ): Map<number, Float32Array[]> {
68
- const vectors = new Map<number, Float32Array[]>();
74
+ documents: StoredVectorDocument[]
75
+ ): Map<number, Float32Array> {
76
+ const vectors = new Map<number, Float32Array>();
69
77
  const add = (documentId: number, blob: Uint8Array): void => {
70
- const existing = vectors.get(documentId);
71
- if (existing && options.firstChunkOnly) return;
72
- const embedding = decodeEmbedding(blob);
73
- if (existing) existing.push(embedding);
74
- else vectors.set(documentId, [embedding]);
78
+ if (!vectors.has(documentId))
79
+ vectors.set(documentId, unitVector(decodeEmbedding(blob)));
75
80
  };
76
81
  if (documents.length === 0) return vectors;
77
82
 
@@ -95,7 +100,7 @@ export function readStoredDocumentVectors(
95
100
  JSON.stringify(documents.map((document) => document.id))
96
101
  );
97
102
  for (const row of rows) {
98
- if (options.firstChunkOnly && vectors.has(row.documentId)) continue;
103
+ if (vectors.has(row.documentId)) continue;
99
104
  const input = formatDocForEmbedding(
100
105
  row.text,
101
106
  row.title ?? undefined,
@@ -1 +0,0 @@
1
- 7bc830276cb923a482b662b980208b0978bf8ddbfbade0873e67d9757cf9c989 gno-browser-clipper-v2.8.2.zip