@gmickel/gno 2.1.0 → 2.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -117,7 +117,7 @@ gno daemon --detach # headless indexing + resident MCP gateway
117
117
 
118
118
  <!-- public-truth:current-version -->
119
119
 
120
- > Current source version: **v2.1.0**. See [CHANGELOG.md](./CHANGELOG.md).
120
+ > Current source version: **v2.2.0**. See [CHANGELOG.md](./CHANGELOG.md).
121
121
 
122
122
  <!-- /public-truth -->
123
123
 
@@ -0,0 +1 @@
1
+ c2367d97e5599b41454becfa7d7e09ba9c1d5962096ce9e4f89b8794f9929f39 gno-browser-clipper-v2.2.0.zip
@@ -21,5 +21,5 @@
21
21
  "content_security_policy": {
22
22
  "extension_pages": "script-src 'self'; object-src 'none'; connect-src http://127.0.0.1:*"
23
23
  },
24
- "version": "2.1.0"
24
+ "version": "2.2.0"
25
25
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@gmickel/gno",
3
- "version": "2.1.0",
3
+ "version": "2.2.0",
4
4
  "description": "Local semantic search for your documents. Index Markdown, PDF, and Office files with hybrid BM25 + vector search.",
5
5
  "keywords": [
6
6
  "embeddings",
package/spec/cli.md CHANGED
@@ -224,6 +224,14 @@ setup and emits `setup-profile-result@1.0`.
224
224
 
225
225
  Display index status and health information.
226
226
 
227
+ Embedding backlog follows the last verified partition for the selected model
228
+ when exact-input storage is authoritative, counting pending document/chunk
229
+ owners. Per-collection chunk totals remain deduplicated by canonical chunk;
230
+ embedded counts require matching current inputs for every active owner within
231
+ that collection. Status reads persisted identity and coverage without loading
232
+ models. Legacy storage remains the fallback before variant authority; ambiguous
233
+ older partition selection is conservative until a normal embed records it.
234
+
227
235
  **Synopsis:**
228
236
 
229
237
  ```bash
@@ -2305,6 +2313,18 @@ MUST fail without an artifact. Encrypted CLI export requires `--passphrase`;
2305
2313
  the local API requires `encryptionPassphrase`. Neither sends that input to
2306
2314
  gno.sh. Export is a local operation, not hosted activation or deletion.
2307
2315
 
2316
+ New exports MUST assign each note a random lowercase UUIDv4 persisted in
2317
+ `publish-identities.json` beside the resolved config file (`--config` and
2318
+ `GNO_CONFIG_DIR` apply). The private registry keys canonical collection roots
2319
+ and source-relative paths, outside the disposable index. The collection root
2320
+ MUST be accessible for canonical-path resolution. V1 notes emit `id`;
2321
+ V2 spaces emit `noteIds` matching decrypted reader cards’ `noteId` values.
2322
+ Legacy artifacts without IDs remain valid. Registry corruption or write failure
2323
+ MUST fail export rather than reset IDs. Content/title edits, published route/slug
2324
+ changes, and index rebuilds preserve IDs for unchanged source paths. Moving a
2325
+ source file or collection root starts a new identity; no move inference or
2326
+ source-Markdown mutation is performed. Registry paths MUST NOT enter artifacts.
2327
+
2308
2328
  Public V1 spaces MUST carry a `manifest` conforming to
2309
2329
  [`publish-artifact.schema.json`](./output-schemas/publish-artifact.schema.json).
2310
2330
  The manifest contains schema version `1.0`, a deterministic projection
@@ -2334,7 +2354,8 @@ insufficient). Asset-free exports omit `assets` /
2334
2354
 
2335
2355
  Secret-link and invite-only V1 spaces MUST NOT contain a manifest or agent
2336
2356
  capability field. Encrypted V2 spaces MUST contain only ciphertext parameters,
2337
- the opaque secret token, route slug, source type, and encrypted visibility; no
2357
+ the opaque secret token, route slug, source type, encrypted visibility, and an
2358
+ optional unique `noteIds` UUIDv4 roster (1–5000 IDs); no
2338
2359
  plaintext manifest or evidence may appear outside the ciphertext. V2 builders
2339
2360
  MUST emit a closed projection, validate payload strings as non-empty bounded
2340
2361
  base64, require a positive safe-integer KDF iteration count, and bound the
@@ -173,11 +173,16 @@
173
173
  ]
174
174
  }
175
175
  },
176
+ "noteId": {
177
+ "type": "string",
178
+ "pattern": "^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$"
179
+ },
176
180
  "note": {
177
181
  "type": "object",
178
182
  "additionalProperties": false,
179
183
  "required": ["markdown", "slug", "summary", "title"],
180
184
  "properties": {
185
+ "id": { "$ref": "#/definitions/noteId" },
181
186
  "markdown": { "type": "string" },
182
187
  "metadata": { "$ref": "#/definitions/metadata" },
183
188
  "slug": { "$ref": "#/definitions/slug" },
@@ -347,6 +352,13 @@
347
352
  "visibility"
348
353
  ],
349
354
  "properties": {
355
+ "noteIds": {
356
+ "type": "array",
357
+ "minItems": 1,
358
+ "maxItems": 5000,
359
+ "uniqueItems": true,
360
+ "items": { "$ref": "#/definitions/noteId" }
361
+ },
350
362
  "encryptedPayload": {
351
363
  "type": "object",
352
364
  "additionalProperties": false,
@@ -98,6 +98,7 @@ export async function publishExport(
98
98
  const { artifact, assetSummary, warnings } = await exportPublishArtifact({
99
99
  collections,
100
100
  options: {
101
+ configPath: initResult.actualConfigPath,
101
102
  routeSlug: options.slug,
102
103
  encryptionPassphrase: options.encryptionPassphrase,
103
104
  summary: options.summary,
@@ -272,6 +272,7 @@ export async function prepareEmbeddingBacklog(
272
272
  truncationPolicy: identity.truncationPolicy,
273
273
  dimensions,
274
274
  });
275
+ variantStore.selectForEmbedding();
275
276
  return ok({
276
277
  ...deps,
277
278
  variantStore,
@@ -6,6 +6,9 @@
6
6
 
7
7
  import { MAX_PUBLISH_UPLOAD_BYTES } from "./artifact-asset-contract";
8
8
 
9
+ export const PUBLISH_NOTE_ID_PATTERN =
10
+ /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/u;
11
+
9
12
  export const MAX_PUBLISH_SLUG_LENGTH = 80;
10
13
  /**
11
14
  * Ciphertext base64 character ceiling aligned to the 100 MiB final-envelope
@@ -33,6 +36,7 @@ const SOURCE_TYPES = new Set(["collection", "note"]);
33
36
  const READER_VISIBILITIES = new Set(["invite-only", "public", "secret-link"]);
34
37
 
35
38
  export interface ValidatedPublishNote {
39
+ id?: string;
36
40
  markdown: string;
37
41
  metadata?: Record<string, string | string[]>;
38
42
  slug: string;
@@ -51,6 +55,7 @@ export interface ValidatedPublishSpaceInput {
51
55
  }
52
56
 
53
57
  export interface ValidatedEncryptedPublishInput {
58
+ noteIds?: string[];
54
59
  encryptedPayload: {
55
60
  ciphertext: string;
56
61
  iterations: number;
@@ -158,6 +163,21 @@ const projectMetadata = (
158
163
  return result;
159
164
  };
160
165
 
166
+ function requireNoteId(value: unknown): string {
167
+ if (typeof value !== "string" || !PUBLISH_NOTE_ID_PATTERN.test(value))
168
+ throw new Error("Publish note ID must be a lowercase UUIDv4");
169
+ return value;
170
+ }
171
+
172
+ function projectNoteIds(value: unknown): string[] {
173
+ if (!Array.isArray(value) || value.length < 1 || value.length > 5000)
174
+ throw new Error("noteIds must contain 1 to 5000 UUIDv4 IDs");
175
+ const ids = value.map(requireNoteId);
176
+ if (new Set(ids).size !== ids.length)
177
+ throw new Error("Duplicate publish note ID");
178
+ return ids;
179
+ }
180
+
161
181
  const projectNote = (value: unknown, index: number): ValidatedPublishNote => {
162
182
  const field = `notes[${index}]`;
163
183
  const input = requireRecord(value, field);
@@ -168,6 +188,7 @@ const projectNote = (value: unknown, index: number): ValidatedPublishNote => {
168
188
  summary: requireString(input.summary, `${field}.summary`),
169
189
  title: requireNonblankString(input.title, `${field}.title`),
170
190
  };
191
+ if (input.id !== undefined) note.id = requireNoteId(input.id);
171
192
  if (metadata !== undefined) note.metadata = metadata;
172
193
  return note;
173
194
  };
@@ -181,6 +202,9 @@ export const validateAndProjectPublishSpaceInput = (
181
202
  }
182
203
 
183
204
  const notes = input.notes.map(projectNote);
205
+ const ids = notes.flatMap((note) => (note.id ? [note.id] : []));
206
+ if (new Set(ids).size !== ids.length)
207
+ throw new Error("Duplicate publish note ID");
184
208
  const noteSlugs = new Set<string>();
185
209
  for (const note of notes) {
186
210
  if (noteSlugs.has(note.slug)) {
@@ -246,6 +270,9 @@ export const validateAndProjectEncryptedPublishInput = (
246
270
  }
247
271
 
248
272
  return {
273
+ ...(input.noteIds === undefined
274
+ ? {}
275
+ : { noteIds: projectNoteIds(input.noteIds) }),
249
276
  encryptedPayload: {
250
277
  ciphertext: requireEncryptedCiphertext(payload.ciphertext),
251
278
  iterations,
@@ -68,6 +68,7 @@ export type PublishVisibility =
68
68
  | "secret-link";
69
69
 
70
70
  export interface PublishArtifactNote {
71
+ id?: string;
71
72
  markdown: string;
72
73
  metadata?: Record<string, string | string[]>;
73
74
  slug: string;
@@ -148,6 +149,7 @@ export interface EncryptedArtifactPayload {
148
149
  }
149
150
 
150
151
  export interface EncryptedPublishArtifactSpace {
152
+ noteIds?: string[];
151
153
  encryptedPayload: EncryptedArtifactPayload;
152
154
  routeSlug: string;
153
155
  secretToken: string;
@@ -441,6 +443,7 @@ export const buildPublishArtifact = (input: {
441
443
  };
442
444
 
443
445
  export const buildEncryptedPublishArtifact = (input: {
446
+ noteIds?: string[];
444
447
  egressLineage?: EgressLineage;
445
448
  encryptedPayload: EncryptedArtifactPayload;
446
449
  requiredCapabilities?: KnownPublishRequiredCapability[];
@@ -464,6 +467,9 @@ export const buildEncryptedPublishArtifact = (input: {
464
467
  source: validated.routeSlug,
465
468
  spaces: [
466
469
  {
470
+ ...(validated.noteIds === undefined
471
+ ? {}
472
+ : { noteIds: validated.noteIds }),
467
473
  encryptedPayload: validated.encryptedPayload,
468
474
  routeSlug: validated.routeSlug,
469
475
  secretToken: validated.secretToken,
@@ -183,7 +183,7 @@ const deriveReaderPayload = (input: {
183
183
  const noteCards: ReaderNoteCard[] = input.notes.map((note) => {
184
184
  const blocks = parseMarkdownBlocks(note.markdown);
185
185
  return {
186
- noteId: `${input.routeSlug}:${note.slug}`,
186
+ noteId: note.id ?? `${input.routeSlug}:${note.slug}`,
187
187
  slug: note.slug,
188
188
  title: note.title,
189
189
  excerpt: deriveExcerpt(note.summary, blocks),
@@ -197,6 +197,9 @@ export async function finalizeEncryptedArtifact(input: {
197
197
  const artifact = buildEncryptedPublishArtifact({
198
198
  egressLineage: input.egressLineage,
199
199
  encryptedPayload: encrypted.encryptedPayload,
200
+ ...(input.notes.every((note) => note.id !== undefined)
201
+ ? { noteIds: input.notes.map((note) => note.id!) }
202
+ : {}),
200
203
  requiredCapabilities:
201
204
  assets.length > 0 ? [BUNDLED_RASTER_ASSETS_CAPABILITY] : undefined,
202
205
  routeSlug: input.routeSlug,
@@ -34,12 +34,15 @@ import {
34
34
  sanitizeNoteMarkdown,
35
35
  type NoteBuildAccumulator,
36
36
  } from "./export-attachments";
37
+ import { resolvePublishNoteIds } from "./identities";
37
38
  import {
38
39
  isPublishDisabledByFrontmatter,
39
40
  type SanitizeWarning,
40
41
  } from "./obsidian-sanitize";
41
42
 
42
43
  export interface PublishExportCoreOptions {
44
+ /** Actual loaded config file; defaults to the platform config directory. */
45
+ configPath?: string;
43
46
  encryptionPassphrase?: string;
44
47
  routeSlug?: string;
45
48
  summary?: string;
@@ -199,6 +202,7 @@ async function exportCollectionArtifact(
199
202
  preDedupRawBytes: 0,
200
203
  };
201
204
  const notes: PublishArtifactNote[] = [];
205
+ const sourceRelPaths: string[] = [];
202
206
 
203
207
  for (const doc of activeDocs) {
204
208
  if (!doc.mirrorHash) {
@@ -230,6 +234,7 @@ async function exportCollectionArtifact(
230
234
  acc.externalCount += sanitized.externalCount;
231
235
  acc.preDedupRawBytes += sanitized.preDedupRawBytes;
232
236
  acc.encodedAssetBytes += mergePayloads(acc.payloads, sanitized.payloads);
237
+ sourceRelPaths.push(doc.relPath);
233
238
  notes.push({
234
239
  markdown: sanitized.markdown,
235
240
  metadata: buildExportedMetadata(
@@ -268,6 +273,13 @@ async function exportCollectionArtifact(
268
273
  store,
269
274
  });
270
275
 
276
+ const noteIds = await resolvePublishNoteIds({
277
+ collectionRoot: collection.path,
278
+ sourceRelPaths,
279
+ configPath: options.configPath,
280
+ });
281
+ for (const [index, note] of notes.entries()) note.id = noteIds[index];
282
+
271
283
  if (visibility === "encrypted") {
272
284
  if (!options.encryptionPassphrase) {
273
285
  throw new Error(
@@ -321,6 +333,8 @@ async function exportDocumentArtifact(
321
333
 
322
334
  const collection =
323
335
  collections.find((entry) => entry.name === doc.collection) ?? null;
336
+ if (!collection)
337
+ throw new Error(`Collection not configured: ${doc.collection}`);
324
338
  const rawMarkdown = await loadDocumentMarkdown(store, doc);
325
339
  if (isPublishDisabledByFrontmatter(rawMarkdown)) {
326
340
  throw new Error(
@@ -362,7 +376,13 @@ async function exportDocumentArtifact(
362
376
  store,
363
377
  });
364
378
 
379
+ const [id] = await resolvePublishNoteIds({
380
+ collectionRoot: collection.path,
381
+ sourceRelPaths: [doc.relPath],
382
+ configPath: options.configPath,
383
+ });
365
384
  const note: PublishArtifactNote = {
385
+ id,
366
386
  markdown,
367
387
  metadata: buildExportedMetadata(doc, frontmatter, tags),
368
388
  slug,
@@ -0,0 +1,153 @@
1
+ /** Private durable publish identities, independent of the disposable index. */
2
+ // node:fs/promises — canonical paths, permissions and atomic rename have no Bun equivalents.
3
+ import {
4
+ chmod,
5
+ lstat,
6
+ mkdir,
7
+ realpath,
8
+ rename,
9
+ unlink,
10
+ } from "node:fs/promises";
11
+ // node:path — Bun has no path utilities.
12
+ import { dirname, isAbsolute, join, normalize } from "node:path";
13
+
14
+ import { getConfigPaths, toAbsolutePath } from "../config/paths";
15
+ import { acquireWriteLock, type WriteLockHandle } from "../core/file-lock";
16
+ import { PUBLISH_NOTE_ID_PATTERN } from "./artifact-validation";
17
+
18
+ const IDENTITY_LOCK_TIMEOUT_MS = 5000;
19
+ const IDENTITY_LOCK_RETRY_MS = 25;
20
+
21
+ type Registry = { version: 1; identities: Record<string, string> };
22
+
23
+ export function publishIdentityRegistryPath(configPath?: string): string {
24
+ return join(
25
+ configPath
26
+ ? dirname(toAbsolutePath(configPath))
27
+ : getConfigPaths().configDir,
28
+ "publish-identities.json"
29
+ );
30
+ }
31
+
32
+ function validateRegistry(value: unknown): Registry {
33
+ if (!value || typeof value !== "object" || Array.isArray(value))
34
+ throw new Error("Invalid registry object");
35
+ const record = value as Record<string, unknown>;
36
+ if (
37
+ record.version !== 1 ||
38
+ Object.keys(record).length !== 2 ||
39
+ !record.identities ||
40
+ typeof record.identities !== "object" ||
41
+ Array.isArray(record.identities)
42
+ )
43
+ throw new Error("Invalid registry version or identities");
44
+ const identities = record.identities as Record<string, unknown>;
45
+ const ids = new Set<string>();
46
+ for (const [key, id] of Object.entries(identities)) {
47
+ const source: unknown = JSON.parse(key);
48
+ if (
49
+ !Array.isArray(source) ||
50
+ source.length !== 2 ||
51
+ !source.every((part) => typeof part === "string" && part.length > 0) ||
52
+ typeof id !== "string" ||
53
+ !PUBLISH_NOTE_ID_PATTERN.test(id) ||
54
+ ids.has(id)
55
+ )
56
+ throw new Error("Invalid or duplicate registry identity");
57
+ ids.add(id);
58
+ }
59
+ return value as Registry;
60
+ }
61
+
62
+ async function readRegistry(path: string): Promise<Registry> {
63
+ try {
64
+ const info = await lstat(path);
65
+ if (!info.isFile() || info.isSymbolicLink())
66
+ throw new Error("Registry must be a regular private file");
67
+ await chmod(path, 0o600);
68
+ return validateRegistry(await Bun.file(path).json());
69
+ } catch (error) {
70
+ if ((error as NodeJS.ErrnoException).code === "ENOENT")
71
+ return { version: 1, identities: {} };
72
+ throw new Error(
73
+ "Cannot read publish identity registry; restore a valid backup before exporting",
74
+ { cause: error }
75
+ );
76
+ }
77
+ }
78
+
79
+ async function acquireIdentityLock(path: string): Promise<WriteLockHandle> {
80
+ const deadline = performance.now() + IDENTITY_LOCK_TIMEOUT_MS;
81
+ // Zero-wait attempts avoid blocking the JS holder on SQLite's busy timeout.
82
+ // flock -w 0 and lockf -t 0 also mean immediate acquisition or failure.
83
+ let lock = await acquireWriteLock(path, 0);
84
+ while (!lock) {
85
+ const remaining = deadline - performance.now();
86
+ if (remaining <= 0)
87
+ throw new Error("Publish identity registry is busy; retry the export");
88
+ await Bun.sleep(Math.min(IDENTITY_LOCK_RETRY_MS, remaining));
89
+ lock = await acquireWriteLock(path, 0);
90
+ }
91
+ return lock;
92
+ }
93
+
94
+ /** Allocate a batch under one OS lock; publish IDs only after the private file commits. */
95
+ export async function resolvePublishNoteIds(input: {
96
+ collectionRoot: string;
97
+ sourceRelPaths: string[];
98
+ configPath?: string;
99
+ }): Promise<string[]> {
100
+ let root: string;
101
+ try {
102
+ root = await realpath(toAbsolutePath(input.collectionRoot));
103
+ } catch (cause) {
104
+ throw new Error(
105
+ "Collection root must exist and be accessible to resolve publish identities",
106
+ { cause }
107
+ );
108
+ }
109
+ const keys = input.sourceRelPaths.map((source) => {
110
+ const rel = normalize(source).replaceAll("\\", "/");
111
+ if (
112
+ !rel ||
113
+ rel === "." ||
114
+ rel === ".." ||
115
+ rel.startsWith("../") ||
116
+ isAbsolute(rel) ||
117
+ rel.includes("\0")
118
+ )
119
+ throw new Error(
120
+ "Publish source path must be relative to its collection root"
121
+ );
122
+ return JSON.stringify([root, rel]);
123
+ });
124
+ const path = publishIdentityRegistryPath(input.configPath);
125
+ await mkdir(dirname(path), { recursive: true, mode: 0o700 });
126
+ const lock = await acquireIdentityLock(`${path}.lock`);
127
+ const temporaryPath = `${path}.tmp.${crypto.randomUUID()}`;
128
+ try {
129
+ const registry = await readRegistry(path);
130
+ let changed = false;
131
+ const ids = keys.map((key) => {
132
+ const existing = registry.identities[key];
133
+ if (existing) return existing;
134
+ const id = crypto.randomUUID();
135
+ registry.identities[key] = id;
136
+ changed = true;
137
+ return id;
138
+ });
139
+ if (changed) {
140
+ await Bun.write(temporaryPath, JSON.stringify(registry), { mode: 0o600 });
141
+ await rename(temporaryPath, path);
142
+ }
143
+ return ids;
144
+ } catch (error) {
145
+ throw new Error(
146
+ "Publish identity registry update failed; export aborted to preserve note identity",
147
+ { cause: error }
148
+ );
149
+ } finally {
150
+ await unlink(temporaryPath).catch(() => undefined);
151
+ await lock.release();
152
+ }
153
+ }
@@ -1266,7 +1266,8 @@ export async function handleEgressAuditPurge(
1266
1266
  export async function handlePublishExport(
1267
1267
  config: Config,
1268
1268
  store: SqliteAdapter,
1269
- req: Request
1269
+ req: Request,
1270
+ configPath?: string
1270
1271
  ): Promise<Response> {
1271
1272
  let body: PublishExportRequestBody;
1272
1273
  try {
@@ -1304,6 +1305,7 @@ export async function handlePublishExport(
1304
1305
  const { artifact, assetSummary, warnings } = await exportPublishArtifact({
1305
1306
  collections: config.collections,
1306
1307
  options: {
1308
+ configPath,
1307
1309
  encryptionPassphrase: body.encryptionPassphrase,
1308
1310
  routeSlug: body.slug,
1309
1311
  summary: body.summary,
@@ -802,7 +802,8 @@ export async function startServer(
802
802
  (dependencies.handlePublishExport ?? handlePublishExport)(
803
803
  ctxHolder.config,
804
804
  store,
805
- req
805
+ req,
806
+ runtime.actualConfigPath
806
807
  )
807
808
  ),
808
809
  isDev
@@ -144,6 +144,7 @@ import { getSchemaVersion, migrations, runMigrations } from "../migrations";
144
144
  import { err, ok } from "../types";
145
145
  import { getStoredEmbeddingFingerprint } from "../vector/freshness";
146
146
  import { modelTableName } from "../vector/sqlite-vec";
147
+ import { getVariantStatus } from "../vector/status";
147
148
  import {
148
149
  deleteSavedCapsuleRegistration as deleteStoredSavedCapsuleRegistration,
149
150
  getSavedCapsuleRegistration as getStoredSavedCapsuleRegistration,
@@ -5697,6 +5698,8 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
5697
5698
  options?.embedFingerprint ??
5698
5699
  (embedModel ? getStoredEmbeddingFingerprint(db, embedModel) : null);
5699
5700
 
5701
+ const variantStatus = getVariantStatus(db, options);
5702
+
5700
5703
  // Get version
5701
5704
  const versionRow = db
5702
5705
  .query<{ value: string }, []>(
@@ -5873,12 +5876,14 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
5873
5876
  errorDocuments: s.errored,
5874
5877
  chunkedDocuments: s.chunked,
5875
5878
  totalChunks: s.chunk_count,
5876
- embeddedChunks: s.embedded_count,
5879
+ embeddedChunks: variantStatus
5880
+ ? (variantStatus.embeddedByCollection.get(s.name) ?? 0)
5881
+ : s.embedded_count,
5877
5882
  })),
5878
5883
  totalDocuments: totalsRow?.total ?? 0,
5879
5884
  activeDocuments: totalsRow?.active ?? 0,
5880
5885
  totalChunks: chunkCount,
5881
- embeddingBacklog: backlogRow?.count ?? 0,
5886
+ embeddingBacklog: variantStatus?.backlog ?? backlogRow?.count ?? 0,
5882
5887
  recentErrors,
5883
5888
  lastUpdatedAt: lastUpdatedRow?.last_updated ?? null,
5884
5889
  healthy,
@@ -0,0 +1,196 @@
1
+ /** Read-only coverage of persisted variant authority; never initializes models. */
2
+ import type { Database } from "bun:sqlite";
3
+
4
+ import { getEmbeddingFingerprint } from "../../embed/fingerprint";
5
+ import { formatDocForEmbedding } from "../../pipeline/contextual";
6
+ import {
7
+ embeddingInputHash,
8
+ SELECTED_VECTOR_PARTITION_PREFIX,
9
+ } from "./variants";
10
+
11
+ interface Partition {
12
+ partition_id: string;
13
+ version: number;
14
+ model: string;
15
+ fingerprint: string;
16
+ dimensions: number;
17
+ state: string;
18
+ activated_epoch: number | null;
19
+ }
20
+
21
+ interface OwnerCoverage {
22
+ document_id: number;
23
+ partition_id: string | null;
24
+ collection: string;
25
+ mirror_hash: string;
26
+ seq: number;
27
+ text: string;
28
+ title: string | null;
29
+ input_hash: string | null;
30
+ embedding_bytes: number | null;
31
+ legacy_embedded: number;
32
+ }
33
+
34
+ /** Null retains legacy counts until a verified selection or activation exists. */
35
+ export function getVariantStatus(
36
+ db: Database,
37
+ options?: { embedModel?: string; embedFingerprint?: string }
38
+ ): { backlog: number; embeddedByCollection: Map<string, number> } | null {
39
+ return db.transaction(() => {
40
+ if (
41
+ !db
42
+ .query(
43
+ "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'vector_partitions'"
44
+ )
45
+ .get()
46
+ )
47
+ return null;
48
+ const partitions = db
49
+ .query<Partition, [string | null, string | null]>(`
50
+ SELECT partition_id, version, model, fingerprint, dimensions, state, activated_epoch
51
+ FROM vector_partitions WHERE (? IS NULL OR model = ?)
52
+ `)
53
+ .all(options?.embedModel ?? null, options?.embedModel ?? null);
54
+ const selections = db
55
+ .query<{ key: string; value: string }, [string]>(
56
+ "SELECT key, value FROM schema_meta WHERE key GLOB ?"
57
+ )
58
+ .all(`${SELECTED_VECTOR_PARTITION_PREFIX}*`);
59
+ const selected = new Map(
60
+ selections.map((row) => [
61
+ row.key.slice(SELECTED_VECTOR_PARTITION_PREFIX.length),
62
+ row.value,
63
+ ])
64
+ );
65
+ const models = new Set(partitions.map((p) => p.model));
66
+ if (options?.embedModel) models.add(options.embedModel);
67
+ else for (const model of selected.keys()) models.add(model);
68
+ const authoritativeModels = new Set<string>();
69
+ const usablePartitions = new Map<string, Partition>();
70
+ for (const model of models) {
71
+ const candidates = partitions.filter((p) => p.model === model);
72
+ const selection = selected.get(model);
73
+ const activated = candidates.some(
74
+ (p) => p.state === "active" && p.activated_epoch !== null
75
+ );
76
+ if (selection === undefined && !activated) continue;
77
+ authoritativeModels.add(model);
78
+ // Resolve one persisted identity per model, never combine alternative
79
+ // partitions of the same model. Unscoped status may accept any model.
80
+ // Stale epochs do not revoke owners whose current inputs still match.
81
+ const partition =
82
+ selection === undefined
83
+ ? candidates.length === 1
84
+ ? candidates[0]
85
+ : undefined
86
+ : candidates.find((p) => p.partition_id === selection);
87
+ if (
88
+ partition &&
89
+ partition.version === 1 &&
90
+ partition.partition_id ===
91
+ embeddingInputHash(
92
+ JSON.stringify([
93
+ partition.model,
94
+ partition.fingerprint,
95
+ partition.dimensions,
96
+ ])
97
+ ) &&
98
+ (options?.embedFingerprint === undefined ||
99
+ options.embedFingerprint ===
100
+ getEmbeddingFingerprint({
101
+ modelUri: partition.model,
102
+ dimensions: partition.dimensions,
103
+ }))
104
+ )
105
+ usablePartitions.set(partition.partition_id, partition);
106
+ }
107
+ if (authoritativeModels.size === 0) return null;
108
+ // Unscoped legacy coverage remains valid only for models that have never
109
+ // selected or activated verified authority; legacy rows cannot repair it.
110
+ const statement = db.prepare<
111
+ OwnerCoverage,
112
+ [string | null, string, string]
113
+ >(`
114
+ WITH legacy_vectors AS (
115
+ SELECT mirror_hash, seq, MAX(embedded_at) AS embedded_at
116
+ FROM content_vectors
117
+ WHERE ? IS NULL AND model NOT IN (SELECT value FROM json_each(?))
118
+ GROUP BY mirror_hash, seq
119
+ )
120
+ SELECT d.id AS document_id, o.partition_id, d.collection, d.mirror_hash, c.seq, c.text, d.title,
121
+ v.input_hash, length(v.embedding) AS embedding_bytes,
122
+ CASE WHEN lv.embedded_at >= c.created_at THEN 1 ELSE 0 END AS legacy_embedded
123
+ FROM documents d
124
+ JOIN content_chunks c ON c.mirror_hash = d.mirror_hash
125
+ LEFT JOIN vector_owners o ON o.document_id = d.id AND o.seq = c.seq
126
+ AND o.mirror_hash = d.mirror_hash
127
+ AND o.partition_id IN (SELECT value FROM json_each(?))
128
+ LEFT JOIN vector_variants v ON v.variant_id = o.variant_id
129
+ AND v.partition_id = o.partition_id
130
+ LEFT JOIN legacy_vectors lv ON lv.mirror_hash = c.mirror_hash AND lv.seq = c.seq
131
+ WHERE d.active = 1
132
+ `);
133
+ const owners = new Map<
134
+ string,
135
+ { collection: string; chunk: string; embedded: boolean }
136
+ >();
137
+ try {
138
+ for (const row of statement.iterate(
139
+ options?.embedModel ?? null,
140
+ JSON.stringify([...authoritativeModels]),
141
+ JSON.stringify([...usablePartitions.keys()])
142
+ )) {
143
+ const partition = row.partition_id
144
+ ? usablePartitions.get(row.partition_id)
145
+ : undefined;
146
+ const embedded = Boolean(
147
+ row.legacy_embedded ||
148
+ (partition &&
149
+ row.embedding_bytes ===
150
+ partition.dimensions * Float32Array.BYTES_PER_ELEMENT &&
151
+ row.input_hash ===
152
+ embeddingInputHash(
153
+ formatDocForEmbedding(
154
+ row.text,
155
+ row.title ?? undefined,
156
+ partition.model
157
+ )
158
+ ))
159
+ );
160
+ const key = `${row.document_id}:${row.seq}`;
161
+ const previous = owners.get(key);
162
+ if (previous) previous.embedded ||= embedded;
163
+ else
164
+ owners.set(key, {
165
+ collection: row.collection,
166
+ chunk: JSON.stringify([row.mirror_hash, row.seq]),
167
+ embedded,
168
+ });
169
+ }
170
+ } finally {
171
+ statement.finalize();
172
+ }
173
+ const collections = new Map<string, Map<string, boolean>>();
174
+ let backlog = 0;
175
+ for (const owner of owners.values()) {
176
+ if (!owner.embedded) backlog++;
177
+ let chunks = collections.get(owner.collection);
178
+ if (!chunks) {
179
+ chunks = new Map();
180
+ collections.set(owner.collection, chunks);
181
+ }
182
+ // Distinct collection chunks are ready only when every active owner is.
183
+ chunks.set(
184
+ owner.chunk,
185
+ (chunks.get(owner.chunk) ?? true) && owner.embedded
186
+ );
187
+ }
188
+ const embeddedByCollection = new Map<string, number>();
189
+ for (const [collection, chunks] of collections) {
190
+ let count = 0;
191
+ for (const embedded of chunks.values()) if (embedded) count++;
192
+ embeddedByCollection.set(collection, count);
193
+ }
194
+ return { backlog, embeddedByCollection };
195
+ })();
196
+ }
@@ -6,6 +6,8 @@ import type { VectorOwnerInput, VectorVariantIdentity } from "./types";
6
6
  import { formatDocForEmbedding } from "../../pipeline/contextual";
7
7
  import { decodeEmbedding, encodeEmbedding } from "./sqlite-vec";
8
8
 
9
+ export const SELECTED_VECTOR_PARTITION_PREFIX = "vector_selected_partition:";
10
+
9
11
  export function embeddingInputHash(input: string): string {
10
12
  return new Bun.CryptoHasher("sha256").update(input).digest("hex");
11
13
  }
@@ -94,6 +96,15 @@ export class VectorVariantStore {
94
96
  }).immediate();
95
97
  }
96
98
 
99
+ /** Persist only after embedding has resolved the actual runtime identity. */
100
+ selectForEmbedding(): void {
101
+ this.db.run(
102
+ `INSERT INTO schema_meta (key, value) VALUES (?, ?)
103
+ ON CONFLICT(key) DO UPDATE SET value = excluded.value, updated_at = datetime('now')`,
104
+ [SELECTED_VECTOR_PARTITION_PREFIX + this.identity.model, this.partitionId]
105
+ );
106
+ }
107
+
97
108
  epoch(): number {
98
109
  return this.db
99
110
  .query<{ epoch: number }, []>(
@@ -1 +0,0 @@
1
- 8dd833b01ebc0a0cb5747de65a9cd60b06cafcf645c2a74d697c918fe7c642d7 gno-browser-clipper-v2.1.0.zip