@gmickel/gno 2.1.0 → 2.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/browser-extension/artifacts/{gno-browser-clipper-v2.1.0.zip → gno-browser-clipper-v2.2.0.zip} +0 -0
- package/browser-extension/artifacts/gno-browser-clipper-v2.2.0.zip.sha256 +1 -0
- package/browser-extension/dist/manifest.json +1 -1
- package/package.json +1 -1
- package/spec/cli.md +22 -1
- package/spec/output-schemas/publish-artifact.schema.json +12 -0
- package/src/cli/commands/publish.ts +1 -0
- package/src/embed/backlog.ts +1 -0
- package/src/publish/artifact-validation.ts +27 -0
- package/src/publish/artifact.ts +6 -0
- package/src/publish/encrypted-export.ts +1 -1
- package/src/publish/export-attachments.ts +3 -0
- package/src/publish/export-service.ts +20 -0
- package/src/publish/identities.ts +153 -0
- package/src/serve/routes/api.ts +3 -1
- package/src/serve/server.ts +2 -1
- package/src/store/sqlite/adapter.ts +7 -2
- package/src/store/vector/status.ts +196 -0
- package/src/store/vector/variants.ts +11 -0
- package/browser-extension/artifacts/gno-browser-clipper-v2.1.0.zip.sha256 +0 -1
package/README.md
CHANGED
|
@@ -117,7 +117,7 @@ gno daemon --detach # headless indexing + resident MCP gateway
|
|
|
117
117
|
|
|
118
118
|
<!-- public-truth:current-version -->
|
|
119
119
|
|
|
120
|
-
> Current source version: **v2.
|
|
120
|
+
> Current source version: **v2.2.0**. See [CHANGELOG.md](./CHANGELOG.md).
|
|
121
121
|
|
|
122
122
|
<!-- /public-truth -->
|
|
123
123
|
|
|
Binary file
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
c2367d97e5599b41454becfa7d7e09ba9c1d5962096ce9e4f89b8794f9929f39 gno-browser-clipper-v2.2.0.zip
|
package/package.json
CHANGED
package/spec/cli.md
CHANGED
|
@@ -224,6 +224,14 @@ setup and emits `setup-profile-result@1.0`.
|
|
|
224
224
|
|
|
225
225
|
Display index status and health information.
|
|
226
226
|
|
|
227
|
+
Embedding backlog follows the last verified partition for the selected model
|
|
228
|
+
when exact-input storage is authoritative, counting pending document/chunk
|
|
229
|
+
owners. Per-collection chunk totals remain deduplicated by canonical chunk;
|
|
230
|
+
embedded counts require matching current inputs for every active owner within
|
|
231
|
+
that collection. Status reads persisted identity and coverage without loading
|
|
232
|
+
models. Legacy storage remains the fallback before variant authority; ambiguous
|
|
233
|
+
older partition selection is conservative until a normal embed records it.
|
|
234
|
+
|
|
227
235
|
**Synopsis:**
|
|
228
236
|
|
|
229
237
|
```bash
|
|
@@ -2305,6 +2313,18 @@ MUST fail without an artifact. Encrypted CLI export requires `--passphrase`;
|
|
|
2305
2313
|
the local API requires `encryptionPassphrase`. Neither sends that input to
|
|
2306
2314
|
gno.sh. Export is a local operation, not hosted activation or deletion.
|
|
2307
2315
|
|
|
2316
|
+
New exports MUST assign each note a random lowercase UUIDv4 persisted in
|
|
2317
|
+
`publish-identities.json` beside the resolved config file (`--config` and
|
|
2318
|
+
`GNO_CONFIG_DIR` apply). The private registry keys canonical collection roots
|
|
2319
|
+
and source-relative paths, outside the disposable index. The collection root
|
|
2320
|
+
MUST be accessible for canonical-path resolution. V1 notes emit `id`;
|
|
2321
|
+
V2 spaces emit `noteIds` matching decrypted reader cards’ `noteId` values.
|
|
2322
|
+
Legacy artifacts without IDs remain valid. Registry corruption or write failure
|
|
2323
|
+
MUST fail export rather than reset IDs. Content/title edits, published route/slug
|
|
2324
|
+
changes, and index rebuilds preserve IDs for unchanged source paths. Moving a
|
|
2325
|
+
source file or collection root starts a new identity; no move inference or
|
|
2326
|
+
source-Markdown mutation is performed. Registry paths MUST NOT enter artifacts.
|
|
2327
|
+
|
|
2308
2328
|
Public V1 spaces MUST carry a `manifest` conforming to
|
|
2309
2329
|
[`publish-artifact.schema.json`](./output-schemas/publish-artifact.schema.json).
|
|
2310
2330
|
The manifest contains schema version `1.0`, a deterministic projection
|
|
@@ -2334,7 +2354,8 @@ insufficient). Asset-free exports omit `assets` /
|
|
|
2334
2354
|
|
|
2335
2355
|
Secret-link and invite-only V1 spaces MUST NOT contain a manifest or agent
|
|
2336
2356
|
capability field. Encrypted V2 spaces MUST contain only ciphertext parameters,
|
|
2337
|
-
the opaque secret token, route slug, source type,
|
|
2357
|
+
the opaque secret token, route slug, source type, encrypted visibility, and an
|
|
2358
|
+
optional unique `noteIds` UUIDv4 roster (1–5000 IDs); no
|
|
2338
2359
|
plaintext manifest or evidence may appear outside the ciphertext. V2 builders
|
|
2339
2360
|
MUST emit a closed projection, validate payload strings as non-empty bounded
|
|
2340
2361
|
base64, require a positive safe-integer KDF iteration count, and bound the
|
|
@@ -173,11 +173,16 @@
|
|
|
173
173
|
]
|
|
174
174
|
}
|
|
175
175
|
},
|
|
176
|
+
"noteId": {
|
|
177
|
+
"type": "string",
|
|
178
|
+
"pattern": "^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$"
|
|
179
|
+
},
|
|
176
180
|
"note": {
|
|
177
181
|
"type": "object",
|
|
178
182
|
"additionalProperties": false,
|
|
179
183
|
"required": ["markdown", "slug", "summary", "title"],
|
|
180
184
|
"properties": {
|
|
185
|
+
"id": { "$ref": "#/definitions/noteId" },
|
|
181
186
|
"markdown": { "type": "string" },
|
|
182
187
|
"metadata": { "$ref": "#/definitions/metadata" },
|
|
183
188
|
"slug": { "$ref": "#/definitions/slug" },
|
|
@@ -347,6 +352,13 @@
|
|
|
347
352
|
"visibility"
|
|
348
353
|
],
|
|
349
354
|
"properties": {
|
|
355
|
+
"noteIds": {
|
|
356
|
+
"type": "array",
|
|
357
|
+
"minItems": 1,
|
|
358
|
+
"maxItems": 5000,
|
|
359
|
+
"uniqueItems": true,
|
|
360
|
+
"items": { "$ref": "#/definitions/noteId" }
|
|
361
|
+
},
|
|
350
362
|
"encryptedPayload": {
|
|
351
363
|
"type": "object",
|
|
352
364
|
"additionalProperties": false,
|
|
@@ -98,6 +98,7 @@ export async function publishExport(
|
|
|
98
98
|
const { artifact, assetSummary, warnings } = await exportPublishArtifact({
|
|
99
99
|
collections,
|
|
100
100
|
options: {
|
|
101
|
+
configPath: initResult.actualConfigPath,
|
|
101
102
|
routeSlug: options.slug,
|
|
102
103
|
encryptionPassphrase: options.encryptionPassphrase,
|
|
103
104
|
summary: options.summary,
|
package/src/embed/backlog.ts
CHANGED
|
@@ -6,6 +6,9 @@
|
|
|
6
6
|
|
|
7
7
|
import { MAX_PUBLISH_UPLOAD_BYTES } from "./artifact-asset-contract";
|
|
8
8
|
|
|
9
|
+
export const PUBLISH_NOTE_ID_PATTERN =
|
|
10
|
+
/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/u;
|
|
11
|
+
|
|
9
12
|
export const MAX_PUBLISH_SLUG_LENGTH = 80;
|
|
10
13
|
/**
|
|
11
14
|
* Ciphertext base64 character ceiling aligned to the 100 MiB final-envelope
|
|
@@ -33,6 +36,7 @@ const SOURCE_TYPES = new Set(["collection", "note"]);
|
|
|
33
36
|
const READER_VISIBILITIES = new Set(["invite-only", "public", "secret-link"]);
|
|
34
37
|
|
|
35
38
|
export interface ValidatedPublishNote {
|
|
39
|
+
id?: string;
|
|
36
40
|
markdown: string;
|
|
37
41
|
metadata?: Record<string, string | string[]>;
|
|
38
42
|
slug: string;
|
|
@@ -51,6 +55,7 @@ export interface ValidatedPublishSpaceInput {
|
|
|
51
55
|
}
|
|
52
56
|
|
|
53
57
|
export interface ValidatedEncryptedPublishInput {
|
|
58
|
+
noteIds?: string[];
|
|
54
59
|
encryptedPayload: {
|
|
55
60
|
ciphertext: string;
|
|
56
61
|
iterations: number;
|
|
@@ -158,6 +163,21 @@ const projectMetadata = (
|
|
|
158
163
|
return result;
|
|
159
164
|
};
|
|
160
165
|
|
|
166
|
+
function requireNoteId(value: unknown): string {
|
|
167
|
+
if (typeof value !== "string" || !PUBLISH_NOTE_ID_PATTERN.test(value))
|
|
168
|
+
throw new Error("Publish note ID must be a lowercase UUIDv4");
|
|
169
|
+
return value;
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
function projectNoteIds(value: unknown): string[] {
|
|
173
|
+
if (!Array.isArray(value) || value.length < 1 || value.length > 5000)
|
|
174
|
+
throw new Error("noteIds must contain 1 to 5000 UUIDv4 IDs");
|
|
175
|
+
const ids = value.map(requireNoteId);
|
|
176
|
+
if (new Set(ids).size !== ids.length)
|
|
177
|
+
throw new Error("Duplicate publish note ID");
|
|
178
|
+
return ids;
|
|
179
|
+
}
|
|
180
|
+
|
|
161
181
|
const projectNote = (value: unknown, index: number): ValidatedPublishNote => {
|
|
162
182
|
const field = `notes[${index}]`;
|
|
163
183
|
const input = requireRecord(value, field);
|
|
@@ -168,6 +188,7 @@ const projectNote = (value: unknown, index: number): ValidatedPublishNote => {
|
|
|
168
188
|
summary: requireString(input.summary, `${field}.summary`),
|
|
169
189
|
title: requireNonblankString(input.title, `${field}.title`),
|
|
170
190
|
};
|
|
191
|
+
if (input.id !== undefined) note.id = requireNoteId(input.id);
|
|
171
192
|
if (metadata !== undefined) note.metadata = metadata;
|
|
172
193
|
return note;
|
|
173
194
|
};
|
|
@@ -181,6 +202,9 @@ export const validateAndProjectPublishSpaceInput = (
|
|
|
181
202
|
}
|
|
182
203
|
|
|
183
204
|
const notes = input.notes.map(projectNote);
|
|
205
|
+
const ids = notes.flatMap((note) => (note.id ? [note.id] : []));
|
|
206
|
+
if (new Set(ids).size !== ids.length)
|
|
207
|
+
throw new Error("Duplicate publish note ID");
|
|
184
208
|
const noteSlugs = new Set<string>();
|
|
185
209
|
for (const note of notes) {
|
|
186
210
|
if (noteSlugs.has(note.slug)) {
|
|
@@ -246,6 +270,9 @@ export const validateAndProjectEncryptedPublishInput = (
|
|
|
246
270
|
}
|
|
247
271
|
|
|
248
272
|
return {
|
|
273
|
+
...(input.noteIds === undefined
|
|
274
|
+
? {}
|
|
275
|
+
: { noteIds: projectNoteIds(input.noteIds) }),
|
|
249
276
|
encryptedPayload: {
|
|
250
277
|
ciphertext: requireEncryptedCiphertext(payload.ciphertext),
|
|
251
278
|
iterations,
|
package/src/publish/artifact.ts
CHANGED
|
@@ -68,6 +68,7 @@ export type PublishVisibility =
|
|
|
68
68
|
| "secret-link";
|
|
69
69
|
|
|
70
70
|
export interface PublishArtifactNote {
|
|
71
|
+
id?: string;
|
|
71
72
|
markdown: string;
|
|
72
73
|
metadata?: Record<string, string | string[]>;
|
|
73
74
|
slug: string;
|
|
@@ -148,6 +149,7 @@ export interface EncryptedArtifactPayload {
|
|
|
148
149
|
}
|
|
149
150
|
|
|
150
151
|
export interface EncryptedPublishArtifactSpace {
|
|
152
|
+
noteIds?: string[];
|
|
151
153
|
encryptedPayload: EncryptedArtifactPayload;
|
|
152
154
|
routeSlug: string;
|
|
153
155
|
secretToken: string;
|
|
@@ -441,6 +443,7 @@ export const buildPublishArtifact = (input: {
|
|
|
441
443
|
};
|
|
442
444
|
|
|
443
445
|
export const buildEncryptedPublishArtifact = (input: {
|
|
446
|
+
noteIds?: string[];
|
|
444
447
|
egressLineage?: EgressLineage;
|
|
445
448
|
encryptedPayload: EncryptedArtifactPayload;
|
|
446
449
|
requiredCapabilities?: KnownPublishRequiredCapability[];
|
|
@@ -464,6 +467,9 @@ export const buildEncryptedPublishArtifact = (input: {
|
|
|
464
467
|
source: validated.routeSlug,
|
|
465
468
|
spaces: [
|
|
466
469
|
{
|
|
470
|
+
...(validated.noteIds === undefined
|
|
471
|
+
? {}
|
|
472
|
+
: { noteIds: validated.noteIds }),
|
|
467
473
|
encryptedPayload: validated.encryptedPayload,
|
|
468
474
|
routeSlug: validated.routeSlug,
|
|
469
475
|
secretToken: validated.secretToken,
|
|
@@ -183,7 +183,7 @@ const deriveReaderPayload = (input: {
|
|
|
183
183
|
const noteCards: ReaderNoteCard[] = input.notes.map((note) => {
|
|
184
184
|
const blocks = parseMarkdownBlocks(note.markdown);
|
|
185
185
|
return {
|
|
186
|
-
noteId: `${input.routeSlug}:${note.slug}`,
|
|
186
|
+
noteId: note.id ?? `${input.routeSlug}:${note.slug}`,
|
|
187
187
|
slug: note.slug,
|
|
188
188
|
title: note.title,
|
|
189
189
|
excerpt: deriveExcerpt(note.summary, blocks),
|
|
@@ -197,6 +197,9 @@ export async function finalizeEncryptedArtifact(input: {
|
|
|
197
197
|
const artifact = buildEncryptedPublishArtifact({
|
|
198
198
|
egressLineage: input.egressLineage,
|
|
199
199
|
encryptedPayload: encrypted.encryptedPayload,
|
|
200
|
+
...(input.notes.every((note) => note.id !== undefined)
|
|
201
|
+
? { noteIds: input.notes.map((note) => note.id!) }
|
|
202
|
+
: {}),
|
|
200
203
|
requiredCapabilities:
|
|
201
204
|
assets.length > 0 ? [BUNDLED_RASTER_ASSETS_CAPABILITY] : undefined,
|
|
202
205
|
routeSlug: input.routeSlug,
|
|
@@ -34,12 +34,15 @@ import {
|
|
|
34
34
|
sanitizeNoteMarkdown,
|
|
35
35
|
type NoteBuildAccumulator,
|
|
36
36
|
} from "./export-attachments";
|
|
37
|
+
import { resolvePublishNoteIds } from "./identities";
|
|
37
38
|
import {
|
|
38
39
|
isPublishDisabledByFrontmatter,
|
|
39
40
|
type SanitizeWarning,
|
|
40
41
|
} from "./obsidian-sanitize";
|
|
41
42
|
|
|
42
43
|
export interface PublishExportCoreOptions {
|
|
44
|
+
/** Actual loaded config file; defaults to the platform config directory. */
|
|
45
|
+
configPath?: string;
|
|
43
46
|
encryptionPassphrase?: string;
|
|
44
47
|
routeSlug?: string;
|
|
45
48
|
summary?: string;
|
|
@@ -199,6 +202,7 @@ async function exportCollectionArtifact(
|
|
|
199
202
|
preDedupRawBytes: 0,
|
|
200
203
|
};
|
|
201
204
|
const notes: PublishArtifactNote[] = [];
|
|
205
|
+
const sourceRelPaths: string[] = [];
|
|
202
206
|
|
|
203
207
|
for (const doc of activeDocs) {
|
|
204
208
|
if (!doc.mirrorHash) {
|
|
@@ -230,6 +234,7 @@ async function exportCollectionArtifact(
|
|
|
230
234
|
acc.externalCount += sanitized.externalCount;
|
|
231
235
|
acc.preDedupRawBytes += sanitized.preDedupRawBytes;
|
|
232
236
|
acc.encodedAssetBytes += mergePayloads(acc.payloads, sanitized.payloads);
|
|
237
|
+
sourceRelPaths.push(doc.relPath);
|
|
233
238
|
notes.push({
|
|
234
239
|
markdown: sanitized.markdown,
|
|
235
240
|
metadata: buildExportedMetadata(
|
|
@@ -268,6 +273,13 @@ async function exportCollectionArtifact(
|
|
|
268
273
|
store,
|
|
269
274
|
});
|
|
270
275
|
|
|
276
|
+
const noteIds = await resolvePublishNoteIds({
|
|
277
|
+
collectionRoot: collection.path,
|
|
278
|
+
sourceRelPaths,
|
|
279
|
+
configPath: options.configPath,
|
|
280
|
+
});
|
|
281
|
+
for (const [index, note] of notes.entries()) note.id = noteIds[index];
|
|
282
|
+
|
|
271
283
|
if (visibility === "encrypted") {
|
|
272
284
|
if (!options.encryptionPassphrase) {
|
|
273
285
|
throw new Error(
|
|
@@ -321,6 +333,8 @@ async function exportDocumentArtifact(
|
|
|
321
333
|
|
|
322
334
|
const collection =
|
|
323
335
|
collections.find((entry) => entry.name === doc.collection) ?? null;
|
|
336
|
+
if (!collection)
|
|
337
|
+
throw new Error(`Collection not configured: ${doc.collection}`);
|
|
324
338
|
const rawMarkdown = await loadDocumentMarkdown(store, doc);
|
|
325
339
|
if (isPublishDisabledByFrontmatter(rawMarkdown)) {
|
|
326
340
|
throw new Error(
|
|
@@ -362,7 +376,13 @@ async function exportDocumentArtifact(
|
|
|
362
376
|
store,
|
|
363
377
|
});
|
|
364
378
|
|
|
379
|
+
const [id] = await resolvePublishNoteIds({
|
|
380
|
+
collectionRoot: collection.path,
|
|
381
|
+
sourceRelPaths: [doc.relPath],
|
|
382
|
+
configPath: options.configPath,
|
|
383
|
+
});
|
|
365
384
|
const note: PublishArtifactNote = {
|
|
385
|
+
id,
|
|
366
386
|
markdown,
|
|
367
387
|
metadata: buildExportedMetadata(doc, frontmatter, tags),
|
|
368
388
|
slug,
|
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
/** Private durable publish identities, independent of the disposable index. */
|
|
2
|
+
// node:fs/promises — canonical paths, permissions and atomic rename have no Bun equivalents.
|
|
3
|
+
import {
|
|
4
|
+
chmod,
|
|
5
|
+
lstat,
|
|
6
|
+
mkdir,
|
|
7
|
+
realpath,
|
|
8
|
+
rename,
|
|
9
|
+
unlink,
|
|
10
|
+
} from "node:fs/promises";
|
|
11
|
+
// node:path — Bun has no path utilities.
|
|
12
|
+
import { dirname, isAbsolute, join, normalize } from "node:path";
|
|
13
|
+
|
|
14
|
+
import { getConfigPaths, toAbsolutePath } from "../config/paths";
|
|
15
|
+
import { acquireWriteLock, type WriteLockHandle } from "../core/file-lock";
|
|
16
|
+
import { PUBLISH_NOTE_ID_PATTERN } from "./artifact-validation";
|
|
17
|
+
|
|
18
|
+
const IDENTITY_LOCK_TIMEOUT_MS = 5000;
|
|
19
|
+
const IDENTITY_LOCK_RETRY_MS = 25;
|
|
20
|
+
|
|
21
|
+
type Registry = { version: 1; identities: Record<string, string> };
|
|
22
|
+
|
|
23
|
+
export function publishIdentityRegistryPath(configPath?: string): string {
|
|
24
|
+
return join(
|
|
25
|
+
configPath
|
|
26
|
+
? dirname(toAbsolutePath(configPath))
|
|
27
|
+
: getConfigPaths().configDir,
|
|
28
|
+
"publish-identities.json"
|
|
29
|
+
);
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
function validateRegistry(value: unknown): Registry {
|
|
33
|
+
if (!value || typeof value !== "object" || Array.isArray(value))
|
|
34
|
+
throw new Error("Invalid registry object");
|
|
35
|
+
const record = value as Record<string, unknown>;
|
|
36
|
+
if (
|
|
37
|
+
record.version !== 1 ||
|
|
38
|
+
Object.keys(record).length !== 2 ||
|
|
39
|
+
!record.identities ||
|
|
40
|
+
typeof record.identities !== "object" ||
|
|
41
|
+
Array.isArray(record.identities)
|
|
42
|
+
)
|
|
43
|
+
throw new Error("Invalid registry version or identities");
|
|
44
|
+
const identities = record.identities as Record<string, unknown>;
|
|
45
|
+
const ids = new Set<string>();
|
|
46
|
+
for (const [key, id] of Object.entries(identities)) {
|
|
47
|
+
const source: unknown = JSON.parse(key);
|
|
48
|
+
if (
|
|
49
|
+
!Array.isArray(source) ||
|
|
50
|
+
source.length !== 2 ||
|
|
51
|
+
!source.every((part) => typeof part === "string" && part.length > 0) ||
|
|
52
|
+
typeof id !== "string" ||
|
|
53
|
+
!PUBLISH_NOTE_ID_PATTERN.test(id) ||
|
|
54
|
+
ids.has(id)
|
|
55
|
+
)
|
|
56
|
+
throw new Error("Invalid or duplicate registry identity");
|
|
57
|
+
ids.add(id);
|
|
58
|
+
}
|
|
59
|
+
return value as Registry;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
async function readRegistry(path: string): Promise<Registry> {
|
|
63
|
+
try {
|
|
64
|
+
const info = await lstat(path);
|
|
65
|
+
if (!info.isFile() || info.isSymbolicLink())
|
|
66
|
+
throw new Error("Registry must be a regular private file");
|
|
67
|
+
await chmod(path, 0o600);
|
|
68
|
+
return validateRegistry(await Bun.file(path).json());
|
|
69
|
+
} catch (error) {
|
|
70
|
+
if ((error as NodeJS.ErrnoException).code === "ENOENT")
|
|
71
|
+
return { version: 1, identities: {} };
|
|
72
|
+
throw new Error(
|
|
73
|
+
"Cannot read publish identity registry; restore a valid backup before exporting",
|
|
74
|
+
{ cause: error }
|
|
75
|
+
);
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
async function acquireIdentityLock(path: string): Promise<WriteLockHandle> {
|
|
80
|
+
const deadline = performance.now() + IDENTITY_LOCK_TIMEOUT_MS;
|
|
81
|
+
// Zero-wait attempts avoid blocking the JS holder on SQLite's busy timeout.
|
|
82
|
+
// flock -w 0 and lockf -t 0 also mean immediate acquisition or failure.
|
|
83
|
+
let lock = await acquireWriteLock(path, 0);
|
|
84
|
+
while (!lock) {
|
|
85
|
+
const remaining = deadline - performance.now();
|
|
86
|
+
if (remaining <= 0)
|
|
87
|
+
throw new Error("Publish identity registry is busy; retry the export");
|
|
88
|
+
await Bun.sleep(Math.min(IDENTITY_LOCK_RETRY_MS, remaining));
|
|
89
|
+
lock = await acquireWriteLock(path, 0);
|
|
90
|
+
}
|
|
91
|
+
return lock;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/** Allocate a batch under one OS lock; publish IDs only after the private file commits. */
|
|
95
|
+
export async function resolvePublishNoteIds(input: {
|
|
96
|
+
collectionRoot: string;
|
|
97
|
+
sourceRelPaths: string[];
|
|
98
|
+
configPath?: string;
|
|
99
|
+
}): Promise<string[]> {
|
|
100
|
+
let root: string;
|
|
101
|
+
try {
|
|
102
|
+
root = await realpath(toAbsolutePath(input.collectionRoot));
|
|
103
|
+
} catch (cause) {
|
|
104
|
+
throw new Error(
|
|
105
|
+
"Collection root must exist and be accessible to resolve publish identities",
|
|
106
|
+
{ cause }
|
|
107
|
+
);
|
|
108
|
+
}
|
|
109
|
+
const keys = input.sourceRelPaths.map((source) => {
|
|
110
|
+
const rel = normalize(source).replaceAll("\\", "/");
|
|
111
|
+
if (
|
|
112
|
+
!rel ||
|
|
113
|
+
rel === "." ||
|
|
114
|
+
rel === ".." ||
|
|
115
|
+
rel.startsWith("../") ||
|
|
116
|
+
isAbsolute(rel) ||
|
|
117
|
+
rel.includes("\0")
|
|
118
|
+
)
|
|
119
|
+
throw new Error(
|
|
120
|
+
"Publish source path must be relative to its collection root"
|
|
121
|
+
);
|
|
122
|
+
return JSON.stringify([root, rel]);
|
|
123
|
+
});
|
|
124
|
+
const path = publishIdentityRegistryPath(input.configPath);
|
|
125
|
+
await mkdir(dirname(path), { recursive: true, mode: 0o700 });
|
|
126
|
+
const lock = await acquireIdentityLock(`${path}.lock`);
|
|
127
|
+
const temporaryPath = `${path}.tmp.${crypto.randomUUID()}`;
|
|
128
|
+
try {
|
|
129
|
+
const registry = await readRegistry(path);
|
|
130
|
+
let changed = false;
|
|
131
|
+
const ids = keys.map((key) => {
|
|
132
|
+
const existing = registry.identities[key];
|
|
133
|
+
if (existing) return existing;
|
|
134
|
+
const id = crypto.randomUUID();
|
|
135
|
+
registry.identities[key] = id;
|
|
136
|
+
changed = true;
|
|
137
|
+
return id;
|
|
138
|
+
});
|
|
139
|
+
if (changed) {
|
|
140
|
+
await Bun.write(temporaryPath, JSON.stringify(registry), { mode: 0o600 });
|
|
141
|
+
await rename(temporaryPath, path);
|
|
142
|
+
}
|
|
143
|
+
return ids;
|
|
144
|
+
} catch (error) {
|
|
145
|
+
throw new Error(
|
|
146
|
+
"Publish identity registry update failed; export aborted to preserve note identity",
|
|
147
|
+
{ cause: error }
|
|
148
|
+
);
|
|
149
|
+
} finally {
|
|
150
|
+
await unlink(temporaryPath).catch(() => undefined);
|
|
151
|
+
await lock.release();
|
|
152
|
+
}
|
|
153
|
+
}
|
package/src/serve/routes/api.ts
CHANGED
|
@@ -1266,7 +1266,8 @@ export async function handleEgressAuditPurge(
|
|
|
1266
1266
|
export async function handlePublishExport(
|
|
1267
1267
|
config: Config,
|
|
1268
1268
|
store: SqliteAdapter,
|
|
1269
|
-
req: Request
|
|
1269
|
+
req: Request,
|
|
1270
|
+
configPath?: string
|
|
1270
1271
|
): Promise<Response> {
|
|
1271
1272
|
let body: PublishExportRequestBody;
|
|
1272
1273
|
try {
|
|
@@ -1304,6 +1305,7 @@ export async function handlePublishExport(
|
|
|
1304
1305
|
const { artifact, assetSummary, warnings } = await exportPublishArtifact({
|
|
1305
1306
|
collections: config.collections,
|
|
1306
1307
|
options: {
|
|
1308
|
+
configPath,
|
|
1307
1309
|
encryptionPassphrase: body.encryptionPassphrase,
|
|
1308
1310
|
routeSlug: body.slug,
|
|
1309
1311
|
summary: body.summary,
|
package/src/serve/server.ts
CHANGED
|
@@ -144,6 +144,7 @@ import { getSchemaVersion, migrations, runMigrations } from "../migrations";
|
|
|
144
144
|
import { err, ok } from "../types";
|
|
145
145
|
import { getStoredEmbeddingFingerprint } from "../vector/freshness";
|
|
146
146
|
import { modelTableName } from "../vector/sqlite-vec";
|
|
147
|
+
import { getVariantStatus } from "../vector/status";
|
|
147
148
|
import {
|
|
148
149
|
deleteSavedCapsuleRegistration as deleteStoredSavedCapsuleRegistration,
|
|
149
150
|
getSavedCapsuleRegistration as getStoredSavedCapsuleRegistration,
|
|
@@ -5697,6 +5698,8 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
|
|
|
5697
5698
|
options?.embedFingerprint ??
|
|
5698
5699
|
(embedModel ? getStoredEmbeddingFingerprint(db, embedModel) : null);
|
|
5699
5700
|
|
|
5701
|
+
const variantStatus = getVariantStatus(db, options);
|
|
5702
|
+
|
|
5700
5703
|
// Get version
|
|
5701
5704
|
const versionRow = db
|
|
5702
5705
|
.query<{ value: string }, []>(
|
|
@@ -5873,12 +5876,14 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
|
|
|
5873
5876
|
errorDocuments: s.errored,
|
|
5874
5877
|
chunkedDocuments: s.chunked,
|
|
5875
5878
|
totalChunks: s.chunk_count,
|
|
5876
|
-
embeddedChunks:
|
|
5879
|
+
embeddedChunks: variantStatus
|
|
5880
|
+
? (variantStatus.embeddedByCollection.get(s.name) ?? 0)
|
|
5881
|
+
: s.embedded_count,
|
|
5877
5882
|
})),
|
|
5878
5883
|
totalDocuments: totalsRow?.total ?? 0,
|
|
5879
5884
|
activeDocuments: totalsRow?.active ?? 0,
|
|
5880
5885
|
totalChunks: chunkCount,
|
|
5881
|
-
embeddingBacklog: backlogRow?.count ?? 0,
|
|
5886
|
+
embeddingBacklog: variantStatus?.backlog ?? backlogRow?.count ?? 0,
|
|
5882
5887
|
recentErrors,
|
|
5883
5888
|
lastUpdatedAt: lastUpdatedRow?.last_updated ?? null,
|
|
5884
5889
|
healthy,
|
|
@@ -0,0 +1,196 @@
|
|
|
1
|
+
/** Read-only coverage of persisted variant authority; never initializes models. */
|
|
2
|
+
import type { Database } from "bun:sqlite";
|
|
3
|
+
|
|
4
|
+
import { getEmbeddingFingerprint } from "../../embed/fingerprint";
|
|
5
|
+
import { formatDocForEmbedding } from "../../pipeline/contextual";
|
|
6
|
+
import {
|
|
7
|
+
embeddingInputHash,
|
|
8
|
+
SELECTED_VECTOR_PARTITION_PREFIX,
|
|
9
|
+
} from "./variants";
|
|
10
|
+
|
|
11
|
+
interface Partition {
|
|
12
|
+
partition_id: string;
|
|
13
|
+
version: number;
|
|
14
|
+
model: string;
|
|
15
|
+
fingerprint: string;
|
|
16
|
+
dimensions: number;
|
|
17
|
+
state: string;
|
|
18
|
+
activated_epoch: number | null;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
interface OwnerCoverage {
|
|
22
|
+
document_id: number;
|
|
23
|
+
partition_id: string | null;
|
|
24
|
+
collection: string;
|
|
25
|
+
mirror_hash: string;
|
|
26
|
+
seq: number;
|
|
27
|
+
text: string;
|
|
28
|
+
title: string | null;
|
|
29
|
+
input_hash: string | null;
|
|
30
|
+
embedding_bytes: number | null;
|
|
31
|
+
legacy_embedded: number;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/** Null retains legacy counts until a verified selection or activation exists. */
|
|
35
|
+
export function getVariantStatus(
|
|
36
|
+
db: Database,
|
|
37
|
+
options?: { embedModel?: string; embedFingerprint?: string }
|
|
38
|
+
): { backlog: number; embeddedByCollection: Map<string, number> } | null {
|
|
39
|
+
return db.transaction(() => {
|
|
40
|
+
if (
|
|
41
|
+
!db
|
|
42
|
+
.query(
|
|
43
|
+
"SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'vector_partitions'"
|
|
44
|
+
)
|
|
45
|
+
.get()
|
|
46
|
+
)
|
|
47
|
+
return null;
|
|
48
|
+
const partitions = db
|
|
49
|
+
.query<Partition, [string | null, string | null]>(`
|
|
50
|
+
SELECT partition_id, version, model, fingerprint, dimensions, state, activated_epoch
|
|
51
|
+
FROM vector_partitions WHERE (? IS NULL OR model = ?)
|
|
52
|
+
`)
|
|
53
|
+
.all(options?.embedModel ?? null, options?.embedModel ?? null);
|
|
54
|
+
const selections = db
|
|
55
|
+
.query<{ key: string; value: string }, [string]>(
|
|
56
|
+
"SELECT key, value FROM schema_meta WHERE key GLOB ?"
|
|
57
|
+
)
|
|
58
|
+
.all(`${SELECTED_VECTOR_PARTITION_PREFIX}*`);
|
|
59
|
+
const selected = new Map(
|
|
60
|
+
selections.map((row) => [
|
|
61
|
+
row.key.slice(SELECTED_VECTOR_PARTITION_PREFIX.length),
|
|
62
|
+
row.value,
|
|
63
|
+
])
|
|
64
|
+
);
|
|
65
|
+
const models = new Set(partitions.map((p) => p.model));
|
|
66
|
+
if (options?.embedModel) models.add(options.embedModel);
|
|
67
|
+
else for (const model of selected.keys()) models.add(model);
|
|
68
|
+
const authoritativeModels = new Set<string>();
|
|
69
|
+
const usablePartitions = new Map<string, Partition>();
|
|
70
|
+
for (const model of models) {
|
|
71
|
+
const candidates = partitions.filter((p) => p.model === model);
|
|
72
|
+
const selection = selected.get(model);
|
|
73
|
+
const activated = candidates.some(
|
|
74
|
+
(p) => p.state === "active" && p.activated_epoch !== null
|
|
75
|
+
);
|
|
76
|
+
if (selection === undefined && !activated) continue;
|
|
77
|
+
authoritativeModels.add(model);
|
|
78
|
+
// Resolve one persisted identity per model, never combine alternative
|
|
79
|
+
// partitions of the same model. Unscoped status may accept any model.
|
|
80
|
+
// Stale epochs do not revoke owners whose current inputs still match.
|
|
81
|
+
const partition =
|
|
82
|
+
selection === undefined
|
|
83
|
+
? candidates.length === 1
|
|
84
|
+
? candidates[0]
|
|
85
|
+
: undefined
|
|
86
|
+
: candidates.find((p) => p.partition_id === selection);
|
|
87
|
+
if (
|
|
88
|
+
partition &&
|
|
89
|
+
partition.version === 1 &&
|
|
90
|
+
partition.partition_id ===
|
|
91
|
+
embeddingInputHash(
|
|
92
|
+
JSON.stringify([
|
|
93
|
+
partition.model,
|
|
94
|
+
partition.fingerprint,
|
|
95
|
+
partition.dimensions,
|
|
96
|
+
])
|
|
97
|
+
) &&
|
|
98
|
+
(options?.embedFingerprint === undefined ||
|
|
99
|
+
options.embedFingerprint ===
|
|
100
|
+
getEmbeddingFingerprint({
|
|
101
|
+
modelUri: partition.model,
|
|
102
|
+
dimensions: partition.dimensions,
|
|
103
|
+
}))
|
|
104
|
+
)
|
|
105
|
+
usablePartitions.set(partition.partition_id, partition);
|
|
106
|
+
}
|
|
107
|
+
if (authoritativeModels.size === 0) return null;
|
|
108
|
+
// Unscoped legacy coverage remains valid only for models that have never
|
|
109
|
+
// selected or activated verified authority; legacy rows cannot repair it.
|
|
110
|
+
const statement = db.prepare<
|
|
111
|
+
OwnerCoverage,
|
|
112
|
+
[string | null, string, string]
|
|
113
|
+
>(`
|
|
114
|
+
WITH legacy_vectors AS (
|
|
115
|
+
SELECT mirror_hash, seq, MAX(embedded_at) AS embedded_at
|
|
116
|
+
FROM content_vectors
|
|
117
|
+
WHERE ? IS NULL AND model NOT IN (SELECT value FROM json_each(?))
|
|
118
|
+
GROUP BY mirror_hash, seq
|
|
119
|
+
)
|
|
120
|
+
SELECT d.id AS document_id, o.partition_id, d.collection, d.mirror_hash, c.seq, c.text, d.title,
|
|
121
|
+
v.input_hash, length(v.embedding) AS embedding_bytes,
|
|
122
|
+
CASE WHEN lv.embedded_at >= c.created_at THEN 1 ELSE 0 END AS legacy_embedded
|
|
123
|
+
FROM documents d
|
|
124
|
+
JOIN content_chunks c ON c.mirror_hash = d.mirror_hash
|
|
125
|
+
LEFT JOIN vector_owners o ON o.document_id = d.id AND o.seq = c.seq
|
|
126
|
+
AND o.mirror_hash = d.mirror_hash
|
|
127
|
+
AND o.partition_id IN (SELECT value FROM json_each(?))
|
|
128
|
+
LEFT JOIN vector_variants v ON v.variant_id = o.variant_id
|
|
129
|
+
AND v.partition_id = o.partition_id
|
|
130
|
+
LEFT JOIN legacy_vectors lv ON lv.mirror_hash = c.mirror_hash AND lv.seq = c.seq
|
|
131
|
+
WHERE d.active = 1
|
|
132
|
+
`);
|
|
133
|
+
const owners = new Map<
|
|
134
|
+
string,
|
|
135
|
+
{ collection: string; chunk: string; embedded: boolean }
|
|
136
|
+
>();
|
|
137
|
+
try {
|
|
138
|
+
for (const row of statement.iterate(
|
|
139
|
+
options?.embedModel ?? null,
|
|
140
|
+
JSON.stringify([...authoritativeModels]),
|
|
141
|
+
JSON.stringify([...usablePartitions.keys()])
|
|
142
|
+
)) {
|
|
143
|
+
const partition = row.partition_id
|
|
144
|
+
? usablePartitions.get(row.partition_id)
|
|
145
|
+
: undefined;
|
|
146
|
+
const embedded = Boolean(
|
|
147
|
+
row.legacy_embedded ||
|
|
148
|
+
(partition &&
|
|
149
|
+
row.embedding_bytes ===
|
|
150
|
+
partition.dimensions * Float32Array.BYTES_PER_ELEMENT &&
|
|
151
|
+
row.input_hash ===
|
|
152
|
+
embeddingInputHash(
|
|
153
|
+
formatDocForEmbedding(
|
|
154
|
+
row.text,
|
|
155
|
+
row.title ?? undefined,
|
|
156
|
+
partition.model
|
|
157
|
+
)
|
|
158
|
+
))
|
|
159
|
+
);
|
|
160
|
+
const key = `${row.document_id}:${row.seq}`;
|
|
161
|
+
const previous = owners.get(key);
|
|
162
|
+
if (previous) previous.embedded ||= embedded;
|
|
163
|
+
else
|
|
164
|
+
owners.set(key, {
|
|
165
|
+
collection: row.collection,
|
|
166
|
+
chunk: JSON.stringify([row.mirror_hash, row.seq]),
|
|
167
|
+
embedded,
|
|
168
|
+
});
|
|
169
|
+
}
|
|
170
|
+
} finally {
|
|
171
|
+
statement.finalize();
|
|
172
|
+
}
|
|
173
|
+
const collections = new Map<string, Map<string, boolean>>();
|
|
174
|
+
let backlog = 0;
|
|
175
|
+
for (const owner of owners.values()) {
|
|
176
|
+
if (!owner.embedded) backlog++;
|
|
177
|
+
let chunks = collections.get(owner.collection);
|
|
178
|
+
if (!chunks) {
|
|
179
|
+
chunks = new Map();
|
|
180
|
+
collections.set(owner.collection, chunks);
|
|
181
|
+
}
|
|
182
|
+
// Distinct collection chunks are ready only when every active owner is.
|
|
183
|
+
chunks.set(
|
|
184
|
+
owner.chunk,
|
|
185
|
+
(chunks.get(owner.chunk) ?? true) && owner.embedded
|
|
186
|
+
);
|
|
187
|
+
}
|
|
188
|
+
const embeddedByCollection = new Map<string, number>();
|
|
189
|
+
for (const [collection, chunks] of collections) {
|
|
190
|
+
let count = 0;
|
|
191
|
+
for (const embedded of chunks.values()) if (embedded) count++;
|
|
192
|
+
embeddedByCollection.set(collection, count);
|
|
193
|
+
}
|
|
194
|
+
return { backlog, embeddedByCollection };
|
|
195
|
+
})();
|
|
196
|
+
}
|
|
@@ -6,6 +6,8 @@ import type { VectorOwnerInput, VectorVariantIdentity } from "./types";
|
|
|
6
6
|
import { formatDocForEmbedding } from "../../pipeline/contextual";
|
|
7
7
|
import { decodeEmbedding, encodeEmbedding } from "./sqlite-vec";
|
|
8
8
|
|
|
9
|
+
export const SELECTED_VECTOR_PARTITION_PREFIX = "vector_selected_partition:";
|
|
10
|
+
|
|
9
11
|
export function embeddingInputHash(input: string): string {
|
|
10
12
|
return new Bun.CryptoHasher("sha256").update(input).digest("hex");
|
|
11
13
|
}
|
|
@@ -94,6 +96,15 @@ export class VectorVariantStore {
|
|
|
94
96
|
}).immediate();
|
|
95
97
|
}
|
|
96
98
|
|
|
99
|
+
/** Persist only after embedding has resolved the actual runtime identity. */
|
|
100
|
+
selectForEmbedding(): void {
|
|
101
|
+
this.db.run(
|
|
102
|
+
`INSERT INTO schema_meta (key, value) VALUES (?, ?)
|
|
103
|
+
ON CONFLICT(key) DO UPDATE SET value = excluded.value, updated_at = datetime('now')`,
|
|
104
|
+
[SELECTED_VECTOR_PARTITION_PREFIX + this.identity.model, this.partitionId]
|
|
105
|
+
);
|
|
106
|
+
}
|
|
107
|
+
|
|
97
108
|
epoch(): number {
|
|
98
109
|
return this.db
|
|
99
110
|
.query<{ epoch: number }, []>(
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
8dd833b01ebc0a0cb5747de65a9cd60b06cafcf645c2a74d697c918fe7c642d7 gno-browser-clipper-v2.1.0.zip
|