@gmickel/gno 2.0.0 → 2.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -5
- package/assets/spa-production.json.gz +0 -0
- package/browser-extension/artifacts/{gno-browser-clipper-v2.0.0.zip → gno-browser-clipper-v2.1.1.zip} +0 -0
- package/browser-extension/artifacts/gno-browser-clipper-v2.1.1.zip.sha256 +1 -0
- package/browser-extension/dist/manifest.json +1 -1
- package/package.json +1 -1
- package/spec/cli.md +16 -0
- package/src/embed/backlog.ts +1 -0
- package/src/serve/public/components/PublishExportDialog.tsx +266 -0
- package/src/serve/public/globals.built.css +1 -1
- package/src/serve/public/globals.css +35 -0
- package/src/serve/public/lib/publish-export.ts +81 -1
- package/src/serve/public/pages/Collections.tsx +12 -46
- package/src/serve/public/pages/DocView.tsx +14 -52
- package/src/store/sqlite/adapter.ts +7 -2
- package/src/store/vector/status.ts +196 -0
- package/src/store/vector/variants.ts +11 -0
- package/browser-extension/artifacts/gno-browser-clipper-v2.0.0.zip.sha256 +0 -1
|
@@ -568,3 +568,38 @@ mark {
|
|
|
568
568
|
animation: none;
|
|
569
569
|
}
|
|
570
570
|
}
|
|
571
|
+
|
|
572
|
+
/* Keep the review and actions inside the viewport, including short windows and
|
|
573
|
+
mobile keyboards. Only the form body scrolls; the close/export controls stay put. */
|
|
574
|
+
.publish-export-dialog {
|
|
575
|
+
display: flex;
|
|
576
|
+
flex-direction: column;
|
|
577
|
+
width: min(36rem, calc(100vw - 2rem));
|
|
578
|
+
max-width: 36rem;
|
|
579
|
+
max-height: calc(100dvh - 2rem);
|
|
580
|
+
gap: 0;
|
|
581
|
+
overflow: hidden;
|
|
582
|
+
padding: 0;
|
|
583
|
+
}
|
|
584
|
+
|
|
585
|
+
.publish-export-dialog-header {
|
|
586
|
+
flex-shrink: 0;
|
|
587
|
+
padding: 1.25rem 2.5rem 1rem 1.5rem;
|
|
588
|
+
overflow-wrap: anywhere;
|
|
589
|
+
}
|
|
590
|
+
|
|
591
|
+
.publish-export-dialog-body {
|
|
592
|
+
display: flex;
|
|
593
|
+
min-height: 0;
|
|
594
|
+
flex-direction: column;
|
|
595
|
+
gap: 1rem;
|
|
596
|
+
overflow-y: auto;
|
|
597
|
+
overscroll-behavior: contain;
|
|
598
|
+
padding: 0.25rem 1.5rem 1.25rem;
|
|
599
|
+
}
|
|
600
|
+
|
|
601
|
+
.publish-export-dialog-footer {
|
|
602
|
+
flex-shrink: 0;
|
|
603
|
+
border-top: 1px solid hsl(var(--border));
|
|
604
|
+
padding: 1rem 1.5rem;
|
|
605
|
+
}
|
|
@@ -1,4 +1,7 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type {
|
|
2
|
+
PublishArtifact,
|
|
3
|
+
PublishVisibility,
|
|
4
|
+
} from "../../../publish/artifact";
|
|
2
5
|
|
|
3
6
|
export interface PublishExportResponse {
|
|
4
7
|
artifact: PublishArtifact;
|
|
@@ -19,3 +22,80 @@ export function downloadPublishArtifactFile(
|
|
|
19
22
|
anchor.click();
|
|
20
23
|
URL.revokeObjectURL(href);
|
|
21
24
|
}
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* Local help metadata, checked against gno.sh src/lib/server/{billing,entitlements}.ts
|
|
28
|
+
* on 2026-09-05. Keep in sync when hosted plan capabilities change. These labels
|
|
29
|
+
* explain availability; only the hosted entitlement service authorizes publishing.
|
|
30
|
+
*/
|
|
31
|
+
export const PUBLISH_ACCESS_OPTIONS: ReadonlyArray<{
|
|
32
|
+
value: PublishVisibility;
|
|
33
|
+
label: string;
|
|
34
|
+
audience: string;
|
|
35
|
+
availability: string;
|
|
36
|
+
}> = [
|
|
37
|
+
{
|
|
38
|
+
value: "public",
|
|
39
|
+
label: "Public",
|
|
40
|
+
audience:
|
|
41
|
+
"Anyone can read, discover, and download the published content, including search engines and agents.",
|
|
42
|
+
availability: "Free and paid plans",
|
|
43
|
+
},
|
|
44
|
+
{
|
|
45
|
+
value: "secret-link",
|
|
46
|
+
label: "Secret link",
|
|
47
|
+
audience:
|
|
48
|
+
"Anyone with the link can read and forward it. Readers do not need an invitation.",
|
|
49
|
+
availability: "Paid plans with private publishing",
|
|
50
|
+
},
|
|
51
|
+
{
|
|
52
|
+
value: "invite-only",
|
|
53
|
+
label: "Invite only",
|
|
54
|
+
audience:
|
|
55
|
+
"Personal shares start with owner access only. Choose recipients or the intended organization audience in Studio before publishing.",
|
|
56
|
+
availability: "Paid plans with private publishing",
|
|
57
|
+
},
|
|
58
|
+
{
|
|
59
|
+
value: "encrypted",
|
|
60
|
+
label: "Encrypted",
|
|
61
|
+
audience:
|
|
62
|
+
"Anyone with both the link and passphrase can decrypt the content in their browser. Share the passphrase separately.",
|
|
63
|
+
availability: "Paid plans with encrypted publishing",
|
|
64
|
+
},
|
|
65
|
+
];
|
|
66
|
+
|
|
67
|
+
export function buildPublishExportRequest(input: {
|
|
68
|
+
target: string;
|
|
69
|
+
visibility: string | null;
|
|
70
|
+
passphrase: string;
|
|
71
|
+
passphraseConfirmation: string;
|
|
72
|
+
audienceConfirmed: boolean;
|
|
73
|
+
}): {
|
|
74
|
+
target: string;
|
|
75
|
+
visibility: PublishVisibility;
|
|
76
|
+
encryptionPassphrase?: string;
|
|
77
|
+
} {
|
|
78
|
+
const option = PUBLISH_ACCESS_OPTIONS.find(
|
|
79
|
+
({ value }) => value === input.visibility
|
|
80
|
+
);
|
|
81
|
+
if (!option) {
|
|
82
|
+
throw new Error("Choose who can read this export.");
|
|
83
|
+
}
|
|
84
|
+
if (!input.audienceConfirmed) {
|
|
85
|
+
throw new Error("Review and confirm the audience before exporting.");
|
|
86
|
+
}
|
|
87
|
+
if (option.value === "encrypted") {
|
|
88
|
+
if (!input.passphrase.trim()) {
|
|
89
|
+
throw new Error("Enter a passphrase for the encrypted export.");
|
|
90
|
+
}
|
|
91
|
+
if (input.passphrase !== input.passphraseConfirmation) {
|
|
92
|
+
throw new Error("The passphrases do not match.");
|
|
93
|
+
}
|
|
94
|
+
return {
|
|
95
|
+
target: input.target,
|
|
96
|
+
visibility: option.value,
|
|
97
|
+
encryptionPassphrase: input.passphrase,
|
|
98
|
+
};
|
|
99
|
+
}
|
|
100
|
+
return { target: input.target, visibility: option.value };
|
|
101
|
+
}
|
|
@@ -37,6 +37,7 @@ import {
|
|
|
37
37
|
} from "../components/CollectionModelDialog";
|
|
38
38
|
import { CollectionsEmptyState } from "../components/CollectionsEmptyState";
|
|
39
39
|
import { IndexingProgress } from "../components/IndexingProgress";
|
|
40
|
+
import { PublishExportDialog } from "../components/PublishExportDialog";
|
|
40
41
|
import { Badge } from "../components/ui/badge";
|
|
41
42
|
import { Button } from "../components/ui/button";
|
|
42
43
|
import {
|
|
@@ -67,10 +68,6 @@ import {
|
|
|
67
68
|
TooltipTrigger,
|
|
68
69
|
} from "../components/ui/tooltip";
|
|
69
70
|
import { apiFetch } from "../hooks/use-api";
|
|
70
|
-
import {
|
|
71
|
-
downloadPublishArtifactFile,
|
|
72
|
-
type PublishExportResponse,
|
|
73
|
-
} from "../lib/publish-export";
|
|
74
71
|
|
|
75
72
|
interface PageProps {
|
|
76
73
|
navigate: (to: string | number) => void;
|
|
@@ -129,7 +126,6 @@ interface CollectionCardProps {
|
|
|
129
126
|
onModelSettings: () => void;
|
|
130
127
|
onReindex: () => void;
|
|
131
128
|
onRemove: () => void;
|
|
132
|
-
isExporting: boolean;
|
|
133
129
|
isReindexing: boolean;
|
|
134
130
|
}
|
|
135
131
|
|
|
@@ -161,7 +157,6 @@ function CollectionCard({
|
|
|
161
157
|
onModelSettings,
|
|
162
158
|
onReindex,
|
|
163
159
|
onRemove,
|
|
164
|
-
isExporting,
|
|
165
160
|
isReindexing,
|
|
166
161
|
}: CollectionCardProps) {
|
|
167
162
|
const embedPercent =
|
|
@@ -227,17 +222,13 @@ function CollectionCard({
|
|
|
227
222
|
Embedding cleanup
|
|
228
223
|
</DropdownMenuItem>
|
|
229
224
|
<DropdownMenuItem
|
|
230
|
-
disabled={actionsDisabled
|
|
225
|
+
disabled={actionsDisabled}
|
|
231
226
|
onClick={(event) => {
|
|
232
227
|
event.stopPropagation();
|
|
233
228
|
onExport();
|
|
234
229
|
}}
|
|
235
230
|
>
|
|
236
|
-
|
|
237
|
-
<Loader2Icon className="mr-2 size-4 animate-spin" />
|
|
238
|
-
) : (
|
|
239
|
-
<Share2Icon className="mr-2 size-4" />
|
|
240
|
-
)}
|
|
231
|
+
<Share2Icon className="mr-2 size-4" />
|
|
241
232
|
Export for gno.sh
|
|
242
233
|
</DropdownMenuItem>
|
|
243
234
|
<DropdownMenuSeparator />
|
|
@@ -361,8 +352,7 @@ export default function Collections({ navigate }: PageProps) {
|
|
|
361
352
|
const [syncJobId, setSyncJobId] = useState<string | null>(null);
|
|
362
353
|
const [syncTarget, setSyncTarget] = useState<SyncTarget>(null);
|
|
363
354
|
const [syncError, setSyncError] = useState<string | null>(null);
|
|
364
|
-
const [
|
|
365
|
-
const [exportingCollectionName, setExportingCollectionName] = useState<
|
|
355
|
+
const [exportCollectionName, setExportCollectionName] = useState<
|
|
366
356
|
string | null
|
|
367
357
|
>(null);
|
|
368
358
|
const [removeDialog, setRemoveDialog] = useState<CollectionStats | null>(
|
|
@@ -499,30 +489,6 @@ export default function Collections({ navigate }: PageProps) {
|
|
|
499
489
|
await loadCollections();
|
|
500
490
|
};
|
|
501
491
|
|
|
502
|
-
const handleExport = async (name: string) => {
|
|
503
|
-
setExportError(null);
|
|
504
|
-
setExportingCollectionName(name);
|
|
505
|
-
|
|
506
|
-
const { data, error: err } = await apiFetch<PublishExportResponse>(
|
|
507
|
-
"/api/publish/export",
|
|
508
|
-
{
|
|
509
|
-
body: JSON.stringify({ target: name }),
|
|
510
|
-
method: "POST",
|
|
511
|
-
}
|
|
512
|
-
);
|
|
513
|
-
|
|
514
|
-
setExportingCollectionName(null);
|
|
515
|
-
|
|
516
|
-
if (err) {
|
|
517
|
-
setExportError(err);
|
|
518
|
-
return;
|
|
519
|
-
}
|
|
520
|
-
|
|
521
|
-
if (data) {
|
|
522
|
-
downloadPublishArtifactFile(data);
|
|
523
|
-
}
|
|
524
|
-
};
|
|
525
|
-
|
|
526
492
|
// Loading state
|
|
527
493
|
if (loading) {
|
|
528
494
|
return (
|
|
@@ -643,12 +609,13 @@ export default function Collections({ navigate }: PageProps) {
|
|
|
643
609
|
</Card>
|
|
644
610
|
)}
|
|
645
611
|
|
|
646
|
-
{
|
|
647
|
-
<
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
612
|
+
{exportCollectionName !== null && (
|
|
613
|
+
<PublishExportDialog
|
|
614
|
+
key={exportCollectionName}
|
|
615
|
+
onClose={() => setExportCollectionName(null)}
|
|
616
|
+
target={exportCollectionName}
|
|
617
|
+
title={exportCollectionName}
|
|
618
|
+
/>
|
|
652
619
|
)}
|
|
653
620
|
|
|
654
621
|
{/* Error */}
|
|
@@ -687,7 +654,6 @@ export default function Collections({ navigate }: PageProps) {
|
|
|
687
654
|
<CollectionCard
|
|
688
655
|
actionsDisabled={Boolean(syncJobId)}
|
|
689
656
|
collection={collection}
|
|
690
|
-
isExporting={exportingCollectionName === collection.name}
|
|
691
657
|
isReindexing={
|
|
692
658
|
Boolean(syncJobId) &&
|
|
693
659
|
syncTarget?.kind === "collection" &&
|
|
@@ -704,7 +670,7 @@ export default function Collections({ navigate }: PageProps) {
|
|
|
704
670
|
setEmbeddingCleanupNote(null);
|
|
705
671
|
}}
|
|
706
672
|
onExport={() => {
|
|
707
|
-
|
|
673
|
+
setExportCollectionName(collection.name);
|
|
708
674
|
}}
|
|
709
675
|
onModelSettings={() => setModelDialogCollection(collection)}
|
|
710
676
|
onReindex={() => void handleReindex(collection.name)}
|
|
@@ -47,6 +47,7 @@ import {
|
|
|
47
47
|
OutgoingLinksPanel,
|
|
48
48
|
type OutgoingLink,
|
|
49
49
|
} from "../components/OutgoingLinksPanel";
|
|
50
|
+
import { PublishExportDialog } from "../components/PublishExportDialog";
|
|
50
51
|
import { RefactorImpactPreview } from "../components/RefactorImpactPreview";
|
|
51
52
|
import { RelatedNotesSidebar } from "../components/RelatedNotesSidebar";
|
|
52
53
|
import { TagInput } from "../components/TagInput";
|
|
@@ -81,10 +82,6 @@ import {
|
|
|
81
82
|
isPdfDocument,
|
|
82
83
|
} from "../lib/doc-asset-url";
|
|
83
84
|
import { waitForDocumentAvailability } from "../lib/document-availability";
|
|
84
|
-
import {
|
|
85
|
-
downloadPublishArtifactFile,
|
|
86
|
-
type PublishExportResponse,
|
|
87
|
-
} from "../lib/publish-export";
|
|
88
85
|
import {
|
|
89
86
|
buildReadableSectionUrl,
|
|
90
87
|
createCitationSectionUrl,
|
|
@@ -411,11 +408,7 @@ export default function DocView({ navigate }: PageProps) {
|
|
|
411
408
|
const [externalChangeNotice, setExternalChangeNotice] = useState<
|
|
412
409
|
string | null
|
|
413
410
|
>(null);
|
|
414
|
-
const [
|
|
415
|
-
useState(false);
|
|
416
|
-
const [publishExportError, setPublishExportError] = useState<string | null>(
|
|
417
|
-
null
|
|
418
|
-
);
|
|
411
|
+
const [publishExportOpen, setPublishExportOpen] = useState(false);
|
|
419
412
|
|
|
420
413
|
// Tag editing state
|
|
421
414
|
const [editingTags, setEditingTags] = useState(false);
|
|
@@ -879,32 +872,6 @@ export default function DocView({ navigate }: PageProps) {
|
|
|
879
872
|
}
|
|
880
873
|
}, [doc, navigate]);
|
|
881
874
|
|
|
882
|
-
const handlePublishExport = useCallback(async () => {
|
|
883
|
-
if (!doc) {
|
|
884
|
-
return;
|
|
885
|
-
}
|
|
886
|
-
|
|
887
|
-
setPublishExportError(null);
|
|
888
|
-
setExportingPublishArtifact(true);
|
|
889
|
-
const { data, error: err } = await apiFetch<PublishExportResponse>(
|
|
890
|
-
"/api/publish/export",
|
|
891
|
-
{
|
|
892
|
-
body: JSON.stringify({ target: doc.uri }),
|
|
893
|
-
method: "POST",
|
|
894
|
-
}
|
|
895
|
-
);
|
|
896
|
-
setExportingPublishArtifact(false);
|
|
897
|
-
|
|
898
|
-
if (err) {
|
|
899
|
-
setPublishExportError(err);
|
|
900
|
-
return;
|
|
901
|
-
}
|
|
902
|
-
|
|
903
|
-
if (data) {
|
|
904
|
-
downloadPublishArtifactFile(data);
|
|
905
|
-
}
|
|
906
|
-
}, [doc]);
|
|
907
|
-
|
|
908
875
|
const handleDelete = async () => {
|
|
909
876
|
if (!doc) return;
|
|
910
877
|
|
|
@@ -1784,18 +1751,13 @@ export default function DocView({ navigate }: PageProps) {
|
|
|
1784
1751
|
</Button>
|
|
1785
1752
|
<Button
|
|
1786
1753
|
className="gap-1.5"
|
|
1787
|
-
disabled={exportingPublishArtifact}
|
|
1788
1754
|
onClick={() => {
|
|
1789
|
-
|
|
1755
|
+
setPublishExportOpen(true);
|
|
1790
1756
|
}}
|
|
1791
1757
|
size="sm"
|
|
1792
1758
|
variant="outline"
|
|
1793
1759
|
>
|
|
1794
|
-
|
|
1795
|
-
<Loader2Icon className="size-4 animate-spin" />
|
|
1796
|
-
) : (
|
|
1797
|
-
<Share2Icon className="size-4" />
|
|
1798
|
-
)}
|
|
1760
|
+
<Share2Icon className="size-4" />
|
|
1799
1761
|
Export for gno.sh
|
|
1800
1762
|
</Button>
|
|
1801
1763
|
<Button
|
|
@@ -1847,18 +1809,13 @@ export default function DocView({ navigate }: PageProps) {
|
|
|
1847
1809
|
)}
|
|
1848
1810
|
<Button
|
|
1849
1811
|
className="gap-1.5"
|
|
1850
|
-
disabled={exportingPublishArtifact}
|
|
1851
1812
|
onClick={() => {
|
|
1852
|
-
|
|
1813
|
+
setPublishExportOpen(true);
|
|
1853
1814
|
}}
|
|
1854
1815
|
size="sm"
|
|
1855
1816
|
variant="outline"
|
|
1856
1817
|
>
|
|
1857
|
-
|
|
1858
|
-
<Loader2Icon className="size-4 animate-spin" />
|
|
1859
|
-
) : (
|
|
1860
|
-
<Share2Icon className="size-4" />
|
|
1861
|
-
)}
|
|
1818
|
+
<Share2Icon className="size-4" />
|
|
1862
1819
|
Export for gno.sh
|
|
1863
1820
|
</Button>
|
|
1864
1821
|
{localClient && doc.source.absPath && (
|
|
@@ -1945,11 +1902,16 @@ export default function DocView({ navigate }: PageProps) {
|
|
|
1945
1902
|
</>
|
|
1946
1903
|
)}
|
|
1947
1904
|
</div>
|
|
1948
|
-
{publishExportError && (
|
|
1949
|
-
<p className="pt-2 text-destructive text-sm">{publishExportError}</p>
|
|
1950
|
-
)}
|
|
1951
1905
|
</header>
|
|
1952
1906
|
|
|
1907
|
+
{publishExportOpen && doc && (
|
|
1908
|
+
<PublishExportDialog
|
|
1909
|
+
key={doc.uri}
|
|
1910
|
+
onClose={() => setPublishExportOpen(false)}
|
|
1911
|
+
target={doc.uri}
|
|
1912
|
+
title={doc.title ?? doc.uri}
|
|
1913
|
+
/>
|
|
1914
|
+
)}
|
|
1953
1915
|
<div className="mx-auto flex max-w-[1800px] gap-5 px-6 xl:px-8">
|
|
1954
1916
|
{/* Left rail — metadata + outline */}
|
|
1955
1917
|
{doc && (
|
|
@@ -144,6 +144,7 @@ import { getSchemaVersion, migrations, runMigrations } from "../migrations";
|
|
|
144
144
|
import { err, ok } from "../types";
|
|
145
145
|
import { getStoredEmbeddingFingerprint } from "../vector/freshness";
|
|
146
146
|
import { modelTableName } from "../vector/sqlite-vec";
|
|
147
|
+
import { getVariantStatus } from "../vector/status";
|
|
147
148
|
import {
|
|
148
149
|
deleteSavedCapsuleRegistration as deleteStoredSavedCapsuleRegistration,
|
|
149
150
|
getSavedCapsuleRegistration as getStoredSavedCapsuleRegistration,
|
|
@@ -5697,6 +5698,8 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
|
|
|
5697
5698
|
options?.embedFingerprint ??
|
|
5698
5699
|
(embedModel ? getStoredEmbeddingFingerprint(db, embedModel) : null);
|
|
5699
5700
|
|
|
5701
|
+
const variantStatus = getVariantStatus(db, options);
|
|
5702
|
+
|
|
5700
5703
|
// Get version
|
|
5701
5704
|
const versionRow = db
|
|
5702
5705
|
.query<{ value: string }, []>(
|
|
@@ -5873,12 +5876,14 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
|
|
|
5873
5876
|
errorDocuments: s.errored,
|
|
5874
5877
|
chunkedDocuments: s.chunked,
|
|
5875
5878
|
totalChunks: s.chunk_count,
|
|
5876
|
-
embeddedChunks:
|
|
5879
|
+
embeddedChunks: variantStatus
|
|
5880
|
+
? (variantStatus.embeddedByCollection.get(s.name) ?? 0)
|
|
5881
|
+
: s.embedded_count,
|
|
5877
5882
|
})),
|
|
5878
5883
|
totalDocuments: totalsRow?.total ?? 0,
|
|
5879
5884
|
activeDocuments: totalsRow?.active ?? 0,
|
|
5880
5885
|
totalChunks: chunkCount,
|
|
5881
|
-
embeddingBacklog: backlogRow?.count ?? 0,
|
|
5886
|
+
embeddingBacklog: variantStatus?.backlog ?? backlogRow?.count ?? 0,
|
|
5882
5887
|
recentErrors,
|
|
5883
5888
|
lastUpdatedAt: lastUpdatedRow?.last_updated ?? null,
|
|
5884
5889
|
healthy,
|
|
@@ -0,0 +1,196 @@
|
|
|
1
|
+
/** Read-only coverage of persisted variant authority; never initializes models. */
|
|
2
|
+
import type { Database } from "bun:sqlite";
|
|
3
|
+
|
|
4
|
+
import { getEmbeddingFingerprint } from "../../embed/fingerprint";
|
|
5
|
+
import { formatDocForEmbedding } from "../../pipeline/contextual";
|
|
6
|
+
import {
|
|
7
|
+
embeddingInputHash,
|
|
8
|
+
SELECTED_VECTOR_PARTITION_PREFIX,
|
|
9
|
+
} from "./variants";
|
|
10
|
+
|
|
11
|
+
interface Partition {
|
|
12
|
+
partition_id: string;
|
|
13
|
+
version: number;
|
|
14
|
+
model: string;
|
|
15
|
+
fingerprint: string;
|
|
16
|
+
dimensions: number;
|
|
17
|
+
state: string;
|
|
18
|
+
activated_epoch: number | null;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
interface OwnerCoverage {
|
|
22
|
+
document_id: number;
|
|
23
|
+
partition_id: string | null;
|
|
24
|
+
collection: string;
|
|
25
|
+
mirror_hash: string;
|
|
26
|
+
seq: number;
|
|
27
|
+
text: string;
|
|
28
|
+
title: string | null;
|
|
29
|
+
input_hash: string | null;
|
|
30
|
+
embedding_bytes: number | null;
|
|
31
|
+
legacy_embedded: number;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/** Null retains legacy counts until a verified selection or activation exists. */
|
|
35
|
+
export function getVariantStatus(
|
|
36
|
+
db: Database,
|
|
37
|
+
options?: { embedModel?: string; embedFingerprint?: string }
|
|
38
|
+
): { backlog: number; embeddedByCollection: Map<string, number> } | null {
|
|
39
|
+
return db.transaction(() => {
|
|
40
|
+
if (
|
|
41
|
+
!db
|
|
42
|
+
.query(
|
|
43
|
+
"SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'vector_partitions'"
|
|
44
|
+
)
|
|
45
|
+
.get()
|
|
46
|
+
)
|
|
47
|
+
return null;
|
|
48
|
+
const partitions = db
|
|
49
|
+
.query<Partition, [string | null, string | null]>(`
|
|
50
|
+
SELECT partition_id, version, model, fingerprint, dimensions, state, activated_epoch
|
|
51
|
+
FROM vector_partitions WHERE (? IS NULL OR model = ?)
|
|
52
|
+
`)
|
|
53
|
+
.all(options?.embedModel ?? null, options?.embedModel ?? null);
|
|
54
|
+
const selections = db
|
|
55
|
+
.query<{ key: string; value: string }, [string]>(
|
|
56
|
+
"SELECT key, value FROM schema_meta WHERE key GLOB ?"
|
|
57
|
+
)
|
|
58
|
+
.all(`${SELECTED_VECTOR_PARTITION_PREFIX}*`);
|
|
59
|
+
const selected = new Map(
|
|
60
|
+
selections.map((row) => [
|
|
61
|
+
row.key.slice(SELECTED_VECTOR_PARTITION_PREFIX.length),
|
|
62
|
+
row.value,
|
|
63
|
+
])
|
|
64
|
+
);
|
|
65
|
+
const models = new Set(partitions.map((p) => p.model));
|
|
66
|
+
if (options?.embedModel) models.add(options.embedModel);
|
|
67
|
+
else for (const model of selected.keys()) models.add(model);
|
|
68
|
+
const authoritativeModels = new Set<string>();
|
|
69
|
+
const usablePartitions = new Map<string, Partition>();
|
|
70
|
+
for (const model of models) {
|
|
71
|
+
const candidates = partitions.filter((p) => p.model === model);
|
|
72
|
+
const selection = selected.get(model);
|
|
73
|
+
const activated = candidates.some(
|
|
74
|
+
(p) => p.state === "active" && p.activated_epoch !== null
|
|
75
|
+
);
|
|
76
|
+
if (selection === undefined && !activated) continue;
|
|
77
|
+
authoritativeModels.add(model);
|
|
78
|
+
// Resolve one persisted identity per model, never combine alternative
|
|
79
|
+
// partitions of the same model. Unscoped status may accept any model.
|
|
80
|
+
// Stale epochs do not revoke owners whose current inputs still match.
|
|
81
|
+
const partition =
|
|
82
|
+
selection === undefined
|
|
83
|
+
? candidates.length === 1
|
|
84
|
+
? candidates[0]
|
|
85
|
+
: undefined
|
|
86
|
+
: candidates.find((p) => p.partition_id === selection);
|
|
87
|
+
if (
|
|
88
|
+
partition &&
|
|
89
|
+
partition.version === 1 &&
|
|
90
|
+
partition.partition_id ===
|
|
91
|
+
embeddingInputHash(
|
|
92
|
+
JSON.stringify([
|
|
93
|
+
partition.model,
|
|
94
|
+
partition.fingerprint,
|
|
95
|
+
partition.dimensions,
|
|
96
|
+
])
|
|
97
|
+
) &&
|
|
98
|
+
(options?.embedFingerprint === undefined ||
|
|
99
|
+
options.embedFingerprint ===
|
|
100
|
+
getEmbeddingFingerprint({
|
|
101
|
+
modelUri: partition.model,
|
|
102
|
+
dimensions: partition.dimensions,
|
|
103
|
+
}))
|
|
104
|
+
)
|
|
105
|
+
usablePartitions.set(partition.partition_id, partition);
|
|
106
|
+
}
|
|
107
|
+
if (authoritativeModels.size === 0) return null;
|
|
108
|
+
// Unscoped legacy coverage remains valid only for models that have never
|
|
109
|
+
// selected or activated verified authority; legacy rows cannot repair it.
|
|
110
|
+
const statement = db.prepare<
|
|
111
|
+
OwnerCoverage,
|
|
112
|
+
[string | null, string, string]
|
|
113
|
+
>(`
|
|
114
|
+
WITH legacy_vectors AS (
|
|
115
|
+
SELECT mirror_hash, seq, MAX(embedded_at) AS embedded_at
|
|
116
|
+
FROM content_vectors
|
|
117
|
+
WHERE ? IS NULL AND model NOT IN (SELECT value FROM json_each(?))
|
|
118
|
+
GROUP BY mirror_hash, seq
|
|
119
|
+
)
|
|
120
|
+
SELECT d.id AS document_id, o.partition_id, d.collection, d.mirror_hash, c.seq, c.text, d.title,
|
|
121
|
+
v.input_hash, length(v.embedding) AS embedding_bytes,
|
|
122
|
+
CASE WHEN lv.embedded_at >= c.created_at THEN 1 ELSE 0 END AS legacy_embedded
|
|
123
|
+
FROM documents d
|
|
124
|
+
JOIN content_chunks c ON c.mirror_hash = d.mirror_hash
|
|
125
|
+
LEFT JOIN vector_owners o ON o.document_id = d.id AND o.seq = c.seq
|
|
126
|
+
AND o.mirror_hash = d.mirror_hash
|
|
127
|
+
AND o.partition_id IN (SELECT value FROM json_each(?))
|
|
128
|
+
LEFT JOIN vector_variants v ON v.variant_id = o.variant_id
|
|
129
|
+
AND v.partition_id = o.partition_id
|
|
130
|
+
LEFT JOIN legacy_vectors lv ON lv.mirror_hash = c.mirror_hash AND lv.seq = c.seq
|
|
131
|
+
WHERE d.active = 1
|
|
132
|
+
`);
|
|
133
|
+
const owners = new Map<
|
|
134
|
+
string,
|
|
135
|
+
{ collection: string; chunk: string; embedded: boolean }
|
|
136
|
+
>();
|
|
137
|
+
try {
|
|
138
|
+
for (const row of statement.iterate(
|
|
139
|
+
options?.embedModel ?? null,
|
|
140
|
+
JSON.stringify([...authoritativeModels]),
|
|
141
|
+
JSON.stringify([...usablePartitions.keys()])
|
|
142
|
+
)) {
|
|
143
|
+
const partition = row.partition_id
|
|
144
|
+
? usablePartitions.get(row.partition_id)
|
|
145
|
+
: undefined;
|
|
146
|
+
const embedded = Boolean(
|
|
147
|
+
row.legacy_embedded ||
|
|
148
|
+
(partition &&
|
|
149
|
+
row.embedding_bytes ===
|
|
150
|
+
partition.dimensions * Float32Array.BYTES_PER_ELEMENT &&
|
|
151
|
+
row.input_hash ===
|
|
152
|
+
embeddingInputHash(
|
|
153
|
+
formatDocForEmbedding(
|
|
154
|
+
row.text,
|
|
155
|
+
row.title ?? undefined,
|
|
156
|
+
partition.model
|
|
157
|
+
)
|
|
158
|
+
))
|
|
159
|
+
);
|
|
160
|
+
const key = `${row.document_id}:${row.seq}`;
|
|
161
|
+
const previous = owners.get(key);
|
|
162
|
+
if (previous) previous.embedded ||= embedded;
|
|
163
|
+
else
|
|
164
|
+
owners.set(key, {
|
|
165
|
+
collection: row.collection,
|
|
166
|
+
chunk: JSON.stringify([row.mirror_hash, row.seq]),
|
|
167
|
+
embedded,
|
|
168
|
+
});
|
|
169
|
+
}
|
|
170
|
+
} finally {
|
|
171
|
+
statement.finalize();
|
|
172
|
+
}
|
|
173
|
+
const collections = new Map<string, Map<string, boolean>>();
|
|
174
|
+
let backlog = 0;
|
|
175
|
+
for (const owner of owners.values()) {
|
|
176
|
+
if (!owner.embedded) backlog++;
|
|
177
|
+
let chunks = collections.get(owner.collection);
|
|
178
|
+
if (!chunks) {
|
|
179
|
+
chunks = new Map();
|
|
180
|
+
collections.set(owner.collection, chunks);
|
|
181
|
+
}
|
|
182
|
+
// Distinct collection chunks are ready only when every active owner is.
|
|
183
|
+
chunks.set(
|
|
184
|
+
owner.chunk,
|
|
185
|
+
(chunks.get(owner.chunk) ?? true) && owner.embedded
|
|
186
|
+
);
|
|
187
|
+
}
|
|
188
|
+
const embeddedByCollection = new Map<string, number>();
|
|
189
|
+
for (const [collection, chunks] of collections) {
|
|
190
|
+
let count = 0;
|
|
191
|
+
for (const embedded of chunks.values()) if (embedded) count++;
|
|
192
|
+
embeddedByCollection.set(collection, count);
|
|
193
|
+
}
|
|
194
|
+
return { backlog, embeddedByCollection };
|
|
195
|
+
})();
|
|
196
|
+
}
|
|
@@ -6,6 +6,8 @@ import type { VectorOwnerInput, VectorVariantIdentity } from "./types";
|
|
|
6
6
|
import { formatDocForEmbedding } from "../../pipeline/contextual";
|
|
7
7
|
import { decodeEmbedding, encodeEmbedding } from "./sqlite-vec";
|
|
8
8
|
|
|
9
|
+
export const SELECTED_VECTOR_PARTITION_PREFIX = "vector_selected_partition:";
|
|
10
|
+
|
|
9
11
|
export function embeddingInputHash(input: string): string {
|
|
10
12
|
return new Bun.CryptoHasher("sha256").update(input).digest("hex");
|
|
11
13
|
}
|
|
@@ -94,6 +96,15 @@ export class VectorVariantStore {
|
|
|
94
96
|
}).immediate();
|
|
95
97
|
}
|
|
96
98
|
|
|
99
|
+
/** Persist only after embedding has resolved the actual runtime identity. */
|
|
100
|
+
selectForEmbedding(): void {
|
|
101
|
+
this.db.run(
|
|
102
|
+
`INSERT INTO schema_meta (key, value) VALUES (?, ?)
|
|
103
|
+
ON CONFLICT(key) DO UPDATE SET value = excluded.value, updated_at = datetime('now')`,
|
|
104
|
+
[SELECTED_VECTOR_PARTITION_PREFIX + this.identity.model, this.partitionId]
|
|
105
|
+
);
|
|
106
|
+
}
|
|
107
|
+
|
|
97
108
|
epoch(): number {
|
|
98
109
|
return this.db
|
|
99
110
|
.query<{ epoch: number }, []>(
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
cb9a683d5e8bddd30f0a2aaf00e252065c85e7779ff3da50e9e87636a855fd8d gno-browser-clipper-v2.0.0.zip
|