@hraness/sponge 0.2.6 → 0.3.0

package/README.md

@@ -79,6 +79,9 @@ to that repository's contents. Sponge never saves the token and sends it only to
 GitHub's API; public model downloads do not receive it. With an authorized GitHub
 CLI session, for example: `GH_TOKEN="$(gh auth token)" sponge setup --asr`.
 Repeat with `--embedding` for search. Already verified caches work without a token.
+Installed helpers and models are identified by content, so an upgrade that ships
+byte-identical native artifacts keeps working without rerunning setup; a release
+that changes them reports `runtime-missing` until setup runs again.
 
 ```sh
 sponge setup --asr
@@ -87,7 +90,7 @@ sponge refresh ITEM --wait --language en --force-local
 sponge setup --embedding
 sponge index create
 sponge index run
-sponge index status GENERATION
+sponge index status
 sponge search 'your query'
 ```
 
@@ -98,11 +101,16 @@ saves the English ASR choice for background jobs. Search combines local
 full-text and semantic results when a compatible index and model are available,
 and reports when semantic search is unavailable. `--mode lexical` uses only
 full-text search. Use `index create --profile NAME` and `search QUERY --profile NAME`
-to keep separate search profiles. Model changes create separate index generations; a new one
-becomes active only after it covers the current library. `index rollback
+to keep separate search profiles. The active index follows the library: each new capture,
+refresh or delete queues only its new chunks, which the worker (or `index run`) embeds, and
+identical chunks are embedded once. `index status` shows the active generation's coverage
+and what is pending; `index status GENERATION` shows one generation. Model changes create
+separate index generations; a new one becomes active only after it covers the current
+library. `index rollback
 GENERATION` requires a complete generation compatible with the current revisions.
 `index prune --dry-run` lists what pruning would delete; `index prune` deletes the
-vectors of cancelled, failed and retired generations, keeping the active and any
+coverage of cancelled, failed and retired generations, then the vectors no kept generation
+covers (including those of deleted items), keeping the active and any
 building generation plus the most recent rollback-ready one per profile (`--keep N`,
 0-16). It refuses while index work, cleanup or a backup is running. Freed pages are
 reused; `--vacuum` also shrinks the file after checking free space. Rollback to a
@@ -146,9 +154,11 @@ sponge worker status
 sponge worker uninstall
 ```
 
-Exports contain Markdown, images and saved notes. Backups preserve revisions,
-assets, reading state and notes. Restore requires a new directory, verifies every
-file and leaves connections unresolved; runtimes and credentials are excluded.
+Exports contain Markdown, images and saved notes, never the publishing key. Backups
+preserve revisions, assets, reading state, notes and the publishing key, so a
+restored library publishes under the same identity; keep backups private. Restore
+requires a new directory, verifies every file and leaves connections unresolved;
+runtimes and credentials are excluded.
 Garbage collection removes only unreferenced blobs after rechecking the report.
 `gc --runtimes` removes capture runtimes and model packages this Sponge no longer
 pins, plus settled worker journals; `--dry-run` only lists them. It holds worker
@@ -156,8 +166,10 @@ custody, keeps models an active or building index generation needs, and refuses
 symlinks and anything in use. Successful `setup` runs the same collection.
 
 The background worker is opt-in and processes work sequentially. Its LaunchAgent
-pins the installed command; after an upgrade, reinstall the worker to update that
-binding. Uninstall stops the worker and preserves the library. In-session startup
+pins the installed command and Node; after a Sponge or Node upgrade, `worker status`
+and `doctor` report the drift and `sponge worker install` reinstalls it in place.
+Its output goes to `runtime/service/launchd.log` in the library, rotated at 1 MiB.
+Uninstall stops the worker and preserves the library. In-session startup
 and cleanup are checked; fresh-login startup remains unqualified. See
 [importing existing captures](docs/importing.md) for a preview-first import flow.
 

package/bin/sponge.js

@@ -1,10 +1,11 @@
 #!/usr/bin/env node
 import { createRequire } from 'node:module';
-const [major,minor]=process.versions.node.split('.').map(Number);
-if(major<24||(major===24&&minor<20)){
-  process.stderr.write('Sponge requires Node.js 24.20 or newer.\n');process.exitCode=2;
-}else{
-  const {version}=createRequire(import.meta.url)('../package.json');
-  const {main}=await import('../dist/src/cli/main.js');
-  process.exitCode=await main(process.argv.slice(2),version);
+const [major, minor] = process.versions.node.split('.').map(Number);
+if (major < 24 || (major === 24 && minor < 20)) {
+  process.stderr.write('Sponge requires Node.js 24.20 or newer.\n');
+  process.exitCode = 2;
+} else {
+  const { version } = createRequire(import.meta.url)('../package.json');
+  const { main } = await import('../dist/src/cli/main.js');
+  process.exitCode = await main(process.argv.slice(2), version);
 }

package/dist/contracts/index.js

@@ -1,6 +1,24 @@
-export const exitCodes = { success: 0, failure: 1, 'invalid-input': 2, partial: 3, 'action-required': 4, cancelled: 130 };
-export const limits = Object.freeze({ pdfBytes: 100 * 1024 ** 2, imageBytes: 20 * 1024 ** 2,
-    imagePixels: 40_000_000, imagesPerItem: 200, mediaDurationMs: 3_600_000,
-    temporaryMediaBytes: 1024 ** 3, stagingBytes: 2 * 1024 ** 3, freeReserveBytes: 2 * 1024 ** 3,
-    metadataBytes: 1024 ** 2, metadataClaims: 512, markdownBytes: 16 * 1024 ** 2,
-    captureDeadlineMs: 120_000, helperFrameBytes: 64 * 1024, helperOutputBytes: 16 * 1024 ** 2 });
+export const exitCodes = {
+    success: 0,
+    failure: 1,
+    'invalid-input': 2,
+    partial: 3,
+    'action-required': 4,
+    cancelled: 130,
+};
+export const limits = Object.freeze({
+    pdfBytes: 100 * 1024 ** 2,
+    imageBytes: 20 * 1024 ** 2,
+    imagePixels: 40_000_000,
+    imagesPerItem: 200,
+    mediaDurationMs: 3_600_000,
+    temporaryMediaBytes: 1024 ** 3,
+    stagingBytes: 2 * 1024 ** 3,
+    freeReserveBytes: 2 * 1024 ** 3,
+    metadataBytes: 1024 ** 2,
+    metadataClaims: 512,
+    markdownBytes: 16 * 1024 ** 2,
+    captureDeadlineMs: 120_000,
+    helperFrameBytes: 64 * 1024,
+    helperOutputBytes: 16 * 1024 ** 2,
+});

package/dist/contracts/validate.js

@@ -2,18 +2,32 @@ import { isAbsolute } from 'node:path';
 import { createHash } from 'node:crypto';
 import { limits } from './index.js';
 const digest = /^[a-f0-9]{64}$/;
-function requireValue(ok) { if (!ok)
-    throw new Error('invalid-contract'); }
-function record(value) { requireValue(value !== null && typeof value === 'object' && !Array.isArray(value)); }
-function boundedString(value, max = 4096) { requireValue(typeof value === 'string' && value.length > 0 && value.length <= max && !value.includes('\0')); }
-function onlyKeys(value, keys) { requireValue(Object.keys(value).every(key => keys.includes(key))); }
-function envelope(value) { requireValue(value.schemaVersion === 1); boundedString(value.requestId, 128); }
+function requireValue(ok) {
+    if (!ok)
+        throw new Error('invalid-contract');
+}
+function record(value) {
+    requireValue(value !== null && typeof value === 'object' && !Array.isArray(value));
+}
+function boundedString(value, max = 4096) {
+    requireValue(typeof value === 'string' && value.length > 0 && value.length <= max && !value.includes('\0'));
+}
+function onlyKeys(value, keys) {
+    requireValue(Object.keys(value).every((key) => keys.includes(key)));
+}
+function envelope(value) {
+    requireValue(value.schemaVersion === 1);
+    boundedString(value.requestId, 128);
+}
 /** Lexical confinement only. Consumers MUST additionally no-follow open/hash the bytes. */
 export function validateAsset(value) {
     record(value);
     requireValue(typeof value.sha256 === 'string' && digest.test(value.sha256));
     boundedString(value.relativePath);
-    requireValue(!value.relativePath.startsWith('/') && !value.relativePath.includes('\\') && !value.relativePath.includes(':') && value.relativePath.split('/').every(p => p !== '' && p !== '.' && p !== '..'));
+    requireValue(!value.relativePath.startsWith('/') &&
+        !value.relativePath.includes('\\') &&
+        !value.relativePath.includes(':') &&
+        value.relativePath.split('/').every((p) => p !== '' && p !== '.' && p !== '..'));
     boundedString(value.mediaType, 128);
     requireValue(Number.isSafeInteger(value.bytes) && Number(value.bytes) >= 0);
 }
@@ -27,7 +41,10 @@ export function validateCapture(value) {
     validateAsset(value.markdown);
     requireValue(value.markdown.bytes <= limits.markdownBytes && value.markdown.mediaType === 'text/markdown');
     requireValue(Array.isArray(value.assets) && value.assets.length <= limits.imagesPerItem);
-    value.assets.forEach(asset => { validateAsset(asset); requireValue(asset.mediaType.startsWith('image/') && asset.bytes <= limits.imageBytes); });
+    value.assets.forEach((asset) => {
+        validateAsset(asset);
+        requireValue(asset.mediaType.startsWith('image/') && asset.bytes <= limits.imageBytes);
+    });
     if (value.sourcePdf !== undefined) {
         validateAsset(value.sourcePdf);
         requireValue(value.sourcePdf.mediaType === 'application/pdf' && value.sourcePdf.bytes <= limits.pdfBytes);
@@ -35,30 +52,38 @@ export function validateCapture(value) {
     requireValue(value.kind !== 'pdf' || value.sourcePdf !== undefined);
     record(value.completeness);
     requireValue(typeof value.completeness.text === 'string' && ['complete', 'partial'].includes(String(value.completeness.text)));
-    requireValue(typeof value.completeness.images === 'string' && ['complete', 'partial', 'not-requested'].includes(String(value.completeness.images)));
+    requireValue(typeof value.completeness.images === 'string' &&
+        ['complete', 'partial', 'not-requested'].includes(String(value.completeness.images)));
     requireValue(value.status !== 'complete' || (value.completeness.text === 'complete' && value.completeness.images !== 'partial'));
     record(value.extractor);
     boundedString(value.extractor.name, 128);
     boundedString(value.extractor.version, 128);
     requireValue(Array.isArray(value.warnings) && value.warnings.length <= 128);
-    value.warnings.forEach(v => requireValue(errorCodes.has(v)));
-    requireValue(Array.isArray(value.metadata) && value.metadata.length <= limits.metadataClaims && Buffer.byteLength(JSON.stringify(value.metadata)) <= limits.metadataBytes);
+    value.warnings.forEach((v) => requireValue(errorCodes.has(v)));
+    requireValue(Array.isArray(value.metadata) &&
+        value.metadata.length <= limits.metadataClaims &&
+        Buffer.byteLength(JSON.stringify(value.metadata)) <= limits.metadataBytes);
     value.metadata.forEach(validateMetadataClaim);
     if (value.metadataWarnings !== undefined)
         validateMetadataWarnings(value.metadataWarnings);
 }
 export function validateMetadataWarnings(value) {
-    requireValue(Array.isArray(value) && value.length <= 5 && new Set(value).size === value.length && value.every(v => typeof v === 'string' && ['input-limit', 'claim-limit', 'value-limit', 'json-limit', 'json-invalid'].includes(v)));
+    requireValue(Array.isArray(value) &&
+        value.length <= 5 &&
+        new Set(value).size === value.length &&
+        value.every((v) => typeof v === 'string' &&
+            ['input-limit', 'claim-limit', 'value-limit', 'json-limit', 'json-invalid'].includes(v)));
 }
 export function validateMetadataClaim(claim) {
     record(claim);
     boundedString(claim.field, 128);
     boundedString(claim.observedAt, 64);
     requireValue(Number.isFinite(Date.parse(claim.observedAt)));
-    requireValue(typeof claim.source === 'string' && ['html', 'opengraph', 'twitter', 'jsonld', 'pdf', 'media'].includes(String(claim.source)));
+    requireValue(typeof claim.source === 'string' &&
+        ['html', 'opengraph', 'twitter', 'jsonld', 'pdf', 'media'].includes(String(claim.source)));
     if (Array.isArray(claim.value)) {
         requireValue(claim.value.length <= 128);
-        claim.value.forEach(v => boundedString(v, 16384));
+        claim.value.forEach((v) => boundedString(v, 16384));
     }
     else
         boundedString(claim.value, 16384);
@@ -69,7 +94,7 @@ export function validateMetadataClaim(claim) {
     if (claim.observed !== undefined) {
         if (Array.isArray(claim.observed)) {
             requireValue(claim.observed.length <= 128);
-            claim.observed.forEach(v => boundedString(v, 16384));
+            claim.observed.forEach((v) => boundedString(v, 16384));
         }
         else
             boundedString(claim.observed, 16384);
@@ -83,27 +108,71 @@ export function validateMetadataClaim(claim) {
     if (claim.datePrecision !== undefined)
         requireValue(typeof claim.datePrecision === 'string' && ['date', 'datetime', 'invalid'].includes(claim.datePrecision));
 }
-const errorCodes = new Set(['grant-permission', 'repair-auth', 'unsupported', 'unavailable', 'rate-limited', 'partial', 'cleanup-required', 'cancelled', 'invalid-input', 'unclassified', 'integrity', 'limit-exceeded', 'runtime-missing', 'schema-newer', 'owner-busy']);
+const errorCodes = new Set([
+    'grant-permission',
+    'repair-auth',
+    'unsupported',
+    'unavailable',
+    'rate-limited',
+    'partial',
+    'cleanup-required',
+    'cancelled',
+    'invalid-input',
+    'unclassified',
+    'integrity',
+    'limit-exceeded',
+    'runtime-missing',
+    'schema-newer',
+    'owner-busy',
+]);
 export function validateError(value) {
     record(value);
     envelope(value);
     onlyKeys(value, ['schemaVersion', 'requestId', 'ok', 'code', 'retryable', 'retryAfterMs', 'phase']);
-    requireValue(value.ok === false && typeof value.code === 'string' && errorCodes.has(value.code) && typeof value.retryable === 'boolean');
+    requireValue(value.ok === false &&
+        typeof value.code === 'string' &&
+        errorCodes.has(value.code) &&
+        typeof value.retryable === 'boolean');
     requireValue(!value.retryable || value.code === 'unavailable' || value.code === 'rate-limited');
     if (value.phase !== undefined)
-        requireValue(typeof value.phase === 'string' && ['setup', 'acquisition', 'verification', 'storage', 'inference', 'cleanup'].includes(String(value.phase)));
+        requireValue(typeof value.phase === 'string' &&
+            ['setup', 'acquisition', 'verification', 'storage', 'inference', 'cleanup'].includes(String(value.phase)));
     if (value.retryAfterMs !== undefined)
-        requireValue(Number.isSafeInteger(value.retryAfterMs) && Number(value.retryAfterMs) >= 0 && Number(value.retryAfterMs) <= 86_400_000);
+        requireValue(Number.isSafeInteger(value.retryAfterMs) &&
+            Number(value.retryAfterMs) >= 0 &&
+            Number(value.retryAfterMs) <= 86_400_000);
 }
 /** Fixed ordered tuple avoids key-order or irrelevant envelope fields changing a space. */
 export function embeddingFingerprint(identity) {
-    const hashes = [identity.modelSha256, identity.runtimeSha256, identity.tokenizerSha256, identity.queryTemplateSha256, identity.documentTemplateSha256, identity.preprocessingSha256];
-    requireValue(hashes.every(h => typeof h === 'string' && digest.test(h)));
+    const hashes = [
+        identity.modelSha256,
+        identity.runtimeSha256,
+        identity.tokenizerSha256,
+        identity.queryTemplateSha256,
+        identity.documentTemplateSha256,
+        identity.preprocessingSha256,
+    ];
+    requireValue(hashes.every((h) => typeof h === 'string' && digest.test(h)));
     requireValue(Number.isSafeInteger(identity.dimensions) && identity.dimensions > 0 && identity.dimensions <= 16384);
     requireValue(identity.metric === 'cosine' && typeof identity.normalized === 'boolean');
-    requireValue(Array.isArray(identity.modalities) && identity.modalities.length > 0 && identity.modalities.every(m => m === 'text' || m === 'image') && new Set(identity.modalities).size === identity.modalities.length);
-    [identity.pooling, identity.quantization, identity.backend].forEach(v => boundedString(v, 256));
-    return createHash('sha256').update(JSON.stringify(['sponge-space-v1', ...hashes, identity.dimensions, [...identity.modalities].sort(), identity.normalized, identity.metric, identity.pooling, identity.quantization, identity.backend])).digest('hex');
+    requireValue(Array.isArray(identity.modalities) &&
+        identity.modalities.length > 0 &&
+        identity.modalities.every((m) => m === 'text' || m === 'image') &&
+        new Set(identity.modalities).size === identity.modalities.length);
+    [identity.pooling, identity.quantization, identity.backend].forEach((v) => boundedString(v, 256));
+    return createHash('sha256')
+        .update(JSON.stringify([
+        'sponge-space-v1',
+        ...hashes,
+        identity.dimensions,
+        [...identity.modalities].sort(),
+        identity.normalized,
+        identity.metric,
+        identity.pooling,
+        identity.quantization,
+        identity.backend,
+    ]))
+        .digest('hex');
 }
 /** Manifest validation is admission metadata, not proof of downloaded bytes. */
 export function validateModelManifest(value) {
@@ -145,8 +214,10 @@ export function validateJobCheckpoint(value) {
     for (const key of ['jobId', 'attemptId', 'ownerIncarnation'])
         boundedString(value[key], 128);
     requireValue(Number.isSafeInteger(value.fence) && Number(value.fence) > 0);
-    requireValue(typeof value.stage === 'string' && ['queued', 'acquiring', 'extracting', 'indexing', 'promoting', 'cleanup', 'settled'].includes(String(value.stage)));
-    requireValue(typeof value.status === 'string' && ['pending', 'running', 'action-required', 'cancelled', 'failed', 'complete'].includes(String(value.status)));
+    requireValue(typeof value.stage === 'string' &&
+        ['queued', 'acquiring', 'extracting', 'indexing', 'promoting', 'cleanup', 'settled'].includes(String(value.stage)));
+    requireValue(typeof value.status === 'string' &&
+        ['pending', 'running', 'action-required', 'cancelled', 'failed', 'complete'].includes(String(value.status)));
     requireValue(value.cleanup === 'settled' || value.cleanup === 'pending');
     requireValue(value.stage !== 'settled' || value.cleanup === 'settled');
     requireValue(value.status !== 'complete' || value.stage === 'settled');
@@ -159,8 +230,12 @@ export function validateHelperRequest(value) {
     envelope(value);
     requireValue(Buffer.byteLength(JSON.stringify(value)) <= limits.helperFrameBytes);
     requireValue(Number.isSafeInteger(value.deadlineMs) && Number(value.deadlineMs) > 0 && Number(value.deadlineMs) <= 86_400_000);
-    requireValue(Number.isSafeInteger(value.maxInputBytes) && Number(value.maxInputBytes) > 0 && Number(value.maxInputBytes) <= limits.temporaryMediaBytes);
-    requireValue(Number.isSafeInteger(value.maxOutputBytes) && Number(value.maxOutputBytes) > 0 && Number(value.maxOutputBytes) <= limits.helperOutputBytes);
+    requireValue(Number.isSafeInteger(value.maxInputBytes) &&
+        Number(value.maxInputBytes) > 0 &&
+        Number(value.maxInputBytes) <= limits.temporaryMediaBytes);
+    requireValue(Number.isSafeInteger(value.maxOutputBytes) &&
+        Number(value.maxOutputBytes) > 0 &&
+        Number(value.maxOutputBytes) <= limits.helperOutputBytes);
     const keys = ['schemaVersion', 'requestId', 'operation', 'deadlineMs', 'maxInputBytes', 'maxOutputBytes'];
     if (value.operation === 'cancel') {
         onlyKeys(value, [...keys, 'targetRequestId']);
@@ -181,7 +256,9 @@ export function validateHelperRequest(value) {
     boundedString(value.input.ownedPath);
     requireValue(isAbsolute(value.input.ownedPath));
     requireValue(typeof value.input.sha256 === 'string' && digest.test(value.input.sha256));
-    requireValue(Number.isSafeInteger(value.input.bytes) && Number(value.input.bytes) > 0 && Number(value.input.bytes) <= Number(value.maxInputBytes));
+    requireValue(Number.isSafeInteger(value.input.bytes) &&
+        Number(value.input.bytes) > 0 &&
+        Number(value.input.bytes) <= Number(value.maxInputBytes));
     if (value.operation === 'embed-image')
         requireValue(Number(value.input.bytes) <= limits.imageBytes);
 }

package/dist/migrations/008.d.ts

@@ -0,0 +1,3 @@
+/** Backup fences name their holder (a process identity), so a fence left by a crashed backup can be expired on exact death proof. Additive: older rows keep NULL and stay unknown. */
+export declare const version = 8;
+export declare const sql = "\nALTER TABLE backup_fences ADD COLUMN owner_json TEXT;\n";

package/dist/migrations/008.js

@@ -0,0 +1,5 @@
+/** Backup fences name their holder (a process identity), so a fence left by a crashed backup can be expired on exact death proof. Additive: older rows keep NULL and stay unknown. */
+export const version = 8;
+export const sql = `
+ALTER TABLE backup_fences ADD COLUMN owner_json TEXT;
+`;

package/dist/migrations/009.d.ts

@@ -0,0 +1,3 @@
+/** Sync readiness: local paths move to a local-only table, and titles split into captured, importer-note and user-override fields (`items.title` stays the effective title). The code step re-keys immutable objects; see docs/decisions/content-ids.md. */
+export declare const version = 9;
+export declare const sql = "\nCREATE TABLE local_sources(item_id TEXT PRIMARY KEY REFERENCES items(id),locator TEXT NOT NULL CHECK(locator LIKE 'file:%')) STRICT;\nALTER TABLE items ADD COLUMN title_captured TEXT NOT NULL DEFAULT '';\nALTER TABLE items ADD COLUMN title_note TEXT;\nALTER TABLE items ADD COLUMN title_override TEXT;\n";

package/dist/migrations/009.js

@@ -0,0 +1,8 @@
+/** Sync readiness: local paths move to a local-only table, and titles split into captured, importer-note and user-override fields (`items.title` stays the effective title). The code step re-keys immutable objects; see docs/decisions/content-ids.md. */
+export const version = 9;
+export const sql = `
+CREATE TABLE local_sources(item_id TEXT PRIMARY KEY REFERENCES items(id),locator TEXT NOT NULL CHECK(locator LIKE 'file:%')) STRICT;
+ALTER TABLE items ADD COLUMN title_captured TEXT NOT NULL DEFAULT '';
+ALTER TABLE items ADD COLUMN title_note TEXT;
+ALTER TABLE items ADD COLUMN title_override TEXT;
+`;

package/dist/migrations/010.d.ts

@@ -0,0 +1,3 @@
+/** Incremental semantic index: vectors are keyed by space fingerprint and content hash, so identical chunks embed once across items, revisions and generations; coverage moves from `coverage_json` to rows a capture or delete can update. `vectors` moves to a connection-private temporary table that the code step converts and drops, so the stored schema is exactly this SQL; see docs/decisions/incremental-index.md. */
+export declare const version = 10;
+export declare const sql = "\nCREATE TABLE embeddings(fingerprint TEXT NOT NULL REFERENCES embedding_spaces(fingerprint),content_hash TEXT NOT NULL CHECK(length(content_hash)=64),input_hash TEXT NOT NULL CHECK(length(input_hash)=64),vector BLOB NOT NULL,PRIMARY KEY(fingerprint,content_hash)) STRICT;\nCREATE TABLE index_items(generation_id TEXT NOT NULL REFERENCES index_generations(id),item_id TEXT NOT NULL REFERENCES items(id),revision_id TEXT NOT NULL REFERENCES revisions(id),PRIMARY KEY(generation_id,item_id)) STRICT,WITHOUT ROWID;\nCREATE TABLE index_inputs(generation_id TEXT NOT NULL REFERENCES index_generations(id),source_id TEXT NOT NULL,revision_id TEXT NOT NULL REFERENCES revisions(id),content_hash TEXT NOT NULL CHECK(length(content_hash)=64),PRIMARY KEY(generation_id,source_id)) STRICT;\nCREATE INDEX index_inputs_revision ON index_inputs(generation_id,revision_id);\nCREATE INDEX index_inputs_content ON index_inputs(content_hash);\nCREATE TEMP TABLE vectors_v9 AS SELECT * FROM main.vectors;\nDROP TABLE main.vectors;\n";

package/dist/migrations/010.js

@@ -0,0 +1,11 @@
+/** Incremental semantic index: vectors are keyed by space fingerprint and content hash, so identical chunks embed once across items, revisions and generations; coverage moves from `coverage_json` to rows a capture or delete can update. `vectors` moves to a connection-private temporary table that the code step converts and drops, so the stored schema is exactly this SQL; see docs/decisions/incremental-index.md. */
+export const version = 10;
+export const sql = `
+CREATE TABLE embeddings(fingerprint TEXT NOT NULL REFERENCES embedding_spaces(fingerprint),content_hash TEXT NOT NULL CHECK(length(content_hash)=64),input_hash TEXT NOT NULL CHECK(length(input_hash)=64),vector BLOB NOT NULL,PRIMARY KEY(fingerprint,content_hash)) STRICT;
+CREATE TABLE index_items(generation_id TEXT NOT NULL REFERENCES index_generations(id),item_id TEXT NOT NULL REFERENCES items(id),revision_id TEXT NOT NULL REFERENCES revisions(id),PRIMARY KEY(generation_id,item_id)) STRICT,WITHOUT ROWID;
+CREATE TABLE index_inputs(generation_id TEXT NOT NULL REFERENCES index_generations(id),source_id TEXT NOT NULL,revision_id TEXT NOT NULL REFERENCES revisions(id),content_hash TEXT NOT NULL CHECK(length(content_hash)=64),PRIMARY KEY(generation_id,source_id)) STRICT;
+CREATE INDEX index_inputs_revision ON index_inputs(generation_id,revision_id);
+CREATE INDEX index_inputs_content ON index_inputs(content_hash);
+CREATE TEMP TABLE vectors_v9 AS SELECT * FROM main.vectors;
+DROP TABLE main.vectors;
+`;

package/dist/migrations/index.d.ts

@@ -0,0 +1,15 @@
+import type { DatabaseSync } from 'node:sqlite';
+/**
+ * One schema step. `sql` is immutable once released: `schema_migrations` stores its digest and every open checks it.
+ * `step` runs after `sql` in the same `BEGIN IMMEDIATE`; code steps are total per row, so one hostile row never makes a
+ * library unopenable.
+ */
+export interface Migration {
+    readonly version: number;
+    readonly sql: string;
+    readonly step?: (db: DatabaseSync) => void;
+}
+/** Every migration, in order. Add a new numbered file and append it here; never edit or reorder a released entry. */
+export declare const migrations: readonly Migration[];
+/** The schema version this build writes. */
+export declare const schemaVersion: number;

package/dist/migrations/index.js

@@ -0,0 +1,39 @@
+import { backfillUris } from '../src/identity/visibility.js';
+import { seedChanges } from '../src/store/changes.js';
+import * as m1 from './001.js';
+import * as m2 from './002.js';
+import * as m3 from './003.js';
+import * as m4 from './004.js';
+import * as m5 from './005.js';
+import * as m6 from './006.js';
+import * as m7 from './007.js';
+import * as m8 from './008.js';
+import * as m9 from './009.js';
+import * as m10 from './010.js';
+import { syncReady } from '../src/store/sync-ready.js';
+import { incrementalIndex } from '../src/indexing/upgrade.js';
+/** Every migration, in order. Add a new numbered file and append it here; never edit or reorder a released entry. */
+export const migrations = Object.freeze([
+    m1,
+    m2,
+    m3,
+    m4,
+    {
+        ...m5,
+        step: (db) => {
+            backfillUris(db);
+            seedChanges(db);
+        },
+    },
+    m6,
+    m7,
+    m8,
+    { ...m9, step: syncReady },
+    { ...m10, step: incrementalIndex },
+].map((m) => Object.freeze({ ...m })));
+migrations.forEach((m, i) => {
+    if (m.version !== i + 1)
+        throw new Error(`migration ${i + 1} declares version ${m.version}`);
+});
+/** The schema version this build writes. */
+export const schemaVersion = migrations.length;