@hraness/sponge 0.2.4 → 0.2.5

package/dist/src/importers/jungle.js

@@ -0,0 +1,134 @@
+import { readdir, realpath } from 'node:fs/promises';
+import { dirname, isAbsolute, join } from 'node:path';
+import { readConfined, hash } from '../acquisition/files.js';
+import { object } from '../acquisition/normalize.js';
+import { localizeMarkdown } from '../app/markdown.js';
+import { checkDigest } from '../digest/schema.js';
+import { accessClass, defaultVisibility } from '../identity/uri.js';
+import { convertJungleDigest, parseJungleEditorial, jungleTags, JungleDigestError } from './jungle-digest.js';
+import { jungleNamespace, jungleRealm, visibilityAction } from './jungle-store.js';
+import { importBundleEntry, importBundleHash } from './apply.js';
+const noteBytes = 4 * 1024 ** 2, captureBytes = 16 * 1024 ** 2;
+const errorCode = (error) => String(error?.code ?? (error instanceof Error ? error.message : 'unclassified'));
+/** A capture link is `<dir>/<name>` under `kb/`, never escaping it; the bundle is its directory. */
+function captureLink(link) { if (isAbsolute(link) || link.includes('\\') || link.split('/').length < 2 || link.split('/').some(p => !p || p === '.' || p === '..'))
+    throw new JungleDigestError('no-capture'); return link; }
+/** Digest v1 only when the note converts with nothing dropped or trimmed and passes every rule against this revision's Markdown. */
+export function jungleV1(editorial, markdown) {
+    if (editorial.form !== 'digest')
+        return { issues: [] };
+    const c = convertJungleDigest(editorial.markdown, markdown), check = checkDigest(c.gold, markdown, { tags: jungleTags }), issues = [...c.lossy, ...c.issues.map(i => `${i.path}:${i.code}`), ...check.issues.map(i => `${i.path}:${i.code}`)];
+    return c.clean && check.ok ? { v1: c.gold, issues: [] } : { issues: [...new Set(issues)] };
+}
+/** What a verified import would store, read from the bundle without the verifier: the same localization, from the manifest's asset list. */
+async function previewCapture(kb, bundle, link) {
+    const manifest = object(JSON.parse((await readConfined(kb, join(bundle, 'capture.json'), captureBytes)).toString('utf8'))), pdf = manifest.kind === 'pdf';
+    const listed = (pdf ? (Array.isArray(manifest.images) ? manifest.images.map(i => object(i).asset) : []) : Array.isArray(manifest.assets) ? manifest.assets : []).flatMap(raw => { const a = raw && typeof raw === 'object' ? raw : {}; return typeof a.path === 'string' && typeof a.sha256 === 'string' ? [{ relativePath: a.path, sha256: a.sha256, bytes: Number(a.bytes), mediaType: String(a.mimeType) }] : []; });
+    const text = (await readConfined(kb, link + '.md', captureBytes)).toString('utf8'), markdown = localizeMarkdown(pdf ? text.replace(/^<!-- pdf-page: ([1-9][0-9]*) -->$/gm, '## Page $1') : text, listed).markdown;
+    const locator = pdf ? String(object(manifest.source ?? {}).requestedUrl ?? '') : String(manifest.sourceUrl ?? '');
+    return { markdown, locator };
+}
+export async function importJungle(store, jungle, options = {}) {
+    if (!isAbsolute(jungle))
+        throw Object.assign(Error('invalid-input'), { code: 'invalid-input' });
+    const kb = await realpath(join(jungle, 'kb')), notesDir = join(kb, 'notes', 'reading'), files = (await readdir(notesDir)).filter(f => /^[a-z0-9][a-z0-9-]*\.md$/.test(f)).sort();
+    const limit = options.limit ?? 1000, offset = options.offset ?? 0, dryRun = options.dryRun !== false;
+    if (!Number.isSafeInteger(limit) || limit < 1 || limit > 1000 || !Number.isSafeInteger(offset) || offset < 0 || offset > files.length)
+        throw Object.assign(Error('invalid-input'), { code: 'invalid-input' });
+    if (!dryRun && !options.verify)
+        throw Object.assign(Error('runtime-missing'), { code: 'runtime-missing' });
+    const report = { schemaVersion: 1, dryRun, captureVerification: dryRun ? 'deferred' : 'verified', notes: files.length, counts: { newItems: 0, existingItems: 0, editorial: {}, v1: {}, visibility: {}, skipped: {} }, entries: [], nextOffset: offset + limit < files.length ? offset + limit : null };
+    const bump = (m, k) => { m[k] = (m[k] ?? 0) + 1; }, seen = new Set();
+    for (const file of files.slice(offset, offset + limit)) {
+        if (options.signal?.aborted)
+            throw Object.assign(Error('cancelled'), { code: 'cancelled' });
+        const slug = file.slice(0, -3);
+        let entry = { slug };
+        try {
+            let editorial;
+            try {
+                editorial = parseJungleEditorial((await readConfined(notesDir, file, noteBytes)).toString('utf8'));
+            }
+            catch (error) {
+                const code = error instanceof JungleDigestError ? error.code : 'malformed';
+                entry.skipped = code === 'not-digest' ? 'not-reading-note' : code === 'no-capture' ? 'no-capture' : 'malformed-note';
+                throw entry;
+            }
+            entry = { slug, form: editorial.form, status: editorial.status };
+            let link;
+            try {
+                link = captureLink(editorial.capture);
+            }
+            catch {
+                entry.skipped = 'no-capture';
+                throw entry;
+            }
+            const bundle = dirname(link);
+            try {
+                await readConfined(kb, join(bundle, 'capture.json'), captureBytes);
+            }
+            catch {
+                entry.skipped = 'capture-missing';
+                throw entry;
+            }
+            const prior = store.importedItem(jungleRealm, hash(JSON.stringify([jungleNamespace, link])));
+            if (prior?.deleted) {
+                entry.skipped = 'item-deleted';
+                entry.itemId = prior.itemId;
+                throw entry;
+            }
+            // Two notes on one capture share its item, in a dry run too.
+            entry.item = prior || seen.has(link) ? 'existing' : 'new';
+            seen.add(link);
+            if (dryRun) {
+                if (prior?.revisionId) {
+                    const markdown = store.revisionMarkdown(prior.revisionId), { v1, issues } = jungleV1(editorial, markdown), plan = store.planJungleNote({ itemId: prior.itemId, revisionId: prior.revisionId, editorial, ...(v1 ? { v1 } : {}) });
+                    Object.assign(entry, { itemId: prior.itemId, editorial: plan.editorial, v1: plan.v1, ...(issues.length ? { v1Issues: issues } : {}), visibility: plan.visibility });
+                }
+                else {
+                    const preview = await previewCapture(kb, bundle, link), { v1, issues } = jungleV1(editorial, preview.markdown), access = accessClass(jungleRealm, preview.locator);
+                    Object.assign(entry, { editorial: 'new', v1: editorial.form !== 'digest' ? 'not-digest' : v1 ? 'new' : 'not-convertible', ...(issues.length ? { v1Issues: issues } : {}), visibility: visibilityAction({ access, default: defaultVisibility(access, 'private'), override: null }, editorial.status, false).action });
+                }
+            }
+            else {
+                let imported;
+                try {
+                    imported = await importBundleEntry(store, kb, jungleNamespace, { id: link, bundle, privacy: 'private', notes: [] }, options.verify, false, options.signal);
+                }
+                catch (error) {
+                    const code = errorCode(error);
+                    if (['cleanup-required', 'owner-busy', 'cancelled'].includes(code))
+                        throw error;
+                    entry.skipped = code === 'unsupported' ? 'capture-unsupported' : 'capture-failed';
+                    entry.code = code;
+                    throw entry;
+                }
+                const itemId = imported.itemId, revisionId = store.importedRevision(itemId, importBundleHash(imported.sourceHash)), markdown = store.revisionMarkdown(revisionId), { v1, issues } = jungleV1(editorial, markdown);
+                const plan = store.applyJungleNote({ itemId, revisionId, editorial, ...(v1 ? { v1 } : {}) });
+                Object.assign(entry, { itemId, editorial: plan.editorial, v1: plan.v1, ...(issues.length ? { v1Issues: issues } : {}), visibility: plan.visibility });
+            }
+            bump(report.counts.editorial, entry.editorial);
+            bump(report.counts.v1, entry.v1);
+            bump(report.counts.visibility, entry.visibility);
+            if (entry.item === 'new')
+                report.counts.newItems++;
+            else
+                report.counts.existingItems++;
+            report.entries.push(entry);
+        }
+        catch (error) {
+            if (error !== entry) {
+                const code = errorCode(error);
+                if (['cleanup-required', 'owner-busy', 'cancelled'].includes(code)) {
+                    report.nextOffset = offset + report.entries.length;
+                    break;
+                }
+                entry.skipped = 'capture-failed';
+                entry.code = code;
+            }
+            bump(report.counts.skipped, entry.skipped);
+            report.entries.push(entry);
+        }
+    }
+    return report;
+}

package/dist/src/indexing/index.d.ts

@@ -23,6 +23,13 @@ export declare class IndexError extends Error {
     code: 'invalid-index' | 'stale-index' | 'incomplete-index' | 'cancelled-index';
     constructor(code: 'invalid-index' | 'stale-index' | 'incomplete-index' | 'cancelled-index');
 }
+export type VectorEncoding = 'f32' | 'int8';
+/** Storage format follows the space identity: no `quantization` (legacy) or `f32` stores float32 LE; `int8-symmetric-maxabs-v1` stores one int8 code per dimension. */
+export declare function spaceEncoding(identity: Record<string, unknown>): VectorEncoding;
+/** int8 codes are round(x*127/max|x|) of a unit vector. The per-vector scale is not stored: cosine is scale-invariant, so codes rank directly and dequantize by renormalizing. */
+export declare function encodeVector(vector: readonly number[], encoding: VectorEncoding): Buffer;
+/** Exact byte length, finite floats, int8 codes within [-127,127] and nonzero; int8 returns the unit dequantized vector. */
+export declare function decodeVector(bytes: Uint8Array, dimensions: number, encoding: VectorEncoding): number[];
 /** Unicode-safe chunks fit the fixed document prompt within the embedding text budget, preferring whitespace breaks. */
 export declare function chunkDocument(text: string): string[];
 /** Identity must already be the verified embedder fingerprint. Existing identities are immutable. */

package/dist/src/indexing/index.js

@@ -21,6 +21,30 @@ catch (e) {
 } }
 function epoch(db) { return Number(db.prepare('SELECT revision_epoch FROM library_info WHERE id=1').get()?.revision_epoch); }
 function revisions(db) { return db.prepare('SELECT id AS itemId,current_revision AS revisionId FROM items WHERE deleted_at IS NULL AND current_revision IS NOT NULL ORDER BY id').all(); }
+/** Storage format follows the space identity: no `quantization` (legacy) or `f32` stores float32 LE; `int8-symmetric-maxabs-v1` stores one int8 code per dimension. */
+export function spaceEncoding(identity) { const q = identity.quantization; if (q === undefined || q === 'f32')
+    return 'f32'; if (q === 'int8-symmetric-maxabs-v1')
+    return 'int8'; return fail(); }
+/** int8 codes are round(x*127/max|x|) of a unit vector. The per-vector scale is not stored: cosine is scale-invariant, so codes rank directly and dequantize by renormalizing. */
+export function encodeVector(vector, encoding) { if (!vector.length || vector.some(n => !Number.isFinite(n)))
+    fail(); if (encoding === 'f32') {
+    const bytes = Buffer.alloc(vector.length * 4);
+    vector.forEach((n, i) => bytes.writeFloatLE(n, i * 4));
+    return bytes;
+} const max = Math.max(...vector.map(Math.abs)); if (!(max > 0))
+    fail(); return Buffer.from(Int8Array.from(vector, n => Math.round(n * 127 / max)).buffer); }
+/** Exact byte length, finite floats, int8 codes within [-127,127] and nonzero; int8 returns the unit dequantized vector. */
+export function decodeVector(bytes, dimensions, encoding) { const b = Buffer.from(bytes); if (encoding === 'f32') {
+    if (b.length !== dimensions * 4)
+        fail();
+    const vector = Array.from({ length: dimensions }, (_, i) => b.readFloatLE(i * 4));
+    if (vector.some(n => !Number.isFinite(n)))
+        fail();
+    return vector;
+} if (b.length !== dimensions)
+    fail(); const codes = Array.from({ length: dimensions }, (_, i) => b.readInt8(i)); if (codes.some(n => n < -127))
+    fail(); const norm = Math.hypot(...codes); if (!(norm > 0))
+    fail(); return codes.map(n => n / norm); }
 const chunkBytes = embeddingTextBytes - Buffer.byteLength('title: none | text: ');
 /** Unicode-safe chunks fit the fixed document prompt within the embedding text budget, preferring whitespace breaks. */
 export function chunkDocument(text) { const chunks = []; let part = '', size = 0, cut = -1, cutSize = 0; for (const char of text) {
@@ -61,15 +85,16 @@ function snapshot(db) {
     }
     return { version: 1, revisions: current, inputs };
 }
-function generation(db, id) { const row = db.prepare('SELECT g.*,s.dimensions FROM index_generations g JOIN embedding_spaces s ON s.fingerprint=g.fingerprint WHERE g.id=?').get(id); if (!row)
+function generation(db, id) { const row = db.prepare('SELECT g.*,s.dimensions,s.identity_json FROM index_generations g JOIN embedding_spaces s ON s.fingerprint=g.fingerprint WHERE g.id=?').get(id); if (!row)
     return fail(); const coverage = JSON.parse(String(row.coverage_json)); const profile = JSON.parse(String(row.profile_json)); if (coverage.version !== 1 || !Array.isArray(coverage.inputs) || !Array.isArray(coverage.revisions) || typeof profile.profileId !== 'string')
-    fail(); return { row, coverage, profile }; }
+    fail(); return { row, coverage, profile, encoding: spaceEncoding(JSON.parse(String(row.identity_json))) }; }
 /** Identity must already be the verified embedder fingerprint. Existing identities are immutable. */
 export function createGeneration(db, options) {
     if (!options.profileId || options.profileId.length > 128 || !/^[a-f0-9]{64}$/.test(options.fingerprint) || !Number.isSafeInteger(options.dimensions) || options.dimensions < 1 || options.dimensions > 4096)
         fail();
     if (spaceFingerprint(options.identity) !== options.fingerprint || options.identity.dimensions !== options.dimensions)
         fail();
+    spaceEncoding(options.identity);
     return atomic(db, () => { const identity = JSON.stringify(Object.fromEntries(Object.keys(options.identity).sort().map(k => [k, options.identity[k]]))); if (!identity || Buffer.byteLength(identity) > 16384)
         fail(); const existing = db.prepare('SELECT * FROM embedding_spaces WHERE fingerprint=?').get(options.fingerprint); if (existing && (existing.identity_json !== identity || Number(existing.dimensions) !== options.dimensions || existing.normalization !== 'unit-l2' || existing.metric !== 'cosine'))
         fail(); if (!existing)
@@ -86,7 +111,7 @@ export function pendingInputs(db, id, limit = 16) { if (!Number.isSafeInteger(li
 export function writeBatch(db, id, fingerprint, batch) {
     if (!batch.length || batch.length > 256)
         fail();
-    atomic(db, () => { const { row, coverage } = generation(db, id); if (row.state === 'cancelled')
+    atomic(db, () => { const { row, coverage, encoding } = generation(db, id); if (row.state === 'cancelled')
         fail('cancelled-index'); if (row.state !== 'building' || row.fingerprint !== fingerprint)
         fail(); const expected = new Map(coverage.inputs.map(i => [i.sourceId, i])); const seen = new Set(); for (const result of batch) {
         const input = expected.get(result.sourceId);
@@ -103,8 +128,7 @@ export function writeBatch(db, id, fingerprint, batch) {
                 fail();
         }
         seen.add(result.sourceId);
-        const bytes = Buffer.alloc(result.vector.length * 4);
-        result.vector.forEach((n, i) => bytes.writeFloatLE(n, i * 4));
+        const bytes = encodeVector(result.vector, encoding);
         const prior = db.prepare('SELECT vector,input_hash FROM vectors WHERE generation_id=? AND source_id=?').get(id, input.sourceId);
         if (prior && prior.input_hash !== null) {
             if (prior.input_hash !== result.inputHash || !bytes.equals(Buffer.from(prior.vector)))
@@ -125,7 +149,6 @@ export function cancelGeneration(db, id) { atomic(db, () => { const { row } = ge
     fail(); db.prepare("UPDATE index_generations SET state='cancelled' WHERE id=?").run(id); }); }
 /** Read-side join removes deleted/refreshed revisions even when an old generation stays active. */
 export function activeVectors(db, profileId) { const active = db.prepare('SELECT generation_id FROM active_indexes WHERE profile_id=?').get(profileId); if (!active)
-    return []; const id = String(active.generation_id), { row, coverage } = generation(db, id), inputs = new Map(coverage.inputs.map(i => [i.sourceId, i])); return db.prepare('SELECT v.* FROM vectors v JOIN revisions r ON r.id=v.revision_id JOIN items i ON i.id=r.item_id WHERE v.generation_id=? AND i.deleted_at IS NULL AND i.current_revision=v.revision_id ORDER BY v.source_id').all(id).map(v => { const bytes = Buffer.from(v.vector), source = inputs.get(String(v.source_id)); if (!source || bytes.length !== Number(row.dimensions) * 4)
-    fail(); const vector = Array.from({ length: Number(row.dimensions) }, (_, i) => bytes.readFloatLE(i * 4)); if (vector.some(n => !Number.isFinite(n)))
-    fail(); return { fingerprint: String(row.fingerprint), generationId: id, source: source, vector }; }); }
+    return []; const id = String(active.generation_id), { row, coverage, encoding } = generation(db, id), inputs = new Map(coverage.inputs.map(i => [i.sourceId, i])); return db.prepare('SELECT v.* FROM vectors v JOIN revisions r ON r.id=v.revision_id JOIN items i ON i.id=r.item_id WHERE v.generation_id=? AND i.deleted_at IS NULL AND i.current_revision=v.revision_id ORDER BY v.source_id').all(id).map(v => { const source = inputs.get(String(v.source_id)); if (!source)
+    fail(); const vector = decodeVector(v.vector, Number(row.dimensions), encoding); return { fingerprint: String(row.fingerprint), generationId: id, source: source, vector }; }); }
 export const rollbackGeneration = activateGeneration;

package/dist/src/indexing/prune-run.d.ts

@@ -0,0 +1,22 @@
+import { type LibraryStore } from '../store/index.js';
+import { type PageStats, type PruneEntry } from './prune.js';
+export interface PruneReport {
+    dryRun: boolean;
+    keep: number;
+    generations: PruneEntry[];
+    prunedVectors: number;
+    estimatedBytes: number;
+    before: PageStats;
+    after: PageStats;
+    reclaimedBytes: number;
+    vacuum: 'skipped' | 'done' | 'would-run';
+    note: string;
+}
+/** Holds worker custody and the store owner fence for the whole run, so no worker, index run or query starts;
+ * each bounded batch rechecks owner, cleanup and backup/GC fences in its own transaction. Refuses while index work runs. */
+export declare function pruneIndex(store: LibraryStore, options?: {
+    keep?: number;
+    dryRun?: boolean;
+    vacuum?: boolean;
+}): Promise<PruneReport>;
+export declare function pruneText(report: PruneReport): string;

package/dist/src/indexing/prune-run.js

@@ -0,0 +1,67 @@
+import { rm } from 'node:fs/promises';
+import { join } from 'node:path';
+import { StoreError } from '../store/index.js';
+import { acquireWorkerCustody } from '../worker/custody.js';
+import { syncDir } from '../worker/journal.js';
+import { maximumKeep } from './prune.js';
+const note = 'Rollback to a pruned generation needs a rebuild (`sponge index create`).';
+const blocked = (store) => { if (store.ownerSnapshot())
+    throw new StoreError('owner-busy'); if (store.pendingCleanupCount())
+    throw Object.assign(Error('cleanup-required'), { code: 'cleanup-required' }); if (store.hasGcFence())
+    throw new StoreError('owner-busy'); };
+const total = (entries, key) => entries.filter(g => g.action === 'prune').reduce((n, g) => n + g[key], 0);
+/** Holds worker custody and the store owner fence for the whole run, so no worker, index run or query starts;
+ * each bounded batch rechecks owner, cleanup and backup/GC fences in its own transaction. Refuses while index work runs. */
+export async function pruneIndex(store, options = {}) {
+    const keep = options.keep ?? 1, dryRun = options.dryRun === true;
+    if (!Number.isSafeInteger(keep) || keep < 0 || keep > maximumKeep)
+        throw new StoreError('invalid-input');
+    if (dryRun) {
+        const { generations, stats } = store.pruneIndexPlan(keep);
+        return { dryRun, keep, generations, prunedVectors: total(generations, 'vectors'), estimatedBytes: total(generations, 'bytes'), before: stats, after: stats, reclaimedBytes: 0, vacuum: options.vacuum ? 'would-run' : 'skipped', note };
+    }
+    blocked(store);
+    const custody = await acquireWorkerCustody(store.root);
+    let owner;
+    try {
+        owner = store.acquireOwner(custody.incarnation);
+        const { generations, stats: before } = store.pruneIndexPlan(keep);
+        let prunedVectors = 0;
+        for (const generation of generations.filter(g => g.action === 'prune'))
+            for (;;) {
+                const n = store.pruneIndexStep(owner, generation.id, keep);
+                prunedVectors += n;
+                if (!n)
+                    break;
+            }
+        // A concurrent rollback can make a planned generation a kept one again; report what actually happened.
+        const now = new Map(store.pruneIndexPlan(keep).generations.map(g => [g.id, g])), final = generations.map(g => { const current = now.get(g.id); return g.action === 'prune' && current?.action === 'keep' ? current : g; });
+        let after = store.pruneIndexPlan(keep).stats, vacuum = 'skipped';
+        if (options.vacuum) {
+            after = store.vacuumLibrary(owner);
+            vacuum = 'done';
+        }
+        return { dryRun, keep, generations: final, prunedVectors, estimatedBytes: total(final, 'bytes'), before, after, reclaimedBytes: vacuum === 'done' ? before.bytes - after.bytes : (after.freePages - before.freePages) * after.pageSize, vacuum, note };
+    }
+    finally {
+        try {
+            if (owner)
+                store.releaseOwner(owner);
+        }
+        finally {
+            await custody.release();
+            try {
+                await rm(custody.directory, { recursive: true });
+                await syncDir(join(store.root, 'runtime', 'workers'));
+            }
+            catch { /* Runtime GC removes this settled journal. */ }
+        }
+    }
+}
+export function pruneText(report) {
+    const lines = report.generations.map(g => `${g.action === 'prune' ? (report.dryRun ? 'would prune' : 'pruned') : 'kept'}\t${g.id}\t${g.profileId}\t${g.space}\t${g.state}\t${g.reason}\t${g.vectors} vectors`);
+    lines.push(report.dryRun ? `Would free about ${report.estimatedBytes} bytes of rows${report.vacuum === 'would-run' ? '; --vacuum would then shrink the file' : ''}` : `Reclaimed ${report.reclaimedBytes} bytes (${report.vacuum === 'done' ? 'file shrank after VACUUM' : 'freed pages are reused; --vacuum shrinks the file'})`);
+    if (report.generations.some(g => g.action === 'prune'))
+        lines.push(report.note);
+    return lines.join('\n');
+}

package/dist/src/indexing/prune.d.ts

@@ -0,0 +1,30 @@
+import type { DatabaseSync } from 'node:sqlite';
+export type PruneReason = 'active' | 'building' | 'queued' | 'running' | 'rollback' | 'unknown' | 'retired' | 'stale' | 'cancelled' | 'failed' | 'pruned';
+export interface PruneEntry {
+    id: string;
+    profileId: string;
+    fingerprint: string;
+    space: string;
+    state: string;
+    action: 'keep' | 'prune';
+    reason: PruneReason;
+    vectors: number;
+    bytes: number;
+}
+export interface PageStats {
+    pageSize: number;
+    pages: number;
+    freePages: number;
+    bytes: number;
+}
+export declare const maximumKeep = 16, pruneBatchRows = 1000;
+/** Caller holds a write transaction. Keeps active, building/resumable and job-referenced work, plus the `keep` most
+ * recent rollback-ready retired generations per profile. Stale retired generations can never activate again (the
+ * revision epoch only grows), so they are pruned. Recency is index job creation; VACUUM may renumber rowids. */
+export declare function planPrune(db: DatabaseSync, keep: number): PruneEntry[];
+/** One bounded step for one planned generation, inside the caller's write transaction. The first step marks it pruned
+ * and drops its coverage, so it is never rollback-ready while partially deleted. Returns rows deleted; 0 means done. */
+export declare function pruneStep(db: DatabaseSync, id: string, keep: number, limit?: number): number;
+export declare function pageStats(db: DatabaseSync): PageStats;
+/** VACUUM rewrites the database through a temporary copy and the WAL: about the file size again on each volume. */
+export declare function vacuumSpace(root: string, bytes: number, margin?: number): void;

package/dist/src/indexing/prune.js

@@ -0,0 +1,95 @@
+import { statSync, statfsSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { StoreError } from '../store/types.js';
+import { activateGeneration, IndexError, spaceEncoding } from './index.js';
+export const maximumKeep = 16, pruneBatchRows = 1000;
+/** Pruned rows keep their id (index_jobs history references it) with no coverage and no vectors. */
+const prunedCoverage = JSON.stringify({ version: 1, revisions: [], inputs: [] });
+/** Exactly the rollback rules: a probe activation inside a savepoint that is always rolled back. */
+function rollbackReady(db, id) { db.exec('SAVEPOINT prune_probe'); try {
+    activateGeneration(db, id);
+    return true;
+}
+catch (error) {
+    if (error instanceof IndexError)
+        return false;
+    throw error;
+}
+finally {
+    db.exec('ROLLBACK TO prune_probe; RELEASE prune_probe');
+} }
+/** Caller holds a write transaction. Keeps active, building/resumable and job-referenced work, plus the `keep` most
+ * recent rollback-ready retired generations per profile. Stale retired generations can never activate again (the
+ * revision epoch only grows), so they are pruned. Recency is index job creation; VACUUM may renumber rowids. */
+export function planPrune(db, keep) {
+    if (!Number.isSafeInteger(keep) || keep < 0 || keep > maximumKeep)
+        throw new StoreError('invalid-input');
+    if (!db.isTransaction)
+        throw new StoreError('invalid-input');
+    const epoch = Number(db.prepare('SELECT revision_epoch FROM library_info WHERE id=1').get()?.revision_epoch), active = new Set(db.prepare('SELECT generation_id FROM active_indexes').all().map(r => String(r.generation_id)));
+    const rows = db.prepare("SELECT g.id,g.fingerprint,g.state,g.revision_epoch,g.profile_json,length(g.coverage_json) AS coverage,s.dimensions,s.identity_json,j.state AS job,(SELECT count(*) FROM vectors v WHERE v.generation_id=g.id) AS vectors,(SELECT COALESCE(sum(length(v.vector)+length(v.source_id)+length(v.revision_id)+length(v.generation_id)+length(v.derivation_hash)+COALESCE(length(v.input_hash),0)+COALESCE(length(v.asset_sha256),0)),0) FROM vectors v WHERE v.generation_id=g.id) AS bytes FROM index_generations g JOIN embedding_spaces s ON s.fingerprint=g.fingerprint LEFT JOIN index_jobs j ON j.generation_id=g.id ORDER BY j.created_at IS NULL,j.created_at DESC,g.rowid DESC").all();
+    const kept = new Map();
+    return rows.flatMap(row => {
+        const id = String(row.id), state = String(row.state), job = row.job === null ? null : String(row.job), vectors = Number(row.vectors);
+        let profileId, space;
+        try {
+            profileId = String(JSON.parse(String(row.profile_json)).profileId);
+            const identity = JSON.parse(String(row.identity_json));
+            space = `${spaceEncoding(identity)}-${Number(row.dimensions)}`;
+        }
+        catch {
+            throw new StoreError('integrity');
+        }
+        if (state === 'pruned' && !vectors && Number(row.coverage) <= prunedCoverage.length)
+            return [];
+        const entry = (action, reason) => [{ id, profileId, fingerprint: String(row.fingerprint), space, state, action, reason, vectors, bytes: Number(row.bytes) + (action === 'prune' ? Math.max(0, Number(row.coverage) - prunedCoverage.length) : 0) }];
+        if (active.has(id))
+            return entry('keep', 'active');
+        if (job === 'running' || job === 'cleanup-pending')
+            return entry('keep', 'running');
+        if (state === 'pruned')
+            return entry('prune', 'pruned');
+        if (state === 'cancelled')
+            return entry('prune', 'cancelled');
+        // A building generation stays resumable until the revision epoch moves; after that it can never activate.
+        if (state === 'building')
+            return job === 'queued' ? entry('keep', 'queued') : Number(row.revision_epoch) === epoch ? entry('keep', 'building') : entry('prune', 'failed');
+        if (state !== 'retired')
+            return entry('keep', 'unknown');
+        if (!rollbackReady(db, id))
+            return entry('prune', 'stale');
+        const n = kept.get(profileId) ?? 0;
+        if (n < keep) {
+            kept.set(profileId, n + 1);
+            return entry('keep', 'rollback');
+        }
+        return entry('prune', 'retired');
+    });
+}
+/** One bounded step for one planned generation, inside the caller's write transaction. The first step marks it pruned
+ * and drops its coverage, so it is never rollback-ready while partially deleted. Returns rows deleted; 0 means done. */
+export function pruneStep(db, id, keep, limit = pruneBatchRows) {
+    if (!Number.isSafeInteger(limit) || limit < 1 || limit > 10000)
+        throw new StoreError('invalid-input');
+    const row = db.prepare('SELECT state FROM index_generations WHERE id=?').get(id);
+    if (!row)
+        throw new StoreError('unavailable');
+    // Once marked, a generation can never become active, building or retired again: no replanning per batch.
+    if (row.state !== 'pruned') {
+        const planned = planPrune(db, keep).find(e => e.id === id);
+        if (!planned || planned.action !== 'prune')
+            return 0;
+        db.prepare("UPDATE index_generations SET state='pruned',coverage_json=? WHERE id=?").run(prunedCoverage, id);
+        db.prepare("UPDATE index_jobs SET state='cancelled',cancel_requested=1 WHERE generation_id=? AND state='queued'").run(id);
+    }
+    else if (db.prepare('SELECT 1 FROM active_indexes WHERE generation_id=?').get(id))
+        throw new StoreError('integrity');
+    return Number(db.prepare('DELETE FROM vectors WHERE rowid IN (SELECT rowid FROM vectors WHERE generation_id=? LIMIT ?)').run(id, limit).changes);
+}
+export function pageStats(db) { const pageSize = Number(db.prepare('PRAGMA page_size').get()?.page_size), pages = Number(db.prepare('PRAGMA page_count').get()?.page_count), freePages = Number(db.prepare('PRAGMA freelist_count').get()?.freelist_count); return { pageSize, pages, freePages, bytes: pages * pageSize }; }
+/** VACUUM rewrites the database through a temporary copy and the WAL: about the file size again on each volume. */
+export function vacuumSpace(root, bytes, margin = 64 * 1024 ** 2) {
+    const free = (path) => { const s = statfsSync(path); return s.bavail * s.bsize; }, temp = tmpdir(), same = statSync(root).dev === statSync(temp).dev;
+    if (same ? free(root) < 2 * bytes + margin : free(root) < bytes + margin || free(temp) < bytes + margin)
+        throw new StoreError('limit-exceeded');
+}

package/dist/src/indexing/runner.js

@@ -3,7 +3,7 @@ import { StoreError } from '../store/index.js';
 import { acquireWorkerCustody, proveAttemptCleanup } from '../worker/custody.js';
 import { supervisedRunner } from '../worker/supervisor.js';
 import { directory, writeJournal } from '../worker/journal.js';
-import { embedBatch, embeddingFingerprint, encodeEmbeddingRequest } from '../inference/embedding/index.js';
+import { embedBatch, embeddingFingerprint, encodeEmbeddingRequest, resolveIndexSpace, projectVector } from '../inference/embedding/index.js';
 import { deriveEmbeddingImage } from './image.js';
 import { failure } from '../acquisition/errors.js';
 /** Same durable worker lock/owner/supervisor as acquisition, with sequential bounded batches. */
@@ -39,8 +39,8 @@ export async function drainIndexQueue(store, options) {
                 const job = store.indexJob(attempt.jobId);
                 if (!job)
                     throw new StoreError('unavailable');
-                const generationId = String(job.generation_id), status = store.indexStatus(generationId);
-                if (embeddingFingerprint(options.runtime.helperSha256) !== status.fingerprint)
+                const generationId = String(job.generation_id), status = store.indexStatus(generationId), space = resolveIndexSpace(options.runtime.helperSha256, status.fingerprint);
+                if (!space)
                     throw new StoreError('integrity');
                 await checkpoint('indexing');
                 for (;;) {
@@ -64,9 +64,9 @@ export async function drainIndexQueue(store, options) {
                         batch.push({ kind: 'image', png: derived.png });
                     }
                     const expected = encodeEmbeddingRequest(batch).inputHashes, embedded = await (options.embed ?? embedBatch)(batch, { runtime: options.runtime, scratchRoot, runProcess, signal: controller.signal });
-                    if (embedded.fingerprint !== status.fingerprint || embedded.inputHashes.length !== expected.length || embedded.inputHashes.some((hash, i) => hash !== expected[i]))
+                    if (embedded.fingerprint !== embeddingFingerprint(options.runtime.helperSha256) || embedded.inputHashes.length !== expected.length || embedded.inputHashes.some((hash, i) => hash !== expected[i]))
                         throw new StoreError('integrity');
-                    store.writeIndexAttemptBatch(attempt, embedded.fingerprint, inputs.map((input, i) => ({ ...input, inputHash: expected[i], vector: embedded.vectors[i] })));
+                    store.writeIndexAttemptBatch(attempt, status.fingerprint, inputs.map((input, i) => ({ ...input, inputHash: expected[i], vector: projectVector(embedded.vectors[i], space.dimensions) })));
                     await options.onBatch?.(attempt, store.indexStatus(generationId).completed);
                 }
                 await checkpoint('promoting');

package/dist/src/inference/embedding/index.d.ts

@@ -32,6 +32,31 @@ export declare class EmbeddingError extends Error {
 }
 export declare function spaceFingerprint(identity: Record<string, string | number>): string;
 export declare function embeddingFingerprint(helperSha256: string): string;
+/** Stored index spaces over the fixed 768-dim helper output. `f32-768` is the legacy identity and keeps its original fingerprint so existing generations stay readable; `int8-768` (default) stores symmetric per-vector max-abs int8 codes of the full vector; `int8-512` also keeps only the renormalized Matryoshka prefix. Quantization alone was near-lossless on real queries while the 512 prefix moved chunk top-1 on a quarter of them (docs/decisions/storage-diet.md), so the default keeps all 768 dimensions for 256 more bytes per row. */
+export declare const indexEncodings: {
+    readonly 'f32-768': {};
+    readonly 'int8-768': {
+        readonly quantization: "int8-symmetric-maxabs-v1";
+    };
+    readonly 'int8-512': {
+        readonly dimensions: 512;
+        readonly truncation: "matryoshka-prefix-renormalized-v1";
+        readonly quantization: "int8-symmetric-maxabs-v1";
+    };
+};
+export type IndexEncoding = keyof typeof indexEncodings;
+export declare const defaultIndexEncoding: IndexEncoding;
+export interface IndexSpace {
+    encoding: IndexEncoding;
+    identity: Record<string, string | number>;
+    fingerprint: string;
+    dimensions: number;
+}
+export declare function indexSpace(helperSha256: string, encoding?: IndexEncoding): IndexSpace;
+/** The stored space for a generation fingerprint built from this helper, if any. */
+export declare function resolveIndexSpace(helperSha256: string, fingerprint: string): IndexSpace | undefined;
+/** Matryoshka prefix truncation of a unit helper vector, renormalized to unit L2. */
+export declare function projectVector(vector: readonly number[], dimensions: number): number[];
 /** Versioned length-delimited bytes. No user content or image pathname enters argv. */
 export declare function encodeEmbeddingRequest(inputs: readonly EmbeddingInput[]): {
     bytes: Buffer;

package/dist/src/inference/embedding/index.js

@@ -19,6 +19,17 @@ export function spaceFingerprint(identity) { const keys = Object.keys(identity).
     fail(); return hash(JSON.stringify(Object.fromEntries(keys.map(k => [k, identity[k]])))); }
 export function embeddingFingerprint(helperSha256) { if (!/^[a-f0-9]{64}$/.test(helperSha256))
     fail(); return spaceFingerprint({ ...embeddingIdentity, helperSha256 }); }
+/** Stored index spaces over the fixed 768-dim helper output. `f32-768` is the legacy identity and keeps its original fingerprint so existing generations stay readable; `int8-768` (default) stores symmetric per-vector max-abs int8 codes of the full vector; `int8-512` also keeps only the renormalized Matryoshka prefix. Quantization alone was near-lossless on real queries while the 512 prefix moved chunk top-1 on a quarter of them (docs/decisions/storage-diet.md), so the default keeps all 768 dimensions for 256 more bytes per row. */
+export const indexEncodings = { 'f32-768': {}, 'int8-768': { quantization: 'int8-symmetric-maxabs-v1' }, 'int8-512': { dimensions: 512, truncation: 'matryoshka-prefix-renormalized-v1', quantization: 'int8-symmetric-maxabs-v1' } };
+export const defaultIndexEncoding = 'int8-768';
+export function indexSpace(helperSha256, encoding = defaultIndexEncoding) { if (!/^[a-f0-9]{64}$/.test(helperSha256) || !Object.hasOwn(indexEncodings, encoding))
+    fail(); const identity = { ...embeddingIdentity, helperSha256, ...indexEncodings[encoding] }; return { encoding, identity, fingerprint: spaceFingerprint(identity), dimensions: Number(identity.dimensions) }; }
+/** The stored space for a generation fingerprint built from this helper, if any. */
+export function resolveIndexSpace(helperSha256, fingerprint) { return Object.keys(indexEncodings).map(e => indexSpace(helperSha256, e)).find(s => s.fingerprint === fingerprint); }
+/** Matryoshka prefix truncation of a unit helper vector, renormalized to unit L2. */
+export function projectVector(vector, dimensions) { if (!Number.isSafeInteger(dimensions) || dimensions < 1 || dimensions > vector.length || vector.some(n => !Number.isFinite(n)))
+    fail(); const prefix = vector.slice(0, dimensions), norm = Math.hypot(...prefix); if (!(norm > 0))
+    fail(); return dimensions === vector.length ? [...vector] : prefix.map(n => n / norm); }
 /** Versioned length-delimited bytes. No user content or image pathname enters argv. */
 export function encodeEmbeddingRequest(inputs) {
     if (!inputs.length || inputs.length > 16)