@hraness/sponge 0.2.6 → 0.3.1

package/dist/src/indexing/jobs.js

@@ -1,49 +1,158 @@
 import { randomUUID } from 'node:crypto';
-import { StoreError } from '../store/types.js';
+import { ownerMatches, pendingCleanup } from '../store/lease.js';
+import { SpongeError } from '../errors.js';
+import { syncGeneration } from './index.js';
 const now = () => new Date().toISOString();
-function atomic(db, fn) { if (db.isTransaction)
-    return fn(); db.exec('BEGIN IMMEDIATE'); try {
-    const result = fn();
-    db.exec('COMMIT');
-    return result;
-}
-catch (error) {
-    db.exec('ROLLBACK');
-    throw error;
-} }
-function owner(db, token) { const row = db.prepare('SELECT * FROM worker_owner WHERE id=1').get(); if (row?.incarnation !== token.ownerIncarnation || row.fence !== token.fence)
-    throw new StoreError('owner-busy'); }
-function attempt(db, token, active = true) { owner(db, token); const row = db.prepare('SELECT a.*,j.state,j.cancel_requested,j.generation_id FROM index_attempts a JOIN index_jobs j ON j.id=a.job_id WHERE a.id=? AND a.job_id=?').get(token.attemptId, token.jobId); if (!row || row.owner_incarnation !== token.ownerIncarnation || row.fence !== token.fence || (active && (row.finished_at !== null || row.state !== 'running')))
-    throw new StoreError('owner-busy'); if (active && row.cancel_requested === 1)
-    throw new StoreError('cancelled'); return row; }
-export function submitIndexJob(db, generationId) { return atomic(db, () => { const generation = db.prepare('SELECT state FROM index_generations WHERE id=?').get(generationId); if (!generation || generation.state !== 'building')
-    throw new StoreError('unavailable'); const prior = db.prepare('SELECT id FROM index_jobs WHERE generation_id=?').get(generationId); if (prior)
-    return String(prior.id); const id = randomUUID(); db.prepare("INSERT INTO index_jobs(id,generation_id,state,created_at) VALUES(?,?,'queued',?)").run(id, generationId, now()); return id; }); }
-export function listIndexJobs(db) { return db.prepare("SELECT *, 'index' AS operation, NULL AS item_id FROM index_jobs ORDER BY created_at,id").all(); }
-export function indexJob(db, id) { return db.prepare('SELECT * FROM index_jobs WHERE id=?').get(id); }
-export function claimIndexJob(db, token, id) { return atomic(db, () => { owner(db, token); const job = id ? indexJob(db, id) : db.prepare("SELECT * FROM index_jobs WHERE state='queued' AND cancel_requested=0 ORDER BY created_at,id LIMIT 1").get(); if (!job) {
-    if (id)
-        throw new StoreError('unavailable');
-    return null;
-} if (job.state !== 'queued' || job.cancel_requested)
-    throw new StoreError('unavailable'); if (db.prepare("SELECT (SELECT count(*) FROM attempts WHERE cleanup='pending')+(SELECT count(*) FROM index_attempts WHERE cleanup='pending') AS n").get()?.n)
-    throw new StoreError('owner-busy'); const attemptId = randomUUID(); db.prepare("UPDATE index_jobs SET state='running',attempt_count=attempt_count+1 WHERE id=?").run(job.id); db.prepare("INSERT INTO index_attempts VALUES(?,?,?,?,'acquiring','pending',?,NULL)").run(attemptId, job.id, token.ownerIncarnation, token.fence, now()); return { ...token, jobId: String(job.id), attemptId }; }); }
-export function withIndexAttempt(db, token, fn) { return atomic(db, () => { const row = attempt(db, token); return fn(String(row.generation_id)); }); }
-export function checkpointIndexAttempt(db, token, stage) { const stages = ['acquiring', 'extracting', 'indexing', 'promoting', 'cleanup']; atomic(db, () => { const row = attempt(db, token); if (!stages.includes(stage) || stages.indexOf(stage) < stages.indexOf(String(row.stage)))
-    throw new StoreError('invalid-input'); db.prepare('UPDATE index_attempts SET stage=? WHERE id=?').run(stage, token.attemptId); }); }
-export function finishIndexAttempt(db, token, reason) { atomic(db, () => { const row = attempt(db, token, false); if (row.finished_at !== null || row.state !== 'running')
-    throw new StoreError('owner-busy'); if (reason !== undefined && !/^[a-z][a-z-]{0,63}$/.test(reason))
-    throw new StoreError('invalid-input'); if (!reason && row.cancel_requested)
-    throw new StoreError('cancelled'); db.prepare('UPDATE index_jobs SET state=?,reason=? WHERE id=?').run(row.cancel_requested ? 'cancelled' : reason ? 'failed' : 'succeeded', reason ?? null, token.jobId); db.prepare("UPDATE index_attempts SET stage='cleanup',finished_at=? WHERE id=?").run(now(), token.attemptId); }); }
-export function settleIndexAttempt(db, token) { atomic(db, () => { const row = attempt(db, token, false); if (row.finished_at === null)
-    throw new StoreError('owner-busy'); db.prepare("UPDATE index_attempts SET cleanup='settled',stage='settled' WHERE id=?").run(token.attemptId); }); }
-export function cancelIndexJob(db, id) { atomic(db, () => { const row = indexJob(db, id); if (!row)
-    throw new StoreError('unavailable'); if (row.state === 'succeeded')
-    return; db.prepare("UPDATE index_jobs SET cancel_requested=1,state=CASE WHEN state IN ('queued','failed','cancelled') THEN 'cancelled' ELSE state END WHERE id=?").run(id); }); }
-export function retryIndexJob(db, id) { atomic(db, () => { const row = indexJob(db, id); if (!row || !['failed', 'cancelled'].includes(String(row.state)))
-    throw new StoreError('unavailable'); if (db.prepare("SELECT count(*) AS n FROM index_attempts WHERE job_id=? AND cleanup='pending'").get(id)?.n)
-    throw new StoreError('owner-busy'); const generation = db.prepare('SELECT state,revision_epoch FROM index_generations WHERE id=?').get(row.generation_id); if (!generation || !['building', 'active'].includes(String(generation.state)) || generation.revision_epoch !== db.prepare('SELECT revision_epoch FROM library_info WHERE id=1').get()?.revision_epoch)
-    throw new StoreError('unavailable'); db.prepare("UPDATE index_jobs SET state='queued',cancel_requested=0,reason=NULL WHERE id=?").run(id); }); }
-export function indexOwnerAttempts(db, token) { owner(db, token); return db.prepare('SELECT a.*,j.state,j.cancel_requested FROM index_attempts a JOIN index_jobs j ON j.id=a.job_id WHERE a.owner_incarnation=? AND a.fence=? ORDER BY a.created_at,a.id').all(token.ownerIncarnation, token.fence).map(a => ({ ...token, kind: 'index', attemptId: String(a.id), jobId: String(a.job_id), stage: String(a.stage), cleanup: a.cleanup, finished: a.finished_at !== null, jobState: String(a.state), cancelRequested: a.cancel_requested === 1 })); }
-export function recoverIndexAttempt(db, token) { const row = attempt(db, token, false); db.prepare("UPDATE index_attempts SET cleanup='settled',stage='settled',finished_at=COALESCE(finished_at,?) WHERE id=?").run(now(), token.attemptId); if (row.finished_at === null)
-    db.prepare("UPDATE index_jobs SET state=CASE WHEN cancel_requested=1 THEN 'cancelled' ELSE 'failed' END,reason='unavailable' WHERE id=?").run(token.jobId); }
+function atomic(db, fn) {
+    if (db.isTransaction)
+        return fn();
+    db.exec('BEGIN IMMEDIATE');
+    try {
+        const result = fn();
+        db.exec('COMMIT');
+        return result;
+    }
+    catch (error) {
+        db.exec('ROLLBACK');
+        throw error;
+    }
+}
+function attempt(db, token, active = true) {
+    ownerMatches(db, token);
+    const row = db
+        .prepare('SELECT a.*,j.state,j.cancel_requested,j.generation_id FROM index_attempts a JOIN index_jobs j ON j.id=a.job_id WHERE a.id=? AND a.job_id=?')
+        .get(token.attemptId, token.jobId);
+    if (!row ||
+        row.owner_incarnation !== token.ownerIncarnation ||
+        row.fence !== token.fence ||
+        (active && (row.finished_at !== null || row.state !== 'running')))
+        throw new SpongeError('owner-busy');
+    if (active && row.cancel_requested === 1)
+        throw new SpongeError('cancelled');
+    return row;
+}
+export function submitIndexJob(db, generationId) {
+    return atomic(db, () => {
+        const generation = db.prepare('SELECT state FROM index_generations WHERE id=?').get(generationId);
+        if (!generation || generation.state !== 'building')
+            throw new SpongeError('unavailable');
+        const prior = db.prepare('SELECT id FROM index_jobs WHERE generation_id=?').get(generationId);
+        if (prior)
+            return String(prior.id);
+        const id = randomUUID();
+        db.prepare("INSERT INTO index_jobs(id,generation_id,state,created_at) VALUES(?,?,'queued',?)").run(id, generationId, now());
+        return id;
+    });
+}
+export function listIndexJobs(db) {
+    return db.prepare("SELECT *, 'index' AS operation, NULL AS item_id FROM index_jobs ORDER BY created_at,id").all();
+}
+export function indexJob(db, id) {
+    return db.prepare('SELECT * FROM index_jobs WHERE id=?').get(id);
+}
+export function claimIndexJob(db, token, id) {
+    return atomic(db, () => {
+        ownerMatches(db, token);
+        const job = id
+            ? indexJob(db, id)
+            : db
+                .prepare("SELECT * FROM index_jobs WHERE state='queued' AND cancel_requested=0 ORDER BY created_at,id LIMIT 1")
+                .get();
+        if (!job) {
+            if (id)
+                throw new SpongeError('unavailable');
+            return null;
+        }
+        if (job.state !== 'queued' || job.cancel_requested)
+            throw new SpongeError('unavailable');
+        if (pendingCleanup(db))
+            throw new SpongeError('owner-busy');
+        const attemptId = randomUUID();
+        db.prepare("UPDATE index_jobs SET state='running',attempt_count=attempt_count+1 WHERE id=?").run(job.id);
+        db.prepare("INSERT INTO index_attempts VALUES(?,?,?,?,'acquiring','pending',?,NULL)").run(attemptId, job.id, token.ownerIncarnation, token.fence, now());
+        return { ...token, jobId: String(job.id), attemptId };
+    });
+}
+export function withIndexAttempt(db, token, fn) {
+    return atomic(db, () => {
+        const row = attempt(db, token);
+        return fn(String(row.generation_id));
+    });
+}
+export function checkpointIndexAttempt(db, token, stage) {
+    const stages = ['acquiring', 'extracting', 'indexing', 'promoting', 'cleanup'];
+    atomic(db, () => {
+        const row = attempt(db, token);
+        if (!stages.includes(stage) || stages.indexOf(stage) < stages.indexOf(String(row.stage)))
+            throw new SpongeError('invalid-input');
+        db.prepare('UPDATE index_attempts SET stage=? WHERE id=?').run(stage, token.attemptId);
+    });
+}
+export function finishIndexAttempt(db, token, reason) {
+    atomic(db, () => {
+        const row = attempt(db, token, false);
+        if (row.finished_at !== null || row.state !== 'running')
+            throw new SpongeError('owner-busy');
+        if (reason !== undefined && !/^[a-z][a-z-]{0,63}$/.test(reason))
+            throw new SpongeError('invalid-input');
+        if (!reason && row.cancel_requested)
+            throw new SpongeError('cancelled');
+        db.prepare('UPDATE index_jobs SET state=?,reason=? WHERE id=?').run(row.cancel_requested ? 'cancelled' : reason ? 'failed' : 'succeeded', reason ?? null, token.jobId);
+        db.prepare("UPDATE index_attempts SET stage='cleanup',finished_at=? WHERE id=?").run(now(), token.attemptId);
+    });
+}
+export function settleIndexAttempt(db, token) {
+    atomic(db, () => {
+        const row = attempt(db, token, false);
+        if (row.finished_at === null)
+            throw new SpongeError('owner-busy');
+        db.prepare("UPDATE index_attempts SET cleanup='settled',stage='settled' WHERE id=?").run(token.attemptId);
+    });
+}
+export function cancelIndexJob(db, id) {
+    atomic(db, () => {
+        const row = indexJob(db, id);
+        if (!row)
+            throw new SpongeError('unavailable');
+        if (row.state === 'succeeded')
+            return;
+        db.prepare("UPDATE index_jobs SET cancel_requested=1,state=CASE WHEN state IN ('queued','failed','cancelled') THEN 'cancelled' ELSE state END WHERE id=?").run(id);
+    });
+}
+export function retryIndexJob(db, id) {
+    atomic(db, () => {
+        const row = indexJob(db, id);
+        if (!row || !['failed', 'cancelled'].includes(String(row.state)))
+            throw new SpongeError('unavailable');
+        if (db.prepare("SELECT count(*) AS n FROM index_attempts WHERE job_id=? AND cleanup='pending'").get(id)?.n)
+            throw new SpongeError('owner-busy');
+        const generation = db.prepare('SELECT state FROM index_generations WHERE id=?').get(row.generation_id);
+        if (!generation || !['building', 'active'].includes(String(generation.state)))
+            throw new SpongeError('unavailable');
+        db.prepare("UPDATE index_jobs SET state='queued',cancel_requested=0,reason=NULL WHERE id=?").run(id); // A retried generation catches up with captures made while it was stopped.
+        syncGeneration(db, String(row.generation_id));
+    });
+}
+export function indexOwnerAttempts(db, token) {
+    ownerMatches(db, token);
+    return db
+        .prepare('SELECT a.*,j.state,j.cancel_requested FROM index_attempts a JOIN index_jobs j ON j.id=a.job_id WHERE a.owner_incarnation=? AND a.fence=? ORDER BY a.created_at,a.id')
+        .all(token.ownerIncarnation, token.fence)
+        .map((a) => ({
+        ...token,
+        kind: 'index',
+        attemptId: String(a.id),
+        jobId: String(a.job_id),
+        stage: String(a.stage),
+        cleanup: a.cleanup,
+        finished: a.finished_at !== null,
+        jobState: String(a.state),
+        cancelRequested: a.cancel_requested === 1,
+    }));
+}
+export function recoverIndexAttempt(db, token) {
+    const row = attempt(db, token, false);
+    db.prepare("UPDATE index_attempts SET cleanup='settled',stage='settled',finished_at=COALESCE(finished_at,?) WHERE id=?").run(now(), token.attemptId);
+    if (row.finished_at === null)
+        db.prepare("UPDATE index_jobs SET state=CASE WHEN cancel_requested=1 THEN 'cancelled' ELSE 'failed' END,reason='unavailable' WHERE id=?").run(token.jobId);
+}

package/dist/src/indexing/prune-run.d.ts

@@ -1,9 +1,11 @@
 import { type LibraryStore } from '../store/index.js';
 import { type PageStats, type PruneEntry } from './prune.js';
+/** `prunedInputs` are coverage rows of pruned generations; `prunedVectors` embeddings no retained generation covers (deleted items' too). */
 export interface PruneReport {
     dryRun: boolean;
     keep: number;
     generations: PruneEntry[];
+    prunedInputs: number;
     prunedVectors: number;
     estimatedBytes: number;
     before: PageStats;

package/dist/src/indexing/prune-run.js

@@ -1,67 +1,110 @@
 import { rm } from 'node:fs/promises';
 import { join } from 'node:path';
-import { StoreError } from '../store/index.js';
 import { acquireWorkerCustody } from '../worker/custody.js';
-import { syncDir } from '../worker/journal.js';
+import { syncDirectoryAsync } from '../util/fsync.js';
 import { maximumKeep } from './prune.js';
+import { SpongeError } from '../errors.js';
 const note = 'Rollback to a pruned generation needs a rebuild (`sponge index create`).';
-const blocked = (store) => { if (store.ownerSnapshot())
-    throw new StoreError('owner-busy'); if (store.pendingCleanupCount())
-    throw Object.assign(Error('cleanup-required'), { code: 'cleanup-required' }); if (store.hasGcFence())
-    throw new StoreError('owner-busy'); };
-const total = (entries, key) => entries.filter(g => g.action === 'prune').reduce((n, g) => n + g[key], 0);
+const blocked = (store) => {
+    if (store.queue.ownerSnapshot())
+        throw new SpongeError('owner-busy');
+    if (store.queue.pendingCleanup())
+        throw new SpongeError('cleanup-required');
+    if (store.blobs.hasGcFence())
+        throw new SpongeError('owner-busy');
+};
+const total = (entries, key) => entries.filter((g) => g.action === 'prune').reduce((n, g) => n + g[key], 0);
 /** Holds worker custody and the store owner fence for the whole run, so no worker, index run or query starts;
  * each bounded batch rechecks owner, cleanup and backup/GC fences in its own transaction. Refuses while index work runs. */
 export async function pruneIndex(store, options = {}) {
     const keep = options.keep ?? 1, dryRun = options.dryRun === true;
     if (!Number.isSafeInteger(keep) || keep < 0 || keep > maximumKeep)
-        throw new StoreError('invalid-input');
+        throw new SpongeError('invalid-input');
     if (dryRun) {
-        const { generations, stats } = store.pruneIndexPlan(keep);
-        return { dryRun, keep, generations, prunedVectors: total(generations, 'vectors'), estimatedBytes: total(generations, 'bytes'), before: stats, after: stats, reclaimedBytes: 0, vacuum: options.vacuum ? 'would-run' : 'skipped', note };
+        const { generations, orphans, stats } = store.index.prunePlan(keep);
+        return {
+            dryRun,
+            keep,
+            generations,
+            prunedInputs: total(generations, 'inputs'),
+            prunedVectors: orphans.vectors,
+            estimatedBytes: total(generations, 'bytes') + orphans.bytes,
+            before: stats,
+            after: stats,
+            reclaimedBytes: 0,
+            vacuum: options.vacuum ? 'would-run' : 'skipped',
+            note,
+        };
     }
+    await store.backups.expireFences();
     blocked(store);
     const custody = await acquireWorkerCustody(store.root);
     let owner;
     try {
-        owner = store.acquireOwner(custody.incarnation);
-        const { generations, stats: before } = store.pruneIndexPlan(keep);
-        let prunedVectors = 0;
-        for (const generation of generations.filter(g => g.action === 'prune'))
+        owner = store.queue.acquireOwner(custody.incarnation);
+        const { generations, orphans, stats: before } = store.index.prunePlan(keep);
+        let prunedInputs = 0, prunedVectors = 0;
+        for (const generation of generations.filter((g) => g.action === 'prune'))
             for (;;) {
-                const n = store.pruneIndexStep(owner, generation.id, keep);
-                prunedVectors += n;
+                const n = store.index.pruneStep(owner, generation.id, keep);
+                prunedInputs += n;
                 if (!n)
                     break;
             }
+        for (;;) {
+            const n = store.index.gcStep(owner);
+            prunedVectors += n;
+            if (!n)
+                break;
+        }
         // A concurrent rollback can make a planned generation a kept one again; report what actually happened.
-        const now = new Map(store.pruneIndexPlan(keep).generations.map(g => [g.id, g])), final = generations.map(g => { const current = now.get(g.id); return g.action === 'prune' && current?.action === 'keep' ? current : g; });
-        let after = store.pruneIndexPlan(keep).stats, vacuum = 'skipped';
+        const now = new Map(store.index.prunePlan(keep).generations.map((g) => [g.id, g])), final = generations.map((g) => {
+            const current = now.get(g.id);
+            return g.action === 'prune' && current?.action === 'keep' ? current : g;
+        });
+        let after = store.index.prunePlan(keep).stats, vacuum = 'skipped';
         if (options.vacuum) {
-            after = store.vacuumLibrary(owner);
+            after = store.index.vacuum(owner);
             vacuum = 'done';
         }
-        return { dryRun, keep, generations: final, prunedVectors, estimatedBytes: total(final, 'bytes'), before, after, reclaimedBytes: vacuum === 'done' ? before.bytes - after.bytes : (after.freePages - before.freePages) * after.pageSize, vacuum, note };
+        return {
+            dryRun,
+            keep,
+            generations: final,
+            prunedInputs,
+            prunedVectors,
+            estimatedBytes: total(final, 'bytes') + orphans.bytes,
+            before,
+            after,
+            reclaimedBytes: vacuum === 'done' ? before.bytes - after.bytes : (after.freePages - before.freePages) * after.pageSize,
+            vacuum,
+            note,
+        };
     }
     finally {
         try {
             if (owner)
-                store.releaseOwner(owner);
+                store.queue.releaseOwner(owner);
         }
         finally {
             await custody.release();
             try {
                 await rm(custody.directory, { recursive: true });
-                await syncDir(join(store.root, 'runtime', 'workers'));
+                await syncDirectoryAsync(join(store.root, 'runtime', 'workers'));
+            }
+            catch {
+                /* Runtime GC removes this settled journal. */
             }
-            catch { /* Runtime GC removes this settled journal. */ }
         }
     }
 }
 export function pruneText(report) {
-    const lines = report.generations.map(g => `${g.action === 'prune' ? (report.dryRun ? 'would prune' : 'pruned') : 'kept'}\t${g.id}\t${g.profileId}\t${g.space}\t${g.state}\t${g.reason}\t${g.vectors} vectors`);
-    lines.push(report.dryRun ? `Would free about ${report.estimatedBytes} bytes of rows${report.vacuum === 'would-run' ? '; --vacuum would then shrink the file' : ''}` : `Reclaimed ${report.reclaimedBytes} bytes (${report.vacuum === 'done' ? 'file shrank after VACUUM' : 'freed pages are reused; --vacuum shrinks the file'})`);
-    if (report.generations.some(g => g.action === 'prune'))
+    const lines = report.generations.map((g) => `${g.action === 'prune' ? (report.dryRun ? 'would prune' : 'pruned') : 'kept'}\t${g.id}\t${g.profileId}\t${g.space}\t${g.state}\t${g.reason}\t${g.inputs} inputs`);
+    lines.push(`${report.dryRun ? 'Would delete' : 'Deleted'} ${report.prunedVectors} uncovered vectors`);
+    lines.push(report.dryRun
+        ? `Would free about ${report.estimatedBytes} bytes of rows${report.vacuum === 'would-run' ? '; --vacuum would then shrink the file' : ''}`
+        : `Reclaimed ${report.reclaimedBytes} bytes (${report.vacuum === 'done' ? 'file shrank after VACUUM' : 'freed pages are reused; --vacuum shrinks the file'})`);
+    if (report.generations.some((g) => g.action === 'prune'))
         lines.push(report.note);
     return lines.join('\n');
 }

package/dist/src/indexing/prune.d.ts

@@ -1,5 +1,6 @@
 import type { DatabaseSync } from 'node:sqlite';
-export type PruneReason = 'active' | 'building' | 'queued' | 'running' | 'rollback' | 'unknown' | 'retired' | 'stale' | 'cancelled' | 'failed' | 'pruned';
+type PruneReason = 'active' | 'building' | 'queued' | 'running' | 'rollback' | 'unknown' | 'retired' | 'stale' | 'cancelled' | 'failed' | 'pruned';
+/** `inputs` are the generation's coverage rows; `bytes` their size. Vectors are shared by content and counted by `orphanVectors`. */
 export interface PruneEntry {
     id: string;
     profileId: string;
@@ -8,7 +9,7 @@ export interface PruneEntry {
     state: string;
     action: 'keep' | 'prune';
     reason: PruneReason;
-    vectors: number;
+    inputs: number;
     bytes: number;
 }
 export interface PageStats {
@@ -17,7 +18,7 @@ export interface PageStats {
     freePages: number;
     bytes: number;
 }
-export declare const maximumKeep = 16, pruneBatchRows = 1000;
+export declare const maximumKeep = 16;
 /** Caller holds a write transaction. Keeps active, building/resumable and job-referenced work, plus the `keep` most
  * recent rollback-ready retired generations per profile. Stale retired generations can never activate again (the
  * revision epoch only grows), so they are pruned. Recency is index job creation; VACUUM may renumber rowids. */
@@ -25,6 +26,14 @@ export declare function planPrune(db: DatabaseSync, keep: number): PruneEntry[];
 /** One bounded step for one planned generation, inside the caller's write transaction. The first step marks it pruned
  * and drops its coverage, so it is never rollback-ready while partially deleted. Returns rows deleted; 0 means done. */
 export declare function pruneStep(db: DatabaseSync, id: string, keep: number, limit?: number): number;
+/** Embeddings, and their bytes, that pruning these generations leaves without coverage (deleted items' included). */
+export declare function orphanVectors(db: DatabaseSync, pruning: readonly string[]): {
+    vectors: number;
+    bytes: number;
+};
+/** One bounded batch of orphan embeddings, inside the caller's write transaction. Returns rows deleted; 0 means done. */
+export declare function gcStep(db: DatabaseSync, limit?: number): number;
 export declare function pageStats(db: DatabaseSync): PageStats;
 /** VACUUM rewrites the database through a temporary copy and the WAL: about the file size again on each volume. */
 export declare function vacuumSpace(root: string, bytes: number, margin?: number): void;
+export {};

package/dist/src/indexing/prune.js

@@ -1,36 +1,44 @@
 import { statSync, statfsSync } from 'node:fs';
 import { tmpdir } from 'node:os';
-import { StoreError } from '../store/types.js';
-import { activateGeneration, IndexError, spaceEncoding } from './index.js';
-export const maximumKeep = 16, pruneBatchRows = 1000;
-/** Pruned rows keep their id (index_jobs history references it) with no coverage and no vectors. */
-const prunedCoverage = JSON.stringify({ version: 1, revisions: [], inputs: [] });
+import { activateGeneration, coverageMarker, spaceEncoding } from './index.js';
+import { SpongeError, indexCodes, codeOf } from '../errors.js';
+export const maximumKeep = 16;
+const pruneBatchRows = 1000;
+/** Pruned rows keep their id (index_jobs history references it) with no coverage. */
 /** Exactly the rollback rules: a probe activation inside a savepoint that is always rolled back. */
-function rollbackReady(db, id) { db.exec('SAVEPOINT prune_probe'); try {
-    activateGeneration(db, id);
-    return true;
-}
-catch (error) {
-    if (error instanceof IndexError)
-        return false;
-    throw error;
+function rollbackReady(db, id) {
+    db.exec('SAVEPOINT prune_probe');
+    try {
+        activateGeneration(db, id);
+        return true;
+    }
+    catch (error) {
+        if (indexCodes.includes(codeOf(error)))
+            return false;
+        throw error;
+    }
+    finally {
+        db.exec('ROLLBACK TO prune_probe; RELEASE prune_probe');
+    }
 }
-finally {
-    db.exec('ROLLBACK TO prune_probe; RELEASE prune_probe');
-} }
 /** Caller holds a write transaction. Keeps active, building/resumable and job-referenced work, plus the `keep` most
  * recent rollback-ready retired generations per profile. Stale retired generations can never activate again (the
  * revision epoch only grows), so they are pruned. Recency is index job creation; VACUUM may renumber rowids. */
 export function planPrune(db, keep) {
     if (!Number.isSafeInteger(keep) || keep < 0 || keep > maximumKeep)
-        throw new StoreError('invalid-input');
+        throw new SpongeError('invalid-input');
     if (!db.isTransaction)
-        throw new StoreError('invalid-input');
-    const epoch = Number(db.prepare('SELECT revision_epoch FROM library_info WHERE id=1').get()?.revision_epoch), active = new Set(db.prepare('SELECT generation_id FROM active_indexes').all().map(r => String(r.generation_id)));
-    const rows = db.prepare("SELECT g.id,g.fingerprint,g.state,g.revision_epoch,g.profile_json,length(g.coverage_json) AS coverage,s.dimensions,s.identity_json,j.state AS job,(SELECT count(*) FROM vectors v WHERE v.generation_id=g.id) AS vectors,(SELECT COALESCE(sum(length(v.vector)+length(v.source_id)+length(v.revision_id)+length(v.generation_id)+length(v.derivation_hash)+COALESCE(length(v.input_hash),0)+COALESCE(length(v.asset_sha256),0)),0) FROM vectors v WHERE v.generation_id=g.id) AS bytes FROM index_generations g JOIN embedding_spaces s ON s.fingerprint=g.fingerprint LEFT JOIN index_jobs j ON j.generation_id=g.id ORDER BY j.created_at IS NULL,j.created_at DESC,g.rowid DESC").all();
+        throw new SpongeError('invalid-input');
+    const epoch = Number(db.prepare('SELECT revision_epoch FROM library_info WHERE id=1').get()?.revision_epoch), active = new Set(db
+        .prepare('SELECT generation_id FROM active_indexes')
+        .all()
+        .map((r) => String(r.generation_id)));
+    const rows = db
+        .prepare('SELECT g.id,g.fingerprint,g.state,g.revision_epoch,g.profile_json,length(g.coverage_json) AS coverage,s.dimensions,s.identity_json,j.state AS job,(SELECT count(*) FROM index_inputs c WHERE c.generation_id=g.id) AS inputs,(SELECT COALESCE(sum(length(c.generation_id)+length(c.source_id)+length(c.revision_id)+length(c.content_hash)),0) FROM index_inputs c WHERE c.generation_id=g.id) AS bytes FROM index_generations g JOIN embedding_spaces s ON s.fingerprint=g.fingerprint LEFT JOIN index_jobs j ON j.generation_id=g.id ORDER BY j.created_at IS NULL,j.created_at DESC,g.rowid DESC')
+        .all();
     const kept = new Map();
-    return rows.flatMap(row => {
-        const id = String(row.id), state = String(row.state), job = row.job === null ? null : String(row.job), vectors = Number(row.vectors);
+    return rows.flatMap((row) => {
+        const id = String(row.id), state = String(row.state), job = row.job === null ? null : String(row.job), inputs = Number(row.inputs);
         let profileId, space;
         try {
             profileId = String(JSON.parse(String(row.profile_json)).profileId);
@@ -38,11 +46,23 @@ export function planPrune(db, keep) {
             space = `${spaceEncoding(identity)}-${Number(row.dimensions)}`;
         }
         catch {
-            throw new StoreError('integrity');
+            throw new SpongeError('integrity');
         }
-        if (state === 'pruned' && !vectors && Number(row.coverage) <= prunedCoverage.length)
+        if (state === 'pruned' && !inputs && Number(row.coverage) <= coverageMarker.length)
             return [];
-        const entry = (action, reason) => [{ id, profileId, fingerprint: String(row.fingerprint), space, state, action, reason, vectors, bytes: Number(row.bytes) + (action === 'prune' ? Math.max(0, Number(row.coverage) - prunedCoverage.length) : 0) }];
+        const entry = (action, reason) => [
+            {
+                id,
+                profileId,
+                fingerprint: String(row.fingerprint),
+                space,
+                state,
+                action,
+                reason,
+                inputs,
+                bytes: Number(row.bytes) + (action === 'prune' ? Math.max(0, Number(row.coverage) - coverageMarker.length) : 0),
+            },
+        ];
         if (active.has(id))
             return entry('keep', 'active');
         if (job === 'running' || job === 'cleanup-pending')
@@ -53,7 +73,11 @@ export function planPrune(db, keep) {
             return entry('prune', 'cancelled');
         // A building generation stays resumable until the revision epoch moves; after that it can never activate.
         if (state === 'building')
-            return job === 'queued' ? entry('keep', 'queued') : Number(row.revision_epoch) === epoch ? entry('keep', 'building') : entry('prune', 'failed');
+            return job === 'queued'
+                ? entry('keep', 'queued')
+                : Number(row.revision_epoch) === epoch
+                    ? entry('keep', 'building')
+                    : entry('prune', 'failed');
         if (state !== 'retired')
             return entry('keep', 'unknown');
         if (!rollbackReady(db, id))
@@ -70,26 +94,52 @@ export function planPrune(db, keep) {
  * and drops its coverage, so it is never rollback-ready while partially deleted. Returns rows deleted; 0 means done. */
 export function pruneStep(db, id, keep, limit = pruneBatchRows) {
     if (!Number.isSafeInteger(limit) || limit < 1 || limit > 10000)
-        throw new StoreError('invalid-input');
+        throw new SpongeError('invalid-input');
     const row = db.prepare('SELECT state FROM index_generations WHERE id=?').get(id);
     if (!row)
-        throw new StoreError('unavailable');
+        throw new SpongeError('unavailable');
     // Once marked, a generation can never become active, building or retired again: no replanning per batch.
     if (row.state !== 'pruned') {
-        const planned = planPrune(db, keep).find(e => e.id === id);
+        const planned = planPrune(db, keep).find((e) => e.id === id);
         if (!planned || planned.action !== 'prune')
             return 0;
-        db.prepare("UPDATE index_generations SET state='pruned',coverage_json=? WHERE id=?").run(prunedCoverage, id);
+        db.prepare("UPDATE index_generations SET state='pruned',coverage_json=? WHERE id=?").run(coverageMarker, id);
+        db.prepare('DELETE FROM index_items WHERE generation_id=?').run(id);
         db.prepare("UPDATE index_jobs SET state='cancelled',cancel_requested=1 WHERE generation_id=? AND state='queued'").run(id);
     }
     else if (db.prepare('SELECT 1 FROM active_indexes WHERE generation_id=?').get(id))
-        throw new StoreError('integrity');
-    return Number(db.prepare('DELETE FROM vectors WHERE rowid IN (SELECT rowid FROM vectors WHERE generation_id=? LIMIT ?)').run(id, limit).changes);
+        throw new SpongeError('integrity');
+    return Number(db
+        .prepare('DELETE FROM index_inputs WHERE rowid IN (SELECT rowid FROM index_inputs WHERE generation_id=? LIMIT ?)')
+        .run(id, limit).changes);
+}
+/** An embedding is an orphan when no generation outside `pruning` covers its content in its space. */
+const orphan = 'NOT EXISTS(SELECT 1 FROM index_inputs c JOIN index_generations g ON g.id=c.generation_id WHERE c.content_hash=e.content_hash AND g.fingerprint=e.fingerprint AND g.id NOT IN (SELECT value FROM json_each(?)))';
+/** Embeddings, and their bytes, that pruning these generations leaves without coverage (deleted items' included). */
+export function orphanVectors(db, pruning) {
+    const row = db
+        .prepare(`SELECT count(*) n,COALESCE(sum(length(e.fingerprint)+length(e.content_hash)+length(e.input_hash)+length(e.vector)),0) b FROM embeddings e WHERE ${orphan}`)
+        .get(JSON.stringify(pruning));
+    return { vectors: Number(row?.n), bytes: Number(row?.b) };
+}
+/** One bounded batch of orphan embeddings, inside the caller's write transaction. Returns rows deleted; 0 means done. */
+export function gcStep(db, limit = pruneBatchRows) {
+    if (!Number.isSafeInteger(limit) || limit < 1 || limit > 10000)
+        throw new SpongeError('invalid-input');
+    return Number(db
+        .prepare(`DELETE FROM embeddings WHERE rowid IN (SELECT e.rowid FROM embeddings e WHERE ${orphan} LIMIT ?)`)
+        .run('[]', limit).changes);
+}
+export function pageStats(db) {
+    const pageSize = Number(db.prepare('PRAGMA page_size').get()?.page_size), pages = Number(db.prepare('PRAGMA page_count').get()?.page_count), freePages = Number(db.prepare('PRAGMA freelist_count').get()?.freelist_count);
+    return { pageSize, pages, freePages, bytes: pages * pageSize };
 }
-export function pageStats(db) { const pageSize = Number(db.prepare('PRAGMA page_size').get()?.page_size), pages = Number(db.prepare('PRAGMA page_count').get()?.page_count), freePages = Number(db.prepare('PRAGMA freelist_count').get()?.freelist_count); return { pageSize, pages, freePages, bytes: pages * pageSize }; }
 /** VACUUM rewrites the database through a temporary copy and the WAL: about the file size again on each volume. */
 export function vacuumSpace(root, bytes, margin = 64 * 1024 ** 2) {
-    const free = (path) => { const s = statfsSync(path); return s.bavail * s.bsize; }, temp = tmpdir(), same = statSync(root).dev === statSync(temp).dev;
+    const free = (path) => {
+        const s = statfsSync(path);
+        return s.bavail * s.bsize;
+    }, temp = tmpdir(), same = statSync(root).dev === statSync(temp).dev;
     if (same ? free(root) < 2 * bytes + margin : free(root) < bytes + margin || free(temp) < bytes + margin)
-        throw new StoreError('limit-exceeded');
+        throw new SpongeError('limit-exceeded');
 }

package/dist/src/indexing/runner.d.ts

@@ -19,5 +19,6 @@ export interface IndexRunOptions {
     onCheckpoint?: (attempt: AttemptToken, stage: string) => Promise<void>;
     onBatch?: (attempt: AttemptToken, completed: number) => Promise<void>;
 }
-/** Same durable worker lock/owner/supervisor as acquisition, with sequential bounded batches. */
+/** Same durable worker lock/owner/supervisor as acquisition, with sequential bounded batches. Only inputs whose content the space
+ * has not embedded run: a delta after captures, or everything new for a new fingerprint. Captures during a run join it. */
 export declare function drainIndexQueue(store: LibraryStore, options: IndexRunOptions): Promise<IndexRunResult[]>;