@hraness/sponge 0.2.6 → 0.3.0

package/dist/src/indexing/index.js

@@ -1,154 +1,390 @@
-import { createHash, randomUUID } from 'node:crypto';
-import { spaceFingerprint, encodeEmbeddingRequest, embeddingTextBytes } from '../inference/embedding/index.js';
-export class IndexError extends Error {
-    code;
-    constructor(code) {
-        super(code);
-        this.code = code;
+import { randomUUID } from 'node:crypto';
+import { sha256 as hash } from '../util/hash.js';
+import { spaceFingerprint, embeddingTextBytes } from '../inference/embedding/index.js';
+import { SpongeError } from '../errors.js';
+const fail = (code = 'invalid-index') => {
+    throw new SpongeError(code);
+};
+function atomic(db, fn) {
+    if (db.isTransaction)
+        return fn();
+    db.exec('BEGIN IMMEDIATE');
+    try {
+        const value = fn();
+        db.exec('COMMIT');
+        return value;
     }
+    catch (e) {
+        db.exec('ROLLBACK');
+        throw e;
+    }
+}
+function epoch(db) {
+    return Number(db.prepare('SELECT revision_epoch FROM library_info WHERE id=1').get()?.revision_epoch);
 }
-const fail = (code = 'invalid-index') => { throw new IndexError(code); };
-const hash = (value) => createHash('sha256').update(value).digest('hex');
-function atomic(db, fn) { if (db.isTransaction)
-    return fn(); db.exec('BEGIN IMMEDIATE'); try {
-    const value = fn();
-    db.exec('COMMIT');
-    return value;
-}
-catch (e) {
-    db.exec('ROLLBACK');
-    throw e;
-} }
-function epoch(db) { return Number(db.prepare('SELECT revision_epoch FROM library_info WHERE id=1').get()?.revision_epoch); }
-function revisions(db) { return db.prepare('SELECT id AS itemId,current_revision AS revisionId FROM items WHERE deleted_at IS NULL AND current_revision IS NOT NULL ORDER BY id').all(); }
+/** Schema 10 keeps coverage in `index_items` and `index_inputs`; a generation whose JSON coverage could not be converted keeps it and never reads. */
+export const coverageMarker = '{"version":2}';
 /** Storage format follows the space identity: no `quantization` (legacy) or `f32` stores float32 LE; `int8-symmetric-maxabs-v1` stores one int8 code per dimension. */
-export function spaceEncoding(identity) { const q = identity.quantization; if (q === undefined || q === 'f32')
-    return 'f32'; if (q === 'int8-symmetric-maxabs-v1')
-    return 'int8'; return fail(); }
+export function spaceEncoding(identity) {
+    const q = identity.quantization;
+    if (q === undefined || q === 'f32')
+        return 'f32';
+    if (q === 'int8-symmetric-maxabs-v1')
+        return 'int8';
+    return fail();
+}
 /** int8 codes are round(x*127/max|x|) of a unit vector. The per-vector scale is not stored: cosine is scale-invariant, so codes rank directly and dequantize by renormalizing. */
-export function encodeVector(vector, encoding) { if (!vector.length || vector.some(n => !Number.isFinite(n)))
-    fail(); if (encoding === 'f32') {
-    const bytes = Buffer.alloc(vector.length * 4);
-    vector.forEach((n, i) => bytes.writeFloatLE(n, i * 4));
-    return bytes;
-} const max = Math.max(...vector.map(Math.abs)); if (!(max > 0))
-    fail(); return Buffer.from(Int8Array.from(vector, n => Math.round(n * 127 / max)).buffer); }
+export function encodeVector(vector, encoding) {
+    if (!vector.length || vector.some((n) => !Number.isFinite(n)))
+        fail();
+    if (encoding === 'f32') {
+        const bytes = Buffer.alloc(vector.length * 4);
+        vector.forEach((n, i) => bytes.writeFloatLE(n, i * 4));
+        return bytes;
+    }
+    const max = Math.max(...vector.map(Math.abs));
+    if (!(max > 0))
+        fail();
+    return Buffer.from(Int8Array.from(vector, (n) => Math.round((n * 127) / max)).buffer);
+}
 /** Exact byte length, finite floats, int8 codes within [-127,127] and nonzero; int8 returns the unit dequantized vector. */
-export function decodeVector(bytes, dimensions, encoding) { const b = Buffer.from(bytes); if (encoding === 'f32') {
-    if (b.length !== dimensions * 4)
+export function decodeVector(bytes, dimensions, encoding) {
+    const b = Buffer.from(bytes);
+    if (encoding === 'f32') {
+        if (b.length !== dimensions * 4)
+            fail();
+        const vector = Array.from({ length: dimensions }, (_, i) => b.readFloatLE(i * 4));
+        if (vector.some((n) => !Number.isFinite(n)))
+            fail();
+        return vector;
+    }
+    if (b.length !== dimensions)
+        fail();
+    const codes = Array.from({ length: dimensions }, (_, i) => b.readInt8(i));
+    if (codes.some((n) => n < -127))
         fail();
-    const vector = Array.from({ length: dimensions }, (_, i) => b.readFloatLE(i * 4));
-    if (vector.some(n => !Number.isFinite(n)))
+    const norm = Math.hypot(...codes);
+    if (!(norm > 0))
         fail();
-    return vector;
-} if (b.length !== dimensions)
-    fail(); const codes = Array.from({ length: dimensions }, (_, i) => b.readInt8(i)); if (codes.some(n => n < -127))
-    fail(); const norm = Math.hypot(...codes); if (!(norm > 0))
-    fail(); return codes.map(n => n / norm); }
+    return codes.map((n) => n / norm);
+}
+/** What one document chunk's vector was derived from before schema 10: its representation, position and text. Migrations 9 and 10 still read it. */
+export const chunkHash = (representationId, ordinal, text) => hash(JSON.stringify(['sponge-chunk-v2', representationId, ordinal, text]));
 const chunkBytes = embeddingTextBytes - Buffer.byteLength('title: none | text: ');
 /** Unicode-safe chunks fit the fixed document prompt within the embedding text budget, preferring whitespace breaks. */
-export function chunkDocument(text) { const chunks = []; let part = '', size = 0, cut = -1, cutSize = 0; for (const char of text) {
-    const bytes = Buffer.byteLength(char);
-    if (size + bytes > chunkBytes) {
-        const at = cut > 0 && cutSize * 2 >= chunkBytes ? cut : part.length;
-        const head = part.slice(0, at);
-        if (head.trim())
-            chunks.push(head);
-        part = part.slice(at);
-        size = Buffer.byteLength(part);
-        cut = -1;
+export function chunkDocument(text) {
+    const chunks = [];
+    let part = '', size = 0, cut = -1, cutSize = 0;
+    for (const char of text) {
+        const bytes = Buffer.byteLength(char);
+        if (size + bytes > chunkBytes) {
+            const at = cut > 0 && cutSize * 2 >= chunkBytes ? cut : part.length;
+            const head = part.slice(0, at);
+            if (head.trim())
+                chunks.push(head);
+            part = part.slice(at);
+            size = Buffer.byteLength(part);
+            cut = -1;
+        }
+        part += char;
+        size += bytes;
+        if (/\s/u.test(char)) {
+            cut = part.length;
+            cutSize = size;
+        }
+    }
+    if (part.trim())
+        chunks.push(part);
+    return chunks;
+}
+/** The helper's input hash of a document chunk: `encodeEmbeddingRequest([{kind:'document',text}]).inputHashes[0]` without its validation, so a commit never fails on text the embedder refuses. */
+export function documentHash(text) {
+    const payload = Buffer.from('title: none | text: ' + text), header = Buffer.alloc(5);
+    header[0] = 1;
+    header.writeUInt32BE(payload.length, 1);
+    return hash(Buffer.concat([header, payload]));
+}
+export const imageHash = (assetSha256) => hash(JSON.stringify(['sponge-image-source-v1', assetSha256]));
+/** A source id is `[revisionId,'document',representationId,ordinal]` or `[revisionId,'image',assetSha256]`. */
+export function parseSource(sourceId, revisionId) {
+    let d;
+    try {
+        d = JSON.parse(sourceId);
     }
-    part += char;
-    size += bytes;
-    if (/\s/u.test(char)) {
-        cut = part.length;
-        cutSize = size;
+    catch {
+        return fail();
     }
-} if (part.trim())
-    chunks.push(part); return chunks; }
-function snapshot(db) {
-    const current = revisions(db), inputs = [];
-    for (const r of current) {
-        const docs = db.prepare('SELECT id,markdown FROM representations WHERE revision_id=? AND markdown IS NOT NULL ORDER BY id').all(r.revisionId);
-        for (const doc of docs) {
-            const chunks = chunkDocument(String(doc.markdown));
-            for (let ordinal = 0; ordinal < chunks.length; ordinal++) {
-                const text = chunks[ordinal], derivationHash = hash(JSON.stringify(['sponge-chunk-v2', String(doc.id), ordinal, text]));
-                inputs.push({ ...r, sourceId: JSON.stringify([r.revisionId, 'document', String(doc.id), ordinal]), kind: 'document', representationId: String(doc.id), ordinal, derivationHash });
+    if (!Array.isArray(d) || d[0] !== revisionId)
+        return fail();
+    if (d[1] === 'document' && d.length === 4 && typeof d[2] === 'string' && Number.isSafeInteger(d[3]) && d[3] >= 0)
+        return { kind: 'document', representationId: d[2], ordinal: d[3] };
+    if (d[1] === 'image' && d.length === 3 && typeof d[2] === 'string' && /^[a-f0-9]{64}$/.test(d[2]))
+        return { kind: 'image', assetSha256: d[2] };
+    return fail();
+}
+export const sourceInput = (row) => ({
+    sourceId: String(row.source_id),
+    revisionId: String(row.revision_id),
+    itemId: String(row.item_id),
+    derivationHash: String(row.content_hash),
+    ...parseSource(String(row.source_id), String(row.revision_id)),
+});
+/** Chunks and images of one revision. A chunk the helper refuses (a NUL byte) is left out rather than failing every later delta. */
+function revisionInputs(db, revisionId) {
+    const inputs = [];
+    for (const doc of db
+        .prepare('SELECT id,markdown FROM representations WHERE revision_id=? AND markdown IS NOT NULL ORDER BY id')
+        .all(revisionId))
+        chunkDocument(String(doc.markdown)).forEach((text, ordinal) => {
+            if (!text.includes('\0'))
+                inputs.push({
+                    sourceId: JSON.stringify([revisionId, 'document', String(doc.id), ordinal]),
+                    contentHash: documentHash(text),
+                });
+        });
+    for (const image of db
+        .prepare("SELECT DISTINCT a.sha256 FROM revision_assets ra JOIN assets a ON a.sha256=ra.sha256 WHERE ra.revision_id=? AND a.media_type LIKE 'image/%' ORDER BY a.sha256")
+        .all(revisionId))
+        inputs.push({
+            sourceId: JSON.stringify([revisionId, 'image', String(image.sha256)]),
+            contentHash: imageHash(String(image.sha256)),
+        });
+    return inputs;
+}
+function generation(db, id) {
+    const row = db
+        .prepare('SELECT g.*,s.dimensions,s.identity_json FROM index_generations g JOIN embedding_spaces s ON s.fingerprint=g.fingerprint WHERE g.id=?')
+        .get(id);
+    if (!row)
+        return fail();
+    if (row.coverage_json !== coverageMarker)
+        fail();
+    const profile = JSON.parse(String(row.profile_json));
+    if (typeof profile.profileId !== 'string')
+        fail();
+    return { row, profile, encoding: spaceEncoding(JSON.parse(String(row.identity_json))) };
+}
+const missing = 'NOT EXISTS(SELECT 1 FROM embeddings e WHERE e.fingerprint=? AND e.content_hash=c.content_hash)';
+/** Inputs whose content has no embedding in the space yet. */
+const pendingCount = (db, id, fingerprint) => Number(db.prepare(`SELECT count(*) n FROM index_inputs c WHERE c.generation_id=? AND ${missing}`).get(id, fingerprint)?.n);
+/** Items whose current, non-deleted revision differs from the one a generation covers. */
+const drift = `FROM items i LEFT JOIN index_items x ON x.generation_id=? AND x.item_id=i.id WHERE (CASE WHEN i.deleted_at IS NULL THEN i.current_revision END) IS NOT x.revision_id`;
+/** Brings a building or active generation's coverage to the current revisions: inputs of refreshed or deleted items go, the
+ * new revision's come. Embeddings stay; an input is pending while its space has none for its content. Returns items changed. */
+export function syncGeneration(db, id) {
+    return atomic(db, () => {
+        const g = db.prepare('SELECT state,coverage_json FROM index_generations WHERE id=?').get(id);
+        if (!g || g.coverage_json !== coverageMarker || !['building', 'active'].includes(String(g.state)))
+            return 0;
+        const changed = db
+            .prepare(`SELECT i.id,CASE WHEN i.deleted_at IS NULL THEN i.current_revision END AS current,x.revision_id AS covered ${drift}`)
+            .all(id), insert = db.prepare('INSERT OR IGNORE INTO index_inputs VALUES(?,?,?,?)');
+        for (const item of changed) {
+            if (item.covered !== null) {
+                db.prepare('DELETE FROM index_inputs WHERE generation_id=? AND revision_id=?').run(id, item.covered);
+                db.prepare('DELETE FROM index_items WHERE generation_id=? AND item_id=?').run(id, item.id);
+            }
+            if (item.current !== null) {
+                const revisionId = String(item.current);
+                db.prepare('INSERT INTO index_items VALUES(?,?,?)').run(id, item.id, revisionId);
+                for (const input of revisionInputs(db, revisionId))
+                    insert.run(id, input.sourceId, revisionId, input.contentHash);
             }
         }
-        const images = db.prepare("SELECT DISTINCT a.sha256 FROM revision_assets ra JOIN assets a ON a.sha256=ra.sha256 WHERE ra.revision_id=? AND a.media_type LIKE 'image/%' ORDER BY a.sha256").all(r.revisionId);
-        for (const image of images) {
-            const assetSha256 = String(image.sha256);
-            inputs.push({ ...r, sourceId: JSON.stringify([r.revisionId, 'image', assetSha256]), kind: 'image', assetSha256, derivationHash: hash(JSON.stringify(['sponge-image-source-v1', assetSha256])) });
-        }
+        db.prepare('UPDATE index_generations SET revision_epoch=? WHERE id=?').run(epoch(db), id);
+        return changed.length;
+    });
+}
+/** Queues a generation's index job again (or its first) unless it is queued, running or awaiting cleanup. */
+export function armJob(db, id) {
+    const job = db.prepare('SELECT id,state FROM index_jobs WHERE generation_id=?').get(id);
+    if (!job) {
+        db.prepare("INSERT INTO index_jobs(id,generation_id,state,created_at) VALUES(?,?,'queued',?)").run(randomUUID(), id, new Date().toISOString());
+        return;
     }
-    return { version: 1, revisions: current, inputs };
+    if (['succeeded', 'failed', 'cancelled'].includes(String(job.state)) &&
+        !db.prepare("SELECT 1 FROM index_attempts WHERE job_id=? AND cleanup='pending'").get(job.id))
+        db.prepare("UPDATE index_jobs SET state='queued',cancel_requested=0,reason=NULL WHERE id=?").run(job.id);
+}
+/** Inside the transaction that changed a current revision or deleted an item: every active generation, and every building one
+ * whose job is live, follows the library. An active generation with content to embed gets its job queued: only the delta runs. */
+export function trackLibrary(db) {
+    atomic(db, () => {
+        for (const g of db
+            .prepare("SELECT id,fingerprint,state FROM index_generations g WHERE coverage_json=? AND (id IN (SELECT generation_id FROM active_indexes) OR state='building' AND EXISTS(SELECT 1 FROM index_jobs j WHERE j.generation_id=g.id AND j.state IN ('queued','running','cleanup-pending'))) ORDER BY rowid")
+            .all(coverageMarker)) {
+            const id = String(g.id);
+            syncGeneration(db, id);
+            if (g.state === 'active' && pendingCount(db, id, String(g.fingerprint)))
+                armJob(db, id);
+        }
+    });
 }
-function generation(db, id) { const row = db.prepare('SELECT g.*,s.dimensions,s.identity_json FROM index_generations g JOIN embedding_spaces s ON s.fingerprint=g.fingerprint WHERE g.id=?').get(id); if (!row)
-    return fail(); const coverage = JSON.parse(String(row.coverage_json)); const profile = JSON.parse(String(row.profile_json)); if (coverage.version !== 1 || !Array.isArray(coverage.inputs) || !Array.isArray(coverage.revisions) || typeof profile.profileId !== 'string')
-    fail(); return { row, coverage, profile, encoding: spaceEncoding(JSON.parse(String(row.identity_json))) }; }
-/** Identity must already be the verified embedder fingerprint. Existing identities are immutable. */
+/** Identity must already be the verified embedder fingerprint. Existing identities are immutable. Every input whose content the
+ * space already embedded (an earlier generation, another item) is complete at once. */
 export function createGeneration(db, options) {
-    if (!options.profileId || options.profileId.length > 128 || !/^[a-f0-9]{64}$/.test(options.fingerprint) || !Number.isSafeInteger(options.dimensions) || options.dimensions < 1 || options.dimensions > 4096)
+    if (!options.profileId ||
+        options.profileId.length > 128 ||
+        !/^[a-f0-9]{64}$/.test(options.fingerprint) ||
+        !Number.isSafeInteger(options.dimensions) ||
+        options.dimensions < 1 ||
+        options.dimensions > 4096)
         fail();
     if (spaceFingerprint(options.identity) !== options.fingerprint || options.identity.dimensions !== options.dimensions)
         fail();
     spaceEncoding(options.identity);
-    return atomic(db, () => { const identity = JSON.stringify(Object.fromEntries(Object.keys(options.identity).sort().map(k => [k, options.identity[k]]))); if (!identity || Buffer.byteLength(identity) > 16384)
-        fail(); const existing = db.prepare('SELECT * FROM embedding_spaces WHERE fingerprint=?').get(options.fingerprint); if (existing && (existing.identity_json !== identity || Number(existing.dimensions) !== options.dimensions || existing.normalization !== 'unit-l2' || existing.metric !== 'cosine'))
-        fail(); if (!existing)
-        db.prepare('INSERT INTO embedding_spaces VALUES(?,?,?,?,?)').run(options.fingerprint, identity, options.dimensions, 'unit-l2', 'cosine'); const id = randomUUID(); db.prepare('INSERT INTO index_generations VALUES(?,?,?,?,?,?)').run(id, options.fingerprint, JSON.stringify({ profileId: options.profileId }), 'building', epoch(db), JSON.stringify(snapshot(db))); return generationStatus(db, id); });
-}
-export function generationStatus(db, id) { const { row, coverage, profile } = generation(db, id); return { id, fingerprint: String(row.fingerprint), state: String(row.state), revisionEpoch: Number(row.revision_epoch), expected: coverage.inputs.length, completed: Number(db.prepare('SELECT count(*) AS n FROM vectors WHERE generation_id=?').get(id)?.n), profileId: profile.profileId }; }
-export function pendingInputs(db, id, limit = 16) { if (!Number.isSafeInteger(limit) || limit < 1 || limit > 256)
-    fail(); const { row, coverage } = generation(db, id); if (row.state === 'cancelled')
-    fail('cancelled-index'); if (row.state !== 'building')
-    return []; const done = new Set(db.prepare('SELECT source_id FROM vectors WHERE generation_id=? AND input_hash IS NOT NULL').all(id).map(r => String(r.source_id))); return coverage.inputs.filter(i => !done.has(i.sourceId)).slice(0, limit).map(input => { if (input.kind === 'image')
-    return input; const doc = db.prepare('SELECT markdown FROM representations WHERE id=? AND revision_id=?').get(input.representationId, input.revisionId); if (!doc || typeof doc.markdown !== 'string')
-    return fail(); const text = chunkDocument(doc.markdown)[input.ordinal]; if (text === undefined || hash(JSON.stringify(['sponge-chunk-v2', input.representationId, input.ordinal, text])) !== input.derivationHash)
-    return fail(); return { ...input, text }; }); }
+    return atomic(db, () => {
+        const identity = JSON.stringify(Object.fromEntries(Object.keys(options.identity)
+            .sort()
+            .map((k) => [k, options.identity[k]])));
+        if (!identity || Buffer.byteLength(identity) > 16384)
+            fail();
+        const existing = db.prepare('SELECT * FROM embedding_spaces WHERE fingerprint=?').get(options.fingerprint);
+        if (existing &&
+            (existing.identity_json !== identity ||
+                Number(existing.dimensions) !== options.dimensions ||
+                existing.normalization !== 'unit-l2' ||
+                existing.metric !== 'cosine'))
+            fail();
+        if (!existing)
+            db.prepare('INSERT INTO embedding_spaces VALUES(?,?,?,?,?)').run(options.fingerprint, identity, options.dimensions, 'unit-l2', 'cosine');
+        const id = randomUUID();
+        db.prepare('INSERT INTO index_generations VALUES(?,?,?,?,?,?)').run(id, options.fingerprint, JSON.stringify({ profileId: options.profileId }), 'building', epoch(db), coverageMarker);
+        syncGeneration(db, id);
+        return generationStatus(db, id);
+    });
+}
+export function generationStatus(db, id) {
+    const { row, profile } = generation(db, id), fingerprint = String(row.fingerprint), expected = Number(db.prepare('SELECT count(*) n FROM index_inputs WHERE generation_id=?').get(id)?.n), pending = pendingCount(db, id, fingerprint);
+    return {
+        id,
+        fingerprint,
+        state: String(row.state),
+        revisionEpoch: Number(row.revision_epoch),
+        items: Number(db.prepare('SELECT count(*) n FROM index_items WHERE generation_id=?').get(id)?.n),
+        expected,
+        completed: expected - pending,
+        pending,
+        profileId: profile.profileId,
+    };
+}
+/** One input per content hash still to embed, oldest coverage first: identical chunks embed once. */
+export function pendingInputs(db, id, limit = 16) {
+    if (!Number.isSafeInteger(limit) || limit < 1 || limit > 256)
+        fail();
+    const { row } = generation(db, id);
+    if (row.state === 'cancelled')
+        fail('cancelled-index');
+    if (row.state !== 'building' && row.state !== 'active')
+        return [];
+    const chunks = new Map();
+    return db
+        .prepare(`SELECT c.source_id,c.revision_id,c.content_hash,r.item_id,min(c.rowid) AS first FROM index_inputs c JOIN revisions r ON r.id=c.revision_id WHERE c.generation_id=? AND ${missing} GROUP BY c.content_hash ORDER BY first LIMIT ?`)
+        .all(id, row.fingerprint, limit)
+        .map((r) => {
+        const input = sourceInput(r);
+        if (input.kind === 'image')
+            return input;
+        let parts = chunks.get(input.representationId);
+        if (!parts) {
+            const doc = db
+                .prepare('SELECT markdown FROM representations WHERE id=? AND revision_id=?')
+                .get(input.representationId, input.revisionId);
+            if (!doc || typeof doc.markdown !== 'string')
+                return fail();
+            parts = chunkDocument(doc.markdown);
+            chunks.set(input.representationId, parts);
+        }
+        const text = parts[input.ordinal];
+        if (text === undefined || documentHash(text) !== input.derivationHash)
+            return fail();
+        return { ...input, text };
+    });
+}
+/** Stores each result's vector under its content hash in the space. A source dropped by a refresh or delete since it was read is skipped. */
 export function writeBatch(db, id, fingerprint, batch) {
     if (!batch.length || batch.length > 256)
         fail();
-    atomic(db, () => { const { row, coverage, encoding } = generation(db, id); if (row.state === 'cancelled')
-        fail('cancelled-index'); if (row.state !== 'building' || row.fingerprint !== fingerprint)
-        fail(); const expected = new Map(coverage.inputs.map(i => [i.sourceId, i])); const seen = new Set(); for (const result of batch) {
-        const input = expected.get(result.sourceId);
-        if (!input)
-            return fail();
-        if (seen.has(result.sourceId) || input.revisionId !== result.revisionId || input.derivationHash !== result.derivationHash || !/^[a-f0-9]{64}$/.test(result.inputHash) || result.vector.length !== Number(row.dimensions) || result.vector.some(n => !Number.isFinite(n)) || Math.abs(Math.hypot(...result.vector) - 1) > .001)
+    atomic(db, () => {
+        const { row, encoding } = generation(db, id);
+        if (row.state === 'cancelled')
+            fail('cancelled-index');
+        if (!['building', 'active'].includes(String(row.state)) || row.fingerprint !== fingerprint)
             fail();
-        if (input.kind === 'document') {
-            const doc = db.prepare('SELECT markdown FROM representations WHERE id=? AND revision_id=?').get(input.representationId, input.revisionId);
-            if (!doc || typeof doc.markdown !== 'string')
-                return fail();
-            const text = chunkDocument(String(doc.markdown))[input.ordinal];
-            if (text === undefined || encodeEmbeddingRequest([{ kind: 'document', text }]).inputHashes[0] !== result.inputHash)
+        const seen = new Set();
+        for (const result of batch) {
+            if (seen.has(result.sourceId) ||
+                !/^[a-f0-9]{64}$/.test(result.inputHash) ||
+                result.vector.length !== Number(row.dimensions) ||
+                result.vector.some((n) => !Number.isFinite(n)) ||
+                Math.abs(Math.hypot(...result.vector) - 1) > 0.001)
                 fail();
-        }
-        seen.add(result.sourceId);
-        const bytes = encodeVector(result.vector, encoding);
-        const prior = db.prepare('SELECT vector,input_hash FROM vectors WHERE generation_id=? AND source_id=?').get(id, input.sourceId);
-        if (prior && prior.input_hash !== null) {
-            if (prior.input_hash !== result.inputHash || !bytes.equals(Buffer.from(prior.vector)))
+            seen.add(result.sourceId);
+            const input = db
+                .prepare('SELECT revision_id,content_hash FROM index_inputs WHERE generation_id=? AND source_id=?')
+                .get(id, result.sourceId);
+            if (!input)
+                continue;
+            if (input.revision_id !== result.revisionId ||
+                input.content_hash !== result.derivationHash ||
+                (parseSource(result.sourceId, result.revisionId).kind === 'document' && result.inputHash !== input.content_hash))
                 fail();
-            continue;
+            const bytes = encodeVector(result.vector, encoding), prior = db
+                .prepare('SELECT input_hash,vector FROM embeddings WHERE fingerprint=? AND content_hash=?')
+                .get(fingerprint, input.content_hash);
+            if (prior) {
+                if (prior.input_hash !== result.inputHash || !bytes.equals(Buffer.from(prior.vector)))
+                    fail();
+                continue;
+            }
+            db.prepare('INSERT INTO embeddings VALUES(?,?,?,?)').run(fingerprint, input.content_hash, result.inputHash, bytes);
         }
-        db.prepare('INSERT INTO vectors(generation_id,source_id,revision_id,asset_sha256,derivation_hash,vector,input_hash) VALUES(?,?,?,?,?,?,?) ON CONFLICT(generation_id,source_id) DO UPDATE SET vector=excluded.vector,input_hash=excluded.input_hash').run(id, input.sourceId, input.revisionId, input.assetSha256 ?? null, input.derivationHash, bytes, result.inputHash);
-    } });
+    });
 }
+/** Complete and current: no pending input, and coverage names exactly the current revision of every live item. */
 export function activateGeneration(db, id) {
-    return atomic(db, () => { const { row, coverage, profile } = generation(db, id); if (!['building', 'active', 'retired'].includes(String(row.state)))
-        fail('cancelled-index'); if (Number(row.revision_epoch) !== epoch(db) || JSON.stringify(coverage.revisions) !== JSON.stringify(revisions(db)))
-        fail('stale-index'); const actual = db.prepare('SELECT source_id,revision_id,derivation_hash,input_hash FROM vectors WHERE generation_id=?').all(id); const expected = new Map(coverage.inputs.map(i => [i.sourceId, i])); if (actual.length !== expected.size || actual.some(v => { const input = expected.get(String(v.source_id)); return !input || input.revisionId !== v.revision_id || input.derivationHash !== v.derivation_hash || typeof v.input_hash !== 'string' || !/^[a-f0-9]{64}$/.test(v.input_hash); }))
-        fail('incomplete-index'); const prior = db.prepare('SELECT generation_id FROM active_indexes WHERE profile_id=?').get(profile.profileId); if (prior && prior.generation_id !== id)
-        db.prepare("UPDATE index_generations SET state='retired' WHERE id=?").run(prior.generation_id); db.prepare('INSERT INTO active_indexes VALUES(?,?) ON CONFLICT(profile_id) DO UPDATE SET generation_id=excluded.generation_id').run(profile.profileId, id); db.prepare("UPDATE index_generations SET state='active' WHERE id=?").run(id); return generationStatus(db, id); });
-}
-export function cancelGeneration(db, id) { atomic(db, () => { const { row } = generation(db, id); if (row.state === 'active' || row.state === 'retired')
-    fail(); db.prepare("UPDATE index_generations SET state='cancelled' WHERE id=?").run(id); }); }
+    return atomic(db, () => {
+        const { row, profile } = generation(db, id);
+        if (!['building', 'active', 'retired'].includes(String(row.state)))
+            fail('cancelled-index');
+        if (Number(row.revision_epoch) !== epoch(db) || db.prepare(`SELECT 1 ${drift} LIMIT 1`).get(id))
+            fail('stale-index');
+        if (pendingCount(db, id, String(row.fingerprint)))
+            fail('incomplete-index');
+        const prior = db.prepare('SELECT generation_id FROM active_indexes WHERE profile_id=?').get(profile.profileId);
+        if (prior && prior.generation_id !== id)
+            db.prepare("UPDATE index_generations SET state='retired' WHERE id=?").run(prior.generation_id);
+        db.prepare('INSERT INTO active_indexes VALUES(?,?) ON CONFLICT(profile_id) DO UPDATE SET generation_id=excluded.generation_id').run(profile.profileId, id);
+        db.prepare("UPDATE index_generations SET state='active' WHERE id=?").run(id);
+        return generationStatus(db, id);
+    });
+}
+export function cancelGeneration(db, id) {
+    atomic(db, () => {
+        const { row } = generation(db, id);
+        if (row.state === 'active' || row.state === 'retired')
+            fail();
+        db.prepare("UPDATE index_generations SET state='cancelled' WHERE id=?").run(id);
+    });
+}
+/** The active generation of a profile, if any. */
+export function activeGeneration(db, profileId) {
+    const row = db.prepare('SELECT generation_id FROM active_indexes WHERE profile_id=?').get(profileId);
+    return row ? String(row.generation_id) : undefined;
+}
 /** Read-side join removes deleted/refreshed revisions even when an old generation stays active. */
-export function activeVectors(db, profileId) { const active = db.prepare('SELECT generation_id FROM active_indexes WHERE profile_id=?').get(profileId); if (!active)
-    return []; const id = String(active.generation_id), { row, coverage, encoding } = generation(db, id), inputs = new Map(coverage.inputs.map(i => [i.sourceId, i])); return db.prepare('SELECT v.* FROM vectors v JOIN revisions r ON r.id=v.revision_id JOIN items i ON i.id=r.item_id WHERE v.generation_id=? AND i.deleted_at IS NULL AND i.current_revision=v.revision_id ORDER BY v.source_id').all(id).map(v => { const source = inputs.get(String(v.source_id)); if (!source)
-    fail(); const vector = decodeVector(v.vector, Number(row.dimensions), encoding); return { fingerprint: String(row.fingerprint), generationId: id, source: source, vector }; }); }
-export const rollbackGeneration = activateGeneration;
+export function activeVectors(db, profileId) {
+    const id = activeGeneration(db, profileId);
+    if (!id)
+        return [];
+    const { row, encoding } = generation(db, id), fingerprint = String(row.fingerprint);
+    return db
+        .prepare('SELECT c.source_id,c.revision_id,c.content_hash,r.item_id,e.vector FROM index_inputs c JOIN embeddings e ON e.fingerprint=? AND e.content_hash=c.content_hash JOIN revisions r ON r.id=c.revision_id JOIN items i ON i.id=r.item_id WHERE c.generation_id=? AND i.deleted_at IS NULL AND i.current_revision=c.revision_id ORDER BY c.source_id')
+        .all(fingerprint, id)
+        .map((v) => ({
+        fingerprint,
+        generationId: id,
+        source: sourceInput(v),
+        vector: decodeVector(v.vector, Number(row.dimensions), encoding),
+    }));
+}