@hraness/sponge 0.2.6 → 0.3.0

package/dist/src/importers/manifest.js

@@ -1,43 +1,79 @@
 import { dirname, isAbsolute, join } from 'node:path';
 import { realpath, lstat } from 'node:fs/promises';
-import { readConfined, hash } from '../acquisition/files.js';
+import { readConfined } from '../acquisition/files.js';
+import { sha256 as hash } from '../util/hash.js';
 import { object } from '../acquisition/normalize.js';
-function bounded(value, max) { if (typeof value !== 'string' || !value || Buffer.byteLength(value) > max || value.includes('\0'))
-    throw Error('invalid-input'); return value; }
-function keys(value, allowed) { if (Object.keys(value).some(k => !allowed.includes(k)))
-    throw Error('invalid-input'); }
+import { SpongeError } from '../errors.js';
+function bounded(value, max) {
+    if (typeof value !== 'string' || !value || Buffer.byteLength(value) > max || value.includes('\0'))
+        throw new SpongeError('invalid-input');
+    return value;
+}
+function keys(value, allowed) {
+    if (Object.keys(value).some((k) => !allowed.includes(k)))
+        throw new SpongeError('invalid-input');
+}
+function parseJson(bytes) {
+    try {
+        return JSON.parse(bytes.toString('utf8'));
+    }
+    catch (error) {
+        throw new SpongeError('invalid-input', { cause: error });
+    }
+}
 /** Explicit local source manifest; no recursive discovery or implicit public selection. */
 export async function readImportManifest(path) {
-    if (!isAbsolute(path) || await realpath(path) !== path)
-        throw Error('invalid-input');
-    const root = dirname(path), bytes = await readConfined(root, path.slice(root.length + 1), 1024 ** 2), raw = object(JSON.parse(bytes.toString('utf8')));
+    if (!isAbsolute(path) || (await realpath(path)) !== path)
+        throw new SpongeError('invalid-input');
+    const root = dirname(path), bytes = await readConfined(root, path.slice(root.length + 1), 1024 ** 2), raw = object(parseJson(bytes));
     keys(raw, ['schemaVersion', 'namespace', 'entries']);
     if (raw.schemaVersion !== 1 || !Array.isArray(raw.entries) || raw.entries.length > 1000)
-        throw Error('unsupported');
+        throw new SpongeError('unsupported');
     const namespace = bounded(raw.namespace, 128);
     if (!/^[A-Za-z0-9_.-]+$/.test(namespace))
-        throw Error('invalid-input');
+        throw new SpongeError('invalid-input');
     const ids = new Set();
-    const entries = raw.entries.map(value => {
+    const entries = raw.entries.map((value) => {
         const entry = object(value);
         keys(entry, ['id', 'bundle', 'privacy', 'notes', 'readingState']);
         const id = bounded(entry.id, 256), bundle = bounded(entry.bundle, 4096);
-        if (ids.has(id) || isAbsolute(bundle) || bundle.includes('\\') || bundle.split('/').some(p => !p || p === '.' || p === '..') || !['private', 'published'].includes(String(entry.privacy)))
-            throw Error('invalid-input');
+        if (ids.has(id) ||
+            isAbsolute(bundle) ||
+            bundle.includes('\\') ||
+            bundle.split('/').some((p) => !p || p === '.' || p === '..') ||
+            !['private', 'published'].includes(String(entry.privacy)))
+            throw new SpongeError('invalid-input');
         ids.add(id);
         if (entry.notes !== undefined && !Array.isArray(entry.notes))
-            throw Error('invalid-input');
+            throw new SpongeError('invalid-input');
         const noteIds = new Set();
-        const notes = (entry.notes ?? []).map(value => { const note = object(value); keys(note, ['id', 'body']); const id = bounded(note.id, 256), body = bounded(note.body, 65536); if (noteIds.has(id))
-            throw Error('invalid-input'); noteIds.add(id); return { id, body }; });
+        const notes = (entry.notes ?? []).map((value) => {
+            const note = object(value);
+            keys(note, ['id', 'body']);
+            const id = bounded(note.id, 256), body = bounded(note.body, 65536);
+            if (noteIds.has(id))
+                throw new SpongeError('invalid-input');
+            noteIds.add(id);
+            return { id, body };
+        });
         if (notes.length > 100)
-            throw Error('limit-exceeded');
+            throw new SpongeError('limit-exceeded');
         const readingState = entry.readingState === undefined ? undefined : object(entry.readingState);
         if (readingState && Buffer.byteLength(JSON.stringify(readingState)) > 65536)
-            throw Error('limit-exceeded');
-        return { id, bundle, privacy: entry.privacy, notes, ...(readingState ? { readingState } : {}) };
+            throw new SpongeError('limit-exceeded');
+        return {
+            id,
+            bundle,
+            privacy: entry.privacy,
+            notes,
+            ...(readingState ? { readingState } : {}),
+        };
     });
     return { root, manifest: { schemaVersion: 1, namespace, entries }, sha256: hash(bytes) };
 }
-export async function importBundleRoot(root, entry) { const path = join(root, entry.bundle); if (await realpath(path) !== path || (await lstat(path)).isSymbolicLink() || !(await lstat(path)).isDirectory())
-    throw Error('integrity'); return path; }
+export async function importBundleRoot(root, entry) {
+    const path = join(root, entry.bundle);
+    if ((await realpath(path)) !== path || (await lstat(path)).isSymbolicLink() || !(await lstat(path)).isDirectory())
+        throw new SpongeError('integrity');
+    return path;
+}

package/dist/src/importers/verify-helper.js

@@ -1,27 +1,41 @@
+import { SpongeError } from '../errors.js';
 /** Only released Wordcell public verifiers, with no source or document text in output. */
 const runtime = process.argv[2], bundle = process.argv[3], kind = process.argv[4];
 try {
     let result;
     if (kind === 'pdf') {
         const { verifyPdfCaptureBundle } = await import(import.meta.resolve('@hraness/wordcell/pdf', `${runtime}/package.json`));
-        result = await verifyPdfCaptureBundle(bundle, { maxDocumentBytes: 16 * 1024 ** 2, maxSourceBytes: 100 * 1024 ** 2, maxVerifiedAssets: 200, maxAssetBytes: 20 * 1024 ** 2, maxTotalAssetBytes: 500 * 1024 ** 2, maxVerificationMs: 30000 });
+        result = await verifyPdfCaptureBundle(bundle, {
+            maxDocumentBytes: 16 * 1024 ** 2,
+            maxSourceBytes: 100 * 1024 ** 2,
+            maxVerifiedAssets: 200,
+            maxAssetBytes: 20 * 1024 ** 2,
+            maxTotalAssetBytes: 500 * 1024 ** 2,
+            maxVerificationMs: 30000,
+        });
         const { markdown, ...document } = result.inspection.markdown;
         result = { ...result, inspection: { ...result.inspection, markdown: document } };
     }
     else {
         const { verifyCaptureBundle } = await import(import.meta.resolve('@hraness/wordcell/clip/bundle-reader', `${runtime}/package.json`));
-        result = await verifyCaptureBundle(bundle, { verifyAssets: true, maxDocumentBytes: 16 * 1024 ** 2, maxVerifiedAssets: 200, maxAssetBytes: 20 * 1024 ** 2, maxTotalAssetBytes: 500 * 1024 ** 2, maxAssetVerificationMs: 30000 });
+        result = await verifyCaptureBundle(bundle, {
+            verifyAssets: true,
+            maxDocumentBytes: 16 * 1024 ** 2,
+            maxVerifiedAssets: 200,
+            maxAssetBytes: 20 * 1024 ** 2,
+            maxTotalAssetBytes: 500 * 1024 ** 2,
+            maxAssetVerificationMs: 30000,
+        });
         const { markdown, ...document } = result.inspection.document;
         const { metadata, ...inspection } = result.inspection;
         result = { ...result, inspection: { ...inspection, document } };
     }
     const output = JSON.stringify(result);
     if (Buffer.byteLength(output) > 65536)
-        throw Error('limit');
+        throw new SpongeError('limit-exceeded');
     process.stdout.write(output);
 }
 catch {
     process.stdout.write(JSON.stringify({ ok: false, code: 'unsupported' }));
     process.exitCode = 1;
 }
-export {};

package/dist/src/indexing/image.js

@@ -1,27 +1,46 @@
 import { mkdtemp, writeFile, rm } from 'node:fs/promises';
 import { join } from 'node:path';
-import { privateRoot, readConfined, hash } from '../acquisition/files.js';
+import { privateRoot, readConfined } from '../acquisition/files.js';
+import { sha256 as hash } from '../util/hash.js';
 import { runOwned } from '../acquisition/process.js';
-import { encodeEmbeddingRequest, EmbeddingError } from '../inference/embedding/index.js';
+import { encodeEmbeddingRequest } from '../inference/embedding/index.js';
+import { SpongeError } from '../errors.js';
 /** Derived pixels are disposable; source image blobs remain unchanged. */
 export async function deriveEmbeddingImage(bytes, options) {
     if (process.platform !== 'darwin' || process.arch !== 'arm64')
-        throw new EmbeddingError('unsupported');
+        throw new SpongeError('unsupported');
     if (!bytes.length || bytes.length > 20 * 1024 ** 2)
-        throw new EmbeddingError('limit-exceeded');
+        throw new SpongeError('limit-exceeded');
     const root = await privateRoot(options.scratchRoot), scratch = await mkdtemp(join(root, 'image-derive-'));
     let settled = true;
     try {
         const input = join(scratch, 'source'), output = join(scratch, 'image.png');
         await writeFile(input, bytes, { mode: 0o600, flag: 'wx' });
-        const run = async (args) => { settled = false; const result = await (options.runProcess ?? runOwned)('/usr/bin/sandbox-exec', ['-p', '(version 1) (allow default) (deny network*)', '/usr/bin/sips', ...args], { env: { PATH: '/usr/bin:/bin', HOME: scratch, TMPDIR: scratch, LANG: 'C' }, deadlineMs: 30000, maxOutputBytes: 65536, signal: options.signal }); settled = result.cleanup === 'settled'; if (!settled)
-            throw new EmbeddingError('cleanup-required'); if (result.stopped)
-            throw new EmbeddingError(result.stopped); if (result.code !== 0)
-            throw new EmbeddingError('unsupported'); return result.stdout; };
+        const run = async (args) => {
+            settled = false;
+            const result = await (options.runProcess ?? runOwned)('/usr/bin/sandbox-exec', ['-p', '(version 1) (allow default) (deny network*)', '/usr/bin/sips', ...args], {
+                env: { PATH: '/usr/bin:/bin', HOME: scratch, TMPDIR: scratch, LANG: 'C' },
+                deadlineMs: 30000,
+                maxOutputBytes: 65536,
+                signal: options.signal,
+            });
+            settled = result.cleanup === 'settled';
+            if (!settled)
+                throw new SpongeError('cleanup-required');
+            if (result.stopped)
+                throw new SpongeError(result.stopped);
+            if (result.code !== 0)
+                throw new SpongeError('unsupported');
+            return result.stdout;
+        };
         const metadata = await run(['-g', 'pixelWidth', '-g', 'pixelHeight', input]);
         const width = Number(metadata.match(/pixelWidth:\s*(\d+)/)?.[1]), height = Number(metadata.match(/pixelHeight:\s*(\d+)/)?.[1]);
-        if (!Number.isSafeInteger(width) || !Number.isSafeInteger(height) || width < 1 || height < 1 || width * height > 40_000_000)
-            throw new EmbeddingError('limit-exceeded');
+        if (!Number.isSafeInteger(width) ||
+            !Number.isSafeInteger(height) ||
+            width < 1 ||
+            height < 1 ||
+            width * height > 40_000_000)
+            throw new SpongeError('limit-exceeded');
         await run(['--resampleHeightWidthMax', '512', '--setProperty', 'format', 'png', input, '--out', output]);
         const png = await readConfined(scratch, 'image.png', 2 * 1024 ** 2);
         encodeEmbeddingRequest([{ kind: 'image', png }]);

package/dist/src/indexing/index.d.ts

@@ -1,4 +1,5 @@
 import type { DatabaseSync } from 'node:sqlite';
+/** One embeddable source of a revision. `derivationHash` is its content key in the space: a document chunk's embedding input hash, or an image's source hash. */
 export interface IndexInput {
     sourceId: string;
     revisionId: string;
@@ -15,14 +16,14 @@ export interface GenerationStatus {
     fingerprint: string;
     state: string;
     revisionEpoch: number;
+    items: number;
     expected: number;
     completed: number;
+    pending: number;
     profileId: string;
 }
-export declare class IndexError extends Error {
-    code: 'invalid-index' | 'stale-index' | 'incomplete-index' | 'cancelled-index';
-    constructor(code: 'invalid-index' | 'stale-index' | 'incomplete-index' | 'cancelled-index');
-}
+/** Schema 10 keeps coverage in `index_items` and `index_inputs`; a generation whose JSON coverage could not be converted keeps it and never reads. */
+export declare const coverageMarker = "{\"version\":2}";
 export type VectorEncoding = 'f32' | 'int8';
 /** Storage format follows the space identity: no `quantization` (legacy) or `f32` stores float32 LE; `int8-symmetric-maxabs-v1` stores one int8 code per dimension. */
 export declare function spaceEncoding(identity: Record<string, unknown>): VectorEncoding;
@@ -30,9 +31,34 @@ export declare function spaceEncoding(identity: Record<string, unknown>): Vector
 export declare function encodeVector(vector: readonly number[], encoding: VectorEncoding): Buffer;
 /** Exact byte length, finite floats, int8 codes within [-127,127] and nonzero; int8 returns the unit dequantized vector. */
 export declare function decodeVector(bytes: Uint8Array, dimensions: number, encoding: VectorEncoding): number[];
+/** What one document chunk's vector was derived from before schema 10: its representation, position and text. Migrations 9 and 10 still read it. */
+export declare const chunkHash: (representationId: string, ordinal: number, text: string) => string;
 /** Unicode-safe chunks fit the fixed document prompt within the embedding text budget, preferring whitespace breaks. */
 export declare function chunkDocument(text: string): string[];
-/** Identity must already be the verified embedder fingerprint. Existing identities are immutable. */
+/** The helper's input hash of a document chunk: `encodeEmbeddingRequest([{kind:'document',text}]).inputHashes[0]` without its validation, so a commit never fails on text the embedder refuses. */
+export declare function documentHash(text: string): string;
+export declare const imageHash: (assetSha256: string) => string;
+type Source = {
+    kind: 'document';
+    representationId: string;
+    ordinal: number;
+} | {
+    kind: 'image';
+    assetSha256: string;
+};
+/** A source id is `[revisionId,'document',representationId,ordinal]` or `[revisionId,'image',assetSha256]`. */
+export declare function parseSource(sourceId: string, revisionId: string): Source;
+export declare const sourceInput: (row: Record<string, unknown>) => IndexInput;
+/** Brings a building or active generation's coverage to the current revisions: inputs of refreshed or deleted items go, the
+ * new revision's come. Embeddings stay; an input is pending while its space has none for its content. Returns items changed. */
+export declare function syncGeneration(db: DatabaseSync, id: string): number;
+/** Queues a generation's index job again (or its first) unless it is queued, running or awaiting cleanup. */
+export declare function armJob(db: DatabaseSync, id: string): void;
+/** Inside the transaction that changed a current revision or deleted an item: every active generation, and every building one
+ * whose job is live, follows the library. An active generation with content to embed gets its job queued: only the delta runs. */
+export declare function trackLibrary(db: DatabaseSync): void;
+/** Identity must already be the verified embedder fingerprint. Existing identities are immutable. Every input whose content the
+ * space already embedded (an earlier generation, another item) is complete at once. */
 export declare function createGeneration(db: DatabaseSync, options: {
     profileId: string;
     fingerprint: string;
@@ -40,7 +66,9 @@ export declare function createGeneration(db: DatabaseSync, options: {
     dimensions: number;
 }): GenerationStatus;
 export declare function generationStatus(db: DatabaseSync, id: string): GenerationStatus;
+/** One input per content hash still to embed, oldest coverage first: identical chunks embed once. */
 export declare function pendingInputs(db: DatabaseSync, id: string, limit?: number): IndexInput[];
+/** Stores each result's vector under its content hash in the space. A source dropped by a refresh or delete since it was read is skipped. */
 export declare function writeBatch(db: DatabaseSync, id: string, fingerprint: string, batch: readonly {
     sourceId: string;
     revisionId: string;
@@ -48,8 +76,11 @@ export declare function writeBatch(db: DatabaseSync, id: string, fingerprint: st
     inputHash: string;
     vector: readonly number[];
 }[]): void;
+/** Complete and current: no pending input, and coverage names exactly the current revision of every live item. */
 export declare function activateGeneration(db: DatabaseSync, id: string): GenerationStatus;
 export declare function cancelGeneration(db: DatabaseSync, id: string): void;
+/** The active generation of a profile, if any. */
+export declare function activeGeneration(db: DatabaseSync, profileId: string): string | undefined;
 /** Read-side join removes deleted/refreshed revisions even when an old generation stays active. */
 export declare function activeVectors(db: DatabaseSync, profileId: string): {
     fingerprint: string;
@@ -57,4 +88,4 @@ export declare function activeVectors(db: DatabaseSync, profileId: string): {
     source: IndexInput;
     vector: number[];
 }[];
-export declare const rollbackGeneration: typeof activateGeneration;
+export {};