@hraness/sponge 0.1.0 → 0.2.0

package/dist/src/cli/worker.js

@@ -0,0 +1,62 @@
+import { join } from 'node:path';
+import { inspectNative } from '../models/runtime.js';
+import { drainIndexQueue } from '../indexing/runner.js';
+import { drainLibrary } from '../worker/basic.js';
+import { recoverServiceLifetime } from '../service/lifetime.js';
+import { StoreError } from '../store/types.js';
+import { runWorkerService } from '../service/loop.js';
+import { installService, uninstallService, serviceStatus, installedServiceId, serviceEnvironment } from '../service/launch-agent.js';
+export async function workerAction(store, action, version, environment, options = {}) {
+    const env = environment ?? serviceEnvironment(version);
+    if (action === 'install')
+        return installService(store, env, options);
+    if (action === 'uninstall')
+        return uninstallService(store, env);
+    return serviceStatus(store, env);
+}
+/** Dispatch only work that claimNext can admit; delayed retries retain their due time. */
+export function hasDueAcquisition(store) {
+    const now = new Date().toISOString();
+    return store.listJobs().some(job => job.operation !== 'index' && ['queued', 'delayed'].includes(String(job.state)) && job.cancel_requested !== 1 && (job.not_before === null || job.not_before === undefined || String(job.not_before) <= now));
+}
+export async function drainServiceQueues(store, options, version, signal) {
+    if (store.pendingCleanupCount())
+        throw Object.assign(Error('cleanup-required'), { code: 'cleanup-required' });
+    const results = [];
+    let acquisitionError;
+    if (hasDueAcquisition(store)) {
+        try {
+            const runtime = options.runtime ?? await options.resolveCaptureRuntime?.();
+            if (!runtime)
+                throw Object.assign(Error('runtime-missing'), { code: 'runtime-missing' });
+            let asrRuntime = options.asrRuntime;
+            try {
+                asrRuntime ??= await inspectNative('asr', join(store.root, 'runtime', 'models'), version);
+            }
+            catch (error) {
+                if (!['runtime-missing', 'unsupported'].includes(String(error.code)))
+                    throw error;
+            }
+            results.push(...await drainLibrary(store, { ...options, runtime, asrRuntime, signal, allowPrivateNetwork: false }));
+        }
+        catch (error) {
+            acquisitionError = error;
+        }
+    }
+    // A missing capture setup must not starve already queued, independently runnable indexes.
+    if (!signal.aborted && store.pendingCleanupCount() === 0 && store.listJobs().some(job => job.operation === 'index' && job.state === 'queued')) {
+        const runtime = await inspectNative('embedding', join(store.root, 'runtime', 'models'), version);
+        results.push(...await drainIndexQueue(store, { runtime, signal }));
+    }
+    if (acquisitionError)
+        throw acquisitionError;
+    return results;
+}
+export async function serveWorker(store, options, version, environment) {
+    const env = environment ?? serviceEnvironment(version), installationId = installedServiceId(store, env);
+    if (!options.signal)
+        throw Error('signal-required');
+    if (!await recoverServiceLifetime(store, installationId, env.alive))
+        throw new StoreError('owner-busy');
+    return runWorkerService(store, { installationId, signal: options.signal, drain: signal => drainServiceQueues(store, options, version, signal) });
+}

package/dist/src/importers/apply.d.ts

@@ -0,0 +1,29 @@
+import type { LibraryStore } from '../store/index.js';
+import { type BundleVerifier } from './inspect.js';
+import type { CaptureRuntime } from '../acquisition/runtime.js';
+export interface ImportOptions {
+    runtime: CaptureRuntime;
+    dryRun?: boolean;
+    limit?: number;
+    offset?: number;
+    signal?: AbortSignal;
+    verify?: BundleVerifier;
+}
+export interface ImportReport {
+    schemaVersion: 1;
+    manifestSha256: string;
+    namespace: string;
+    dryRun: boolean;
+    entries: {
+        id: string;
+        status: 'ready' | 'imported' | 'duplicate' | 'unsupported' | 'failed';
+        sourceHash?: string;
+        historicalIntegrity?: 'verified' | 'unavailable';
+        itemId?: string;
+        code?: string;
+        privacy: 'private' | 'published';
+    }[];
+    nextOffset: number | null;
+}
+/** Additive bounded batches. Every applied entry is reverified; source bundles are read-only. */
+export declare function importManifest(store: LibraryStore, path: string, options: ImportOptions): Promise<ImportReport>;

package/dist/src/importers/apply.js

@@ -0,0 +1,85 @@
+import { hash } from '../acquisition/files.js';
+import { failure } from '../acquisition/errors.js';
+import { bundleBytes, bundleText } from '../app/bundle-files.js';
+import { localizeMarkdown } from '../app/markdown.js';
+import { selectedMetadata } from '../metadata/article.js';
+import { drainQueue } from '../worker/engine.js';
+import { readImportManifest, importBundleRoot } from './manifest.js';
+import { inspectImportBundle, releasedBundleVerifier } from './inspect.js';
+const errorCode = (error) => { const code = error?.code ?? (error instanceof Error ? error.message : ''); return ['unsupported', 'invalid-input', 'integrity', 'limit-exceeded', 'cancelled', 'cleanup-required', 'runtime-missing', 'owner-busy', 'unavailable'].includes(code) ? code : 'unclassified'; };
+async function promote(context, bundle, namespace, entry, sourceHash) {
+    const { store, attempt, signal, checkpoint } = context;
+    let committed = false;
+    let result = failure(attempt.attemptId, 'unclassified');
+    try {
+        const source = store.sourceForJob(attempt.jobId);
+        if (!source)
+            throw Error('integrity');
+        await checkpoint('extracting');
+        const markdown = await bundleText(bundle.root, bundle.markdown), localized = localizeMarkdown(bundle.kind === 'pdf' ? markdown.replace(/^<!-- pdf-page: ([1-9][0-9]*) -->$/gm, '## Page $1') : markdown, bundle.assets), assets = [];
+        for (const asset of bundle.assets) {
+            if (signal.aborted)
+                throw Error('cancelled');
+            const stored = await store.putBlob(attempt, asset.sha256, asset.bytes, bundleBytes(bundle.root, asset));
+            assets.push({ ...stored, mediaType: asset.mediaType, relationship: asset.relationship });
+        }
+        if (signal.aborted)
+            throw Error('cancelled');
+        await checkpoint('promoting');
+        const title = selectedMetadata(bundle.metadata, 'title')?.value;
+        store.commitRevision({ ...attempt, itemId: source.itemId, bundleHash: hash(JSON.stringify(['wordcell-import-v1', sourceHash])), markdown: localized.markdown, title: typeof title === 'string' ? title : undefined, extractor: { name: 'sponge-wordcell-import', version: '1' }, status: bundle.status === 'partial' || localized.missingImages ? 'partial' : 'complete', metadata: [...bundle.metadata, ...Object.entries({ importNamespace: namespace, externalSourceId: entry.id, sourcePrivacy: entry.privacy }).map(([field, value]) => ({ field, value, source: bundle.kind === 'pdf' ? 'pdf' : 'html', observedAt: new Date().toISOString(), rule: 'sponge-import-v1' }))], assets, warnings: bundle.status === 'partial' || localized.missingImages ? ['partial'] : [], importedState: { namespace, sourceKey: entry.id, sourceHash, importerVersion: '1', notes: entry.notes, readingState: entry.readingState, privacy: entry.privacy } });
+        committed = true;
+        result = { ok: true, itemId: source.itemId };
+    }
+    catch (error) {
+        result = failure(attempt.attemptId, errorCode(error), 'storage');
+    }
+    if (!committed)
+        store.failAttempt(attempt, result.ok ? 'unclassified' : result.code);
+    store.settleAttempt(attempt);
+    return { ...result, cleanup: 'settled' };
+}
+/** Additive bounded batches. Every applied entry is reverified; source bundles are read-only. */
+export async function importManifest(store, path, options) {
+    const { root, manifest, sha256 } = await readImportManifest(path), limit = options.limit ?? 20, offset = options.offset ?? 0;
+    if (!Number.isSafeInteger(limit) || limit < 1 || limit > 100 || !Number.isSafeInteger(offset) || offset < 0 || offset > manifest.entries.length)
+        throw Error('invalid-input');
+    const dryRun = options.dryRun !== false, report = { schemaVersion: 1, manifestSha256: sha256, namespace: manifest.namespace, dryRun, entries: [], nextOffset: offset + limit < manifest.entries.length ? offset + limit : null };
+    const verify = options.verify ?? releasedBundleVerifier(options.runtime, store.root, options.signal);
+    for (const entry of manifest.entries.slice(offset, offset + limit)) {
+        if (options.signal?.aborted)
+            throw Object.assign(Error('cancelled'), { code: 'cancelled' });
+        try {
+            const bundle = await inspectImportBundle(await importBundleRoot(root, entry), verify), sourceHash = hash(JSON.stringify([bundle.importHash, entry])), receipt = store.lookupImportReceipt(manifest.namespace, entry.id, sourceHash);
+            if (options.signal?.aborted)
+                throw Object.assign(Error('cancelled'), { code: 'cancelled' });
+            const base = { id: entry.id, sourceHash, historicalIntegrity: bundle.historicalIntegrity, privacy: entry.privacy };
+            if (receipt) {
+                report.entries.push({ ...base, status: 'duplicate', itemId: receipt.itemId });
+                continue;
+            }
+            if (dryRun) {
+                report.entries.push({ ...base, status: 'ready' });
+                continue;
+            }
+            const submission = store.submitSource({ kind: bundle.kind, sourceKey: hash(JSON.stringify([manifest.namespace, entry.id])), locator: bundle.sourceUrl, realm: 'import:' + manifest.namespace, idempotencyKey: 'import:' + hash(JSON.stringify([manifest.namespace, entry.id, sourceHash])), intent: 'refresh' });
+            const job = store.listJobs().find(row => row.id === submission.jobId);
+            if (job && ['failed', 'needs-auth', 'needs-permission', 'unsupported'].includes(String(job.state)))
+                store.retryJob(submission.jobId, { automatic: false });
+            const results = await drainQueue(store, { jobId: submission.jobId, signal: options.signal, handlers: { [bundle.kind]: context => promote(context, bundle, manifest.namespace, entry, sourceHash) } });
+            const result = results[0];
+            if (!result || !result.ok)
+                throw Object.assign(Error(result && !result.ok ? result.code : 'integrity'), { code: result && !result.ok ? result.code : 'integrity' });
+            report.entries.push({ ...base, status: 'imported', itemId: submission.itemId });
+        }
+        catch (error) {
+            const code = errorCode(error);
+            report.entries.push({ id: entry.id, status: code === 'unsupported' ? 'unsupported' : 'failed', code, privacy: entry.privacy });
+            if (code === 'cleanup-required' || code === 'owner-busy' || code === 'cancelled') {
+                report.nextOffset = offset + report.entries.length - 1;
+                break;
+            }
+        }
+    }
+    return report;
+}

package/dist/src/importers/custody.d.ts

@@ -0,0 +1,6 @@
+import { supervisedRunner } from '../worker/supervisor.js';
+/** Read-only verification still owns subprocesses, including during an import preview. */
+export declare function withImportVerification<T>(libraryRoot: string, verify: (context: {
+    scratchRoot: string;
+    runProcess: ReturnType<typeof supervisedRunner>;
+}) => Promise<T>): Promise<T>;

package/dist/src/importers/custody.js

@@ -0,0 +1,35 @@
+import { randomUUID } from 'node:crypto';
+import { join } from 'node:path';
+import { rm } from 'node:fs/promises';
+import { acquireWorkerCustody, proveAttemptCleanup } from '../worker/custody.js';
+import { directory, writeJournal, readJournal, syncDir } from '../worker/journal.js';
+import { supervisedRunner } from '../worker/supervisor.js';
+const pending = () => Object.assign(Error('cleanup-required'), { code: 'cleanup-required' });
+/** Read-only verification still owns subprocesses, including during an import preview. */
+export async function withImportVerification(libraryRoot, verify) {
+    const custody = await acquireWorkerCustody(libraryRoot), attemptId = randomUUID();
+    let scratchRoot, unknown = false;
+    try {
+        const attempt = await directory(join(custody.directory, 'attempts', attemptId));
+        await writeJournal(join(attempt, 'import-verification.json'), { schemaVersion: 1, kind: 'import-verification', attemptId, ownerIncarnation: custody.incarnation });
+        scratchRoot = await directory(join(libraryRoot, 'staging', 'import-verification-' + attemptId));
+        await writeJournal(join(scratchRoot, 'ownership.json'), { schemaVersion: 1, kind: 'import-verification', requestId: attemptId, ownerIncarnation: custody.incarnation });
+        return await verify({ scratchRoot, runProcess: supervisedRunner(custody.directory, custody.incarnation, attemptId) });
+    }
+    catch (error) {
+        unknown = error?.code === 'cleanup-required' || (error instanceof Error && error.message === 'cleanup-required');
+        throw error;
+    }
+    finally {
+        if (unknown || !await proveAttemptCleanup(custody.directory, custody.incarnation, attemptId))
+            throw pending();
+        if (scratchRoot) {
+            const marker = await readJournal(join(scratchRoot, 'ownership.json'));
+            if (marker.schemaVersion !== 1 || marker.kind !== 'import-verification' || marker.requestId !== attemptId || marker.ownerIncarnation !== custody.incarnation)
+                throw pending();
+            await rm(scratchRoot, { recursive: true });
+            await syncDir(join(libraryRoot, 'staging'));
+        }
+        await custody.release();
+    }
+}

package/dist/src/importers/inspect.d.ts

@@ -0,0 +1,21 @@
+import type { Asset, MetadataClaim } from '../../contracts/index.js';
+import { type CaptureRuntime } from '../acquisition/runtime.js';
+export interface ImportedBundle {
+    root: string;
+    kind: 'article' | 'pdf';
+    sourceUrl: string;
+    schemaVersion: number;
+    status: 'complete' | 'partial';
+    markdown: Asset;
+    assets: (Asset & {
+        relationship: 'image' | 'source-pdf';
+    })[];
+    metadata: MetadataClaim[];
+    manifestSha256: string;
+    historicalIntegrity: 'verified' | 'unavailable';
+    importHash: string;
+}
+export type BundleVerifier = (root: string, kind: 'article' | 'pdf') => Promise<unknown>;
+export declare function releasedBundleVerifier(runtime: CaptureRuntime, libraryRoot: string, signal?: AbortSignal): BundleVerifier;
+/** An import-time digest never upgrades a missing historical document digest. */
+export declare function inspectImportBundle(root: string, verify: BundleVerifier): Promise<ImportedBundle>;

package/dist/src/importers/inspect.js

@@ -0,0 +1,97 @@
+import { fileURLToPath, pathToFileURL } from 'node:url';
+import { join } from 'node:path';
+import { limits } from '../../contracts/index.js';
+import { validateAsset } from '../../contracts/validate.js';
+import { object, metadataFromMarkdown } from '../acquisition/normalize.js';
+import { readConfined, hash } from '../acquisition/files.js';
+import { withImportVerification } from './custody.js';
+import { inspectCaptureRuntime } from '../acquisition/runtime.js';
+import { articleMetadata } from '../metadata/article.js';
+import { bundleBytes } from '../app/bundle-files.js';
+export function releasedBundleVerifier(runtime, libraryRoot, signal) {
+    return async (root, kind) => {
+        const checked = await inspectCaptureRuntime(runtime.root, runtime.pins), helper = fileURLToPath(new URL(import.meta.url.endsWith('.ts') ? './verify-helper.ts' : './verify-helper.js', import.meta.url));
+        return withImportVerification(libraryRoot, async ({ scratchRoot, runProcess }) => {
+            const args = [helper, checked.root, root, kind];
+            const result = await runProcess(process.platform === 'darwin' ? '/usr/bin/sandbox-exec' : checked.bun, process.platform === 'darwin' ? ['-p', '(version 1) (allow default) (deny network*)', checked.bun, ...args] : args, { env: { PATH: '/usr/bin:/bin', HOME: checked.root, TMPDIR: scratchRoot, LANG: 'en_US.UTF-8' }, deadlineMs: 45000, maxOutputBytes: 65536, signal });
+            if (result.cleanup !== 'settled')
+                throw Error('cleanup-required');
+            if (result.stopped)
+                throw Error(result.stopped);
+            if (result.code !== 0)
+                throw Error('unsupported');
+            return JSON.parse(result.stdout);
+        });
+    };
+}
+function imageType(bytes) { if (bytes[0] === 255 && bytes[1] === 216)
+    return 'image/jpeg'; if (bytes.subarray(0, 8).equals(Buffer.from([137, 80, 78, 71, 13, 10, 26, 10])))
+    return 'image/png'; if (bytes.toString('ascii', 0, 4) === 'RIFF' && bytes.toString('ascii', 8, 12) === 'WEBP')
+    return 'image/webp'; if (/^GIF8[79]a/.test(bytes.toString('ascii', 0, 6)))
+    return 'image/gif'; if (bytes.toString('ascii', 4, 12) === 'ftypavif')
+    return 'image/avif'; throw Error('unsupported'); }
+/** An import-time digest never upgrades a missing historical document digest. */
+export async function inspectImportBundle(root, verify) {
+    const manifestBytes = await readConfined(root, 'capture.json', limits.metadataBytes), manifest = object(JSON.parse(manifestBytes.toString('utf8'))), kind = manifest.kind === 'pdf' ? 'pdf' : 'article';
+    if (kind === 'pdf' ? manifest.schemaVersion !== 1 : ![1, 2, 3, 4].includes(Number(manifest.schemaVersion)))
+        throw Error('unsupported');
+    const result = object(await verify(root, kind)), inspection = object(result.inspection), document = object(kind === 'pdf' ? inspection.markdown : inspection.document);
+    if (inspection.schemaVersion !== manifest.schemaVersion || !['verified', 'unavailable'].includes(String(document.integrity)) || !Array.isArray(inspection.assets) || inspection.assets.length > limits.imagesPerItem)
+        throw Error('integrity');
+    if (result.ok !== true && (!Array.isArray(result.issues) || !result.issues.length || result.issues.some(raw => { const issue = object(raw); return !(issue.kind === 'document-integrity' && issue.path === document.path && document.integrity === 'unavailable' && (kind === 'article' || issue.integrity === 'unavailable')); })))
+        throw Error('integrity');
+    if (inspection.status !== undefined && inspection.status !== null && !['complete', 'partial'].includes(String(inspection.status)))
+        throw Error('unsupported');
+    const markdownBytes = await readConfined(root, String(document.path), limits.markdownBytes), markdown = { relativePath: String(document.path), bytes: markdownBytes.length, sha256: hash(markdownBytes), mediaType: 'text/markdown' };
+    validateAsset(markdown);
+    if (markdown.sha256 !== document.sha256 || markdown.bytes !== document.bytes)
+        throw Error('integrity');
+    let unavailable = document.integrity === 'unavailable';
+    const assets = [];
+    let total = 0;
+    for (const raw of inspection.assets) {
+        const asset = object(raw);
+        if (!['verified', 'unavailable'].includes(String(asset.integrity)))
+            throw Error('integrity');
+        unavailable ||= asset.integrity === 'unavailable';
+        const bytes = await readConfined(root, String(asset.path), limits.imageBytes);
+        total += bytes.length;
+        if (total > 500 * 1024 ** 2)
+            throw Error('limit-exceeded');
+        const value = { relativePath: String(asset.path), bytes: bytes.length, sha256: hash(bytes), mediaType: imageType(bytes), relationship: 'image' };
+        validateAsset(value);
+        const expectedHash = kind === 'pdf' ? asset.sha256 : asset.expectedSha256, expectedBytes = kind === 'pdf' ? asset.bytes : asset.expectedBytes;
+        if (asset.integrity === 'verified' && (expectedHash !== value.sha256 || expectedBytes !== value.bytes))
+            throw Error('integrity');
+        assets.push(value);
+    }
+    if (kind === 'pdf') {
+        const source = object(inspection.source);
+        if (source.integrity !== 'verified')
+            throw Error('integrity');
+        const asset = { relativePath: String(source.path), sha256: String(source.sha256), bytes: Number(source.bytes), mediaType: 'application/pdf', relationship: 'source-pdf' };
+        validateAsset(asset);
+        for await (const _ of bundleBytes(root, asset)) { }
+        assets.push(asset);
+    }
+    if (assets.length !== new Set(assets.map(a => a.relativePath)).size)
+        throw Error('integrity');
+    const sourceUrl = kind === 'pdf' ? (object(manifest.source).requestedUrl ?? pathToFileURL(join(root, String(object(inspection.source).path))).href) : inspection.sourceUrl;
+    if (typeof sourceUrl !== 'string' || sourceUrl.length > 16384 || !['http:', 'https:', 'file:'].includes(new URL(sourceUrl).protocol))
+        throw Error('integrity');
+    const observedAt = typeof manifest.capturedAt === 'string' && Number.isFinite(Date.parse(manifest.capturedAt)) ? manifest.capturedAt : '1970-01-01T00:00:00.000Z';
+    const metadata = kind === 'article' ? (manifest.metadata === undefined ? metadataFromMarkdown(markdownBytes.toString('utf8'), observedAt) : articleMetadata(manifest.metadata, observedAt)) : [];
+    if (kind === 'pdf') {
+        const document = object(manifest.document ?? {});
+        for (const field of ['title', 'author', 'subject', 'creator', 'producer', 'createdAt', 'modifiedAt']) {
+            const value = document[field];
+            if (typeof value === 'string' && value && Buffer.byteLength(value) <= 16384)
+                metadata.push({ field, value, source: 'pdf', observedAt, rule: 'wordcell-pdf-v1', selected: true });
+        }
+    }
+    metadata.push({ field: 'historicalIntegrity', value: unavailable ? 'unavailable' : 'verified', source: kind === 'pdf' ? 'pdf' : 'html', observedAt, rule: 'sponge-import-v1' });
+    if (hash(await readConfined(root, 'capture.json', limits.metadataBytes)) !== hash(manifestBytes))
+        throw Error('integrity');
+    const manifestSha256 = hash(manifestBytes), importHash = hash(JSON.stringify([manifestSha256, markdown.sha256, ...assets.map(a => [a.relativePath, a.sha256, a.bytes])]));
+    return { root, kind, sourceUrl, schemaVersion: Number(manifest.schemaVersion), status: unavailable || inspection.status !== 'complete' ? 'partial' : 'complete', markdown, assets, metadata, manifestSha256, historicalIntegrity: unavailable ? 'unavailable' : 'verified', importHash };
+}

package/dist/src/importers/manifest.d.ts

@@ -0,0 +1,22 @@
+export interface ImportEntry {
+    id: string;
+    bundle: string;
+    privacy: 'private' | 'published';
+    notes: {
+        id: string;
+        body: string;
+    }[];
+    readingState?: Record<string, unknown>;
+}
+export interface ImportManifest {
+    schemaVersion: 1;
+    namespace: string;
+    entries: ImportEntry[];
+}
+/** Explicit local source manifest; no recursive discovery or implicit public selection. */
+export declare function readImportManifest(path: string): Promise<{
+    root: string;
+    manifest: ImportManifest;
+    sha256: string;
+}>;
+export declare function importBundleRoot(root: string, entry: ImportEntry): Promise<string>;

package/dist/src/importers/manifest.js

@@ -0,0 +1,43 @@
+import { dirname, isAbsolute, join } from 'node:path';
+import { realpath, lstat } from 'node:fs/promises';
+import { readConfined, hash } from '../acquisition/files.js';
+import { object } from '../acquisition/normalize.js';
+function bounded(value, max) { if (typeof value !== 'string' || !value || Buffer.byteLength(value) > max || value.includes('\0'))
+    throw Error('invalid-input'); return value; }
+function keys(value, allowed) { if (Object.keys(value).some(k => !allowed.includes(k)))
+    throw Error('invalid-input'); }
+/** Explicit local source manifest; no recursive discovery or implicit public selection. */
+export async function readImportManifest(path) {
+    if (!isAbsolute(path) || await realpath(path) !== path)
+        throw Error('invalid-input');
+    const root = dirname(path), bytes = await readConfined(root, path.slice(root.length + 1), 1024 ** 2), raw = object(JSON.parse(bytes.toString('utf8')));
+    keys(raw, ['schemaVersion', 'namespace', 'entries']);
+    if (raw.schemaVersion !== 1 || !Array.isArray(raw.entries) || raw.entries.length > 1000)
+        throw Error('unsupported');
+    const namespace = bounded(raw.namespace, 128);
+    if (!/^[A-Za-z0-9_.-]+$/.test(namespace))
+        throw Error('invalid-input');
+    const ids = new Set();
+    const entries = raw.entries.map(value => {
+        const entry = object(value);
+        keys(entry, ['id', 'bundle', 'privacy', 'notes', 'readingState']);
+        const id = bounded(entry.id, 256), bundle = bounded(entry.bundle, 4096);
+        if (ids.has(id) || isAbsolute(bundle) || bundle.includes('\\') || bundle.split('/').some(p => !p || p === '.' || p === '..') || !['private', 'published'].includes(String(entry.privacy)))
+            throw Error('invalid-input');
+        ids.add(id);
+        if (entry.notes !== undefined && !Array.isArray(entry.notes))
+            throw Error('invalid-input');
+        const noteIds = new Set();
+        const notes = (entry.notes ?? []).map(value => { const note = object(value); keys(note, ['id', 'body']); const id = bounded(note.id, 256), body = bounded(note.body, 65536); if (noteIds.has(id))
+            throw Error('invalid-input'); noteIds.add(id); return { id, body }; });
+        if (notes.length > 100)
+            throw Error('limit-exceeded');
+        const readingState = entry.readingState === undefined ? undefined : object(entry.readingState);
+        if (readingState && Buffer.byteLength(JSON.stringify(readingState)) > 65536)
+            throw Error('limit-exceeded');
+        return { id, bundle, privacy: entry.privacy, notes, ...(readingState ? { readingState } : {}) };
+    });
+    return { root, manifest: { schemaVersion: 1, namespace, entries }, sha256: hash(bytes) };
+}
+export async function importBundleRoot(root, entry) { const path = join(root, entry.bundle); if (await realpath(path) !== path || (await lstat(path)).isSymbolicLink() || !(await lstat(path)).isDirectory())
+    throw Error('integrity'); return path; }

package/dist/src/importers/verify-helper.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/src/importers/verify-helper.js

@@ -0,0 +1,27 @@
+/** Only released Wordcell public verifiers, with no source or document text in output. */
+const runtime = process.argv[2], bundle = process.argv[3], kind = process.argv[4];
+try {
+    let result;
+    if (kind === 'pdf') {
+        const { verifyPdfCaptureBundle } = await import(import.meta.resolve('@hraness/wordcell/pdf', `${runtime}/package.json`));
+        result = await verifyPdfCaptureBundle(bundle, { maxDocumentBytes: 16 * 1024 ** 2, maxSourceBytes: 100 * 1024 ** 2, maxVerifiedAssets: 200, maxAssetBytes: 20 * 1024 ** 2, maxTotalAssetBytes: 500 * 1024 ** 2, maxVerificationMs: 30000 });
+        const { markdown, ...document } = result.inspection.markdown;
+        result = { ...result, inspection: { ...result.inspection, markdown: document } };
+    }
+    else {
+        const { verifyCaptureBundle } = await import(import.meta.resolve('@hraness/wordcell/clip/bundle-reader', `${runtime}/package.json`));
+        result = await verifyCaptureBundle(bundle, { verifyAssets: true, maxDocumentBytes: 16 * 1024 ** 2, maxVerifiedAssets: 200, maxAssetBytes: 20 * 1024 ** 2, maxTotalAssetBytes: 500 * 1024 ** 2, maxAssetVerificationMs: 30000 });
+        const { markdown, ...document } = result.inspection.document;
+        const { metadata, ...inspection } = result.inspection;
+        result = { ...result, inspection: { ...inspection, document } };
+    }
+    const output = JSON.stringify(result);
+    if (Buffer.byteLength(output) > 65536)
+        throw Error('limit');
+    process.stdout.write(output);
+}
+catch {
+    process.stdout.write(JSON.stringify({ ok: false, code: 'unsupported' }));
+    process.exitCode = 1;
+}
+export {};

package/dist/src/indexing/image.d.ts

@@ -0,0 +1,10 @@
+import { runOwned } from '../acquisition/process.js';
+/** Derived pixels are disposable; source image blobs remain unchanged. */
+export declare function deriveEmbeddingImage(bytes: Uint8Array, options: {
+    scratchRoot: string;
+    runProcess?: typeof runOwned;
+    signal?: AbortSignal;
+}): Promise<{
+    png: Buffer;
+    sha256: string;
+}>;

package/dist/src/indexing/image.js

@@ -0,0 +1,34 @@
+import { mkdtemp, writeFile, rm } from 'node:fs/promises';
+import { join } from 'node:path';
+import { privateRoot, readConfined, hash } from '../acquisition/files.js';
+import { runOwned } from '../acquisition/process.js';
+import { encodeEmbeddingRequest, EmbeddingError } from '../inference/embedding/index.js';
+/** Derived pixels are disposable; source image blobs remain unchanged. */
+export async function deriveEmbeddingImage(bytes, options) {
+    if (process.platform !== 'darwin' || process.arch !== 'arm64')
+        throw new EmbeddingError('unsupported');
+    if (!bytes.length || bytes.length > 20 * 1024 ** 2)
+        throw new EmbeddingError('limit-exceeded');
+    const root = await privateRoot(options.scratchRoot), scratch = await mkdtemp(join(root, 'image-derive-'));
+    let settled = true;
+    try {
+        const input = join(scratch, 'source'), output = join(scratch, 'image.png');
+        await writeFile(input, bytes, { mode: 0o600, flag: 'wx' });
+        const run = async (args) => { settled = false; const result = await (options.runProcess ?? runOwned)('/usr/bin/sandbox-exec', ['-p', '(version 1) (allow default) (deny network*)', '/usr/bin/sips', ...args], { env: { PATH: '/usr/bin:/bin', HOME: scratch, TMPDIR: scratch, LANG: 'C' }, deadlineMs: 30000, maxOutputBytes: 65536, signal: options.signal }); settled = result.cleanup === 'settled'; if (!settled)
+            throw new EmbeddingError('cleanup-required'); if (result.stopped)
+            throw new EmbeddingError(result.stopped); if (result.code !== 0)
+            throw new EmbeddingError('unsupported'); return result.stdout; };
+        const metadata = await run(['-g', 'pixelWidth', '-g', 'pixelHeight', input]);
+        const width = Number(metadata.match(/pixelWidth:\s*(\d+)/)?.[1]), height = Number(metadata.match(/pixelHeight:\s*(\d+)/)?.[1]);
+        if (!Number.isSafeInteger(width) || !Number.isSafeInteger(height) || width < 1 || height < 1 || width * height > 40_000_000)
+            throw new EmbeddingError('limit-exceeded');
+        await run(['--resampleHeightWidthMax', '512', '--setProperty', 'format', 'png', input, '--out', output]);
+        const png = await readConfined(scratch, 'image.png', 2 * 1024 ** 2);
+        encodeEmbeddingRequest([{ kind: 'image', png }]);
+        return { png, sha256: hash(png) };
+    }
+    finally {
+        if (settled)
+            await rm(scratch, { recursive: true, force: true });
+    }
+}

package/dist/src/indexing/index.d.ts

@@ -0,0 +1,53 @@
+import type { DatabaseSync } from 'node:sqlite';
+export interface IndexInput {
+    sourceId: string;
+    revisionId: string;
+    itemId: string;
+    derivationHash: string;
+    kind: 'document' | 'image';
+    text?: string;
+    representationId?: string;
+    ordinal?: number;
+    assetSha256?: string;
+}
+export interface GenerationStatus {
+    id: string;
+    fingerprint: string;
+    state: string;
+    revisionEpoch: number;
+    expected: number;
+    completed: number;
+    profileId: string;
+}
+export declare class IndexError extends Error {
+    code: 'invalid-index' | 'stale-index' | 'incomplete-index' | 'cancelled-index';
+    constructor(code: 'invalid-index' | 'stale-index' | 'incomplete-index' | 'cancelled-index');
+}
+/** Unicode-safe chunks leave room for the fixed document prompt within 8192 UTF-8 bytes. */
+export declare function chunkDocument(text: string): string[];
+/** Identity must already be the verified embedder fingerprint. Existing identities are immutable. */
+export declare function createGeneration(db: DatabaseSync, options: {
+    profileId: string;
+    fingerprint: string;
+    identity: Record<string, string | number>;
+    dimensions: number;
+}): GenerationStatus;
+export declare function generationStatus(db: DatabaseSync, id: string): GenerationStatus;
+export declare function pendingInputs(db: DatabaseSync, id: string, limit?: number): IndexInput[];
+export declare function writeBatch(db: DatabaseSync, id: string, fingerprint: string, batch: readonly {
+    sourceId: string;
+    revisionId: string;
+    derivationHash: string;
+    inputHash: string;
+    vector: readonly number[];
+}[]): void;
+export declare function activateGeneration(db: DatabaseSync, id: string): GenerationStatus;
+export declare function cancelGeneration(db: DatabaseSync, id: string): void;
+/** Read-side join removes deleted/refreshed revisions even when an old generation stays active. */
+export declare function activeVectors(db: DatabaseSync, profileId: string): {
+    fingerprint: string;
+    generationId: string;
+    source: IndexInput;
+    vector: number[];
+}[];
+export declare const rollbackGeneration: typeof activateGeneration;