@hraness/sponge 0.1.2 → 0.2.1

package/dist/src/service/loop.js

@@ -0,0 +1,94 @@
+import { constants, openSync, closeSync, fstatSync, writeSync, renameSync, lstatSync, existsSync } from 'node:fs';
+import { join } from 'node:path';
+import { safeDirectory, regularFile, syncDirectory } from '../store/files.js';
+import { writePrivateExclusive } from '../portability/files.js';
+import { processIdentity } from '../worker/identity.js';
+import { randomUUID } from 'node:crypto';
+import { acquireServiceLifetime } from './lifetime.js';
+/** Fixed event vocabulary; source URLs, credentials, content and raw errors never enter logs. */
+export function serviceLog(root, event) {
+    const directory = safeDirectory(join(root, 'runtime', 'service'), true), path = join(directory, 'worker.log');
+    if (existsSync(path)) {
+        regularFile(path);
+        if (lstatSync(path).size >= 1024 * 1024) {
+            for (let i = 3; i >= 1; i--) {
+                const old = i === 1 ? path : path + '.' + (i - 1), next = path + '.' + i;
+                if (existsSync(next))
+                    regularFile(next);
+                if (existsSync(old)) {
+                    regularFile(old);
+                    renameSync(old, next);
+                }
+            }
+        }
+    }
+    const fd = openSync(path, constants.O_WRONLY | constants.O_CREAT | constants.O_APPEND | constants.O_NOFOLLOW, 0o600);
+    try {
+        const stat = fstatSync(fd);
+        if (!stat.isFile() || stat.nlink !== 1)
+            throw Error('integrity');
+        writeSync(fd, JSON.stringify({ at: new Date().toISOString(), event }) + '\n');
+    }
+    finally {
+        closeSync(fd);
+    }
+}
+export function idleWait(ms, signal) { return new Promise(resolve => { if (signal.aborted) {
+    resolve();
+    return;
+} const done = () => { clearTimeout(timer); signal.removeEventListener('abort', done); resolve(); }; const timer = setTimeout(done, ms); signal.addEventListener('abort', done, { once: true }); }); }
+/** Uses the existing drain engine; it owns locking, cancellation and descendant collection. */
+export async function runWorkerService(store, options) {
+    const idleMs = options.idleMs ?? 15000;
+    if (!Number.isSafeInteger(idleMs) || idleMs < 1000 || idleMs > 60000 || !/^[a-f0-9-]{36}$/u.test(options.installationId))
+        throw Error('invalid-input');
+    const root = safeDirectory(join(store.root, 'runtime', 'service'), true), identity = await (options.identity ?? (() => processIdentity(process.pid)))();
+    if (!identity)
+        throw Error('unavailable');
+    const lifetime = await acquireServiceLifetime(root, options.installationId, identity);
+    const runId = randomUUID(), runPath = join(root, 'run.json');
+    const record = (state) => { const temp = join(root, `run-${runId}.tmp`); writePrivateExclusive(temp, JSON.stringify({ schemaVersion: 1, installationId: options.installationId, runId, identity, state }) + '\n'); if (existsSync(runPath))
+        regularFile(runPath); renameSync(temp, runPath); syncDirectory(root); };
+    const log = options.log ?? (event => serviceLog(store.root, event));
+    let cycles = 0, cleanup = 'settled';
+    try {
+        record('running');
+        log('started');
+        while (!options.signal.aborted) {
+            try {
+                const results = await options.drain(options.signal);
+                cycles++;
+                log(results.length ? 'drained' : 'idle');
+            }
+            catch (error) {
+                const code = error.code;
+                if (code === 'owner-busy')
+                    log('busy');
+                else {
+                    log('repair-required');
+                    if (code === 'cleanup-required')
+                        cleanup = 'pending';
+                    break;
+                }
+            }
+            await options.onIdle?.();
+            if (options.signal.aborted)
+                break;
+            await (options.wait ?? idleWait)(idleMs, options.signal);
+        }
+        if (store.pendingCleanupCount() > 0)
+            cleanup = 'pending';
+        return { cycles, cleanup };
+    }
+    finally {
+        try {
+            if (store.pendingCleanupCount() > 0)
+                cleanup = 'pending';
+            record(cleanup === 'settled' ? 'stopped' : 'cleanup-required');
+            log('stopped');
+        }
+        finally {
+            await lifetime.release();
+        }
+    }
+}

package/dist/src/store/files.d.ts

@@ -0,0 +1,12 @@
+export declare const digest: (value: string | Uint8Array) => string;
+export declare function hash(value: string): void;
+export declare function text(value: string, max?: number): void;
+export declare function integer(value: number, maximum?: number): void;
+export declare function syncDirectory(path: string): void;
+/** Reject symlinks in every existing component; callers use canonical /private/tmp on macOS. */
+export declare function safeDirectory(path: string, create?: boolean): string;
+export declare function confinedFile(root: string, path: string): string;
+export declare function regularFile(path: string): void;
+export declare function pathExists(path: string): boolean;
+export declare function verifyFile(path: string, expectedHash: string, expectedBytes: number): void;
+export declare function readVerifiedFile(path: string, expectedHash: string, expectedBytes: number): Buffer;

package/dist/src/store/files.js

@@ -0,0 +1,101 @@
+import { constants, closeSync, fstatSync, fsyncSync, lstatSync, mkdirSync, openSync, readSync, realpathSync } from 'node:fs';
+import { dirname, isAbsolute, join, parse, relative, sep } from 'node:path';
+import { createHash } from 'node:crypto';
+import { StoreError } from './types.js';
+export const digest = (value) => createHash('sha256').update(value).digest('hex');
+export function hash(value) {
+    if (!/^[a-f0-9]{64}$/.test(value))
+        throw new StoreError('invalid-input');
+}
+export function text(value, max = 4096) {
+    if (typeof value !== 'string' || value.length === 0 || Buffer.byteLength(value) > max || value.includes('\0'))
+        throw new StoreError('invalid-input');
+}
+export function integer(value, maximum = Number.MAX_SAFE_INTEGER) {
+    if (!Number.isSafeInteger(value) || value < 0 || value > maximum)
+        throw new StoreError('invalid-input');
+}
+export function syncDirectory(path) {
+    const fd = openSync(path, constants.O_RDONLY);
+    try {
+        fsyncSync(fd);
+    }
+    finally {
+        closeSync(fd);
+    }
+}
+/** Reject symlinks in every existing component; callers use canonical /private/tmp on macOS. */
+export function safeDirectory(path, create = false) {
+    if (!isAbsolute(path) || path.includes('\0') || path.split(sep).includes('..'))
+        throw new StoreError('invalid-input');
+    let current = parse(path).root;
+    for (const part of path.slice(current.length).split(sep).filter(Boolean)) {
+        current = join(current, part);
+        try {
+            const stat = lstatSync(current);
+            if (!stat.isDirectory() || stat.isSymbolicLink())
+                throw new StoreError('integrity');
+        }
+        catch (error) {
+            if (error.code !== 'ENOENT' || !create)
+                throw error;
+            mkdirSync(current, { mode: 0o700 });
+            syncDirectory(dirname(current));
+        }
+    }
+    return realpathSync(current);
+}
+export function confinedFile(root, path) {
+    const rel = relative(root, path);
+    if (!rel || rel.startsWith('..' + sep) || rel === '..' || isAbsolute(rel))
+        throw new StoreError('integrity');
+    safeDirectory(dirname(path));
+    return path;
+}
+export function regularFile(path) {
+    const stat = lstatSync(path);
+    if (!stat.isFile() || stat.isSymbolicLink() || stat.nlink !== 1)
+        throw new StoreError('integrity');
+}
+function checkedFile(path, expectedHash, expectedBytes, retain) {
+    regularFile(path);
+    const fd = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW);
+    try {
+        const stat = fstatSync(fd);
+        if (!stat.isFile() || stat.size !== expectedBytes || stat.nlink !== 1)
+            throw new StoreError('integrity');
+        const h = createHash('sha256'), buffer = Buffer.allocUnsafe(64 * 1024);
+        let count = 0;
+        const chunks = [];
+        for (;;) {
+            const read = readSync(fd, buffer, 0, buffer.length, null);
+            if (!read)
+                break;
+            count += read;
+            if (count > expectedBytes)
+                throw new StoreError('integrity');
+            h.update(buffer.subarray(0, read));
+            if (retain)
+                chunks.push(Buffer.from(buffer.subarray(0, read)));
+        }
+        if (count !== expectedBytes || h.digest('hex') !== expectedHash)
+            throw new StoreError('integrity');
+        return Buffer.concat(chunks);
+    }
+    finally {
+        closeSync(fd);
+    }
+}
+export function pathExists(path) {
+    try {
+        lstatSync(path);
+        return true;
+    }
+    catch (error) {
+        if (error.code === 'ENOENT')
+            return false;
+        throw error;
+    }
+}
+export function verifyFile(path, expectedHash, expectedBytes) { checkedFile(path, expectedHash, expectedBytes, false); }
+export function readVerifiedFile(path, expectedHash, expectedBytes) { return checkedFile(path, expectedHash, expectedBytes, true); }

package/dist/src/store/index.d.ts

@@ -0,0 +1,118 @@
+import { DatabaseSync } from 'node:sqlite';
+import type { JobCheckpoint } from '../../contracts/index.js';
+import { type AttemptToken, type OwnedAttempt, type BlobReference, type CommitRevision, type OwnerToken, type ReadableItem, type SourceSubmission, type StoreOptions, type Submission } from './types.js';
+import * as indexing from '../indexing/index.js';
+import { hybridSearch } from '../search/hybrid.js';
+export * from './types.js';
+export declare const libraryMigrations: readonly string[];
+type Row = Record<string, any>;
+export declare function openLibrary(path: string, options?: StoreOptions): Promise<LibraryStore>;
+/** Storage CAS authority only. The worker separately establishes OS/process custody. */
+export declare class LibraryStore {
+    readonly root: string;
+    private readonly db;
+    private readonly options;
+    private closed;
+    constructor(root: string, db: DatabaseSync, options: StoreOptions);
+    createIndexGeneration(options: Parameters<typeof indexing.createGeneration>[1]): indexing.GenerationStatus;
+    submitIndexJob(generationId: string): string;
+    indexJob(id: string): Record<string, import("node:sqlite").SQLOutputValue> | undefined;
+    claimIndexJob(owner: OwnerToken, id?: string): AttemptToken | null;
+    checkpointIndexAttempt(token: AttemptToken, stage: string): void;
+    writeIndexAttemptBatch(token: AttemptToken, fingerprint: string, batch: Parameters<typeof indexing.writeBatch>[3]): void;
+    completeIndexAttempt(token: AttemptToken): indexing.GenerationStatus;
+    failIndexAttempt(token: AttemptToken, reason: string): void;
+    settleIndexAttempt(token: AttemptToken): void;
+    private checkIndexStaging;
+    indexStatus(id: string): indexing.GenerationStatus;
+    indexInputs(id: string, limit?: number): indexing.IndexInput[];
+    writeIndexBatch(id: string, fingerprint: string, batch: Parameters<typeof indexing.writeBatch>[3]): void;
+    activateIndex(id: string): indexing.GenerationStatus;
+    cancelIndex(id: string): void;
+    searchHybrid(query: string, options?: Parameters<typeof hybridSearch>[2]): Promise<import("../search/hybrid.js").SearchResult>;
+    readAnnotations(itemId: string): {
+        annotations: {
+            id: string;
+            revisionId: string | null;
+            anchor: unknown;
+            body: string;
+            createdAt: string;
+        }[];
+        readingState: {
+            state: unknown;
+            updatedAt: string;
+        } | null;
+    };
+    lookupImportReceipt(namespace: string, sourceKey: string, sourceHash: string): {
+        itemId: string;
+        outcome: string;
+    } | null;
+    close(): void;
+    private transaction;
+    get revisionEpoch(): number;
+    private bumpEpoch;
+    acquireOwner(ownerIncarnation: string): OwnerToken;
+    releaseOwner(token: OwnerToken): void;
+    ownerSnapshot(): OwnerToken | null;
+    ownerAttempts(token: OwnerToken): OwnedAttempt[];
+    /** Requires external custody proof for EVERY pending attempt, even after its job committed.
+     * IDs are acknowledgements from the custody owner, never evidence inferred from PID/timeouts here. */
+    recoverOwner(expected: OwnerToken, provedAttemptIds: readonly string[]): void;
+    checkpointAttempt(token: AttemptToken, stage: JobCheckpoint['stage']): void;
+    jobCancellationRequested(jobId: string): boolean;
+    retryJob(jobId: string, options?: {
+        notBefore?: string;
+        automatic?: boolean;
+    }): void;
+    private checkOwner;
+    submitSource(input: SourceSubmission): Submission;
+    listJobs(): Row[];
+    sourceForJob(jobId: string): {
+        itemId: string;
+        kind: string;
+        sourceKey: string;
+        locator: string;
+        realm: string;
+    } | null;
+    claimNext(token: OwnerToken): AttemptToken | null;
+    claimJob(token: OwnerToken, jobId: string): AttemptToken;
+    private checkAttempt;
+    requestCancellation(jobId: string): void;
+    failAttempt(token: AttemptToken, reason?: string, options?: {
+        retryable?: boolean;
+        retryAfterMs?: number;
+    }): void;
+    /** Caller must first settle owned external children. This only releases storage staging pins. */
+    settleAttempt(token: AttemptToken): void;
+    putBlob(token: AttemptToken, expectedHash: string, expectedBytes: number, source: AsyncIterable<Uint8Array>): Promise<BlobReference>;
+    commitRevision(input: CommitRevision): {
+        revisionId: string;
+        duplicate: boolean;
+    };
+    list(): {
+        itemId: string;
+        kind: string;
+        title: string;
+        revisionId: string | null;
+    }[];
+    read(itemId: string): ReadableItem | null;
+    readBlob(reference: BlobReference): Uint8Array;
+    search(query: string, count?: number): {
+        itemId: string;
+        title: string;
+        revisionId: string;
+    }[];
+    delete(itemId: string): void;
+    /** Online DB snapshot only; P12 pairs this with blobs while the returned fence remains active. */
+    createBackupSnapshot(): Promise<{
+        id: string;
+        path: string;
+        revisionEpoch: number;
+    }>;
+    finishBackup(id: string): void;
+    /** Hold the SQLite writer fence across synchronous GC inspection and removal. */
+    withGcFence<T>(expectedEpoch: number, callback: (references: readonly BlobReference[]) => T): T;
+    /** GC must honor both durable staging pins and active backup fences. */
+    pendingCleanupCount(): number;
+    hasGcFence(): boolean;
+}