@hraness/sponge 0.2.4 → 0.2.6

package/dist/src/worker/runtime-gc.js

@@ -0,0 +1,296 @@
+import { constants } from 'node:fs';
+import { lstat, readdir, realpath, rename, rm, link, unlink, open } from 'node:fs/promises';
+import { join } from 'node:path';
+import { createHash, randomUUID } from 'node:crypto';
+import { StoreError } from '../store/index.js';
+import { modelSetupPlan } from '../models/artifacts.js';
+import { acquireWorkerCustody } from './custody.js';
+import { readJournal, writeJournal, syncDir } from './journal.js';
+import { processIdentity, identityAlive } from './identity.js';
+const uuid = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
+const patterns = { capture: /^capture-[a-z0-9][a-z0-9.-]{0,99}$/, model: /^[a-f0-9]{64}$/, worker: uuid };
+const helperNames = ['sponge-speech', 'sponge-embedding'];
+const trash = /^\.gc-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
+// A worker writes owner.json just after creating its directory; a young unreadable directory may be mid-acquisition.
+const youngMs = 10 * 60 * 1000;
+const integrity = () => new StoreError('integrity');
+const missing = (error) => error.code === 'ENOENT';
+/** Disk blocks removing one tree frees, never following links: each inode once, and a file only when all of its
+ * hard links are inside the tree (a model artifact shared with a kept package frees nothing). */
+async function diskBytes(root) {
+    let total = 0;
+    const files = new Map();
+    const visit = async (path) => {
+        let stat;
+        try {
+            stat = await lstat(path);
+        }
+        catch (error) {
+            if (missing(error))
+                return;
+            throw error;
+        }
+        if (stat.isDirectory()) {
+            total += stat.blocks * 512;
+            for (const name of await readdir(path))
+                await visit(join(path, name));
+            return;
+        }
+        const key = `${stat.dev}:${stat.ino}`, file = files.get(key);
+        if (file)
+            file.seen++;
+        else
+            files.set(key, { blocks: stat.blocks, nlink: stat.nlink, seen: 1 });
+    };
+    await visit(root);
+    for (const file of files.values())
+        if (file.seen >= file.nlink)
+            total += file.blocks * 512;
+    return total;
+}
+/** The expected parent: a real, private, owned directory at its canonical path, or absent. */
+async function parent(path) {
+    let stat;
+    try {
+        stat = await lstat(path);
+    }
+    catch (error) {
+        if (missing(error))
+            return null;
+        throw error;
+    }
+    if (!stat.isDirectory() || stat.isSymbolicLink() || stat.uid !== process.getuid?.() || (stat.mode & 0o077) !== 0 || await realpath(path) !== path)
+        throw integrity();
+    return path;
+}
+async function exists(path) { try {
+    await lstat(path);
+    return true;
+}
+catch (error) {
+    if (missing(error))
+        return false;
+    throw error;
+} }
+async function fileSha256(path, maxBytes) {
+    let file;
+    try {
+        file = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW);
+    }
+    catch {
+        return null;
+    }
+    try {
+        const stat = await file.stat();
+        if (!stat.isFile() || stat.size > maxBytes)
+            return null;
+        return createHash('sha256').update(await file.readFile()).digest('hex');
+    }
+    finally {
+        await file.close();
+    }
+}
+/** Direct children whose names match the kind's pattern; symlinks and foreign entries are reported, never followed. */
+async function children(kind, dir, kept) {
+    if (!dir)
+        return [];
+    const result = [];
+    for (const name of (await readdir(dir)).sort()) {
+        if (!patterns[kind].test(name) && !trash.test(name))
+            continue;
+        const stat = await lstat(join(dir, name));
+        if (stat.isSymbolicLink()) {
+            kept.push({ kind, name, reason: 'symlink' });
+            continue;
+        }
+        if (!stat.isDirectory() || stat.uid !== process.getuid?.()) {
+            kept.push({ kind, name, reason: 'foreign' });
+            continue;
+        }
+        result.push({ kind, dir, name });
+    }
+    return result;
+}
+/** Rename first so a swapped-in link is caught before recursive removal; rm never follows links inside. */
+async function remove(candidate) {
+    const path = join(candidate.dir, candidate.name), moved = join(candidate.dir, `.gc-${randomUUID()}`);
+    const before = await lstat(path);
+    if (before.isSymbolicLink() || !before.isDirectory())
+        return false;
+    await rename(path, moved);
+    const after = await lstat(moved);
+    if (after.isSymbolicLink() || !after.isDirectory() || after.ino !== before.ino || after.dev !== before.dev) {
+        await rename(moved, path);
+        return false;
+    }
+    await syncDir(candidate.dir);
+    await rm(moved, { recursive: true });
+    await syncDir(candidate.dir);
+    return true;
+}
+async function workerInUse(dir, name) {
+    try {
+        const owner = await readJournal(join(dir, name, 'owner.json'));
+        if (owner?.schemaVersion !== 1 || owner.incarnation !== name || !owner.identity)
+            return true;
+        return await identityAlive(owner.identity);
+    }
+    catch (error) {
+        if (!missing(error))
+            return true;
+        try {
+            return Date.now() - (await lstat(join(dir, name))).mtimeMs < youngMs;
+        }
+        catch {
+            return true;
+        }
+    }
+}
+/** Remove only superseded capture runtimes, model packages and settled worker journals.
+ * Applying holds worker custody, so no worker, capture, import verification, index run or query
+ * can start or be running; each model directory is additionally taken under its setup lock. */
+export async function collectRuntimes(store, options) {
+    const dryRun = options.dryRun === true, base = await realpath(store.root), runtime = await parent(join(base, 'runtime'));
+    if (store.ownerSnapshot())
+        throw new StoreError('owner-busy');
+    if (store.pendingCleanupCount())
+        throw Object.assign(Error('cleanup-required'), { code: 'cleanup-required' });
+    const lockPath = join(base, 'runtime', 'worker.lock');
+    // Checked first so a refused collection leaves no custody journal; acquisition stays the authority.
+    if (await exists(lockPath))
+        throw new StoreError('owner-busy');
+    const custody = dryRun || !runtime ? undefined : await acquireWorkerCustody(base);
+    const removed = [], kept = [];
+    try {
+        if (!runtime)
+            return { dryRun, removed, kept, bytes: 0 };
+        // Recheck under custody: an owner that appeared before our lock still blocks.
+        if (store.ownerSnapshot())
+            throw new StoreError('owner-busy');
+        if (store.pendingCleanupCount())
+            throw Object.assign(Error('cleanup-required'), { code: 'cleanup-required' });
+        const dirs = { capture: await parent(join(runtime, 'capture')), model: await parent(join(runtime, 'models')), worker: await parent(join(runtime, 'workers')) };
+        const references = store.runtimeReferences(), models = options.models;
+        const modelKeep = new Set(models?.fingerprints ?? []);
+        for (const artifact of references.modelArtifacts) {
+            if (patterns.model.test(artifact.id))
+                modelKeep.add(artifact.id);
+            try {
+                modelKeep.add(modelSetupPlan(JSON.parse(artifact.manifest)).fingerprint);
+            }
+            catch { /* An unparsable row still pins its id. */ }
+        }
+        // A generation's helper is satisfied only by an installed kept package, never by an uninstalled catalog entry.
+        const helpers = new Set(references.embeddingHelpers), installed = new Set();
+        for (const fp of modelKeep) {
+            if (!dirs.model || !await exists(join(dirs.model, fp)))
+                continue;
+            for (const helper of helperNames) {
+                if (await exists(join(dirs.model, fp, helper)))
+                    installed.add(helper);
+            }
+            helpers.delete(await fileSha256(join(dirs.model, fp, 'sponge-embedding'), 32 * 1024 ** 2) ?? '');
+        }
+        const take = async (candidate) => {
+            const bytes = await diskBytes(join(candidate.dir, candidate.name));
+            if (dryRun || await remove(candidate))
+                removed.push({ kind: candidate.kind, name: candidate.name, bytes });
+            else
+                kept.push({ kind: candidate.kind, name: candidate.name, reason: 'symlink' });
+        };
+        for (const candidate of await children('capture', dirs.capture, kept)) {
+            if (options.captureIds.includes(candidate.name))
+                kept.push({ kind: 'capture', name: candidate.name, reason: 'referenced' });
+            else
+                await take(candidate);
+        }
+        for (const candidate of await children('model', dirs.model, kept)) {
+            const { name, dir } = candidate;
+            if (trash.test(name)) {
+                await take(candidate);
+                continue;
+            }
+            if (!models) {
+                kept.push({ kind: 'model', name, reason: 'catalog-unavailable' });
+                continue;
+            }
+            if (modelKeep.has(name)) {
+                kept.push({ kind: 'model', name, reason: 'referenced' });
+                continue;
+            }
+            const helper = join(dir, name, 'sponge-embedding');
+            if ((references.unmappedSpaces && await exists(helper)) || helpers.size && helpers.has(await fileSha256(helper, 32 * 1024 ** 2) ?? '')) {
+                kept.push({ kind: 'model', name, reason: 'referenced' });
+                continue;
+            }
+            // Until this Sponge's package of a kind is installed, older packages of that kind are its only local copy.
+            let missingKind = false;
+            for (const helper of helperNames)
+                if (!installed.has(helper) && await exists(join(dir, name, helper)))
+                    missingKind = true;
+            if (missingKind) {
+                kept.push({ kind: 'model', name, reason: 'current-missing' });
+                continue;
+            }
+            const lock = join(dir, `.${name}.lock`);
+            if (await exists(lock) || await exists(join(dir, `.${name}.recovery-lock`))) {
+                kept.push({ kind: 'model', name, reason: 'in-use' });
+                continue;
+            }
+            if (dryRun) {
+                await take(candidate);
+                continue;
+            }
+            // The model setup lock: a concurrent setup or recovery of this fingerprint sees owner-busy.
+            const identity = await processIdentity(process.pid);
+            if (!identity)
+                throw integrity();
+            const incarnation = randomUUID(), ownerFile = join(dir, `.model-owner-${incarnation}.json`);
+            await writeJournal(ownerFile, { schemaVersion: 1, incarnation, fingerprint: name, stage: '.model-install-' + incarnation, identity });
+            try {
+                try {
+                    await link(ownerFile, lock);
+                }
+                catch (error) {
+                    if (error.code === 'EEXIST') {
+                        kept.push({ kind: 'model', name, reason: 'in-use' });
+                        continue;
+                    }
+                    throw error;
+                }
+                try {
+                    await syncDir(dir);
+                    await take(candidate);
+                }
+                finally {
+                    await unlink(lock);
+                }
+            }
+            finally {
+                await unlink(ownerFile);
+                await syncDir(dir);
+            }
+        }
+        for (const candidate of await children('worker', dirs.worker, kept)) {
+            const { name, dir } = candidate;
+            if (name === custody?.incarnation)
+                continue;
+            if (!trash.test(name) && await workerInUse(dir, name)) {
+                kept.push({ kind: 'worker', name, reason: 'in-use' });
+                continue;
+            }
+            await take(candidate);
+        }
+        return { dryRun, removed, kept, bytes: removed.reduce((n, e) => n + e.bytes, 0) };
+    }
+    finally {
+        if (custody) {
+            await custody.release();
+            try {
+                await rm(custody.directory, { recursive: true });
+                await syncDir(join(base, 'runtime', 'workers'));
+            }
+            catch { /* The next collection removes this settled journal. */ }
+        }
+    }
+}

package/native/runtime-manifest.json

@@ -27,14 +27,14 @@
     }
   ],
   "build": {
-    "asrSourceSha256": "c81c23b78e379019e53f9f8e0160e96de7da6cd0b9f940feaccfdc7ae8834b84",
+    "asrSourceSha256": "e944e4e0b1c2cf571e78a83fcfbf56a91a14fd08dcc701562d32555cf090ec40",
     "clangVersion": "Apple clang version 21.0.0 (clang-2100.1.1.101)",
-    "embeddingSourceSha256": "8f5a5ff17e39a4c90e1b22a2789ca081fbd0c9fe19ca81ade8c36b294ae2bfde",
+    "embeddingSourceSha256": "7d58718911725b3c3dce35a09a253704b4e5e732cf1e23c12389485127534b10",
     "swiftVersion": "Apple Swift version 6.3.3 (swiftlang-6.3.3.1.3 clang-2100.1.1.101)"
   },
   "minimumMacOS": "26.0",
-  "packageVersion": "0.2.4",
+  "packageVersion": "0.2.6",
   "platform": "darwin",
   "schemaVersion": 1,
-  "sourceCommit": "0248c746db995e126bdb8713542cadfa49ec2303"
+  "sourceCommit": "1190f23ad793985b6934583c5d8b0957d79a4ce0"
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hraness/sponge",
-  "version": "0.2.4",
+  "version": "0.2.6",
   "description": "A local library for captured articles and verified offline assets.",
   "repository": {
     "type": "git",