@cairn-tool/cairn 2.0.0

package/dist/archive/run.js

@@ -0,0 +1,331 @@
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { loadSqlite } from "../sqlite.js";
+import { closeQuietly, transact } from "../sqlite-store.js";
+import { openArchive, segmentsDirectory } from "./db.js";
+import { profileFor } from "./sets.js";
+import { DEFAULT_SEGMENT_BYTES, hashBuffer, SegmentWriter } from "./segments.js";
+export function emptyCounters() {
+    return {
+        discovered: 0,
+        unchanged: 0,
+        hashed: 0,
+        duplicate: 0,
+        stored: 0,
+        skipped: 0,
+        bytes: 0,
+    };
+}
+/**
+ * Walks one set, yielding what it matches.
+ *
+ * Symbolic links are not followed. A link into a directory the sets deliberately
+ * exclude would otherwise pull it in anyway, which is the failure mode the
+ * allowlist in `sets.ts` exists to prevent.
+ */
+export function collectSet(root, provider, set) {
+    const base = set.root ? path.join(root, set.root) : root;
+    const found = [];
+    const walk = (directory, prefix) => {
+        let entries;
+        try {
+            entries = fs.readdirSync(directory, { withFileTypes: true });
+        }
+        catch {
+            // A directory that is absent or unreadable simply contributes nothing.
+            return;
+        }
+        for (const entry of entries) {
+            const relative = prefix ? `${prefix}/${entry.name}` : entry.name;
+            const absolute = path.join(directory, entry.name);
+            if (entry.isDirectory()) {
+                if (set.recursive)
+                    walk(absolute, relative);
+                continue;
+            }
+            if (!entry.isFile())
+                continue;
+            if (!set.match(relative))
+                continue;
+            let stats;
+            try {
+                stats = fs.statSync(absolute);
+            }
+            catch {
+                continue;
+            }
+            found.push({
+                provider,
+                set,
+                file: absolute,
+                recordPath: absolute,
+                size: stats.size,
+                mtimeMs: stats.mtimeMs,
+                mode: stats.mode & 0o777,
+            });
+        }
+    };
+    walk(base, "");
+    // Byte comparison, never localeCompare: segment membership must not depend on
+    // the ICU build of the machine that ran the archive.
+    found.sort((a, b) => (a.file < b.file ? -1 : a.file > b.file ? 1 : 0));
+    return found;
+}
+/** Everything the selected classes match, across the selected providers. */
+export function collect(sources, classes) {
+    const wanted = new Set(classes);
+    const found = [];
+    for (const source of sources) {
+        const profile = profileFor(source.provider.name);
+        if (!profile)
+            continue;
+        for (const set of profile.sets) {
+            if (!wanted.has(set.class))
+                continue;
+            found.push(...collectSet(source.root, source.provider.name, set));
+        }
+    }
+    return found;
+}
+/**
+ * Reads a candidate's bytes.
+ *
+ * A set marked `snapshot: "sqlite"` is copied through the online backup API into
+ * a temporary file rather than read directly: those databases carry live `-wal`
+ * sidecars, and a byte copy of the main file alone can capture a page image torn
+ * mid-write. The snapshot is a single consistent file, which is also what makes
+ * it worth archiving — a `.db` without its `-wal` may be missing recent writes.
+ */
+async function readCandidate(candidate) {
+    if (candidate.set.snapshot !== "sqlite")
+        return fs.readFileSync(candidate.file);
+    const sqlite = loadSqlite();
+    if (!sqlite)
+        return fs.readFileSync(candidate.file);
+    const scratch = fs.mkdtempSync(path.join(os.tmpdir(), "cairn-snapshot-"));
+    const target = path.join(scratch, "snapshot.db");
+    let source;
+    try {
+        source = new sqlite.DatabaseSync(candidate.file, { readOnly: true });
+        const backup = sqlite
+            .backup;
+        await backup(source, target);
+        return fs.readFileSync(target);
+    }
+    finally {
+        closeQuietly(source);
+        try {
+            fs.rmSync(scratch, { recursive: true, force: true });
+        }
+        catch {
+            // Best-effort.
+        }
+    }
+}
+/**
+ * The key a candidate and its stored row are matched on.
+ *
+ * One helper for both sides on purpose: they were briefly written separately,
+ * with different separators, and the result was an archive that re-hashed its
+ * entire corpus on every run while still producing correct output — a bug with
+ * no wrong answer to give it away.
+ *
+ * The separator is a NUL, for the same reason `sessionKey` in
+ * `src/usage/events.ts` uses one: it cannot occur in either half. A space can
+ * and does occur in a path.
+ */
+function artifactKey(provider, file) {
+    return `${provider}\u0000${file}`;
+}
+/** What the index already holds, keyed by {@link artifactKey}. */
+function indexedArtifacts(db) {
+    const found = new Map();
+    // The newest row per path is the one whose size and mtime a freshness test
+    // should compare against; an older row describes a version since replaced.
+    for (const row of db
+        .prepare(`SELECT provider, path, size, mtime_ms FROM artifact
+        WHERE id IN (SELECT MAX(id) FROM artifact GROUP BY provider, path)`)
+        .all()) {
+        found.set(artifactKey(String(row.provider), String(row.path)), {
+            size: Number(row.size),
+            mtimeMs: Number(row.mtime_ms),
+        });
+    }
+    return found;
+}
+export async function runArchive(options) {
+    const counters = emptyCounters();
+    const failures = [];
+    const byClass = {};
+    const bump = (name, field, amount = 1) => {
+        const entry = (byClass[name] ??= { discovered: 0, stored: 0, bytes: 0 });
+        entry[field] += amount;
+    };
+    const candidates = collect(options.sources, options.classes);
+    counters.discovered = candidates.length;
+    let plannedBytes = 0;
+    for (const candidate of candidates) {
+        bump(candidate.set.class, "discovered");
+        plannedBytes += candidate.size;
+    }
+    const reporter = options.reporter;
+    reporter?.start?.({ files: candidates.length, bytes: plannedBytes });
+    if (options.dryRun) {
+        // Nothing is opened: a dry run reports what the sets matched and how much it
+        // is, from the stat the walk already did.
+        for (const candidate of candidates) {
+            counters.bytes += candidate.size;
+            (byClass[candidate.set.class] ??= { discovered: 0, stored: 0, bytes: 0 }).bytes +=
+                candidate.size;
+        }
+        reporter?.finish?.();
+        return { counters, failures, segments: [], byClass };
+    }
+    const opened = openArchive({ root: options.archiveRoot });
+    const db = opened.db;
+    try {
+        const indexed = indexedArtifacts(db);
+        /**
+         * Blobs the index already holds, so an artifact row may reference them now.
+         */
+        const storedBlobs = new Set(db
+            .prepare("SELECT sha256 FROM blob")
+            .all()
+            .map((row) => String(row.sha256)));
+        /**
+         * Blobs buffered for the segment currently being built.
+         *
+         * Kept apart from {@link storedBlobs} because the two demand opposite
+         * handling, and conflating them is a foreign key violation: a second file
+         * with the same content as one still in the buffer cannot have its row
+         * written yet, because the `blob` row it would reference does not exist
+         * until that segment is sealed. It waits instead.
+         */
+        const pendingBlobs = new Set();
+        const nextSequence = Number(db.prepare("SELECT COUNT(*) AS n FROM segment").get()?.n ?? 0) + 1;
+        const writer = new SegmentWriter(segmentsDirectory(options.archiveRoot), nextSequence, options.segmentBytes ?? DEFAULT_SEGMENT_BYTES);
+        const now = new Date().toISOString();
+        const insertSegment = db.prepare("INSERT INTO segment(name, bytes, blobs, sha256, sealed_at) VALUES(?,?,?,?,?)");
+        const insertBlob = db.prepare("INSERT OR IGNORE INTO blob(sha256, size, segment_id, offset, stored_at) VALUES(?,?,?,?,?)");
+        const insertArtifact = db.prepare(`INSERT INTO artifact(provider, set_id, class, path, sha256, size, mtime_ms, mode,
+         first_seen, last_seen)
+       VALUES(?,?,?,?,?,?,?,?,?,?)
+       ON CONFLICT(provider, path, sha256) DO UPDATE SET last_seen = excluded.last_seen`);
+        /** Rows waiting on the segment their content is being written into. */
+        let waiting = [];
+        const writeArtifact = (candidate, sha256) => {
+            insertArtifact.run(candidate.provider, candidate.set.id, candidate.set.class, candidate.recordPath, sha256, candidate.size, candidate.mtimeMs, candidate.mode, now, now);
+        };
+        /**
+         * Seals whatever is buffered and writes the rows that were waiting on it.
+         *
+         * Segment, blobs and artifact rows go in together: an artifact row pointing
+         * at a blob whose segment was never recorded would be an index that cannot
+         * find its own content.
+         */
+        const commitSegment = () => {
+            const segment = writer.flush();
+            if (!segment)
+                return;
+            transact(db, () => {
+                const inserted = insertSegment.run(segment.name, segment.bytes, segment.blobs.length, segment.sha256, now);
+                const segmentId = Number(inserted.lastInsertRowid);
+                for (const blob of segment.blobs) {
+                    insertBlob.run(blob.sha256, blob.size, segmentId, blob.offset, now);
+                    storedBlobs.add(blob.sha256);
+                }
+                for (const row of waiting)
+                    writeArtifact(row.candidate, row.sha256);
+            });
+            waiting = [];
+            pendingBlobs.clear();
+            reporter?.segment?.({
+                name: segment.name,
+                bytes: segment.bytes,
+                blobs: segment.blobs.length,
+            });
+        };
+        for (const candidate of candidates) {
+            const previous = indexed.get(artifactKey(candidate.provider, candidate.recordPath));
+            if (previous && previous.size === candidate.size && previous.mtimeMs === candidate.mtimeMs) {
+                counters.unchanged += 1;
+                reporter?.file?.({
+                    path: candidate.recordPath,
+                    class: candidate.set.class,
+                    size: candidate.size,
+                    disposition: "unchanged",
+                });
+                continue;
+            }
+            let content;
+            try {
+                content = await readCandidate(candidate);
+            }
+            catch (error) {
+                counters.skipped += 1;
+                const reason = error.message;
+                failures.push({ file: candidate.file, reason });
+                reporter?.failure?.({ file: candidate.file, reason });
+                reporter?.file?.({
+                    path: candidate.recordPath,
+                    class: candidate.set.class,
+                    size: candidate.size,
+                    disposition: "skipped",
+                });
+                continue;
+            }
+            counters.hashed += 1;
+            const sha256 = hashBuffer(content);
+            const seen = (disposition) => reporter?.file?.({
+                path: candidate.recordPath,
+                class: candidate.set.class,
+                size: candidate.size,
+                disposition,
+                sha256,
+            });
+            if (storedBlobs.has(sha256)) {
+                // Same bytes under a new path, or a file touched without being changed.
+                counters.duplicate += 1;
+                transact(db, () => writeArtifact(candidate, sha256));
+                seen("duplicate");
+                continue;
+            }
+            if (pendingBlobs.has(sha256)) {
+                // Content this same run has already buffered. One member covers both
+                // paths, so only the row is added — once the segment is sealed.
+                counters.duplicate += 1;
+                waiting.push({ candidate, sha256 });
+                seen("duplicate");
+                continue;
+            }
+            counters.stored += 1;
+            counters.bytes += content.length;
+            bump(candidate.set.class, "stored");
+            (byClass[candidate.set.class] ??= { discovered: 0, stored: 0, bytes: 0 }).bytes +=
+                content.length;
+            pendingBlobs.add(sha256);
+            waiting.push({ candidate, sha256 });
+            seen("stored");
+            writer.add({ sha256, content, mode: candidate.mode });
+            if (writer.shouldSeal)
+                commitSegment();
+        }
+        commitSegment();
+        reporter?.finish?.();
+        return {
+            counters,
+            failures,
+            segments: writer.sealed.map((segment) => ({
+                name: segment.name,
+                bytes: segment.bytes,
+                blobs: segment.blobs.length,
+            })),
+            byClass,
+        };
+    }
+    finally {
+        closeQuietly(db);
+    }
+}
+//# sourceMappingURL=run.js.map

package/dist/archive/run.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"run.js","sourceRoot":"","sources":["../../src/archive/run.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAC1C,OAAO,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAE5D,OAAO,EAAE,WAAW,EAAE,iBAAiB,EAAE,MAAM,SAAS,CAAC;AAEzD,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAEvC,OAAO,EAAE,qBAAqB,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAsDjF,MAAM,UAAU,aAAa;IAC3B,OAAO;QACL,UAAU,EAAE,CAAC;QACb,SAAS,EAAE,CAAC;QACZ,MAAM,EAAE,CAAC;QACT,SAAS,EAAE,CAAC;QACZ,MAAM,EAAE,CAAC;QACT,OAAO,EAAE,CAAC;QACV,KAAK,EAAE,CAAC;KACT,CAAC;AACJ,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,UAAU,CAAC,IAAY,EAAE,QAAgB,EAAE,GAAgB;IACzE,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IACzD,MAAM,KAAK,GAAgB,EAAE,CAAC;IAE9B,MAAM,IAAI,GAAG,CAAC,SAAiB,EAAE,MAAc,EAAQ,EAAE;QACvD,IAAI,OAAoB,CAAC;QACzB,IAAI,CAAC;YACH,OAAO,GAAG,EAAE,CAAC,WAAW,CAAC,SAAS,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;QAC/D,CAAC;QAAC,MAAM,CAAC;YACP,uEAAuE;YACvE,OAAO;QACT,CAAC;QACD,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,MAAM,QAAQ,GAAG,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC;YACjE,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;YAClD,IAAI,KAAK,CAAC,WAAW,EAAE,EAAE,CAAC;gBACxB,IAAI,GAAG,CAAC,SAAS;oBAAE,IAAI,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;gBAC5C,SAAS;YACX,CAAC;YACD,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE;gBAAE,SAAS;YAC9B,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC;gBAAE,SAAS;YACnC,IAAI,KAAe,CAAC;YACpB,IAAI,CAAC;gBACH,KAAK,GAAG,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;YAChC,CAAC;YAAC,MAAM,CAAC;gBACP,SAAS;YACX,CAAC;YACD,KAAK,CAAC,IAAI,CAAC;gBACT,QAAQ;gBACR,GAAG;gBACH,IAAI,EAAE,QAAQ;gBACd,UAAU,EAAE,QAAQ;gBACpB,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,OAAO,EAAE,KAAK,CAAC,OAAO;gBACtB,IAAI,EAAE,KAAK,CAAC,IAAI,GAAG,KAAK;aACzB,CAAC,CAAC;QACL,CAAC;IACH,CAAC,CAAC;IAEF,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;IACf,8EAA8E;IAC9E,qDAAqD;IACrD,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACvE,OAAO,KAAK,CAAC;AACf,CAAC;AAED,4EAA4E;AAC5E,MAAM,UAAU,OAAO,CACrB,OAAiE,EACjE,OAAiC;IAEjC,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC;IAChC,MAAM,KAAK,GAAgB,EAAE,CAAC;IAC9B,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,MAAM,OAAO,GAAG,UAAU,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACjD,IAAI,CAAC,OAAO;YAAE,SAAS;QACvB,KAAK,MAAM,GAAG,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;YAC/B,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC;gBAAE,SAAS;YACrC,KAAK,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,QAAQ,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC;QACpE,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;;;;;GAQG;AACH,KAAK,UAAU,aAAa,CAAC,SAAoB;IAC/C,IAAI,SAAS,CAAC,GAAG,CAAC,QAAQ,KAAK,QAAQ;QAAE,OAAO,EAAE,CAAC,YAAY,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;IAEhF,MAAM,MAAM,GAAG,UAAU,EAAE,CAAC;IAC5B,IAAI,CAAC,MAAM;QAAE,OAAO,EAAE,CAAC,YAAY,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;IAEpD,MAAM,OAAO,GAAG,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,iBAAiB,CAAC,CAAC,CAAC;IAC1E,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;IACjD,IAAI,MAAkC,CAAC;IACvC,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,MAAM,CAAC,YAAY,CAAC,SAAS,CAAC,IAAI,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;QACrE,MAAM,MAAM,GAAI,MAA8E;aAC3F,MAAM,CAAC;QACV,MAAM,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC7B,OAAO,EAAE,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;IACjC,CAAC;YAAS,CAAC;QACT,YAAY,CAAC,MAAM,CAAC,CAAC;QACrB,IAAI,CAAC;YACH,EAAE,CAAC,MAAM,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACvD,CAAC;QAAC,MAAM,CAAC;YACP,eAAe;QACjB,CAAC;IACH,CAAC;AACH,CAAC;AAOD;;;;;;;;;;;GAWG;AACH,SAAS,WAAW,CAAC,QAAgB,EAAE,IAAY;IACjD,OAAO,GAAG,QAAQ,SAAS,IAAI,EAAE,CAAC;AACpC,CAAC;AAED,kEAAkE;AAClE,SAAS,gBAAgB,CAAC,EAAkB;IAC1C,MAAM,KAAK,GAAG,IAAI,GAAG,EAA2B,CAAC;IACjD,2EAA2E;IAC3E,2EAA2E;IAC3E,KAAK,MAAM,GAAG,IAAI,EAAE;SACjB,OAAO,CACN;2EACqE,CACtE;SACA,GAAG,EAAE,EAAE,CAAC;QACT,KAAK,CAAC,GAAG,CAAC,WAAW,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,EAAE;YAC7D,IAAI,EAAE,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC;YACtB,OAAO,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;SAC9B,CAAC,CAAC;IACL,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAkBD,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,OAAmB;IAClD,MAAM,QAAQ,GAAG,aAAa,EAAE,CAAC;IACjC,MAAM,QAAQ,GAAiB,EAAE,CAAC;IAClC,MAAM,OAAO,GAAyB,EAAE,CAAC;IACzC,MAAM,IAAI,GAAG,CAAC,IAAY,EAAE,KAA8B,EAAE,MAAM,GAAG,CAAC,EAAQ,EAAE;QAC9E,MAAM,KAAK,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,EAAE,UAAU,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;QACzE,KAAK,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC;IACzB,CAAC,CAAC;IAEF,MAAM,UAAU,GAAG,OAAO,CAAC,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;IAC7D,QAAQ,CAAC,UAAU,GAAG,UAAU,CAAC,MAAM,CAAC;IACxC,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACnC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,EAAE,YAAY,CAAC,CAAC;QACxC,YAAY,IAAI,SAAS,CAAC,IAAI,CAAC;IACjC,CAAC;IACD,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC;IAClC,QAAQ,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,UAAU,CAAC,MAAM,EAAE,KAAK,EAAE,YAAY,EAAE,CAAC,CAAC;IAErE,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QACnB,6EAA6E;QAC7E,0CAA0C;QAC1C,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;YACnC,QAAQ,CAAC,KAAK,IAAI,SAAS,CAAC,IAAI,CAAC;YACjC,CAAC,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,EAAE,UAAU,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC,KAAK;gBAC7E,SAAS,CAAC,IAAI,CAAC;QACnB,CAAC;QACD,QAAQ,EAAE,MAAM,EAAE,EAAE,CAAC;QACrB,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,EAAE,EAAE,OAAO,EAAE,CAAC;IACvD,CAAC;IAED,MAAM,MAAM,GAAG,WAAW,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC;IAC1D,MAAM,EAAE,GAAG,MAAM,CAAC,EAAE,CAAC;IACrB,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,gBAAgB,CAAC,EAAE,CAAC,CAAC;QACrC;;WAEG;QACH,MAAM,WAAW,GAAG,IAAI,GAAG,CACzB,EAAE;aACC,OAAO,CAAC,yBAAyB,CAAC;aAClC,GAAG,EAAE;aACL,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CACpC,CAAC;QACF;;;;;;;;WAQG;QACH,MAAM,YAAY,GAAG,IAAI,GAAG,EAAU,CAAC;QACvC,MAAM,YAAY,GAAG,MAAM,CAAC,EAAE,CAAC,OAAO,CAAC,mCAAmC,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;QAE/F,MAAM,MAAM,GAAG,IAAI,aAAa,CAC9B,iBAAiB,CAAC,OAAO,CAAC,WAAW,CAAC,EACtC,YAAY,EACZ,OAAO,CAAC,YAAY,IAAI,qBAAqB,CAC9C,CAAC;QAEF,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QACrC,MAAM,aAAa,GAAG,EAAE,CAAC,OAAO,CAC9B,8EAA8E,CAC/E,CAAC;QACF,MAAM,UAAU,GAAG,EAAE,CAAC,OAAO,CAC3B,2FAA2F,CAC5F,CAAC;QACF,MAAM,cAAc,GAAG,EAAE,CAAC,OAAO,CAC/B;;;wFAGkF,CACnF,CAAC;QAEF,uEAAuE;QACvE,IAAI,OAAO,GAAoD,EAAE,CAAC;QAElE,MAAM,aAAa,GAAG,CAAC,SAAoB,EAAE,MAAc,EAAQ,EAAE;YACnE,cAAc,CAAC,GAAG,CAChB,SAAS,CAAC,QAAQ,EAClB,SAAS,CAAC,GAAG,CAAC,EAAE,EAChB,SAAS,CAAC,GAAG,CAAC,KAAK,EACnB,SAAS,CAAC,UAAU,EACpB,MAAM,EACN,SAAS,CAAC,IAAI,EACd,SAAS,CAAC,OAAO,EACjB,SAAS,CAAC,IAAI,EACd,GAAG,EACH,GAAG,CACJ,CAAC;QACJ,CAAC,CAAC;QAEF;;;;;;WAMG;QACH,MAAM,aAAa,GAAG,GAAS,EAAE;YAC/B,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,EAAE,CAAC;YAC/B,IAAI,CAAC,OAAO;gBAAE,OAAO;YACrB,QAAQ,CAAC,EAAE,EAAE,GAAG,EAAE;gBAChB,MAAM,QAAQ,GAAG,aAAa,CAAC,GAAG,CAChC,OAAO,CAAC,IAAI,EACZ,OAAO,CAAC,KAAK,EACb,OAAO,CAAC,KAAK,CAAC,MAAM,EACpB,OAAO,CAAC,MAAM,EACd,GAAG,CACJ,CAAC;gBACF,MAAM,SAAS,GAAG,MAAM,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC;gBACnD,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;oBACjC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,EAAE,SAAS,EAAE,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;oBACpE,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBAC/B,CAAC;gBACD,KAAK,MAAM,GAAG,IAAI,OAAO;oBAAE,aAAa,CAAC,GAAG,CAAC,SAAS,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;YACtE,CAAC,CAAC,CAAC;YACH,OAAO,GAAG,EAAE,CAAC;YACb,YAAY,CAAC,KAAK,EAAE,CAAC;YACrB,QAAQ,EAAE,OAAO,EAAE,CAAC;gBAClB,IAAI,EAAE,OAAO,CAAC,IAAI;gBAClB,KAAK,EAAE,OAAO,CAAC,KAAK;gBACpB,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC,MAAM;aAC5B,CAAC,CAAC;QACL,CAAC,CAAC;QAEF,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;YACnC,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,SAAS,CAAC,QAAQ,EAAE,SAAS,CAAC,UAAU,CAAC,CAAC,CAAC;YACpF,IAAI,QAAQ,IAAI,QAAQ,CAAC,IAAI,KAAK,SAAS,CAAC,IAAI,IAAI,QAAQ,CAAC,OAAO,KAAK,SAAS,CAAC,OAAO,EAAE,CAAC;gBAC3F,QAAQ,CAAC,SAAS,IAAI,CAAC,CAAC;gBACxB,QAAQ,EAAE,IAAI,EAAE,CAAC;oBACf,IAAI,EAAE,SAAS,CAAC,UAAU;oBAC1B,KAAK,EAAE,SAAS,CAAC,GAAG,CAAC,KAAK;oBAC1B,IAAI,EAAE,SAAS,CAAC,IAAI;oBACpB,WAAW,EAAE,WAAW;iBACzB,CAAC,CAAC;gBACH,SAAS;YACX,CAAC;YAED,IAAI,OAAe,CAAC;YACpB,IAAI,CAAC;gBACH,OAAO,GAAG,MAAM,aAAa,CAAC,SAAS,CAAC,CAAC;YAC3C,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,QAAQ,CAAC,OAAO,IAAI,CAAC,CAAC;gBACtB,MAAM,MAAM,GAAI,KAAe,CAAC,OAAO,CAAC;gBACxC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC;gBAChD,QAAQ,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC;gBACtD,QAAQ,EAAE,IAAI,EAAE,CAAC;oBACf,IAAI,EAAE,SAAS,CAAC,UAAU;oBAC1B,KAAK,EAAE,SAAS,CAAC,GAAG,CAAC,KAAK;oBAC1B,IAAI,EAAE,SAAS,CAAC,IAAI;oBACpB,WAAW,EAAE,SAAS;iBACvB,CAAC,CAAC;gBACH,SAAS;YACX,CAAC;YAED,QAAQ,CAAC,MAAM,IAAI,CAAC,CAAC;YACrB,MAAM,MAAM,GAAG,UAAU,CAAC,OAAO,CAAC,CAAC;YAEnC,MAAM,IAAI,GAAG,CAAC,WAAmC,EAAQ,EAAE,CACzD,QAAQ,EAAE,IAAI,EAAE,CAAC;gBACf,IAAI,EAAE,SAAS,CAAC,UAAU;gBAC1B,KAAK,EAAE,SAAS,CAAC,GAAG,CAAC,KAAK;gBAC1B,IAAI,EAAE,SAAS,CAAC,IAAI;gBACpB,WAAW;gBACX,MAAM;aACP,CAAC,CAAC;YAEL,IAAI,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC5B,wEAAwE;gBACxE,QAAQ,CAAC,SAAS,IAAI,CAAC,CAAC;gBACxB,QAAQ,CAAC,EAAE,EAAE,GAAG,EAAE,CAAC,aAAa,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC;gBACrD,IAAI,CAAC,WAAW,CAAC,CAAC;gBAClB,SAAS;YACX,CAAC;YAED,IAAI,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC7B,qEAAqE;gBACrE,gEAAgE;gBAChE,QAAQ,CAAC,SAAS,IAAI,CAAC,CAAC;gBACxB,OAAO,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC;gBACpC,IAAI,CAAC,WAAW,CAAC,CAAC;gBAClB,SAAS;YACX,CAAC;YAED,QAAQ,CAAC,MAAM,IAAI,CAAC,CAAC;YACrB,QAAQ,CAAC,KAAK,IAAI,OAAO,CAAC,MAAM,CAAC;YACjC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;YACpC,CAAC,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,EAAE,UAAU,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC,KAAK;gBAC7E,OAAO,CAAC,MAAM,CAAC;YAEjB,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACzB,OAAO,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC;YACpC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACf,MAAM,CAAC,GAAG,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC;YACtD,IAAI,MAAM,CAAC,UAAU;gBAAE,aAAa,EAAE,CAAC;QACzC,CAAC;QAED,aAAa,EAAE,CAAC;QAChB,QAAQ,EAAE,MAAM,EAAE,EAAE,CAAC;QAErB,OAAO;YACL,QAAQ;YACR,QAAQ;YACR,QAAQ,EAAE,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;gBACxC,IAAI,EAAE,OAAO,CAAC,IAAI;gBAClB,KAAK,EAAE,OAAO,CAAC,KAAK;gBACpB,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC,MAAM;aAC5B,CAAC,CAAC;YACH,OAAO;SACR,CAAC;IACJ,CAAC;YAAS,CAAC;QACT,YAAY,CAAC,EAAE,CAAC,CAAC;IACnB,CAAC;AACH,CAAC"}

package/dist/archive/segments.d.ts

@@ -0,0 +1,89 @@
+/**
+ * Append-only compressed segments.
+ *
+ * A segment is a deterministic `.tar.gz` built by the writer in
+ * `src/agent/package/tar.ts`, which already pins mtime, uid, gid, uname and
+ * entry order. Once sealed it is never rewritten: a later run adds a new
+ * segment, which is what makes the archive safe to copy to slower storage and
+ * what keeps an interrupted run from corrupting what came before.
+ *
+ * Members are named `blobs/<aa>/<sha256>` rather than by their original path,
+ * for two reasons. Deduplication falls out of it — two identical files are one
+ * member — and every name is then 73 characters, comfortably inside the ustar
+ * `name` field. Real paths would not be: the source corpus nests seven deep
+ * under project slugs that are themselves absolute paths with the separators
+ * replaced, and `tarball` refuses a path that will not fit rather than escalating
+ * to a PAX header.
+ */
+/** Default seal threshold, in uncompressed bytes. */
+export declare const DEFAULT_SEGMENT_BYTES: number;
+export interface PendingBlob {
+    sha256: string;
+    content: Buffer;
+    mode: number;
+}
+export interface StoredBlob {
+    sha256: string;
+    size: number;
+    /** Byte offset of the member's data within the uncompressed segment. */
+    offset: number;
+}
+export interface SealedSegment {
+    name: string;
+    /** Size of the `.tar.gz` on disk. */
+    bytes: number;
+    /** Hash of the `.tar.gz`, which is what `archive verify` checks. */
+    sha256: string;
+    blobs: StoredBlob[];
+}
+export declare function hashBuffer(content: Buffer): string;
+export declare function memberPath(sha256: string): string;
+export declare function segmentName(sequence: number): string;
+/**
+ * Writes pending blobs as one sealed segment.
+ *
+ * The archive is built once and then read to find each member's offset, rather
+ * than the offsets being predicted from the entry sizes. Predicting them would
+ * duplicate the writer's padding and header rules in a second place, where they
+ * could drift; reading them back cannot be wrong about a file this same process
+ * just produced.
+ */
+export declare function sealSegment(directory: string, sequence: number, pending: readonly PendingBlob[]): SealedSegment;
+/** Decompresses a segment. Callers that want several members read it once. */
+export declare function readSegment(directory: string, name: string): Buffer;
+/**
+ * Extracts one blob from an already-decompressed segment.
+ *
+ * The offset comes from the index, so this is a slice rather than a walk. The
+ * hash is re-checked against the content because an archive whose index and
+ * bytes disagree should say so rather than hand back the wrong file.
+ */
+export declare function extractBlob(uncompressed: Buffer, blob: StoredBlob): Buffer;
+/**
+ * Accumulates blobs and seals a segment when the threshold is reached.
+ *
+ * The threshold is on *uncompressed* bytes because that is what bounds memory:
+ * the tar is built whole before it is compressed. A blob larger than the
+ * threshold gets a segment to itself rather than being split, since a member
+ * spanning two archives could not be read by `tar`.
+ */
+export declare class SegmentWriter {
+    private readonly directory;
+    private readonly threshold;
+    private pending;
+    private pendingBytes;
+    private sequence;
+    readonly sealed: SealedSegment[];
+    constructor(directory: string, firstSequence: number, threshold?: number);
+    add(blob: PendingBlob): void;
+    /**
+     * Whether the buffer has reached the threshold.
+     *
+     * Sealing is the caller's move rather than a side effect of {@link add},
+     * because the index rows for the blobs in a segment can only be written once
+     * that segment has an id — so the caller has to know exactly when it happened.
+     */
+    get shouldSeal(): boolean;
+    get buffered(): number;
+    flush(): SealedSegment | null;
+}

package/dist/archive/segments.js

@@ -0,0 +1,144 @@
+import crypto from "node:crypto";
+import fs from "node:fs";
+import path from "node:path";
+import zlib from "node:zlib";
+import { archive as tarGz, tarball } from "../agent/package/tar.js";
+import { listMembers, readMember } from "./tar-read.js";
+/**
+ * Append-only compressed segments.
+ *
+ * A segment is a deterministic `.tar.gz` built by the writer in
+ * `src/agent/package/tar.ts`, which already pins mtime, uid, gid, uname and
+ * entry order. Once sealed it is never rewritten: a later run adds a new
+ * segment, which is what makes the archive safe to copy to slower storage and
+ * what keeps an interrupted run from corrupting what came before.
+ *
+ * Members are named `blobs/<aa>/<sha256>` rather than by their original path,
+ * for two reasons. Deduplication falls out of it — two identical files are one
+ * member — and every name is then 73 characters, comfortably inside the ustar
+ * `name` field. Real paths would not be: the source corpus nests seven deep
+ * under project slugs that are themselves absolute paths with the separators
+ * replaced, and `tarball` refuses a path that will not fit rather than escalating
+ * to a PAX header.
+ */
+/** Default seal threshold, in uncompressed bytes. */
+export const DEFAULT_SEGMENT_BYTES = 64 * 1024 * 1024;
+export function hashBuffer(content) {
+    return crypto.createHash("sha256").update(content).digest("hex");
+}
+export function memberPath(sha256) {
+    return `blobs/${sha256.slice(0, 2)}/${sha256}`;
+}
+export function segmentName(sequence) {
+    return `seg-${String(sequence).padStart(6, "0")}.tar.gz`;
+}
+/**
+ * Writes pending blobs as one sealed segment.
+ *
+ * The archive is built once and then read to find each member's offset, rather
+ * than the offsets being predicted from the entry sizes. Predicting them would
+ * duplicate the writer's padding and header rules in a second place, where they
+ * could drift; reading them back cannot be wrong about a file this same process
+ * just produced.
+ */
+export function sealSegment(directory, sequence, pending) {
+    const entries = pending.map((blob) => ({
+        path: memberPath(blob.sha256),
+        content: blob.content,
+        mode: blob.mode,
+    }));
+    const uncompressed = tarball(entries);
+    const members = new Map(listMembers(uncompressed)
+        .filter((member) => !member.directory)
+        .map((member) => [member.path, member]));
+    const blobs = pending.map((blob) => {
+        const member = members.get(memberPath(blob.sha256));
+        if (!member)
+            throw new Error(`Segment is missing the member it was built with: ${blob.sha256}`);
+        return { sha256: blob.sha256, size: member.size, offset: member.offset };
+    });
+    const name = segmentName(sequence);
+    const payload = tarGz(entries);
+    fs.mkdirSync(directory, { recursive: true });
+    const target = path.join(directory, name);
+    // Written to a temporary name and renamed, so an interrupted run never leaves
+    // a half-written segment that the index would then claim is complete.
+    const temporary = `${target}.${process.pid}.tmp`;
+    fs.writeFileSync(temporary, payload);
+    fs.renameSync(temporary, target);
+    return { name, bytes: payload.length, sha256: hashBuffer(payload), blobs };
+}
+/** Decompresses a segment. Callers that want several members read it once. */
+export function readSegment(directory, name) {
+    return zlib.gunzipSync(fs.readFileSync(path.join(directory, name)));
+}
+/**
+ * Extracts one blob from an already-decompressed segment.
+ *
+ * The offset comes from the index, so this is a slice rather than a walk. The
+ * hash is re-checked against the content because an archive whose index and
+ * bytes disagree should say so rather than hand back the wrong file.
+ */
+export function extractBlob(uncompressed, blob) {
+    const content = readMember(uncompressed, {
+        path: memberPath(blob.sha256),
+        mode: 0,
+        size: blob.size,
+        offset: blob.offset,
+        directory: false,
+    });
+    const actual = hashBuffer(content);
+    if (actual !== blob.sha256) {
+        throw new Error(`Archive corruption: expected ${blob.sha256}, segment holds ${actual}`);
+    }
+    return content;
+}
+/**
+ * Accumulates blobs and seals a segment when the threshold is reached.
+ *
+ * The threshold is on *uncompressed* bytes because that is what bounds memory:
+ * the tar is built whole before it is compressed. A blob larger than the
+ * threshold gets a segment to itself rather than being split, since a member
+ * spanning two archives could not be read by `tar`.
+ */
+export class SegmentWriter {
+    directory;
+    threshold;
+    pending = [];
+    pendingBytes = 0;
+    sequence;
+    sealed = [];
+    constructor(directory, firstSequence, threshold = DEFAULT_SEGMENT_BYTES) {
+        this.directory = directory;
+        this.threshold = threshold;
+        this.sequence = firstSequence;
+    }
+    add(blob) {
+        this.pending.push(blob);
+        this.pendingBytes += blob.content.length;
+    }
+    /**
+     * Whether the buffer has reached the threshold.
+     *
+     * Sealing is the caller's move rather than a side effect of {@link add},
+     * because the index rows for the blobs in a segment can only be written once
+     * that segment has an id — so the caller has to know exactly when it happened.
+     */
+    get shouldSeal() {
+        return this.pendingBytes >= this.threshold;
+    }
+    get buffered() {
+        return this.pending.length;
+    }
+    flush() {
+        if (this.pending.length === 0)
+            return null;
+        const segment = sealSegment(this.directory, this.sequence, this.pending);
+        this.sealed.push(segment);
+        this.sequence += 1;
+        this.pending = [];
+        this.pendingBytes = 0;
+        return segment;
+    }
+}
+//# sourceMappingURL=segments.js.map

package/dist/archive/segments.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"segments.js","sourceRoot":"","sources":["../../src/archive/segments.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,aAAa,CAAC;AACjC,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,OAAO,IAAI,KAAK,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAEpE,OAAO,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAExD;;;;;;;;;;;;;;;;GAgBG;AAEH,qDAAqD;AACrD,MAAM,CAAC,MAAM,qBAAqB,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC;AAwBtD,MAAM,UAAU,UAAU,CAAC,OAAe;IACxC,OAAO,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AACnE,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,MAAc;IACvC,OAAO,SAAS,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,EAAE,CAAC;AACjD,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,QAAgB;IAC1C,OAAO,OAAO,MAAM,CAAC,QAAQ,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,SAAS,CAAC;AAC3D,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,WAAW,CACzB,SAAiB,EACjB,QAAgB,EAChB,OAA+B;IAE/B,MAAM,OAAO,GAAe,OAAO,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QACjD,IAAI,EAAE,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC;QAC7B,OAAO,EAAE,IAAI,CAAC,OAAO;QACrB,IAAI,EAAE,IAAI,CAAC,IAAI;KAChB,CAAC,CAAC,CAAC;IAEJ,MAAM,YAAY,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IACtC,MAAM,OAAO,GAAG,IAAI,GAAG,CACrB,WAAW,CAAC,YAAY,CAAC;SACtB,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC;SACrC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAC1C,CAAC;IAEF,MAAM,KAAK,GAAiB,OAAO,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QAC/C,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;QACpD,IAAI,CAAC,MAAM;YAAE,MAAM,IAAI,KAAK,CAAC,oDAAoD,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;QAChG,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC;IAC3E,CAAC,CAAC,CAAC;IAEH,MAAM,IAAI,GAAG,WAAW,CAAC,QAAQ,CAAC,CAAC;IACnC,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC;IAC/B,EAAE,CAAC,SAAS,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC7C,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;IAC1C,8EAA8E;IAC9E,sEAAsE;IACtE,MAAM,SAAS,GAAG,GAAG,MAAM,IAAI,OAAO,CAAC,GAAG,MAAM,CAAC;IACjD,EAAE,CAAC,aAAa,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;IACrC,EAAE,CAAC,UAAU,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAEjC,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,CAAC,OAAO,CAAC,EAAE,KAAK,EAAE,CAAC;AAC7E,CAAC;AAED,8EAA8E;AAC9E,MAAM,UAAU,WAAW,CAAC,SAAiB,EAAE,IAAY;IACzD,OAAO,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC;AACtE,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,WAAW,CAAC,YAAoB,EAAE,IAAgB;IAChE,MAAM,OAAO,GAAG,UAAU,CAAC,YAAY,EAAE;QACvC,IAAI,EAAE,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC;QAC7B,IAAI,EAAE,CAAC;QACP,IAAI,EAAE,IAAI,CAAC,IAAI;QACf,MAAM,EAAE,IAAI,CAAC,MAAM;QACnB,SAAS,EAAE,KAAK;KACjB,CAAC,CAAC;IACH,MAAM,MAAM,GAAG,UAAU,CAAC,OAAO,CAAC,CAAC;IACnC,IAAI,MAAM,KAAK,IAAI,CAAC,MAAM,EAAE,CAAC;QAC3B,MAAM,IAAI,KAAK,CAAC,gCAAgC,IAAI,CAAC,MAAM,mBAAmB,MAAM,EAAE,CAAC,CAAC;IAC1F,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,OAAO,aAAa;IAOL;IAEA;IARX,OAAO,GAAkB,EAAE,CAAC;IAC5B,YAAY,GAAG,CAAC,CAAC;IACjB,QAAQ,CAAS;IAChB,MAAM,GAAoB,EAAE,CAAC;IAEtC,YACmB,SAAiB,EAClC,aAAqB,EACJ,YAAoB,qBAAqB;QAFzC,cAAS,GAAT,SAAS,CAAQ;QAEjB,cAAS,GAAT,SAAS,CAAgC;QAE1D,IAAI,CAAC,QAAQ,GAAG,aAAa,CAAC;IAChC,CAAC;IAED,GAAG,CAAC,IAAiB;QACnB,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACxB,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;IAC3C,CAAC;IAED;;;;;;OAMG;IACH,IAAI,UAAU;QACZ,OAAO,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC,SAAS,CAAC;IAC7C,CAAC;IAED,IAAI,QAAQ;QACV,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;IAC7B,CAAC;IAED,KAAK;QACH,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,IAAI,CAAC;QAC3C,MAAM,OAAO,GAAG,WAAW,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QACzE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAC1B,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC;QACnB,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC;QAClB,IAAI,CAAC,YAAY,GAAG,CAAC,CAAC;QACtB,OAAO,OAAO,CAAC;IACjB,CAAC;CACF"}

package/dist/archive/sets.d.ts

@@ -0,0 +1,58 @@
+/**
+ * What each provider leaves on disk that is worth keeping.
+ *
+ * This is data the archiver reads, exactly as `src/agent/targets/*.ts` is for
+ * the renderer and `ProviderCapabilities` is for the usage reports.
+ * `src/commands/archive.ts` must never branch on a provider name: a provider
+ * that stores plans somewhere unusual says so here, and adding a fourth
+ * assistant is a new entry in this list plus the usage provider it names.
+ *
+ * Selection is an **allowlist of directories**, not a filter over a whole home
+ * directory, and that is the design. `~/.claude` alone holds 340 MB of plugin
+ * payloads, a 71 MB downloaded binary, and a `jobs/` tree that on this machine
+ * was 343 MB of Rust build output — none of it conversation data, all of it
+ * things a blocklist would eventually fail to exclude. Walking only what is
+ * named here cannot pick them up by accident.
+ */
+/**
+ * What an artifact is, which is what `--include` selects on.
+ *
+ * `plan` and `artifact` are archived by default: they are the durable output of
+ * a session, they are small, and nothing else keeps them. `transcript` and `log`
+ * are opt-in because they are the other three orders of magnitude — the same
+ * corpus is roughly 150 MB of the first two and over 9 GB with the rest.
+ */
+export type ArtifactClass = "plan" | "artifact" | "transcript" | "log";
+export declare const ARTIFACT_CLASSES: readonly ArtifactClass[];
+/** The classes `archive run` takes when `--include` is not given. */
+export declare const DEFAULT_CLASSES: readonly ArtifactClass[];
+export interface ArtifactSet {
+    /** Stable identifier, reported per set so a caller can see what matched. */
+    id: string;
+    class: ArtifactClass;
+    description: string;
+    /** Directory under the provider's log root to walk; "" is the root itself. */
+    root: string;
+    /** Whether to descend into subdirectories. */
+    recursive: boolean;
+    /** Decides membership, given a path relative to {@link root}, POSIX-separated. */
+    match(relative: string): boolean;
+    /**
+     * How to read a file whose bytes are not safe to copy directly.
+     *
+     * Every Codex `.sqlite` and all 501 Antigravity `.db` files carry live `-wal`
+     * sidecars, so copying the main file alone can capture a torn page image.
+     * `sqlite` routes them through the online backup API, which produces a
+     * consistent snapshot of a database being written to.
+     */
+    snapshot?: "sqlite";
+}
+export interface ArchiveProfile {
+    /** Matches a `UsageProvider.name`, which is what resolves the log root. */
+    provider: string;
+    sets: readonly ArtifactSet[];
+}
+export declare const ARCHIVE_PROFILES: readonly ArchiveProfile[];
+export declare function profileFor(provider: string): ArchiveProfile | undefined;
+/** Parses `--include plans,transcripts` into the classes it names. */
+export declare function parseClasses(value: string | undefined): ArtifactClass[];