@hyperdrive.bot/fleet-server 0.3.164 → 0.3.165

package/dist/server/server/state/legacy-import.js

@@ -0,0 +1,682 @@
+import { promises as fs } from "node:fs";
+import path from "node:path";
+import { listAgentRecordFiles, parseStoredAgentRecord, } from "../agent/agent-storage.js";
+import { TOMBSTONED_TABLES } from "./state-schema.js";
+/**
+ * First-boot copy of the legacy JSON stores into `state.sqlite`.
+ *
+ * - COPY, never move: no legacy file is deleted or renamed, so a rollback to an
+ *   older daemon still finds everything (and the mirror keeps it current).
+ * - Batched and yielding: each batch is one transaction and then a
+ *   `setImmediate`, so the event loop is never held for long.
+ * - Resumable: each store keeps a marker whose cursor commits in the same
+ *   transaction as the rows it covers. A crash mid-import resumes after the
+ *   last committed batch; re-running a finished store is a no-op.
+ * - Re-sync by content, not by clock: every file read here or written by the
+ *   mirror has a row in the legacy manifest (`legacy-manifest.ts`). At the next
+ *   boot a file is foreign only when its content no longer matches that row; a
+ *   foreign file is reconciled row by row (the file wins, unless the database
+ *   row changed after the file was written), and a file that is gone deletes
+ *   what it held. Files this daemon wrote and nobody touched are never read.
+ */
+/**
+ * Files per import batch. Parsing and validating 50 agent records is about 5ms
+ * of main-thread work; the commit itself runs on the state worker.
+ */
+export const AGENT_IMPORT_BATCH = 50;
+/**
+ * Runs one batch and records how long it held the event loop: the synchronous
+ * part of `fn` (parsing, building statements, and the commit itself when there
+ * is no worker). A commit on the worker is awaited but not counted, because the
+ * loop is free while it runs.
+ */
+export class BatchClock {
+    constructor() {
+        this.maxBatchMs = 0;
+        /** Every batch's loop-holding time, for percentiles in the boot log. */
+        this.batchMs = [];
+    }
+    async batch(fn) {
+        const started = performance.now();
+        const pending = fn();
+        const held = performance.now() - started;
+        this.batchMs.push(held);
+        this.maxBatchMs = Math.max(this.maxBatchMs, held);
+        const result = await pending;
+        await new Promise((resolve) => setImmediate(resolve));
+        return result;
+    }
+}
+function writeDocRows(state, rows) {
+    for (const row of rows) {
+        state.doc(row.table).putNow(row.scope, row.id, JSON.stringify(row.value), row.ord ?? 0);
+    }
+}
+function docRowStatements(state, rows) {
+    return rows.map((row) => state.doc(row.table).putStatement(row.scope, row.id, JSON.stringify(row.value), row.ord ?? 0));
+}
+async function mapLimit(items, limit, fn) {
+    const out = [];
+    let next = 0;
+    await Promise.all(Array.from({ length: Math.min(limit, items.length) }, async () => {
+        while (next < items.length) {
+            const index = next++;
+            out[index] = await fn(items[index]);
+        }
+    }));
+    return out;
+}
+async function exists(filePath) {
+    try {
+        await fs.stat(filePath);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+const rowKey = (table, scope, id) => `${table}\u0000${scope}\u0000${id}`;
+/**
+ * Reconcile the rows one foreign file describes, inside the caller's
+ * transaction. `fileRows` null means the file was deleted. For each row: when
+ * the database row's rev is above `mirroredRev` it changed after the file was
+ * written, so it wins (counted as a conflict when the two disagree); otherwise
+ * the file's content replaces it. Rows the file no longer has are removed
+ * (tombstoned where the table keeps tombstones) under the same rule.
+ */
+function ownedRows(state, owned, tombstoned) {
+    const existing = new Map();
+    for (const pattern of owned) {
+        const table = state.doc(pattern.table, { tombstones: tombstoned?.has(pattern.table) ?? false });
+        for (const row of table.storedRows({ scope: pattern.scope, id: pattern.id })) {
+            existing.set(rowKey(pattern.table, row.scope, row.id), { table: pattern.table, row });
+        }
+    }
+    return existing;
+}
+function reconcileRows(state, owned, fileRows, mirroredRev, options = {}) {
+    const outcome = { applied: 0, removed: 0, conflicts: 0 };
+    const existing = ownedRows(state, owned, options.tombstoned);
+    const seen = new Set();
+    for (const fileRow of fileRows ?? []) {
+        const key = rowKey(fileRow.table, fileRow.scope, fileRow.id);
+        seen.add(key);
+        const body = JSON.stringify(fileRow.value);
+        const current = existing.get(key)?.row;
+        if (current && current.deletedAt === null && current.body === body)
+            continue;
+        if (current && current.rev > mirroredRev) {
+            outcome.conflicts += 1;
+            continue;
+        }
+        state.doc(fileRow.table).putNow(fileRow.scope, fileRow.id, body, fileRow.ord ?? 0);
+        outcome.applied += 1;
+    }
+    const deleted = fileRows === null;
+    for (const [key, { table, row }] of existing) {
+        if (seen.has(key) || row.deletedAt !== null)
+            continue;
+        if (row.rev > mirroredRev && !(deleted && options.deleteWins)) {
+            outcome.conflicts += 1;
+            continue;
+        }
+        state.doc(table).removeNow(row.scope, row.id);
+        outcome.removed += 1;
+    }
+    return outcome;
+}
+function logConflicts(logger, store, filePath, conflicts) {
+    if (conflicts > 0) {
+        logger.info({ store, filePath, conflicts }, "Legacy file changed by someone else; rows the daemon changed after writing it keep the daemon's version");
+    }
+}
+async function importWholeFile(state, source, clock, logger) {
+    const started = performance.now();
+    const marker = state.getImportMarker(source.store);
+    const resync = marker?.status === "done";
+    const file = await state.manifest.classify(source.filePath);
+    if (resync && (file.status === "ours" || file.status === "absent")) {
+        return { store: source.store, imported: 0, skipped: 0, ms: 0, mode: "skip" };
+    }
+    let rows = null;
+    let skipped = 0;
+    if (file.status === "changed" || file.status === "created") {
+        try {
+            rows = source.toRows(JSON.parse(file.content));
+        }
+        catch (error) {
+            skipped = 1;
+            logger.warn({ err: error, filePath: source.filePath }, "Legacy store unreadable; not imported");
+        }
+    }
+    else if (file.status === "ours") {
+        // First import of a file the manifest already knows: read it plainly.
+        try {
+            rows = source.toRows(JSON.parse(await fs.readFile(source.filePath, "utf8")));
+        }
+        catch (error) {
+            skipped = 1;
+            logger.warn({ err: error, filePath: source.filePath }, "Legacy store unreadable; not imported");
+        }
+    }
+    if (resync && skipped > 0) {
+        if (file.status === "changed" || file.status === "created") {
+            const seen = state.manifest.statement(source.store, source.filePath, file.content, file.stat, file.entry?.rev ?? 0);
+            state.transaction(() => state.manifest.runNow(seen));
+        }
+        return {
+            store: source.store,
+            imported: 0,
+            skipped,
+            ms: performance.now() - started,
+            mode: "resync",
+        };
+    }
+    let imported = 0;
+    await clock.batch(() => state.transaction(() => {
+        if (resync) {
+            const outcome = reconcileRows(state, source.tables.map((table) => ({ table })), rows, file.status === "deleted" || file.status === "changed" ? (file.entry?.rev ?? 0) : 0, { tombstoned: TOMBSTONED_TABLES });
+            imported = outcome.applied + outcome.removed;
+            logConflicts(logger, source.store, source.filePath, outcome.conflicts);
+            if (file.status === "deleted") {
+                state.manifest.runNow(state.manifest.forgetStatement(source.filePath));
+            }
+            else if (outcome.conflicts === 0 &&
+                (file.status === "changed" || file.status === "created")) {
+                state.manifest.runNow(state.manifest.statement(source.store, source.filePath, file.content, file.stat, state.currentRev));
+            }
+        }
+        else {
+            writeDocRows(state, rows ?? []);
+            imported = rows?.length ?? 0;
+            if (file.status === "changed" || file.status === "created") {
+                state.manifest.runNow(state.manifest.statement(source.store, source.filePath, file.content, file.stat, state.currentRev));
+            }
+        }
+        state.putImportMarker(source.store, { status: "done", cursor: null, imported, skipped });
+    }));
+    return {
+        store: source.store,
+        imported,
+        skipped,
+        ms: performance.now() - started,
+        mode: resync ? "resync" : "import",
+    };
+}
+/** Parse one batch of a directory store on its first import. */
+async function readDirectoryBatch(state, source, names, logger) {
+    const rows = [];
+    const files = [];
+    let imported = 0;
+    let skipped = 0;
+    for (const name of names) {
+        const filePath = path.join(source.dir, name);
+        try {
+            const content = await fs.readFile(filePath, "utf8");
+            const stat = await fs.stat(filePath);
+            rows.push(...source.toRows(JSON.parse(content), name));
+            files.push({ name, content, size: stat.size, mtimeMs: stat.mtimeMs });
+            imported += 1;
+        }
+        catch (error) {
+            skipped += 1;
+            logger.warn({ err: error, filePath }, "Legacy record unreadable; not imported");
+        }
+    }
+    void state;
+    return { rows, files, imported, skipped };
+}
+async function listJson(dir) {
+    try {
+        return (await fs.readdir(dir))
+            .filter((name) => name.endsWith(".json") && !name.startsWith("."))
+            .sort();
+    }
+    catch {
+        return null;
+    }
+}
+async function importDirectory(state, source, clock, logger) {
+    const started = performance.now();
+    const marker = state.getImportMarker(source.store);
+    if (marker?.status === "done") {
+        return resyncDirectory(state, source, clock, logger, started);
+    }
+    const names = (await listJson(source.dir)) ?? [];
+    const cursor = marker?.cursor ?? null;
+    let imported = marker?.imported ?? 0;
+    let skipped = marker?.skipped ?? 0;
+    const todo = cursor === null ? names : names.filter((name) => name > cursor);
+    for (let index = 0; index < todo.length; index += AGENT_IMPORT_BATCH) {
+        const batch = todo.slice(index, index + AGENT_IMPORT_BATCH);
+        const read = await readDirectoryBatch(state, source, batch, logger);
+        imported += read.imported;
+        skipped += read.skipped;
+        const last = batch[batch.length - 1];
+        await clock.batch(() => {
+            const statements = docRowStatements(state, read.rows);
+            const rev = state.currentRev;
+            return state.commitBatch([
+                ...statements,
+                ...read.files.map((file) => state.manifest.statement(source.store, path.join(source.dir, file.name), file.content, file, rev)),
+                state.importMarkerStatement(source.store, {
+                    status: "running",
+                    cursor: last,
+                    imported,
+                    skipped,
+                }),
+            ]);
+        });
+    }
+    state.transaction(() => state.putImportMarker(source.store, { status: "done", cursor: null, imported, skipped }));
+    return {
+        store: source.store,
+        imported,
+        skipped,
+        ms: performance.now() - started,
+        mode: "import",
+    };
+}
+/**
+ * Directory store after the first import: classify every file on disk and
+ * every manifest entry, reconcile the foreign ones. A missing directory is
+ * treated as "nothing to say" rather than "everything deleted".
+ */
+async function resyncDirectory(state, source, clock, logger, started) {
+    const names = await listJson(source.dir);
+    const entries = state.manifest.entries(source.store);
+    const paths = new Set((names ?? []).map((name) => path.join(source.dir, name)));
+    if (names !== null) {
+        for (const entryPath of entries.keys())
+            paths.add(entryPath);
+    }
+    const states = await mapLimit([...paths].sort(), 32, async (filePath) => ({
+        filePath,
+        state: await state.manifest.classify(filePath, entries.get(filePath) ?? null),
+    }));
+    let imported = 0;
+    let skipped = 0;
+    const foreign = states.filter((item) => item.state.status !== "ours" && item.state.status !== "absent");
+    // Catch-up: a file we wrote whose rows the database has since deleted (the
+    // process died before the mirror's unlink ran). Remove it, so a rollback
+    // does not bring the deleted entry back.
+    if (state.mirror.enabled) {
+        for (const { filePath, state: file } of states) {
+            if (file.status !== "ours")
+                continue;
+            const owned = ownedRows(state, source.owns(path.basename(filePath)), undefined);
+            const live = [...owned.values()].some(({ row }) => row.deletedAt === null);
+            if (!live) {
+                await fs.rm(filePath, { force: true });
+                await state.manifest.forget(filePath);
+            }
+        }
+    }
+    for (let index = 0; index < foreign.length; index += AGENT_IMPORT_BATCH) {
+        const batch = foreign.slice(index, index + AGENT_IMPORT_BATCH);
+        await clock.batch(() => state.transaction(() => {
+            for (const { filePath, state: file } of batch) {
+                const name = path.basename(filePath);
+                let rows = null;
+                if (file.status === "changed" || file.status === "created") {
+                    try {
+                        rows = source.toRows(JSON.parse(file.content), name);
+                    }
+                    catch (error) {
+                        skipped += 1;
+                        logger.warn({ err: error, filePath }, "Legacy record unreadable; not imported");
+                        state.manifest.runNow(state.manifest.statement(source.store, filePath, file.content, file.stat, file.entry?.rev ?? 0));
+                        continue;
+                    }
+                }
+                const mirroredRev = "entry" in file ? (file.entry?.rev ?? 0) : 0;
+                const outcome = reconcileRows(state, source.owns(name), rows, mirroredRev, {
+                    deleteWins: source.deleteWins,
+                });
+                imported += outcome.applied + outcome.removed > 0 ? 1 : 0;
+                logConflicts(logger, source.store, filePath, outcome.conflicts);
+                if (file.status === "deleted") {
+                    state.manifest.runNow(state.manifest.forgetStatement(filePath));
+                }
+                else if (outcome.conflicts === 0 &&
+                    (file.status === "changed" || file.status === "created")) {
+                    state.manifest.runNow(state.manifest.statement(source.store, filePath, file.content, file.stat, state.currentRev));
+                }
+            }
+        }));
+    }
+    return {
+        store: source.store,
+        imported,
+        skipped,
+        ms: performance.now() - started,
+        mode: foreign.length > 0 ? "resync" : "skip",
+    };
+}
+/**
+ * Validate one batch of a first import and build its upserts; newer
+ * `updatedAt` wins within the batch and against what an earlier batch stored
+ * (a cwd move that left the old file behind). Every file read is recorded in
+ * the manifest.
+ */
+function agentBatchStatements(state, table, contents, logger) {
+    let imported = 0;
+    let skipped = 0;
+    const chosen = new Map();
+    const read = [];
+    for (const item of contents) {
+        let record;
+        try {
+            if (item.content === undefined)
+                throw item.error;
+            record = parseStoredAgentRecord(JSON.parse(item.content));
+        }
+        catch (error) {
+            skipped += 1;
+            logger.warn({ err: error, filePath: item.filePath }, "Skipping invalid agent record");
+            // Remembered as seen, so it is only read again once someone changes it.
+            if (item.content !== undefined) {
+                read.push({
+                    filePath: item.filePath,
+                    content: item.content,
+                    size: item.size,
+                    mtimeMs: item.mtimeMs,
+                });
+            }
+            continue;
+        }
+        read.push({
+            filePath: item.filePath,
+            content: item.content,
+            size: item.size,
+            mtimeMs: item.mtimeMs,
+        });
+        const inBatch = chosen.get(record.id);
+        const stored = inBatch ? null : table.getInTransaction("", record.id);
+        const current = inBatch ?? (stored === null ? null : JSON.parse(stored));
+        if (current && current.updatedAt >= record.updatedAt) {
+            continue;
+        }
+        chosen.set(record.id, record);
+        imported += 1;
+    }
+    const statements = [...chosen.values()].map((record) => table.putStatement("", record.id, JSON.stringify(record)));
+    const rev = state.currentRev;
+    for (const file of read) {
+        statements.push(state.manifest.statement("agents", file.filePath, file.content, file, rev));
+    }
+    return { statements, imported, skipped };
+}
+function agentIdOf(filePath) {
+    return path.basename(filePath, ".json");
+}
+/** Classify every agent file and manifest entry, grouped by agent id. */
+async function classifyAgentFiles(state, baseDir, logger) {
+    const entries = state.manifest.entries("agents");
+    const dirPresent = await exists(baseDir);
+    const paths = new Set(dirPresent ? await listAgentRecordFiles(baseDir) : []);
+    if (dirPresent) {
+        for (const entryPath of entries.keys())
+            paths.add(entryPath);
+    }
+    else if (entries.size > 0) {
+        logger.warn({ baseDir }, "Agent directory missing; not treating its files as deleted");
+    }
+    const classified = await mapLimit([...paths], 64, async (filePath) => ({
+        filePath,
+        file: await state.manifest.classify(filePath, entries.get(filePath) ?? null),
+    }));
+    const byId = new Map();
+    const invalid = [];
+    let skipped = 0;
+    for (const { filePath, file } of classified) {
+        const id = agentIdOf(filePath);
+        const current = byId.get(id) ?? { foreign: [], deleted: [], live: 0, mirroredRev: 0 };
+        byId.set(id, current);
+        const entry = "entry" in file ? file.entry : null;
+        if (entry)
+            current.mirroredRev = Math.max(current.mirroredRev, entry.rev);
+        if (file.status === "ours")
+            current.live += 1;
+        if (file.status === "deleted")
+            current.deleted.push(filePath);
+        if (file.status !== "changed" && file.status !== "created")
+            continue;
+        try {
+            const record = parseStoredAgentRecord(JSON.parse(file.content));
+            current.foreign.push({ filePath, record, file });
+            current.live += 1;
+        }
+        catch (error) {
+            skipped += 1;
+            logger.warn({ err: error, filePath }, "Skipping invalid agent record");
+            invalid.push(state.manifest.statement("agents", filePath, file.content, file.stat, file.entry?.rev ?? 0));
+        }
+    }
+    return { byId, invalid, skipped };
+}
+/**
+ * The statements that reconcile one agent id. Returns whether a row changed
+ * and whether the database won a conflict.
+ */
+function reconcileAgent(state, table, id, current, statements) {
+    const row = table.storedRows({ scope: "", id })[0] ?? null;
+    const sqliteWins = row !== null && row.rev > current.mirroredRev;
+    let changed = false;
+    let conflict = false;
+    if (current.foreign.length > 0) {
+        const best = current.foreign.reduce((left, right) => right.record.updatedAt > left.record.updatedAt ||
+            (right.record.updatedAt === left.record.updatedAt &&
+                right.file.stat.mtimeMs > left.file.stat.mtimeMs)
+            ? right
+            : left);
+        const body = JSON.stringify(best.record);
+        const same = row !== null && row.deletedAt === null && row.body === body;
+        if (sqliteWins && !same) {
+            conflict = true;
+        }
+        else {
+            if (!same) {
+                statements.push(table.putStatement("", id, body));
+                changed = true;
+            }
+            const rev = state.currentRev;
+            for (const item of current.foreign) {
+                statements.push(state.manifest.statement("agents", item.filePath, item.file.content, item.file.stat, rev));
+            }
+        }
+    }
+    else if (current.live === 0 && row !== null && row.deletedAt === null) {
+        if (sqliteWins) {
+            conflict = true;
+        }
+        else {
+            statements.push(table.removeStatement("", id));
+            changed = true;
+        }
+    }
+    for (const filePath of current.deleted) {
+        statements.push(state.manifest.forgetStatement(filePath));
+    }
+    return { changed, conflict };
+}
+async function resyncAgents(state, baseDir, clock, logger, started, catchUp) {
+    const store = "agents";
+    const table = state.doc("agents", { tombstones: true });
+    const { byId, invalid, skipped } = await classifyAgentFiles(state, baseDir, logger);
+    const statements = [...invalid];
+    let imported = 0;
+    let conflicts = 0;
+    const flush = async () => {
+        if (statements.length === 0)
+            return;
+        const batch = statements.splice(0);
+        await clock.batch(() => state.commitBatch(batch));
+    };
+    for (const [id, current] of byId) {
+        if (current.foreign.length === 0 && current.deleted.length === 0)
+            continue;
+        const outcome = reconcileAgent(state, table, id, current, statements);
+        if (outcome.changed)
+            imported += 1;
+        if (outcome.conflict)
+            conflicts += 1;
+        if (statements.length >= AGENT_IMPORT_BATCH)
+            await flush();
+    }
+    await flush();
+    if (conflicts > 0) {
+        logger.info({ store, conflicts }, "Agent files changed by someone else; agents the daemon changed after writing them keep the daemon's version");
+    }
+    collectAgentCatchUp(state, table, catchUp);
+    return {
+        store,
+        imported,
+        skipped,
+        ms: performance.now() - started,
+        mode: imported > 0 || skipped > 0 || conflicts > 0 ? "resync" : "skip",
+    };
+}
+/**
+ * Rows whose legacy file lags the database: the process died inside the
+ * mirror's debounce window. Rewritten (or unlinked) at boot, so a later
+ * rollback does not land on files from before the crash.
+ */
+function collectAgentCatchUp(state, table, catchUp) {
+    const byId = new Map();
+    for (const entry of state.manifest.entries("agents").values()) {
+        const id = agentIdOf(entry.path);
+        const current = byId.get(id) ?? { rev: 0, paths: [] };
+        current.rev = Math.max(current.rev, entry.rev);
+        current.paths.push(entry.path);
+        byId.set(id, current);
+    }
+    for (const row of table.storedRows({ scope: "" })) {
+        const mirrored = byId.get(row.id);
+        if (row.deletedAt !== null) {
+            if (mirrored && row.rev > mirrored.rev)
+                catchUp.unlink.push(...mirrored.paths);
+            continue;
+        }
+        if (!mirrored || row.rev > mirrored.rev)
+            catchUp.rewrite.push(row.id);
+    }
+}
+/**
+ * Agents: a directory of project directories. The first import is resumable
+ * by sorted path; when the same agent id shows up twice (a cwd move that left
+ * the old file behind), the newer `updatedAt` wins. Afterwards, see `resyncAgents`.
+ */
+export async function importLegacyAgents(state, baseDir, clock, logger, catchUp = { rewrite: [], unlink: [] }) {
+    const store = "agents";
+    const started = performance.now();
+    const marker = state.getImportMarker(store);
+    if (marker?.status === "done") {
+        return resyncAgents(state, baseDir, clock, logger, started, catchUp);
+    }
+    const files = (await listAgentRecordFiles(baseDir)).sort();
+    const cursor = marker?.cursor ?? null;
+    const todo = cursor === null ? files : files.filter((file) => file > cursor);
+    let imported = marker?.imported ?? 0;
+    let skipped = marker?.skipped ?? 0;
+    const table = state.doc("agents", { tombstones: true });
+    for (let index = 0; index < todo.length; index += AGENT_IMPORT_BATCH) {
+        const batch = todo.slice(index, index + AGENT_IMPORT_BATCH);
+        const contents = await Promise.all(batch.map(async (filePath) => {
+            try {
+                const content = await fs.readFile(filePath, "utf8");
+                const stat = await fs.stat(filePath);
+                return { filePath, content, size: stat.size, mtimeMs: stat.mtimeMs };
+            }
+            catch (error) {
+                return { filePath, error };
+            }
+        }));
+        await clock.batch(() => {
+            const built = agentBatchStatements(state, table, contents, logger);
+            imported += built.imported;
+            skipped += built.skipped;
+            return state.commitBatch([
+                ...built.statements,
+                state.importMarkerStatement(store, {
+                    status: "running",
+                    cursor: batch[batch.length - 1],
+                    imported,
+                    skipped,
+                }),
+            ]);
+        });
+    }
+    state.transaction(() => state.putImportMarker(store, { status: "done", cursor: null, imported, skipped }));
+    return { store, imported, skipped, ms: performance.now() - started, mode: "import" };
+}
+/**
+ * How long a deleted row keeps its tombstone. A tombstone matters for as long
+ * as a stale legacy file could still come back to contradict it: a rollback to
+ * the file-based release and a re-upgrade, or a crash before the mirror's
+ * unlink. A week covers the updater's rollback and any realistic
+ * downgrade-and-return; past it the tombstone is only boot-time cost (every
+ * create/delete cycle of a short-lived agent leaves one). Revisit when the
+ * delta protocol exposes `rev` to clients: then keep them past the oldest
+ * client cursor too.
+ */
+export const TOMBSTONE_RETENTION_MS = 7 * 24 * 60 * 60 * 1000;
+/**
+ * Purge tombstones past `TOMBSTONE_RETENTION_MS`. An agent whose legacy file
+ * the manifest still tracks keeps its tombstone: that file is the one thing
+ * that could resurrect it, and the boot catch-up still owes its unlink.
+ */
+function purgeOldTombstones(state, logger) {
+    const before = Date.now() - TOMBSTONE_RETENTION_MS;
+    const tracked = new Set([...state.manifest.entries("agents").keys()].map((filePath) => agentIdOf(filePath)));
+    let purged = 0;
+    for (const table of TOMBSTONED_TABLES) {
+        purged += state.doc(table).purgeTombstones(before, table === "agents" ? tracked : new Set());
+    }
+    if (purged > 0)
+        logger.info({ purged }, "Purged old tombstones from state.sqlite");
+}
+/**
+ * Import (or re-sync) every legacy store. Never throws for a single store: a
+ * store that fails is logged and left for the next boot, its marker still
+ * "running" so it resumes where it stopped.
+ */
+export async function importLegacyState(state, options) {
+    const started = performance.now();
+    const clock = new BatchClock();
+    const results = [];
+    const logger = options.logger;
+    state.pauseCheckpoints();
+    const run = async (store, fn) => {
+        try {
+            results.push(await fn());
+        }
+        catch (error) {
+            logger.error({ err: error, store }, "Legacy import failed; will resume on next boot");
+        }
+    };
+    const agentCatchUp = { rewrite: [], unlink: [] };
+    await run("agents", () => importLegacyAgents(state, options.agentStoragePath, clock, logger, agentCatchUp));
+    for (const source of options.sources) {
+        await run(source.store, () => {
+            switch (source.kind) {
+                case "whole-file":
+                    return importWholeFile(state, source, clock, logger);
+                case "directory":
+                    return importDirectory(state, source, clock, logger);
+            }
+        });
+    }
+    state.resumeCheckpoints();
+    purgeOldTombstones(state, logger);
+    const sorted = [...clock.batchMs].sort((left, right) => left - right);
+    return {
+        totalMs: performance.now() - started,
+        maxBatchMs: clock.maxBatchMs,
+        p99BatchMs: sorted[Math.floor(sorted.length * 0.99)] ?? 0,
+        batches: sorted.length,
+        stores: results,
+        agentCatchUp,
+    };
+}
+//# sourceMappingURL=legacy-import.js.map