@hyperdrive.bot/fleet-server 0.3.164 → 0.3.165

package/dist/server/server/state/daemon-state.js

@@ -0,0 +1,99 @@
+import { existsSync, promises as fs } from "node:fs";
+import path from "node:path";
+import { SqliteAgentRecordBackend } from "./agent-state.js";
+import { importLegacyState } from "./legacy-import.js";
+import { legacySources } from "./legacy-sources.js";
+import { attachStateDb, detachStateDb, openStateDb, STATE_DB_FILENAME, STATE_SQLITE_ENV, } from "./state-db.js";
+/**
+ * Free space the first import needs, as a multiple of the legacy state it
+ * copies: the database itself (about 1x on a real install) plus the WAL, which
+ * grows to about 1.4x while checkpoints are paused during the import.
+ */
+export const IMPORT_SPACE_FACTOR = 2.5;
+async function freeBytesAt(dir) {
+    const stats = await fs.statfs(dir);
+    return Number(stats.bavail) * Number(stats.bsize);
+}
+async function dirBytes(dir, depth) {
+    let total = 0;
+    let entries;
+    try {
+        entries = await fs.readdir(dir, { withFileTypes: true });
+    }
+    catch {
+        return 0;
+    }
+    for (const entry of entries) {
+        const full = path.join(dir, entry.name);
+        if (entry.isDirectory() && depth > 0)
+            total += await dirBytes(full, depth - 1);
+        else if (entry.isFile() && entry.name.endsWith(".json")) {
+            total += await fs.stat(full).then((stat) => stat.size, () => 0);
+        }
+    }
+    return total;
+}
+/** Bytes of every legacy store the first import copies. */
+async function legacyStateBytes(paseoHome, agentStoragePath) {
+    let total = await dirBytes(agentStoragePath, 1);
+    for (const source of legacySources(paseoHome)) {
+        total +=
+            source.kind === "directory"
+                ? await dirBytes(source.dir, 0)
+                : await fs.stat(source.filePath).then((stat) => stat.size, () => 0);
+    }
+    return total;
+}
+/**
+ * Open `state.sqlite` for a daemon, copy the legacy stores in (first boot) or
+ * re-sync files an older daemon changed (after a rollback), and attach it for
+ * the stores constructed deeper down. See data-layer-rework.md section 5.
+ */
+export async function openDaemonState(options) {
+    const { paseoHome, agentStoragePath, logger } = options;
+    const none = {
+        stateDb: null,
+        agentBackend: undefined,
+        close: async () => undefined,
+    };
+    const firstImport = !existsSync(path.join(paseoHome, STATE_DB_FILENAME));
+    if (firstImport && process.env[STATE_SQLITE_ENV] !== "0") {
+        const need = Math.ceil(IMPORT_SPACE_FACTOR * (await legacyStateBytes(paseoHome, agentStoragePath)));
+        const free = await (options.freeBytes ?? (() => freeBytesAt(paseoHome)))().catch(() => Infinity);
+        if (free < need) {
+            logger.warn({ freeBytes: free, neededBytes: need }, "Not enough free disk space to move daemon state into state.sqlite; staying on the legacy file stores");
+            return none;
+        }
+    }
+    const stateDb = openStateDb({ paseoHome, logger });
+    if (!stateDb) {
+        return none;
+    }
+    const report = await importLegacyState(stateDb, {
+        agentStoragePath,
+        sources: legacySources(paseoHome),
+        logger,
+    });
+    attachStateDb(paseoHome, stateDb);
+    logger.info({
+        elapsed: options.elapsed(),
+        importMs: Math.round(report.totalMs),
+        maxBatchMs: Math.round(report.maxBatchMs),
+        p99BatchMs: Math.round(report.p99BatchMs),
+        stores: report.stores
+            .filter((store) => store.mode !== "skip")
+            .map((store) => `${store.store}:${store.mode}:${store.imported}/${store.skipped}`),
+        mirror: stateDb.mirror.enabled,
+    }, "State store ready");
+    return {
+        stateDb,
+        agentBackend: new SqliteAgentRecordBackend(stateDb, agentStoragePath, logger, report.agentCatchUp),
+        close: async () => {
+            detachStateDb(paseoHome);
+            await stateDb.close().catch((error) => {
+                logger.warn({ err: error }, "Failed to close state.sqlite cleanly");
+            });
+        },
+    };
+}
+//# sourceMappingURL=daemon-state.js.map

package/dist/server/server/state/legacy-import.d.ts

@@ -0,0 +1,124 @@
+import type { Logger } from "pino";
+import type { StateDb } from "./state-db.js";
+import { type DocTableName } from "./state-schema.js";
+/**
+ * First-boot copy of the legacy JSON stores into `state.sqlite`.
+ *
+ * - COPY, never move: no legacy file is deleted or renamed, so a rollback to an
+ *   older daemon still finds everything (and the mirror keeps it current).
+ * - Batched and yielding: each batch is one transaction and then a
+ *   `setImmediate`, so the event loop is never held for long.
+ * - Resumable: each store keeps a marker whose cursor commits in the same
+ *   transaction as the rows it covers. A crash mid-import resumes after the
+ *   last committed batch; re-running a finished store is a no-op.
+ * - Re-sync by content, not by clock: every file read here or written by the
+ *   mirror has a row in the legacy manifest (`legacy-manifest.ts`). At the next
+ *   boot a file is foreign only when its content no longer matches that row; a
+ *   foreign file is reconciled row by row (the file wins, unless the database
+ *   row changed after the file was written), and a file that is gone deletes
+ *   what it held. Files this daemon wrote and nobody touched are never read.
+ */
+/**
+ * Files per import batch. Parsing and validating 50 agent records is about 5ms
+ * of main-thread work; the commit itself runs on the state worker.
+ */
+export declare const AGENT_IMPORT_BATCH = 50;
+export interface StoreImportResult {
+    store: string;
+    imported: number;
+    skipped: number;
+    ms: number;
+    /** "import" on a first copy, "resync" when re-reading changed files, "skip" when current. */
+    mode: "import" | "resync" | "skip";
+}
+export interface LegacyImportReport {
+    totalMs: number;
+    maxBatchMs: number;
+    p99BatchMs: number;
+    batches: number;
+    stores: StoreImportResult[];
+}
+export interface DocImportRow {
+    table: DocTableName;
+    scope: string;
+    id: string;
+    value: unknown;
+    ord?: number;
+}
+/** A store whose whole state is one legacy JSON file (loops.json, rooms.json, ...). */
+export interface WholeFileSource {
+    kind: "whole-file";
+    store: string;
+    filePath: string;
+    /** Tables this file fully describes; a re-sync removes rows the file no longer has. */
+    tables: DocTableName[];
+    toRows(json: unknown): DocImportRow[];
+}
+/** A store kept as one JSON file per entity in a directory. */
+export interface DirectorySource {
+    kind: "directory";
+    store: string;
+    dir: string;
+    toRows(json: unknown, fileName: string): DocImportRow[];
+    /** The rows one file fully describes (an omitted id means the whole scope). */
+    owns(fileName: string): RowOwnership[];
+    /** A foreign deletion beats a database edit made after the last mirror (schedules). */
+    deleteWins?: boolean;
+}
+export interface RowOwnership {
+    table: DocTableName;
+    scope?: string;
+    id?: string;
+}
+export type LegacySource = WholeFileSource | DirectorySource;
+/**
+ * Runs one batch and records how long it held the event loop: the synchronous
+ * part of `fn` (parsing, building statements, and the commit itself when there
+ * is no worker). A commit on the worker is awaited but not counted, because the
+ * loop is free while it runs.
+ */
+export declare class BatchClock {
+    maxBatchMs: number;
+    /** Every batch's loop-holding time, for percentiles in the boot log. */
+    readonly batchMs: number[];
+    batch<T>(fn: () => T | Promise<T>): Promise<T>;
+}
+/** What the agent import wants the storage backend to bring back in line with the database. */
+export interface AgentMirrorCatchUp {
+    /** Agent ids whose row changed after their legacy file was last written. */
+    rewrite: string[];
+    /** Legacy files of agents the database deleted after the file was written. */
+    unlink: string[];
+}
+/**
+ * Agents: a directory of project directories. The first import is resumable
+ * by sorted path; when the same agent id shows up twice (a cwd move that left
+ * the old file behind), the newer `updatedAt` wins. Afterwards, see `resyncAgents`.
+ */
+export declare function importLegacyAgents(state: StateDb, baseDir: string, clock: BatchClock, logger: Logger, catchUp?: AgentMirrorCatchUp): Promise<StoreImportResult>;
+/**
+ * How long a deleted row keeps its tombstone. A tombstone matters for as long
+ * as a stale legacy file could still come back to contradict it: a rollback to
+ * the file-based release and a re-upgrade, or a crash before the mirror's
+ * unlink. A week covers the updater's rollback and any realistic
+ * downgrade-and-return; past it the tombstone is only boot-time cost (every
+ * create/delete cycle of a short-lived agent leaves one). Revisit when the
+ * delta protocol exposes `rev` to clients: then keep them past the oldest
+ * client cursor too.
+ */
+export declare const TOMBSTONE_RETENTION_MS: number;
+export interface ImportLegacyStateOptions {
+    agentStoragePath: string;
+    sources: LegacySource[];
+    logger: Logger;
+}
+export interface LegacyImportResult extends LegacyImportReport {
+    agentCatchUp: AgentMirrorCatchUp;
+}
+/**
+ * Import (or re-sync) every legacy store. Never throws for a single store: a
+ * store that fails is logged and left for the next boot, its marker still
+ * "running" so it resumes where it stopped.
+ */
+export declare function importLegacyState(state: StateDb, options: ImportLegacyStateOptions): Promise<LegacyImportResult>;
+//# sourceMappingURL=legacy-import.d.ts.map