@hyperdrive.bot/fleet-server 0.3.164 → 0.3.165

package/dist/server/server/state/legacy-manifest.d.ts

@@ -0,0 +1,73 @@
+import { type Stats } from "node:fs";
+import type { StateDb } from "./state-db.js";
+import type { WorkerStatement } from "./state-worker.js";
+/**
+ * What this daemon last left in each legacy JSON file.
+ *
+ * Every file the mirror writes, and every file the import reads, gets a row:
+ * content hash, size, mtime, and the daemon-wide `rev` the file reflects. That
+ * row is how the daemon tells its own files from everyone else's, file by file,
+ * with no global clock:
+ *
+ *   - the file matches its row (same mtime and size, or same content hash):
+ *     ours, never re-imported, whatever its mtime says;
+ *   - the file differs from its row, or has no row: changed by someone else (a
+ *     rolled-back older daemon, the daemon itself under PASEO_STATE_SQLITE=0, a
+ *     hand edit), reconciled into the database by content;
+ *   - the row's file is gone: deleted by someone else.
+ *
+ * `rev` settles conflicts: a database row whose rev is greater than the
+ * manifest rev of its file changed after that file was written, so on a
+ * conflict the database wins (and the next mirror write brings the file up).
+ *
+ * mtime + size is only a fast path to skip hashing a file that was not
+ * touched; a differing mtime alone never makes a file foreign.
+ */
+export interface ManifestEntry {
+    /** Absolute path. */
+    path: string;
+    store: string;
+    hash: string;
+    size: number;
+    mtimeMs: number;
+    rev: number;
+}
+export type LegacyFileState = {
+    status: "ours";
+    entry: ManifestEntry;
+} | {
+    status: "changed" | "created";
+    entry: ManifestEntry | null;
+    content: string;
+    stat: Stats;
+} | {
+    status: "deleted";
+    entry: ManifestEntry;
+} | {
+    status: "absent";
+};
+export declare function hashContent(content: string | Buffer): string;
+export declare class LegacyManifest {
+    private readonly state;
+    private readonly home;
+    constructor(state: StateDb, home: string);
+    private key;
+    private toEntry;
+    get(filePath: string): ManifestEntry | null;
+    /** Every entry of a store, keyed by absolute path. */
+    entries(store: string): Map<string, ManifestEntry>;
+    /** The upsert for a file whose `content` was just read or written, as a batch statement. */
+    statement(store: string, filePath: string, content: string | Buffer, stat: Pick<Stats, "size" | "mtimeMs">, rev: number): WorkerStatement;
+    forgetStatement(filePath: string): WorkerStatement;
+    /** Run a statement inside the caller's open transaction. */
+    runNow(statement: WorkerStatement): void;
+    /**
+     * Record the file as this daemon just left it, reflecting `rev`. Reads the
+     * file back, so the hash is exactly what is on disk. Resolves once committed.
+     */
+    record(store: string, filePath: string, rev: number): Promise<void>;
+    forget(filePath: string): Promise<void>;
+    /** Compare a file on disk with its entry (looked up when not given). */
+    classify(filePath: string, entry?: ManifestEntry | null): Promise<LegacyFileState>;
+}
+//# sourceMappingURL=legacy-manifest.d.ts.map

package/dist/server/server/state/legacy-manifest.js

@@ -0,0 +1,112 @@
+import { createHash } from "node:crypto";
+import { promises as fs } from "node:fs";
+import path from "node:path";
+export function hashContent(content) {
+    return createHash("sha1").update(content).digest("hex");
+}
+const UPSERT_SQL = `INSERT INTO legacy_manifest (path, store, hash, size, mtime_ms, rev)
+  VALUES (?, ?, ?, ?, ?, ?)
+  ON CONFLICT(path) DO UPDATE SET store = excluded.store, hash = excluded.hash,
+    size = excluded.size, mtime_ms = excluded.mtime_ms, rev = excluded.rev`;
+export class LegacyManifest {
+    constructor(state, home) {
+        this.state = state;
+        this.home = home;
+    }
+    key(filePath) {
+        return path.relative(this.home, path.resolve(filePath));
+    }
+    toEntry(row) {
+        return {
+            path: path.resolve(this.home, String(row.path)),
+            store: String(row.store),
+            hash: String(row.hash),
+            size: Number(row.size),
+            mtimeMs: Number(row.mtime_ms),
+            rev: Number(row.rev),
+        };
+    }
+    get(filePath) {
+        this.state.flushSync();
+        const row = this.state
+            .prepare("SELECT * FROM legacy_manifest WHERE path = ?")
+            .get(this.key(filePath));
+        return row ? this.toEntry(row) : null;
+    }
+    /** Every entry of a store, keyed by absolute path. */
+    entries(store) {
+        this.state.flushSync();
+        const rows = this.state.prepare("SELECT * FROM legacy_manifest WHERE store = ?").all(store);
+        return new Map(rows.map((row) => [path.resolve(this.home, String(row.path)), this.toEntry(row)]));
+    }
+    /** The upsert for a file whose `content` was just read or written, as a batch statement. */
+    statement(store, filePath, content, stat, rev) {
+        return {
+            sql: UPSERT_SQL,
+            params: [this.key(filePath), store, hashContent(content), stat.size, stat.mtimeMs, rev],
+        };
+    }
+    forgetStatement(filePath) {
+        return { sql: "DELETE FROM legacy_manifest WHERE path = ?", params: [this.key(filePath)] };
+    }
+    /** Run a statement inside the caller's open transaction. */
+    runNow(statement) {
+        this.state.prepare(statement.sql).run(...statement.params);
+    }
+    /**
+     * Record the file as this daemon just left it, reflecting `rev`. Reads the
+     * file back, so the hash is exactly what is on disk. Resolves once committed.
+     */
+    async record(store, filePath, rev) {
+        let content;
+        let stat;
+        try {
+            content = await fs.readFile(filePath);
+            stat = await fs.stat(filePath);
+        }
+        catch {
+            await this.forget(filePath);
+            return;
+        }
+        const statement = this.statement(store, filePath, content, stat, rev);
+        await this.state.enqueue(() => this.runNow(statement));
+    }
+    async forget(filePath) {
+        const statement = this.forgetStatement(filePath);
+        await this.state.enqueue(() => this.runNow(statement));
+    }
+    /** Compare a file on disk with its entry (looked up when not given). */
+    async classify(filePath, entry) {
+        const known = entry === undefined ? this.get(filePath) : entry;
+        let stat;
+        try {
+            stat = await fs.stat(filePath);
+        }
+        catch {
+            return known ? { status: "deleted", entry: known } : { status: "absent" };
+        }
+        if (known && known.size === stat.size && known.mtimeMs === stat.mtimeMs) {
+            return { status: "ours", entry: known };
+        }
+        let content;
+        try {
+            content = await fs.readFile(filePath, "utf8");
+        }
+        catch {
+            return known ? { status: "deleted", entry: known } : { status: "absent" };
+        }
+        if (known && hashContent(content) === known.hash) {
+            // Same bytes, new mtime (a copy, a touch): still ours. Remember the mtime
+            // so the next look is a stat again.
+            const refreshed = this.statement(known.store, filePath, content, stat, known.rev);
+            void this.state
+                .enqueue(() => this.runNow(refreshed))
+                .catch((error) => {
+                this.state.logger.warn({ err: error, filePath }, "Failed to refresh a legacy manifest entry");
+            });
+            return { status: "ours", entry: known };
+        }
+        return { status: known ? "changed" : "created", entry: known, content, stat };
+    }
+}
+//# sourceMappingURL=legacy-manifest.js.map

package/dist/server/server/state/legacy-mirror.d.ts

@@ -0,0 +1,57 @@
+import type { Logger } from "pino";
+/**
+ * Keeps the legacy JSON files written after `state.sqlite` took over, for one
+ * release, so that:
+ *   - the updater's automatic rollback (cli daemon/update.ts, 180s readiness
+ *     budget) lands on an older daemon that reads CURRENT files, not a frozen
+ *     snapshot from the upgrade moment;
+ *   - scripts that read the files directly keep working (week-report
+ *     extract.py reads agents/, loops run-agent.sh reads
+ *     projects/workspaces.json, paseo-new-host provision.sh greps projects.json).
+ *
+ * Every write is off the request path: debounced per key (with a max wait, so
+ * a hot key still flushes), latest task wins, serialized per key, failures
+ * logged and never thrown. On SIGTERM/SIGINT the daemon switches the mirror to
+ * immediate mode and flushes before anything else shuts down; only a SIGKILL
+ * (or a power cut) inside the last debounce window, at most about a second,
+ * can leave a file behind the database. The next boot of this daemon catches
+ * those files up. Remove in the release after next (see
+ * docs/architecture/data-layer-rework.md section 5).
+ */
+export declare class LegacyMirror {
+    readonly enabled: boolean;
+    private readonly logger;
+    private readonly debounceMs;
+    private readonly maxWaitMs;
+    private readonly queued;
+    private readonly running;
+    private immediate;
+    constructor(options: {
+        enabled: boolean;
+        logger: Logger;
+        debounceMs?: number;
+        maxWaitMs?: number;
+    });
+    /**
+     * Debounce `task` under `key`; a later call for the same key replaces it.
+     * The first call of a burst fixes a deadline `maxWaitMs` out, which later
+     * calls never push back.
+     */
+    write(key: string, task: () => Promise<void>): void;
+    /** True while a write for `key` is debounced or still running. */
+    isPending(key: string): boolean;
+    /** True while a write for `key` is running (not merely debounced). */
+    isRunning(key: string): boolean;
+    /**
+     * Shutdown: run everything queued now, and every later write without a
+     * debounce, so nothing written on the way down waits on a timer.
+     */
+    beginShutdown(): Promise<void>;
+    /** Drop queued writes without running them. Tests use it to simulate a crash. */
+    discard(): void;
+    get pendingCount(): number;
+    /** Run every debounced task now and wait for all of them. */
+    flush(): Promise<void>;
+    private run;
+}
+//# sourceMappingURL=legacy-mirror.d.ts.map

package/dist/server/server/state/legacy-mirror.js

@@ -0,0 +1,114 @@
+const DEFAULT_DEBOUNCE_MS = 250;
+/**
+ * A key written more often than the debounce still reaches its file at least
+ * this often (or every four debounces, when that is sooner), so a streaming agent's legacy file is never more than about a
+ * second behind.
+ */
+const DEFAULT_MAX_WAIT_MS = 1000;
+/**
+ * Keeps the legacy JSON files written after `state.sqlite` took over, for one
+ * release, so that:
+ *   - the updater's automatic rollback (cli daemon/update.ts, 180s readiness
+ *     budget) lands on an older daemon that reads CURRENT files, not a frozen
+ *     snapshot from the upgrade moment;
+ *   - scripts that read the files directly keep working (week-report
+ *     extract.py reads agents/, loops run-agent.sh reads
+ *     projects/workspaces.json, paseo-new-host provision.sh greps projects.json).
+ *
+ * Every write is off the request path: debounced per key (with a max wait, so
+ * a hot key still flushes), latest task wins, serialized per key, failures
+ * logged and never thrown. On SIGTERM/SIGINT the daemon switches the mirror to
+ * immediate mode and flushes before anything else shuts down; only a SIGKILL
+ * (or a power cut) inside the last debounce window, at most about a second,
+ * can leave a file behind the database. The next boot of this daemon catches
+ * those files up. Remove in the release after next (see
+ * docs/architecture/data-layer-rework.md section 5).
+ */
+export class LegacyMirror {
+    constructor(options) {
+        this.queued = new Map();
+        this.running = new Map();
+        this.immediate = false;
+        this.enabled = options.enabled;
+        this.logger = options.logger;
+        this.debounceMs = options.debounceMs ?? DEFAULT_DEBOUNCE_MS;
+        this.maxWaitMs = Math.max(this.debounceMs, options.maxWaitMs ?? Math.min(DEFAULT_MAX_WAIT_MS, 4 * this.debounceMs));
+    }
+    /**
+     * Debounce `task` under `key`; a later call for the same key replaces it.
+     * The first call of a burst fixes a deadline `maxWaitMs` out, which later
+     * calls never push back.
+     */
+    write(key, task) {
+        if (!this.enabled) {
+            return;
+        }
+        const now = Date.now();
+        const existing = this.queued.get(key);
+        if (existing) {
+            clearTimeout(existing.timer);
+        }
+        const firstQueuedAt = existing?.firstQueuedAt ?? now;
+        const delay = this.immediate
+            ? 0
+            : Math.max(0, Math.min(this.debounceMs, firstQueuedAt + this.maxWaitMs - now));
+        const timer = setTimeout(() => this.run(key), delay);
+        timer.unref?.();
+        this.queued.set(key, { task, timer, firstQueuedAt });
+    }
+    /** True while a write for `key` is debounced or still running. */
+    isPending(key) {
+        return this.queued.has(key) || this.running.has(key);
+    }
+    /** True while a write for `key` is running (not merely debounced). */
+    isRunning(key) {
+        return this.running.has(key);
+    }
+    /**
+     * Shutdown: run everything queued now, and every later write without a
+     * debounce, so nothing written on the way down waits on a timer.
+     */
+    beginShutdown() {
+        this.immediate = true;
+        return this.flush();
+    }
+    /** Drop queued writes without running them. Tests use it to simulate a crash. */
+    discard() {
+        for (const entry of this.queued.values())
+            clearTimeout(entry.timer);
+        this.queued.clear();
+    }
+    get pendingCount() {
+        return this.queued.size + this.running.size;
+    }
+    /** Run every debounced task now and wait for all of them. */
+    async flush() {
+        for (const [key, entry] of Array.from(this.queued)) {
+            clearTimeout(entry.timer);
+            this.run(key);
+        }
+        while (this.running.size > 0) {
+            await Promise.allSettled(this.running.values());
+        }
+    }
+    run(key) {
+        const entry = this.queued.get(key);
+        if (!entry) {
+            return;
+        }
+        this.queued.delete(key);
+        const previous = this.running.get(key) ?? Promise.resolve();
+        const next = previous
+            .then(() => entry.task())
+            .catch((error) => {
+            this.logger.warn({ err: error, key }, "Legacy file mirror write failed");
+        })
+            .finally(() => {
+            if (this.running.get(key) === next) {
+                this.running.delete(key);
+            }
+        });
+        this.running.set(key, next);
+    }
+}
+//# sourceMappingURL=legacy-mirror.js.map

package/dist/server/server/state/legacy-sources.d.ts

@@ -0,0 +1,9 @@
+import type { LegacySource } from "./legacy-import.js";
+/**
+ * Every legacy store `state.sqlite` takes over, except agents (imported by
+ * `importLegacyAgents` from the agent storage directory). Each parser is the
+ * store's own zod schema, so an import accepts exactly what the file store
+ * would have loaded.
+ */
+export declare function legacySources(paseoHome: string): LegacySource[];
+//# sourceMappingURL=legacy-sources.d.ts.map

package/dist/server/server/state/legacy-sources.js

@@ -0,0 +1,163 @@
+import path from "node:path";
+import { LoopDecisionSchema } from "@hyperdrive.bot/fleet-protocol/fleet/decisions";
+import { StoredScheduleSchema } from "@hyperdrive.bot/fleet-protocol/schedule/types";
+import { ChatStorePayloadSchema } from "../chat/chat-service.js";
+import { LOOP_LOGS_RETAINED, StoredLoopsSchema } from "../loop-service.js";
+import { capScheduleRuns } from "../schedule/store.js";
+import { PersistedProjectRecordListSchema, PersistedWorkspaceRecordListSchema, } from "../workspace-registry.js";
+/**
+ * Every legacy store `state.sqlite` takes over, except agents (imported by
+ * `importLegacyAgents` from the agent storage directory). Each parser is the
+ * store's own zod schema, so an import accepts exactly what the file store
+ * would have loaded.
+ */
+export function legacySources(paseoHome) {
+    return [
+        {
+            kind: "directory",
+            store: "schedules",
+            dir: path.join(paseoHome, "schedules"),
+            deleteWins: true,
+            owns: (fileName) => {
+                const id = fileName.slice(0, -".json".length);
+                return [
+                    { table: "schedules", scope: "", id },
+                    { table: "schedule_runs", scope: id },
+                ];
+            },
+            toRows: (json) => {
+                const { runs, ...schedule } = capScheduleRuns(StoredScheduleSchema.parse(json));
+                return [
+                    { table: "schedules", scope: "", id: schedule.id, value: schedule },
+                    ...runs.map((run, index) => ({
+                        table: "schedule_runs",
+                        scope: schedule.id,
+                        id: run.id,
+                        value: run,
+                        ord: index,
+                    })),
+                ];
+            },
+        },
+        {
+            kind: "whole-file",
+            store: "loops",
+            filePath: path.join(paseoHome, "loops", "loops.json"),
+            tables: ["loops", "loop_logs"],
+            toRows: (json) => {
+                const rows = [];
+                for (const { logs, ...loop } of StoredLoopsSchema.parse(json)) {
+                    rows.push({ table: "loops", scope: "", id: loop.id, value: loop });
+                    logs.slice(-LOOP_LOGS_RETAINED).forEach((entry, index) => {
+                        rows.push({
+                            table: "loop_logs",
+                            scope: loop.id,
+                            id: String(entry.seq).padStart(10, "0"),
+                            value: entry,
+                            ord: index,
+                        });
+                    });
+                }
+                return rows;
+            },
+        },
+        {
+            kind: "whole-file",
+            store: "chat",
+            filePath: path.join(paseoHome, "chat", "rooms.json"),
+            tables: ["chat_rooms", "chat_messages"],
+            toRows: (json) => {
+                const payload = ChatStorePayloadSchema.parse(json);
+                const order = new Map();
+                return [
+                    ...payload.rooms.map((room) => ({
+                        table: "chat_rooms",
+                        scope: "",
+                        id: room.id,
+                        value: room,
+                    })),
+                    ...payload.messages.map((message) => {
+                        const ord = order.get(message.roomId) ?? 0;
+                        order.set(message.roomId, ord + 1);
+                        return {
+                            table: "chat_messages",
+                            scope: message.roomId,
+                            id: message.id,
+                            value: message,
+                            ord,
+                        };
+                    }),
+                ];
+            },
+        },
+        {
+            kind: "whole-file",
+            store: "projects",
+            filePath: path.join(paseoHome, "projects", "projects.json"),
+            tables: ["projects"],
+            toRows: (json) => PersistedProjectRecordListSchema.parse(json).map((record) => ({
+                table: "projects",
+                scope: "",
+                id: record.projectId,
+                value: record,
+            })),
+        },
+        {
+            kind: "whole-file",
+            store: "workspaces",
+            filePath: path.join(paseoHome, "projects", "workspaces.json"),
+            tables: ["workspaces"],
+            toRows: (json) => PersistedWorkspaceRecordListSchema.parse(json).map((record) => ({
+                table: "workspaces",
+                scope: "",
+                id: record.workspaceId,
+                value: record,
+            })),
+        },
+        {
+            kind: "whole-file",
+            store: "push_tokens",
+            filePath: path.join(paseoHome, "push-tokens.json"),
+            tables: ["push_tokens"],
+            toRows: (json) => {
+                const tokens = json?.tokens;
+                return (Array.isArray(tokens) ? tokens : [])
+                    .filter((token) => typeof token === "string" && token.trim() !== "")
+                    .map((token) => ({
+                    table: "push_tokens",
+                    scope: "",
+                    id: token.trim(),
+                    value: { token: token.trim() },
+                }));
+            },
+        },
+        {
+            kind: "directory",
+            store: "fleet_decisions",
+            dir: path.join(paseoHome, "fleet", "decisions"),
+            owns: (fileName) => [
+                { table: "fleet_decisions", scope: "", id: fileName.slice(0, -".json".length) },
+            ],
+            toRows: (json) => {
+                const decision = LoopDecisionSchema.parse(json);
+                return [{ table: "fleet_decisions", scope: "", id: decision.id, value: decision }];
+            },
+        },
+        {
+            kind: "directory",
+            store: "extension_kv",
+            dir: path.join(paseoHome, "extension-storage"),
+            owns: (fileName) => [{ table: "extension_kv", scope: fileName.slice(0, -".json".length) }],
+            toRows: (json, fileName) => {
+                const extensionId = fileName.slice(0, -".json".length);
+                return Object.entries((json ?? {})).map(([key, value]) => ({
+                    table: "extension_kv",
+                    scope: extensionId,
+                    id: key,
+                    value: value ?? null,
+                }));
+            },
+        },
+    ];
+}
+//# sourceMappingURL=legacy-sources.js.map