@wopr-network/defcon 1.11.0 → 1.13.0

package/dist/src/execution/cli.js

@@ -15,9 +15,11 @@ import { resolveCorsOrigin } from "../cors.js";
 import { DomainEventPersistAdapter } from "../engine/domain-event-adapter.js";
 import { Engine } from "../engine/engine.js";
 import { EventEmitter } from "../engine/event-emitter.js";
+import { buildConfigFromEnv, isLitestreamEnabled, LitestreamManager } from "../litestream/manager.js";
 import { withTransaction } from "../main.js";
 import { DrizzleDomainEventRepository } from "../repositories/drizzle/domain-event.repo.js";
 import { DrizzleEntityRepository } from "../repositories/drizzle/entity.repo.js";
+import { DrizzleEntitySnapshotRepository } from "../repositories/drizzle/entity-snapshot.repo.js";
 import { DrizzleEventRepository } from "../repositories/drizzle/event.repo.js";
 import { DrizzleFlowRepository } from "../repositories/drizzle/flow.repo.js";
 import { DrizzleGateRepository } from "../repositories/drizzle/gate.repo.js";
@@ -25,6 +27,8 @@ import { DrizzleInvocationRepository } from "../repositories/drizzle/invocation.
 import * as schema from "../repositories/drizzle/schema.js";
 import { entities, entityHistory, flowDefinitions, flowVersions, gateDefinitions, gateResults, invocations, stateDefinitions, transitionRules, } from "../repositories/drizzle/schema.js";
 import { DrizzleTransitionLogRepository } from "../repositories/drizzle/transition-log.repo.js";
+import { EventSourcedEntityRepository } from "../repositories/event-sourced/entity.repo.js";
+import { EventSourcedInvocationRepository } from "../repositories/event-sourced/invocation.repo.js";
 import { UiSseAdapter } from "../ui/sse.js";
 import { WebSocketBroadcaster } from "../ws/broadcast.js";
 import { createMcpServer, startStdioServer } from "./mcp-server.js";
@@ -158,13 +162,38 @@ program
     .option("--http-host <address>", "Host for HTTP REST API", "127.0.0.1")
     .option("--ui", "Enable built-in web UI at /ui")
     .action(async (opts) => {
+    // Litestream: restore from replica if DB missing and replication configured
+    let litestreamMgr;
+    if (isLitestreamEnabled()) {
+        const lsConfig = buildConfigFromEnv(opts.db);
+        litestreamMgr = new LitestreamManager(lsConfig);
+        litestreamMgr.restore();
+    }
     const { db, sqlite } = openDb(opts.db);
-    const entityRepo = new DrizzleEntityRepository(db);
+    // Litestream: start continuous replication
+    if (litestreamMgr) {
+        litestreamMgr.start();
+    }
+    const mutableEntityRepo = new DrizzleEntityRepository(db);
     const flowRepo = new DrizzleFlowRepository(db);
-    const invocationRepo = new DrizzleInvocationRepository(db);
+    const mutableInvocationRepo = new DrizzleInvocationRepository(db);
     const gateRepo = new DrizzleGateRepository(db);
     const transitionLogRepo = new DrizzleTransitionLogRepository(db);
     const domainEventRepo = new DrizzleDomainEventRepository(db);
+    const useEventSourced = process.env.DEFCON_EVENT_SOURCED === "true";
+    const snapshotInterval = parseInt(process.env.DEFCON_SNAPSHOT_INTERVAL ?? "10", 10);
+    let entityRepo;
+    let invocationRepo;
+    if (useEventSourced) {
+        const snapshotRepo = new DrizzleEntitySnapshotRepository(db);
+        entityRepo = new EventSourcedEntityRepository(mutableEntityRepo, domainEventRepo, snapshotRepo, snapshotInterval);
+        invocationRepo = new EventSourcedInvocationRepository(mutableInvocationRepo, domainEventRepo);
+        process.stderr.write("[defcon] Event-sourced repositories enabled\n");
+    }
+    else {
+        entityRepo = mutableEntityRepo;
+        invocationRepo = mutableInvocationRepo;
+    }
     const eventEmitter = new EventEmitter();
     eventEmitter.register({
         emit: async (event) => {
@@ -359,7 +388,12 @@ program
         });
         const shutdown = makeShutdownHandler({
             stopReaper,
-            closeables: [{ close: () => restHttpServer?.close() }, httpServer, sqlite],
+            closeables: [
+                ...(litestreamMgr ? [litestreamMgr] : []),
+                { close: () => restHttpServer?.close() },
+                httpServer,
+                sqlite,
+            ],
         });
         process.once("SIGINT", shutdown);
         process.once("SIGTERM", shutdown);
@@ -367,7 +401,10 @@ program
     else if (startMcp) {
         // stdio (default)
         console.error("Starting MCP server on stdio...");
-        const cleanup = makeShutdownHandler({ stopReaper, closeables: [sqlite] });
+        const cleanup = makeShutdownHandler({
+            stopReaper,
+            closeables: [...(litestreamMgr ? [litestreamMgr] : []), sqlite],
+        });
         process.once("SIGINT", cleanup);
         process.once("SIGTERM", cleanup);
         const mcpOpts = { adminToken, workerToken, stdioTrusted: true };
@@ -377,7 +414,7 @@ program
         // HTTP-only mode — keep process alive
         const cleanup = makeShutdownHandler({
             stopReaper,
-            closeables: [{ close: () => restHttpServer?.close() }, sqlite],
+            closeables: [...(litestreamMgr ? [litestreamMgr] : []), { close: () => restHttpServer?.close() }, sqlite],
         });
         process.once("SIGINT", cleanup);
         process.once("SIGTERM", cleanup);

package/dist/src/litestream/cli-integration.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/src/litestream/cli-integration.test.js

@@ -0,0 +1,35 @@
+import { afterEach, beforeEach, describe, expect, it } from "vitest";
+import { buildConfigFromEnv } from "./manager.js";
+describe("buildConfigFromEnv", () => {
+    const originalEnv = process.env;
+    beforeEach(() => {
+        process.env = { ...originalEnv };
+    });
+    afterEach(() => {
+        process.env = originalEnv;
+    });
+    it("builds config from env vars with defaults", () => {
+        process.env.DEFCON_LITESTREAM_REPLICA_URL = "s3://bucket/db";
+        process.env.DEFCON_LITESTREAM_ACCESS_KEY_ID = "AKIA";
+        process.env.DEFCON_LITESTREAM_SECRET_ACCESS_KEY = "secret";
+        const config = buildConfigFromEnv("./defcon.db");
+        expect(config.replicaUrl).toBe("s3://bucket/db");
+        expect(config.region).toBe("us-east-1");
+        expect(config.retention).toBe("24h");
+        expect(config.syncInterval).toBe("1s");
+        expect(config.endpoint).toBeUndefined();
+    });
+    it("throws if access keys missing", () => {
+        process.env.DEFCON_LITESTREAM_REPLICA_URL = "s3://bucket/db";
+        delete process.env.DEFCON_LITESTREAM_ACCESS_KEY_ID;
+        expect(() => buildConfigFromEnv("./defcon.db")).toThrow("DEFCON_LITESTREAM_ACCESS_KEY_ID");
+    });
+    it("includes custom endpoint", () => {
+        process.env.DEFCON_LITESTREAM_REPLICA_URL = "s3://bucket/db";
+        process.env.DEFCON_LITESTREAM_ACCESS_KEY_ID = "AKIA";
+        process.env.DEFCON_LITESTREAM_SECRET_ACCESS_KEY = "secret";
+        process.env.DEFCON_LITESTREAM_ENDPOINT = "https://r2.example.com";
+        const config = buildConfigFromEnv("./defcon.db");
+        expect(config.endpoint).toBe("https://r2.example.com");
+    });
+});

package/dist/src/litestream/manager.d.ts

@@ -0,0 +1,23 @@
+export interface LitestreamConfig {
+    dbPath: string;
+    replicaUrl: string;
+    accessKeyId: string;
+    secretAccessKey: string;
+    endpoint?: string;
+    region: string;
+    retention: string;
+    syncInterval: string;
+}
+export declare function isLitestreamEnabled(): boolean;
+export declare function buildConfigFromEnv(dbPath: string): LitestreamConfig;
+export declare class LitestreamManager {
+    private config;
+    private configPath;
+    private child;
+    constructor(config: LitestreamConfig);
+    generateConfig(): string;
+    restore(): void;
+    start(): void;
+    close(): Promise<void>;
+    private writeConfig;
+}

package/dist/src/litestream/manager.js

@@ -0,0 +1,123 @@
+import { spawn, spawnSync } from "node:child_process";
+import { existsSync, mkdirSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { dirname, join, resolve } from "node:path";
+export function isLitestreamEnabled() {
+    return !!process.env.DEFCON_LITESTREAM_REPLICA_URL?.trim();
+}
+export function buildConfigFromEnv(dbPath) {
+    const replicaUrl = process.env.DEFCON_LITESTREAM_REPLICA_URL ?? "";
+    const accessKeyId = process.env.DEFCON_LITESTREAM_ACCESS_KEY_ID;
+    const secretAccessKey = process.env.DEFCON_LITESTREAM_SECRET_ACCESS_KEY;
+    if (!accessKeyId || !secretAccessKey) {
+        throw new Error("DEFCON_LITESTREAM_ACCESS_KEY_ID and DEFCON_LITESTREAM_SECRET_ACCESS_KEY must be set when DEFCON_LITESTREAM_REPLICA_URL is configured");
+    }
+    return {
+        dbPath: resolve(dbPath),
+        replicaUrl,
+        accessKeyId,
+        secretAccessKey,
+        endpoint: process.env.DEFCON_LITESTREAM_ENDPOINT || undefined,
+        region: process.env.DEFCON_LITESTREAM_REGION || "us-east-1",
+        retention: process.env.DEFCON_LITESTREAM_RETENTION || "24h",
+        syncInterval: process.env.DEFCON_LITESTREAM_SYNC_INTERVAL || "1s",
+    };
+}
+export class LitestreamManager {
+    config;
+    configPath;
+    child = null;
+    constructor(config) {
+        this.config = config;
+        this.configPath = join(tmpdir(), `litestream-${process.pid}.yml`);
+    }
+    generateConfig() {
+        const q = (v) => `'${v.replace(/'/g, "''")}'`;
+        const endpoint = this.config.endpoint ? `        endpoint: ${q(this.config.endpoint)}\n` : "";
+        return `dbs:
+  - path: ${q(this.config.dbPath)}
+    replicas:
+      - type: s3
+        url: ${q(this.config.replicaUrl)}
+${endpoint}        region: ${q(this.config.region)}
+        retention: ${q(this.config.retention)}
+        sync-interval: ${q(this.config.syncInterval)}
+`;
+    }
+    restore() {
+        if (existsSync(this.config.dbPath)) {
+            process.stderr.write(`[litestream] DB exists at ${this.config.dbPath}, skipping restore\n`);
+            return;
+        }
+        const dir = dirname(this.config.dbPath);
+        if (!existsSync(dir)) {
+            mkdirSync(dir, { recursive: true });
+        }
+        this.writeConfig();
+        process.stderr.write(`[litestream] Restoring from ${this.config.replicaUrl}...\n`);
+        const result = spawnSync("litestream", ["restore", "-config", this.configPath, this.config.dbPath], {
+            stdio: ["ignore", "pipe", "pipe"],
+            timeout: 120_000,
+            env: {
+                ...process.env,
+                LITESTREAM_ACCESS_KEY_ID: this.config.accessKeyId,
+                LITESTREAM_SECRET_ACCESS_KEY: this.config.secretAccessKey,
+            },
+        });
+        if (result.status !== 0) {
+            const stderr = result.stderr?.toString() ?? "";
+            if (stderr.includes("no generations found")) {
+                process.stderr.write(`[litestream] No replica found, starting fresh\n`);
+            }
+            else {
+                throw new Error(`[litestream] Restore failed: ${stderr}`);
+            }
+        }
+        else {
+            process.stderr.write(`[litestream] Restore complete\n`);
+        }
+    }
+    start() {
+        this.writeConfig();
+        process.stderr.write(`[litestream] Starting replication to ${this.config.replicaUrl}\n`);
+        this.child = spawn("litestream", ["replicate", "-config", this.configPath], {
+            stdio: ["ignore", "pipe", "pipe"],
+            env: {
+                ...process.env,
+                LITESTREAM_ACCESS_KEY_ID: this.config.accessKeyId,
+                LITESTREAM_SECRET_ACCESS_KEY: this.config.secretAccessKey,
+            },
+        });
+        this.child.stdout?.on("data", (chunk) => {
+            process.stderr.write(`[litestream] ${chunk.toString()}`);
+        });
+        this.child.stderr?.on("data", (chunk) => {
+            process.stderr.write(`[litestream] ${chunk.toString()}`);
+        });
+        this.child.on("exit", (code) => {
+            process.stderr.write(`[litestream] Process exited with code ${code}\n`);
+            this.child = null;
+        });
+        this.child.on("error", (err) => {
+            process.stderr.write(`[litestream] Process error: ${err.message}\n`);
+            this.child = null;
+        });
+    }
+    close() {
+        if (!this.child) {
+            return Promise.resolve();
+        }
+        const child = this.child;
+        return new Promise((resolve) => {
+            process.stderr.write(`[litestream] Stopping replication\n`);
+            child.once("exit", () => {
+                this.child = null;
+                resolve();
+            });
+            child.kill("SIGTERM");
+        });
+    }
+    writeConfig() {
+        writeFileSync(this.configPath, this.generateConfig());
+    }
+}

package/dist/src/litestream/manager.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/src/litestream/manager.test.js

@@ -0,0 +1,46 @@
+import { describe, expect, it } from "vitest";
+import { isLitestreamEnabled, LitestreamManager } from "./manager.js";
+describe("isLitestreamEnabled", () => {
+    it("returns false when DEFCON_LITESTREAM_REPLICA_URL is not set", () => {
+        delete process.env.DEFCON_LITESTREAM_REPLICA_URL;
+        expect(isLitestreamEnabled()).toBe(false);
+    });
+    it("returns true when DEFCON_LITESTREAM_REPLICA_URL is set", () => {
+        process.env.DEFCON_LITESTREAM_REPLICA_URL = "s3://bucket/defcon.db";
+        expect(isLitestreamEnabled()).toBe(true);
+        delete process.env.DEFCON_LITESTREAM_REPLICA_URL;
+    });
+});
+describe("LitestreamManager", () => {
+    it("generates correct YAML config", () => {
+        const mgr = new LitestreamManager({
+            dbPath: "/data/defcon.db",
+            replicaUrl: "s3://bucket/defcon.db",
+            accessKeyId: "AKIA...",
+            secretAccessKey: "secret",
+            region: "us-east-1",
+            retention: "24h",
+            syncInterval: "1s",
+        });
+        const yaml = mgr.generateConfig();
+        expect(yaml).toContain("'/data/defcon.db'");
+        expect(yaml).toContain("'s3://bucket/defcon.db'");
+        expect(yaml).not.toContain("AKIA...");
+        expect(yaml).toContain("retention: '24h'");
+        expect(yaml).toContain("sync-interval: '1s'");
+    });
+    it("generates config with custom endpoint for R2", () => {
+        const mgr = new LitestreamManager({
+            dbPath: "/data/defcon.db",
+            replicaUrl: "s3://bucket/defcon.db",
+            accessKeyId: "key",
+            secretAccessKey: "secret",
+            endpoint: "https://account.r2.cloudflarestorage.com",
+            region: "auto",
+            retention: "24h",
+            syncInterval: "1s",
+        });
+        const yaml = mgr.generateConfig();
+        expect(yaml).toContain("endpoint: 'https://account.r2.cloudflarestorage.com'");
+    });
+});

package/dist/src/repositories/drizzle/domain-event.repo.d.ts

@@ -11,6 +11,7 @@ export declare class DrizzleDomainEventRepository implements IDomainEventReposit
     list(entityId: string, opts?: {
         type?: string;
         limit?: number;
+        minSequence?: number;
     }): Promise<DomainEvent[]>;
 }
 export {};

package/dist/src/repositories/drizzle/domain-event.repo.js

@@ -1,5 +1,5 @@
 import { randomUUID } from "node:crypto";
-import { and, eq, sql } from "drizzle-orm";
+import { and, eq, gt, sql } from "drizzle-orm";
 import { domainEvents } from "./schema.js";
 export class DrizzleDomainEventRepository {
     db;
@@ -62,6 +62,9 @@ export class DrizzleDomainEventRepository {
         if (opts?.type) {
             conditions.push(eq(domainEvents.type, opts.type));
         }
+        if (opts?.minSequence !== undefined) {
+            conditions.push(gt(domainEvents.sequence, opts.minSequence));
+        }
         const rows = this.db
             .select()
             .from(domainEvents)

package/dist/src/repositories/drizzle/entity-snapshot.repo.d.ts

@@ -0,0 +1,14 @@
+import type { BetterSQLite3Database } from "drizzle-orm/better-sqlite3";
+import type { Entity, IEntitySnapshotRepository } from "../interfaces.js";
+import type * as schema from "./schema.js";
+type Db = BetterSQLite3Database<typeof schema>;
+export declare class DrizzleEntitySnapshotRepository implements IEntitySnapshotRepository {
+    private readonly db;
+    constructor(db: Db);
+    save(entityId: string, sequence: number, state: Entity): Promise<void>;
+    loadLatest(entityId: string): Promise<{
+        sequence: number;
+        state: Entity;
+    } | null>;
+}
+export {};

package/dist/src/repositories/drizzle/entity-snapshot.repo.js

@@ -0,0 +1,68 @@
+import { randomUUID } from "node:crypto";
+import { desc, eq } from "drizzle-orm";
+import { entitySnapshots } from "./schema.js";
+export class DrizzleEntitySnapshotRepository {
+    db;
+    constructor(db) {
+        this.db = db;
+    }
+    async save(entityId, sequence, state) {
+        const id = randomUUID();
+        this.db
+            .insert(entitySnapshots)
+            .values({
+            id,
+            entityId,
+            sequence,
+            state: state.state,
+            flowId: state.flowId,
+            refs: state.refs,
+            artifacts: state.artifacts,
+            claimedBy: state.claimedBy,
+            claimedAt: state.claimedAt?.getTime() ?? null,
+            flowVersion: state.flowVersion,
+            priority: state.priority,
+            affinityWorkerId: state.affinityWorkerId,
+            affinityRole: state.affinityRole,
+            affinityExpiresAt: state.affinityExpiresAt?.getTime() ?? null,
+            createdAt: state.createdAt.getTime(),
+            updatedAt: state.updatedAt.getTime(),
+            snapshotAt: Date.now(),
+            parentEntityId: state.parentEntityId,
+        })
+            .onConflictDoNothing()
+            .run();
+    }
+    async loadLatest(entityId) {
+        const rows = this.db
+            .select()
+            .from(entitySnapshots)
+            .where(eq(entitySnapshots.entityId, entityId))
+            .orderBy(desc(entitySnapshots.sequence))
+            .limit(1)
+            .all();
+        if (rows.length === 0)
+            return null;
+        const row = rows[0];
+        return {
+            sequence: row.sequence,
+            state: {
+                id: entityId,
+                flowId: row.flowId,
+                state: row.state,
+                refs: row.refs,
+                artifacts: row.artifacts,
+                claimedBy: row.claimedBy,
+                claimedAt: row.claimedAt ? new Date(row.claimedAt) : null,
+                flowVersion: row.flowVersion ?? 1,
+                priority: row.priority ?? 0,
+                createdAt: new Date(row.createdAt ?? 0),
+                updatedAt: new Date(row.updatedAt ?? 0),
+                affinityWorkerId: row.affinityWorkerId ?? null,
+                affinityRole: row.affinityRole ?? null,
+                affinityExpiresAt: row.affinityExpiresAt ? new Date(row.affinityExpiresAt) : null,
+                parentEntityId: row.parentEntityId ?? null,
+            },
+        };
+    }
+}

package/dist/src/repositories/drizzle/index.d.ts

@@ -1,5 +1,6 @@
 export { DrizzleDomainEventRepository } from "./domain-event.repo.js";
 export { DrizzleEntityRepository } from "./entity.repo.js";
+export { DrizzleEntitySnapshotRepository } from "./entity-snapshot.repo.js";
 export { DrizzleEventRepository } from "./event.repo.js";
 export { DrizzleFlowRepository } from "./flow.repo.js";
 export { DrizzleGateRepository } from "./gate.repo.js";

package/dist/src/repositories/drizzle/index.js

@@ -1,6 +1,7 @@
 // Drizzle ORM implementations of repository interfaces
 export { DrizzleDomainEventRepository } from "./domain-event.repo.js";
 export { DrizzleEntityRepository } from "./entity.repo.js";
+export { DrizzleEntitySnapshotRepository } from "./entity-snapshot.repo.js";
 export { DrizzleEventRepository } from "./event.repo.js";
 export { DrizzleFlowRepository } from "./flow.repo.js";
 export { DrizzleGateRepository } from "./gate.repo.js";