@wopr-network/defcon 1.10.0 → 1.12.0

package/dist/src/engine/engine.d.ts

@@ -1,5 +1,5 @@
 import type { Logger } from "../logger.js";
-import type { Artifacts, Entity, IEntityRepository, IFlowRepository, IGateRepository, IInvocationRepository, ITransitionLogRepository } from "../repositories/interfaces.js";
+import type { Artifacts, Entity, IDomainEventRepository, IEntityRepository, IFlowRepository, IGateRepository, IInvocationRepository, ITransitionLogRepository } from "../repositories/interfaces.js";
 import type { IEventBusAdapter } from "./event-types.js";
 export interface ProcessSignalResult {
     newState?: string;
@@ -41,6 +41,8 @@ export interface EngineDeps {
     logger?: Logger;
     /** Optional transaction wrapper from the database layer. When provided, processSignal runs inside a single transaction and events are flushed only after successful commit. */
     withTransaction?: <T>(fn: () => T | Promise<T>) => Promise<T>;
+    /** Optional domain event repository. When provided, claimWork uses CAS on the event log to prevent concurrent claim races. */
+    domainEvents?: IDomainEventRepository;
 }
 export declare class Engine {
     private entityRepo;
@@ -52,6 +54,7 @@ export declare class Engine {
     private eventEmitter;
     private readonly logger;
     private readonly withTransactionFn;
+    private readonly domainEventRepo;
     private drainingWorkers;
     constructor(deps: EngineDeps);
     drainWorker(workerId: string): void;

package/dist/src/engine/engine.js

@@ -18,6 +18,7 @@ export class Engine {
     eventEmitter;
     logger;
     withTransactionFn;
+    domainEventRepo;
     drainingWorkers = new Set();
     constructor(deps) {
         this.entityRepo = deps.entityRepo;
@@ -29,6 +30,7 @@ export class Engine {
         this.eventEmitter = deps.eventEmitter;
         this.logger = deps.logger ?? consoleLogger;
         this.withTransactionFn = deps.withTransaction ?? null;
+        this.domainEventRepo = deps.domainEvents ?? null;
     }
     drainWorker(workerId) {
         this.drainingWorkers.add(workerId);
@@ -476,9 +478,30 @@ export class Engine {
             if (entity.state !== pending.stage)
                 continue;
             const entityClaimToken = worker_id ?? `agent:${role}`;
+            // CAS guard: atomically append an invocation.claimed event using optimistic concurrency.
+            // Only one writer wins the unique (entityId, sequence) constraint — losers move to the next candidate.
+            if (this.domainEventRepo) {
+                const lastSeq = await this.domainEventRepo.getLastSequence(entity.id);
+                const casResult = await this.domainEventRepo.appendCas("invocation.claim_attempted", entity.id, { agentId: entityClaimToken, invocationId: pending.id, stage: pending.stage }, lastSeq);
+                if (!casResult)
+                    continue; // Another agent won the race
+            }
             const claimed = await this.entityRepo.claimById(entity.id, entityClaimToken);
-            if (!claimed)
+            if (!claimed) {
+                // claimById failed after CAS success — record rollback event for observability
+                if (this.domainEventRepo) {
+                    try {
+                        await this.domainEventRepo.append("invocation.claim_released", entity.id, {
+                            agentId: entityClaimToken,
+                            reason: "claimById_failed",
+                        });
+                    }
+                    catch (err) {
+                        this.logger.warn("[engine] CAS claim rollback event failed", { err });
+                    }
+                }
                 continue;
+            }
             // Post-claim state validation — entity may have transitioned between guard check and claim
             if (claimed.state !== pending.stage) {
                 try {

package/dist/src/execution/cli.js

@@ -15,6 +15,7 @@ import { resolveCorsOrigin } from "../cors.js";
 import { DomainEventPersistAdapter } from "../engine/domain-event-adapter.js";
 import { Engine } from "../engine/engine.js";
 import { EventEmitter } from "../engine/event-emitter.js";
+import { buildConfigFromEnv, isLitestreamEnabled, LitestreamManager } from "../litestream/manager.js";
 import { withTransaction } from "../main.js";
 import { DrizzleDomainEventRepository } from "../repositories/drizzle/domain-event.repo.js";
 import { DrizzleEntityRepository } from "../repositories/drizzle/entity.repo.js";
@@ -158,7 +159,18 @@ program
     .option("--http-host <address>", "Host for HTTP REST API", "127.0.0.1")
     .option("--ui", "Enable built-in web UI at /ui")
     .action(async (opts) => {
+    // Litestream: restore from replica if DB missing and replication configured
+    let litestreamMgr;
+    if (isLitestreamEnabled()) {
+        const lsConfig = buildConfigFromEnv(opts.db);
+        litestreamMgr = new LitestreamManager(lsConfig);
+        litestreamMgr.restore();
+    }
     const { db, sqlite } = openDb(opts.db);
+    // Litestream: start continuous replication
+    if (litestreamMgr) {
+        litestreamMgr.start();
+    }
     const entityRepo = new DrizzleEntityRepository(db);
     const flowRepo = new DrizzleFlowRepository(db);
     const invocationRepo = new DrizzleInvocationRepository(db);
@@ -181,6 +193,7 @@ program
         adapters: new Map(),
         eventEmitter,
         withTransaction: (fn) => withTransaction(sqlite, fn),
+        domainEvents: domainEventRepo,
     });
     const deps = {
         entities: entityRepo,
@@ -358,7 +371,12 @@ program
         });
         const shutdown = makeShutdownHandler({
             stopReaper,
-            closeables: [{ close: () => restHttpServer?.close() }, httpServer, sqlite],
+            closeables: [
+                ...(litestreamMgr ? [litestreamMgr] : []),
+                { close: () => restHttpServer?.close() },
+                httpServer,
+                sqlite,
+            ],
         });
         process.once("SIGINT", shutdown);
         process.once("SIGTERM", shutdown);
@@ -366,7 +384,10 @@ program
     else if (startMcp) {
         // stdio (default)
         console.error("Starting MCP server on stdio...");
-        const cleanup = makeShutdownHandler({ stopReaper, closeables: [sqlite] });
+        const cleanup = makeShutdownHandler({
+            stopReaper,
+            closeables: [...(litestreamMgr ? [litestreamMgr] : []), sqlite],
+        });
         process.once("SIGINT", cleanup);
         process.once("SIGTERM", cleanup);
         const mcpOpts = { adminToken, workerToken, stdioTrusted: true };
@@ -376,7 +397,7 @@ program
         // HTTP-only mode — keep process alive
         const cleanup = makeShutdownHandler({
             stopReaper,
-            closeables: [{ close: () => restHttpServer?.close() }, sqlite],
+            closeables: [...(litestreamMgr ? [litestreamMgr] : []), { close: () => restHttpServer?.close() }, sqlite],
         });
         process.once("SIGINT", cleanup);
         process.once("SIGTERM", cleanup);

package/dist/src/litestream/cli-integration.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/src/litestream/cli-integration.test.js

@@ -0,0 +1,35 @@
+import { afterEach, beforeEach, describe, expect, it } from "vitest";
+import { buildConfigFromEnv } from "./manager.js";
+describe("buildConfigFromEnv", () => {
+    const originalEnv = process.env;
+    beforeEach(() => {
+        process.env = { ...originalEnv };
+    });
+    afterEach(() => {
+        process.env = originalEnv;
+    });
+    it("builds config from env vars with defaults", () => {
+        process.env.DEFCON_LITESTREAM_REPLICA_URL = "s3://bucket/db";
+        process.env.DEFCON_LITESTREAM_ACCESS_KEY_ID = "AKIA";
+        process.env.DEFCON_LITESTREAM_SECRET_ACCESS_KEY = "secret";
+        const config = buildConfigFromEnv("./defcon.db");
+        expect(config.replicaUrl).toBe("s3://bucket/db");
+        expect(config.region).toBe("us-east-1");
+        expect(config.retention).toBe("24h");
+        expect(config.syncInterval).toBe("1s");
+        expect(config.endpoint).toBeUndefined();
+    });
+    it("throws if access keys missing", () => {
+        process.env.DEFCON_LITESTREAM_REPLICA_URL = "s3://bucket/db";
+        delete process.env.DEFCON_LITESTREAM_ACCESS_KEY_ID;
+        expect(() => buildConfigFromEnv("./defcon.db")).toThrow("DEFCON_LITESTREAM_ACCESS_KEY_ID");
+    });
+    it("includes custom endpoint", () => {
+        process.env.DEFCON_LITESTREAM_REPLICA_URL = "s3://bucket/db";
+        process.env.DEFCON_LITESTREAM_ACCESS_KEY_ID = "AKIA";
+        process.env.DEFCON_LITESTREAM_SECRET_ACCESS_KEY = "secret";
+        process.env.DEFCON_LITESTREAM_ENDPOINT = "https://r2.example.com";
+        const config = buildConfigFromEnv("./defcon.db");
+        expect(config.endpoint).toBe("https://r2.example.com");
+    });
+});

package/dist/src/litestream/manager.d.ts

@@ -0,0 +1,23 @@
+export interface LitestreamConfig {
+    dbPath: string;
+    replicaUrl: string;
+    accessKeyId: string;
+    secretAccessKey: string;
+    endpoint?: string;
+    region: string;
+    retention: string;
+    syncInterval: string;
+}
+export declare function isLitestreamEnabled(): boolean;
+export declare function buildConfigFromEnv(dbPath: string): LitestreamConfig;
+export declare class LitestreamManager {
+    private config;
+    private configPath;
+    private child;
+    constructor(config: LitestreamConfig);
+    generateConfig(): string;
+    restore(): void;
+    start(): void;
+    close(): Promise<void>;
+    private writeConfig;
+}

package/dist/src/litestream/manager.js

@@ -0,0 +1,123 @@
+import { spawn, spawnSync } from "node:child_process";
+import { existsSync, mkdirSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { dirname, join, resolve } from "node:path";
+export function isLitestreamEnabled() {
+    return !!process.env.DEFCON_LITESTREAM_REPLICA_URL?.trim();
+}
+export function buildConfigFromEnv(dbPath) {
+    const replicaUrl = process.env.DEFCON_LITESTREAM_REPLICA_URL ?? "";
+    const accessKeyId = process.env.DEFCON_LITESTREAM_ACCESS_KEY_ID;
+    const secretAccessKey = process.env.DEFCON_LITESTREAM_SECRET_ACCESS_KEY;
+    if (!accessKeyId || !secretAccessKey) {
+        throw new Error("DEFCON_LITESTREAM_ACCESS_KEY_ID and DEFCON_LITESTREAM_SECRET_ACCESS_KEY must be set when DEFCON_LITESTREAM_REPLICA_URL is configured");
+    }
+    return {
+        dbPath: resolve(dbPath),
+        replicaUrl,
+        accessKeyId,
+        secretAccessKey,
+        endpoint: process.env.DEFCON_LITESTREAM_ENDPOINT || undefined,
+        region: process.env.DEFCON_LITESTREAM_REGION || "us-east-1",
+        retention: process.env.DEFCON_LITESTREAM_RETENTION || "24h",
+        syncInterval: process.env.DEFCON_LITESTREAM_SYNC_INTERVAL || "1s",
+    };
+}
+export class LitestreamManager {
+    config;
+    configPath;
+    child = null;
+    constructor(config) {
+        this.config = config;
+        this.configPath = join(tmpdir(), `litestream-${process.pid}.yml`);
+    }
+    generateConfig() {
+        const q = (v) => `'${v.replace(/'/g, "''")}'`;
+        const endpoint = this.config.endpoint ? `        endpoint: ${q(this.config.endpoint)}\n` : "";
+        return `dbs:
+  - path: ${q(this.config.dbPath)}
+    replicas:
+      - type: s3
+        url: ${q(this.config.replicaUrl)}
+${endpoint}        region: ${q(this.config.region)}
+        retention: ${q(this.config.retention)}
+        sync-interval: ${q(this.config.syncInterval)}
+`;
+    }
+    restore() {
+        if (existsSync(this.config.dbPath)) {
+            process.stderr.write(`[litestream] DB exists at ${this.config.dbPath}, skipping restore\n`);
+            return;
+        }
+        const dir = dirname(this.config.dbPath);
+        if (!existsSync(dir)) {
+            mkdirSync(dir, { recursive: true });
+        }
+        this.writeConfig();
+        process.stderr.write(`[litestream] Restoring from ${this.config.replicaUrl}...\n`);
+        const result = spawnSync("litestream", ["restore", "-config", this.configPath, this.config.dbPath], {
+            stdio: ["ignore", "pipe", "pipe"],
+            timeout: 120_000,
+            env: {
+                ...process.env,
+                LITESTREAM_ACCESS_KEY_ID: this.config.accessKeyId,
+                LITESTREAM_SECRET_ACCESS_KEY: this.config.secretAccessKey,
+            },
+        });
+        if (result.status !== 0) {
+            const stderr = result.stderr?.toString() ?? "";
+            if (stderr.includes("no generations found")) {
+                process.stderr.write(`[litestream] No replica found, starting fresh\n`);
+            }
+            else {
+                throw new Error(`[litestream] Restore failed: ${stderr}`);
+            }
+        }
+        else {
+            process.stderr.write(`[litestream] Restore complete\n`);
+        }
+    }
+    start() {
+        this.writeConfig();
+        process.stderr.write(`[litestream] Starting replication to ${this.config.replicaUrl}\n`);
+        this.child = spawn("litestream", ["replicate", "-config", this.configPath], {
+            stdio: ["ignore", "pipe", "pipe"],
+            env: {
+                ...process.env,
+                LITESTREAM_ACCESS_KEY_ID: this.config.accessKeyId,
+                LITESTREAM_SECRET_ACCESS_KEY: this.config.secretAccessKey,
+            },
+        });
+        this.child.stdout?.on("data", (chunk) => {
+            process.stderr.write(`[litestream] ${chunk.toString()}`);
+        });
+        this.child.stderr?.on("data", (chunk) => {
+            process.stderr.write(`[litestream] ${chunk.toString()}`);
+        });
+        this.child.on("exit", (code) => {
+            process.stderr.write(`[litestream] Process exited with code ${code}\n`);
+            this.child = null;
+        });
+        this.child.on("error", (err) => {
+            process.stderr.write(`[litestream] Process error: ${err.message}\n`);
+            this.child = null;
+        });
+    }
+    close() {
+        if (!this.child) {
+            return Promise.resolve();
+        }
+        const child = this.child;
+        return new Promise((resolve) => {
+            process.stderr.write(`[litestream] Stopping replication\n`);
+            child.once("exit", () => {
+                this.child = null;
+                resolve();
+            });
+            child.kill("SIGTERM");
+        });
+    }
+    writeConfig() {
+        writeFileSync(this.configPath, this.generateConfig());
+    }
+}

package/dist/src/litestream/manager.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/src/litestream/manager.test.js

@@ -0,0 +1,46 @@
+import { describe, expect, it } from "vitest";
+import { isLitestreamEnabled, LitestreamManager } from "./manager.js";
+describe("isLitestreamEnabled", () => {
+    it("returns false when DEFCON_LITESTREAM_REPLICA_URL is not set", () => {
+        delete process.env.DEFCON_LITESTREAM_REPLICA_URL;
+        expect(isLitestreamEnabled()).toBe(false);
+    });
+    it("returns true when DEFCON_LITESTREAM_REPLICA_URL is set", () => {
+        process.env.DEFCON_LITESTREAM_REPLICA_URL = "s3://bucket/defcon.db";
+        expect(isLitestreamEnabled()).toBe(true);
+        delete process.env.DEFCON_LITESTREAM_REPLICA_URL;
+    });
+});
+describe("LitestreamManager", () => {
+    it("generates correct YAML config", () => {
+        const mgr = new LitestreamManager({
+            dbPath: "/data/defcon.db",
+            replicaUrl: "s3://bucket/defcon.db",
+            accessKeyId: "AKIA...",
+            secretAccessKey: "secret",
+            region: "us-east-1",
+            retention: "24h",
+            syncInterval: "1s",
+        });
+        const yaml = mgr.generateConfig();
+        expect(yaml).toContain("'/data/defcon.db'");
+        expect(yaml).toContain("'s3://bucket/defcon.db'");
+        expect(yaml).not.toContain("AKIA...");
+        expect(yaml).toContain("retention: '24h'");
+        expect(yaml).toContain("sync-interval: '1s'");
+    });
+    it("generates config with custom endpoint for R2", () => {
+        const mgr = new LitestreamManager({
+            dbPath: "/data/defcon.db",
+            replicaUrl: "s3://bucket/defcon.db",
+            accessKeyId: "key",
+            secretAccessKey: "secret",
+            endpoint: "https://account.r2.cloudflarestorage.com",
+            region: "auto",
+            retention: "24h",
+            syncInterval: "1s",
+        });
+        const yaml = mgr.generateConfig();
+        expect(yaml).toContain("endpoint: 'https://account.r2.cloudflarestorage.com'");
+    });
+});

package/dist/src/repositories/drizzle/domain-event.repo.d.ts

@@ -6,6 +6,8 @@ export declare class DrizzleDomainEventRepository implements IDomainEventReposit
     private readonly db;
     constructor(db: Db);
     append(type: string, entityId: string, payload: Record<string, unknown>): Promise<DomainEvent>;
+    getLastSequence(entityId: string): Promise<number>;
+    appendCas(type: string, entityId: string, payload: Record<string, unknown>, expectedSequence: number): Promise<DomainEvent | null>;
     list(entityId: string, opts?: {
         type?: string;
         limit?: number;

package/dist/src/repositories/drizzle/domain-event.repo.js

@@ -20,6 +20,43 @@ export class DrizzleDomainEventRepository {
             return { id, type, entityId, payload, sequence, emittedAt };
         });
     }
+    async getLastSequence(entityId) {
+        const row = this.db
+            .select({ maxSeq: sql `coalesce(max(${domainEvents.sequence}), 0)` })
+            .from(domainEvents)
+            .where(eq(domainEvents.entityId, entityId))
+            .get();
+        return row?.maxSeq ?? 0;
+    }
+    async appendCas(type, entityId, payload, expectedSequence) {
+        try {
+            return this.db.transaction((tx) => {
+                const currentRow = tx
+                    .select({ maxSeq: sql `coalesce(max(${domainEvents.sequence}), 0)` })
+                    .from(domainEvents)
+                    .where(eq(domainEvents.entityId, entityId))
+                    .get();
+                const currentSeq = currentRow?.maxSeq ?? 0;
+                if (currentSeq !== expectedSequence) {
+                    return null;
+                }
+                const newSequence = expectedSequence + 1;
+                const id = randomUUID();
+                const emittedAt = Date.now();
+                tx.insert(domainEvents).values({ id, type, entityId, payload, sequence: newSequence, emittedAt }).run();
+                return { id, type, entityId, payload, sequence: newSequence, emittedAt };
+            });
+        }
+        catch (err) {
+            if (err instanceof Error &&
+                (("code" in err && err.code === "SQLITE_CONSTRAINT_UNIQUE") ||
+                    ("code" in err && err.code === "23505") ||
+                    err.message.includes("UNIQUE constraint failed"))) {
+                return null;
+            }
+            throw err;
+        }
+    }
     async list(entityId, opts) {
         const conditions = [eq(domainEvents.entityId, entityId)];
         if (opts?.type) {

package/dist/src/repositories/interfaces.d.ts

@@ -374,6 +374,11 @@ export interface IDomainEventRepository {
         type?: string;
         limit?: number;
     }): Promise<DomainEvent[]>;
+    /** Get the current max sequence number for an entity (0 if no events exist). */
+    getLastSequence(entityId: string): Promise<number>;
+    /** Append a domain event only if the entity's current max sequence equals expectedSequence.
+     *  Returns the event on success, or null if another writer won (unique constraint violation). */
+    appendCas(type: string, entityId: string, payload: Record<string, unknown>, expectedSequence: number): Promise<DomainEvent | null>;
 }
 /** Data-access contract for gate definitions and result recording. */
 export interface IGateRepository {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@wopr-network/defcon",
-  "version": "1.10.0",
+  "version": "1.12.0",
   "type": "module",
   "packageManager": "pnpm@9.15.4",
   "engines": {