@adamancyzhang/claude-orchestrator 0.2.8 → 0.3.1

package/dist/leader/decision-engine.d.ts

@@ -0,0 +1,35 @@
+import { ZkClient } from "../zk/client.js";
+import { TaskQueue } from "../modules/task-queue.js";
+import { MessageRouter } from "../modules/message-router.js";
+import type { Message } from "../models/schemas.js";
+export interface DecisionContext {
+    teamStatus: object;
+    taskQueues: object;
+    chainStatus: object;
+}
+export interface Decision {
+    decision: "pass" | "feedback" | "reject";
+    reason: string;
+    feedbackToWorker?: string;
+    nextAction: {
+        action: "activate_next_link" | "reassign" | "close_chain" | "broadcast_help" | "none";
+        nextLink: string | null;
+        suggestedWorker: string | null;
+        messageToWorker: string | null;
+    };
+}
+export declare class DecisionEngine {
+    private zk;
+    private taskQueue;
+    private messageRouter;
+    private command;
+    private cacheDir;
+    private leaderInstanceId;
+    private template;
+    private templatePath;
+    constructor(zk: ZkClient, taskQueue: TaskQueue, messageRouter: MessageRouter, command: string, cacheDir: string, leaderInstanceId: string, templatePath?: string);
+    private loadTemplate;
+    evaluate(report: Message, context: DecisionContext): Promise<Decision>;
+    private executeDecision;
+    private parseOutput;
+}

package/dist/leader/decision-engine.js

@@ -0,0 +1,102 @@
+import * as fs from "node:fs";
+import * as path from "node:path";
+import { expandHomeDir } from "../config.js";
+import { execAndCapture } from "../utils/exec.js";
+const NEXT_LINKS = {
+    plan: "build",
+    build: "verify",
+    verify: "review",
+    review: "accept",
+    accept: null,
+};
+export class DecisionEngine {
+    zk;
+    taskQueue;
+    messageRouter;
+    command;
+    cacheDir;
+    leaderInstanceId;
+    template = null;
+    templatePath;
+    constructor(zk, taskQueue, messageRouter, command, cacheDir, leaderInstanceId, templatePath) {
+        this.zk = zk;
+        this.taskQueue = taskQueue;
+        this.messageRouter = messageRouter;
+        this.command = command;
+        this.cacheDir = cacheDir;
+        this.leaderInstanceId = leaderInstanceId;
+        this.templatePath = templatePath ?? path.join(process.cwd(), ".claude-orchestrator", "agents", "leader-decide.md");
+    }
+    async loadTemplate() {
+        if (this.template)
+            return this.template;
+        try {
+            this.template = await fs.promises.readFile(this.templatePath, "utf-8");
+        }
+        catch {
+            throw new Error(`Decide template not found at ${this.templatePath}. Run setup first.`);
+        }
+        return this.template;
+    }
+    async evaluate(report, context) {
+        const template = await this.loadTemplate();
+        const prompt = template
+            .replace("{{team_status}}", JSON.stringify(context.teamStatus, null, 2))
+            .replace("{{task_queues}}", JSON.stringify(context.taskQueues, null, 2))
+            .replace("{{chain_status}}", JSON.stringify(context.chainStatus, null, 2))
+            .replace("{{content}}", report.content);
+        const uniqueKey = `decide-${Date.now().toString(36)}`;
+        const resolvedCacheDir = expandHomeDir(path.join(this.cacheDir, this.leaderInstanceId));
+        const logPath = path.join(resolvedCacheDir, `${uniqueKey}.log`);
+        const { stdout } = await execAndCapture(this.command, prompt, logPath);
+        const decision = this.parseOutput(stdout);
+        await this.executeDecision(decision, report);
+        return decision;
+    }
+    async executeDecision(decision, report) {
+        const nextLink = decision.nextAction.nextLink;
+        const linkOrder = ["plan", "build", "verify", "review", "accept"];
+        switch (decision.nextAction.action) {
+            case "activate_next_link": {
+                if (!nextLink)
+                    break;
+                const pending = await this.zk.listPendingTasks();
+                for (const [taskId, data] of pending) {
+                    if (data.link === nextLink && data.chain_id === report.chain_id) {
+                        // Task already exists in pending, no need to create
+                        break;
+                    }
+                }
+                break;
+            }
+            case "close_chain":
+                break;
+            case "reassign":
+            case "broadcast_help":
+            case "none":
+                break;
+        }
+        if (decision.decision === "feedback" && decision.feedbackToWorker && report.from_instance) {
+            await this.messageRouter.send(this.leaderInstanceId, "Leader", decision.feedbackToWorker, report.from_instance);
+        }
+    }
+    parseOutput(output) {
+        const cleaned = output
+            .replace(/```json\s*/g, "")
+            .replace(/```\s*/g, "")
+            .trim();
+        const parsed = JSON.parse(cleaned);
+        return {
+            decision: parsed.decision ?? "pass",
+            reason: parsed.reason ?? "",
+            feedbackToWorker: parsed.feedback_to_worker,
+            nextAction: {
+                action: parsed.next_action?.action ?? "none",
+                nextLink: parsed.next_action?.next_link ?? null,
+                suggestedWorker: parsed.next_action?.suggested_worker ?? null,
+                messageToWorker: parsed.next_action?.message_to_worker ?? null,
+            },
+        };
+    }
+}
+//# sourceMappingURL=decision-engine.js.map

package/dist/leader/decision-engine.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"decision-engine.js","sourceRoot":"","sources":["../../src/leader/decision-engine.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAKlC,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAC7C,OAAO,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAElD,MAAM,UAAU,GAAkC;IAChD,IAAI,EAAE,OAAO;IACb,KAAK,EAAE,QAAQ;IACf,MAAM,EAAE,QAAQ;IAChB,MAAM,EAAE,QAAQ;IAChB,MAAM,EAAE,IAAI;CACb,CAAC;AAoBF,MAAM,OAAO,cAAc;IAKf;IACA;IACA;IACA;IACA;IACA;IATF,QAAQ,GAAkB,IAAI,CAAC;IAC/B,YAAY,CAAS;IAE7B,YACU,EAAY,EACZ,SAAoB,EACpB,aAA4B,EAC5B,OAAe,EACf,QAAgB,EAChB,gBAAwB,EAChC,YAAqB;QANb,OAAE,GAAF,EAAE,CAAU;QACZ,cAAS,GAAT,SAAS,CAAW;QACpB,kBAAa,GAAb,aAAa,CAAe;QAC5B,YAAO,GAAP,OAAO,CAAQ;QACf,aAAQ,GAAR,QAAQ,CAAQ;QAChB,qBAAgB,GAAhB,gBAAgB,CAAQ;QAGhC,IAAI,CAAC,YAAY,GAAG,YAAY,IAAI,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,sBAAsB,EAAE,QAAQ,EAAE,kBAAkB,CAAC,CAAC;IACrH,CAAC;IAEO,KAAK,CAAC,YAAY;QACxB,IAAI,IAAI,CAAC,QAAQ;YAAE,OAAO,IAAI,CAAC,QAAQ,CAAC;QACxC,IAAI,CAAC;YACH,IAAI,CAAC,QAAQ,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;QACzE,CAAC;QAAC,MAAM,CAAC;YACP,MAAM,IAAI,KAAK,CAAC,gCAAgC,IAAI,CAAC,YAAY,oBAAoB,CAAC,CAAC;QACzF,CAAC;QACD,OAAO,IAAI,CAAC,QAAQ,CAAC;IACvB,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,MAAe,EAAE,OAAwB;QACtD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;QAE3C,MAAM,MAAM,GAAG,QAAQ;aACpB,OAAO,CAAC,iBAAiB,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;aACvE,OAAO,CAAC,iBAAiB,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;aACvE,OAAO,CAAC,kBAAkB,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;aACzE,OAAO,CAAC,aAAa,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC;QAE1C,MAAM,SAAS,GAAG,UAAU,IAAI,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,CAAC;QACtD,MAAM,gBAAgB,GAAG,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,gBAAgB,CAAC,CAAC,CAAC;QACxF,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,GAAG,SAAS,MAAM,CAAC,CAAC;QAEhE,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,cAAc,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;QACvE,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;QAE1C,MAAM,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QAE7C,OAAO,QAAQ,CAAC;IAClB,CAAC;IAEO,KAAK,CAAC,eAAe,CAAC,QAAkB,EAAE,MAAe;QAC/D,MAAM,QAAQ,GAAG,QAAQ,CAAC,UAAU,CAAC,QAAQ,CAAC;QAC9C,MAAM,SAAS,GAAG,CAAC,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC;QAElE,QAAQ,QAAQ,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC;YACnC,KAAK,oBAAoB,CAAC,CAAC,CAAC;gBAC1B,IAAI,CAAC,QAAQ;oBAAE,MAAM;gBACrB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,gBAAgB,EAAE,CAAC;gBACjD,KAAK,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,OAAO,EAAE,CAAC;oBACrC,IAAI,IAAI,CAAC,IAAI,KAAK,QAAQ,IAAI,IAAI,CAAC,QAAQ,KAAM,MAAkC,CAAC,QAAQ,EAAE,CAAC;wBAC7F,oDAAoD;wBACpD,MAAM;oBACR,CAAC;gBACH,CAAC;gBACD,MAAM;YACR,CAAC;YACD,KAAK,aAAa;gBAChB,MAAM;YACR,KAAK,UAAU,CAAC;YAChB,KAAK,gBAAgB,CAAC;YACtB,KAAK,MAAM;gBACT,MAAM;QACV,CAAC;QAED,IAAI,QAAQ,CAAC,QAAQ,KAAK,UAAU,IAAI,QAAQ,CAAC,gBAAgB,IAAI,MAAM,CAAC,aAAa,EAAE,CAAC;YAC1F,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,CAC3B,IAAI,CAAC,gBAAgB,EACrB,QAAQ,EACR,QAAQ,CAAC,gBAAgB,EACzB,MAAM,CAAC,aAAa,CACrB,CAAC;QACJ,CAAC;IACH,CAAC;IAEO,WAAW,CAAC,MAAc;QAChC,MAAM,OAAO,GAAG,MAAM;aACnB,OAAO,CAAC,aAAa,EAAE,EAAE,CAAC;aAC1B,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC;aACtB,IAAI,EAAE,CAAC;QAEV,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAEnC,OAAO;YACL,QAAQ,EAAE,MAAM,CAAC,QAAQ,IAAI,MAAM;YACnC,MAAM,EAAE,MAAM,CAAC,MAAM,IAAI,EAAE;YAC3B,gBAAgB,EAAE,MAAM,CAAC,kBAAkB;YAC3C,UAAU,EAAE;gBACV,MAAM,EAAE,MAAM,CAAC,WAAW,EAAE,MAAM,IAAI,MAAM;gBAC5C,QAAQ,EAAE,MAAM,CAAC,WAAW,EAAE,SAAS,IAAI,IAAI;gBAC/C,eAAe,EAAE,MAAM,CAAC,WAAW,EAAE,gBAAgB,IAAI,IAAI;gBAC7D,eAAe,EAAE,MAAM,CAAC,WAAW,EAAE,iBAAiB,IAAI,IAAI;aAC/D;SACF,CAAC;IACJ,CAAC;CACF"}

package/dist/leader/event-bus.d.ts

@@ -0,0 +1,11 @@
+export type LeaderEventType = "worker_joined" | "worker_left" | "worker_status_changed" | "task_created" | "task_claimed" | "task_completed" | "task_blocked" | "task_failed" | "task_recovered" | "message_received" | "message_processed";
+export interface LeaderEvent {
+    type: LeaderEventType;
+    [key: string]: unknown;
+}
+export declare class LeaderEventBus {
+    private emitter;
+    onAll(handler: (event: LeaderEvent) => void): void;
+    emit(event: LeaderEvent): void;
+    on(type: LeaderEventType, handler: (event: LeaderEvent) => void): void;
+}

package/dist/leader/event-bus.js

@@ -0,0 +1,21 @@
+import { EventEmitter } from "node:events";
+const ALL_EVENT_TYPES = [
+    "worker_joined", "worker_left", "worker_status_changed",
+    "task_created", "task_claimed", "task_completed", "task_blocked", "task_failed", "task_recovered",
+    "message_received", "message_processed",
+];
+export class LeaderEventBus {
+    emitter = new EventEmitter();
+    onAll(handler) {
+        for (const t of ALL_EVENT_TYPES) {
+            this.emitter.on(t, handler);
+        }
+    }
+    emit(event) {
+        this.emitter.emit(event.type, event);
+    }
+    on(type, handler) {
+        this.emitter.on(type, handler);
+    }
+}
+//# sourceMappingURL=event-bus.js.map

package/dist/leader/event-bus.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"event-bus.js","sourceRoot":"","sources":["../../src/leader/event-bus.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAoB3C,MAAM,eAAe,GAAsB;IACzC,eAAe,EAAE,aAAa,EAAE,uBAAuB;IACvD,cAAc,EAAE,cAAc,EAAE,gBAAgB,EAAE,cAAc,EAAE,aAAa,EAAE,gBAAgB;IACjG,kBAAkB,EAAE,mBAAmB;CACxC,CAAC;AAEF,MAAM,OAAO,cAAc;IACjB,OAAO,GAAG,IAAI,YAAY,EAAE,CAAC;IAErC,KAAK,CAAC,OAAqC;QACzC,KAAK,MAAM,CAAC,IAAI,eAAe,EAAE,CAAC;YAChC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;QAC9B,CAAC;IACH,CAAC;IAED,IAAI,CAAC,KAAkB;QACrB,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IACvC,CAAC;IAED,EAAE,CAAC,IAAqB,EAAE,OAAqC;QAC7D,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IACjC,CAAC;CACF"}

package/dist/leader/index.d.ts

@@ -0,0 +1,7 @@
+export declare function startLeader(config: {
+    zkHosts: string;
+    name?: string;
+    instanceId?: string;
+    command?: string;
+    cacheDir?: string;
+}): Promise<void>;

package/dist/leader/index.js

@@ -0,0 +1,96 @@
+import * as os from "node:os";
+import * as path from "node:path";
+import * as fs from "node:fs";
+import { ZkClient, isNodeExists } from "../zk/client.js";
+import { InstanceRegistry } from "../modules/registry.js";
+import { LeaderEventBus } from "./event-bus.js";
+import { LeaderState } from "./state.js";
+import { WorkerMonitor } from "./monitor.js";
+import { TaskOrchestrator } from "./orchestrator.js";
+import { TaskRecovery } from "./recovery.js";
+import { LeaderWatcher } from "./watcher.js";
+import { TaskGenerator } from "./task-generator.js";
+import { DecisionEngine } from "./decision-engine.js";
+import { TaskQueue } from "../modules/task-queue.js";
+import { MessageRouter } from "../modules/message-router.js";
+import { expandHomeDir, loadGlobalConfig, loadInstanceConfig, saveInstanceId } from "../config.js";
+import { LeaderTui } from "./tui.js";
+export async function startLeader(config) {
+    const zk = new ZkClient(config.zkHosts);
+    await zk.connect();
+    const instanceConfig = loadInstanceConfig();
+    const globalConfig = loadGlobalConfig();
+    const leaderName = config.name || instanceConfig.name || "Leader";
+    const command = config.command || globalConfig.commands?.["claude-cli"] || "claude --dangerously-skip-permissions --permission-mode dontAsk";
+    const cacheDir = config.cacheDir || globalConfig.cache_dir || "~/.claude-orchestrator/sessions";
+    // Create /leader EPHEMERAL node
+    const leaderId = crypto.randomUUID().replace(/-/g, "");
+    try {
+        await zk.createLeader({
+            instance_id: leaderId,
+            name: leaderName,
+            role: "leader",
+            started_at: new Date().toISOString(),
+            host: os.hostname(),
+            pid: process.pid,
+            version: "0.3.0",
+        });
+    }
+    catch (err) {
+        if (isNodeExists(err)) {
+            console.error("Another leader is already running.");
+            process.exit(1);
+        }
+        throw err;
+    }
+    // Register own instance
+    const registry = new InstanceRegistry(zk);
+    const instance = await registry.register(leaderName, "leader", leaderId);
+    saveInstanceId(instance.id);
+    // Initialize CACHE_DIR
+    const resolvedCache = expandHomeDir(cacheDir);
+    const myCacheDir = path.join(resolvedCache, instance.id);
+    await fs.promises.mkdir(myCacheDir, { recursive: true });
+    // Initialize EventBus + State
+    const eventBus = new LeaderEventBus();
+    const state = new LeaderState();
+    state.leaderName = leaderName;
+    state.leaderInstanceId = instance.id;
+    state.cacheDir = resolvedCache;
+    eventBus.onAll((event) => state.apply(event));
+    // Initialize TaskQueue and MessageRouter for decision engine
+    const taskQueue = new TaskQueue(zk);
+    const messageRouter = new MessageRouter(zk);
+    const taskGenerator = new TaskGenerator(zk, taskQueue, command, resolvedCache, instance.id);
+    const decisionEngine = new DecisionEngine(zk, taskQueue, messageRouter, command, resolvedCache, instance.id);
+    // Start subsystems
+    const leaderWatcher = new LeaderWatcher(zk, eventBus, instance.id, command, resolvedCache, decisionEngine);
+    await leaderWatcher.start();
+    const monitor = new WorkerMonitor(zk, eventBus);
+    await monitor.start();
+    const orchestrator = new TaskOrchestrator(zk, eventBus);
+    await orchestrator.start();
+    const recovery = new TaskRecovery(zk, eventBus);
+    recovery.start();
+    await recovery.scanOrphans();
+    // Initialize TUI
+    const tui = new LeaderTui();
+    eventBus.onAll(() => tui.render(state));
+    tui.render(state);
+    // Block on SIGINT
+    await new Promise((resolve) => {
+        const cleanup = () => {
+            tui.destroy();
+            resolve();
+        };
+        process.once("SIGINT", cleanup);
+        process.once("SIGTERM", cleanup);
+    });
+    // Shutdown: stop watchers first, then disconnect
+    leaderWatcher.stop();
+    monitor.stop();
+    orchestrator.stop();
+    await new Promise(r => setTimeout(r, 100)); // let pending callbacks drain
+    await zk.disconnect();
+}
+//# sourceMappingURL=index.js.map

package/dist/leader/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/leader/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AACzD,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAC1D,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAChD,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACzC,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAC7C,OAAO,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAC7C,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAC7C,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAE,SAAS,EAAE,MAAM,0BAA0B,CAAC;AACrD,OAAO,EAAE,aAAa,EAAE,MAAM,8BAA8B,CAAC;AAC7D,OAAO,EAAE,aAAa,EAAE,gBAAgB,EAAE,kBAAkB,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AACnG,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AAErC,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,MAMjC;IACC,MAAM,EAAE,GAAG,IAAI,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IACxC,MAAM,EAAE,CAAC,OAAO,EAAE,CAAC;IAEnB,MAAM,cAAc,GAAG,kBAAkB,EAAE,CAAC;IAC5C,MAAM,YAAY,GAAG,gBAAgB,EAAE,CAAC;IACxC,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,IAAI,cAAc,CAAC,IAAI,IAAI,QAAQ,CAAC;IAClE,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,YAAY,CAAC,QAAQ,EAAE,CAAC,YAAY,CAAC,IAAI,iEAAiE,CAAC;IAC7I,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,IAAI,YAAY,CAAC,SAAS,IAAI,iCAAiC,CAAC;IAEhG,gCAAgC;IAChC,MAAM,QAAQ,GAAG,MAAM,CAAC,UAAU,EAAE,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;IACvD,IAAI,CAAC;QACH,MAAM,EAAE,CAAC,YAAY,CAAC;YACpB,WAAW,EAAE,QAAQ;YACrB,IAAI,EAAE,UAAU;YAChB,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACpC,IAAI,EAAE,EAAE,CAAC,QAAQ,EAAE;YACnB,GAAG,EAAE,OAAO,CAAC,GAAG;YAChB,OAAO,EAAE,OAAO;SACjB,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,IAAI,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC;YACtB,OAAO,CAAC,KAAK,CAAC,oCAAoC,CAAC,CAAC;YACpD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;QACD,MAAM,GAAG,CAAC;IACZ,CAAC;IAED,wBAAwB;IACxB,MAAM,QAAQ,GAAG,IAAI,gBAAgB,CAAC,EAAE,CAAC,CAAC;IAC1C,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC,UAAU,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC;IACzE,cAAc,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;IAE5B,uBAAuB;IACvB,MAAM,aAAa,GAAG,aAAa,CAAC,QAAQ,CAAC,CAAC;IAC9C,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,QAAQ,CAAC,EAAE,CAAC,CAAC;IACzD,MAAM,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAEzD,8BAA8B;IAC9B,MAAM,QAAQ,GAAG,IAAI,cAAc,EAAE,CAAC;IACtC,MAAM,KAAK,GAAG,IAAI,WAAW,EAAE,CAAC;IAChC,KAAK,CAAC,UAAU,GAAG,UAAU,CAAC;IAC9B,KAAK,CAAC,gBAAgB,GAAG,QAAQ,CAAC,EAAE,CAAC;IACrC,KAAK,CAAC,QAAQ,GAAG,aAAa,CAAC;IAE/B,QAAQ,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;IAE9C,6DAA6D;IAC7D,MAAM,SAAS,GAAG,IAAI,SAAS,CAAC,EAAE,CAAC,CAAC;IACpC,MAAM,aAAa,GAAG,IAAI,aAAa,CAAC,EAAE,CAAC,CAAC;IAE5C,MAAM,aAAa,GAAG,IAAI,aAAa,CAAC,EAAE,EAAE,SAAS,EAAE,OAAO,EAAE,aAAa,EAAE,QAAQ,CAAC,EAAE,CAAC,CAAC;IAC5F,MAAM,cAAc,GAAG,IAAI,cAAc,CAAC,EAAE,EAAE,SAAS,EAAE,aAAa,EAAE,OAAO,EAAE,aAAa,EAAE,QAAQ,CAAC,EAAE,CAAC,CAAC;IAE7G,mBAAmB;IACnB,MAAM,aAAa,GAAG,IAAI,aAAa,CAAC,EAAE,EAAE,QAAQ,EAAE,QAAQ,CAAC,EAAE,EAAE,OAAO,EAAE,aAAa,EAAE,cAAc,CAAC,CAAC;IAC3G,MAAM,aAAa,CAAC,KAAK,EAAE,CAAC;IAE5B,MAAM,OAAO,GAAG,IAAI,aAAa,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC;IAChD,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;IAEtB,MAAM,YAAY,GAAG,IAAI,gBAAgB,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC;IACxD,MAAM,YAAY,CAAC,KAAK,EAAE,CAAC;IAE3B,MAAM,QAAQ,GAAG,IAAI,YAAY,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC;IAChD,QAAQ,CAAC,KAAK,EAAE,CAAC;IACjB,MAAM,QAAQ,CAAC,WAAW,EAAE,CAAC;IAE7B,iBAAiB;IACjB,MAAM,GAAG,GAAG,IAAI,SAAS,EAAE,CAAC;IAC5B,QAAQ,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;IACxC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAElB,kBAAkB;IAClB,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;QAClC,MAAM,OAAO,GAAG,GAAG,EAAE;YACnB,GAAG,CAAC,OAAO,EAAE,CAAC;YACd,OAAO,EAAE,CAAC;QACZ,CAAC,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAChC,OAAO,CAAC,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;IACnC,CAAC,CAAC,CAAC;IAEH,iDAAiD;IACjD,aAAa,CAAC,IAAI,EAAE,CAAC;IACrB,OAAO,CAAC,IAAI,EAAE,CAAC;IACf,YAAY,CAAC,IAAI,EAAE,CAAC;IACpB,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,8BAA8B;IAC1E,MAAM,EAAE,CAAC,UAAU,EAAE,CAAC;AACxB,CAAC"}

package/dist/leader/monitor.d.ts

@@ -0,0 +1,14 @@
+import { ZkClient } from "../zk/client.js";
+import { LeaderEventBus } from "./event-bus.js";
+export declare class WorkerMonitor {
+    private zk;
+    private eventBus;
+    private knownInstances;
+    private instanceNames;
+    private stopped;
+    constructor(zk: ZkClient, eventBus: LeaderEventBus);
+    start(): Promise<void>;
+    stop(): void;
+    private watchLoop;
+    private onChildrenChanged;
+}

package/dist/leader/monitor.js

@@ -0,0 +1,55 @@
+export class WorkerMonitor {
+    zk;
+    eventBus;
+    knownInstances = new Set();
+    instanceNames = new Map();
+    stopped = false;
+    constructor(zk, eventBus) {
+        this.zk = zk;
+        this.eventBus = eventBus;
+    }
+    async start() {
+        await this.watchLoop();
+    }
+    stop() {
+        this.stopped = true;
+    }
+    async watchLoop() {
+        if (this.stopped)
+            return;
+        const children = await this.zk.watchInstances(async (newChildren) => {
+            if (this.stopped)
+                return;
+            await this.onChildrenChanged(newChildren);
+            this.watchLoop();
+        });
+        await this.onChildrenChanged(children);
+    }
+    async onChildrenChanged(children) {
+        const curr = new Set(children);
+        for (const id of curr) {
+            if (!this.knownInstances.has(id)) {
+                const data = await this.zk.getInstance(id);
+                if (data && data.role !== "leader") {
+                    const instName = data.name;
+                    this.eventBus.emit({
+                        type: "worker_joined",
+                        instance: data,
+                        instanceId: id,
+                        name: instName,
+                    });
+                    this.instanceNames.set(id, instName);
+                }
+            }
+        }
+        for (const id of this.knownInstances) {
+            if (!curr.has(id)) {
+                const name = this.instanceNames.get(id) ?? id.slice(0, 8);
+                this.eventBus.emit({ type: "worker_left", instanceId: id, name });
+                this.instanceNames.delete(id);
+            }
+        }
+        this.knownInstances = curr;
+    }
+}
+//# sourceMappingURL=monitor.js.map

package/dist/leader/monitor.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"monitor.js","sourceRoot":"","sources":["../../src/leader/monitor.ts"],"names":[],"mappings":"AAGA,MAAM,OAAO,aAAa;IAMd;IACA;IANF,cAAc,GAAG,IAAI,GAAG,EAAU,CAAC;IACnC,aAAa,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC1C,OAAO,GAAG,KAAK,CAAC;IAExB,YACU,EAAY,EACZ,QAAwB;QADxB,OAAE,GAAF,EAAE,CAAU;QACZ,aAAQ,GAAR,QAAQ,CAAgB;IAC/B,CAAC;IAEJ,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;IACzB,CAAC;IAED,IAAI;QACF,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;IACtB,CAAC;IAEO,KAAK,CAAC,SAAS;QACrB,IAAI,IAAI,CAAC,OAAO;YAAE,OAAO;QACzB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,cAAc,CAAC,KAAK,EAAE,WAAW,EAAE,EAAE;YAClE,IAAI,IAAI,CAAC,OAAO;gBAAE,OAAO;YACzB,MAAM,IAAI,CAAC,iBAAiB,CAAC,WAAW,CAAC,CAAC;YAC1C,IAAI,CAAC,SAAS,EAAE,CAAC;QACnB,CAAC,CAAC,CAAC;QACH,MAAM,IAAI,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAC;IACzC,CAAC;IAEO,KAAK,CAAC,iBAAiB,CAAC,QAAkB;QAChD,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC;QAE/B,KAAK,MAAM,EAAE,IAAI,IAAI,EAAE,CAAC;YACtB,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;gBACjC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;gBAC3C,IAAI,IAAI,IAAI,IAAI,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;oBACnC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAc,CAAC;oBACrC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;wBACjB,IAAI,EAAE,eAAe;wBACrB,QAAQ,EAAE,IAAI;wBACd,UAAU,EAAE,EAAE;wBACd,IAAI,EAAE,QAAQ;qBACf,CAAC,CAAC;oBACH,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC;gBACvC,CAAC;YACH,CAAC;QACH,CAAC;QAED,KAAK,MAAM,EAAE,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACrC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;gBAClB,MAAM,IAAI,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;gBAC1D,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,aAAa,EAAE,UAAU,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;gBAClE,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;YAChC,CAAC;QACH,CAAC;QAED,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC;IAC7B,CAAC;CACF"}

package/dist/leader/orchestrator.d.ts

@@ -0,0 +1,14 @@
+import { ZkClient } from "../zk/client.js";
+import { LeaderEventBus } from "./event-bus.js";
+export declare class TaskOrchestrator {
+    private zk;
+    private eventBus;
+    private knownPending;
+    private knownClaimed;
+    private stopped;
+    constructor(zk: ZkClient, eventBus: LeaderEventBus);
+    start(): Promise<void>;
+    stop(): void;
+    private watchPending;
+    private watchClaimed;
+}

package/dist/leader/orchestrator.js

@@ -0,0 +1,83 @@
+export class TaskOrchestrator {
+    zk;
+    eventBus;
+    knownPending = new Set();
+    knownClaimed = new Set();
+    stopped = false;
+    constructor(zk, eventBus) {
+        this.zk = zk;
+        this.eventBus = eventBus;
+    }
+    async start() {
+        await this.watchPending();
+        await this.watchClaimed();
+    }
+    stop() {
+        this.stopped = true;
+    }
+    async watchPending() {
+        if (this.stopped)
+            return;
+        const children = await this.zk.watchPendingTasks(async (newChildren) => {
+            if (this.stopped)
+                return;
+            for (const id of newChildren) {
+                if (!this.knownPending.has(id)) {
+                    const data = await this.zk.getPendingTask(id);
+                    if (data) {
+                        this.eventBus.emit({ type: "task_created", task: { ...data, id }, taskId: id });
+                    }
+                }
+            }
+            this.knownPending = new Set(newChildren);
+            this.watchPending();
+        });
+        for (const id of children) {
+            const data = await this.zk.getPendingTask(id);
+            if (data) {
+                this.eventBus.emit({ type: "task_created", task: { ...data, id }, taskId: id });
+            }
+        }
+        this.knownPending = new Set(children);
+    }
+    async watchClaimed() {
+        if (this.stopped)
+            return;
+        const children = await this.zk.watchClaimedTasks(async (newChildren) => {
+            if (this.stopped)
+                return;
+            for (const name of newChildren) {
+                if (!this.knownClaimed.has(name)) {
+                    const idx = name.indexOf("-");
+                    if (idx === -1)
+                        continue;
+                    const insId = name.substring(0, idx);
+                    const taskId = name.substring(idx + 1);
+                    this.eventBus.emit({ type: "task_claimed", taskId, instanceId: insId });
+                }
+            }
+            // Detect completed/released tasks
+            for (const name of this.knownClaimed) {
+                if (!newChildren.includes(name)) {
+                    const idx = name.indexOf("-");
+                    if (idx !== -1) {
+                        const taskId = name.substring(idx + 1);
+                        this.eventBus.emit({ type: "task_completed", taskId });
+                    }
+                }
+            }
+            this.knownClaimed = new Set(newChildren);
+            this.watchClaimed();
+        });
+        for (const name of children) {
+            const idx = name.indexOf("-");
+            if (idx === -1)
+                continue;
+            const insId = name.substring(0, idx);
+            const taskId = name.substring(idx + 1);
+            this.eventBus.emit({ type: "task_claimed", taskId, instanceId: insId });
+        }
+        this.knownClaimed = new Set(children);
+    }
+}
+//# sourceMappingURL=orchestrator.js.map

package/dist/leader/orchestrator.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"orchestrator.js","sourceRoot":"","sources":["../../src/leader/orchestrator.ts"],"names":[],"mappings":"AAGA,MAAM,OAAO,gBAAgB;IAMjB;IACA;IANF,YAAY,GAAG,IAAI,GAAG,EAAU,CAAC;IACjC,YAAY,GAAG,IAAI,GAAG,EAAU,CAAC;IACjC,OAAO,GAAG,KAAK,CAAC;IAExB,YACU,EAAY,EACZ,QAAwB;QADxB,OAAE,GAAF,EAAE,CAAU;QACZ,aAAQ,GAAR,QAAQ,CAAgB;IAC/B,CAAC;IAEJ,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;QAC1B,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;IAC5B,CAAC;IAED,IAAI;QACF,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;IACtB,CAAC;IAEO,KAAK,CAAC,YAAY;QACxB,IAAI,IAAI,CAAC,OAAO;YAAE,OAAO;QACzB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,iBAAiB,CAAC,KAAK,EAAE,WAAW,EAAE,EAAE;YACrE,IAAI,IAAI,CAAC,OAAO;gBAAE,OAAO;YACzB,KAAK,MAAM,EAAE,IAAI,WAAW,EAAE,CAAC;gBAC7B,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;oBAC/B,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC;oBAC9C,IAAI,IAAI,EAAE,CAAC;wBACT,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,cAAc,EAAE,IAAI,EAAE,EAAE,GAAG,IAAI,EAAE,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,CAAC;oBAClF,CAAC;gBACH,CAAC;YACH,CAAC;YACD,IAAI,CAAC,YAAY,GAAG,IAAI,GAAG,CAAC,WAAW,CAAC,CAAC;YACzC,IAAI,CAAC,YAAY,EAAE,CAAC;QACtB,CAAC,CAAC,CAAC;QACH,KAAK,MAAM,EAAE,IAAI,QAAQ,EAAE,CAAC;YAC1B,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC;YAC9C,IAAI,IAAI,EAAE,CAAC;gBACT,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,cAAc,EAAE,IAAI,EAAE,EAAE,GAAG,IAAI,EAAE,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,CAAC;YAClF,CAAC;QACH,CAAC;QACD,IAAI,CAAC,YAAY,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC;IACxC,CAAC;IAEO,KAAK,CAAC,YAAY;QACxB,IAAI,IAAI,CAAC,OAAO;YAAE,OAAO;QACzB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,iBAAiB,CAAC,KAAK,EAAE,WAAW,EAAE,EAAE;YACrE,IAAI,IAAI,CAAC,OAAO;gBAAE,OAAO;YACzB,KAAK,MAAM,IAAI,IAAI,WAAW,EAAE,CAAC;gBAC/B,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;oBACjC,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;oBAC9B,IAAI,GAAG,KAAK,CAAC,CAAC;wBAAE,SAAS;oBACzB,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;oBACrC,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC;oBACvC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,cAAc,EAAE,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC,CAAC;gBAC1E,CAAC;YACH,CAAC;YACD,kCAAkC;YAClC,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;gBACrC,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;oBAChC,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;oBAC9B,IAAI,GAAG,KAAK,CAAC,CAAC,EAAE,CAAC;wBACf,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC;wBACvC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,gBAAgB,EAAE,MAAM,EAAE,CAAC,CAAC;oBACzD,CAAC;gBACH,CAAC;YACH,CAAC;YACD,IAAI,CAAC,YAAY,GAAG,IAAI,GAAG,CAAC,WAAW,CAAC,CAAC;YACzC,IAAI,CAAC,YAAY,EAAE,CAAC;QACtB,CAAC,CAAC,CAAC;QACH,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;YAC5B,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;YAC9B,IAAI,GAAG,KAAK,CAAC,CAAC;gBAAE,SAAS;YACzB,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;YACrC,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC;YACvC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,cAAc,EAAE,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC,CAAC;QAC1E,CAAC;QACD,IAAI,CAAC,YAAY,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC;IACxC,CAAC;CACF"}

package/dist/leader/recovery.d.ts

@@ -0,0 +1,11 @@
+import { ZkClient } from "../zk/client.js";
+import { LeaderEventBus } from "./event-bus.js";
+export declare class TaskRecovery {
+    private zk;
+    private eventBus;
+    constructor(zk: ZkClient, eventBus: LeaderEventBus);
+    start(): void;
+    scanOrphans(): Promise<void>;
+    private recoverOrphan;
+    private recoverOrphanedTasks;
+}

package/dist/leader/recovery.js

@@ -0,0 +1,61 @@
+const MAX_RETRIES = 3;
+export class TaskRecovery {
+    zk;
+    eventBus;
+    constructor(zk, eventBus) {
+        this.zk = zk;
+        this.eventBus = eventBus;
+    }
+    start() {
+        this.eventBus.on("worker_left", (event) => {
+            this.recoverOrphanedTasks(event.instanceId);
+        });
+    }
+    async scanOrphans() {
+        if (!this.zk.connected)
+            return;
+        const instances = await this.zk.listInstances();
+        const onlineIds = new Set(instances.map((i) => i.id));
+        const claimed = await this.zk.listClaimedTasks();
+        for (const [insId, taskId, data] of claimed) {
+            if (!onlineIds.has(insId)) {
+                await this.recoverOrphan(insId, taskId, data);
+            }
+        }
+    }
+    async recoverOrphan(workerId, taskId, data) {
+        const retryCount = (data.retry_count ?? 0) + 1;
+        if (retryCount > MAX_RETRIES) {
+            await this.zk.saveCompletedTask(taskId, {
+                ...data,
+                status: "failed",
+                completed_at: new Date().toISOString(),
+                retry_count: retryCount,
+                fail_reason: `Max retries (${MAX_RETRIES}) exceeded after worker disconnect`,
+            });
+            await this.zk.deleteClaimedTask(workerId, taskId);
+            this.eventBus.emit({ type: "task_failed", taskId, reason: "Max retries exceeded" });
+        }
+        else {
+            const taskData = { ...data };
+            taskData.retry_count = retryCount;
+            taskData.status = "pending";
+            delete taskData.claimed_by;
+            delete taskData.claimed_at;
+            const newTaskId = await this.zk.createPendingTask(taskData);
+            await this.zk.deleteClaimedTask(workerId, taskId);
+            this.eventBus.emit({ type: "task_recovered", taskId, newTaskId, retryCount });
+        }
+    }
+    async recoverOrphanedTasks(workerId) {
+        if (!this.zk.connected)
+            return;
+        const claimed = await this.zk.listClaimedTasks();
+        for (const [insId, taskId, data] of claimed) {
+            if (insId !== workerId)
+                continue;
+            await this.recoverOrphan(insId, taskId, data);
+        }
+    }
+}
+//# sourceMappingURL=recovery.js.map

package/dist/leader/recovery.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"recovery.js","sourceRoot":"","sources":["../../src/leader/recovery.ts"],"names":[],"mappings":"AAGA,MAAM,WAAW,GAAG,CAAC,CAAC;AAEtB,MAAM,OAAO,YAAY;IAEb;IACA;IAFV,YACU,EAAY,EACZ,QAAwB;QADxB,OAAE,GAAF,EAAE,CAAU;QACZ,aAAQ,GAAR,QAAQ,CAAgB;IAC/B,CAAC;IAEJ,KAAK;QACH,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,aAAa,EAAE,CAAC,KAAK,EAAE,EAAE;YACxC,IAAI,CAAC,oBAAoB,CAAC,KAAK,CAAC,UAAoB,CAAC,CAAC;QACxD,CAAC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,WAAW;QACf,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,SAAS;YAAE,OAAO;QAC/B,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,aAAa,EAAE,CAAC;QAChD,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAY,CAAC,CAAC,CAAC;QAChE,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,gBAAgB,EAAE,CAAC;QACjD,KAAK,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,IAAI,OAAO,EAAE,CAAC;YAC5C,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC1B,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC;YAChD,CAAC;QACH,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,aAAa,CACzB,QAAgB,EAChB,MAAc,EACd,IAA6B;QAE7B,MAAM,UAAU,GAAG,CAAC,IAAI,CAAC,WAAqB,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;QACzD,IAAI,UAAU,GAAG,WAAW,EAAE,CAAC;YAC7B,MAAM,IAAI,CAAC,EAAE,CAAC,iBAAiB,CAAC,MAAM,EAAE;gBACtC,GAAG,IAAI;gBACP,MAAM,EAAE,QAAQ;gBAChB,YAAY,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACtC,WAAW,EAAE,UAAU;gBACvB,WAAW,EAAE,gBAAgB,WAAW,oCAAoC;aAC7E,CAAC,CAAC;YACH,MAAM,IAAI,CAAC,EAAE,CAAC,iBAAiB,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;YAClD,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,aAAa,EAAE,MAAM,EAAE,MAAM,EAAE,sBAAsB,EAAE,CAAC,CAAC;QACtF,CAAC;aAAM,CAAC;YACN,MAAM,QAAQ,GAAG,EAAE,GAAG,IAAI,EAAE,CAAC;YAC7B,QAAQ,CAAC,WAAW,GAAG,UAAU,CAAC;YAClC,QAAQ,CAAC,MAAM,GAAG,SAAS,CAAC;YAC5B,OAAO,QAAQ,CAAC,UAAU,CAAC;YAC3B,OAAO,QAAQ,CAAC,UAAU,CAAC;YAC3B,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAC;YAC5D,MAAM,IAAI,CAAC,EAAE,CAAC,iBAAiB,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;YAClD,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,gBAAgB,EAAE,MAAM,EAAE,SAAS,EAAE,UAAU,EAAE,CAAC,CAAC;QAChF,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,oBAAoB,CAAC,QAAgB;QACjD,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,SAAS;YAAE,OAAO;QAC/B,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,gBAAgB,EAAE,CAAC;QACjD,KAAK,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,IAAI,OAAO,EAAE,CAAC;YAC5C,IAAI,KAAK,KAAK,QAAQ;gBAAE,SAAS;YACjC,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC;QAChD,CAAC;IACH,CAAC;CACF"}

package/dist/leader/state.d.ts

@@ -0,0 +1,24 @@
+import type { LeaderEvent } from "./event-bus.js";
+export interface WorkerInfo {
+    id: string;
+    name: string;
+    presetRole: string;
+    currentRole: string | null;
+    status: string;
+    currentTaskId: string | null;
+}
+export interface EventLogEntry {
+    timestamp: string;
+    message: string;
+}
+export declare class LeaderState {
+    workers: WorkerInfo[];
+    pendingTasks: Record<string, unknown>[];
+    claimedTasks: Record<string, unknown>[];
+    completedTasks: Record<string, unknown>[];
+    events: EventLogEntry[];
+    leaderName: string;
+    leaderInstanceId: string;
+    cacheDir: string;
+    apply(event: LeaderEvent): void;
+}