agent-relay 1.1.0 → 1.2.3

package/dist/resiliency/stateless-lead.d.ts

@@ -0,0 +1,149 @@
+/**
+ * Stateless Lead Coordinator
+ *
+ * Implements P0: Lead reads from Beads, no in-memory task queue.
+ * All task state lives in Beads. If lead crashes, new lead picks up.
+ *
+ * Key principles:
+ * - Lead is a coordinator, not a state holder
+ * - Beads is the single source of truth for task state
+ * - Any agent can become lead by reading from Beads
+ * - Tasks are assigned by updating Beads, not in-memory
+ */
+import { EventEmitter } from 'events';
+/**
+ * Task from Beads
+ */
+export interface BeadsTask {
+    id: string;
+    title: string;
+    description?: string;
+    status: 'open' | 'in_progress' | 'blocked' | 'closed';
+    priority: number;
+    assignee?: string;
+    leaseExpires?: number;
+    tags?: string[];
+    created_at: string;
+    updated_at: string;
+}
+/**
+ * Lead heartbeat (P2)
+ */
+export interface LeadHeartbeat {
+    leadName: string;
+    leadId: string;
+    timestamp: number;
+    activeTaskCount: number;
+    assignedAgents: string[];
+}
+/**
+ * Configuration for stateless lead
+ */
+export interface StatelessLeadConfig {
+    /** Path to .beads directory */
+    beadsDir: string;
+    /** Agent name for this lead */
+    agentName: string;
+    /** Unique agent ID */
+    agentId: string;
+    /** How often to poll Beads for ready tasks (ms) */
+    pollIntervalMs: number;
+    /** Heartbeat interval (ms) */
+    heartbeatIntervalMs: number;
+    /** Lease duration for assigned tasks (ms) - P1 */
+    leaseDurationMs: number;
+    /** Callback to send relay messages */
+    sendRelay: (to: string, message: string) => Promise<void>;
+    /** Callback to get available workers */
+    getAvailableWorkers: () => Promise<string[]>;
+}
+/**
+ * Stateless Lead Coordinator
+ *
+ * Reads tasks from Beads JSONL, assigns to workers, tracks via Beads updates.
+ * No in-memory task queue - all state persisted to Beads.
+ */
+export declare class StatelessLeadCoordinator extends EventEmitter {
+    private config;
+    private issuesPath;
+    private heartbeatPath;
+    private pollInterval?;
+    private heartbeatInterval?;
+    private isRunning;
+    constructor(config: StatelessLeadConfig);
+    /**
+     * Start the lead coordinator loop
+     */
+    start(): Promise<void>;
+    /**
+     * Stop the lead coordinator
+     */
+    stop(): Promise<void>;
+    /**
+     * Read all tasks from Beads JSONL
+     */
+    private readTasks;
+    /**
+     * Update a task in Beads JSONL
+     */
+    private updateTask;
+    /**
+     * Get tasks that are ready to be assigned
+     * Ready = open status, not assigned, not blocked, sorted by priority
+     */
+    private getReadyTasks;
+    /**
+     * Get tasks currently assigned to agents
+     */
+    private getAssignedTasks;
+    /**
+     * Poll for ready tasks and assign to available workers
+     */
+    private pollAndAssign;
+    /**
+     * Assign a task to a worker
+     */
+    private assignTask;
+    /**
+     * Handle task completion from worker
+     */
+    completeTask(taskId: string, worker: string, reason?: string): Promise<void>;
+    /**
+     * Handle task blocked by worker
+     */
+    blockTask(taskId: string, worker: string, reason: string): Promise<void>;
+    /**
+     * Renew lease for a task (worker signals still working)
+     */
+    renewLease(taskId: string, worker: string): Promise<void>;
+    /**
+     * Write leader heartbeat to file (P2)
+     */
+    private writeHeartbeat;
+    /**
+     * Read current leader heartbeat (for watchdog - P3)
+     */
+    static readHeartbeat(beadsDir: string): Promise<LeadHeartbeat | null>;
+    /**
+     * Check if leader is stale (for watchdog - P3)
+     */
+    static isLeaderStale(beadsDir: string, staleThresholdMs?: number): Promise<boolean>;
+    /**
+     * Get current status
+     */
+    getStatus(): Promise<{
+        isRunning: boolean;
+        leadName: string;
+        readyTasks: number;
+        assignedTasks: number;
+        lastHeartbeat: number | null;
+    }>;
+}
+/**
+ * Create a stateless lead coordinator with defaults
+ */
+export declare function createStatelessLead(beadsDir: string, agentName: string, agentId: string, callbacks: {
+    sendRelay: (to: string, message: string) => Promise<void>;
+    getAvailableWorkers: () => Promise<string[]>;
+}): StatelessLeadCoordinator;
+//# sourceMappingURL=stateless-lead.d.ts.map

package/dist/resiliency/stateless-lead.js

@@ -0,0 +1,308 @@
+/**
+ * Stateless Lead Coordinator
+ *
+ * Implements P0: Lead reads from Beads, no in-memory task queue.
+ * All task state lives in Beads. If lead crashes, new lead picks up.
+ *
+ * Key principles:
+ * - Lead is a coordinator, not a state holder
+ * - Beads is the single source of truth for task state
+ * - Any agent can become lead by reading from Beads
+ * - Tasks are assigned by updating Beads, not in-memory
+ */
+import * as fs from 'fs';
+import * as path from 'path';
+import { EventEmitter } from 'events';
+const DEFAULT_CONFIG = {
+    pollIntervalMs: 5000,
+    heartbeatIntervalMs: 10000,
+    leaseDurationMs: 300000, // 5 minutes
+};
+/**
+ * Stateless Lead Coordinator
+ *
+ * Reads tasks from Beads JSONL, assigns to workers, tracks via Beads updates.
+ * No in-memory task queue - all state persisted to Beads.
+ */
+export class StatelessLeadCoordinator extends EventEmitter {
+    config;
+    issuesPath;
+    heartbeatPath;
+    pollInterval;
+    heartbeatInterval;
+    isRunning = false;
+    constructor(config) {
+        super();
+        this.config = { ...DEFAULT_CONFIG, ...config };
+        this.issuesPath = path.join(this.config.beadsDir, 'issues.jsonl');
+        this.heartbeatPath = path.join(this.config.beadsDir, 'leader-heartbeat.json');
+    }
+    /**
+     * Start the lead coordinator loop
+     */
+    async start() {
+        if (this.isRunning)
+            return;
+        this.isRunning = true;
+        console.log(`[stateless-lead] Starting lead coordinator: ${this.config.agentName}`);
+        // Write initial heartbeat
+        await this.writeHeartbeat();
+        // Start polling for ready tasks
+        this.pollInterval = setInterval(async () => {
+            try {
+                await this.pollAndAssign();
+            }
+            catch (err) {
+                console.error('[stateless-lead] Poll error:', err);
+                this.emit('error', err);
+            }
+        }, this.config.pollIntervalMs);
+        // Start heartbeat
+        this.heartbeatInterval = setInterval(async () => {
+            try {
+                await this.writeHeartbeat();
+            }
+            catch (err) {
+                console.error('[stateless-lead] Heartbeat error:', err);
+            }
+        }, this.config.heartbeatIntervalMs);
+        // Initial poll
+        await this.pollAndAssign();
+        this.emit('started', { leadName: this.config.agentName });
+    }
+    /**
+     * Stop the lead coordinator
+     */
+    async stop() {
+        if (!this.isRunning)
+            return;
+        this.isRunning = false;
+        if (this.pollInterval) {
+            clearInterval(this.pollInterval);
+            this.pollInterval = undefined;
+        }
+        if (this.heartbeatInterval) {
+            clearInterval(this.heartbeatInterval);
+            this.heartbeatInterval = undefined;
+        }
+        console.log(`[stateless-lead] Stopped lead coordinator: ${this.config.agentName}`);
+        this.emit('stopped', { leadName: this.config.agentName });
+    }
+    /**
+     * Read all tasks from Beads JSONL
+     */
+    async readTasks() {
+        if (!fs.existsSync(this.issuesPath)) {
+            return [];
+        }
+        const content = await fs.promises.readFile(this.issuesPath, 'utf-8');
+        const tasks = [];
+        for (const line of content.split('\n')) {
+            if (!line.trim())
+                continue;
+            try {
+                tasks.push(JSON.parse(line));
+            }
+            catch {
+                // Skip malformed lines
+            }
+        }
+        return tasks;
+    }
+    /**
+     * Update a task in Beads JSONL
+     */
+    async updateTask(taskId, updates) {
+        const tasks = await this.readTasks();
+        const updated = tasks.map((task) => {
+            if (task.id === taskId) {
+                return {
+                    ...task,
+                    ...updates,
+                    updated_at: new Date().toISOString(),
+                };
+            }
+            return task;
+        });
+        const content = updated.map((t) => JSON.stringify(t)).join('\n') + '\n';
+        await fs.promises.writeFile(this.issuesPath, content, 'utf-8');
+    }
+    /**
+     * Get tasks that are ready to be assigned
+     * Ready = open status, not assigned, not blocked, sorted by priority
+     */
+    async getReadyTasks() {
+        const tasks = await this.readTasks();
+        const now = Date.now();
+        return tasks
+            .filter((task) => {
+            // Must be open
+            if (task.status !== 'open')
+                return false;
+            // Not assigned, or lease expired (P1)
+            if (task.assignee) {
+                if (task.leaseExpires && task.leaseExpires > now) {
+                    return false; // Still leased
+                }
+                // Lease expired - task is available again
+            }
+            return true;
+        })
+            .sort((a, b) => a.priority - b.priority); // Lower priority number = higher priority
+    }
+    /**
+     * Get tasks currently assigned to agents
+     */
+    async getAssignedTasks() {
+        const tasks = await this.readTasks();
+        const now = Date.now();
+        return tasks.filter((task) => {
+            if (task.status !== 'in_progress')
+                return false;
+            if (!task.assignee)
+                return false;
+            // Check lease not expired
+            if (task.leaseExpires && task.leaseExpires <= now)
+                return false;
+            return true;
+        });
+    }
+    /**
+     * Poll for ready tasks and assign to available workers
+     */
+    async pollAndAssign() {
+        const readyTasks = await this.getReadyTasks();
+        if (readyTasks.length === 0)
+            return;
+        const workers = await this.config.getAvailableWorkers();
+        if (workers.length === 0) {
+            console.log('[stateless-lead] No available workers');
+            return;
+        }
+        // Assign one task per available worker
+        for (const worker of workers) {
+            const task = readyTasks.shift();
+            if (!task)
+                break;
+            await this.assignTask(task, worker);
+        }
+    }
+    /**
+     * Assign a task to a worker
+     */
+    async assignTask(task, worker) {
+        const leaseExpires = Date.now() + this.config.leaseDurationMs;
+        // Update Beads first (source of truth)
+        await this.updateTask(task.id, {
+            status: 'in_progress',
+            assignee: worker,
+            leaseExpires,
+        });
+        // Send task to worker via relay
+        const message = `TASK [${task.id}]: ${task.title}${task.description ? '\n\n' + task.description : ''}`;
+        await this.config.sendRelay(worker, message);
+        console.log(`[stateless-lead] Assigned ${task.id} to ${worker} (lease expires in ${this.config.leaseDurationMs / 1000}s)`);
+        this.emit('assigned', { taskId: task.id, worker, leaseExpires });
+    }
+    /**
+     * Handle task completion from worker
+     */
+    async completeTask(taskId, worker, reason) {
+        await this.updateTask(taskId, {
+            status: 'closed',
+            assignee: worker,
+        });
+        console.log(`[stateless-lead] Task ${taskId} completed by ${worker}${reason ? ': ' + reason : ''}`);
+        this.emit('completed', { taskId, worker, reason });
+    }
+    /**
+     * Handle task blocked by worker
+     */
+    async blockTask(taskId, worker, reason) {
+        await this.updateTask(taskId, {
+            status: 'blocked',
+            assignee: worker,
+        });
+        console.log(`[stateless-lead] Task ${taskId} blocked by ${worker}: ${reason}`);
+        this.emit('blocked', { taskId, worker, reason });
+    }
+    /**
+     * Renew lease for a task (worker signals still working)
+     */
+    async renewLease(taskId, worker) {
+        const leaseExpires = Date.now() + this.config.leaseDurationMs;
+        await this.updateTask(taskId, { leaseExpires });
+        console.log(`[stateless-lead] Renewed lease for ${taskId} (${worker})`);
+        this.emit('leaseRenewed', { taskId, worker, leaseExpires });
+    }
+    /**
+     * Write leader heartbeat to file (P2)
+     */
+    async writeHeartbeat() {
+        const assignedTasks = await this.getAssignedTasks();
+        const assignedAgents = [...new Set(assignedTasks.map((t) => t.assignee).filter(Boolean))];
+        const heartbeat = {
+            leadName: this.config.agentName,
+            leadId: this.config.agentId,
+            timestamp: Date.now(),
+            activeTaskCount: assignedTasks.length,
+            assignedAgents,
+        };
+        await fs.promises.writeFile(this.heartbeatPath, JSON.stringify(heartbeat, null, 2), 'utf-8');
+    }
+    /**
+     * Read current leader heartbeat (for watchdog - P3)
+     */
+    static async readHeartbeat(beadsDir) {
+        const heartbeatPath = path.join(beadsDir, 'leader-heartbeat.json');
+        if (!fs.existsSync(heartbeatPath)) {
+            return null;
+        }
+        try {
+            const content = await fs.promises.readFile(heartbeatPath, 'utf-8');
+            return JSON.parse(content);
+        }
+        catch {
+            return null;
+        }
+    }
+    /**
+     * Check if leader is stale (for watchdog - P3)
+     */
+    static async isLeaderStale(beadsDir, staleThresholdMs = 30000) {
+        const heartbeat = await StatelessLeadCoordinator.readHeartbeat(beadsDir);
+        if (!heartbeat)
+            return true;
+        return Date.now() - heartbeat.timestamp > staleThresholdMs;
+    }
+    /**
+     * Get current status
+     */
+    async getStatus() {
+        const readyTasks = await this.getReadyTasks();
+        const assignedTasks = await this.getAssignedTasks();
+        const heartbeat = await StatelessLeadCoordinator.readHeartbeat(this.config.beadsDir);
+        return {
+            isRunning: this.isRunning,
+            leadName: this.config.agentName,
+            readyTasks: readyTasks.length,
+            assignedTasks: assignedTasks.length,
+            lastHeartbeat: heartbeat?.timestamp ?? null,
+        };
+    }
+}
+/**
+ * Create a stateless lead coordinator with defaults
+ */
+export function createStatelessLead(beadsDir, agentName, agentId, callbacks) {
+    return new StatelessLeadCoordinator({
+        beadsDir,
+        agentName,
+        agentId,
+        ...callbacks,
+        pollIntervalMs: 5000,
+        heartbeatIntervalMs: 10000,
+        leaseDurationMs: 300000,
+    });
+}
+//# sourceMappingURL=stateless-lead.js.map

package/dist/resiliency/supervisor.d.ts

@@ -9,6 +9,7 @@ import { AgentHealthMonitor, HealthMonitorConfig } from './health-monitor.js';
 import { LogLevel } from './logger.js';
 import { metrics } from './metrics.js';
 import { ProviderType } from './provider-context.js';
+import { LeadHeartbeat } from './stateless-lead.js';
 export interface SupervisedAgent {
     name: string;
     cli: string;
@@ -33,6 +34,14 @@ export interface SupervisorConfig {
         baseDir?: string;
         autoInjectOnRestart: boolean;
     };
+    /** Leader coordination (P0-P3) */
+    leaderCoordination?: {
+        enabled: boolean;
+        /** Path to .beads directory */
+        beadsDir: string;
+        /** Callback to send relay messages */
+        sendRelay?: (to: string, message: string) => Promise<void>;
+    };
 }
 export declare class AgentSupervisor extends EventEmitter {
     private config;
@@ -42,6 +51,9 @@ export declare class AgentSupervisor extends EventEmitter {
     private restarters;
     private contextPersistence?;
     private contextHandlers;
+    private leaderWatchdog?;
+    private leadCoordinator?;
+    private supervisorAgentId;
     constructor(config?: Partial<SupervisorConfig>);
     /**
      * Start supervising agents
@@ -51,6 +63,32 @@ export declare class AgentSupervisor extends EventEmitter {
      * Stop supervising agents
      */
     stop(): void;
+    /**
+     * Enable leader coordination with watchdog
+     * This allows this supervisor to participate in leader election and
+     * potentially become the lead coordinator for task distribution.
+     */
+    enableLeaderCoordination(beadsDir: string, sendRelay: (to: string, message: string) => Promise<void>): void;
+    /**
+     * Called when this supervisor wins leader election
+     */
+    private becomeLeader;
+    /**
+     * Get healthy agents for leader election
+     */
+    private getHealthyAgents;
+    /**
+     * Get available worker names (healthy agents not currently busy)
+     */
+    private getAvailableWorkerNames;
+    /**
+     * Check if this supervisor is currently the leader
+     */
+    isLeader(): boolean;
+    /**
+     * Get current leader info
+     */
+    getCurrentLeader(): LeadHeartbeat | null;
     /**
      * Add an agent to supervision
      */

package/dist/resiliency/supervisor.js

@@ -10,6 +10,8 @@ import { createLogger } from './logger.js';
 import { metrics } from './metrics.js';
 import { getContextPersistence } from './context-persistence.js';
 import { createContextHandler, detectProvider } from './provider-context.js';
+import { LeaderWatchdog } from './leader-watchdog.js';
+import { StatelessLeadCoordinator } from './stateless-lead.js';
 const DEFAULT_CONFIG = {
     healthCheck: {
         checkIntervalMs: 5000,
@@ -34,9 +36,13 @@ export class AgentSupervisor extends EventEmitter {
     restarters = new Map();
     contextPersistence;
     contextHandlers = new Map();
+    leaderWatchdog;
+    leadCoordinator;
+    supervisorAgentId;
     constructor(config = {}) {
         super();
         this.config = { ...DEFAULT_CONFIG, ...config };
+        this.supervisorAgentId = `supervisor-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
         this.logger = createLogger('supervisor', {
             level: this.config.logging.level,
             file: this.config.logging.file,
@@ -66,6 +72,15 @@ export class AgentSupervisor extends EventEmitter {
     stop() {
         this.logger.info('Agent supervisor stopping');
         this.healthMonitor.stop();
+        // Stop leader coordination
+        if (this.leaderWatchdog) {
+            this.leaderWatchdog.stop();
+        }
+        if (this.leadCoordinator) {
+            this.leadCoordinator.stop().catch((err) => {
+                this.logger.error('Error stopping lead coordinator', { error: String(err) });
+            });
+        }
         // Stop context persistence
         if (this.contextPersistence) {
             this.contextPersistence.stopAutoSave();
@@ -77,6 +92,113 @@ export class AgentSupervisor extends EventEmitter {
             });
         });
     }
+    /**
+     * Enable leader coordination with watchdog
+     * This allows this supervisor to participate in leader election and
+     * potentially become the lead coordinator for task distribution.
+     */
+    enableLeaderCoordination(beadsDir, sendRelay) {
+        if (this.leaderWatchdog) {
+            this.logger.warn('Leader coordination already enabled');
+            return;
+        }
+        this.logger.info('Enabling leader coordination', { beadsDir });
+        // Create watchdog to monitor leader health
+        this.leaderWatchdog = new LeaderWatchdog({
+            beadsDir,
+            agentName: 'supervisor',
+            agentId: this.supervisorAgentId,
+            checkIntervalMs: 5000,
+            staleThresholdMs: 30000,
+            onBecomeLeader: async () => {
+                await this.becomeLeader(beadsDir, sendRelay);
+            },
+            getHealthyAgents: async () => {
+                return this.getHealthyAgents();
+            },
+        });
+        // Forward watchdog events
+        this.leaderWatchdog.on('electionStarted', (data) => this.emit('electionStarted', data));
+        this.leaderWatchdog.on('electionComplete', (data) => this.emit('electionComplete', data));
+        this.leaderWatchdog.on('becameLeader', () => this.emit('becameLeader'));
+        this.leaderWatchdog.on('leaderStale', (data) => this.emit('leaderStale', data));
+        this.leaderWatchdog.start();
+    }
+    /**
+     * Called when this supervisor wins leader election
+     */
+    async becomeLeader(beadsDir, sendRelay) {
+        this.logger.info('This supervisor is becoming the lead coordinator');
+        // Stop existing coordinator if any
+        if (this.leadCoordinator) {
+            await this.leadCoordinator.stop();
+        }
+        // Create stateless lead coordinator
+        this.leadCoordinator = new StatelessLeadCoordinator({
+            beadsDir,
+            agentName: 'supervisor',
+            agentId: this.supervisorAgentId,
+            pollIntervalMs: 5000,
+            heartbeatIntervalMs: 10000,
+            leaseDurationMs: 300000,
+            sendRelay,
+            getAvailableWorkers: async () => {
+                return this.getAvailableWorkerNames();
+            },
+        });
+        // Forward coordinator events
+        this.leadCoordinator.on('assigned', (data) => this.emit('taskAssigned', data));
+        this.leadCoordinator.on('completed', (data) => this.emit('taskCompleted', data));
+        await this.leadCoordinator.start();
+    }
+    /**
+     * Get healthy agents for leader election
+     */
+    getHealthyAgents() {
+        const healthyAgents = [];
+        for (const [name, agent] of this.agents) {
+            const health = this.healthMonitor.get(name);
+            if (health && health.status === 'healthy') {
+                healthyAgents.push({
+                    name: agent.name,
+                    id: `${agent.name}-${agent.pid}`,
+                    spawnedAt: agent.spawnedAt,
+                });
+            }
+        }
+        // Include supervisor itself as a candidate
+        healthyAgents.push({
+            name: 'supervisor',
+            id: this.supervisorAgentId,
+            spawnedAt: new Date(0), // Supervisor is always oldest
+        });
+        return healthyAgents;
+    }
+    /**
+     * Get available worker names (healthy agents not currently busy)
+     */
+    getAvailableWorkerNames() {
+        const available = [];
+        for (const [name, agent] of this.agents) {
+            const health = this.healthMonitor.get(name);
+            if (health && health.status === 'healthy') {
+                available.push(agent.name);
+            }
+        }
+        return available;
+    }
+    /**
+     * Check if this supervisor is currently the leader
+     */
+    isLeader() {
+        return this.leaderWatchdog?.isCurrentLeader() ?? false;
+    }
+    /**
+     * Get current leader info
+     */
+    getCurrentLeader() {
+        return this.leaderWatchdog?.getCurrentLeader() ?? null;
+    }
     /**
      * Add an agent to supervision
      */