@claude-flow/cli 3.32.1 → 3.32.3

package/dist/src/services/worker-queue.d.ts

@@ -1,194 +0,0 @@
-/**
- * Worker Queue Service
- * Redis-based task queue for distributed headless worker execution.
- *
- * ADR-020: Headless Worker Integration Architecture - Phase 4
- * - Priority-based task scheduling
- * - Result persistence and retrieval
- * - Distributed locking for task assignment
- * - Dead letter queue for failed tasks
- * - Metrics and monitoring
- *
- * Key Features:
- * - FIFO with priority levels (critical, high, normal, low)
- * - Automatic retry with exponential backoff
- * - Task timeout detection
- * - Result caching with TTL
- * - Worker heartbeat monitoring
- */
-import { EventEmitter } from 'events';
-import type { HeadlessWorkerType, HeadlessExecutionResult, WorkerPriority } from './headless-worker-executor.js';
-/**
- * Task status
- */
-export type TaskStatus = 'pending' | 'processing' | 'completed' | 'failed' | 'timeout' | 'cancelled';
-/**
- * Queue task
- */
-export interface QueueTask {
-    id: string;
-    workerType: HeadlessWorkerType;
-    priority: WorkerPriority;
-    payload: {
-        prompt?: string;
-        contextPatterns?: string[];
-        sandbox?: string;
-        model?: string;
-        timeoutMs?: number;
-    };
-    status: TaskStatus;
-    createdAt: Date;
-    startedAt?: Date;
-    completedAt?: Date;
-    workerId?: string;
-    retryCount: number;
-    maxRetries: number;
-    result?: HeadlessExecutionResult;
-    error?: string;
-}
-/**
- * Queue configuration
- */
-export interface WorkerQueueConfig {
-    /** Redis connection URL */
-    redisUrl: string;
-    /** Queue name prefix */
-    queuePrefix: string;
-    /** Default task timeout in ms */
-    defaultTimeoutMs: number;
-    /** Maximum retries for failed tasks */
-    maxRetries: number;
-    /** Task result TTL in seconds */
-    resultTtlSeconds: number;
-    /** Worker heartbeat interval in ms */
-    heartbeatIntervalMs: number;
-    /** Dead letter queue enabled */
-    deadLetterEnabled: boolean;
-    /** Visibility timeout in ms (task processing lock) */
-    visibilityTimeoutMs: number;
-}
-/**
- * Queue statistics
- */
-export interface QueueStats {
-    pending: number;
-    processing: number;
-    completed: number;
-    failed: number;
-    deadLetter: number;
-    byPriority: Record<WorkerPriority, number>;
-    byWorkerType: Partial<Record<HeadlessWorkerType, number>>;
-    averageWaitTimeMs: number;
-    averageProcessingTimeMs: number;
-}
-/**
- * Worker registration info
- */
-export interface WorkerRegistration {
-    workerId: string;
-    workerTypes: HeadlessWorkerType[];
-    maxConcurrent: number;
-    currentTasks: number;
-    lastHeartbeat: Date;
-    registeredAt: Date;
-    hostname?: string;
-    containerId?: string;
-}
-/**
- * WorkerQueue - Redis-based task queue for distributed worker execution
- */
-export declare class WorkerQueue extends EventEmitter {
-    private config;
-    private store;
-    private workerId;
-    private heartbeatTimer?;
-    private processingTasks;
-    private isShuttingDown;
-    private maxConcurrent;
-    private initialized;
-    constructor(config?: Partial<WorkerQueueConfig>);
-    /**
-     * Initialize the queue (starts cleanup timers)
-     */
-    initialize(): Promise<void>;
-    /**
-     * Enqueue a new task
-     */
-    enqueue(workerType: HeadlessWorkerType, payload?: QueueTask['payload'], options?: {
-        priority?: WorkerPriority;
-        maxRetries?: number;
-        timeoutMs?: number;
-    }): Promise<string>;
-    /**
-     * Dequeue a task for processing
-     */
-    dequeue(workerTypes: HeadlessWorkerType[]): Promise<QueueTask | null>;
-    /**
-     * Complete a task with result
-     */
-    complete(taskId: string, result: HeadlessExecutionResult): Promise<void>;
-    /**
-     * Fail a task with error
-     */
-    fail(taskId: string, error: string, retryable?: boolean): Promise<void>;
-    /**
-     * Get task status
-     */
-    getTask(taskId: string): Promise<QueueTask | null>;
-    /**
-     * Get task result
-     */
-    getResult(taskId: string): Promise<HeadlessExecutionResult | null>;
-    /**
-     * Cancel a pending task
-     */
-    cancel(taskId: string): Promise<boolean>;
-    /**
-     * Register this instance as a worker
-     */
-    registerWorker(workerTypes: HeadlessWorkerType[], options?: {
-        maxConcurrent?: number;
-        hostname?: string;
-        containerId?: string;
-    }): Promise<string>;
-    /**
-     * Unregister this worker
-     */
-    unregisterWorker(): Promise<void>;
-    /**
-     * Get all registered workers
-     */
-    getWorkers(): Promise<WorkerRegistration[]>;
-    /**
-     * Get queue statistics
-     */
-    getStats(): Promise<QueueStats>;
-    /**
-     * Start processing tasks
-     */
-    start(workerTypes: HeadlessWorkerType[], handler: (task: QueueTask) => Promise<HeadlessExecutionResult>, options?: {
-        maxConcurrent?: number;
-    }): Promise<void>;
-    /**
-     * Process a single task
-     */
-    private processTask;
-    /**
-     * Shutdown the queue gracefully
-     */
-    shutdown(): Promise<void>;
-    /**
-     * Get queue name for worker type
-     */
-    private getQueueName;
-    /**
-     * Start heartbeat timer
-     */
-    private startHeartbeat;
-    /**
-     * Stop heartbeat timer
-     */
-    private stopHeartbeat;
-}
-export default WorkerQueue;
-//# sourceMappingURL=worker-queue.d.ts.map

package/dist/src/services/worker-queue.js

@@ -1,513 +0,0 @@
-/**
- * Worker Queue Service
- * Redis-based task queue for distributed headless worker execution.
- *
- * ADR-020: Headless Worker Integration Architecture - Phase 4
- * - Priority-based task scheduling
- * - Result persistence and retrieval
- * - Distributed locking for task assignment
- * - Dead letter queue for failed tasks
- * - Metrics and monitoring
- *
- * Key Features:
- * - FIFO with priority levels (critical, high, normal, low)
- * - Automatic retry with exponential backoff
- * - Task timeout detection
- * - Result caching with TTL
- * - Worker heartbeat monitoring
- */
-import { EventEmitter } from 'events';
-import { randomUUID } from 'crypto';
-// ============================================
-// Constants
-// ============================================
-const DEFAULT_CONFIG = {
-    redisUrl: 'redis://localhost:6379',
-    queuePrefix: 'claude-flow:queue',
-    defaultTimeoutMs: 300000, // 5 minutes
-    maxRetries: 3,
-    resultTtlSeconds: 86400, // 24 hours
-    heartbeatIntervalMs: 30000, // 30 seconds
-    deadLetterEnabled: true,
-    visibilityTimeoutMs: 60000, // 1 minute
-};
-const PRIORITY_SCORES = {
-    critical: 4,
-    high: 3,
-    normal: 2,
-    low: 1,
-};
-// ============================================
-// In-Memory Redis Simulation (for non-Redis environments)
-// ============================================
-/**
- * Simple in-memory queue implementation for environments without Redis
- * Production should use actual Redis connection
- */
-class InMemoryStore {
-    tasks = new Map();
-    queues = new Map();
-    workers = new Map();
-    results = new Map();
-    cleanupTimer;
-    /**
-     * Start cleanup timer (called after initialization)
-     */
-    startCleanup() {
-        if (this.cleanupTimer)
-            return;
-        this.cleanupTimer = setInterval(() => this.cleanupExpired(), 60000);
-        this.cleanupTimer.unref();
-    }
-    /**
-     * Stop cleanup timer
-     */
-    stopCleanup() {
-        if (this.cleanupTimer) {
-            clearInterval(this.cleanupTimer);
-            this.cleanupTimer = undefined;
-        }
-    }
-    // Task operations
-    setTask(id, task) {
-        this.tasks.set(id, task);
-    }
-    getTask(id) {
-        return this.tasks.get(id);
-    }
-    deleteTask(id) {
-        this.tasks.delete(id);
-    }
-    // Queue operations
-    pushToQueue(queue, taskId, priority) {
-        const queueTasks = this.queues.get(queue) || [];
-        // Insert based on priority (higher priority = earlier in queue)
-        let insertIndex = queueTasks.length;
-        for (let i = 0; i < queueTasks.length; i++) {
-            const task = this.tasks.get(queueTasks[i]);
-            if (task && PRIORITY_SCORES[task.priority] < priority) {
-                insertIndex = i;
-                break;
-            }
-        }
-        queueTasks.splice(insertIndex, 0, taskId);
-        this.queues.set(queue, queueTasks);
-    }
-    popFromQueue(queue) {
-        const queueTasks = this.queues.get(queue) || [];
-        if (queueTasks.length === 0)
-            return null;
-        return queueTasks.shift() || null;
-    }
-    getQueueLength(queue) {
-        return (this.queues.get(queue) || []).length;
-    }
-    // Worker operations
-    setWorker(workerId, registration) {
-        this.workers.set(workerId, registration);
-    }
-    getWorker(workerId) {
-        return this.workers.get(workerId);
-    }
-    deleteWorker(workerId) {
-        this.workers.delete(workerId);
-    }
-    getAllWorkers() {
-        return Array.from(this.workers.values());
-    }
-    // Result operations
-    setResult(taskId, result, ttlSeconds) {
-        this.results.set(taskId, {
-            result,
-            expiresAt: Date.now() + ttlSeconds * 1000,
-        });
-    }
-    getResult(taskId) {
-        const entry = this.results.get(taskId);
-        if (!entry)
-            return undefined;
-        if (Date.now() > entry.expiresAt) {
-            this.results.delete(taskId);
-            return undefined;
-        }
-        return entry.result;
-    }
-    // Stats
-    getStats() {
-        return {
-            tasks: this.tasks.size,
-            workers: this.workers.size,
-            results: this.results.size,
-        };
-    }
-    // Cleanup
-    cleanupExpired() {
-        const now = Date.now();
-        for (const [id, entry] of this.results) {
-            if (now > entry.expiresAt) {
-                this.results.delete(id);
-            }
-        }
-    }
-}
-// ============================================
-// WorkerQueue Class
-// ============================================
-/**
- * WorkerQueue - Redis-based task queue for distributed worker execution
- */
-export class WorkerQueue extends EventEmitter {
-    config;
-    store;
-    workerId;
-    heartbeatTimer;
-    processingTasks = new Set();
-    isShuttingDown = false;
-    maxConcurrent = 1;
-    initialized = false;
-    constructor(config) {
-        super();
-        this.config = { ...DEFAULT_CONFIG, ...config };
-        this.store = new InMemoryStore();
-        this.workerId = `worker-${randomUUID().slice(0, 8)}`;
-    }
-    /**
-     * Initialize the queue (starts cleanup timers)
-     */
-    async initialize() {
-        if (this.initialized)
-            return;
-        this.store.startCleanup();
-        this.initialized = true;
-        this.emit('initialized', { workerId: this.workerId });
-    }
-    // ============================================
-    // Public API - Task Management
-    // ============================================
-    /**
-     * Enqueue a new task
-     */
-    async enqueue(workerType, payload = {}, options) {
-        // Initialize if needed
-        if (!this.initialized) {
-            await this.initialize();
-        }
-        // Validate worker type
-        if (!workerType || typeof workerType !== 'string') {
-            throw new Error('Invalid worker type');
-        }
-        // Validate priority
-        const priority = options?.priority || 'normal';
-        if (!['critical', 'high', 'normal', 'low'].includes(priority)) {
-            throw new Error(`Invalid priority: ${priority}`);
-        }
-        const taskId = `task-${Date.now()}-${randomUUID().slice(0, 8)}`;
-        const task = {
-            id: taskId,
-            workerType,
-            priority,
-            payload: {
-                ...payload,
-                timeoutMs: options?.timeoutMs || this.config.defaultTimeoutMs,
-            },
-            status: 'pending',
-            createdAt: new Date(),
-            retryCount: 0,
-            maxRetries: options?.maxRetries ?? this.config.maxRetries,
-        };
-        // Store task
-        this.store.setTask(taskId, task);
-        // Add to priority queue
-        const queueName = this.getQueueName(workerType);
-        this.store.pushToQueue(queueName, taskId, PRIORITY_SCORES[priority]);
-        this.emit('taskEnqueued', { taskId, workerType, priority });
-        return taskId;
-    }
-    /**
-     * Dequeue a task for processing
-     */
-    async dequeue(workerTypes) {
-        if (this.isShuttingDown)
-            return null;
-        // Check queues in priority order
-        for (const workerType of workerTypes) {
-            const queueName = this.getQueueName(workerType);
-            const taskId = this.store.popFromQueue(queueName);
-            if (taskId) {
-                const task = this.store.getTask(taskId);
-                if (task && task.status === 'pending') {
-                    task.status = 'processing';
-                    task.startedAt = new Date();
-                    task.workerId = this.workerId;
-                    this.store.setTask(taskId, task);
-                    this.processingTasks.add(taskId);
-                    this.emit('taskDequeued', { taskId, workerType });
-                    return task;
-                }
-            }
-        }
-        return null;
-    }
-    /**
-     * Complete a task with result
-     */
-    async complete(taskId, result) {
-        const task = this.store.getTask(taskId);
-        if (!task) {
-            this.emit('warning', { message: `Task ${taskId} not found for completion` });
-            return;
-        }
-        task.status = 'completed';
-        task.completedAt = new Date();
-        task.result = result;
-        this.store.setTask(taskId, task);
-        // Store result with TTL
-        this.store.setResult(taskId, result, this.config.resultTtlSeconds);
-        this.processingTasks.delete(taskId);
-        this.emit('taskCompleted', { taskId, result, duration: task.completedAt.getTime() - (task.startedAt?.getTime() || 0) });
-    }
-    /**
-     * Fail a task with error
-     */
-    async fail(taskId, error, retryable = true) {
-        const task = this.store.getTask(taskId);
-        if (!task) {
-            this.emit('warning', { message: `Task ${taskId} not found for failure` });
-            return;
-        }
-        this.processingTasks.delete(taskId);
-        // Check if we should retry
-        if (retryable && task.retryCount < task.maxRetries) {
-            task.retryCount++;
-            task.status = 'pending';
-            task.startedAt = undefined;
-            task.workerId = undefined;
-            task.error = error;
-            this.store.setTask(taskId, task);
-            // Re-queue with delay (exponential backoff)
-            const delay = Math.min(30000, 1000 * Math.pow(2, task.retryCount));
-            setTimeout(() => {
-                const queueName = this.getQueueName(task.workerType);
-                this.store.pushToQueue(queueName, taskId, PRIORITY_SCORES[task.priority]);
-            }, delay);
-            this.emit('taskRetrying', { taskId, retryCount: task.retryCount, delay });
-        }
-        else {
-            // Move to failed/dead letter
-            task.status = 'failed';
-            task.completedAt = new Date();
-            task.error = error;
-            this.store.setTask(taskId, task);
-            if (this.config.deadLetterEnabled) {
-                const dlqName = `${this.config.queuePrefix}:dlq`;
-                this.store.pushToQueue(dlqName, taskId, 0);
-            }
-            this.emit('taskFailed', { taskId, error, retryCount: task.retryCount });
-        }
-    }
-    /**
-     * Get task status
-     */
-    async getTask(taskId) {
-        return this.store.getTask(taskId) || null;
-    }
-    /**
-     * Get task result
-     */
-    async getResult(taskId) {
-        return this.store.getResult(taskId) || null;
-    }
-    /**
-     * Cancel a pending task
-     */
-    async cancel(taskId) {
-        const task = this.store.getTask(taskId);
-        if (!task || task.status !== 'pending') {
-            return false;
-        }
-        task.status = 'cancelled';
-        task.completedAt = new Date();
-        this.store.setTask(taskId, task);
-        this.emit('taskCancelled', { taskId });
-        return true;
-    }
-    // ============================================
-    // Public API - Worker Management
-    // ============================================
-    /**
-     * Register this instance as a worker
-     */
-    async registerWorker(workerTypes, options) {
-        // Initialize if needed
-        if (!this.initialized) {
-            await this.initialize();
-        }
-        // Validate worker types
-        if (!Array.isArray(workerTypes) || workerTypes.length === 0) {
-            throw new Error('Worker types must be a non-empty array');
-        }
-        this.maxConcurrent = options?.maxConcurrent || 1;
-        const registration = {
-            workerId: this.workerId,
-            workerTypes,
-            maxConcurrent: this.maxConcurrent,
-            currentTasks: 0,
-            lastHeartbeat: new Date(),
-            registeredAt: new Date(),
-            hostname: options?.hostname,
-            containerId: options?.containerId,
-        };
-        this.store.setWorker(this.workerId, registration);
-        // Start heartbeat
-        this.startHeartbeat();
-        this.emit('workerRegistered', { workerId: this.workerId, workerTypes });
-        return this.workerId;
-    }
-    /**
-     * Unregister this worker
-     */
-    async unregisterWorker() {
-        this.stopHeartbeat();
-        this.store.deleteWorker(this.workerId);
-        this.emit('workerUnregistered', { workerId: this.workerId });
-    }
-    /**
-     * Get all registered workers
-     */
-    async getWorkers() {
-        return this.store.getAllWorkers();
-    }
-    // ============================================
-    // Public API - Statistics
-    // ============================================
-    /**
-     * Get queue statistics
-     */
-    async getStats() {
-        const storeStats = this.store.getStats();
-        // This is a simplified implementation
-        // Full implementation would aggregate across all queues
-        const stats = {
-            pending: 0,
-            processing: this.processingTasks.size,
-            completed: 0,
-            failed: 0,
-            deadLetter: 0,
-            byPriority: { critical: 0, high: 0, normal: 0, low: 0 },
-            byWorkerType: {},
-            averageWaitTimeMs: 0,
-            averageProcessingTimeMs: 0,
-        };
-        return stats;
-    }
-    // ============================================
-    // Public API - Lifecycle
-    // ============================================
-    /**
-     * Start processing tasks
-     */
-    async start(workerTypes, handler, options) {
-        await this.registerWorker(workerTypes, { maxConcurrent: options?.maxConcurrent });
-        const processLoop = async () => {
-            while (!this.isShuttingDown) {
-                try {
-                    // Respect concurrency limit
-                    if (this.processingTasks.size >= this.maxConcurrent) {
-                        await new Promise(resolve => setTimeout(resolve, 100));
-                        continue;
-                    }
-                    const task = await this.dequeue(workerTypes);
-                    if (task) {
-                        // Process task without blocking the loop (allows concurrency)
-                        this.processTask(task, handler).catch(error => {
-                            this.emit('error', { taskId: task.id, error: String(error) });
-                        });
-                    }
-                    else {
-                        // No task available, wait before polling again
-                        await new Promise(resolve => setTimeout(resolve, 1000));
-                    }
-                }
-                catch (error) {
-                    this.emit('error', { error: error instanceof Error ? error.message : String(error) });
-                    // Wait before retrying to avoid tight error loop
-                    await new Promise(resolve => setTimeout(resolve, 5000));
-                }
-            }
-        };
-        // Start processing
-        processLoop().catch(error => {
-            this.emit('error', { error: error instanceof Error ? error.message : String(error) });
-        });
-    }
-    /**
-     * Process a single task
-     */
-    async processTask(task, handler) {
-        try {
-            const result = await handler(task);
-            await this.complete(task.id, result);
-        }
-        catch (error) {
-            await this.fail(task.id, error instanceof Error ? error.message : String(error));
-        }
-    }
-    /**
-     * Shutdown the queue gracefully
-     */
-    async shutdown() {
-        if (this.isShuttingDown)
-            return;
-        this.isShuttingDown = true;
-        // Wait for processing tasks to complete (with timeout)
-        const timeout = 30000;
-        const start = Date.now();
-        while (this.processingTasks.size > 0 && Date.now() - start < timeout) {
-            await new Promise(resolve => setTimeout(resolve, 1000));
-        }
-        // Force fail remaining tasks
-        for (const taskId of this.processingTasks) {
-            await this.fail(taskId, 'Worker shutdown', false);
-        }
-        // Stop store cleanup
-        this.store.stopCleanup();
-        await this.unregisterWorker();
-        this.initialized = false;
-        this.emit('shutdown', {});
-    }
-    // ============================================
-    // Private Methods
-    // ============================================
-    /**
-     * Get queue name for worker type
-     */
-    getQueueName(workerType) {
-        return `${this.config.queuePrefix}:${workerType}`;
-    }
-    /**
-     * Start heartbeat timer
-     */
-    startHeartbeat() {
-        this.heartbeatTimer = setInterval(() => {
-            const registration = this.store.getWorker(this.workerId);
-            if (registration) {
-                registration.lastHeartbeat = new Date();
-                registration.currentTasks = this.processingTasks.size;
-                this.store.setWorker(this.workerId, registration);
-            }
-        }, this.config.heartbeatIntervalMs);
-        this.heartbeatTimer.unref();
-    }
-    /**
-     * Stop heartbeat timer
-     */
-    stopHeartbeat() {
-        if (this.heartbeatTimer) {
-            clearInterval(this.heartbeatTimer);
-            this.heartbeatTimer = undefined;
-        }
-    }
-}
-// Export default
-export default WorkerQueue;
-//# sourceMappingURL=worker-queue.js.map