@overlordai/server 1.0.1

package/dist/dispatcher/reconciler.js

@@ -0,0 +1,556 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+var Reconciler_1;
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Reconciler = void 0;
+const common_1 = require("@nestjs/common");
+const crypto = __importStar(require("node:crypto"));
+const protocol_1 = require("@overlordai/protocol");
+const task_repository_1 = require("../database/repositories/task.repository");
+const machine_repository_1 = require("../database/repositories/machine.repository");
+const audit_log_repository_1 = require("../database/repositories/audit-log.repository");
+const workspace_repository_1 = require("../database/repositories/workspace.repository");
+const scheduler_service_1 = require("./scheduler.service");
+const cleanup_service_1 = require("./cleanup.service");
+const worker_connection_manager_1 = require("./worker-connection.manager");
+let Reconciler = Reconciler_1 = class Reconciler {
+    taskRepo;
+    machineRepo;
+    auditLogRepo;
+    workspaceRepo;
+    schedulerService;
+    cleanupService;
+    workerConnectionManager;
+    logger = new common_1.Logger(Reconciler_1.name);
+    constructor(taskRepo, machineRepo, auditLogRepo, workspaceRepo, schedulerService, cleanupService, workerConnectionManager) {
+        this.taskRepo = taskRepo;
+        this.machineRepo = machineRepo;
+        this.auditLogRepo = auditLogRepo;
+        this.workspaceRepo = workspaceRepo;
+        this.schedulerService = schedulerService;
+        this.cleanupService = cleanupService;
+        this.workerConnectionManager = workerConnectionManager;
+    }
+    async onModuleInit() {
+        this.logger.log('Running boot reconciliation...');
+        await this.onBoot();
+        this.logger.log('Boot reconciliation complete');
+    }
+    // ---------------------------------------------------------------------------
+    // onBoot — startup reconciliation
+    // ---------------------------------------------------------------------------
+    async onBoot() {
+        const now = Date.now();
+        let reEnqueuedCount = 0;
+        let failedStaleCount = 0;
+        // Step 1: Re-enqueue QUEUED/ASSIGNED tasks missing from BullMQ.
+        // On boot we conservatively re-enqueue all QUEUED tasks since BullMQ
+        // may have lost state during a restart.
+        const queuedTasks = this.taskRepo.findByStatus([protocol_1.TaskStatus.QUEUED]);
+        for (const task of queuedTasks) {
+            await this.schedulerService.enqueueTask(task.id);
+            reEnqueuedCount++;
+            this.logger.log(`Re-enqueued QUEUED task #${task.id}`);
+        }
+        // ASSIGNED tasks with offline/missing machines get reset to QUEUED
+        const assignedTasks = this.taskRepo.findByStatus([protocol_1.TaskStatus.ASSIGNED]);
+        for (const task of assignedTasks) {
+            if (task.machineId) {
+                const machine = this.machineRepo.findById(task.machineId);
+                if (!machine || machine.status === protocol_1.MachineStatus.OFFLINE) {
+                    this.updateTaskStatusSafe(task.id, protocol_1.TaskStatus.QUEUED, task.revision, {
+                        machineId: null,
+                        assignedAt: null,
+                    });
+                    await this.schedulerService.enqueueTask(task.id);
+                    reEnqueuedCount++;
+                    this.logger.warn(`Reset ASSIGNED task #${task.id} to QUEUED (machine ${task.machineId} offline)`);
+                }
+                else {
+                    // Machine online — re-enqueue so the scheduler can verify and re-dispatch
+                    await this.schedulerService.enqueueTask(task.id);
+                    reEnqueuedCount++;
+                }
+            }
+            else {
+                // No machine assigned — reset to QUEUED
+                this.updateTaskStatusSafe(task.id, protocol_1.TaskStatus.QUEUED, task.revision);
+                await this.schedulerService.enqueueTask(task.id);
+                reEnqueuedCount++;
+            }
+        }
+        // Step 2: Fail stale RUNNING/SUSPENDED tasks (grace 10 min).
+        const runningTasks = this.taskRepo.findByStatus([protocol_1.TaskStatus.RUNNING]);
+        for (const task of runningTasks) {
+            if (!task.machineId) {
+                this.updateTaskStatusSafe(task.id, protocol_1.TaskStatus.FAILED, task.revision, {
+                    errorMessage: 'No machine assigned during boot reconciliation',
+                    completedAt: new Date().toISOString(),
+                });
+                failedStaleCount++;
+                continue;
+            }
+            const machine = this.machineRepo.findById(task.machineId);
+            if (!machine || machine.status === protocol_1.MachineStatus.OFFLINE) {
+                const lastHeartbeat = machine?.lastHeartbeat
+                    ? new Date(machine.lastHeartbeat).getTime()
+                    : 0;
+                if (now - lastHeartbeat > protocol_1.GRACE_WINDOW_MS) {
+                    this.updateTaskStatusSafe(task.id, protocol_1.TaskStatus.FAILED, task.revision, {
+                        errorMessage: `Machine '${machine?.name ?? task.machineId}' offline beyond grace window (${Math.round((now - lastHeartbeat) / 1000)}s)`,
+                        completedAt: new Date().toISOString(),
+                    });
+                    if (task.machineId)
+                        this.scheduleWorkspaceCleanup(task.id, task.machineId);
+                    failedStaleCount++;
+                    this.logger.warn(`Failed stale RUNNING task #${task.id} (machine ${task.machineId} offline > ${protocol_1.GRACE_WINDOW_MS / 1000}s)`);
+                }
+                else {
+                    // Within grace window: suspend and await Worker recovery
+                    this.updateTaskStatusSafe(task.id, protocol_1.TaskStatus.SUSPENDED, task.revision);
+                    this.logger.warn(`Suspended RUNNING task #${task.id} (machine ${task.machineId} offline, within grace window)`);
+                }
+            }
+        }
+        const suspendedTasks = this.taskRepo.findByStatus([protocol_1.TaskStatus.SUSPENDED]);
+        for (const task of suspendedTasks) {
+            if (!task.machineId) {
+                this.updateTaskStatusSafe(task.id, protocol_1.TaskStatus.FAILED, task.revision, {
+                    errorMessage: 'No machine assigned during boot reconciliation',
+                    completedAt: new Date().toISOString(),
+                });
+                failedStaleCount++;
+                continue;
+            }
+            const machine = this.machineRepo.findById(task.machineId);
+            if (!machine || machine.status === protocol_1.MachineStatus.OFFLINE) {
+                const lastHeartbeat = machine?.lastHeartbeat
+                    ? new Date(machine.lastHeartbeat).getTime()
+                    : 0;
+                if (now - lastHeartbeat > protocol_1.GRACE_WINDOW_MS) {
+                    this.updateTaskStatusSafe(task.id, protocol_1.TaskStatus.FAILED, task.revision, {
+                        errorMessage: `Machine '${machine?.name ?? task.machineId}' offline beyond grace window`,
+                        completedAt: new Date().toISOString(),
+                    });
+                    if (task.machineId)
+                        this.scheduleWorkspaceCleanup(task.id, task.machineId);
+                    failedStaleCount++;
+                    this.logger.warn(`Failed stale SUSPENDED task #${task.id} (machine ${task.machineId} offline > grace)`);
+                }
+                // Within grace: keep SUSPENDED, await Worker recovery
+            }
+        }
+        // Step 3: Remove stale BullMQ jobs for terminal tasks.
+        // BullMQ removeOnComplete/removeOnFail handles most cleanup.
+        // The scheduler's processJob re-checks status and skips stale tasks.
+        // Step 4: Recalculate active_slots for all machines and update DB if mismatched.
+        const allMachines = this.machineRepo.findAll();
+        for (const machine of allMachines) {
+            const actualSlots = this.machineRepo.countActiveSlots(machine.id);
+            if (machine.activeSlots !== actualSlots) {
+                this.machineRepo.updateActiveSlots(machine.id, actualSlots);
+                this.logger.log(`Machine '${machine.name}' active_slots corrected: ${machine.activeSlots} → ${actualSlots}`);
+            }
+        }
+        // Step 5: Write audit_log.
+        const globalActiveCount = this.taskRepo.countActive();
+        this.auditLogRepo.create({
+            action: 'dispatcher_reconcile',
+            resource: 'system',
+            detail: JSON.stringify({
+                reEnqueued: reEnqueuedCount,
+                failedStale: failedStaleCount,
+                globalActiveCount,
+                machinesChecked: allMachines.length,
+            }),
+        });
+        // Step 6 (branch g): Re-enqueue cleanup for terminal tasks with active workspaces.
+        // If a task reached a terminal state but its workspace was never cleaned up
+        // (e.g. the cleanup job was lost during a restart), schedule immediate cleanup.
+        const orphanedWorkspaces = this.workspaceRepo.findActiveForTerminalTasks();
+        let cleanupReEnqueuedCount = 0;
+        for (const workspace of orphanedWorkspaces) {
+            try {
+                await this.cleanupService.scheduleCleanup(workspace.taskId, workspace.machineId, workspace.path, 0);
+                cleanupReEnqueuedCount++;
+                this.logger.warn(`Re-enqueued cleanup for workspace #${workspace.id} (task #${workspace.taskId}, machine ${workspace.machineId})`);
+            }
+            catch (err) {
+                this.logger.error(`Failed to re-enqueue cleanup for workspace #${workspace.id} (task #${workspace.taskId}): ${err instanceof Error ? err.message : String(err)}`);
+            }
+        }
+        this.logger.log(`Boot reconciliation: re-enqueued=${reEnqueuedCount}, failed_stale=${failedStaleCount}, active_tasks=${globalActiveCount}, cleanup_re-enqueued=${cleanupReEnqueuedCount}`);
+    }
+    // ---------------------------------------------------------------------------
+    // handleReconnect — Worker reconnection reconciliation
+    //
+    // Processes 6 branches (a-f):
+    //   a. SUSPENDED + still_running → RUNNING (if admission ok)
+    //   b. SUSPENDED + completed_locally → COMPLETED
+    //   c. FAILED + still_running → notify Worker cleanup
+    //   d. FAILED + completed_locally → COMPLETED
+    //   e. CANCELLED → notify Worker stop
+    //   f. Not reported → FAILED
+    // ---------------------------------------------------------------------------
+    async handleReconnect(machineId, tasks) {
+        this.logger.log(`Reconciling reconnect for machine ${machineId} with ${tasks.length} reported task(s)`);
+        const machine = this.machineRepo.findById(machineId);
+        if (!machine) {
+            this.logger.warn(`Reconnect from unknown machine ${machineId}, ignoring`);
+            return [];
+        }
+        const actions = [];
+        const reportedTaskIds = new Set(tasks.map((t) => t.taskId));
+        const reportedTaskMap = new Map(tasks.map((t) => [t.taskId, t]));
+        // Get all tasks assigned to this machine in active or recent terminal states
+        const activeMachineTasks = this.taskRepo.findByMachineId(machineId, [
+            protocol_1.TaskStatus.RUNNING,
+            protocol_1.TaskStatus.SUSPENDED,
+            protocol_1.TaskStatus.ASSIGNED,
+        ]);
+        const terminalMachineTasks = this.taskRepo.findByMachineId(machineId, [
+            protocol_1.TaskStatus.FAILED,
+            protocol_1.TaskStatus.CANCELLED,
+        ]);
+        const allMachineTasks = [...activeMachineTasks, ...terminalMachineTasks];
+        // Process each machine task against the Worker's report
+        for (const task of allMachineTasks) {
+            const reported = reportedTaskMap.get(task.id);
+            if (reported) {
+                const action = this.reconcileReportedTask(machine, task, reported);
+                actions.push({ taskId: task.id, action });
+            }
+            else if (task.status === protocol_1.TaskStatus.RUNNING ||
+                task.status === protocol_1.TaskStatus.SUSPENDED) {
+                // Branch (f): Dispatcher has task but Worker did not report it.
+                // Worker lost execution context (process exited / Worker restarted).
+                this.updateTaskStatusSafe(task.id, protocol_1.TaskStatus.FAILED, task.revision, {
+                    errorMessage: `Worker '${machine.name}' reconnected but did not report this task (execution context lost)`,
+                    completedAt: new Date().toISOString(),
+                });
+                if (task.machineId)
+                    this.scheduleWorkspaceCleanup(task.id, task.machineId);
+                this.auditLogRepo.create({
+                    action: 'reconcile_unreported_failed',
+                    resource: `task:${task.id}`,
+                    detail: `Task not reported by Worker '${machine.name}' during reconnect`,
+                });
+                actions.push({ taskId: task.id, action: 'failed_unreported' });
+                this.logger.warn(`Task #${task.id} (${task.status}) not reported by Worker, marked FAILED`);
+            }
+            else if (task.status === protocol_1.TaskStatus.ASSIGNED) {
+                // ASSIGNED but not reported — reset to QUEUED for rescheduling
+                this.updateTaskStatusSafe(task.id, protocol_1.TaskStatus.QUEUED, task.revision, {
+                    machineId: null,
+                    assignedAt: null,
+                });
+                await this.schedulerService.enqueueTask(task.id);
+                actions.push({ taskId: task.id, action: 'reset_to_queued' });
+                this.logger.warn(`Task #${task.id} (ASSIGNED) not reported by Worker, reset to QUEUED`);
+            }
+        }
+        // Handle tasks reported by Worker that Dispatcher doesn't have on this machine
+        for (const reported of tasks) {
+            if (allMachineTasks.some((t) => t.id === reported.taskId))
+                continue;
+            const task = this.taskRepo.findById(reported.taskId);
+            if (!task) {
+                this.logger.warn(`Worker reported unknown task #${reported.taskId}, requesting cleanup`);
+                this.sendCleanupToWorker(machineId, reported.taskId);
+                actions.push({ taskId: reported.taskId, action: 'cleanup_unknown' });
+                continue;
+            }
+            // Task exists but is assigned to a different machine or already processed
+            const action = this.reconcileReportedTask(machine, task, reported);
+            actions.push({ taskId: task.id, action });
+        }
+        this.logger.log(`Reconnect reconciliation for machine ${machineId}: ${actions.length} tasks processed`);
+        return actions;
+    }
+    // ---------------------------------------------------------------------------
+    // Private: reconcile a single reported task (branches a-e)
+    // ---------------------------------------------------------------------------
+    reconcileReportedTask(machine, task, reported) {
+        const { taskId, localStatus, currentStage, branch, mrUrl, errorMessage } = reported;
+        switch (task.status) {
+            // Branch (a): SUSPENDED + still_running → RUNNING (if admission ok)
+            // Branch (b): SUSPENDED + completed_locally → COMPLETED
+            case protocol_1.TaskStatus.SUSPENDED: {
+                if (localStatus === 'still_running') {
+                    const activeCount = this.taskRepo.countActive();
+                    if (activeCount < protocol_1.MAX_CONCURRENT_TASKS) {
+                        const extra = {};
+                        if (currentStage)
+                            extra['currentStage'] = currentStage;
+                        this.updateTaskStatusSafe(taskId, protocol_1.TaskStatus.RUNNING, task.revision, extra);
+                        // Send resume frame to Worker
+                        const resumeFrame = {
+                            type: 'resume',
+                            msgId: crypto.randomUUID(),
+                            taskId,
+                        };
+                        this.workerConnectionManager.send(machine.id, resumeFrame);
+                        this.logger.log(`Task #${taskId}: SUSPENDED → RUNNING (reconnect, admission ok)`);
+                        return 'resumed';
+                    }
+                    // Admission blocked — keep SUSPENDED
+                    this.logger.log(`Task #${taskId}: kept SUSPENDED (admission blocked, active=${activeCount}/${protocol_1.MAX_CONCURRENT_TASKS})`);
+                    this.auditLogRepo.create({
+                        action: 'reconcile_admission_blocked',
+                        resource: `task:${taskId}`,
+                        detail: `Admission control blocked resume (active=${activeCount}/${protocol_1.MAX_CONCURRENT_TASKS})`,
+                    });
+                    return 'admission_blocked';
+                }
+                if (localStatus === 'completed_locally') {
+                    const extra = {
+                        completedAt: new Date().toISOString(),
+                    };
+                    if (branch)
+                        extra['branch'] = branch;
+                    if (mrUrl)
+                        extra['mrUrl'] = mrUrl;
+                    this.updateTaskStatusSafe(taskId, protocol_1.TaskStatus.COMPLETED, task.revision, extra);
+                    if (task.machineId)
+                        this.scheduleWorkspaceCleanup(taskId, task.machineId);
+                    this.logger.log(`Task #${taskId}: SUSPENDED → COMPLETED (completed during disconnect)`);
+                    return 'completed';
+                }
+                if (localStatus === 'failed_locally') {
+                    this.updateTaskStatusSafe(taskId, protocol_1.TaskStatus.FAILED, task.revision, {
+                        errorMessage: errorMessage ?? 'Task failed locally during disconnect',
+                        completedAt: new Date().toISOString(),
+                    });
+                    if (task.machineId)
+                        this.scheduleWorkspaceCleanup(taskId, task.machineId);
+                    this.logger.log(`Task #${taskId}: SUSPENDED → FAILED (failed during disconnect)`);
+                    return 'failed';
+                }
+                return 'no_op';
+            }
+            // Branch (c): FAILED + still_running → notify Worker cleanup
+            // Branch (d): FAILED + completed_locally → restore to COMPLETED
+            case protocol_1.TaskStatus.FAILED: {
+                if (localStatus === 'still_running') {
+                    this.logger.log(`Task #${taskId}: FAILED but still running on Worker, sending cancel`);
+                    this.sendCancelToWorker(machine.id, taskId);
+                    return 'send_cancel';
+                }
+                if (localStatus === 'completed_locally') {
+                    const extra = {
+                        completedAt: new Date().toISOString(),
+                        errorMessage: null,
+                    };
+                    if (branch)
+                        extra['branch'] = branch;
+                    if (mrUrl)
+                        extra['mrUrl'] = mrUrl;
+                    this.updateTaskStatusSafe(taskId, protocol_1.TaskStatus.COMPLETED, task.revision, extra);
+                    if (task.machineId)
+                        this.scheduleWorkspaceCleanup(taskId, task.machineId);
+                    this.auditLogRepo.create({
+                        action: 'reconcile_restore_completed',
+                        resource: `task:${taskId}`,
+                        detail: 'Task restored from FAILED to COMPLETED (completed during disconnect)',
+                    });
+                    this.logger.log(`Task #${taskId}: FAILED → COMPLETED (actually completed during disconnect)`);
+                    return 'restored_completed';
+                }
+                return 'no_op';
+            }
+            // Branch (e): CANCELLED → notify Worker to stop
+            case protocol_1.TaskStatus.CANCELLED: {
+                if (localStatus === 'still_running' ||
+                    localStatus === 'completed_locally') {
+                    // Keep CANCELLED (user intent priority), notify Worker to stop
+                    this.logger.log(`Task #${taskId}: CANCELLED, notifying Worker to stop (localStatus=${localStatus})`);
+                    this.sendCancelToWorker(machine.id, taskId);
+                    if (localStatus === 'completed_locally') {
+                        this.auditLogRepo.create({
+                            action: 'reconcile_cancelled_but_completed',
+                            resource: `task:${taskId}`,
+                            detail: 'Task was CANCELLED but Worker reports completed_locally. Keeping CANCELLED (user intent).',
+                        });
+                    }
+                    return 'send_cancel';
+                }
+                return 'already_terminal';
+            }
+            // RUNNING: Worker confirms still running or reports completion/failure
+            case protocol_1.TaskStatus.RUNNING: {
+                if (localStatus === 'still_running') {
+                    if (currentStage && currentStage !== task.currentStage) {
+                        this.updateTaskStatusSafe(taskId, protocol_1.TaskStatus.RUNNING, task.revision, { currentStage });
+                    }
+                    return 'no_op';
+                }
+                if (localStatus === 'completed_locally') {
+                    const extra = {
+                        completedAt: new Date().toISOString(),
+                    };
+                    if (branch)
+                        extra['branch'] = branch;
+                    if (mrUrl)
+                        extra['mrUrl'] = mrUrl;
+                    this.updateTaskStatusSafe(taskId, protocol_1.TaskStatus.COMPLETED, task.revision, extra);
+                    if (task.machineId)
+                        this.scheduleWorkspaceCleanup(taskId, task.machineId);
+                    this.logger.log(`Task #${taskId}: RUNNING → COMPLETED (completed during disconnect)`);
+                    return 'completed';
+                }
+                if (localStatus === 'failed_locally') {
+                    this.updateTaskStatusSafe(taskId, protocol_1.TaskStatus.FAILED, task.revision, {
+                        errorMessage: errorMessage ?? 'Task failed locally during disconnect',
+                        completedAt: new Date().toISOString(),
+                    });
+                    if (task.machineId)
+                        this.scheduleWorkspaceCleanup(taskId, task.machineId);
+                    this.logger.log(`Task #${taskId}: RUNNING → FAILED (failed during disconnect)`);
+                    return 'failed';
+                }
+                return 'no_op';
+            }
+            // ASSIGNED: Worker may have started executing before we recorded RUNNING
+            case protocol_1.TaskStatus.ASSIGNED: {
+                if (localStatus === 'still_running') {
+                    this.updateTaskStatusSafe(taskId, protocol_1.TaskStatus.RUNNING, task.revision, {
+                        startedAt: new Date().toISOString(),
+                    });
+                    this.logger.log(`Task #${taskId}: ASSIGNED → RUNNING (Worker reports still_running)`);
+                    return 'promoted_to_running';
+                }
+                if (localStatus === 'completed_locally') {
+                    const extra = {
+                        completedAt: new Date().toISOString(),
+                        startedAt: task.startedAt ?? new Date().toISOString(),
+                    };
+                    if (branch)
+                        extra['branch'] = branch;
+                    if (mrUrl)
+                        extra['mrUrl'] = mrUrl;
+                    this.updateTaskStatusSafe(taskId, protocol_1.TaskStatus.COMPLETED, task.revision, extra);
+                    if (task.machineId)
+                        this.scheduleWorkspaceCleanup(taskId, task.machineId);
+                    this.logger.log(`Task #${taskId}: ASSIGNED → COMPLETED (completed locally)`);
+                    return 'completed';
+                }
+                return 'no_op';
+            }
+            case protocol_1.TaskStatus.COMPLETED:
+                return 'already_terminal';
+            default:
+                this.logger.debug(`Task #${taskId} in status ${task.status} — no reconciliation action`);
+                return 'no_op';
+        }
+    }
+    // ---------------------------------------------------------------------------
+    // Private helpers
+    // ---------------------------------------------------------------------------
+    /**
+     * Update task status with CAS retry, logging on failure instead of throwing.
+     * Best-effort for reconciliation — continues processing other tasks on failure.
+     */
+    updateTaskStatusSafe(taskId, status, initialRevision, extra) {
+        let revision = initialRevision;
+        for (let attempt = 0; attempt < protocol_1.CAS_MAX_RETRIES; attempt++) {
+            const success = this.taskRepo.updateStatus(taskId, status, revision, extra);
+            if (success)
+                return true;
+            const freshTask = this.taskRepo.findById(taskId);
+            if (!freshTask) {
+                this.logger.error(`Task #${taskId} disappeared during CAS retry (reconciliation)`);
+                return false;
+            }
+            // Already at target status — success
+            if (freshTask.status === status)
+                return true;
+            revision = freshTask.revision;
+        }
+        this.logger.error(`CAS retry exhausted for task #${taskId} → ${status} during reconciliation (${protocol_1.CAS_MAX_RETRIES} attempts)`);
+        return false;
+    }
+    /** Send cancel frame to a Worker for a specific task. */
+    sendCancelToWorker(machineId, taskId) {
+        const frame = {
+            type: 'cancel',
+            msgId: crypto.randomUUID(),
+            taskId,
+        };
+        this.workerConnectionManager.send(machineId, frame);
+    }
+    /** Send cleanup instruction to Worker for a specific task. */
+    sendCleanupToWorker(machineId, taskId) {
+        const workspace = this.workspaceRepo.findByTaskId(taskId);
+        const frame = {
+            type: 'cleanup_workspace',
+            msgId: crypto.randomUUID(),
+            taskId,
+            workspacePath: workspace?.path ?? '',
+        };
+        this.workerConnectionManager.send(machineId, frame);
+    }
+    /** Schedule workspace cleanup for a task that reached a terminal state. */
+    scheduleWorkspaceCleanup(taskId, machineId) {
+        const workspace = this.workspaceRepo.findByTaskId(taskId);
+        if (workspace) {
+            this.cleanupService
+                .scheduleCleanup(taskId, machineId, workspace.path)
+                .catch((err) => {
+                this.logger.error(`Failed to schedule cleanup for task #${taskId}: ${err instanceof Error ? err.message : String(err)}`);
+            });
+        }
+    }
+};
+exports.Reconciler = Reconciler;
+exports.Reconciler = Reconciler = Reconciler_1 = __decorate([
+    (0, common_1.Injectable)(),
+    __metadata("design:paramtypes", [task_repository_1.TaskRepository,
+        machine_repository_1.MachineRepository,
+        audit_log_repository_1.AuditLogRepository,
+        workspace_repository_1.WorkspaceRepository,
+        scheduler_service_1.SchedulerService,
+        cleanup_service_1.CleanupService,
+        worker_connection_manager_1.WorkerConnectionManager])
+], Reconciler);
+//# sourceMappingURL=reconciler.js.map

package/dist/dispatcher/reconciler.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"reconciler.js","sourceRoot":"","sources":["../../src/dispatcher/reconciler.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,2CAAkE;AAClE,oDAAsC;AAStC,mDAM8B;AAE9B,8EAA0E;AAC1E,oFAAgF;AAChF,wFAAmF;AACnF,wFAAoF;AAEpF,2DAAuD;AACvD,uDAAmD;AACnD,2EAAsE;AAG/D,IAAM,UAAU,kBAAhB,MAAM,UAAU;IAIF;IACA;IACA;IACA;IACA;IACA;IACA;IATF,MAAM,GAAG,IAAI,eAAM,CAAC,YAAU,CAAC,IAAI,CAAC,CAAC;IAEtD,YACmB,QAAwB,EACxB,WAA8B,EAC9B,YAAgC,EAChC,aAAkC,EAClC,gBAAkC,EAClC,cAA8B,EAC9B,uBAAgD;QANhD,aAAQ,GAAR,QAAQ,CAAgB;QACxB,gBAAW,GAAX,WAAW,CAAmB;QAC9B,iBAAY,GAAZ,YAAY,CAAoB;QAChC,kBAAa,GAAb,aAAa,CAAqB;QAClC,qBAAgB,GAAhB,gBAAgB,CAAkB;QAClC,mBAAc,GAAd,cAAc,CAAgB;QAC9B,4BAAuB,GAAvB,uBAAuB,CAAyB;IAChE,CAAC;IAEJ,KAAK,CAAC,YAAY;QAChB,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,gCAAgC,CAAC,CAAC;QAClD,MAAM,IAAI,CAAC,MAAM,EAAE,CAAC;QACpB,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,8BAA8B,CAAC,CAAC;IAClD,CAAC;IAED,8EAA8E;IAC9E,kCAAkC;IAClC,8EAA8E;IAE9E,KAAK,CAAC,MAAM;QACV,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,IAAI,eAAe,GAAG,CAAC,CAAC;QACxB,IAAI,gBAAgB,GAAG,CAAC,CAAC;QAEzB,gEAAgE;QAChE,qEAAqE;QACrE,wCAAwC;QACxC,MAAM,WAAW,GAAG,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC,qBAAU,CAAC,MAAM,CAAC,CAAC,CAAC;QACpE,KAAK,MAAM,IAAI,IAAI,WAAW,EAAE,CAAC;YAC/B,MAAM,IAAI,CAAC,gBAAgB,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACjD,eAAe,EAAE,CAAC;YAClB,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,4BAA4B,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;QACzD,CAAC;QAED,mEAAmE;QACnE,MAAM,aAAa,GAAG,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC,qBAAU,CAAC,QAAQ,CAAC,CAAC,CAAC;QACxE,KAAK,MAAM,IAAI,IAAI,aAAa,EAAE,CAAC;YACjC,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;gBACnB,MAAM,OAAO,GAAG,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;gBAC1D,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,MAAM,KAAK,wBAAa,CAAC,OAAO,EAAE,CAAC;oBACzD,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,EAAE,EAAE,qBAAU,CAAC,MAAM,EAAE,IAAI,CAAC,QAAQ,EAAE;wBACnE,SAAS,EAAE,IAAI;wBACf,UAAU,EAAE,IAAI;qBACjB,CAAC,CAAC;oBACH,MAAM,IAAI,CAAC,gBAAgB,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;oBACjD,eAAe,EAAE,CAAC;oBAClB,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,wBAAwB,IAAI,CAAC,EAAE,uBAAuB,IAAI,CAAC,SAAS,WAAW,CAChF,CAAC;gBACJ,CAAC;qBAAM,CAAC;oBACN,0EAA0E;oBAC1E,MAAM,IAAI,CAAC,gBAAgB,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;oBACjD,eAAe,EAAE,CAAC;gBACpB,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,wCAAwC;gBACxC,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,EAAE,EAAE,qBAAU,CAAC,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;gBACrE,MAAM,IAAI,CAAC,gBAAgB,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBACjD,eAAe,EAAE,CAAC;YACpB,CAAC;QACH,CAAC;QAED,6DAA6D;QAC7D,MAAM,YAAY,GAAG,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC,qBAAU,CAAC,OAAO,CAAC,CAAC,CAAC;QACtE,KAAK,MAAM,IAAI,IAAI,YAAY,EAAE,CAAC;YAChC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;gBACpB,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,EAAE,EAAE,qBAAU,CAAC,MAAM,EAAE,IAAI,CAAC,QAAQ,EAAE;oBACnE,YAAY,EAAE,gDAAgD;oBAC9D,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;iBACtC,CAAC,CAAC;gBACH,gBAAgB,EAAE,CAAC;gBACnB,SAAS;YACX,CAAC;YAED,MAAM,OAAO,GAAG,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAC1D,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,MAAM,KAAK,wBAAa,CAAC,OAAO,EAAE,CAAC;gBACzD,MAAM,aAAa,GAAG,OAAO,EAAE,aAAa;oBAC1C,CAAC,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC,OAAO,EAAE;oBAC3C,CAAC,CAAC,CAAC,CAAC;gBAEN,IAAI,GAAG,GAAG,aAAa,GAAG,0BAAe,EAAE,CAAC;oBAC1C,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,EAAE,EAAE,qBAAU,CAAC,MAAM,EAAE,IAAI,CAAC,QAAQ,EAAE;wBACnE,YAAY,EAAE,YAAY,OAAO,EAAE,IAAI,IAAI,IAAI,CAAC,SAAS,kCAAkC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,GAAG,aAAa,CAAC,GAAG,IAAI,CAAC,IAAI;wBACvI,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;qBACtC,CAAC,CAAC;oBACH,IAAI,IAAI,CAAC,SAAS;wBAAE,IAAI,CAAC,wBAAwB,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;oBAC3E,gBAAgB,EAAE,CAAC;oBACnB,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,8BAA8B,IAAI,CAAC,EAAE,aAAa,IAAI,CAAC,SAAS,cAAc,0BAAe,GAAG,IAAI,IAAI,CACzG,CAAC;gBACJ,CAAC;qBAAM,CAAC;oBACN,yDAAyD;oBACzD,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,EAAE,EAAE,qBAAU,CAAC,SAAS,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;oBACxE,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,2BAA2B,IAAI,CAAC,EAAE,aAAa,IAAI,CAAC,SAAS,gCAAgC,CAC9F,CAAC;gBACJ,CAAC;YACH,CAAC;QACH,CAAC;QAED,MAAM,cAAc,GAAG,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC,qBAAU,CAAC,SAAS,CAAC,CAAC,CAAC;QAC1E,KAAK,MAAM,IAAI,IAAI,cAAc,EAAE,CAAC;YAClC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;gBACpB,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,EAAE,EAAE,qBAAU,CAAC,MAAM,EAAE,IAAI,CAAC,QAAQ,EAAE;oBACnE,YAAY,EAAE,gDAAgD;oBAC9D,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;iBACtC,CAAC,CAAC;gBACH,gBAAgB,EAAE,CAAC;gBACnB,SAAS;YACX,CAAC;YAED,MAAM,OAAO,GAAG,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAC1D,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,MAAM,KAAK,wBAAa,CAAC,OAAO,EAAE,CAAC;gBACzD,MAAM,aAAa,GAAG,OAAO,EAAE,aAAa;oBAC1C,CAAC,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC,OAAO,EAAE;oBAC3C,CAAC,CAAC,CAAC,CAAC;gBAEN,IAAI,GAAG,GAAG,aAAa,GAAG,0BAAe,EAAE,CAAC;oBAC1C,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,EAAE,EAAE,qBAAU,CAAC,MAAM,EAAE,IAAI,CAAC,QAAQ,EAAE;wBACnE,YAAY,EAAE,YAAY,OAAO,EAAE,IAAI,IAAI,IAAI,CAAC,SAAS,+BAA+B;wBACxF,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;qBACtC,CAAC,CAAC;oBACH,IAAI,IAAI,CAAC,SAAS;wBAAE,IAAI,CAAC,wBAAwB,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;oBAC3E,gBAAgB,EAAE,CAAC;oBACnB,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,gCAAgC,IAAI,CAAC,EAAE,aAAa,IAAI,CAAC,SAAS,mBAAmB,CACtF,CAAC;gBACJ,CAAC;gBACD,sDAAsD;YACxD,CAAC;QACH,CAAC;QAED,uDAAuD;QACvD,6DAA6D;QAC7D,qEAAqE;QAErE,iFAAiF;QACjF,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC;QAC/C,KAAK,MAAM,OAAO,IAAI,WAAW,EAAE,CAAC;YAClC,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC,gBAAgB,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;YAClE,IAAI,OAAO,CAAC,WAAW,KAAK,WAAW,EAAE,CAAC;gBACxC,IAAI,CAAC,WAAW,CAAC,iBAAiB,CAAC,OAAO,CAAC,EAAE,EAAE,WAAW,CAAC,CAAC;gBAC5D,IAAI,CAAC,MAAM,CAAC,GAAG,CACb,YAAY,OAAO,CAAC,IAAI,6BAA6B,OAAO,CAAC,WAAW,MAAM,WAAW,EAAE,CAC5F,CAAC;YACJ,CAAC;QACH,CAAC;QAED,2BAA2B;QAC3B,MAAM,iBAAiB,GAAG,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;QACtD,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC;YACvB,MAAM,EAAE,sBAAsB;YAC9B,QAAQ,EAAE,QAAQ;YAClB,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC;gBACrB,UAAU,EAAE,eAAe;gBAC3B,WAAW,EAAE,gBAAgB;gBAC7B,iBAAiB;gBACjB,eAAe,EAAE,WAAW,CAAC,MAAM;aACpC,CAAC;SACH,CAAC,CAAC;QAEH,mFAAmF;QACnF,4EAA4E;QAC5E,gFAAgF;QAChF,MAAM,kBAAkB,GAAG,IAAI,CAAC,aAAa,CAAC,0BAA0B,EAAE,CAAC;QAC3E,IAAI,sBAAsB,GAAG,CAAC,CAAC;QAC/B,KAAK,MAAM,SAAS,IAAI,kBAAkB,EAAE,CAAC;YAC3C,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,cAAc,CAAC,eAAe,CACvC,SAAS,CAAC,MAAM,EAChB,SAAS,CAAC,SAAS,EACnB,SAAS,CAAC,IAAI,EACd,CAAC,CACF,CAAC;gBACF,sBAAsB,EAAE,CAAC;gBACzB,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,sCAAsC,SAAS,CAAC,EAAE,WAAW,SAAS,CAAC,MAAM,aAAa,SAAS,CAAC,SAAS,GAAG,CACjH,CAAC;YACJ,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,+CAA+C,SAAS,CAAC,EAAE,WAAW,SAAS,CAAC,MAAM,MACpF,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CACjD,EAAE,CACH,CAAC;YACJ,CAAC;QACH,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,GAAG,CACb,oCAAoC,eAAe,kBAAkB,gBAAgB,kBAAkB,iBAAiB,yBAAyB,sBAAsB,EAAE,CAC1K,CAAC;IACJ,CAAC;IAED,8EAA8E;IAC9E,uDAAuD;IACvD,EAAE;IACF,8BAA8B;IAC9B,6DAA6D;IAC7D,iDAAiD;IACjD,sDAAsD;IACtD,8CAA8C;IAC9C,sCAAsC;IACtC,6BAA6B;IAC7B,8EAA8E;IAE9E,KAAK,CAAC,eAAe,CACnB,SAAiB,EACjB,KAAoC;QAEpC,IAAI,CAAC,MAAM,CAAC,GAAG,CACb,qCAAqC,SAAS,SAAS,KAAK,CAAC,MAAM,mBAAmB,CACvF,CAAC;QAEF,MAAM,OAAO,GAAG,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;QACrD,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,kCAAkC,SAAS,YAAY,CAAC,CAAC;YAC1E,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,MAAM,OAAO,GAA8C,EAAE,CAAC;QAC9D,MAAM,eAAe,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;QAC5D,MAAM,eAAe,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAEjE,6EAA6E;QAC7E,MAAM,kBAAkB,GAAG,IAAI,CAAC,QAAQ,CAAC,eAAe,CAAC,SAAS,EAAE;YAClE,qBAAU,CAAC,OAAO;YAClB,qBAAU,CAAC,SAAS;YACpB,qBAAU,CAAC,QAAQ;SACpB,CAAC,CAAC;QACH,MAAM,oBAAoB,GAAG,IAAI,CAAC,QAAQ,CAAC,eAAe,CAAC,SAAS,EAAE;YACpE,qBAAU,CAAC,MAAM;YACjB,qBAAU,CAAC,SAAS;SACrB,CAAC,CAAC;QACH,MAAM,eAAe,GAAG,CAAC,GAAG,kBAAkB,EAAE,GAAG,oBAAoB,CAAC,CAAC;QAEzE,wDAAwD;QACxD,KAAK,MAAM,IAAI,IAAI,eAAe,EAAE,CAAC;YACnC,MAAM,QAAQ,GAAG,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAE9C,IAAI,QAAQ,EAAE,CAAC;gBACb,MAAM,MAAM,GAAG,IAAI,CAAC,qBAAqB,CAAC,OAAO,EAAE,IAAI,EAAE,QAAQ,CAAC,CAAC;gBACnE,OAAO,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;YAC5C,CAAC;iBAAM,IACL,IAAI,CAAC,MAAM,KAAK,qBAAU,CAAC,OAAO;gBAClC,IAAI,CAAC,MAAM,KAAK,qBAAU,CAAC,SAAS,EACpC,CAAC;gBACD,gEAAgE;gBAChE,qEAAqE;gBACrE,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,EAAE,EAAE,qBAAU,CAAC,MAAM,EAAE,IAAI,CAAC,QAAQ,EAAE;oBACnE,YAAY,EAAE,WAAW,OAAO,CAAC,IAAI,qEAAqE;oBAC1G,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;iBACtC,CAAC,CAAC;gBACH,IAAI,IAAI,CAAC,SAAS;oBAAE,IAAI,CAAC,wBAAwB,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;gBAC3E,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC;oBACvB,MAAM,EAAE,6BAA6B;oBACrC,QAAQ,EAAE,QAAQ,IAAI,CAAC,EAAE,EAAE;oBAC3B,MAAM,EAAE,gCAAgC,OAAO,CAAC,IAAI,oBAAoB;iBACzE,CAAC,CAAC;gBACH,OAAO,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,EAAE,MAAM,EAAE,mBAAmB,EAAE,CAAC,CAAC;gBAC/D,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,SAAS,IAAI,CAAC,EAAE,KAAK,IAAI,CAAC,MAAM,yCAAyC,CAC1E,CAAC;YACJ,CAAC;iBAAM,IAAI,IAAI,CAAC,MAAM,KAAK,qBAAU,CAAC,QAAQ,EAAE,CAAC;gBAC/C,+DAA+D;gBAC/D,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,EAAE,EAAE,qBAAU,CAAC,MAAM,EAAE,IAAI,CAAC,QAAQ,EAAE;oBACnE,SAAS,EAAE,IAAI;oBACf,UAAU,EAAE,IAAI;iBACjB,CAAC,CAAC;gBACH,MAAM,IAAI,CAAC,gBAAgB,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBACjD,OAAO,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,EAAE,MAAM,EAAE,iBAAiB,EAAE,CAAC,CAAC;gBAC7D,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,SAAS,IAAI,CAAC,EAAE,qDAAqD,CACtE,CAAC;YACJ,CAAC;QACH,CAAC;QAED,+EAA+E;QAC/E,KAAK,MAAM,QAAQ,IAAI,KAAK,EAAE,CAAC;YAC7B,IAAI,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,QAAQ,CAAC,MAAM,CAAC;gBAAE,SAAS;YAEpE,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YACrD,IAAI,CAAC,IAAI,EAAE,CAAC;gBACV,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,iCAAiC,QAAQ,CAAC,MAAM,sBAAsB,CACvE,CAAC;gBACF,IAAI,CAAC,mBAAmB,CAAC,SAAS,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;gBACrD,OAAO,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,MAAM,EAAE,iBAAiB,EAAE,CAAC,CAAC;gBACrE,SAAS;YACX,CAAC;YAED,0EAA0E;YAC1E,MAAM,MAAM,GAAG,IAAI,CAAC,qBAAqB,CAAC,OAAO,EAAE,IAAI,EAAE,QAAQ,CAAC,CAAC;YACnE,OAAO,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;QAC5C,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,GAAG,CACb,wCAAwC,SAAS,KAAK,OAAO,CAAC,MAAM,kBAAkB,CACvF,CAAC;QAEF,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,8EAA8E;IAC9E,2DAA2D;IAC3D,8EAA8E;IAEtE,qBAAqB,CAC3B,OAAgB,EAChB,IAAU,EACV,QAA+C;QAE/C,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,EAAE,KAAK,EAAE,YAAY,EAAE,GACtE,QAAQ,CAAC;QAEX,QAAQ,IAAI,CAAC,MAAM,EAAE,CAAC;YACpB,oEAAoE;YACpE,wDAAwD;YACxD,KAAK,qBAAU,CAAC,SAAS,CAAC,CAAC,CAAC;gBAC1B,IAAI,WAAW,KAAK,eAAe,EAAE,CAAC;oBACpC,MAAM,WAAW,GAAG,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;oBAChD,IAAI,WAAW,GAAG,+BAAoB,EAAE,CAAC;wBACvC,MAAM,KAAK,GAA4B,EAAE,CAAC;wBAC1C,IAAI,YAAY;4BAAE,KAAK,CAAC,cAAc,CAAC,GAAG,YAAY,CAAC;wBACvD,IAAI,CAAC,oBAAoB,CACvB,MAAM,EACN,qBAAU,CAAC,OAAO,EAClB,IAAI,CAAC,QAAQ,EACb,KAAK,CACN,CAAC;wBAEF,8BAA8B;wBAC9B,MAAM,WAAW,GAAoB;4BACnC,IAAI,EAAE,QAAQ;4BACd,KAAK,EAAE,MAAM,CAAC,UAAU,EAAE;4BAC1B,MAAM;yBACP,CAAC;wBACF,IAAI,CAAC,uBAAuB,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,EAAE,WAAW,CAAC,CAAC;wBAE3D,IAAI,CAAC,MAAM,CAAC,GAAG,CACb,SAAS,MAAM,iDAAiD,CACjE,CAAC;wBACF,OAAO,SAAS,CAAC;oBACnB,CAAC;oBACD,qCAAqC;oBACrC,IAAI,CAAC,MAAM,CAAC,GAAG,CACb,SAAS,MAAM,+CAA+C,WAAW,IAAI,+BAAoB,GAAG,CACrG,CAAC;oBACF,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC;wBACvB,MAAM,EAAE,6BAA6B;wBACrC,QAAQ,EAAE,QAAQ,MAAM,EAAE;wBAC1B,MAAM,EAAE,4CAA4C,WAAW,IAAI,+BAAoB,GAAG;qBAC3F,CAAC,CAAC;oBACH,OAAO,mBAAmB,CAAC;gBAC7B,CAAC;gBAED,IAAI,WAAW,KAAK,mBAAmB,EAAE,CAAC;oBACxC,MAAM,KAAK,GAA4B;wBACrC,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;qBACtC,CAAC;oBACF,IAAI,MAAM;wBAAE,KAAK,CAAC,QAAQ,CAAC,GAAG,MAAM,CAAC;oBACrC,IAAI,KAAK;wBAAE,KAAK,CAAC,OAAO,CAAC,GAAG,KAAK,CAAC;oBAClC,IAAI,CAAC,oBAAoB,CACvB,MAAM,EACN,qBAAU,CAAC,SAAS,EACpB,IAAI,CAAC,QAAQ,EACb,KAAK,CACN,CAAC;oBACF,IAAI,IAAI,CAAC,SAAS;wBAAE,IAAI,CAAC,wBAAwB,CAAC,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;oBAC1E,IAAI,CAAC,MAAM,CAAC,GAAG,CACb,SAAS,MAAM,uDAAuD,CACvE,CAAC;oBACF,OAAO,WAAW,CAAC;gBACrB,CAAC;gBAED,IAAI,WAAW,KAAK,gBAAgB,EAAE,CAAC;oBACrC,IAAI,CAAC,oBAAoB,CAAC,MAAM,EAAE,qBAAU,CAAC,MAAM,EAAE,IAAI,CAAC,QAAQ,EAAE;wBAClE,YAAY,EACV,YAAY,IAAI,uCAAuC;wBACzD,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;qBACtC,CAAC,CAAC;oBACH,IAAI,IAAI,CAAC,SAAS;wBAAE,IAAI,CAAC,wBAAwB,CAAC,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;oBAC1E,IAAI,CAAC,MAAM,CAAC,GAAG,CACb,SAAS,MAAM,iDAAiD,CACjE,CAAC;oBACF,OAAO,QAAQ,CAAC;gBAClB,CAAC;gBAED,OAAO,OAAO,CAAC;YACjB,CAAC;YAED,6DAA6D;YAC7D,gEAAgE;YAChE,KAAK,qBAAU,CAAC,MAAM,CAAC,CAAC,CAAC;gBACvB,IAAI,WAAW,KAAK,eAAe,EAAE,CAAC;oBACpC,IAAI,CAAC,MAAM,CAAC,GAAG,CACb,SAAS,MAAM,sDAAsD,CACtE,CAAC;oBACF,IAAI,CAAC,kBAAkB,CAAC,OAAO,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;oBAC5C,OAAO,aAAa,CAAC;gBACvB,CAAC;gBAED,IAAI,WAAW,KAAK,mBAAmB,EAAE,CAAC;oBACxC,MAAM,KAAK,GAA4B;wBACrC,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;wBACrC,YAAY,EAAE,IAAI;qBACnB,CAAC;oBACF,IAAI,MAAM;wBAAE,KAAK,CAAC,QAAQ,CAAC,GAAG,MAAM,CAAC;oBACrC,IAAI,KAAK;wBAAE,KAAK,CAAC,OAAO,CAAC,GAAG,KAAK,CAAC;oBAElC,IAAI,CAAC,oBAAoB,CACvB,MAAM,EACN,qBAAU,CAAC,SAAS,EACpB,IAAI,CAAC,QAAQ,EACb,KAAK,CACN,CAAC;oBACF,IAAI,IAAI,CAAC,SAAS;wBAAE,IAAI,CAAC,wBAAwB,CAAC,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;oBAC1E,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC;wBACvB,MAAM,EAAE,6BAA6B;wBACrC,QAAQ,EAAE,QAAQ,MAAM,EAAE;wBAC1B,MAAM,EACJ,sEAAsE;qBACzE,CAAC,CAAC;oBACH,IAAI,CAAC,MAAM,CAAC,GAAG,CACb,SAAS,MAAM,6DAA6D,CAC7E,CAAC;oBACF,OAAO,oBAAoB,CAAC;gBAC9B,CAAC;gBAED,OAAO,OAAO,CAAC;YACjB,CAAC;YAED,gDAAgD;YAChD,KAAK,qBAAU,CAAC,SAAS,CAAC,CAAC,CAAC;gBAC1B,IACE,WAAW,KAAK,eAAe;oBAC/B,WAAW,KAAK,mBAAmB,EACnC,CAAC;oBACD,+DAA+D;oBAC/D,IAAI,CAAC,MAAM,CAAC,GAAG,CACb,SAAS,MAAM,sDAAsD,WAAW,GAAG,CACpF,CAAC;oBACF,IAAI,CAAC,kBAAkB,CAAC,OAAO,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;oBAE5C,IAAI,WAAW,KAAK,mBAAmB,EAAE,CAAC;wBACxC,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC;4BACvB,MAAM,EAAE,mCAAmC;4BAC3C,QAAQ,EAAE,QAAQ,MAAM,EAAE;4BAC1B,MAAM,EACJ,2FAA2F;yBAC9F,CAAC,CAAC;oBACL,CAAC;oBACD,OAAO,aAAa,CAAC;gBACvB,CAAC;gBACD,OAAO,kBAAkB,CAAC;YAC5B,CAAC;YAED,uEAAuE;YACvE,KAAK,qBAAU,CAAC,OAAO,CAAC,CAAC,CAAC;gBACxB,IAAI,WAAW,KAAK,eAAe,EAAE,CAAC;oBACpC,IAAI,YAAY,IAAI,YAAY,KAAK,IAAI,CAAC,YAAY,EAAE,CAAC;wBACvD,IAAI,CAAC,oBAAoB,CACvB,MAAM,EACN,qBAAU,CAAC,OAAO,EAClB,IAAI,CAAC,QAAQ,EACb,EAAE,YAAY,EAAE,CACjB,CAAC;oBACJ,CAAC;oBACD,OAAO,OAAO,CAAC;gBACjB,CAAC;gBAED,IAAI,WAAW,KAAK,mBAAmB,EAAE,CAAC;oBACxC,MAAM,KAAK,GAA4B;wBACrC,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;qBACtC,CAAC;oBACF,IAAI,MAAM;wBAAE,KAAK,CAAC,QAAQ,CAAC,GAAG,MAAM,CAAC;oBACrC,IAAI,KAAK;wBAAE,KAAK,CAAC,OAAO,CAAC,GAAG,KAAK,CAAC;oBAClC,IAAI,CAAC,oBAAoB,CACvB,MAAM,EACN,qBAAU,CAAC,SAAS,EACpB,IAAI,CAAC,QAAQ,EACb,KAAK,CACN,CAAC;oBACF,IAAI,IAAI,CAAC,SAAS;wBAAE,IAAI,CAAC,wBAAwB,CAAC,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;oBAC1E,IAAI,CAAC,MAAM,CAAC,GAAG,CACb,SAAS,MAAM,qDAAqD,CACrE,CAAC;oBACF,OAAO,WAAW,CAAC;gBACrB,CAAC;gBAED,IAAI,WAAW,KAAK,gBAAgB,EAAE,CAAC;oBACrC,IAAI,CAAC,oBAAoB,CAAC,MAAM,EAAE,qBAAU,CAAC,MAAM,EAAE,IAAI,CAAC,QAAQ,EAAE;wBAClE,YAAY,EACV,YAAY,IAAI,uCAAuC;wBACzD,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;qBACtC,CAAC,CAAC;oBACH,IAAI,IAAI,CAAC,SAAS;wBAAE,IAAI,CAAC,wBAAwB,CAAC,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;oBAC1E,IAAI,CAAC,MAAM,CAAC,GAAG,CACb,SAAS,MAAM,+CAA+C,CAC/D,CAAC;oBACF,OAAO,QAAQ,CAAC;gBAClB,CAAC;gBAED,OAAO,OAAO,CAAC;YACjB,CAAC;YAED,yEAAyE;YACzE,KAAK,qBAAU,CAAC,QAAQ,CAAC,CAAC,CAAC;gBACzB,IAAI,WAAW,KAAK,eAAe,EAAE,CAAC;oBACpC,IAAI,CAAC,oBAAoB,CAAC,MAAM,EAAE,qBAAU,CAAC,OAAO,EAAE,IAAI,CAAC,QAAQ,EAAE;wBACnE,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;qBACpC,CAAC,CAAC;oBACH,IAAI,CAAC,MAAM,CAAC,GAAG,CACb,SAAS,MAAM,qDAAqD,CACrE,CAAC;oBACF,OAAO,qBAAqB,CAAC;gBAC/B,CAAC;gBAED,IAAI,WAAW,KAAK,mBAAmB,EAAE,CAAC;oBACxC,MAAM,KAAK,GAA4B;wBACrC,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;wBACrC,SAAS,EAAE,IAAI,CAAC,SAAS,IAAI,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;qBACtD,CAAC;oBACF,IAAI,MAAM;wBAAE,KAAK,CAAC,QAAQ,CAAC,GAAG,MAAM,CAAC;oBACrC,IAAI,KAAK;wBAAE,KAAK,CAAC,OAAO,CAAC,GAAG,KAAK,CAAC;oBAClC,IAAI,CAAC,oBAAoB,CACvB,MAAM,EACN,qBAAU,CAAC,SAAS,EACpB,IAAI,CAAC,QAAQ,EACb,KAAK,CACN,CAAC;oBACF,IAAI,IAAI,CAAC,SAAS;wBAAE,IAAI,CAAC,wBAAwB,CAAC,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;oBAC1E,IAAI,CAAC,MAAM,CAAC,GAAG,CACb,SAAS,MAAM,4CAA4C,CAC5D,CAAC;oBACF,OAAO,WAAW,CAAC;gBACrB,CAAC;gBAED,OAAO,OAAO,CAAC;YACjB,CAAC;YAED,KAAK,qBAAU,CAAC,SAAS;gBACvB,OAAO,kBAAkB,CAAC;YAE5B;gBACE,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,SAAS,MAAM,cAAc,IAAI,CAAC,MAAM,6BAA6B,CACtE,CAAC;gBACF,OAAO,OAAO,CAAC;QACnB,CAAC;IACH,CAAC;IAED,8EAA8E;IAC9E,kBAAkB;IAClB,8EAA8E;IAE9E;;;OAGG;IACK,oBAAoB,CAC1B,MAAc,EACd,MAAkB,EAClB,eAAuB,EACvB,KAA+B;QAE/B,IAAI,QAAQ,GAAG,eAAe,CAAC;QAE/B,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,GAAG,0BAAe,EAAE,OAAO,EAAE,EAAE,CAAC;YAC3D,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,YAAY,CACxC,MAAM,EACN,MAAM,EACN,QAAQ,EACR,KAAK,CACN,CAAC;YACF,IAAI,OAAO;gBAAE,OAAO,IAAI,CAAC;YAEzB,MAAM,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YACjD,IAAI,CAAC,SAAS,EAAE,CAAC;gBACf,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,SAAS,MAAM,gDAAgD,CAChE,CAAC;gBACF,OAAO,KAAK,CAAC;YACf,CAAC;YAED,qCAAqC;YACrC,IAAI,SAAS,CAAC,MAAM,KAAK,MAAM;gBAAE,OAAO,IAAI,CAAC;YAE7C,QAAQ,GAAG,SAAS,CAAC,QAAQ,CAAC;QAChC,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,iCAAiC,MAAM,MAAM,MAAM,2BAA2B,0BAAe,YAAY,CAC1G,CAAC;QACF,OAAO,KAAK,CAAC;IACf,CAAC;IAED,yDAAyD;IACjD,kBAAkB,CAAC,SAAiB,EAAE,MAAc;QAC1D,MAAM,KAAK,GAAoB;YAC7B,IAAI,EAAE,QAAQ;YACd,KAAK,EAAE,MAAM,CAAC,UAAU,EAAE;YAC1B,MAAM;SACP,CAAC;QACF,IAAI,CAAC,uBAAuB,CAAC,IAAI,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;IACtD,CAAC;IAED,8DAA8D;IACtD,mBAAmB,CAAC,SAAiB,EAAE,MAAc;QAC3D,MAAM,SAAS,GAAG,IAAI,CAAC,aAAa,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;QAC1D,MAAM,KAAK,GAA0B;YACnC,IAAI,EAAE,mBAAmB;YACzB,KAAK,EAAE,MAAM,CAAC,UAAU,EAAE;YAC1B,MAAM;YACN,aAAa,EAAE,SAAS,EAAE,IAAI,IAAI,EAAE;SACrC,CAAC;QACF,IAAI,CAAC,uBAAuB,CAAC,IAAI,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;IACtD,CAAC;IAED,2EAA2E;IACnE,wBAAwB,CAAC,MAAc,EAAE,SAAiB;QAChE,MAAM,SAAS,GAAG,IAAI,CAAC,aAAa,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;QAC1D,IAAI,SAAS,EAAE,CAAC;YACd,IAAI,CAAC,cAAc;iBAChB,eAAe,CAAC,MAAM,EAAE,SAAS,EAAE,SAAS,CAAC,IAAI,CAAC;iBAClD,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;gBACb,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,wCAAwC,MAAM,KAC5C,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CACjD,EAAE,CACH,CAAC;YACJ,CAAC,CAAC,CAAC;QACP,CAAC;IACH,CAAC;CACF,CAAA;AA5nBY,gCAAU;qBAAV,UAAU;IADtB,IAAA,mBAAU,GAAE;qCAKkB,gCAAc;QACX,sCAAiB;QAChB,yCAAkB;QACjB,0CAAmB;QAChB,oCAAgB;QAClB,gCAAc;QACL,mDAAuB;GAVxD,UAAU,CA4nBtB"}

package/dist/dispatcher/scheduler.service.d.ts

@@ -0,0 +1,50 @@
+import { OnModuleInit, OnModuleDestroy } from '@nestjs/common';
+import { RedisService } from '../redis/redis.service';
+import { TaskRepository } from '../database/repositories/task.repository';
+import { MachineRepository } from '../database/repositories/machine.repository';
+import { DeveloperRepository } from '../database/repositories/developer.repository';
+import { AuthService } from '../auth/auth.service';
+import { MachineSelector } from './machine-selector';
+import { WorkerConnectionManager } from './worker-connection.manager';
+export declare class SchedulerService implements OnModuleInit, OnModuleDestroy {
+    private readonly redis;
+    private readonly taskRepo;
+    private readonly machineRepo;
+    private readonly machineSelector;
+    private readonly workerConnMgr;
+    private readonly authService;
+    private readonly developerRepo;
+    private readonly logger;
+    private queue;
+    private worker;
+    private pollTimer;
+    constructor(redis: RedisService, taskRepo: TaskRepository, machineRepo: MachineRepository, machineSelector: MachineSelector, workerConnMgr: WorkerConnectionManager, authService: AuthService, developerRepo: DeveloperRepository);
+    onModuleInit(): Promise<void>;
+    onModuleDestroy(): Promise<void>;
+    /**
+     * Enqueue a task for immediate scheduling (used when a new task is created).
+     */
+    enqueueTask(taskId: number): Promise<void>;
+    /**
+     * Enqueue a task for resume (used when a suspended task's machine comes back online).
+     */
+    enqueueResume(taskId: number): Promise<void>;
+    /**
+     * Poll for QUEUED tasks and enqueue them for processing.
+     * Also check SUSPENDED tasks whose machines are back online.
+     */
+    private pollQueuedTasks;
+    /**
+     * Process a single scheduler job.
+     */
+    private processJob;
+    /**
+     * Attempt to assign a QUEUED task to a machine and transition to ASSIGNED.
+     */
+    private processScheduleTask;
+    /**
+     * Attempt to resume a SUSPENDED task on its assigned machine.
+     */
+    private processResumeTask;
+}
+//# sourceMappingURL=scheduler.service.d.ts.map

package/dist/dispatcher/scheduler.service.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"scheduler.service.d.ts","sourceRoot":"","sources":["../../src/dispatcher/scheduler.service.ts"],"names":[],"mappings":"AAAA,OAAO,EAGL,YAAY,EACZ,eAAe,EAChB,MAAM,gBAAgB,CAAC;AAWxB,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AACtD,OAAO,EAAE,cAAc,EAAE,MAAM,0CAA0C,CAAC;AAC1E,OAAO,EAAE,iBAAiB,EAAE,MAAM,6CAA6C,CAAC;AAChF,OAAO,EAAE,mBAAmB,EAAE,MAAM,+CAA+C,CAAC;AACpF,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AACnD,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,EAAE,uBAAuB,EAAE,MAAM,6BAA6B,CAAC;AAStE,qBACa,gBAAiB,YAAW,YAAY,EAAE,eAAe;IAOlE,OAAO,CAAC,QAAQ,CAAC,KAAK;IACtB,OAAO,CAAC,QAAQ,CAAC,QAAQ;IACzB,OAAO,CAAC,QAAQ,CAAC,WAAW;IAC5B,OAAO,CAAC,QAAQ,CAAC,eAAe;IAChC,OAAO,CAAC,QAAQ,CAAC,aAAa;IAC9B,OAAO,CAAC,QAAQ,CAAC,WAAW;IAC5B,OAAO,CAAC,QAAQ,CAAC,aAAa;IAZhC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAqC;IAC5D,OAAO,CAAC,KAAK,CAAS;IACtB,OAAO,CAAC,MAAM,CAAc;IAC5B,OAAO,CAAC,SAAS,CAA+C;gBAG7C,KAAK,EAAE,YAAY,EACnB,QAAQ,EAAE,cAAc,EACxB,WAAW,EAAE,iBAAiB,EAC9B,eAAe,EAAE,eAAe,EAChC,aAAa,EAAE,uBAAuB,EACtC,WAAW,EAAE,WAAW,EACxB,aAAa,EAAE,mBAAmB;IAG/C,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC;IA+C7B,eAAe,IAAI,OAAO,CAAC,IAAI,CAAC;IAiBtC;;OAEG;IACG,WAAW,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAMhD;;OAEG;IACG,aAAa,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAMlD;;;OAGG;YACW,eAAe;IAoC7B;;OAEG;YACW,UAAU;IAgBxB;;OAEG;YACW,mBAAmB;IA6IjC;;OAEG;YACW,iBAAiB;CA0ChC"}