@overlordai/server 1.0.53 → 1.0.55

package/dist/dispatcher/reconciler.js

@@ -44,38 +44,79 @@ var __metadata = (this && this.__metadata) || function (k, v) {
 var Reconciler_1;
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.Reconciler = void 0;
+const protocol_1 = require("@overlordai/protocol");
 const common_1 = require("@nestjs/common");
 const crypto = __importStar(require("node:crypto"));
-const protocol_1 = require("@overlordai/protocol");
 const task_repository_1 = require("../database/repositories/task.repository");
-const machine_repository_1 = require("../database/repositories/machine.repository");
+const worker_repository_1 = require("../database/repositories/worker.repository");
 const audit_log_repository_1 = require("../database/repositories/audit-log.repository");
 const workspace_repository_1 = require("../database/repositories/workspace.repository");
+const event_emitter_1 = require("@nestjs/event-emitter");
+const event_types_1 = require("../events/event-types");
 const scheduler_service_1 = require("./scheduler.service");
 const cleanup_service_1 = require("./cleanup.service");
 const worker_connection_manager_1 = require("./worker-connection.manager");
-let Reconciler = Reconciler_1 = class Reconciler {
+let Reconciler = class Reconciler {
+    static { Reconciler_1 = this; }
     taskRepo;
-    machineRepo;
+    workerRepo;
     auditLogRepo;
     workspaceRepo;
     schedulerService;
     cleanupService;
     workerConnectionManager;
+    eventEmitter;
     logger = new common_1.Logger(Reconciler_1.name);
-    constructor(taskRepo, machineRepo, auditLogRepo, workspaceRepo, schedulerService, cleanupService, workerConnectionManager) {
+    failedCleanups = new Map();
+    cleanupRetryTimer = null;
+    static CLEANUP_RETRY_INTERVAL_MS = 5 * 60 * 1000; // 5 minutes
+    constructor(taskRepo, workerRepo, auditLogRepo, workspaceRepo, schedulerService, cleanupService, workerConnectionManager, eventEmitter) {
         this.taskRepo = taskRepo;
-        this.machineRepo = machineRepo;
+        this.workerRepo = workerRepo;
         this.auditLogRepo = auditLogRepo;
         this.workspaceRepo = workspaceRepo;
         this.schedulerService = schedulerService;
         this.cleanupService = cleanupService;
         this.workerConnectionManager = workerConnectionManager;
+        this.eventEmitter = eventEmitter;
     }
     async onModuleInit() {
         this.logger.log('Running boot reconciliation...');
         await this.onBoot();
         this.logger.log('Boot reconciliation complete');
+        this.startCleanupRetryLoop();
+    }
+    onModuleDestroy() {
+        if (this.cleanupRetryTimer) {
+            clearInterval(this.cleanupRetryTimer);
+            this.cleanupRetryTimer = null;
+        }
+    }
+    startCleanupRetryLoop() {
+        this.cleanupRetryTimer = setInterval(() => {
+            void this.retryFailedCleanups();
+        }, Reconciler_1.CLEANUP_RETRY_INTERVAL_MS);
+    }
+    async retryFailedCleanups() {
+        if (this.failedCleanups.size === 0)
+            return;
+        this.logger.log(`Retrying ${this.failedCleanups.size} failed workspace cleanup(s)...`);
+        const entries = [...this.failedCleanups.entries()];
+        for (const [key, entry] of entries) {
+            const workspace = this.workspaceRepo.findByTaskId(entry.taskId);
+            if (!workspace) {
+                this.failedCleanups.delete(key);
+                continue;
+            }
+            try {
+                await this.cleanupService.scheduleCleanup(entry.taskId, entry.workerId, workspace.path);
+                this.failedCleanups.delete(key);
+                this.logger.log(`Successfully retried cleanup for task #${entry.taskId}`);
+            }
+            catch (err) {
+                this.logger.error(`Retry cleanup still failing for task #${entry.taskId}: ${(0, protocol_1.getErrorMessage)(err)}`);
+            }
+        }
     }
     // ---------------------------------------------------------------------------
     // onBoot — startup reconciliation
@@ -84,99 +125,137 @@ let Reconciler = Reconciler_1 = class Reconciler {
         const now = Date.now();
         let reEnqueuedCount = 0;
         let failedStaleCount = 0;
+        // Single query to fetch all active tasks instead of 4 separate findByStatus calls
+        const allActiveTasks = this.taskRepo.findByStatus([
+            protocol_1.TaskStatus.QUEUED,
+            protocol_1.TaskStatus.ASSIGNED,
+            protocol_1.TaskStatus.RUNNING,
+            protocol_1.TaskStatus.SUSPENDED,
+        ]);
+        // Group tasks by status in memory
+        const queuedTasks = [];
+        const assignedTasks = [];
+        const runningTasks = [];
+        const suspendedTasks = [];
+        for (const task of allActiveTasks) {
+            switch (task.status) {
+                case protocol_1.TaskStatus.QUEUED:
+                    queuedTasks.push(task);
+                    break;
+                case protocol_1.TaskStatus.ASSIGNED:
+                    assignedTasks.push(task);
+                    break;
+                case protocol_1.TaskStatus.RUNNING:
+                    runningTasks.push(task);
+                    break;
+                case protocol_1.TaskStatus.SUSPENDED:
+                    suspendedTasks.push(task);
+                    break;
+            }
+        }
+        // Pre-fetch all referenced workers in a single batch lookup
+        const workerIds = new Set();
+        for (const task of allActiveTasks) {
+            if (task.workerId)
+                workerIds.add(task.workerId);
+        }
+        const workerCache = new Map();
+        for (const wid of workerIds) {
+            workerCache.set(wid, this.workerRepo.findById(wid));
+        }
         // Step 1: Re-enqueue QUEUED/ASSIGNED tasks missing from BullMQ.
         // On boot we conservatively re-enqueue all QUEUED tasks since BullMQ
         // may have lost state during a restart.
-        const queuedTasks = this.taskRepo.findByStatus([protocol_1.TaskStatus.QUEUED]);
         for (const task of queuedTasks) {
             await this.schedulerService.enqueueTask(task.id);
             reEnqueuedCount++;
             this.logger.log(`Re-enqueued QUEUED task #${task.id}`);
         }
-        // ASSIGNED tasks with offline/missing machines get reset to QUEUED
-        const assignedTasks = this.taskRepo.findByStatus([protocol_1.TaskStatus.ASSIGNED]);
+        // ASSIGNED tasks with offline/missing workers get reset to QUEUED
         for (const task of assignedTasks) {
-            if (task.machineId) {
-                const machine = this.machineRepo.findById(task.machineId);
-                if (this.isMachineUnavailable(machine)) {
+            if (task.workerId) {
+                const worker = workerCache.get(task.workerId) ?? null;
+                if (this.isWorkerUnavailable(worker)) {
                     this.updateTaskStatusSafe(task.id, protocol_1.TaskStatus.QUEUED, task.revision, {
-                        machineId: null,
+                        workerId: null,
                         assignedAt: null,
                     });
                     await this.schedulerService.enqueueTask(task.id);
                     reEnqueuedCount++;
-                    this.logger.warn(`Reset ASSIGNED task #${task.id} to QUEUED (machine ${task.machineId} offline)`);
+                    this.logger.warn(`Reset ASSIGNED task #${task.id} to QUEUED (worker ${task.workerId} offline)`);
                 }
                 else {
-                    // Machine online — re-enqueue so the scheduler can verify and re-dispatch
+                    // Worker online — re-enqueue so the scheduler can verify and re-dispatch
                     await this.schedulerService.enqueueTask(task.id);
                     reEnqueuedCount++;
                 }
             }
             else {
-                // No machine assigned — reset to QUEUED
+                // No worker assigned — reset to QUEUED
                 this.updateTaskStatusSafe(task.id, protocol_1.TaskStatus.QUEUED, task.revision);
                 await this.schedulerService.enqueueTask(task.id);
                 reEnqueuedCount++;
             }
         }
         // Step 2: Fail stale RUNNING/SUSPENDED tasks (grace 10 min).
-        const runningTasks = this.taskRepo.findByStatus([protocol_1.TaskStatus.RUNNING]);
         for (const task of runningTasks) {
-            if (!task.machineId) {
+            if (!task.workerId) {
                 this.updateTaskStatusSafe(task.id, protocol_1.TaskStatus.FAILED, task.revision, {
-                    errorMessage: 'No machine assigned during boot reconciliation',
+                    errorMessage: 'No worker assigned during boot reconciliation',
                     completedAt: new Date().toISOString(),
                 });
+                this.emitTaskStatusChanged(task.id, protocol_1.TaskStatus.FAILED, task.status, task.workerId, task.projectKey);
                 failedStaleCount++;
                 continue;
             }
-            const machine = this.machineRepo.findById(task.machineId);
-            if (this.isMachineUnavailable(machine)) {
-                const lastHeartbeat = machine?.lastHeartbeat
-                    ? new Date(machine.lastHeartbeat).getTime()
+            const worker = workerCache.get(task.workerId) ?? null;
+            if (this.isWorkerUnavailable(worker)) {
+                const lastHeartbeat = worker?.lastHeartbeat
+                    ? new Date(worker.lastHeartbeat).getTime()
                     : 0;
                 if (now - lastHeartbeat > protocol_1.GRACE_WINDOW_MS) {
                     this.updateTaskStatusSafe(task.id, protocol_1.TaskStatus.FAILED, task.revision, {
-                        errorMessage: `Machine '${machine?.name ?? task.machineId}' offline beyond grace window (${Math.round((now - lastHeartbeat) / 1000)}s)`,
+                        errorMessage: `Worker '${worker?.name ?? task.workerId}' offline beyond grace window (${Math.round((now - lastHeartbeat) / 1000)}s)`,
                         completedAt: new Date().toISOString(),
                     });
-                    if (task.machineId)
-                        this.scheduleWorkspaceCleanup(task.id, task.machineId);
+                    this.emitTaskStatusChanged(task.id, protocol_1.TaskStatus.FAILED, task.status, task.workerId, task.projectKey);
+                    if (task.workerId)
+                        this.scheduleWorkspaceCleanup(task.id, task.workerId);
                     failedStaleCount++;
-                    this.logger.warn(`Failed stale RUNNING task #${task.id} (machine ${task.machineId} offline > ${protocol_1.GRACE_WINDOW_MS / 1000}s)`);
+                    this.logger.warn(`Failed stale RUNNING task #${task.id} (worker ${task.workerId} offline > ${protocol_1.GRACE_WINDOW_MS / 1000}s)`);
                 }
                 else {
                     // Within grace window: suspend and await Worker recovery
                     this.updateTaskStatusSafe(task.id, protocol_1.TaskStatus.SUSPENDED, task.revision);
-                    this.logger.warn(`Suspended RUNNING task #${task.id} (machine ${task.machineId} offline, within grace window)`);
+                    this.logger.warn(`Suspended RUNNING task #${task.id} (worker ${task.workerId} offline, within grace window)`);
                 }
             }
         }
-        const suspendedTasks = this.taskRepo.findByStatus([protocol_1.TaskStatus.SUSPENDED]);
         for (const task of suspendedTasks) {
-            if (!task.machineId) {
+            if (!task.workerId) {
                 this.updateTaskStatusSafe(task.id, protocol_1.TaskStatus.FAILED, task.revision, {
-                    errorMessage: 'No machine assigned during boot reconciliation',
+                    errorMessage: 'No worker assigned during boot reconciliation',
                     completedAt: new Date().toISOString(),
                 });
+                this.emitTaskStatusChanged(task.id, protocol_1.TaskStatus.FAILED, task.status, task.workerId, task.projectKey);
                 failedStaleCount++;
                 continue;
             }
-            const machine = this.machineRepo.findById(task.machineId);
-            if (this.isMachineUnavailable(machine)) {
-                const lastHeartbeat = machine?.lastHeartbeat
-                    ? new Date(machine.lastHeartbeat).getTime()
+            const worker = workerCache.get(task.workerId) ?? null;
+            if (this.isWorkerUnavailable(worker)) {
+                const lastHeartbeat = worker?.lastHeartbeat
+                    ? new Date(worker.lastHeartbeat).getTime()
                     : 0;
                 if (now - lastHeartbeat > protocol_1.GRACE_WINDOW_MS) {
                     this.updateTaskStatusSafe(task.id, protocol_1.TaskStatus.FAILED, task.revision, {
-                        errorMessage: `Machine '${machine?.name ?? task.machineId}' offline beyond grace window`,
+                        errorMessage: `Worker '${worker?.name ?? task.workerId}' offline beyond grace window`,
                         completedAt: new Date().toISOString(),
                     });
-                    if (task.machineId)
-                        this.scheduleWorkspaceCleanup(task.id, task.machineId);
+                    this.emitTaskStatusChanged(task.id, protocol_1.TaskStatus.FAILED, task.status, task.workerId, task.projectKey);
+                    if (task.workerId)
+                        this.scheduleWorkspaceCleanup(task.id, task.workerId);
                     failedStaleCount++;
-                    this.logger.warn(`Failed stale SUSPENDED task #${task.id} (machine ${task.machineId} offline > grace)`);
+                    this.logger.warn(`Failed stale SUSPENDED task #${task.id} (worker ${task.workerId} offline > grace)`);
                 }
                 // Within grace: keep SUSPENDED, await Worker recovery
             }
@@ -184,13 +263,13 @@ let Reconciler = Reconciler_1 = class Reconciler {
         // Step 3: Remove stale BullMQ jobs for terminal tasks.
         // BullMQ removeOnComplete/removeOnFail handles most cleanup.
         // The scheduler's processJob re-checks status and skips stale tasks.
-        // Step 4: Recalculate active_slots for all machines and update DB if mismatched.
-        const allMachines = this.machineRepo.findAll();
-        for (const machine of allMachines) {
-            const actualSlots = this.machineRepo.countActiveSlots(machine.id);
-            if (machine.activeSlots !== actualSlots) {
-                this.machineRepo.updateActiveSlots(machine.id, actualSlots);
-                this.logger.log(`Machine '${machine.name}' active_slots corrected: ${machine.activeSlots} → ${actualSlots}`);
+        // Step 4: Recalculate active_slots for all workers and update DB if mismatched.
+        const allWorkers = this.workerRepo.findAll();
+        for (const worker of allWorkers) {
+            const actualSlots = this.workerRepo.countActiveSlots(worker.id);
+            if (worker.activeSlots !== actualSlots) {
+                this.workerRepo.updateActiveSlots(worker.id, actualSlots);
+                this.logger.log(`Worker '${worker.name}' active_slots corrected: ${worker.activeSlots} → ${actualSlots}`);
             }
         }
         // Step 5: Write audit_log.
@@ -202,7 +281,7 @@ let Reconciler = Reconciler_1 = class Reconciler {
                 reEnqueued: reEnqueuedCount,
                 failedStale: failedStaleCount,
                 globalActiveCount,
-                machinesChecked: allMachines.length,
+                workersChecked: allWorkers.length,
             }),
         });
         // Step 6 (branch g): Re-enqueue cleanup for terminal tasks with active workspaces.
@@ -212,12 +291,12 @@ let Reconciler = Reconciler_1 = class Reconciler {
         let cleanupReEnqueuedCount = 0;
         for (const workspace of orphanedWorkspaces) {
             try {
-                await this.cleanupService.scheduleCleanup(workspace.taskId, workspace.machineId, workspace.path, 0);
+                await this.cleanupService.scheduleCleanup(workspace.taskId, workspace.workerId, workspace.path, 0);
                 cleanupReEnqueuedCount++;
-                this.logger.warn(`Re-enqueued cleanup for workspace #${workspace.id} (task #${workspace.taskId}, machine ${workspace.machineId})`);
+                this.logger.warn(`Re-enqueued cleanup for workspace #${workspace.id} (task #${workspace.taskId}, worker ${workspace.workerId})`);
             }
             catch (err) {
-                this.logger.error(`Failed to re-enqueue cleanup for workspace #${workspace.id} (task #${workspace.taskId}): ${err instanceof Error ? err.message : String(err)}`);
+                this.logger.error(`Failed to re-enqueue cleanup for workspace #${workspace.id} (task #${workspace.taskId}): ${(0, protocol_1.getErrorMessage)(err)}`);
             }
         }
         this.logger.log(`Boot reconciliation: re-enqueued=${reEnqueuedCount}, failed_stale=${failedStaleCount}, active_tasks=${globalActiveCount}, cleanup_re-enqueued=${cleanupReEnqueuedCount}`);
@@ -233,32 +312,32 @@ let Reconciler = Reconciler_1 = class Reconciler {
     //   e. CANCELLED → notify Worker stop
     //   f. Not reported → FAILED
     // ---------------------------------------------------------------------------
-    async handleReconnect(machineId, tasks) {
-        this.logger.log(`Reconciling reconnect for machine ${machineId} with ${tasks.length} reported task(s)`);
-        const machine = this.machineRepo.findById(machineId);
-        if (!machine) {
-            this.logger.warn(`Reconnect from unknown machine ${machineId}, ignoring`);
+    async handleReconnect(workerId, tasks) {
+        this.logger.log(`Reconciling reconnect for worker ${workerId} with ${tasks.length} reported task(s)`);
+        const worker = this.workerRepo.findById(workerId);
+        if (!worker) {
+            this.logger.warn(`Reconnect from unknown worker ${workerId}, ignoring`);
             return [];
         }
         const actions = [];
         const reportedTaskIds = new Set(tasks.map((t) => t.taskId));
         const reportedTaskMap = new Map(tasks.map((t) => [t.taskId, t]));
-        // Get all tasks assigned to this machine in active or recent terminal states
-        const activeMachineTasks = this.taskRepo.findByMachineId(machineId, [
+        // Get all tasks assigned to this worker in active or recent terminal states
+        const activeWorkerTasks = this.taskRepo.findByWorkerId(workerId, [
             protocol_1.TaskStatus.RUNNING,
             protocol_1.TaskStatus.SUSPENDED,
             protocol_1.TaskStatus.ASSIGNED,
         ]);
-        const terminalMachineTasks = this.taskRepo.findByMachineId(machineId, [
+        const terminalWorkerTasks = this.taskRepo.findByWorkerId(workerId, [
             protocol_1.TaskStatus.FAILED,
             protocol_1.TaskStatus.CANCELLED,
         ]);
-        const allMachineTasks = [...activeMachineTasks, ...terminalMachineTasks];
-        // Process each machine task against the Worker's report
-        for (const task of allMachineTasks) {
+        const allWorkerTasks = [...activeWorkerTasks, ...terminalWorkerTasks];
+        // Process each worker task against the Worker's report
+        for (const task of allWorkerTasks) {
             const reported = reportedTaskMap.get(task.id);
             if (reported) {
-                const action = this.reconcileReportedTask(machine, task, reported);
+                const action = this.reconcileReportedTask(worker, task, reported);
                 actions.push({ taskId: task.id, action });
             }
             else if (task.status === protocol_1.TaskStatus.RUNNING ||
@@ -266,15 +345,16 @@ let Reconciler = Reconciler_1 = class Reconciler {
                 // Branch (f): Dispatcher has task but Worker did not report it.
                 // Worker lost execution context (process exited / Worker restarted).
                 this.updateTaskStatusSafe(task.id, protocol_1.TaskStatus.FAILED, task.revision, {
-                    errorMessage: `Worker '${machine.name}' reconnected but did not report this task (execution context lost)`,
+                    errorMessage: `Worker '${worker.name}' reconnected but did not report this task (execution context lost)`,
                     completedAt: new Date().toISOString(),
                 });
-                if (task.machineId)
-                    this.scheduleWorkspaceCleanup(task.id, task.machineId);
+                if (task.workerId)
+                    this.scheduleWorkspaceCleanup(task.id, task.workerId);
+                this.emitTaskStatusChanged(task.id, protocol_1.TaskStatus.FAILED, task.status, task.workerId, task.projectKey);
                 this.auditLogRepo.create({
                     action: 'reconcile_unreported_failed',
                     resource: `task:${task.id}`,
-                    detail: `Task not reported by Worker '${machine.name}' during reconnect`,
+                    detail: `Task not reported by Worker '${worker.name}' during reconnect`,
                 });
                 actions.push({ taskId: task.id, action: 'failed_unreported' });
                 this.logger.warn(`Task #${task.id} (${task.status}) not reported by Worker, marked FAILED`);
@@ -282,7 +362,7 @@ let Reconciler = Reconciler_1 = class Reconciler {
             else if (task.status === protocol_1.TaskStatus.ASSIGNED) {
                 // ASSIGNED but not reported — reset to QUEUED for rescheduling
                 this.updateTaskStatusSafe(task.id, protocol_1.TaskStatus.QUEUED, task.revision, {
-                    machineId: null,
+                    workerId: null,
                     assignedAt: null,
                 });
                 await this.schedulerService.enqueueTask(task.id);
@@ -290,34 +370,34 @@ let Reconciler = Reconciler_1 = class Reconciler {
                 this.logger.warn(`Task #${task.id} (ASSIGNED) not reported by Worker, reset to QUEUED`);
             }
         }
-        // Handle tasks reported by Worker that Dispatcher doesn't have on this machine
+        // Handle tasks reported by Worker that Dispatcher doesn't have on this worker
         for (const reported of tasks) {
-            if (allMachineTasks.some((t) => t.id === reported.taskId))
+            if (allWorkerTasks.some((t) => t.id === reported.taskId))
                 continue;
             const task = this.taskRepo.findById(reported.taskId);
             if (!task) {
                 this.logger.warn(`Worker reported unknown task #${reported.taskId}, requesting cleanup`);
-                this.sendCleanupToWorker(machineId, reported.taskId);
+                this.sendCleanupToWorker(workerId, reported.taskId);
                 actions.push({ taskId: reported.taskId, action: 'cleanup_unknown' });
                 continue;
             }
-            // Task exists but is assigned to a different machine or already processed
-            const action = this.reconcileReportedTask(machine, task, reported);
+            // Task exists but is assigned to a different worker or already processed
+            const action = this.reconcileReportedTask(worker, task, reported);
             actions.push({ taskId: task.id, action });
         }
-        this.logger.log(`Reconnect reconciliation for machine ${machineId}: ${actions.length} tasks processed`);
+        this.logger.log(`Reconnect reconciliation for worker ${workerId}: ${actions.length} tasks processed`);
         return actions;
     }
     // ---------------------------------------------------------------------------
     // Private: reconcile a single reported task (branches a-e)
     // ---------------------------------------------------------------------------
-    reconcileReportedTask(machine, task, reported) {
+    reconcileReportedTask(worker, task, reported) {
         const { taskId, localStatus, currentStage, branch, mrUrl, errorMessage } = reported;
         switch (task.status) {
             // Branch (a): SUSPENDED + still_running → RUNNING (if admission ok)
             // Branch (b): SUSPENDED + completed_locally → COMPLETED
             case protocol_1.TaskStatus.SUSPENDED: {
-                if (localStatus === 'still_running') {
+                if (localStatus === protocol_1.LocalStatus.STILL_RUNNING) {
                     const activeCount = this.taskRepo.countActive();
                     if (activeCount < protocol_1.MAX_CONCURRENT_TASKS) {
                         const extra = {};
@@ -330,7 +410,8 @@ let Reconciler = Reconciler_1 = class Reconciler {
                             msgId: crypto.randomUUID(),
                             taskId,
                         };
-                        this.workerConnectionManager.send(machine.id, resumeFrame);
+                        this.workerConnectionManager.send(worker.id, resumeFrame);
+                        this.emitTaskStatusChanged(taskId, protocol_1.TaskStatus.RUNNING, task.status, task.workerId, task.projectKey);
                         this.logger.log(`Task #${taskId}: SUSPENDED → RUNNING (reconnect, admission ok)`);
                         return 'resumed';
                     }
@@ -343,7 +424,7 @@ let Reconciler = Reconciler_1 = class Reconciler {
                     });
                     return 'admission_blocked';
                 }
-                if (localStatus === 'completed_locally') {
+                if (localStatus === protocol_1.LocalStatus.COMPLETED_LOCALLY) {
                     const extra = {
                         completedAt: new Date().toISOString(),
                     };
@@ -352,18 +433,20 @@ let Reconciler = Reconciler_1 = class Reconciler {
                     if (mrUrl)
                         extra['mrUrl'] = mrUrl;
                     this.updateTaskStatusSafe(taskId, protocol_1.TaskStatus.COMPLETED, task.revision, extra);
-                    if (task.machineId)
-                        this.scheduleWorkspaceCleanup(taskId, task.machineId);
+                    if (task.workerId)
+                        this.scheduleWorkspaceCleanup(taskId, task.workerId);
+                    this.emitTaskStatusChanged(taskId, protocol_1.TaskStatus.COMPLETED, task.status, task.workerId, task.projectKey);
                     this.logger.log(`Task #${taskId}: SUSPENDED → COMPLETED (completed during disconnect)`);
                     return 'completed';
                 }
-                if (localStatus === 'failed_locally') {
+                if (localStatus === protocol_1.LocalStatus.FAILED_LOCALLY) {
                     this.updateTaskStatusSafe(taskId, protocol_1.TaskStatus.FAILED, task.revision, {
                         errorMessage: errorMessage ?? 'Task failed locally during disconnect',
                         completedAt: new Date().toISOString(),
                     });
-                    if (task.machineId)
-                        this.scheduleWorkspaceCleanup(taskId, task.machineId);
+                    if (task.workerId)
+                        this.scheduleWorkspaceCleanup(taskId, task.workerId);
+                    this.emitTaskStatusChanged(taskId, protocol_1.TaskStatus.FAILED, task.status, task.workerId, task.projectKey);
                     this.logger.log(`Task #${taskId}: SUSPENDED → FAILED (failed during disconnect)`);
                     return 'failed';
                 }
@@ -372,12 +455,12 @@ let Reconciler = Reconciler_1 = class Reconciler {
             // Branch (c): FAILED + still_running → notify Worker cleanup
             // Branch (d): FAILED + completed_locally → restore to COMPLETED
             case protocol_1.TaskStatus.FAILED: {
-                if (localStatus === 'still_running') {
+                if (localStatus === protocol_1.LocalStatus.STILL_RUNNING) {
                     this.logger.log(`Task #${taskId}: FAILED but still running on Worker, sending cancel`);
-                    this.sendCancelToWorker(machine.id, taskId);
+                    this.sendCancelToWorker(worker.id, taskId);
                     return 'send_cancel';
                 }
-                if (localStatus === 'completed_locally') {
+                if (localStatus === protocol_1.LocalStatus.COMPLETED_LOCALLY) {
                     const extra = {
                         completedAt: new Date().toISOString(),
                         errorMessage: null,
@@ -387,8 +470,9 @@ let Reconciler = Reconciler_1 = class Reconciler {
                     if (mrUrl)
                         extra['mrUrl'] = mrUrl;
                     this.updateTaskStatusSafe(taskId, protocol_1.TaskStatus.COMPLETED, task.revision, extra);
-                    if (task.machineId)
-                        this.scheduleWorkspaceCleanup(taskId, task.machineId);
+                    if (task.workerId)
+                        this.scheduleWorkspaceCleanup(taskId, task.workerId);
+                    this.emitTaskStatusChanged(taskId, protocol_1.TaskStatus.COMPLETED, task.status, task.workerId, task.projectKey);
                     this.auditLogRepo.create({
                         action: 'reconcile_restore_completed',
                         resource: `task:${taskId}`,
@@ -401,12 +485,12 @@ let Reconciler = Reconciler_1 = class Reconciler {
             }
             // Branch (e): CANCELLED → notify Worker to stop
             case protocol_1.TaskStatus.CANCELLED: {
-                if (localStatus === 'still_running' ||
-                    localStatus === 'completed_locally') {
+                if (localStatus === protocol_1.LocalStatus.STILL_RUNNING ||
+                    localStatus === protocol_1.LocalStatus.COMPLETED_LOCALLY) {
                     // Keep CANCELLED (user intent priority), notify Worker to stop
                     this.logger.log(`Task #${taskId}: CANCELLED, notifying Worker to stop (localStatus=${localStatus})`);
-                    this.sendCancelToWorker(machine.id, taskId);
-                    if (localStatus === 'completed_locally') {
+                    this.sendCancelToWorker(worker.id, taskId);
+                    if (localStatus === protocol_1.LocalStatus.COMPLETED_LOCALLY) {
                         this.auditLogRepo.create({
                             action: 'reconcile_cancelled_but_completed',
                             resource: `task:${taskId}`,
@@ -419,13 +503,13 @@ let Reconciler = Reconciler_1 = class Reconciler {
             }
             // RUNNING: Worker confirms still running or reports completion/failure
             case protocol_1.TaskStatus.RUNNING: {
-                if (localStatus === 'still_running') {
+                if (localStatus === protocol_1.LocalStatus.STILL_RUNNING) {
                     if (currentStage && currentStage !== task.currentStage) {
                         this.updateTaskStatusSafe(taskId, protocol_1.TaskStatus.RUNNING, task.revision, { currentStage });
                     }
                     return 'no_op';
                 }
-                if (localStatus === 'completed_locally') {
+                if (localStatus === protocol_1.LocalStatus.COMPLETED_LOCALLY) {
                     const extra = {
                         completedAt: new Date().toISOString(),
                     };
@@ -434,18 +518,20 @@ let Reconciler = Reconciler_1 = class Reconciler {
                     if (mrUrl)
                         extra['mrUrl'] = mrUrl;
                     this.updateTaskStatusSafe(taskId, protocol_1.TaskStatus.COMPLETED, task.revision, extra);
-                    if (task.machineId)
-                        this.scheduleWorkspaceCleanup(taskId, task.machineId);
+                    if (task.workerId)
+                        this.scheduleWorkspaceCleanup(taskId, task.workerId);
+                    this.emitTaskStatusChanged(taskId, protocol_1.TaskStatus.COMPLETED, task.status, task.workerId, task.projectKey);
                     this.logger.log(`Task #${taskId}: RUNNING → COMPLETED (completed during disconnect)`);
                     return 'completed';
                 }
-                if (localStatus === 'failed_locally') {
+                if (localStatus === protocol_1.LocalStatus.FAILED_LOCALLY) {
                     this.updateTaskStatusSafe(taskId, protocol_1.TaskStatus.FAILED, task.revision, {
                         errorMessage: errorMessage ?? 'Task failed locally during disconnect',
                         completedAt: new Date().toISOString(),
                     });
-                    if (task.machineId)
-                        this.scheduleWorkspaceCleanup(taskId, task.machineId);
+                    if (task.workerId)
+                        this.scheduleWorkspaceCleanup(taskId, task.workerId);
+                    this.emitTaskStatusChanged(taskId, protocol_1.TaskStatus.FAILED, task.status, task.workerId, task.projectKey);
                     this.logger.log(`Task #${taskId}: RUNNING → FAILED (failed during disconnect)`);
                     return 'failed';
                 }
@@ -453,14 +539,15 @@ let Reconciler = Reconciler_1 = class Reconciler {
             }
             // ASSIGNED: Worker may have started executing before we recorded RUNNING
             case protocol_1.TaskStatus.ASSIGNED: {
-                if (localStatus === 'still_running') {
+                if (localStatus === protocol_1.LocalStatus.STILL_RUNNING) {
                     this.updateTaskStatusSafe(taskId, protocol_1.TaskStatus.RUNNING, task.revision, {
                         startedAt: new Date().toISOString(),
                     });
+                    this.emitTaskStatusChanged(taskId, protocol_1.TaskStatus.RUNNING, task.status, task.workerId, task.projectKey);
                     this.logger.log(`Task #${taskId}: ASSIGNED → RUNNING (Worker reports still_running)`);
                     return 'promoted_to_running';
                 }
-                if (localStatus === 'completed_locally') {
+                if (localStatus === protocol_1.LocalStatus.COMPLETED_LOCALLY) {
                     const extra = {
                         completedAt: new Date().toISOString(),
                         startedAt: task.startedAt ?? new Date().toISOString(),
@@ -470,8 +557,9 @@ let Reconciler = Reconciler_1 = class Reconciler {
                     if (mrUrl)
                         extra['mrUrl'] = mrUrl;
                     this.updateTaskStatusSafe(taskId, protocol_1.TaskStatus.COMPLETED, task.revision, extra);
-                    if (task.machineId)
-                        this.scheduleWorkspaceCleanup(taskId, task.machineId);
+                    if (task.workerId)
+                        this.scheduleWorkspaceCleanup(taskId, task.workerId);
+                    this.emitTaskStatusChanged(taskId, protocol_1.TaskStatus.COMPLETED, task.status, task.workerId, task.projectKey);
                     this.logger.log(`Task #${taskId}: ASSIGNED → COMPLETED (completed locally)`);
                     return 'completed';
                 }
@@ -492,35 +580,23 @@ let Reconciler = Reconciler_1 = class Reconciler {
      * Best-effort for reconciliation — continues processing other tasks on failure.
      */
     updateTaskStatusSafe(taskId, status, initialRevision, extra) {
-        let revision = initialRevision;
-        for (let attempt = 0; attempt < protocol_1.CAS_MAX_RETRIES; attempt++) {
-            const success = this.taskRepo.updateStatus(taskId, status, revision, extra);
-            if (success)
-                return true;
-            const freshTask = this.taskRepo.findById(taskId);
-            if (!freshTask) {
-                this.logger.error(`Task #${taskId} disappeared during CAS retry (reconciliation)`);
-                return false;
-            }
-            // Already at target status — success
-            if (freshTask.status === status)
-                return true;
-            revision = freshTask.revision;
+        const { success, task } = this.taskRepo.updateStatusWithCasRetry(taskId, status, initialRevision, extra);
+        if (!success) {
+            this.logger.error(`CAS retry exhausted for task #${taskId} → ${status} during reconciliation`);
         }
-        this.logger.error(`CAS retry exhausted for task #${taskId} → ${status} during reconciliation (${protocol_1.CAS_MAX_RETRIES} attempts)`);
-        return false;
+        return success;
     }
     /** Send cancel frame to a Worker for a specific task. */
-    sendCancelToWorker(machineId, taskId) {
+    sendCancelToWorker(workerId, taskId) {
         const frame = {
             type: 'cancel',
             msgId: crypto.randomUUID(),
             taskId,
         };
-        this.workerConnectionManager.send(machineId, frame);
+        this.workerConnectionManager.send(workerId, frame);
     }
     /** Send cleanup instruction to Worker for a specific task. */
-    sendCleanupToWorker(machineId, taskId) {
+    sendCleanupToWorker(workerId, taskId) {
         const workspace = this.workspaceRepo.findByTaskId(taskId);
         const frame = {
             type: 'cleanup_workspace',
@@ -528,20 +604,32 @@ let Reconciler = Reconciler_1 = class Reconciler {
             taskId,
             workspacePath: workspace?.path ?? '',
         };
-        this.workerConnectionManager.send(machineId, frame);
+        this.workerConnectionManager.send(workerId, frame);
+    }
+    /** Emit a TASK_STATUS_CHANGED domain event. */
+    emitTaskStatusChanged(taskId, status, previousStatus, workerId, projectKey) {
+        this.eventEmitter.emit(event_types_1.DomainEvents.TASK_STATUS_CHANGED, {
+            taskId,
+            status,
+            previousStatus,
+            workerId,
+            projectKey,
+        });
     }
-    /** Check if a machine is unavailable (missing, offline, or decommissioned). */
-    isMachineUnavailable(machine) {
-        return !machine || machine.status === protocol_1.MachineStatus.OFFLINE || machine.status === protocol_1.MachineStatus.DECOMMISSIONED;
+    /** Check if a worker is unavailable (missing, offline, or decommissioned). */
+    isWorkerUnavailable(worker) {
+        return !worker || worker.status === protocol_1.WorkerStatus.OFFLINE || worker.status === protocol_1.WorkerStatus.DECOMMISSIONED;
     }
     /** Schedule workspace cleanup for a task that reached a terminal state. */
-    scheduleWorkspaceCleanup(taskId, machineId) {
+    scheduleWorkspaceCleanup(taskId, workerId) {
         const workspace = this.workspaceRepo.findByTaskId(taskId);
         if (workspace) {
             this.cleanupService
-                .scheduleCleanup(taskId, machineId, workspace.path)
+                .scheduleCleanup(taskId, workerId, workspace.path)
                 .catch((err) => {
-                this.logger.error(`Failed to schedule cleanup for task #${taskId}: ${err instanceof Error ? err.message : String(err)}`);
+                this.logger.error(`Failed to schedule cleanup for task #${taskId}: ${(0, protocol_1.getErrorMessage)(err)}. Marking workspace for cleanup and adding to retry set.`);
+                this.workspaceRepo.markForCleanup(taskId);
+                this.failedCleanups.set(`${taskId}:${workerId}`, { taskId, workerId });
             });
         }
     }
@@ -550,11 +638,12 @@ exports.Reconciler = Reconciler;
 exports.Reconciler = Reconciler = Reconciler_1 = __decorate([
     (0, common_1.Injectable)(),
     __metadata("design:paramtypes", [task_repository_1.TaskRepository,
-        machine_repository_1.MachineRepository,
+        worker_repository_1.WorkerRepository,
         audit_log_repository_1.AuditLogRepository,
         workspace_repository_1.WorkspaceRepository,
         scheduler_service_1.SchedulerService,
         cleanup_service_1.CleanupService,
-        worker_connection_manager_1.WorkerConnectionManager])
+        worker_connection_manager_1.WorkerConnectionManager,
+        event_emitter_1.EventEmitter2])
 ], Reconciler);
 //# sourceMappingURL=reconciler.js.map