n8n 2.33.2 → 2.34.0

package/dist/task-runners/task-broker/task-broker-ws-server.js

@@ -14,6 +14,7 @@ const backend_common_1 = require("@n8n/backend-common");
 const config_1 = require("@n8n/config");
 const constants_1 = require("@n8n/constants");
 const di_1 = require("@n8n/di");
+const sleep_1 = require("@n8n/utils/sleep");
 const n8n_workflow_1 = require("n8n-workflow");
 const constants_2 = require("../../constants");
 const default_task_runner_disconnect_analyzer_1 = require("../../task-runners/default-task-runner-disconnect-analyzer");
@@ -40,17 +41,27 @@ let TaskBrokerWsServer = class TaskBrokerWsServer {
         if (heartbeatInterval <= 0) {
             throw new n8n_workflow_1.UserError('Heartbeat interval must be greater than 0');
         }
-        this.heartbeatTimer = setInterval(() => {
-            for (const [runnerId, connection] of this.runnerConnections.entries()) {
-                if (!connection.isAlive) {
-                    void this.removeConnection(runnerId, 'failed-heartbeat-check', constants_2.WsStatusCodes.CloseProtocolError);
-                    this.runnerLifecycleEvents.emit('runner:failed-heartbeat-check');
-                    return;
-                }
+        this.heartbeatTimer = setInterval(() => this.checkConnectionLiveness(), heartbeatInterval * constants_1.Time.seconds.toMilliseconds);
+    }
+    checkConnectionLiveness() {
+        let anyDead = false;
+        for (const [runnerId, connection] of this.runnerConnections) {
+            if (connection.isAlive) {
                 connection.isAlive = false;
                 connection.ping();
             }
-        }, heartbeatInterval * constants_1.Time.seconds.toMilliseconds);
+            else {
+                anyDead = true;
+                void this.removeConnection(runnerId, {
+                    reason: 'failed-heartbeat-check',
+                    code: constants_2.WsStatusCodes.CloseProtocolError,
+                    expectedConnection: connection,
+                });
+            }
+        }
+        if (anyDead) {
+            this.runnerLifecycleEvents.emit('runner:failed-heartbeat-check');
+        }
     }
     async stop() {
         if (this.heartbeatTimer) {
@@ -80,23 +91,23 @@ let TaskBrokerWsServer = class TaskBrokerWsServer {
                         ? Buffer.from(data)
                         : data;
                 const message = JSON.parse(buffer.toString('utf8'));
-                if (!isConnected && message.type !== 'runner:info') {
-                    return;
+                if (!isConnected) {
+                    if (message.type === 'runner:info') {
+                        await this.removeConnection(id);
+                        isConnected = true;
+                        this.runnerConnections.set(id, connection);
+                        this.taskBroker.registerRunner({
+                            id,
+                            taskTypes: message.types,
+                            lastSeen: new Date(),
+                            name: message.name,
+                        }, this.sendMessage.bind(this, id), () => this.isRunnerReachable(id, connection));
+                        this.logger.info(`Registered runner "${message.name}" (${id}) `);
+                    }
                 }
-                else if (!isConnected && message.type === 'runner:info') {
-                    await this.removeConnection(id);
-                    isConnected = true;
-                    this.runnerConnections.set(id, connection);
-                    this.taskBroker.registerRunner({
-                        id,
-                        taskTypes: message.types,
-                        lastSeen: new Date(),
-                        name: message.name,
-                    }, this.sendMessage.bind(this, id));
-                    this.logger.info(`Registered runner "${message.name}" (${id}) `);
-                    return;
+                else if (this.isCurrentConnection(id, connection)) {
+                    void this.taskBroker.onRunnerMessage(id, message);
                 }
-                void this.taskBroker.onRunnerMessage(id, message);
             }
             catch (error) {
                 this.logger.error(`Couldn't parse message from runner "${id}"`, {
@@ -109,31 +120,49 @@ let TaskBrokerWsServer = class TaskBrokerWsServer {
         connection.once('close', async () => {
             connection.off('pong', heartbeat);
             connection.off('message', onMessage);
-            await this.removeConnection(id);
+            await this.removeConnection(id, { expectedConnection: connection });
         });
         connection.on('message', onMessage);
         connection.send(JSON.stringify({ type: 'broker:inforequest' }));
     }
-    async removeConnection(id, reason = 'unknown', code = constants_2.WsStatusCodes.CloseNormal) {
+    async removeConnection(id, { reason = 'unknown', code = constants_2.WsStatusCodes.CloseNormal, expectedConnection, } = {}) {
         const connection = this.runnerConnections.get(id);
-        if (connection) {
+        const isStaleRemoval = expectedConnection !== undefined && !this.isCurrentConnection(id, expectedConnection);
+        if (connection && !isStaleRemoval) {
+            this.runnerConnections.delete(id);
+            connection.close(code);
+            const inFlightTaskIds = this.taskBroker.getInFlightTaskIds(id);
             const disconnectError = await this.disconnectAnalyzer.toDisconnectError({
                 runnerId: id,
                 reason,
                 heartbeatInterval: this.taskRunnersConfig.heartbeatInterval,
             });
-            this.taskBroker.deregisterRunner(id, disconnectError);
-            this.logger.debug(`Deregistered runner "${id}"`);
-            connection.close(code);
-            this.runnerConnections.delete(id);
+            const hasReconnected = this.runnerConnections.has(id);
+            if (hasReconnected) {
+                this.taskBroker.failTasks(inFlightTaskIds, disconnectError);
+            }
+            else {
+                this.taskBroker.deregisterRunner(id, disconnectError);
+                this.logger.debug(`Deregistered runner "${id}"`);
+            }
         }
     }
     handleRequest(req, _res) {
         this.add(req.query.id, req.ws);
     }
+    isCurrentConnection(id, connection) {
+        return this.runnerConnections.get(id) === connection;
+    }
+    isRunnerReachable(id, connection) {
+        return this.isCurrentConnection(id, connection) && connection.readyState === connection.OPEN;
+    }
     async stopConnectedRunners() {
         await this.drainActiveTasks();
-        await Promise.all(Array.from(this.runnerConnections.keys()).map(async (id) => await this.removeConnection(id, 'shutting-down', constants_2.WsStatusCodes.CloseGoingAway)));
+        await Promise.all(Array.from(this.runnerConnections.entries()).map(async ([id, connection]) => await this.removeConnection(id, {
+            reason: 'shutting-down',
+            code: constants_2.WsStatusCodes.CloseGoingAway,
+            expectedConnection: connection,
+        })));
     }
     async drainActiveTasks() {
         const drainTimeout = Math.floor(this.globalConfig.generic.gracefulShutdownTimeout * 0.8);
@@ -148,7 +177,7 @@ let TaskBrokerWsServer = class TaskBrokerWsServer {
         }
         const start = Date.now();
         while (this.taskBroker.hasActiveTasks() && Date.now() - start < drainTimeoutMs) {
-            await (0, n8n_workflow_1.sleep)(100);
+            await (0, sleep_1.sleep)(100);
         }
         if (this.taskBroker.hasActiveTasks()) {
             this.logger.warn(`Drain timeout reached after ${drainTimeout}s, will force-shutdown with active tasks...`);

package/dist/task-runners/task-broker/task-broker-ws-server.js.map

@@ -1 +1 @@
-{"version":3,"file":"task-broker-ws-server.js","sourceRoot":"","sources":["../../../src/task-runners/task-broker/task-broker-ws-server.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,wDAA6C;AAC7C,wCAA8D;AAC9D,8CAAsC;AACtC,gCAAkC;AAElC,+CAA+D;AAG/D,2CAA4C;AAC5C,oHAA6G;AAO7G,8FAAwF;AAExF,+DAA0F;AAE1F,SAAS,SAAS;IACjB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;AACrB,CAAC;AASM,IAAM,kBAAkB,GAAxB,MAAM,kBAAkB;IAK9B,YACkB,MAAc,EACd,UAAsB,EAC/B,kBAAuD,EAC9C,iBAAoC,EACpC,qBAAgD,EAChD,YAA0B;QAL1B,WAAM,GAAN,MAAM,CAAQ;QACd,eAAU,GAAV,UAAU,CAAY;QAC/B,uBAAkB,GAAlB,kBAAkB,CAAqC;QAC9C,sBAAiB,GAAjB,iBAAiB,CAAmB;QACpC,0BAAqB,GAArB,qBAAqB,CAA2B;QAChD,iBAAY,GAAZ,YAAY,CAAc;QAV5C,sBAAiB,GAAqC,IAAI,GAAG,EAAE,CAAC;IAW7D,CAAC;IAEJ,KAAK;QACJ,IAAI,CAAC,oBAAoB,EAAE,CAAC;IAC7B,CAAC;IAEO,oBAAoB;QAC3B,MAAM,EAAE,iBAAiB,EAAE,GAAG,IAAI,CAAC,iBAAiB,CAAC;QAErD,IAAI,iBAAiB,IAAI,CAAC,EAAE,CAAC;YAC5B,MAAM,IAAI,wBAAS,CAAC,2CAA2C,CAAC,CAAC;QAClE,CAAC;QAED,IAAI,CAAC,cAAc,GAAG,WAAW,CAAC,GAAG,EAAE;YACtC,KAAK,MAAM,CAAC,QAAQ,EAAE,UAAU,CAAC,IAAI,IAAI,CAAC,iBAAiB,CAAC,OAAO,EAAE,EAAE,CAAC;gBACvE,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC;oBACzB,KAAK,IAAI,CAAC,gBAAgB,CACzB,QAAQ,EACR,wBAAwB,EACxB,yBAAa,CAAC,kBAAkB,CAChC,CAAC;oBACF,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,+BAA+B,CAAC,CAAC;oBACjE,OAAO;gBACR,CAAC;gBACD,UAAU,CAAC,OAAO,GAAG,KAAK,CAAC;gBAC3B,UAAU,CAAC,IAAI,EAAE,CAAC;YACnB,CAAC;QACF,CAAC,EAAE,iBAAiB,GAAG,gBAAI,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;IACrD,CAAC;IAED,KAAK,CAAC,IAAI;QACT,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACzB,aAAa,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;YACnC,IAAI,CAAC,cAAc,GAAG,SAAS,CAAC;QACjC,CAAC;QAED,MAAM,IAAI,CAAC,oBAAoB,EAAE,CAAC;IACnC,CAAC;IAED,qBAAqB,CAAC,kBAAsC;QAC3D,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;IAC9C,CAAC;IAED,qBAAqB;QACpB,OAAO,IAAI,CAAC,kBAAkB,CAAC;IAChC,CAAC;IAED,WAAW,CAAC,EAAoB,EAAE,OAAmC;QACpE,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,IAAI,CAAC,IAAA,4BAAa,EAAC,OAAO,EAAE,EAAE,mBAAmB,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;IAC7F,CAAC;IAED,GAAG,CAAC,EAAoB,EAAE,UAAqB;QAC9C,UAAU,CAAC,OAAO,GAAG,IAAI,CAAC;QAC1B,UAAU,CAAC,EAAE,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;QAEjC,IAAI,WAAW,GAAG,KAAK,CAAC;QAExB,MAAM,SAAS,GAAG,KAAK,EAAE,IAAuB,EAAE,EAAE;YACnD,IAAI,CAAC;gBACJ,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC;oBACjC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC;oBACrB,CAAC,CAAC,IAAI,YAAY,WAAW;wBAC5B,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC;wBACnB,CAAC,CAAC,IAAI,CAAC;gBAET,MAAM,OAAO,GAA+B,IAAI,CAAC,KAAK,CACrD,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CACO,CAAC;gBAEhC,IAAI,CAAC,WAAW,IAAI,OAAO,CAAC,IAAI,KAAK,aAAa,EAAE,CAAC;oBACpD,OAAO;gBACR,CAAC;qBAAM,IAAI,CAAC,WAAW,IAAI,OAAO,CAAC,IAAI,KAAK,aAAa,EAAE,CAAC;oBAC3D,MAAM,IAAI,CAAC,gBAAgB,CAAC,EAAE,CAAC,CAAC;oBAChC,WAAW,GAAG,IAAI,CAAC;oBAEnB,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,EAAE,EAAE,UAAU,CAAC,CAAC;oBAE3C,IAAI,CAAC,UAAU,CAAC,cAAc,CAC7B;wBACC,EAAE;wBACF,SAAS,EAAE,OAAO,CAAC,KAAK;wBACxB,QAAQ,EAAE,IAAI,IAAI,EAAE;wBACpB,IAAI,EAAE,OAAO,CAAC,IAAI;qBAClB,EACD,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,CAAoB,CAClD,CAAC;oBAEF,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,sBAAsB,OAAO,CAAC,IAAI,MAAM,EAAE,IAAI,CAAC,CAAC;oBACjE,OAAO;gBACR,CAAC;gBAED,KAAK,IAAI,CAAC,UAAU,CAAC,eAAe,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;YACnD,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBAChB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,uCAAuC,EAAE,GAAG,EAAE;oBAC/D,KAAK,EAAE,KAAgB;oBACvB,EAAE;oBACF,IAAI;iBACJ,CAAC,CAAC;YACJ,CAAC;QACF,CAAC,CAAC;QAGF,UAAU,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,IAAI,EAAE;YACnC,UAAU,CAAC,GAAG,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;YAClC,UAAU,CAAC,GAAG,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;YACrC,MAAM,IAAI,CAAC,gBAAgB,CAAC,EAAE,CAAC,CAAC;QACjC,CAAC,CAAC,CAAC;QAEH,UAAU,CAAC,EAAE,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;QACpC,UAAU,CAAC,IAAI,CACd,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,oBAAoB,EAAwC,CAAC,CACpF,CAAC;IACH,CAAC;IAED,KAAK,CAAC,gBAAgB,CACrB,EAAoB,EACpB,MAAM,GAAqB,SAAS,EACpC,IAAI,GAAiB,yBAAa,CAAC,WAAW;QAE9C,MAAM,UAAU,GAAG,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAClD,IAAI,UAAU,EAAE,CAAC;YAChB,MAAM,eAAe,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,iBAAiB,CAAC;gBACvE,QAAQ,EAAE,EAAE;gBACZ,MAAM;gBACN,iBAAiB,EAAE,IAAI,CAAC,iBAAiB,CAAC,iBAAiB;aAC3D,CAAC,CAAC;YACH,IAAI,CAAC,UAAU,CAAC,gBAAgB,CAAC,EAAE,EAAE,eAAe,CAAC,CAAC;YACtD,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,wBAAwB,EAAE,GAAG,CAAC,CAAC;YACjD,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YACvB,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QACnC,CAAC;IACF,CAAC;IAED,aAAa,CAAC,GAAgC,EAAE,IAAkC;QACjF,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;IAChC,CAAC;IAEO,KAAK,CAAC,oBAAoB;QACjC,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAE9B,MAAM,OAAO,CAAC,GAAG,CAChB,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,CAC5C,KAAK,EAAE,EAAE,EAAE,EAAE,CACZ,MAAM,IAAI,CAAC,gBAAgB,CAAC,EAAE,EAAE,eAAe,EAAE,yBAAa,CAAC,cAAc,CAAC,CAC/E,CACD,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,gBAAgB;QAC7B,MAAM,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,uBAAuB,GAAG,GAAG,CAAC,CAAC;QAEzF,MAAM,cAAc,GAAG,YAAY,GAAG,gBAAI,CAAC,OAAO,CAAC,cAAc,CAAC;QAElE,IAAI,CAAC,UAAU,CAAC,aAAa,EAAE,CAAC;QAEhC,KAAK,MAAM,UAAU,IAAI,IAAI,CAAC,iBAAiB,CAAC,MAAM,EAAE,EAAE,CAAC;YAC1D,IAAI,CAAC;gBACJ,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,cAAc,EAAE,CAAC,CAAC,CAAC;YAC3D,CAAC;YAAC,MAAM,CAAC;YAET,CAAC;QACF,CAAC;QAED,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACzB,OAAO,IAAI,CAAC,UAAU,CAAC,cAAc,EAAE,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,GAAG,cAAc,EAAE,CAAC;YAChF,MAAM,IAAA,oBAAK,EAAC,GAAG,CAAC,CAAC;QAClB,CAAC;QAED,IAAI,IAAI,CAAC,UAAU,CAAC,cAAc,EAAE,EAAE,CAAC;YACtC,IAAI,CAAC,MAAM,CAAC,IAAI,CACf,+BAA+B,YAAY,6CAA6C,CACxF,CAAC;QACH,CAAC;IACF,CAAC;CACD,CAAA;;6BA1LY,kBAAkB;IAD9B,IAAA,YAAO,GAAE;qCAOiB,uBAAM,EACF,gCAAU,EACX,6EAAmC,EAC3B,0BAAiB,EACb,wDAAyB,EAClC,qBAAY;GAXhC,kBAAkB,CA0L9B"}
+{"version":3,"file":"task-broker-ws-server.js","sourceRoot":"","sources":["../../../src/task-runners/task-broker/task-broker-ws-server.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,wDAA6C;AAC7C,wCAA8D;AAC9D,8CAAsC;AACtC,gCAAkC;AAElC,4CAAyC;AACzC,+CAAwD;AAGxD,2CAA4C;AAC5C,oHAA6G;AAO7G,8FAAwF;AAExF,+DAA0F;AAE1F,SAAS,SAAS;IACjB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;AACrB,CAAC;AAoBM,IAAM,kBAAkB,GAAxB,MAAM,kBAAkB;IAK9B,YACkB,MAAc,EACd,UAAsB,EAC/B,kBAAuD,EAC9C,iBAAoC,EACpC,qBAAgD,EAChD,YAA0B;QAL1B,WAAM,GAAN,MAAM,CAAQ;QACd,eAAU,GAAV,UAAU,CAAY;QAC/B,uBAAkB,GAAlB,kBAAkB,CAAqC;QAC9C,sBAAiB,GAAjB,iBAAiB,CAAmB;QACpC,0BAAqB,GAArB,qBAAqB,CAA2B;QAChD,iBAAY,GAAZ,YAAY,CAAc;QAV5C,sBAAiB,GAAqC,IAAI,GAAG,EAAE,CAAC;IAW7D,CAAC;IAEJ,KAAK;QACJ,IAAI,CAAC,oBAAoB,EAAE,CAAC;IAC7B,CAAC;IAEO,oBAAoB;QAC3B,MAAM,EAAE,iBAAiB,EAAE,GAAG,IAAI,CAAC,iBAAiB,CAAC;QAErD,IAAI,iBAAiB,IAAI,CAAC,EAAE,CAAC;YAC5B,MAAM,IAAI,wBAAS,CAAC,2CAA2C,CAAC,CAAC;QAClE,CAAC;QAED,IAAI,CAAC,cAAc,GAAG,WAAW,CAChC,GAAG,EAAE,CAAC,IAAI,CAAC,uBAAuB,EAAE,EACpC,iBAAiB,GAAG,gBAAI,CAAC,OAAO,CAAC,cAAc,CAC/C,CAAC;IACH,CAAC;IAEO,uBAAuB;QAC9B,IAAI,OAAO,GAAG,KAAK,CAAC;QAEpB,KAAK,MAAM,CAAC,QAAQ,EAAE,UAAU,CAAC,IAAI,IAAI,CAAC,iBAAiB,EAAE,CAAC;YAC7D,IAAI,UAAU,CAAC,OAAO,EAAE,CAAC;gBACxB,UAAU,CAAC,OAAO,GAAG,KAAK,CAAC;gBAC3B,UAAU,CAAC,IAAI,EAAE,CAAC;YACnB,CAAC;iBAAM,CAAC;gBACP,OAAO,GAAG,IAAI,CAAC;gBACf,KAAK,IAAI,CAAC,gBAAgB,CAAC,QAAQ,EAAE;oBACpC,MAAM,EAAE,wBAAwB;oBAChC,IAAI,EAAE,yBAAa,CAAC,kBAAkB;oBACtC,kBAAkB,EAAE,UAAU;iBAC9B,CAAC,CAAC;YACJ,CAAC;QACF,CAAC;QAED,IAAI,OAAO,EAAE,CAAC;YACb,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,+BAA+B,CAAC,CAAC;QAClE,CAAC;IACF,CAAC;IAED,KAAK,CAAC,IAAI;QACT,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACzB,aAAa,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;YACnC,IAAI,CAAC,cAAc,GAAG,SAAS,CAAC;QACjC,CAAC;QAED,MAAM,IAAI,CAAC,oBAAoB,EAAE,CAAC;IACnC,CAAC;IAED,qBAAqB,CAAC,kBAAsC;QAC3D,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;IAC9C,CAAC;IAED,qBAAqB;QACpB,OAAO,IAAI,CAAC,kBAAkB,CAAC;IAChC,CAAC;IAED,WAAW,CAAC,EAAoB,EAAE,OAAmC;QACpE,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,IAAI,CAAC,IAAA,4BAAa,EAAC,OAAO,EAAE,EAAE,mBAAmB,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;IAC7F,CAAC;IAED,GAAG,CAAC,EAAoB,EAAE,UAAqB;QAC9C,UAAU,CAAC,OAAO,GAAG,IAAI,CAAC;QAC1B,UAAU,CAAC,EAAE,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;QAEjC,IAAI,WAAW,GAAG,KAAK,CAAC;QAExB,MAAM,SAAS,GAAG,KAAK,EAAE,IAAuB,EAAE,EAAE;YACnD,IAAI,CAAC;gBACJ,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC;oBACjC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC;oBACrB,CAAC,CAAC,IAAI,YAAY,WAAW;wBAC5B,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC;wBACnB,CAAC,CAAC,IAAI,CAAC;gBAET,MAAM,OAAO,GAA+B,IAAI,CAAC,KAAK,CACrD,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CACO,CAAC;gBAEhC,IAAI,CAAC,WAAW,EAAE,CAAC;oBAClB,IAAI,OAAO,CAAC,IAAI,KAAK,aAAa,EAAE,CAAC;wBACpC,MAAM,IAAI,CAAC,gBAAgB,CAAC,EAAE,CAAC,CAAC;wBAChC,WAAW,GAAG,IAAI,CAAC;wBAEnB,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,EAAE,EAAE,UAAU,CAAC,CAAC;wBAE3C,IAAI,CAAC,UAAU,CAAC,cAAc,CAC7B;4BACC,EAAE;4BACF,SAAS,EAAE,OAAO,CAAC,KAAK;4BACxB,QAAQ,EAAE,IAAI,IAAI,EAAE;4BACpB,IAAI,EAAE,OAAO,CAAC,IAAI;yBAClB,EACD,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,CAAoB,EAClD,GAAG,EAAE,CAAC,IAAI,CAAC,iBAAiB,CAAC,EAAE,EAAE,UAAU,CAAC,CAC5C,CAAC;wBAEF,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,sBAAsB,OAAO,CAAC,IAAI,MAAM,EAAE,IAAI,CAAC,CAAC;oBAClE,CAAC;gBACF,CAAC;qBAAM,IAAI,IAAI,CAAC,mBAAmB,CAAC,EAAE,EAAE,UAAU,CAAC,EAAE,CAAC;oBACrD,KAAK,IAAI,CAAC,UAAU,CAAC,eAAe,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;gBACnD,CAAC;YACF,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBAChB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,uCAAuC,EAAE,GAAG,EAAE;oBAC/D,KAAK,EAAE,KAAgB;oBACvB,EAAE;oBACF,IAAI;iBACJ,CAAC,CAAC;YACJ,CAAC;QACF,CAAC,CAAC;QAGF,UAAU,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,IAAI,EAAE;YACnC,UAAU,CAAC,GAAG,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;YAClC,UAAU,CAAC,GAAG,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;YACrC,MAAM,IAAI,CAAC,gBAAgB,CAAC,EAAE,EAAE,EAAE,kBAAkB,EAAE,UAAU,EAAE,CAAC,CAAC;QACrE,CAAC,CAAC,CAAC;QAEH,UAAU,CAAC,EAAE,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;QACpC,UAAU,CAAC,IAAI,CACd,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,oBAAoB,EAAwC,CAAC,CACpF,CAAC;IACH,CAAC;IAED,KAAK,CAAC,gBAAgB,CACrB,EAAoB,EACpB,EACC,MAAM,GAAG,SAAS,EAClB,IAAI,GAAG,yBAAa,CAAC,WAAW,EAChC,kBAAkB,GAClB,GAA4B,EAAE;QAE/B,MAAM,UAAU,GAAG,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAClD,MAAM,cAAc,GACnB,kBAAkB,KAAK,SAAS,IAAI,CAAC,IAAI,CAAC,mBAAmB,CAAC,EAAE,EAAE,kBAAkB,CAAC,CAAC;QAEvF,IAAI,UAAU,IAAI,CAAC,cAAc,EAAE,CAAC;YAEnC,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;YAClC,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAEvB,MAAM,eAAe,GAAG,IAAI,CAAC,UAAU,CAAC,kBAAkB,CAAC,EAAE,CAAC,CAAC;YAE/D,MAAM,eAAe,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,iBAAiB,CAAC;gBACvE,QAAQ,EAAE,EAAE;gBACZ,MAAM;gBACN,iBAAiB,EAAE,IAAI,CAAC,iBAAiB,CAAC,iBAAiB;aAC3D,CAAC,CAAC;YAEH,MAAM,cAAc,GAAG,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAEtD,IAAI,cAAc,EAAE,CAAC;gBACpB,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,eAAe,EAAE,eAAe,CAAC,CAAC;YAC7D,CAAC;iBAAM,CAAC;gBACP,IAAI,CAAC,UAAU,CAAC,gBAAgB,CAAC,EAAE,EAAE,eAAe,CAAC,CAAC;gBACtD,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,wBAAwB,EAAE,GAAG,CAAC,CAAC;YAClD,CAAC;QACF,CAAC;IACF,CAAC;IAED,aAAa,CAAC,GAAgC,EAAE,IAAkC;QACjF,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;IAChC,CAAC;IAQO,mBAAmB,CAAC,EAAoB,EAAE,UAAqB;QACtE,OAAO,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,EAAE,CAAC,KAAK,UAAU,CAAC;IACtD,CAAC;IAMO,iBAAiB,CAAC,EAAoB,EAAE,UAAqB;QACpE,OAAO,IAAI,CAAC,mBAAmB,CAAC,EAAE,EAAE,UAAU,CAAC,IAAI,UAAU,CAAC,UAAU,KAAK,UAAU,CAAC,IAAI,CAAC;IAC9F,CAAC;IAEO,KAAK,CAAC,oBAAoB;QACjC,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAE9B,MAAM,OAAO,CAAC,GAAG,CAChB,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,OAAO,EAAE,CAAC,CAAC,GAAG,CAC/C,KAAK,EAAE,CAAC,EAAE,EAAE,UAAU,CAAC,EAAE,EAAE,CAC1B,MAAM,IAAI,CAAC,gBAAgB,CAAC,EAAE,EAAE;YAC/B,MAAM,EAAE,eAAe;YACvB,IAAI,EAAE,yBAAa,CAAC,cAAc;YAClC,kBAAkB,EAAE,UAAU;SAC9B,CAAC,CACH,CACD,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,gBAAgB;QAC7B,MAAM,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,uBAAuB,GAAG,GAAG,CAAC,CAAC;QAEzF,MAAM,cAAc,GAAG,YAAY,GAAG,gBAAI,CAAC,OAAO,CAAC,cAAc,CAAC;QAElE,IAAI,CAAC,UAAU,CAAC,aAAa,EAAE,CAAC;QAEhC,KAAK,MAAM,UAAU,IAAI,IAAI,CAAC,iBAAiB,CAAC,MAAM,EAAE,EAAE,CAAC;YAC1D,IAAI,CAAC;gBACJ,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,cAAc,EAAE,CAAC,CAAC,CAAC;YAC3D,CAAC;YAAC,MAAM,CAAC;YAET,CAAC;QACF,CAAC;QAED,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACzB,OAAO,IAAI,CAAC,UAAU,CAAC,cAAc,EAAE,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,GAAG,cAAc,EAAE,CAAC;YAChF,MAAM,IAAA,aAAK,EAAC,GAAG,CAAC,CAAC;QAClB,CAAC;QAED,IAAI,IAAI,CAAC,UAAU,CAAC,cAAc,EAAE,EAAE,CAAC;YACtC,IAAI,CAAC,MAAM,CAAC,IAAI,CACf,+BAA+B,YAAY,6CAA6C,CACxF,CAAC;QACH,CAAC;IACF,CAAC;CACD,CAAA;;6BA5OY,kBAAkB;IAD9B,IAAA,YAAO,GAAE;qCAOiB,uBAAM,EACF,gCAAU,EACX,6EAAmC,EAC3B,0BAAiB,EACb,wDAAyB,EAClC,qBAAY;GAXhC,kBAAkB,CA4O9B"}

package/dist/task-runners/task-broker/task-broker.service.d.ts

@@ -30,11 +30,14 @@ export interface TaskRequest {
     taskType: string;
     timeout?: NodeJS.Timeout;
     acceptInProgress?: boolean;
+    timeoutRefreshes?: number;
 }
 export type MessageCallback = (message: BrokerMessage.ToRunner.All) => Promise<void> | void;
 export type RequesterMessageCallback = (message: BrokerMessage.ToRequester.All) => Promise<void> | void;
 type RunnerAcceptCallback = () => void;
+type RequesterAcceptCallback = (settings: RequesterMessage.ToBroker.TaskSettings['settings']) => void;
 type TaskRejectCallback = (reason: TaskRejectError | TaskDeferredError) => void;
+export type RunnerReachabilityCheck = () => boolean;
 export declare class TaskBroker {
     private readonly logger;
     private readonly taskRunnersConfig;
@@ -51,10 +54,16 @@ export declare class TaskBroker {
     private mismatchWarned;
     constructor(logger: Logger, taskRunnersConfig: TaskRunnersConfig, taskRunnerLifecycleEvents: TaskRunnerLifecycleEvents, globalConfig: GlobalConfig);
     private createRequestTimeout;
+    private refreshRequestTimeout;
     private handleRequestTimeout;
+    private discardOffers;
     expireTasks(): void;
-    registerRunner(runner: TaskRunner, messageCallback: MessageCallback): void;
+    private discardOffersFrom;
+    private discardOffersFromUnreachableRunners;
+    registerRunner(runner: TaskRunner, messageCallback: MessageCallback, isRunnerReachable?: RunnerReachabilityCheck): void;
     deregisterRunner(runnerId: string, error: Error): void;
+    getInFlightTaskIds(runnerId: TaskRunner['id']): Array<Task['id']>;
+    failTasks(taskIds: Array<Task['id']>, error: Error): void;
     registerRequester(requesterId: string, messageCallback: RequesterMessageCallback): void;
     deregisterRequester(requesterId: string): void;
     private messageRunner;
@@ -73,6 +82,7 @@ export declare class TaskBroker {
     handleRequesterNodeTypesResponse(taskId: Task['id'], requestId: RequesterMessage.ToBroker.NodeTypesResponse['requestId'], nodeTypes: RequesterMessage.ToBroker.NodeTypesResponse['nodeTypes']): Promise<void>;
     handleRequesterAccept(taskId: Task['id'], settings: RequesterMessage.ToBroker.TaskSettings['settings']): void;
     handleRequesterReject(taskId: Task['id'], reason: string): void;
+    private untrackTask;
     private cancelTask;
     private failTask;
     private getRunnerOrFailTask;
@@ -94,11 +104,17 @@ export declare class TaskBroker {
     getKnownRunners(): Map<string, {
         runner: TaskRunner;
         messageCallback: MessageCallback;
+        isRunnerReachable: RunnerReachabilityCheck;
     }>;
     getKnownRequesters(): Map<string, RequesterMessageCallback>;
     getRunnerAcceptRejects(): Map<string, {
         accept: RunnerAcceptCallback;
         reject: TaskRejectCallback;
+        runnerId: TaskRunner['id'];
+    }>;
+    getRequesterAcceptRejects(): Map<string, {
+        accept: RequesterAcceptCallback;
+        reject: TaskRejectCallback;
     }>;
     setTasks(tasks: Record<string, Task>): void;
     setPendingTaskOffers(pendingTaskOffers: TaskOffer[]): void;
@@ -106,6 +122,7 @@ export declare class TaskBroker {
     setRunnerAcceptRejects(runnerAcceptRejects: Record<string, {
         accept: RunnerAcceptCallback;
         reject: TaskRejectCallback;
+        runnerId: TaskRunner['id'];
     }>): void;
 }
 export {};

package/dist/task-runners/task-broker/task-broker.service.js

@@ -18,9 +18,12 @@ const n8n_workflow_1 = require("n8n-workflow");
 const nanoid_1 = require("nanoid");
 const task_deferred_error_1 = require("../../task-runners/task-broker/errors/task-deferred.error");
 const task_reject_error_1 = require("../../task-runners/task-broker/errors/task-reject.error");
+const task_requester_accept_timeout_error_1 = require("../../task-runners/task-broker/errors/task-requester-accept-timeout.error");
 const task_runner_accept_timeout_error_1 = require("../../task-runners/task-broker/errors/task-runner-accept-timeout.error");
 const task_runner_execution_timeout_error_1 = require("../../task-runners/task-broker/errors/task-runner-execution-timeout.error");
+const task_runner_unreachable_error_1 = require("../../task-runners/task-broker/errors/task-runner-unreachable.error");
 const task_runner_lifecycle_events_1 = require("../../task-runners/task-runner-lifecycle-events");
+const MAX_REQUEST_TIMEOUT_REFRESHES = 3;
 let TaskBroker = class TaskBroker {
     constructor(logger, taskRunnersConfig, taskRunnerLifecycleEvents, globalConfig) {
         this.logger = logger;
@@ -39,12 +42,27 @@ let TaskBroker = class TaskBroker {
         if (this.taskRunnersConfig.taskTimeout <= 0) {
             throw new n8n_workflow_1.UserError('Task timeout must be greater than 0');
         }
+        if (this.taskRunnersConfig.taskRequestTimeout <= 0) {
+            throw new n8n_workflow_1.UserError('Task request timeout must be greater than 0');
+        }
+        if (this.taskRunnersConfig.taskAcceptTimeout <= 0) {
+            throw new n8n_workflow_1.UserError('Task accept timeout must be greater than 0');
+        }
     }
     createRequestTimeout(requestId) {
         return setTimeout(() => {
             this.handleRequestTimeout(requestId);
         }, this.taskRunnersConfig.taskRequestTimeout * constants_1.Time.seconds.toMilliseconds);
     }
+    refreshRequestTimeout(request) {
+        const isPending = this.pendingTaskRequests.some((r) => r.requestId === request.requestId);
+        const refreshes = request.timeoutRefreshes ?? 0;
+        if (isPending && refreshes < MAX_REQUEST_TIMEOUT_REFRESHES) {
+            request.timeoutRefreshes = refreshes + 1;
+            clearTimeout(request.timeout);
+            request.timeout = this.createRequestTimeout(request.requestId);
+        }
+    }
     handleRequestTimeout(requestId) {
         const requestIndex = this.pendingTaskRequests.findIndex((r) => r.requestId === requestId);
         if (requestIndex === -1)
@@ -59,33 +77,45 @@ let TaskBroker = class TaskBroker {
             reason: 'timeout',
         });
     }
-    expireTasks() {
-        const now = process.hrtime.bigint();
+    discardOffers(shouldDiscard) {
         for (let i = this.pendingTaskOffers.length - 1; i >= 0; i--) {
-            const offer = this.pendingTaskOffers[i];
-            if (offer.validFor === -1)
-                continue;
-            if (offer.validUntil < now) {
+            if (shouldDiscard(this.pendingTaskOffers[i])) {
                 this.pendingTaskOffers.splice(i, 1);
             }
         }
     }
-    registerRunner(runner, messageCallback) {
-        this.knownRunners.set(runner.id, { runner, messageCallback });
+    expireTasks() {
+        const now = process.hrtime.bigint();
+        this.discardOffers(({ validFor, validUntil }) => validFor !== -1 && validUntil < now);
+    }
+    discardOffersFrom(runnerId) {
+        this.discardOffers((offer) => offer.runnerId === runnerId);
+    }
+    discardOffersFromUnreachableRunners() {
+        this.discardOffers((offer) => this.knownRunners.get(offer.runnerId)?.isRunnerReachable() === false);
+    }
+    registerRunner(runner, messageCallback, isRunnerReachable = () => true) {
+        this.knownRunners.set(runner.id, { runner, messageCallback, isRunnerReachable });
         void this.knownRunners.get(runner.id).messageCallback({ type: 'broker:runnerregistered' });
     }
     deregisterRunner(runnerId, error) {
         this.knownRunners.delete(runnerId);
-        for (let i = this.pendingTaskOffers.length - 1; i >= 0; i--) {
-            if (this.pendingTaskOffers[i].runnerId === runnerId) {
-                this.pendingTaskOffers.splice(i, 1);
+        this.discardOffersFrom(runnerId);
+        for (const [taskId, acceptReject] of this.runnerAcceptRejects) {
+            if (acceptReject.runnerId === runnerId) {
+                this.handleRunnerReject(taskId, `The Task Runner (${runnerId}) has disconnected: ${error.message}`);
             }
         }
-        for (const task of this.tasks.values()) {
-            if (task.runnerId === runnerId) {
-                void this.failTask(task.id, error);
-                this.handleRunnerReject(task.id, `The Task Runner (${runnerId}) has disconnected: ${error.message}`);
-            }
+        this.failTasks(this.getInFlightTaskIds(runnerId), error);
+    }
+    getInFlightTaskIds(runnerId) {
+        return [...this.tasks.values()]
+            .filter((task) => task.runnerId === runnerId)
+            .map((task) => task.id);
+    }
+    failTasks(taskIds, error) {
+        for (const taskId of taskIds) {
+            void this.failTask(taskId, error);
         }
     }
     registerRequester(requesterId, messageCallback) {
@@ -290,29 +320,33 @@ let TaskBroker = class TaskBroker {
             this.requesterAcceptRejects.delete(taskId);
         }
     }
-    async cancelTask(taskId, reason) {
+    untrackTask(taskId) {
         const task = this.tasks.get(taskId);
-        if (!task) {
-            return;
+        if (task) {
+            clearTimeout(task.timeout);
+            this.tasks.delete(taskId);
+        }
+        return task;
+    }
+    async cancelTask(taskId, reason) {
+        const task = this.untrackTask(taskId);
+        if (task) {
+            await this.messageRunner(task.runnerId, {
+                type: 'broker:taskcancel',
+                taskId,
+                reason,
+            });
         }
-        this.tasks.delete(taskId);
-        await this.messageRunner(task.runnerId, {
-            type: 'broker:taskcancel',
-            taskId,
-            reason,
-        });
     }
     async failTask(taskId, error) {
-        const task = this.tasks.get(taskId);
-        if (!task) {
-            return;
+        const task = this.untrackTask(taskId);
+        if (task) {
+            await this.messageRequester(task.requesterId, {
+                type: 'broker:taskerror',
+                taskId,
+                error,
+            });
         }
-        this.tasks.delete(taskId);
-        await this.messageRequester(task.requesterId, {
-            type: 'broker:taskerror',
-            taskId,
-            error,
-        });
     }
     async getRunnerOrFailTask(taskId) {
         const task = this.tasks.get(taskId);
@@ -325,6 +359,11 @@ let TaskBroker = class TaskBroker {
             await this.failTask(taskId, error);
             throw error;
         }
+        if (!runner.isRunnerReachable()) {
+            const error = new task_runner_unreachable_error_1.TaskRunnerUnreachableError(taskId, task.runnerId);
+            await this.failTask(taskId, error);
+            throw error;
+        }
         return runner.runner;
     }
     async sendTaskSettings(taskId, settings) {
@@ -365,37 +404,38 @@ let TaskBroker = class TaskBroker {
         }));
     }
     async taskDoneHandler(taskId, data) {
-        const task = this.tasks.get(taskId);
-        if (!task)
-            return;
-        clearTimeout(task.timeout);
-        await this.requesters.get(task.requesterId)?.({
-            type: 'broker:taskdone',
-            taskId: task.id,
-            data,
-        });
-        this.tasks.delete(task.id);
+        const task = this.untrackTask(taskId);
+        if (task) {
+            await this.requesters.get(task.requesterId)?.({
+                type: 'broker:taskdone',
+                taskId: task.id,
+                data,
+            });
+        }
     }
     async taskErrorHandler(taskId, error) {
-        const task = this.tasks.get(taskId);
-        if (!task)
-            return;
-        clearTimeout(task.timeout);
-        await this.requesters.get(task.requesterId)?.({
-            type: 'broker:taskerror',
-            taskId: task.id,
-            error,
-        });
-        this.tasks.delete(task.id);
+        const task = this.untrackTask(taskId);
+        if (task) {
+            await this.requesters.get(task.requesterId)?.({
+                type: 'broker:taskerror',
+                taskId: task.id,
+                error,
+            });
+        }
     }
     async acceptOffer(offer, request) {
         const taskId = (0, nanoid_1.nanoid)(8);
+        let runnerAcceptTimer;
         try {
             const acceptPromise = new Promise((resolve, reject) => {
-                this.runnerAcceptRejects.set(taskId, { accept: resolve, reject });
-                setTimeout(() => {
+                this.runnerAcceptRejects.set(taskId, {
+                    accept: resolve,
+                    reject,
+                    runnerId: offer.runnerId,
+                });
+                runnerAcceptTimer = setTimeout(() => {
                     reject(new task_runner_accept_timeout_error_1.TaskRunnerAcceptTimeoutError(taskId, offer.runnerId));
-                }, 2000);
+                }, this.taskRunnersConfig.taskAcceptTimeout * constants_1.Time.seconds.toMilliseconds);
             });
             await this.messageRunner(offer.runnerId, {
                 type: 'broker:taskofferaccept',
@@ -408,20 +448,32 @@ let TaskBroker = class TaskBroker {
             request.acceptInProgress = false;
             if (e instanceof task_reject_error_1.TaskRejectError) {
                 this.logger.info(`Task (${taskId}) rejected by Runner with reason "${e.reason}"`);
+                this.settleTasks();
                 return;
             }
             if (e instanceof task_deferred_error_1.TaskDeferredError) {
                 this.logger.debug(`Task (${taskId}) deferred until runner is ready`);
-                clearTimeout(request.timeout);
-                request.timeout = this.createRequestTimeout(request.requestId);
+                this.refreshRequestTimeout(request);
                 return;
             }
             if (e instanceof task_runner_accept_timeout_error_1.TaskRunnerAcceptTimeoutError) {
                 this.logger.warn(e.message);
+                this.runnerAcceptRejects.delete(taskId);
+                this.refreshRequestTimeout(request);
+                await this.messageRunner(offer.runnerId, {
+                    type: 'broker:taskcancel',
+                    taskId,
+                    reason: 'Acknowledgement window elapsed',
+                });
+                this.discardOffersFrom(offer.runnerId);
+                this.settleTasks();
                 return;
             }
             throw e;
         }
+        finally {
+            clearTimeout(runnerAcceptTimer);
+        }
         clearTimeout(request.timeout);
         const task = {
             id: taskId,
@@ -432,19 +484,21 @@ let TaskBroker = class TaskBroker {
         this.tasks.set(taskId, task);
         const requestIndex = this.pendingTaskRequests.findIndex((r) => r.requestId === request.requestId);
         if (requestIndex === -1) {
-            this.logger.error(`Failed to find task request (${request.requestId}) after a task was accepted. This shouldn't happen, and might be a race condition.`);
+            this.logger.warn(`Task request (${request.requestId}) expired while runner (${offer.runnerId}) was accepting task (${taskId}), canceling task`);
+            await this.cancelTask(taskId, 'Task request expired');
             return;
         }
         this.pendingTaskRequests.splice(requestIndex, 1);
+        let requesterAcceptTimer;
         try {
             const acceptPromise = new Promise((resolve, reject) => {
                 this.requesterAcceptRejects.set(taskId, {
                     accept: resolve,
                     reject,
                 });
-                setTimeout(() => {
-                    reject('Requester timed out');
-                }, 2000);
+                requesterAcceptTimer = setTimeout(() => {
+                    reject(new task_requester_accept_timeout_error_1.TaskRequesterAcceptTimeoutError(taskId, request.requesterId));
+                }, this.taskRunnersConfig.taskAcceptTimeout * constants_1.Time.seconds.toMilliseconds);
             });
             await this.messageRequester(request.requesterId, {
                 type: 'broker:taskready',
@@ -460,12 +514,26 @@ let TaskBroker = class TaskBroker {
                 this.logger.info(`Task (${taskId}) rejected by Requester with reason "${e.reason}"`);
                 return;
             }
+            if (e instanceof task_runner_unreachable_error_1.TaskRunnerUnreachableError) {
+                this.logger.warn(e.message);
+                return;
+            }
+            if (e instanceof task_requester_accept_timeout_error_1.TaskRequesterAcceptTimeoutError) {
+                this.logger.warn(e.message);
+                this.requesterAcceptRejects.delete(taskId);
+                await this.cancelTask(task.id, 'Requester took too long to acknowledge the task');
+                return;
+            }
             await this.cancelTask(task.id, 'Unknown reason');
             throw e;
         }
+        finally {
+            clearTimeout(requesterAcceptTimer);
+        }
     }
     settleTasks() {
         this.expireTasks();
+        this.discardOffersFromUnreachableRunners();
         for (const request of this.pendingTaskRequests) {
             if (request.acceptInProgress) {
                 continue;
@@ -535,6 +603,9 @@ let TaskBroker = class TaskBroker {
     getRunnerAcceptRejects() {
         return this.runnerAcceptRejects;
     }
+    getRequesterAcceptRejects() {
+        return this.requesterAcceptRejects;
+    }
     setTasks(tasks) {
         this.tasks = new Map(Object.entries(tasks));
     }