@hatchet-dev/typescript-sdk 1.15.2 → 1.17.0

package/v1/client/worker/worker-internal.js

@@ -1,4 +1,37 @@
 "use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
 var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
     function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
     return new (P || (P = Promise))(function (resolve, reject) {
@@ -21,11 +54,12 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.InternalWorker = void 0;
 exports.mapRateLimitPb = mapRateLimitPb;
-/* eslint-disable no-underscore-dangle */
-/* eslint-disable no-nested-ternary */
+exports.resolveExecutionTimeout = resolveExecutionTimeout;
+exports.resolveScheduleTimeout = resolveScheduleTimeout;
 const hatchet_error_1 = __importDefault(require("../../../util/errors/hatchet-error"));
+const task_run_terminated_error_1 = require("../../../util/errors/task-run-terminated-error");
 const dispatcher_1 = require("../../../protoc/dispatcher");
-const hatchet_promise_1 = __importDefault(require("../../../util/hatchet-promise/hatchet-promise"));
+const hatchet_promise_1 = __importStar(require("../../../util/hatchet-promise/hatchet-promise"));
 const workflows_1 = require("../../../protoc/workflows");
 const logger_1 = require("../../../util/logger");
 const task_1 = require("../../task");
@@ -34,12 +68,18 @@ const zod_to_json_schema_1 = require("zod-to-json-schema");
 const apply_namespace_1 = require("../../../util/apply-namespace");
 const sleep_1 = __importDefault(require("../../../util/sleep"));
 const abort_error_1 = require("../../../util/abort-error");
+const duration_1 = require("../duration");
 const context_1 = require("./context");
 const parent_run_context_vars_1 = require("../../parent-run-context-vars");
 const health_server_1 = require("./health-server");
+const eviction_manager_1 = require("./eviction/eviction-manager");
+const eviction_policy_1 = require("./eviction/eviction-policy");
+const engine_version_1 = require("./engine-version");
 class InternalWorker {
     constructor(client, options) {
         var _a, _b, _c, _d;
+        this.durable_action_set = new Set();
+        this.eviction_policies = new Map();
         this.workflow_registry = [];
         this.futures = {};
         this.contexts = {};
@@ -97,7 +137,12 @@ class InternalWorker {
         const newActions = workflow._durableTasks
             .filter((task) => !!task.fn)
             .reduce((acc, task) => {
-            acc[`${(0, apply_namespace_1.applyNamespace)(workflow.name, this.client.config.namespace).toLowerCase()}:${task.name.toLowerCase()}`] = (ctx) => task.fn(ctx.input, ctx);
+            const actionId = `${(0, apply_namespace_1.applyNamespace)(workflow.name, this.client.config.namespace).toLowerCase()}:${task.name.toLowerCase()}`;
+            acc[actionId] = (ctx) => task.fn(ctx.input, ctx);
+            this.durable_action_set.add(actionId);
+            this.eviction_policies.set(actionId, task.evictionPolicy !== undefined
+                ? task.evictionPolicy
+                : eviction_policy_1.DEFAULT_DURABLE_TASK_EVICTION_POLICY);
             return acc;
         }, {});
         this.action_registry = Object.assign(Object.assign({}, this.action_registry), newActions);
@@ -123,7 +168,7 @@ class InternalWorker {
     }
     registerWorkflow(initWorkflow_1) {
         return __awaiter(this, arguments, void 0, function* (initWorkflow, durable = false) {
-            var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o, _p, _q, _r, _s, _t, _u, _v;
+            var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o, _p, _q, _r, _s, _t, _u, _v, _w;
             // patch the namespace
             const workflow = Object.assign(Object.assign({}, initWorkflow.definition), { name: (0, apply_namespace_1.applyNamespace)(initWorkflow.definition.name, this.client.config.namespace).toLowerCase() });
             try {
@@ -146,19 +191,20 @@ class InternalWorker {
                 }
                 if (workflow.onFailure && typeof workflow.onFailure === 'object') {
                     const onFailure = workflow.onFailure;
+                    const scheduleTimeout = (_a = onFailure.scheduleTimeout) !== null && _a !== void 0 ? _a : (_b = workflow.taskDefaults) === null || _b === void 0 ? void 0 : _b.scheduleTimeout;
                     onFailureTask = {
                         readableId: 'on-failure-task',
                         action: onFailureTaskName(workflow),
-                        timeout: onFailure.executionTimeout || ((_a = workflow.taskDefaults) === null || _a === void 0 ? void 0 : _a.executionTimeout) || '60s',
-                        scheduleTimeout: onFailure.scheduleTimeout || ((_b = workflow.taskDefaults) === null || _b === void 0 ? void 0 : _b.scheduleTimeout),
+                        timeout: (0, duration_1.durationToString)(onFailure.executionTimeout || ((_c = workflow.taskDefaults) === null || _c === void 0 ? void 0 : _c.executionTimeout) || '60s'),
+                        scheduleTimeout: scheduleTimeout ? (0, duration_1.durationToString)(scheduleTimeout) : undefined,
                         inputs: '{}',
                         parents: [],
-                        retries: onFailure.retries || ((_c = workflow.taskDefaults) === null || _c === void 0 ? void 0 : _c.retries) || 0,
-                        rateLimits: mapRateLimitPb(onFailure.rateLimits || ((_d = workflow.taskDefaults) === null || _d === void 0 ? void 0 : _d.rateLimits)),
-                        workerLabels: mapWorkerLabelPb(onFailure.desiredWorkerLabels || ((_e = workflow.taskDefaults) === null || _e === void 0 ? void 0 : _e.workerLabels)),
+                        retries: onFailure.retries || ((_d = workflow.taskDefaults) === null || _d === void 0 ? void 0 : _d.retries) || 0,
+                        rateLimits: mapRateLimitPb(onFailure.rateLimits || ((_e = workflow.taskDefaults) === null || _e === void 0 ? void 0 : _e.rateLimits)),
+                        workerLabels: mapWorkerLabelPb(onFailure.desiredWorkerLabels || ((_f = workflow.taskDefaults) === null || _f === void 0 ? void 0 : _f.workerLabels)),
                         concurrency: [],
-                        backoffFactor: ((_f = onFailure.backoff) === null || _f === void 0 ? void 0 : _f.factor) || ((_h = (_g = workflow.taskDefaults) === null || _g === void 0 ? void 0 : _g.backoff) === null || _h === void 0 ? void 0 : _h.factor),
-                        backoffMaxSeconds: ((_j = onFailure.backoff) === null || _j === void 0 ? void 0 : _j.maxSeconds) || ((_l = (_k = workflow.taskDefaults) === null || _k === void 0 ? void 0 : _k.backoff) === null || _l === void 0 ? void 0 : _l.maxSeconds),
+                        backoffFactor: ((_g = onFailure.backoff) === null || _g === void 0 ? void 0 : _g.factor) || ((_j = (_h = workflow.taskDefaults) === null || _h === void 0 ? void 0 : _h.backoff) === null || _j === void 0 ? void 0 : _j.factor),
+                        backoffMaxSeconds: ((_k = onFailure.backoff) === null || _k === void 0 ? void 0 : _k.maxSeconds) || ((_m = (_l = workflow.taskDefaults) === null || _l === void 0 ? void 0 : _l.backoff) === null || _m === void 0 ? void 0 : _m.maxSeconds),
                         isDurable: false,
                         slotRequests: { default: 1 },
                     };
@@ -183,14 +229,14 @@ class InternalWorker {
                     onSuccessTask = {
                         name: 'on-success-task',
                         fn: onSuccess.fn,
-                        executionTimeout: onSuccess.executionTimeout || ((_m = workflow.taskDefaults) === null || _m === void 0 ? void 0 : _m.executionTimeout) || '60s',
-                        scheduleTimeout: onSuccess.scheduleTimeout || ((_o = workflow.taskDefaults) === null || _o === void 0 ? void 0 : _o.scheduleTimeout),
+                        executionTimeout: onSuccess.executionTimeout || ((_o = workflow.taskDefaults) === null || _o === void 0 ? void 0 : _o.executionTimeout) || '60s',
+                        scheduleTimeout: onSuccess.scheduleTimeout || ((_p = workflow.taskDefaults) === null || _p === void 0 ? void 0 : _p.scheduleTimeout),
                         parents,
-                        retries: onSuccess.retries || ((_p = workflow.taskDefaults) === null || _p === void 0 ? void 0 : _p.retries) || 0,
-                        rateLimits: onSuccess.rateLimits || ((_q = workflow.taskDefaults) === null || _q === void 0 ? void 0 : _q.rateLimits),
-                        desiredWorkerLabels: onSuccess.desiredWorkerLabels || ((_r = workflow.taskDefaults) === null || _r === void 0 ? void 0 : _r.workerLabels),
-                        concurrency: onSuccess.concurrency || ((_s = workflow.taskDefaults) === null || _s === void 0 ? void 0 : _s.concurrency),
-                        backoff: onSuccess.backoff || ((_t = workflow.taskDefaults) === null || _t === void 0 ? void 0 : _t.backoff),
+                        retries: onSuccess.retries || ((_q = workflow.taskDefaults) === null || _q === void 0 ? void 0 : _q.retries) || 0,
+                        rateLimits: onSuccess.rateLimits || ((_r = workflow.taskDefaults) === null || _r === void 0 ? void 0 : _r.rateLimits),
+                        desiredWorkerLabels: onSuccess.desiredWorkerLabels || ((_s = workflow.taskDefaults) === null || _s === void 0 ? void 0 : _s.workerLabels),
+                        concurrency: onSuccess.concurrency || ((_t = workflow.taskDefaults) === null || _t === void 0 ? void 0 : _t.concurrency),
+                        backoff: onSuccess.backoff || ((_u = workflow.taskDefaults) === null || _u === void 0 ? void 0 : _u.backoff),
                     };
                 }
                 if (onSuccessTask) {
@@ -217,7 +263,6 @@ class InternalWorker {
                 // Convert Zod schema to JSON Schema if provided
                 let inputJsonSchema;
                 if (workflow.inputValidator) {
-                    // eslint-disable-next-line @typescript-eslint/no-explicit-any
                     const jsonSchema = (0, zod_to_json_schema_1.zodToJsonSchema)(workflow.inputValidator);
                     inputJsonSchema = new TextEncoder().encode(JSON.stringify(jsonSchema));
                 }
@@ -255,23 +300,20 @@ class InternalWorker {
                     defaultPriority: workflow.defaultPriority,
                     inputJsonSchema,
                     tasks: [...workflow._tasks, ...workflow._durableTasks].map((task) => {
-                        var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o, _p;
+                        var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m;
                         return ({
                             readableId: task.name,
                             action: `${workflow.name}:${task.name}`,
-                            timeout: task.executionTimeout ||
-                                task.timeout ||
-                                ((_a = workflow.taskDefaults) === null || _a === void 0 ? void 0 : _a.executionTimeout) ||
-                                '60s',
-                            scheduleTimeout: task.scheduleTimeout || ((_b = workflow.taskDefaults) === null || _b === void 0 ? void 0 : _b.scheduleTimeout),
+                            timeout: resolveExecutionTimeout(task, workflow.taskDefaults),
+                            scheduleTimeout: resolveScheduleTimeout(task, workflow.taskDefaults),
                             inputs: '{}',
-                            parents: (_d = (_c = task.parents) === null || _c === void 0 ? void 0 : _c.map((p) => p.name)) !== null && _d !== void 0 ? _d : [],
+                            parents: (_b = (_a = task.parents) === null || _a === void 0 ? void 0 : _a.map((p) => p.name)) !== null && _b !== void 0 ? _b : [],
                             userData: '{}',
-                            retries: task.retries || ((_e = workflow.taskDefaults) === null || _e === void 0 ? void 0 : _e.retries) || 0,
-                            rateLimits: mapRateLimitPb(task.rateLimits || ((_f = workflow.taskDefaults) === null || _f === void 0 ? void 0 : _f.rateLimits)),
-                            workerLabels: mapWorkerLabelPb(task.desiredWorkerLabels || ((_g = workflow.taskDefaults) === null || _g === void 0 ? void 0 : _g.workerLabels)),
-                            backoffFactor: ((_h = task.backoff) === null || _h === void 0 ? void 0 : _h.factor) || ((_k = (_j = workflow.taskDefaults) === null || _j === void 0 ? void 0 : _j.backoff) === null || _k === void 0 ? void 0 : _k.factor),
-                            backoffMaxSeconds: ((_l = task.backoff) === null || _l === void 0 ? void 0 : _l.maxSeconds) || ((_o = (_m = workflow.taskDefaults) === null || _m === void 0 ? void 0 : _m.backoff) === null || _o === void 0 ? void 0 : _o.maxSeconds),
+                            retries: task.retries || ((_c = workflow.taskDefaults) === null || _c === void 0 ? void 0 : _c.retries) || 0,
+                            rateLimits: mapRateLimitPb(task.rateLimits || ((_d = workflow.taskDefaults) === null || _d === void 0 ? void 0 : _d.rateLimits)),
+                            workerLabels: mapWorkerLabelPb(task.desiredWorkerLabels || ((_e = workflow.taskDefaults) === null || _e === void 0 ? void 0 : _e.workerLabels)),
+                            backoffFactor: ((_f = task.backoff) === null || _f === void 0 ? void 0 : _f.factor) || ((_h = (_g = workflow.taskDefaults) === null || _g === void 0 ? void 0 : _g.backoff) === null || _h === void 0 ? void 0 : _h.factor),
+                            backoffMaxSeconds: ((_j = task.backoff) === null || _j === void 0 ? void 0 : _j.maxSeconds) || ((_l = (_k = workflow.taskDefaults) === null || _k === void 0 ? void 0 : _k.backoff) === null || _l === void 0 ? void 0 : _l.maxSeconds),
                             conditions: (0, transformer_1.taskConditionsToPb)(task, this.client.config.namespace),
                             isDurable: durableTaskSet.has(task),
                             slotRequests: task.slotRequests || (durableTaskSet.has(task) ? { durable: 1 } : { default: 1 }),
@@ -279,7 +321,7 @@ class InternalWorker {
                                 ? Array.isArray(task.concurrency)
                                     ? task.concurrency
                                     : [task.concurrency]
-                                : ((_p = workflow.taskDefaults) === null || _p === void 0 ? void 0 : _p.concurrency)
+                                : ((_m = workflow.taskDefaults) === null || _m === void 0 ? void 0 : _m.concurrency)
                                     ? Array.isArray(workflow.taskDefaults.concurrency)
                                         ? workflow.taskDefaults.concurrency
                                         : [workflow.taskDefaults.concurrency]
@@ -287,11 +329,11 @@ class InternalWorker {
                         });
                     }),
                     concurrency: concurrencySolo,
-                    defaultFilters: (_v = (_u = workflow.defaultFilters) === null || _u === void 0 ? void 0 : _u.map((f) => ({
+                    defaultFilters: (_w = (_v = workflow.defaultFilters) === null || _v === void 0 ? void 0 : _v.map((f) => ({
                         scope: f.scope,
                         expression: f.expression,
                         payload: f.payload ? new TextEncoder().encode(JSON.stringify(f.payload)) : undefined,
-                    }))) !== null && _v !== void 0 ? _v : [],
+                    }))) !== null && _w !== void 0 ? _w : [],
                 });
                 this.registeredWorkflowPromises.push(registeredWorkflow);
                 yield registeredWorkflow;
@@ -303,17 +345,83 @@ class InternalWorker {
             this.registerActions(workflow);
         });
     }
+    ensureEvictionManager() {
+        var _a, _b, _c;
+        if (this.evictionManager)
+            return this.evictionManager;
+        const totalDurableSlots = (_c = (_b = (_a = this.slotConfig) === null || _a === void 0 ? void 0 : _a.durable) !== null && _b !== void 0 ? _b : this.durableSlots) !== null && _c !== void 0 ? _c : 0;
+        this.evictionManager = new eviction_manager_1.DurableEvictionManager({
+            durableSlots: totalDurableSlots,
+            cancelLocal: (key) => {
+                var _a;
+                const err = new task_run_terminated_error_1.TaskRunTerminatedError('evicted');
+                const ctx = this.contexts[key];
+                if (ctx) {
+                    const invocationCount = (_a = ctx.invocationCount) !== null && _a !== void 0 ? _a : 1;
+                    this.client.durableListener.cleanupTaskState(ctx.action.taskRunExternalId, invocationCount);
+                    if (ctx.abortController) {
+                        ctx.abortController.abort(err);
+                    }
+                }
+                const future = this.futures[key];
+                if (future) {
+                    future.promise.catch(() => undefined);
+                    future.cancel(hatchet_promise_1.CancellationReason.EVICTED_BY_WORKER);
+                }
+            },
+            requestEvictionWithAck: (key, rec) => __awaiter(this, void 0, void 0, function* () {
+                var _a;
+                const ctx = this.contexts[key];
+                const invocationCount = (_a = ctx === null || ctx === void 0 ? void 0 : ctx.invocationCount) !== null && _a !== void 0 ? _a : 1;
+                yield this.client.durableListener.sendEvictInvocation(rec.taskRunExternalId, invocationCount, rec.evictionReason);
+            }),
+            logger: this.logger,
+        });
+        this.client.durableListener.onServerEvict = (durableTaskExternalId, invocationCount) => {
+            var _a;
+            (_a = this.evictionManager) === null || _a === void 0 ? void 0 : _a.handleServerEviction(durableTaskExternalId, invocationCount);
+        };
+        this.evictionManager.start();
+        return this.evictionManager;
+    }
+    cleanupRun(key) {
+        var _a;
+        const ctx = this.contexts[key];
+        if (ctx instanceof context_1.DurableContext) {
+            this.client.durableListener.cleanupTaskState(ctx.action.taskRunExternalId, ctx.invocationCount);
+        }
+        (_a = this.evictionManager) === null || _a === void 0 ? void 0 : _a.unregisterRun(key);
+        delete this.futures[key];
+        delete this.contexts[key];
+    }
     handleStartStepRun(action) {
         return __awaiter(this, void 0, void 0, function* () {
+            var _a;
             const { actionId, taskRunExternalId, taskName } = action;
+            const actionKey = action.key;
             try {
-                // Note: we always use a DurableContext since its a superset of the Context class
-                const context = new context_1.DurableContext(action, this.client, this);
-                this.contexts[taskRunExternalId] = context;
+                const isDurable = this.durable_action_set.has(actionId);
+                let context;
+                if (isDurable) {
+                    const { durableListener } = this.client;
+                    let mgr;
+                    if ((0, engine_version_1.supportsEviction)(this.engineVersion)) {
+                        yield durableListener.ensureStarted(this.workerId || '');
+                        mgr = this.ensureEvictionManager();
+                        const evictionPolicy = this.eviction_policies.get(actionId);
+                        mgr.registerRun(actionKey, taskRunExternalId, (_a = action.durableTaskInvocationCount) !== null && _a !== void 0 ? _a : 1, evictionPolicy);
+                    }
+                    context = new context_1.DurableContext(action, this.client, this, durableListener, mgr, this.engineVersion);
+                }
+                else {
+                    context = new context_1.Context(action, this.client, this);
+                }
+                this.contexts[actionKey] = context;
                 const step = this.action_registry[actionId];
                 if (!step) {
                     this.logger.error(`Registered actions: '${Object.keys(this.action_registry).join(', ')}'`);
                     this.logger.error(`Could not find step '${actionId}'`);
+                    this.cleanupRun(actionKey);
                     return;
                 }
                 const run = () => __awaiter(this, void 0, void 0, function* () {
@@ -337,8 +445,8 @@ class InternalWorker {
                         childIndex: 0,
                         desiredWorkerId: this.workerId || '',
                         signal: context.abortController.signal,
+                        durableContext: isDurable && context instanceof context_1.DurableContext ? context : undefined,
                     }, () => {
-                        // Precheck: if cancellation already happened, don't execute user code.
                         (0, abort_error_1.throwIfAborted)(context.abortController.signal);
                         return step(context);
                     });
@@ -359,13 +467,11 @@ class InternalWorker {
                             return;
                         }
                         this.logger.info((0, logger_1.taskRunLog)(taskName, taskRunExternalId, 'completed'));
-                        // Send the action event to the dispatcher
                         const event = this.getStepActionEvent(action, dispatcher_1.StepActionEventType.STEP_EVENT_TYPE_COMPLETED, false, result || null, action.retryCount);
                         yield this.client.dispatcher.sendStepActionEvent(event);
                     }
                     catch (actionEventError) {
                         this.logger.error(`Could not send completed action event: ${actionEventError.message || actionEventError}`);
-                        // send a failure event
                         const failureEvent = this.getStepActionEvent(action, dispatcher_1.StepActionEventType.STEP_EVENT_TYPE_FAILED, false, actionEventError.message, action.retryCount);
                         try {
                             yield this.client.dispatcher.sendStepActionEvent(failureEvent);
@@ -376,9 +482,7 @@ class InternalWorker {
                         this.logger.error(`Could not send action event: ${actionEventError.message || actionEventError}`);
                     }
                     finally {
-                        // delete the run from the futures
-                        delete this.futures[taskRunExternalId];
-                        delete this.contexts[taskRunExternalId];
+                        this.cleanupRun(actionKey);
                     }
                 });
                 const failure = (error) => __awaiter(this, void 0, void 0, function* () {
@@ -391,7 +495,6 @@ class InternalWorker {
                         if (error.stack) {
                             this.logger.error(error.stack);
                         }
-                        // Send the action event to the dispatcher
                         const event = this.getStepActionEvent(action, dispatcher_1.StepActionEventType.STEP_EVENT_TYPE_FAILED, shouldNotRetry, {
                             message: error === null || error === void 0 ? void 0 : error.message,
                             stack: error === null || error === void 0 ? void 0 : error.stack,
@@ -402,9 +505,7 @@ class InternalWorker {
                         this.logger.error(`Could not send action event: ${e.message}`);
                     }
                     finally {
-                        // delete the run from the futures
-                        delete this.futures[taskRunExternalId];
-                        delete this.contexts[taskRunExternalId];
+                        this.cleanupRun(actionKey);
                     }
                 });
                 const future = new hatchet_promise_1.default((() => __awaiter(this, void 0, void 0, function* () {
@@ -427,7 +528,7 @@ class InternalWorker {
                     (0, abort_error_1.throwIfAborted)(context.abortController.signal);
                     yield success(result);
                 }))());
-                this.futures[taskRunExternalId] = future;
+                this.futures[actionKey] = future;
                 // Send the action event to the dispatcher
                 const event = this.getStepActionEvent(action, dispatcher_1.StepActionEventType.STEP_EVENT_TYPE_STARTED, false, undefined, action.retryCount);
                 this.client.dispatcher.sendStepActionEvent(event).catch((e) => {
@@ -437,87 +538,18 @@ class InternalWorker {
                     yield future.promise;
                 }
                 catch (e) {
-                    const message = (e === null || e === void 0 ? void 0 : e.message) || String(e);
-                    // TODO is this cased correctly...
-                    if (!message.includes('Cancelled')) {
+                    if (!(0, task_run_terminated_error_1.isTaskRunTerminatedError)(e)) {
                         this.logger.error(`Could not wait for task run ${taskRunExternalId} to finish. ` +
                             `See https://docs.hatchet.run/home/cancellation for best practices on handling cancellation: `, e);
                     }
                 }
-            }
-            catch (e) {
-                this.logger.error('Could not send action event (outer): ', e);
-            }
-        });
-    }
-    handleStartGroupKeyRun(action) {
-        return __awaiter(this, void 0, void 0, function* () {
-            const { actionId, getGroupKeyRunId, taskRunExternalId, taskName } = action;
-            this.logger.error('Concurrency Key Functions have been deprecated and will be removed in a future release. Use Concurrency Expressions instead.');
-            try {
-                const context = new context_1.Context(action, this.client, this);
-                const key = getGroupKeyRunId;
-                if (!key) {
-                    this.logger.error(`No group key run id provided for action ${actionId}`);
-                    return;
-                }
-                this.contexts[key] = context;
-                this.logger.debug(`Starting group key run ${key}`);
-                const step = this.action_registry[actionId];
-                if (!step) {
-                    this.logger.error(`Could not find step '${actionId}'`);
-                    return;
+                finally {
+                    this.cleanupRun(actionKey);
                 }
-                const run = () => __awaiter(this, void 0, void 0, function* () {
-                    return step(context);
-                });
-                const success = (result) => {
-                    this.logger.info((0, logger_1.taskRunLog)(taskName, taskRunExternalId, 'completed'));
-                    try {
-                        // Send the action event to the dispatcher
-                        const event = this.getGroupKeyActionEvent(action, dispatcher_1.GroupKeyActionEventType.GROUP_KEY_EVENT_TYPE_COMPLETED, result);
-                        this.client.dispatcher.sendGroupKeyActionEvent(event).catch((e) => {
-                            this.logger.error(`Could not send action event: ${e.message}`);
-                        });
-                    }
-                    catch (e) {
-                        this.logger.error(`Could not send action event: ${e.message}`);
-                    }
-                    finally {
-                        // delete the run from the futures
-                        delete this.futures[key];
-                        delete this.contexts[key];
-                    }
-                };
-                const failure = (error) => {
-                    this.logger.error((0, logger_1.taskRunLog)(taskName, taskRunExternalId, `failed: ${error.message}`));
-                    try {
-                        // Send the action event to the dispatcher
-                        const event = this.getGroupKeyActionEvent(action, dispatcher_1.GroupKeyActionEventType.GROUP_KEY_EVENT_TYPE_FAILED, error);
-                        this.client.dispatcher.sendGroupKeyActionEvent(event).catch((e) => {
-                            this.logger.error(`Could not send action event: ${e.message}`);
-                        });
-                    }
-                    catch (e) {
-                        this.logger.error(`Could not send action event: ${e.message}`);
-                    }
-                    finally {
-                        // delete the run from the futures
-                        delete this.futures[key];
-                        delete this.contexts[key];
-                    }
-                };
-                const future = new hatchet_promise_1.default(run().then(success).catch(failure));
-                this.futures[key] = future;
-                // Send the action event to the dispatcher
-                const event = this.getGroupKeyActionEvent(action, dispatcher_1.GroupKeyActionEventType.GROUP_KEY_EVENT_TYPE_STARTED);
-                this.client.dispatcher.sendGroupKeyActionEvent(event).catch((e) => {
-                    this.logger.error(`Could not send action event: ${e.message}`);
-                });
-                yield future.promise;
             }
             catch (e) {
-                this.logger.error(`Could not send action event: ${e.message}`);
+                this.cleanupRun(actionKey);
+                this.logger.error('Could not send action event (outer): ', e);
             }
         });
     }
@@ -554,11 +586,13 @@ class InternalWorker {
         return __awaiter(this, void 0, void 0, function* () {
             var _a, _b, _c;
             const { taskRunExternalId, taskName } = action;
+            const actionKey = action.key;
             try {
-                const future = this.futures[taskRunExternalId];
-                const context = this.contexts[taskRunExternalId];
+                const future = this.futures[actionKey];
+                const context = this.contexts[actionKey];
+                const cancelErr = new task_run_terminated_error_1.TaskRunTerminatedError('cancelled', 'Cancelled by worker');
                 if (context && context.abortController) {
-                    context.abortController.abort('Cancelled by worker'); // TODO this reason is nonsensical
+                    context.abortController.abort(cancelErr);
                 }
                 if (future) {
                     const start = Date.now();
@@ -570,7 +604,7 @@ class InternalWorker {
                     // where the main action handler isn't currently awaiting `future.promise`.
                     future.promise.catch(() => undefined);
                     // Cancel the future (rejects the wrapper); user code must still cooperate with AbortSignal.
-                    future.cancel('Cancelled by worker'); // TODO this reason is nonsensical
+                    future.cancel(hatchet_promise_1.CancellationReason.CANCELLED_BY_WORKER);
                     // Track completion of the underlying work (not the cancelable wrapper).
                     // Ensure this promise never throws into our supervision flow.
                     const completion = ((_c = future.inner) !== null && _c !== void 0 ? _c : future.promise).catch(() => undefined);
@@ -608,8 +642,7 @@ class InternalWorker {
                 this.logger.error(`Cancellation: error while supervising cancellation for task run ${taskRunExternalId}: ${(e === null || e === void 0 ? void 0 : e.message) || e}`);
             }
             finally {
-                delete this.futures[taskRunExternalId];
-                delete this.contexts[taskRunExternalId];
+                this.cleanupRun(actionKey);
             }
         });
     }
@@ -624,6 +657,33 @@ class InternalWorker {
             this.killing = true;
             this.setStatus(health_server_1.workerStatus.UNHEALTHY);
             this.logger.info('Starting to exit...');
+            // Pause the worker on the server so it stops receiving new task assignments
+            // before we evict waiting durable runs, mirroring Python's pause_task_assignment().
+            if (this.workerId) {
+                try {
+                    yield this.client.workers.pause(this.workerId);
+                }
+                catch (e) {
+                    this.logger.error(`Could not pause worker: ${e.message}`);
+                }
+            }
+            if (this.evictionManager) {
+                try {
+                    const evicted = yield this.evictionManager.evictAllWaiting();
+                    if (evicted > 0) {
+                        this.logger.info(`Evicted ${evicted} waiting durable run(s) during shutdown`);
+                    }
+                }
+                catch (e) {
+                    this.logger.error(`Could not evict waiting runs: ${e.message}`);
+                }
+            }
+            try {
+                yield this.client.durableListener.stop();
+            }
+            catch (e) {
+                this.logger.error(`Could not stop durable listener: ${e.message}`);
+            }
             try {
                 yield ((_a = this.listener) === null || _a === void 0 ? void 0 : _a.unregister());
             }
@@ -719,20 +779,22 @@ class InternalWorker {
     }
     handleAction(action) {
         return __awaiter(this, void 0, void 0, function* () {
-            const type = action.actionType
-                ? (0, dispatcher_1.actionTypeFromJSON)(action.actionType)
-                : dispatcher_1.ActionType.START_STEP_RUN;
-            if (type === dispatcher_1.ActionType.START_STEP_RUN) {
-                yield this.handleStartStepRun(action);
-            }
-            else if (type === dispatcher_1.ActionType.CANCEL_STEP_RUN) {
-                yield this.handleCancelStepRun(action);
-            }
-            else if (type === dispatcher_1.ActionType.START_GET_GROUP_KEY) {
-                yield this.handleStartGroupKeyRun(action);
-            }
-            else {
-                this.logger.error(`Worker ${this.name} received unknown action type ${type}`);
+            const type = (0, dispatcher_1.actionTypeFromJSON)(action.actionType) || dispatcher_1.ActionType.START_STEP_RUN;
+            switch (type) {
+                case dispatcher_1.ActionType.START_STEP_RUN:
+                    return this.handleStartStepRun(action);
+                case dispatcher_1.ActionType.CANCEL_STEP_RUN:
+                    return this.handleCancelStepRun(action);
+                case dispatcher_1.ActionType.START_GET_GROUP_KEY:
+                    this.logger.error(`Worker ${this.name} received unsupported action type START_GET_GROUP_KEY, please upgrade to V1...`);
+                    return Promise.resolve();
+                case dispatcher_1.ActionType.UNRECOGNIZED:
+                    this.logger.error(`Worker ${this.name} received unrecognized action type ${action.actionType}`);
+                    return Promise.resolve();
+                default: {
+                    const _ = type;
+                    throw new Error(`Unhandled action type: ${_}`);
+                }
             }
         });
     }
@@ -791,15 +853,15 @@ function isLeafTask(task, allTasks) {
     return !allTasks.some((t) => { var _a; return (_a = t.parents) === null || _a === void 0 ? void 0 : _a.some((p) => p.name === task.name); });
 }
 function mapRateLimitPb(limits) {
-    if (!limits)
+    if (!limits) {
         return [];
+    }
     return limits.map((l) => {
         let key = l.staticKey;
         const keyExpression = l.dynamicKey;
         if (l.key !== undefined) {
-            // eslint-disable-next-line no-console
             console.warn('key is deprecated and will be removed in a future release, please use staticKey instead');
-            key = l.key;
+            ({ key } = l);
         }
         if (keyExpression !== undefined) {
             if (key !== undefined) {
@@ -816,7 +878,7 @@ function mapRateLimitPb(limits) {
         let units;
         let unitsExpression;
         if (typeof l.units === 'number') {
-            units = l.units;
+            ({ units } = l);
         }
         else {
             if (!validateCelExpression(l.units)) {
@@ -853,9 +915,15 @@ function mapRateLimitPb(limits) {
     });
 }
 // Helper function to validate CEL expressions
-// eslint-disable-next-line @typescript-eslint/no-unused-vars
 function validateCelExpression(_expr) {
     // FIXME: this is a placeholder. In a real implementation, you'd need to use a CEL parser or validator.
     // For now, we'll just return true to mimic the behavior.
     return true;
 }
+function resolveExecutionTimeout(task, workflowDefaults) {
+    return (0, duration_1.durationToString)(task.executionTimeout || task.timeout || (workflowDefaults === null || workflowDefaults === void 0 ? void 0 : workflowDefaults.executionTimeout) || '60s');
+}
+function resolveScheduleTimeout(task, workflowDefaults) {
+    const value = task.scheduleTimeout || (workflowDefaults === null || workflowDefaults === void 0 ? void 0 : workflowDefaults.scheduleTimeout);
+    return value ? (0, duration_1.durationToString)(value) : undefined;
+}

package/v1/client/worker/worker.d.ts

@@ -61,6 +61,7 @@ export declare class Worker {
      * @returns Promise that resolves when the worker is stopped or killed
      */
     start(): Promise<void>;
+    private _checkEvictionSupport;
     /**
      * Stops the worker
      * @returns Promise that resolves when the worker stops