@hatchet-dev/typescript-sdk 1.16.0 → 1.17.0

package/v1/client/worker/worker-internal.js

@@ -54,7 +54,10 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.InternalWorker = void 0;
 exports.mapRateLimitPb = mapRateLimitPb;
+exports.resolveExecutionTimeout = resolveExecutionTimeout;
+exports.resolveScheduleTimeout = resolveScheduleTimeout;
 const hatchet_error_1 = __importDefault(require("../../../util/errors/hatchet-error"));
+const task_run_terminated_error_1 = require("../../../util/errors/task-run-terminated-error");
 const dispatcher_1 = require("../../../protoc/dispatcher");
 const hatchet_promise_1 = __importStar(require("../../../util/hatchet-promise/hatchet-promise"));
 const workflows_1 = require("../../../protoc/workflows");
@@ -65,12 +68,18 @@ const zod_to_json_schema_1 = require("zod-to-json-schema");
 const apply_namespace_1 = require("../../../util/apply-namespace");
 const sleep_1 = __importDefault(require("../../../util/sleep"));
 const abort_error_1 = require("../../../util/abort-error");
+const duration_1 = require("../duration");
 const context_1 = require("./context");
 const parent_run_context_vars_1 = require("../../parent-run-context-vars");
 const health_server_1 = require("./health-server");
+const eviction_manager_1 = require("./eviction/eviction-manager");
+const eviction_policy_1 = require("./eviction/eviction-policy");
+const engine_version_1 = require("./engine-version");
 class InternalWorker {
     constructor(client, options) {
         var _a, _b, _c, _d;
+        this.durable_action_set = new Set();
+        this.eviction_policies = new Map();
         this.workflow_registry = [];
         this.futures = {};
         this.contexts = {};
@@ -128,7 +137,12 @@ class InternalWorker {
         const newActions = workflow._durableTasks
             .filter((task) => !!task.fn)
             .reduce((acc, task) => {
-            acc[`${(0, apply_namespace_1.applyNamespace)(workflow.name, this.client.config.namespace).toLowerCase()}:${task.name.toLowerCase()}`] = (ctx) => task.fn(ctx.input, ctx);
+            const actionId = `${(0, apply_namespace_1.applyNamespace)(workflow.name, this.client.config.namespace).toLowerCase()}:${task.name.toLowerCase()}`;
+            acc[actionId] = (ctx) => task.fn(ctx.input, ctx);
+            this.durable_action_set.add(actionId);
+            this.eviction_policies.set(actionId, task.evictionPolicy !== undefined
+                ? task.evictionPolicy
+                : eviction_policy_1.DEFAULT_DURABLE_TASK_EVICTION_POLICY);
             return acc;
         }, {});
         this.action_registry = Object.assign(Object.assign({}, this.action_registry), newActions);
@@ -154,7 +168,7 @@ class InternalWorker {
     }
     registerWorkflow(initWorkflow_1) {
         return __awaiter(this, arguments, void 0, function* (initWorkflow, durable = false) {
-            var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o, _p, _q, _r, _s, _t, _u, _v;
+            var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o, _p, _q, _r, _s, _t, _u, _v, _w;
             // patch the namespace
             const workflow = Object.assign(Object.assign({}, initWorkflow.definition), { name: (0, apply_namespace_1.applyNamespace)(initWorkflow.definition.name, this.client.config.namespace).toLowerCase() });
             try {
@@ -177,19 +191,20 @@ class InternalWorker {
                 }
                 if (workflow.onFailure && typeof workflow.onFailure === 'object') {
                     const onFailure = workflow.onFailure;
+                    const scheduleTimeout = (_a = onFailure.scheduleTimeout) !== null && _a !== void 0 ? _a : (_b = workflow.taskDefaults) === null || _b === void 0 ? void 0 : _b.scheduleTimeout;
                     onFailureTask = {
                         readableId: 'on-failure-task',
                         action: onFailureTaskName(workflow),
-                        timeout: onFailure.executionTimeout || ((_a = workflow.taskDefaults) === null || _a === void 0 ? void 0 : _a.executionTimeout) || '60s',
-                        scheduleTimeout: onFailure.scheduleTimeout || ((_b = workflow.taskDefaults) === null || _b === void 0 ? void 0 : _b.scheduleTimeout),
+                        timeout: (0, duration_1.durationToString)(onFailure.executionTimeout || ((_c = workflow.taskDefaults) === null || _c === void 0 ? void 0 : _c.executionTimeout) || '60s'),
+                        scheduleTimeout: scheduleTimeout ? (0, duration_1.durationToString)(scheduleTimeout) : undefined,
                         inputs: '{}',
                         parents: [],
-                        retries: onFailure.retries || ((_c = workflow.taskDefaults) === null || _c === void 0 ? void 0 : _c.retries) || 0,
-                        rateLimits: mapRateLimitPb(onFailure.rateLimits || ((_d = workflow.taskDefaults) === null || _d === void 0 ? void 0 : _d.rateLimits)),
-                        workerLabels: mapWorkerLabelPb(onFailure.desiredWorkerLabels || ((_e = workflow.taskDefaults) === null || _e === void 0 ? void 0 : _e.workerLabels)),
+                        retries: onFailure.retries || ((_d = workflow.taskDefaults) === null || _d === void 0 ? void 0 : _d.retries) || 0,
+                        rateLimits: mapRateLimitPb(onFailure.rateLimits || ((_e = workflow.taskDefaults) === null || _e === void 0 ? void 0 : _e.rateLimits)),
+                        workerLabels: mapWorkerLabelPb(onFailure.desiredWorkerLabels || ((_f = workflow.taskDefaults) === null || _f === void 0 ? void 0 : _f.workerLabels)),
                         concurrency: [],
-                        backoffFactor: ((_f = onFailure.backoff) === null || _f === void 0 ? void 0 : _f.factor) || ((_h = (_g = workflow.taskDefaults) === null || _g === void 0 ? void 0 : _g.backoff) === null || _h === void 0 ? void 0 : _h.factor),
-                        backoffMaxSeconds: ((_j = onFailure.backoff) === null || _j === void 0 ? void 0 : _j.maxSeconds) || ((_l = (_k = workflow.taskDefaults) === null || _k === void 0 ? void 0 : _k.backoff) === null || _l === void 0 ? void 0 : _l.maxSeconds),
+                        backoffFactor: ((_g = onFailure.backoff) === null || _g === void 0 ? void 0 : _g.factor) || ((_j = (_h = workflow.taskDefaults) === null || _h === void 0 ? void 0 : _h.backoff) === null || _j === void 0 ? void 0 : _j.factor),
+                        backoffMaxSeconds: ((_k = onFailure.backoff) === null || _k === void 0 ? void 0 : _k.maxSeconds) || ((_m = (_l = workflow.taskDefaults) === null || _l === void 0 ? void 0 : _l.backoff) === null || _m === void 0 ? void 0 : _m.maxSeconds),
                         isDurable: false,
                         slotRequests: { default: 1 },
                     };
@@ -214,14 +229,14 @@ class InternalWorker {
                     onSuccessTask = {
                         name: 'on-success-task',
                         fn: onSuccess.fn,
-                        executionTimeout: onSuccess.executionTimeout || ((_m = workflow.taskDefaults) === null || _m === void 0 ? void 0 : _m.executionTimeout) || '60s',
-                        scheduleTimeout: onSuccess.scheduleTimeout || ((_o = workflow.taskDefaults) === null || _o === void 0 ? void 0 : _o.scheduleTimeout),
+                        executionTimeout: onSuccess.executionTimeout || ((_o = workflow.taskDefaults) === null || _o === void 0 ? void 0 : _o.executionTimeout) || '60s',
+                        scheduleTimeout: onSuccess.scheduleTimeout || ((_p = workflow.taskDefaults) === null || _p === void 0 ? void 0 : _p.scheduleTimeout),
                         parents,
-                        retries: onSuccess.retries || ((_p = workflow.taskDefaults) === null || _p === void 0 ? void 0 : _p.retries) || 0,
-                        rateLimits: onSuccess.rateLimits || ((_q = workflow.taskDefaults) === null || _q === void 0 ? void 0 : _q.rateLimits),
-                        desiredWorkerLabels: onSuccess.desiredWorkerLabels || ((_r = workflow.taskDefaults) === null || _r === void 0 ? void 0 : _r.workerLabels),
-                        concurrency: onSuccess.concurrency || ((_s = workflow.taskDefaults) === null || _s === void 0 ? void 0 : _s.concurrency),
-                        backoff: onSuccess.backoff || ((_t = workflow.taskDefaults) === null || _t === void 0 ? void 0 : _t.backoff),
+                        retries: onSuccess.retries || ((_q = workflow.taskDefaults) === null || _q === void 0 ? void 0 : _q.retries) || 0,
+                        rateLimits: onSuccess.rateLimits || ((_r = workflow.taskDefaults) === null || _r === void 0 ? void 0 : _r.rateLimits),
+                        desiredWorkerLabels: onSuccess.desiredWorkerLabels || ((_s = workflow.taskDefaults) === null || _s === void 0 ? void 0 : _s.workerLabels),
+                        concurrency: onSuccess.concurrency || ((_t = workflow.taskDefaults) === null || _t === void 0 ? void 0 : _t.concurrency),
+                        backoff: onSuccess.backoff || ((_u = workflow.taskDefaults) === null || _u === void 0 ? void 0 : _u.backoff),
                     };
                 }
                 if (onSuccessTask) {
@@ -285,23 +300,20 @@ class InternalWorker {
                     defaultPriority: workflow.defaultPriority,
                     inputJsonSchema,
                     tasks: [...workflow._tasks, ...workflow._durableTasks].map((task) => {
-                        var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o, _p;
+                        var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m;
                         return ({
                             readableId: task.name,
                             action: `${workflow.name}:${task.name}`,
-                            timeout: task.executionTimeout ||
-                                task.timeout ||
-                                ((_a = workflow.taskDefaults) === null || _a === void 0 ? void 0 : _a.executionTimeout) ||
-                                '60s',
-                            scheduleTimeout: task.scheduleTimeout || ((_b = workflow.taskDefaults) === null || _b === void 0 ? void 0 : _b.scheduleTimeout),
+                            timeout: resolveExecutionTimeout(task, workflow.taskDefaults),
+                            scheduleTimeout: resolveScheduleTimeout(task, workflow.taskDefaults),
                             inputs: '{}',
-                            parents: (_d = (_c = task.parents) === null || _c === void 0 ? void 0 : _c.map((p) => p.name)) !== null && _d !== void 0 ? _d : [],
+                            parents: (_b = (_a = task.parents) === null || _a === void 0 ? void 0 : _a.map((p) => p.name)) !== null && _b !== void 0 ? _b : [],
                             userData: '{}',
-                            retries: task.retries || ((_e = workflow.taskDefaults) === null || _e === void 0 ? void 0 : _e.retries) || 0,
-                            rateLimits: mapRateLimitPb(task.rateLimits || ((_f = workflow.taskDefaults) === null || _f === void 0 ? void 0 : _f.rateLimits)),
-                            workerLabels: mapWorkerLabelPb(task.desiredWorkerLabels || ((_g = workflow.taskDefaults) === null || _g === void 0 ? void 0 : _g.workerLabels)),
-                            backoffFactor: ((_h = task.backoff) === null || _h === void 0 ? void 0 : _h.factor) || ((_k = (_j = workflow.taskDefaults) === null || _j === void 0 ? void 0 : _j.backoff) === null || _k === void 0 ? void 0 : _k.factor),
-                            backoffMaxSeconds: ((_l = task.backoff) === null || _l === void 0 ? void 0 : _l.maxSeconds) || ((_o = (_m = workflow.taskDefaults) === null || _m === void 0 ? void 0 : _m.backoff) === null || _o === void 0 ? void 0 : _o.maxSeconds),
+                            retries: task.retries || ((_c = workflow.taskDefaults) === null || _c === void 0 ? void 0 : _c.retries) || 0,
+                            rateLimits: mapRateLimitPb(task.rateLimits || ((_d = workflow.taskDefaults) === null || _d === void 0 ? void 0 : _d.rateLimits)),
+                            workerLabels: mapWorkerLabelPb(task.desiredWorkerLabels || ((_e = workflow.taskDefaults) === null || _e === void 0 ? void 0 : _e.workerLabels)),
+                            backoffFactor: ((_f = task.backoff) === null || _f === void 0 ? void 0 : _f.factor) || ((_h = (_g = workflow.taskDefaults) === null || _g === void 0 ? void 0 : _g.backoff) === null || _h === void 0 ? void 0 : _h.factor),
+                            backoffMaxSeconds: ((_j = task.backoff) === null || _j === void 0 ? void 0 : _j.maxSeconds) || ((_l = (_k = workflow.taskDefaults) === null || _k === void 0 ? void 0 : _k.backoff) === null || _l === void 0 ? void 0 : _l.maxSeconds),
                             conditions: (0, transformer_1.taskConditionsToPb)(task, this.client.config.namespace),
                             isDurable: durableTaskSet.has(task),
                             slotRequests: task.slotRequests || (durableTaskSet.has(task) ? { durable: 1 } : { default: 1 }),
@@ -309,7 +321,7 @@ class InternalWorker {
                                 ? Array.isArray(task.concurrency)
                                     ? task.concurrency
                                     : [task.concurrency]
-                                : ((_p = workflow.taskDefaults) === null || _p === void 0 ? void 0 : _p.concurrency)
+                                : ((_m = workflow.taskDefaults) === null || _m === void 0 ? void 0 : _m.concurrency)
                                     ? Array.isArray(workflow.taskDefaults.concurrency)
                                         ? workflow.taskDefaults.concurrency
                                         : [workflow.taskDefaults.concurrency]
@@ -317,11 +329,11 @@ class InternalWorker {
                         });
                     }),
                     concurrency: concurrencySolo,
-                    defaultFilters: (_v = (_u = workflow.defaultFilters) === null || _u === void 0 ? void 0 : _u.map((f) => ({
+                    defaultFilters: (_w = (_v = workflow.defaultFilters) === null || _v === void 0 ? void 0 : _v.map((f) => ({
                         scope: f.scope,
                         expression: f.expression,
                         payload: f.payload ? new TextEncoder().encode(JSON.stringify(f.payload)) : undefined,
-                    }))) !== null && _v !== void 0 ? _v : [],
+                    }))) !== null && _w !== void 0 ? _w : [],
                 });
                 this.registeredWorkflowPromises.push(registeredWorkflow);
                 yield registeredWorkflow;
@@ -333,17 +345,83 @@ class InternalWorker {
             this.registerActions(workflow);
         });
     }
+    ensureEvictionManager() {
+        var _a, _b, _c;
+        if (this.evictionManager)
+            return this.evictionManager;
+        const totalDurableSlots = (_c = (_b = (_a = this.slotConfig) === null || _a === void 0 ? void 0 : _a.durable) !== null && _b !== void 0 ? _b : this.durableSlots) !== null && _c !== void 0 ? _c : 0;
+        this.evictionManager = new eviction_manager_1.DurableEvictionManager({
+            durableSlots: totalDurableSlots,
+            cancelLocal: (key) => {
+                var _a;
+                const err = new task_run_terminated_error_1.TaskRunTerminatedError('evicted');
+                const ctx = this.contexts[key];
+                if (ctx) {
+                    const invocationCount = (_a = ctx.invocationCount) !== null && _a !== void 0 ? _a : 1;
+                    this.client.durableListener.cleanupTaskState(ctx.action.taskRunExternalId, invocationCount);
+                    if (ctx.abortController) {
+                        ctx.abortController.abort(err);
+                    }
+                }
+                const future = this.futures[key];
+                if (future) {
+                    future.promise.catch(() => undefined);
+                    future.cancel(hatchet_promise_1.CancellationReason.EVICTED_BY_WORKER);
+                }
+            },
+            requestEvictionWithAck: (key, rec) => __awaiter(this, void 0, void 0, function* () {
+                var _a;
+                const ctx = this.contexts[key];
+                const invocationCount = (_a = ctx === null || ctx === void 0 ? void 0 : ctx.invocationCount) !== null && _a !== void 0 ? _a : 1;
+                yield this.client.durableListener.sendEvictInvocation(rec.taskRunExternalId, invocationCount, rec.evictionReason);
+            }),
+            logger: this.logger,
+        });
+        this.client.durableListener.onServerEvict = (durableTaskExternalId, invocationCount) => {
+            var _a;
+            (_a = this.evictionManager) === null || _a === void 0 ? void 0 : _a.handleServerEviction(durableTaskExternalId, invocationCount);
+        };
+        this.evictionManager.start();
+        return this.evictionManager;
+    }
+    cleanupRun(key) {
+        var _a;
+        const ctx = this.contexts[key];
+        if (ctx instanceof context_1.DurableContext) {
+            this.client.durableListener.cleanupTaskState(ctx.action.taskRunExternalId, ctx.invocationCount);
+        }
+        (_a = this.evictionManager) === null || _a === void 0 ? void 0 : _a.unregisterRun(key);
+        delete this.futures[key];
+        delete this.contexts[key];
+    }
     handleStartStepRun(action) {
         return __awaiter(this, void 0, void 0, function* () {
+            var _a;
             const { actionId, taskRunExternalId, taskName } = action;
+            const actionKey = action.key;
             try {
-                // Note: we always use a DurableContext since its a superset of the Context class
-                const context = new context_1.DurableContext(action, this.client, this);
-                this.contexts[taskRunExternalId] = context;
+                const isDurable = this.durable_action_set.has(actionId);
+                let context;
+                if (isDurable) {
+                    const { durableListener } = this.client;
+                    let mgr;
+                    if ((0, engine_version_1.supportsEviction)(this.engineVersion)) {
+                        yield durableListener.ensureStarted(this.workerId || '');
+                        mgr = this.ensureEvictionManager();
+                        const evictionPolicy = this.eviction_policies.get(actionId);
+                        mgr.registerRun(actionKey, taskRunExternalId, (_a = action.durableTaskInvocationCount) !== null && _a !== void 0 ? _a : 1, evictionPolicy);
+                    }
+                    context = new context_1.DurableContext(action, this.client, this, durableListener, mgr, this.engineVersion);
+                }
+                else {
+                    context = new context_1.Context(action, this.client, this);
+                }
+                this.contexts[actionKey] = context;
                 const step = this.action_registry[actionId];
                 if (!step) {
                     this.logger.error(`Registered actions: '${Object.keys(this.action_registry).join(', ')}'`);
                     this.logger.error(`Could not find step '${actionId}'`);
+                    this.cleanupRun(actionKey);
                     return;
                 }
                 const run = () => __awaiter(this, void 0, void 0, function* () {
@@ -367,8 +445,8 @@ class InternalWorker {
                         childIndex: 0,
                         desiredWorkerId: this.workerId || '',
                         signal: context.abortController.signal,
+                        durableContext: isDurable && context instanceof context_1.DurableContext ? context : undefined,
                     }, () => {
-                        // Precheck: if cancellation already happened, don't execute user code.
                         (0, abort_error_1.throwIfAborted)(context.abortController.signal);
                         return step(context);
                     });
@@ -389,13 +467,11 @@ class InternalWorker {
                             return;
                         }
                         this.logger.info((0, logger_1.taskRunLog)(taskName, taskRunExternalId, 'completed'));
-                        // Send the action event to the dispatcher
                         const event = this.getStepActionEvent(action, dispatcher_1.StepActionEventType.STEP_EVENT_TYPE_COMPLETED, false, result || null, action.retryCount);
                         yield this.client.dispatcher.sendStepActionEvent(event);
                     }
                     catch (actionEventError) {
                         this.logger.error(`Could not send completed action event: ${actionEventError.message || actionEventError}`);
-                        // send a failure event
                         const failureEvent = this.getStepActionEvent(action, dispatcher_1.StepActionEventType.STEP_EVENT_TYPE_FAILED, false, actionEventError.message, action.retryCount);
                         try {
                             yield this.client.dispatcher.sendStepActionEvent(failureEvent);
@@ -406,9 +482,7 @@ class InternalWorker {
                         this.logger.error(`Could not send action event: ${actionEventError.message || actionEventError}`);
                     }
                     finally {
-                        // delete the run from the futures
-                        delete this.futures[taskRunExternalId];
-                        delete this.contexts[taskRunExternalId];
+                        this.cleanupRun(actionKey);
                     }
                 });
                 const failure = (error) => __awaiter(this, void 0, void 0, function* () {
@@ -421,7 +495,6 @@ class InternalWorker {
                         if (error.stack) {
                             this.logger.error(error.stack);
                         }
-                        // Send the action event to the dispatcher
                         const event = this.getStepActionEvent(action, dispatcher_1.StepActionEventType.STEP_EVENT_TYPE_FAILED, shouldNotRetry, {
                             message: error === null || error === void 0 ? void 0 : error.message,
                             stack: error === null || error === void 0 ? void 0 : error.stack,
@@ -432,9 +505,7 @@ class InternalWorker {
                         this.logger.error(`Could not send action event: ${e.message}`);
                     }
                     finally {
-                        // delete the run from the futures
-                        delete this.futures[taskRunExternalId];
-                        delete this.contexts[taskRunExternalId];
+                        this.cleanupRun(actionKey);
                     }
                 });
                 const future = new hatchet_promise_1.default((() => __awaiter(this, void 0, void 0, function* () {
@@ -457,7 +528,7 @@ class InternalWorker {
                     (0, abort_error_1.throwIfAborted)(context.abortController.signal);
                     yield success(result);
                 }))());
-                this.futures[taskRunExternalId] = future;
+                this.futures[actionKey] = future;
                 // Send the action event to the dispatcher
                 const event = this.getStepActionEvent(action, dispatcher_1.StepActionEventType.STEP_EVENT_TYPE_STARTED, false, undefined, action.retryCount);
                 this.client.dispatcher.sendStepActionEvent(event).catch((e) => {
@@ -467,87 +538,18 @@ class InternalWorker {
                     yield future.promise;
                 }
                 catch (e) {
-                    const message = (e === null || e === void 0 ? void 0 : e.message) || String(e);
-                    // TODO is this cased correctly...
-                    if (!message.includes('Cancelled')) {
+                    if (!(0, task_run_terminated_error_1.isTaskRunTerminatedError)(e)) {
                         this.logger.error(`Could not wait for task run ${taskRunExternalId} to finish. ` +
                             `See https://docs.hatchet.run/home/cancellation for best practices on handling cancellation: `, e);
                     }
                 }
-            }
-            catch (e) {
-                this.logger.error('Could not send action event (outer): ', e);
-            }
-        });
-    }
-    handleStartGroupKeyRun(action) {
-        return __awaiter(this, void 0, void 0, function* () {
-            const { actionId, getGroupKeyRunId, taskRunExternalId, taskName } = action;
-            this.logger.error('Concurrency Key Functions have been deprecated and will be removed in a future release. Use Concurrency Expressions instead.');
-            try {
-                const context = new context_1.Context(action, this.client, this);
-                const key = getGroupKeyRunId;
-                if (!key) {
-                    this.logger.error(`No group key run id provided for action ${actionId}`);
-                    return;
-                }
-                this.contexts[key] = context;
-                this.logger.debug(`Starting group key run ${key}`);
-                const step = this.action_registry[actionId];
-                if (!step) {
-                    this.logger.error(`Could not find step '${actionId}'`);
-                    return;
+                finally {
+                    this.cleanupRun(actionKey);
                 }
-                const run = () => __awaiter(this, void 0, void 0, function* () {
-                    return step(context);
-                });
-                const success = (result) => {
-                    this.logger.info((0, logger_1.taskRunLog)(taskName, taskRunExternalId, 'completed'));
-                    try {
-                        // Send the action event to the dispatcher
-                        const event = this.getGroupKeyActionEvent(action, dispatcher_1.GroupKeyActionEventType.GROUP_KEY_EVENT_TYPE_COMPLETED, result);
-                        this.client.dispatcher.sendGroupKeyActionEvent(event).catch((e) => {
-                            this.logger.error(`Could not send action event: ${e.message}`);
-                        });
-                    }
-                    catch (e) {
-                        this.logger.error(`Could not send action event: ${e.message}`);
-                    }
-                    finally {
-                        // delete the run from the futures
-                        delete this.futures[key];
-                        delete this.contexts[key];
-                    }
-                };
-                const failure = (error) => {
-                    this.logger.error((0, logger_1.taskRunLog)(taskName, taskRunExternalId, `failed: ${error.message}`));
-                    try {
-                        // Send the action event to the dispatcher
-                        const event = this.getGroupKeyActionEvent(action, dispatcher_1.GroupKeyActionEventType.GROUP_KEY_EVENT_TYPE_FAILED, error);
-                        this.client.dispatcher.sendGroupKeyActionEvent(event).catch((e) => {
-                            this.logger.error(`Could not send action event: ${e.message}`);
-                        });
-                    }
-                    catch (e) {
-                        this.logger.error(`Could not send action event: ${e.message}`);
-                    }
-                    finally {
-                        // delete the run from the futures
-                        delete this.futures[key];
-                        delete this.contexts[key];
-                    }
-                };
-                const future = new hatchet_promise_1.default(run().then(success).catch(failure));
-                this.futures[key] = future;
-                // Send the action event to the dispatcher
-                const event = this.getGroupKeyActionEvent(action, dispatcher_1.GroupKeyActionEventType.GROUP_KEY_EVENT_TYPE_STARTED);
-                this.client.dispatcher.sendGroupKeyActionEvent(event).catch((e) => {
-                    this.logger.error(`Could not send action event: ${e.message}`);
-                });
-                yield future.promise;
             }
             catch (e) {
-                this.logger.error(`Could not send action event: ${e.message}`);
+                this.cleanupRun(actionKey);
+                this.logger.error('Could not send action event (outer): ', e);
             }
         });
     }
@@ -584,11 +586,13 @@ class InternalWorker {
         return __awaiter(this, void 0, void 0, function* () {
             var _a, _b, _c;
             const { taskRunExternalId, taskName } = action;
+            const actionKey = action.key;
             try {
-                const future = this.futures[taskRunExternalId];
-                const context = this.contexts[taskRunExternalId];
+                const future = this.futures[actionKey];
+                const context = this.contexts[actionKey];
+                const cancelErr = new task_run_terminated_error_1.TaskRunTerminatedError('cancelled', 'Cancelled by worker');
                 if (context && context.abortController) {
-                    context.abortController.abort('Cancelled by worker'); // TODO this reason is nonsensical
+                    context.abortController.abort(cancelErr);
                 }
                 if (future) {
                     const start = Date.now();
@@ -638,8 +642,7 @@ class InternalWorker {
                 this.logger.error(`Cancellation: error while supervising cancellation for task run ${taskRunExternalId}: ${(e === null || e === void 0 ? void 0 : e.message) || e}`);
             }
             finally {
-                delete this.futures[taskRunExternalId];
-                delete this.contexts[taskRunExternalId];
+                this.cleanupRun(actionKey);
             }
         });
     }
@@ -654,6 +657,33 @@ class InternalWorker {
             this.killing = true;
             this.setStatus(health_server_1.workerStatus.UNHEALTHY);
             this.logger.info('Starting to exit...');
+            // Pause the worker on the server so it stops receiving new task assignments
+            // before we evict waiting durable runs, mirroring Python's pause_task_assignment().
+            if (this.workerId) {
+                try {
+                    yield this.client.workers.pause(this.workerId);
+                }
+                catch (e) {
+                    this.logger.error(`Could not pause worker: ${e.message}`);
+                }
+            }
+            if (this.evictionManager) {
+                try {
+                    const evicted = yield this.evictionManager.evictAllWaiting();
+                    if (evicted > 0) {
+                        this.logger.info(`Evicted ${evicted} waiting durable run(s) during shutdown`);
+                    }
+                }
+                catch (e) {
+                    this.logger.error(`Could not evict waiting runs: ${e.message}`);
+                }
+            }
+            try {
+                yield this.client.durableListener.stop();
+            }
+            catch (e) {
+                this.logger.error(`Could not stop durable listener: ${e.message}`);
+            }
             try {
                 yield ((_a = this.listener) === null || _a === void 0 ? void 0 : _a.unregister());
             }
@@ -749,20 +779,22 @@ class InternalWorker {
     }
     handleAction(action) {
         return __awaiter(this, void 0, void 0, function* () {
-            const type = action.actionType
-                ? (0, dispatcher_1.actionTypeFromJSON)(action.actionType)
-                : dispatcher_1.ActionType.START_STEP_RUN;
-            if (type === dispatcher_1.ActionType.START_STEP_RUN) {
-                yield this.handleStartStepRun(action);
-            }
-            else if (type === dispatcher_1.ActionType.CANCEL_STEP_RUN) {
-                yield this.handleCancelStepRun(action);
-            }
-            else if (type === dispatcher_1.ActionType.START_GET_GROUP_KEY) {
-                yield this.handleStartGroupKeyRun(action);
-            }
-            else {
-                this.logger.error(`Worker ${this.name} received unknown action type ${type}`);
+            const type = (0, dispatcher_1.actionTypeFromJSON)(action.actionType) || dispatcher_1.ActionType.START_STEP_RUN;
+            switch (type) {
+                case dispatcher_1.ActionType.START_STEP_RUN:
+                    return this.handleStartStepRun(action);
+                case dispatcher_1.ActionType.CANCEL_STEP_RUN:
+                    return this.handleCancelStepRun(action);
+                case dispatcher_1.ActionType.START_GET_GROUP_KEY:
+                    this.logger.error(`Worker ${this.name} received unsupported action type START_GET_GROUP_KEY, please upgrade to V1...`);
+                    return Promise.resolve();
+                case dispatcher_1.ActionType.UNRECOGNIZED:
+                    this.logger.error(`Worker ${this.name} received unrecognized action type ${action.actionType}`);
+                    return Promise.resolve();
+                default: {
+                    const _ = type;
+                    throw new Error(`Unhandled action type: ${_}`);
+                }
             }
         });
     }
@@ -888,3 +920,10 @@ function validateCelExpression(_expr) {
     // For now, we'll just return true to mimic the behavior.
     return true;
 }
+function resolveExecutionTimeout(task, workflowDefaults) {
+    return (0, duration_1.durationToString)(task.executionTimeout || task.timeout || (workflowDefaults === null || workflowDefaults === void 0 ? void 0 : workflowDefaults.executionTimeout) || '60s');
+}
+function resolveScheduleTimeout(task, workflowDefaults) {
+    const value = task.scheduleTimeout || (workflowDefaults === null || workflowDefaults === void 0 ? void 0 : workflowDefaults.scheduleTimeout);
+    return value ? (0, duration_1.durationToString)(value) : undefined;
+}

package/v1/client/worker/worker.d.ts

@@ -61,6 +61,7 @@ export declare class Worker {
      * @returns Promise that resolves when the worker is stopped or killed
      */
     start(): Promise<void>;
+    private _checkEvictionSupport;
     /**
      * Stops the worker
      * @returns Promise that resolves when the worker stops

package/v1/client/worker/worker.js

@@ -14,10 +14,12 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.__testing = exports.Worker = void 0;
 const sleep_1 = __importDefault(require("../../../util/sleep"));
+const declaration_1 = require("../../declaration");
 const legacy_transformer_1 = require("../../../legacy/legacy-transformer");
 const worker_internal_1 = require("./worker-internal");
 const slot_utils_1 = require("./slot-utils");
 const deprecated_1 = require("./deprecated");
+const engine_version_1 = require("./engine-version");
 /**
  * HatchetWorker class for workflow execution runtime
  */
@@ -97,9 +99,35 @@ class Worker {
                 this._legacyWorker = yield deprecated_1.LegacyDualWorker.create(this._v1, this.name, legacyConfig);
                 return this._legacyWorker.start();
             }
+            const engineVersion = yield (0, deprecated_1.fetchEngineVersion)(this._v1).catch(() => undefined);
+            this._checkEvictionSupport(engineVersion);
+            this._internal.engineVersion = engineVersion;
             return this._internal.start();
         });
     }
+    _checkEvictionSupport(engineVersion) {
+        if ((0, engine_version_1.supportsEviction)(engineVersion))
+            return;
+        const workflows = (this.config.workflows || []);
+        const tasksWithEviction = [];
+        for (const wf of workflows) {
+            if (!(wf instanceof declaration_1.BaseWorkflowDeclaration))
+                continue;
+            for (const task of wf.definition._durableTasks) {
+                if (task.evictionPolicy) {
+                    tasksWithEviction.push(`${wf.definition.name}:${task.name}`);
+                }
+            }
+        }
+        if (tasksWithEviction.length === 0)
+            return;
+        const names = tasksWithEviction.join(', ');
+        const logger = this._v1.config.logger('Worker', this._v1.config.log_level);
+        (0, deprecated_1.emitDeprecationNotice)('pre-eviction-engine', `Engine ${engineVersion || 'unknown'} does not support durable eviction ` +
+            `(requires >= ${engine_version_1.MinEngineVersion.DURABLE_EVICTION}). ` +
+            `Eviction policies will be ignored for tasks: ${names}. ` +
+            `Please upgrade your Hatchet engine.`, new Date('2026-03-01T00:00:00Z'), logger);
+    }
     /**
      * Stops the worker
      * @returns Promise that resolves when the worker stops
@@ -163,9 +191,14 @@ class Worker {
             const pollInterval = 200;
             const start = Date.now();
             while (Date.now() - start < timeoutMs) {
-                if ((_a = this._internal) === null || _a === void 0 ? void 0 : _a.workerId) {
+                // start() may asynchronously detect a legacy engine and set _legacyWorker
+                // after waitUntilReady has already entered this loop
+                if (this._legacyWorker) {
+                    yield (0, sleep_1.default)(2000);
                     return;
                 }
+                if ((_a = this._internal) === null || _a === void 0 ? void 0 : _a.workerId)
+                    return;
                 yield (0, sleep_1.default)(pollInterval);
             }
             throw new Error(`Worker ${this.name} did not become ready within ${timeoutMs}ms`);

package/v1/conditions/sleep-condition.js

@@ -1,6 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.SleepCondition = void 0;
+const duration_1 = require("../client/duration");
 const base_1 = require("./base");
 /**
  * Represents a condition that waits for a specified duration before proceeding.
@@ -27,7 +28,7 @@ class SleepCondition extends base_1.Condition {
      */
     constructor(sleepFor, readableDataKey, action) {
         super({
-            readableDataKey: readableDataKey || `sleep-${sleepFor}`,
+            readableDataKey: readableDataKey || `sleep-${(0, duration_1.durationToString)(sleepFor)}`,
             action,
             orGroupId: '',
             expression: '',

package/v1/conditions/transformer.js

@@ -6,6 +6,7 @@ const apply_namespace_1 = require("../../util/apply-namespace");
 const _1 = require(".");
 const base_1 = require("./base");
 const parent_condition_1 = require("./parent-condition");
+const duration_1 = require("../client/duration");
 function taskConditionsToPb(task, namespace) {
     const waitForConditions = (0, _1.Render)(base_1.Action.QUEUE, task.waitFor);
     const cancelIfConditions = (0, _1.Render)(base_1.Action.CANCEL, task.cancelIf);
@@ -21,7 +22,7 @@ function conditionsToPb(conditions, namespace) {
         if (condition instanceof _1.SleepCondition) {
             sleepConditions.push({
                 base: baseToPb(condition.base),
-                sleepFor: condition.sleepFor,
+                sleepFor: (0, duration_1.durationToString)(condition.sleepFor),
             });
         }
         else if (condition instanceof _1.UserEventCondition) {