@hatchet-dev/typescript-sdk 1.30.0 → 1.31.0

package/clients/dispatcher/action-listener.d.ts

@@ -7,7 +7,7 @@ declare enum ListenStrategy {
     LISTEN_STRATEGY_V1 = 1,
     LISTEN_STRATEGY_V2 = 2
 }
-export type ActionKey = `${string}/${number}`;
+export type ActionKey = `${string}/${number}` | `${string}/${number}/${number}`;
 export type Action = AssignedAction & {
     readonly key: ActionKey;
 };

package/clients/dispatcher/action-listener.js

@@ -90,6 +90,12 @@ function createAction(assignedAction) {
     const action = assignedAction;
     Object.defineProperty(action, 'key', {
         get() {
+            // Durable task invocations each get a distinct key so a restored
+            // invocation never collides with the one it replaces in contexts,
+            // futures, or eviction state.
+            if (this.durableTaskInvocationCount !== undefined) {
+                return `${this.taskRunExternalId}/${this.retryCount}/${this.durableTaskInvocationCount}`;
+            }
             return `${this.taskRunExternalId}/${this.retryCount}`;
         },
         enumerable: true,

package/clients/listeners/durable-listener/durable-listener-client.d.ts

@@ -65,7 +65,7 @@ export declare class DurableListenerClient {
     private _requestNotify;
     private _pendingEventAcks;
     private _pendingCallbacks;
-    private _bufferedCompletions;
+    private _orderedCompletions;
     private _pendingEvictionAcks;
     private _receiveAbort;
     private _statusInterval;
@@ -90,6 +90,11 @@ export declare class DurableListenerClient {
     sendEvent(durableTaskExternalId: string, invocationCount: number, event: RunChildrenEvent): Promise<DurableTaskEventRunAck>;
     sendEvent(durableTaskExternalId: string, invocationCount: number, event: WaitForEvent): Promise<DurableTaskEventWaitForAck>;
     sendEvent(durableTaskExternalId: string, invocationCount: number, event: MemoEvent): Promise<DurableTaskEventMemoAck>;
+    private _drainsInProgress;
+    private _drainRerunRequests;
+    private _drainOrderedCompletions;
+    private _drainLoop;
+    consumeCallbackWithoutBlocking(durableTaskExternalId: string, invocationCount: number, branchId: number, nodeId: number): void;
     waitForCallback(durableTaskExternalId: string, invocationCount: number, branchId: number, nodeId: number, opts?: {
         signal?: AbortSignal;
     }): Promise<DurableTaskEventLogEntryResult>;

package/clients/listeners/durable-listener/durable-listener-client.js

@@ -38,6 +38,7 @@ const abort_controller_x_1 = require("abort-controller-x");
 const hatchet_error_1 = require("../../../util/errors/hatchet-error");
 const dispatcher_1 = require("../../../protoc/v1/dispatcher");
 const non_determinism_error_1 = require("../../../util/errors/non-determinism-error");
+const task_run_terminated_error_1 = require("../../../util/errors/task-run-terminated-error");
 const abort_error_1 = require("../../../util/abort-error");
 const sleep_1 = __importDefault(require("../../../util/sleep"));
 const listener_severity_1 = require("../../dispatcher/listener-severity");
@@ -110,6 +111,9 @@ function eventLogEntryResultFromProto(proto) {
 function ackKey(taskExtId, invocationCount) {
     return `${taskExtId}:${invocationCount}`;
 }
+function completionOrderKey(taskExtId, invocationCount) {
+    return `${taskExtId}:${invocationCount}`;
+}
 function callbackKey(taskExtId, invocationCount, branchId, nodeId) {
     return `${taskExtId}:${invocationCount}:${branchId}:${nodeId}`;
 }
@@ -131,13 +135,15 @@ class DurableListenerClient {
         this._requestQueue = [];
         this._pendingEventAcks = new Map();
         this._pendingCallbacks = new Map();
-        // Completions that arrived before waitForCallback() registered a deferred
-        // in _pendingCallbacks. This happens when the server delivers an
-        // entryCompleted between the event ack and the waitForCallback call
-        // (e.g. an already-satisfied sleep delivered via polling).
-        this._bufferedCompletions = new TTLMap(10000);
+        // Completions held in server delivery order until their waiters can
+        // consume them without overtaking an earlier-delivered completion.
+        // The TTL only garbage-collects queues for invocations that died
+        // without consuming everything; the server stall-evicts long before.
+        this._orderedCompletions = new TTLMap(300000);
         this._pendingEvictionAcks = new Map();
         this._consecutiveFailures = 0;
+        this._drainsInProgress = new Set();
+        this._drainRerunRequests = new Set();
         this.config = config;
         this.client = factory.create(dispatcher_1.V1DispatcherDefinition, channel);
         this.logger = config.logger(`DurableListener`, config.log_level);
@@ -185,7 +191,7 @@ class DurableListenerClient {
                 this._receiveAbort.abort();
             }
             this._failPendingAcks(new Error('DurableListener stopped'));
-            this._bufferedCompletions.destroy();
+            this._orderedCompletions.destroy();
         });
     }
     _connect() {
@@ -332,10 +338,10 @@ class DurableListenerClient {
             d.reject(exc);
         }
         this._pendingCallbacks.clear();
-        this._bufferedCompletions.clear();
+        this._orderedCompletions.clear();
     }
     _handleResponse(response) {
-        var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o, _p, _q, _r, _s, _t, _u, _v, _w, _x, _y, _z, _0, _1, _2;
+        var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o, _p, _q, _r, _s, _t, _u, _v, _w, _x, _y, _z, _0, _1, _2, _3, _4, _5;
         if (response.registerWorker) {
             // registration acknowledged
         }
@@ -396,14 +402,28 @@ class DurableListenerClient {
             const { ref } = completed;
             const key = callbackKey((_o = ref === null || ref === void 0 ? void 0 : ref.durableTaskExternalId) !== null && _o !== void 0 ? _o : '', (_p = ref === null || ref === void 0 ? void 0 : ref.invocationCount) !== null && _p !== void 0 ? _p : 0, (_q = ref === null || ref === void 0 ? void 0 : ref.branchId) !== null && _q !== void 0 ? _q : 0, (_r = ref === null || ref === void 0 ? void 0 : ref.nodeId) !== null && _r !== void 0 ? _r : 0);
             const result = eventLogEntryResultFromProto(completed);
-            const pending = this._pendingCallbacks.get(key);
-            if (pending) {
-                pending.resolve(result);
-                this._pendingCallbacks.delete(key);
-            }
-            else {
-                this._bufferedCompletions.set(key, result);
+            const orderKey = completionOrderKey((_s = ref === null || ref === void 0 ? void 0 : ref.durableTaskExternalId) !== null && _s !== void 0 ? _s : '', (_t = ref === null || ref === void 0 ? void 0 : ref.invocationCount) !== null && _t !== void 0 ? _t : 0);
+            const queue = (_u = this._orderedCompletions.get(orderKey)) !== null && _u !== void 0 ? _u : {
+                pending: [],
+                delivered: new Set(),
+            };
+            if (!queue.delivered.has(key)) {
+                queue.delivered.add(key);
+                queue.pending.push({ key, result });
+            }
+            else if (queue.pending.every((entry) => entry.key !== key)) {
+                // Re-delivery of a completion that already drained (reconnect,
+                // worker-status re-send, or a repeated wait on a node deduped by
+                // child key). Its satisfied order was released before anything
+                // still queued, so handing it straight to a waiter keeps order.
+                const redelivered = this._pendingCallbacks.get(key);
+                if (redelivered) {
+                    this._pendingCallbacks.delete(key);
+                    redelivered.resolve(result);
+                }
             }
+            this._orderedCompletions.set(orderKey, queue);
+            this._drainOrderedCompletions(orderKey);
         }
         else if (response.evictionAck) {
             const ack = response.evictionAck;
@@ -418,34 +438,36 @@ class DurableListenerClient {
             const evict = response.serverEvict;
             this.logger.info(`received server eviction notification for task ${evict.durableTaskExternalId} ` +
                 `invocation ${evict.invocationCount}: ${evict.reason}`);
-            this.cleanupTaskState(evict.durableTaskExternalId, evict.invocationCount);
+            // onServerEvict aborts the run first so waiters settle as aborted
+            // (eviction) rather than with cleanup's generic rejection.
             if (this.onServerEvict) {
                 this.onServerEvict(evict.durableTaskExternalId, evict.invocationCount);
             }
+            this.cleanupTaskState(evict.durableTaskExternalId, evict.invocationCount);
         }
         else if (response.error) {
             const { error } = response;
             const { ref } = error;
             let exc;
             if (error.errorType === dispatcher_1.DurableTaskErrorType.DURABLE_TASK_ERROR_TYPE_NONDETERMINISM) {
-                exc = new non_determinism_error_1.NonDeterminismError((_s = ref === null || ref === void 0 ? void 0 : ref.durableTaskExternalId) !== null && _s !== void 0 ? _s : '', (_t = ref === null || ref === void 0 ? void 0 : ref.invocationCount) !== null && _t !== void 0 ? _t : 0, (_u = ref === null || ref === void 0 ? void 0 : ref.nodeId) !== null && _u !== void 0 ? _u : 0, error.errorMessage);
+                exc = new non_determinism_error_1.NonDeterminismError((_v = ref === null || ref === void 0 ? void 0 : ref.durableTaskExternalId) !== null && _v !== void 0 ? _v : '', (_w = ref === null || ref === void 0 ? void 0 : ref.invocationCount) !== null && _w !== void 0 ? _w : 0, (_x = ref === null || ref === void 0 ? void 0 : ref.nodeId) !== null && _x !== void 0 ? _x : 0, error.errorMessage);
             }
             else {
                 exc = new Error(`Unspecified durable task error: ${error.errorMessage} (type: ${error.errorType})`);
             }
-            const eAckKey = ackKey((_v = ref === null || ref === void 0 ? void 0 : ref.durableTaskExternalId) !== null && _v !== void 0 ? _v : '', (_w = ref === null || ref === void 0 ? void 0 : ref.invocationCount) !== null && _w !== void 0 ? _w : 0);
+            const eAckKey = ackKey((_y = ref === null || ref === void 0 ? void 0 : ref.durableTaskExternalId) !== null && _y !== void 0 ? _y : '', (_z = ref === null || ref === void 0 ? void 0 : ref.invocationCount) !== null && _z !== void 0 ? _z : 0);
             const pendingAck = this._pendingEventAcks.get(eAckKey);
             if (pendingAck) {
                 pendingAck.reject(exc);
                 this._pendingEventAcks.delete(eAckKey);
             }
-            const eCbKey = callbackKey((_x = ref === null || ref === void 0 ? void 0 : ref.durableTaskExternalId) !== null && _x !== void 0 ? _x : '', (_y = ref === null || ref === void 0 ? void 0 : ref.invocationCount) !== null && _y !== void 0 ? _y : 0, (_z = ref === null || ref === void 0 ? void 0 : ref.branchId) !== null && _z !== void 0 ? _z : 0, (_0 = ref === null || ref === void 0 ? void 0 : ref.nodeId) !== null && _0 !== void 0 ? _0 : 0);
+            const eCbKey = callbackKey((_0 = ref === null || ref === void 0 ? void 0 : ref.durableTaskExternalId) !== null && _0 !== void 0 ? _0 : '', (_1 = ref === null || ref === void 0 ? void 0 : ref.invocationCount) !== null && _1 !== void 0 ? _1 : 0, (_2 = ref === null || ref === void 0 ? void 0 : ref.branchId) !== null && _2 !== void 0 ? _2 : 0, (_3 = ref === null || ref === void 0 ? void 0 : ref.nodeId) !== null && _3 !== void 0 ? _3 : 0);
             const pendingCb = this._pendingCallbacks.get(eCbKey);
             if (pendingCb) {
                 pendingCb.reject(exc);
                 this._pendingCallbacks.delete(eCbKey);
             }
-            const eEvKey = evictionKey((_1 = ref === null || ref === void 0 ? void 0 : ref.durableTaskExternalId) !== null && _1 !== void 0 ? _1 : '', (_2 = ref === null || ref === void 0 ? void 0 : ref.invocationCount) !== null && _2 !== void 0 ? _2 : 0);
+            const eEvKey = evictionKey((_4 = ref === null || ref === void 0 ? void 0 : ref.durableTaskExternalId) !== null && _4 !== void 0 ? _4 : '', (_5 = ref === null || ref === void 0 ? void 0 : ref.invocationCount) !== null && _5 !== void 0 ? _5 : 0);
             const pendingEv = this._pendingEvictionAcks.get(eEvKey);
             if (pendingEv) {
                 pendingEv.reject(exc);
@@ -498,26 +520,87 @@ class DurableListenerClient {
             return d.promise;
         });
     }
+    // Hands queued completions to their waiters strictly in delivery order.
+    // Stops at the first completion whose waiter has not registered yet:
+    // releasing a later completion first would resume its coroutine ahead of
+    // the recorded order and diverge the re-emitted event sequence.
+    _drainOrderedCompletions(orderKey) {
+        if (this._drainsInProgress.has(orderKey)) {
+            this._drainRerunRequests.add(orderKey);
+            return;
+        }
+        this._drainsInProgress.add(orderKey);
+        void this._drainLoop(orderKey).finally(() => {
+            this._drainsInProgress.delete(orderKey);
+            if (this._drainRerunRequests.delete(orderKey)) {
+                this._drainOrderedCompletions(orderKey);
+            }
+        });
+    }
+    _drainLoop(orderKey) {
+        return __awaiter(this, void 0, void 0, function* () {
+            let released = false;
+            while (true) {
+                if (released) {
+                    // A macrotask boundary between releases lets the just-resumed waiter
+                    // run to its next suspension point (emitting its next durable event)
+                    // before the following completion is released. Resolving several
+                    // deferreds back-to-back would instead resume them in the order their
+                    // continuations happened to be attached, not in release order.
+                    yield new Promise((resolve) => {
+                        setImmediate(resolve);
+                    });
+                }
+                const queue = this._orderedCompletions.get(orderKey);
+                if (!queue || queue.pending.length === 0)
+                    return;
+                const [head] = queue.pending;
+                const waiter = this._pendingCallbacks.get(head.key);
+                if (!waiter)
+                    return;
+                queue.pending.shift();
+                this._pendingCallbacks.delete(head.key);
+                waiter.resolve(head.result);
+                released = true;
+            }
+        });
+    }
+    // A memo's value arrives in its ack and its completion is never awaited,
+    // but the server still delivers that completion on replay, in recorded
+    // order. Register a waiter that nothing blocks on so the drain can hand the
+    // completion through instead of stalling the queue at it forever.
+    consumeCallbackWithoutBlocking(durableTaskExternalId, invocationCount, branchId, nodeId) {
+        const key = callbackKey(durableTaskExternalId, invocationCount, branchId, nodeId);
+        if (this._pendingCallbacks.has(key))
+            return;
+        const d = deferred();
+        d.promise.catch(() => { });
+        this._pendingCallbacks.set(key, d);
+        this._drainOrderedCompletions(completionOrderKey(durableTaskExternalId, invocationCount));
+    }
     waitForCallback(durableTaskExternalId, invocationCount, branchId, nodeId, opts) {
         return __awaiter(this, void 0, void 0, function* () {
-            const key = callbackKey(durableTaskExternalId, invocationCount, branchId, nodeId);
-            const early = this._bufferedCompletions.get(key);
-            if (early) {
-                this._bufferedCompletions.delete(key);
-                return early;
+            const signal = opts === null || opts === void 0 ? void 0 : opts.signal;
+            if (signal === null || signal === void 0 ? void 0 : signal.aborted) {
+                return Promise.reject((0, abort_error_1.createAbortError)('Operation cancelled by AbortSignal'));
             }
-            if (!this._pendingCallbacks.has(key)) {
-                this._pendingCallbacks.set(key, deferred());
-                this._pollWorkerStatus();
+            const key = callbackKey(durableTaskExternalId, invocationCount, branchId, nodeId);
+            let d = this._pendingCallbacks.get(key);
+            if (!d) {
+                d = deferred();
+                // A deferred can outlive its waiters (eviction aborts the caller while
+                // the entry stays registered); a later cleanupTaskState rejection must
+                // not crash the process as an unhandled rejection.
+                d.promise.catch(() => { });
+                this._pendingCallbacks.set(key, d);
+                this._drainOrderedCompletions(completionOrderKey(durableTaskExternalId, invocationCount));
+                if (this._pendingCallbacks.has(key)) {
+                    this._pollWorkerStatus();
+                }
             }
-            const d = this._pendingCallbacks.get(key);
-            const signal = opts === null || opts === void 0 ? void 0 : opts.signal;
             if (!signal) {
                 return d.promise;
             }
-            if (signal.aborted) {
-                return Promise.reject((0, abort_error_1.createAbortError)('Operation cancelled by AbortSignal'));
-            }
             return new Promise((resolve, reject) => {
                 let settled = false;
                 const onAbort = () => {
@@ -544,24 +627,28 @@ class DurableListenerClient {
         });
     }
     cleanupTaskState(durableTaskExternalId, invocationCount) {
+        // Rejecting with TaskRunTerminatedError marks any coroutine still blocked
+        // on this invocation's state as evicted rather than failed, matching
+        // Python's future cancellation on cleanup.
+        const evicted = () => new task_run_terminated_error_1.TaskRunTerminatedError('evicted', 'task state cleaned up');
         for (const [k, d] of this._pendingCallbacks) {
             const parts = k.split(':');
             if (parts[0] === durableTaskExternalId && parseInt(parts[1], 10) <= invocationCount) {
-                d.reject(new Error('task state cleaned up'));
+                d.reject(evicted());
                 this._pendingCallbacks.delete(k);
             }
         }
         for (const [k, d] of this._pendingEventAcks) {
             const parts = k.split(':');
             if (parts[0] === durableTaskExternalId && parseInt(parts[1], 10) <= invocationCount) {
-                d.reject(new Error('task state cleaned up'));
+                d.reject(evicted());
                 this._pendingEventAcks.delete(k);
             }
         }
-        for (const k of this._bufferedCompletions.keys()) {
+        for (const k of this._orderedCompletions.keys()) {
             const parts = k.split(':');
             if (parts[0] === durableTaskExternalId && parseInt(parts[1], 10) <= invocationCount) {
-                this._bufferedCompletions.delete(k);
+                this._orderedCompletions.delete(k);
             }
         }
     }

package/clients/rest/generated/Api.d.ts

@@ -350,7 +350,10 @@ export declare class Api<SecurityDataType = unknown> extends HttpClient<Security
      * @request GET:/api/v1/stable/workflow-runs/{v1-workflow-run}
      * @secure
      */
-    v1WorkflowRunGet: (v1WorkflowRun: string, params?: RequestParams) => Promise<import("axios").AxiosResponse<V1WorkflowRunDetails, any, {}>>;
+    v1WorkflowRunGet: (v1WorkflowRun: string, query?: {
+        /** Whether to include the DAG orchestrator's task events, which are hidden by default */
+        includeOrchestratorEvents?: boolean;
+    }, params?: RequestParams) => Promise<import("axios").AxiosResponse<V1WorkflowRunDetails, any, {}>>;
     /**
      * @description Get the status of a workflow run.
      *
@@ -2024,6 +2027,8 @@ export declare class Api<SecurityDataType = unknown> extends HttpClient<Security
         statuses?: WorkerStatus[];
         /** Filter by worker labels */
         labels?: string[];
+        /** Whether to include engine-managed operator workers, which are hidden by default */
+        includeOperators?: boolean;
     }, params?: RequestParams) => Promise<import("axios").AxiosResponse<WorkerList, any, {}>>;
     /**
      * @description Update a worker

package/clients/rest/generated/Api.js

@@ -175,7 +175,7 @@ class Api extends http_client_1.HttpClient {
          * @request GET:/api/v1/stable/workflow-runs/{v1-workflow-run}
          * @secure
          */
-        this.v1WorkflowRunGet = (v1WorkflowRun, params = {}) => this.request(Object.assign({ path: `/api/v1/stable/workflow-runs/${v1WorkflowRun}`, method: 'GET', secure: true, format: 'json' }, params));
+        this.v1WorkflowRunGet = (v1WorkflowRun, query, params = {}) => this.request(Object.assign({ path: `/api/v1/stable/workflow-runs/${v1WorkflowRun}`, method: 'GET', query: query, secure: true, format: 'json' }, params));
         /**
          * @description Get the status of a workflow run.
          *

package/clients/rest/generated/data-contracts.d.ts

@@ -25,7 +25,8 @@ export declare enum PullRequestState {
 export declare enum FeatureFlagId {
     TenantLogWorkflowFilterEnabled = "tenant-log-workflow-filter-enabled",
     TraceMinimapEnabled = "trace-minimap-enabled",
-    OrganizationSsoEnabled = "organization-sso-enabled"
+    OrganizationSsoEnabled = "organization-sso-enabled",
+    OperatorDetailsEnabled = "operator-details-enabled"
 }
 export declare enum WebhookWorkerRequestMethod {
     GET = "GET",
@@ -2328,6 +2329,13 @@ export interface Worker {
      */
     webhookId?: string;
     runtimeInfo?: WorkerRuntimeInfo;
+    /**
+     * The id of the operator that owns this worker, if it is an engine-managed operator worker.
+     * @format uuid
+     */
+    operatorId?: string;
+    /** The number of durable task runs owned by this operator that are currently evicted while waiting on durable events. Evicted runs hold no slots. Only set for operator workers. */
+    evictedDurableTaskCount?: number;
 }
 export interface WorkerList {
     pagination?: PaginationResponse;

package/clients/rest/generated/data-contracts.js

@@ -46,6 +46,7 @@ var FeatureFlagId;
     FeatureFlagId["TenantLogWorkflowFilterEnabled"] = "tenant-log-workflow-filter-enabled";
     FeatureFlagId["TraceMinimapEnabled"] = "trace-minimap-enabled";
     FeatureFlagId["OrganizationSsoEnabled"] = "organization-sso-enabled";
+    FeatureFlagId["OperatorDetailsEnabled"] = "operator-details-enabled";
 })(FeatureFlagId || (exports.FeatureFlagId = FeatureFlagId = {}));
 var WebhookWorkerRequestMethod;
 (function (WebhookWorkerRequestMethod) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hatchet-dev/typescript-sdk",
-  "version": "1.30.0",
+  "version": "1.31.0",
   "description": "Background task orchestration & visibility for developers",
   "types": "dist/index.d.ts",
   "files": [

package/protoc/v1/workflows.d.ts

@@ -170,6 +170,15 @@ export interface DefaultFilter {
     /** (optional) the payload for the filter, if any. A JSON object as a string. */
     payload?: Uint8Array | undefined;
 }
+/**
+ * Concurrency declares one entry in a concurrency chain. Entries are processed in array
+ * order, so a tenant-scoped entry may come before or after a workflow-scoped one.
+ *
+ * A tenant-scoped entry (is_tenant_scoped = true) defines (or updates in place) a strategy
+ * shared across workflows, keyed by name: every task declaring the same name consumes the
+ * same concurrency limit. Registrations whose chains order the same tenant-scoped
+ * strategies inconsistently are rejected, since inconsistent orders can deadlock.
+ */
 export interface Concurrency {
     /** (required) the expression to use for concurrency */
     expression: string;
@@ -177,6 +186,12 @@ export interface Concurrency {
     maxRuns?: number | undefined;
     /** (optional) the strategy to use when the concurrency limit is reached, default CANCEL_IN_PROGRESS */
     limitStrategy?: ConcurrencyLimitStrategy | undefined;
+    /** (required when is_tenant_scoped) the strategy name; unique per tenant for tenant-scoped strategies */
+    name?: string | undefined;
+    /** (optional) when true, the entry is a tenant-scoped strategy shared across workflows, default false */
+    isTenantScoped?: boolean | undefined;
+    /** (optional) CEL expression over task input returning the max runs for that task's concurrency group; the group's effective limit is the value from its most recently created task. Overrides max_runs per group; a non-integer or negative result fails the task, and 0 holds the group until a newer task raises the limit */
+    maxRunsExpression?: string | undefined;
 }
 export interface TaskBatchConfig {
     /** (required) maximum items per batch */

package/protoc/v1/workflows.js

@@ -1794,7 +1794,14 @@ exports.DefaultFilter = {
     },
 };
 function createBaseConcurrency() {
-    return { expression: '', maxRuns: undefined, limitStrategy: undefined };
+    return {
+        expression: '',
+        maxRuns: undefined,
+        limitStrategy: undefined,
+        name: undefined,
+        isTenantScoped: undefined,
+        maxRunsExpression: undefined,
+    };
 }
 exports.Concurrency = {
     encode(message, writer = new wire_1.BinaryWriter()) {
@@ -1807,6 +1814,15 @@ exports.Concurrency = {
         if (message.limitStrategy !== undefined) {
             writer.uint32(24).int32(message.limitStrategy);
         }
+        if (message.name !== undefined) {
+            writer.uint32(34).string(message.name);
+        }
+        if (message.isTenantScoped !== undefined) {
+            writer.uint32(40).bool(message.isTenantScoped);
+        }
+        if (message.maxRunsExpression !== undefined) {
+            writer.uint32(50).string(message.maxRunsExpression);
+        }
         return writer;
     },
     decode(input, length) {
@@ -1837,6 +1853,27 @@ exports.Concurrency = {
                     message.limitStrategy = reader.int32();
                     continue;
                 }
+                case 4: {
+                    if (tag !== 34) {
+                        break;
+                    }
+                    message.name = reader.string();
+                    continue;
+                }
+                case 5: {
+                    if (tag !== 40) {
+                        break;
+                    }
+                    message.isTenantScoped = reader.bool();
+                    continue;
+                }
+                case 6: {
+                    if (tag !== 50) {
+                        break;
+                    }
+                    message.maxRunsExpression = reader.string();
+                    continue;
+                }
             }
             if ((tag & 7) === 4 || tag === 0) {
                 break;
@@ -1858,6 +1895,17 @@ exports.Concurrency = {
                 : isSet(object.limit_strategy)
                     ? concurrencyLimitStrategyFromJSON(object.limit_strategy)
                     : undefined,
+            name: isSet(object.name) ? globalThis.String(object.name) : undefined,
+            isTenantScoped: isSet(object.isTenantScoped)
+                ? globalThis.Boolean(object.isTenantScoped)
+                : isSet(object.is_tenant_scoped)
+                    ? globalThis.Boolean(object.is_tenant_scoped)
+                    : undefined,
+            maxRunsExpression: isSet(object.maxRunsExpression)
+                ? globalThis.String(object.maxRunsExpression)
+                : isSet(object.max_runs_expression)
+                    ? globalThis.String(object.max_runs_expression)
+                    : undefined,
         };
     },
     toJSON(message) {
@@ -1871,17 +1919,29 @@ exports.Concurrency = {
         if (message.limitStrategy !== undefined) {
             obj.limitStrategy = concurrencyLimitStrategyToJSON(message.limitStrategy);
         }
+        if (message.name !== undefined) {
+            obj.name = message.name;
+        }
+        if (message.isTenantScoped !== undefined) {
+            obj.isTenantScoped = message.isTenantScoped;
+        }
+        if (message.maxRunsExpression !== undefined) {
+            obj.maxRunsExpression = message.maxRunsExpression;
+        }
         return obj;
     },
     create(base) {
         return exports.Concurrency.fromPartial(base !== null && base !== void 0 ? base : {});
     },
     fromPartial(object) {
-        var _a, _b, _c;
+        var _a, _b, _c, _d, _e, _f;
         const message = createBaseConcurrency();
         message.expression = (_a = object.expression) !== null && _a !== void 0 ? _a : '';
         message.maxRuns = (_b = object.maxRuns) !== null && _b !== void 0 ? _b : undefined;
         message.limitStrategy = (_c = object.limitStrategy) !== null && _c !== void 0 ? _c : undefined;
+        message.name = (_d = object.name) !== null && _d !== void 0 ? _d : undefined;
+        message.isTenantScoped = (_e = object.isTenantScoped) !== null && _e !== void 0 ? _e : undefined;
+        message.maxRunsExpression = (_f = object.maxRunsExpression) !== null && _f !== void 0 ? _f : undefined;
         return message;
     },
 };

package/v1/client/worker/context.d.ts

@@ -381,6 +381,8 @@ export declare class DurableContext<T, K = {}> extends Context<T, K> {
     private _evictionManager;
     private _engineVersion;
     private _waitKey;
+    private _sendEventLock;
+    private _serializeSendEvent;
     constructor(action: Action, v1: HatchetClient, worker: InternalWorker, durableListener: DurableListenerClient, evictionManager?: DurableEvictionManager, engineVersion?: string);
     get supportsEviction(): boolean;
     get durableListener(): DurableListenerClient;

package/v1/client/worker/context.js

@@ -692,9 +692,27 @@ exports.Context = Context;
  * It extends the Context class and includes additional methods for durable execution like sleepFor and waitFor.
  */
 class DurableContext extends Context {
+    _serializeSendEvent(send) {
+        // Re-check cancellation when the queued send actually executes: once this
+        // invocation is aborted (evicted), a queued send must not reach the
+        // server, where it could race the eviction and append to the event log
+        // out of recorded order. Python gets this for free from task
+        // cancellation killing coroutines queued on the send lock.
+        const runSend = () => {
+            this.throwIfCancelled();
+            return send();
+        };
+        const result = this._sendEventLock.then(runSend, runSend);
+        this._sendEventLock = result.then(() => undefined, () => undefined);
+        return result;
+    }
     constructor(action, v1, worker, durableListener, evictionManager, engineVersion) {
         super(action, v1, worker);
         this._waitKey = 0;
+        // Serializes sendEvent calls from concurrent coroutines in this invocation.
+        // The listener keys pending acks by (task, invocation), so overlapping sends
+        // would overwrite each other's ack and cross-wire branch/node assignments.
+        this._sendEventLock = Promise.resolve();
         this._durableListener = durableListener;
         this._evictionManager = evictionManager;
         this._engineVersion = engineVersion;
@@ -767,14 +785,14 @@ class DurableContext extends Context {
             }
             const rendered = (0, conditions_1.Render)(condition_1.Action.CREATE, conditions);
             const pbConditions = (0, transformer_1.conditionsToPb)(rendered, this.v1.config.namespace);
-            const ack = yield this._durableListener.sendEvent(this.action.taskRunExternalId, this.invocationCount, {
+            const ack = yield this._serializeSendEvent(() => this._durableListener.sendEvent(this.action.taskRunExternalId, this.invocationCount, {
                 kind: 'waitFor',
                 waitForConditions: {
                     sleepConditions: pbConditions.sleepConditions,
                     userEventConditions: pbConditions.userEventConditions,
                 },
                 label,
-            });
+            }));
             const resourceId = rendered
                 .map((c) => c.base.readableDataKey)
                 .filter(Boolean)
@@ -787,9 +805,8 @@ class DurableContext extends Context {
     }
     waitForEvent(key, expression, payloadSchema, scope, lookbackWindow, label) {
         return __awaiter(this, void 0, void 0, function* () {
-            const now = yield this.now();
             const considerEventsSince = lookbackWindow
-                ? new Date(now.getTime() - (0, duration_1.durationToMs)(lookbackWindow)).toISOString()
+                ? new Date((yield this.now()).getTime() - (0, duration_1.durationToMs)(lookbackWindow)).toISOString()
                 : undefined;
             const res = yield this.waitFor({
                 eventKey: key,
@@ -918,10 +935,10 @@ class DurableContext extends Context {
                 const { triggerOpts } = this._buildTriggerOpts(child.workflow, child.input, child.options);
                 return triggerOpts;
             });
-            const ack = yield this._durableListener.sendEvent(this.action.taskRunExternalId, this.invocationCount, {
+            const ack = yield this._serializeSendEvent(() => this._durableListener.sendEvent(this.action.taskRunExternalId, this.invocationCount, {
                 kind: 'runChildren',
                 triggerOpts: triggerOptsList,
-            });
+            }));
             const results = yield Promise.all(ack.runEntries.map((entry) => this.withEvictionWait('runChild', `workflow:bulk-child`, () => __awaiter(this, void 0, void 0, function* () {
                 const result = yield this._durableListener.waitForCallback(this.action.taskRunExternalId, this.invocationCount, entry.branchId, entry.nodeId, { signal: this.abortController.signal });
                 if (result.isFailure) {
@@ -946,10 +963,11 @@ class DurableContext extends Context {
                 return fn();
             }
             const memoKey = computeMemoKey(this.action.taskRunExternalId, deps);
-            const ack = yield this._durableListener.sendEvent(this.action.taskRunExternalId, this.invocationCount, {
+            const ack = yield this._serializeSendEvent(() => this._durableListener.sendEvent(this.action.taskRunExternalId, this.invocationCount, {
                 kind: 'memo',
                 memoKey,
-            });
+            }));
+            this._durableListener.consumeCallbackWithoutBlocking(this.action.taskRunExternalId, this.invocationCount, ack.branchId, ack.nodeId);
             if (ack.memoAlreadyExisted && ack.memoResultPayload && ack.memoResultPayload.length > 0) {
                 const serialized = new TextDecoder().decode(ack.memoResultPayload);
                 return JSON.parse(serialized);

package/v1/client/worker/eviction/eviction-manager.d.ts

@@ -21,7 +21,7 @@ export declare class DurableEvictionManager {
     private _ticking;
     constructor(opts: {
         durableSlots: number;
-        cancelLocal: (key: ActionKey) => void;
+        cancelLocal: (key: ActionKey, invocationCount: number) => void;
         requestEvictionWithAck: (key: ActionKey, rec: DurableRunRecord) => Promise<void>;
         config?: DurableEvictionConfig;
         cache?: DurableEvictionCache;