@hatchet-dev/typescript-sdk 1.34.0 → 1.34.1

package/clients/dispatcher/action-listener.d.ts

@@ -28,5 +28,6 @@ export declare class ActionListener {
     getListenStrategy(): Promise<ListenStrategy>;
     incrementRetries(): Promise<void>;
     getListenClient(): Promise<AsyncIterable<AssignedAction>>;
+    stopStream(): void;
     unregister(): Promise<import("../../protoc/dispatcher").WorkerUnsubscribeResponse>;
 }

package/clients/dispatcher/action-listener.js

@@ -225,14 +225,19 @@ class ActionListener {
             }
         });
     }
+    // Leaves the heartbeat running: tasks that are already in flight still need
+    // the engine to see this worker as alive until they finish.
+    stopStream() {
+        this.done = true;
+        // Abort the gRPC stream to immediately cancel the generator
+        if (this.abortController) {
+            this.abortController.abort('Worker stopping');
+        }
+    }
     unregister() {
         return __awaiter(this, void 0, void 0, function* () {
-            this.done = true;
+            this.stopStream();
             this.heartbeat.stop();
-            // Abort the gRPC stream to immediately cancel the generator
-            if (this.abortController) {
-                this.abortController.abort('Worker stopping');
-            }
             try {
                 return yield this.client.unsubscribe({
                     workerId: this.workerId,

package/clients/listeners/durable-listener/durable-listener-client.d.ts

@@ -18,6 +18,9 @@ export declare class DurableListenerClient {
     private _pendingCallbacks;
     private _orderedCompletions;
     private _pendingEvictionAcks;
+    private _newlyWaitingKeys;
+    private _waitingSince;
+    private _workerStatusSendScheduled;
     private _receiveAbort;
     private _statusInterval;
     private _startLock;
@@ -34,13 +37,20 @@ export declare class DurableListenerClient {
     private _requestIterator;
     private _enqueueRequest;
     private _startStatusPolling;
-    private _pollWorkerStatus;
+    private _registerWaitingKey;
+    private _scheduleWorkerStatusSend;
+    private _enqueueWorkerStatusForNewlyWaitingKeys;
+    private _enqueueWorkerStatusForLongPendingWaiters;
+    private _enqueueFullWorkerStatus;
+    private _enqueueWorkerStatusFor;
     private _failPendingAcks;
     private _failAllPending;
     private _handleResponse;
     sendEvent(durableTaskExternalId: string, invocationCount: number, event: RunChildrenEvent): Promise<DurableTaskEventRunAck>;
     sendEvent(durableTaskExternalId: string, invocationCount: number, event: WaitForEvent): Promise<DurableTaskEventWaitForAck>;
     sendEvent(durableTaskExternalId: string, invocationCount: number, event: MemoEvent): Promise<DurableTaskEventMemoAck>;
+    private _sendRequestAndAwaitAck;
+    private _sendRunChildrenEvent;
     private _drainsInProgress;
     private _drainRerunRequests;
     private _drainOrderedCompletions;

package/clients/listeners/durable-listener/durable-listener-client.js

@@ -94,7 +94,10 @@ class TTLMap {
 }
 const DEFAULT_RECONNECT_INTERVAL = 3000;
 const EVICTION_ACK_TIMEOUT_MS = 30000;
-const WORKER_STATUS_POLL_INTERVAL_MS = 1000;
+const WORKER_STATUS_RESEND_INTERVAL_MS = 5000;
+const WORKER_STATUS_RESEND_MIN_PENDING_MS = 2000;
+const WORKER_STATUS_MAX_ENTRIES_PER_REQUEST = 10000;
+const RUN_CHILDREN_MAX_CHILDREN_PER_REQUEST = 100;
 function eventLogEntryResultFromProto(proto) {
     var _a, _b, _c, _d;
     let payload;
@@ -109,6 +112,15 @@ function eventLogEntryResultFromProto(proto) {
         errorMessage: proto.errorMessage,
     };
 }
+function awaitedCompletedEntryFromCallbackKey(key) {
+    const [durableTaskExternalId, invocationCount, branchId, nodeId] = key.split(':');
+    return {
+        durableTaskExternalId,
+        invocationCount: parseInt(invocationCount, 10),
+        branchId: parseInt(branchId, 10),
+        nodeId: parseInt(nodeId, 10),
+    };
+}
 function ackKey(taskExtId, invocationCount) {
     return `${taskExtId}:${invocationCount}`;
 }
@@ -142,6 +154,9 @@ class DurableListenerClient {
         // without consuming everything; the server stall-evicts long before.
         this._orderedCompletions = new TTLMap(300000);
         this._pendingEvictionAcks = new Map();
+        this._newlyWaitingKeys = new Set();
+        this._waitingSince = new Map();
+        this._workerStatusSendScheduled = false;
         this._consecutiveFailures = 0;
         this._drainsInProgress = new Set();
         this._drainRerunRequests = new Set();
@@ -205,7 +220,7 @@ class DurableListenerClient {
             this._enqueueRequest({
                 registerWorker: { workerId: this._workerId },
             });
-            this._pollWorkerStatus();
+            this._enqueueFullWorkerStatus();
             void this._streamLoop();
             this.logger.info('durable event listener connected');
         });
@@ -300,28 +315,57 @@ class DurableListenerClient {
             clearInterval(this._statusInterval);
         }
         this._statusInterval = setInterval(() => {
-            this._pollWorkerStatus();
-        }, WORKER_STATUS_POLL_INTERVAL_MS);
+            this._enqueueWorkerStatusForLongPendingWaiters();
+        }, WORKER_STATUS_RESEND_INTERVAL_MS);
     }
-    _pollWorkerStatus() {
-        if (!this._workerId || this._pendingCallbacks.size === 0)
+    _registerWaitingKey(key) {
+        this._waitingSince.set(key, Date.now());
+        this._newlyWaitingKeys.add(key);
+    }
+    // Waiters registered in the same turn of the event loop share one request,
+    // instead of each sending its own.
+    _scheduleWorkerStatusSend() {
+        if (this._workerStatusSendScheduled)
             return;
-        const waitingEntries = [];
-        for (const key of this._pendingCallbacks.keys()) {
-            const parts = key.split(':');
-            waitingEntries.push({
-                durableTaskExternalId: parts[0],
-                invocationCount: parseInt(parts[1], 10),
-                branchId: parseInt(parts[2], 10),
-                nodeId: parseInt(parts[3], 10),
+        this._workerStatusSendScheduled = true;
+        setImmediate(() => this._enqueueWorkerStatusForNewlyWaitingKeys());
+    }
+    _enqueueWorkerStatusForNewlyWaitingKeys() {
+        this._workerStatusSendScheduled = false;
+        const keys = [...this._newlyWaitingKeys].filter((key) => this._pendingCallbacks.has(key));
+        this._newlyWaitingKeys.clear();
+        this._enqueueWorkerStatusFor(keys);
+    }
+    _enqueueWorkerStatusForLongPendingWaiters() {
+        const registeredBefore = Date.now() - WORKER_STATUS_RESEND_MIN_PENDING_MS;
+        const keys = [];
+        for (const [key, waitingSince] of this._waitingSince) {
+            if (!this._pendingCallbacks.has(key)) {
+                this._waitingSince.delete(key);
+            }
+            else if (waitingSince <= registeredBefore) {
+                keys.push(key);
+            }
+        }
+        this._enqueueWorkerStatusFor(keys);
+    }
+    _enqueueFullWorkerStatus() {
+        this._newlyWaitingKeys.clear();
+        this._enqueueWorkerStatusFor([...this._pendingCallbacks.keys()]);
+    }
+    _enqueueWorkerStatusFor(keys, maxEntriesPerRequest = WORKER_STATUS_MAX_ENTRIES_PER_REQUEST) {
+        if (!this._workerId)
+            return;
+        for (let start = 0; start < keys.length; start += maxEntriesPerRequest) {
+            this._enqueueRequest({
+                workerStatus: {
+                    workerId: this._workerId,
+                    waitingEntries: keys
+                        .slice(start, start + maxEntriesPerRequest)
+                        .map(awaitedCompletedEntryFromCallbackKey),
+                },
             });
         }
-        this._enqueueRequest({
-            workerStatus: {
-                workerId: this._workerId,
-                waitingEntries,
-            },
-        });
     }
     _failPendingAcks(exc) {
         for (const d of this._pendingEventAcks.values()) {
@@ -406,13 +450,15 @@ class DurableListenerClient {
             const orderKey = completionOrderKey((_s = ref === null || ref === void 0 ? void 0 : ref.durableTaskExternalId) !== null && _s !== void 0 ? _s : '', (_t = ref === null || ref === void 0 ? void 0 : ref.invocationCount) !== null && _t !== void 0 ? _t : 0);
             const queue = (_u = this._orderedCompletions.get(orderKey)) !== null && _u !== void 0 ? _u : {
                 pending: [],
+                pendingKeys: new Set(),
                 delivered: new Set(),
             };
             if (!queue.delivered.has(key)) {
                 queue.delivered.add(key);
+                queue.pendingKeys.add(key);
                 queue.pending.push({ key, result });
             }
-            else if (queue.pending.every((entry) => entry.key !== key)) {
+            else if (!queue.pendingKeys.has(key)) {
                 // Re-delivery of a completion that already drained (reconnect,
                 // worker-status re-send, or a repeated wait on a node deduped by
                 // child key). Its satisfied order was released before anything
@@ -478,20 +524,10 @@ class DurableListenerClient {
     }
     sendEvent(durableTaskExternalId, invocationCount, event) {
         return __awaiter(this, void 0, void 0, function* () {
-            const key = ackKey(durableTaskExternalId, invocationCount);
-            const d = deferred();
-            this._pendingEventAcks.set(key, d);
             let request;
             switch (event.kind) {
-                case 'runChildren': {
-                    const triggerRunsReq = {
-                        invocationCount,
-                        durableTaskExternalId,
-                        triggerOpts: event.triggerOpts,
-                    };
-                    request = { triggerRuns: triggerRunsReq };
-                    break;
-                }
+                case 'runChildren':
+                    return this._sendRunChildrenEvent(durableTaskExternalId, invocationCount, event);
                 case 'waitFor': {
                     const waitForReq = {
                         invocationCount,
@@ -517,8 +553,33 @@ class DurableListenerClient {
                     throw new Error(`Unknown durable task send event: ${_}`);
                 }
             }
-            this._enqueueRequest(request);
-            return d.promise;
+            return this._sendRequestAndAwaitAck(durableTaskExternalId, invocationCount, request);
+        });
+    }
+    _sendRequestAndAwaitAck(durableTaskExternalId, invocationCount, request) {
+        const d = deferred();
+        this._pendingEventAcks.set(ackKey(durableTaskExternalId, invocationCount), d);
+        this._enqueueRequest(request);
+        return d.promise;
+    }
+    _sendRunChildrenEvent(durableTaskExternalId, invocationCount, event) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const runEntries = [];
+            for (let start = 0; start < event.triggerOpts.length; start += RUN_CHILDREN_MAX_CHILDREN_PER_REQUEST) {
+                const triggerRunsReq = {
+                    invocationCount,
+                    durableTaskExternalId,
+                    triggerOpts: event.triggerOpts.slice(start, start + RUN_CHILDREN_MAX_CHILDREN_PER_REQUEST),
+                };
+                const ack = yield this._sendRequestAndAwaitAck(durableTaskExternalId, invocationCount, {
+                    triggerRuns: triggerRunsReq,
+                });
+                if (ack.ackType !== 'run') {
+                    throw new Error(`Expected a run ack for a trigger runs request, got ${ack.ackType}`);
+                }
+                runEntries.push(...ack.runEntries);
+            }
+            return { ackType: 'run', invocationCount, durableTaskExternalId, runEntries };
         });
     }
     // Hands queued completions to their waiters strictly in delivery order.
@@ -560,6 +621,7 @@ class DurableListenerClient {
                 if (!waiter)
                     return;
                 queue.pending.shift();
+                queue.pendingKeys.delete(head.key);
                 this._pendingCallbacks.delete(head.key);
                 waiter.resolve(head.result);
                 released = true;
@@ -577,6 +639,7 @@ class DurableListenerClient {
         const d = deferred();
         d.promise.catch(() => { });
         this._pendingCallbacks.set(key, d);
+        this._registerWaitingKey(key);
         this._drainOrderedCompletions(completionOrderKey(durableTaskExternalId, invocationCount));
     }
     waitForCallback(durableTaskExternalId, invocationCount, branchId, nodeId, opts) {
@@ -594,9 +657,10 @@ class DurableListenerClient {
                 // not crash the process as an unhandled rejection.
                 d.promise.catch(() => { });
                 this._pendingCallbacks.set(key, d);
+                this._registerWaitingKey(key);
                 this._drainOrderedCompletions(completionOrderKey(durableTaskExternalId, invocationCount));
                 if (this._pendingCallbacks.has(key)) {
-                    this._pollWorkerStatus();
+                    this._scheduleWorkerStatusSend();
                 }
             }
             if (!signal) {
@@ -656,6 +720,10 @@ class DurableListenerClient {
     sendEvictInvocation(durableTaskExternalId, invocationCount, reason) {
         return __awaiter(this, void 0, void 0, function* () {
             const key = evictionKey(durableTaskExternalId, invocationCount);
+            const pending = this._pendingEvictionAcks.get(key);
+            if (pending) {
+                return pending.promise;
+            }
             const d = deferred();
             this._pendingEvictionAcks.set(key, d);
             const req = {
@@ -664,15 +732,17 @@ class DurableListenerClient {
                 reason,
             };
             this._enqueueRequest({ evictInvocation: req });
-            const timeout = (0, sleep_1.default)(EVICTION_ACK_TIMEOUT_MS).then(() => {
-                throw new Error(`Eviction ack timed out after ${EVICTION_ACK_TIMEOUT_MS}ms for task ${durableTaskExternalId} invocation ${invocationCount}`);
-            });
+            const timeout = setTimeout(() => {
+                d.reject(new Error(`Eviction ack timed out after ${EVICTION_ACK_TIMEOUT_MS}ms for task ${durableTaskExternalId} invocation ${invocationCount}`));
+            }, EVICTION_ACK_TIMEOUT_MS);
             try {
-                yield Promise.race([d.promise, timeout]);
+                yield d.promise;
             }
-            catch (err) {
-                this._pendingEvictionAcks.delete(key);
-                throw err;
+            finally {
+                clearTimeout(timeout);
+                if (this._pendingEvictionAcks.get(key) === d) {
+                    this._pendingEvictionAcks.delete(key);
+                }
             }
         });
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hatchet-dev/typescript-sdk",
-  "version": "1.34.0",
+  "version": "1.34.1",
   "engines": {
     "node": ">=20"
   },

package/v1/client/worker/worker-internal.d.ts

@@ -45,6 +45,7 @@ export declare class InternalWorker {
     enableHealthServer: boolean;
     private healthServer;
     private status;
+    private shutdownPromise;
     constructor(client: HatchetClient, options: {
         name: string;
         handleKill?: boolean;
@@ -96,6 +97,7 @@ export declare class InternalWorker {
     handleCancelStepRun(action: Action): Promise<void>;
     stop(): Promise<void>;
     exitGracefully(handleKill: boolean): Promise<void>;
+    private shutdown;
     /**
      * Creates an action listener by registering the worker with the dispatcher.
      * Override in subclasses to change registration behavior (e.g. legacy engines).

package/v1/client/worker/worker-internal.js

@@ -628,6 +628,18 @@ class InternalWorker {
     exitGracefully(handleKill) {
         return __awaiter(this, void 0, void 0, function* () {
             var _a;
+            // Signals and an explicit stop must await the same eviction and listener teardown.
+            (_a = this.shutdownPromise) !== null && _a !== void 0 ? _a : (this.shutdownPromise = this.shutdown());
+            yield this.shutdownPromise;
+            if (handleKill) {
+                this.logger.info('Exiting hatchet worker...');
+                process.exit(0);
+            }
+        });
+    }
+    shutdown() {
+        return __awaiter(this, void 0, void 0, function* () {
+            var _a, _b;
             this.killing = true;
             this.setStatus(health_server_1.workerStatus.UNHEALTHY);
             this.logger.info('Starting to exit...');
@@ -658,16 +670,19 @@ class InternalWorker {
             catch (e) {
                 this.logger.error(`Could not stop durable listener: ${e.message}`);
             }
+            (_a = this.listener) === null || _a === void 0 ? void 0 : _a.stopStream();
+            this.logger.info('Gracefully exiting hatchet worker, running tasks will attempt to finish...');
+            // attempt to wait for futures to finish
+            yield Promise.all(Object.values(this.futures).map(({ promise }) => promise));
+            this.logger.info('Successfully finished pending tasks.');
+            // Unregistering stops the heartbeat, so it has to wait until the running tasks are done:
+            // otherwise the engine sees a dead worker and reassigns tasks that are about to complete.
             try {
-                yield ((_a = this.listener) === null || _a === void 0 ? void 0 : _a.unregister());
+                yield ((_b = this.listener) === null || _b === void 0 ? void 0 : _b.unregister());
             }
             catch (e) {
                 this.logger.error(`Could not unregister listener: ${e.message}`);
             }
-            this.logger.info('Gracefully exiting hatchet worker, running tasks will attempt to finish...');
-            // attempt to wait for futures to finish
-            yield Promise.all(Object.values(this.futures).map(({ promise }) => promise));
-            this.logger.info('Successfully finished pending tasks.');
             if (this.healthServer) {
                 try {
                     yield this.healthServer.stop();
@@ -676,10 +691,6 @@ class InternalWorker {
                     this.logger.error(`Could not stop health server: ${e.message}`);
                 }
             }
-            if (handleKill) {
-                this.logger.info('Exiting hatchet worker...');
-                process.exit(0);
-            }
         });
     }
     /**
@@ -700,10 +711,18 @@ class InternalWorker {
     start() {
         return __awaiter(this, void 0, void 0, function* () {
             var _a, e_1, _b, _c;
+            if (this.killing) {
+                return;
+            }
             this.setStatus(health_server_1.workerStatus.STARTING);
             if (this.healthServer) {
                 try {
                     yield this.healthServer.start();
+                    if (this.killing) {
+                        yield this.shutdownPromise;
+                        yield this.healthServer.stop();
+                        return;
+                    }
                 }
                 catch (e) {
                     this.logger.error(`Could not start health server: ${e.message}`);
@@ -713,11 +732,17 @@ class InternalWorker {
             }
             // ensure all workflows are registered
             yield Promise.all(this.registeredWorkflowPromises);
-            if (Object.keys(this.action_registry).length === 0) {
+            if (this.killing || Object.keys(this.action_registry).length === 0) {
                 return;
             }
             try {
-                this.listener = yield this.createListener();
+                const listener = yield this.createListener();
+                if (this.killing) {
+                    // Shutdown cannot clean up a listener that registration has not returned yet.
+                    yield listener.unregister();
+                    return;
+                }
+                this.listener = listener;
                 this.workerId = this.listener.workerId;
                 this.setStatus(health_server_1.workerStatus.HEALTHY);
                 const generator = this.listener.actions();

package/version.d.ts

@@ -1 +1 @@
-export declare const HATCHET_VERSION = "1.34.0";
+export declare const HATCHET_VERSION = "1.34.1";

package/version.js

@@ -1,4 +1,4 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.HATCHET_VERSION = void 0;
-exports.HATCHET_VERSION = '1.34.0';
+exports.HATCHET_VERSION = '1.34.1';