@hatchet-dev/typescript-sdk 1.28.0 → 1.28.1

package/clients/dispatcher/action-listener.js

@@ -74,6 +74,7 @@ const abort_controller_x_1 = require("abort-controller-x");
 const sleep_1 = __importDefault(require("../../util/sleep"));
 const hatchet_error_1 = __importStar(require("../../util/errors/hatchet-error"));
 const heartbeat_controller_1 = require("./heartbeat/heartbeat-controller");
+const listener_severity_1 = require("./listener-severity");
 const DEFAULT_ACTION_LISTENER_RETRY_INTERVAL = 5000; // milliseconds
 const DEFAULT_ACTION_LISTENER_RETRY_COUNT = 20;
 var ListenStrategy;
@@ -145,7 +146,11 @@ class ActionListener {
                             client.setListenStrategy(ListenStrategy.LISTEN_STRATEGY_V1);
                         }
                         client.incrementRetries();
-                        client.logger.error(`Listener encountered an error: ${(0, hatchet_error_1.getErrorMessage)(e)}`);
+                        const message = `Listener encountered an error: ${(0, hatchet_error_1.getErrorMessage)(e)}`;
+                        const severity = (0, listener_severity_1.classifyListenerFailure)(e, client.retries);
+                        if (severity !== 'silent') {
+                            client.logger[severity](message);
+                        }
                         if (client.retries > 1) {
                             client.logger.info(`Retrying in ${client.retryInterval}ms...`);
                             yield __await((0, sleep_1.default)(client.retryInterval));
@@ -218,7 +223,11 @@ class ActionListener {
             }
             catch (e) {
                 this.retries += 1;
-                this.logger.error(`Attempt ${this.retries}: Failed to connect, retrying...`);
+                const message = `Attempt ${this.retries}: Failed to connect, retrying...`;
+                const severity = (0, listener_severity_1.classifyListenerFailure)(e, this.retries);
+                if (severity !== 'silent') {
+                    this.logger[severity](message);
+                }
                 if ((0, grpc_error_1.getGrpcErrorCode)(e) === nice_grpc_1.Status.UNAVAILABLE) {
                     // Connection lost, reset heartbeat interval and retry connection
                     this.heartbeat.stop();

package/clients/dispatcher/heartbeat/heartbeat-controller.d.ts

@@ -3,10 +3,18 @@ import { DispatcherClient as PbDispatcherClient } from '../../../protoc/dispatch
 import { Worker } from 'worker_threads';
 import { ClientConfig } from '../../hatchet-client';
 import { DispatcherClient } from '../dispatcher-client';
-export interface HeartbeatMessage {
-    type: 'info' | 'warn' | 'error' | 'debug';
-    message: string;
-}
+import { z } from 'zod/v4';
+declare const HeartbeatMessageSchema: z.ZodObject<{
+    type: z.ZodEnum<{
+        error: "error";
+        warn: "warn";
+        info: "info";
+        debug: "debug";
+    }>;
+    message: z.ZodString;
+}, z.core.$strip>;
+export type HeartbeatMessage = z.infer<typeof HeartbeatMessageSchema>;
+export declare const isHeartbeatMessage: (message: unknown) => message is HeartbeatMessage;
 export declare const STOP_HEARTBEAT = "stop";
 export declare class Heartbeat {
     config: ClientConfig;
@@ -18,3 +26,4 @@ export declare class Heartbeat {
     start(): Promise<void>;
     stop(): Promise<void>;
 }
+export {};

package/clients/dispatcher/heartbeat/heartbeat-controller.js

@@ -23,9 +23,16 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.Heartbeat = exports.STOP_HEARTBEAT = void 0;
+exports.Heartbeat = exports.STOP_HEARTBEAT = exports.isHeartbeatMessage = void 0;
 const path_1 = __importDefault(require("path"));
 const thread_helper_1 = require("../../../util/thread-helper");
+const v4_1 = require("zod/v4");
+const HeartbeatMessageSchema = v4_1.z.object({
+    type: v4_1.z.enum(['info', 'warn', 'error', 'debug']),
+    message: v4_1.z.string(),
+});
+const isHeartbeatMessage = (message) => HeartbeatMessageSchema.safeParse(message).success;
+exports.isHeartbeatMessage = isHeartbeatMessage;
 exports.STOP_HEARTBEAT = 'stop';
 class Heartbeat {
     constructor(client, workerId) {
@@ -45,6 +52,9 @@ class Heartbeat {
                     },
                 });
                 this.heartbeatWorker.on('message', (message) => {
+                    if (!(0, exports.isHeartbeatMessage)(message)) {
+                        return;
+                    }
                     this.logger[message.type](message.message);
                 });
             }

package/clients/dispatcher/heartbeat/heartbeat-severity.d.ts

@@ -0,0 +1,3 @@
+import { FailureSeverity } from '../../../util/failure-severity';
+export declare const MAX_MISSED_HEARTBEATS = 3;
+export declare function classifyHeartbeatFailure(code: number | undefined, missedHeartbeats: number): FailureSeverity;

package/clients/dispatcher/heartbeat/heartbeat-severity.js

@@ -0,0 +1,11 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MAX_MISSED_HEARTBEATS = void 0;
+exports.classifyHeartbeatFailure = classifyHeartbeatFailure;
+const grpc_error_1 = require("../../../util/grpc-error");
+const failure_severity_1 = require("../../../util/failure-severity");
+exports.MAX_MISSED_HEARTBEATS = 3;
+// determines whether to immediately error log or wait for additional errors
+function classifyHeartbeatFailure(code, missedHeartbeats) {
+    return (0, failure_severity_1.classifyRepeatedFailure)((0, grpc_error_1.isConnectionError)(code), missedHeartbeats, exports.MAX_MISSED_HEARTBEATS);
+}

package/clients/dispatcher/heartbeat/heartbeat-worker.js

@@ -18,6 +18,7 @@ const grpc_error_1 = require("../../../util/grpc-error");
 const grpc_helpers_1 = require("../../../util/grpc-helpers");
 const dispatcher_client_1 = require("../dispatcher-client");
 const heartbeat_controller_1 = require("./heartbeat-controller");
+const heartbeat_severity_1 = require("./heartbeat-severity");
 const HEARTBEAT_INTERVAL = 4000;
 const postMessage = (message) => {
     worker_threads_1.parentPort === null || worker_threads_1.parentPort === void 0 ? void 0 : worker_threads_1.parentPort.postMessage(message);
@@ -25,6 +26,7 @@ const postMessage = (message) => {
 class HeartbeatWorker {
     constructor(config, workerId) {
         this.timeLastHeartbeat = new Date().getTime();
+        this.missedHeartbeats = 0;
         this.workerId = workerId;
         this.logger = new hatchet_client_1.HatchetLogger(`HeartbeatThread`, config.log_level);
         this.logger.debug('Heartbeat thread starting...');
@@ -69,6 +71,7 @@ class HeartbeatWorker {
                         message: `Heartbeat sent ${actualInterval}ms ago`,
                     });
                     this.timeLastHeartbeat = now;
+                    this.missedHeartbeats = 0;
                 }
                 catch (e) {
                     if ((0, grpc_error_1.getGrpcErrorCode)(e) === nice_grpc_1.Status.UNIMPLEMENTED) {
@@ -82,12 +85,16 @@ class HeartbeatWorker {
                         this.stop();
                         return;
                     }
+                    this.missedHeartbeats += 1;
                     const message = `Failed to send heartbeat: ${(0, hatchet_error_1.getErrorMessage)(e)}`;
                     this.logger.debug(message);
-                    postMessage({
-                        type: 'error',
-                        message,
-                    });
+                    const severity = (0, heartbeat_severity_1.classifyHeartbeatFailure)((0, grpc_error_1.getGrpcErrorCode)(e), this.missedHeartbeats);
+                    if (severity !== 'silent') {
+                        postMessage({
+                            type: severity,
+                            message,
+                        });
+                    }
                 }
             });
             // start with a heartbeat

package/clients/dispatcher/listener-severity.d.ts

@@ -0,0 +1,3 @@
+import { FailureSeverity } from '../../util/failure-severity';
+export declare const MAX_TRANSIENT_LISTENER_RETRIES = 3;
+export declare function classifyListenerFailure(e: unknown, retries: number): FailureSeverity;

package/clients/dispatcher/listener-severity.js

@@ -0,0 +1,11 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MAX_TRANSIENT_LISTENER_RETRIES = void 0;
+exports.classifyListenerFailure = classifyListenerFailure;
+const grpc_error_1 = require("../../util/grpc-error");
+const failure_severity_1 = require("../../util/failure-severity");
+exports.MAX_TRANSIENT_LISTENER_RETRIES = 3;
+function classifyListenerFailure(e, retries) {
+    const isTransient = e === undefined || (0, grpc_error_1.isConnectionError)((0, grpc_error_1.getGrpcErrorCode)(e));
+    return (0, failure_severity_1.classifyRepeatedFailure)(isTransient, retries, exports.MAX_TRANSIENT_LISTENER_RETRIES);
+}

package/clients/listeners/durable-listener/durable-listener-client.d.ts

@@ -70,6 +70,7 @@ export declare class DurableListenerClient {
     private _receiveAbort;
     private _statusInterval;
     private _startLock;
+    private _consecutiveFailures;
     onServerEvict: ((durableTaskExternalId: string, invocationCount: number) => void) | undefined;
     constructor(config: ClientConfig, channel: Channel, factory: ClientFactory);
     get workerId(): string | undefined;

package/clients/listeners/durable-listener/durable-listener-client.js

@@ -40,6 +40,7 @@ const dispatcher_1 = require("../../../protoc/v1/dispatcher");
 const non_determinism_error_1 = require("../../../util/errors/non-determinism-error");
 const abort_error_1 = require("../../../util/abort-error");
 const sleep_1 = __importDefault(require("../../../util/sleep"));
+const listener_severity_1 = require("../../dispatcher/listener-severity");
 class TTLMap {
     constructor(ttlMs) {
         this.ttlMs = ttlMs;
@@ -136,6 +137,7 @@ class DurableListenerClient {
         // (e.g. an already-satisfied sleep delivered via polling).
         this._bufferedCompletions = new TTLMap(10000);
         this._pendingEvictionAcks = new Map();
+        this._consecutiveFailures = 0;
         this.config = config;
         this.client = factory.create(dispatcher_1.V1DispatcherDefinition, channel);
         this.logger = config.logger(`DurableListener`, config.log_level);
@@ -159,6 +161,7 @@ class DurableListenerClient {
                 return;
             this._workerId = workerId;
             this._running = true;
+            this._consecutiveFailures = 0;
             yield this._connect();
             this._startStatusPolling();
         });
@@ -209,11 +212,16 @@ class DurableListenerClient {
                     const stream = this.client.durableTask(this._requestIterator(), {
                         signal: (_d = this._receiveAbort) === null || _d === void 0 ? void 0 : _d.signal,
                     });
+                    let receivedResponse = false;
                     try {
                         for (var _e = true, stream_1 = (e_1 = void 0, __asyncValues(stream)), stream_1_1; stream_1_1 = yield stream_1.next(), _a = stream_1_1.done, !_a; _e = true) {
                             _c = stream_1_1.value;
                             _e = false;
                             const response = _c;
+                            if (!receivedResponse) {
+                                receivedResponse = true;
+                                this._consecutiveFailures = 0;
+                            }
                             this._handleResponse(response);
                         }
                     }
@@ -225,7 +233,12 @@ class DurableListenerClient {
                         finally { if (e_1) throw e_1.error; }
                     }
                     if (this._running) {
-                        this.logger.warn(`durable event listener disconnected (EOF), reconnecting in ${DEFAULT_RECONNECT_INTERVAL}ms...`);
+                        this._consecutiveFailures += 1;
+                        const message = `durable event listener disconnected (EOF), reconnecting in ${DEFAULT_RECONNECT_INTERVAL}ms...`;
+                        const eofSeverity = (0, listener_severity_1.classifyListenerFailure)(undefined, this._consecutiveFailures);
+                        if (eofSeverity !== 'silent') {
+                            this.logger[eofSeverity](message);
+                        }
                         this._failPendingAcks(new Error('durable stream disconnected'));
                         yield (0, sleep_1.default)(DEFAULT_RECONNECT_INTERVAL);
                         yield this._connect();
@@ -237,7 +250,12 @@ class DurableListenerClient {
                         this.logger.debug('durable event listener aborted');
                         return;
                     }
-                    this.logger.error(`error in durable event listener: ${(0, hatchet_error_1.getErrorMessage)(e)}`);
+                    this._consecutiveFailures += 1;
+                    const message = `error in durable event listener: ${(0, hatchet_error_1.getErrorMessage)(e)}`;
+                    const errSeverity = (0, listener_severity_1.classifyListenerFailure)(e, this._consecutiveFailures);
+                    if (errSeverity !== 'silent') {
+                        this.logger[errSeverity](message);
+                    }
                     if (this._running) {
                         this._failPendingAcks(new Error(`durable stream error: ${(0, hatchet_error_1.getErrorMessage)(e)}`));
                         yield (0, sleep_1.default)(DEFAULT_RECONNECT_INTERVAL);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hatchet-dev/typescript-sdk",
-  "version": "1.28.0",
+  "version": "1.28.1",
   "description": "Background task orchestration & visibility for developers",
   "types": "dist/index.d.ts",
   "files": [
@@ -59,7 +59,7 @@
     "@anthropic-ai/claude-agent-sdk": "^0.3.148",
     "@grpc/grpc-js": "^1.14.3",
     "@modelcontextprotocol/sdk": "^1.29.0",
-    "@openai/agents": "0.13.2",
+    "@openai/agents": "0.13.5",
     "@opentelemetry/api": "^1.9.0",
     "@opentelemetry/core": "^2.0.0",
     "@opentelemetry/exporter-trace-otlp-grpc": "^0.220.0",
@@ -91,7 +91,7 @@
     "@anthropic-ai/claude-agent-sdk": "^0.3.148",
     "@grpc/grpc-js": "^1.14.3",
     "@modelcontextprotocol/sdk": "^1.29.0",
-    "@openai/agents": "0.13.2",
+    "@openai/agents": "0.13.5",
     "@opentelemetry/api": "^1.9.0",
     "@opentelemetry/core": "^2.0.0",
     "@opentelemetry/exporter-trace-otlp-grpc": "^0.220.0",

package/util/failure-severity.d.ts

@@ -0,0 +1,2 @@
+export type FailureSeverity = 'silent' | 'warn' | 'error';
+export declare function classifyRepeatedFailure(isTransient: boolean, attempt: number, threshold: number): FailureSeverity;

package/util/failure-severity.js

@@ -0,0 +1,15 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.classifyRepeatedFailure = classifyRepeatedFailure;
+function classifyRepeatedFailure(isTransient, attempt, threshold) {
+    if (!isTransient) {
+        return 'error';
+    }
+    if (attempt >= threshold) {
+        return 'error';
+    }
+    if (attempt > 1) {
+        return 'warn';
+    }
+    return 'silent';
+}

package/util/grpc-error.d.ts

@@ -1,3 +1,9 @@
+/**
+ * gRPC codes that typically indicate a transient connectivity problem
+ * (server unreachable/restarting) rather than an application-level error.
+ */
+export declare const CONNECTION_ERROR_CODES: number[];
+export declare function isConnectionError(code: number | undefined): boolean;
 /**
  * Returns the gRPC status code from an unknown value (e.g. from a catch block).
  * Used for checking Status.CANCELLED, Status.UNAVAILABLE, etc.

package/util/grpc-error.js

@@ -1,7 +1,18 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.CONNECTION_ERROR_CODES = void 0;
+exports.isConnectionError = isConnectionError;
 exports.getGrpcErrorCode = getGrpcErrorCode;
 exports.getGrpcErrorDetails = getGrpcErrorDetails;
+const nice_grpc_1 = require("nice-grpc");
+/**
+ * gRPC codes that typically indicate a transient connectivity problem
+ * (server unreachable/restarting) rather than an application-level error.
+ */
+exports.CONNECTION_ERROR_CODES = [nice_grpc_1.Status.UNAVAILABLE, nice_grpc_1.Status.FAILED_PRECONDITION];
+function isConnectionError(code) {
+    return code !== undefined && exports.CONNECTION_ERROR_CODES.includes(code);
+}
 /**
  * Returns the gRPC status code from an unknown value (e.g. from a catch block).
  * Used for checking Status.CANCELLED, Status.UNAVAILABLE, etc.

package/version.d.ts

@@ -1 +1 @@
-export declare const HATCHET_VERSION = "1.28.0";
+export declare const HATCHET_VERSION = "1.28.1";

package/version.js

@@ -1,4 +1,4 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.HATCHET_VERSION = void 0;
-exports.HATCHET_VERSION = '1.28.0';
+exports.HATCHET_VERSION = '1.28.1';