@hatchet-dev/typescript-sdk 1.29.1 → 1.29.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/clients/dispatcher/heartbeat/heartbeat-severity.d.ts +1 -1
- package/clients/dispatcher/heartbeat/heartbeat-severity.js +4 -2
- package/clients/dispatcher/heartbeat/heartbeat-worker.js +6 -2
- package/package.json +1 -1
- package/util/grpc-error.d.ts +7 -0
- package/util/grpc-error.js +10 -0
- package/version.d.ts +1 -1
- package/version.js +1 -1
|
@@ -1,3 +1,3 @@
|
|
|
1
1
|
import { FailureSeverity } from '../../../util/failure-severity';
|
|
2
2
|
export declare const MAX_MISSED_HEARTBEATS = 3;
|
|
3
|
-
export declare function classifyHeartbeatFailure(
|
|
3
|
+
export declare function classifyHeartbeatFailure(e: unknown, missedHeartbeats: number): FailureSeverity;
|
|
@@ -6,6 +6,8 @@ const grpc_error_1 = require("../../../util/grpc-error");
|
|
|
6
6
|
const failure_severity_1 = require("../../../util/failure-severity");
|
|
7
7
|
exports.MAX_MISSED_HEARTBEATS = 3;
|
|
8
8
|
// determines whether to immediately error log or wait for additional errors
|
|
9
|
-
function classifyHeartbeatFailure(
|
|
10
|
-
|
|
9
|
+
function classifyHeartbeatFailure(e, missedHeartbeats) {
|
|
10
|
+
const code = (0, grpc_error_1.getGrpcErrorCode)(e);
|
|
11
|
+
const isTransient = (0, grpc_error_1.isConnectionError)(code) || (0, grpc_error_1.isTimeoutOrAbortError)(e);
|
|
12
|
+
return (0, failure_severity_1.classifyRepeatedFailure)(isTransient, missedHeartbeats, exports.MAX_MISSED_HEARTBEATS);
|
|
11
13
|
}
|
|
@@ -20,6 +20,10 @@ const dispatcher_client_1 = require("../dispatcher-client");
|
|
|
20
20
|
const heartbeat_controller_1 = require("./heartbeat-controller");
|
|
21
21
|
const heartbeat_severity_1 = require("./heartbeat-severity");
|
|
22
22
|
const HEARTBEAT_INTERVAL = 4000;
|
|
23
|
+
// Fail fast on a stalled connection: without a deadline, a heartbeat RPC on a dead
|
|
24
|
+
// TCP connection can hang for 50+ seconds, well past the engine's 30s reassignment
|
|
25
|
+
// window, while setInterval stacks overlapping in-flight requests behind it.
|
|
26
|
+
const HEARTBEAT_TIMEOUT = 5000;
|
|
23
27
|
const postMessage = (message) => {
|
|
24
28
|
worker_threads_1.parentPort === null || worker_threads_1.parentPort === void 0 ? void 0 : worker_threads_1.parentPort.postMessage(message);
|
|
25
29
|
};
|
|
@@ -54,7 +58,7 @@ class HeartbeatWorker {
|
|
|
54
58
|
yield this.client.heartbeat({
|
|
55
59
|
workerId: this.workerId,
|
|
56
60
|
heartbeatAt: new Date(),
|
|
57
|
-
});
|
|
61
|
+
}, { signal: AbortSignal.timeout(HEARTBEAT_TIMEOUT) });
|
|
58
62
|
const now = new Date().getTime();
|
|
59
63
|
const actualInterval = now - this.timeLastHeartbeat;
|
|
60
64
|
if (actualInterval > HEARTBEAT_INTERVAL * 1.2) {
|
|
@@ -88,7 +92,7 @@ class HeartbeatWorker {
|
|
|
88
92
|
this.missedHeartbeats += 1;
|
|
89
93
|
const message = `Failed to send heartbeat: ${(0, hatchet_error_1.getErrorMessage)(e)}`;
|
|
90
94
|
this.logger.debug(message);
|
|
91
|
-
const severity = (0, heartbeat_severity_1.classifyHeartbeatFailure)(
|
|
95
|
+
const severity = (0, heartbeat_severity_1.classifyHeartbeatFailure)(e, this.missedHeartbeats);
|
|
92
96
|
if (severity !== 'silent') {
|
|
93
97
|
postMessage({
|
|
94
98
|
type: severity,
|
package/package.json
CHANGED
package/util/grpc-error.d.ts
CHANGED
|
@@ -4,6 +4,13 @@
|
|
|
4
4
|
*/
|
|
5
5
|
export declare const CONNECTION_ERROR_CODES: number[];
|
|
6
6
|
export declare function isConnectionError(code: number | undefined): boolean;
|
|
7
|
+
/**
|
|
8
|
+
* Returns true if `e` is the DOMException thrown when an AbortSignal (e.g. from
|
|
9
|
+
* AbortSignal.timeout()) fires. This rejects the underlying call directly rather than
|
|
10
|
+
* surfacing as a gRPC status code, so it needs to be classified as transient separately
|
|
11
|
+
* from isConnectionError.
|
|
12
|
+
*/
|
|
13
|
+
export declare function isTimeoutOrAbortError(e: unknown): boolean;
|
|
7
14
|
/**
|
|
8
15
|
* Returns the gRPC status code from an unknown value (e.g. from a catch block).
|
|
9
16
|
* Used for checking Status.CANCELLED, Status.UNAVAILABLE, etc.
|
package/util/grpc-error.js
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.CONNECTION_ERROR_CODES = void 0;
|
|
4
4
|
exports.isConnectionError = isConnectionError;
|
|
5
|
+
exports.isTimeoutOrAbortError = isTimeoutOrAbortError;
|
|
5
6
|
exports.getGrpcErrorCode = getGrpcErrorCode;
|
|
6
7
|
exports.getGrpcErrorDetails = getGrpcErrorDetails;
|
|
7
8
|
const nice_grpc_1 = require("nice-grpc");
|
|
@@ -13,6 +14,15 @@ exports.CONNECTION_ERROR_CODES = [nice_grpc_1.Status.UNAVAILABLE, nice_grpc_1.St
|
|
|
13
14
|
function isConnectionError(code) {
|
|
14
15
|
return code !== undefined && exports.CONNECTION_ERROR_CODES.includes(code);
|
|
15
16
|
}
|
|
17
|
+
/**
|
|
18
|
+
* Returns true if `e` is the DOMException thrown when an AbortSignal (e.g. from
|
|
19
|
+
* AbortSignal.timeout()) fires. This rejects the underlying call directly rather than
|
|
20
|
+
* surfacing as a gRPC status code, so it needs to be classified as transient separately
|
|
21
|
+
* from isConnectionError.
|
|
22
|
+
*/
|
|
23
|
+
function isTimeoutOrAbortError(e) {
|
|
24
|
+
return e instanceof DOMException && (e.name === 'TimeoutError' || e.name === 'AbortError');
|
|
25
|
+
}
|
|
16
26
|
/**
|
|
17
27
|
* Returns the gRPC status code from an unknown value (e.g. from a catch block).
|
|
18
28
|
* Used for checking Status.CANCELLED, Status.UNAVAILABLE, etc.
|
package/version.d.ts
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
export declare const HATCHET_VERSION = "1.29.
|
|
1
|
+
export declare const HATCHET_VERSION = "1.29.2";
|
package/version.js
CHANGED