@spooled/sdk 1.2.1 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +119 -18
- package/dist/{index-CWVI9LRo.d.cts → index-DzTC8ld3.d.cts} +741 -134
- package/dist/{index-CWVI9LRo.d.ts → index-DzTC8ld3.d.ts} +741 -134
- package/dist/index.cjs +758 -304
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +9 -151
- package/dist/index.d.ts +9 -151
- package/dist/index.js +757 -305
- package/dist/index.js.map +1 -1
- package/dist/worker/index.cjs +164 -45
- package/dist/worker/index.cjs.map +1 -1
- package/dist/worker/index.d.cts +1 -1
- package/dist/worker/index.d.ts +1 -1
- package/dist/worker/index.js +164 -45
- package/dist/worker/index.js.map +1 -1
- package/package.json +4 -3
- package/proto/spooled.proto +19 -1
package/dist/index.cjs
CHANGED
|
@@ -1,7 +1,5 @@
|
|
|
1
1
|
'use strict';
|
|
2
2
|
|
|
3
|
-
var crypto = require('crypto');
|
|
4
|
-
var buffer = require('buffer');
|
|
5
3
|
var grpc2 = require('@grpc/grpc-js');
|
|
6
4
|
var protoLoader = require('@grpc/proto-loader');
|
|
7
5
|
var fs = require('fs');
|
|
@@ -50,10 +48,10 @@ var DEFAULT_CONFIG = {
|
|
|
50
48
|
successThreshold: 3,
|
|
51
49
|
timeout: 3e4
|
|
52
50
|
},
|
|
53
|
-
userAgent: "@spooled/sdk-nodejs/1.
|
|
51
|
+
userAgent: "@spooled/sdk-nodejs/1.4.0",
|
|
54
52
|
autoRefreshToken: true
|
|
55
53
|
};
|
|
56
|
-
var SDK_VERSION = "1.
|
|
54
|
+
var SDK_VERSION = "1.4.0";
|
|
57
55
|
var API_VERSION = "v1";
|
|
58
56
|
var API_BASE_PATH = `/api/${API_VERSION}`;
|
|
59
57
|
function normalizeCredential(value) {
|
|
@@ -110,9 +108,9 @@ function resolveConfig(options) {
|
|
|
110
108
|
};
|
|
111
109
|
}
|
|
112
110
|
function validateConfig(config) {
|
|
113
|
-
if (!config.apiKey && !config.accessToken) {
|
|
111
|
+
if (!config.apiKey && !config.accessToken && !config.adminKey) {
|
|
114
112
|
throw new Error(
|
|
115
|
-
"SpooledClient requires either apiKey or accessToken for authentication"
|
|
113
|
+
"SpooledClient requires either apiKey or accessToken for authentication (or adminKey for admin-only use)"
|
|
116
114
|
);
|
|
117
115
|
}
|
|
118
116
|
if (config.apiKey && !config.apiKey.startsWith("sp_") && !config.apiKey.startsWith("sk_")) {
|
|
@@ -175,9 +173,15 @@ var SpooledError = class extends Error {
|
|
|
175
173
|
Error.captureStackTrace(this, this.constructor);
|
|
176
174
|
}
|
|
177
175
|
}
|
|
178
|
-
/**
|
|
176
|
+
/**
|
|
177
|
+
* Whether this error is retryable.
|
|
178
|
+
*
|
|
179
|
+
* Only 408 (Request Timeout) is retryable among the plain 4xx statuses: the
|
|
180
|
+
* server gave up waiting for the request, so it never ran. Subclasses
|
|
181
|
+
* override this (5xx, network, timeout, rate limit).
|
|
182
|
+
*/
|
|
179
183
|
isRetryable() {
|
|
180
|
-
return
|
|
184
|
+
return this.statusCode === 408;
|
|
181
185
|
}
|
|
182
186
|
/** Convert to plain object for logging */
|
|
183
187
|
toJSON() {
|
|
@@ -216,8 +220,8 @@ var ConflictError = class extends SpooledError {
|
|
|
216
220
|
}
|
|
217
221
|
};
|
|
218
222
|
var ValidationError = class extends SpooledError {
|
|
219
|
-
constructor(message = "Validation failed", code = "VALIDATION_ERROR", details, requestId) {
|
|
220
|
-
super(message,
|
|
223
|
+
constructor(message = "Validation failed", code = "VALIDATION_ERROR", details, requestId, statusCode = 400) {
|
|
224
|
+
super(message, statusCode, code, details, requestId);
|
|
221
225
|
this.name = "ValidationError";
|
|
222
226
|
}
|
|
223
227
|
};
|
|
@@ -227,6 +231,7 @@ var PayloadTooLargeError = class extends SpooledError {
|
|
|
227
231
|
this.name = "PayloadTooLargeError";
|
|
228
232
|
}
|
|
229
233
|
};
|
|
234
|
+
var QUOTA_MESSAGE = /\blimit reached\b/i;
|
|
230
235
|
var RateLimitError = class extends SpooledError {
|
|
231
236
|
/** Rate limit information */
|
|
232
237
|
rateLimitInfo;
|
|
@@ -235,8 +240,27 @@ var RateLimitError = class extends SpooledError {
|
|
|
235
240
|
this.name = "RateLimitError";
|
|
236
241
|
this.rateLimitInfo = rateLimitInfo;
|
|
237
242
|
}
|
|
243
|
+
/**
|
|
244
|
+
* True when this 429 is a plan quota or limit (daily job cap, active jobs,
|
|
245
|
+
* queue/worker/webhook caps) rather than per-second rate limiting.
|
|
246
|
+
*
|
|
247
|
+
* A quota does not clear for minutes or hours, so retrying it only blocks the
|
|
248
|
+
* caller for the whole backoff and, via the circuit breaker, takes every
|
|
249
|
+
* other call on the client down with it. It is never retried and never
|
|
250
|
+
* counted as a circuit breaker failure.
|
|
251
|
+
*
|
|
252
|
+
* Detection: the `QUOTA_EXCEEDED` code, or, for older plan-limit paths that
|
|
253
|
+
* answer with plain text wrapped as `RATE_LIMIT_EXCEEDED`, a "... limit
|
|
254
|
+
* reached" message that carries no `Retry-After`.
|
|
255
|
+
*/
|
|
256
|
+
get isQuota() {
|
|
257
|
+
if (this.code === "QUOTA_EXCEEDED") {
|
|
258
|
+
return true;
|
|
259
|
+
}
|
|
260
|
+
return this.rateLimitInfo.retryAfter === void 0 && QUOTA_MESSAGE.test(this.message);
|
|
261
|
+
}
|
|
238
262
|
isRetryable() {
|
|
239
|
-
return
|
|
263
|
+
return !this.isQuota;
|
|
240
264
|
}
|
|
241
265
|
/** Get the number of seconds to wait before retrying */
|
|
242
266
|
getRetryAfter() {
|
|
@@ -296,6 +320,20 @@ var CircuitBreakerOpenError = class extends SpooledError {
|
|
|
296
320
|
return false;
|
|
297
321
|
}
|
|
298
322
|
};
|
|
323
|
+
var UnsupportedOperationError = class extends SpooledError {
|
|
324
|
+
/** The SDK method that is not supported, e.g. `ingest.github` */
|
|
325
|
+
operation;
|
|
326
|
+
constructor(operation, reason) {
|
|
327
|
+
super(
|
|
328
|
+
`${operation} is not supported by the Spooled API: ${reason}`,
|
|
329
|
+
0,
|
|
330
|
+
"UNSUPPORTED_OPERATION",
|
|
331
|
+
{ operation }
|
|
332
|
+
);
|
|
333
|
+
this.name = "UnsupportedOperationError";
|
|
334
|
+
this.operation = operation;
|
|
335
|
+
}
|
|
336
|
+
};
|
|
299
337
|
function parseRateLimitHeaders(headers) {
|
|
300
338
|
const info = {};
|
|
301
339
|
const retryAfter = headers.get("Retry-After");
|
|
@@ -326,6 +364,10 @@ function parseRateLimitHeaders(headers) {
|
|
|
326
364
|
info.reset = new Date(timestamp * 1e3);
|
|
327
365
|
}
|
|
328
366
|
}
|
|
367
|
+
const scope = headers.get("X-RateLimit-Scope");
|
|
368
|
+
if (scope) {
|
|
369
|
+
info.scope = scope.trim();
|
|
370
|
+
}
|
|
329
371
|
return info;
|
|
330
372
|
}
|
|
331
373
|
var MAX_RAW_ERROR_BODY_CHARS = 500;
|
|
@@ -367,6 +409,10 @@ async function createErrorFromResponse(response) {
|
|
|
367
409
|
switch (response.status) {
|
|
368
410
|
case 400:
|
|
369
411
|
return new ValidationError(message, code, details, requestId);
|
|
412
|
+
// A missing or mistyped JSON field is a 422 from the API's extractor; it is
|
|
413
|
+
// the same class of caller error as a 400 field-rule failure.
|
|
414
|
+
case 422:
|
|
415
|
+
return new ValidationError(message, code, details, requestId, 422);
|
|
370
416
|
case 401:
|
|
371
417
|
return new AuthenticationError(message, code, details, requestId);
|
|
372
418
|
case 403:
|
|
@@ -595,8 +641,14 @@ var HttpClient = class {
|
|
|
595
641
|
authToken;
|
|
596
642
|
/** Optional function to mint a fresh access token (set when auto-refresh is enabled) */
|
|
597
643
|
refreshTokenFn = null;
|
|
598
|
-
/**
|
|
599
|
-
|
|
644
|
+
/**
|
|
645
|
+
* The refresh currently in flight, shared by every request that gets a 401
|
|
646
|
+
* meanwhile. A boolean guard here made the second concurrent 401 skip the
|
|
647
|
+
* refresh and fail even though the token was about to be replaced.
|
|
648
|
+
*/
|
|
649
|
+
refreshInFlight = null;
|
|
650
|
+
/** Rate-limit headers of the most recent response that carried them */
|
|
651
|
+
lastRateLimit;
|
|
600
652
|
constructor(config, circuitBreaker) {
|
|
601
653
|
this.config = config;
|
|
602
654
|
this.circuitBreaker = circuitBreaker;
|
|
@@ -609,6 +661,14 @@ var HttpClient = class {
|
|
|
609
661
|
setAuthToken(token) {
|
|
610
662
|
this.authToken = token;
|
|
611
663
|
}
|
|
664
|
+
/**
|
|
665
|
+
* The `X-RateLimit-*` (and `Retry-After`) values of the most recent
|
|
666
|
+
* response that carried them, successful or not. `undefined` until such a
|
|
667
|
+
* response arrives. With concurrent requests it is the last one to finish.
|
|
668
|
+
*/
|
|
669
|
+
getLastRateLimitInfo() {
|
|
670
|
+
return this.lastRateLimit ? { ...this.lastRateLimit } : void 0;
|
|
671
|
+
}
|
|
612
672
|
/**
|
|
613
673
|
* Set the token refresh function.
|
|
614
674
|
*
|
|
@@ -618,6 +678,30 @@ var HttpClient = class {
|
|
|
618
678
|
setRefreshTokenFn(fn) {
|
|
619
679
|
this.refreshTokenFn = fn;
|
|
620
680
|
}
|
|
681
|
+
/**
|
|
682
|
+
* Refresh the access token once for all callers.
|
|
683
|
+
*
|
|
684
|
+
* Resolves true when a new token is installed, false when the refresh failed
|
|
685
|
+
* (the caller then surfaces its original 401).
|
|
686
|
+
*/
|
|
687
|
+
refreshAuthToken() {
|
|
688
|
+
if (!this.refreshInFlight) {
|
|
689
|
+
const refresh = this.refreshTokenFn;
|
|
690
|
+
this.refreshInFlight = (async () => {
|
|
691
|
+
try {
|
|
692
|
+
this.setAuthToken(await refresh());
|
|
693
|
+
return true;
|
|
694
|
+
} catch (refreshError) {
|
|
695
|
+
const message = refreshError instanceof Error ? refreshError.message : "unknown";
|
|
696
|
+
this.debug?.("Token refresh failed", { error: message });
|
|
697
|
+
return false;
|
|
698
|
+
} finally {
|
|
699
|
+
this.refreshInFlight = null;
|
|
700
|
+
}
|
|
701
|
+
})();
|
|
702
|
+
}
|
|
703
|
+
return this.refreshInFlight;
|
|
704
|
+
}
|
|
621
705
|
/**
|
|
622
706
|
* Build full URL with query parameters
|
|
623
707
|
*/
|
|
@@ -693,6 +777,7 @@ var HttpClient = class {
|
|
|
693
777
|
*/
|
|
694
778
|
async executeRequest(url, options, hasRefreshed = false) {
|
|
695
779
|
const method = options.method ?? "GET";
|
|
780
|
+
const tokenUsed = this.authToken;
|
|
696
781
|
const headers = this.buildHeaders(options.headers);
|
|
697
782
|
const timeout = options.timeout ?? this.config.timeout;
|
|
698
783
|
const init = {
|
|
@@ -707,6 +792,9 @@ var HttpClient = class {
|
|
|
707
792
|
init.body = JSON.stringify(body);
|
|
708
793
|
}
|
|
709
794
|
}
|
|
795
|
+
if (init.body === void 0) {
|
|
796
|
+
headers.delete("Content-Type");
|
|
797
|
+
}
|
|
710
798
|
this.debug?.(`${method} ${url}`, { timeout, hasBody: !!options.body });
|
|
711
799
|
let response;
|
|
712
800
|
try {
|
|
@@ -727,21 +815,11 @@ var HttpClient = class {
|
|
|
727
815
|
);
|
|
728
816
|
}
|
|
729
817
|
this.debug?.(`Response: ${response.status}`, { url });
|
|
818
|
+
this.recordRateLimit(response.headers);
|
|
730
819
|
if (!response.ok) {
|
|
731
|
-
if (response.status === 401 && this.refreshTokenFn && !hasRefreshed && !
|
|
732
|
-
|
|
733
|
-
this.
|
|
734
|
-
try {
|
|
735
|
-
const newToken = await this.refreshTokenFn();
|
|
736
|
-
this.setAuthToken(newToken);
|
|
737
|
-
refreshed = true;
|
|
738
|
-
} catch (refreshError) {
|
|
739
|
-
const message = refreshError instanceof Error ? refreshError.message : "unknown";
|
|
740
|
-
this.debug?.("Token refresh failed", { error: message });
|
|
741
|
-
} finally {
|
|
742
|
-
this.isRefreshing = false;
|
|
743
|
-
}
|
|
744
|
-
if (refreshed) {
|
|
820
|
+
if (response.status === 401 && this.refreshTokenFn && !hasRefreshed && !options.skipAuthRefresh) {
|
|
821
|
+
const alreadyRefreshed = this.authToken !== tokenUsed;
|
|
822
|
+
if (alreadyRefreshed || await this.refreshAuthToken()) {
|
|
745
823
|
return this.executeRequest(url, options, true);
|
|
746
824
|
}
|
|
747
825
|
}
|
|
@@ -764,6 +842,13 @@ var HttpClient = class {
|
|
|
764
842
|
data
|
|
765
843
|
};
|
|
766
844
|
}
|
|
845
|
+
/** Remember the rate-limit headers of a response that has them. */
|
|
846
|
+
recordRateLimit(headers) {
|
|
847
|
+
if (!headers || typeof headers.get !== "function" || headers.get("X-RateLimit-Limit") === null && headers.get("X-RateLimit-Remaining") === null) {
|
|
848
|
+
return;
|
|
849
|
+
}
|
|
850
|
+
this.lastRateLimit = parseRateLimitHeaders(headers);
|
|
851
|
+
}
|
|
767
852
|
/**
|
|
768
853
|
* Make an HTTP request with retry and circuit breaker
|
|
769
854
|
*/
|
|
@@ -803,22 +888,26 @@ var HttpClient = class {
|
|
|
803
888
|
/**
|
|
804
889
|
* Resolve the retry configuration for a request.
|
|
805
890
|
*
|
|
806
|
-
* Idempotent methods (GET/PUT/DELETE)
|
|
807
|
-
*
|
|
808
|
-
*
|
|
891
|
+
* Idempotent methods (GET/PUT/DELETE), any request carrying an idempotency
|
|
892
|
+
* key and any request marked `idempotent` retry normally. Non-idempotent
|
|
893
|
+
* methods (POST/PATCH) otherwise must NOT be retried on ambiguous failures
|
|
809
894
|
* (network/timeout/5xx) — doing so risks duplicate jobs, double claims,
|
|
810
|
-
* etc. They are still retried on 429, which is rejected before
|
|
811
|
-
* server processes the request and is therefore side-effect-free.
|
|
895
|
+
* etc. They are still retried on a rate-limit 429, which is rejected before
|
|
896
|
+
* the server processes the request and is therefore side-effect-free. A
|
|
897
|
+
* quota 429 is never retried: it does not clear for hours.
|
|
812
898
|
*/
|
|
813
899
|
resolveRetryConfig(method, options) {
|
|
814
900
|
const base = this.config.retry;
|
|
815
|
-
if (IDEMPOTENT_METHODS.has(method) || this.hasIdempotencyKey(options)) {
|
|
901
|
+
if (options.idempotent || IDEMPOTENT_METHODS.has(method) || this.hasIdempotencyKey(options)) {
|
|
816
902
|
return base;
|
|
817
903
|
}
|
|
818
904
|
return {
|
|
819
905
|
...base,
|
|
820
906
|
retryOn: (error, attempt) => {
|
|
821
907
|
if (error instanceof RateLimitError) {
|
|
908
|
+
if (error.isQuota) {
|
|
909
|
+
return false;
|
|
910
|
+
}
|
|
822
911
|
return base.retryOn ? base.retryOn(error, attempt) : true;
|
|
823
912
|
}
|
|
824
913
|
return false;
|
|
@@ -887,6 +976,12 @@ var CircuitState = /* @__PURE__ */ ((CircuitState2) => {
|
|
|
887
976
|
CircuitState2["HALF_OPEN"] = "HALF_OPEN";
|
|
888
977
|
return CircuitState2;
|
|
889
978
|
})(CircuitState || {});
|
|
979
|
+
function isCircuitBreakerFailure(error) {
|
|
980
|
+
if (error instanceof RateLimitError) {
|
|
981
|
+
return false;
|
|
982
|
+
}
|
|
983
|
+
return isRetryableError(error);
|
|
984
|
+
}
|
|
890
985
|
var CircuitBreaker = class {
|
|
891
986
|
state = "CLOSED" /* CLOSED */;
|
|
892
987
|
failureCount = 0;
|
|
@@ -947,7 +1042,7 @@ var CircuitBreaker = class {
|
|
|
947
1042
|
if (!this.config.enabled) {
|
|
948
1043
|
return;
|
|
949
1044
|
}
|
|
950
|
-
const shouldCount = this.config.isFailure ? this.config.isFailure(error) :
|
|
1045
|
+
const shouldCount = this.config.isFailure ? this.config.isFailure(error) : isCircuitBreakerFailure(error);
|
|
951
1046
|
if (!shouldCount) {
|
|
952
1047
|
return;
|
|
953
1048
|
}
|
|
@@ -1049,13 +1144,17 @@ var AuthResource = class {
|
|
|
1049
1144
|
* Exchange API key for JWT tokens
|
|
1050
1145
|
*/
|
|
1051
1146
|
async login(params) {
|
|
1052
|
-
return this.http.post("/auth/login", params
|
|
1147
|
+
return this.http.post("/auth/login", params, {
|
|
1148
|
+
skipAuthRefresh: true
|
|
1149
|
+
});
|
|
1053
1150
|
}
|
|
1054
1151
|
/**
|
|
1055
1152
|
* Refresh access token using refresh token
|
|
1056
1153
|
*/
|
|
1057
1154
|
async refresh(params) {
|
|
1058
|
-
return this.http.post("/auth/refresh", params
|
|
1155
|
+
return this.http.post("/auth/refresh", params, {
|
|
1156
|
+
skipAuthRefresh: true
|
|
1157
|
+
});
|
|
1059
1158
|
}
|
|
1060
1159
|
/**
|
|
1061
1160
|
* Logout and revoke the refresh token server-side.
|
|
@@ -1126,12 +1225,46 @@ function mapValidate(raw) {
|
|
|
1126
1225
|
};
|
|
1127
1226
|
}
|
|
1128
1227
|
|
|
1228
|
+
// src/utils/failure.ts
|
|
1229
|
+
var MAX_FAILURE_MESSAGE_CHARS = 2048;
|
|
1230
|
+
var DEFAULT_FAILURE_MESSAGE = "Job failed";
|
|
1231
|
+
function coerceFailureMessage(message) {
|
|
1232
|
+
const text = typeof message === "string" ? message : message == null ? "" : `${message}`;
|
|
1233
|
+
if (text === "") {
|
|
1234
|
+
return DEFAULT_FAILURE_MESSAGE;
|
|
1235
|
+
}
|
|
1236
|
+
if (text.length <= MAX_FAILURE_MESSAGE_CHARS) {
|
|
1237
|
+
return text;
|
|
1238
|
+
}
|
|
1239
|
+
const codePoints = Array.from(text);
|
|
1240
|
+
return codePoints.length <= MAX_FAILURE_MESSAGE_CHARS ? text : codePoints.slice(0, MAX_FAILURE_MESSAGE_CHARS).join("");
|
|
1241
|
+
}
|
|
1242
|
+
|
|
1243
|
+
// src/utils/paging.ts
|
|
1244
|
+
var HAS_MORE_HEADER = "X-Has-More";
|
|
1245
|
+
function hasMoreFromHeaders(headers, count, limit, defaultLimit, maxLimit) {
|
|
1246
|
+
const header = headers.get(HAS_MORE_HEADER);
|
|
1247
|
+
if (header !== null) {
|
|
1248
|
+
return header.trim().toLowerCase() === "true";
|
|
1249
|
+
}
|
|
1250
|
+
const applied = Math.min(Math.max(limit ?? defaultLimit, 1), maxLimit);
|
|
1251
|
+
return count >= applied;
|
|
1252
|
+
}
|
|
1253
|
+
function toQueryTimestamp(value) {
|
|
1254
|
+
return value instanceof Date ? value.toISOString() : value;
|
|
1255
|
+
}
|
|
1256
|
+
|
|
1129
1257
|
// src/resources/jobs.ts
|
|
1258
|
+
var JOB_LIST_DEFAULT_LIMIT = 50;
|
|
1259
|
+
var JOB_LIST_MAX_LIMIT = 100;
|
|
1260
|
+
var JOB_HISTORY_DEFAULT_LIMIT = 100;
|
|
1261
|
+
var JOB_HISTORY_MAX_LIMIT = 500;
|
|
1130
1262
|
var JobsResource = class {
|
|
1131
1263
|
constructor(http) {
|
|
1132
1264
|
this.http = http;
|
|
1133
1265
|
this.dlq = {
|
|
1134
1266
|
list: this.listDlq.bind(this),
|
|
1267
|
+
listPage: this.listDlqPage.bind(this),
|
|
1135
1268
|
retry: this.retryDlq.bind(this),
|
|
1136
1269
|
purge: this.purgeDlq.bind(this)
|
|
1137
1270
|
};
|
|
@@ -1142,7 +1275,9 @@ var JobsResource = class {
|
|
|
1142
1275
|
* Create a new job
|
|
1143
1276
|
*/
|
|
1144
1277
|
async create(params) {
|
|
1145
|
-
return this.http.post("/jobs", params
|
|
1278
|
+
return this.http.post("/jobs", params, {
|
|
1279
|
+
idempotent: params.idempotencyKey ? true : void 0
|
|
1280
|
+
});
|
|
1146
1281
|
}
|
|
1147
1282
|
/**
|
|
1148
1283
|
* Create a job and return the full job object
|
|
@@ -1153,15 +1288,36 @@ var JobsResource = class {
|
|
|
1153
1288
|
}
|
|
1154
1289
|
/**
|
|
1155
1290
|
* List jobs with optional filtering
|
|
1291
|
+
*
|
|
1292
|
+
* `status` takes one status or several, and `q`, `jobType`, `createdAfter`
|
|
1293
|
+
* and `createdBefore` filter on the server (API 0.1.122+). Use
|
|
1294
|
+
* {@link JobsResource.listPage} to also learn whether another page follows.
|
|
1156
1295
|
*/
|
|
1157
1296
|
async list(params) {
|
|
1158
|
-
|
|
1159
|
-
|
|
1160
|
-
|
|
1161
|
-
|
|
1162
|
-
|
|
1163
|
-
|
|
1164
|
-
|
|
1297
|
+
return (await this.listPage(params)).data;
|
|
1298
|
+
}
|
|
1299
|
+
/**
|
|
1300
|
+
* List one page of jobs with `hasMore` from the `X-Has-More` header
|
|
1301
|
+
* (API 0.1.122+). Takes the same filters as {@link JobsResource.list}.
|
|
1302
|
+
*
|
|
1303
|
+
* @example
|
|
1304
|
+
* ```typescript
|
|
1305
|
+
* let offset = 0;
|
|
1306
|
+
* for (;;) {
|
|
1307
|
+
* const page = await client.jobs.listPage({
|
|
1308
|
+
* status: ["failed", "deadletter"],
|
|
1309
|
+
* createdAfter: new Date(Date.now() - 86_400_000),
|
|
1310
|
+
* limit: 100,
|
|
1311
|
+
* offset,
|
|
1312
|
+
* });
|
|
1313
|
+
* handle(page.data);
|
|
1314
|
+
* if (!page.hasMore) break;
|
|
1315
|
+
* offset += page.data.length;
|
|
1316
|
+
* }
|
|
1317
|
+
* ```
|
|
1318
|
+
*/
|
|
1319
|
+
async listPage(params) {
|
|
1320
|
+
return this.listSummaries("/jobs", params);
|
|
1165
1321
|
}
|
|
1166
1322
|
/**
|
|
1167
1323
|
* Get a job by ID
|
|
@@ -1191,6 +1347,46 @@ var JobsResource = class {
|
|
|
1191
1347
|
priority
|
|
1192
1348
|
});
|
|
1193
1349
|
}
|
|
1350
|
+
/**
|
|
1351
|
+
* Get the recorded lifecycle events of a job (`GET /jobs/{id}/history`,
|
|
1352
|
+
* API 0.1.122+), oldest first unless `order: "desc"`.
|
|
1353
|
+
*
|
|
1354
|
+
* The history is sparse: the server records REST claims (`processing`),
|
|
1355
|
+
* completions (`completed`) and failures (`retry_scheduled`,
|
|
1356
|
+
* `deadlettered`) and the scheduler's dead-letter sweeps, so a job settled
|
|
1357
|
+
* over gRPC may have few or no events. An unknown job is a `NotFoundError`.
|
|
1358
|
+
*/
|
|
1359
|
+
async history(id, params) {
|
|
1360
|
+
return (await this.historyPage(id, params)).data;
|
|
1361
|
+
}
|
|
1362
|
+
/**
|
|
1363
|
+
* One page of {@link JobsResource.history} with `hasMore` from the
|
|
1364
|
+
* `X-Has-More` header. `limit` defaults to 100 (at most 500).
|
|
1365
|
+
*/
|
|
1366
|
+
async historyPage(id, params) {
|
|
1367
|
+
const response = await this.http.request(
|
|
1368
|
+
`/jobs/${id}/history`,
|
|
1369
|
+
{
|
|
1370
|
+
method: "GET",
|
|
1371
|
+
params: {
|
|
1372
|
+
order: params?.order,
|
|
1373
|
+
limit: params?.limit,
|
|
1374
|
+
offset: params?.offset
|
|
1375
|
+
}
|
|
1376
|
+
}
|
|
1377
|
+
);
|
|
1378
|
+
const data = response.data ?? [];
|
|
1379
|
+
return {
|
|
1380
|
+
data,
|
|
1381
|
+
hasMore: hasMoreFromHeaders(
|
|
1382
|
+
response.headers,
|
|
1383
|
+
data.length,
|
|
1384
|
+
params?.limit,
|
|
1385
|
+
JOB_HISTORY_DEFAULT_LIMIT,
|
|
1386
|
+
JOB_HISTORY_MAX_LIMIT
|
|
1387
|
+
)
|
|
1388
|
+
};
|
|
1389
|
+
}
|
|
1194
1390
|
/**
|
|
1195
1391
|
* Get job statistics
|
|
1196
1392
|
*/
|
|
@@ -1226,37 +1422,71 @@ var JobsResource = class {
|
|
|
1226
1422
|
}
|
|
1227
1423
|
/**
|
|
1228
1424
|
* Complete a job (worker ack)
|
|
1425
|
+
*
|
|
1426
|
+
* Lease-fenced and idempotent on the server, so a network error, timeout or
|
|
1427
|
+
* 5xx is retried: a lost ack would otherwise re-run a finished job.
|
|
1229
1428
|
*/
|
|
1230
1429
|
async complete(id, params) {
|
|
1231
|
-
return this.http.post(
|
|
1430
|
+
return this.http.post(
|
|
1431
|
+
`/jobs/${id}/complete`,
|
|
1432
|
+
params,
|
|
1433
|
+
{ idempotent: true }
|
|
1434
|
+
);
|
|
1232
1435
|
}
|
|
1233
1436
|
/**
|
|
1234
1437
|
* Fail a job (worker nack)
|
|
1438
|
+
*
|
|
1439
|
+
* `params.error` is made acceptable first: empty text becomes "Job failed"
|
|
1440
|
+
* and text over 2048 characters is truncated, because the API rejects both
|
|
1441
|
+
* with a 400 that would leave the job processing until its lease expires.
|
|
1442
|
+
* Retried like {@link JobsResource.complete}.
|
|
1235
1443
|
*/
|
|
1236
1444
|
async fail(id, params) {
|
|
1237
1445
|
return this.http.post(
|
|
1238
1446
|
`/jobs/${id}/fail`,
|
|
1239
|
-
params
|
|
1447
|
+
{ ...params, error: coerceFailureMessage(params.error) },
|
|
1448
|
+
{ idempotent: true }
|
|
1240
1449
|
);
|
|
1241
1450
|
}
|
|
1242
1451
|
/**
|
|
1243
1452
|
* Extend job lease (heartbeat)
|
|
1453
|
+
*
|
|
1454
|
+
* Retried like {@link JobsResource.complete}: with the default 30 s lease and
|
|
1455
|
+
* 15 s heartbeat, one lost heartbeat leaves no slack.
|
|
1244
1456
|
*/
|
|
1245
1457
|
async heartbeat(id, params) {
|
|
1246
1458
|
return this.http.post(
|
|
1247
1459
|
`/jobs/${id}/heartbeat`,
|
|
1248
|
-
params
|
|
1460
|
+
params,
|
|
1461
|
+
{ idempotent: true }
|
|
1249
1462
|
);
|
|
1250
1463
|
}
|
|
1251
1464
|
// DLQ operations (private implementations)
|
|
1252
1465
|
async listDlq(params) {
|
|
1253
|
-
|
|
1254
|
-
|
|
1466
|
+
return (await this.listDlqPage(params)).data;
|
|
1467
|
+
}
|
|
1468
|
+
async listDlqPage(params) {
|
|
1469
|
+
return this.listSummaries("/jobs/dlq", params);
|
|
1470
|
+
}
|
|
1471
|
+
async listSummaries(path, params) {
|
|
1472
|
+
const response = await this.http.request(path, {
|
|
1473
|
+
method: "GET",
|
|
1474
|
+
params: jobListQuery(params)
|
|
1255
1475
|
});
|
|
1256
|
-
|
|
1257
|
-
|
|
1258
|
-
|
|
1259
|
-
|
|
1476
|
+
const rows = response.data ?? [];
|
|
1477
|
+
return {
|
|
1478
|
+
data: rows.map((row) => ({
|
|
1479
|
+
...row,
|
|
1480
|
+
jobType: row.jobType ?? jobTypeFromPayload(row)
|
|
1481
|
+
})),
|
|
1482
|
+
hasMore: hasMoreFromHeaders(
|
|
1483
|
+
response.headers,
|
|
1484
|
+
rows.length,
|
|
1485
|
+
params?.limit,
|
|
1486
|
+
JOB_LIST_DEFAULT_LIMIT,
|
|
1487
|
+
JOB_LIST_MAX_LIMIT
|
|
1488
|
+
)
|
|
1489
|
+
};
|
|
1260
1490
|
}
|
|
1261
1491
|
async retryDlq(params) {
|
|
1262
1492
|
return this.http.post("/jobs/dlq/retry", params);
|
|
@@ -1265,6 +1495,24 @@ var JobsResource = class {
|
|
|
1265
1495
|
return this.http.post("/jobs/dlq/purge", params);
|
|
1266
1496
|
}
|
|
1267
1497
|
};
|
|
1498
|
+
function jobListQuery(params) {
|
|
1499
|
+
if (!params) {
|
|
1500
|
+
return void 0;
|
|
1501
|
+
}
|
|
1502
|
+
const { status: status2, createdAfter, createdBefore, ...rest } = params;
|
|
1503
|
+
let statusParam;
|
|
1504
|
+
if (typeof status2 === "string") {
|
|
1505
|
+
statusParam = status2;
|
|
1506
|
+
} else if (status2 && status2.length > 0) {
|
|
1507
|
+
statusParam = status2.join(",");
|
|
1508
|
+
}
|
|
1509
|
+
return {
|
|
1510
|
+
...rest,
|
|
1511
|
+
status: statusParam,
|
|
1512
|
+
createdAfter: toQueryTimestamp(createdAfter),
|
|
1513
|
+
createdBefore: toQueryTimestamp(createdBefore)
|
|
1514
|
+
};
|
|
1515
|
+
}
|
|
1268
1516
|
function withJobType(job) {
|
|
1269
1517
|
return {
|
|
1270
1518
|
...job,
|
|
@@ -1341,9 +1589,17 @@ var QueuesResource = class {
|
|
|
1341
1589
|
}
|
|
1342
1590
|
/**
|
|
1343
1591
|
* Delete a queue configuration
|
|
1344
|
-
|
|
1345
|
-
|
|
1346
|
-
|
|
1592
|
+
*
|
|
1593
|
+
* Without `deleteJobs` the API only deletes an explicit queue config of an
|
|
1594
|
+
* idle queue, and answers 409 otherwise: a queue that exists only because
|
|
1595
|
+
* jobs were posted to it has no config to delete, and a queue with pending
|
|
1596
|
+
* or processing jobs is not idle. Pass `{ deleteJobs: true }` to delete the
|
|
1597
|
+
* queue's jobs (and DLQ rows) along with it.
|
|
1598
|
+
*/
|
|
1599
|
+
async delete(name, options) {
|
|
1600
|
+
await this.http.delete(`/queues/${encodeURIComponent(name)}`, {
|
|
1601
|
+
params: options?.deleteJobs ? { deleteJobs: true } : void 0
|
|
1602
|
+
});
|
|
1347
1603
|
}
|
|
1348
1604
|
};
|
|
1349
1605
|
|
|
@@ -1372,9 +1628,13 @@ var WorkersResource = class {
|
|
|
1372
1628
|
}
|
|
1373
1629
|
/**
|
|
1374
1630
|
* Send worker heartbeat
|
|
1631
|
+
*
|
|
1632
|
+
* Idempotent on the server, so network errors, timeouts and 5xx are retried.
|
|
1375
1633
|
*/
|
|
1376
1634
|
async heartbeat(id, params) {
|
|
1377
|
-
await this.http.post(`/workers/${id}/heartbeat`, params
|
|
1635
|
+
await this.http.post(`/workers/${id}/heartbeat`, params, {
|
|
1636
|
+
idempotent: true
|
|
1637
|
+
});
|
|
1378
1638
|
}
|
|
1379
1639
|
/**
|
|
1380
1640
|
* Deregister a worker
|
|
@@ -1523,7 +1783,7 @@ var WorkflowsResource = class {
|
|
|
1523
1783
|
const jobs = await this.listWorkflowJobs(workflowId);
|
|
1524
1784
|
return jobs.map((job) => ({
|
|
1525
1785
|
jobId: job.id,
|
|
1526
|
-
key: job.key,
|
|
1786
|
+
...job.key !== void 0 && { key: job.key },
|
|
1527
1787
|
status: job.status
|
|
1528
1788
|
}));
|
|
1529
1789
|
}
|
|
@@ -1581,7 +1841,10 @@ function mapWorkflowDetailJobs(detail) {
|
|
|
1581
1841
|
return {
|
|
1582
1842
|
id,
|
|
1583
1843
|
workflowId: String(job.workflowId ?? workflowId),
|
|
1584
|
-
key
|
|
1844
|
+
// The API does not return the creation-time key (jobs store only a
|
|
1845
|
+
// workflow step), so there is nothing to put here unless a future
|
|
1846
|
+
// version sends one. Never fabricate "".
|
|
1847
|
+
...typeof job.key === "string" && job.key !== "" && { key: job.key },
|
|
1585
1848
|
queueName: String(job.queueName ?? job.queue ?? ""),
|
|
1586
1849
|
status: job.status,
|
|
1587
1850
|
payload: job.payload ?? {},
|
|
@@ -1793,19 +2056,18 @@ var OrganizationsResource = class {
|
|
|
1793
2056
|
/**
|
|
1794
2057
|
* Clear/delete the webhook token for the current organization.
|
|
1795
2058
|
*
|
|
1796
|
-
*
|
|
1797
|
-
*
|
|
1798
|
-
*
|
|
1799
|
-
*
|
|
1800
|
-
* @
|
|
1801
|
-
*
|
|
1802
|
-
* await client.organizations.clearWebhookToken();
|
|
1803
|
-
* ```
|
|
2059
|
+
* @deprecated The API refuses this operation on purpose (an org without a
|
|
2060
|
+
* webhook token would have an open public ingestion endpoint) and always
|
|
2061
|
+
* answers with an error. This method now throws
|
|
2062
|
+
* {@link UnsupportedOperationError} without a network call; rotate the token
|
|
2063
|
+
* with {@link OrganizationsResource.regenerateWebhookToken} instead. It will
|
|
2064
|
+
* be removed in the next major release.
|
|
1804
2065
|
*/
|
|
1805
2066
|
async clearWebhookToken() {
|
|
1806
|
-
|
|
1807
|
-
|
|
1808
|
-
|
|
2067
|
+
throw new UnsupportedOperationError(
|
|
2068
|
+
"organizations.clearWebhookToken",
|
|
2069
|
+
"clearing the webhook token is not allowed; use organizations.regenerateWebhookToken() to rotate it"
|
|
2070
|
+
);
|
|
1809
2071
|
}
|
|
1810
2072
|
};
|
|
1811
2073
|
|
|
@@ -2013,24 +2275,8 @@ var AdminResource = class {
|
|
|
2013
2275
|
});
|
|
2014
2276
|
}
|
|
2015
2277
|
};
|
|
2016
|
-
|
|
2017
|
-
|
|
2018
|
-
return buffer.Buffer.from(body, "utf8");
|
|
2019
|
-
}
|
|
2020
|
-
if (body instanceof Uint8Array) {
|
|
2021
|
-
return body;
|
|
2022
|
-
}
|
|
2023
|
-
return new Uint8Array(body);
|
|
2024
|
-
}
|
|
2025
|
-
function githubSignature(secret, body) {
|
|
2026
|
-
const hex = crypto.createHmac("sha256", secret).update(body).digest("hex");
|
|
2027
|
-
return `sha256=${hex}`;
|
|
2028
|
-
}
|
|
2029
|
-
function stripeSignature(secret, body, timestamp) {
|
|
2030
|
-
const payload = `${timestamp}.${buffer.Buffer.from(body).toString("utf8")}`;
|
|
2031
|
-
const hex = crypto.createHmac("sha256", secret).update(payload, "utf8").digest("hex");
|
|
2032
|
-
return `t=${timestamp},v1=${hex}`;
|
|
2033
|
-
}
|
|
2278
|
+
|
|
2279
|
+
// src/resources/webhook-ingestion.ts
|
|
2034
2280
|
var WebhookIngestionResource = class {
|
|
2035
2281
|
constructor(http) {
|
|
2036
2282
|
this.http = http;
|
|
@@ -2038,66 +2284,32 @@ var WebhookIngestionResource = class {
|
|
|
2038
2284
|
/**
|
|
2039
2285
|
* Ingest a GitHub webhook.
|
|
2040
2286
|
*
|
|
2041
|
-
*
|
|
2042
|
-
|
|
2043
|
-
|
|
2044
|
-
|
|
2045
|
-
|
|
2046
|
-
|
|
2047
|
-
|
|
2048
|
-
|
|
2049
|
-
|
|
2050
|
-
|
|
2051
|
-
|
|
2052
|
-
"Content-Type": "application/json",
|
|
2053
|
-
"X-GitHub-Event": options.githubEvent,
|
|
2054
|
-
"X-Hub-Signature-256": signature
|
|
2055
|
-
};
|
|
2056
|
-
if (options.webhookToken) {
|
|
2057
|
-
headers["X-Webhook-Token"] = options.webhookToken;
|
|
2058
|
-
}
|
|
2059
|
-
if (options.forwardedProto) {
|
|
2060
|
-
headers["X-Forwarded-Proto"] = options.forwardedProto;
|
|
2061
|
-
}
|
|
2062
|
-
await this.http.request(`/webhooks/${orgId}/github`, {
|
|
2063
|
-
method: "POST",
|
|
2064
|
-
rawBody: bytes,
|
|
2065
|
-
headers,
|
|
2066
|
-
skipRequestConversion: true,
|
|
2067
|
-
skipResponseConversion: true
|
|
2068
|
-
});
|
|
2287
|
+
* @deprecated The API has no `/webhooks/{org_id}/github` route (it always
|
|
2288
|
+
* answered 404). This method now throws {@link UnsupportedOperationError}
|
|
2289
|
+
* without a network call; point the GitHub webhook at `ingest.custom` /
|
|
2290
|
+
* `POST /api/v1/webhooks/{org_id}/custom` instead. It will be removed in the
|
|
2291
|
+
* next major release.
|
|
2292
|
+
*/
|
|
2293
|
+
async github(_orgId, _body, _options) {
|
|
2294
|
+
throw new UnsupportedOperationError(
|
|
2295
|
+
"ingest.github",
|
|
2296
|
+
"the API has no GitHub webhook route; use ingest.custom (POST /webhooks/{org_id}/custom)"
|
|
2297
|
+
);
|
|
2069
2298
|
}
|
|
2070
2299
|
/**
|
|
2071
2300
|
* Ingest a Stripe webhook.
|
|
2072
2301
|
*
|
|
2073
|
-
*
|
|
2074
|
-
|
|
2075
|
-
|
|
2076
|
-
|
|
2077
|
-
|
|
2078
|
-
|
|
2079
|
-
|
|
2080
|
-
|
|
2081
|
-
|
|
2082
|
-
)
|
|
2083
|
-
|
|
2084
|
-
const headers = {
|
|
2085
|
-
"Content-Type": "application/json",
|
|
2086
|
-
"Stripe-Signature": signature
|
|
2087
|
-
};
|
|
2088
|
-
if (options.webhookToken) {
|
|
2089
|
-
headers["X-Webhook-Token"] = options.webhookToken;
|
|
2090
|
-
}
|
|
2091
|
-
if (options.forwardedProto) {
|
|
2092
|
-
headers["X-Forwarded-Proto"] = options.forwardedProto;
|
|
2093
|
-
}
|
|
2094
|
-
await this.http.request(`/webhooks/${orgId}/stripe`, {
|
|
2095
|
-
method: "POST",
|
|
2096
|
-
rawBody: bytes,
|
|
2097
|
-
headers,
|
|
2098
|
-
skipRequestConversion: true,
|
|
2099
|
-
skipResponseConversion: true
|
|
2100
|
-
});
|
|
2302
|
+
* @deprecated The API has no `/webhooks/{org_id}/stripe` route (it always
|
|
2303
|
+
* answered 404). This method now throws {@link UnsupportedOperationError}
|
|
2304
|
+
* without a network call; point the Stripe webhook at `ingest.custom` /
|
|
2305
|
+
* `POST /api/v1/webhooks/{org_id}/custom` instead. It will be removed in the
|
|
2306
|
+
* next major release.
|
|
2307
|
+
*/
|
|
2308
|
+
async stripe(_orgId, _body, _options = {}) {
|
|
2309
|
+
throw new UnsupportedOperationError(
|
|
2310
|
+
"ingest.stripe",
|
|
2311
|
+
"the API has no Stripe webhook route; use ingest.custom (POST /webhooks/{org_id}/custom)"
|
|
2312
|
+
);
|
|
2101
2313
|
}
|
|
2102
2314
|
/**
|
|
2103
2315
|
* Ingest a custom webhook.
|
|
@@ -2124,6 +2336,31 @@ var WebhookIngestionResource = class {
|
|
|
2124
2336
|
status: raw.status
|
|
2125
2337
|
};
|
|
2126
2338
|
}
|
|
2339
|
+
/**
|
|
2340
|
+
* The inbound webhook log: accepted requests to
|
|
2341
|
+
* `POST /webhooks/{org_id}/custom`, newest first
|
|
2342
|
+
* (`GET /webhooks/deliveries`, API 0.1.122+).
|
|
2343
|
+
*
|
|
2344
|
+
* Needs an API key without a queue restriction (a queue-scoped key gets a
|
|
2345
|
+
* 403 `AuthorizationError`). Entries carry no payload and no signature;
|
|
2346
|
+
* requests that were rejected are not logged. `limit` defaults to 50 (at
|
|
2347
|
+
* most 100); an invalid filter is a 400 `ValidationError`.
|
|
2348
|
+
*/
|
|
2349
|
+
async listDeliveries(params) {
|
|
2350
|
+
const rows = await this.http.get(
|
|
2351
|
+
"/webhooks/deliveries",
|
|
2352
|
+
{
|
|
2353
|
+
params: {
|
|
2354
|
+
status: params?.status,
|
|
2355
|
+
since: toQueryTimestamp(params?.since),
|
|
2356
|
+
queueName: params?.queueName,
|
|
2357
|
+
limit: params?.limit,
|
|
2358
|
+
offset: params?.offset
|
|
2359
|
+
}
|
|
2360
|
+
}
|
|
2361
|
+
);
|
|
2362
|
+
return rows ?? [];
|
|
2363
|
+
}
|
|
2127
2364
|
};
|
|
2128
2365
|
|
|
2129
2366
|
// src/realtime/event-map.ts
|
|
@@ -2474,6 +2711,13 @@ var SSE_EVENT_NAMES = [
|
|
|
2474
2711
|
"ping",
|
|
2475
2712
|
"error"
|
|
2476
2713
|
];
|
|
2714
|
+
var TERMINAL_JOB_STATUSES = /* @__PURE__ */ new Set([
|
|
2715
|
+
"completed",
|
|
2716
|
+
"failed",
|
|
2717
|
+
"deadletter",
|
|
2718
|
+
"cancelled"
|
|
2719
|
+
]);
|
|
2720
|
+
var STABLE_CONNECTION_MS = 3e4;
|
|
2477
2721
|
var SseRealtimeClient = class {
|
|
2478
2722
|
options;
|
|
2479
2723
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
@@ -2482,6 +2726,12 @@ var SseRealtimeClient = class {
|
|
|
2482
2726
|
reconnectAttempts = 0;
|
|
2483
2727
|
reconnectTimer = null;
|
|
2484
2728
|
filter = null;
|
|
2729
|
+
/** When the current connection opened (for the stable-connection check). */
|
|
2730
|
+
connectedAt = null;
|
|
2731
|
+
/** The server finished this stream on purpose (terminal job); do not reconnect. */
|
|
2732
|
+
streamEnded = false;
|
|
2733
|
+
/** The server rejected our token; mint a fresh one for the next connect. */
|
|
2734
|
+
forceTokenRefresh = false;
|
|
2485
2735
|
// Event handlers
|
|
2486
2736
|
eventHandlers = /* @__PURE__ */ new Map();
|
|
2487
2737
|
allEventsHandlers = /* @__PURE__ */ new Set();
|
|
@@ -2520,6 +2770,7 @@ var SseRealtimeClient = class {
|
|
|
2520
2770
|
return;
|
|
2521
2771
|
}
|
|
2522
2772
|
this.filter = filter || null;
|
|
2773
|
+
this.streamEnded = false;
|
|
2523
2774
|
this.setState("connecting");
|
|
2524
2775
|
const sseUrl = this.buildSseUrl(filter);
|
|
2525
2776
|
this.options.debug(`Connecting to SSE: ${sseUrl}`);
|
|
@@ -2539,7 +2790,9 @@ var SseRealtimeClient = class {
|
|
|
2539
2790
|
}
|
|
2540
2791
|
let token;
|
|
2541
2792
|
try {
|
|
2542
|
-
|
|
2793
|
+
const force = this.forceTokenRefresh;
|
|
2794
|
+
this.forceTokenRefresh = false;
|
|
2795
|
+
token = await this.options.tokenProvider(force);
|
|
2543
2796
|
} catch (error) {
|
|
2544
2797
|
this.setState("disconnected");
|
|
2545
2798
|
throw new Error(`Failed to acquire realtime token: ${error}`);
|
|
@@ -2559,7 +2812,7 @@ var SseRealtimeClient = class {
|
|
|
2559
2812
|
this.eventSource.onopen = () => {
|
|
2560
2813
|
this.options.debug("SSE connected");
|
|
2561
2814
|
this.setState("connected");
|
|
2562
|
-
this.
|
|
2815
|
+
this.connectedAt = Date.now();
|
|
2563
2816
|
resolve();
|
|
2564
2817
|
};
|
|
2565
2818
|
this.eventSource.onmessage = (event) => {
|
|
@@ -2570,8 +2823,11 @@ var SseRealtimeClient = class {
|
|
|
2570
2823
|
if (typeof event?.data === "string") this.handleMessage(event.data);
|
|
2571
2824
|
});
|
|
2572
2825
|
}
|
|
2573
|
-
this.eventSource.onerror = () => {
|
|
2826
|
+
this.eventSource.onerror = (event) => {
|
|
2574
2827
|
this.options.debug("SSE error");
|
|
2828
|
+
if (event?.code === 401 || event?.code === 403) {
|
|
2829
|
+
this.forceTokenRefresh = true;
|
|
2830
|
+
}
|
|
2575
2831
|
if (this.state === "connecting") {
|
|
2576
2832
|
this.setState("disconnected");
|
|
2577
2833
|
if (this.eventSource) {
|
|
@@ -2603,6 +2859,8 @@ var SseRealtimeClient = class {
|
|
|
2603
2859
|
}
|
|
2604
2860
|
this.setState("disconnected");
|
|
2605
2861
|
this.filter = null;
|
|
2862
|
+
this.connectedAt = null;
|
|
2863
|
+
this.streamEnded = false;
|
|
2606
2864
|
}
|
|
2607
2865
|
/**
|
|
2608
2866
|
* Reconnect with a new filter (SSE requires reconnection to change subscriptions)
|
|
@@ -2667,6 +2925,9 @@ var SseRealtimeClient = class {
|
|
|
2667
2925
|
parsed.type = eventType;
|
|
2668
2926
|
const event = parsed;
|
|
2669
2927
|
this.options.debug(`Received SSE event: ${eventType}`, event);
|
|
2928
|
+
if (this.filter?.jobId && isStreamEndingEvent(eventType, parsed.data)) {
|
|
2929
|
+
this.streamEnded = true;
|
|
2930
|
+
}
|
|
2670
2931
|
const handlers = this.eventHandlers.get(eventType);
|
|
2671
2932
|
if (handlers) {
|
|
2672
2933
|
const eventData = event.data;
|
|
@@ -2694,6 +2955,15 @@ var SseRealtimeClient = class {
|
|
|
2694
2955
|
this.eventSource.close();
|
|
2695
2956
|
this.eventSource = null;
|
|
2696
2957
|
}
|
|
2958
|
+
if (this.streamEnded) {
|
|
2959
|
+
this.connectedAt = null;
|
|
2960
|
+
this.setState("disconnected");
|
|
2961
|
+
return;
|
|
2962
|
+
}
|
|
2963
|
+
if (this.connectedAt !== null && Date.now() - this.connectedAt >= STABLE_CONNECTION_MS) {
|
|
2964
|
+
this.reconnectAttempts = 0;
|
|
2965
|
+
}
|
|
2966
|
+
this.connectedAt = null;
|
|
2697
2967
|
if (this.options.autoReconnect && this.reconnectAttempts < this.options.maxReconnectAttempts) {
|
|
2698
2968
|
this.scheduleReconnect();
|
|
2699
2969
|
} else {
|
|
@@ -2725,6 +2995,19 @@ var SseRealtimeClient = class {
|
|
|
2725
2995
|
}, delay);
|
|
2726
2996
|
}
|
|
2727
2997
|
};
|
|
2998
|
+
function isStreamEndingEvent(eventType, data) {
|
|
2999
|
+
if (eventType === "error" || eventType === "job.completed") {
|
|
3000
|
+
return true;
|
|
3001
|
+
}
|
|
3002
|
+
const record = data ?? {};
|
|
3003
|
+
if (eventType === "job.failed") {
|
|
3004
|
+
return record.willRetry === false;
|
|
3005
|
+
}
|
|
3006
|
+
if (eventType === "job.status_changed" || eventType === "job.status") {
|
|
3007
|
+
return typeof record.newStatus === "string" && TERMINAL_JOB_STATUSES.has(record.newStatus);
|
|
3008
|
+
}
|
|
3009
|
+
return false;
|
|
3010
|
+
}
|
|
2728
3011
|
|
|
2729
3012
|
// src/realtime/index.ts
|
|
2730
3013
|
var SpooledRealtime = class {
|
|
@@ -2911,114 +3194,10 @@ function clearProtoCache() {
|
|
|
2911
3194
|
cachedDefinition = null;
|
|
2912
3195
|
}
|
|
2913
3196
|
|
|
2914
|
-
// src/grpc/
|
|
2915
|
-
|
|
2916
|
-
|
|
2917
|
-
if (signal) {
|
|
2918
|
-
signal.addEventListener("abort", () => {
|
|
2919
|
-
stream.cancel();
|
|
2920
|
-
});
|
|
2921
|
-
}
|
|
2922
|
-
return yield* new AsyncIterableStream(stream, {
|
|
2923
|
-
onConnected,
|
|
2924
|
-
onError,
|
|
2925
|
-
onEnd
|
|
2926
|
-
});
|
|
3197
|
+
// src/grpc/encode.ts
|
|
3198
|
+
function encodeFailRequest(params) {
|
|
3199
|
+
return { ...params, retry: params.retry ?? true };
|
|
2927
3200
|
}
|
|
2928
|
-
var AsyncIterableStream = class {
|
|
2929
|
-
stream;
|
|
2930
|
-
options;
|
|
2931
|
-
constructor(stream, options = {}) {
|
|
2932
|
-
this.stream = stream;
|
|
2933
|
-
this.options = options;
|
|
2934
|
-
}
|
|
2935
|
-
async *[Symbol.asyncIterator]() {
|
|
2936
|
-
const { onConnected, onError, onEnd } = this.options;
|
|
2937
|
-
const queue = [];
|
|
2938
|
-
let error = null;
|
|
2939
|
-
let ended = false;
|
|
2940
|
-
let resolve = null;
|
|
2941
|
-
this.stream.on("data", (data) => {
|
|
2942
|
-
queue.push(data);
|
|
2943
|
-
resolve?.();
|
|
2944
|
-
});
|
|
2945
|
-
this.stream.on("error", (err) => {
|
|
2946
|
-
error = err;
|
|
2947
|
-
onError?.(err);
|
|
2948
|
-
resolve?.();
|
|
2949
|
-
});
|
|
2950
|
-
this.stream.on("end", () => {
|
|
2951
|
-
ended = true;
|
|
2952
|
-
onEnd?.();
|
|
2953
|
-
resolve?.();
|
|
2954
|
-
});
|
|
2955
|
-
this.stream.on("status", (status2) => {
|
|
2956
|
-
if (status2.code === 0) {
|
|
2957
|
-
onConnected?.();
|
|
2958
|
-
}
|
|
2959
|
-
});
|
|
2960
|
-
while (true) {
|
|
2961
|
-
if (queue.length > 0) {
|
|
2962
|
-
yield queue.shift();
|
|
2963
|
-
continue;
|
|
2964
|
-
}
|
|
2965
|
-
if (error) {
|
|
2966
|
-
throw error;
|
|
2967
|
-
}
|
|
2968
|
-
if (ended) {
|
|
2969
|
-
return;
|
|
2970
|
-
}
|
|
2971
|
-
await new Promise((r) => {
|
|
2972
|
-
resolve = r;
|
|
2973
|
-
});
|
|
2974
|
-
}
|
|
2975
|
-
}
|
|
2976
|
-
};
|
|
2977
|
-
function createProcessJobsStream(call, options) {
|
|
2978
|
-
const { signal, onError, onEnd } = options ?? {};
|
|
2979
|
-
if (signal) {
|
|
2980
|
-
signal.addEventListener("abort", () => {
|
|
2981
|
-
call.cancel();
|
|
2982
|
-
});
|
|
2983
|
-
}
|
|
2984
|
-
return {
|
|
2985
|
-
send(request) {
|
|
2986
|
-
call.write(request);
|
|
2987
|
-
},
|
|
2988
|
-
receive() {
|
|
2989
|
-
return new AsyncIterableStream(call, { onError, onEnd });
|
|
2990
|
-
},
|
|
2991
|
-
end() {
|
|
2992
|
-
call.end();
|
|
2993
|
-
},
|
|
2994
|
-
cancel() {
|
|
2995
|
-
call.cancel();
|
|
2996
|
-
}
|
|
2997
|
-
};
|
|
2998
|
-
}
|
|
2999
|
-
function createJobStream(call, options) {
|
|
3000
|
-
const { signal, onConnected, onError, onEnd } = options ?? {};
|
|
3001
|
-
if (signal) {
|
|
3002
|
-
signal.addEventListener("abort", () => {
|
|
3003
|
-
call.cancel();
|
|
3004
|
-
});
|
|
3005
|
-
}
|
|
3006
|
-
const iterable = new AsyncIterableStream(call, {
|
|
3007
|
-
onConnected,
|
|
3008
|
-
onError,
|
|
3009
|
-
onEnd
|
|
3010
|
-
});
|
|
3011
|
-
return {
|
|
3012
|
-
[Symbol.asyncIterator]() {
|
|
3013
|
-
return iterable[Symbol.asyncIterator]();
|
|
3014
|
-
},
|
|
3015
|
-
cancel() {
|
|
3016
|
-
call.cancel();
|
|
3017
|
-
}
|
|
3018
|
-
};
|
|
3019
|
-
}
|
|
3020
|
-
|
|
3021
|
-
// src/grpc/client.ts
|
|
3022
3201
|
function encodeStruct(value) {
|
|
3023
3202
|
if (value == null) {
|
|
3024
3203
|
return void 0;
|
|
@@ -3113,10 +3292,157 @@ function encodeCompleteRequest(params) {
|
|
|
3113
3292
|
function decodeJob(job) {
|
|
3114
3293
|
return {
|
|
3115
3294
|
...job,
|
|
3295
|
+
// proto-loader keeps an enum number it does not know (a status added to
|
|
3296
|
+
// the server's proto after this SDK) as a number; keep it a string so
|
|
3297
|
+
// `status` always matches its type.
|
|
3298
|
+
status: typeof job.status === "number" ? String(job.status) : job.status,
|
|
3116
3299
|
payload: decodeStruct(job.payload),
|
|
3117
3300
|
result: decodeStruct(job.result)
|
|
3118
3301
|
};
|
|
3119
3302
|
}
|
|
3303
|
+
function encodeProcessRequest(request) {
|
|
3304
|
+
if (request.fail) {
|
|
3305
|
+
return { ...request, fail: encodeFailRequest(request.fail) };
|
|
3306
|
+
}
|
|
3307
|
+
if (request.complete) {
|
|
3308
|
+
return { ...request, complete: encodeCompleteRequest(request.complete) };
|
|
3309
|
+
}
|
|
3310
|
+
return request;
|
|
3311
|
+
}
|
|
3312
|
+
function decodeProcessResponse(response) {
|
|
3313
|
+
return response.job ? { ...response, job: decodeJob(response.job) } : response;
|
|
3314
|
+
}
|
|
3315
|
+
|
|
3316
|
+
// src/grpc/streaming.ts
|
|
3317
|
+
async function* asyncIterableFromStream(stream, options) {
|
|
3318
|
+
const { signal, onConnected, onError, onEnd } = options ?? {};
|
|
3319
|
+
if (signal) {
|
|
3320
|
+
signal.addEventListener("abort", () => {
|
|
3321
|
+
stream.cancel();
|
|
3322
|
+
});
|
|
3323
|
+
}
|
|
3324
|
+
return yield* new AsyncIterableStream(stream, {
|
|
3325
|
+
onConnected,
|
|
3326
|
+
onError,
|
|
3327
|
+
onEnd
|
|
3328
|
+
});
|
|
3329
|
+
}
|
|
3330
|
+
var AsyncIterableStream = class {
|
|
3331
|
+
stream;
|
|
3332
|
+
options;
|
|
3333
|
+
decode;
|
|
3334
|
+
constructor(stream, options = {}, decode = (item) => item) {
|
|
3335
|
+
this.stream = stream;
|
|
3336
|
+
this.options = options;
|
|
3337
|
+
this.decode = decode;
|
|
3338
|
+
}
|
|
3339
|
+
async *[Symbol.asyncIterator]() {
|
|
3340
|
+
const { onConnected, onError, onEnd } = this.options;
|
|
3341
|
+
const queue = [];
|
|
3342
|
+
let error = null;
|
|
3343
|
+
let ended = false;
|
|
3344
|
+
let decodeFailed = false;
|
|
3345
|
+
let resolve = null;
|
|
3346
|
+
this.stream.on("data", (data) => {
|
|
3347
|
+
if (decodeFailed) {
|
|
3348
|
+
return;
|
|
3349
|
+
}
|
|
3350
|
+
try {
|
|
3351
|
+
queue.push(this.decode(data));
|
|
3352
|
+
} catch (decodeError) {
|
|
3353
|
+
decodeFailed = true;
|
|
3354
|
+
error = decodeError instanceof Error ? decodeError : new Error(String(decodeError));
|
|
3355
|
+
try {
|
|
3356
|
+
this.stream.cancel?.();
|
|
3357
|
+
} catch {
|
|
3358
|
+
}
|
|
3359
|
+
}
|
|
3360
|
+
resolve?.();
|
|
3361
|
+
});
|
|
3362
|
+
this.stream.on("error", (err) => {
|
|
3363
|
+
if (decodeFailed) {
|
|
3364
|
+
return;
|
|
3365
|
+
}
|
|
3366
|
+
error = err;
|
|
3367
|
+
onError?.(err);
|
|
3368
|
+
resolve?.();
|
|
3369
|
+
});
|
|
3370
|
+
this.stream.on("end", () => {
|
|
3371
|
+
ended = true;
|
|
3372
|
+
onEnd?.();
|
|
3373
|
+
resolve?.();
|
|
3374
|
+
});
|
|
3375
|
+
this.stream.on("status", (status2) => {
|
|
3376
|
+
if (status2.code === 0) {
|
|
3377
|
+
onConnected?.();
|
|
3378
|
+
}
|
|
3379
|
+
});
|
|
3380
|
+
while (true) {
|
|
3381
|
+
if (queue.length > 0) {
|
|
3382
|
+
yield queue.shift();
|
|
3383
|
+
continue;
|
|
3384
|
+
}
|
|
3385
|
+
if (error) {
|
|
3386
|
+
throw error;
|
|
3387
|
+
}
|
|
3388
|
+
if (ended) {
|
|
3389
|
+
return;
|
|
3390
|
+
}
|
|
3391
|
+
await new Promise((r) => {
|
|
3392
|
+
resolve = r;
|
|
3393
|
+
});
|
|
3394
|
+
}
|
|
3395
|
+
}
|
|
3396
|
+
};
|
|
3397
|
+
function createProcessJobsStream(call, options) {
|
|
3398
|
+
const { signal, onError, onEnd } = options ?? {};
|
|
3399
|
+
if (signal) {
|
|
3400
|
+
signal.addEventListener("abort", () => {
|
|
3401
|
+
call.cancel();
|
|
3402
|
+
});
|
|
3403
|
+
}
|
|
3404
|
+
return {
|
|
3405
|
+
send(request) {
|
|
3406
|
+
call.write(encodeProcessRequest(request));
|
|
3407
|
+
},
|
|
3408
|
+
receive() {
|
|
3409
|
+
return new AsyncIterableStream(
|
|
3410
|
+
call,
|
|
3411
|
+
{ onError, onEnd },
|
|
3412
|
+
decodeProcessResponse
|
|
3413
|
+
);
|
|
3414
|
+
},
|
|
3415
|
+
end() {
|
|
3416
|
+
call.end();
|
|
3417
|
+
},
|
|
3418
|
+
cancel() {
|
|
3419
|
+
call.cancel();
|
|
3420
|
+
}
|
|
3421
|
+
};
|
|
3422
|
+
}
|
|
3423
|
+
function createJobStream(call, options) {
|
|
3424
|
+
const { signal, onConnected, onError, onEnd } = options ?? {};
|
|
3425
|
+
if (signal) {
|
|
3426
|
+
signal.addEventListener("abort", () => {
|
|
3427
|
+
call.cancel();
|
|
3428
|
+
});
|
|
3429
|
+
}
|
|
3430
|
+
const iterable = new AsyncIterableStream(
|
|
3431
|
+
call,
|
|
3432
|
+
{ onConnected, onError, onEnd },
|
|
3433
|
+
decodeJob
|
|
3434
|
+
);
|
|
3435
|
+
return {
|
|
3436
|
+
[Symbol.asyncIterator]() {
|
|
3437
|
+
return iterable[Symbol.asyncIterator]();
|
|
3438
|
+
},
|
|
3439
|
+
cancel() {
|
|
3440
|
+
call.cancel();
|
|
3441
|
+
}
|
|
3442
|
+
};
|
|
3443
|
+
}
|
|
3444
|
+
|
|
3445
|
+
// src/grpc/client.ts
|
|
3120
3446
|
function promisify(client, method, request, metadata, timeoutMs) {
|
|
3121
3447
|
return new Promise((resolve, reject) => {
|
|
3122
3448
|
const callback = (error, response) => {
|
|
@@ -3171,12 +3497,19 @@ function mapGrpcError(error, timeoutMs) {
|
|
|
3171
3497
|
return new ServerError(message, 500, "GRPC_ERROR");
|
|
3172
3498
|
}
|
|
3173
3499
|
}
|
|
3500
|
+
function grpcHost(address) {
|
|
3501
|
+
if (address.startsWith("[")) {
|
|
3502
|
+
const end = address.indexOf("]");
|
|
3503
|
+
return end === -1 ? address : address.slice(1, end);
|
|
3504
|
+
}
|
|
3505
|
+
return address.split(":").length > 2 ? address : address.split(":")[0];
|
|
3506
|
+
}
|
|
3174
3507
|
function shouldUseTls(address, explicitTls) {
|
|
3175
3508
|
if (explicitTls !== void 0) {
|
|
3176
3509
|
return explicitTls;
|
|
3177
3510
|
}
|
|
3178
|
-
const host = address
|
|
3179
|
-
return host !== "localhost" && host !== "127.0.0.1" &&
|
|
3511
|
+
const host = grpcHost(address).toLowerCase();
|
|
3512
|
+
return host !== "localhost" && host !== "127.0.0.1" && host !== "::1";
|
|
3180
3513
|
}
|
|
3181
3514
|
var GrpcQueueOperations = class {
|
|
3182
3515
|
constructor(client, metadata, timeoutMs) {
|
|
@@ -3228,7 +3561,7 @@ var GrpcQueueOperations = class {
|
|
|
3228
3561
|
return promisify(
|
|
3229
3562
|
this.client,
|
|
3230
3563
|
this.client.Fail,
|
|
3231
|
-
params,
|
|
3564
|
+
encodeFailRequest(params),
|
|
3232
3565
|
this.metadata,
|
|
3233
3566
|
this.timeoutMs
|
|
3234
3567
|
);
|
|
@@ -3618,6 +3951,13 @@ var SpooledClient = class _SpooledClient {
|
|
|
3618
3951
|
/**
|
|
3619
3952
|
* Exchange the API key for a JWT via /auth/login and cache the result.
|
|
3620
3953
|
*
|
|
3954
|
+
* The JWT belongs to the realtime transport only. It must NOT replace the
|
|
3955
|
+
* REST bearer: an API-key client has no refresh token wired into its HTTP
|
|
3956
|
+
* layer, so after the JWT's 24 h lifetime every REST call (claim, complete,
|
|
3957
|
+
* heartbeat, create) would have failed with 401 for good. REST keeps using
|
|
3958
|
+
* the API key; realtime re-mints its own JWT near expiry (see
|
|
3959
|
+
* `getJwtToken`).
|
|
3960
|
+
*
|
|
3621
3961
|
* Concurrent callers share a single in-flight login so a burst of reconnects
|
|
3622
3962
|
* (e.g. WebSocket and SSE at once) issues just one request.
|
|
3623
3963
|
*/
|
|
@@ -3627,9 +3967,6 @@ var SpooledClient = class _SpooledClient {
|
|
|
3627
3967
|
}
|
|
3628
3968
|
this.jwtLoginPromise = (async () => {
|
|
3629
3969
|
const response = await this.auth.login({ apiKey: this.config.apiKey });
|
|
3630
|
-
this.http.setAuthToken(response.accessToken);
|
|
3631
|
-
this.tokenExpiresAt = Date.now() + response.expiresIn * 1e3;
|
|
3632
|
-
this.config.refreshToken = response.refreshToken;
|
|
3633
3970
|
this.cachedJwt = response.accessToken;
|
|
3634
3971
|
return response.accessToken;
|
|
3635
3972
|
})();
|
|
@@ -3685,6 +4022,17 @@ var SpooledClient = class _SpooledClient {
|
|
|
3685
4022
|
getConfig() {
|
|
3686
4023
|
return this.config;
|
|
3687
4024
|
}
|
|
4025
|
+
/**
|
|
4026
|
+
* Rate-limit state from the most recent response that carried
|
|
4027
|
+
* `X-RateLimit-*` headers (API 0.1.122+): `limit`, `remaining`, `reset`
|
|
4028
|
+
* (from the Unix-seconds header), `scope` and, after a 429, `retryAfter`.
|
|
4029
|
+
* `undefined` until such a response arrives, and for servers or routes that
|
|
4030
|
+
* do not send the headers. A 429 also carries the same values on
|
|
4031
|
+
* `RateLimitError.rateLimitInfo`.
|
|
4032
|
+
*/
|
|
4033
|
+
getRateLimitInfo() {
|
|
4034
|
+
return this.http.getLastRateLimitInfo();
|
|
4035
|
+
}
|
|
3688
4036
|
/**
|
|
3689
4037
|
* Get circuit breaker statistics
|
|
3690
4038
|
*/
|
|
@@ -3825,6 +4173,9 @@ var DEFAULT_OPTIONS = {
|
|
|
3825
4173
|
autoStart: false,
|
|
3826
4174
|
shutdownTimeout: 3e4
|
|
3827
4175
|
};
|
|
4176
|
+
var JOB_TIMEOUT_MESSAGE = "Job timeout exceeded";
|
|
4177
|
+
var SHUTDOWN_TIMEOUT_MESSAGE = "Worker shutdown timeout";
|
|
4178
|
+
var MAX_TIMER_MS = 2 ** 31 - 1;
|
|
3828
4179
|
var SpooledWorker = class {
|
|
3829
4180
|
client;
|
|
3830
4181
|
/**
|
|
@@ -3841,6 +4192,10 @@ var SpooledWorker = class {
|
|
|
3841
4192
|
pollTimer = null;
|
|
3842
4193
|
workerHeartbeatTimer = null;
|
|
3843
4194
|
shutdownPromise = null;
|
|
4195
|
+
/** The claim request currently on the wire, so a stop can wait for its jobs. */
|
|
4196
|
+
pollInFlight = null;
|
|
4197
|
+
/** Wakes the shutdown drain when the last active job has been acked. */
|
|
4198
|
+
drainWaiter = null;
|
|
3844
4199
|
// Event handlers
|
|
3845
4200
|
eventHandlers = /* @__PURE__ */ new Map();
|
|
3846
4201
|
constructor(client, options) {
|
|
@@ -3857,7 +4212,11 @@ var SpooledWorker = class {
|
|
|
3857
4212
|
this.debug = config.debug ?? (() => {
|
|
3858
4213
|
});
|
|
3859
4214
|
if (this.options.autoStart) {
|
|
3860
|
-
setImmediate(() =>
|
|
4215
|
+
setImmediate(() => {
|
|
4216
|
+
this.start().catch((error) => {
|
|
4217
|
+
this.debug("Auto-start failed", error);
|
|
4218
|
+
});
|
|
4219
|
+
});
|
|
3861
4220
|
}
|
|
3862
4221
|
}
|
|
3863
4222
|
/**
|
|
@@ -3928,7 +4287,15 @@ var SpooledWorker = class {
|
|
|
3928
4287
|
}
|
|
3929
4288
|
}
|
|
3930
4289
|
/**
|
|
3931
|
-
* Stop the worker gracefully
|
|
4290
|
+
* Stop the worker gracefully.
|
|
4291
|
+
*
|
|
4292
|
+
* Drains: the worker stops claiming, then lets running handlers finish for
|
|
4293
|
+
* up to `shutdownTimeout`. A handler that returns completes its job and one
|
|
4294
|
+
* that throws fails it, even though a stop was requested meanwhile; its
|
|
4295
|
+
* `signal` is NOT aborted during the grace period. Only handlers still
|
|
4296
|
+
* running when the timeout ends are aborted and failed with "Worker shutdown
|
|
4297
|
+
* timeout". The worker deregisters last, because deregistering returns every
|
|
4298
|
+
* job the worker still holds to the queue.
|
|
3932
4299
|
*/
|
|
3933
4300
|
async stop() {
|
|
3934
4301
|
if (this.state !== "running") {
|
|
@@ -3941,39 +4308,37 @@ var SpooledWorker = class {
|
|
|
3941
4308
|
return this.shutdownPromise;
|
|
3942
4309
|
}
|
|
3943
4310
|
async doStop() {
|
|
4311
|
+
const deadline = Date.now() + this.options.shutdownTimeout;
|
|
3944
4312
|
this.state = "stopping";
|
|
3945
4313
|
this.debug("Stopping worker...");
|
|
3946
4314
|
if (this.pollTimer) {
|
|
3947
4315
|
clearTimeout(this.pollTimer);
|
|
3948
4316
|
this.pollTimer = null;
|
|
3949
4317
|
}
|
|
3950
|
-
if (this.
|
|
3951
|
-
|
|
3952
|
-
this.workerHeartbeatTimer = null;
|
|
3953
|
-
}
|
|
3954
|
-
for (const active of this.activeJobs.values()) {
|
|
3955
|
-
active.abortController.abort();
|
|
4318
|
+
if (this.pollInFlight) {
|
|
4319
|
+
await this.pollInFlight;
|
|
3956
4320
|
}
|
|
3957
4321
|
if (this.activeJobs.size > 0) {
|
|
3958
4322
|
this.debug(
|
|
3959
4323
|
`Waiting for ${this.activeJobs.size} active jobs to complete...`
|
|
3960
4324
|
);
|
|
3961
|
-
await
|
|
3962
|
-
|
|
3963
|
-
|
|
3964
|
-
|
|
3965
|
-
|
|
3966
|
-
|
|
3967
|
-
this.
|
|
3968
|
-
|
|
3969
|
-
);
|
|
3970
|
-
await this.failJob(active.job, "Worker shutdown timeout");
|
|
3971
|
-
this.cleanupJob(active);
|
|
3972
|
-
}
|
|
4325
|
+
await this.waitForDrain(Math.max(0, deadline - Date.now()));
|
|
4326
|
+
const stragglers = Array.from(this.activeJobs.values()).filter(
|
|
4327
|
+
(active) => !active.settled
|
|
4328
|
+
);
|
|
4329
|
+
await Promise.all(stragglers.map((active) => this.abandonJob(active)));
|
|
4330
|
+
await Promise.all(
|
|
4331
|
+
Array.from(this.activeJobs.values()).map((active) => active.ack)
|
|
4332
|
+
);
|
|
3973
4333
|
}
|
|
3974
|
-
if (this.
|
|
4334
|
+
if (this.workerHeartbeatTimer) {
|
|
4335
|
+
clearInterval(this.workerHeartbeatTimer);
|
|
4336
|
+
this.workerHeartbeatTimer = null;
|
|
4337
|
+
}
|
|
4338
|
+
const workerId = this.workerId;
|
|
4339
|
+
if (workerId) {
|
|
3975
4340
|
try {
|
|
3976
|
-
await this.client.workers.deregister(
|
|
4341
|
+
await this.client.workers.deregister(workerId);
|
|
3977
4342
|
this.debug("Worker deregistered");
|
|
3978
4343
|
} catch (error) {
|
|
3979
4344
|
this.debug("Failed to deregister worker", error);
|
|
@@ -3981,7 +4346,7 @@ var SpooledWorker = class {
|
|
|
3981
4346
|
}
|
|
3982
4347
|
this.state = "stopped";
|
|
3983
4348
|
this.workerId = null;
|
|
3984
|
-
this.emit("stopped", { workerId:
|
|
4349
|
+
this.emit("stopped", { workerId: workerId ?? "", reason: "graceful" });
|
|
3985
4350
|
this.shutdownPromise = null;
|
|
3986
4351
|
}
|
|
3987
4352
|
/**
|
|
@@ -4020,7 +4385,17 @@ var SpooledWorker = class {
|
|
|
4020
4385
|
if (this.state !== "running") {
|
|
4021
4386
|
return;
|
|
4022
4387
|
}
|
|
4023
|
-
this.pollTimer = setTimeout(() =>
|
|
4388
|
+
this.pollTimer = setTimeout(() => {
|
|
4389
|
+
this.pollTimer = null;
|
|
4390
|
+
const run = this.poll();
|
|
4391
|
+
this.pollInFlight = run;
|
|
4392
|
+
const done = () => {
|
|
4393
|
+
if (this.pollInFlight === run) {
|
|
4394
|
+
this.pollInFlight = null;
|
|
4395
|
+
}
|
|
4396
|
+
};
|
|
4397
|
+
run.then(done, done);
|
|
4398
|
+
}, this.options.pollInterval);
|
|
4024
4399
|
}
|
|
4025
4400
|
async poll() {
|
|
4026
4401
|
if (this.state !== "running" || !this.workerId) {
|
|
@@ -4054,7 +4429,8 @@ var SpooledWorker = class {
|
|
|
4054
4429
|
executionId: Symbol(job.id),
|
|
4055
4430
|
job,
|
|
4056
4431
|
startedAt: /* @__PURE__ */ new Date(),
|
|
4057
|
-
abortController
|
|
4432
|
+
abortController,
|
|
4433
|
+
settled: false
|
|
4058
4434
|
};
|
|
4059
4435
|
this.activeJobs.set(activeJob.executionId, activeJob);
|
|
4060
4436
|
const heartbeatInterval = this.options.leaseDuration * this.options.heartbeatFraction * 1e3;
|
|
@@ -4083,23 +4459,72 @@ var SpooledWorker = class {
|
|
|
4083
4459
|
}
|
|
4084
4460
|
};
|
|
4085
4461
|
try {
|
|
4086
|
-
const
|
|
4087
|
-
if (
|
|
4088
|
-
this.debug(`Job ${job.id} was
|
|
4462
|
+
const outcome = await this.runHandler(activeJob, context);
|
|
4463
|
+
if (activeJob.settled) {
|
|
4464
|
+
this.debug(`Job ${job.id} was already settled; dropping late result`);
|
|
4089
4465
|
return;
|
|
4090
4466
|
}
|
|
4091
|
-
|
|
4092
|
-
|
|
4093
|
-
|
|
4094
|
-
|
|
4095
|
-
|
|
4467
|
+
activeJob.settled = true;
|
|
4468
|
+
switch (outcome.kind) {
|
|
4469
|
+
case "returned":
|
|
4470
|
+
activeJob.ack = this.completeJob(job, outcome.value ?? void 0);
|
|
4471
|
+
break;
|
|
4472
|
+
case "threw":
|
|
4473
|
+
activeJob.ack = this.failJob(
|
|
4474
|
+
job,
|
|
4475
|
+
outcome.error instanceof Error ? outcome.error.message : String(outcome.error)
|
|
4476
|
+
);
|
|
4477
|
+
break;
|
|
4478
|
+
case "timeout":
|
|
4479
|
+
abortController.abort(new Error(JOB_TIMEOUT_MESSAGE));
|
|
4480
|
+
this.emit("job:timeout", {
|
|
4481
|
+
jobId: job.id,
|
|
4482
|
+
queueName: job.queueName
|
|
4483
|
+
});
|
|
4484
|
+
activeJob.ack = this.failJob(job, JOB_TIMEOUT_MESSAGE);
|
|
4485
|
+
break;
|
|
4096
4486
|
}
|
|
4097
|
-
|
|
4098
|
-
await this.failJob(job, errorMessage);
|
|
4487
|
+
await activeJob.ack;
|
|
4099
4488
|
} finally {
|
|
4100
4489
|
this.cleanupJob(activeJob);
|
|
4101
4490
|
}
|
|
4102
4491
|
}
|
|
4492
|
+
/**
|
|
4493
|
+
* Run the handler, racing it against the job's `timeoutSeconds`.
|
|
4494
|
+
* Never rejects: a throw (sync or async) is reported as an outcome.
|
|
4495
|
+
*/
|
|
4496
|
+
runHandler(activeJob, context) {
|
|
4497
|
+
const run = (async () => {
|
|
4498
|
+
try {
|
|
4499
|
+
return { kind: "returned", value: await this.handler(context) };
|
|
4500
|
+
} catch (error) {
|
|
4501
|
+
return { kind: "threw", error };
|
|
4502
|
+
}
|
|
4503
|
+
})();
|
|
4504
|
+
const { timeoutSeconds } = activeJob.job;
|
|
4505
|
+
if (typeof timeoutSeconds !== "number" || !(timeoutSeconds > 0)) {
|
|
4506
|
+
return run;
|
|
4507
|
+
}
|
|
4508
|
+
return new Promise((resolve) => {
|
|
4509
|
+
activeJob.timeoutTimer = setTimeout(
|
|
4510
|
+
() => resolve({ kind: "timeout" }),
|
|
4511
|
+
Math.min(timeoutSeconds * 1e3, MAX_TIMER_MS)
|
|
4512
|
+
);
|
|
4513
|
+
run.then((outcome) => {
|
|
4514
|
+
this.clearTimeoutTimer(activeJob);
|
|
4515
|
+
resolve(outcome);
|
|
4516
|
+
});
|
|
4517
|
+
});
|
|
4518
|
+
}
|
|
4519
|
+
/** Abort a handler that outlived the shutdown grace period and fail its job. */
|
|
4520
|
+
async abandonJob(activeJob) {
|
|
4521
|
+
this.debug(`Force-failing job ${activeJob.job.id} due to shutdown timeout`);
|
|
4522
|
+
activeJob.settled = true;
|
|
4523
|
+
activeJob.abortController.abort(new Error(SHUTDOWN_TIMEOUT_MESSAGE));
|
|
4524
|
+
activeJob.ack = this.failJob(activeJob.job, SHUTDOWN_TIMEOUT_MESSAGE);
|
|
4525
|
+
await activeJob.ack;
|
|
4526
|
+
this.cleanupJob(activeJob);
|
|
4527
|
+
}
|
|
4103
4528
|
async completeJob(job, result) {
|
|
4104
4529
|
if (!this.workerId) return;
|
|
4105
4530
|
try {
|
|
@@ -4115,25 +4540,34 @@ var SpooledWorker = class {
|
|
|
4115
4540
|
});
|
|
4116
4541
|
} catch (error) {
|
|
4117
4542
|
this.debug(`Failed to complete job ${job.id}`, error);
|
|
4543
|
+
this.emit("error", { error });
|
|
4118
4544
|
}
|
|
4119
4545
|
}
|
|
4120
4546
|
async failJob(job, errorMessage) {
|
|
4121
4547
|
if (!this.workerId) return;
|
|
4122
4548
|
const willRetry = job.retryCount < job.maxRetries;
|
|
4549
|
+
const message = coerceFailureMessage(errorMessage);
|
|
4123
4550
|
try {
|
|
4124
4551
|
await this.client.jobs.fail(job.id, {
|
|
4125
4552
|
workerId: this.workerId,
|
|
4126
|
-
error:
|
|
4553
|
+
error: message,
|
|
4127
4554
|
...job.leaseId != null && { leaseId: job.leaseId }
|
|
4128
4555
|
});
|
|
4129
4556
|
this.emit("job:failed", {
|
|
4130
4557
|
jobId: job.id,
|
|
4131
4558
|
queueName: job.queueName,
|
|
4132
|
-
error:
|
|
4559
|
+
error: message,
|
|
4133
4560
|
willRetry
|
|
4134
4561
|
});
|
|
4135
4562
|
} catch (error) {
|
|
4136
4563
|
this.debug(`Failed to fail job ${job.id}`, error);
|
|
4564
|
+
this.emit("error", { error });
|
|
4565
|
+
}
|
|
4566
|
+
}
|
|
4567
|
+
clearTimeoutTimer(activeJob) {
|
|
4568
|
+
if (activeJob.timeoutTimer) {
|
|
4569
|
+
clearTimeout(activeJob.timeoutTimer);
|
|
4570
|
+
activeJob.timeoutTimer = void 0;
|
|
4137
4571
|
}
|
|
4138
4572
|
}
|
|
4139
4573
|
cleanupJob(activeJob) {
|
|
@@ -4141,8 +4575,12 @@ var SpooledWorker = class {
|
|
|
4141
4575
|
clearInterval(activeJob.heartbeatTimer);
|
|
4142
4576
|
activeJob.heartbeatTimer = void 0;
|
|
4143
4577
|
}
|
|
4578
|
+
this.clearTimeoutTimer(activeJob);
|
|
4144
4579
|
if (this.activeJobs.get(activeJob.executionId) === activeJob) {
|
|
4145
4580
|
this.activeJobs.delete(activeJob.executionId);
|
|
4581
|
+
if (this.activeJobs.size === 0) {
|
|
4582
|
+
this.drainWaiter?.();
|
|
4583
|
+
}
|
|
4146
4584
|
}
|
|
4147
4585
|
}
|
|
4148
4586
|
async sendJobHeartbeat(activeJob) {
|
|
@@ -4159,7 +4597,9 @@ var SpooledWorker = class {
|
|
|
4159
4597
|
}
|
|
4160
4598
|
}
|
|
4161
4599
|
async sendWorkerHeartbeat() {
|
|
4162
|
-
if (!this.workerId || this.state !== "running")
|
|
4600
|
+
if (!this.workerId || this.state !== "running" && this.state !== "stopping") {
|
|
4601
|
+
return;
|
|
4602
|
+
}
|
|
4163
4603
|
try {
|
|
4164
4604
|
await this.client.workers.heartbeat(this.workerId, {
|
|
4165
4605
|
currentJobs: this.activeJobs.size,
|
|
@@ -4169,13 +4609,25 @@ var SpooledWorker = class {
|
|
|
4169
4609
|
this.debug("Worker heartbeat failed", error);
|
|
4170
4610
|
}
|
|
4171
4611
|
}
|
|
4172
|
-
|
|
4173
|
-
|
|
4174
|
-
|
|
4612
|
+
/**
|
|
4613
|
+
* Resolve once no job is active, or after `timeoutMs`. The timer is always
|
|
4614
|
+
* cleared, so a clean stop leaves nothing that keeps the process alive.
|
|
4615
|
+
*/
|
|
4616
|
+
waitForDrain(timeoutMs) {
|
|
4617
|
+
if (this.activeJobs.size === 0) {
|
|
4618
|
+
return Promise.resolve();
|
|
4175
4619
|
}
|
|
4176
|
-
|
|
4177
|
-
|
|
4178
|
-
|
|
4620
|
+
return new Promise((resolve) => {
|
|
4621
|
+
const timer = setTimeout(() => {
|
|
4622
|
+
this.drainWaiter = null;
|
|
4623
|
+
resolve();
|
|
4624
|
+
}, timeoutMs);
|
|
4625
|
+
this.drainWaiter = () => {
|
|
4626
|
+
clearTimeout(timer);
|
|
4627
|
+
this.drainWaiter = null;
|
|
4628
|
+
resolve();
|
|
4629
|
+
};
|
|
4630
|
+
});
|
|
4179
4631
|
}
|
|
4180
4632
|
};
|
|
4181
4633
|
|
|
@@ -4217,6 +4669,7 @@ exports.SpooledRealtime = SpooledRealtime;
|
|
|
4217
4669
|
exports.SpooledWorker = SpooledWorker;
|
|
4218
4670
|
exports.SseRealtimeClient = SseRealtimeClient;
|
|
4219
4671
|
exports.TimeoutError = TimeoutError;
|
|
4672
|
+
exports.UnsupportedOperationError = UnsupportedOperationError;
|
|
4220
4673
|
exports.ValidationError = ValidationError;
|
|
4221
4674
|
exports.WebSocketRealtimeClient = WebSocketRealtimeClient;
|
|
4222
4675
|
exports.WebhookIngestionResource = WebhookIngestionResource;
|
|
@@ -4240,6 +4693,7 @@ exports.createProcessJobsStream = createProcessJobsStream;
|
|
|
4240
4693
|
exports.createRetryWrapper = createRetryWrapper;
|
|
4241
4694
|
exports.dateToTimestamp = dateToTimestamp;
|
|
4242
4695
|
exports.getProtoPath = getProtoPath;
|
|
4696
|
+
exports.isCircuitBreakerFailure = isCircuitBreakerFailure;
|
|
4243
4697
|
exports.isRetryableError = isRetryableError;
|
|
4244
4698
|
exports.isSpooledError = isSpooledError;
|
|
4245
4699
|
exports.loadProtoDefinition = loadProtoDefinition;
|