@spooled/sdk 1.2.1 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +119 -18
- package/dist/{index-CWVI9LRo.d.cts → index-DzTC8ld3.d.cts} +741 -134
- package/dist/{index-CWVI9LRo.d.ts → index-DzTC8ld3.d.ts} +741 -134
- package/dist/index.cjs +758 -304
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +9 -151
- package/dist/index.d.ts +9 -151
- package/dist/index.js +757 -305
- package/dist/index.js.map +1 -1
- package/dist/worker/index.cjs +164 -45
- package/dist/worker/index.cjs.map +1 -1
- package/dist/worker/index.d.cts +1 -1
- package/dist/worker/index.d.ts +1 -1
- package/dist/worker/index.js +164 -45
- package/dist/worker/index.js.map +1 -1
- package/package.json +4 -3
- package/proto/spooled.proto +19 -1
package/dist/index.js
CHANGED
|
@@ -1,5 +1,3 @@
|
|
|
1
|
-
import { createHmac } from 'crypto';
|
|
2
|
-
import { Buffer } from 'buffer';
|
|
3
1
|
import * as grpc2 from '@grpc/grpc-js';
|
|
4
2
|
import * as protoLoader from '@grpc/proto-loader';
|
|
5
3
|
import { existsSync } from 'fs';
|
|
@@ -26,10 +24,10 @@ var DEFAULT_CONFIG = {
|
|
|
26
24
|
successThreshold: 3,
|
|
27
25
|
timeout: 3e4
|
|
28
26
|
},
|
|
29
|
-
userAgent: "@spooled/sdk-nodejs/1.
|
|
27
|
+
userAgent: "@spooled/sdk-nodejs/1.4.0",
|
|
30
28
|
autoRefreshToken: true
|
|
31
29
|
};
|
|
32
|
-
var SDK_VERSION = "1.
|
|
30
|
+
var SDK_VERSION = "1.4.0";
|
|
33
31
|
var API_VERSION = "v1";
|
|
34
32
|
var API_BASE_PATH = `/api/${API_VERSION}`;
|
|
35
33
|
function normalizeCredential(value) {
|
|
@@ -86,9 +84,9 @@ function resolveConfig(options) {
|
|
|
86
84
|
};
|
|
87
85
|
}
|
|
88
86
|
function validateConfig(config) {
|
|
89
|
-
if (!config.apiKey && !config.accessToken) {
|
|
87
|
+
if (!config.apiKey && !config.accessToken && !config.adminKey) {
|
|
90
88
|
throw new Error(
|
|
91
|
-
"SpooledClient requires either apiKey or accessToken for authentication"
|
|
89
|
+
"SpooledClient requires either apiKey or accessToken for authentication (or adminKey for admin-only use)"
|
|
92
90
|
);
|
|
93
91
|
}
|
|
94
92
|
if (config.apiKey && !config.apiKey.startsWith("sp_") && !config.apiKey.startsWith("sk_")) {
|
|
@@ -151,9 +149,15 @@ var SpooledError = class extends Error {
|
|
|
151
149
|
Error.captureStackTrace(this, this.constructor);
|
|
152
150
|
}
|
|
153
151
|
}
|
|
154
|
-
/**
|
|
152
|
+
/**
|
|
153
|
+
* Whether this error is retryable.
|
|
154
|
+
*
|
|
155
|
+
* Only 408 (Request Timeout) is retryable among the plain 4xx statuses: the
|
|
156
|
+
* server gave up waiting for the request, so it never ran. Subclasses
|
|
157
|
+
* override this (5xx, network, timeout, rate limit).
|
|
158
|
+
*/
|
|
155
159
|
isRetryable() {
|
|
156
|
-
return
|
|
160
|
+
return this.statusCode === 408;
|
|
157
161
|
}
|
|
158
162
|
/** Convert to plain object for logging */
|
|
159
163
|
toJSON() {
|
|
@@ -192,8 +196,8 @@ var ConflictError = class extends SpooledError {
|
|
|
192
196
|
}
|
|
193
197
|
};
|
|
194
198
|
var ValidationError = class extends SpooledError {
|
|
195
|
-
constructor(message = "Validation failed", code = "VALIDATION_ERROR", details, requestId) {
|
|
196
|
-
super(message,
|
|
199
|
+
constructor(message = "Validation failed", code = "VALIDATION_ERROR", details, requestId, statusCode = 400) {
|
|
200
|
+
super(message, statusCode, code, details, requestId);
|
|
197
201
|
this.name = "ValidationError";
|
|
198
202
|
}
|
|
199
203
|
};
|
|
@@ -203,6 +207,7 @@ var PayloadTooLargeError = class extends SpooledError {
|
|
|
203
207
|
this.name = "PayloadTooLargeError";
|
|
204
208
|
}
|
|
205
209
|
};
|
|
210
|
+
var QUOTA_MESSAGE = /\blimit reached\b/i;
|
|
206
211
|
var RateLimitError = class extends SpooledError {
|
|
207
212
|
/** Rate limit information */
|
|
208
213
|
rateLimitInfo;
|
|
@@ -211,8 +216,27 @@ var RateLimitError = class extends SpooledError {
|
|
|
211
216
|
this.name = "RateLimitError";
|
|
212
217
|
this.rateLimitInfo = rateLimitInfo;
|
|
213
218
|
}
|
|
219
|
+
/**
|
|
220
|
+
* True when this 429 is a plan quota or limit (daily job cap, active jobs,
|
|
221
|
+
* queue/worker/webhook caps) rather than per-second rate limiting.
|
|
222
|
+
*
|
|
223
|
+
* A quota does not clear for minutes or hours, so retrying it only blocks the
|
|
224
|
+
* caller for the whole backoff and, via the circuit breaker, takes every
|
|
225
|
+
* other call on the client down with it. It is never retried and never
|
|
226
|
+
* counted as a circuit breaker failure.
|
|
227
|
+
*
|
|
228
|
+
* Detection: the `QUOTA_EXCEEDED` code, or, for older plan-limit paths that
|
|
229
|
+
* answer with plain text wrapped as `RATE_LIMIT_EXCEEDED`, a "... limit
|
|
230
|
+
* reached" message that carries no `Retry-After`.
|
|
231
|
+
*/
|
|
232
|
+
get isQuota() {
|
|
233
|
+
if (this.code === "QUOTA_EXCEEDED") {
|
|
234
|
+
return true;
|
|
235
|
+
}
|
|
236
|
+
return this.rateLimitInfo.retryAfter === void 0 && QUOTA_MESSAGE.test(this.message);
|
|
237
|
+
}
|
|
214
238
|
isRetryable() {
|
|
215
|
-
return
|
|
239
|
+
return !this.isQuota;
|
|
216
240
|
}
|
|
217
241
|
/** Get the number of seconds to wait before retrying */
|
|
218
242
|
getRetryAfter() {
|
|
@@ -272,6 +296,20 @@ var CircuitBreakerOpenError = class extends SpooledError {
|
|
|
272
296
|
return false;
|
|
273
297
|
}
|
|
274
298
|
};
|
|
299
|
+
var UnsupportedOperationError = class extends SpooledError {
|
|
300
|
+
/** The SDK method that is not supported, e.g. `ingest.github` */
|
|
301
|
+
operation;
|
|
302
|
+
constructor(operation, reason) {
|
|
303
|
+
super(
|
|
304
|
+
`${operation} is not supported by the Spooled API: ${reason}`,
|
|
305
|
+
0,
|
|
306
|
+
"UNSUPPORTED_OPERATION",
|
|
307
|
+
{ operation }
|
|
308
|
+
);
|
|
309
|
+
this.name = "UnsupportedOperationError";
|
|
310
|
+
this.operation = operation;
|
|
311
|
+
}
|
|
312
|
+
};
|
|
275
313
|
function parseRateLimitHeaders(headers) {
|
|
276
314
|
const info = {};
|
|
277
315
|
const retryAfter = headers.get("Retry-After");
|
|
@@ -302,6 +340,10 @@ function parseRateLimitHeaders(headers) {
|
|
|
302
340
|
info.reset = new Date(timestamp * 1e3);
|
|
303
341
|
}
|
|
304
342
|
}
|
|
343
|
+
const scope = headers.get("X-RateLimit-Scope");
|
|
344
|
+
if (scope) {
|
|
345
|
+
info.scope = scope.trim();
|
|
346
|
+
}
|
|
305
347
|
return info;
|
|
306
348
|
}
|
|
307
349
|
var MAX_RAW_ERROR_BODY_CHARS = 500;
|
|
@@ -343,6 +385,10 @@ async function createErrorFromResponse(response) {
|
|
|
343
385
|
switch (response.status) {
|
|
344
386
|
case 400:
|
|
345
387
|
return new ValidationError(message, code, details, requestId);
|
|
388
|
+
// A missing or mistyped JSON field is a 422 from the API's extractor; it is
|
|
389
|
+
// the same class of caller error as a 400 field-rule failure.
|
|
390
|
+
case 422:
|
|
391
|
+
return new ValidationError(message, code, details, requestId, 422);
|
|
346
392
|
case 401:
|
|
347
393
|
return new AuthenticationError(message, code, details, requestId);
|
|
348
394
|
case 403:
|
|
@@ -571,8 +617,14 @@ var HttpClient = class {
|
|
|
571
617
|
authToken;
|
|
572
618
|
/** Optional function to mint a fresh access token (set when auto-refresh is enabled) */
|
|
573
619
|
refreshTokenFn = null;
|
|
574
|
-
/**
|
|
575
|
-
|
|
620
|
+
/**
|
|
621
|
+
* The refresh currently in flight, shared by every request that gets a 401
|
|
622
|
+
* meanwhile. A boolean guard here made the second concurrent 401 skip the
|
|
623
|
+
* refresh and fail even though the token was about to be replaced.
|
|
624
|
+
*/
|
|
625
|
+
refreshInFlight = null;
|
|
626
|
+
/** Rate-limit headers of the most recent response that carried them */
|
|
627
|
+
lastRateLimit;
|
|
576
628
|
constructor(config, circuitBreaker) {
|
|
577
629
|
this.config = config;
|
|
578
630
|
this.circuitBreaker = circuitBreaker;
|
|
@@ -585,6 +637,14 @@ var HttpClient = class {
|
|
|
585
637
|
setAuthToken(token) {
|
|
586
638
|
this.authToken = token;
|
|
587
639
|
}
|
|
640
|
+
/**
|
|
641
|
+
* The `X-RateLimit-*` (and `Retry-After`) values of the most recent
|
|
642
|
+
* response that carried them, successful or not. `undefined` until such a
|
|
643
|
+
* response arrives. With concurrent requests it is the last one to finish.
|
|
644
|
+
*/
|
|
645
|
+
getLastRateLimitInfo() {
|
|
646
|
+
return this.lastRateLimit ? { ...this.lastRateLimit } : void 0;
|
|
647
|
+
}
|
|
588
648
|
/**
|
|
589
649
|
* Set the token refresh function.
|
|
590
650
|
*
|
|
@@ -594,6 +654,30 @@ var HttpClient = class {
|
|
|
594
654
|
setRefreshTokenFn(fn) {
|
|
595
655
|
this.refreshTokenFn = fn;
|
|
596
656
|
}
|
|
657
|
+
/**
|
|
658
|
+
* Refresh the access token once for all callers.
|
|
659
|
+
*
|
|
660
|
+
* Resolves true when a new token is installed, false when the refresh failed
|
|
661
|
+
* (the caller then surfaces its original 401).
|
|
662
|
+
*/
|
|
663
|
+
refreshAuthToken() {
|
|
664
|
+
if (!this.refreshInFlight) {
|
|
665
|
+
const refresh = this.refreshTokenFn;
|
|
666
|
+
this.refreshInFlight = (async () => {
|
|
667
|
+
try {
|
|
668
|
+
this.setAuthToken(await refresh());
|
|
669
|
+
return true;
|
|
670
|
+
} catch (refreshError) {
|
|
671
|
+
const message = refreshError instanceof Error ? refreshError.message : "unknown";
|
|
672
|
+
this.debug?.("Token refresh failed", { error: message });
|
|
673
|
+
return false;
|
|
674
|
+
} finally {
|
|
675
|
+
this.refreshInFlight = null;
|
|
676
|
+
}
|
|
677
|
+
})();
|
|
678
|
+
}
|
|
679
|
+
return this.refreshInFlight;
|
|
680
|
+
}
|
|
597
681
|
/**
|
|
598
682
|
* Build full URL with query parameters
|
|
599
683
|
*/
|
|
@@ -669,6 +753,7 @@ var HttpClient = class {
|
|
|
669
753
|
*/
|
|
670
754
|
async executeRequest(url, options, hasRefreshed = false) {
|
|
671
755
|
const method = options.method ?? "GET";
|
|
756
|
+
const tokenUsed = this.authToken;
|
|
672
757
|
const headers = this.buildHeaders(options.headers);
|
|
673
758
|
const timeout = options.timeout ?? this.config.timeout;
|
|
674
759
|
const init = {
|
|
@@ -683,6 +768,9 @@ var HttpClient = class {
|
|
|
683
768
|
init.body = JSON.stringify(body);
|
|
684
769
|
}
|
|
685
770
|
}
|
|
771
|
+
if (init.body === void 0) {
|
|
772
|
+
headers.delete("Content-Type");
|
|
773
|
+
}
|
|
686
774
|
this.debug?.(`${method} ${url}`, { timeout, hasBody: !!options.body });
|
|
687
775
|
let response;
|
|
688
776
|
try {
|
|
@@ -703,21 +791,11 @@ var HttpClient = class {
|
|
|
703
791
|
);
|
|
704
792
|
}
|
|
705
793
|
this.debug?.(`Response: ${response.status}`, { url });
|
|
794
|
+
this.recordRateLimit(response.headers);
|
|
706
795
|
if (!response.ok) {
|
|
707
|
-
if (response.status === 401 && this.refreshTokenFn && !hasRefreshed && !
|
|
708
|
-
|
|
709
|
-
this.
|
|
710
|
-
try {
|
|
711
|
-
const newToken = await this.refreshTokenFn();
|
|
712
|
-
this.setAuthToken(newToken);
|
|
713
|
-
refreshed = true;
|
|
714
|
-
} catch (refreshError) {
|
|
715
|
-
const message = refreshError instanceof Error ? refreshError.message : "unknown";
|
|
716
|
-
this.debug?.("Token refresh failed", { error: message });
|
|
717
|
-
} finally {
|
|
718
|
-
this.isRefreshing = false;
|
|
719
|
-
}
|
|
720
|
-
if (refreshed) {
|
|
796
|
+
if (response.status === 401 && this.refreshTokenFn && !hasRefreshed && !options.skipAuthRefresh) {
|
|
797
|
+
const alreadyRefreshed = this.authToken !== tokenUsed;
|
|
798
|
+
if (alreadyRefreshed || await this.refreshAuthToken()) {
|
|
721
799
|
return this.executeRequest(url, options, true);
|
|
722
800
|
}
|
|
723
801
|
}
|
|
@@ -740,6 +818,13 @@ var HttpClient = class {
|
|
|
740
818
|
data
|
|
741
819
|
};
|
|
742
820
|
}
|
|
821
|
+
/** Remember the rate-limit headers of a response that has them. */
|
|
822
|
+
recordRateLimit(headers) {
|
|
823
|
+
if (!headers || typeof headers.get !== "function" || headers.get("X-RateLimit-Limit") === null && headers.get("X-RateLimit-Remaining") === null) {
|
|
824
|
+
return;
|
|
825
|
+
}
|
|
826
|
+
this.lastRateLimit = parseRateLimitHeaders(headers);
|
|
827
|
+
}
|
|
743
828
|
/**
|
|
744
829
|
* Make an HTTP request with retry and circuit breaker
|
|
745
830
|
*/
|
|
@@ -779,22 +864,26 @@ var HttpClient = class {
|
|
|
779
864
|
/**
|
|
780
865
|
* Resolve the retry configuration for a request.
|
|
781
866
|
*
|
|
782
|
-
* Idempotent methods (GET/PUT/DELETE)
|
|
783
|
-
*
|
|
784
|
-
*
|
|
867
|
+
* Idempotent methods (GET/PUT/DELETE), any request carrying an idempotency
|
|
868
|
+
* key and any request marked `idempotent` retry normally. Non-idempotent
|
|
869
|
+
* methods (POST/PATCH) otherwise must NOT be retried on ambiguous failures
|
|
785
870
|
* (network/timeout/5xx) — doing so risks duplicate jobs, double claims,
|
|
786
|
-
* etc. They are still retried on 429, which is rejected before
|
|
787
|
-
* server processes the request and is therefore side-effect-free.
|
|
871
|
+
* etc. They are still retried on a rate-limit 429, which is rejected before
|
|
872
|
+
* the server processes the request and is therefore side-effect-free. A
|
|
873
|
+
* quota 429 is never retried: it does not clear for hours.
|
|
788
874
|
*/
|
|
789
875
|
resolveRetryConfig(method, options) {
|
|
790
876
|
const base = this.config.retry;
|
|
791
|
-
if (IDEMPOTENT_METHODS.has(method) || this.hasIdempotencyKey(options)) {
|
|
877
|
+
if (options.idempotent || IDEMPOTENT_METHODS.has(method) || this.hasIdempotencyKey(options)) {
|
|
792
878
|
return base;
|
|
793
879
|
}
|
|
794
880
|
return {
|
|
795
881
|
...base,
|
|
796
882
|
retryOn: (error, attempt) => {
|
|
797
883
|
if (error instanceof RateLimitError) {
|
|
884
|
+
if (error.isQuota) {
|
|
885
|
+
return false;
|
|
886
|
+
}
|
|
798
887
|
return base.retryOn ? base.retryOn(error, attempt) : true;
|
|
799
888
|
}
|
|
800
889
|
return false;
|
|
@@ -863,6 +952,12 @@ var CircuitState = /* @__PURE__ */ ((CircuitState2) => {
|
|
|
863
952
|
CircuitState2["HALF_OPEN"] = "HALF_OPEN";
|
|
864
953
|
return CircuitState2;
|
|
865
954
|
})(CircuitState || {});
|
|
955
|
+
function isCircuitBreakerFailure(error) {
|
|
956
|
+
if (error instanceof RateLimitError) {
|
|
957
|
+
return false;
|
|
958
|
+
}
|
|
959
|
+
return isRetryableError(error);
|
|
960
|
+
}
|
|
866
961
|
var CircuitBreaker = class {
|
|
867
962
|
state = "CLOSED" /* CLOSED */;
|
|
868
963
|
failureCount = 0;
|
|
@@ -923,7 +1018,7 @@ var CircuitBreaker = class {
|
|
|
923
1018
|
if (!this.config.enabled) {
|
|
924
1019
|
return;
|
|
925
1020
|
}
|
|
926
|
-
const shouldCount = this.config.isFailure ? this.config.isFailure(error) :
|
|
1021
|
+
const shouldCount = this.config.isFailure ? this.config.isFailure(error) : isCircuitBreakerFailure(error);
|
|
927
1022
|
if (!shouldCount) {
|
|
928
1023
|
return;
|
|
929
1024
|
}
|
|
@@ -1025,13 +1120,17 @@ var AuthResource = class {
|
|
|
1025
1120
|
* Exchange API key for JWT tokens
|
|
1026
1121
|
*/
|
|
1027
1122
|
async login(params) {
|
|
1028
|
-
return this.http.post("/auth/login", params
|
|
1123
|
+
return this.http.post("/auth/login", params, {
|
|
1124
|
+
skipAuthRefresh: true
|
|
1125
|
+
});
|
|
1029
1126
|
}
|
|
1030
1127
|
/**
|
|
1031
1128
|
* Refresh access token using refresh token
|
|
1032
1129
|
*/
|
|
1033
1130
|
async refresh(params) {
|
|
1034
|
-
return this.http.post("/auth/refresh", params
|
|
1131
|
+
return this.http.post("/auth/refresh", params, {
|
|
1132
|
+
skipAuthRefresh: true
|
|
1133
|
+
});
|
|
1035
1134
|
}
|
|
1036
1135
|
/**
|
|
1037
1136
|
* Logout and revoke the refresh token server-side.
|
|
@@ -1102,12 +1201,46 @@ function mapValidate(raw) {
|
|
|
1102
1201
|
};
|
|
1103
1202
|
}
|
|
1104
1203
|
|
|
1204
|
+
// src/utils/failure.ts
|
|
1205
|
+
var MAX_FAILURE_MESSAGE_CHARS = 2048;
|
|
1206
|
+
var DEFAULT_FAILURE_MESSAGE = "Job failed";
|
|
1207
|
+
function coerceFailureMessage(message) {
|
|
1208
|
+
const text = typeof message === "string" ? message : message == null ? "" : `${message}`;
|
|
1209
|
+
if (text === "") {
|
|
1210
|
+
return DEFAULT_FAILURE_MESSAGE;
|
|
1211
|
+
}
|
|
1212
|
+
if (text.length <= MAX_FAILURE_MESSAGE_CHARS) {
|
|
1213
|
+
return text;
|
|
1214
|
+
}
|
|
1215
|
+
const codePoints = Array.from(text);
|
|
1216
|
+
return codePoints.length <= MAX_FAILURE_MESSAGE_CHARS ? text : codePoints.slice(0, MAX_FAILURE_MESSAGE_CHARS).join("");
|
|
1217
|
+
}
|
|
1218
|
+
|
|
1219
|
+
// src/utils/paging.ts
|
|
1220
|
+
var HAS_MORE_HEADER = "X-Has-More";
|
|
1221
|
+
function hasMoreFromHeaders(headers, count, limit, defaultLimit, maxLimit) {
|
|
1222
|
+
const header = headers.get(HAS_MORE_HEADER);
|
|
1223
|
+
if (header !== null) {
|
|
1224
|
+
return header.trim().toLowerCase() === "true";
|
|
1225
|
+
}
|
|
1226
|
+
const applied = Math.min(Math.max(limit ?? defaultLimit, 1), maxLimit);
|
|
1227
|
+
return count >= applied;
|
|
1228
|
+
}
|
|
1229
|
+
function toQueryTimestamp(value) {
|
|
1230
|
+
return value instanceof Date ? value.toISOString() : value;
|
|
1231
|
+
}
|
|
1232
|
+
|
|
1105
1233
|
// src/resources/jobs.ts
|
|
1234
|
+
var JOB_LIST_DEFAULT_LIMIT = 50;
|
|
1235
|
+
var JOB_LIST_MAX_LIMIT = 100;
|
|
1236
|
+
var JOB_HISTORY_DEFAULT_LIMIT = 100;
|
|
1237
|
+
var JOB_HISTORY_MAX_LIMIT = 500;
|
|
1106
1238
|
var JobsResource = class {
|
|
1107
1239
|
constructor(http) {
|
|
1108
1240
|
this.http = http;
|
|
1109
1241
|
this.dlq = {
|
|
1110
1242
|
list: this.listDlq.bind(this),
|
|
1243
|
+
listPage: this.listDlqPage.bind(this),
|
|
1111
1244
|
retry: this.retryDlq.bind(this),
|
|
1112
1245
|
purge: this.purgeDlq.bind(this)
|
|
1113
1246
|
};
|
|
@@ -1118,7 +1251,9 @@ var JobsResource = class {
|
|
|
1118
1251
|
* Create a new job
|
|
1119
1252
|
*/
|
|
1120
1253
|
async create(params) {
|
|
1121
|
-
return this.http.post("/jobs", params
|
|
1254
|
+
return this.http.post("/jobs", params, {
|
|
1255
|
+
idempotent: params.idempotencyKey ? true : void 0
|
|
1256
|
+
});
|
|
1122
1257
|
}
|
|
1123
1258
|
/**
|
|
1124
1259
|
* Create a job and return the full job object
|
|
@@ -1129,15 +1264,36 @@ var JobsResource = class {
|
|
|
1129
1264
|
}
|
|
1130
1265
|
/**
|
|
1131
1266
|
* List jobs with optional filtering
|
|
1267
|
+
*
|
|
1268
|
+
* `status` takes one status or several, and `q`, `jobType`, `createdAfter`
|
|
1269
|
+
* and `createdBefore` filter on the server (API 0.1.122+). Use
|
|
1270
|
+
* {@link JobsResource.listPage} to also learn whether another page follows.
|
|
1132
1271
|
*/
|
|
1133
1272
|
async list(params) {
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
|
|
1137
|
-
|
|
1138
|
-
|
|
1139
|
-
|
|
1140
|
-
|
|
1273
|
+
return (await this.listPage(params)).data;
|
|
1274
|
+
}
|
|
1275
|
+
/**
|
|
1276
|
+
* List one page of jobs with `hasMore` from the `X-Has-More` header
|
|
1277
|
+
* (API 0.1.122+). Takes the same filters as {@link JobsResource.list}.
|
|
1278
|
+
*
|
|
1279
|
+
* @example
|
|
1280
|
+
* ```typescript
|
|
1281
|
+
* let offset = 0;
|
|
1282
|
+
* for (;;) {
|
|
1283
|
+
* const page = await client.jobs.listPage({
|
|
1284
|
+
* status: ["failed", "deadletter"],
|
|
1285
|
+
* createdAfter: new Date(Date.now() - 86_400_000),
|
|
1286
|
+
* limit: 100,
|
|
1287
|
+
* offset,
|
|
1288
|
+
* });
|
|
1289
|
+
* handle(page.data);
|
|
1290
|
+
* if (!page.hasMore) break;
|
|
1291
|
+
* offset += page.data.length;
|
|
1292
|
+
* }
|
|
1293
|
+
* ```
|
|
1294
|
+
*/
|
|
1295
|
+
async listPage(params) {
|
|
1296
|
+
return this.listSummaries("/jobs", params);
|
|
1141
1297
|
}
|
|
1142
1298
|
/**
|
|
1143
1299
|
* Get a job by ID
|
|
@@ -1167,6 +1323,46 @@ var JobsResource = class {
|
|
|
1167
1323
|
priority
|
|
1168
1324
|
});
|
|
1169
1325
|
}
|
|
1326
|
+
/**
|
|
1327
|
+
* Get the recorded lifecycle events of a job (`GET /jobs/{id}/history`,
|
|
1328
|
+
* API 0.1.122+), oldest first unless `order: "desc"`.
|
|
1329
|
+
*
|
|
1330
|
+
* The history is sparse: the server records REST claims (`processing`),
|
|
1331
|
+
* completions (`completed`) and failures (`retry_scheduled`,
|
|
1332
|
+
* `deadlettered`) and the scheduler's dead-letter sweeps, so a job settled
|
|
1333
|
+
* over gRPC may have few or no events. An unknown job is a `NotFoundError`.
|
|
1334
|
+
*/
|
|
1335
|
+
async history(id, params) {
|
|
1336
|
+
return (await this.historyPage(id, params)).data;
|
|
1337
|
+
}
|
|
1338
|
+
/**
|
|
1339
|
+
* One page of {@link JobsResource.history} with `hasMore` from the
|
|
1340
|
+
* `X-Has-More` header. `limit` defaults to 100 (at most 500).
|
|
1341
|
+
*/
|
|
1342
|
+
async historyPage(id, params) {
|
|
1343
|
+
const response = await this.http.request(
|
|
1344
|
+
`/jobs/${id}/history`,
|
|
1345
|
+
{
|
|
1346
|
+
method: "GET",
|
|
1347
|
+
params: {
|
|
1348
|
+
order: params?.order,
|
|
1349
|
+
limit: params?.limit,
|
|
1350
|
+
offset: params?.offset
|
|
1351
|
+
}
|
|
1352
|
+
}
|
|
1353
|
+
);
|
|
1354
|
+
const data = response.data ?? [];
|
|
1355
|
+
return {
|
|
1356
|
+
data,
|
|
1357
|
+
hasMore: hasMoreFromHeaders(
|
|
1358
|
+
response.headers,
|
|
1359
|
+
data.length,
|
|
1360
|
+
params?.limit,
|
|
1361
|
+
JOB_HISTORY_DEFAULT_LIMIT,
|
|
1362
|
+
JOB_HISTORY_MAX_LIMIT
|
|
1363
|
+
)
|
|
1364
|
+
};
|
|
1365
|
+
}
|
|
1170
1366
|
/**
|
|
1171
1367
|
* Get job statistics
|
|
1172
1368
|
*/
|
|
@@ -1202,37 +1398,71 @@ var JobsResource = class {
|
|
|
1202
1398
|
}
|
|
1203
1399
|
/**
|
|
1204
1400
|
* Complete a job (worker ack)
|
|
1401
|
+
*
|
|
1402
|
+
* Lease-fenced and idempotent on the server, so a network error, timeout or
|
|
1403
|
+
* 5xx is retried: a lost ack would otherwise re-run a finished job.
|
|
1205
1404
|
*/
|
|
1206
1405
|
async complete(id, params) {
|
|
1207
|
-
return this.http.post(
|
|
1406
|
+
return this.http.post(
|
|
1407
|
+
`/jobs/${id}/complete`,
|
|
1408
|
+
params,
|
|
1409
|
+
{ idempotent: true }
|
|
1410
|
+
);
|
|
1208
1411
|
}
|
|
1209
1412
|
/**
|
|
1210
1413
|
* Fail a job (worker nack)
|
|
1414
|
+
*
|
|
1415
|
+
* `params.error` is made acceptable first: empty text becomes "Job failed"
|
|
1416
|
+
* and text over 2048 characters is truncated, because the API rejects both
|
|
1417
|
+
* with a 400 that would leave the job processing until its lease expires.
|
|
1418
|
+
* Retried like {@link JobsResource.complete}.
|
|
1211
1419
|
*/
|
|
1212
1420
|
async fail(id, params) {
|
|
1213
1421
|
return this.http.post(
|
|
1214
1422
|
`/jobs/${id}/fail`,
|
|
1215
|
-
params
|
|
1423
|
+
{ ...params, error: coerceFailureMessage(params.error) },
|
|
1424
|
+
{ idempotent: true }
|
|
1216
1425
|
);
|
|
1217
1426
|
}
|
|
1218
1427
|
/**
|
|
1219
1428
|
* Extend job lease (heartbeat)
|
|
1429
|
+
*
|
|
1430
|
+
* Retried like {@link JobsResource.complete}: with the default 30 s lease and
|
|
1431
|
+
* 15 s heartbeat, one lost heartbeat leaves no slack.
|
|
1220
1432
|
*/
|
|
1221
1433
|
async heartbeat(id, params) {
|
|
1222
1434
|
return this.http.post(
|
|
1223
1435
|
`/jobs/${id}/heartbeat`,
|
|
1224
|
-
params
|
|
1436
|
+
params,
|
|
1437
|
+
{ idempotent: true }
|
|
1225
1438
|
);
|
|
1226
1439
|
}
|
|
1227
1440
|
// DLQ operations (private implementations)
|
|
1228
1441
|
async listDlq(params) {
|
|
1229
|
-
|
|
1230
|
-
|
|
1442
|
+
return (await this.listDlqPage(params)).data;
|
|
1443
|
+
}
|
|
1444
|
+
async listDlqPage(params) {
|
|
1445
|
+
return this.listSummaries("/jobs/dlq", params);
|
|
1446
|
+
}
|
|
1447
|
+
async listSummaries(path, params) {
|
|
1448
|
+
const response = await this.http.request(path, {
|
|
1449
|
+
method: "GET",
|
|
1450
|
+
params: jobListQuery(params)
|
|
1231
1451
|
});
|
|
1232
|
-
|
|
1233
|
-
|
|
1234
|
-
|
|
1235
|
-
|
|
1452
|
+
const rows = response.data ?? [];
|
|
1453
|
+
return {
|
|
1454
|
+
data: rows.map((row) => ({
|
|
1455
|
+
...row,
|
|
1456
|
+
jobType: row.jobType ?? jobTypeFromPayload(row)
|
|
1457
|
+
})),
|
|
1458
|
+
hasMore: hasMoreFromHeaders(
|
|
1459
|
+
response.headers,
|
|
1460
|
+
rows.length,
|
|
1461
|
+
params?.limit,
|
|
1462
|
+
JOB_LIST_DEFAULT_LIMIT,
|
|
1463
|
+
JOB_LIST_MAX_LIMIT
|
|
1464
|
+
)
|
|
1465
|
+
};
|
|
1236
1466
|
}
|
|
1237
1467
|
async retryDlq(params) {
|
|
1238
1468
|
return this.http.post("/jobs/dlq/retry", params);
|
|
@@ -1241,6 +1471,24 @@ var JobsResource = class {
|
|
|
1241
1471
|
return this.http.post("/jobs/dlq/purge", params);
|
|
1242
1472
|
}
|
|
1243
1473
|
};
|
|
1474
|
+
function jobListQuery(params) {
|
|
1475
|
+
if (!params) {
|
|
1476
|
+
return void 0;
|
|
1477
|
+
}
|
|
1478
|
+
const { status: status2, createdAfter, createdBefore, ...rest } = params;
|
|
1479
|
+
let statusParam;
|
|
1480
|
+
if (typeof status2 === "string") {
|
|
1481
|
+
statusParam = status2;
|
|
1482
|
+
} else if (status2 && status2.length > 0) {
|
|
1483
|
+
statusParam = status2.join(",");
|
|
1484
|
+
}
|
|
1485
|
+
return {
|
|
1486
|
+
...rest,
|
|
1487
|
+
status: statusParam,
|
|
1488
|
+
createdAfter: toQueryTimestamp(createdAfter),
|
|
1489
|
+
createdBefore: toQueryTimestamp(createdBefore)
|
|
1490
|
+
};
|
|
1491
|
+
}
|
|
1244
1492
|
function withJobType(job) {
|
|
1245
1493
|
return {
|
|
1246
1494
|
...job,
|
|
@@ -1317,9 +1565,17 @@ var QueuesResource = class {
|
|
|
1317
1565
|
}
|
|
1318
1566
|
/**
|
|
1319
1567
|
* Delete a queue configuration
|
|
1320
|
-
|
|
1321
|
-
|
|
1322
|
-
|
|
1568
|
+
*
|
|
1569
|
+
* Without `deleteJobs` the API only deletes an explicit queue config of an
|
|
1570
|
+
* idle queue, and answers 409 otherwise: a queue that exists only because
|
|
1571
|
+
* jobs were posted to it has no config to delete, and a queue with pending
|
|
1572
|
+
* or processing jobs is not idle. Pass `{ deleteJobs: true }` to delete the
|
|
1573
|
+
* queue's jobs (and DLQ rows) along with it.
|
|
1574
|
+
*/
|
|
1575
|
+
async delete(name, options) {
|
|
1576
|
+
await this.http.delete(`/queues/${encodeURIComponent(name)}`, {
|
|
1577
|
+
params: options?.deleteJobs ? { deleteJobs: true } : void 0
|
|
1578
|
+
});
|
|
1323
1579
|
}
|
|
1324
1580
|
};
|
|
1325
1581
|
|
|
@@ -1348,9 +1604,13 @@ var WorkersResource = class {
|
|
|
1348
1604
|
}
|
|
1349
1605
|
/**
|
|
1350
1606
|
* Send worker heartbeat
|
|
1607
|
+
*
|
|
1608
|
+
* Idempotent on the server, so network errors, timeouts and 5xx are retried.
|
|
1351
1609
|
*/
|
|
1352
1610
|
async heartbeat(id, params) {
|
|
1353
|
-
await this.http.post(`/workers/${id}/heartbeat`, params
|
|
1611
|
+
await this.http.post(`/workers/${id}/heartbeat`, params, {
|
|
1612
|
+
idempotent: true
|
|
1613
|
+
});
|
|
1354
1614
|
}
|
|
1355
1615
|
/**
|
|
1356
1616
|
* Deregister a worker
|
|
@@ -1499,7 +1759,7 @@ var WorkflowsResource = class {
|
|
|
1499
1759
|
const jobs = await this.listWorkflowJobs(workflowId);
|
|
1500
1760
|
return jobs.map((job) => ({
|
|
1501
1761
|
jobId: job.id,
|
|
1502
|
-
key: job.key,
|
|
1762
|
+
...job.key !== void 0 && { key: job.key },
|
|
1503
1763
|
status: job.status
|
|
1504
1764
|
}));
|
|
1505
1765
|
}
|
|
@@ -1557,7 +1817,10 @@ function mapWorkflowDetailJobs(detail) {
|
|
|
1557
1817
|
return {
|
|
1558
1818
|
id,
|
|
1559
1819
|
workflowId: String(job.workflowId ?? workflowId),
|
|
1560
|
-
key
|
|
1820
|
+
// The API does not return the creation-time key (jobs store only a
|
|
1821
|
+
// workflow step), so there is nothing to put here unless a future
|
|
1822
|
+
// version sends one. Never fabricate "".
|
|
1823
|
+
...typeof job.key === "string" && job.key !== "" && { key: job.key },
|
|
1561
1824
|
queueName: String(job.queueName ?? job.queue ?? ""),
|
|
1562
1825
|
status: job.status,
|
|
1563
1826
|
payload: job.payload ?? {},
|
|
@@ -1769,19 +2032,18 @@ var OrganizationsResource = class {
|
|
|
1769
2032
|
/**
|
|
1770
2033
|
* Clear/delete the webhook token for the current organization.
|
|
1771
2034
|
*
|
|
1772
|
-
*
|
|
1773
|
-
*
|
|
1774
|
-
*
|
|
1775
|
-
*
|
|
1776
|
-
* @
|
|
1777
|
-
*
|
|
1778
|
-
* await client.organizations.clearWebhookToken();
|
|
1779
|
-
* ```
|
|
2035
|
+
* @deprecated The API refuses this operation on purpose (an org without a
|
|
2036
|
+
* webhook token would have an open public ingestion endpoint) and always
|
|
2037
|
+
* answers with an error. This method now throws
|
|
2038
|
+
* {@link UnsupportedOperationError} without a network call; rotate the token
|
|
2039
|
+
* with {@link OrganizationsResource.regenerateWebhookToken} instead. It will
|
|
2040
|
+
* be removed in the next major release.
|
|
1780
2041
|
*/
|
|
1781
2042
|
async clearWebhookToken() {
|
|
1782
|
-
|
|
1783
|
-
|
|
1784
|
-
|
|
2043
|
+
throw new UnsupportedOperationError(
|
|
2044
|
+
"organizations.clearWebhookToken",
|
|
2045
|
+
"clearing the webhook token is not allowed; use organizations.regenerateWebhookToken() to rotate it"
|
|
2046
|
+
);
|
|
1785
2047
|
}
|
|
1786
2048
|
};
|
|
1787
2049
|
|
|
@@ -1989,24 +2251,8 @@ var AdminResource = class {
|
|
|
1989
2251
|
});
|
|
1990
2252
|
}
|
|
1991
2253
|
};
|
|
1992
|
-
|
|
1993
|
-
|
|
1994
|
-
return Buffer.from(body, "utf8");
|
|
1995
|
-
}
|
|
1996
|
-
if (body instanceof Uint8Array) {
|
|
1997
|
-
return body;
|
|
1998
|
-
}
|
|
1999
|
-
return new Uint8Array(body);
|
|
2000
|
-
}
|
|
2001
|
-
function githubSignature(secret, body) {
|
|
2002
|
-
const hex = createHmac("sha256", secret).update(body).digest("hex");
|
|
2003
|
-
return `sha256=${hex}`;
|
|
2004
|
-
}
|
|
2005
|
-
function stripeSignature(secret, body, timestamp) {
|
|
2006
|
-
const payload = `${timestamp}.${Buffer.from(body).toString("utf8")}`;
|
|
2007
|
-
const hex = createHmac("sha256", secret).update(payload, "utf8").digest("hex");
|
|
2008
|
-
return `t=${timestamp},v1=${hex}`;
|
|
2009
|
-
}
|
|
2254
|
+
|
|
2255
|
+
// src/resources/webhook-ingestion.ts
|
|
2010
2256
|
var WebhookIngestionResource = class {
|
|
2011
2257
|
constructor(http) {
|
|
2012
2258
|
this.http = http;
|
|
@@ -2014,66 +2260,32 @@ var WebhookIngestionResource = class {
|
|
|
2014
2260
|
/**
|
|
2015
2261
|
* Ingest a GitHub webhook.
|
|
2016
2262
|
*
|
|
2017
|
-
*
|
|
2018
|
-
|
|
2019
|
-
|
|
2020
|
-
|
|
2021
|
-
|
|
2022
|
-
|
|
2023
|
-
|
|
2024
|
-
|
|
2025
|
-
|
|
2026
|
-
|
|
2027
|
-
|
|
2028
|
-
"Content-Type": "application/json",
|
|
2029
|
-
"X-GitHub-Event": options.githubEvent,
|
|
2030
|
-
"X-Hub-Signature-256": signature
|
|
2031
|
-
};
|
|
2032
|
-
if (options.webhookToken) {
|
|
2033
|
-
headers["X-Webhook-Token"] = options.webhookToken;
|
|
2034
|
-
}
|
|
2035
|
-
if (options.forwardedProto) {
|
|
2036
|
-
headers["X-Forwarded-Proto"] = options.forwardedProto;
|
|
2037
|
-
}
|
|
2038
|
-
await this.http.request(`/webhooks/${orgId}/github`, {
|
|
2039
|
-
method: "POST",
|
|
2040
|
-
rawBody: bytes,
|
|
2041
|
-
headers,
|
|
2042
|
-
skipRequestConversion: true,
|
|
2043
|
-
skipResponseConversion: true
|
|
2044
|
-
});
|
|
2263
|
+
* @deprecated The API has no `/webhooks/{org_id}/github` route (it always
|
|
2264
|
+
* answered 404). This method now throws {@link UnsupportedOperationError}
|
|
2265
|
+
* without a network call; point the GitHub webhook at `ingest.custom` /
|
|
2266
|
+
* `POST /api/v1/webhooks/{org_id}/custom` instead. It will be removed in the
|
|
2267
|
+
* next major release.
|
|
2268
|
+
*/
|
|
2269
|
+
async github(_orgId, _body, _options) {
|
|
2270
|
+
throw new UnsupportedOperationError(
|
|
2271
|
+
"ingest.github",
|
|
2272
|
+
"the API has no GitHub webhook route; use ingest.custom (POST /webhooks/{org_id}/custom)"
|
|
2273
|
+
);
|
|
2045
2274
|
}
|
|
2046
2275
|
/**
|
|
2047
2276
|
* Ingest a Stripe webhook.
|
|
2048
2277
|
*
|
|
2049
|
-
*
|
|
2050
|
-
|
|
2051
|
-
|
|
2052
|
-
|
|
2053
|
-
|
|
2054
|
-
|
|
2055
|
-
|
|
2056
|
-
|
|
2057
|
-
|
|
2058
|
-
)
|
|
2059
|
-
|
|
2060
|
-
const headers = {
|
|
2061
|
-
"Content-Type": "application/json",
|
|
2062
|
-
"Stripe-Signature": signature
|
|
2063
|
-
};
|
|
2064
|
-
if (options.webhookToken) {
|
|
2065
|
-
headers["X-Webhook-Token"] = options.webhookToken;
|
|
2066
|
-
}
|
|
2067
|
-
if (options.forwardedProto) {
|
|
2068
|
-
headers["X-Forwarded-Proto"] = options.forwardedProto;
|
|
2069
|
-
}
|
|
2070
|
-
await this.http.request(`/webhooks/${orgId}/stripe`, {
|
|
2071
|
-
method: "POST",
|
|
2072
|
-
rawBody: bytes,
|
|
2073
|
-
headers,
|
|
2074
|
-
skipRequestConversion: true,
|
|
2075
|
-
skipResponseConversion: true
|
|
2076
|
-
});
|
|
2278
|
+
* @deprecated The API has no `/webhooks/{org_id}/stripe` route (it always
|
|
2279
|
+
* answered 404). This method now throws {@link UnsupportedOperationError}
|
|
2280
|
+
* without a network call; point the Stripe webhook at `ingest.custom` /
|
|
2281
|
+
* `POST /api/v1/webhooks/{org_id}/custom` instead. It will be removed in the
|
|
2282
|
+
* next major release.
|
|
2283
|
+
*/
|
|
2284
|
+
async stripe(_orgId, _body, _options = {}) {
|
|
2285
|
+
throw new UnsupportedOperationError(
|
|
2286
|
+
"ingest.stripe",
|
|
2287
|
+
"the API has no Stripe webhook route; use ingest.custom (POST /webhooks/{org_id}/custom)"
|
|
2288
|
+
);
|
|
2077
2289
|
}
|
|
2078
2290
|
/**
|
|
2079
2291
|
* Ingest a custom webhook.
|
|
@@ -2100,6 +2312,31 @@ var WebhookIngestionResource = class {
|
|
|
2100
2312
|
status: raw.status
|
|
2101
2313
|
};
|
|
2102
2314
|
}
|
|
2315
|
+
/**
|
|
2316
|
+
* The inbound webhook log: accepted requests to
|
|
2317
|
+
* `POST /webhooks/{org_id}/custom`, newest first
|
|
2318
|
+
* (`GET /webhooks/deliveries`, API 0.1.122+).
|
|
2319
|
+
*
|
|
2320
|
+
* Needs an API key without a queue restriction (a queue-scoped key gets a
|
|
2321
|
+
* 403 `AuthorizationError`). Entries carry no payload and no signature;
|
|
2322
|
+
* requests that were rejected are not logged. `limit` defaults to 50 (at
|
|
2323
|
+
* most 100); an invalid filter is a 400 `ValidationError`.
|
|
2324
|
+
*/
|
|
2325
|
+
async listDeliveries(params) {
|
|
2326
|
+
const rows = await this.http.get(
|
|
2327
|
+
"/webhooks/deliveries",
|
|
2328
|
+
{
|
|
2329
|
+
params: {
|
|
2330
|
+
status: params?.status,
|
|
2331
|
+
since: toQueryTimestamp(params?.since),
|
|
2332
|
+
queueName: params?.queueName,
|
|
2333
|
+
limit: params?.limit,
|
|
2334
|
+
offset: params?.offset
|
|
2335
|
+
}
|
|
2336
|
+
}
|
|
2337
|
+
);
|
|
2338
|
+
return rows ?? [];
|
|
2339
|
+
}
|
|
2103
2340
|
};
|
|
2104
2341
|
|
|
2105
2342
|
// src/realtime/event-map.ts
|
|
@@ -2450,6 +2687,13 @@ var SSE_EVENT_NAMES = [
|
|
|
2450
2687
|
"ping",
|
|
2451
2688
|
"error"
|
|
2452
2689
|
];
|
|
2690
|
+
var TERMINAL_JOB_STATUSES = /* @__PURE__ */ new Set([
|
|
2691
|
+
"completed",
|
|
2692
|
+
"failed",
|
|
2693
|
+
"deadletter",
|
|
2694
|
+
"cancelled"
|
|
2695
|
+
]);
|
|
2696
|
+
var STABLE_CONNECTION_MS = 3e4;
|
|
2453
2697
|
var SseRealtimeClient = class {
|
|
2454
2698
|
options;
|
|
2455
2699
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
@@ -2458,6 +2702,12 @@ var SseRealtimeClient = class {
|
|
|
2458
2702
|
reconnectAttempts = 0;
|
|
2459
2703
|
reconnectTimer = null;
|
|
2460
2704
|
filter = null;
|
|
2705
|
+
/** When the current connection opened (for the stable-connection check). */
|
|
2706
|
+
connectedAt = null;
|
|
2707
|
+
/** The server finished this stream on purpose (terminal job); do not reconnect. */
|
|
2708
|
+
streamEnded = false;
|
|
2709
|
+
/** The server rejected our token; mint a fresh one for the next connect. */
|
|
2710
|
+
forceTokenRefresh = false;
|
|
2461
2711
|
// Event handlers
|
|
2462
2712
|
eventHandlers = /* @__PURE__ */ new Map();
|
|
2463
2713
|
allEventsHandlers = /* @__PURE__ */ new Set();
|
|
@@ -2496,6 +2746,7 @@ var SseRealtimeClient = class {
|
|
|
2496
2746
|
return;
|
|
2497
2747
|
}
|
|
2498
2748
|
this.filter = filter || null;
|
|
2749
|
+
this.streamEnded = false;
|
|
2499
2750
|
this.setState("connecting");
|
|
2500
2751
|
const sseUrl = this.buildSseUrl(filter);
|
|
2501
2752
|
this.options.debug(`Connecting to SSE: ${sseUrl}`);
|
|
@@ -2515,7 +2766,9 @@ var SseRealtimeClient = class {
|
|
|
2515
2766
|
}
|
|
2516
2767
|
let token;
|
|
2517
2768
|
try {
|
|
2518
|
-
|
|
2769
|
+
const force = this.forceTokenRefresh;
|
|
2770
|
+
this.forceTokenRefresh = false;
|
|
2771
|
+
token = await this.options.tokenProvider(force);
|
|
2519
2772
|
} catch (error) {
|
|
2520
2773
|
this.setState("disconnected");
|
|
2521
2774
|
throw new Error(`Failed to acquire realtime token: ${error}`);
|
|
@@ -2535,7 +2788,7 @@ var SseRealtimeClient = class {
|
|
|
2535
2788
|
this.eventSource.onopen = () => {
|
|
2536
2789
|
this.options.debug("SSE connected");
|
|
2537
2790
|
this.setState("connected");
|
|
2538
|
-
this.
|
|
2791
|
+
this.connectedAt = Date.now();
|
|
2539
2792
|
resolve();
|
|
2540
2793
|
};
|
|
2541
2794
|
this.eventSource.onmessage = (event) => {
|
|
@@ -2546,8 +2799,11 @@ var SseRealtimeClient = class {
|
|
|
2546
2799
|
if (typeof event?.data === "string") this.handleMessage(event.data);
|
|
2547
2800
|
});
|
|
2548
2801
|
}
|
|
2549
|
-
this.eventSource.onerror = () => {
|
|
2802
|
+
this.eventSource.onerror = (event) => {
|
|
2550
2803
|
this.options.debug("SSE error");
|
|
2804
|
+
if (event?.code === 401 || event?.code === 403) {
|
|
2805
|
+
this.forceTokenRefresh = true;
|
|
2806
|
+
}
|
|
2551
2807
|
if (this.state === "connecting") {
|
|
2552
2808
|
this.setState("disconnected");
|
|
2553
2809
|
if (this.eventSource) {
|
|
@@ -2579,6 +2835,8 @@ var SseRealtimeClient = class {
|
|
|
2579
2835
|
}
|
|
2580
2836
|
this.setState("disconnected");
|
|
2581
2837
|
this.filter = null;
|
|
2838
|
+
this.connectedAt = null;
|
|
2839
|
+
this.streamEnded = false;
|
|
2582
2840
|
}
|
|
2583
2841
|
/**
|
|
2584
2842
|
* Reconnect with a new filter (SSE requires reconnection to change subscriptions)
|
|
@@ -2643,6 +2901,9 @@ var SseRealtimeClient = class {
|
|
|
2643
2901
|
parsed.type = eventType;
|
|
2644
2902
|
const event = parsed;
|
|
2645
2903
|
this.options.debug(`Received SSE event: ${eventType}`, event);
|
|
2904
|
+
if (this.filter?.jobId && isStreamEndingEvent(eventType, parsed.data)) {
|
|
2905
|
+
this.streamEnded = true;
|
|
2906
|
+
}
|
|
2646
2907
|
const handlers = this.eventHandlers.get(eventType);
|
|
2647
2908
|
if (handlers) {
|
|
2648
2909
|
const eventData = event.data;
|
|
@@ -2670,6 +2931,15 @@ var SseRealtimeClient = class {
|
|
|
2670
2931
|
this.eventSource.close();
|
|
2671
2932
|
this.eventSource = null;
|
|
2672
2933
|
}
|
|
2934
|
+
if (this.streamEnded) {
|
|
2935
|
+
this.connectedAt = null;
|
|
2936
|
+
this.setState("disconnected");
|
|
2937
|
+
return;
|
|
2938
|
+
}
|
|
2939
|
+
if (this.connectedAt !== null && Date.now() - this.connectedAt >= STABLE_CONNECTION_MS) {
|
|
2940
|
+
this.reconnectAttempts = 0;
|
|
2941
|
+
}
|
|
2942
|
+
this.connectedAt = null;
|
|
2673
2943
|
if (this.options.autoReconnect && this.reconnectAttempts < this.options.maxReconnectAttempts) {
|
|
2674
2944
|
this.scheduleReconnect();
|
|
2675
2945
|
} else {
|
|
@@ -2701,6 +2971,19 @@ var SseRealtimeClient = class {
|
|
|
2701
2971
|
}, delay);
|
|
2702
2972
|
}
|
|
2703
2973
|
};
|
|
2974
|
+
function isStreamEndingEvent(eventType, data) {
|
|
2975
|
+
if (eventType === "error" || eventType === "job.completed") {
|
|
2976
|
+
return true;
|
|
2977
|
+
}
|
|
2978
|
+
const record = data ?? {};
|
|
2979
|
+
if (eventType === "job.failed") {
|
|
2980
|
+
return record.willRetry === false;
|
|
2981
|
+
}
|
|
2982
|
+
if (eventType === "job.status_changed" || eventType === "job.status") {
|
|
2983
|
+
return typeof record.newStatus === "string" && TERMINAL_JOB_STATUSES.has(record.newStatus);
|
|
2984
|
+
}
|
|
2985
|
+
return false;
|
|
2986
|
+
}
|
|
2704
2987
|
|
|
2705
2988
|
// src/realtime/index.ts
|
|
2706
2989
|
var SpooledRealtime = class {
|
|
@@ -2887,114 +3170,10 @@ function clearProtoCache() {
|
|
|
2887
3170
|
cachedDefinition = null;
|
|
2888
3171
|
}
|
|
2889
3172
|
|
|
2890
|
-
// src/grpc/
|
|
2891
|
-
|
|
2892
|
-
|
|
2893
|
-
if (signal) {
|
|
2894
|
-
signal.addEventListener("abort", () => {
|
|
2895
|
-
stream.cancel();
|
|
2896
|
-
});
|
|
2897
|
-
}
|
|
2898
|
-
return yield* new AsyncIterableStream(stream, {
|
|
2899
|
-
onConnected,
|
|
2900
|
-
onError,
|
|
2901
|
-
onEnd
|
|
2902
|
-
});
|
|
3173
|
+
// src/grpc/encode.ts
|
|
3174
|
+
function encodeFailRequest(params) {
|
|
3175
|
+
return { ...params, retry: params.retry ?? true };
|
|
2903
3176
|
}
|
|
2904
|
-
var AsyncIterableStream = class {
|
|
2905
|
-
stream;
|
|
2906
|
-
options;
|
|
2907
|
-
constructor(stream, options = {}) {
|
|
2908
|
-
this.stream = stream;
|
|
2909
|
-
this.options = options;
|
|
2910
|
-
}
|
|
2911
|
-
async *[Symbol.asyncIterator]() {
|
|
2912
|
-
const { onConnected, onError, onEnd } = this.options;
|
|
2913
|
-
const queue = [];
|
|
2914
|
-
let error = null;
|
|
2915
|
-
let ended = false;
|
|
2916
|
-
let resolve = null;
|
|
2917
|
-
this.stream.on("data", (data) => {
|
|
2918
|
-
queue.push(data);
|
|
2919
|
-
resolve?.();
|
|
2920
|
-
});
|
|
2921
|
-
this.stream.on("error", (err) => {
|
|
2922
|
-
error = err;
|
|
2923
|
-
onError?.(err);
|
|
2924
|
-
resolve?.();
|
|
2925
|
-
});
|
|
2926
|
-
this.stream.on("end", () => {
|
|
2927
|
-
ended = true;
|
|
2928
|
-
onEnd?.();
|
|
2929
|
-
resolve?.();
|
|
2930
|
-
});
|
|
2931
|
-
this.stream.on("status", (status2) => {
|
|
2932
|
-
if (status2.code === 0) {
|
|
2933
|
-
onConnected?.();
|
|
2934
|
-
}
|
|
2935
|
-
});
|
|
2936
|
-
while (true) {
|
|
2937
|
-
if (queue.length > 0) {
|
|
2938
|
-
yield queue.shift();
|
|
2939
|
-
continue;
|
|
2940
|
-
}
|
|
2941
|
-
if (error) {
|
|
2942
|
-
throw error;
|
|
2943
|
-
}
|
|
2944
|
-
if (ended) {
|
|
2945
|
-
return;
|
|
2946
|
-
}
|
|
2947
|
-
await new Promise((r) => {
|
|
2948
|
-
resolve = r;
|
|
2949
|
-
});
|
|
2950
|
-
}
|
|
2951
|
-
}
|
|
2952
|
-
};
|
|
2953
|
-
function createProcessJobsStream(call, options) {
|
|
2954
|
-
const { signal, onError, onEnd } = options ?? {};
|
|
2955
|
-
if (signal) {
|
|
2956
|
-
signal.addEventListener("abort", () => {
|
|
2957
|
-
call.cancel();
|
|
2958
|
-
});
|
|
2959
|
-
}
|
|
2960
|
-
return {
|
|
2961
|
-
send(request) {
|
|
2962
|
-
call.write(request);
|
|
2963
|
-
},
|
|
2964
|
-
receive() {
|
|
2965
|
-
return new AsyncIterableStream(call, { onError, onEnd });
|
|
2966
|
-
},
|
|
2967
|
-
end() {
|
|
2968
|
-
call.end();
|
|
2969
|
-
},
|
|
2970
|
-
cancel() {
|
|
2971
|
-
call.cancel();
|
|
2972
|
-
}
|
|
2973
|
-
};
|
|
2974
|
-
}
|
|
2975
|
-
function createJobStream(call, options) {
|
|
2976
|
-
const { signal, onConnected, onError, onEnd } = options ?? {};
|
|
2977
|
-
if (signal) {
|
|
2978
|
-
signal.addEventListener("abort", () => {
|
|
2979
|
-
call.cancel();
|
|
2980
|
-
});
|
|
2981
|
-
}
|
|
2982
|
-
const iterable = new AsyncIterableStream(call, {
|
|
2983
|
-
onConnected,
|
|
2984
|
-
onError,
|
|
2985
|
-
onEnd
|
|
2986
|
-
});
|
|
2987
|
-
return {
|
|
2988
|
-
[Symbol.asyncIterator]() {
|
|
2989
|
-
return iterable[Symbol.asyncIterator]();
|
|
2990
|
-
},
|
|
2991
|
-
cancel() {
|
|
2992
|
-
call.cancel();
|
|
2993
|
-
}
|
|
2994
|
-
};
|
|
2995
|
-
}
|
|
2996
|
-
|
|
2997
|
-
// src/grpc/client.ts
|
|
2998
3177
|
function encodeStruct(value) {
|
|
2999
3178
|
if (value == null) {
|
|
3000
3179
|
return void 0;
|
|
@@ -3089,10 +3268,157 @@ function encodeCompleteRequest(params) {
|
|
|
3089
3268
|
function decodeJob(job) {
|
|
3090
3269
|
return {
|
|
3091
3270
|
...job,
|
|
3271
|
+
// proto-loader keeps an enum number it does not know (a status added to
|
|
3272
|
+
// the server's proto after this SDK) as a number; keep it a string so
|
|
3273
|
+
// `status` always matches its type.
|
|
3274
|
+
status: typeof job.status === "number" ? String(job.status) : job.status,
|
|
3092
3275
|
payload: decodeStruct(job.payload),
|
|
3093
3276
|
result: decodeStruct(job.result)
|
|
3094
3277
|
};
|
|
3095
3278
|
}
|
|
3279
|
+
function encodeProcessRequest(request) {
|
|
3280
|
+
if (request.fail) {
|
|
3281
|
+
return { ...request, fail: encodeFailRequest(request.fail) };
|
|
3282
|
+
}
|
|
3283
|
+
if (request.complete) {
|
|
3284
|
+
return { ...request, complete: encodeCompleteRequest(request.complete) };
|
|
3285
|
+
}
|
|
3286
|
+
return request;
|
|
3287
|
+
}
|
|
3288
|
+
function decodeProcessResponse(response) {
|
|
3289
|
+
return response.job ? { ...response, job: decodeJob(response.job) } : response;
|
|
3290
|
+
}
|
|
3291
|
+
|
|
3292
|
+
// src/grpc/streaming.ts
|
|
3293
|
+
async function* asyncIterableFromStream(stream, options) {
|
|
3294
|
+
const { signal, onConnected, onError, onEnd } = options ?? {};
|
|
3295
|
+
if (signal) {
|
|
3296
|
+
signal.addEventListener("abort", () => {
|
|
3297
|
+
stream.cancel();
|
|
3298
|
+
});
|
|
3299
|
+
}
|
|
3300
|
+
return yield* new AsyncIterableStream(stream, {
|
|
3301
|
+
onConnected,
|
|
3302
|
+
onError,
|
|
3303
|
+
onEnd
|
|
3304
|
+
});
|
|
3305
|
+
}
|
|
3306
|
+
var AsyncIterableStream = class {
|
|
3307
|
+
stream;
|
|
3308
|
+
options;
|
|
3309
|
+
decode;
|
|
3310
|
+
constructor(stream, options = {}, decode = (item) => item) {
|
|
3311
|
+
this.stream = stream;
|
|
3312
|
+
this.options = options;
|
|
3313
|
+
this.decode = decode;
|
|
3314
|
+
}
|
|
3315
|
+
async *[Symbol.asyncIterator]() {
|
|
3316
|
+
const { onConnected, onError, onEnd } = this.options;
|
|
3317
|
+
const queue = [];
|
|
3318
|
+
let error = null;
|
|
3319
|
+
let ended = false;
|
|
3320
|
+
let decodeFailed = false;
|
|
3321
|
+
let resolve = null;
|
|
3322
|
+
this.stream.on("data", (data) => {
|
|
3323
|
+
if (decodeFailed) {
|
|
3324
|
+
return;
|
|
3325
|
+
}
|
|
3326
|
+
try {
|
|
3327
|
+
queue.push(this.decode(data));
|
|
3328
|
+
} catch (decodeError) {
|
|
3329
|
+
decodeFailed = true;
|
|
3330
|
+
error = decodeError instanceof Error ? decodeError : new Error(String(decodeError));
|
|
3331
|
+
try {
|
|
3332
|
+
this.stream.cancel?.();
|
|
3333
|
+
} catch {
|
|
3334
|
+
}
|
|
3335
|
+
}
|
|
3336
|
+
resolve?.();
|
|
3337
|
+
});
|
|
3338
|
+
this.stream.on("error", (err) => {
|
|
3339
|
+
if (decodeFailed) {
|
|
3340
|
+
return;
|
|
3341
|
+
}
|
|
3342
|
+
error = err;
|
|
3343
|
+
onError?.(err);
|
|
3344
|
+
resolve?.();
|
|
3345
|
+
});
|
|
3346
|
+
this.stream.on("end", () => {
|
|
3347
|
+
ended = true;
|
|
3348
|
+
onEnd?.();
|
|
3349
|
+
resolve?.();
|
|
3350
|
+
});
|
|
3351
|
+
this.stream.on("status", (status2) => {
|
|
3352
|
+
if (status2.code === 0) {
|
|
3353
|
+
onConnected?.();
|
|
3354
|
+
}
|
|
3355
|
+
});
|
|
3356
|
+
while (true) {
|
|
3357
|
+
if (queue.length > 0) {
|
|
3358
|
+
yield queue.shift();
|
|
3359
|
+
continue;
|
|
3360
|
+
}
|
|
3361
|
+
if (error) {
|
|
3362
|
+
throw error;
|
|
3363
|
+
}
|
|
3364
|
+
if (ended) {
|
|
3365
|
+
return;
|
|
3366
|
+
}
|
|
3367
|
+
await new Promise((r) => {
|
|
3368
|
+
resolve = r;
|
|
3369
|
+
});
|
|
3370
|
+
}
|
|
3371
|
+
}
|
|
3372
|
+
};
|
|
3373
|
+
function createProcessJobsStream(call, options) {
|
|
3374
|
+
const { signal, onError, onEnd } = options ?? {};
|
|
3375
|
+
if (signal) {
|
|
3376
|
+
signal.addEventListener("abort", () => {
|
|
3377
|
+
call.cancel();
|
|
3378
|
+
});
|
|
3379
|
+
}
|
|
3380
|
+
return {
|
|
3381
|
+
send(request) {
|
|
3382
|
+
call.write(encodeProcessRequest(request));
|
|
3383
|
+
},
|
|
3384
|
+
receive() {
|
|
3385
|
+
return new AsyncIterableStream(
|
|
3386
|
+
call,
|
|
3387
|
+
{ onError, onEnd },
|
|
3388
|
+
decodeProcessResponse
|
|
3389
|
+
);
|
|
3390
|
+
},
|
|
3391
|
+
end() {
|
|
3392
|
+
call.end();
|
|
3393
|
+
},
|
|
3394
|
+
cancel() {
|
|
3395
|
+
call.cancel();
|
|
3396
|
+
}
|
|
3397
|
+
};
|
|
3398
|
+
}
|
|
3399
|
+
function createJobStream(call, options) {
|
|
3400
|
+
const { signal, onConnected, onError, onEnd } = options ?? {};
|
|
3401
|
+
if (signal) {
|
|
3402
|
+
signal.addEventListener("abort", () => {
|
|
3403
|
+
call.cancel();
|
|
3404
|
+
});
|
|
3405
|
+
}
|
|
3406
|
+
const iterable = new AsyncIterableStream(
|
|
3407
|
+
call,
|
|
3408
|
+
{ onConnected, onError, onEnd },
|
|
3409
|
+
decodeJob
|
|
3410
|
+
);
|
|
3411
|
+
return {
|
|
3412
|
+
[Symbol.asyncIterator]() {
|
|
3413
|
+
return iterable[Symbol.asyncIterator]();
|
|
3414
|
+
},
|
|
3415
|
+
cancel() {
|
|
3416
|
+
call.cancel();
|
|
3417
|
+
}
|
|
3418
|
+
};
|
|
3419
|
+
}
|
|
3420
|
+
|
|
3421
|
+
// src/grpc/client.ts
|
|
3096
3422
|
function promisify(client, method, request, metadata, timeoutMs) {
|
|
3097
3423
|
return new Promise((resolve, reject) => {
|
|
3098
3424
|
const callback = (error, response) => {
|
|
@@ -3147,12 +3473,19 @@ function mapGrpcError(error, timeoutMs) {
|
|
|
3147
3473
|
return new ServerError(message, 500, "GRPC_ERROR");
|
|
3148
3474
|
}
|
|
3149
3475
|
}
|
|
3476
|
+
function grpcHost(address) {
|
|
3477
|
+
if (address.startsWith("[")) {
|
|
3478
|
+
const end = address.indexOf("]");
|
|
3479
|
+
return end === -1 ? address : address.slice(1, end);
|
|
3480
|
+
}
|
|
3481
|
+
return address.split(":").length > 2 ? address : address.split(":")[0];
|
|
3482
|
+
}
|
|
3150
3483
|
function shouldUseTls(address, explicitTls) {
|
|
3151
3484
|
if (explicitTls !== void 0) {
|
|
3152
3485
|
return explicitTls;
|
|
3153
3486
|
}
|
|
3154
|
-
const host = address
|
|
3155
|
-
return host !== "localhost" && host !== "127.0.0.1" &&
|
|
3487
|
+
const host = grpcHost(address).toLowerCase();
|
|
3488
|
+
return host !== "localhost" && host !== "127.0.0.1" && host !== "::1";
|
|
3156
3489
|
}
|
|
3157
3490
|
var GrpcQueueOperations = class {
|
|
3158
3491
|
constructor(client, metadata, timeoutMs) {
|
|
@@ -3204,7 +3537,7 @@ var GrpcQueueOperations = class {
|
|
|
3204
3537
|
return promisify(
|
|
3205
3538
|
this.client,
|
|
3206
3539
|
this.client.Fail,
|
|
3207
|
-
params,
|
|
3540
|
+
encodeFailRequest(params),
|
|
3208
3541
|
this.metadata,
|
|
3209
3542
|
this.timeoutMs
|
|
3210
3543
|
);
|
|
@@ -3594,6 +3927,13 @@ var SpooledClient = class _SpooledClient {
|
|
|
3594
3927
|
/**
|
|
3595
3928
|
* Exchange the API key for a JWT via /auth/login and cache the result.
|
|
3596
3929
|
*
|
|
3930
|
+
* The JWT belongs to the realtime transport only. It must NOT replace the
|
|
3931
|
+
* REST bearer: an API-key client has no refresh token wired into its HTTP
|
|
3932
|
+
* layer, so after the JWT's 24 h lifetime every REST call (claim, complete,
|
|
3933
|
+
* heartbeat, create) would have failed with 401 for good. REST keeps using
|
|
3934
|
+
* the API key; realtime re-mints its own JWT near expiry (see
|
|
3935
|
+
* `getJwtToken`).
|
|
3936
|
+
*
|
|
3597
3937
|
* Concurrent callers share a single in-flight login so a burst of reconnects
|
|
3598
3938
|
* (e.g. WebSocket and SSE at once) issues just one request.
|
|
3599
3939
|
*/
|
|
@@ -3603,9 +3943,6 @@ var SpooledClient = class _SpooledClient {
|
|
|
3603
3943
|
}
|
|
3604
3944
|
this.jwtLoginPromise = (async () => {
|
|
3605
3945
|
const response = await this.auth.login({ apiKey: this.config.apiKey });
|
|
3606
|
-
this.http.setAuthToken(response.accessToken);
|
|
3607
|
-
this.tokenExpiresAt = Date.now() + response.expiresIn * 1e3;
|
|
3608
|
-
this.config.refreshToken = response.refreshToken;
|
|
3609
3946
|
this.cachedJwt = response.accessToken;
|
|
3610
3947
|
return response.accessToken;
|
|
3611
3948
|
})();
|
|
@@ -3661,6 +3998,17 @@ var SpooledClient = class _SpooledClient {
|
|
|
3661
3998
|
getConfig() {
|
|
3662
3999
|
return this.config;
|
|
3663
4000
|
}
|
|
4001
|
+
/**
|
|
4002
|
+
* Rate-limit state from the most recent response that carried
|
|
4003
|
+
* `X-RateLimit-*` headers (API 0.1.122+): `limit`, `remaining`, `reset`
|
|
4004
|
+
* (from the Unix-seconds header), `scope` and, after a 429, `retryAfter`.
|
|
4005
|
+
* `undefined` until such a response arrives, and for servers or routes that
|
|
4006
|
+
* do not send the headers. A 429 also carries the same values on
|
|
4007
|
+
* `RateLimitError.rateLimitInfo`.
|
|
4008
|
+
*/
|
|
4009
|
+
getRateLimitInfo() {
|
|
4010
|
+
return this.http.getLastRateLimitInfo();
|
|
4011
|
+
}
|
|
3664
4012
|
/**
|
|
3665
4013
|
* Get circuit breaker statistics
|
|
3666
4014
|
*/
|
|
@@ -3801,6 +4149,9 @@ var DEFAULT_OPTIONS = {
|
|
|
3801
4149
|
autoStart: false,
|
|
3802
4150
|
shutdownTimeout: 3e4
|
|
3803
4151
|
};
|
|
4152
|
+
var JOB_TIMEOUT_MESSAGE = "Job timeout exceeded";
|
|
4153
|
+
var SHUTDOWN_TIMEOUT_MESSAGE = "Worker shutdown timeout";
|
|
4154
|
+
var MAX_TIMER_MS = 2 ** 31 - 1;
|
|
3804
4155
|
var SpooledWorker = class {
|
|
3805
4156
|
client;
|
|
3806
4157
|
/**
|
|
@@ -3817,6 +4168,10 @@ var SpooledWorker = class {
|
|
|
3817
4168
|
pollTimer = null;
|
|
3818
4169
|
workerHeartbeatTimer = null;
|
|
3819
4170
|
shutdownPromise = null;
|
|
4171
|
+
/** The claim request currently on the wire, so a stop can wait for its jobs. */
|
|
4172
|
+
pollInFlight = null;
|
|
4173
|
+
/** Wakes the shutdown drain when the last active job has been acked. */
|
|
4174
|
+
drainWaiter = null;
|
|
3820
4175
|
// Event handlers
|
|
3821
4176
|
eventHandlers = /* @__PURE__ */ new Map();
|
|
3822
4177
|
constructor(client, options) {
|
|
@@ -3833,7 +4188,11 @@ var SpooledWorker = class {
|
|
|
3833
4188
|
this.debug = config.debug ?? (() => {
|
|
3834
4189
|
});
|
|
3835
4190
|
if (this.options.autoStart) {
|
|
3836
|
-
setImmediate(() =>
|
|
4191
|
+
setImmediate(() => {
|
|
4192
|
+
this.start().catch((error) => {
|
|
4193
|
+
this.debug("Auto-start failed", error);
|
|
4194
|
+
});
|
|
4195
|
+
});
|
|
3837
4196
|
}
|
|
3838
4197
|
}
|
|
3839
4198
|
/**
|
|
@@ -3904,7 +4263,15 @@ var SpooledWorker = class {
|
|
|
3904
4263
|
}
|
|
3905
4264
|
}
|
|
3906
4265
|
/**
|
|
3907
|
-
* Stop the worker gracefully
|
|
4266
|
+
* Stop the worker gracefully.
|
|
4267
|
+
*
|
|
4268
|
+
* Drains: the worker stops claiming, then lets running handlers finish for
|
|
4269
|
+
* up to `shutdownTimeout`. A handler that returns completes its job and one
|
|
4270
|
+
* that throws fails it, even though a stop was requested meanwhile; its
|
|
4271
|
+
* `signal` is NOT aborted during the grace period. Only handlers still
|
|
4272
|
+
* running when the timeout ends are aborted and failed with "Worker shutdown
|
|
4273
|
+
* timeout". The worker deregisters last, because deregistering returns every
|
|
4274
|
+
* job the worker still holds to the queue.
|
|
3908
4275
|
*/
|
|
3909
4276
|
async stop() {
|
|
3910
4277
|
if (this.state !== "running") {
|
|
@@ -3917,39 +4284,37 @@ var SpooledWorker = class {
|
|
|
3917
4284
|
return this.shutdownPromise;
|
|
3918
4285
|
}
|
|
3919
4286
|
async doStop() {
|
|
4287
|
+
const deadline = Date.now() + this.options.shutdownTimeout;
|
|
3920
4288
|
this.state = "stopping";
|
|
3921
4289
|
this.debug("Stopping worker...");
|
|
3922
4290
|
if (this.pollTimer) {
|
|
3923
4291
|
clearTimeout(this.pollTimer);
|
|
3924
4292
|
this.pollTimer = null;
|
|
3925
4293
|
}
|
|
3926
|
-
if (this.
|
|
3927
|
-
|
|
3928
|
-
this.workerHeartbeatTimer = null;
|
|
3929
|
-
}
|
|
3930
|
-
for (const active of this.activeJobs.values()) {
|
|
3931
|
-
active.abortController.abort();
|
|
4294
|
+
if (this.pollInFlight) {
|
|
4295
|
+
await this.pollInFlight;
|
|
3932
4296
|
}
|
|
3933
4297
|
if (this.activeJobs.size > 0) {
|
|
3934
4298
|
this.debug(
|
|
3935
4299
|
`Waiting for ${this.activeJobs.size} active jobs to complete...`
|
|
3936
4300
|
);
|
|
3937
|
-
await
|
|
3938
|
-
|
|
3939
|
-
|
|
3940
|
-
|
|
3941
|
-
|
|
3942
|
-
|
|
3943
|
-
this.
|
|
3944
|
-
|
|
3945
|
-
);
|
|
3946
|
-
await this.failJob(active.job, "Worker shutdown timeout");
|
|
3947
|
-
this.cleanupJob(active);
|
|
3948
|
-
}
|
|
4301
|
+
await this.waitForDrain(Math.max(0, deadline - Date.now()));
|
|
4302
|
+
const stragglers = Array.from(this.activeJobs.values()).filter(
|
|
4303
|
+
(active) => !active.settled
|
|
4304
|
+
);
|
|
4305
|
+
await Promise.all(stragglers.map((active) => this.abandonJob(active)));
|
|
4306
|
+
await Promise.all(
|
|
4307
|
+
Array.from(this.activeJobs.values()).map((active) => active.ack)
|
|
4308
|
+
);
|
|
3949
4309
|
}
|
|
3950
|
-
if (this.
|
|
4310
|
+
if (this.workerHeartbeatTimer) {
|
|
4311
|
+
clearInterval(this.workerHeartbeatTimer);
|
|
4312
|
+
this.workerHeartbeatTimer = null;
|
|
4313
|
+
}
|
|
4314
|
+
const workerId = this.workerId;
|
|
4315
|
+
if (workerId) {
|
|
3951
4316
|
try {
|
|
3952
|
-
await this.client.workers.deregister(
|
|
4317
|
+
await this.client.workers.deregister(workerId);
|
|
3953
4318
|
this.debug("Worker deregistered");
|
|
3954
4319
|
} catch (error) {
|
|
3955
4320
|
this.debug("Failed to deregister worker", error);
|
|
@@ -3957,7 +4322,7 @@ var SpooledWorker = class {
|
|
|
3957
4322
|
}
|
|
3958
4323
|
this.state = "stopped";
|
|
3959
4324
|
this.workerId = null;
|
|
3960
|
-
this.emit("stopped", { workerId:
|
|
4325
|
+
this.emit("stopped", { workerId: workerId ?? "", reason: "graceful" });
|
|
3961
4326
|
this.shutdownPromise = null;
|
|
3962
4327
|
}
|
|
3963
4328
|
/**
|
|
@@ -3996,7 +4361,17 @@ var SpooledWorker = class {
|
|
|
3996
4361
|
if (this.state !== "running") {
|
|
3997
4362
|
return;
|
|
3998
4363
|
}
|
|
3999
|
-
this.pollTimer = setTimeout(() =>
|
|
4364
|
+
this.pollTimer = setTimeout(() => {
|
|
4365
|
+
this.pollTimer = null;
|
|
4366
|
+
const run = this.poll();
|
|
4367
|
+
this.pollInFlight = run;
|
|
4368
|
+
const done = () => {
|
|
4369
|
+
if (this.pollInFlight === run) {
|
|
4370
|
+
this.pollInFlight = null;
|
|
4371
|
+
}
|
|
4372
|
+
};
|
|
4373
|
+
run.then(done, done);
|
|
4374
|
+
}, this.options.pollInterval);
|
|
4000
4375
|
}
|
|
4001
4376
|
async poll() {
|
|
4002
4377
|
if (this.state !== "running" || !this.workerId) {
|
|
@@ -4030,7 +4405,8 @@ var SpooledWorker = class {
|
|
|
4030
4405
|
executionId: Symbol(job.id),
|
|
4031
4406
|
job,
|
|
4032
4407
|
startedAt: /* @__PURE__ */ new Date(),
|
|
4033
|
-
abortController
|
|
4408
|
+
abortController,
|
|
4409
|
+
settled: false
|
|
4034
4410
|
};
|
|
4035
4411
|
this.activeJobs.set(activeJob.executionId, activeJob);
|
|
4036
4412
|
const heartbeatInterval = this.options.leaseDuration * this.options.heartbeatFraction * 1e3;
|
|
@@ -4059,23 +4435,72 @@ var SpooledWorker = class {
|
|
|
4059
4435
|
}
|
|
4060
4436
|
};
|
|
4061
4437
|
try {
|
|
4062
|
-
const
|
|
4063
|
-
if (
|
|
4064
|
-
this.debug(`Job ${job.id} was
|
|
4438
|
+
const outcome = await this.runHandler(activeJob, context);
|
|
4439
|
+
if (activeJob.settled) {
|
|
4440
|
+
this.debug(`Job ${job.id} was already settled; dropping late result`);
|
|
4065
4441
|
return;
|
|
4066
4442
|
}
|
|
4067
|
-
|
|
4068
|
-
|
|
4069
|
-
|
|
4070
|
-
|
|
4071
|
-
|
|
4443
|
+
activeJob.settled = true;
|
|
4444
|
+
switch (outcome.kind) {
|
|
4445
|
+
case "returned":
|
|
4446
|
+
activeJob.ack = this.completeJob(job, outcome.value ?? void 0);
|
|
4447
|
+
break;
|
|
4448
|
+
case "threw":
|
|
4449
|
+
activeJob.ack = this.failJob(
|
|
4450
|
+
job,
|
|
4451
|
+
outcome.error instanceof Error ? outcome.error.message : String(outcome.error)
|
|
4452
|
+
);
|
|
4453
|
+
break;
|
|
4454
|
+
case "timeout":
|
|
4455
|
+
abortController.abort(new Error(JOB_TIMEOUT_MESSAGE));
|
|
4456
|
+
this.emit("job:timeout", {
|
|
4457
|
+
jobId: job.id,
|
|
4458
|
+
queueName: job.queueName
|
|
4459
|
+
});
|
|
4460
|
+
activeJob.ack = this.failJob(job, JOB_TIMEOUT_MESSAGE);
|
|
4461
|
+
break;
|
|
4072
4462
|
}
|
|
4073
|
-
|
|
4074
|
-
await this.failJob(job, errorMessage);
|
|
4463
|
+
await activeJob.ack;
|
|
4075
4464
|
} finally {
|
|
4076
4465
|
this.cleanupJob(activeJob);
|
|
4077
4466
|
}
|
|
4078
4467
|
}
|
|
4468
|
+
/**
|
|
4469
|
+
* Run the handler, racing it against the job's `timeoutSeconds`.
|
|
4470
|
+
* Never rejects: a throw (sync or async) is reported as an outcome.
|
|
4471
|
+
*/
|
|
4472
|
+
runHandler(activeJob, context) {
|
|
4473
|
+
const run = (async () => {
|
|
4474
|
+
try {
|
|
4475
|
+
return { kind: "returned", value: await this.handler(context) };
|
|
4476
|
+
} catch (error) {
|
|
4477
|
+
return { kind: "threw", error };
|
|
4478
|
+
}
|
|
4479
|
+
})();
|
|
4480
|
+
const { timeoutSeconds } = activeJob.job;
|
|
4481
|
+
if (typeof timeoutSeconds !== "number" || !(timeoutSeconds > 0)) {
|
|
4482
|
+
return run;
|
|
4483
|
+
}
|
|
4484
|
+
return new Promise((resolve) => {
|
|
4485
|
+
activeJob.timeoutTimer = setTimeout(
|
|
4486
|
+
() => resolve({ kind: "timeout" }),
|
|
4487
|
+
Math.min(timeoutSeconds * 1e3, MAX_TIMER_MS)
|
|
4488
|
+
);
|
|
4489
|
+
run.then((outcome) => {
|
|
4490
|
+
this.clearTimeoutTimer(activeJob);
|
|
4491
|
+
resolve(outcome);
|
|
4492
|
+
});
|
|
4493
|
+
});
|
|
4494
|
+
}
|
|
4495
|
+
/** Abort a handler that outlived the shutdown grace period and fail its job. */
|
|
4496
|
+
async abandonJob(activeJob) {
|
|
4497
|
+
this.debug(`Force-failing job ${activeJob.job.id} due to shutdown timeout`);
|
|
4498
|
+
activeJob.settled = true;
|
|
4499
|
+
activeJob.abortController.abort(new Error(SHUTDOWN_TIMEOUT_MESSAGE));
|
|
4500
|
+
activeJob.ack = this.failJob(activeJob.job, SHUTDOWN_TIMEOUT_MESSAGE);
|
|
4501
|
+
await activeJob.ack;
|
|
4502
|
+
this.cleanupJob(activeJob);
|
|
4503
|
+
}
|
|
4079
4504
|
async completeJob(job, result) {
|
|
4080
4505
|
if (!this.workerId) return;
|
|
4081
4506
|
try {
|
|
@@ -4091,25 +4516,34 @@ var SpooledWorker = class {
|
|
|
4091
4516
|
});
|
|
4092
4517
|
} catch (error) {
|
|
4093
4518
|
this.debug(`Failed to complete job ${job.id}`, error);
|
|
4519
|
+
this.emit("error", { error });
|
|
4094
4520
|
}
|
|
4095
4521
|
}
|
|
4096
4522
|
async failJob(job, errorMessage) {
|
|
4097
4523
|
if (!this.workerId) return;
|
|
4098
4524
|
const willRetry = job.retryCount < job.maxRetries;
|
|
4525
|
+
const message = coerceFailureMessage(errorMessage);
|
|
4099
4526
|
try {
|
|
4100
4527
|
await this.client.jobs.fail(job.id, {
|
|
4101
4528
|
workerId: this.workerId,
|
|
4102
|
-
error:
|
|
4529
|
+
error: message,
|
|
4103
4530
|
...job.leaseId != null && { leaseId: job.leaseId }
|
|
4104
4531
|
});
|
|
4105
4532
|
this.emit("job:failed", {
|
|
4106
4533
|
jobId: job.id,
|
|
4107
4534
|
queueName: job.queueName,
|
|
4108
|
-
error:
|
|
4535
|
+
error: message,
|
|
4109
4536
|
willRetry
|
|
4110
4537
|
});
|
|
4111
4538
|
} catch (error) {
|
|
4112
4539
|
this.debug(`Failed to fail job ${job.id}`, error);
|
|
4540
|
+
this.emit("error", { error });
|
|
4541
|
+
}
|
|
4542
|
+
}
|
|
4543
|
+
clearTimeoutTimer(activeJob) {
|
|
4544
|
+
if (activeJob.timeoutTimer) {
|
|
4545
|
+
clearTimeout(activeJob.timeoutTimer);
|
|
4546
|
+
activeJob.timeoutTimer = void 0;
|
|
4113
4547
|
}
|
|
4114
4548
|
}
|
|
4115
4549
|
cleanupJob(activeJob) {
|
|
@@ -4117,8 +4551,12 @@ var SpooledWorker = class {
|
|
|
4117
4551
|
clearInterval(activeJob.heartbeatTimer);
|
|
4118
4552
|
activeJob.heartbeatTimer = void 0;
|
|
4119
4553
|
}
|
|
4554
|
+
this.clearTimeoutTimer(activeJob);
|
|
4120
4555
|
if (this.activeJobs.get(activeJob.executionId) === activeJob) {
|
|
4121
4556
|
this.activeJobs.delete(activeJob.executionId);
|
|
4557
|
+
if (this.activeJobs.size === 0) {
|
|
4558
|
+
this.drainWaiter?.();
|
|
4559
|
+
}
|
|
4122
4560
|
}
|
|
4123
4561
|
}
|
|
4124
4562
|
async sendJobHeartbeat(activeJob) {
|
|
@@ -4135,7 +4573,9 @@ var SpooledWorker = class {
|
|
|
4135
4573
|
}
|
|
4136
4574
|
}
|
|
4137
4575
|
async sendWorkerHeartbeat() {
|
|
4138
|
-
if (!this.workerId || this.state !== "running")
|
|
4576
|
+
if (!this.workerId || this.state !== "running" && this.state !== "stopping") {
|
|
4577
|
+
return;
|
|
4578
|
+
}
|
|
4139
4579
|
try {
|
|
4140
4580
|
await this.client.workers.heartbeat(this.workerId, {
|
|
4141
4581
|
currentJobs: this.activeJobs.size,
|
|
@@ -4145,16 +4585,28 @@ var SpooledWorker = class {
|
|
|
4145
4585
|
this.debug("Worker heartbeat failed", error);
|
|
4146
4586
|
}
|
|
4147
4587
|
}
|
|
4148
|
-
|
|
4149
|
-
|
|
4150
|
-
|
|
4588
|
+
/**
|
|
4589
|
+
* Resolve once no job is active, or after `timeoutMs`. The timer is always
|
|
4590
|
+
* cleared, so a clean stop leaves nothing that keeps the process alive.
|
|
4591
|
+
*/
|
|
4592
|
+
waitForDrain(timeoutMs) {
|
|
4593
|
+
if (this.activeJobs.size === 0) {
|
|
4594
|
+
return Promise.resolve();
|
|
4151
4595
|
}
|
|
4152
|
-
|
|
4153
|
-
|
|
4154
|
-
|
|
4596
|
+
return new Promise((resolve) => {
|
|
4597
|
+
const timer = setTimeout(() => {
|
|
4598
|
+
this.drainWaiter = null;
|
|
4599
|
+
resolve();
|
|
4600
|
+
}, timeoutMs);
|
|
4601
|
+
this.drainWaiter = () => {
|
|
4602
|
+
clearTimeout(timer);
|
|
4603
|
+
this.drainWaiter = null;
|
|
4604
|
+
resolve();
|
|
4605
|
+
};
|
|
4606
|
+
});
|
|
4155
4607
|
}
|
|
4156
4608
|
};
|
|
4157
4609
|
|
|
4158
|
-
export { API_VERSION, AdminResource, ApiKeysResource, AuthResource, AuthenticationError, AuthorizationError, BillingResource, CircuitBreaker, CircuitBreakerOpenError, CircuitState, ConflictError, DEFAULT_CONFIG, DashboardResource, GrpcJobStatus, GrpcJobStatusNumber, GrpcQueueOperations, GrpcWorkerOperations, HealthResource, HttpClient, JobsResource, MetricsResource, NetworkError, NotFoundError, OrganizationsResource, PROTO_LOADER_OPTIONS, PayloadTooLargeError, QueuesResource, RateLimitError, SDK_VERSION, SchedulesResource, ServerError, SpooledClient, SpooledError, SpooledGrpcClient, SpooledRealtime, SpooledWorker, SseRealtimeClient, TimeoutError, ValidationError, WebSocketRealtimeClient, WebhookIngestionResource, WebhooksResource, WorkersResource, WorkflowsResource, asyncIterableFromStream, calculateDelay, camelToSnake, clearProtoCache, convertQueryParams, convertRequest, convertResponse, createCircuitBreaker, createClient, createErrorFromResponse, createHttpClient, createJobStream, createPaginatedResponse, createProcessJobsStream, createRetryWrapper, dateToTimestamp, getProtoPath, isRetryableError, isSpooledError, loadProtoDefinition, loadProtoDefinitionAsync, parseRateLimitHeaders, resolveConfig, shouldRetry, sleep, snakeToCamel, timestampToDate, toCamelCase, toSnakeCase, validateConfig, withRetry };
|
|
4610
|
+
export { API_VERSION, AdminResource, ApiKeysResource, AuthResource, AuthenticationError, AuthorizationError, BillingResource, CircuitBreaker, CircuitBreakerOpenError, CircuitState, ConflictError, DEFAULT_CONFIG, DashboardResource, GrpcJobStatus, GrpcJobStatusNumber, GrpcQueueOperations, GrpcWorkerOperations, HealthResource, HttpClient, JobsResource, MetricsResource, NetworkError, NotFoundError, OrganizationsResource, PROTO_LOADER_OPTIONS, PayloadTooLargeError, QueuesResource, RateLimitError, SDK_VERSION, SchedulesResource, ServerError, SpooledClient, SpooledError, SpooledGrpcClient, SpooledRealtime, SpooledWorker, SseRealtimeClient, TimeoutError, UnsupportedOperationError, ValidationError, WebSocketRealtimeClient, WebhookIngestionResource, WebhooksResource, WorkersResource, WorkflowsResource, asyncIterableFromStream, calculateDelay, camelToSnake, clearProtoCache, convertQueryParams, convertRequest, convertResponse, createCircuitBreaker, createClient, createErrorFromResponse, createHttpClient, createJobStream, createPaginatedResponse, createProcessJobsStream, createRetryWrapper, dateToTimestamp, getProtoPath, isCircuitBreakerFailure, isRetryableError, isSpooledError, loadProtoDefinition, loadProtoDefinitionAsync, parseRateLimitHeaders, resolveConfig, shouldRetry, sleep, snakeToCamel, timestampToDate, toCamelCase, toSnakeCase, validateConfig, withRetry };
|
|
4159
4611
|
//# sourceMappingURL=index.js.map
|
|
4160
4612
|
//# sourceMappingURL=index.js.map
|