@camunda8/sdk 8.9.0-alpha.16 → 8.9.0-alpha.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.eslintrc.json +79 -0
- package/AGENT.md +9 -0
- package/AGENTS.md +22 -0
- package/CHANGELOG.md +14 -0
- package/MAINTAINER.md +2 -1
- package/README.md +10 -1
- package/dist/zeebe/index.d.ts +1 -0
- package/dist/zeebe/index.js.map +1 -1
- package/dist/zeebe/lib/GrpcClient.d.ts +6 -0
- package/dist/zeebe/lib/GrpcClient.js +93 -85
- package/dist/zeebe/lib/GrpcClient.js.map +1 -1
- package/dist/zeebe/lib/StreamLifecycle.d.ts +189 -0
- package/dist/zeebe/lib/StreamLifecycle.js +142 -0
- package/dist/zeebe/lib/StreamLifecycle.js.map +1 -0
- package/dist/zeebe/lib/ZBStreamWorker.d.ts +17 -3
- package/dist/zeebe/lib/ZBStreamWorker.js +364 -137
- package/dist/zeebe/lib/ZBStreamWorker.js.map +1 -1
- package/dist/zeebe/zb/ZeebeGrpcClient.d.ts +6 -3
- package/dist/zeebe/zb/ZeebeGrpcClient.js +4 -0
- package/dist/zeebe/zb/ZeebeGrpcClient.js.map +1 -1
- package/package.json +6 -1
- package/scripts/assert-no-skipped-tests.d.ts +17 -0
- package/scripts/assert-no-skipped-tests.js +118 -0
- package/scripts/required-integration-specs.json +4 -0
- package/stryker.config.json +20 -0
- package/vitest.mutation.config.js +18 -0
|
@@ -1,38 +1,65 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
-
};
|
|
5
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
exports.ZBStreamWorker = void 0;
|
|
7
|
-
const
|
|
3
|
+
exports.ZBStreamWorker = exports.drainRetryDelayMs = exports.isTransientActivationError = exports.DRAIN_RETRY_MAX_DELAY_MS = exports.DRAIN_RETRY_MIN_DELAY_MS = void 0;
|
|
4
|
+
const GrpcError_1 = require("./GrpcError");
|
|
5
|
+
const StreamLifecycle_1 = require("./StreamLifecycle");
|
|
8
6
|
const interfaces_1_0_1 = require("./interfaces-1.0");
|
|
9
7
|
const _1 = require(".");
|
|
10
8
|
const DEFAULT_RECONNECT_INITIAL_DELAY_MS = 1000;
|
|
11
9
|
const DEFAULT_RECONNECT_MAX_DELAY_MS = 30000;
|
|
12
|
-
|
|
10
|
+
// Backoff for retrying a backlog activation that failed with a temporary
|
|
11
|
+
// error. Matches the Java client's default ExponentialBackoff: min 50ms,
|
|
12
|
+
// max 5s, factor 1.6, jitter 0.1.
|
|
13
|
+
exports.DRAIN_RETRY_MIN_DELAY_MS = 50;
|
|
14
|
+
exports.DRAIN_RETRY_MAX_DELAY_MS = 5000;
|
|
15
|
+
const DRAIN_RETRY_FACTOR = 1.6;
|
|
16
|
+
const DRAIN_RETRY_JITTER = 0.1;
|
|
17
|
+
const TRANSIENT_GRPC_CODES = new Set([
|
|
18
|
+
GrpcError_1.GrpcError.RESOURCE_EXHAUSTED,
|
|
19
|
+
GrpcError_1.GrpcError.UNAVAILABLE,
|
|
20
|
+
GrpcError_1.GrpcError.DEADLINE_EXCEEDED,
|
|
21
|
+
]);
|
|
22
|
+
// Only the leading gRPC status prefix ("14 UNAVAILABLE: ...") is trusted, so
|
|
23
|
+
// a status name appearing later in free text cannot mark an error transient.
|
|
24
|
+
const TRANSIENT_MESSAGE = /^\s*(?:\d+\s+)?(?:RESOURCE_EXHAUSTED|UNAVAILABLE|DEADLINE_EXCEEDED)\b/;
|
|
13
25
|
/**
|
|
14
|
-
*
|
|
15
|
-
*
|
|
26
|
+
* Whether an activation error is temporary (broker backpressure, gateway
|
|
27
|
+
* unavailable, deadline exceeded) and the activation should be retried.
|
|
16
28
|
*/
|
|
17
|
-
const
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
if (typeof
|
|
21
|
-
return
|
|
29
|
+
const isTransientActivationError = (e) => {
|
|
30
|
+
const code = e?.code;
|
|
31
|
+
// A numeric gRPC status is authoritative; the message is only a fallback.
|
|
32
|
+
if (typeof code === 'number') {
|
|
33
|
+
return TRANSIENT_GRPC_CODES.has(code);
|
|
22
34
|
}
|
|
23
|
-
return
|
|
35
|
+
return TRANSIENT_MESSAGE.test((0, StreamLifecycle_1.toError)(e).message);
|
|
36
|
+
};
|
|
37
|
+
exports.isTransientActivationError = isTransientActivationError;
|
|
38
|
+
/**
|
|
39
|
+
* Delay before the `attempt`-th retry (1-based) of a failed backlog
|
|
40
|
+
* activation: exponential from 50ms by a factor of 1.6, capped at 5s, with
|
|
41
|
+
* +/-10% jitter.
|
|
42
|
+
*/
|
|
43
|
+
const drainRetryDelayMs = (attempt, random = Math.random) => {
|
|
44
|
+
const base = Math.min(exports.DRAIN_RETRY_MIN_DELAY_MS * DRAIN_RETRY_FACTOR ** (attempt - 1), exports.DRAIN_RETRY_MAX_DELAY_MS);
|
|
45
|
+
const jitter = base * DRAIN_RETRY_JITTER * (2 * random() - 1);
|
|
46
|
+
return Math.max(0, Math.round(base + jitter));
|
|
24
47
|
};
|
|
48
|
+
exports.drainRetryDelayMs = drainRetryDelayMs;
|
|
25
49
|
class ZBStreamWorker {
|
|
26
50
|
constructor({ grpcClient, log, zbClient, }) {
|
|
27
51
|
this.streams = [];
|
|
52
|
+
this.machines = new Set();
|
|
28
53
|
this.closed = false;
|
|
29
54
|
this.pollTimers = [];
|
|
30
55
|
this.reconnectTimers = new Set();
|
|
56
|
+
// Pending activation-retry waits; close() wakes them so they exit.
|
|
57
|
+
this.retryWaits = new Set();
|
|
31
58
|
this.grpcClient = grpcClient;
|
|
32
59
|
this.logger = log;
|
|
33
60
|
this.zbClient = zbClient;
|
|
34
61
|
if (!this.zbClient) {
|
|
35
|
-
|
|
62
|
+
this.logger.logError(`Missing ZBClient`);
|
|
36
63
|
}
|
|
37
64
|
}
|
|
38
65
|
streamJobs(req) {
|
|
@@ -41,27 +68,36 @@ class ZBStreamWorker {
|
|
|
41
68
|
const reconnectOptions = typeof streamReconnect === 'object' ? streamReconnect : {};
|
|
42
69
|
const initialDelayMs = reconnectOptions.initialDelayMs ?? DEFAULT_RECONNECT_INITIAL_DELAY_MS;
|
|
43
70
|
const maxDelayMs = Math.max(reconnectOptions.maxDelayMs ?? DEFAULT_RECONNECT_MAX_DELAY_MS, initialDelayMs);
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
71
|
+
// Returns the handler's promise, which never rejects: a failure is
|
|
72
|
+
// logged here. The backlog drain awaits it for backpressure.
|
|
73
|
+
const handleJob = (job) =>
|
|
74
|
+
// Invoke the handler inside a promise continuation: a synchronous
|
|
75
|
+
// throw must reject this promise rather than escape handleJob,
|
|
76
|
+
// which would abort jobs.map(handleJob) mid-batch — leaving later
|
|
77
|
+
// jobs undispatched and releasing the drain's reservation before
|
|
78
|
+
// the handlers already started have settled.
|
|
79
|
+
//
|
|
80
|
+
// Observe the handler's returned promise so that a rejected job
|
|
81
|
+
// action (for example, a completion command that fails because the
|
|
82
|
+
// job was already completed, failed, or the process instance was
|
|
83
|
+
// cancelled) is handled here rather than surfacing as an unhandled
|
|
84
|
+
// promise rejection that could terminate the process.
|
|
85
|
+
Promise.resolve()
|
|
86
|
+
.then(() => taskHandler({
|
|
87
|
+
...job,
|
|
88
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
89
|
+
...this.makeCompleteHandlers(job, req.type),
|
|
90
|
+
}, this))
|
|
91
|
+
.catch((e) => {
|
|
92
|
+
const message = e instanceof Error ? e.message : String(e);
|
|
93
|
+
this.logger.logError(`Unhandled exception in stream worker task handler for job ${job.key}: ${message}`);
|
|
94
|
+
});
|
|
59
95
|
// Invoke a caller-supplied lifecycle callback. A throwing callback must
|
|
60
96
|
// not break stream recovery (or crash the process from an event handler).
|
|
61
97
|
// Callbacks may be async, so a returned promise is observed as well: a
|
|
62
98
|
// rejection is logged rather than surfacing as an unhandled rejection.
|
|
63
99
|
const notify = (name, callback) => {
|
|
64
|
-
const logFailure = (e) => this.logger.logError(`Exception in ${name} callback of stream worker: ${toError(e).message}`);
|
|
100
|
+
const logFailure = (e) => this.logger.logError(`Exception in ${name} callback of stream worker: ${(0, StreamLifecycle_1.toError)(e).message}`);
|
|
65
101
|
try {
|
|
66
102
|
Promise.resolve(callback()).catch(logFailure);
|
|
67
103
|
}
|
|
@@ -73,44 +109,211 @@ class ZBStreamWorker {
|
|
|
73
109
|
// stream lifecycle handlers below.
|
|
74
110
|
let handleClosed = false; // set by the returned handle's close()
|
|
75
111
|
let currentStream;
|
|
112
|
+
const machine = (0, StreamLifecycle_1.createStreamStateMachine)();
|
|
113
|
+
this.machines.add(machine);
|
|
114
|
+
void machine.done.then(() => this.machines.delete(machine));
|
|
115
|
+
const transition = (next) => {
|
|
116
|
+
if (machine.transition(next)) {
|
|
117
|
+
this.logger.logDebug(`Job stream for ${req.type}: ${(0, StreamLifecycle_1.describeStreamState)(next)}`);
|
|
118
|
+
}
|
|
119
|
+
};
|
|
76
120
|
let reconnectTimer;
|
|
77
121
|
let sidecarTimer;
|
|
78
122
|
let consecutiveFailures = 0; // drives the exponential backoff
|
|
79
123
|
const isClosed = () => this.closed || handleClosed;
|
|
80
|
-
//
|
|
81
|
-
const
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
124
|
+
// Activate up to pollMaxJobsToActivate jobs.
|
|
125
|
+
const activate = () => this.zbClient.activateJobs({
|
|
126
|
+
type: req.type,
|
|
127
|
+
worker: req.worker,
|
|
128
|
+
timeout: req.timeout,
|
|
129
|
+
maxJobsToActivate: pollMaxJobsToActivate,
|
|
130
|
+
tenantIds: req.tenantIds,
|
|
131
|
+
fetchVariable: req.fetchVariable,
|
|
132
|
+
inputVariableDto,
|
|
133
|
+
customHeadersDto,
|
|
134
|
+
requestTimeout: -1,
|
|
135
|
+
});
|
|
136
|
+
// Wait before retrying an activation. Resolves early when the handle
|
|
137
|
+
// or the client is closed, so a pending retry never outlives close().
|
|
138
|
+
const handleRetryWaits = new Set();
|
|
139
|
+
const retryWait = (ms) => new Promise((resolve) => {
|
|
140
|
+
const wake = () => {
|
|
141
|
+
clearTimeout(timer);
|
|
142
|
+
handleRetryWaits.delete(wake);
|
|
143
|
+
this.retryWaits.delete(wake);
|
|
144
|
+
resolve();
|
|
145
|
+
};
|
|
146
|
+
const timer = setTimeout(wake, ms);
|
|
147
|
+
handleRetryWaits.add(wake);
|
|
148
|
+
this.retryWaits.add(wake);
|
|
149
|
+
});
|
|
150
|
+
// Activate, retrying temporary failures (RESOURCE_EXHAUSTED,
|
|
151
|
+
// UNAVAILABLE, DEADLINE_EXCEEDED) with exponential backoff while the
|
|
152
|
+
// handle is open, like the Java client's poller. Abandoning the drain
|
|
153
|
+
// on broker backpressure would leave the backlog untouched until the
|
|
154
|
+
// next sidecar poll — up to a whole pollInterval (30s by default).
|
|
155
|
+
// Returns an empty batch if the handle closes while retrying.
|
|
156
|
+
// Non-transient errors are rethrown.
|
|
157
|
+
const activateWithRetry = async () => {
|
|
158
|
+
for (let attempt = 1;; attempt++) {
|
|
159
|
+
try {
|
|
160
|
+
return await activate();
|
|
161
|
+
}
|
|
162
|
+
catch (e) {
|
|
163
|
+
if (isClosed() || !(0, exports.isTransientActivationError)(e)) {
|
|
164
|
+
throw e;
|
|
165
|
+
}
|
|
166
|
+
const delay = (0, exports.drainRetryDelayMs)(attempt);
|
|
167
|
+
this.logger.logDebug(`Stream worker backlog poll for ${req.type} failed with a temporary error, retrying in ${delay}ms: ${(0, StreamLifecycle_1.toError)(e).message}`);
|
|
168
|
+
await retryWait(delay);
|
|
169
|
+
if (isClosed()) {
|
|
170
|
+
return [];
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
};
|
|
175
|
+
// Hand a polled batch to the handler, then keep polling while polls
|
|
176
|
+
// come back full. The stream only pushes jobs that become activatable
|
|
177
|
+
// after it opens; a backlog that already exists is reachable only by
|
|
178
|
+
// polling. A full batch means more jobs are probably waiting, so poll
|
|
179
|
+
// again as soon as this batch is handled, rather than waiting a whole
|
|
180
|
+
// pollInterval per batch. Waiting for the batch's handlers to settle
|
|
181
|
+
// is the backpressure: the worker never holds more than one batch of
|
|
182
|
+
// polled jobs, so they don't sit activated until their timeout.
|
|
183
|
+
//
|
|
184
|
+
// Iterative, not recursive: a recursion would retain one pending
|
|
185
|
+
// promise per backlog batch until the whole backlog finishes, so a
|
|
186
|
+
// large or continuously replenished backlog would grow memory without
|
|
187
|
+
// bound. The loop holds only the current batch.
|
|
188
|
+
//
|
|
189
|
+
// A reconnect backfill requested while this drain was in flight is
|
|
190
|
+
// honoured before the drain exits: jobs created while the stream was
|
|
191
|
+
// down must not be stranded when recurring polling is disabled.
|
|
192
|
+
const drain = async (firstBatch) => {
|
|
193
|
+
let jobs = firstBatch;
|
|
194
|
+
for (;;) {
|
|
195
|
+
await Promise.all(jobs.map(handleJob));
|
|
196
|
+
if (backfillRequested && !isClosed()) {
|
|
197
|
+
// The stream reconnected while this drain was in flight and
|
|
198
|
+
// its backfill was deferred (see backfill). Run it now, even
|
|
199
|
+
// if this batch was partial: jobs created while the stream
|
|
200
|
+
// was down are not visible to any other poll when recurring
|
|
201
|
+
// polling is disabled. If the activation fails, restore the
|
|
202
|
+
// deferral before re-throwing so runDrain's finally can
|
|
203
|
+
// re-run it once the reservation is released — the backfill
|
|
204
|
+
// must not be discarded while the handle is open.
|
|
205
|
+
backfillRequested = false;
|
|
206
|
+
try {
|
|
207
|
+
jobs = await activateWithRetry();
|
|
208
|
+
}
|
|
209
|
+
catch (e) {
|
|
210
|
+
backfillRequested = true;
|
|
211
|
+
throw e;
|
|
212
|
+
}
|
|
213
|
+
continue;
|
|
214
|
+
}
|
|
215
|
+
if (jobs.length < pollMaxJobsToActivate || isClosed()) {
|
|
216
|
+
return;
|
|
217
|
+
}
|
|
218
|
+
jobs = await activateWithRetry();
|
|
219
|
+
}
|
|
220
|
+
};
|
|
221
|
+
// Only one activation is ever in flight. The flag is reserved *before*
|
|
222
|
+
// the activation call and released only when the drain it fed finishes
|
|
223
|
+
// (or the activation fails), so a reconnect backfill can never start a
|
|
224
|
+
// second activation while a sidecar activation is still pending — that
|
|
225
|
+
// would launch two independent drains, exceed the one-polled-batch
|
|
226
|
+
// limit, and let either drain clear the flag while the other runs.
|
|
227
|
+
let activationReserved = false; // mutated by runDrain/backfill/sidecar
|
|
228
|
+
// Set when a reconnect backfill arrives while an activation is in
|
|
229
|
+
// flight; the in-flight drain runs the backfill before exiting, or
|
|
230
|
+
// re-runs it from runDrain's finally if the drain's activation failed
|
|
231
|
+
// while the handle is still open.
|
|
232
|
+
let backfillRequested = false;
|
|
233
|
+
const runDrain = (jobs) => drain(jobs)
|
|
234
|
+
.catch((e) => {
|
|
235
|
+
// Temporary failures were already retried in activateWithRetry;
|
|
236
|
+
// this is a non-transient error. The next sidecar poll retries.
|
|
237
|
+
this.logger.logError(`Stream worker backlog poll failed: ${(0, StreamLifecycle_1.toError)(e).message}`);
|
|
238
|
+
})
|
|
239
|
+
.finally(() => {
|
|
240
|
+
activationReserved = false;
|
|
241
|
+
// A backfill deferred to this drain that never ran — the
|
|
242
|
+
// drain's activation failed, either before the deferral was
|
|
243
|
+
// consumed or in the activation that consumed it (drain
|
|
244
|
+
// restores the flag before re-throwing) — must not be
|
|
245
|
+
// dropped while the handle is open: with recurring polling
|
|
246
|
+
// disabled there is no other poll to pick up jobs created
|
|
247
|
+
// while the stream was down. Run it now that the reservation
|
|
248
|
+
// is released. If the handle is closed the reservation is
|
|
249
|
+
// stale instead: clear it so a later backfill is free to
|
|
250
|
+
// reserve and run its own poll.
|
|
251
|
+
if (backfillRequested) {
|
|
252
|
+
backfillRequested = false;
|
|
253
|
+
if (!isClosed()) {
|
|
254
|
+
// Observe the rejection: backfill rethrows a failed
|
|
255
|
+
// initial activation after releasing the reservation,
|
|
256
|
+
// and this call is fire-and-forget.
|
|
257
|
+
void backfill().catch((e) => this.logger.logError(`Stream worker backfill poll failed: ${(0, StreamLifecycle_1.toError)(e).message}`));
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
});
|
|
261
|
+
// Run a backfill poll. Only the first activation is awaited (so its
|
|
262
|
+
// errors reach the caller); the rest of the backlog drains in the
|
|
263
|
+
// background. If an activation is already in flight, the backfill is
|
|
264
|
+
// not dropped: the in-flight drain runs it before exiting, so jobs
|
|
265
|
+
// created while the stream was down are picked up even when recurring
|
|
266
|
+
// polling is disabled (pollInterval 0 or -1).
|
|
267
|
+
const backfill = async () => {
|
|
268
|
+
if (isClosed()) {
|
|
269
|
+
return;
|
|
270
|
+
}
|
|
271
|
+
if (activationReserved) {
|
|
272
|
+
backfillRequested = true;
|
|
273
|
+
return;
|
|
274
|
+
}
|
|
275
|
+
activationReserved = true;
|
|
276
|
+
try {
|
|
277
|
+
void runDrain(await activate());
|
|
278
|
+
}
|
|
279
|
+
catch (e) {
|
|
280
|
+
activationReserved = false;
|
|
281
|
+
throw e;
|
|
95
282
|
}
|
|
96
283
|
};
|
|
97
|
-
const discardStream = (
|
|
98
|
-
const index = this.streams.indexOf(
|
|
284
|
+
const discardStream = (attachment) => {
|
|
285
|
+
const index = this.streams.indexOf(attachment);
|
|
99
286
|
if (index !== -1) {
|
|
100
287
|
this.streams.splice(index, 1);
|
|
101
288
|
}
|
|
102
289
|
};
|
|
290
|
+
// The stream is down. Re-open it with backoff, or, if reconnection is
|
|
291
|
+
// disabled, record that it is closed for good.
|
|
292
|
+
const onStreamDown = (cause) => {
|
|
293
|
+
if (isClosed()) {
|
|
294
|
+
return;
|
|
295
|
+
}
|
|
296
|
+
if (!reconnectEnabled) {
|
|
297
|
+
transition({
|
|
298
|
+
status: 'closed',
|
|
299
|
+
reason: { kind: 'stream-failed', failure: cause },
|
|
300
|
+
});
|
|
301
|
+
return;
|
|
302
|
+
}
|
|
303
|
+
scheduleReconnect(cause);
|
|
304
|
+
};
|
|
103
305
|
// Schedule a single re-open of the stream with exponential backoff. Only
|
|
104
306
|
// one reconnect timer is ever pending, so an `error` and an `end` for the
|
|
105
307
|
// same dead stream (or a failed open racing a stream failure) cannot
|
|
106
308
|
// result in duplicate re-opens.
|
|
107
|
-
const scheduleReconnect = () => {
|
|
309
|
+
const scheduleReconnect = (cause) => {
|
|
108
310
|
if (!reconnectEnabled || isClosed() || reconnectTimer) {
|
|
109
311
|
return;
|
|
110
312
|
}
|
|
111
313
|
consecutiveFailures++;
|
|
112
314
|
const attempt = consecutiveFailures;
|
|
113
|
-
const delay =
|
|
315
|
+
const delay = (0, StreamLifecycle_1.backoffDelayMs)(attempt, initialDelayMs, maxDelayMs);
|
|
316
|
+
transition({ status: 'reconnecting', attempt, cause });
|
|
114
317
|
const timer = setTimeout(() => {
|
|
115
318
|
reconnectTimer = undefined;
|
|
116
319
|
this.reconnectTimers.delete(timer);
|
|
@@ -118,7 +321,7 @@ class ZBStreamWorker {
|
|
|
118
321
|
return;
|
|
119
322
|
}
|
|
120
323
|
notify('onStreamReconnect', () => onStreamReconnect?.(attempt));
|
|
121
|
-
reconnect().catch((e) => this.logger.logError(`Unexpected error re-opening job stream: ${toError(e).message}`));
|
|
324
|
+
reconnect().catch((e) => this.logger.logError(`Unexpected error re-opening job stream: ${(0, StreamLifecycle_1.toError)(e).message}`));
|
|
122
325
|
}, delay);
|
|
123
326
|
reconnectTimer = timer;
|
|
124
327
|
this.reconnectTimers.add(timer);
|
|
@@ -126,7 +329,7 @@ class ZBStreamWorker {
|
|
|
126
329
|
const handleOpenFailure = (error) => {
|
|
127
330
|
this.logger.logError(`Failed to open job stream for ${req.type}: ${error.message}`);
|
|
128
331
|
notify('onStreamError', () => onStreamError?.(error));
|
|
129
|
-
|
|
332
|
+
onStreamDown({ kind: 'open-failed', error });
|
|
130
333
|
};
|
|
131
334
|
// Open the stream. Returns an Error if it could not be opened (the
|
|
132
335
|
// caller decides what to do), otherwise attaches the lifecycle handlers.
|
|
@@ -139,74 +342,79 @@ class ZBStreamWorker {
|
|
|
139
342
|
return { error: e };
|
|
140
343
|
}
|
|
141
344
|
})();
|
|
142
|
-
const openError = getOpenError(result);
|
|
345
|
+
const openError = (0, StreamLifecycle_1.getOpenError)(result);
|
|
143
346
|
if (openError) {
|
|
144
347
|
return openError;
|
|
145
348
|
}
|
|
146
|
-
const stream = result;
|
|
349
|
+
const stream = (0, StreamLifecycle_1.asStream)(result);
|
|
147
350
|
if (isClosed()) {
|
|
148
351
|
// close() was called while the stream was being opened.
|
|
149
|
-
|
|
150
|
-
stream.destroy();
|
|
352
|
+
(0, StreamLifecycle_1.cancelStream)(stream);
|
|
151
353
|
return undefined;
|
|
152
354
|
}
|
|
153
355
|
const openedAt = Date.now();
|
|
154
356
|
let receivedData = false;
|
|
155
|
-
let
|
|
156
|
-
//
|
|
157
|
-
// the
|
|
158
|
-
const
|
|
159
|
-
if (dead) {
|
|
160
|
-
return;
|
|
161
|
-
}
|
|
162
|
-
dead = true;
|
|
163
|
-
discardStream(stream);
|
|
164
|
-
if (currentStream === stream) {
|
|
165
|
-
currentStream = undefined;
|
|
166
|
-
}
|
|
167
|
-
if (isClosed()) {
|
|
168
|
-
// Expected: we cancelled it ourselves.
|
|
169
|
-
return;
|
|
170
|
-
}
|
|
171
|
-
// A stream that delivered a job, or stayed up for a while, was
|
|
172
|
-
// healthy: restart the backoff from the beginning.
|
|
357
|
+
let cause = { kind: 'end' };
|
|
358
|
+
// A stream that delivered a job, or stayed up for a while, was
|
|
359
|
+
// healthy: restart the backoff from the beginning.
|
|
360
|
+
const resetBackoffIfHealthy = () => {
|
|
173
361
|
if (receivedData || Date.now() - openedAt >= maxDelayMs) {
|
|
174
362
|
consecutiveFailures = 0;
|
|
175
363
|
}
|
|
176
|
-
notifyCaller();
|
|
177
|
-
scheduleReconnect();
|
|
178
364
|
};
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
365
|
+
// The adapter is the only code that attaches listeners to the raw
|
|
366
|
+
// stream. It delivers exactly one of `error`/`end`, then `close`.
|
|
367
|
+
const attachment = (0, StreamLifecycle_1.attachStream)(stream, {
|
|
368
|
+
data: (res) => {
|
|
369
|
+
if (!receivedData) {
|
|
370
|
+
receivedData = true;
|
|
371
|
+
consecutiveFailures = 0;
|
|
372
|
+
}
|
|
373
|
+
try {
|
|
374
|
+
const parsedJob = (0, _1.parseVariablesAndCustomHeadersToJSON)(res, inputVariableDto, customHeadersDto);
|
|
375
|
+
void handleJob(parsedJob);
|
|
376
|
+
}
|
|
377
|
+
catch (e) {
|
|
378
|
+
this.zbClient
|
|
379
|
+
.failJob({
|
|
380
|
+
jobKey: res.key,
|
|
381
|
+
errorMessage: `Error parsing variable payload ${e}`,
|
|
382
|
+
retries: res.retries - 1,
|
|
383
|
+
retryBackOff: 0,
|
|
384
|
+
})
|
|
385
|
+
.catch((failError) => this.logger.logError(`Failed to fail job ${res.key} after a payload parse error: ${(0, StreamLifecycle_1.toError)(failError).message}`));
|
|
386
|
+
}
|
|
387
|
+
},
|
|
388
|
+
error: (error) => {
|
|
389
|
+
cause = { kind: 'error', error };
|
|
390
|
+
if (isClosed()) {
|
|
391
|
+
// Expected: we cancelled it ourselves.
|
|
392
|
+
return;
|
|
393
|
+
}
|
|
394
|
+
resetBackoffIfHealthy();
|
|
395
|
+
this.logger.logError(`Job stream for ${req.type} failed: ${error.message}`);
|
|
396
|
+
notify('onStreamError', () => onStreamError?.(error));
|
|
397
|
+
},
|
|
398
|
+
end: () => {
|
|
399
|
+
cause = { kind: 'end' };
|
|
400
|
+
if (isClosed()) {
|
|
401
|
+
return;
|
|
402
|
+
}
|
|
403
|
+
resetBackoffIfHealthy();
|
|
404
|
+
this.logger.logError(`Job stream for ${req.type} ended`);
|
|
405
|
+
notify('onStreamEnd', () => onStreamEnd?.());
|
|
406
|
+
},
|
|
407
|
+
close: () => {
|
|
408
|
+
discardStream(attachment);
|
|
409
|
+
if (currentStream === attachment) {
|
|
410
|
+
currentStream = undefined;
|
|
411
|
+
}
|
|
412
|
+
onStreamDown(cause);
|
|
413
|
+
},
|
|
207
414
|
});
|
|
208
|
-
this.streams.push(
|
|
209
|
-
currentStream =
|
|
415
|
+
this.streams.push(attachment);
|
|
416
|
+
currentStream = attachment;
|
|
417
|
+
transition({ status: 'open' });
|
|
210
418
|
return undefined;
|
|
211
419
|
};
|
|
212
420
|
// Re-open a dead stream, then backfill: jobs created while the stream
|
|
@@ -222,9 +430,35 @@ class ZBStreamWorker {
|
|
|
222
430
|
if (openError) {
|
|
223
431
|
return handleOpenFailure(openError);
|
|
224
432
|
}
|
|
225
|
-
await
|
|
433
|
+
await backfill().catch((e) => this.logger.logError(`Backfill poll after job stream reconnect failed: ${(0, StreamLifecycle_1.toError)(e).message}`));
|
|
226
434
|
};
|
|
435
|
+
// The handle returned to the caller. `close()` is idempotent.
|
|
436
|
+
const closeHandle = () => {
|
|
437
|
+
handleClosed = true;
|
|
438
|
+
transition({ status: 'closed', reason: { kind: 'closed-by-caller' } });
|
|
439
|
+
if (sidecarTimer) {
|
|
440
|
+
clearTimeout(sidecarTimer);
|
|
441
|
+
}
|
|
442
|
+
handleRetryWaits.forEach((wake) => wake());
|
|
443
|
+
if (reconnectTimer) {
|
|
444
|
+
clearTimeout(reconnectTimer);
|
|
445
|
+
this.reconnectTimers.delete(reconnectTimer);
|
|
446
|
+
reconnectTimer = undefined;
|
|
447
|
+
}
|
|
448
|
+
if (currentStream) {
|
|
449
|
+
discardStream(currentStream);
|
|
450
|
+
currentStream.dispose();
|
|
451
|
+
currentStream = undefined;
|
|
452
|
+
}
|
|
453
|
+
};
|
|
454
|
+
const handle = (0, StreamLifecycle_1.createStreamHandle)(machine, closeHandle);
|
|
227
455
|
const pollAndStream = async () => {
|
|
456
|
+
if (this.closed) {
|
|
457
|
+
// The client was already closed: open nothing, and give the caller a
|
|
458
|
+
// handle that says so, instead of one that reports `open` forever.
|
|
459
|
+
transition({ status: 'closed', reason: { kind: 'client-closed' } });
|
|
460
|
+
return handle;
|
|
461
|
+
}
|
|
228
462
|
// Optional jitter to stagger multiple workers
|
|
229
463
|
if (jitter) {
|
|
230
464
|
await new Promise((resolve) => setTimeout(resolve, Math.random() * jitter));
|
|
@@ -233,7 +467,7 @@ class ZBStreamWorker {
|
|
|
233
467
|
const openError = await openStream();
|
|
234
468
|
if (isClosed()) {
|
|
235
469
|
// close() was called while the stream was being opened.
|
|
236
|
-
return
|
|
470
|
+
return handle;
|
|
237
471
|
}
|
|
238
472
|
if (openError) {
|
|
239
473
|
handleOpenFailure(openError);
|
|
@@ -245,7 +479,7 @@ class ZBStreamWorker {
|
|
|
245
479
|
// Initial backfill poll: pick up jobs that existed before the stream.
|
|
246
480
|
// The broker guarantees single activation, so there is no risk of
|
|
247
481
|
// duplicate delivery between the poll and the stream.
|
|
248
|
-
await
|
|
482
|
+
await backfill();
|
|
249
483
|
}
|
|
250
484
|
// Recurring sidecar poll: a low-frequency safety net that catches
|
|
251
485
|
// jobs the stream may have missed (e.g. jobs re-queued after a
|
|
@@ -271,10 +505,22 @@ class ZBStreamWorker {
|
|
|
271
505
|
if (index !== -1) {
|
|
272
506
|
this.pollTimers.splice(index, 1);
|
|
273
507
|
}
|
|
274
|
-
|
|
275
|
-
|
|
508
|
+
// An activation already in flight is doing this poll's job.
|
|
509
|
+
if (activationReserved || isClosed()) {
|
|
510
|
+
schedulePoll();
|
|
511
|
+
return;
|
|
512
|
+
}
|
|
513
|
+
// Reserve before activating: a reconnect backfill arriving
|
|
514
|
+
// while this activation is pending must not start a second,
|
|
515
|
+
// overlapping drain. The reservation is released by
|
|
516
|
+
// runDrain's finally, or here if the activation fails.
|
|
517
|
+
activationReserved = true;
|
|
518
|
+
activateWithRetry()
|
|
519
|
+
.then(runDrain, (e) => {
|
|
520
|
+
activationReserved = false;
|
|
276
521
|
// Swallow errors — the stream is the primary channel.
|
|
277
522
|
// The next poll cycle will retry.
|
|
523
|
+
this.logger.logDebug(`Stream worker sidecar poll failed: ${(0, StreamLifecycle_1.toError)(e).message}`);
|
|
278
524
|
})
|
|
279
525
|
.finally(schedulePoll);
|
|
280
526
|
}, pollInterval);
|
|
@@ -282,25 +528,7 @@ class ZBStreamWorker {
|
|
|
282
528
|
};
|
|
283
529
|
schedulePoll();
|
|
284
530
|
}
|
|
285
|
-
return
|
|
286
|
-
close: () => {
|
|
287
|
-
handleClosed = true;
|
|
288
|
-
if (sidecarTimer) {
|
|
289
|
-
clearTimeout(sidecarTimer);
|
|
290
|
-
}
|
|
291
|
-
if (reconnectTimer) {
|
|
292
|
-
clearTimeout(reconnectTimer);
|
|
293
|
-
this.reconnectTimers.delete(reconnectTimer);
|
|
294
|
-
reconnectTimer = undefined;
|
|
295
|
-
}
|
|
296
|
-
if (currentStream) {
|
|
297
|
-
discardStream(currentStream);
|
|
298
|
-
currentStream.cancel();
|
|
299
|
-
currentStream.destroy();
|
|
300
|
-
currentStream = undefined;
|
|
301
|
-
}
|
|
302
|
-
},
|
|
303
|
-
};
|
|
531
|
+
return handle;
|
|
304
532
|
};
|
|
305
533
|
return pollAndStream();
|
|
306
534
|
}
|
|
@@ -310,10 +538,10 @@ class ZBStreamWorker {
|
|
|
310
538
|
this.pollTimers = [];
|
|
311
539
|
this.reconnectTimers.forEach((t) => clearTimeout(t));
|
|
312
540
|
this.reconnectTimers.clear();
|
|
313
|
-
this.
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
541
|
+
this.retryWaits.forEach((wake) => wake());
|
|
542
|
+
const clientClosed = { kind: 'client-closed' };
|
|
543
|
+
this.machines.forEach((m) => m.transition({ status: 'closed', reason: clientClosed }));
|
|
544
|
+
this.streams.forEach((s) => s.dispose());
|
|
317
545
|
this.streams = [];
|
|
318
546
|
return this.grpcClient.close();
|
|
319
547
|
}
|
|
@@ -335,9 +563,8 @@ class ZBStreamWorker {
|
|
|
335
563
|
const errorMsgOnPriorMessageCall = (thisMethod, wrappedFunction) => {
|
|
336
564
|
return (...args) => {
|
|
337
565
|
if (methodCalled !== undefined) {
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
You should call only one job action method in the worker handler. This is a bug in a Stream worker handler.`));
|
|
566
|
+
this.logger.logError(`WARNING: Call to ${thisMethod}() after ${methodCalled}() was called.
|
|
567
|
+
You should call only one job action method in the worker handler. This is a bug in a Stream worker handler.`);
|
|
341
568
|
return wrappedFunction(...args);
|
|
342
569
|
}
|
|
343
570
|
methodCalled = thisMethod;
|