@camunda8/sdk 8.9.0-alpha.16 → 8.9.0-alpha.18

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,38 +1,65 @@
1
1
  "use strict";
2
- var __importDefault = (this && this.__importDefault) || function (mod) {
3
- return (mod && mod.__esModule) ? mod : { "default": mod };
4
- };
5
2
  Object.defineProperty(exports, "__esModule", { value: true });
6
- exports.ZBStreamWorker = void 0;
7
- const chalk_1 = __importDefault(require("chalk"));
3
+ exports.ZBStreamWorker = exports.drainRetryDelayMs = exports.isTransientActivationError = exports.DRAIN_RETRY_MAX_DELAY_MS = exports.DRAIN_RETRY_MIN_DELAY_MS = void 0;
4
+ const GrpcError_1 = require("./GrpcError");
5
+ const StreamLifecycle_1 = require("./StreamLifecycle");
8
6
  const interfaces_1_0_1 = require("./interfaces-1.0");
9
7
  const _1 = require(".");
10
8
  const DEFAULT_RECONNECT_INITIAL_DELAY_MS = 1000;
11
9
  const DEFAULT_RECONNECT_MAX_DELAY_MS = 30000;
12
- const toError = (e) => e instanceof Error ? e : new Error(String(e));
10
+ // Backoff for retrying a backlog activation that failed with a temporary
11
+ // error. Matches the Java client's default ExponentialBackoff: min 50ms,
12
+ // max 5s, factor 1.6, jitter 0.1.
13
+ exports.DRAIN_RETRY_MIN_DELAY_MS = 50;
14
+ exports.DRAIN_RETRY_MAX_DELAY_MS = 5000;
15
+ const DRAIN_RETRY_FACTOR = 1.6;
16
+ const DRAIN_RETRY_JITTER = 0.1;
17
+ const TRANSIENT_GRPC_CODES = new Set([
18
+ GrpcError_1.GrpcError.RESOURCE_EXHAUSTED,
19
+ GrpcError_1.GrpcError.UNAVAILABLE,
20
+ GrpcError_1.GrpcError.DEADLINE_EXCEEDED,
21
+ ]);
22
+ // Only the leading gRPC status prefix ("14 UNAVAILABLE: ...") is trusted, so
23
+ // a status name appearing later in free text cannot mark an error transient.
24
+ const TRANSIENT_MESSAGE = /^\s*(?:\d+\s+)?(?:RESOURCE_EXHAUSTED|UNAVAILABLE|DEADLINE_EXCEEDED)\b/;
13
25
  /**
14
- * `streamActivatedJobsStream` resolves to `{ error }` (or `undefined` when the
15
- * client is closing) instead of a stream when the stream cannot be opened.
26
+ * Whether an activation error is temporary (broker backpressure, gateway
27
+ * unavailable, deadline exceeded) and the activation should be retried.
16
28
  */
17
- const getOpenError = (result) => {
18
- if (!result)
19
- return new Error('Job stream could not be opened');
20
- if (typeof result === 'object' && 'error' in result) {
21
- return toError(result.error);
29
+ const isTransientActivationError = (e) => {
30
+ const code = e?.code;
31
+ // A numeric gRPC status is authoritative; the message is only a fallback.
32
+ if (typeof code === 'number') {
33
+ return TRANSIENT_GRPC_CODES.has(code);
22
34
  }
23
- return undefined;
35
+ return TRANSIENT_MESSAGE.test((0, StreamLifecycle_1.toError)(e).message);
36
+ };
37
+ exports.isTransientActivationError = isTransientActivationError;
38
+ /**
39
+ * Delay before the `attempt`-th retry (1-based) of a failed backlog
40
+ * activation: exponential from 50ms by a factor of 1.6, capped at 5s, with
41
+ * +/-10% jitter.
42
+ */
43
+ const drainRetryDelayMs = (attempt, random = Math.random) => {
44
+ const base = Math.min(exports.DRAIN_RETRY_MIN_DELAY_MS * DRAIN_RETRY_FACTOR ** (attempt - 1), exports.DRAIN_RETRY_MAX_DELAY_MS);
45
+ const jitter = base * DRAIN_RETRY_JITTER * (2 * random() - 1);
46
+ return Math.max(0, Math.round(base + jitter));
24
47
  };
48
+ exports.drainRetryDelayMs = drainRetryDelayMs;
25
49
  class ZBStreamWorker {
26
50
  constructor({ grpcClient, log, zbClient, }) {
27
51
  this.streams = [];
52
+ this.machines = new Set();
28
53
  this.closed = false;
29
54
  this.pollTimers = [];
30
55
  this.reconnectTimers = new Set();
56
+ // Pending activation-retry waits; close() wakes them so they exit.
57
+ this.retryWaits = new Set();
31
58
  this.grpcClient = grpcClient;
32
59
  this.logger = log;
33
60
  this.zbClient = zbClient;
34
61
  if (!this.zbClient) {
35
- console.log(`Missing ZBClient`);
62
+ this.logger.logError(`Missing ZBClient`);
36
63
  }
37
64
  }
38
65
  streamJobs(req) {
@@ -41,27 +68,36 @@ class ZBStreamWorker {
41
68
  const reconnectOptions = typeof streamReconnect === 'object' ? streamReconnect : {};
42
69
  const initialDelayMs = reconnectOptions.initialDelayMs ?? DEFAULT_RECONNECT_INITIAL_DELAY_MS;
43
70
  const maxDelayMs = Math.max(reconnectOptions.maxDelayMs ?? DEFAULT_RECONNECT_MAX_DELAY_MS, initialDelayMs);
44
- const handleJob = (job) => {
45
- // Observe the handler's returned promise so that a rejected job
46
- // action (for example, a completion command that fails because the
47
- // job was already completed, failed, or the process instance was
48
- // cancelled) is handled here rather than surfacing as an unhandled
49
- // promise rejection that could terminate the process.
50
- Promise.resolve(taskHandler({
51
- ...job,
52
- // eslint-disable-next-line @typescript-eslint/no-explicit-any
53
- ...this.makeCompleteHandlers(job, req.type),
54
- }, this)).catch((e) => {
55
- const message = e instanceof Error ? e.message : String(e);
56
- this.logger.logError(`Unhandled exception in stream worker task handler for job ${job.key}: ${message}`);
57
- });
58
- };
71
+ // Returns the handler's promise, which never rejects: a failure is
72
+ // logged here. The backlog drain awaits it for backpressure.
73
+ const handleJob = (job) =>
74
+ // Invoke the handler inside a promise continuation: a synchronous
75
+ // throw must reject this promise rather than escape handleJob,
76
+ // which would abort jobs.map(handleJob) mid-batch — leaving later
77
+ // jobs undispatched and releasing the drain's reservation before
78
+ // the handlers already started have settled.
79
+ //
80
+ // Observe the handler's returned promise so that a rejected job
81
+ // action (for example, a completion command that fails because the
82
+ // job was already completed, failed, or the process instance was
83
+ // cancelled) is handled here rather than surfacing as an unhandled
84
+ // promise rejection that could terminate the process.
85
+ Promise.resolve()
86
+ .then(() => taskHandler({
87
+ ...job,
88
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
89
+ ...this.makeCompleteHandlers(job, req.type),
90
+ }, this))
91
+ .catch((e) => {
92
+ const message = e instanceof Error ? e.message : String(e);
93
+ this.logger.logError(`Unhandled exception in stream worker task handler for job ${job.key}: ${message}`);
94
+ });
59
95
  // Invoke a caller-supplied lifecycle callback. A throwing callback must
60
96
  // not break stream recovery (or crash the process from an event handler).
61
97
  // Callbacks may be async, so a returned promise is observed as well: a
62
98
  // rejection is logged rather than surfacing as an unhandled rejection.
63
99
  const notify = (name, callback) => {
64
- const logFailure = (e) => this.logger.logError(`Exception in ${name} callback of stream worker: ${toError(e).message}`);
100
+ const logFailure = (e) => this.logger.logError(`Exception in ${name} callback of stream worker: ${(0, StreamLifecycle_1.toError)(e).message}`);
65
101
  try {
66
102
  Promise.resolve(callback()).catch(logFailure);
67
103
  }
@@ -73,44 +109,211 @@ class ZBStreamWorker {
73
109
  // stream lifecycle handlers below.
74
110
  let handleClosed = false; // set by the returned handle's close()
75
111
  let currentStream;
112
+ const machine = (0, StreamLifecycle_1.createStreamStateMachine)();
113
+ this.machines.add(machine);
114
+ void machine.done.then(() => this.machines.delete(machine));
115
+ const transition = (next) => {
116
+ if (machine.transition(next)) {
117
+ this.logger.logDebug(`Job stream for ${req.type}: ${(0, StreamLifecycle_1.describeStreamState)(next)}`);
118
+ }
119
+ };
76
120
  let reconnectTimer;
77
121
  let sidecarTimer;
78
122
  let consecutiveFailures = 0; // drives the exponential backoff
79
123
  const isClosed = () => this.closed || handleClosed;
80
- // Helper to run a single poll cycle and feed results to handleJob.
81
- const runPoll = async () => {
82
- const jobs = await this.zbClient.activateJobs({
83
- type: req.type,
84
- worker: req.worker,
85
- timeout: req.timeout,
86
- maxJobsToActivate: pollMaxJobsToActivate,
87
- tenantIds: req.tenantIds,
88
- fetchVariable: req.fetchVariable,
89
- inputVariableDto,
90
- customHeadersDto,
91
- requestTimeout: -1,
92
- });
93
- for (const job of jobs) {
94
- handleJob(job);
124
+ // Activate up to pollMaxJobsToActivate jobs.
125
+ const activate = () => this.zbClient.activateJobs({
126
+ type: req.type,
127
+ worker: req.worker,
128
+ timeout: req.timeout,
129
+ maxJobsToActivate: pollMaxJobsToActivate,
130
+ tenantIds: req.tenantIds,
131
+ fetchVariable: req.fetchVariable,
132
+ inputVariableDto,
133
+ customHeadersDto,
134
+ requestTimeout: -1,
135
+ });
136
+ // Wait before retrying an activation. Resolves early when the handle
137
+ // or the client is closed, so a pending retry never outlives close().
138
+ const handleRetryWaits = new Set();
139
+ const retryWait = (ms) => new Promise((resolve) => {
140
+ const wake = () => {
141
+ clearTimeout(timer);
142
+ handleRetryWaits.delete(wake);
143
+ this.retryWaits.delete(wake);
144
+ resolve();
145
+ };
146
+ const timer = setTimeout(wake, ms);
147
+ handleRetryWaits.add(wake);
148
+ this.retryWaits.add(wake);
149
+ });
150
+ // Activate, retrying temporary failures (RESOURCE_EXHAUSTED,
151
+ // UNAVAILABLE, DEADLINE_EXCEEDED) with exponential backoff while the
152
+ // handle is open, like the Java client's poller. Abandoning the drain
153
+ // on broker backpressure would leave the backlog untouched until the
154
+ // next sidecar poll — up to a whole pollInterval (30s by default).
155
+ // Returns an empty batch if the handle closes while retrying.
156
+ // Non-transient errors are rethrown.
157
+ const activateWithRetry = async () => {
158
+ for (let attempt = 1;; attempt++) {
159
+ try {
160
+ return await activate();
161
+ }
162
+ catch (e) {
163
+ if (isClosed() || !(0, exports.isTransientActivationError)(e)) {
164
+ throw e;
165
+ }
166
+ const delay = (0, exports.drainRetryDelayMs)(attempt);
167
+ this.logger.logDebug(`Stream worker backlog poll for ${req.type} failed with a temporary error, retrying in ${delay}ms: ${(0, StreamLifecycle_1.toError)(e).message}`);
168
+ await retryWait(delay);
169
+ if (isClosed()) {
170
+ return [];
171
+ }
172
+ }
173
+ }
174
+ };
175
+ // Hand a polled batch to the handler, then keep polling while polls
176
+ // come back full. The stream only pushes jobs that become activatable
177
+ // after it opens; a backlog that already exists is reachable only by
178
+ // polling. A full batch means more jobs are probably waiting, so poll
179
+ // again as soon as this batch is handled, rather than waiting a whole
180
+ // pollInterval per batch. Waiting for the batch's handlers to settle
181
+ // is the backpressure: the worker never holds more than one batch of
182
+ // polled jobs, so they don't sit activated until their timeout.
183
+ //
184
+ // Iterative, not recursive: a recursion would retain one pending
185
+ // promise per backlog batch until the whole backlog finishes, so a
186
+ // large or continuously replenished backlog would grow memory without
187
+ // bound. The loop holds only the current batch.
188
+ //
189
+ // A reconnect backfill requested while this drain was in flight is
190
+ // honoured before the drain exits: jobs created while the stream was
191
+ // down must not be stranded when recurring polling is disabled.
192
+ const drain = async (firstBatch) => {
193
+ let jobs = firstBatch;
194
+ for (;;) {
195
+ await Promise.all(jobs.map(handleJob));
196
+ if (backfillRequested && !isClosed()) {
197
+ // The stream reconnected while this drain was in flight and
198
+ // its backfill was deferred (see backfill). Run it now, even
199
+ // if this batch was partial: jobs created while the stream
200
+ // was down are not visible to any other poll when recurring
201
+ // polling is disabled. If the activation fails, restore the
202
+ // deferral before re-throwing so runDrain's finally can
203
+ // re-run it once the reservation is released — the backfill
204
+ // must not be discarded while the handle is open.
205
+ backfillRequested = false;
206
+ try {
207
+ jobs = await activateWithRetry();
208
+ }
209
+ catch (e) {
210
+ backfillRequested = true;
211
+ throw e;
212
+ }
213
+ continue;
214
+ }
215
+ if (jobs.length < pollMaxJobsToActivate || isClosed()) {
216
+ return;
217
+ }
218
+ jobs = await activateWithRetry();
219
+ }
220
+ };
221
+ // Only one activation is ever in flight. The flag is reserved *before*
222
+ // the activation call and released only when the drain it fed finishes
223
+ // (or the activation fails), so a reconnect backfill can never start a
224
+ // second activation while a sidecar activation is still pending — that
225
+ // would launch two independent drains, exceed the one-polled-batch
226
+ // limit, and let either drain clear the flag while the other runs.
227
+ let activationReserved = false; // mutated by runDrain/backfill/sidecar
228
+ // Set when a reconnect backfill arrives while an activation is in
229
+ // flight; the in-flight drain runs the backfill before exiting, or
230
+ // re-runs it from runDrain's finally if the drain's activation failed
231
+ // while the handle is still open.
232
+ let backfillRequested = false;
233
+ const runDrain = (jobs) => drain(jobs)
234
+ .catch((e) => {
235
+ // Temporary failures were already retried in activateWithRetry;
236
+ // this is a non-transient error. The next sidecar poll retries.
237
+ this.logger.logError(`Stream worker backlog poll failed: ${(0, StreamLifecycle_1.toError)(e).message}`);
238
+ })
239
+ .finally(() => {
240
+ activationReserved = false;
241
+ // A backfill deferred to this drain that never ran — the
242
+ // drain's activation failed, either before the deferral was
243
+ // consumed or in the activation that consumed it (drain
244
+ // restores the flag before re-throwing) — must not be
245
+ // dropped while the handle is open: with recurring polling
246
+ // disabled there is no other poll to pick up jobs created
247
+ // while the stream was down. Run it now that the reservation
248
+ // is released. If the handle is closed the reservation is
249
+ // stale instead: clear it so a later backfill is free to
250
+ // reserve and run its own poll.
251
+ if (backfillRequested) {
252
+ backfillRequested = false;
253
+ if (!isClosed()) {
254
+ // Observe the rejection: backfill rethrows a failed
255
+ // initial activation after releasing the reservation,
256
+ // and this call is fire-and-forget.
257
+ void backfill().catch((e) => this.logger.logError(`Stream worker backfill poll failed: ${(0, StreamLifecycle_1.toError)(e).message}`));
258
+ }
259
+ }
260
+ });
261
+ // Run a backfill poll. Only the first activation is awaited (so its
262
+ // errors reach the caller); the rest of the backlog drains in the
263
+ // background. If an activation is already in flight, the backfill is
264
+ // not dropped: the in-flight drain runs it before exiting, so jobs
265
+ // created while the stream was down are picked up even when recurring
266
+ // polling is disabled (pollInterval 0 or -1).
267
+ const backfill = async () => {
268
+ if (isClosed()) {
269
+ return;
270
+ }
271
+ if (activationReserved) {
272
+ backfillRequested = true;
273
+ return;
274
+ }
275
+ activationReserved = true;
276
+ try {
277
+ void runDrain(await activate());
278
+ }
279
+ catch (e) {
280
+ activationReserved = false;
281
+ throw e;
95
282
  }
96
283
  };
97
- const discardStream = (stream) => {
98
- const index = this.streams.indexOf(stream);
284
+ const discardStream = (attachment) => {
285
+ const index = this.streams.indexOf(attachment);
99
286
  if (index !== -1) {
100
287
  this.streams.splice(index, 1);
101
288
  }
102
289
  };
290
+ // The stream is down. Re-open it with backoff, or, if reconnection is
291
+ // disabled, record that it is closed for good.
292
+ const onStreamDown = (cause) => {
293
+ if (isClosed()) {
294
+ return;
295
+ }
296
+ if (!reconnectEnabled) {
297
+ transition({
298
+ status: 'closed',
299
+ reason: { kind: 'stream-failed', failure: cause },
300
+ });
301
+ return;
302
+ }
303
+ scheduleReconnect(cause);
304
+ };
103
305
  // Schedule a single re-open of the stream with exponential backoff. Only
104
306
  // one reconnect timer is ever pending, so an `error` and an `end` for the
105
307
  // same dead stream (or a failed open racing a stream failure) cannot
106
308
  // result in duplicate re-opens.
107
- const scheduleReconnect = () => {
309
+ const scheduleReconnect = (cause) => {
108
310
  if (!reconnectEnabled || isClosed() || reconnectTimer) {
109
311
  return;
110
312
  }
111
313
  consecutiveFailures++;
112
314
  const attempt = consecutiveFailures;
113
- const delay = Math.min(initialDelayMs * 2 ** (attempt - 1), maxDelayMs);
315
+ const delay = (0, StreamLifecycle_1.backoffDelayMs)(attempt, initialDelayMs, maxDelayMs);
316
+ transition({ status: 'reconnecting', attempt, cause });
114
317
  const timer = setTimeout(() => {
115
318
  reconnectTimer = undefined;
116
319
  this.reconnectTimers.delete(timer);
@@ -118,7 +321,7 @@ class ZBStreamWorker {
118
321
  return;
119
322
  }
120
323
  notify('onStreamReconnect', () => onStreamReconnect?.(attempt));
121
- reconnect().catch((e) => this.logger.logError(`Unexpected error re-opening job stream: ${toError(e).message}`));
324
+ reconnect().catch((e) => this.logger.logError(`Unexpected error re-opening job stream: ${(0, StreamLifecycle_1.toError)(e).message}`));
122
325
  }, delay);
123
326
  reconnectTimer = timer;
124
327
  this.reconnectTimers.add(timer);
@@ -126,7 +329,7 @@ class ZBStreamWorker {
126
329
  const handleOpenFailure = (error) => {
127
330
  this.logger.logError(`Failed to open job stream for ${req.type}: ${error.message}`);
128
331
  notify('onStreamError', () => onStreamError?.(error));
129
- scheduleReconnect();
332
+ onStreamDown({ kind: 'open-failed', error });
130
333
  };
131
334
  // Open the stream. Returns an Error if it could not be opened (the
132
335
  // caller decides what to do), otherwise attaches the lifecycle handlers.
@@ -139,74 +342,79 @@ class ZBStreamWorker {
139
342
  return { error: e };
140
343
  }
141
344
  })();
142
- const openError = getOpenError(result);
345
+ const openError = (0, StreamLifecycle_1.getOpenError)(result);
143
346
  if (openError) {
144
347
  return openError;
145
348
  }
146
- const stream = result;
349
+ const stream = (0, StreamLifecycle_1.asStream)(result);
147
350
  if (isClosed()) {
148
351
  // close() was called while the stream was being opened.
149
- stream.cancel();
150
- stream.destroy();
352
+ (0, StreamLifecycle_1.cancelStream)(stream);
151
353
  return undefined;
152
354
  }
153
355
  const openedAt = Date.now();
154
356
  let receivedData = false;
155
- let dead = false;
156
- // `error`, `end` and `close` can all fire for one dead stream; only
157
- // the first is acted on.
158
- const onStreamDead = (notifyCaller) => {
159
- if (dead) {
160
- return;
161
- }
162
- dead = true;
163
- discardStream(stream);
164
- if (currentStream === stream) {
165
- currentStream = undefined;
166
- }
167
- if (isClosed()) {
168
- // Expected: we cancelled it ourselves.
169
- return;
170
- }
171
- // A stream that delivered a job, or stayed up for a while, was
172
- // healthy: restart the backoff from the beginning.
357
+ let cause = { kind: 'end' };
358
+ // A stream that delivered a job, or stayed up for a while, was
359
+ // healthy: restart the backoff from the beginning.
360
+ const resetBackoffIfHealthy = () => {
173
361
  if (receivedData || Date.now() - openedAt >= maxDelayMs) {
174
362
  consecutiveFailures = 0;
175
363
  }
176
- notifyCaller();
177
- scheduleReconnect();
178
364
  };
179
- stream.on('error', (e) => onStreamDead(() => {
180
- const error = toError(e);
181
- this.logger.logError(`Job stream for ${req.type} failed: ${error.message}`);
182
- notify('onStreamError', () => onStreamError?.(error));
183
- }));
184
- const onEnd = () => onStreamDead(() => {
185
- this.logger.logError(`Job stream for ${req.type} ended`);
186
- notify('onStreamEnd', () => onStreamEnd?.());
187
- });
188
- stream.on('end', onEnd);
189
- stream.on('close', onEnd);
190
- stream.on('data', (res) => {
191
- if (!receivedData) {
192
- receivedData = true;
193
- consecutiveFailures = 0;
194
- }
195
- try {
196
- const parsedJob = (0, _1.parseVariablesAndCustomHeadersToJSON)(res, inputVariableDto, customHeadersDto);
197
- handleJob(parsedJob);
198
- }
199
- catch (e) {
200
- this.zbClient.failJob({
201
- jobKey: res.key,
202
- errorMessage: `Error parsing variable payload ${e}`,
203
- retries: res.retries - 1,
204
- retryBackOff: 0,
205
- });
206
- }
365
+ // The adapter is the only code that attaches listeners to the raw
366
+ // stream. It delivers exactly one of `error`/`end`, then `close`.
367
+ const attachment = (0, StreamLifecycle_1.attachStream)(stream, {
368
+ data: (res) => {
369
+ if (!receivedData) {
370
+ receivedData = true;
371
+ consecutiveFailures = 0;
372
+ }
373
+ try {
374
+ const parsedJob = (0, _1.parseVariablesAndCustomHeadersToJSON)(res, inputVariableDto, customHeadersDto);
375
+ void handleJob(parsedJob);
376
+ }
377
+ catch (e) {
378
+ this.zbClient
379
+ .failJob({
380
+ jobKey: res.key,
381
+ errorMessage: `Error parsing variable payload ${e}`,
382
+ retries: res.retries - 1,
383
+ retryBackOff: 0,
384
+ })
385
+ .catch((failError) => this.logger.logError(`Failed to fail job ${res.key} after a payload parse error: ${(0, StreamLifecycle_1.toError)(failError).message}`));
386
+ }
387
+ },
388
+ error: (error) => {
389
+ cause = { kind: 'error', error };
390
+ if (isClosed()) {
391
+ // Expected: we cancelled it ourselves.
392
+ return;
393
+ }
394
+ resetBackoffIfHealthy();
395
+ this.logger.logError(`Job stream for ${req.type} failed: ${error.message}`);
396
+ notify('onStreamError', () => onStreamError?.(error));
397
+ },
398
+ end: () => {
399
+ cause = { kind: 'end' };
400
+ if (isClosed()) {
401
+ return;
402
+ }
403
+ resetBackoffIfHealthy();
404
+ this.logger.logError(`Job stream for ${req.type} ended`);
405
+ notify('onStreamEnd', () => onStreamEnd?.());
406
+ },
407
+ close: () => {
408
+ discardStream(attachment);
409
+ if (currentStream === attachment) {
410
+ currentStream = undefined;
411
+ }
412
+ onStreamDown(cause);
413
+ },
207
414
  });
208
- this.streams.push(stream);
209
- currentStream = stream;
415
+ this.streams.push(attachment);
416
+ currentStream = attachment;
417
+ transition({ status: 'open' });
210
418
  return undefined;
211
419
  };
212
420
  // Re-open a dead stream, then backfill: jobs created while the stream
@@ -222,9 +430,35 @@ class ZBStreamWorker {
222
430
  if (openError) {
223
431
  return handleOpenFailure(openError);
224
432
  }
225
- await runPoll().catch((e) => this.logger.logError(`Backfill poll after job stream reconnect failed: ${toError(e).message}`));
433
+ await backfill().catch((e) => this.logger.logError(`Backfill poll after job stream reconnect failed: ${(0, StreamLifecycle_1.toError)(e).message}`));
226
434
  };
435
+ // The handle returned to the caller. `close()` is idempotent.
436
+ const closeHandle = () => {
437
+ handleClosed = true;
438
+ transition({ status: 'closed', reason: { kind: 'closed-by-caller' } });
439
+ if (sidecarTimer) {
440
+ clearTimeout(sidecarTimer);
441
+ }
442
+ handleRetryWaits.forEach((wake) => wake());
443
+ if (reconnectTimer) {
444
+ clearTimeout(reconnectTimer);
445
+ this.reconnectTimers.delete(reconnectTimer);
446
+ reconnectTimer = undefined;
447
+ }
448
+ if (currentStream) {
449
+ discardStream(currentStream);
450
+ currentStream.dispose();
451
+ currentStream = undefined;
452
+ }
453
+ };
454
+ const handle = (0, StreamLifecycle_1.createStreamHandle)(machine, closeHandle);
227
455
  const pollAndStream = async () => {
456
+ if (this.closed) {
457
+ // The client was already closed: open nothing, and give the caller a
458
+ // handle that says so, instead of one that reports `open` forever.
459
+ transition({ status: 'closed', reason: { kind: 'client-closed' } });
460
+ return handle;
461
+ }
228
462
  // Optional jitter to stagger multiple workers
229
463
  if (jitter) {
230
464
  await new Promise((resolve) => setTimeout(resolve, Math.random() * jitter));
@@ -233,7 +467,7 @@ class ZBStreamWorker {
233
467
  const openError = await openStream();
234
468
  if (isClosed()) {
235
469
  // close() was called while the stream was being opened.
236
- return { close: () => { } };
470
+ return handle;
237
471
  }
238
472
  if (openError) {
239
473
  handleOpenFailure(openError);
@@ -245,7 +479,7 @@ class ZBStreamWorker {
245
479
  // Initial backfill poll: pick up jobs that existed before the stream.
246
480
  // The broker guarantees single activation, so there is no risk of
247
481
  // duplicate delivery between the poll and the stream.
248
- await runPoll();
482
+ await backfill();
249
483
  }
250
484
  // Recurring sidecar poll: a low-frequency safety net that catches
251
485
  // jobs the stream may have missed (e.g. jobs re-queued after a
@@ -271,10 +505,22 @@ class ZBStreamWorker {
271
505
  if (index !== -1) {
272
506
  this.pollTimers.splice(index, 1);
273
507
  }
274
- runPoll()
275
- .catch(() => {
508
+ // An activation already in flight is doing this poll's job.
509
+ if (activationReserved || isClosed()) {
510
+ schedulePoll();
511
+ return;
512
+ }
513
+ // Reserve before activating: a reconnect backfill arriving
514
+ // while this activation is pending must not start a second,
515
+ // overlapping drain. The reservation is released by
516
+ // runDrain's finally, or here if the activation fails.
517
+ activationReserved = true;
518
+ activateWithRetry()
519
+ .then(runDrain, (e) => {
520
+ activationReserved = false;
276
521
  // Swallow errors — the stream is the primary channel.
277
522
  // The next poll cycle will retry.
523
+ this.logger.logDebug(`Stream worker sidecar poll failed: ${(0, StreamLifecycle_1.toError)(e).message}`);
278
524
  })
279
525
  .finally(schedulePoll);
280
526
  }, pollInterval);
@@ -282,25 +528,7 @@ class ZBStreamWorker {
282
528
  };
283
529
  schedulePoll();
284
530
  }
285
- return {
286
- close: () => {
287
- handleClosed = true;
288
- if (sidecarTimer) {
289
- clearTimeout(sidecarTimer);
290
- }
291
- if (reconnectTimer) {
292
- clearTimeout(reconnectTimer);
293
- this.reconnectTimers.delete(reconnectTimer);
294
- reconnectTimer = undefined;
295
- }
296
- if (currentStream) {
297
- discardStream(currentStream);
298
- currentStream.cancel();
299
- currentStream.destroy();
300
- currentStream = undefined;
301
- }
302
- },
303
- };
531
+ return handle;
304
532
  };
305
533
  return pollAndStream();
306
534
  }
@@ -310,10 +538,10 @@ class ZBStreamWorker {
310
538
  this.pollTimers = [];
311
539
  this.reconnectTimers.forEach((t) => clearTimeout(t));
312
540
  this.reconnectTimers.clear();
313
- this.streams.forEach((s) => {
314
- s.cancel();
315
- s.destroy();
316
- });
541
+ this.retryWaits.forEach((wake) => wake());
542
+ const clientClosed = { kind: 'client-closed' };
543
+ this.machines.forEach((m) => m.transition({ status: 'closed', reason: clientClosed }));
544
+ this.streams.forEach((s) => s.dispose());
317
545
  this.streams = [];
318
546
  return this.grpcClient.close();
319
547
  }
@@ -335,9 +563,8 @@ class ZBStreamWorker {
335
563
  const errorMsgOnPriorMessageCall = (thisMethod, wrappedFunction) => {
336
564
  return (...args) => {
337
565
  if (methodCalled !== undefined) {
338
- // tslint:disable-next-line: no-console
339
- console.log(chalk_1.default.red(`WARNING: Call to ${thisMethod}() after ${methodCalled}() was called.
340
- You should call only one job action method in the worker handler. This is a bug in a Stream worker handler.`));
566
+ this.logger.logError(`WARNING: Call to ${thisMethod}() after ${methodCalled}() was called.
567
+ You should call only one job action method in the worker handler. This is a bug in a Stream worker handler.`);
341
568
  return wrappedFunction(...args);
342
569
  }
343
570
  methodCalled = thisMethod;