@camunda8/sdk 8.9.0-alpha.15 → 8.9.0-alpha.17

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,14 +1,54 @@
1
1
  import { ZeebeGrpcClient } from '../zb/ZeebeGrpcClient';
2
2
  import { StatefulLogInterceptor } from './StatefulLogInterceptor';
3
+ import { StreamHandle } from './StreamLifecycle';
3
4
  import { IZBJobWorker, Job, JobCompletionInterface, ZBGrpc, ZBWorkerTaskHandler } from './interfaces-1.0';
4
5
  import { StreamActivatedJobsRequest } from './interfaces-grpc-1.0';
6
+ /**
7
+ * Options controlling how `streamJobs` reports and recovers from a job stream
8
+ * that dies (stream error, server-side end/close, or a failed open).
9
+ */
10
+ export interface StreamJobsRecoveryOptions {
11
+ /**
12
+ * Called when the job stream fails: a stream `error` event, or a failed
13
+ * attempt to open the stream. Exceptions thrown by the callback, and
14
+ * rejections of a returned promise, are caught and logged.
15
+ */
16
+ onStreamError?: (error: Error) => void | Promise<void>;
17
+ /**
18
+ * Called when the server ends or closes the job stream without an error.
19
+ * Exceptions thrown by the callback, and rejections of a returned
20
+ * promise, are caught and logged.
21
+ */
22
+ onStreamEnd?: () => void | Promise<void>;
23
+ /**
24
+ * Called just before each attempt to re-open a dead stream. `attempt` is
25
+ * the number of consecutive failed streams or open attempts, starting at 1.
26
+ * It resets once a re-opened stream delivers a job or stays up for at least
27
+ * `maxDelayMs`. Exceptions thrown by the callback, and rejections of a
28
+ * returned promise, are caught and logged.
29
+ */
30
+ onStreamReconnect?: (attempt: number) => void | Promise<void>;
31
+ /**
32
+ * Re-open the stream when it dies. Defaults to `true`, with exponential
33
+ * backoff starting at 1000ms and capped at 30000ms. Pass `false` to disable
34
+ * reconnection (the `onStream*` callbacks still fire), or an object to tune
35
+ * the backoff. After a re-opened stream is established, one backfill poll
36
+ * runs to pick up jobs that were created while the stream was down.
37
+ */
38
+ streamReconnect?: boolean | {
39
+ initialDelayMs?: number;
40
+ maxDelayMs?: number;
41
+ };
42
+ }
5
43
  export declare class ZBStreamWorker implements IZBJobWorker {
6
44
  private grpcClient;
7
45
  private logger;
8
46
  private zbClient;
9
47
  private streams;
48
+ private machines;
10
49
  private closed;
11
50
  private pollTimers;
51
+ private reconnectTimers;
12
52
  constructor({ grpcClient, log, zbClient, }: {
13
53
  grpcClient: ZBGrpc;
14
54
  log: StatefulLogInterceptor;
@@ -43,9 +83,7 @@ export declare class ZBStreamWorker implements IZBJobWorker {
43
83
  * polling (the initial backfill poll still runs).
44
84
  */
45
85
  pollInterval?: number;
46
- }): Promise<{
47
- close: () => void;
48
- }>;
86
+ } & StreamJobsRecoveryOptions): Promise<StreamHandle>;
49
87
  close(): Promise<void>;
50
88
  log(msg: JSON): void;
51
89
  debug(msg: JSON): void;
@@ -1,102 +1,417 @@
1
1
  "use strict";
2
- var __importDefault = (this && this.__importDefault) || function (mod) {
3
- return (mod && mod.__esModule) ? mod : { "default": mod };
4
- };
5
2
  Object.defineProperty(exports, "__esModule", { value: true });
6
3
  exports.ZBStreamWorker = void 0;
7
- const chalk_1 = __importDefault(require("chalk"));
4
+ const StreamLifecycle_1 = require("./StreamLifecycle");
8
5
  const interfaces_1_0_1 = require("./interfaces-1.0");
9
6
  const _1 = require(".");
7
+ const DEFAULT_RECONNECT_INITIAL_DELAY_MS = 1000;
8
+ const DEFAULT_RECONNECT_MAX_DELAY_MS = 30000;
10
9
  class ZBStreamWorker {
11
10
  constructor({ grpcClient, log, zbClient, }) {
12
11
  this.streams = [];
12
+ this.machines = new Set();
13
13
  this.closed = false;
14
14
  this.pollTimers = [];
15
+ this.reconnectTimers = new Set();
15
16
  this.grpcClient = grpcClient;
16
17
  this.logger = log;
17
18
  this.zbClient = zbClient;
18
19
  if (!this.zbClient) {
19
- console.log(`Missing ZBClient`);
20
+ this.logger.logError(`Missing ZBClient`);
20
21
  }
21
22
  }
22
23
  streamJobs(req) {
23
- const { taskHandler, inputVariableDto, customHeadersDto, jitter, pollMaxJobsToActivate = 32, pollInterval = 30000, ...streamReq } = req;
24
- const handleJob = (job) => {
25
- // Observe the handler's returned promise so that a rejected job
26
- // action (for example, a completion command that fails because the
27
- // job was already completed, failed, or the process instance was
28
- // cancelled) is handled here rather than surfacing as an unhandled
29
- // promise rejection that could terminate the process.
30
- Promise.resolve(taskHandler({
31
- ...job,
32
- // eslint-disable-next-line @typescript-eslint/no-explicit-any
33
- ...this.makeCompleteHandlers(job, req.type),
34
- }, this)).catch((e) => {
35
- const message = e instanceof Error ? e.message : String(e);
36
- this.logger.logError(`Unhandled exception in stream worker task handler for job ${job.key}: ${message}`);
24
+ const { taskHandler, inputVariableDto, customHeadersDto, jitter, pollMaxJobsToActivate = 32, pollInterval = 30000, onStreamError, onStreamEnd, onStreamReconnect, streamReconnect = true, ...streamReq } = req;
25
+ const reconnectEnabled = streamReconnect !== false;
26
+ const reconnectOptions = typeof streamReconnect === 'object' ? streamReconnect : {};
27
+ const initialDelayMs = reconnectOptions.initialDelayMs ?? DEFAULT_RECONNECT_INITIAL_DELAY_MS;
28
+ const maxDelayMs = Math.max(reconnectOptions.maxDelayMs ?? DEFAULT_RECONNECT_MAX_DELAY_MS, initialDelayMs);
29
+ // Returns the handler's promise, which never rejects: a failure is
30
+ // logged here. The backlog drain awaits it for backpressure.
31
+ const handleJob = (job) =>
32
+ // Invoke the handler inside a promise continuation: a synchronous
33
+ // throw must reject this promise rather than escape handleJob,
34
+ // which would abort jobs.map(handleJob) mid-batch — leaving later
35
+ // jobs undispatched and releasing the drain's reservation before
36
+ // the handlers already started have settled.
37
+ //
38
+ // Observe the handler's returned promise so that a rejected job
39
+ // action (for example, a completion command that fails because the
40
+ // job was already completed, failed, or the process instance was
41
+ // cancelled) is handled here rather than surfacing as an unhandled
42
+ // promise rejection that could terminate the process.
43
+ Promise.resolve()
44
+ .then(() => taskHandler({
45
+ ...job,
46
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
47
+ ...this.makeCompleteHandlers(job, req.type),
48
+ }, this))
49
+ .catch((e) => {
50
+ const message = e instanceof Error ? e.message : String(e);
51
+ this.logger.logError(`Unhandled exception in stream worker task handler for job ${job.key}: ${message}`);
52
+ });
53
+ // Invoke a caller-supplied lifecycle callback. A throwing callback must
54
+ // not break stream recovery (or crash the process from an event handler).
55
+ // Callbacks may be async, so a returned promise is observed as well: a
56
+ // rejection is logged rather than surfacing as an unhandled rejection.
57
+ const notify = (name, callback) => {
58
+ const logFailure = (e) => this.logger.logError(`Exception in ${name} callback of stream worker: ${(0, StreamLifecycle_1.toError)(e).message}`);
59
+ try {
60
+ Promise.resolve(callback()).catch(logFailure);
61
+ }
62
+ catch (e) {
63
+ logFailure(e);
64
+ }
65
+ };
66
+ // Mutable per-streamJobs-call state. Mutation is confined to the
67
+ // stream lifecycle handlers below.
68
+ let handleClosed = false; // set by the returned handle's close()
69
+ let currentStream;
70
+ const machine = (0, StreamLifecycle_1.createStreamStateMachine)();
71
+ this.machines.add(machine);
72
+ void machine.done.then(() => this.machines.delete(machine));
73
+ const transition = (next) => {
74
+ if (machine.transition(next)) {
75
+ this.logger.logDebug(`Job stream for ${req.type}: ${(0, StreamLifecycle_1.describeStreamState)(next)}`);
76
+ }
77
+ };
78
+ let reconnectTimer;
79
+ let sidecarTimer;
80
+ let consecutiveFailures = 0; // drives the exponential backoff
81
+ const isClosed = () => this.closed || handleClosed;
82
+ // Activate up to pollMaxJobsToActivate jobs.
83
+ const activate = () => this.zbClient.activateJobs({
84
+ type: req.type,
85
+ worker: req.worker,
86
+ timeout: req.timeout,
87
+ maxJobsToActivate: pollMaxJobsToActivate,
88
+ tenantIds: req.tenantIds,
89
+ fetchVariable: req.fetchVariable,
90
+ inputVariableDto,
91
+ customHeadersDto,
92
+ requestTimeout: -1,
93
+ });
94
+ // Hand a polled batch to the handler, then keep polling while polls
95
+ // come back full. The stream only pushes jobs that become activatable
96
+ // after it opens; a backlog that already exists is reachable only by
97
+ // polling. A full batch means more jobs are probably waiting, so poll
98
+ // again as soon as this batch is handled, rather than waiting a whole
99
+ // pollInterval per batch. Waiting for the batch's handlers to settle
100
+ // is the backpressure: the worker never holds more than one batch of
101
+ // polled jobs, so they don't sit activated until their timeout.
102
+ //
103
+ // Iterative, not recursive: a recursion would retain one pending
104
+ // promise per backlog batch until the whole backlog finishes, so a
105
+ // large or continuously replenished backlog would grow memory without
106
+ // bound. The loop holds only the current batch.
107
+ //
108
+ // A reconnect backfill requested while this drain was in flight is
109
+ // honoured before the drain exits: jobs created while the stream was
110
+ // down must not be stranded when recurring polling is disabled.
111
+ const drain = async (firstBatch) => {
112
+ let jobs = firstBatch;
113
+ for (;;) {
114
+ await Promise.all(jobs.map(handleJob));
115
+ if (backfillRequested && !isClosed()) {
116
+ // The stream reconnected while this drain was in flight and
117
+ // its backfill was deferred (see backfill). Run it now, even
118
+ // if this batch was partial: jobs created while the stream
119
+ // was down are not visible to any other poll when recurring
120
+ // polling is disabled. If the activation fails, restore the
121
+ // deferral before re-throwing so runDrain's finally can
122
+ // re-run it once the reservation is released — the backfill
123
+ // must not be discarded while the handle is open.
124
+ backfillRequested = false;
125
+ try {
126
+ jobs = await activate();
127
+ }
128
+ catch (e) {
129
+ backfillRequested = true;
130
+ throw e;
131
+ }
132
+ continue;
133
+ }
134
+ if (jobs.length < pollMaxJobsToActivate || isClosed()) {
135
+ return;
136
+ }
137
+ jobs = await activate();
138
+ }
139
+ };
140
+ // Only one activation is ever in flight. The flag is reserved *before*
141
+ // the activation call and released only when the drain it fed finishes
142
+ // (or the activation fails), so a reconnect backfill can never start a
143
+ // second activation while a sidecar activation is still pending — that
144
+ // would launch two independent drains, exceed the one-polled-batch
145
+ // limit, and let either drain clear the flag while the other runs.
146
+ let activationReserved = false; // mutated by runDrain/backfill/sidecar
147
+ // Set when a reconnect backfill arrives while an activation is in
148
+ // flight; the in-flight drain runs the backfill before exiting, or
149
+ // re-runs it from runDrain's finally if the drain's activation failed
150
+ // while the handle is still open.
151
+ let backfillRequested = false;
152
+ const runDrain = (jobs) => drain(jobs)
153
+ .catch((e) => {
154
+ // The stream is the primary channel; the next sidecar poll retries.
155
+ this.logger.logError(`Stream worker backlog poll failed: ${(0, StreamLifecycle_1.toError)(e).message}`);
156
+ })
157
+ .finally(() => {
158
+ activationReserved = false;
159
+ // A backfill deferred to this drain that never ran — the
160
+ // drain's activation failed, either before the deferral was
161
+ // consumed or in the activation that consumed it (drain
162
+ // restores the flag before re-throwing) — must not be
163
+ // dropped while the handle is open: with recurring polling
164
+ // disabled there is no other poll to pick up jobs created
165
+ // while the stream was down. Run it now that the reservation
166
+ // is released. If the handle is closed the reservation is
167
+ // stale instead: clear it so a later backfill is free to
168
+ // reserve and run its own poll.
169
+ if (backfillRequested) {
170
+ backfillRequested = false;
171
+ if (!isClosed()) {
172
+ // Observe the rejection: backfill rethrows a failed
173
+ // initial activation after releasing the reservation,
174
+ // and this call is fire-and-forget.
175
+ void backfill().catch((e) => this.logger.logError(`Stream worker backfill poll failed: ${(0, StreamLifecycle_1.toError)(e).message}`));
176
+ }
177
+ }
178
+ });
179
+ // Run a backfill poll. Only the first activation is awaited (so its
180
+ // errors reach the caller); the rest of the backlog drains in the
181
+ // background. If an activation is already in flight, the backfill is
182
+ // not dropped: the in-flight drain runs it before exiting, so jobs
183
+ // created while the stream was down are picked up even when recurring
184
+ // polling is disabled (pollInterval 0 or -1).
185
+ const backfill = async () => {
186
+ if (isClosed()) {
187
+ return;
188
+ }
189
+ if (activationReserved) {
190
+ backfillRequested = true;
191
+ return;
192
+ }
193
+ activationReserved = true;
194
+ try {
195
+ void runDrain(await activate());
196
+ }
197
+ catch (e) {
198
+ activationReserved = false;
199
+ throw e;
200
+ }
201
+ };
202
+ const discardStream = (attachment) => {
203
+ const index = this.streams.indexOf(attachment);
204
+ if (index !== -1) {
205
+ this.streams.splice(index, 1);
206
+ }
207
+ };
208
+ // The stream is down. Re-open it with backoff, or, if reconnection is
209
+ // disabled, record that it is closed for good.
210
+ const onStreamDown = (cause) => {
211
+ if (isClosed()) {
212
+ return;
213
+ }
214
+ if (!reconnectEnabled) {
215
+ transition({
216
+ status: 'closed',
217
+ reason: { kind: 'stream-failed', failure: cause },
218
+ });
219
+ return;
220
+ }
221
+ scheduleReconnect(cause);
222
+ };
223
+ // Schedule a single re-open of the stream with exponential backoff. Only
224
+ // one reconnect timer is ever pending, so an `error` and an `end` for the
225
+ // same dead stream (or a failed open racing a stream failure) cannot
226
+ // result in duplicate re-opens.
227
+ const scheduleReconnect = (cause) => {
228
+ if (!reconnectEnabled || isClosed() || reconnectTimer) {
229
+ return;
230
+ }
231
+ consecutiveFailures++;
232
+ const attempt = consecutiveFailures;
233
+ const delay = (0, StreamLifecycle_1.backoffDelayMs)(attempt, initialDelayMs, maxDelayMs);
234
+ transition({ status: 'reconnecting', attempt, cause });
235
+ const timer = setTimeout(() => {
236
+ reconnectTimer = undefined;
237
+ this.reconnectTimers.delete(timer);
238
+ if (isClosed()) {
239
+ return;
240
+ }
241
+ notify('onStreamReconnect', () => onStreamReconnect?.(attempt));
242
+ reconnect().catch((e) => this.logger.logError(`Unexpected error re-opening job stream: ${(0, StreamLifecycle_1.toError)(e).message}`));
243
+ }, delay);
244
+ reconnectTimer = timer;
245
+ this.reconnectTimers.add(timer);
246
+ };
247
+ const handleOpenFailure = (error) => {
248
+ this.logger.logError(`Failed to open job stream for ${req.type}: ${error.message}`);
249
+ notify('onStreamError', () => onStreamError?.(error));
250
+ onStreamDown({ kind: 'open-failed', error });
251
+ };
252
+ // Open the stream. Returns an Error if it could not be opened (the
253
+ // caller decides what to do), otherwise attaches the lifecycle handlers.
254
+ const openStream = async () => {
255
+ const result = await (async () => {
256
+ try {
257
+ return await this.grpcClient.streamActivatedJobsStream(streamReq);
258
+ }
259
+ catch (e) {
260
+ return { error: e };
261
+ }
262
+ })();
263
+ const openError = (0, StreamLifecycle_1.getOpenError)(result);
264
+ if (openError) {
265
+ return openError;
266
+ }
267
+ const stream = (0, StreamLifecycle_1.asStream)(result);
268
+ if (isClosed()) {
269
+ // close() was called while the stream was being opened.
270
+ (0, StreamLifecycle_1.cancelStream)(stream);
271
+ return undefined;
272
+ }
273
+ const openedAt = Date.now();
274
+ let receivedData = false;
275
+ let cause = { kind: 'end' };
276
+ // A stream that delivered a job, or stayed up for a while, was
277
+ // healthy: restart the backoff from the beginning.
278
+ const resetBackoffIfHealthy = () => {
279
+ if (receivedData || Date.now() - openedAt >= maxDelayMs) {
280
+ consecutiveFailures = 0;
281
+ }
282
+ };
283
+ // The adapter is the only code that attaches listeners to the raw
284
+ // stream. It delivers exactly one of `error`/`end`, then `close`.
285
+ const attachment = (0, StreamLifecycle_1.attachStream)(stream, {
286
+ data: (res) => {
287
+ if (!receivedData) {
288
+ receivedData = true;
289
+ consecutiveFailures = 0;
290
+ }
291
+ try {
292
+ const parsedJob = (0, _1.parseVariablesAndCustomHeadersToJSON)(res, inputVariableDto, customHeadersDto);
293
+ void handleJob(parsedJob);
294
+ }
295
+ catch (e) {
296
+ this.zbClient
297
+ .failJob({
298
+ jobKey: res.key,
299
+ errorMessage: `Error parsing variable payload ${e}`,
300
+ retries: res.retries - 1,
301
+ retryBackOff: 0,
302
+ })
303
+ .catch((failError) => this.logger.logError(`Failed to fail job ${res.key} after a payload parse error: ${(0, StreamLifecycle_1.toError)(failError).message}`));
304
+ }
305
+ },
306
+ error: (error) => {
307
+ cause = { kind: 'error', error };
308
+ if (isClosed()) {
309
+ // Expected: we cancelled it ourselves.
310
+ return;
311
+ }
312
+ resetBackoffIfHealthy();
313
+ this.logger.logError(`Job stream for ${req.type} failed: ${error.message}`);
314
+ notify('onStreamError', () => onStreamError?.(error));
315
+ },
316
+ end: () => {
317
+ cause = { kind: 'end' };
318
+ if (isClosed()) {
319
+ return;
320
+ }
321
+ resetBackoffIfHealthy();
322
+ this.logger.logError(`Job stream for ${req.type} ended`);
323
+ notify('onStreamEnd', () => onStreamEnd?.());
324
+ },
325
+ close: () => {
326
+ discardStream(attachment);
327
+ if (currentStream === attachment) {
328
+ currentStream = undefined;
329
+ }
330
+ onStreamDown(cause);
331
+ },
37
332
  });
333
+ this.streams.push(attachment);
334
+ currentStream = attachment;
335
+ transition({ status: 'open' });
336
+ return undefined;
337
+ };
338
+ // Re-open a dead stream, then backfill: jobs created while the stream
339
+ // was down were never delivered to it. The broker guarantees single
340
+ // activation, so there is no risk of duplicate delivery.
341
+ const reconnect = async () => {
342
+ const openError = await openStream();
343
+ if (isClosed()) {
344
+ // close() was called while the stream was being opened: a failed
345
+ // open is expected fallout, so neither report it nor retry.
346
+ return;
347
+ }
348
+ if (openError) {
349
+ return handleOpenFailure(openError);
350
+ }
351
+ await backfill().catch((e) => this.logger.logError(`Backfill poll after job stream reconnect failed: ${(0, StreamLifecycle_1.toError)(e).message}`));
38
352
  };
353
+ // The handle returned to the caller. `close()` is idempotent.
354
+ const closeHandle = () => {
355
+ handleClosed = true;
356
+ transition({ status: 'closed', reason: { kind: 'closed-by-caller' } });
357
+ if (sidecarTimer) {
358
+ clearTimeout(sidecarTimer);
359
+ }
360
+ if (reconnectTimer) {
361
+ clearTimeout(reconnectTimer);
362
+ this.reconnectTimers.delete(reconnectTimer);
363
+ reconnectTimer = undefined;
364
+ }
365
+ if (currentStream) {
366
+ discardStream(currentStream);
367
+ currentStream.dispose();
368
+ currentStream = undefined;
369
+ }
370
+ };
371
+ const handle = (0, StreamLifecycle_1.createStreamHandle)(machine, closeHandle);
39
372
  const pollAndStream = async () => {
373
+ if (this.closed) {
374
+ // The client was already closed: open nothing, and give the caller a
375
+ // handle that says so, instead of one that reports `open` forever.
376
+ transition({ status: 'closed', reason: { kind: 'client-closed' } });
377
+ return handle;
378
+ }
40
379
  // Optional jitter to stagger multiple workers
41
380
  if (jitter) {
42
381
  await new Promise((resolve) => setTimeout(resolve, Math.random() * jitter));
43
382
  }
44
383
  // Open the stream first so no new jobs are missed
45
- const stream = await this.grpcClient.streamActivatedJobsStream(streamReq);
46
- stream.on('error', (e) => {
47
- console.error(e);
48
- });
49
- stream.on('data', (res) => {
50
- try {
51
- const parsedJob = (0, _1.parseVariablesAndCustomHeadersToJSON)(res, inputVariableDto, customHeadersDto);
52
- handleJob(parsedJob);
53
- }
54
- catch (e) {
55
- this.zbClient.failJob({
56
- jobKey: res.key,
57
- errorMessage: `Error parsing variable payload ${e}`,
58
- retries: res.retries - 1,
59
- retryBackOff: 0,
60
- });
61
- }
62
- });
63
- this.streams.push(stream);
64
- // Helper to run a single poll cycle and feed results to handleJob.
65
- const runPoll = async () => {
66
- const jobs = await this.zbClient.activateJobs({
67
- type: req.type,
68
- worker: req.worker,
69
- timeout: req.timeout,
70
- maxJobsToActivate: pollMaxJobsToActivate,
71
- tenantIds: req.tenantIds,
72
- fetchVariable: req.fetchVariable,
73
- inputVariableDto,
74
- customHeadersDto,
75
- requestTimeout: -1,
76
- });
77
- for (const job of jobs) {
78
- handleJob(job);
384
+ const openError = await openStream();
385
+ if (isClosed()) {
386
+ // close() was called while the stream was being opened.
387
+ return handle;
388
+ }
389
+ if (openError) {
390
+ handleOpenFailure(openError);
391
+ if (!reconnectEnabled) {
392
+ throw openError;
79
393
  }
80
- };
81
- // Initial backfill poll: pick up jobs that existed before the stream.
82
- // The broker guarantees single activation, so there is no risk of
83
- // duplicate delivery between the poll and the stream.
84
- await runPoll();
394
+ }
395
+ else {
396
+ // Initial backfill poll: pick up jobs that existed before the stream.
397
+ // The broker guarantees single activation, so there is no risk of
398
+ // duplicate delivery between the poll and the stream.
399
+ await backfill();
400
+ }
85
401
  // Recurring sidecar poll: a low-frequency safety net that catches
86
402
  // jobs the stream may have missed (e.g. jobs re-queued after a
87
- // timeout or during a brief stream reconnect). Uses a setTimeout
88
- // chain so each cycle waits for the previous poll to finish before
89
- // scheduling the next, preventing overlapping polls.
403
+ // timeout or while the stream is down and being re-opened). Uses a
404
+ // setTimeout chain so each cycle waits for the previous poll to
405
+ // finish before scheduling the next, preventing overlapping polls.
90
406
  //
91
407
  // The live timer is tracked in `this.pollTimers` so that `close()`
92
408
  // can cancel it. Each cycle removes the timer that just fired
93
409
  // before scheduling the next one, so the array holds at most one
94
410
  // live timer per stream rather than growing by one per poll cycle.
95
- let sidecarTimer;
96
411
  if (pollInterval > 0) {
97
412
  const schedulePoll = () => {
98
413
  // A poll in flight when close() is called must not schedule another.
99
- if (this.closed) {
414
+ if (isClosed()) {
100
415
  return;
101
416
  }
102
417
  sidecarTimer = setTimeout(() => {
@@ -107,10 +422,22 @@ class ZBStreamWorker {
107
422
  if (index !== -1) {
108
423
  this.pollTimers.splice(index, 1);
109
424
  }
110
- runPoll()
111
- .catch(() => {
425
+ // An activation already in flight is doing this poll's job.
426
+ if (activationReserved || isClosed()) {
427
+ schedulePoll();
428
+ return;
429
+ }
430
+ // Reserve before activating: a reconnect backfill arriving
431
+ // while this activation is pending must not start a second,
432
+ // overlapping drain. The reservation is released by
433
+ // runDrain's finally, or here if the activation fails.
434
+ activationReserved = true;
435
+ activate()
436
+ .then(runDrain, (e) => {
437
+ activationReserved = false;
112
438
  // Swallow errors — the stream is the primary channel.
113
439
  // The next poll cycle will retry.
440
+ this.logger.logDebug(`Stream worker sidecar poll failed: ${(0, StreamLifecycle_1.toError)(e).message}`);
114
441
  })
115
442
  .finally(schedulePoll);
116
443
  }, pollInterval);
@@ -118,15 +445,7 @@ class ZBStreamWorker {
118
445
  };
119
446
  schedulePoll();
120
447
  }
121
- return {
122
- close: () => {
123
- if (sidecarTimer) {
124
- clearTimeout(sidecarTimer);
125
- }
126
- stream.cancel();
127
- stream.destroy();
128
- },
129
- };
448
+ return handle;
130
449
  };
131
450
  return pollAndStream();
132
451
  }
@@ -134,10 +453,12 @@ class ZBStreamWorker {
134
453
  this.closed = true;
135
454
  this.pollTimers.forEach((t) => clearTimeout(t));
136
455
  this.pollTimers = [];
137
- this.streams.forEach((s) => {
138
- s.cancel();
139
- s.destroy();
140
- });
456
+ this.reconnectTimers.forEach((t) => clearTimeout(t));
457
+ this.reconnectTimers.clear();
458
+ const clientClosed = { kind: 'client-closed' };
459
+ this.machines.forEach((m) => m.transition({ status: 'closed', reason: clientClosed }));
460
+ this.streams.forEach((s) => s.dispose());
461
+ this.streams = [];
141
462
  return this.grpcClient.close();
142
463
  }
143
464
  log(msg) {
@@ -158,9 +479,8 @@ class ZBStreamWorker {
158
479
  const errorMsgOnPriorMessageCall = (thisMethod, wrappedFunction) => {
159
480
  return (...args) => {
160
481
  if (methodCalled !== undefined) {
161
- // tslint:disable-next-line: no-console
162
- console.log(chalk_1.default.red(`WARNING: Call to ${thisMethod}() after ${methodCalled}() was called.
163
- You should call only one job action method in the worker handler. This is a bug in a Stream worker handler.`));
482
+ this.logger.logError(`WARNING: Call to ${thisMethod}() after ${methodCalled}() was called.
483
+ You should call only one job action method in the worker handler. This is a bug in a Stream worker handler.`);
164
484
  return wrappedFunction(...args);
165
485
  }
166
486
  methodCalled = thisMethod;