@camunda8/sdk 8.9.0-alpha.15 → 8.9.0-alpha.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.eslintrc.json +79 -0
- package/AGENT.md +9 -0
- package/CHANGELOG.md +14 -0
- package/MAINTAINER.md +10 -10
- package/README.md +36 -2
- package/dist/test-support/socketTap.d.ts +23 -0
- package/dist/test-support/socketTap.js +183 -0
- package/dist/test-support/socketTap.js.map +1 -0
- package/dist/zeebe/index.d.ts +1 -0
- package/dist/zeebe/index.js.map +1 -1
- package/dist/zeebe/lib/GrpcClient.d.ts +6 -0
- package/dist/zeebe/lib/GrpcClient.js +93 -85
- package/dist/zeebe/lib/GrpcClient.js.map +1 -1
- package/dist/zeebe/lib/StreamLifecycle.d.ts +189 -0
- package/dist/zeebe/lib/StreamLifecycle.js +142 -0
- package/dist/zeebe/lib/StreamLifecycle.js.map +1 -0
- package/dist/zeebe/lib/ZBStreamWorker.d.ts +41 -3
- package/dist/zeebe/lib/ZBStreamWorker.js +401 -81
- package/dist/zeebe/lib/ZBStreamWorker.js.map +1 -1
- package/dist/zeebe/lib/cancelProcesses.js +8 -0
- package/dist/zeebe/lib/cancelProcesses.js.map +1 -1
- package/dist/zeebe/zb/ZeebeGrpcClient.d.ts +17 -5
- package/dist/zeebe/zb/ZeebeGrpcClient.js +14 -2
- package/dist/zeebe/zb/ZeebeGrpcClient.js.map +1 -1
- package/package.json +8 -2
- package/scripts/assert-no-skipped-tests.d.ts +17 -0
- package/scripts/assert-no-skipped-tests.js +118 -0
- package/scripts/required-integration-specs.json +4 -0
- package/stryker.config.json +20 -0
- package/vitest.mutation.config.js +18 -0
|
@@ -1,14 +1,54 @@
|
|
|
1
1
|
import { ZeebeGrpcClient } from '../zb/ZeebeGrpcClient';
|
|
2
2
|
import { StatefulLogInterceptor } from './StatefulLogInterceptor';
|
|
3
|
+
import { StreamHandle } from './StreamLifecycle';
|
|
3
4
|
import { IZBJobWorker, Job, JobCompletionInterface, ZBGrpc, ZBWorkerTaskHandler } from './interfaces-1.0';
|
|
4
5
|
import { StreamActivatedJobsRequest } from './interfaces-grpc-1.0';
|
|
6
|
+
/**
|
|
7
|
+
* Options controlling how `streamJobs` reports and recovers from a job stream
|
|
8
|
+
* that dies (stream error, server-side end/close, or a failed open).
|
|
9
|
+
*/
|
|
10
|
+
export interface StreamJobsRecoveryOptions {
|
|
11
|
+
/**
|
|
12
|
+
* Called when the job stream fails: a stream `error` event, or a failed
|
|
13
|
+
* attempt to open the stream. Exceptions thrown by the callback, and
|
|
14
|
+
* rejections of a returned promise, are caught and logged.
|
|
15
|
+
*/
|
|
16
|
+
onStreamError?: (error: Error) => void | Promise<void>;
|
|
17
|
+
/**
|
|
18
|
+
* Called when the server ends or closes the job stream without an error.
|
|
19
|
+
* Exceptions thrown by the callback, and rejections of a returned
|
|
20
|
+
* promise, are caught and logged.
|
|
21
|
+
*/
|
|
22
|
+
onStreamEnd?: () => void | Promise<void>;
|
|
23
|
+
/**
|
|
24
|
+
* Called just before each attempt to re-open a dead stream. `attempt` is
|
|
25
|
+
* the number of consecutive failed streams or open attempts, starting at 1.
|
|
26
|
+
* It resets once a re-opened stream delivers a job or stays up for at least
|
|
27
|
+
* `maxDelayMs`. Exceptions thrown by the callback, and rejections of a
|
|
28
|
+
* returned promise, are caught and logged.
|
|
29
|
+
*/
|
|
30
|
+
onStreamReconnect?: (attempt: number) => void | Promise<void>;
|
|
31
|
+
/**
|
|
32
|
+
* Re-open the stream when it dies. Defaults to `true`, with exponential
|
|
33
|
+
* backoff starting at 1000ms and capped at 30000ms. Pass `false` to disable
|
|
34
|
+
* reconnection (the `onStream*` callbacks still fire), or an object to tune
|
|
35
|
+
* the backoff. After a re-opened stream is established, one backfill poll
|
|
36
|
+
* runs to pick up jobs that were created while the stream was down.
|
|
37
|
+
*/
|
|
38
|
+
streamReconnect?: boolean | {
|
|
39
|
+
initialDelayMs?: number;
|
|
40
|
+
maxDelayMs?: number;
|
|
41
|
+
};
|
|
42
|
+
}
|
|
5
43
|
export declare class ZBStreamWorker implements IZBJobWorker {
|
|
6
44
|
private grpcClient;
|
|
7
45
|
private logger;
|
|
8
46
|
private zbClient;
|
|
9
47
|
private streams;
|
|
48
|
+
private machines;
|
|
10
49
|
private closed;
|
|
11
50
|
private pollTimers;
|
|
51
|
+
private reconnectTimers;
|
|
12
52
|
constructor({ grpcClient, log, zbClient, }: {
|
|
13
53
|
grpcClient: ZBGrpc;
|
|
14
54
|
log: StatefulLogInterceptor;
|
|
@@ -43,9 +83,7 @@ export declare class ZBStreamWorker implements IZBJobWorker {
|
|
|
43
83
|
* polling (the initial backfill poll still runs).
|
|
44
84
|
*/
|
|
45
85
|
pollInterval?: number;
|
|
46
|
-
}): Promise<
|
|
47
|
-
close: () => void;
|
|
48
|
-
}>;
|
|
86
|
+
} & StreamJobsRecoveryOptions): Promise<StreamHandle>;
|
|
49
87
|
close(): Promise<void>;
|
|
50
88
|
log(msg: JSON): void;
|
|
51
89
|
debug(msg: JSON): void;
|
|
@@ -1,102 +1,417 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
-
};
|
|
5
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
3
|
exports.ZBStreamWorker = void 0;
|
|
7
|
-
const
|
|
4
|
+
const StreamLifecycle_1 = require("./StreamLifecycle");
|
|
8
5
|
const interfaces_1_0_1 = require("./interfaces-1.0");
|
|
9
6
|
const _1 = require(".");
|
|
7
|
+
const DEFAULT_RECONNECT_INITIAL_DELAY_MS = 1000;
|
|
8
|
+
const DEFAULT_RECONNECT_MAX_DELAY_MS = 30000;
|
|
10
9
|
class ZBStreamWorker {
|
|
11
10
|
constructor({ grpcClient, log, zbClient, }) {
|
|
12
11
|
this.streams = [];
|
|
12
|
+
this.machines = new Set();
|
|
13
13
|
this.closed = false;
|
|
14
14
|
this.pollTimers = [];
|
|
15
|
+
this.reconnectTimers = new Set();
|
|
15
16
|
this.grpcClient = grpcClient;
|
|
16
17
|
this.logger = log;
|
|
17
18
|
this.zbClient = zbClient;
|
|
18
19
|
if (!this.zbClient) {
|
|
19
|
-
|
|
20
|
+
this.logger.logError(`Missing ZBClient`);
|
|
20
21
|
}
|
|
21
22
|
}
|
|
22
23
|
streamJobs(req) {
|
|
23
|
-
const { taskHandler, inputVariableDto, customHeadersDto, jitter, pollMaxJobsToActivate = 32, pollInterval = 30000, ...streamReq } = req;
|
|
24
|
-
const
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
24
|
+
const { taskHandler, inputVariableDto, customHeadersDto, jitter, pollMaxJobsToActivate = 32, pollInterval = 30000, onStreamError, onStreamEnd, onStreamReconnect, streamReconnect = true, ...streamReq } = req;
|
|
25
|
+
const reconnectEnabled = streamReconnect !== false;
|
|
26
|
+
const reconnectOptions = typeof streamReconnect === 'object' ? streamReconnect : {};
|
|
27
|
+
const initialDelayMs = reconnectOptions.initialDelayMs ?? DEFAULT_RECONNECT_INITIAL_DELAY_MS;
|
|
28
|
+
const maxDelayMs = Math.max(reconnectOptions.maxDelayMs ?? DEFAULT_RECONNECT_MAX_DELAY_MS, initialDelayMs);
|
|
29
|
+
// Returns the handler's promise, which never rejects: a failure is
|
|
30
|
+
// logged here. The backlog drain awaits it for backpressure.
|
|
31
|
+
const handleJob = (job) =>
|
|
32
|
+
// Invoke the handler inside a promise continuation: a synchronous
|
|
33
|
+
// throw must reject this promise rather than escape handleJob,
|
|
34
|
+
// which would abort jobs.map(handleJob) mid-batch — leaving later
|
|
35
|
+
// jobs undispatched and releasing the drain's reservation before
|
|
36
|
+
// the handlers already started have settled.
|
|
37
|
+
//
|
|
38
|
+
// Observe the handler's returned promise so that a rejected job
|
|
39
|
+
// action (for example, a completion command that fails because the
|
|
40
|
+
// job was already completed, failed, or the process instance was
|
|
41
|
+
// cancelled) is handled here rather than surfacing as an unhandled
|
|
42
|
+
// promise rejection that could terminate the process.
|
|
43
|
+
Promise.resolve()
|
|
44
|
+
.then(() => taskHandler({
|
|
45
|
+
...job,
|
|
46
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
47
|
+
...this.makeCompleteHandlers(job, req.type),
|
|
48
|
+
}, this))
|
|
49
|
+
.catch((e) => {
|
|
50
|
+
const message = e instanceof Error ? e.message : String(e);
|
|
51
|
+
this.logger.logError(`Unhandled exception in stream worker task handler for job ${job.key}: ${message}`);
|
|
52
|
+
});
|
|
53
|
+
// Invoke a caller-supplied lifecycle callback. A throwing callback must
|
|
54
|
+
// not break stream recovery (or crash the process from an event handler).
|
|
55
|
+
// Callbacks may be async, so a returned promise is observed as well: a
|
|
56
|
+
// rejection is logged rather than surfacing as an unhandled rejection.
|
|
57
|
+
const notify = (name, callback) => {
|
|
58
|
+
const logFailure = (e) => this.logger.logError(`Exception in ${name} callback of stream worker: ${(0, StreamLifecycle_1.toError)(e).message}`);
|
|
59
|
+
try {
|
|
60
|
+
Promise.resolve(callback()).catch(logFailure);
|
|
61
|
+
}
|
|
62
|
+
catch (e) {
|
|
63
|
+
logFailure(e);
|
|
64
|
+
}
|
|
65
|
+
};
|
|
66
|
+
// Mutable per-streamJobs-call state. Mutation is confined to the
|
|
67
|
+
// stream lifecycle handlers below.
|
|
68
|
+
let handleClosed = false; // set by the returned handle's close()
|
|
69
|
+
let currentStream;
|
|
70
|
+
const machine = (0, StreamLifecycle_1.createStreamStateMachine)();
|
|
71
|
+
this.machines.add(machine);
|
|
72
|
+
void machine.done.then(() => this.machines.delete(machine));
|
|
73
|
+
const transition = (next) => {
|
|
74
|
+
if (machine.transition(next)) {
|
|
75
|
+
this.logger.logDebug(`Job stream for ${req.type}: ${(0, StreamLifecycle_1.describeStreamState)(next)}`);
|
|
76
|
+
}
|
|
77
|
+
};
|
|
78
|
+
let reconnectTimer;
|
|
79
|
+
let sidecarTimer;
|
|
80
|
+
let consecutiveFailures = 0; // drives the exponential backoff
|
|
81
|
+
const isClosed = () => this.closed || handleClosed;
|
|
82
|
+
// Activate up to pollMaxJobsToActivate jobs.
|
|
83
|
+
const activate = () => this.zbClient.activateJobs({
|
|
84
|
+
type: req.type,
|
|
85
|
+
worker: req.worker,
|
|
86
|
+
timeout: req.timeout,
|
|
87
|
+
maxJobsToActivate: pollMaxJobsToActivate,
|
|
88
|
+
tenantIds: req.tenantIds,
|
|
89
|
+
fetchVariable: req.fetchVariable,
|
|
90
|
+
inputVariableDto,
|
|
91
|
+
customHeadersDto,
|
|
92
|
+
requestTimeout: -1,
|
|
93
|
+
});
|
|
94
|
+
// Hand a polled batch to the handler, then keep polling while polls
|
|
95
|
+
// come back full. The stream only pushes jobs that become activatable
|
|
96
|
+
// after it opens; a backlog that already exists is reachable only by
|
|
97
|
+
// polling. A full batch means more jobs are probably waiting, so poll
|
|
98
|
+
// again as soon as this batch is handled, rather than waiting a whole
|
|
99
|
+
// pollInterval per batch. Waiting for the batch's handlers to settle
|
|
100
|
+
// is the backpressure: the worker never holds more than one batch of
|
|
101
|
+
// polled jobs, so they don't sit activated until their timeout.
|
|
102
|
+
//
|
|
103
|
+
// Iterative, not recursive: a recursion would retain one pending
|
|
104
|
+
// promise per backlog batch until the whole backlog finishes, so a
|
|
105
|
+
// large or continuously replenished backlog would grow memory without
|
|
106
|
+
// bound. The loop holds only the current batch.
|
|
107
|
+
//
|
|
108
|
+
// A reconnect backfill requested while this drain was in flight is
|
|
109
|
+
// honoured before the drain exits: jobs created while the stream was
|
|
110
|
+
// down must not be stranded when recurring polling is disabled.
|
|
111
|
+
const drain = async (firstBatch) => {
|
|
112
|
+
let jobs = firstBatch;
|
|
113
|
+
for (;;) {
|
|
114
|
+
await Promise.all(jobs.map(handleJob));
|
|
115
|
+
if (backfillRequested && !isClosed()) {
|
|
116
|
+
// The stream reconnected while this drain was in flight and
|
|
117
|
+
// its backfill was deferred (see backfill). Run it now, even
|
|
118
|
+
// if this batch was partial: jobs created while the stream
|
|
119
|
+
// was down are not visible to any other poll when recurring
|
|
120
|
+
// polling is disabled. If the activation fails, restore the
|
|
121
|
+
// deferral before re-throwing so runDrain's finally can
|
|
122
|
+
// re-run it once the reservation is released — the backfill
|
|
123
|
+
// must not be discarded while the handle is open.
|
|
124
|
+
backfillRequested = false;
|
|
125
|
+
try {
|
|
126
|
+
jobs = await activate();
|
|
127
|
+
}
|
|
128
|
+
catch (e) {
|
|
129
|
+
backfillRequested = true;
|
|
130
|
+
throw e;
|
|
131
|
+
}
|
|
132
|
+
continue;
|
|
133
|
+
}
|
|
134
|
+
if (jobs.length < pollMaxJobsToActivate || isClosed()) {
|
|
135
|
+
return;
|
|
136
|
+
}
|
|
137
|
+
jobs = await activate();
|
|
138
|
+
}
|
|
139
|
+
};
|
|
140
|
+
// Only one activation is ever in flight. The flag is reserved *before*
|
|
141
|
+
// the activation call and released only when the drain it fed finishes
|
|
142
|
+
// (or the activation fails), so a reconnect backfill can never start a
|
|
143
|
+
// second activation while a sidecar activation is still pending — that
|
|
144
|
+
// would launch two independent drains, exceed the one-polled-batch
|
|
145
|
+
// limit, and let either drain clear the flag while the other runs.
|
|
146
|
+
let activationReserved = false; // mutated by runDrain/backfill/sidecar
|
|
147
|
+
// Set when a reconnect backfill arrives while an activation is in
|
|
148
|
+
// flight; the in-flight drain runs the backfill before exiting, or
|
|
149
|
+
// re-runs it from runDrain's finally if the drain's activation failed
|
|
150
|
+
// while the handle is still open.
|
|
151
|
+
let backfillRequested = false;
|
|
152
|
+
const runDrain = (jobs) => drain(jobs)
|
|
153
|
+
.catch((e) => {
|
|
154
|
+
// The stream is the primary channel; the next sidecar poll retries.
|
|
155
|
+
this.logger.logError(`Stream worker backlog poll failed: ${(0, StreamLifecycle_1.toError)(e).message}`);
|
|
156
|
+
})
|
|
157
|
+
.finally(() => {
|
|
158
|
+
activationReserved = false;
|
|
159
|
+
// A backfill deferred to this drain that never ran — the
|
|
160
|
+
// drain's activation failed, either before the deferral was
|
|
161
|
+
// consumed or in the activation that consumed it (drain
|
|
162
|
+
// restores the flag before re-throwing) — must not be
|
|
163
|
+
// dropped while the handle is open: with recurring polling
|
|
164
|
+
// disabled there is no other poll to pick up jobs created
|
|
165
|
+
// while the stream was down. Run it now that the reservation
|
|
166
|
+
// is released. If the handle is closed the reservation is
|
|
167
|
+
// stale instead: clear it so a later backfill is free to
|
|
168
|
+
// reserve and run its own poll.
|
|
169
|
+
if (backfillRequested) {
|
|
170
|
+
backfillRequested = false;
|
|
171
|
+
if (!isClosed()) {
|
|
172
|
+
// Observe the rejection: backfill rethrows a failed
|
|
173
|
+
// initial activation after releasing the reservation,
|
|
174
|
+
// and this call is fire-and-forget.
|
|
175
|
+
void backfill().catch((e) => this.logger.logError(`Stream worker backfill poll failed: ${(0, StreamLifecycle_1.toError)(e).message}`));
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
});
|
|
179
|
+
// Run a backfill poll. Only the first activation is awaited (so its
|
|
180
|
+
// errors reach the caller); the rest of the backlog drains in the
|
|
181
|
+
// background. If an activation is already in flight, the backfill is
|
|
182
|
+
// not dropped: the in-flight drain runs it before exiting, so jobs
|
|
183
|
+
// created while the stream was down are picked up even when recurring
|
|
184
|
+
// polling is disabled (pollInterval 0 or -1).
|
|
185
|
+
const backfill = async () => {
|
|
186
|
+
if (isClosed()) {
|
|
187
|
+
return;
|
|
188
|
+
}
|
|
189
|
+
if (activationReserved) {
|
|
190
|
+
backfillRequested = true;
|
|
191
|
+
return;
|
|
192
|
+
}
|
|
193
|
+
activationReserved = true;
|
|
194
|
+
try {
|
|
195
|
+
void runDrain(await activate());
|
|
196
|
+
}
|
|
197
|
+
catch (e) {
|
|
198
|
+
activationReserved = false;
|
|
199
|
+
throw e;
|
|
200
|
+
}
|
|
201
|
+
};
|
|
202
|
+
const discardStream = (attachment) => {
|
|
203
|
+
const index = this.streams.indexOf(attachment);
|
|
204
|
+
if (index !== -1) {
|
|
205
|
+
this.streams.splice(index, 1);
|
|
206
|
+
}
|
|
207
|
+
};
|
|
208
|
+
// The stream is down. Re-open it with backoff, or, if reconnection is
|
|
209
|
+
// disabled, record that it is closed for good.
|
|
210
|
+
const onStreamDown = (cause) => {
|
|
211
|
+
if (isClosed()) {
|
|
212
|
+
return;
|
|
213
|
+
}
|
|
214
|
+
if (!reconnectEnabled) {
|
|
215
|
+
transition({
|
|
216
|
+
status: 'closed',
|
|
217
|
+
reason: { kind: 'stream-failed', failure: cause },
|
|
218
|
+
});
|
|
219
|
+
return;
|
|
220
|
+
}
|
|
221
|
+
scheduleReconnect(cause);
|
|
222
|
+
};
|
|
223
|
+
// Schedule a single re-open of the stream with exponential backoff. Only
|
|
224
|
+
// one reconnect timer is ever pending, so an `error` and an `end` for the
|
|
225
|
+
// same dead stream (or a failed open racing a stream failure) cannot
|
|
226
|
+
// result in duplicate re-opens.
|
|
227
|
+
const scheduleReconnect = (cause) => {
|
|
228
|
+
if (!reconnectEnabled || isClosed() || reconnectTimer) {
|
|
229
|
+
return;
|
|
230
|
+
}
|
|
231
|
+
consecutiveFailures++;
|
|
232
|
+
const attempt = consecutiveFailures;
|
|
233
|
+
const delay = (0, StreamLifecycle_1.backoffDelayMs)(attempt, initialDelayMs, maxDelayMs);
|
|
234
|
+
transition({ status: 'reconnecting', attempt, cause });
|
|
235
|
+
const timer = setTimeout(() => {
|
|
236
|
+
reconnectTimer = undefined;
|
|
237
|
+
this.reconnectTimers.delete(timer);
|
|
238
|
+
if (isClosed()) {
|
|
239
|
+
return;
|
|
240
|
+
}
|
|
241
|
+
notify('onStreamReconnect', () => onStreamReconnect?.(attempt));
|
|
242
|
+
reconnect().catch((e) => this.logger.logError(`Unexpected error re-opening job stream: ${(0, StreamLifecycle_1.toError)(e).message}`));
|
|
243
|
+
}, delay);
|
|
244
|
+
reconnectTimer = timer;
|
|
245
|
+
this.reconnectTimers.add(timer);
|
|
246
|
+
};
|
|
247
|
+
const handleOpenFailure = (error) => {
|
|
248
|
+
this.logger.logError(`Failed to open job stream for ${req.type}: ${error.message}`);
|
|
249
|
+
notify('onStreamError', () => onStreamError?.(error));
|
|
250
|
+
onStreamDown({ kind: 'open-failed', error });
|
|
251
|
+
};
|
|
252
|
+
// Open the stream. Returns an Error if it could not be opened (the
|
|
253
|
+
// caller decides what to do), otherwise attaches the lifecycle handlers.
|
|
254
|
+
const openStream = async () => {
|
|
255
|
+
const result = await (async () => {
|
|
256
|
+
try {
|
|
257
|
+
return await this.grpcClient.streamActivatedJobsStream(streamReq);
|
|
258
|
+
}
|
|
259
|
+
catch (e) {
|
|
260
|
+
return { error: e };
|
|
261
|
+
}
|
|
262
|
+
})();
|
|
263
|
+
const openError = (0, StreamLifecycle_1.getOpenError)(result);
|
|
264
|
+
if (openError) {
|
|
265
|
+
return openError;
|
|
266
|
+
}
|
|
267
|
+
const stream = (0, StreamLifecycle_1.asStream)(result);
|
|
268
|
+
if (isClosed()) {
|
|
269
|
+
// close() was called while the stream was being opened.
|
|
270
|
+
(0, StreamLifecycle_1.cancelStream)(stream);
|
|
271
|
+
return undefined;
|
|
272
|
+
}
|
|
273
|
+
const openedAt = Date.now();
|
|
274
|
+
let receivedData = false;
|
|
275
|
+
let cause = { kind: 'end' };
|
|
276
|
+
// A stream that delivered a job, or stayed up for a while, was
|
|
277
|
+
// healthy: restart the backoff from the beginning.
|
|
278
|
+
const resetBackoffIfHealthy = () => {
|
|
279
|
+
if (receivedData || Date.now() - openedAt >= maxDelayMs) {
|
|
280
|
+
consecutiveFailures = 0;
|
|
281
|
+
}
|
|
282
|
+
};
|
|
283
|
+
// The adapter is the only code that attaches listeners to the raw
|
|
284
|
+
// stream. It delivers exactly one of `error`/`end`, then `close`.
|
|
285
|
+
const attachment = (0, StreamLifecycle_1.attachStream)(stream, {
|
|
286
|
+
data: (res) => {
|
|
287
|
+
if (!receivedData) {
|
|
288
|
+
receivedData = true;
|
|
289
|
+
consecutiveFailures = 0;
|
|
290
|
+
}
|
|
291
|
+
try {
|
|
292
|
+
const parsedJob = (0, _1.parseVariablesAndCustomHeadersToJSON)(res, inputVariableDto, customHeadersDto);
|
|
293
|
+
void handleJob(parsedJob);
|
|
294
|
+
}
|
|
295
|
+
catch (e) {
|
|
296
|
+
this.zbClient
|
|
297
|
+
.failJob({
|
|
298
|
+
jobKey: res.key,
|
|
299
|
+
errorMessage: `Error parsing variable payload ${e}`,
|
|
300
|
+
retries: res.retries - 1,
|
|
301
|
+
retryBackOff: 0,
|
|
302
|
+
})
|
|
303
|
+
.catch((failError) => this.logger.logError(`Failed to fail job ${res.key} after a payload parse error: ${(0, StreamLifecycle_1.toError)(failError).message}`));
|
|
304
|
+
}
|
|
305
|
+
},
|
|
306
|
+
error: (error) => {
|
|
307
|
+
cause = { kind: 'error', error };
|
|
308
|
+
if (isClosed()) {
|
|
309
|
+
// Expected: we cancelled it ourselves.
|
|
310
|
+
return;
|
|
311
|
+
}
|
|
312
|
+
resetBackoffIfHealthy();
|
|
313
|
+
this.logger.logError(`Job stream for ${req.type} failed: ${error.message}`);
|
|
314
|
+
notify('onStreamError', () => onStreamError?.(error));
|
|
315
|
+
},
|
|
316
|
+
end: () => {
|
|
317
|
+
cause = { kind: 'end' };
|
|
318
|
+
if (isClosed()) {
|
|
319
|
+
return;
|
|
320
|
+
}
|
|
321
|
+
resetBackoffIfHealthy();
|
|
322
|
+
this.logger.logError(`Job stream for ${req.type} ended`);
|
|
323
|
+
notify('onStreamEnd', () => onStreamEnd?.());
|
|
324
|
+
},
|
|
325
|
+
close: () => {
|
|
326
|
+
discardStream(attachment);
|
|
327
|
+
if (currentStream === attachment) {
|
|
328
|
+
currentStream = undefined;
|
|
329
|
+
}
|
|
330
|
+
onStreamDown(cause);
|
|
331
|
+
},
|
|
37
332
|
});
|
|
333
|
+
this.streams.push(attachment);
|
|
334
|
+
currentStream = attachment;
|
|
335
|
+
transition({ status: 'open' });
|
|
336
|
+
return undefined;
|
|
337
|
+
};
|
|
338
|
+
// Re-open a dead stream, then backfill: jobs created while the stream
|
|
339
|
+
// was down were never delivered to it. The broker guarantees single
|
|
340
|
+
// activation, so there is no risk of duplicate delivery.
|
|
341
|
+
const reconnect = async () => {
|
|
342
|
+
const openError = await openStream();
|
|
343
|
+
if (isClosed()) {
|
|
344
|
+
// close() was called while the stream was being opened: a failed
|
|
345
|
+
// open is expected fallout, so neither report it nor retry.
|
|
346
|
+
return;
|
|
347
|
+
}
|
|
348
|
+
if (openError) {
|
|
349
|
+
return handleOpenFailure(openError);
|
|
350
|
+
}
|
|
351
|
+
await backfill().catch((e) => this.logger.logError(`Backfill poll after job stream reconnect failed: ${(0, StreamLifecycle_1.toError)(e).message}`));
|
|
38
352
|
};
|
|
353
|
+
// The handle returned to the caller. `close()` is idempotent.
|
|
354
|
+
const closeHandle = () => {
|
|
355
|
+
handleClosed = true;
|
|
356
|
+
transition({ status: 'closed', reason: { kind: 'closed-by-caller' } });
|
|
357
|
+
if (sidecarTimer) {
|
|
358
|
+
clearTimeout(sidecarTimer);
|
|
359
|
+
}
|
|
360
|
+
if (reconnectTimer) {
|
|
361
|
+
clearTimeout(reconnectTimer);
|
|
362
|
+
this.reconnectTimers.delete(reconnectTimer);
|
|
363
|
+
reconnectTimer = undefined;
|
|
364
|
+
}
|
|
365
|
+
if (currentStream) {
|
|
366
|
+
discardStream(currentStream);
|
|
367
|
+
currentStream.dispose();
|
|
368
|
+
currentStream = undefined;
|
|
369
|
+
}
|
|
370
|
+
};
|
|
371
|
+
const handle = (0, StreamLifecycle_1.createStreamHandle)(machine, closeHandle);
|
|
39
372
|
const pollAndStream = async () => {
|
|
373
|
+
if (this.closed) {
|
|
374
|
+
// The client was already closed: open nothing, and give the caller a
|
|
375
|
+
// handle that says so, instead of one that reports `open` forever.
|
|
376
|
+
transition({ status: 'closed', reason: { kind: 'client-closed' } });
|
|
377
|
+
return handle;
|
|
378
|
+
}
|
|
40
379
|
// Optional jitter to stagger multiple workers
|
|
41
380
|
if (jitter) {
|
|
42
381
|
await new Promise((resolve) => setTimeout(resolve, Math.random() * jitter));
|
|
43
382
|
}
|
|
44
383
|
// Open the stream first so no new jobs are missed
|
|
45
|
-
const
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
catch (e) {
|
|
55
|
-
this.zbClient.failJob({
|
|
56
|
-
jobKey: res.key,
|
|
57
|
-
errorMessage: `Error parsing variable payload ${e}`,
|
|
58
|
-
retries: res.retries - 1,
|
|
59
|
-
retryBackOff: 0,
|
|
60
|
-
});
|
|
61
|
-
}
|
|
62
|
-
});
|
|
63
|
-
this.streams.push(stream);
|
|
64
|
-
// Helper to run a single poll cycle and feed results to handleJob.
|
|
65
|
-
const runPoll = async () => {
|
|
66
|
-
const jobs = await this.zbClient.activateJobs({
|
|
67
|
-
type: req.type,
|
|
68
|
-
worker: req.worker,
|
|
69
|
-
timeout: req.timeout,
|
|
70
|
-
maxJobsToActivate: pollMaxJobsToActivate,
|
|
71
|
-
tenantIds: req.tenantIds,
|
|
72
|
-
fetchVariable: req.fetchVariable,
|
|
73
|
-
inputVariableDto,
|
|
74
|
-
customHeadersDto,
|
|
75
|
-
requestTimeout: -1,
|
|
76
|
-
});
|
|
77
|
-
for (const job of jobs) {
|
|
78
|
-
handleJob(job);
|
|
384
|
+
const openError = await openStream();
|
|
385
|
+
if (isClosed()) {
|
|
386
|
+
// close() was called while the stream was being opened.
|
|
387
|
+
return handle;
|
|
388
|
+
}
|
|
389
|
+
if (openError) {
|
|
390
|
+
handleOpenFailure(openError);
|
|
391
|
+
if (!reconnectEnabled) {
|
|
392
|
+
throw openError;
|
|
79
393
|
}
|
|
80
|
-
}
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
394
|
+
}
|
|
395
|
+
else {
|
|
396
|
+
// Initial backfill poll: pick up jobs that existed before the stream.
|
|
397
|
+
// The broker guarantees single activation, so there is no risk of
|
|
398
|
+
// duplicate delivery between the poll and the stream.
|
|
399
|
+
await backfill();
|
|
400
|
+
}
|
|
85
401
|
// Recurring sidecar poll: a low-frequency safety net that catches
|
|
86
402
|
// jobs the stream may have missed (e.g. jobs re-queued after a
|
|
87
|
-
// timeout or
|
|
88
|
-
// chain so each cycle waits for the previous poll to
|
|
89
|
-
// scheduling the next, preventing overlapping polls.
|
|
403
|
+
// timeout or while the stream is down and being re-opened). Uses a
|
|
404
|
+
// setTimeout chain so each cycle waits for the previous poll to
|
|
405
|
+
// finish before scheduling the next, preventing overlapping polls.
|
|
90
406
|
//
|
|
91
407
|
// The live timer is tracked in `this.pollTimers` so that `close()`
|
|
92
408
|
// can cancel it. Each cycle removes the timer that just fired
|
|
93
409
|
// before scheduling the next one, so the array holds at most one
|
|
94
410
|
// live timer per stream rather than growing by one per poll cycle.
|
|
95
|
-
let sidecarTimer;
|
|
96
411
|
if (pollInterval > 0) {
|
|
97
412
|
const schedulePoll = () => {
|
|
98
413
|
// A poll in flight when close() is called must not schedule another.
|
|
99
|
-
if (
|
|
414
|
+
if (isClosed()) {
|
|
100
415
|
return;
|
|
101
416
|
}
|
|
102
417
|
sidecarTimer = setTimeout(() => {
|
|
@@ -107,10 +422,22 @@ class ZBStreamWorker {
|
|
|
107
422
|
if (index !== -1) {
|
|
108
423
|
this.pollTimers.splice(index, 1);
|
|
109
424
|
}
|
|
110
|
-
|
|
111
|
-
|
|
425
|
+
// An activation already in flight is doing this poll's job.
|
|
426
|
+
if (activationReserved || isClosed()) {
|
|
427
|
+
schedulePoll();
|
|
428
|
+
return;
|
|
429
|
+
}
|
|
430
|
+
// Reserve before activating: a reconnect backfill arriving
|
|
431
|
+
// while this activation is pending must not start a second,
|
|
432
|
+
// overlapping drain. The reservation is released by
|
|
433
|
+
// runDrain's finally, or here if the activation fails.
|
|
434
|
+
activationReserved = true;
|
|
435
|
+
activate()
|
|
436
|
+
.then(runDrain, (e) => {
|
|
437
|
+
activationReserved = false;
|
|
112
438
|
// Swallow errors — the stream is the primary channel.
|
|
113
439
|
// The next poll cycle will retry.
|
|
440
|
+
this.logger.logDebug(`Stream worker sidecar poll failed: ${(0, StreamLifecycle_1.toError)(e).message}`);
|
|
114
441
|
})
|
|
115
442
|
.finally(schedulePoll);
|
|
116
443
|
}, pollInterval);
|
|
@@ -118,15 +445,7 @@ class ZBStreamWorker {
|
|
|
118
445
|
};
|
|
119
446
|
schedulePoll();
|
|
120
447
|
}
|
|
121
|
-
return
|
|
122
|
-
close: () => {
|
|
123
|
-
if (sidecarTimer) {
|
|
124
|
-
clearTimeout(sidecarTimer);
|
|
125
|
-
}
|
|
126
|
-
stream.cancel();
|
|
127
|
-
stream.destroy();
|
|
128
|
-
},
|
|
129
|
-
};
|
|
448
|
+
return handle;
|
|
130
449
|
};
|
|
131
450
|
return pollAndStream();
|
|
132
451
|
}
|
|
@@ -134,10 +453,12 @@ class ZBStreamWorker {
|
|
|
134
453
|
this.closed = true;
|
|
135
454
|
this.pollTimers.forEach((t) => clearTimeout(t));
|
|
136
455
|
this.pollTimers = [];
|
|
137
|
-
this.
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
});
|
|
456
|
+
this.reconnectTimers.forEach((t) => clearTimeout(t));
|
|
457
|
+
this.reconnectTimers.clear();
|
|
458
|
+
const clientClosed = { kind: 'client-closed' };
|
|
459
|
+
this.machines.forEach((m) => m.transition({ status: 'closed', reason: clientClosed }));
|
|
460
|
+
this.streams.forEach((s) => s.dispose());
|
|
461
|
+
this.streams = [];
|
|
141
462
|
return this.grpcClient.close();
|
|
142
463
|
}
|
|
143
464
|
log(msg) {
|
|
@@ -158,9 +479,8 @@ class ZBStreamWorker {
|
|
|
158
479
|
const errorMsgOnPriorMessageCall = (thisMethod, wrappedFunction) => {
|
|
159
480
|
return (...args) => {
|
|
160
481
|
if (methodCalled !== undefined) {
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
You should call only one job action method in the worker handler. This is a bug in a Stream worker handler.`));
|
|
482
|
+
this.logger.logError(`WARNING: Call to ${thisMethod}() after ${methodCalled}() was called.
|
|
483
|
+
You should call only one job action method in the worker handler. This is a bug in a Stream worker handler.`);
|
|
164
484
|
return wrappedFunction(...args);
|
|
165
485
|
}
|
|
166
486
|
methodCalled = thisMethod;
|