notifkit 0.1.9 → 0.1.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +66 -72
- package/dashboard/dist/assets/{index-7mvXb4bS.js → index-Bsrmo9LE.js} +1 -1
- package/dashboard/dist/assets/index-s-SXGIwv.css +1 -0
- package/dashboard/dist/index.html +2 -2
- package/dist/bootstrap-GRXzgPvW.mjs +89 -0
- package/dist/bulk-BvRfz6Of.mjs +77 -0
- package/dist/index.d.mts +1107 -8885
- package/dist/index.mjs +2 -38
- package/dist/{main-DSmh1e-V.mjs → main-BENOY7HG.mjs} +49 -75
- package/dist/{main-BPbPnBIG.mjs → main-BpnuFX62.mjs} +119 -112
- package/dist/{main-4Wyge9ms.mjs → main-BqVH2BHj.mjs} +57 -54
- package/dist/{main-D19Wxs0x.mjs → main-C_5VFIwp.mjs} +135 -79
- package/dist/main-D6od9N8h.mjs +312 -0
- package/dist/main-D84OX0ux.mjs +811 -0
- package/dist/{main-JdOvJ70b.mjs → main-HH-WwsMM.mjs} +395 -184
- package/dist/{main-AO0JaY5z.mjs → main-HY_TDs61.mjs} +149 -134
- package/dist/{src-CGmXRfsM.mjs → src-CI7FGf3h.mjs} +1223 -651
- package/drizzle/0006_message_logs_redundant_indexes.sql +7 -0
- package/drizzle/0007_workflow_status_canceled.sql +3 -0
- package/drizzle/meta/_journal.json +14 -0
- package/package.json +17 -17
- package/scripts/create-admin.mjs +8 -11
- package/src/client.ts +15 -5
- package/src/config/index.ts +26 -3
- package/src/contracts/events/notification-ai-pending.ts +12 -0
- package/src/contracts/events/notification-delivered.ts +7 -0
- package/src/contracts/events/notification-dispatched.ts +21 -2
- package/src/contracts/events/notification-enriched.ts +16 -0
- package/src/contracts/events/notification-failed.ts +2 -0
- package/src/contracts/events/notification-requested.ts +2 -0
- package/src/contracts/registry.ts +2 -2
- package/src/contracts/sdk.ts +77 -29
- package/src/db/bulk.ts +132 -0
- package/src/db/index.ts +2 -2
- package/src/db/schema.ts +3 -50
- package/src/idempotency/index.ts +60 -6
- package/src/logger/index.ts +0 -12
- package/src/queue/index.ts +255 -76
- package/src/rate-limiter/index.ts +217 -16
- package/src/redis/index.ts +265 -0
- package/src/repositories/index.ts +171 -82
- package/src/server.ts +71 -41
- package/src/services/ai/main.ts +83 -70
- package/src/services/api/admin-static.ts +41 -12
- package/src/services/api/handlers.ts +357 -159
- package/src/services/api/http.ts +34 -7
- package/src/services/api/main.ts +104 -67
- package/src/services/api/router.ts +18 -5
- package/src/services/delivery/main.ts +495 -230
- package/src/services/delivery/throttle.ts +42 -59
- package/src/services/engine/main.ts +206 -152
- package/src/services/enricher/main.ts +155 -107
- package/src/services/events/main.ts +100 -55
- package/src/services/scheduler/main.ts +151 -79
- package/src/services/scheduler/send-gate.ts +117 -0
- package/src/services/workflow/main.ts +220 -100
- package/src/shared/batch-processor.ts +65 -34
- package/src/shared/dataloader.ts +58 -22
- package/src/shared/events.ts +3 -0
- package/src/shared/index.ts +0 -1
- package/src/shared/utils.ts +2 -47
- package/src/templates/index.ts +0 -67
- package/src/transport/index.ts +6 -0
- package/src/workers/bootstrap.ts +124 -0
- package/src/workers/index.ts +99 -24
- package/src/workflows/sdk.ts +2 -0
- package/dashboard/dist/assets/index-o7L-f3-c.css +0 -1
- package/dist/index.d.mts.map +0 -1
- package/dist/index.mjs.map +0 -1
- package/dist/main-4Wyge9ms.mjs.map +0 -1
- package/dist/main-AO0JaY5z.mjs.map +0 -1
- package/dist/main-BMYSYcv2.mjs +0 -224
- package/dist/main-BMYSYcv2.mjs.map +0 -1
- package/dist/main-BPbPnBIG.mjs.map +0 -1
- package/dist/main-CyeQ_yDt.mjs +0 -632
- package/dist/main-CyeQ_yDt.mjs.map +0 -1
- package/dist/main-D19Wxs0x.mjs.map +0 -1
- package/dist/main-DSmh1e-V.mjs.map +0 -1
- package/dist/main-JdOvJ70b.mjs.map +0 -1
- package/dist/src-CGmXRfsM.mjs.map +0 -1
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { K as StreamProducer, Ut as buildStreamEvent, Wt as CONSUMER_GROUPS, Yt as STREAMS, a as SuspendExecutionError, c as BaseWorker, cn as loadEnv, ct as workflowInstances, d as startHealthReporter, i as workflowRegistry, lt as workflowSteps, o as buildStepNotifyPayload, st as workflowDefinitions, un as readBaseConfig, ut as workflowWaiters } from "./src-CI7FGf3h.mjs";
|
|
2
|
+
import { a as uniqueConsumerId, i as shutdownWorker, n as createStreamConsumer, r as createWorkerRuntime } from "./bootstrap-GRXzgPvW.mjs";
|
|
2
3
|
import { randomUUID } from "node:crypto";
|
|
3
4
|
import { and, eq } from "drizzle-orm";
|
|
4
5
|
//#region src/services/workflow/main.ts
|
|
@@ -8,6 +9,40 @@ const WORKFLOW_LOCK_TTL_SECONDS = 60;
|
|
|
8
9
|
const WORKFLOW_LOCK_RENEW_MS = WORKFLOW_LOCK_TTL_SECONDS / 3 * 1e3;
|
|
9
10
|
/** How long a claimed workflow timer stays invisible to other pollers. */
|
|
10
11
|
const WORKFLOW_TIMER_VISIBILITY_MS = 6e4;
|
|
12
|
+
/** Wake-ups for sleeping instances: waits, event timeouts, deferred resumes. */
|
|
13
|
+
const WORKFLOW_TIMERS_KEY = "notif:workflow:timers";
|
|
14
|
+
/** How soon a resume that found its instance busy is tried again. */
|
|
15
|
+
const WORKFLOW_RESUME_RETRY_MS = 2e3;
|
|
16
|
+
const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
|
17
|
+
/** Seconds per unit. Everything is read as seconds first, then scheduled in ms. */
|
|
18
|
+
const DURATION_UNIT_SECONDS = {
|
|
19
|
+
ms: .001,
|
|
20
|
+
s: 1,
|
|
21
|
+
m: 60,
|
|
22
|
+
h: 3600,
|
|
23
|
+
d: 86400
|
|
24
|
+
};
|
|
25
|
+
/**
|
|
26
|
+
* `"500ms"`, `"0.5s"`, `"15m"`, `"1.5h"`, `"2.5d"` to seconds. Decimals are
|
|
27
|
+
* read exactly; anything that is not a number and a unit throws. The old
|
|
28
|
+
* parser matched on the last character, so "500ms" meant 500 seconds, and ran
|
|
29
|
+
* parseInt, so "1.5h" meant one hour.
|
|
30
|
+
*/
|
|
31
|
+
function parseDurationSeconds(value, what) {
|
|
32
|
+
const match = /^(\d+(?:\.\d+)?|\.\d+)(ms|s|m|h|d)$/.exec(value.trim());
|
|
33
|
+
if (!match) throw new Error(`Invalid ${what}: ${value}`);
|
|
34
|
+
return Number(match[1]) * DURATION_UNIT_SECONDS[match[2]];
|
|
35
|
+
}
|
|
36
|
+
function durationToMs(value, what) {
|
|
37
|
+
return Math.round(parseDurationSeconds(value, what) * 1e3);
|
|
38
|
+
}
|
|
39
|
+
/** Thrown at a step boundary once the instance has been canceled. */
|
|
40
|
+
var WorkflowCanceledError = class extends Error {
|
|
41
|
+
constructor() {
|
|
42
|
+
super("Workflow instance was canceled");
|
|
43
|
+
this.name = "WorkflowCanceledError";
|
|
44
|
+
}
|
|
45
|
+
};
|
|
11
46
|
loadEnv();
|
|
12
47
|
const config = readBaseConfig();
|
|
13
48
|
let logger;
|
|
@@ -38,12 +73,12 @@ var WorkflowWorker = class extends BaseWorker {
|
|
|
38
73
|
this.flushTimer = setInterval(() => void this.flushWorkerBuffers(), 100);
|
|
39
74
|
}
|
|
40
75
|
async stop() {
|
|
76
|
+
await super.stop();
|
|
41
77
|
if (this.flushTimer) {
|
|
42
78
|
clearInterval(this.flushTimer);
|
|
43
79
|
this.flushTimer = null;
|
|
44
80
|
}
|
|
45
81
|
await this.flushWorkerBuffers();
|
|
46
|
-
await super.stop();
|
|
47
82
|
}
|
|
48
83
|
async flushWorkerBuffers() {
|
|
49
84
|
if (this.eventBuffer.length === 0) return;
|
|
@@ -71,6 +106,24 @@ var WorkflowWorker = class extends BaseWorker {
|
|
|
71
106
|
for (const e of events) e.reject(err);
|
|
72
107
|
}
|
|
73
108
|
}
|
|
109
|
+
/**
|
|
110
|
+
* Re-delivers a resume through the timer ZSET a few seconds from now. The
|
|
111
|
+
* poll loop publishes it once the instance is pending again, and drops it if
|
|
112
|
+
* the instance has finished. The member carries no timestamp, so repeated
|
|
113
|
+
* deferrals of one instance collapse into a single entry. It is marked
|
|
114
|
+
* `deferred` so it never equals a `wait` timer's member: sharing one let the
|
|
115
|
+
* ZADD here re-score a sleep's wake-up to seconds from now, and the early
|
|
116
|
+
* resume then re-suspended with no timer left to wake it.
|
|
117
|
+
*/
|
|
118
|
+
async deferResume(payload) {
|
|
119
|
+
await this.redisCli.zadd(WORKFLOW_TIMERS_KEY, Date.now() + WORKFLOW_RESUME_RETRY_MS, JSON.stringify({
|
|
120
|
+
instanceId: payload.instanceId,
|
|
121
|
+
name: payload.name,
|
|
122
|
+
projectId: payload.projectId,
|
|
123
|
+
input: payload.input ?? {},
|
|
124
|
+
deferred: true
|
|
125
|
+
}));
|
|
126
|
+
}
|
|
74
127
|
async process(message) {
|
|
75
128
|
const { event } = message;
|
|
76
129
|
const publishPromises = [];
|
|
@@ -107,11 +160,12 @@ var WorkflowWorker = class extends BaseWorker {
|
|
|
107
160
|
const lockKey = `lock:workflow:${instanceId}`;
|
|
108
161
|
const lockToken = randomUUID();
|
|
109
162
|
if (!await this.redisCli.set(lockKey, lockToken, "EX", WORKFLOW_LOCK_TTL_SECONDS, "NX")) {
|
|
110
|
-
this.logger.info({ instanceId }, "Workflow is locked by another process,
|
|
163
|
+
this.logger.info({ instanceId }, "Workflow is locked by another process, deferring resume");
|
|
164
|
+
await this.deferResume(payload);
|
|
111
165
|
return;
|
|
112
166
|
}
|
|
113
167
|
const renewTimer = setInterval(() => {
|
|
114
|
-
this.redisCli.
|
|
168
|
+
this.redisCli.renewLock(lockKey, lockToken, WORKFLOW_LOCK_TTL_SECONDS).catch((err) => {
|
|
115
169
|
this.logger.warn({
|
|
116
170
|
err,
|
|
117
171
|
instanceId
|
|
@@ -127,11 +181,22 @@ var WorkflowWorker = class extends BaseWorker {
|
|
|
127
181
|
status: "pending",
|
|
128
182
|
input: payload.input || {}
|
|
129
183
|
}).returning())[0];
|
|
184
|
+
if (instance.status === "running") {
|
|
185
|
+
await this.deferResume(payload);
|
|
186
|
+
return;
|
|
187
|
+
}
|
|
130
188
|
if (instance.status !== "pending") {
|
|
131
189
|
this.logger.info({ instanceId }, "Workflow is not pending, skipping");
|
|
132
190
|
return;
|
|
133
191
|
}
|
|
134
|
-
await this.dbConn.update(workflowInstances).set({ status: "running" }).where(eq(workflowInstances.id, instanceId))
|
|
192
|
+
if ((await this.dbConn.update(workflowInstances).set({ status: "running" }).where(and(eq(workflowInstances.id, instanceId), eq(workflowInstances.status, "pending"))).returning({ id: workflowInstances.id })).length === 0) {
|
|
193
|
+
this.logger.info({ instanceId }, "Workflow is no longer pending, skipping");
|
|
194
|
+
return;
|
|
195
|
+
}
|
|
196
|
+
const stopIfCanceled = async () => {
|
|
197
|
+
const [row] = await this.dbConn.select({ status: workflowInstances.status }).from(workflowInstances).where(eq(workflowInstances.id, instanceId)).limit(1);
|
|
198
|
+
if (row?.status === "canceled") throw new WorkflowCanceledError();
|
|
199
|
+
};
|
|
135
200
|
const existingSteps = await this.dbConn.select().from(workflowSteps).where(eq(workflowSteps.instanceId, instanceId));
|
|
136
201
|
const stepOutputMap = /* @__PURE__ */ new Map();
|
|
137
202
|
for (const s of existingSteps) stepOutputMap.set(s.stepIndex, s.output);
|
|
@@ -141,7 +206,8 @@ var WorkflowWorker = class extends BaseWorker {
|
|
|
141
206
|
notify: async (args) => {
|
|
142
207
|
const stepId = String(currentStepIndex++);
|
|
143
208
|
if (stepOutputMap.has(stepId)) return stepOutputMap.get(stepId);
|
|
144
|
-
|
|
209
|
+
await stopIfCanceled();
|
|
210
|
+
const requested = buildStepNotifyPayload(args, instance.input, payload.projectId, `wf-${instanceId}-${stepId}`, UUID_RE.test(instanceId) ? instanceId : void 0);
|
|
145
211
|
const result = await new Promise((resolve, reject) => {
|
|
146
212
|
this.eventBuffer.push({
|
|
147
213
|
producer: this.notificationProducer,
|
|
@@ -166,15 +232,15 @@ var WorkflowWorker = class extends BaseWorker {
|
|
|
166
232
|
},
|
|
167
233
|
wait: async (duration) => {
|
|
168
234
|
const stepId = String(currentStepIndex++);
|
|
169
|
-
if (stepOutputMap.has(stepId))
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
235
|
+
if (stepOutputMap.has(stepId)) {
|
|
236
|
+
const until = Number(stepOutputMap.get(stepId)?.scheduledAt);
|
|
237
|
+
if (Number.isFinite(until) && until > Date.now()) throw new SuspendExecutionError("wait", { duration });
|
|
238
|
+
return;
|
|
239
|
+
}
|
|
240
|
+
await stopIfCanceled();
|
|
241
|
+
const ms = durationToMs(duration, "wait duration");
|
|
176
242
|
const resumeAt = Date.now() + ms;
|
|
177
|
-
await this.redisCli.zadd(
|
|
243
|
+
await this.redisCli.zadd(WORKFLOW_TIMERS_KEY, resumeAt, JSON.stringify({
|
|
178
244
|
instanceId: instance.id,
|
|
179
245
|
name: instance.name,
|
|
180
246
|
projectId: payload.projectId,
|
|
@@ -193,20 +259,17 @@ var WorkflowWorker = class extends BaseWorker {
|
|
|
193
259
|
const stepId = String(currentStepIndex++);
|
|
194
260
|
if (stepOutputMap.has(stepId)) {
|
|
195
261
|
const out = stepOutputMap.get(stepId);
|
|
196
|
-
if (out
|
|
262
|
+
if (out === null || out === void 0) throw new SuspendExecutionError("waitForEvent", { eventName });
|
|
263
|
+
if (typeof out === "object" && out.timedOut === true) return null;
|
|
197
264
|
return out;
|
|
198
265
|
}
|
|
266
|
+
await stopIfCanceled();
|
|
199
267
|
options = options || {};
|
|
200
268
|
options.timeout = options.timeout || "24h";
|
|
201
269
|
options.match = options.match || {};
|
|
202
|
-
|
|
203
|
-
if (options.timeout.endsWith("d")) ms = parseInt(options.timeout) * 24 * 60 * 60 * 1e3;
|
|
204
|
-
else if (options.timeout.endsWith("h")) ms = parseInt(options.timeout) * 60 * 60 * 1e3;
|
|
205
|
-
else if (options.timeout.endsWith("m")) ms = parseInt(options.timeout) * 60 * 1e3;
|
|
206
|
-
else if (options.timeout.endsWith("s")) ms = parseInt(options.timeout) * 1e3;
|
|
207
|
-
else throw new Error(`Invalid waitForEvent timeout: ${options.timeout}`);
|
|
270
|
+
const ms = durationToMs(options.timeout, "waitForEvent timeout");
|
|
208
271
|
const resumeAt = Date.now() + ms;
|
|
209
|
-
await this.redisCli.zadd(
|
|
272
|
+
await this.redisCli.zadd(WORKFLOW_TIMERS_KEY, resumeAt, JSON.stringify({
|
|
210
273
|
instanceId: instance.id,
|
|
211
274
|
name: instance.name,
|
|
212
275
|
projectId: payload.projectId,
|
|
@@ -215,13 +278,6 @@ var WorkflowWorker = class extends BaseWorker {
|
|
|
215
278
|
eventName,
|
|
216
279
|
stepId
|
|
217
280
|
}));
|
|
218
|
-
await this.dbConn.insert(workflowWaiters).values({
|
|
219
|
-
instanceId: instance.id,
|
|
220
|
-
projectId: payload.projectId,
|
|
221
|
-
eventName,
|
|
222
|
-
matchCriteria: options.match,
|
|
223
|
-
expiresAt: new Date(resumeAt)
|
|
224
|
-
});
|
|
225
281
|
await this.dbConn.insert(workflowSteps).values({
|
|
226
282
|
instanceId: instance.id,
|
|
227
283
|
projectId: payload.projectId,
|
|
@@ -229,11 +285,19 @@ var WorkflowWorker = class extends BaseWorker {
|
|
|
229
285
|
action: "waitForEvent",
|
|
230
286
|
output: null
|
|
231
287
|
});
|
|
288
|
+
await this.dbConn.insert(workflowWaiters).values({
|
|
289
|
+
instanceId: instance.id,
|
|
290
|
+
projectId: payload.projectId,
|
|
291
|
+
eventName,
|
|
292
|
+
matchCriteria: options.match,
|
|
293
|
+
expiresAt: new Date(resumeAt)
|
|
294
|
+
});
|
|
232
295
|
throw new SuspendExecutionError("waitForEvent", { eventName });
|
|
233
296
|
},
|
|
234
297
|
run: async (stepName, fn) => {
|
|
235
298
|
const stepId = String(currentStepIndex++);
|
|
236
299
|
if (stepOutputMap.has(stepId)) return stepOutputMap.get(stepId);
|
|
300
|
+
await stopIfCanceled();
|
|
237
301
|
const result = await fn();
|
|
238
302
|
await this.dbConn.insert(workflowSteps).values({
|
|
239
303
|
instanceId: instance.id,
|
|
@@ -247,13 +311,15 @@ var WorkflowWorker = class extends BaseWorker {
|
|
|
247
311
|
},
|
|
248
312
|
event: instance.input || { user: { id: "unknown" } }
|
|
249
313
|
};
|
|
314
|
+
const finish = (status) => this.dbConn.update(workflowInstances).set({ status }).where(and(eq(workflowInstances.id, instance.id), eq(workflowInstances.status, "running")));
|
|
250
315
|
try {
|
|
251
316
|
await handler(ctx);
|
|
252
|
-
await
|
|
317
|
+
await finish("completed");
|
|
253
318
|
this.logger.info({ instanceId: instance.id }, "Workflow completed successfully");
|
|
254
319
|
} catch (err) {
|
|
255
|
-
if (err instanceof
|
|
256
|
-
|
|
320
|
+
if (err instanceof WorkflowCanceledError) this.logger.info({ instanceId: instance.id }, "Workflow canceled mid-run, stopping");
|
|
321
|
+
else if (err instanceof SuspendExecutionError || err.name === "SuspendExecutionError") {
|
|
322
|
+
await finish("pending");
|
|
257
323
|
this.logger.info({
|
|
258
324
|
instanceId: instance.id,
|
|
259
325
|
reason: err.reason
|
|
@@ -263,33 +329,18 @@ var WorkflowWorker = class extends BaseWorker {
|
|
|
263
329
|
err,
|
|
264
330
|
instanceId: instance.id
|
|
265
331
|
}, "Workflow failed");
|
|
266
|
-
await
|
|
332
|
+
await finish("failed");
|
|
267
333
|
}
|
|
268
334
|
}
|
|
269
335
|
} finally {
|
|
270
336
|
clearInterval(renewTimer);
|
|
271
|
-
await this.redisCli.
|
|
337
|
+
await this.redisCli.releaseLock(lockKey, lockToken);
|
|
272
338
|
}
|
|
273
339
|
await Promise.all(publishPromises);
|
|
274
340
|
}
|
|
275
341
|
};
|
|
276
342
|
async function startWorkflowWorker() {
|
|
277
|
-
logger =
|
|
278
|
-
name: "workflow-worker",
|
|
279
|
-
level: config.LOG_LEVEL
|
|
280
|
-
});
|
|
281
|
-
redis = new RedisClient({
|
|
282
|
-
url: config.REDIS_URL,
|
|
283
|
-
name: "workflow",
|
|
284
|
-
logger
|
|
285
|
-
});
|
|
286
|
-
const dbData = createDatabase({
|
|
287
|
-
url: config.DATABASE_URL,
|
|
288
|
-
applicationName: "workflow",
|
|
289
|
-
logger
|
|
290
|
-
});
|
|
291
|
-
sql$1 = dbData.sql;
|
|
292
|
-
db = dbData.db;
|
|
343
|
+
({logger, redis, sql: sql$1, db} = createWorkerRuntime(config, "workflow", "workflow-worker"));
|
|
293
344
|
notificationProducer = new StreamProducer({
|
|
294
345
|
redis: redis.native,
|
|
295
346
|
stream: STREAMS.INBOUND_NORMAL,
|
|
@@ -300,22 +351,15 @@ async function startWorkflowWorker() {
|
|
|
300
351
|
stream: STREAMS.WORKFLOW_INBOUND,
|
|
301
352
|
logger
|
|
302
353
|
});
|
|
303
|
-
consumer =
|
|
304
|
-
redis
|
|
305
|
-
stream: STREAMS.WORKFLOW_INBOUND,
|
|
306
|
-
group: CONSUMER_GROUPS.WORKFLOW,
|
|
307
|
-
consumer: `workflow-${process.pid}`,
|
|
308
|
-
dlqStream: STREAMS.DEAD_LETTER,
|
|
309
|
-
batchSize: config.WORKER_CONCURRENCY,
|
|
354
|
+
({consumer, pendingScanner} = createStreamConsumer({
|
|
355
|
+
redis,
|
|
310
356
|
logger
|
|
311
|
-
}
|
|
312
|
-
pendingScanner = new PendingMessageScanner({
|
|
313
|
-
redis: redis.native,
|
|
357
|
+
}, {
|
|
314
358
|
stream: STREAMS.WORKFLOW_INBOUND,
|
|
315
359
|
group: CONSUMER_GROUPS.WORKFLOW,
|
|
316
|
-
consumer:
|
|
317
|
-
|
|
318
|
-
});
|
|
360
|
+
consumer: uniqueConsumerId("workflow"),
|
|
361
|
+
batchSize: config.WORKER_CONCURRENCY
|
|
362
|
+
}));
|
|
319
363
|
worker = new WorkflowWorker({
|
|
320
364
|
consumer,
|
|
321
365
|
pendingScanner,
|
|
@@ -330,25 +374,33 @@ async function startWorkflowWorker() {
|
|
|
330
374
|
(async () => {
|
|
331
375
|
try {
|
|
332
376
|
const now = Date.now();
|
|
333
|
-
const tasks = await redis.native.
|
|
377
|
+
const tasks = await redis.native.schedulerPoll(WORKFLOW_TIMERS_KEY, now, 100, WORKFLOW_TIMER_VISIBILITY_MS);
|
|
334
378
|
for (const taskStr of tasks) {
|
|
335
379
|
const task = JSON.parse(taskStr);
|
|
336
380
|
const inst = (await db.select().from(workflowInstances).where(eq(workflowInstances.id, task.instanceId)).limit(1))[0];
|
|
337
|
-
if (!inst || inst.status === "completed" || inst.status === "failed") {
|
|
338
|
-
await redis.native.zrem(
|
|
381
|
+
if (!inst || inst.status === "completed" || inst.status === "failed" || inst.status === "canceled") {
|
|
382
|
+
await redis.native.zrem(WORKFLOW_TIMERS_KEY, taskStr);
|
|
339
383
|
continue;
|
|
340
384
|
}
|
|
341
385
|
if (inst.status === "running") continue;
|
|
342
386
|
if (task.isEventTimeout) {
|
|
387
|
+
if (task.stepId !== void 0) {
|
|
388
|
+
const [own] = await db.select({ output: workflowSteps.output }).from(workflowSteps).where(and(eq(workflowSteps.instanceId, task.instanceId), eq(workflowSteps.stepIndex, String(task.stepId)))).limit(1);
|
|
389
|
+
if (!own || own.output !== null) {
|
|
390
|
+
await redis.native.zrem(WORKFLOW_TIMERS_KEY, taskStr);
|
|
391
|
+
continue;
|
|
392
|
+
}
|
|
393
|
+
}
|
|
343
394
|
if ((await db.delete(workflowWaiters).where(and(eq(workflowWaiters.instanceId, task.instanceId), eq(workflowWaiters.eventName, task.eventName))).returning()).length === 0) {
|
|
344
|
-
await redis.native.zrem(
|
|
395
|
+
await redis.native.zrem(WORKFLOW_TIMERS_KEY, taskStr);
|
|
345
396
|
continue;
|
|
346
397
|
}
|
|
347
|
-
const
|
|
398
|
+
const steps = await db.select().from(workflowSteps).where(and(eq(workflowSteps.instanceId, task.instanceId), eq(workflowSteps.action, "waitForEvent")));
|
|
399
|
+
const pendingStep = task.stepId !== void 0 ? steps.find((s) => s.stepIndex === String(task.stepId) && s.output === null) : steps.find((s) => s.output === null);
|
|
348
400
|
if (pendingStep) await db.update(workflowSteps).set({ output: { timedOut: true } }).where(eq(workflowSteps.id, pendingStep.id));
|
|
349
401
|
}
|
|
350
402
|
await workflowProducer.publish(buildStreamEvent("workflow.resumed", task, "scheduler", void 0));
|
|
351
|
-
await redis.native.zrem(
|
|
403
|
+
await redis.native.zrem(WORKFLOW_TIMERS_KEY, taskStr);
|
|
352
404
|
logger.info({ instanceId: task.instanceId }, "Workflow resumed from timer");
|
|
353
405
|
}
|
|
354
406
|
} catch (err) {
|
|
@@ -377,16 +429,20 @@ async function startWorkflowWorker() {
|
|
|
377
429
|
await worker.start();
|
|
378
430
|
}
|
|
379
431
|
async function stopWorkflowWorker() {
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
432
|
+
await shutdownWorker("workflow worker", {
|
|
433
|
+
logger,
|
|
434
|
+
timers: [
|
|
435
|
+
healthInterval,
|
|
436
|
+
pollInterval,
|
|
437
|
+
reaperInterval
|
|
438
|
+
],
|
|
439
|
+
worker,
|
|
440
|
+
sql: sql$1,
|
|
441
|
+
redis
|
|
442
|
+
});
|
|
443
|
+
healthInterval = null;
|
|
444
|
+
pollInterval = null;
|
|
445
|
+
reaperInterval = null;
|
|
388
446
|
}
|
|
389
447
|
//#endregion
|
|
390
448
|
export { startWorkflowWorker, stopWorkflowWorker };
|
|
391
|
-
|
|
392
|
-
//# sourceMappingURL=main-D19Wxs0x.mjs.map
|
|
@@ -0,0 +1,312 @@
|
|
|
1
|
+
import { C as ContactRepository, D as TemplateRepository, O as UserRepository, Sn as normaliseTarget, Ut as buildStreamEvent, Wt as CONSUMER_GROUPS, Yt as STREAMS, c as BaseWorker, cn as loadEnv, d as startHealthReporter, en as registry, it as suppressions, rt as scheduledPayloads, un as readBaseConfig, wn as globalEmitter, xn as getPriorityBucket } from "./src-CI7FGf3h.mjs";
|
|
2
|
+
import { a as uniqueConsumerId, i as shutdownWorker, n as createStreamConsumer, r as createWorkerRuntime, t as createPriorityProducers } from "./bootstrap-GRXzgPvW.mjs";
|
|
3
|
+
import { randomUUID } from "node:crypto";
|
|
4
|
+
import { and, asc, eq, gt, inArray, or, sql } from "drizzle-orm";
|
|
5
|
+
//#region src/services/scheduler/send-gate.ts
|
|
6
|
+
/**
|
|
7
|
+
* Re-checks parked sends at the moment they are released.
|
|
8
|
+
*
|
|
9
|
+
* The engine clears a message against suppressions, opt-outs and contacts when
|
|
10
|
+
* it first sees it. A send parked for later — a `sendAt`, a quiet-hours
|
|
11
|
+
* deferral, a retry — can sit for hours or days, and whatever the person did in
|
|
12
|
+
* between (unsubscribed, complained, hard-bounced, removed the address) must
|
|
13
|
+
* still stop it. Without this the scheduler sent to them anyway.
|
|
14
|
+
*
|
|
15
|
+
* Reads Postgres directly rather than through the repositories' caches: this
|
|
16
|
+
* runs once per poll, not once per message, and a stale "still subscribed" is
|
|
17
|
+
* the very thing it exists to prevent.
|
|
18
|
+
*
|
|
19
|
+
* Returns the blocked tasks with the reason. Throws if a lookup fails, so the
|
|
20
|
+
* caller holds the batch rather than sending it unchecked.
|
|
21
|
+
*/
|
|
22
|
+
async function findBlockedAtRelease(db, tasks) {
|
|
23
|
+
const blocked = /* @__PURE__ */ new Map();
|
|
24
|
+
const byProject = /* @__PURE__ */ new Map();
|
|
25
|
+
for (const t of tasks) {
|
|
26
|
+
if (!t?.projectId || !t.recipientId) continue;
|
|
27
|
+
if (!byProject.has(t.projectId)) byProject.set(t.projectId, []);
|
|
28
|
+
byProject.get(t.projectId).push(t);
|
|
29
|
+
}
|
|
30
|
+
for (const [projectId, group] of byProject) {
|
|
31
|
+
const recipientIds = [...new Set(group.map((t) => t.recipientId))];
|
|
32
|
+
const templateIds = [...new Set(group.map((t) => t.templateId).filter(Boolean))];
|
|
33
|
+
const targets = [...new Set(group.map((t) => t.destination && normaliseTarget(t.destination)).filter(Boolean))];
|
|
34
|
+
const templateRepo = new TemplateRepository(db);
|
|
35
|
+
const [suppressedRows, users, contactsByUser, templates] = await Promise.all([
|
|
36
|
+
targets.length > 0 ? db.select({
|
|
37
|
+
channel: suppressions.channel,
|
|
38
|
+
target: suppressions.target
|
|
39
|
+
}).from(suppressions).where(and(eq(suppressions.projectId, projectId), inArray(suppressions.target, targets))) : [],
|
|
40
|
+
new UserRepository(db).findRecordsByIds(projectId, recipientIds),
|
|
41
|
+
new ContactRepository(db).findActiveByUserIds(projectId, recipientIds),
|
|
42
|
+
Promise.all(templateIds.map((id) => templateRepo.findById(projectId, id)))
|
|
43
|
+
]);
|
|
44
|
+
const suppressed = new Set(suppressedRows.map((r) => `${r.channel}:${normaliseTarget(r.target)}`));
|
|
45
|
+
const userById = new Map(users.map((u) => [u.userId, u]));
|
|
46
|
+
const topicsByTemplate = /* @__PURE__ */ new Map();
|
|
47
|
+
templateIds.forEach((id, i) => topicsByTemplate.set(id, templates[i]?.topics ?? []));
|
|
48
|
+
for (const t of group) {
|
|
49
|
+
const reason = decide(t, t.templateId ? topicsByTemplate.get(t.templateId) ?? [] : [], suppressed, userById.get(t.recipientId), contactsByUser);
|
|
50
|
+
if (reason) blocked.set(t.taskId, reason);
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
return blocked;
|
|
54
|
+
}
|
|
55
|
+
function decide(t, topics, suppressed, user, contactsByUser) {
|
|
56
|
+
if (t.destination && suppressed.has(`${t.channel}:${normaliseTarget(t.destination)}`)) return "suppressed";
|
|
57
|
+
if (!user) return "recipient_not_found";
|
|
58
|
+
if (topics.some((topic) => user.preferences.topics?.[topic] === false)) return "user_opted_out";
|
|
59
|
+
if (user.preferences.channels?.[t.channel] === false) return "channel_disabled";
|
|
60
|
+
if (t.destination) {
|
|
61
|
+
const contact = (contactsByUser.get(t.recipientId) ?? []).find((c) => c.channel === t.channel && c.target === t.destination);
|
|
62
|
+
if (!contact) return "contact_inactive";
|
|
63
|
+
if (topics.some((topic) => contact.preferences?.topics?.[topic] === false)) return "user_opted_out";
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
//#endregion
|
|
67
|
+
//#region src/services/scheduler/main.ts
|
|
68
|
+
loadEnv();
|
|
69
|
+
const config = readBaseConfig();
|
|
70
|
+
let logger;
|
|
71
|
+
let redis;
|
|
72
|
+
let sql$1;
|
|
73
|
+
let db;
|
|
74
|
+
let consumer;
|
|
75
|
+
let pendingScanner;
|
|
76
|
+
let worker;
|
|
77
|
+
let healthInterval = null;
|
|
78
|
+
let pollTimeout = null;
|
|
79
|
+
let isPolling = false;
|
|
80
|
+
var SchedulerWorker = class extends BaseWorker {
|
|
81
|
+
registry;
|
|
82
|
+
redisCli;
|
|
83
|
+
constructor(options) {
|
|
84
|
+
super(options);
|
|
85
|
+
this.registry = options.registry;
|
|
86
|
+
this.redisCli = options.redis;
|
|
87
|
+
}
|
|
88
|
+
async process(message) {
|
|
89
|
+
const { event } = message;
|
|
90
|
+
const payloadResult = this.registry.safeParsePayload("notification.scheduled", event.payload);
|
|
91
|
+
if (!payloadResult.success) {
|
|
92
|
+
this.logger.warn({
|
|
93
|
+
messageId: message.id,
|
|
94
|
+
issues: payloadResult.error.issues
|
|
95
|
+
}, "invalid notification.scheduled payload — skipping");
|
|
96
|
+
return;
|
|
97
|
+
}
|
|
98
|
+
const scheduled = payloadResult.data;
|
|
99
|
+
const scheduledAt = new Date(scheduled.scheduledAt).getTime();
|
|
100
|
+
const taskData = JSON.stringify({
|
|
101
|
+
taskId: scheduled.taskId,
|
|
102
|
+
enrichedEventId: scheduled.enrichedEventId,
|
|
103
|
+
traceId: event.metadata.traceId
|
|
104
|
+
});
|
|
105
|
+
const shard = parseInt(scheduled.taskId.slice(-1), 16) || 0;
|
|
106
|
+
const zsetKey = `notif:scheduled:zset:${shard}`;
|
|
107
|
+
await this.redisCli.zadd(zsetKey, scheduledAt, taskData);
|
|
108
|
+
this.logger.debug({
|
|
109
|
+
taskId: scheduled.taskId,
|
|
110
|
+
scheduledAt: scheduled.scheduledAt,
|
|
111
|
+
shard
|
|
112
|
+
}, "scheduled task queued in ZSET");
|
|
113
|
+
}
|
|
114
|
+
};
|
|
115
|
+
async function executeSchedulerPoll(redis, outboundProducers, logger, db) {
|
|
116
|
+
const lockKey = "notif:lock:scheduler:poll";
|
|
117
|
+
const lockOwner = randomUUID();
|
|
118
|
+
let acquired = false;
|
|
119
|
+
try {
|
|
120
|
+
if (await redis.set(lockKey, lockOwner, "EX", 30, "NX") !== "OK") return false;
|
|
121
|
+
acquired = true;
|
|
122
|
+
const now = Date.now();
|
|
123
|
+
const visibilityTimeout = 6e4;
|
|
124
|
+
const perShardLimit = 100;
|
|
125
|
+
const pipeline = redis.pipeline();
|
|
126
|
+
for (let i = 0; i < 16; i++) {
|
|
127
|
+
const shardKey = `notif:scheduled:zset:${i}`;
|
|
128
|
+
pipeline.schedulerPoll(shardKey, now, perShardLimit, visibilityTimeout);
|
|
129
|
+
}
|
|
130
|
+
const perShard = (await pipeline.exec())?.map((res) => res[0] ? [] : res[1] ?? []) ?? [];
|
|
131
|
+
const tasks = perShard.flat();
|
|
132
|
+
const anyShardFull = perShard.some((shard) => shard.length >= perShardLimit);
|
|
133
|
+
if (tasks.length === 0) return false;
|
|
134
|
+
const parsedTasks = tasks.map((t) => {
|
|
135
|
+
try {
|
|
136
|
+
return {
|
|
137
|
+
taskStr: t,
|
|
138
|
+
...JSON.parse(t)
|
|
139
|
+
};
|
|
140
|
+
} catch (err) {
|
|
141
|
+
logger.error({
|
|
142
|
+
taskStr: t,
|
|
143
|
+
err
|
|
144
|
+
}, "failed to parse scheduled task JSON");
|
|
145
|
+
return null;
|
|
146
|
+
}
|
|
147
|
+
}).filter(Boolean);
|
|
148
|
+
if (parsedTasks.length === 0) return false;
|
|
149
|
+
const taskIds = parsedTasks.map((t) => t.taskId);
|
|
150
|
+
const dbPayloads = await db.select().from(scheduledPayloads).where(inArray(scheduledPayloads.taskId, taskIds));
|
|
151
|
+
const payloadMap = new Map(dbPayloads.map((row) => [row.taskId, row.payload]));
|
|
152
|
+
const blocked = await findBlockedAtRelease(db, [...payloadMap.values()]);
|
|
153
|
+
const batchedEvents = {
|
|
154
|
+
critical: [],
|
|
155
|
+
normal: [],
|
|
156
|
+
low: []
|
|
157
|
+
};
|
|
158
|
+
const cleanupPipeline = redis.pipeline();
|
|
159
|
+
const dbCleanupIds = [];
|
|
160
|
+
const released = /* @__PURE__ */ new Set();
|
|
161
|
+
for (let i = 0; i < parsedTasks.length; i++) {
|
|
162
|
+
const { taskStr, taskId, traceId } = parsedTasks[i];
|
|
163
|
+
if (released.has(taskId)) {
|
|
164
|
+
cleanupPipeline.zrem(`notif:scheduled:zset:${parseInt(taskId.slice(-1), 16) || 0}`, taskStr);
|
|
165
|
+
continue;
|
|
166
|
+
}
|
|
167
|
+
const payload = payloadMap.get(taskId);
|
|
168
|
+
if (!payload) {
|
|
169
|
+
logger.warn({ taskId }, "scheduled payload not found in Postgres — skipping release");
|
|
170
|
+
const shard = parseInt(taskId.slice(-1), 16) || 0;
|
|
171
|
+
cleanupPipeline.zrem(`notif:scheduled:zset:${shard}`, taskStr);
|
|
172
|
+
continue;
|
|
173
|
+
}
|
|
174
|
+
const dispatchPayload = payload;
|
|
175
|
+
const blockReason = blocked.get(taskId);
|
|
176
|
+
if (blockReason) {
|
|
177
|
+
logger.info({
|
|
178
|
+
taskId,
|
|
179
|
+
reason: blockReason
|
|
180
|
+
}, "parked send no longer allowed — dropping");
|
|
181
|
+
globalEmitter.emit("notification:skipped", {
|
|
182
|
+
projectId: dispatchPayload.projectId,
|
|
183
|
+
eventId: dispatchPayload.enrichedEventId,
|
|
184
|
+
recipientId: dispatchPayload.recipientId,
|
|
185
|
+
reason: blockReason
|
|
186
|
+
});
|
|
187
|
+
released.add(taskId);
|
|
188
|
+
dbCleanupIds.push(taskId);
|
|
189
|
+
const shard = parseInt(taskId.slice(-1), 16) || 0;
|
|
190
|
+
cleanupPipeline.zrem(`notif:scheduled:zset:${shard}`, taskStr);
|
|
191
|
+
continue;
|
|
192
|
+
}
|
|
193
|
+
batchedEvents[getPriorityBucket(dispatchPayload.priority)].push(buildStreamEvent("notification.dispatched", dispatchPayload, "scheduler", traceId));
|
|
194
|
+
released.add(taskId);
|
|
195
|
+
dbCleanupIds.push(taskId);
|
|
196
|
+
const shard = parseInt(taskId.slice(-1), 16) || 0;
|
|
197
|
+
cleanupPipeline.zrem(`notif:scheduled:zset:${shard}`, taskStr);
|
|
198
|
+
}
|
|
199
|
+
for (const p of Object.keys(batchedEvents)) if (batchedEvents[p].length > 0) await (outboundProducers[p] ?? outboundProducers["normal"]).publishBatch(batchedEvents[p]);
|
|
200
|
+
if (dbCleanupIds.length > 0) try {
|
|
201
|
+
const sendAt = sql`${scheduledPayloads.payload}->>'scheduledAt'`;
|
|
202
|
+
await db.delete(scheduledPayloads).where(or(...dbCleanupIds.map((taskId) => {
|
|
203
|
+
const released = payloadMap.get(taskId)?.scheduledAt;
|
|
204
|
+
return and(eq(scheduledPayloads.taskId, taskId), typeof released === "string" ? sql`${sendAt} = ${released}` : sql`${sendAt} IS NULL`);
|
|
205
|
+
})));
|
|
206
|
+
} catch (err) {
|
|
207
|
+
logger.error({ err }, "failed to delete scheduled payloads from Postgres, continuing with Redis cleanup");
|
|
208
|
+
}
|
|
209
|
+
await cleanupPipeline.exec();
|
|
210
|
+
logger.info({ count: parsedTasks.length }, "scheduled tasks released to outbound");
|
|
211
|
+
return anyShardFull;
|
|
212
|
+
} catch (err) {
|
|
213
|
+
logger.error({ err }, "error in scheduler polling loop");
|
|
214
|
+
return false;
|
|
215
|
+
} finally {
|
|
216
|
+
if (acquired) await redis.releaseLock(lockKey, lockOwner).catch(() => {});
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
/**
|
|
220
|
+
* Present while the scheduler ZSETs are known to be intact. Redis losing its
|
|
221
|
+
* data (a flush, a restart without persistence, failover to an empty replica)
|
|
222
|
+
* takes this key with it, which is how the poll loop notices.
|
|
223
|
+
*/
|
|
224
|
+
const SCHEDULER_SENTINEL_KEY = "notif:scheduler:intact";
|
|
225
|
+
/**
|
|
226
|
+
* Re-queues every scheduled send still waiting in Postgres. Each row keeps its
|
|
227
|
+
* payload, and with it the send time, so the ZSETs can be rebuilt exactly;
|
|
228
|
+
* a legacy row without a send time is released on the next poll.
|
|
229
|
+
*
|
|
230
|
+
* Only run when the sentinel is missing: a rebuilt member has no trace id and
|
|
231
|
+
* so never matches a surviving original, and the poll only de-duplicates
|
|
232
|
+
* copies that happen to be claimed together.
|
|
233
|
+
*/
|
|
234
|
+
async function rebuildScheduledQueue(redis, db, logger, pageSize = 1e3) {
|
|
235
|
+
let after;
|
|
236
|
+
let total = 0;
|
|
237
|
+
for (;;) {
|
|
238
|
+
const rows = await db.select({
|
|
239
|
+
taskId: scheduledPayloads.taskId,
|
|
240
|
+
payload: scheduledPayloads.payload
|
|
241
|
+
}).from(scheduledPayloads).where(after === void 0 ? void 0 : gt(scheduledPayloads.taskId, after)).orderBy(asc(scheduledPayloads.taskId)).limit(pageSize);
|
|
242
|
+
if (rows.length === 0) break;
|
|
243
|
+
const pipeline = redis.pipeline();
|
|
244
|
+
for (const { taskId, payload } of rows) {
|
|
245
|
+
const sendAt = Date.parse(payload?.scheduledAt ?? "");
|
|
246
|
+
const member = JSON.stringify({
|
|
247
|
+
taskId,
|
|
248
|
+
enrichedEventId: payload?.enrichedEventId
|
|
249
|
+
});
|
|
250
|
+
const shard = parseInt(taskId.slice(-1), 16) || 0;
|
|
251
|
+
pipeline.zadd(`notif:scheduled:zset:${shard}`, "NX", Number.isNaN(sendAt) ? Date.now() : sendAt, member);
|
|
252
|
+
}
|
|
253
|
+
await pipeline.exec();
|
|
254
|
+
total += rows.length;
|
|
255
|
+
after = rows[rows.length - 1].taskId;
|
|
256
|
+
if (rows.length < pageSize) break;
|
|
257
|
+
}
|
|
258
|
+
await redis.set(SCHEDULER_SENTINEL_KEY, "1");
|
|
259
|
+
if (total > 0) logger.warn({ count: total }, "rebuilt scheduled queue from Postgres");
|
|
260
|
+
return total;
|
|
261
|
+
}
|
|
262
|
+
async function startSchedulerWorker() {
|
|
263
|
+
({logger, redis, sql: sql$1, db} = createWorkerRuntime(config, "scheduler"));
|
|
264
|
+
({consumer, pendingScanner} = createStreamConsumer({
|
|
265
|
+
redis,
|
|
266
|
+
logger
|
|
267
|
+
}, {
|
|
268
|
+
stream: STREAMS.SCHEDULED,
|
|
269
|
+
group: CONSUMER_GROUPS.SCHEDULER,
|
|
270
|
+
consumer: uniqueConsumerId("scheduler"),
|
|
271
|
+
batchSize: config.WORKER_CONCURRENCY
|
|
272
|
+
}));
|
|
273
|
+
const outboundProducers = createPriorityProducers(redis.native, logger, "OUTBOUND");
|
|
274
|
+
worker = new SchedulerWorker({
|
|
275
|
+
consumer,
|
|
276
|
+
pendingScanner,
|
|
277
|
+
logger,
|
|
278
|
+
concurrency: config.WORKER_CONCURRENCY,
|
|
279
|
+
registry,
|
|
280
|
+
redis: redis.native
|
|
281
|
+
});
|
|
282
|
+
isPolling = true;
|
|
283
|
+
const pollLoop = async () => {
|
|
284
|
+
if (!isPolling) return;
|
|
285
|
+
try {
|
|
286
|
+
if (!await redis.native.exists("notif:scheduler:intact")) await rebuildScheduledQueue(redis.native, db, logger);
|
|
287
|
+
const hasMore = await executeSchedulerPoll(redis.native, outboundProducers, logger, db);
|
|
288
|
+
if (isPolling) pollTimeout = setTimeout(() => void pollLoop(), hasMore ? 0 : 5e3);
|
|
289
|
+
} catch (err) {
|
|
290
|
+
logger.error({ err }, "scheduler poll loop error");
|
|
291
|
+
if (isPolling) pollTimeout = setTimeout(() => void pollLoop(), 5e3);
|
|
292
|
+
}
|
|
293
|
+
};
|
|
294
|
+
pollLoop();
|
|
295
|
+
healthInterval = startHealthReporter("scheduler", worker, redis, logger);
|
|
296
|
+
logger.info({ env: config.NODE_ENV }, "scheduler starting");
|
|
297
|
+
await worker.start();
|
|
298
|
+
}
|
|
299
|
+
async function stopSchedulerWorker() {
|
|
300
|
+
isPolling = false;
|
|
301
|
+
await shutdownWorker("scheduler", {
|
|
302
|
+
logger,
|
|
303
|
+
timers: [healthInterval, pollTimeout],
|
|
304
|
+
worker,
|
|
305
|
+
sql: sql$1,
|
|
306
|
+
redis
|
|
307
|
+
});
|
|
308
|
+
healthInterval = null;
|
|
309
|
+
pollTimeout = null;
|
|
310
|
+
}
|
|
311
|
+
//#endregion
|
|
312
|
+
export { startSchedulerWorker, stopSchedulerWorker };
|