notifkit 0.1.15 → 0.1.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dashboard/dist/assets/index-C-DJYu0_.css +1 -0
- package/dashboard/dist/assets/index-CzFZcEon.js +318 -0
- package/dashboard/dist/index.html +2 -2
- package/dist/{bootstrap-BOmT69OH.mjs → bootstrap-BMDRPjgp.mjs} +1 -1
- package/dist/{bulk-ZPiOJRx-.mjs → bulk-CJ5QXiPa.mjs} +1 -1
- package/dist/index.d.mts +71 -27
- package/dist/index.mjs +2 -2
- package/dist/{main-CCqcFDDe.mjs → main-6_bXQBDH.mjs} +21 -41
- package/dist/{main-BdZvBgV7.mjs → main-8nP3qmCT.mjs} +22 -48
- package/dist/{main-C0m9-Eos.mjs → main-B82bsFoy.mjs} +183 -334
- package/dist/{main-CTjaZ7ZD.mjs → main-CpaHF1Zw.mjs} +13 -15
- package/dist/{main-JEXBLP2A.mjs → main-DULD8-th.mjs} +8 -6
- package/dist/{main-C6S3EsJ3.mjs → main-ZW5RaiOK.mjs} +77 -64
- package/dist/{main-g33jpia4.mjs → main-g1KEcXXp.mjs} +676 -582
- package/dist/{main-24J4U8yP.mjs → main-mGIT8jv4.mjs} +4 -4
- package/dist/schedule-or-dispatch-Cd7GLfc3.mjs +52 -0
- package/dist/{src-9LXITcNi.mjs → src-BE_EUpZ6.mjs} +93 -93
- package/package.json +1 -1
- package/scripts/create-project.mjs +6 -2
- package/src/client.ts +3 -3
- package/src/config/index.ts +1 -0
- package/src/contracts/common.ts +2 -0
- package/src/contracts/envelope.ts +1 -1
- package/src/contracts/events/notification-created.ts +2 -0
- package/src/contracts/events/notification-delivered.ts +1 -1
- package/src/contracts/events/notification-failed.ts +1 -1
- package/src/contracts/events/notification-requested.ts +3 -2
- package/src/contracts/helpers.ts +1 -2
- package/src/contracts/index.ts +1 -0
- package/src/contracts/registry.ts +1 -0
- package/src/contracts/sdk.ts +8 -37
- package/src/db/schema.ts +8 -13
- package/src/idempotency/index.ts +6 -8
- package/src/metrics/index.ts +2 -0
- package/src/rate-limiter/api-rate-limiter.ts +208 -0
- package/src/rate-limiter/index.ts +3 -411
- package/src/rate-limiter/project-settings-cache.ts +99 -0
- package/src/rate-limiter/user-throttle.ts +100 -0
- package/src/redis/index.ts +10 -3
- package/src/repositories/admin-user.ts +83 -0
- package/src/repositories/contact.ts +359 -0
- package/src/repositories/index.ts +9 -1416
- package/src/repositories/preference.ts +35 -0
- package/src/repositories/project.ts +110 -0
- package/src/repositories/segment.ts +16 -0
- package/src/repositories/template.ts +72 -0
- package/src/repositories/types.ts +49 -0
- package/src/repositories/user.ts +644 -0
- package/src/repositories/workflow.ts +64 -0
- package/src/services/ai/main.ts +29 -71
- package/src/services/api/admin-static.ts +4 -1
- package/src/services/api/handlers/auth.ts +109 -0
- package/src/services/api/handlers/campaigns.ts +191 -0
- package/src/services/api/handlers/events.ts +99 -0
- package/src/services/api/handlers/notifications.ts +356 -0
- package/src/services/api/handlers/projects.ts +116 -0
- package/src/services/api/handlers/shared.ts +262 -0
- package/src/services/api/handlers/suppressions.ts +95 -0
- package/src/services/api/handlers/system.ts +314 -0
- package/src/services/api/handlers/templates.ts +97 -0
- package/src/services/api/handlers/unsubscribe.ts +215 -0
- package/src/services/api/handlers/users.ts +262 -0
- package/src/services/api/handlers/workflows.ts +150 -0
- package/src/services/api/handlers.ts +25 -2190
- package/src/services/api/health.ts +133 -0
- package/src/services/api/http.ts +12 -0
- package/src/services/api/main.ts +32 -322
- package/src/services/api/router.ts +1 -1
- package/src/services/api/webhooks.ts +172 -0
- package/src/services/delivery/main.ts +256 -498
- package/src/services/engine/main.ts +31 -247
- package/src/services/engine/quiet-hours.ts +151 -0
- package/src/services/engine/schedule-or-dispatch.ts +104 -0
- package/src/services/enricher/main.ts +9 -21
- package/src/services/events/main.ts +2 -2
- package/src/services/scheduler/main.ts +16 -25
- package/src/services/workflow/duration.ts +24 -0
- package/src/services/workflow/main.ts +6 -164
- package/src/services/workflow/timers.ts +157 -0
- package/src/shared/events.ts +3 -2
- package/src/templates/render.ts +1 -0
- package/src/workers/index.ts +7 -4
- package/src/workflows/sdk.ts +1 -0
- package/dashboard/dist/assets/index-ggpJKZqZ.js +0 -315
- package/dashboard/dist/assets/index-s-SXGIwv.css +0 -1
|
@@ -15,12 +15,7 @@ import {
|
|
|
15
15
|
type StreamEvent,
|
|
16
16
|
} from "@/index.js";
|
|
17
17
|
import { IdempotencyGuard } from "@/index.js";
|
|
18
|
-
import {
|
|
19
|
-
UserRepository,
|
|
20
|
-
PreferenceRepository,
|
|
21
|
-
TemplateRepository,
|
|
22
|
-
ContactRepository,
|
|
23
|
-
} from "@/index.js";
|
|
18
|
+
import { UserRepository, TemplateRepository, ContactRepository } from "@/index.js";
|
|
24
19
|
import { TemplateCache } from "@/templates/index.js";
|
|
25
20
|
import { getPriorityBucket, globalEmitter, type WorkerOptions } from "@/shared/index.js";
|
|
26
21
|
import { startHealthReporter } from "@/workers/index.js";
|
|
@@ -60,7 +55,7 @@ export interface EnricherWorkerOptions extends WorkerOptions {
|
|
|
60
55
|
producers: any;
|
|
61
56
|
idempotency: any;
|
|
62
57
|
userRepo: any;
|
|
63
|
-
prefRepo
|
|
58
|
+
prefRepo?: any;
|
|
64
59
|
contactRepo: any;
|
|
65
60
|
templateCache: TemplateCache;
|
|
66
61
|
}
|
|
@@ -69,7 +64,6 @@ export class EnricherWorker extends BaseWorker {
|
|
|
69
64
|
private readonly producers: any;
|
|
70
65
|
private readonly idempotency: any;
|
|
71
66
|
private readonly userRepo: any;
|
|
72
|
-
private readonly prefRepo: any;
|
|
73
67
|
private readonly contactRepo: any;
|
|
74
68
|
private readonly templateCache: TemplateCache;
|
|
75
69
|
/** Engine in this process, when the pipeline is fused. See `setInlineNext`. */
|
|
@@ -140,7 +134,6 @@ export class EnricherWorker extends BaseWorker {
|
|
|
140
134
|
this.producers = options.producers;
|
|
141
135
|
this.idempotency = options.idempotency;
|
|
142
136
|
this.userRepo = options.userRepo;
|
|
143
|
-
this.prefRepo = options.prefRepo;
|
|
144
137
|
this.contactRepo = options.contactRepo;
|
|
145
138
|
this.templateCache = options.templateCache;
|
|
146
139
|
|
|
@@ -248,6 +241,7 @@ export class EnricherWorker extends BaseWorker {
|
|
|
248
241
|
const requestedResult = registry.safeParsePayload("notification.requested", event.payload);
|
|
249
242
|
let createdResult: any = null;
|
|
250
243
|
if (!requestedResult.success) {
|
|
244
|
+
// @deprecated fallback to the legacy `notification.created` payload (see below).
|
|
251
245
|
createdResult = registry.safeParsePayload("notification.created", event.payload);
|
|
252
246
|
isRequested = false;
|
|
253
247
|
}
|
|
@@ -260,7 +254,12 @@ export class EnricherWorker extends BaseWorker {
|
|
|
260
254
|
return;
|
|
261
255
|
}
|
|
262
256
|
|
|
263
|
-
|
|
257
|
+
/**
|
|
258
|
+
* Legacy `notification.created` path.
|
|
259
|
+
* @deprecated Nothing in notifkit publishes `notification.created` any more;
|
|
260
|
+
* this branch only serves callers writing straight to the inbound stream.
|
|
261
|
+
* Removed in the next minor version.
|
|
262
|
+
*/
|
|
264
263
|
if (!isRequested) {
|
|
265
264
|
const raw = createdResult.data as NotificationCreatedPayload;
|
|
266
265
|
const dedupeId = `${raw.projectId}:${raw.idempotencyKey ?? event.id}`;
|
|
@@ -595,19 +594,9 @@ export async function startEnricherWorker() {
|
|
|
595
594
|
});
|
|
596
595
|
|
|
597
596
|
const userRepo = new UserRepository(db);
|
|
598
|
-
const prefRepo = new PreferenceRepository(db);
|
|
599
597
|
const contactRepo = new ContactRepository(db);
|
|
600
598
|
const templateCache = new TemplateCache(new TemplateRepository(db));
|
|
601
599
|
|
|
602
|
-
// ─── Stage 1: Context Enricher ──────────────────────────────────────────────
|
|
603
|
-
//
|
|
604
|
-
// Pipeline:
|
|
605
|
-
// 1. Parse payload as notification.created
|
|
606
|
-
// 2. Idempotency check — drop if already processed
|
|
607
|
-
// 3. Load user profile (language, timezone) from DB
|
|
608
|
-
// 4. Load all stored preferences for this user
|
|
609
|
-
// 5. Publish notification.enriched to ENRICHED stream
|
|
610
|
-
|
|
611
600
|
worker = new EnricherWorker({
|
|
612
601
|
consumer,
|
|
613
602
|
pendingScanner,
|
|
@@ -617,7 +606,6 @@ export async function startEnricherWorker() {
|
|
|
617
606
|
producers,
|
|
618
607
|
idempotency,
|
|
619
608
|
userRepo,
|
|
620
|
-
prefRepo,
|
|
621
609
|
contactRepo,
|
|
622
610
|
templateCache,
|
|
623
611
|
});
|
|
@@ -124,7 +124,7 @@ export class EventWorker extends BaseWorker {
|
|
|
124
124
|
this.logger.error(
|
|
125
125
|
{
|
|
126
126
|
batchSize: batch.length,
|
|
127
|
-
|
|
127
|
+
err: error,
|
|
128
128
|
},
|
|
129
129
|
"failed to flush delivery logs — dropping batch",
|
|
130
130
|
);
|
|
@@ -314,7 +314,7 @@ export class EventWorker extends BaseWorker {
|
|
|
314
314
|
),
|
|
315
315
|
);
|
|
316
316
|
|
|
317
|
-
//
|
|
317
|
+
// The step still waiting. An instance waits on one event at a time.
|
|
318
318
|
const pendingStep = steps.find((s: any) => s.output === null);
|
|
319
319
|
|
|
320
320
|
if (pendingStep) {
|
|
@@ -8,8 +8,8 @@ import {
|
|
|
8
8
|
STREAMS,
|
|
9
9
|
CONSUMER_GROUPS,
|
|
10
10
|
registry,
|
|
11
|
+
type EventRegistry,
|
|
11
12
|
buildStreamEvent,
|
|
12
|
-
type NotificationScheduledPayload,
|
|
13
13
|
} from "@/index.js";
|
|
14
14
|
import { getPriorityBucket, globalEmitter, type WorkerOptions } from "@/shared/index.js";
|
|
15
15
|
import { scheduledPayloads } from "@/db/schema.js";
|
|
@@ -40,13 +40,17 @@ let healthInterval: NodeJS.Timeout | null = null;
|
|
|
40
40
|
let pollTimeout: ReturnType<typeof setTimeout> | null = null;
|
|
41
41
|
let isPolling = false;
|
|
42
42
|
|
|
43
|
+
/** Scheduled tasks are spread over 16 sorted sets by the last hex digit of the task id. */
|
|
44
|
+
const shardOf = (taskId: string): number => parseInt(taskId.slice(-1), 16) || 0;
|
|
45
|
+
const zsetKey = (shard: number): string => `notif:scheduled:zset:${shard}`;
|
|
46
|
+
|
|
43
47
|
export interface SchedulerWorkerOptions extends WorkerOptions {
|
|
44
|
-
registry:
|
|
48
|
+
registry: EventRegistry;
|
|
45
49
|
redis: Redis;
|
|
46
50
|
}
|
|
47
51
|
|
|
48
52
|
export class SchedulerWorker extends BaseWorker {
|
|
49
|
-
private readonly registry:
|
|
53
|
+
private readonly registry: EventRegistry;
|
|
50
54
|
private readonly redisCli: Redis;
|
|
51
55
|
|
|
52
56
|
constructor(options: SchedulerWorkerOptions) {
|
|
@@ -66,7 +70,7 @@ export class SchedulerWorker extends BaseWorker {
|
|
|
66
70
|
return;
|
|
67
71
|
}
|
|
68
72
|
|
|
69
|
-
const scheduled = payloadResult.data
|
|
73
|
+
const scheduled = payloadResult.data;
|
|
70
74
|
const scheduledAt = new Date(scheduled.scheduledAt).getTime();
|
|
71
75
|
|
|
72
76
|
const taskData = JSON.stringify({
|
|
@@ -75,9 +79,8 @@ export class SchedulerWorker extends BaseWorker {
|
|
|
75
79
|
traceId: event.metadata.traceId,
|
|
76
80
|
});
|
|
77
81
|
|
|
78
|
-
const shard =
|
|
79
|
-
|
|
80
|
-
await this.redisCli.zadd(zsetKey, scheduledAt, taskData);
|
|
82
|
+
const shard = shardOf(scheduled.taskId);
|
|
83
|
+
await this.redisCli.zadd(zsetKey(shard), scheduledAt, taskData);
|
|
81
84
|
this.logger.debug(
|
|
82
85
|
{ taskId: scheduled.taskId, scheduledAt: scheduled.scheduledAt, shard },
|
|
83
86
|
"scheduled task queued in ZSET",
|
|
@@ -107,7 +110,7 @@ export async function executeSchedulerPoll(
|
|
|
107
110
|
const pipeline = redis.pipeline();
|
|
108
111
|
|
|
109
112
|
for (let i = 0; i < 16; i++) {
|
|
110
|
-
const shardKey =
|
|
113
|
+
const shardKey = zsetKey(i);
|
|
111
114
|
pipeline.schedulerPoll(shardKey, now, perShardLimit, visibilityTimeout);
|
|
112
115
|
}
|
|
113
116
|
|
|
@@ -161,18 +164,14 @@ export async function executeSchedulerPoll(
|
|
|
161
164
|
for (let i = 0; i < parsedTasks.length; i++) {
|
|
162
165
|
const { taskStr, taskId, traceId } = parsedTasks[i];
|
|
163
166
|
if (released.has(taskId)) {
|
|
164
|
-
cleanupPipeline.zrem(
|
|
165
|
-
`notif:scheduled:zset:${parseInt(taskId.slice(-1), 16) || 0}`,
|
|
166
|
-
taskStr,
|
|
167
|
-
);
|
|
167
|
+
cleanupPipeline.zrem(zsetKey(shardOf(taskId)), taskStr);
|
|
168
168
|
continue;
|
|
169
169
|
}
|
|
170
170
|
const payload = payloadMap.get(taskId);
|
|
171
171
|
|
|
172
172
|
if (!payload) {
|
|
173
173
|
logger.warn({ taskId }, "scheduled payload not found in Postgres — skipping release");
|
|
174
|
-
|
|
175
|
-
cleanupPipeline.zrem(`notif:scheduled:zset:${shard}`, taskStr);
|
|
174
|
+
cleanupPipeline.zrem(zsetKey(shardOf(taskId)), taskStr);
|
|
176
175
|
continue;
|
|
177
176
|
}
|
|
178
177
|
|
|
@@ -189,8 +188,7 @@ export async function executeSchedulerPoll(
|
|
|
189
188
|
});
|
|
190
189
|
released.add(taskId);
|
|
191
190
|
dbCleanupIds.push(taskId);
|
|
192
|
-
|
|
193
|
-
cleanupPipeline.zrem(`notif:scheduled:zset:${shard}`, taskStr);
|
|
191
|
+
cleanupPipeline.zrem(zsetKey(shardOf(taskId)), taskStr);
|
|
194
192
|
continue;
|
|
195
193
|
}
|
|
196
194
|
|
|
@@ -202,8 +200,7 @@ export async function executeSchedulerPoll(
|
|
|
202
200
|
|
|
203
201
|
released.add(taskId);
|
|
204
202
|
dbCleanupIds.push(taskId);
|
|
205
|
-
|
|
206
|
-
cleanupPipeline.zrem(`notif:scheduled:zset:${shard}`, taskStr);
|
|
203
|
+
cleanupPipeline.zrem(zsetKey(shardOf(taskId)), taskStr);
|
|
207
204
|
}
|
|
208
205
|
|
|
209
206
|
// Every bucket, not a hardcoded subset: a task binned into one that is not
|
|
@@ -298,9 +295,8 @@ export async function rebuildScheduledQueue(
|
|
|
298
295
|
for (const { taskId, payload } of rows) {
|
|
299
296
|
const sendAt = Date.parse(payload?.scheduledAt ?? "");
|
|
300
297
|
const member = JSON.stringify({ taskId, enrichedEventId: payload?.enrichedEventId });
|
|
301
|
-
const shard = parseInt(taskId.slice(-1), 16) || 0;
|
|
302
298
|
pipeline.zadd(
|
|
303
|
-
|
|
299
|
+
zsetKey(shardOf(taskId)),
|
|
304
300
|
"NX",
|
|
305
301
|
Number.isNaN(sendAt) ? Date.now() : sendAt,
|
|
306
302
|
member,
|
|
@@ -332,11 +328,6 @@ export async function startSchedulerWorker() {
|
|
|
332
328
|
|
|
333
329
|
const outboundProducers = createPriorityProducers(redis.native, logger, "OUTBOUND");
|
|
334
330
|
|
|
335
|
-
// ─── Stage 4a: Scheduled notification processor ─────────────────────────────
|
|
336
|
-
//
|
|
337
|
-
// Reads from SCHEDULED stream, parses, and queues tasks in a Redis ZSET.
|
|
338
|
-
// A background polling interval releases tasks when they are due.
|
|
339
|
-
|
|
340
331
|
worker = new SchedulerWorker({
|
|
341
332
|
consumer,
|
|
342
333
|
pendingScanner,
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
/** Seconds per unit. Everything is read as seconds first, then scheduled in ms. */
|
|
2
|
+
const DURATION_UNIT_SECONDS: Record<string, number> = {
|
|
3
|
+
ms: 0.001,
|
|
4
|
+
s: 1,
|
|
5
|
+
m: 60,
|
|
6
|
+
h: 3_600,
|
|
7
|
+
d: 86_400,
|
|
8
|
+
};
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* `"500ms"`, `"0.5s"`, `"15m"`, `"1.5h"`, `"2.5d"` to seconds. Decimals are
|
|
12
|
+
* read exactly; anything that is not a number and a unit throws. The old
|
|
13
|
+
* parser matched on the last character, so "500ms" meant 500 seconds, and ran
|
|
14
|
+
* parseInt, so "1.5h" meant one hour.
|
|
15
|
+
*/
|
|
16
|
+
export function parseDurationSeconds(value: string, what: string): number {
|
|
17
|
+
const match = /^(\d+(?:\.\d+)?|\.\d+)(ms|s|m|h|d)$/.exec(value.trim());
|
|
18
|
+
if (!match) throw new Error(`Invalid ${what}: ${value}`);
|
|
19
|
+
return Number(match[1]) * DURATION_UNIT_SECONDS[match[2]!]!;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
export function durationToMs(value: string, what: string): number {
|
|
23
|
+
return Math.round(parseDurationSeconds(value, what) * 1000);
|
|
24
|
+
}
|
|
@@ -32,44 +32,19 @@ import {
|
|
|
32
32
|
shutdownWorker,
|
|
33
33
|
uniqueConsumerId,
|
|
34
34
|
} from "@/workers/bootstrap.js";
|
|
35
|
+
import { WORKFLOW_TIMERS_KEY, pollWorkflowTimers, reapStuckInstances } from "./timers.js";
|
|
36
|
+
import { durationToMs, parseDurationSeconds } from "./duration.js";
|
|
37
|
+
|
|
38
|
+
export { WORKFLOW_TIMERS_KEY, parseDurationSeconds };
|
|
35
39
|
|
|
36
40
|
/** How long a workflow instance lock is held before it self-expires. */
|
|
37
41
|
const WORKFLOW_LOCK_TTL_SECONDS = 60;
|
|
38
42
|
/** Renew the lock well inside its TTL so long-running handlers keep it. */
|
|
39
43
|
const WORKFLOW_LOCK_RENEW_MS = (WORKFLOW_LOCK_TTL_SECONDS / 3) * 1000;
|
|
40
|
-
/** How long a claimed workflow timer stays invisible to other pollers. */
|
|
41
|
-
const WORKFLOW_TIMER_VISIBILITY_MS = 60_000;
|
|
42
|
-
/** Wake-ups for sleeping instances: waits, event timeouts, deferred resumes. */
|
|
43
|
-
export const WORKFLOW_TIMERS_KEY = "notif:workflow:timers";
|
|
44
44
|
/** How soon a resume that found its instance busy is tried again. */
|
|
45
45
|
const WORKFLOW_RESUME_RETRY_MS = 2_000;
|
|
46
46
|
const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
|
47
47
|
|
|
48
|
-
/** Seconds per unit. Everything is read as seconds first, then scheduled in ms. */
|
|
49
|
-
const DURATION_UNIT_SECONDS: Record<string, number> = {
|
|
50
|
-
ms: 0.001,
|
|
51
|
-
s: 1,
|
|
52
|
-
m: 60,
|
|
53
|
-
h: 3_600,
|
|
54
|
-
d: 86_400,
|
|
55
|
-
};
|
|
56
|
-
|
|
57
|
-
/**
|
|
58
|
-
* `"500ms"`, `"0.5s"`, `"15m"`, `"1.5h"`, `"2.5d"` to seconds. Decimals are
|
|
59
|
-
* read exactly; anything that is not a number and a unit throws. The old
|
|
60
|
-
* parser matched on the last character, so "500ms" meant 500 seconds, and ran
|
|
61
|
-
* parseInt, so "1.5h" meant one hour.
|
|
62
|
-
*/
|
|
63
|
-
export function parseDurationSeconds(value: string, what: string): number {
|
|
64
|
-
const match = /^(\d+(?:\.\d+)?|\.\d+)(ms|s|m|h|d)$/.exec(value.trim());
|
|
65
|
-
if (!match) throw new Error(`Invalid ${what}: ${value}`);
|
|
66
|
-
return Number(match[1]) * DURATION_UNIT_SECONDS[match[2]!]!;
|
|
67
|
-
}
|
|
68
|
-
|
|
69
|
-
function durationToMs(value: string, what: string): number {
|
|
70
|
-
return Math.round(parseDurationSeconds(value, what) * 1000);
|
|
71
|
-
}
|
|
72
|
-
|
|
73
48
|
/** Thrown at a step boundary once the instance has been canceled. */
|
|
74
49
|
class WorkflowCanceledError extends Error {
|
|
75
50
|
constructor() {
|
|
@@ -105,7 +80,6 @@ export interface WorkflowWorkerOptions extends WorkerOptions {
|
|
|
105
80
|
export class WorkflowWorker extends BaseWorker {
|
|
106
81
|
private readonly redisCli: Redis;
|
|
107
82
|
private readonly dbConn: any;
|
|
108
|
-
private readonly workflowProducer: any;
|
|
109
83
|
private readonly notificationProducer: any;
|
|
110
84
|
|
|
111
85
|
private eventBuffer: {
|
|
@@ -120,7 +94,6 @@ export class WorkflowWorker extends BaseWorker {
|
|
|
120
94
|
super(options);
|
|
121
95
|
this.redisCli = options.redis;
|
|
122
96
|
this.dbConn = options.db;
|
|
123
|
-
this.workflowProducer = options.workflowProducer;
|
|
124
97
|
this.notificationProducer = options.notificationProducer;
|
|
125
98
|
|
|
126
99
|
this.flushTimer = setInterval(() => void this.flushWorkerBuffers(), 100);
|
|
@@ -190,8 +163,6 @@ export class WorkflowWorker extends BaseWorker {
|
|
|
190
163
|
|
|
191
164
|
async process(message: StreamMessage): Promise<void> {
|
|
192
165
|
const { event } = message;
|
|
193
|
-
const publishPromises: Promise<void>[] = [];
|
|
194
|
-
|
|
195
166
|
if (event.type !== "workflow.triggered" && event.type !== "workflow.resumed") {
|
|
196
167
|
return;
|
|
197
168
|
}
|
|
@@ -547,8 +518,6 @@ export class WorkflowWorker extends BaseWorker {
|
|
|
547
518
|
// Compare-and-delete: never release a lock a later process re-acquired.
|
|
548
519
|
await this.redisCli.releaseLock(lockKey, lockToken);
|
|
549
520
|
}
|
|
550
|
-
|
|
551
|
-
await Promise.all(publishPromises);
|
|
552
521
|
}
|
|
553
522
|
}
|
|
554
523
|
|
|
@@ -592,139 +561,12 @@ export async function startWorkflowWorker() {
|
|
|
592
561
|
notificationProducer,
|
|
593
562
|
});
|
|
594
563
|
|
|
595
|
-
// Polling loop for timers
|
|
596
564
|
pollInterval = setInterval(() => {
|
|
597
|
-
void (
|
|
598
|
-
try {
|
|
599
|
-
const now = Date.now();
|
|
600
|
-
const tasks = await redis.native.schedulerPoll(
|
|
601
|
-
WORKFLOW_TIMERS_KEY,
|
|
602
|
-
now,
|
|
603
|
-
100,
|
|
604
|
-
WORKFLOW_TIMER_VISIBILITY_MS,
|
|
605
|
-
);
|
|
606
|
-
|
|
607
|
-
for (const taskStr of tasks) {
|
|
608
|
-
const task = JSON.parse(taskStr);
|
|
609
|
-
|
|
610
|
-
// Check if workflow is already completed/failed
|
|
611
|
-
const inst = (
|
|
612
|
-
await db
|
|
613
|
-
.select()
|
|
614
|
-
.from(workflowInstances)
|
|
615
|
-
.where(eq(workflowInstances.id, task.instanceId))
|
|
616
|
-
.limit(1)
|
|
617
|
-
)[0];
|
|
618
|
-
if (
|
|
619
|
-
!inst ||
|
|
620
|
-
inst.status === "completed" ||
|
|
621
|
-
inst.status === "failed" ||
|
|
622
|
-
inst.status === "canceled"
|
|
623
|
-
) {
|
|
624
|
-
await redis.native.zrem(WORKFLOW_TIMERS_KEY, taskStr);
|
|
625
|
-
continue;
|
|
626
|
-
}
|
|
627
|
-
// A timer can be observed while the worker is still persisting the
|
|
628
|
-
// corresponding suspension. Leave the claimed member in place; its
|
|
629
|
-
// visibility timeout will make it eligible once the instance becomes
|
|
630
|
-
// pending instead of losing the wake-up signal.
|
|
631
|
-
if (inst.status === "running") continue;
|
|
632
|
-
|
|
633
|
-
if (task.isEventTimeout) {
|
|
634
|
-
// A timeout for one waitForEvent step. If that step already has an
|
|
635
|
-
// output the event came first and this timer is stale; acting on it
|
|
636
|
-
// would time out a later wait on the same event name.
|
|
637
|
-
if (task.stepId !== undefined) {
|
|
638
|
-
const [own] = await db
|
|
639
|
-
.select({ output: workflowSteps.output })
|
|
640
|
-
.from(workflowSteps)
|
|
641
|
-
.where(
|
|
642
|
-
and(
|
|
643
|
-
eq(workflowSteps.instanceId, task.instanceId),
|
|
644
|
-
eq(workflowSteps.stepIndex, String(task.stepId)),
|
|
645
|
-
),
|
|
646
|
-
)
|
|
647
|
-
.limit(1);
|
|
648
|
-
if (!own || own.output !== null) {
|
|
649
|
-
await redis.native.zrem(WORKFLOW_TIMERS_KEY, taskStr);
|
|
650
|
-
continue;
|
|
651
|
-
}
|
|
652
|
-
}
|
|
653
|
-
|
|
654
|
-
// Clean up only this step's waiter
|
|
655
|
-
const deleted = await db
|
|
656
|
-
.delete(workflowWaiters)
|
|
657
|
-
.where(
|
|
658
|
-
and(
|
|
659
|
-
eq(workflowWaiters.instanceId, task.instanceId),
|
|
660
|
-
eq(workflowWaiters.eventName, task.eventName),
|
|
661
|
-
),
|
|
662
|
-
)
|
|
663
|
-
.returning();
|
|
664
|
-
|
|
665
|
-
// If the waiter was already deleted (by EventWorker when event arrived before timeout),
|
|
666
|
-
// this timer is stale. Clean up from Redis and skip re-resuming the workflow.
|
|
667
|
-
if (deleted.length === 0) {
|
|
668
|
-
await redis.native.zrem(WORKFLOW_TIMERS_KEY, taskStr);
|
|
669
|
-
continue;
|
|
670
|
-
}
|
|
671
|
-
|
|
672
|
-
// Record timedOut state on the pending waitForEvent step
|
|
673
|
-
const steps = await db
|
|
674
|
-
.select()
|
|
675
|
-
.from(workflowSteps)
|
|
676
|
-
.where(
|
|
677
|
-
and(
|
|
678
|
-
eq(workflowSteps.instanceId, task.instanceId),
|
|
679
|
-
eq(workflowSteps.action, "waitForEvent"),
|
|
680
|
-
),
|
|
681
|
-
);
|
|
682
|
-
const pendingStep =
|
|
683
|
-
task.stepId !== undefined
|
|
684
|
-
? steps.find((s: any) => s.stepIndex === String(task.stepId) && s.output === null)
|
|
685
|
-
: steps.find((s: any) => s.output === null);
|
|
686
|
-
if (pendingStep) {
|
|
687
|
-
await db
|
|
688
|
-
.update(workflowSteps)
|
|
689
|
-
.set({ output: { timedOut: true } })
|
|
690
|
-
.where(eq(workflowSteps.id, pendingStep.id));
|
|
691
|
-
}
|
|
692
|
-
}
|
|
693
|
-
|
|
694
|
-
await workflowProducer.publish(
|
|
695
|
-
buildStreamEvent("workflow.resumed", task, "scheduler", undefined),
|
|
696
|
-
);
|
|
697
|
-
await redis.native.zrem(WORKFLOW_TIMERS_KEY, taskStr);
|
|
698
|
-
logger.info({ instanceId: task.instanceId }, "Workflow resumed from timer");
|
|
699
|
-
}
|
|
700
|
-
} catch (err) {
|
|
701
|
-
logger.error({ err }, "error in workflow polling loop");
|
|
702
|
-
}
|
|
703
|
-
})();
|
|
565
|
+
void pollWorkflowTimers({ redis, db, logger, workflowProducer });
|
|
704
566
|
}, 5000);
|
|
705
567
|
|
|
706
568
|
reaperInterval = setInterval(() => {
|
|
707
|
-
void (
|
|
708
|
-
try {
|
|
709
|
-
const runningInstances = await db
|
|
710
|
-
.select({ id: workflowInstances.id })
|
|
711
|
-
.from(workflowInstances)
|
|
712
|
-
.where(eq(workflowInstances.status, "running"));
|
|
713
|
-
for (const inst of runningInstances) {
|
|
714
|
-
const lockKey = `lock:workflow:${inst.id}`;
|
|
715
|
-
const hasLock = await redis.native.exists(lockKey);
|
|
716
|
-
if (!hasLock) {
|
|
717
|
-
await db
|
|
718
|
-
.update(workflowInstances)
|
|
719
|
-
.set({ status: "pending" })
|
|
720
|
-
.where(eq(workflowInstances.id, inst.id));
|
|
721
|
-
logger.info({ instanceId: inst.id }, "Reaped stuck workflow instance (lock expired)");
|
|
722
|
-
}
|
|
723
|
-
}
|
|
724
|
-
} catch (err) {
|
|
725
|
-
logger.error({ err }, "error in stuck workflow reaper");
|
|
726
|
-
}
|
|
727
|
-
})();
|
|
569
|
+
void reapStuckInstances({ redis, db, logger });
|
|
728
570
|
}, 60000);
|
|
729
571
|
|
|
730
572
|
healthInterval = startHealthReporter("workflow", worker, redis, logger);
|
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
import type { Logger, RedisClient, StreamProducer } from "@/index.js";
|
|
2
|
+
import { buildStreamEvent } from "@/index.js";
|
|
3
|
+
import { workflowInstances, workflowSteps, workflowWaiters } from "@/db/schema.js";
|
|
4
|
+
import { eq, and } from "drizzle-orm";
|
|
5
|
+
|
|
6
|
+
/** How long a claimed workflow timer stays invisible to other pollers. */
|
|
7
|
+
const WORKFLOW_TIMER_VISIBILITY_MS = 60_000;
|
|
8
|
+
/** Wake-ups for sleeping instances: waits, event timeouts, deferred resumes. */
|
|
9
|
+
export const WORKFLOW_TIMERS_KEY = "notif:workflow:timers";
|
|
10
|
+
|
|
11
|
+
export interface WorkflowTimerDeps {
|
|
12
|
+
redis: RedisClient;
|
|
13
|
+
db: any;
|
|
14
|
+
logger: Logger;
|
|
15
|
+
workflowProducer: StreamProducer;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
/** Resumes instances whose wait, event timeout or deferred resume is due. */
|
|
19
|
+
export async function pollWorkflowTimers({
|
|
20
|
+
redis,
|
|
21
|
+
db,
|
|
22
|
+
logger,
|
|
23
|
+
workflowProducer,
|
|
24
|
+
}: WorkflowTimerDeps): Promise<void> {
|
|
25
|
+
try {
|
|
26
|
+
const now = Date.now();
|
|
27
|
+
const tasks = await redis.native.schedulerPoll(
|
|
28
|
+
WORKFLOW_TIMERS_KEY,
|
|
29
|
+
now,
|
|
30
|
+
100,
|
|
31
|
+
WORKFLOW_TIMER_VISIBILITY_MS,
|
|
32
|
+
);
|
|
33
|
+
|
|
34
|
+
for (const taskStr of tasks) {
|
|
35
|
+
const task = JSON.parse(taskStr);
|
|
36
|
+
|
|
37
|
+
// Check if workflow is already completed/failed
|
|
38
|
+
const inst = (
|
|
39
|
+
await db
|
|
40
|
+
.select()
|
|
41
|
+
.from(workflowInstances)
|
|
42
|
+
.where(eq(workflowInstances.id, task.instanceId))
|
|
43
|
+
.limit(1)
|
|
44
|
+
)[0];
|
|
45
|
+
if (
|
|
46
|
+
!inst ||
|
|
47
|
+
inst.status === "completed" ||
|
|
48
|
+
inst.status === "failed" ||
|
|
49
|
+
inst.status === "canceled"
|
|
50
|
+
) {
|
|
51
|
+
await redis.native.zrem(WORKFLOW_TIMERS_KEY, taskStr);
|
|
52
|
+
continue;
|
|
53
|
+
}
|
|
54
|
+
// A timer can be observed while the worker is still persisting the
|
|
55
|
+
// corresponding suspension. Leave the claimed member in place; its
|
|
56
|
+
// visibility timeout will make it eligible once the instance becomes
|
|
57
|
+
// pending instead of losing the wake-up signal.
|
|
58
|
+
if (inst.status === "running") continue;
|
|
59
|
+
|
|
60
|
+
if (task.isEventTimeout) {
|
|
61
|
+
// A timeout for one waitForEvent step. If that step already has an
|
|
62
|
+
// output the event came first and this timer is stale; acting on it
|
|
63
|
+
// would time out a later wait on the same event name.
|
|
64
|
+
if (task.stepId !== undefined) {
|
|
65
|
+
const [own] = await db
|
|
66
|
+
.select({ output: workflowSteps.output })
|
|
67
|
+
.from(workflowSteps)
|
|
68
|
+
.where(
|
|
69
|
+
and(
|
|
70
|
+
eq(workflowSteps.instanceId, task.instanceId),
|
|
71
|
+
eq(workflowSteps.stepIndex, String(task.stepId)),
|
|
72
|
+
),
|
|
73
|
+
)
|
|
74
|
+
.limit(1);
|
|
75
|
+
if (!own || own.output !== null) {
|
|
76
|
+
await redis.native.zrem(WORKFLOW_TIMERS_KEY, taskStr);
|
|
77
|
+
continue;
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
// Clean up only this step's waiter
|
|
82
|
+
const deleted = await db
|
|
83
|
+
.delete(workflowWaiters)
|
|
84
|
+
.where(
|
|
85
|
+
and(
|
|
86
|
+
eq(workflowWaiters.instanceId, task.instanceId),
|
|
87
|
+
eq(workflowWaiters.eventName, task.eventName),
|
|
88
|
+
),
|
|
89
|
+
)
|
|
90
|
+
.returning();
|
|
91
|
+
|
|
92
|
+
// If the waiter was already deleted (by EventWorker when event arrived before timeout),
|
|
93
|
+
// this timer is stale. Clean up from Redis and skip re-resuming the workflow.
|
|
94
|
+
if (deleted.length === 0) {
|
|
95
|
+
await redis.native.zrem(WORKFLOW_TIMERS_KEY, taskStr);
|
|
96
|
+
continue;
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
// Record timedOut state on the pending waitForEvent step
|
|
100
|
+
const steps = await db
|
|
101
|
+
.select()
|
|
102
|
+
.from(workflowSteps)
|
|
103
|
+
.where(
|
|
104
|
+
and(
|
|
105
|
+
eq(workflowSteps.instanceId, task.instanceId),
|
|
106
|
+
eq(workflowSteps.action, "waitForEvent"),
|
|
107
|
+
),
|
|
108
|
+
);
|
|
109
|
+
const pendingStep =
|
|
110
|
+
task.stepId !== undefined
|
|
111
|
+
? steps.find((s: any) => s.stepIndex === String(task.stepId) && s.output === null)
|
|
112
|
+
: steps.find((s: any) => s.output === null);
|
|
113
|
+
if (pendingStep) {
|
|
114
|
+
await db
|
|
115
|
+
.update(workflowSteps)
|
|
116
|
+
.set({ output: { timedOut: true } })
|
|
117
|
+
.where(eq(workflowSteps.id, pendingStep.id));
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
await workflowProducer.publish(
|
|
122
|
+
buildStreamEvent("workflow.resumed", task, "scheduler", undefined),
|
|
123
|
+
);
|
|
124
|
+
await redis.native.zrem(WORKFLOW_TIMERS_KEY, taskStr);
|
|
125
|
+
logger.info({ instanceId: task.instanceId }, "Workflow resumed from timer");
|
|
126
|
+
}
|
|
127
|
+
} catch (err) {
|
|
128
|
+
logger.error({ err }, "error in workflow polling loop");
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
/** Sets `running` instances whose lock has expired back to `pending`. */
|
|
133
|
+
export async function reapStuckInstances({
|
|
134
|
+
redis,
|
|
135
|
+
db,
|
|
136
|
+
logger,
|
|
137
|
+
}: Omit<WorkflowTimerDeps, "workflowProducer">): Promise<void> {
|
|
138
|
+
try {
|
|
139
|
+
const runningInstances = await db
|
|
140
|
+
.select({ id: workflowInstances.id })
|
|
141
|
+
.from(workflowInstances)
|
|
142
|
+
.where(eq(workflowInstances.status, "running"));
|
|
143
|
+
for (const inst of runningInstances) {
|
|
144
|
+
const lockKey = `lock:workflow:${inst.id}`;
|
|
145
|
+
const hasLock = await redis.native.exists(lockKey);
|
|
146
|
+
if (!hasLock) {
|
|
147
|
+
await db
|
|
148
|
+
.update(workflowInstances)
|
|
149
|
+
.set({ status: "pending" })
|
|
150
|
+
.where(eq(workflowInstances.id, inst.id));
|
|
151
|
+
logger.info({ instanceId: inst.id }, "Reaped stuck workflow instance (lock expired)");
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
} catch (err) {
|
|
155
|
+
logger.error({ err }, "error in stuck workflow reaper");
|
|
156
|
+
}
|
|
157
|
+
}
|
package/src/shared/events.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { EventEmitter } from "node:events";
|
|
2
2
|
|
|
3
3
|
export const globalEmitter = new EventEmitter();
|
|
4
|
-
//
|
|
5
|
-
//
|
|
4
|
+
// Listeners: each NotifkitServer's event forwarding (src/server.ts) and
|
|
5
|
+
// delivery's live-event publisher. Several servers in one process (tests)
|
|
6
|
+
// can pass the default limit without leaking.
|
|
6
7
|
globalEmitter.setMaxListeners(0);
|
package/src/templates/render.ts
CHANGED
|
@@ -14,6 +14,7 @@ export function escapeHeader(unsafe: string): string {
|
|
|
14
14
|
.trim();
|
|
15
15
|
}
|
|
16
16
|
|
|
17
|
+
/** @deprecated Unused; templates render through renderWithTemplate. Removed in the next minor version. */
|
|
17
18
|
export function interpolate(
|
|
18
19
|
tmpl: string,
|
|
19
20
|
variables: Record<string, unknown>,
|
package/src/workers/index.ts
CHANGED
|
@@ -134,12 +134,15 @@ export abstract class BaseWorker {
|
|
|
134
134
|
this.stopRecoveryLoop();
|
|
135
135
|
|
|
136
136
|
if (this.runLoop) {
|
|
137
|
+
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
137
138
|
await Promise.race([
|
|
138
139
|
this.runLoop,
|
|
139
|
-
new Promise((_, reject) =>
|
|
140
|
-
setTimeout(() => reject(new Error("Worker stop timeout")), 30_000)
|
|
141
|
-
),
|
|
142
|
-
])
|
|
140
|
+
new Promise((_, reject) => {
|
|
141
|
+
timer = setTimeout(() => reject(new Error("Worker stop timeout")), 30_000);
|
|
142
|
+
}),
|
|
143
|
+
])
|
|
144
|
+
.catch((err) => this.logger.warn({ err }, "Worker shutdown timeout or error"))
|
|
145
|
+
.finally(() => clearTimeout(timer));
|
|
143
146
|
}
|
|
144
147
|
|
|
145
148
|
this.logger.info("worker shutdown complete");
|
package/src/workflows/sdk.ts
CHANGED