notifkit 0.1.10 → 0.1.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +37 -1
- package/dashboard/dist/assets/{index-7mvXb4bS.js → index-Bsrmo9LE.js} +1 -1
- package/dashboard/dist/assets/index-s-SXGIwv.css +1 -0
- package/dashboard/dist/index.html +2 -2
- package/dist/bootstrap-omGPIBny.mjs +89 -0
- package/dist/bulk-C4hTcnLu.mjs +77 -0
- package/dist/index.d.mts +1160 -8886
- package/dist/index.mjs +2 -38
- package/dist/{main-C9s-2ArH.mjs → main-B-AZ-xoN.mjs} +49 -75
- package/dist/main-B5BwjIMu.mjs +811 -0
- package/dist/{main-B3iQrl3t.mjs → main-BU27VjDC.mjs} +149 -134
- package/dist/main-Bes11Ad3.mjs +312 -0
- package/dist/{main-BiFiOX9D.mjs → main-BtnGmyaT.mjs} +119 -112
- package/dist/{main-39DtKrX4.mjs → main-BxAeC_k6.mjs} +395 -184
- package/dist/{main-Dz2sUh3c.mjs → main-DAXO-MiH.mjs} +57 -54
- package/dist/{main-BgmT1MPe.mjs → main-DYGCl7Ja.mjs} +135 -79
- package/dist/{src-QiTu5e_a.mjs → src-CIPTzq9i.mjs} +1351 -661
- package/drizzle/0006_message_logs_redundant_indexes.sql +7 -0
- package/drizzle/0007_workflow_status_canceled.sql +3 -0
- package/drizzle/meta/_journal.json +14 -0
- package/package.json +21 -18
- package/scripts/create-admin.mjs +8 -1
- package/scripts/migrate.mjs +43 -0
- package/src/client.ts +15 -5
- package/src/config/index.ts +37 -3
- package/src/contracts/events/notification-ai-pending.ts +12 -0
- package/src/contracts/events/notification-delivered.ts +7 -0
- package/src/contracts/events/notification-dispatched.ts +21 -2
- package/src/contracts/events/notification-enriched.ts +16 -0
- package/src/contracts/events/notification-failed.ts +2 -0
- package/src/contracts/events/notification-requested.ts +2 -0
- package/src/contracts/registry.ts +2 -2
- package/src/contracts/sdk.ts +77 -29
- package/src/db/bulk.ts +132 -0
- package/src/db/index.ts +174 -7
- package/src/db/schema.ts +3 -50
- package/src/idempotency/index.ts +60 -6
- package/src/logger/index.ts +0 -12
- package/src/queue/index.ts +255 -76
- package/src/rate-limiter/index.ts +217 -16
- package/src/redis/index.ts +265 -0
- package/src/repositories/index.ts +171 -82
- package/src/server.ts +64 -41
- package/src/services/ai/main.ts +83 -70
- package/src/services/api/admin-static.ts +41 -12
- package/src/services/api/handlers.ts +357 -159
- package/src/services/api/http.ts +34 -7
- package/src/services/api/main.ts +104 -67
- package/src/services/api/router.ts +18 -5
- package/src/services/delivery/main.ts +495 -230
- package/src/services/delivery/throttle.ts +42 -59
- package/src/services/engine/main.ts +206 -152
- package/src/services/enricher/main.ts +155 -107
- package/src/services/events/main.ts +100 -55
- package/src/services/scheduler/main.ts +151 -79
- package/src/services/scheduler/send-gate.ts +117 -0
- package/src/services/workflow/main.ts +220 -100
- package/src/shared/batch-processor.ts +65 -34
- package/src/shared/dataloader.ts +58 -22
- package/src/shared/events.ts +3 -0
- package/src/shared/index.ts +0 -1
- package/src/shared/utils.ts +2 -47
- package/src/templates/index.ts +0 -67
- package/src/transport/index.ts +6 -0
- package/src/workers/bootstrap.ts +124 -0
- package/src/workers/index.ts +99 -24
- package/src/workflows/sdk.ts +2 -0
- package/dashboard/dist/assets/index-o7L-f3-c.css +0 -1
- package/dist/index.d.mts.map +0 -1
- package/dist/index.mjs.map +0 -1
- package/dist/main-39DtKrX4.mjs.map +0 -1
- package/dist/main-B3iQrl3t.mjs.map +0 -1
- package/dist/main-BM8DH3v3.mjs +0 -632
- package/dist/main-BM8DH3v3.mjs.map +0 -1
- package/dist/main-BgmT1MPe.mjs.map +0 -1
- package/dist/main-BiFiOX9D.mjs.map +0 -1
- package/dist/main-C9s-2ArH.mjs.map +0 -1
- package/dist/main-Cqo3rSb5.mjs +0 -224
- package/dist/main-Cqo3rSb5.mjs.map +0 -1
- package/dist/main-Dz2sUh3c.mjs.map +0 -1
- package/dist/src-QiTu5e_a.mjs.map +0 -1
|
@@ -1,4 +1,6 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { $t as STREAMS, Jt as CONSUMER_GROUPS, K as StreamProducer, c as BaseWorker, d as startHealthReporter, fn as loadEnv, ft as workflowInstances, mn as readBaseConfig, mt as workflowWaiters, pt as workflowSteps, qt as buildStreamEvent } from "./src-CIPTzq9i.mjs";
|
|
2
|
+
import { a as uniqueConsumerId, i as shutdownWorker, n as createStreamConsumer, r as createWorkerRuntime } from "./bootstrap-omGPIBny.mjs";
|
|
3
|
+
import { t as insertMessageLogs } from "./bulk-C4hTcnLu.mjs";
|
|
2
4
|
import { and, eq, gt, isNull, or } from "drizzle-orm";
|
|
3
5
|
//#region src/services/events/main.ts
|
|
4
6
|
loadEnv();
|
|
@@ -12,17 +14,28 @@ let pendingScanner;
|
|
|
12
14
|
let worker;
|
|
13
15
|
let healthInterval = null;
|
|
14
16
|
let workflowProducer;
|
|
15
|
-
var EventWorker = class extends BaseWorker {
|
|
17
|
+
var EventWorker = class EventWorker extends BaseWorker {
|
|
16
18
|
dbConn;
|
|
17
19
|
workflowProducer;
|
|
18
20
|
messageLogBuffer = [];
|
|
19
21
|
eventBuffer = [];
|
|
20
22
|
flushInterval = null;
|
|
21
|
-
|
|
23
|
+
flushesInFlight = 0;
|
|
24
|
+
static MAX_CONCURRENT_FLUSHES = 4;
|
|
25
|
+
/**
|
|
26
|
+
* Buffered logs that trigger a flush without waiting for the interval.
|
|
27
|
+
*
|
|
28
|
+
* Every message holds its worker slot until its log row is written, so a
|
|
29
|
+
* purely timed flush caps the worker at `concurrency / LOG_FLUSH_INTERVAL_MS`
|
|
30
|
+
* rows per second — far below what one delivery worker produces. Flushing
|
|
31
|
+
* once half the slots are waiting keeps the other half reading.
|
|
32
|
+
*/
|
|
33
|
+
flushThreshold;
|
|
22
34
|
constructor(options) {
|
|
23
35
|
super(options);
|
|
24
36
|
this.dbConn = options.db;
|
|
25
37
|
this.workflowProducer = options.workflowProducer;
|
|
38
|
+
this.flushThreshold = Math.max(1, Math.min(500, Math.floor(this.concurrency / 2)));
|
|
26
39
|
this.startFlushInterval();
|
|
27
40
|
}
|
|
28
41
|
startFlushInterval() {
|
|
@@ -31,8 +44,8 @@ var EventWorker = class extends BaseWorker {
|
|
|
31
44
|
}, config.LOG_FLUSH_INTERVAL_MS);
|
|
32
45
|
}
|
|
33
46
|
async flushLogs() {
|
|
34
|
-
if (this.
|
|
35
|
-
this.
|
|
47
|
+
if (this.flushesInFlight >= EventWorker.MAX_CONCURRENT_FLUSHES || this.messageLogBuffer.length === 0 && this.eventBuffer.length === 0) return;
|
|
48
|
+
this.flushesInFlight++;
|
|
36
49
|
const startTime = Date.now();
|
|
37
50
|
const batch = this.messageLogBuffer;
|
|
38
51
|
this.messageLogBuffer = [];
|
|
@@ -40,9 +53,9 @@ var EventWorker = class extends BaseWorker {
|
|
|
40
53
|
this.eventBuffer = [];
|
|
41
54
|
try {
|
|
42
55
|
if (batch.length > 0) {
|
|
43
|
-
await this.dbConn
|
|
56
|
+
await insertMessageLogs(this.dbConn, batch.map((b) => b.log));
|
|
44
57
|
for (const b of batch) b.resolve();
|
|
45
|
-
this.logger.
|
|
58
|
+
this.logger.debug({
|
|
46
59
|
batchSize: batch.length,
|
|
47
60
|
latencyMs: Date.now() - startTime
|
|
48
61
|
}, "flushed delivery logs to database");
|
|
@@ -66,13 +79,18 @@ var EventWorker = class extends BaseWorker {
|
|
|
66
79
|
for (const b of batch) b.reject(error);
|
|
67
80
|
for (const e of events) e.reject(error);
|
|
68
81
|
} finally {
|
|
69
|
-
this.
|
|
82
|
+
this.flushesInFlight--;
|
|
83
|
+
if (this.messageLogBuffer.length >= this.flushThreshold) this.flushLogs();
|
|
70
84
|
}
|
|
71
85
|
}
|
|
72
86
|
async stop() {
|
|
73
|
-
if (this.flushInterval) clearInterval(this.flushInterval);
|
|
74
87
|
await super.stop();
|
|
75
|
-
|
|
88
|
+
if (this.flushInterval) clearInterval(this.flushInterval);
|
|
89
|
+
const deadline = Date.now() + 1e4;
|
|
90
|
+
while ((this.messageLogBuffer.length > 0 || this.eventBuffer.length > 0 || this.flushesInFlight > 0) && Date.now() < deadline) {
|
|
91
|
+
await this.flushLogs();
|
|
92
|
+
await new Promise((resolve) => setTimeout(resolve, 10));
|
|
93
|
+
}
|
|
76
94
|
}
|
|
77
95
|
async process(message) {
|
|
78
96
|
const { event } = message;
|
|
@@ -89,24 +107,28 @@ var EventWorker = class extends BaseWorker {
|
|
|
89
107
|
}
|
|
90
108
|
const status = event.type === "notification.dispatched" ? "dispatched" : event.type === "notification.delivered" ? "delivered" : "failed";
|
|
91
109
|
const kind = event.type === "notification.dispatched" ? "dispatched" : "attempt";
|
|
92
|
-
|
|
110
|
+
const row = (rowKind, rowStatus, providerMessageId, timestamp) => new Promise((resolve, reject) => {
|
|
93
111
|
this.messageLogBuffer.push({
|
|
94
112
|
log: {
|
|
95
113
|
projectId: payload.projectId,
|
|
96
114
|
taskId: payload.taskId,
|
|
97
|
-
providerMessageId
|
|
115
|
+
providerMessageId,
|
|
98
116
|
channel: payload.channel,
|
|
99
117
|
attempt: payload.attempt || 1,
|
|
100
|
-
kind,
|
|
101
|
-
status,
|
|
118
|
+
kind: rowKind,
|
|
119
|
+
status: rowStatus,
|
|
102
120
|
templateId: payload.templateId || null,
|
|
103
121
|
workflowInstanceId: payload.workflowInstanceId || null,
|
|
104
|
-
campaignId: payload.campaignId || null
|
|
122
|
+
campaignId: payload.campaignId || null,
|
|
123
|
+
timestamp
|
|
105
124
|
},
|
|
106
125
|
resolve,
|
|
107
126
|
reject
|
|
108
127
|
});
|
|
109
|
-
})
|
|
128
|
+
});
|
|
129
|
+
if (kind === "attempt" && payload.dispatchedAt) publishPromises.push(row("dispatched", "dispatched", null, payload.dispatchedAt));
|
|
130
|
+
publishPromises.push(row(kind, status, payload.providerMessageId || null));
|
|
131
|
+
if (this.messageLogBuffer.length >= this.flushThreshold) this.flushLogs();
|
|
110
132
|
await Promise.all(publishPromises);
|
|
111
133
|
return;
|
|
112
134
|
}
|
|
@@ -171,48 +193,29 @@ var EventWorker = class extends BaseWorker {
|
|
|
171
193
|
}
|
|
172
194
|
};
|
|
173
195
|
async function startEventWorker() {
|
|
174
|
-
logger =
|
|
175
|
-
name: "event-worker",
|
|
176
|
-
level: config.LOG_LEVEL
|
|
177
|
-
});
|
|
178
|
-
redis = new RedisClient({
|
|
179
|
-
url: config.REDIS_URL,
|
|
180
|
-
name: "events",
|
|
181
|
-
logger
|
|
182
|
-
});
|
|
183
|
-
const dbData = createDatabase({
|
|
184
|
-
url: config.DATABASE_URL,
|
|
185
|
-
applicationName: "events",
|
|
186
|
-
logger
|
|
187
|
-
});
|
|
188
|
-
sql$1 = dbData.sql;
|
|
189
|
-
db = dbData.db;
|
|
196
|
+
({logger, redis, sql: sql$1, db} = createWorkerRuntime(config, "events", "event-worker"));
|
|
190
197
|
workflowProducer = new StreamProducer({
|
|
191
198
|
redis: redis.native,
|
|
192
199
|
stream: STREAMS.WORKFLOW_INBOUND,
|
|
193
200
|
logger
|
|
194
201
|
});
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
group: CONSUMER_GROUPS.EVENTS,
|
|
199
|
-
consumer: `events-${process.pid}`,
|
|
200
|
-
dlqStream: STREAMS.DEAD_LETTER,
|
|
201
|
-
batchSize: config.WORKER_CONCURRENCY,
|
|
202
|
+
const eventsConcurrency = Math.max(config.WORKER_CONCURRENCY, config.DELIVERY_CONCURRENCY ?? config.WORKER_CONCURRENCY);
|
|
203
|
+
({consumer, pendingScanner} = createStreamConsumer({
|
|
204
|
+
redis,
|
|
202
205
|
logger
|
|
203
|
-
}
|
|
204
|
-
pendingScanner = new PendingMessageScanner({
|
|
205
|
-
redis: redis.native,
|
|
206
|
+
}, {
|
|
206
207
|
stream: STREAMS.EVENTS_INBOUND,
|
|
207
208
|
group: CONSUMER_GROUPS.EVENTS,
|
|
208
|
-
consumer:
|
|
209
|
-
|
|
210
|
-
|
|
209
|
+
consumer: uniqueConsumerId("events"),
|
|
210
|
+
batchSize: eventsConcurrency,
|
|
211
|
+
bufferAcks: true,
|
|
212
|
+
coalesceMs: 10
|
|
213
|
+
}));
|
|
211
214
|
worker = new EventWorker({
|
|
212
215
|
consumer,
|
|
213
216
|
pendingScanner,
|
|
214
217
|
logger,
|
|
215
|
-
concurrency:
|
|
218
|
+
concurrency: eventsConcurrency,
|
|
216
219
|
db,
|
|
217
220
|
workflowProducer
|
|
218
221
|
});
|
|
@@ -221,14 +224,14 @@ async function startEventWorker() {
|
|
|
221
224
|
await worker.start();
|
|
222
225
|
}
|
|
223
226
|
async function stopEventWorker() {
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
227
|
+
await shutdownWorker("event worker", {
|
|
228
|
+
logger,
|
|
229
|
+
timers: [healthInterval],
|
|
230
|
+
worker,
|
|
231
|
+
sql: sql$1,
|
|
232
|
+
redis
|
|
233
|
+
});
|
|
234
|
+
healthInterval = null;
|
|
230
235
|
}
|
|
231
236
|
//#endregion
|
|
232
237
|
export { startEventWorker, stopEventWorker };
|
|
233
|
-
|
|
234
|
-
//# sourceMappingURL=main-Dz2sUh3c.mjs.map
|
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { $t as STREAMS, Jt as CONSUMER_GROUPS, K as StreamProducer, a as SuspendExecutionError, c as BaseWorker, d as startHealthReporter, dt as workflowDefinitions, fn as loadEnv, ft as workflowInstances, i as workflowRegistry, mn as readBaseConfig, mt as workflowWaiters, o as buildStepNotifyPayload, pt as workflowSteps, qt as buildStreamEvent } from "./src-CIPTzq9i.mjs";
|
|
2
|
+
import { a as uniqueConsumerId, i as shutdownWorker, n as createStreamConsumer, r as createWorkerRuntime } from "./bootstrap-omGPIBny.mjs";
|
|
2
3
|
import { randomUUID } from "node:crypto";
|
|
3
4
|
import { and, eq } from "drizzle-orm";
|
|
4
5
|
//#region src/services/workflow/main.ts
|
|
@@ -8,6 +9,40 @@ const WORKFLOW_LOCK_TTL_SECONDS = 60;
|
|
|
8
9
|
const WORKFLOW_LOCK_RENEW_MS = WORKFLOW_LOCK_TTL_SECONDS / 3 * 1e3;
|
|
9
10
|
/** How long a claimed workflow timer stays invisible to other pollers. */
|
|
10
11
|
const WORKFLOW_TIMER_VISIBILITY_MS = 6e4;
|
|
12
|
+
/** Wake-ups for sleeping instances: waits, event timeouts, deferred resumes. */
|
|
13
|
+
const WORKFLOW_TIMERS_KEY = "notif:workflow:timers";
|
|
14
|
+
/** How soon a resume that found its instance busy is tried again. */
|
|
15
|
+
const WORKFLOW_RESUME_RETRY_MS = 2e3;
|
|
16
|
+
const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
|
17
|
+
/** Seconds per unit. Everything is read as seconds first, then scheduled in ms. */
|
|
18
|
+
const DURATION_UNIT_SECONDS = {
|
|
19
|
+
ms: .001,
|
|
20
|
+
s: 1,
|
|
21
|
+
m: 60,
|
|
22
|
+
h: 3600,
|
|
23
|
+
d: 86400
|
|
24
|
+
};
|
|
25
|
+
/**
|
|
26
|
+
* `"500ms"`, `"0.5s"`, `"15m"`, `"1.5h"`, `"2.5d"` to seconds. Decimals are
|
|
27
|
+
* read exactly; anything that is not a number and a unit throws. The old
|
|
28
|
+
* parser matched on the last character, so "500ms" meant 500 seconds, and ran
|
|
29
|
+
* parseInt, so "1.5h" meant one hour.
|
|
30
|
+
*/
|
|
31
|
+
function parseDurationSeconds(value, what) {
|
|
32
|
+
const match = /^(\d+(?:\.\d+)?|\.\d+)(ms|s|m|h|d)$/.exec(value.trim());
|
|
33
|
+
if (!match) throw new Error(`Invalid ${what}: ${value}`);
|
|
34
|
+
return Number(match[1]) * DURATION_UNIT_SECONDS[match[2]];
|
|
35
|
+
}
|
|
36
|
+
function durationToMs(value, what) {
|
|
37
|
+
return Math.round(parseDurationSeconds(value, what) * 1e3);
|
|
38
|
+
}
|
|
39
|
+
/** Thrown at a step boundary once the instance has been canceled. */
|
|
40
|
+
var WorkflowCanceledError = class extends Error {
|
|
41
|
+
constructor() {
|
|
42
|
+
super("Workflow instance was canceled");
|
|
43
|
+
this.name = "WorkflowCanceledError";
|
|
44
|
+
}
|
|
45
|
+
};
|
|
11
46
|
loadEnv();
|
|
12
47
|
const config = readBaseConfig();
|
|
13
48
|
let logger;
|
|
@@ -38,12 +73,12 @@ var WorkflowWorker = class extends BaseWorker {
|
|
|
38
73
|
this.flushTimer = setInterval(() => void this.flushWorkerBuffers(), 100);
|
|
39
74
|
}
|
|
40
75
|
async stop() {
|
|
76
|
+
await super.stop();
|
|
41
77
|
if (this.flushTimer) {
|
|
42
78
|
clearInterval(this.flushTimer);
|
|
43
79
|
this.flushTimer = null;
|
|
44
80
|
}
|
|
45
81
|
await this.flushWorkerBuffers();
|
|
46
|
-
await super.stop();
|
|
47
82
|
}
|
|
48
83
|
async flushWorkerBuffers() {
|
|
49
84
|
if (this.eventBuffer.length === 0) return;
|
|
@@ -71,6 +106,24 @@ var WorkflowWorker = class extends BaseWorker {
|
|
|
71
106
|
for (const e of events) e.reject(err);
|
|
72
107
|
}
|
|
73
108
|
}
|
|
109
|
+
/**
|
|
110
|
+
* Re-delivers a resume through the timer ZSET a few seconds from now. The
|
|
111
|
+
* poll loop publishes it once the instance is pending again, and drops it if
|
|
112
|
+
* the instance has finished. The member carries no timestamp, so repeated
|
|
113
|
+
* deferrals of one instance collapse into a single entry. It is marked
|
|
114
|
+
* `deferred` so it never equals a `wait` timer's member: sharing one let the
|
|
115
|
+
* ZADD here re-score a sleep's wake-up to seconds from now, and the early
|
|
116
|
+
* resume then re-suspended with no timer left to wake it.
|
|
117
|
+
*/
|
|
118
|
+
async deferResume(payload) {
|
|
119
|
+
await this.redisCli.zadd(WORKFLOW_TIMERS_KEY, Date.now() + WORKFLOW_RESUME_RETRY_MS, JSON.stringify({
|
|
120
|
+
instanceId: payload.instanceId,
|
|
121
|
+
name: payload.name,
|
|
122
|
+
projectId: payload.projectId,
|
|
123
|
+
input: payload.input ?? {},
|
|
124
|
+
deferred: true
|
|
125
|
+
}));
|
|
126
|
+
}
|
|
74
127
|
async process(message) {
|
|
75
128
|
const { event } = message;
|
|
76
129
|
const publishPromises = [];
|
|
@@ -107,11 +160,12 @@ var WorkflowWorker = class extends BaseWorker {
|
|
|
107
160
|
const lockKey = `lock:workflow:${instanceId}`;
|
|
108
161
|
const lockToken = randomUUID();
|
|
109
162
|
if (!await this.redisCli.set(lockKey, lockToken, "EX", WORKFLOW_LOCK_TTL_SECONDS, "NX")) {
|
|
110
|
-
this.logger.info({ instanceId }, "Workflow is locked by another process,
|
|
163
|
+
this.logger.info({ instanceId }, "Workflow is locked by another process, deferring resume");
|
|
164
|
+
await this.deferResume(payload);
|
|
111
165
|
return;
|
|
112
166
|
}
|
|
113
167
|
const renewTimer = setInterval(() => {
|
|
114
|
-
this.redisCli.
|
|
168
|
+
this.redisCli.renewLock(lockKey, lockToken, WORKFLOW_LOCK_TTL_SECONDS).catch((err) => {
|
|
115
169
|
this.logger.warn({
|
|
116
170
|
err,
|
|
117
171
|
instanceId
|
|
@@ -127,11 +181,22 @@ var WorkflowWorker = class extends BaseWorker {
|
|
|
127
181
|
status: "pending",
|
|
128
182
|
input: payload.input || {}
|
|
129
183
|
}).returning())[0];
|
|
184
|
+
if (instance.status === "running") {
|
|
185
|
+
await this.deferResume(payload);
|
|
186
|
+
return;
|
|
187
|
+
}
|
|
130
188
|
if (instance.status !== "pending") {
|
|
131
189
|
this.logger.info({ instanceId }, "Workflow is not pending, skipping");
|
|
132
190
|
return;
|
|
133
191
|
}
|
|
134
|
-
await this.dbConn.update(workflowInstances).set({ status: "running" }).where(eq(workflowInstances.id, instanceId))
|
|
192
|
+
if ((await this.dbConn.update(workflowInstances).set({ status: "running" }).where(and(eq(workflowInstances.id, instanceId), eq(workflowInstances.status, "pending"))).returning({ id: workflowInstances.id })).length === 0) {
|
|
193
|
+
this.logger.info({ instanceId }, "Workflow is no longer pending, skipping");
|
|
194
|
+
return;
|
|
195
|
+
}
|
|
196
|
+
const stopIfCanceled = async () => {
|
|
197
|
+
const [row] = await this.dbConn.select({ status: workflowInstances.status }).from(workflowInstances).where(eq(workflowInstances.id, instanceId)).limit(1);
|
|
198
|
+
if (row?.status === "canceled") throw new WorkflowCanceledError();
|
|
199
|
+
};
|
|
135
200
|
const existingSteps = await this.dbConn.select().from(workflowSteps).where(eq(workflowSteps.instanceId, instanceId));
|
|
136
201
|
const stepOutputMap = /* @__PURE__ */ new Map();
|
|
137
202
|
for (const s of existingSteps) stepOutputMap.set(s.stepIndex, s.output);
|
|
@@ -141,7 +206,8 @@ var WorkflowWorker = class extends BaseWorker {
|
|
|
141
206
|
notify: async (args) => {
|
|
142
207
|
const stepId = String(currentStepIndex++);
|
|
143
208
|
if (stepOutputMap.has(stepId)) return stepOutputMap.get(stepId);
|
|
144
|
-
|
|
209
|
+
await stopIfCanceled();
|
|
210
|
+
const requested = buildStepNotifyPayload(args, instance.input, payload.projectId, `wf-${instanceId}-${stepId}`, UUID_RE.test(instanceId) ? instanceId : void 0);
|
|
145
211
|
const result = await new Promise((resolve, reject) => {
|
|
146
212
|
this.eventBuffer.push({
|
|
147
213
|
producer: this.notificationProducer,
|
|
@@ -166,15 +232,15 @@ var WorkflowWorker = class extends BaseWorker {
|
|
|
166
232
|
},
|
|
167
233
|
wait: async (duration) => {
|
|
168
234
|
const stepId = String(currentStepIndex++);
|
|
169
|
-
if (stepOutputMap.has(stepId))
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
235
|
+
if (stepOutputMap.has(stepId)) {
|
|
236
|
+
const until = Number(stepOutputMap.get(stepId)?.scheduledAt);
|
|
237
|
+
if (Number.isFinite(until) && until > Date.now()) throw new SuspendExecutionError("wait", { duration });
|
|
238
|
+
return;
|
|
239
|
+
}
|
|
240
|
+
await stopIfCanceled();
|
|
241
|
+
const ms = durationToMs(duration, "wait duration");
|
|
176
242
|
const resumeAt = Date.now() + ms;
|
|
177
|
-
await this.redisCli.zadd(
|
|
243
|
+
await this.redisCli.zadd(WORKFLOW_TIMERS_KEY, resumeAt, JSON.stringify({
|
|
178
244
|
instanceId: instance.id,
|
|
179
245
|
name: instance.name,
|
|
180
246
|
projectId: payload.projectId,
|
|
@@ -193,20 +259,17 @@ var WorkflowWorker = class extends BaseWorker {
|
|
|
193
259
|
const stepId = String(currentStepIndex++);
|
|
194
260
|
if (stepOutputMap.has(stepId)) {
|
|
195
261
|
const out = stepOutputMap.get(stepId);
|
|
196
|
-
if (out
|
|
262
|
+
if (out === null || out === void 0) throw new SuspendExecutionError("waitForEvent", { eventName });
|
|
263
|
+
if (typeof out === "object" && out.timedOut === true) return null;
|
|
197
264
|
return out;
|
|
198
265
|
}
|
|
266
|
+
await stopIfCanceled();
|
|
199
267
|
options = options || {};
|
|
200
268
|
options.timeout = options.timeout || "24h";
|
|
201
269
|
options.match = options.match || {};
|
|
202
|
-
|
|
203
|
-
if (options.timeout.endsWith("d")) ms = parseInt(options.timeout) * 24 * 60 * 60 * 1e3;
|
|
204
|
-
else if (options.timeout.endsWith("h")) ms = parseInt(options.timeout) * 60 * 60 * 1e3;
|
|
205
|
-
else if (options.timeout.endsWith("m")) ms = parseInt(options.timeout) * 60 * 1e3;
|
|
206
|
-
else if (options.timeout.endsWith("s")) ms = parseInt(options.timeout) * 1e3;
|
|
207
|
-
else throw new Error(`Invalid waitForEvent timeout: ${options.timeout}`);
|
|
270
|
+
const ms = durationToMs(options.timeout, "waitForEvent timeout");
|
|
208
271
|
const resumeAt = Date.now() + ms;
|
|
209
|
-
await this.redisCli.zadd(
|
|
272
|
+
await this.redisCli.zadd(WORKFLOW_TIMERS_KEY, resumeAt, JSON.stringify({
|
|
210
273
|
instanceId: instance.id,
|
|
211
274
|
name: instance.name,
|
|
212
275
|
projectId: payload.projectId,
|
|
@@ -215,13 +278,6 @@ var WorkflowWorker = class extends BaseWorker {
|
|
|
215
278
|
eventName,
|
|
216
279
|
stepId
|
|
217
280
|
}));
|
|
218
|
-
await this.dbConn.insert(workflowWaiters).values({
|
|
219
|
-
instanceId: instance.id,
|
|
220
|
-
projectId: payload.projectId,
|
|
221
|
-
eventName,
|
|
222
|
-
matchCriteria: options.match,
|
|
223
|
-
expiresAt: new Date(resumeAt)
|
|
224
|
-
});
|
|
225
281
|
await this.dbConn.insert(workflowSteps).values({
|
|
226
282
|
instanceId: instance.id,
|
|
227
283
|
projectId: payload.projectId,
|
|
@@ -229,11 +285,19 @@ var WorkflowWorker = class extends BaseWorker {
|
|
|
229
285
|
action: "waitForEvent",
|
|
230
286
|
output: null
|
|
231
287
|
});
|
|
288
|
+
await this.dbConn.insert(workflowWaiters).values({
|
|
289
|
+
instanceId: instance.id,
|
|
290
|
+
projectId: payload.projectId,
|
|
291
|
+
eventName,
|
|
292
|
+
matchCriteria: options.match,
|
|
293
|
+
expiresAt: new Date(resumeAt)
|
|
294
|
+
});
|
|
232
295
|
throw new SuspendExecutionError("waitForEvent", { eventName });
|
|
233
296
|
},
|
|
234
297
|
run: async (stepName, fn) => {
|
|
235
298
|
const stepId = String(currentStepIndex++);
|
|
236
299
|
if (stepOutputMap.has(stepId)) return stepOutputMap.get(stepId);
|
|
300
|
+
await stopIfCanceled();
|
|
237
301
|
const result = await fn();
|
|
238
302
|
await this.dbConn.insert(workflowSteps).values({
|
|
239
303
|
instanceId: instance.id,
|
|
@@ -247,13 +311,15 @@ var WorkflowWorker = class extends BaseWorker {
|
|
|
247
311
|
},
|
|
248
312
|
event: instance.input || { user: { id: "unknown" } }
|
|
249
313
|
};
|
|
314
|
+
const finish = (status) => this.dbConn.update(workflowInstances).set({ status }).where(and(eq(workflowInstances.id, instance.id), eq(workflowInstances.status, "running")));
|
|
250
315
|
try {
|
|
251
316
|
await handler(ctx);
|
|
252
|
-
await
|
|
317
|
+
await finish("completed");
|
|
253
318
|
this.logger.info({ instanceId: instance.id }, "Workflow completed successfully");
|
|
254
319
|
} catch (err) {
|
|
255
|
-
if (err instanceof
|
|
256
|
-
|
|
320
|
+
if (err instanceof WorkflowCanceledError) this.logger.info({ instanceId: instance.id }, "Workflow canceled mid-run, stopping");
|
|
321
|
+
else if (err instanceof SuspendExecutionError || err.name === "SuspendExecutionError") {
|
|
322
|
+
await finish("pending");
|
|
257
323
|
this.logger.info({
|
|
258
324
|
instanceId: instance.id,
|
|
259
325
|
reason: err.reason
|
|
@@ -263,33 +329,18 @@ var WorkflowWorker = class extends BaseWorker {
|
|
|
263
329
|
err,
|
|
264
330
|
instanceId: instance.id
|
|
265
331
|
}, "Workflow failed");
|
|
266
|
-
await
|
|
332
|
+
await finish("failed");
|
|
267
333
|
}
|
|
268
334
|
}
|
|
269
335
|
} finally {
|
|
270
336
|
clearInterval(renewTimer);
|
|
271
|
-
await this.redisCli.
|
|
337
|
+
await this.redisCli.releaseLock(lockKey, lockToken);
|
|
272
338
|
}
|
|
273
339
|
await Promise.all(publishPromises);
|
|
274
340
|
}
|
|
275
341
|
};
|
|
276
342
|
async function startWorkflowWorker() {
|
|
277
|
-
logger =
|
|
278
|
-
name: "workflow-worker",
|
|
279
|
-
level: config.LOG_LEVEL
|
|
280
|
-
});
|
|
281
|
-
redis = new RedisClient({
|
|
282
|
-
url: config.REDIS_URL,
|
|
283
|
-
name: "workflow",
|
|
284
|
-
logger
|
|
285
|
-
});
|
|
286
|
-
const dbData = createDatabase({
|
|
287
|
-
url: config.DATABASE_URL,
|
|
288
|
-
applicationName: "workflow",
|
|
289
|
-
logger
|
|
290
|
-
});
|
|
291
|
-
sql$1 = dbData.sql;
|
|
292
|
-
db = dbData.db;
|
|
343
|
+
({logger, redis, sql: sql$1, db} = createWorkerRuntime(config, "workflow", "workflow-worker"));
|
|
293
344
|
notificationProducer = new StreamProducer({
|
|
294
345
|
redis: redis.native,
|
|
295
346
|
stream: STREAMS.INBOUND_NORMAL,
|
|
@@ -300,22 +351,15 @@ async function startWorkflowWorker() {
|
|
|
300
351
|
stream: STREAMS.WORKFLOW_INBOUND,
|
|
301
352
|
logger
|
|
302
353
|
});
|
|
303
|
-
consumer =
|
|
304
|
-
redis
|
|
305
|
-
stream: STREAMS.WORKFLOW_INBOUND,
|
|
306
|
-
group: CONSUMER_GROUPS.WORKFLOW,
|
|
307
|
-
consumer: `workflow-${process.pid}`,
|
|
308
|
-
dlqStream: STREAMS.DEAD_LETTER,
|
|
309
|
-
batchSize: config.WORKER_CONCURRENCY,
|
|
354
|
+
({consumer, pendingScanner} = createStreamConsumer({
|
|
355
|
+
redis,
|
|
310
356
|
logger
|
|
311
|
-
}
|
|
312
|
-
pendingScanner = new PendingMessageScanner({
|
|
313
|
-
redis: redis.native,
|
|
357
|
+
}, {
|
|
314
358
|
stream: STREAMS.WORKFLOW_INBOUND,
|
|
315
359
|
group: CONSUMER_GROUPS.WORKFLOW,
|
|
316
|
-
consumer:
|
|
317
|
-
|
|
318
|
-
});
|
|
360
|
+
consumer: uniqueConsumerId("workflow"),
|
|
361
|
+
batchSize: config.WORKER_CONCURRENCY
|
|
362
|
+
}));
|
|
319
363
|
worker = new WorkflowWorker({
|
|
320
364
|
consumer,
|
|
321
365
|
pendingScanner,
|
|
@@ -330,25 +374,33 @@ async function startWorkflowWorker() {
|
|
|
330
374
|
(async () => {
|
|
331
375
|
try {
|
|
332
376
|
const now = Date.now();
|
|
333
|
-
const tasks = await redis.native.
|
|
377
|
+
const tasks = await redis.native.schedulerPoll(WORKFLOW_TIMERS_KEY, now, 100, WORKFLOW_TIMER_VISIBILITY_MS);
|
|
334
378
|
for (const taskStr of tasks) {
|
|
335
379
|
const task = JSON.parse(taskStr);
|
|
336
380
|
const inst = (await db.select().from(workflowInstances).where(eq(workflowInstances.id, task.instanceId)).limit(1))[0];
|
|
337
|
-
if (!inst || inst.status === "completed" || inst.status === "failed") {
|
|
338
|
-
await redis.native.zrem(
|
|
381
|
+
if (!inst || inst.status === "completed" || inst.status === "failed" || inst.status === "canceled") {
|
|
382
|
+
await redis.native.zrem(WORKFLOW_TIMERS_KEY, taskStr);
|
|
339
383
|
continue;
|
|
340
384
|
}
|
|
341
385
|
if (inst.status === "running") continue;
|
|
342
386
|
if (task.isEventTimeout) {
|
|
387
|
+
if (task.stepId !== void 0) {
|
|
388
|
+
const [own] = await db.select({ output: workflowSteps.output }).from(workflowSteps).where(and(eq(workflowSteps.instanceId, task.instanceId), eq(workflowSteps.stepIndex, String(task.stepId)))).limit(1);
|
|
389
|
+
if (!own || own.output !== null) {
|
|
390
|
+
await redis.native.zrem(WORKFLOW_TIMERS_KEY, taskStr);
|
|
391
|
+
continue;
|
|
392
|
+
}
|
|
393
|
+
}
|
|
343
394
|
if ((await db.delete(workflowWaiters).where(and(eq(workflowWaiters.instanceId, task.instanceId), eq(workflowWaiters.eventName, task.eventName))).returning()).length === 0) {
|
|
344
|
-
await redis.native.zrem(
|
|
395
|
+
await redis.native.zrem(WORKFLOW_TIMERS_KEY, taskStr);
|
|
345
396
|
continue;
|
|
346
397
|
}
|
|
347
|
-
const
|
|
398
|
+
const steps = await db.select().from(workflowSteps).where(and(eq(workflowSteps.instanceId, task.instanceId), eq(workflowSteps.action, "waitForEvent")));
|
|
399
|
+
const pendingStep = task.stepId !== void 0 ? steps.find((s) => s.stepIndex === String(task.stepId) && s.output === null) : steps.find((s) => s.output === null);
|
|
348
400
|
if (pendingStep) await db.update(workflowSteps).set({ output: { timedOut: true } }).where(eq(workflowSteps.id, pendingStep.id));
|
|
349
401
|
}
|
|
350
402
|
await workflowProducer.publish(buildStreamEvent("workflow.resumed", task, "scheduler", void 0));
|
|
351
|
-
await redis.native.zrem(
|
|
403
|
+
await redis.native.zrem(WORKFLOW_TIMERS_KEY, taskStr);
|
|
352
404
|
logger.info({ instanceId: task.instanceId }, "Workflow resumed from timer");
|
|
353
405
|
}
|
|
354
406
|
} catch (err) {
|
|
@@ -377,16 +429,20 @@ async function startWorkflowWorker() {
|
|
|
377
429
|
await worker.start();
|
|
378
430
|
}
|
|
379
431
|
async function stopWorkflowWorker() {
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
432
|
+
await shutdownWorker("workflow worker", {
|
|
433
|
+
logger,
|
|
434
|
+
timers: [
|
|
435
|
+
healthInterval,
|
|
436
|
+
pollInterval,
|
|
437
|
+
reaperInterval
|
|
438
|
+
],
|
|
439
|
+
worker,
|
|
440
|
+
sql: sql$1,
|
|
441
|
+
redis
|
|
442
|
+
});
|
|
443
|
+
healthInterval = null;
|
|
444
|
+
pollInterval = null;
|
|
445
|
+
reaperInterval = null;
|
|
388
446
|
}
|
|
389
447
|
//#endregion
|
|
390
448
|
export { startWorkflowWorker, stopWorkflowWorker };
|
|
391
|
-
|
|
392
|
-
//# sourceMappingURL=main-BgmT1MPe.mjs.map
|