flowfn 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.mts +1305 -0
- package/dist/index.d.ts +1305 -0
- package/dist/index.js +3180 -0
- package/dist/index.js.map +1 -0
- package/dist/index.mjs +3088 -0
- package/dist/index.mjs.map +1 -0
- package/docs/API.md +801 -0
- package/docs/USAGE.md +619 -0
- package/package.json +75 -0
- package/src/adapters/base.ts +46 -0
- package/src/adapters/memory.ts +183 -0
- package/src/adapters/postgres/index.ts +383 -0
- package/src/adapters/postgres/postgres.test.ts +100 -0
- package/src/adapters/postgres/schema.ts +110 -0
- package/src/adapters/redis.test.ts +124 -0
- package/src/adapters/redis.ts +331 -0
- package/src/core/flow-fn.test.ts +70 -0
- package/src/core/flow-fn.ts +198 -0
- package/src/core/metrics.ts +198 -0
- package/src/core/scheduler.test.ts +80 -0
- package/src/core/scheduler.ts +154 -0
- package/src/index.ts +57 -0
- package/src/monitoring/health.ts +261 -0
- package/src/patterns/backoff.ts +30 -0
- package/src/patterns/batching.ts +248 -0
- package/src/patterns/circuit-breaker.test.ts +52 -0
- package/src/patterns/circuit-breaker.ts +52 -0
- package/src/patterns/priority.ts +146 -0
- package/src/patterns/rate-limit.ts +290 -0
- package/src/patterns/retry.test.ts +62 -0
- package/src/queue/batch.test.ts +35 -0
- package/src/queue/dependencies.test.ts +33 -0
- package/src/queue/dlq.ts +222 -0
- package/src/queue/job.ts +67 -0
- package/src/queue/queue.ts +243 -0
- package/src/queue/types.ts +153 -0
- package/src/queue/worker.ts +66 -0
- package/src/storage/event-log.ts +205 -0
- package/src/storage/job-storage.ts +206 -0
- package/src/storage/workflow-storage.ts +182 -0
- package/src/stream/stream.ts +194 -0
- package/src/stream/types.ts +81 -0
- package/src/utils/hashing.ts +29 -0
- package/src/utils/id-generator.ts +109 -0
- package/src/utils/serialization.ts +142 -0
- package/src/utils/time.ts +167 -0
- package/src/workflow/advanced.test.ts +43 -0
- package/src/workflow/events.test.ts +39 -0
- package/src/workflow/types.ts +132 -0
- package/src/workflow/workflow.test.ts +55 -0
- package/src/workflow/workflow.ts +422 -0
- package/tests/dlq.test.ts +205 -0
- package/tests/health.test.ts +228 -0
- package/tests/integration.test.ts +253 -0
- package/tests/stream.test.ts +233 -0
- package/tests/workflow.test.ts +286 -0
- package/tsconfig.json +17 -0
- package/tsup.config.ts +10 -0
- package/vitest.config.ts +15 -0
package/dist/index.js
ADDED
|
@@ -0,0 +1,3180 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __create = Object.create;
|
|
3
|
+
var __defProp = Object.defineProperty;
|
|
4
|
+
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
5
|
+
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
6
|
+
var __getProtoOf = Object.getPrototypeOf;
|
|
7
|
+
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
8
|
+
var __export = (target, all) => {
|
|
9
|
+
for (var name in all)
|
|
10
|
+
__defProp(target, name, { get: all[name], enumerable: true });
|
|
11
|
+
};
|
|
12
|
+
var __copyProps = (to, from, except, desc) => {
|
|
13
|
+
if (from && typeof from === "object" || typeof from === "function") {
|
|
14
|
+
for (let key of __getOwnPropNames(from))
|
|
15
|
+
if (!__hasOwnProp.call(to, key) && key !== except)
|
|
16
|
+
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
|
|
17
|
+
}
|
|
18
|
+
return to;
|
|
19
|
+
};
|
|
20
|
+
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
|
|
21
|
+
// If the importer is in node compatibility mode or this is not an ESM
|
|
22
|
+
// file that has been converted to a CommonJS file using a Babel-
|
|
23
|
+
// compatible transform (i.e. "__esModule" has not been set), then set
|
|
24
|
+
// "default" to the CommonJS "module.exports" for node compatibility.
|
|
25
|
+
isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
|
|
26
|
+
mod
|
|
27
|
+
));
|
|
28
|
+
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
29
|
+
|
|
30
|
+
// src/index.ts
|
|
31
|
+
var index_exports = {};
|
|
32
|
+
__export(index_exports, {
|
|
33
|
+
BatchAccumulator: () => BatchAccumulator,
|
|
34
|
+
BatchWriter: () => BatchWriter,
|
|
35
|
+
FlowFnImpl: () => FlowFnImpl,
|
|
36
|
+
HealthCheckerImpl: () => HealthCheckerImpl,
|
|
37
|
+
MemoryAdapter: () => MemoryAdapter,
|
|
38
|
+
MemoryDLQManager: () => MemoryDLQManager,
|
|
39
|
+
MemoryEventLog: () => MemoryEventLog,
|
|
40
|
+
MemoryEventTracker: () => MemoryEventTracker,
|
|
41
|
+
MemoryJobStorage: () => MemoryJobStorage,
|
|
42
|
+
MemoryWorkflowStorage: () => MemoryWorkflowStorage,
|
|
43
|
+
MetricsManager: () => MetricsManager,
|
|
44
|
+
PostgresAdapter: () => PostgresAdapter,
|
|
45
|
+
PriorityQueue: () => PriorityQueue,
|
|
46
|
+
RateLimiter: () => RateLimiter,
|
|
47
|
+
RedisAdapter: () => RedisAdapter,
|
|
48
|
+
Scheduler: () => Scheduler,
|
|
49
|
+
SlidingWindowRateLimiter: () => SlidingWindowRateLimiter,
|
|
50
|
+
TokenBucketRateLimiter: () => TokenBucketRateLimiter,
|
|
51
|
+
Worker: () => Worker,
|
|
52
|
+
addDuration: () => addDuration,
|
|
53
|
+
areJobsEquivalent: () => areJobsEquivalent,
|
|
54
|
+
batch: () => batch,
|
|
55
|
+
batchByKey: () => batchByKey,
|
|
56
|
+
calculateBackoff: () => calculateBackoff,
|
|
57
|
+
chunk: () => chunk,
|
|
58
|
+
circuitBreaker: () => circuitBreaker,
|
|
59
|
+
cloneViaSerialization: () => cloneViaSerialization,
|
|
60
|
+
createFlow: () => createFlow,
|
|
61
|
+
createRateLimiter: () => createRateLimiter,
|
|
62
|
+
createWorker: () => createWorker,
|
|
63
|
+
delayUntil: () => delayUntil,
|
|
64
|
+
deserialize: () => deserialize,
|
|
65
|
+
deserializeCompressed: () => deserializeCompressed,
|
|
66
|
+
formatDuration: () => formatDuration,
|
|
67
|
+
fromMilliseconds: () => fromMilliseconds,
|
|
68
|
+
generateDeduplicationKey: () => generateDeduplicationKey,
|
|
69
|
+
generateDeterministicId: () => generateDeterministicId,
|
|
70
|
+
generateExecutionId: () => generateExecutionId,
|
|
71
|
+
generateId: () => generateId,
|
|
72
|
+
generateJobId: () => generateJobId,
|
|
73
|
+
generateMessageId: () => generateMessageId,
|
|
74
|
+
getSerializedSize: () => getSerializedSize,
|
|
75
|
+
hashJob: () => hashJob,
|
|
76
|
+
isFuture: () => isFuture,
|
|
77
|
+
isPast: () => isPast,
|
|
78
|
+
isSerializable: () => isSerializable,
|
|
79
|
+
now: () => now,
|
|
80
|
+
parseDuration: () => parseDuration,
|
|
81
|
+
processBatches: () => processBatches,
|
|
82
|
+
serialize: () => serialize,
|
|
83
|
+
serializeCompressed: () => serializeCompressed,
|
|
84
|
+
serializeSafe: () => serializeSafe,
|
|
85
|
+
sleep: () => sleep,
|
|
86
|
+
sleepDuration: () => sleepDuration,
|
|
87
|
+
timeout: () => timeout,
|
|
88
|
+
toMilliseconds: () => toMilliseconds
|
|
89
|
+
});
|
|
90
|
+
module.exports = __toCommonJS(index_exports);
|
|
91
|
+
|
|
92
|
+
// src/adapters/memory.ts
|
|
93
|
+
var import_eventemitter3 = require("eventemitter3");
|
|
94
|
+
var MemoryAdapter = class {
|
|
95
|
+
constructor() {
|
|
96
|
+
this.queues = /* @__PURE__ */ new Map();
|
|
97
|
+
this.allJobs = /* @__PURE__ */ new Map();
|
|
98
|
+
this.streams = /* @__PURE__ */ new Map();
|
|
99
|
+
this.streamEmitters = /* @__PURE__ */ new Map();
|
|
100
|
+
this.workflowStates = /* @__PURE__ */ new Map();
|
|
101
|
+
}
|
|
102
|
+
async enqueue(queueName, job) {
|
|
103
|
+
if (!this.queues.has(queueName)) {
|
|
104
|
+
this.queues.set(queueName, []);
|
|
105
|
+
}
|
|
106
|
+
const queue = this.queues.get(queueName);
|
|
107
|
+
queue.push(job);
|
|
108
|
+
this.allJobs.set(job.id, job);
|
|
109
|
+
return job.id;
|
|
110
|
+
}
|
|
111
|
+
async dequeue(queueName) {
|
|
112
|
+
const queue = this.queues.get(queueName);
|
|
113
|
+
if (!queue || queue.length === 0) return null;
|
|
114
|
+
return queue.shift() || null;
|
|
115
|
+
}
|
|
116
|
+
async ack(queue, jobId) {
|
|
117
|
+
const job = this.allJobs.get(jobId);
|
|
118
|
+
if (job) {
|
|
119
|
+
job.state = "completed";
|
|
120
|
+
job.finishedOn = Date.now();
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
async nack(queueName, jobId, requeue = true) {
|
|
124
|
+
const job = this.allJobs.get(jobId);
|
|
125
|
+
if (job && requeue) {
|
|
126
|
+
job.state = "waiting";
|
|
127
|
+
const queue = this.queues.get(queueName);
|
|
128
|
+
if (queue) queue.push(job);
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
async getJob(queueName, jobId) {
|
|
132
|
+
return this.allJobs.get(jobId) || null;
|
|
133
|
+
}
|
|
134
|
+
async getJobs(queue, status) {
|
|
135
|
+
return Array.from(this.allJobs.values()).filter(
|
|
136
|
+
(job) => job.state === status
|
|
137
|
+
);
|
|
138
|
+
}
|
|
139
|
+
async getAllJobs(queue) {
|
|
140
|
+
return Array.from(this.allJobs.values());
|
|
141
|
+
}
|
|
142
|
+
async cleanJobs(queue, grace, status) {
|
|
143
|
+
const now2 = Date.now();
|
|
144
|
+
const jobsToClean = [];
|
|
145
|
+
for (const [id, job] of this.allJobs.entries()) {
|
|
146
|
+
if (job.state === status) {
|
|
147
|
+
const timestamp2 = job.finishedOn || job.timestamp;
|
|
148
|
+
if (now2 - timestamp2 > grace) {
|
|
149
|
+
jobsToClean.push(id);
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
for (const id of jobsToClean) {
|
|
154
|
+
this.allJobs.delete(id);
|
|
155
|
+
}
|
|
156
|
+
return jobsToClean.length;
|
|
157
|
+
}
|
|
158
|
+
async publish(streamName, message) {
|
|
159
|
+
if (!this.streams.has(streamName)) {
|
|
160
|
+
this.streams.set(streamName, []);
|
|
161
|
+
this.streamEmitters.set(streamName, new import_eventemitter3.EventEmitter());
|
|
162
|
+
}
|
|
163
|
+
const stream = this.streams.get(streamName);
|
|
164
|
+
stream.push(message);
|
|
165
|
+
this.streamEmitters.get(streamName).emit("message", message);
|
|
166
|
+
return message.id;
|
|
167
|
+
}
|
|
168
|
+
async subscribe(streamName, handler) {
|
|
169
|
+
if (!this.streamEmitters.has(streamName)) {
|
|
170
|
+
this.streamEmitters.set(streamName, new import_eventemitter3.EventEmitter());
|
|
171
|
+
this.streams.set(streamName, []);
|
|
172
|
+
}
|
|
173
|
+
const emitter = this.streamEmitters.get(streamName);
|
|
174
|
+
const wrapper = (msg) => {
|
|
175
|
+
handler(msg).catch((err) => console.error("Stream handler error", err));
|
|
176
|
+
};
|
|
177
|
+
emitter.on("message", wrapper);
|
|
178
|
+
return {
|
|
179
|
+
unsubscribe: async () => {
|
|
180
|
+
emitter.off("message", wrapper);
|
|
181
|
+
}
|
|
182
|
+
};
|
|
183
|
+
}
|
|
184
|
+
async consume(stream, group, consumer, handler) {
|
|
185
|
+
return this.subscribe(stream, handler);
|
|
186
|
+
}
|
|
187
|
+
async createConsumerGroup(stream, group) {
|
|
188
|
+
}
|
|
189
|
+
async saveWorkflowState(workflowId, state) {
|
|
190
|
+
this.workflowStates.set(workflowId, state);
|
|
191
|
+
}
|
|
192
|
+
async loadWorkflowState(workflowId) {
|
|
193
|
+
return this.workflowStates.get(workflowId) || null;
|
|
194
|
+
}
|
|
195
|
+
async getQueueStats(queueName) {
|
|
196
|
+
const length = this.queues.get(queueName)?.length || 0;
|
|
197
|
+
const allInQueue = Array.from(this.allJobs.values()).filter(
|
|
198
|
+
(j) => j.state === "completed"
|
|
199
|
+
).length;
|
|
200
|
+
return {
|
|
201
|
+
waiting: length,
|
|
202
|
+
active: 0,
|
|
203
|
+
completed: allInQueue,
|
|
204
|
+
failed: 0,
|
|
205
|
+
delayed: 0,
|
|
206
|
+
paused: 0
|
|
207
|
+
};
|
|
208
|
+
}
|
|
209
|
+
async getStreamInfo(streamName) {
|
|
210
|
+
const length = this.streams.get(streamName)?.length || 0;
|
|
211
|
+
return {
|
|
212
|
+
name: streamName,
|
|
213
|
+
length,
|
|
214
|
+
groups: 0
|
|
215
|
+
};
|
|
216
|
+
}
|
|
217
|
+
async cleanup() {
|
|
218
|
+
this.queues.clear();
|
|
219
|
+
this.allJobs.clear();
|
|
220
|
+
this.streams.clear();
|
|
221
|
+
this.workflowStates.clear();
|
|
222
|
+
this.streamEmitters.clear();
|
|
223
|
+
}
|
|
224
|
+
};
|
|
225
|
+
|
|
226
|
+
// src/queue/job.ts
|
|
227
|
+
var import_uuid = require("uuid");
|
|
228
|
+
var JobImpl = class {
|
|
229
|
+
constructor(name, data, opts) {
|
|
230
|
+
this.state = "waiting";
|
|
231
|
+
this.progress = 0;
|
|
232
|
+
this.delay = 0;
|
|
233
|
+
this.attemptsMade = 0;
|
|
234
|
+
this.id = opts?.jobId || (0, import_uuid.v4)();
|
|
235
|
+
this.name = name;
|
|
236
|
+
this.data = data;
|
|
237
|
+
this.opts = opts || {};
|
|
238
|
+
this.timestamp = Date.now();
|
|
239
|
+
}
|
|
240
|
+
async update(data) {
|
|
241
|
+
this.data = { ...this.data, ...data };
|
|
242
|
+
}
|
|
243
|
+
async log(message) {
|
|
244
|
+
}
|
|
245
|
+
async updateProgress(progress) {
|
|
246
|
+
this.progress = progress;
|
|
247
|
+
}
|
|
248
|
+
async moveToCompleted(returnValue) {
|
|
249
|
+
this.state = "completed";
|
|
250
|
+
this.returnvalue = returnValue;
|
|
251
|
+
this.finishedOn = Date.now();
|
|
252
|
+
}
|
|
253
|
+
async moveToFailed(error) {
|
|
254
|
+
this.state = "failed";
|
|
255
|
+
this.failedReason = error.message;
|
|
256
|
+
this.finishedOn = Date.now();
|
|
257
|
+
}
|
|
258
|
+
async retry() {
|
|
259
|
+
}
|
|
260
|
+
async discard() {
|
|
261
|
+
}
|
|
262
|
+
async waitUntilFinished() {
|
|
263
|
+
return Promise.resolve();
|
|
264
|
+
}
|
|
265
|
+
};
|
|
266
|
+
|
|
267
|
+
// src/queue/queue.ts
|
|
268
|
+
var import_eventemitter32 = __toESM(require("eventemitter3"));
|
|
269
|
+
|
|
270
|
+
// src/patterns/backoff.ts
|
|
271
|
+
function calculateBackoff(attemptsMade, options) {
|
|
272
|
+
const { type, delay, maxDelay } = options;
|
|
273
|
+
let resultDelay;
|
|
274
|
+
switch (type) {
|
|
275
|
+
case "fixed":
|
|
276
|
+
resultDelay = delay;
|
|
277
|
+
break;
|
|
278
|
+
case "exponential":
|
|
279
|
+
resultDelay = delay * Math.pow(2, attemptsMade - 1);
|
|
280
|
+
break;
|
|
281
|
+
case "custom":
|
|
282
|
+
resultDelay = delay;
|
|
283
|
+
break;
|
|
284
|
+
default:
|
|
285
|
+
resultDelay = delay;
|
|
286
|
+
}
|
|
287
|
+
if (maxDelay && resultDelay > maxDelay) {
|
|
288
|
+
return maxDelay;
|
|
289
|
+
}
|
|
290
|
+
return resultDelay;
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
// src/queue/queue.ts
|
|
294
|
+
var QueueImpl = class extends import_eventemitter32.default {
|
|
295
|
+
constructor(name, adapter, options = {}) {
|
|
296
|
+
super();
|
|
297
|
+
this.isProcessing = false;
|
|
298
|
+
this.currentWorkers = /* @__PURE__ */ new Set();
|
|
299
|
+
this.name = name;
|
|
300
|
+
this.adapter = adapter;
|
|
301
|
+
this.options = options;
|
|
302
|
+
}
|
|
303
|
+
async add(name, data, opts) {
|
|
304
|
+
const job = new JobImpl(name, data, {
|
|
305
|
+
...this.options.defaultJobOptions,
|
|
306
|
+
...opts
|
|
307
|
+
});
|
|
308
|
+
await this.adapter.enqueue(this.name, job);
|
|
309
|
+
this.emit("waiting", job);
|
|
310
|
+
return job;
|
|
311
|
+
}
|
|
312
|
+
async addBulk(jobs2) {
|
|
313
|
+
return Promise.all(jobs2.map((j) => this.add(j.name, j.data, j.opts)));
|
|
314
|
+
}
|
|
315
|
+
process(arg1, arg2, arg3) {
|
|
316
|
+
let handler;
|
|
317
|
+
let concurrency = 1;
|
|
318
|
+
if (typeof arg1 === "number") {
|
|
319
|
+
concurrency = arg1;
|
|
320
|
+
handler = arg2;
|
|
321
|
+
} else if (typeof arg1 === "string") {
|
|
322
|
+
if (typeof arg2 === "number") {
|
|
323
|
+
concurrency = arg2;
|
|
324
|
+
handler = arg3;
|
|
325
|
+
} else {
|
|
326
|
+
handler = arg2;
|
|
327
|
+
}
|
|
328
|
+
} else {
|
|
329
|
+
handler = arg1;
|
|
330
|
+
}
|
|
331
|
+
this.resume();
|
|
332
|
+
for (let i = 0; i < concurrency; i++) {
|
|
333
|
+
const worker = this.startProcessing(handler);
|
|
334
|
+
this.currentWorkers.add(worker);
|
|
335
|
+
worker.finally(() => this.currentWorkers.delete(worker));
|
|
336
|
+
}
|
|
337
|
+
}
|
|
338
|
+
async startProcessing(handler) {
|
|
339
|
+
while (this.isProcessing) {
|
|
340
|
+
const jobData = await this.adapter.dequeue(this.name);
|
|
341
|
+
if (jobData) {
|
|
342
|
+
const job = Object.assign(
|
|
343
|
+
new JobImpl(jobData.name, jobData.data, jobData.opts),
|
|
344
|
+
jobData
|
|
345
|
+
);
|
|
346
|
+
if (job.opts.waitFor && job.opts.waitFor.length > 0) {
|
|
347
|
+
let allDone = true;
|
|
348
|
+
for (const depId of job.opts.waitFor) {
|
|
349
|
+
const depJob = await this.adapter.getJob(this.name, depId);
|
|
350
|
+
if (!depJob || depJob.state !== "completed") {
|
|
351
|
+
allDone = false;
|
|
352
|
+
break;
|
|
353
|
+
}
|
|
354
|
+
}
|
|
355
|
+
if (!allDone) {
|
|
356
|
+
await this.adapter.enqueue(this.name, job);
|
|
357
|
+
await new Promise((resolve) => setTimeout(resolve, 100));
|
|
358
|
+
continue;
|
|
359
|
+
}
|
|
360
|
+
}
|
|
361
|
+
try {
|
|
362
|
+
job.state = "active";
|
|
363
|
+
job.processedOn = Date.now();
|
|
364
|
+
this.emit("active", job);
|
|
365
|
+
const result = await handler(job);
|
|
366
|
+
await job.moveToCompleted(result);
|
|
367
|
+
await this.adapter.ack(this.name, job.id);
|
|
368
|
+
this.emit("completed", job, result);
|
|
369
|
+
} catch (err) {
|
|
370
|
+
job.attemptsMade++;
|
|
371
|
+
job.stacktrace = job.stacktrace || [];
|
|
372
|
+
job.stacktrace.push(err.stack);
|
|
373
|
+
const maxAttempts = job.opts.attempts || 1;
|
|
374
|
+
if (job.attemptsMade < maxAttempts) {
|
|
375
|
+
job.state = "delayed";
|
|
376
|
+
const backoff = job.opts.backoff ? calculateBackoff(job.attemptsMade, job.opts.backoff) : 0;
|
|
377
|
+
job.opts.delay = backoff;
|
|
378
|
+
await this.adapter.enqueue(this.name, job);
|
|
379
|
+
this.emit("failed", job, err);
|
|
380
|
+
} else {
|
|
381
|
+
await job.moveToFailed(err);
|
|
382
|
+
await this.adapter.ack(this.name, job.id);
|
|
383
|
+
this.emit("failed", job, err);
|
|
384
|
+
}
|
|
385
|
+
}
|
|
386
|
+
} else {
|
|
387
|
+
await new Promise((resolve) => setTimeout(resolve, 100));
|
|
388
|
+
}
|
|
389
|
+
}
|
|
390
|
+
}
|
|
391
|
+
processBatch(arg1, arg2, arg3) {
|
|
392
|
+
let handler;
|
|
393
|
+
let batchSize = 10;
|
|
394
|
+
let maxWait = 1e3;
|
|
395
|
+
if (typeof arg2 === "function") {
|
|
396
|
+
handler = arg2;
|
|
397
|
+
} else {
|
|
398
|
+
handler = arg3;
|
|
399
|
+
if (typeof arg2 === "number") {
|
|
400
|
+
batchSize = arg2;
|
|
401
|
+
} else {
|
|
402
|
+
batchSize = arg2.batchSize;
|
|
403
|
+
maxWait = arg2.maxWait || 1e3;
|
|
404
|
+
}
|
|
405
|
+
}
|
|
406
|
+
this.resume();
|
|
407
|
+
this.startBatchProcessing(handler, batchSize, maxWait);
|
|
408
|
+
}
|
|
409
|
+
async startBatchProcessing(handler, batchSize, maxWait) {
|
|
410
|
+
while (this.isProcessing) {
|
|
411
|
+
const batch2 = [];
|
|
412
|
+
const start = Date.now();
|
|
413
|
+
while (batch2.length < batchSize && Date.now() - start < maxWait) {
|
|
414
|
+
const jobData = await this.adapter.dequeue(this.name);
|
|
415
|
+
if (jobData) {
|
|
416
|
+
const job = Object.assign(
|
|
417
|
+
new JobImpl(jobData.name, jobData.data, jobData.opts),
|
|
418
|
+
jobData
|
|
419
|
+
);
|
|
420
|
+
batch2.push(job);
|
|
421
|
+
} else {
|
|
422
|
+
await new Promise((resolve) => setTimeout(resolve, 50));
|
|
423
|
+
}
|
|
424
|
+
}
|
|
425
|
+
if (batch2.length > 0) {
|
|
426
|
+
try {
|
|
427
|
+
for (const job of batch2) {
|
|
428
|
+
job.state = "active";
|
|
429
|
+
job.processedOn = Date.now();
|
|
430
|
+
}
|
|
431
|
+
const results = await handler(batch2);
|
|
432
|
+
for (let i = 0; i < batch2.length; i++) {
|
|
433
|
+
await batch2[i].moveToCompleted(results[i]);
|
|
434
|
+
await this.adapter.ack(this.name, batch2[i].id);
|
|
435
|
+
}
|
|
436
|
+
} catch (err) {
|
|
437
|
+
for (const job of batch2) {
|
|
438
|
+
await job.moveToFailed(err);
|
|
439
|
+
await this.adapter.ack(this.name, job.id);
|
|
440
|
+
}
|
|
441
|
+
}
|
|
442
|
+
}
|
|
443
|
+
}
|
|
444
|
+
}
|
|
445
|
+
async pause() {
|
|
446
|
+
this.isProcessing = false;
|
|
447
|
+
}
|
|
448
|
+
async resume() {
|
|
449
|
+
this.isProcessing = true;
|
|
450
|
+
}
|
|
451
|
+
async drain() {
|
|
452
|
+
await Promise.all(this.currentWorkers);
|
|
453
|
+
}
|
|
454
|
+
async clean(grace, status) {
|
|
455
|
+
return this.adapter.cleanJobs(this.name, grace, status);
|
|
456
|
+
}
|
|
457
|
+
async getJob(jobId) {
|
|
458
|
+
const job = await this.adapter.getJob(this.name, jobId);
|
|
459
|
+
return job;
|
|
460
|
+
}
|
|
461
|
+
async getJobs(status) {
|
|
462
|
+
const jobs2 = await this.adapter.getJobs(this.name, status);
|
|
463
|
+
return jobs2;
|
|
464
|
+
}
|
|
465
|
+
async getJobCounts() {
|
|
466
|
+
return this.adapter.getQueueStats(this.name);
|
|
467
|
+
}
|
|
468
|
+
async close() {
|
|
469
|
+
this.isProcessing = false;
|
|
470
|
+
await this.drain();
|
|
471
|
+
}
|
|
472
|
+
};
|
|
473
|
+
|
|
474
|
+
// src/stream/stream.ts
|
|
475
|
+
var import_uuid2 = require("uuid");
|
|
476
|
+
var StreamImpl = class {
|
|
477
|
+
constructor(name, adapter, options = {}) {
|
|
478
|
+
this.messages = /* @__PURE__ */ new Map();
|
|
479
|
+
this.name = name;
|
|
480
|
+
this.adapter = adapter;
|
|
481
|
+
this.options = options;
|
|
482
|
+
}
|
|
483
|
+
async publish(data, options) {
|
|
484
|
+
const message = {
|
|
485
|
+
id: (0, import_uuid2.v4)(),
|
|
486
|
+
stream: this.name,
|
|
487
|
+
data,
|
|
488
|
+
headers: options?.headers,
|
|
489
|
+
timestamp: Date.now(),
|
|
490
|
+
partition: options?.partition,
|
|
491
|
+
key: options?.key,
|
|
492
|
+
ack: async () => {
|
|
493
|
+
},
|
|
494
|
+
nack: async () => {
|
|
495
|
+
}
|
|
496
|
+
};
|
|
497
|
+
this.messages.set(message.id, message);
|
|
498
|
+
if (this.options.maxLength && this.messages.size > this.options.maxLength) {
|
|
499
|
+
await this.trim({ maxLength: this.options.maxLength });
|
|
500
|
+
}
|
|
501
|
+
return this.adapter.publish(this.name, message);
|
|
502
|
+
}
|
|
503
|
+
async publishBatch(messages2) {
|
|
504
|
+
return Promise.all(messages2.map((m) => this.publish(m.data, m.options)));
|
|
505
|
+
}
|
|
506
|
+
async subscribe(handler, options) {
|
|
507
|
+
return this.adapter.subscribe(this.name, handler);
|
|
508
|
+
}
|
|
509
|
+
createConsumer(consumerId, options) {
|
|
510
|
+
let subscription = null;
|
|
511
|
+
let paused = false;
|
|
512
|
+
return {
|
|
513
|
+
subscribe: async (handler) => {
|
|
514
|
+
if (options.fromBeginning && this.messages.size > 0) {
|
|
515
|
+
const sortedMessages = Array.from(this.messages.values()).sort(
|
|
516
|
+
(a, b) => a.timestamp - b.timestamp
|
|
517
|
+
);
|
|
518
|
+
for (const msg of sortedMessages) {
|
|
519
|
+
if (!paused) {
|
|
520
|
+
await handler(msg).catch(console.error);
|
|
521
|
+
}
|
|
522
|
+
}
|
|
523
|
+
}
|
|
524
|
+
subscription = await this.adapter.consume(
|
|
525
|
+
this.name,
|
|
526
|
+
options.groupId,
|
|
527
|
+
consumerId,
|
|
528
|
+
handler
|
|
529
|
+
);
|
|
530
|
+
},
|
|
531
|
+
pause: async () => {
|
|
532
|
+
paused = true;
|
|
533
|
+
},
|
|
534
|
+
resume: async () => {
|
|
535
|
+
paused = false;
|
|
536
|
+
},
|
|
537
|
+
close: async () => {
|
|
538
|
+
if (subscription) await subscription.unsubscribe();
|
|
539
|
+
}
|
|
540
|
+
};
|
|
541
|
+
}
|
|
542
|
+
async getInfo() {
|
|
543
|
+
const info = await this.adapter.getStreamInfo(this.name);
|
|
544
|
+
return {
|
|
545
|
+
...info,
|
|
546
|
+
length: this.messages.size
|
|
547
|
+
};
|
|
548
|
+
}
|
|
549
|
+
async trim(strategy) {
|
|
550
|
+
const now2 = Date.now();
|
|
551
|
+
const messagesToDelete = [];
|
|
552
|
+
if (strategy.maxLength) {
|
|
553
|
+
const sortedMessages = Array.from(this.messages.entries()).sort(
|
|
554
|
+
(a, b) => b[1].timestamp - a[1].timestamp
|
|
555
|
+
);
|
|
556
|
+
if (sortedMessages.length > strategy.maxLength) {
|
|
557
|
+
const toRemove = sortedMessages.slice(strategy.maxLength);
|
|
558
|
+
messagesToDelete.push(...toRemove.map(([id]) => id));
|
|
559
|
+
}
|
|
560
|
+
}
|
|
561
|
+
if (strategy.maxAgeSeconds) {
|
|
562
|
+
const maxAge = strategy.maxAgeSeconds * 1e3;
|
|
563
|
+
for (const [id, message] of this.messages.entries()) {
|
|
564
|
+
if (now2 - message.timestamp > maxAge) {
|
|
565
|
+
messagesToDelete.push(id);
|
|
566
|
+
}
|
|
567
|
+
}
|
|
568
|
+
}
|
|
569
|
+
const uniqueToDelete = [...new Set(messagesToDelete)];
|
|
570
|
+
for (const id of uniqueToDelete) {
|
|
571
|
+
this.messages.delete(id);
|
|
572
|
+
}
|
|
573
|
+
return uniqueToDelete.length;
|
|
574
|
+
}
|
|
575
|
+
async getMessages(start, end, count) {
|
|
576
|
+
const allMessages = Array.from(this.messages.values()).sort(
|
|
577
|
+
(a, b) => a.timestamp - b.timestamp
|
|
578
|
+
);
|
|
579
|
+
let filtered = allMessages.filter((m) => m.id >= start && m.id <= end);
|
|
580
|
+
if (count !== void 0 && count > 0) {
|
|
581
|
+
filtered = filtered.slice(0, count);
|
|
582
|
+
}
|
|
583
|
+
return filtered;
|
|
584
|
+
}
|
|
585
|
+
/**
|
|
586
|
+
* Replay messages from a specific timestamp
|
|
587
|
+
*/
|
|
588
|
+
async replay(fromTimestamp, handler) {
|
|
589
|
+
const messages2 = Array.from(this.messages.values()).filter((m) => m.timestamp >= fromTimestamp).sort((a, b) => a.timestamp - b.timestamp);
|
|
590
|
+
for (const message of messages2) {
|
|
591
|
+
await handler(message);
|
|
592
|
+
}
|
|
593
|
+
return messages2.length;
|
|
594
|
+
}
|
|
595
|
+
/**
|
|
596
|
+
* Get message count
|
|
597
|
+
*/
|
|
598
|
+
getMessageCount() {
|
|
599
|
+
return this.messages.size;
|
|
600
|
+
}
|
|
601
|
+
async close() {
|
|
602
|
+
this.messages.clear();
|
|
603
|
+
}
|
|
604
|
+
};
|
|
605
|
+
|
|
606
|
+
// src/workflow/workflow.ts
|
|
607
|
+
var import_uuid3 = require("uuid");
|
|
608
|
+
|
|
609
|
+
// src/storage/workflow-storage.ts
|
|
610
|
+
var MemoryWorkflowStorage = class {
|
|
611
|
+
constructor() {
|
|
612
|
+
this.executions = /* @__PURE__ */ new Map();
|
|
613
|
+
this.workflowIndex = /* @__PURE__ */ new Map();
|
|
614
|
+
}
|
|
615
|
+
// workflowId -> executionIds
|
|
616
|
+
async save(workflowId, execution) {
|
|
617
|
+
this.executions.set(execution.id, { ...execution });
|
|
618
|
+
if (!this.workflowIndex.has(workflowId)) {
|
|
619
|
+
this.workflowIndex.set(workflowId, /* @__PURE__ */ new Set());
|
|
620
|
+
}
|
|
621
|
+
this.workflowIndex.get(workflowId).add(execution.id);
|
|
622
|
+
}
|
|
623
|
+
async get(executionId) {
|
|
624
|
+
return this.executions.get(executionId) || null;
|
|
625
|
+
}
|
|
626
|
+
async list(workflowId, options = {}) {
|
|
627
|
+
const executionIds = this.workflowIndex.get(workflowId) || /* @__PURE__ */ new Set();
|
|
628
|
+
let executions = [];
|
|
629
|
+
for (const id of executionIds) {
|
|
630
|
+
const execution = this.executions.get(id);
|
|
631
|
+
if (execution) {
|
|
632
|
+
if (!options.status || execution.status === options.status) {
|
|
633
|
+
executions.push(execution);
|
|
634
|
+
}
|
|
635
|
+
}
|
|
636
|
+
}
|
|
637
|
+
const sortBy = options.sortBy || "createdAt";
|
|
638
|
+
const sortOrder = options.sortOrder || "desc";
|
|
639
|
+
executions.sort((a, b) => {
|
|
640
|
+
const aVal = a[sortBy] || 0;
|
|
641
|
+
const bVal = b[sortBy] || 0;
|
|
642
|
+
return sortOrder === "asc" ? aVal - bVal : bVal - aVal;
|
|
643
|
+
});
|
|
644
|
+
const offset = options.offset || 0;
|
|
645
|
+
const limit = options.limit || executions.length;
|
|
646
|
+
return executions.slice(offset, offset + limit);
|
|
647
|
+
}
|
|
648
|
+
async updateStatus(executionId, status) {
|
|
649
|
+
const execution = this.executions.get(executionId);
|
|
650
|
+
if (execution) {
|
|
651
|
+
execution.status = status;
|
|
652
|
+
execution.updatedAt = Date.now();
|
|
653
|
+
if (status === "completed" || status === "failed" || status === "cancelled") {
|
|
654
|
+
execution.completedAt = Date.now();
|
|
655
|
+
}
|
|
656
|
+
}
|
|
657
|
+
}
|
|
658
|
+
async updateState(executionId, state) {
|
|
659
|
+
const execution = this.executions.get(executionId);
|
|
660
|
+
if (execution) {
|
|
661
|
+
execution.state = { ...execution.state, ...state };
|
|
662
|
+
execution.updatedAt = Date.now();
|
|
663
|
+
}
|
|
664
|
+
}
|
|
665
|
+
async delete(executionId) {
|
|
666
|
+
const execution = this.executions.get(executionId);
|
|
667
|
+
if (execution) {
|
|
668
|
+
const workflowId = execution.workflowId;
|
|
669
|
+
const executionIds = this.workflowIndex.get(workflowId);
|
|
670
|
+
if (executionIds) {
|
|
671
|
+
executionIds.delete(executionId);
|
|
672
|
+
if (executionIds.size === 0) {
|
|
673
|
+
this.workflowIndex.delete(workflowId);
|
|
674
|
+
}
|
|
675
|
+
}
|
|
676
|
+
this.executions.delete(executionId);
|
|
677
|
+
}
|
|
678
|
+
}
|
|
679
|
+
async clean(workflowId, grace) {
|
|
680
|
+
const now2 = Date.now();
|
|
681
|
+
const executionIds = this.workflowIndex.get(workflowId) || /* @__PURE__ */ new Set();
|
|
682
|
+
const toDelete = [];
|
|
683
|
+
for (const id of executionIds) {
|
|
684
|
+
const execution = this.executions.get(id);
|
|
685
|
+
if (execution && execution.completedAt) {
|
|
686
|
+
if (now2 - execution.completedAt > grace) {
|
|
687
|
+
toDelete.push(id);
|
|
688
|
+
}
|
|
689
|
+
}
|
|
690
|
+
}
|
|
691
|
+
for (const id of toDelete) {
|
|
692
|
+
await this.delete(id);
|
|
693
|
+
}
|
|
694
|
+
return toDelete.length;
|
|
695
|
+
}
|
|
696
|
+
/**
|
|
697
|
+
* Clear all executions
|
|
698
|
+
*/
|
|
699
|
+
clear() {
|
|
700
|
+
this.executions.clear();
|
|
701
|
+
this.workflowIndex.clear();
|
|
702
|
+
}
|
|
703
|
+
};
|
|
704
|
+
|
|
705
|
+
// src/storage/event-log.ts
|
|
706
|
+
var MemoryEventLog = class {
|
|
707
|
+
constructor() {
|
|
708
|
+
this.events = [];
|
|
709
|
+
this.aggregateVersions = /* @__PURE__ */ new Map();
|
|
710
|
+
this.subscribers = [];
|
|
711
|
+
}
|
|
712
|
+
async append(event) {
|
|
713
|
+
const version = (this.aggregateVersions.get(event.aggregateId) || 0) + 1;
|
|
714
|
+
this.aggregateVersions.set(event.aggregateId, version);
|
|
715
|
+
const fullEvent = {
|
|
716
|
+
...event,
|
|
717
|
+
id: `evt_${Date.now()}_${Math.random().toString(36).substring(7)}`,
|
|
718
|
+
timestamp: Date.now(),
|
|
719
|
+
version
|
|
720
|
+
};
|
|
721
|
+
this.events.push(fullEvent);
|
|
722
|
+
for (const subscriber of this.subscribers) {
|
|
723
|
+
if (this.matchesFilter(fullEvent, subscriber.filter)) {
|
|
724
|
+
Promise.resolve(subscriber.handler(fullEvent)).catch(
|
|
725
|
+
(err) => console.error("Event subscriber error:", err)
|
|
726
|
+
);
|
|
727
|
+
}
|
|
728
|
+
}
|
|
729
|
+
return fullEvent;
|
|
730
|
+
}
|
|
731
|
+
async getEvents(filter) {
|
|
732
|
+
let results = [...this.events];
|
|
733
|
+
if (filter.aggregateId) {
|
|
734
|
+
results = results.filter((e) => e.aggregateId === filter.aggregateId);
|
|
735
|
+
}
|
|
736
|
+
if (filter.aggregateType) {
|
|
737
|
+
results = results.filter((e) => e.aggregateType === filter.aggregateType);
|
|
738
|
+
}
|
|
739
|
+
if (filter.types && filter.types.length > 0) {
|
|
740
|
+
results = results.filter((e) => filter.types.includes(e.type));
|
|
741
|
+
}
|
|
742
|
+
if (filter.fromVersion !== void 0) {
|
|
743
|
+
results = results.filter((e) => e.version >= filter.fromVersion);
|
|
744
|
+
}
|
|
745
|
+
if (filter.toVersion !== void 0) {
|
|
746
|
+
results = results.filter((e) => e.version <= filter.toVersion);
|
|
747
|
+
}
|
|
748
|
+
if (filter.fromTimestamp !== void 0) {
|
|
749
|
+
results = results.filter((e) => e.timestamp >= filter.fromTimestamp);
|
|
750
|
+
}
|
|
751
|
+
if (filter.toTimestamp !== void 0) {
|
|
752
|
+
results = results.filter((e) => e.timestamp <= filter.toTimestamp);
|
|
753
|
+
}
|
|
754
|
+
results.sort((a, b) => a.version - b.version);
|
|
755
|
+
const offset = filter.offset || 0;
|
|
756
|
+
const limit = filter.limit || results.length;
|
|
757
|
+
return results.slice(offset, offset + limit);
|
|
758
|
+
}
|
|
759
|
+
async getAggregateEvents(aggregateId, fromVersion) {
|
|
760
|
+
return this.getEvents({
|
|
761
|
+
aggregateId,
|
|
762
|
+
fromVersion
|
|
763
|
+
});
|
|
764
|
+
}
|
|
765
|
+
async getLatestVersion(aggregateId) {
|
|
766
|
+
return this.aggregateVersions.get(aggregateId) || 0;
|
|
767
|
+
}
|
|
768
|
+
subscribe(handler, filter) {
|
|
769
|
+
const subscriber = { handler, filter };
|
|
770
|
+
this.subscribers.push(subscriber);
|
|
771
|
+
return () => {
|
|
772
|
+
const index2 = this.subscribers.indexOf(subscriber);
|
|
773
|
+
if (index2 > -1) {
|
|
774
|
+
this.subscribers.splice(index2, 1);
|
|
775
|
+
}
|
|
776
|
+
};
|
|
777
|
+
}
|
|
778
|
+
matchesFilter(event, filter) {
|
|
779
|
+
if (!filter) return true;
|
|
780
|
+
if (filter.aggregateId && event.aggregateId !== filter.aggregateId)
|
|
781
|
+
return false;
|
|
782
|
+
if (filter.aggregateType && event.aggregateType !== filter.aggregateType)
|
|
783
|
+
return false;
|
|
784
|
+
if (filter.types && !filter.types.includes(event.type)) return false;
|
|
785
|
+
if (filter.fromVersion !== void 0 && event.version < filter.fromVersion)
|
|
786
|
+
return false;
|
|
787
|
+
if (filter.toVersion !== void 0 && event.version > filter.toVersion)
|
|
788
|
+
return false;
|
|
789
|
+
if (filter.fromTimestamp !== void 0 && event.timestamp < filter.fromTimestamp)
|
|
790
|
+
return false;
|
|
791
|
+
if (filter.toTimestamp !== void 0 && event.timestamp > filter.toTimestamp)
|
|
792
|
+
return false;
|
|
793
|
+
return true;
|
|
794
|
+
}
|
|
795
|
+
/**
|
|
796
|
+
* Clear all events
|
|
797
|
+
*/
|
|
798
|
+
clear() {
|
|
799
|
+
this.events = [];
|
|
800
|
+
this.aggregateVersions.clear();
|
|
801
|
+
this.subscribers = [];
|
|
802
|
+
}
|
|
803
|
+
};
|
|
804
|
+
|
|
805
|
+
// src/workflow/workflow.ts
|
|
806
|
+
var WorkflowBuilderImpl = class {
|
|
807
|
+
constructor(name, adapter) {
|
|
808
|
+
this.steps = [];
|
|
809
|
+
this.name = name;
|
|
810
|
+
this.adapter = adapter;
|
|
811
|
+
}
|
|
812
|
+
input() {
|
|
813
|
+
return this;
|
|
814
|
+
}
|
|
815
|
+
step(name, handler) {
|
|
816
|
+
this.steps.push({ type: "step", name, handler });
|
|
817
|
+
return this;
|
|
818
|
+
}
|
|
819
|
+
parallel(steps) {
|
|
820
|
+
this.steps.push({
|
|
821
|
+
type: "parallel",
|
|
822
|
+
steps: steps.map((h, i) => ({
|
|
823
|
+
type: "step",
|
|
824
|
+
name: `parallel-${i}`,
|
|
825
|
+
handler: h
|
|
826
|
+
}))
|
|
827
|
+
});
|
|
828
|
+
return this;
|
|
829
|
+
}
|
|
830
|
+
branch(options) {
|
|
831
|
+
const thenSteps = options.then.steps;
|
|
832
|
+
const elseSteps = options.else ? options.else.steps : void 0;
|
|
833
|
+
this.steps.push({
|
|
834
|
+
type: "branch",
|
|
835
|
+
condition: options.condition,
|
|
836
|
+
steps: thenSteps,
|
|
837
|
+
elseSteps
|
|
838
|
+
});
|
|
839
|
+
return this;
|
|
840
|
+
}
|
|
841
|
+
saga(name, saga) {
|
|
842
|
+
this.steps.push({
|
|
843
|
+
type: "saga",
|
|
844
|
+
name,
|
|
845
|
+
handler: saga.execute,
|
|
846
|
+
compensate: saga.compensate
|
|
847
|
+
});
|
|
848
|
+
return this;
|
|
849
|
+
}
|
|
850
|
+
delay(duration) {
|
|
851
|
+
this.steps.push({ type: "delay", options: duration });
|
|
852
|
+
return this;
|
|
853
|
+
}
|
|
854
|
+
delayUntil(condition) {
|
|
855
|
+
this.steps.push({ type: "delayUntil", condition });
|
|
856
|
+
return this;
|
|
857
|
+
}
|
|
858
|
+
waitForApproval(approver, options) {
|
|
859
|
+
this.steps.push({
|
|
860
|
+
type: "wait",
|
|
861
|
+
name: "approval",
|
|
862
|
+
options: { approver, ...options }
|
|
863
|
+
});
|
|
864
|
+
return this;
|
|
865
|
+
}
|
|
866
|
+
waitForEvent(event, options) {
|
|
867
|
+
this.steps.push({
|
|
868
|
+
type: "wait",
|
|
869
|
+
name: "event",
|
|
870
|
+
options: { event, ...options }
|
|
871
|
+
});
|
|
872
|
+
return this;
|
|
873
|
+
}
|
|
874
|
+
onError(step, handler) {
|
|
875
|
+
const s = this.steps.find((s2) => s2.name === step);
|
|
876
|
+
if (s) {
|
|
877
|
+
}
|
|
878
|
+
return this;
|
|
879
|
+
}
|
|
880
|
+
compensate(step, handler) {
|
|
881
|
+
const s = this.steps.find((s2) => s2.name === step);
|
|
882
|
+
if (s) {
|
|
883
|
+
s.compensate = handler;
|
|
884
|
+
}
|
|
885
|
+
return this;
|
|
886
|
+
}
|
|
887
|
+
build() {
|
|
888
|
+
return new WorkflowImpl(this.name, this.steps, this.adapter);
|
|
889
|
+
}
|
|
890
|
+
};
|
|
891
|
+
var WorkflowImpl = class {
|
|
892
|
+
constructor(name, steps, adapter) {
|
|
893
|
+
this.executionCount = 0;
|
|
894
|
+
this.successCount = 0;
|
|
895
|
+
this.totalDuration = 0;
|
|
896
|
+
this.id = (0, import_uuid3.v4)();
|
|
897
|
+
this.name = name;
|
|
898
|
+
this.steps = steps;
|
|
899
|
+
this.adapter = adapter;
|
|
900
|
+
this.storage = new MemoryWorkflowStorage();
|
|
901
|
+
this.eventLog = new MemoryEventLog();
|
|
902
|
+
}
|
|
903
|
+
async execute(input) {
|
|
904
|
+
const executionId = (0, import_uuid3.v4)();
|
|
905
|
+
const execution = {
|
|
906
|
+
id: executionId,
|
|
907
|
+
workflowId: this.id,
|
|
908
|
+
status: "running",
|
|
909
|
+
input,
|
|
910
|
+
state: {},
|
|
911
|
+
startedAt: Date.now(),
|
|
912
|
+
createdAt: Date.now(),
|
|
913
|
+
updatedAt: Date.now()
|
|
914
|
+
};
|
|
915
|
+
await this.storage.save(this.id, execution);
|
|
916
|
+
await this.logEvent(executionId, "execution.started", { input });
|
|
917
|
+
this.runExecution(execution).catch(console.error);
|
|
918
|
+
return execution;
|
|
919
|
+
}
|
|
920
|
+
async runExecution(execution) {
|
|
921
|
+
const context = {
|
|
922
|
+
workflowId: this.id,
|
|
923
|
+
executionId: execution.id,
|
|
924
|
+
input: execution.input,
|
|
925
|
+
state: execution.state || {},
|
|
926
|
+
metadata: {},
|
|
927
|
+
set: (k, v) => {
|
|
928
|
+
context.state[k] = v;
|
|
929
|
+
this.storage.updateState(execution.id, context.state).catch(console.error);
|
|
930
|
+
},
|
|
931
|
+
get: (k) => context.state[k],
|
|
932
|
+
sleep: (ms) => new Promise((r) => setTimeout(r, ms)),
|
|
933
|
+
waitForEvent: async (event, timeout2) => {
|
|
934
|
+
await new Promise((r) => setTimeout(r, timeout2 || 100));
|
|
935
|
+
}
|
|
936
|
+
};
|
|
937
|
+
const executedSagas = [];
|
|
938
|
+
try {
|
|
939
|
+
await this.executeSteps(this.steps, context, executedSagas);
|
|
940
|
+
execution.status = "completed";
|
|
941
|
+
execution.output = context.state;
|
|
942
|
+
execution.completedAt = Date.now();
|
|
943
|
+
await this.logEvent(execution.id, "execution.completed", {
|
|
944
|
+
output: execution.output
|
|
945
|
+
});
|
|
946
|
+
this.executionCount++;
|
|
947
|
+
this.successCount++;
|
|
948
|
+
this.totalDuration += execution.completedAt - execution.startedAt;
|
|
949
|
+
} catch (error) {
|
|
950
|
+
execution.status = "failed";
|
|
951
|
+
execution.error = error;
|
|
952
|
+
execution.completedAt = Date.now();
|
|
953
|
+
await this.logEvent(execution.id, "execution.failed", {
|
|
954
|
+
error: error.message
|
|
955
|
+
});
|
|
956
|
+
await this.compensate(executedSagas, context);
|
|
957
|
+
this.executionCount++;
|
|
958
|
+
this.totalDuration += execution.completedAt - execution.startedAt;
|
|
959
|
+
}
|
|
960
|
+
await this.storage.save(this.id, execution);
|
|
961
|
+
await this.adapter.saveWorkflowState(execution.id, execution);
|
|
962
|
+
}
|
|
963
|
+
async logEvent(executionId, type, data) {
|
|
964
|
+
await this.eventLog.append({
|
|
965
|
+
type,
|
|
966
|
+
aggregateId: executionId,
|
|
967
|
+
aggregateType: "workflow_execution",
|
|
968
|
+
data
|
|
969
|
+
});
|
|
970
|
+
}
|
|
971
|
+
async executeSteps(steps, context, executedSagas) {
|
|
972
|
+
for (const step of steps) {
|
|
973
|
+
if (step.type === "step" && step.handler) {
|
|
974
|
+
await this.logEvent(context.executionId, "step.started", {
|
|
975
|
+
step: step.name
|
|
976
|
+
});
|
|
977
|
+
await step.handler(context);
|
|
978
|
+
await this.logEvent(context.executionId, "step.completed", {
|
|
979
|
+
step: step.name
|
|
980
|
+
});
|
|
981
|
+
} else if (step.type === "saga" && step.handler) {
|
|
982
|
+
await this.logEvent(context.executionId, "saga.started", {
|
|
983
|
+
step: step.name
|
|
984
|
+
});
|
|
985
|
+
await step.handler(context);
|
|
986
|
+
executedSagas.push(step);
|
|
987
|
+
await this.logEvent(context.executionId, "saga.completed", {
|
|
988
|
+
step: step.name
|
|
989
|
+
});
|
|
990
|
+
} else if (step.type === "parallel" && step.steps) {
|
|
991
|
+
await this.logEvent(context.executionId, "parallel.started", {
|
|
992
|
+
count: step.steps.length
|
|
993
|
+
});
|
|
994
|
+
await Promise.all(
|
|
995
|
+
step.steps.map(
|
|
996
|
+
(s) => s.handler ? s.handler(context) : Promise.resolve()
|
|
997
|
+
)
|
|
998
|
+
);
|
|
999
|
+
await this.logEvent(context.executionId, "parallel.completed", {});
|
|
1000
|
+
} else if (step.type === "branch" && step.condition) {
|
|
1001
|
+
const conditionMet = await step.condition(context);
|
|
1002
|
+
if (conditionMet && step.steps) {
|
|
1003
|
+
await this.executeSteps(step.steps, context, executedSagas);
|
|
1004
|
+
} else if (!conditionMet && step.elseSteps) {
|
|
1005
|
+
await this.executeSteps(step.elseSteps, context, executedSagas);
|
|
1006
|
+
}
|
|
1007
|
+
} else if (step.type === "delay") {
|
|
1008
|
+
const ms = step.options?.ms || 0;
|
|
1009
|
+
await new Promise((r) => setTimeout(r, ms));
|
|
1010
|
+
} else if (step.type === "delayUntil" && step.condition) {
|
|
1011
|
+
const targetTime = await step.condition(context);
|
|
1012
|
+
if (typeof targetTime === "number") {
|
|
1013
|
+
const now2 = Date.now();
|
|
1014
|
+
const delay = Math.max(0, targetTime - now2);
|
|
1015
|
+
if (delay > 0) {
|
|
1016
|
+
await new Promise((r) => setTimeout(r, delay));
|
|
1017
|
+
}
|
|
1018
|
+
}
|
|
1019
|
+
} else if (step.type === "wait") {
|
|
1020
|
+
if (step.options?.timeout) {
|
|
1021
|
+
await new Promise((r) => setTimeout(r, step.options.timeout));
|
|
1022
|
+
}
|
|
1023
|
+
}
|
|
1024
|
+
}
|
|
1025
|
+
}
|
|
1026
|
+
async compensate(executedSagas, context) {
|
|
1027
|
+
for (let i = executedSagas.length - 1; i >= 0; i--) {
|
|
1028
|
+
const step = executedSagas[i];
|
|
1029
|
+
if (step.compensate) {
|
|
1030
|
+
try {
|
|
1031
|
+
await step.compensate(context);
|
|
1032
|
+
} catch (err) {
|
|
1033
|
+
console.error(`Compensation failed for step ${step.name}`, err);
|
|
1034
|
+
}
|
|
1035
|
+
}
|
|
1036
|
+
}
|
|
1037
|
+
}
|
|
1038
|
+
async getExecution(executionId) {
|
|
1039
|
+
const execution = await this.storage.get(executionId);
|
|
1040
|
+
if (!execution) {
|
|
1041
|
+
const adapterExecution = await this.adapter.loadWorkflowState(executionId);
|
|
1042
|
+
if (!adapterExecution) {
|
|
1043
|
+
throw new Error(`Execution ${executionId} not found`);
|
|
1044
|
+
}
|
|
1045
|
+
return adapterExecution;
|
|
1046
|
+
}
|
|
1047
|
+
return execution;
|
|
1048
|
+
}
|
|
1049
|
+
async listExecutions(options) {
|
|
1050
|
+
return this.storage.list(this.id, options);
|
|
1051
|
+
}
|
|
1052
|
+
async cancelExecution(executionId) {
|
|
1053
|
+
const execution = await this.storage.get(executionId);
|
|
1054
|
+
if (!execution) {
|
|
1055
|
+
throw new Error(`Execution ${executionId} not found`);
|
|
1056
|
+
}
|
|
1057
|
+
if (execution.status === "completed" || execution.status === "failed" || execution.status === "cancelled") {
|
|
1058
|
+
throw new Error(`Cannot cancel execution in ${execution.status} state`);
|
|
1059
|
+
}
|
|
1060
|
+
await this.storage.updateStatus(executionId, "cancelled");
|
|
1061
|
+
await this.logEvent(executionId, "execution.cancelled", {});
|
|
1062
|
+
}
|
|
1063
|
+
async retryExecution(executionId) {
|
|
1064
|
+
const execution = await this.storage.get(executionId);
|
|
1065
|
+
if (!execution) {
|
|
1066
|
+
throw new Error(`Execution ${executionId} not found`);
|
|
1067
|
+
}
|
|
1068
|
+
if (execution.status !== "failed") {
|
|
1069
|
+
throw new Error(
|
|
1070
|
+
`Can only retry failed executions, current status: ${execution.status}`
|
|
1071
|
+
);
|
|
1072
|
+
}
|
|
1073
|
+
return this.execute(execution.input);
|
|
1074
|
+
}
|
|
1075
|
+
async getExecutionHistory(executionId) {
|
|
1076
|
+
const events = await this.eventLog.getAggregateEvents(executionId);
|
|
1077
|
+
return events.map((e) => ({
|
|
1078
|
+
timestamp: e.timestamp,
|
|
1079
|
+
type: e.type,
|
|
1080
|
+
step: e.data.step,
|
|
1081
|
+
data: e.data
|
|
1082
|
+
}));
|
|
1083
|
+
}
|
|
1084
|
+
async getMetrics() {
|
|
1085
|
+
const successRate = this.executionCount > 0 ? this.successCount / this.executionCount * 100 : 0;
|
|
1086
|
+
const avgDuration = this.executionCount > 0 ? this.totalDuration / this.executionCount : 0;
|
|
1087
|
+
return {
|
|
1088
|
+
totalExecutions: this.executionCount,
|
|
1089
|
+
successRate,
|
|
1090
|
+
avgDuration
|
|
1091
|
+
};
|
|
1092
|
+
}
|
|
1093
|
+
};
|
|
1094
|
+
|
|
1095
|
+
// src/core/scheduler.ts
|
|
1096
|
+
var import_cron_parser = __toESM(require("cron-parser"));
|
|
1097
|
+
var Scheduler = class {
|
|
1098
|
+
constructor() {
|
|
1099
|
+
this.tasks = /* @__PURE__ */ new Map();
|
|
1100
|
+
}
|
|
1101
|
+
async schedule(name, pattern, handler) {
|
|
1102
|
+
await this.cancel(name);
|
|
1103
|
+
const options = typeof pattern === "string" ? { pattern } : pattern;
|
|
1104
|
+
const now2 = Date.now();
|
|
1105
|
+
let nextRun = 0;
|
|
1106
|
+
if (options.pattern) {
|
|
1107
|
+
const interval = import_cron_parser.default.parseExpression(options.pattern, {
|
|
1108
|
+
currentDate: options.startDate || /* @__PURE__ */ new Date(),
|
|
1109
|
+
tz: options.timezone
|
|
1110
|
+
});
|
|
1111
|
+
nextRun = interval.next().getTime();
|
|
1112
|
+
} else if (options.every) {
|
|
1113
|
+
nextRun = (options.startDate?.getTime() || now2) + options.every;
|
|
1114
|
+
}
|
|
1115
|
+
this.tasks.set(name, {
|
|
1116
|
+
pattern,
|
|
1117
|
+
handler,
|
|
1118
|
+
nextRun,
|
|
1119
|
+
count: 0,
|
|
1120
|
+
paused: false
|
|
1121
|
+
});
|
|
1122
|
+
this.plan(name);
|
|
1123
|
+
}
|
|
1124
|
+
async repeat(name, options, handler) {
|
|
1125
|
+
return this.schedule(name, options, handler);
|
|
1126
|
+
}
|
|
1127
|
+
plan(name) {
|
|
1128
|
+
const task = this.tasks.get(name);
|
|
1129
|
+
if (!task || task.paused) return;
|
|
1130
|
+
const now2 = Date.now();
|
|
1131
|
+
const delay = Math.max(0, task.nextRun - now2);
|
|
1132
|
+
if (task.timer) clearTimeout(task.timer);
|
|
1133
|
+
task.timer = setTimeout(async () => {
|
|
1134
|
+
if (task.paused) return;
|
|
1135
|
+
try {
|
|
1136
|
+
const options = typeof task.pattern === "string" ? { pattern: task.pattern } : task.pattern;
|
|
1137
|
+
await task.handler(options.data);
|
|
1138
|
+
task.count++;
|
|
1139
|
+
if (options.limit && task.count >= options.limit) {
|
|
1140
|
+
this.tasks.delete(name);
|
|
1141
|
+
return;
|
|
1142
|
+
}
|
|
1143
|
+
if (options.pattern) {
|
|
1144
|
+
const interval = import_cron_parser.default.parseExpression(options.pattern, {
|
|
1145
|
+
currentDate: /* @__PURE__ */ new Date(),
|
|
1146
|
+
tz: options.timezone
|
|
1147
|
+
});
|
|
1148
|
+
task.nextRun = interval.next().getTime();
|
|
1149
|
+
} else if (options.every) {
|
|
1150
|
+
task.nextRun = Date.now() + options.every;
|
|
1151
|
+
}
|
|
1152
|
+
if (options.endDate && task.nextRun > options.endDate.getTime()) {
|
|
1153
|
+
this.tasks.delete(name);
|
|
1154
|
+
return;
|
|
1155
|
+
}
|
|
1156
|
+
this.plan(name);
|
|
1157
|
+
} catch (err) {
|
|
1158
|
+
console.error(`Scheduler task "${name}" failed:`, err);
|
|
1159
|
+
this.plan(name);
|
|
1160
|
+
}
|
|
1161
|
+
}, delay);
|
|
1162
|
+
}
|
|
1163
|
+
async list() {
|
|
1164
|
+
return Array.from(this.tasks.entries()).map(([name, task]) => ({
|
|
1165
|
+
name,
|
|
1166
|
+
nextRun: new Date(task.nextRun),
|
|
1167
|
+
count: task.count,
|
|
1168
|
+
paused: task.paused
|
|
1169
|
+
}));
|
|
1170
|
+
}
|
|
1171
|
+
async cancel(name) {
|
|
1172
|
+
const task = this.tasks.get(name);
|
|
1173
|
+
if (task) {
|
|
1174
|
+
if (task.timer) clearTimeout(task.timer);
|
|
1175
|
+
this.tasks.delete(name);
|
|
1176
|
+
}
|
|
1177
|
+
}
|
|
1178
|
+
async pause(name) {
|
|
1179
|
+
const task = this.tasks.get(name);
|
|
1180
|
+
if (task) {
|
|
1181
|
+
task.paused = true;
|
|
1182
|
+
if (task.timer) clearTimeout(task.timer);
|
|
1183
|
+
}
|
|
1184
|
+
}
|
|
1185
|
+
async resume(name) {
|
|
1186
|
+
const task = this.tasks.get(name);
|
|
1187
|
+
if (task && task.paused) {
|
|
1188
|
+
task.paused = false;
|
|
1189
|
+
const options = typeof task.pattern === "string" ? { pattern: task.pattern } : task.pattern;
|
|
1190
|
+
if (options.pattern) {
|
|
1191
|
+
const interval = import_cron_parser.default.parseExpression(options.pattern, {
|
|
1192
|
+
currentDate: /* @__PURE__ */ new Date(),
|
|
1193
|
+
tz: options.timezone
|
|
1194
|
+
});
|
|
1195
|
+
task.nextRun = interval.next().getTime();
|
|
1196
|
+
} else if (options.every) {
|
|
1197
|
+
task.nextRun = Date.now() + options.every;
|
|
1198
|
+
}
|
|
1199
|
+
this.plan(name);
|
|
1200
|
+
}
|
|
1201
|
+
}
|
|
1202
|
+
async close() {
|
|
1203
|
+
for (const task of this.tasks.values()) {
|
|
1204
|
+
if (task.timer) clearTimeout(task.timer);
|
|
1205
|
+
}
|
|
1206
|
+
this.tasks.clear();
|
|
1207
|
+
}
|
|
1208
|
+
};
|
|
1209
|
+
|
|
1210
|
+
// src/core/metrics.ts
|
|
1211
|
+
var MetricsManager = class {
|
|
1212
|
+
constructor(adapter) {
|
|
1213
|
+
this.metrics = /* @__PURE__ */ new Map();
|
|
1214
|
+
this.maxDataPoints = 1e3;
|
|
1215
|
+
this.adapter = adapter;
|
|
1216
|
+
}
|
|
1217
|
+
/**
|
|
1218
|
+
* Record a metric data point
|
|
1219
|
+
*/
|
|
1220
|
+
record(name, value, tags) {
|
|
1221
|
+
const key = this.getMetricKey(name, tags);
|
|
1222
|
+
if (!this.metrics.has(key)) {
|
|
1223
|
+
this.metrics.set(key, []);
|
|
1224
|
+
}
|
|
1225
|
+
const dataPoints = this.metrics.get(key);
|
|
1226
|
+
dataPoints.push({
|
|
1227
|
+
timestamp: Date.now(),
|
|
1228
|
+
value,
|
|
1229
|
+
tags
|
|
1230
|
+
});
|
|
1231
|
+
if (dataPoints.length > this.maxDataPoints) {
|
|
1232
|
+
this.metrics.set(key, dataPoints.slice(-this.maxDataPoints));
|
|
1233
|
+
}
|
|
1234
|
+
}
|
|
1235
|
+
/**
|
|
1236
|
+
* Get time series for a metric
|
|
1237
|
+
*/
|
|
1238
|
+
getTimeSeries(name, options) {
|
|
1239
|
+
const key = this.getMetricKey(name, options?.tags);
|
|
1240
|
+
let dataPoints = this.metrics.get(key) || [];
|
|
1241
|
+
if (options?.since) {
|
|
1242
|
+
dataPoints = dataPoints.filter((dp) => dp.timestamp >= options.since);
|
|
1243
|
+
}
|
|
1244
|
+
if (options?.limit) {
|
|
1245
|
+
dataPoints = dataPoints.slice(-options.limit);
|
|
1246
|
+
}
|
|
1247
|
+
if (dataPoints.length === 0) {
|
|
1248
|
+
return null;
|
|
1249
|
+
}
|
|
1250
|
+
return this.calculateAggregations(dataPoints);
|
|
1251
|
+
}
|
|
1252
|
+
calculateAggregations(dataPoints) {
|
|
1253
|
+
const values = dataPoints.map((dp) => dp.value).sort((a, b) => a - b);
|
|
1254
|
+
const sum = values.reduce((a, b) => a + b, 0);
|
|
1255
|
+
const count = values.length;
|
|
1256
|
+
return {
|
|
1257
|
+
dataPoints,
|
|
1258
|
+
min: values[0],
|
|
1259
|
+
max: values[count - 1],
|
|
1260
|
+
avg: sum / count,
|
|
1261
|
+
sum,
|
|
1262
|
+
count,
|
|
1263
|
+
p50: this.percentile(values, 50),
|
|
1264
|
+
p95: this.percentile(values, 95),
|
|
1265
|
+
p99: this.percentile(values, 99)
|
|
1266
|
+
};
|
|
1267
|
+
}
|
|
1268
|
+
percentile(sorted, p) {
|
|
1269
|
+
const index2 = Math.ceil(sorted.length * p / 100) - 1;
|
|
1270
|
+
return sorted[Math.max(0, index2)];
|
|
1271
|
+
}
|
|
1272
|
+
getMetricKey(name, tags) {
|
|
1273
|
+
if (!tags) return name;
|
|
1274
|
+
const tagStr = Object.entries(tags).sort(([a], [b]) => a.localeCompare(b)).map(([k, v]) => `${k}:${v}`).join(",");
|
|
1275
|
+
return `${name}{${tagStr}}`;
|
|
1276
|
+
}
|
|
1277
|
+
async getQueueMetrics(name) {
|
|
1278
|
+
const stats = await this.adapter.getQueueStats(name);
|
|
1279
|
+
const throughputMetrics = this.getTimeSeries("queue.throughput", {
|
|
1280
|
+
tags: { queue: name },
|
|
1281
|
+
since: Date.now() - 6e4
|
|
1282
|
+
// Last minute
|
|
1283
|
+
});
|
|
1284
|
+
const durationMetrics = this.getTimeSeries("queue.job.duration", {
|
|
1285
|
+
tags: { queue: name }
|
|
1286
|
+
});
|
|
1287
|
+
return {
|
|
1288
|
+
...stats,
|
|
1289
|
+
throughput: throughputMetrics?.avg || 0,
|
|
1290
|
+
avgDuration: durationMetrics?.avg || 0,
|
|
1291
|
+
p95Duration: durationMetrics?.p95 || 0
|
|
1292
|
+
};
|
|
1293
|
+
}
|
|
1294
|
+
async getStreamMetrics(name) {
|
|
1295
|
+
const info = await this.adapter.getStreamInfo(name);
|
|
1296
|
+
const throughputMetrics = this.getTimeSeries("stream.throughput", {
|
|
1297
|
+
tags: { stream: name },
|
|
1298
|
+
since: Date.now() - 1e3
|
|
1299
|
+
// Last second
|
|
1300
|
+
});
|
|
1301
|
+
return {
|
|
1302
|
+
...info,
|
|
1303
|
+
lag: 0,
|
|
1304
|
+
throughput: throughputMetrics?.avg || 0,
|
|
1305
|
+
avgLatency: 0
|
|
1306
|
+
};
|
|
1307
|
+
}
|
|
1308
|
+
async getWorkflowMetrics(name) {
|
|
1309
|
+
const metricsData = this.getTimeSeries("workflow.executions", {
|
|
1310
|
+
tags: { workflow: name }
|
|
1311
|
+
});
|
|
1312
|
+
return {
|
|
1313
|
+
totalExecutions: metricsData?.count || 0,
|
|
1314
|
+
running: 0,
|
|
1315
|
+
completed: 0,
|
|
1316
|
+
failed: 0,
|
|
1317
|
+
successRate: 0,
|
|
1318
|
+
avgDuration: metricsData?.avg || 0
|
|
1319
|
+
};
|
|
1320
|
+
}
|
|
1321
|
+
async getSystemMetrics() {
|
|
1322
|
+
const memUsage = typeof process !== "undefined" && process.memoryUsage ? process.memoryUsage().heapUsed : 0;
|
|
1323
|
+
return {
|
|
1324
|
+
queues: 0,
|
|
1325
|
+
streams: 0,
|
|
1326
|
+
workflows: 0,
|
|
1327
|
+
workers: 0,
|
|
1328
|
+
memoryUsage: memUsage
|
|
1329
|
+
};
|
|
1330
|
+
}
|
|
1331
|
+
/**
|
|
1332
|
+
* Clear old metrics
|
|
1333
|
+
*/
|
|
1334
|
+
cleanup(maxAge) {
|
|
1335
|
+
const now2 = Date.now();
|
|
1336
|
+
for (const [key, dataPoints] of this.metrics.entries()) {
|
|
1337
|
+
const filtered = dataPoints.filter((dp) => now2 - dp.timestamp <= maxAge);
|
|
1338
|
+
if (filtered.length === 0) {
|
|
1339
|
+
this.metrics.delete(key);
|
|
1340
|
+
} else {
|
|
1341
|
+
this.metrics.set(key, filtered);
|
|
1342
|
+
}
|
|
1343
|
+
}
|
|
1344
|
+
}
|
|
1345
|
+
};
|
|
1346
|
+
|
|
1347
|
+
// src/monitoring/health.ts
|
|
1348
|
+
var HealthCheckerImpl = class {
|
|
1349
|
+
constructor() {
|
|
1350
|
+
this.checks = /* @__PURE__ */ new Map();
|
|
1351
|
+
this.startTime = Date.now();
|
|
1352
|
+
this.addCheck("uptime", async () => ({
|
|
1353
|
+
name: "uptime",
|
|
1354
|
+
status: "pass",
|
|
1355
|
+
responseTime: 0,
|
|
1356
|
+
details: {
|
|
1357
|
+
uptime: Date.now() - this.startTime,
|
|
1358
|
+
startTime: this.startTime
|
|
1359
|
+
}
|
|
1360
|
+
}));
|
|
1361
|
+
this.addCheck("memory", async () => {
|
|
1362
|
+
if (typeof process !== "undefined" && process.memoryUsage) {
|
|
1363
|
+
const mem = process.memoryUsage();
|
|
1364
|
+
const usedMB = mem.heapUsed / 1024 / 1024;
|
|
1365
|
+
const totalMB = mem.heapTotal / 1024 / 1024;
|
|
1366
|
+
const usagePercent = usedMB / totalMB * 100;
|
|
1367
|
+
return {
|
|
1368
|
+
name: "memory",
|
|
1369
|
+
status: usagePercent > 90 ? "warn" : "pass",
|
|
1370
|
+
message: usagePercent > 90 ? "High memory usage" : void 0,
|
|
1371
|
+
responseTime: 0,
|
|
1372
|
+
details: {
|
|
1373
|
+
heapUsedMB: Math.round(usedMB),
|
|
1374
|
+
heapTotalMB: Math.round(totalMB),
|
|
1375
|
+
usagePercent: Math.round(usagePercent)
|
|
1376
|
+
}
|
|
1377
|
+
};
|
|
1378
|
+
}
|
|
1379
|
+
return {
|
|
1380
|
+
name: "memory",
|
|
1381
|
+
status: "pass",
|
|
1382
|
+
responseTime: 0
|
|
1383
|
+
};
|
|
1384
|
+
});
|
|
1385
|
+
}
|
|
1386
|
+
addCheck(name, checker) {
|
|
1387
|
+
this.checks.set(name, checker);
|
|
1388
|
+
}
|
|
1389
|
+
removeCheck(name) {
|
|
1390
|
+
this.checks.delete(name);
|
|
1391
|
+
}
|
|
1392
|
+
async check() {
|
|
1393
|
+
const results = [];
|
|
1394
|
+
let allHealthy = true;
|
|
1395
|
+
for (const [name, checker] of this.checks.entries()) {
|
|
1396
|
+
try {
|
|
1397
|
+
const start = Date.now();
|
|
1398
|
+
const result = await checker();
|
|
1399
|
+
result.responseTime = Date.now() - start;
|
|
1400
|
+
results.push(result);
|
|
1401
|
+
if (result.status === "fail") {
|
|
1402
|
+
allHealthy = false;
|
|
1403
|
+
}
|
|
1404
|
+
} catch (error) {
|
|
1405
|
+
results.push({
|
|
1406
|
+
name,
|
|
1407
|
+
status: "fail",
|
|
1408
|
+
message: error.message,
|
|
1409
|
+
responseTime: 0
|
|
1410
|
+
});
|
|
1411
|
+
allHealthy = false;
|
|
1412
|
+
}
|
|
1413
|
+
}
|
|
1414
|
+
return {
|
|
1415
|
+
healthy: allHealthy,
|
|
1416
|
+
timestamp: Date.now(),
|
|
1417
|
+
checks: results
|
|
1418
|
+
};
|
|
1419
|
+
}
|
|
1420
|
+
};
|
|
1421
|
+
var MemoryEventTracker = class {
|
|
1422
|
+
constructor() {
|
|
1423
|
+
this.events = [];
|
|
1424
|
+
this.maxEvents = 1e4;
|
|
1425
|
+
}
|
|
1426
|
+
track(event) {
|
|
1427
|
+
const trackedEvent = {
|
|
1428
|
+
...event,
|
|
1429
|
+
id: `evt_${Date.now()}_${Math.random().toString(36).substring(7)}`,
|
|
1430
|
+
timestamp: Date.now()
|
|
1431
|
+
};
|
|
1432
|
+
this.events.push(trackedEvent);
|
|
1433
|
+
if (this.events.length > this.maxEvents) {
|
|
1434
|
+
this.events = this.events.slice(-this.maxEvents);
|
|
1435
|
+
}
|
|
1436
|
+
}
|
|
1437
|
+
getEvents(filter) {
|
|
1438
|
+
let filtered = [...this.events];
|
|
1439
|
+
if (filter?.category) {
|
|
1440
|
+
filtered = filtered.filter((e) => e.category === filter.category);
|
|
1441
|
+
}
|
|
1442
|
+
if (filter?.severity) {
|
|
1443
|
+
filtered = filtered.filter((e) => e.severity === filter.severity);
|
|
1444
|
+
}
|
|
1445
|
+
if (filter?.since !== void 0) {
|
|
1446
|
+
filtered = filtered.filter((e) => e.timestamp >= filter.since);
|
|
1447
|
+
}
|
|
1448
|
+
if (filter?.limit) {
|
|
1449
|
+
filtered = filtered.slice(-filter.limit);
|
|
1450
|
+
}
|
|
1451
|
+
return filtered;
|
|
1452
|
+
}
|
|
1453
|
+
cleanup(maxAge) {
|
|
1454
|
+
const now2 = Date.now();
|
|
1455
|
+
const before = this.events.length;
|
|
1456
|
+
this.events = this.events.filter((e) => now2 - e.timestamp <= maxAge);
|
|
1457
|
+
return before - this.events.length;
|
|
1458
|
+
}
|
|
1459
|
+
clear() {
|
|
1460
|
+
this.events = [];
|
|
1461
|
+
}
|
|
1462
|
+
};
|
|
1463
|
+
|
|
1464
|
+
// src/core/flow-fn.ts
|
|
1465
|
+
var import_eventemitter33 = __toESM(require("eventemitter3"));
|
|
1466
|
+
var FlowFnImpl = class extends import_eventemitter33.default {
|
|
1467
|
+
constructor(config) {
|
|
1468
|
+
super();
|
|
1469
|
+
this.queues = /* @__PURE__ */ new Map();
|
|
1470
|
+
this.streams = /* @__PURE__ */ new Map();
|
|
1471
|
+
this.startTime = Date.now();
|
|
1472
|
+
if (config.adapter === "memory") {
|
|
1473
|
+
this.adapter = new MemoryAdapter();
|
|
1474
|
+
} else if (typeof config.adapter === "string") {
|
|
1475
|
+
throw new Error(
|
|
1476
|
+
`Adapter ${config.adapter} not automatically initialized yet. Pass an instance.`
|
|
1477
|
+
);
|
|
1478
|
+
} else {
|
|
1479
|
+
this.adapter = config.adapter;
|
|
1480
|
+
}
|
|
1481
|
+
this._metrics = new MetricsManager(this.adapter);
|
|
1482
|
+
this._scheduler = new Scheduler();
|
|
1483
|
+
this.healthChecker = new HealthCheckerImpl();
|
|
1484
|
+
this.eventTracker = new MemoryEventTracker();
|
|
1485
|
+
this.setupHealthChecks();
|
|
1486
|
+
}
|
|
1487
|
+
setupHealthChecks() {
|
|
1488
|
+
this.healthChecker.addCheck("queues", async () => {
|
|
1489
|
+
const queueCount = this.queues.size;
|
|
1490
|
+
return {
|
|
1491
|
+
name: "queues",
|
|
1492
|
+
status: "pass",
|
|
1493
|
+
details: {
|
|
1494
|
+
count: queueCount,
|
|
1495
|
+
active: Array.from(this.queues.keys())
|
|
1496
|
+
}
|
|
1497
|
+
};
|
|
1498
|
+
});
|
|
1499
|
+
this.healthChecker.addCheck("adapter", async () => {
|
|
1500
|
+
try {
|
|
1501
|
+
await this.adapter.getQueueStats("health-check");
|
|
1502
|
+
return {
|
|
1503
|
+
name: "adapter",
|
|
1504
|
+
status: "pass",
|
|
1505
|
+
message: "Adapter responding"
|
|
1506
|
+
};
|
|
1507
|
+
} catch (error) {
|
|
1508
|
+
return {
|
|
1509
|
+
name: "adapter",
|
|
1510
|
+
status: "fail",
|
|
1511
|
+
message: error.message
|
|
1512
|
+
};
|
|
1513
|
+
}
|
|
1514
|
+
});
|
|
1515
|
+
}
|
|
1516
|
+
queue(name, options) {
|
|
1517
|
+
if (!this.queues.has(name)) {
|
|
1518
|
+
this.queues.set(name, new QueueImpl(name, this.adapter, options));
|
|
1519
|
+
}
|
|
1520
|
+
return this.queues.get(name);
|
|
1521
|
+
}
|
|
1522
|
+
async listQueues() {
|
|
1523
|
+
return Array.from(this.queues.keys());
|
|
1524
|
+
}
|
|
1525
|
+
stream(name, options) {
|
|
1526
|
+
if (!this.streams.has(name)) {
|
|
1527
|
+
this.streams.set(name, new StreamImpl(name, this.adapter, options));
|
|
1528
|
+
}
|
|
1529
|
+
return this.streams.get(name);
|
|
1530
|
+
}
|
|
1531
|
+
async listStreams() {
|
|
1532
|
+
return Array.from(this.streams.keys());
|
|
1533
|
+
}
|
|
1534
|
+
workflow(name) {
|
|
1535
|
+
return new WorkflowBuilderImpl(name, this.adapter);
|
|
1536
|
+
}
|
|
1537
|
+
async listWorkflows() {
|
|
1538
|
+
return [];
|
|
1539
|
+
}
|
|
1540
|
+
scheduler() {
|
|
1541
|
+
return this._scheduler;
|
|
1542
|
+
}
|
|
1543
|
+
get metrics() {
|
|
1544
|
+
return this._metrics;
|
|
1545
|
+
}
|
|
1546
|
+
async healthCheck() {
|
|
1547
|
+
const health = await this.healthChecker.check();
|
|
1548
|
+
this.eventTracker.track({
|
|
1549
|
+
type: "health.check",
|
|
1550
|
+
category: "system",
|
|
1551
|
+
severity: health.healthy ? "info" : "warn",
|
|
1552
|
+
message: health.healthy ? "System healthy" : "System unhealthy",
|
|
1553
|
+
metadata: { checksCount: health.checks.length }
|
|
1554
|
+
});
|
|
1555
|
+
return health;
|
|
1556
|
+
}
|
|
1557
|
+
/**
|
|
1558
|
+
* Get event tracker
|
|
1559
|
+
*/
|
|
1560
|
+
getEventTracker() {
|
|
1561
|
+
return this.eventTracker;
|
|
1562
|
+
}
|
|
1563
|
+
async close() {
|
|
1564
|
+
await this.adapter.cleanup();
|
|
1565
|
+
for (const q of this.queues.values()) await q.close();
|
|
1566
|
+
for (const s of this.streams.values()) await s.close();
|
|
1567
|
+
}
|
|
1568
|
+
};
|
|
1569
|
+
function createFlow(config) {
|
|
1570
|
+
return new FlowFnImpl(config);
|
|
1571
|
+
}
|
|
1572
|
+
|
|
1573
|
+
// src/queue/worker.ts
|
|
1574
|
+
var import_eventemitter34 = __toESM(require("eventemitter3"));
|
|
1575
|
+
var Worker = class extends import_eventemitter34.default {
|
|
1576
|
+
constructor(options) {
|
|
1577
|
+
super();
|
|
1578
|
+
this.queues = [];
|
|
1579
|
+
this.flow = options.flow;
|
|
1580
|
+
this.queueNames = options.queues;
|
|
1581
|
+
this.concurrency = options.concurrency || 1;
|
|
1582
|
+
}
|
|
1583
|
+
async run() {
|
|
1584
|
+
for (const name of this.queueNames) {
|
|
1585
|
+
const queue = this.flow.queue(name);
|
|
1586
|
+
this.queues.push(queue);
|
|
1587
|
+
const qConcurrency = typeof this.concurrency === "number" ? this.concurrency : this.concurrency[name] || 1;
|
|
1588
|
+
queue.on("active", (job) => this.emit("active", job));
|
|
1589
|
+
queue.on("completed", (job, result) => this.emit("completed", job, result));
|
|
1590
|
+
queue.on("failed", (job, err) => this.emit("failed", job, err));
|
|
1591
|
+
}
|
|
1592
|
+
this.emit("ready");
|
|
1593
|
+
}
|
|
1594
|
+
// Helper to register handlers
|
|
1595
|
+
register(queueName, handler) {
|
|
1596
|
+
const queue = this.flow.queue(queueName);
|
|
1597
|
+
const qConcurrency = typeof this.concurrency === "number" ? this.concurrency : this.concurrency[queueName] || 1;
|
|
1598
|
+
queue.process(qConcurrency, handler);
|
|
1599
|
+
}
|
|
1600
|
+
async close() {
|
|
1601
|
+
this.emit("closing");
|
|
1602
|
+
await Promise.all(this.queues.map((q) => q.close()));
|
|
1603
|
+
}
|
|
1604
|
+
};
|
|
1605
|
+
function createWorker(options) {
|
|
1606
|
+
return new Worker(options);
|
|
1607
|
+
}
|
|
1608
|
+
|
|
1609
|
+
// src/queue/dlq.ts
|
|
1610
|
+
var MemoryDLQManager = class {
|
|
1611
|
+
constructor(options = {}) {
|
|
1612
|
+
this.dlqJobs = /* @__PURE__ */ new Map();
|
|
1613
|
+
this.options = {
|
|
1614
|
+
queueName: "dlq",
|
|
1615
|
+
maxRetries: 3,
|
|
1616
|
+
ttl: 7 * 24 * 60 * 60 * 1e3,
|
|
1617
|
+
// 7 days
|
|
1618
|
+
...options
|
|
1619
|
+
};
|
|
1620
|
+
}
|
|
1621
|
+
async moveToDLQ(job, reason) {
|
|
1622
|
+
const dlqJob = {
|
|
1623
|
+
...job,
|
|
1624
|
+
originalQueue: job.name,
|
|
1625
|
+
dlqReason: reason,
|
|
1626
|
+
dlqTimestamp: Date.now(),
|
|
1627
|
+
errors: [...job.stacktrace || [], job.failedReason || reason].filter(
|
|
1628
|
+
Boolean
|
|
1629
|
+
)
|
|
1630
|
+
};
|
|
1631
|
+
this.dlqJobs.set(job.id, dlqJob);
|
|
1632
|
+
if (this.options.onDLQ) {
|
|
1633
|
+
await Promise.resolve(this.options.onDLQ(job, reason));
|
|
1634
|
+
}
|
|
1635
|
+
return dlqJob;
|
|
1636
|
+
}
|
|
1637
|
+
async getAll() {
|
|
1638
|
+
return Array.from(this.dlqJobs.values());
|
|
1639
|
+
}
|
|
1640
|
+
async getByQueue(queueName) {
|
|
1641
|
+
return Array.from(this.dlqJobs.values()).filter(
|
|
1642
|
+
(job) => job.originalQueue === queueName
|
|
1643
|
+
);
|
|
1644
|
+
}
|
|
1645
|
+
async retry(jobId) {
|
|
1646
|
+
const dlqJob = this.dlqJobs.get(jobId);
|
|
1647
|
+
if (!dlqJob) {
|
|
1648
|
+
throw new Error(`DLQ job ${jobId} not found`);
|
|
1649
|
+
}
|
|
1650
|
+
this.dlqJobs.delete(jobId);
|
|
1651
|
+
const retriedJob = {
|
|
1652
|
+
...dlqJob,
|
|
1653
|
+
state: "waiting",
|
|
1654
|
+
attemptsMade: 0,
|
|
1655
|
+
failedReason: void 0,
|
|
1656
|
+
stacktrace: [],
|
|
1657
|
+
processedOn: void 0,
|
|
1658
|
+
finishedOn: void 0
|
|
1659
|
+
};
|
|
1660
|
+
return retriedJob;
|
|
1661
|
+
}
|
|
1662
|
+
async retryAll(queueName) {
|
|
1663
|
+
const jobs2 = await this.getByQueue(queueName);
|
|
1664
|
+
let count = 0;
|
|
1665
|
+
for (const job of jobs2) {
|
|
1666
|
+
try {
|
|
1667
|
+
await this.retry(job.id);
|
|
1668
|
+
count++;
|
|
1669
|
+
} catch (err) {
|
|
1670
|
+
console.error(`Failed to retry job ${job.id}:`, err);
|
|
1671
|
+
}
|
|
1672
|
+
}
|
|
1673
|
+
return count;
|
|
1674
|
+
}
|
|
1675
|
+
async delete(jobId) {
|
|
1676
|
+
this.dlqJobs.delete(jobId);
|
|
1677
|
+
}
|
|
1678
|
+
async clean(maxAge) {
|
|
1679
|
+
const now2 = Date.now();
|
|
1680
|
+
const toDelete = [];
|
|
1681
|
+
for (const [id, job] of this.dlqJobs.entries()) {
|
|
1682
|
+
if (now2 - job.dlqTimestamp > maxAge) {
|
|
1683
|
+
toDelete.push(id);
|
|
1684
|
+
}
|
|
1685
|
+
}
|
|
1686
|
+
for (const id of toDelete) {
|
|
1687
|
+
this.dlqJobs.delete(id);
|
|
1688
|
+
}
|
|
1689
|
+
return toDelete.length;
|
|
1690
|
+
}
|
|
1691
|
+
async getStats() {
|
|
1692
|
+
const byQueue = {};
|
|
1693
|
+
for (const job of this.dlqJobs.values()) {
|
|
1694
|
+
byQueue[job.originalQueue] = (byQueue[job.originalQueue] || 0) + 1;
|
|
1695
|
+
}
|
|
1696
|
+
return {
|
|
1697
|
+
total: this.dlqJobs.size,
|
|
1698
|
+
byQueue
|
|
1699
|
+
};
|
|
1700
|
+
}
|
|
1701
|
+
/**
|
|
1702
|
+
* Clear all DLQ jobs (for testing)
|
|
1703
|
+
*/
|
|
1704
|
+
clear() {
|
|
1705
|
+
this.dlqJobs.clear();
|
|
1706
|
+
}
|
|
1707
|
+
};
|
|
1708
|
+
|
|
1709
|
+
// src/adapters/redis.ts
|
|
1710
|
+
var import_ioredis = __toESM(require("ioredis"));
|
|
1711
|
+
var RedisAdapter = class {
|
|
1712
|
+
constructor(options = {}) {
|
|
1713
|
+
this.subscriptions = /* @__PURE__ */ new Map();
|
|
1714
|
+
if (options.connection instanceof import_ioredis.default) {
|
|
1715
|
+
this.redis = options.connection;
|
|
1716
|
+
} else {
|
|
1717
|
+
this.redis = new import_ioredis.default(options.connection || {});
|
|
1718
|
+
}
|
|
1719
|
+
this.prefix = options.prefix || "flowfn:";
|
|
1720
|
+
}
|
|
1721
|
+
key(type, name) {
|
|
1722
|
+
return `${this.prefix}${type}:${name}`;
|
|
1723
|
+
}
|
|
1724
|
+
async enqueue(queueName, job) {
|
|
1725
|
+
const queueKey = this.key("queue", queueName);
|
|
1726
|
+
const jobKey = this.key("job", job.id);
|
|
1727
|
+
await this.redis.set(jobKey, JSON.stringify(job));
|
|
1728
|
+
if (job.opts.delay && job.opts.delay > 0) {
|
|
1729
|
+
const delayedKey = this.key("delayed", queueName);
|
|
1730
|
+
await this.redis.zadd(delayedKey, Date.now() + job.opts.delay, job.id);
|
|
1731
|
+
} else {
|
|
1732
|
+
await this.redis.lpush(queueKey, job.id);
|
|
1733
|
+
}
|
|
1734
|
+
return job.id;
|
|
1735
|
+
}
|
|
1736
|
+
async dequeue(queueName) {
|
|
1737
|
+
const queueKey = this.key("queue", queueName);
|
|
1738
|
+
const delayedKey = this.key("delayed", queueName);
|
|
1739
|
+
const now2 = Date.now();
|
|
1740
|
+
const delayed = await this.redis.zrangebyscore(
|
|
1741
|
+
delayedKey,
|
|
1742
|
+
0,
|
|
1743
|
+
now2,
|
|
1744
|
+
"LIMIT",
|
|
1745
|
+
0,
|
|
1746
|
+
1
|
|
1747
|
+
);
|
|
1748
|
+
if (delayed.length > 0) {
|
|
1749
|
+
const jobId2 = delayed[0];
|
|
1750
|
+
await this.redis.zrem(delayedKey, jobId2);
|
|
1751
|
+
await this.redis.lpush(queueKey, jobId2);
|
|
1752
|
+
}
|
|
1753
|
+
const jobId = await this.redis.rpop(queueKey);
|
|
1754
|
+
if (!jobId) return null;
|
|
1755
|
+
const jobData = await this.redis.get(this.key("job", jobId));
|
|
1756
|
+
if (!jobData) return null;
|
|
1757
|
+
return JSON.parse(jobData);
|
|
1758
|
+
}
|
|
1759
|
+
async ack(queue, jobId) {
|
|
1760
|
+
await this.redis.del(this.key("job", jobId));
|
|
1761
|
+
}
|
|
1762
|
+
async nack(queueName, jobId, requeue = true) {
|
|
1763
|
+
if (requeue) {
|
|
1764
|
+
await this.redis.lpush(this.key("queue", queueName), jobId);
|
|
1765
|
+
}
|
|
1766
|
+
}
|
|
1767
|
+
async publish(streamName, message) {
|
|
1768
|
+
const streamKey = this.key("stream", streamName);
|
|
1769
|
+
const id = await this.redis.xadd(
|
|
1770
|
+
streamKey,
|
|
1771
|
+
"*",
|
|
1772
|
+
"data",
|
|
1773
|
+
JSON.stringify(message.data),
|
|
1774
|
+
"headers",
|
|
1775
|
+
JSON.stringify(message.headers || {})
|
|
1776
|
+
);
|
|
1777
|
+
return id || "";
|
|
1778
|
+
}
|
|
1779
|
+
async subscribe(streamName, handler) {
|
|
1780
|
+
const streamKey = this.key("stream", streamName);
|
|
1781
|
+
let active = true;
|
|
1782
|
+
let lastId = "$";
|
|
1783
|
+
const poll = async () => {
|
|
1784
|
+
if (!active) return;
|
|
1785
|
+
try {
|
|
1786
|
+
const results = await this.redis.xread(
|
|
1787
|
+
"STREAMS",
|
|
1788
|
+
streamKey,
|
|
1789
|
+
lastId,
|
|
1790
|
+
"BLOCK",
|
|
1791
|
+
1e3
|
|
1792
|
+
);
|
|
1793
|
+
if (results) {
|
|
1794
|
+
for (const [stream, messages2] of results) {
|
|
1795
|
+
for (const [id, fields] of messages2) {
|
|
1796
|
+
lastId = id;
|
|
1797
|
+
const dataIdx = fields.indexOf("data");
|
|
1798
|
+
const headersIdx = fields.indexOf("headers");
|
|
1799
|
+
const data = dataIdx > -1 ? JSON.parse(fields[dataIdx + 1]) : {};
|
|
1800
|
+
const headers = headersIdx > -1 ? JSON.parse(fields[headersIdx + 1]) : {};
|
|
1801
|
+
const msg = {
|
|
1802
|
+
id,
|
|
1803
|
+
stream: streamName,
|
|
1804
|
+
data,
|
|
1805
|
+
headers,
|
|
1806
|
+
timestamp: parseInt(id.split("-")[0]),
|
|
1807
|
+
ack: async () => {
|
|
1808
|
+
},
|
|
1809
|
+
nack: async () => {
|
|
1810
|
+
}
|
|
1811
|
+
};
|
|
1812
|
+
handler(msg).catch(console.error);
|
|
1813
|
+
}
|
|
1814
|
+
}
|
|
1815
|
+
}
|
|
1816
|
+
} catch (err) {
|
|
1817
|
+
console.error("Redis stream poll error", err);
|
|
1818
|
+
await new Promise((r) => setTimeout(r, 1e3));
|
|
1819
|
+
}
|
|
1820
|
+
if (active) setTimeout(poll, 0);
|
|
1821
|
+
};
|
|
1822
|
+
setTimeout(poll, 0);
|
|
1823
|
+
return {
|
|
1824
|
+
unsubscribe: async () => {
|
|
1825
|
+
active = false;
|
|
1826
|
+
}
|
|
1827
|
+
};
|
|
1828
|
+
}
|
|
1829
|
+
async consume(streamName, group, consumer, handler) {
|
|
1830
|
+
const streamKey = this.key("stream", streamName);
|
|
1831
|
+
try {
|
|
1832
|
+
await this.redis.xgroup("CREATE", streamKey, group, "$", "MKSTREAM");
|
|
1833
|
+
} catch (e) {
|
|
1834
|
+
if (!e.message.includes("BUSYGROUP")) throw e;
|
|
1835
|
+
}
|
|
1836
|
+
let active = true;
|
|
1837
|
+
const poll = async () => {
|
|
1838
|
+
if (!active) return;
|
|
1839
|
+
try {
|
|
1840
|
+
const results = await this.redis.xreadgroup(
|
|
1841
|
+
"GROUP",
|
|
1842
|
+
group,
|
|
1843
|
+
consumer,
|
|
1844
|
+
"COUNT",
|
|
1845
|
+
10,
|
|
1846
|
+
"BLOCK",
|
|
1847
|
+
1e3,
|
|
1848
|
+
"STREAMS",
|
|
1849
|
+
streamKey,
|
|
1850
|
+
">"
|
|
1851
|
+
);
|
|
1852
|
+
if (results) {
|
|
1853
|
+
for (const [stream, messages2] of results) {
|
|
1854
|
+
for (const [id, fields] of messages2) {
|
|
1855
|
+
const dataIdx = fields.indexOf("data");
|
|
1856
|
+
const headersIdx = fields.indexOf("headers");
|
|
1857
|
+
const data = dataIdx > -1 ? JSON.parse(fields[dataIdx + 1]) : {};
|
|
1858
|
+
const headers = headersIdx > -1 ? JSON.parse(fields[headersIdx + 1]) : {};
|
|
1859
|
+
const msg = {
|
|
1860
|
+
id,
|
|
1861
|
+
stream: streamName,
|
|
1862
|
+
data,
|
|
1863
|
+
headers,
|
|
1864
|
+
timestamp: parseInt(id.split("-")[0]),
|
|
1865
|
+
ack: async () => {
|
|
1866
|
+
await this.redis.xack(streamKey, group, id);
|
|
1867
|
+
},
|
|
1868
|
+
nack: async (requeue = true) => {
|
|
1869
|
+
}
|
|
1870
|
+
};
|
|
1871
|
+
handler(msg).catch(console.error);
|
|
1872
|
+
}
|
|
1873
|
+
}
|
|
1874
|
+
}
|
|
1875
|
+
} catch (err) {
|
|
1876
|
+
console.error("Redis consumer poll error", err);
|
|
1877
|
+
await new Promise((r) => setTimeout(r, 1e3));
|
|
1878
|
+
}
|
|
1879
|
+
if (active) setTimeout(poll, 0);
|
|
1880
|
+
};
|
|
1881
|
+
setTimeout(poll, 0);
|
|
1882
|
+
return {
|
|
1883
|
+
unsubscribe: async () => {
|
|
1884
|
+
active = false;
|
|
1885
|
+
}
|
|
1886
|
+
};
|
|
1887
|
+
}
|
|
1888
|
+
async createConsumerGroup(stream, group) {
|
|
1889
|
+
const streamKey = this.key("stream", stream);
|
|
1890
|
+
try {
|
|
1891
|
+
await this.redis.xgroup("CREATE", streamKey, group, "$", "MKSTREAM");
|
|
1892
|
+
} catch (e) {
|
|
1893
|
+
if (!e.message.includes("BUSYGROUP")) throw e;
|
|
1894
|
+
}
|
|
1895
|
+
}
|
|
1896
|
+
async saveWorkflowState(workflowId, state) {
|
|
1897
|
+
await this.redis.set(
|
|
1898
|
+
this.key("workflow", workflowId),
|
|
1899
|
+
JSON.stringify(state)
|
|
1900
|
+
);
|
|
1901
|
+
}
|
|
1902
|
+
async loadWorkflowState(workflowId) {
|
|
1903
|
+
const data = await this.redis.get(this.key("workflow", workflowId));
|
|
1904
|
+
return data ? JSON.parse(data) : null;
|
|
1905
|
+
}
|
|
1906
|
+
async getJob(queue, jobId) {
|
|
1907
|
+
const jobData = await this.redis.get(this.key("job", jobId));
|
|
1908
|
+
return jobData ? JSON.parse(jobData) : null;
|
|
1909
|
+
}
|
|
1910
|
+
async getJobs(queue, status) {
|
|
1911
|
+
return [];
|
|
1912
|
+
}
|
|
1913
|
+
async getAllJobs(queue) {
|
|
1914
|
+
return [];
|
|
1915
|
+
}
|
|
1916
|
+
async cleanJobs(queue, grace, status) {
|
|
1917
|
+
return 0;
|
|
1918
|
+
}
|
|
1919
|
+
async getQueueStats(queueName) {
|
|
1920
|
+
const length = await this.redis.llen(this.key("queue", queueName));
|
|
1921
|
+
const delayedLength = await this.redis.zcard(
|
|
1922
|
+
this.key("delayed", queueName)
|
|
1923
|
+
);
|
|
1924
|
+
return {
|
|
1925
|
+
waiting: length,
|
|
1926
|
+
active: 0,
|
|
1927
|
+
completed: 0,
|
|
1928
|
+
failed: 0,
|
|
1929
|
+
delayed: delayedLength,
|
|
1930
|
+
paused: 0
|
|
1931
|
+
};
|
|
1932
|
+
}
|
|
1933
|
+
async getStreamInfo(streamName) {
|
|
1934
|
+
return {
|
|
1935
|
+
name: streamName,
|
|
1936
|
+
length: 0,
|
|
1937
|
+
groups: 0
|
|
1938
|
+
};
|
|
1939
|
+
}
|
|
1940
|
+
async cleanup() {
|
|
1941
|
+
for (const subs of this.subscriptions.values()) {
|
|
1942
|
+
for (const sub of subs) {
|
|
1943
|
+
await sub.quit();
|
|
1944
|
+
}
|
|
1945
|
+
}
|
|
1946
|
+
this.subscriptions.clear();
|
|
1947
|
+
await this.redis.quit();
|
|
1948
|
+
}
|
|
1949
|
+
};
|
|
1950
|
+
|
|
1951
|
+
// src/adapters/postgres/schema.ts
|
|
1952
|
+
var import_pg_core = require("drizzle-orm/pg-core");
|
|
1953
|
+
var jobs = (0, import_pg_core.pgTable)("flowfn_jobs", {
|
|
1954
|
+
id: (0, import_pg_core.varchar)("id", { length: 255 }).primaryKey(),
|
|
1955
|
+
queue: (0, import_pg_core.varchar)("queue", { length: 255 }).notNull(),
|
|
1956
|
+
name: (0, import_pg_core.varchar)("name", { length: 255 }).notNull(),
|
|
1957
|
+
data: (0, import_pg_core.jsonb)("data").notNull(),
|
|
1958
|
+
opts: (0, import_pg_core.jsonb)("opts"),
|
|
1959
|
+
state: (0, import_pg_core.varchar)("state", { length: 50 }).notNull(),
|
|
1960
|
+
// waiting, active, completed, failed, delayed, paused
|
|
1961
|
+
priority: (0, import_pg_core.integer)("priority").default(0),
|
|
1962
|
+
progress: (0, import_pg_core.integer)("progress").default(0),
|
|
1963
|
+
returnValue: (0, import_pg_core.jsonb)("return_value"),
|
|
1964
|
+
timestamp: (0, import_pg_core.bigint)("timestamp", { mode: "number" }).notNull(),
|
|
1965
|
+
processedOn: (0, import_pg_core.bigint)("processed_on", { mode: "number" }),
|
|
1966
|
+
finishedOn: (0, import_pg_core.bigint)("finished_on", { mode: "number" }),
|
|
1967
|
+
delay: (0, import_pg_core.bigint)("delay", { mode: "number" }).default(0),
|
|
1968
|
+
attemptsMade: (0, import_pg_core.integer)("attempts_made").default(0),
|
|
1969
|
+
failedReason: (0, import_pg_core.text)("failed_reason"),
|
|
1970
|
+
stacktrace: (0, import_pg_core.jsonb)("stacktrace"),
|
|
1971
|
+
createdAt: (0, import_pg_core.timestamp)("created_at").defaultNow(),
|
|
1972
|
+
updatedAt: (0, import_pg_core.timestamp)("updated_at").defaultNow()
|
|
1973
|
+
}, (table) => ({
|
|
1974
|
+
queueStateIdx: (0, import_pg_core.index)("idx_queue_state").on(table.queue, table.state),
|
|
1975
|
+
queuePriorityIdx: (0, import_pg_core.index)("idx_queue_priority").on(table.queue, table.priority),
|
|
1976
|
+
timestampIdx: (0, import_pg_core.index)("idx_timestamp").on(table.timestamp)
|
|
1977
|
+
}));
|
|
1978
|
+
var messages = (0, import_pg_core.pgTable)("flowfn_messages", {
|
|
1979
|
+
id: (0, import_pg_core.varchar)("id", { length: 255 }).primaryKey(),
|
|
1980
|
+
stream: (0, import_pg_core.varchar)("stream", { length: 255 }).notNull(),
|
|
1981
|
+
data: (0, import_pg_core.jsonb)("data").notNull(),
|
|
1982
|
+
headers: (0, import_pg_core.jsonb)("headers"),
|
|
1983
|
+
partition: (0, import_pg_core.integer)("partition"),
|
|
1984
|
+
offset: (0, import_pg_core.bigint)("offset", { mode: "number" }),
|
|
1985
|
+
key: (0, import_pg_core.varchar)("key", { length: 255 }),
|
|
1986
|
+
timestamp: (0, import_pg_core.bigint)("timestamp", { mode: "number" }).notNull(),
|
|
1987
|
+
createdAt: (0, import_pg_core.timestamp)("created_at").defaultNow()
|
|
1988
|
+
}, (table) => ({
|
|
1989
|
+
streamTimestampIdx: (0, import_pg_core.index)("idx_stream_timestamp").on(table.stream, table.timestamp)
|
|
1990
|
+
}));
|
|
1991
|
+
var consumerGroups = (0, import_pg_core.pgTable)("flowfn_consumer_groups", {
|
|
1992
|
+
id: (0, import_pg_core.varchar)("id", { length: 255 }).primaryKey(),
|
|
1993
|
+
stream: (0, import_pg_core.varchar)("stream", { length: 255 }).notNull(),
|
|
1994
|
+
groupId: (0, import_pg_core.varchar)("group_id", { length: 255 }).notNull(),
|
|
1995
|
+
consumerId: (0, import_pg_core.varchar)("consumer_id", { length: 255 }).notNull(),
|
|
1996
|
+
lastMessageId: (0, import_pg_core.varchar)("last_message_id", { length: 255 }),
|
|
1997
|
+
lastOffset: (0, import_pg_core.bigint)("last_offset", { mode: "number" }),
|
|
1998
|
+
lag: (0, import_pg_core.integer)("lag"),
|
|
1999
|
+
createdAt: (0, import_pg_core.timestamp)("created_at").defaultNow(),
|
|
2000
|
+
updatedAt: (0, import_pg_core.timestamp)("updated_at").defaultNow()
|
|
2001
|
+
}, (table) => ({
|
|
2002
|
+
streamGroupConsumerIdx: (0, import_pg_core.uniqueIndex)("idx_stream_group_consumer").on(table.stream, table.groupId, table.consumerId)
|
|
2003
|
+
}));
|
|
2004
|
+
var workflows = (0, import_pg_core.pgTable)("flowfn_workflows", {
|
|
2005
|
+
id: (0, import_pg_core.varchar)("id", { length: 255 }).primaryKey(),
|
|
2006
|
+
name: (0, import_pg_core.varchar)("name", { length: 255 }).notNull(),
|
|
2007
|
+
definition: (0, import_pg_core.jsonb)("definition").notNull(),
|
|
2008
|
+
version: (0, import_pg_core.integer)("version").default(1),
|
|
2009
|
+
status: (0, import_pg_core.varchar)("status", { length: 50 }).notNull(),
|
|
2010
|
+
metadata: (0, import_pg_core.jsonb)("metadata"),
|
|
2011
|
+
createdAt: (0, import_pg_core.timestamp)("created_at").defaultNow(),
|
|
2012
|
+
updatedAt: (0, import_pg_core.timestamp)("updated_at").defaultNow()
|
|
2013
|
+
});
|
|
2014
|
+
var workflowExecutions = (0, import_pg_core.pgTable)("flowfn_workflow_executions", {
|
|
2015
|
+
id: (0, import_pg_core.varchar)("id", { length: 255 }).primaryKey(),
|
|
2016
|
+
workflowId: (0, import_pg_core.varchar)("workflow_id", { length: 255 }).notNull(),
|
|
2017
|
+
workflowName: (0, import_pg_core.varchar)("workflow_name", { length: 255 }),
|
|
2018
|
+
status: (0, import_pg_core.varchar)("status", { length: 50 }).notNull(),
|
|
2019
|
+
input: (0, import_pg_core.jsonb)("input"),
|
|
2020
|
+
state: (0, import_pg_core.jsonb)("state"),
|
|
2021
|
+
output: (0, import_pg_core.jsonb)("output"),
|
|
2022
|
+
error: (0, import_pg_core.text)("error"),
|
|
2023
|
+
currentStep: (0, import_pg_core.varchar)("current_step", { length: 255 }),
|
|
2024
|
+
completedSteps: (0, import_pg_core.jsonb)("completed_steps"),
|
|
2025
|
+
startedAt: (0, import_pg_core.bigint)("started_at", { mode: "number" }).notNull(),
|
|
2026
|
+
updatedAt: (0, import_pg_core.bigint)("updated_at", { mode: "number" }),
|
|
2027
|
+
completedAt: (0, import_pg_core.bigint)("completed_at", { mode: "number" }),
|
|
2028
|
+
durationMs: (0, import_pg_core.integer)("duration_ms")
|
|
2029
|
+
}, (table) => ({
|
|
2030
|
+
workflowIdx: (0, import_pg_core.index)("idx_workflow").on(table.workflowId),
|
|
2031
|
+
statusIdx: (0, import_pg_core.index)("idx_status").on(table.status)
|
|
2032
|
+
}));
|
|
2033
|
+
var workflowEvents = (0, import_pg_core.pgTable)("flowfn_workflow_events", {
|
|
2034
|
+
id: (0, import_pg_core.varchar)("id", { length: 255 }).primaryKey(),
|
|
2035
|
+
executionId: (0, import_pg_core.varchar)("execution_id", { length: 255 }).notNull(),
|
|
2036
|
+
type: (0, import_pg_core.varchar)("type", { length: 50 }).notNull(),
|
|
2037
|
+
step: (0, import_pg_core.varchar)("step", { length: 255 }),
|
|
2038
|
+
timestamp: (0, import_pg_core.bigint)("timestamp", { mode: "number" }).notNull(),
|
|
2039
|
+
data: (0, import_pg_core.jsonb)("data")
|
|
2040
|
+
}, (table) => ({
|
|
2041
|
+
executionIdx: (0, import_pg_core.index)("idx_execution").on(table.executionId)
|
|
2042
|
+
}));
|
|
2043
|
+
|
|
2044
|
+
// src/adapters/postgres/index.ts
|
|
2045
|
+
var import_drizzle_orm = require("drizzle-orm");
|
|
2046
|
+
var import_uuid4 = require("uuid");
|
|
2047
|
+
var PostgresAdapter = class {
|
|
2048
|
+
constructor(options) {
|
|
2049
|
+
this.activeSubscriptions = /* @__PURE__ */ new Map();
|
|
2050
|
+
this.db = options.db;
|
|
2051
|
+
this.pollInterval = options.pollInterval || 1e3;
|
|
2052
|
+
}
|
|
2053
|
+
async enqueue(queueName, job) {
|
|
2054
|
+
await this.db.insert(jobs).values({
|
|
2055
|
+
id: job.id,
|
|
2056
|
+
queue: queueName,
|
|
2057
|
+
name: job.name,
|
|
2058
|
+
data: job.data,
|
|
2059
|
+
opts: job.opts,
|
|
2060
|
+
state: job.opts.delay ? "delayed" : "waiting",
|
|
2061
|
+
timestamp: job.timestamp,
|
|
2062
|
+
delay: job.opts.delay || 0,
|
|
2063
|
+
priority: job.opts.priority || 0,
|
|
2064
|
+
attemptsMade: 0
|
|
2065
|
+
});
|
|
2066
|
+
return job.id;
|
|
2067
|
+
}
|
|
2068
|
+
async dequeue(queueName) {
|
|
2069
|
+
const now2 = Date.now();
|
|
2070
|
+
return await this.db.transaction(async (tx) => {
|
|
2071
|
+
const result = await tx.execute(import_drizzle_orm.sql`
|
|
2072
|
+
UPDATE flowfn_jobs
|
|
2073
|
+
SET state = 'active', processed_on = ${now2}
|
|
2074
|
+
WHERE id = (
|
|
2075
|
+
SELECT id
|
|
2076
|
+
FROM flowfn_jobs
|
|
2077
|
+
WHERE queue = ${queueName}
|
|
2078
|
+
AND (
|
|
2079
|
+
state = 'waiting'
|
|
2080
|
+
OR (state = 'delayed' AND (timestamp + delay) <= ${now2})
|
|
2081
|
+
)
|
|
2082
|
+
ORDER BY priority DESC, timestamp ASC
|
|
2083
|
+
FOR UPDATE SKIP LOCKED
|
|
2084
|
+
LIMIT 1
|
|
2085
|
+
)
|
|
2086
|
+
RETURNING *
|
|
2087
|
+
`);
|
|
2088
|
+
if (result.length === 0) return null;
|
|
2089
|
+
const row = result[0];
|
|
2090
|
+
return {
|
|
2091
|
+
id: row.id,
|
|
2092
|
+
name: row.name,
|
|
2093
|
+
data: row.data,
|
|
2094
|
+
opts: row.opts,
|
|
2095
|
+
state: row.state,
|
|
2096
|
+
progress: row.progress,
|
|
2097
|
+
returnvalue: row.return_value,
|
|
2098
|
+
timestamp: Number(row.timestamp),
|
|
2099
|
+
processedOn: Number(row.processed_on),
|
|
2100
|
+
finishedOn: Number(row.finished_on),
|
|
2101
|
+
delay: Number(row.delay),
|
|
2102
|
+
attemptsMade: row.attempts_made,
|
|
2103
|
+
failedReason: row.failed_reason,
|
|
2104
|
+
stacktrace: row.stacktrace
|
|
2105
|
+
// Re-bind methods in queue implementation
|
|
2106
|
+
};
|
|
2107
|
+
});
|
|
2108
|
+
}
|
|
2109
|
+
async ack(queue, jobId) {
|
|
2110
|
+
await this.db.update(jobs).set({ state: "completed", finishedOn: Date.now() }).where((0, import_drizzle_orm.eq)(jobs.id, jobId));
|
|
2111
|
+
}
|
|
2112
|
+
async nack(queue, jobId, requeue = true) {
|
|
2113
|
+
if (requeue) {
|
|
2114
|
+
await this.db.update(jobs).set({ state: "waiting", processedOn: null }).where((0, import_drizzle_orm.eq)(jobs.id, jobId));
|
|
2115
|
+
} else {
|
|
2116
|
+
}
|
|
2117
|
+
}
|
|
2118
|
+
async publish(streamName, message) {
|
|
2119
|
+
await this.db.insert(messages).values({
|
|
2120
|
+
id: message.id,
|
|
2121
|
+
stream: streamName,
|
|
2122
|
+
data: message.data,
|
|
2123
|
+
headers: message.headers,
|
|
2124
|
+
timestamp: message.timestamp,
|
|
2125
|
+
partition: message.partition,
|
|
2126
|
+
key: message.key
|
|
2127
|
+
});
|
|
2128
|
+
return message.id;
|
|
2129
|
+
}
|
|
2130
|
+
async subscribe(streamName, handler) {
|
|
2131
|
+
const subId = (0, import_uuid4.v4)();
|
|
2132
|
+
this.activeSubscriptions.set(subId, true);
|
|
2133
|
+
let lastTimestamp = Date.now();
|
|
2134
|
+
const poll = async () => {
|
|
2135
|
+
if (!this.activeSubscriptions.get(subId)) return;
|
|
2136
|
+
try {
|
|
2137
|
+
const msgs = await this.db.select().from(messages).where(
|
|
2138
|
+
(0, import_drizzle_orm.and)(
|
|
2139
|
+
(0, import_drizzle_orm.eq)(messages.stream, streamName),
|
|
2140
|
+
import_drizzle_orm.sql`${messages.timestamp} > ${lastTimestamp}`
|
|
2141
|
+
)
|
|
2142
|
+
).orderBy((0, import_drizzle_orm.asc)(messages.timestamp)).limit(100);
|
|
2143
|
+
for (const row of msgs) {
|
|
2144
|
+
lastTimestamp = Math.max(lastTimestamp, Number(row.timestamp));
|
|
2145
|
+
const msg = {
|
|
2146
|
+
id: row.id,
|
|
2147
|
+
stream: row.stream,
|
|
2148
|
+
data: row.data,
|
|
2149
|
+
headers: row.headers,
|
|
2150
|
+
timestamp: Number(row.timestamp),
|
|
2151
|
+
ack: async () => {
|
|
2152
|
+
},
|
|
2153
|
+
nack: async () => {
|
|
2154
|
+
}
|
|
2155
|
+
};
|
|
2156
|
+
handler(msg).catch(console.error);
|
|
2157
|
+
}
|
|
2158
|
+
} catch (e) {
|
|
2159
|
+
console.error("Postgres stream poll error", e);
|
|
2160
|
+
}
|
|
2161
|
+
if (this.activeSubscriptions.get(subId)) {
|
|
2162
|
+
setTimeout(poll, this.pollInterval);
|
|
2163
|
+
}
|
|
2164
|
+
};
|
|
2165
|
+
setTimeout(poll, 0);
|
|
2166
|
+
return {
|
|
2167
|
+
unsubscribe: async () => {
|
|
2168
|
+
this.activeSubscriptions.set(subId, false);
|
|
2169
|
+
}
|
|
2170
|
+
};
|
|
2171
|
+
}
|
|
2172
|
+
async createConsumerGroup(stream, group) {
|
|
2173
|
+
}
|
|
2174
|
+
async saveWorkflowState(workflowId, state) {
|
|
2175
|
+
await this.db.insert(workflowExecutions).values({
|
|
2176
|
+
id: state.id,
|
|
2177
|
+
workflowId: state.workflowId,
|
|
2178
|
+
status: state.status,
|
|
2179
|
+
input: state.input,
|
|
2180
|
+
output: state.output,
|
|
2181
|
+
error: state.error instanceof Error ? state.error.message : String(state.error),
|
|
2182
|
+
startedAt: state.startedAt,
|
|
2183
|
+
completedAt: state.completedAt
|
|
2184
|
+
}).onConflictDoUpdate({
|
|
2185
|
+
target: workflowExecutions.id,
|
|
2186
|
+
set: {
|
|
2187
|
+
status: state.status,
|
|
2188
|
+
output: state.output,
|
|
2189
|
+
error: state.error instanceof Error ? state.error.message : String(state.error),
|
|
2190
|
+
updatedAt: Date.now(),
|
|
2191
|
+
completedAt: state.completedAt
|
|
2192
|
+
}
|
|
2193
|
+
});
|
|
2194
|
+
}
|
|
2195
|
+
async loadWorkflowState(executionId) {
|
|
2196
|
+
const rows = await this.db.select().from(workflowExecutions).where((0, import_drizzle_orm.eq)(workflowExecutions.id, executionId));
|
|
2197
|
+
if (rows.length === 0) return null;
|
|
2198
|
+
const row = rows[0];
|
|
2199
|
+
return {
|
|
2200
|
+
id: row.id,
|
|
2201
|
+
workflowId: row.workflowId,
|
|
2202
|
+
status: row.status,
|
|
2203
|
+
// Cast from DB string to union type
|
|
2204
|
+
input: row.input,
|
|
2205
|
+
output: row.output,
|
|
2206
|
+
error: row.error,
|
|
2207
|
+
startedAt: Number(row.startedAt),
|
|
2208
|
+
completedAt: row.completedAt ? Number(row.completedAt) : void 0
|
|
2209
|
+
};
|
|
2210
|
+
}
|
|
2211
|
+
async getJob(queue, jobId) {
|
|
2212
|
+
const rows = await this.db.select().from(jobs).where((0, import_drizzle_orm.and)((0, import_drizzle_orm.eq)(jobs.queue, queue), (0, import_drizzle_orm.eq)(jobs.id, jobId)));
|
|
2213
|
+
if (rows.length === 0) return null;
|
|
2214
|
+
const row = rows[0];
|
|
2215
|
+
return {
|
|
2216
|
+
id: row.id,
|
|
2217
|
+
name: row.name,
|
|
2218
|
+
data: row.data,
|
|
2219
|
+
opts: row.opts,
|
|
2220
|
+
state: row.state,
|
|
2221
|
+
timestamp: Number(row.timestamp),
|
|
2222
|
+
attemptsMade: row.attempts_made
|
|
2223
|
+
};
|
|
2224
|
+
}
|
|
2225
|
+
async getQueueStats(queueName) {
|
|
2226
|
+
const counts = await this.db.select({
|
|
2227
|
+
state: jobs.state,
|
|
2228
|
+
count: import_drizzle_orm.sql`count(*)`
|
|
2229
|
+
}).from(jobs).where((0, import_drizzle_orm.eq)(jobs.queue, queueName)).groupBy(jobs.state);
|
|
2230
|
+
const stats = {
|
|
2231
|
+
waiting: 0,
|
|
2232
|
+
active: 0,
|
|
2233
|
+
completed: 0,
|
|
2234
|
+
failed: 0,
|
|
2235
|
+
delayed: 0,
|
|
2236
|
+
paused: 0
|
|
2237
|
+
};
|
|
2238
|
+
for (const c of counts) {
|
|
2239
|
+
if (c.state in stats) {
|
|
2240
|
+
stats[c.state] = Number(c.count);
|
|
2241
|
+
}
|
|
2242
|
+
}
|
|
2243
|
+
return stats;
|
|
2244
|
+
}
|
|
2245
|
+
async getStreamInfo(streamName) {
|
|
2246
|
+
return { name: streamName, length: 0, groups: 0 };
|
|
2247
|
+
}
|
|
2248
|
+
async consume(stream, group, consumer, handler) {
|
|
2249
|
+
return this.subscribe(stream, handler);
|
|
2250
|
+
}
|
|
2251
|
+
async getJobs(queue, status) {
|
|
2252
|
+
const rows = await this.db.select().from(jobs).where((0, import_drizzle_orm.and)((0, import_drizzle_orm.eq)(jobs.queue, queue), (0, import_drizzle_orm.eq)(jobs.state, status)));
|
|
2253
|
+
return rows.map(
|
|
2254
|
+
(row) => ({
|
|
2255
|
+
id: row.id,
|
|
2256
|
+
name: row.name,
|
|
2257
|
+
data: row.data,
|
|
2258
|
+
opts: row.opts,
|
|
2259
|
+
state: row.state,
|
|
2260
|
+
timestamp: Number(row.timestamp),
|
|
2261
|
+
attemptsMade: row.attempts_made
|
|
2262
|
+
})
|
|
2263
|
+
);
|
|
2264
|
+
}
|
|
2265
|
+
async getAllJobs(queue) {
|
|
2266
|
+
const rows = await this.db.select().from(jobs).where((0, import_drizzle_orm.eq)(jobs.queue, queue));
|
|
2267
|
+
return rows.map(
|
|
2268
|
+
(row) => ({
|
|
2269
|
+
id: row.id,
|
|
2270
|
+
name: row.name,
|
|
2271
|
+
data: row.data,
|
|
2272
|
+
opts: row.opts,
|
|
2273
|
+
state: row.state,
|
|
2274
|
+
timestamp: Number(row.timestamp),
|
|
2275
|
+
attemptsMade: row.attempts_made
|
|
2276
|
+
})
|
|
2277
|
+
);
|
|
2278
|
+
}
|
|
2279
|
+
async cleanJobs(queue, grace, status) {
|
|
2280
|
+
const now2 = Date.now();
|
|
2281
|
+
const result = await this.db.delete(jobs).where(
|
|
2282
|
+
(0, import_drizzle_orm.and)(
|
|
2283
|
+
(0, import_drizzle_orm.eq)(jobs.queue, queue),
|
|
2284
|
+
(0, import_drizzle_orm.eq)(jobs.state, status),
|
|
2285
|
+
import_drizzle_orm.sql`${jobs.finishedOn} < ${now2 - grace}`
|
|
2286
|
+
)
|
|
2287
|
+
);
|
|
2288
|
+
return result.rowCount || 0;
|
|
2289
|
+
}
|
|
2290
|
+
async cleanup() {
|
|
2291
|
+
this.activeSubscriptions.clear();
|
|
2292
|
+
}
|
|
2293
|
+
};
|
|
2294
|
+
|
|
2295
|
+
// src/patterns/circuit-breaker.ts
|
|
2296
|
+
function circuitBreaker(options, handler) {
|
|
2297
|
+
let state = "CLOSED";
|
|
2298
|
+
let failures = 0;
|
|
2299
|
+
let lastFailureTime = 0;
|
|
2300
|
+
let lastSuccessTime = 0;
|
|
2301
|
+
return async (job) => {
|
|
2302
|
+
const now2 = Date.now();
|
|
2303
|
+
if (state === "OPEN") {
|
|
2304
|
+
if (now2 - lastFailureTime > options.timeout) {
|
|
2305
|
+
state = "HALF_OPEN";
|
|
2306
|
+
} else {
|
|
2307
|
+
throw new Error("Circuit Breaker is OPEN");
|
|
2308
|
+
}
|
|
2309
|
+
}
|
|
2310
|
+
try {
|
|
2311
|
+
const result = await handler(job);
|
|
2312
|
+
if (state === "HALF_OPEN") {
|
|
2313
|
+
state = "CLOSED";
|
|
2314
|
+
failures = 0;
|
|
2315
|
+
}
|
|
2316
|
+
lastSuccessTime = now2;
|
|
2317
|
+
return result;
|
|
2318
|
+
} catch (err) {
|
|
2319
|
+
failures++;
|
|
2320
|
+
lastFailureTime = now2;
|
|
2321
|
+
if (state === "HALF_OPEN" || failures >= options.threshold) {
|
|
2322
|
+
state = "OPEN";
|
|
2323
|
+
}
|
|
2324
|
+
throw err;
|
|
2325
|
+
}
|
|
2326
|
+
};
|
|
2327
|
+
}
|
|
2328
|
+
|
|
2329
|
+
// src/patterns/rate-limit.ts
|
|
2330
|
+
var RateLimiter = class {
|
|
2331
|
+
constructor(options) {
|
|
2332
|
+
this.counters = /* @__PURE__ */ new Map();
|
|
2333
|
+
this.options = {
|
|
2334
|
+
strategy: "throw",
|
|
2335
|
+
...options
|
|
2336
|
+
};
|
|
2337
|
+
}
|
|
2338
|
+
/**
|
|
2339
|
+
* Check if request is allowed
|
|
2340
|
+
*/
|
|
2341
|
+
async check(key = "default") {
|
|
2342
|
+
const now2 = Date.now();
|
|
2343
|
+
let counter = this.counters.get(key);
|
|
2344
|
+
if (!counter || now2 >= counter.resetAt) {
|
|
2345
|
+
counter = {
|
|
2346
|
+
count: 0,
|
|
2347
|
+
resetAt: now2 + this.options.window
|
|
2348
|
+
};
|
|
2349
|
+
this.counters.set(key, counter);
|
|
2350
|
+
}
|
|
2351
|
+
const allowed = counter.count < this.options.limit;
|
|
2352
|
+
const remaining = Math.max(0, this.options.limit - counter.count);
|
|
2353
|
+
const retryAfter = allowed ? void 0 : counter.resetAt - now2;
|
|
2354
|
+
if (allowed) {
|
|
2355
|
+
counter.count++;
|
|
2356
|
+
}
|
|
2357
|
+
return {
|
|
2358
|
+
allowed,
|
|
2359
|
+
remaining,
|
|
2360
|
+
resetAt: counter.resetAt,
|
|
2361
|
+
retryAfter
|
|
2362
|
+
};
|
|
2363
|
+
}
|
|
2364
|
+
/**
|
|
2365
|
+
* Execute a function with rate limiting
|
|
2366
|
+
*/
|
|
2367
|
+
async execute(fn, key = "default") {
|
|
2368
|
+
const result = await this.check(key);
|
|
2369
|
+
if (!result.allowed) {
|
|
2370
|
+
switch (this.options.strategy) {
|
|
2371
|
+
case "throw":
|
|
2372
|
+
throw new Error(
|
|
2373
|
+
`Rate limit exceeded. Retry after ${result.retryAfter}ms`
|
|
2374
|
+
);
|
|
2375
|
+
case "delay":
|
|
2376
|
+
await new Promise(
|
|
2377
|
+
(resolve) => setTimeout(resolve, result.retryAfter)
|
|
2378
|
+
);
|
|
2379
|
+
return this.execute(fn, key);
|
|
2380
|
+
case "drop":
|
|
2381
|
+
throw new Error("Request dropped due to rate limit");
|
|
2382
|
+
default:
|
|
2383
|
+
throw new Error("Unknown rate limit strategy");
|
|
2384
|
+
}
|
|
2385
|
+
}
|
|
2386
|
+
return fn();
|
|
2387
|
+
}
|
|
2388
|
+
/**
|
|
2389
|
+
* Clear all rate limit counters
|
|
2390
|
+
*/
|
|
2391
|
+
reset() {
|
|
2392
|
+
this.counters.clear();
|
|
2393
|
+
}
|
|
2394
|
+
/**
|
|
2395
|
+
* Clear rate limit counter for specific key
|
|
2396
|
+
*/
|
|
2397
|
+
resetKey(key) {
|
|
2398
|
+
this.counters.delete(key);
|
|
2399
|
+
}
|
|
2400
|
+
/**
|
|
2401
|
+
* Get current limit info for a key
|
|
2402
|
+
*/
|
|
2403
|
+
getInfo(key = "default") {
|
|
2404
|
+
const now2 = Date.now();
|
|
2405
|
+
const counter = this.counters.get(key);
|
|
2406
|
+
if (!counter || now2 >= counter.resetAt) {
|
|
2407
|
+
return {
|
|
2408
|
+
allowed: true,
|
|
2409
|
+
remaining: this.options.limit,
|
|
2410
|
+
resetAt: now2 + this.options.window
|
|
2411
|
+
};
|
|
2412
|
+
}
|
|
2413
|
+
const allowed = counter.count < this.options.limit;
|
|
2414
|
+
const remaining = Math.max(0, this.options.limit - counter.count);
|
|
2415
|
+
return {
|
|
2416
|
+
allowed,
|
|
2417
|
+
remaining,
|
|
2418
|
+
resetAt: counter.resetAt,
|
|
2419
|
+
retryAfter: allowed ? void 0 : counter.resetAt - now2
|
|
2420
|
+
};
|
|
2421
|
+
}
|
|
2422
|
+
};
|
|
2423
|
+
function createRateLimiter(options) {
|
|
2424
|
+
return new RateLimiter(options);
|
|
2425
|
+
}
|
|
2426
|
+
var SlidingWindowRateLimiter = class {
|
|
2427
|
+
constructor(options) {
|
|
2428
|
+
this.requests = /* @__PURE__ */ new Map();
|
|
2429
|
+
this.options = options;
|
|
2430
|
+
}
|
|
2431
|
+
async check(key = "default") {
|
|
2432
|
+
const now2 = Date.now();
|
|
2433
|
+
const windowStart = now2 - this.options.window;
|
|
2434
|
+
let requests = this.requests.get(key) || [];
|
|
2435
|
+
requests = requests.filter((timestamp2) => timestamp2 > windowStart);
|
|
2436
|
+
this.requests.set(key, requests);
|
|
2437
|
+
const allowed = requests.length < this.options.limit;
|
|
2438
|
+
const remaining = Math.max(0, this.options.limit - requests.length);
|
|
2439
|
+
if (allowed) {
|
|
2440
|
+
requests.push(now2);
|
|
2441
|
+
}
|
|
2442
|
+
const oldestRequest = requests[0];
|
|
2443
|
+
const resetAt = oldestRequest ? oldestRequest + this.options.window : now2 + this.options.window;
|
|
2444
|
+
const retryAfter = allowed ? void 0 : resetAt - now2;
|
|
2445
|
+
return {
|
|
2446
|
+
allowed,
|
|
2447
|
+
remaining,
|
|
2448
|
+
resetAt,
|
|
2449
|
+
retryAfter
|
|
2450
|
+
};
|
|
2451
|
+
}
|
|
2452
|
+
async execute(fn, key = "default") {
|
|
2453
|
+
const result = await this.check(key);
|
|
2454
|
+
if (!result.allowed) {
|
|
2455
|
+
switch (this.options.strategy || "throw") {
|
|
2456
|
+
case "throw":
|
|
2457
|
+
throw new Error(
|
|
2458
|
+
`Rate limit exceeded. Retry after ${result.retryAfter}ms`
|
|
2459
|
+
);
|
|
2460
|
+
case "delay":
|
|
2461
|
+
await new Promise(
|
|
2462
|
+
(resolve) => setTimeout(resolve, result.retryAfter)
|
|
2463
|
+
);
|
|
2464
|
+
return this.execute(fn, key);
|
|
2465
|
+
case "drop":
|
|
2466
|
+
throw new Error("Request dropped due to rate limit");
|
|
2467
|
+
}
|
|
2468
|
+
}
|
|
2469
|
+
return fn();
|
|
2470
|
+
}
|
|
2471
|
+
reset() {
|
|
2472
|
+
this.requests.clear();
|
|
2473
|
+
}
|
|
2474
|
+
};
|
|
2475
|
+
var TokenBucketRateLimiter = class {
|
|
2476
|
+
constructor(options) {
|
|
2477
|
+
this.buckets = /* @__PURE__ */ new Map();
|
|
2478
|
+
this.capacity = options.capacity;
|
|
2479
|
+
this.refillRate = options.refillRate;
|
|
2480
|
+
this.refillInterval = options.refillInterval || 1e3;
|
|
2481
|
+
}
|
|
2482
|
+
refill(key) {
|
|
2483
|
+
const now2 = Date.now();
|
|
2484
|
+
let bucket = this.buckets.get(key);
|
|
2485
|
+
if (!bucket) {
|
|
2486
|
+
bucket = { tokens: this.capacity, lastRefill: now2 };
|
|
2487
|
+
this.buckets.set(key, bucket);
|
|
2488
|
+
return;
|
|
2489
|
+
}
|
|
2490
|
+
const timePassed = now2 - bucket.lastRefill;
|
|
2491
|
+
const intervals = timePassed / this.refillInterval;
|
|
2492
|
+
const tokensToAdd = intervals * this.refillRate;
|
|
2493
|
+
bucket.tokens = Math.min(this.capacity, bucket.tokens + tokensToAdd);
|
|
2494
|
+
bucket.lastRefill = now2;
|
|
2495
|
+
}
|
|
2496
|
+
async check(key = "default", cost = 1) {
|
|
2497
|
+
this.refill(key);
|
|
2498
|
+
const bucket = this.buckets.get(key);
|
|
2499
|
+
const allowed = bucket.tokens >= cost;
|
|
2500
|
+
if (allowed) {
|
|
2501
|
+
bucket.tokens -= cost;
|
|
2502
|
+
}
|
|
2503
|
+
return {
|
|
2504
|
+
allowed,
|
|
2505
|
+
remaining: Math.floor(bucket.tokens),
|
|
2506
|
+
resetAt: bucket.lastRefill + this.refillInterval,
|
|
2507
|
+
retryAfter: allowed ? void 0 : Math.ceil(
|
|
2508
|
+
(cost - bucket.tokens) / this.refillRate * this.refillInterval
|
|
2509
|
+
)
|
|
2510
|
+
};
|
|
2511
|
+
}
|
|
2512
|
+
reset() {
|
|
2513
|
+
this.buckets.clear();
|
|
2514
|
+
}
|
|
2515
|
+
};
|
|
2516
|
+
|
|
2517
|
+
// src/patterns/batching.ts
|
|
2518
|
+
var BatchAccumulator = class {
|
|
2519
|
+
constructor(processor, options) {
|
|
2520
|
+
this.batch = [];
|
|
2521
|
+
this.timer = null;
|
|
2522
|
+
this.pending = [];
|
|
2523
|
+
this.processor = processor;
|
|
2524
|
+
this.options = {
|
|
2525
|
+
minSize: 1,
|
|
2526
|
+
maxWait: 1e3,
|
|
2527
|
+
...options
|
|
2528
|
+
};
|
|
2529
|
+
}
|
|
2530
|
+
/**
|
|
2531
|
+
* Add an item to the batch
|
|
2532
|
+
*/
|
|
2533
|
+
async add(item) {
|
|
2534
|
+
return new Promise((resolve, reject) => {
|
|
2535
|
+
this.batch.push(item);
|
|
2536
|
+
this.pending.push({ resolve, reject });
|
|
2537
|
+
if (this.batch.length >= this.options.maxSize) {
|
|
2538
|
+
this.flush();
|
|
2539
|
+
} else if (!this.timer) {
|
|
2540
|
+
this.timer = setTimeout(() => {
|
|
2541
|
+
this.flush();
|
|
2542
|
+
}, this.options.maxWait);
|
|
2543
|
+
}
|
|
2544
|
+
});
|
|
2545
|
+
}
|
|
2546
|
+
/**
|
|
2547
|
+
* Manually flush the current batch
|
|
2548
|
+
*/
|
|
2549
|
+
async flush() {
|
|
2550
|
+
if (this.timer) {
|
|
2551
|
+
clearTimeout(this.timer);
|
|
2552
|
+
this.timer = null;
|
|
2553
|
+
}
|
|
2554
|
+
if (this.batch.length < this.options.minSize) {
|
|
2555
|
+
return;
|
|
2556
|
+
}
|
|
2557
|
+
const currentBatch = this.batch;
|
|
2558
|
+
const currentPending = this.pending;
|
|
2559
|
+
this.batch = [];
|
|
2560
|
+
this.pending = [];
|
|
2561
|
+
try {
|
|
2562
|
+
const results = await this.processor(currentBatch);
|
|
2563
|
+
for (let i = 0; i < currentPending.length; i++) {
|
|
2564
|
+
currentPending[i].resolve(results[i]);
|
|
2565
|
+
}
|
|
2566
|
+
} catch (error) {
|
|
2567
|
+
for (const pending of currentPending) {
|
|
2568
|
+
pending.reject(error);
|
|
2569
|
+
}
|
|
2570
|
+
}
|
|
2571
|
+
}
|
|
2572
|
+
/**
|
|
2573
|
+
* Get current batch size
|
|
2574
|
+
*/
|
|
2575
|
+
size() {
|
|
2576
|
+
return this.batch.length;
|
|
2577
|
+
}
|
|
2578
|
+
/**
|
|
2579
|
+
* Clear the batch without processing
|
|
2580
|
+
*/
|
|
2581
|
+
clear() {
|
|
2582
|
+
if (this.timer) {
|
|
2583
|
+
clearTimeout(this.timer);
|
|
2584
|
+
this.timer = null;
|
|
2585
|
+
}
|
|
2586
|
+
this.batch = [];
|
|
2587
|
+
for (const pending of this.pending) {
|
|
2588
|
+
pending.reject(new Error("Batch cleared"));
|
|
2589
|
+
}
|
|
2590
|
+
this.pending = [];
|
|
2591
|
+
}
|
|
2592
|
+
};
|
|
2593
|
+
function batch(fn, options) {
|
|
2594
|
+
const accumulator = new BatchAccumulator(fn, options);
|
|
2595
|
+
return (item) => accumulator.add(item);
|
|
2596
|
+
}
|
|
2597
|
+
function chunk(array, size) {
|
|
2598
|
+
const chunks = [];
|
|
2599
|
+
for (let i = 0; i < array.length; i += size) {
|
|
2600
|
+
chunks.push(array.slice(i, i + size));
|
|
2601
|
+
}
|
|
2602
|
+
return chunks;
|
|
2603
|
+
}
|
|
2604
|
+
async function processBatches(items, processor, options) {
|
|
2605
|
+
const batches = chunk(items, options.batchSize);
|
|
2606
|
+
const results = [];
|
|
2607
|
+
for (let i = 0; i < batches.length; i++) {
|
|
2608
|
+
const batchResults = await processor(batches[i]);
|
|
2609
|
+
results.push(...batchResults);
|
|
2610
|
+
if (i < batches.length - 1 && options.delayMs) {
|
|
2611
|
+
await new Promise((resolve) => setTimeout(resolve, options.delayMs));
|
|
2612
|
+
}
|
|
2613
|
+
}
|
|
2614
|
+
return results;
|
|
2615
|
+
}
|
|
2616
|
+
var BatchWriter = class {
|
|
2617
|
+
constructor(writer, options) {
|
|
2618
|
+
this.accumulator = new BatchAccumulator(async (items) => {
|
|
2619
|
+
await writer(items);
|
|
2620
|
+
return new Array(items.length).fill(void 0);
|
|
2621
|
+
}, options);
|
|
2622
|
+
}
|
|
2623
|
+
async write(item) {
|
|
2624
|
+
return this.accumulator.add(item);
|
|
2625
|
+
}
|
|
2626
|
+
async flush() {
|
|
2627
|
+
return this.accumulator.flush();
|
|
2628
|
+
}
|
|
2629
|
+
size() {
|
|
2630
|
+
return this.accumulator.size();
|
|
2631
|
+
}
|
|
2632
|
+
clear() {
|
|
2633
|
+
this.accumulator.clear();
|
|
2634
|
+
}
|
|
2635
|
+
};
|
|
2636
|
+
async function batchByKey(items, keyFn, processor, options) {
|
|
2637
|
+
const grouped = /* @__PURE__ */ new Map();
|
|
2638
|
+
for (const item of items) {
|
|
2639
|
+
const key = keyFn(item);
|
|
2640
|
+
if (!grouped.has(key)) {
|
|
2641
|
+
grouped.set(key, []);
|
|
2642
|
+
}
|
|
2643
|
+
grouped.get(key).push(item);
|
|
2644
|
+
}
|
|
2645
|
+
const results = [];
|
|
2646
|
+
const entries = Array.from(grouped.entries());
|
|
2647
|
+
if (options?.concurrency) {
|
|
2648
|
+
for (let i = 0; i < entries.length; i += options.concurrency) {
|
|
2649
|
+
const batch2 = entries.slice(i, i + options.concurrency);
|
|
2650
|
+
const batchResults = await Promise.all(
|
|
2651
|
+
batch2.map(([key, items2]) => processor(key, items2))
|
|
2652
|
+
);
|
|
2653
|
+
results.push(...batchResults.flat());
|
|
2654
|
+
}
|
|
2655
|
+
} else {
|
|
2656
|
+
for (const [key, keyItems] of entries) {
|
|
2657
|
+
const batchResults = await processor(key, keyItems);
|
|
2658
|
+
results.push(...batchResults);
|
|
2659
|
+
}
|
|
2660
|
+
}
|
|
2661
|
+
return results;
|
|
2662
|
+
}
|
|
2663
|
+
|
|
2664
|
+
// src/patterns/priority.ts
|
|
2665
|
+
var PriorityQueue = class {
|
|
2666
|
+
constructor(mode = "max") {
|
|
2667
|
+
this.heap = [];
|
|
2668
|
+
this.compareFn = mode === "max" ? (a, b) => a - b : (a, b) => b - a;
|
|
2669
|
+
}
|
|
2670
|
+
/**
|
|
2671
|
+
* Add item with priority
|
|
2672
|
+
*/
|
|
2673
|
+
enqueue(value, priority) {
|
|
2674
|
+
this.heap.push({ value, priority });
|
|
2675
|
+
this.bubbleUp(this.heap.length - 1);
|
|
2676
|
+
}
|
|
2677
|
+
/**
|
|
2678
|
+
* Remove and return highest priority item
|
|
2679
|
+
*/
|
|
2680
|
+
dequeue() {
|
|
2681
|
+
if (this.heap.length === 0) return void 0;
|
|
2682
|
+
if (this.heap.length === 1) return this.heap.pop().value;
|
|
2683
|
+
const result = this.heap[0];
|
|
2684
|
+
this.heap[0] = this.heap.pop();
|
|
2685
|
+
this.bubbleDown(0);
|
|
2686
|
+
return result.value;
|
|
2687
|
+
}
|
|
2688
|
+
/**
|
|
2689
|
+
* Peek at highest priority item without removing
|
|
2690
|
+
*/
|
|
2691
|
+
peek() {
|
|
2692
|
+
return this.heap[0]?.value;
|
|
2693
|
+
}
|
|
2694
|
+
/**
|
|
2695
|
+
* Get queue size
|
|
2696
|
+
*/
|
|
2697
|
+
size() {
|
|
2698
|
+
return this.heap.length;
|
|
2699
|
+
}
|
|
2700
|
+
/**
|
|
2701
|
+
* Check if queue is empty
|
|
2702
|
+
*/
|
|
2703
|
+
isEmpty() {
|
|
2704
|
+
return this.heap.length === 0;
|
|
2705
|
+
}
|
|
2706
|
+
/**
|
|
2707
|
+
* Clear the queue
|
|
2708
|
+
*/
|
|
2709
|
+
clear() {
|
|
2710
|
+
this.heap = [];
|
|
2711
|
+
}
|
|
2712
|
+
/**
|
|
2713
|
+
* Convert to array (sorted by priority)
|
|
2714
|
+
*/
|
|
2715
|
+
toArray() {
|
|
2716
|
+
const copy = [...this.heap];
|
|
2717
|
+
const result = [];
|
|
2718
|
+
while (this.heap.length > 0) {
|
|
2719
|
+
result.push(this.dequeue());
|
|
2720
|
+
}
|
|
2721
|
+
this.heap = copy;
|
|
2722
|
+
return result;
|
|
2723
|
+
}
|
|
2724
|
+
bubbleUp(index2) {
|
|
2725
|
+
while (index2 > 0) {
|
|
2726
|
+
const parentIndex = Math.floor((index2 - 1) / 2);
|
|
2727
|
+
if (this.compareFn(
|
|
2728
|
+
this.heap[index2].priority,
|
|
2729
|
+
this.heap[parentIndex].priority
|
|
2730
|
+
) <= 0) {
|
|
2731
|
+
break;
|
|
2732
|
+
}
|
|
2733
|
+
[this.heap[index2], this.heap[parentIndex]] = [
|
|
2734
|
+
this.heap[parentIndex],
|
|
2735
|
+
this.heap[index2]
|
|
2736
|
+
];
|
|
2737
|
+
index2 = parentIndex;
|
|
2738
|
+
}
|
|
2739
|
+
}
|
|
2740
|
+
bubbleDown(index2) {
|
|
2741
|
+
while (true) {
|
|
2742
|
+
const leftChild = 2 * index2 + 1;
|
|
2743
|
+
const rightChild = 2 * index2 + 2;
|
|
2744
|
+
let largest = index2;
|
|
2745
|
+
if (leftChild < this.heap.length && this.compareFn(
|
|
2746
|
+
this.heap[leftChild].priority,
|
|
2747
|
+
this.heap[largest].priority
|
|
2748
|
+
) > 0) {
|
|
2749
|
+
largest = leftChild;
|
|
2750
|
+
}
|
|
2751
|
+
if (rightChild < this.heap.length && this.compareFn(
|
|
2752
|
+
this.heap[rightChild].priority,
|
|
2753
|
+
this.heap[largest].priority
|
|
2754
|
+
) > 0) {
|
|
2755
|
+
largest = rightChild;
|
|
2756
|
+
}
|
|
2757
|
+
if (largest === index2) break;
|
|
2758
|
+
[this.heap[index2], this.heap[largest]] = [
|
|
2759
|
+
this.heap[largest],
|
|
2760
|
+
this.heap[index2]
|
|
2761
|
+
];
|
|
2762
|
+
index2 = largest;
|
|
2763
|
+
}
|
|
2764
|
+
}
|
|
2765
|
+
};
|
|
2766
|
+
|
|
2767
|
+
// src/utils/hashing.ts
|
|
2768
|
+
var import_crypto = require("crypto");
|
|
2769
|
+
function hashJob(data, options) {
|
|
2770
|
+
const algorithm = options?.algorithm || "sha256";
|
|
2771
|
+
const hash = (0, import_crypto.createHash)(algorithm);
|
|
2772
|
+
const normalized = JSON.stringify(data, Object.keys(data).sort());
|
|
2773
|
+
hash.update(normalized);
|
|
2774
|
+
return hash.digest("hex");
|
|
2775
|
+
}
|
|
2776
|
+
function generateDeduplicationKey(name, data) {
|
|
2777
|
+
return `${name}:${hashJob(data)}`;
|
|
2778
|
+
}
|
|
2779
|
+
function areJobsEquivalent(job1, job2) {
|
|
2780
|
+
return hashJob(job1) === hashJob(job2);
|
|
2781
|
+
}
|
|
2782
|
+
|
|
2783
|
+
// src/utils/id-generator.ts
|
|
2784
|
+
var import_uuid5 = require("uuid");
|
|
2785
|
+
var import_crypto2 = require("crypto");
|
|
2786
|
+
function generateId(options = {}) {
|
|
2787
|
+
const {
|
|
2788
|
+
strategy = "uuid-v4",
|
|
2789
|
+
namespace,
|
|
2790
|
+
prefix = "",
|
|
2791
|
+
customGenerator
|
|
2792
|
+
} = options;
|
|
2793
|
+
let id;
|
|
2794
|
+
switch (strategy) {
|
|
2795
|
+
case "uuid-v4":
|
|
2796
|
+
id = (0, import_uuid5.v4)();
|
|
2797
|
+
break;
|
|
2798
|
+
case "uuid-v5":
|
|
2799
|
+
if (!namespace) {
|
|
2800
|
+
throw new Error("UUID v5 requires a namespace");
|
|
2801
|
+
}
|
|
2802
|
+
const name = `${Date.now()}-${(0, import_crypto2.randomBytes)(8).toString("hex")}`;
|
|
2803
|
+
id = (0, import_uuid5.v5)(name, namespace);
|
|
2804
|
+
break;
|
|
2805
|
+
case "uuid-v1":
|
|
2806
|
+
id = (0, import_uuid5.v1)();
|
|
2807
|
+
break;
|
|
2808
|
+
case "nanoid":
|
|
2809
|
+
id = (0, import_crypto2.randomBytes)(16).toString("base64url");
|
|
2810
|
+
break;
|
|
2811
|
+
case "incremental":
|
|
2812
|
+
id = `${Date.now()}-${(0, import_crypto2.randomBytes)(4).toString("hex")}`;
|
|
2813
|
+
break;
|
|
2814
|
+
case "custom":
|
|
2815
|
+
if (!customGenerator) {
|
|
2816
|
+
throw new Error("Custom strategy requires customGenerator function");
|
|
2817
|
+
}
|
|
2818
|
+
id = customGenerator();
|
|
2819
|
+
break;
|
|
2820
|
+
default:
|
|
2821
|
+
id = (0, import_uuid5.v4)();
|
|
2822
|
+
}
|
|
2823
|
+
return prefix ? `${prefix}${id}` : id;
|
|
2824
|
+
}
|
|
2825
|
+
function generateJobId(prefix) {
|
|
2826
|
+
return generateId({ prefix: prefix || "job_" });
|
|
2827
|
+
}
|
|
2828
|
+
function generateExecutionId(prefix) {
|
|
2829
|
+
return generateId({ prefix: prefix || "exec_" });
|
|
2830
|
+
}
|
|
2831
|
+
function generateMessageId(prefix) {
|
|
2832
|
+
return generateId({ prefix: prefix || "msg_" });
|
|
2833
|
+
}
|
|
2834
|
+
function generateDeterministicId(content, prefix) {
|
|
2835
|
+
const hash = (0, import_crypto2.createHash)("sha256");
|
|
2836
|
+
const data = typeof content === "string" ? content : JSON.stringify(content);
|
|
2837
|
+
hash.update(data);
|
|
2838
|
+
const id = hash.digest("hex").slice(0, 32);
|
|
2839
|
+
return prefix ? `${prefix}${id}` : id;
|
|
2840
|
+
}
|
|
2841
|
+
|
|
2842
|
+
// src/utils/time.ts
|
|
2843
|
+
function toMilliseconds(duration) {
|
|
2844
|
+
let ms = 0;
|
|
2845
|
+
if (duration.milliseconds) ms += duration.milliseconds;
|
|
2846
|
+
if (duration.seconds) ms += duration.seconds * 1e3;
|
|
2847
|
+
if (duration.minutes) ms += duration.minutes * 60 * 1e3;
|
|
2848
|
+
if (duration.hours) ms += duration.hours * 60 * 60 * 1e3;
|
|
2849
|
+
if (duration.days) ms += duration.days * 24 * 60 * 60 * 1e3;
|
|
2850
|
+
if (duration.weeks) ms += duration.weeks * 7 * 24 * 60 * 60 * 1e3;
|
|
2851
|
+
return ms;
|
|
2852
|
+
}
|
|
2853
|
+
function fromMilliseconds(ms) {
|
|
2854
|
+
const weeks = Math.floor(ms / (7 * 24 * 60 * 60 * 1e3));
|
|
2855
|
+
ms %= 7 * 24 * 60 * 60 * 1e3;
|
|
2856
|
+
const days = Math.floor(ms / (24 * 60 * 60 * 1e3));
|
|
2857
|
+
ms %= 24 * 60 * 60 * 1e3;
|
|
2858
|
+
const hours = Math.floor(ms / (60 * 60 * 1e3));
|
|
2859
|
+
ms %= 60 * 60 * 1e3;
|
|
2860
|
+
const minutes = Math.floor(ms / (60 * 1e3));
|
|
2861
|
+
ms %= 60 * 1e3;
|
|
2862
|
+
const seconds = Math.floor(ms / 1e3);
|
|
2863
|
+
const milliseconds = ms % 1e3;
|
|
2864
|
+
return { weeks, days, hours, minutes, seconds, milliseconds };
|
|
2865
|
+
}
|
|
2866
|
+
function formatDuration(duration) {
|
|
2867
|
+
const parts = [];
|
|
2868
|
+
if (duration.weeks) parts.push(`${duration.weeks}w`);
|
|
2869
|
+
if (duration.days) parts.push(`${duration.days}d`);
|
|
2870
|
+
if (duration.hours) parts.push(`${duration.hours}h`);
|
|
2871
|
+
if (duration.minutes) parts.push(`${duration.minutes}m`);
|
|
2872
|
+
if (duration.seconds) parts.push(`${duration.seconds}s`);
|
|
2873
|
+
if (duration.milliseconds) parts.push(`${duration.milliseconds}ms`);
|
|
2874
|
+
return parts.join(" ") || "0ms";
|
|
2875
|
+
}
|
|
2876
|
+
function sleep(ms) {
|
|
2877
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
2878
|
+
}
|
|
2879
|
+
async function sleepDuration(duration) {
|
|
2880
|
+
return sleep(toMilliseconds(duration));
|
|
2881
|
+
}
|
|
2882
|
+
function timeout(promise, ms, message) {
|
|
2883
|
+
return Promise.race([
|
|
2884
|
+
promise,
|
|
2885
|
+
new Promise(
|
|
2886
|
+
(_, reject) => setTimeout(
|
|
2887
|
+
() => reject(new Error(message || `Timeout after ${ms}ms`)),
|
|
2888
|
+
ms
|
|
2889
|
+
)
|
|
2890
|
+
)
|
|
2891
|
+
]);
|
|
2892
|
+
}
|
|
2893
|
+
function now() {
|
|
2894
|
+
return Date.now();
|
|
2895
|
+
}
|
|
2896
|
+
function delayUntil(timestamp2) {
|
|
2897
|
+
return Math.max(0, timestamp2 - Date.now());
|
|
2898
|
+
}
|
|
2899
|
+
function isPast(timestamp2) {
|
|
2900
|
+
return timestamp2 < Date.now();
|
|
2901
|
+
}
|
|
2902
|
+
function isFuture(timestamp2) {
|
|
2903
|
+
return timestamp2 > Date.now();
|
|
2904
|
+
}
|
|
2905
|
+
function addDuration(timestamp2, duration) {
|
|
2906
|
+
return timestamp2 + toMilliseconds(duration);
|
|
2907
|
+
}
|
|
2908
|
+
function parseDuration(str) {
|
|
2909
|
+
const match = str.match(/^(\d+)(ms|s|m|h|d|w)$/);
|
|
2910
|
+
if (!match) {
|
|
2911
|
+
throw new Error(`Invalid duration string: ${str}`);
|
|
2912
|
+
}
|
|
2913
|
+
const value = parseInt(match[1], 10);
|
|
2914
|
+
const unit = match[2];
|
|
2915
|
+
switch (unit) {
|
|
2916
|
+
case "ms":
|
|
2917
|
+
return value;
|
|
2918
|
+
case "s":
|
|
2919
|
+
return value * 1e3;
|
|
2920
|
+
case "m":
|
|
2921
|
+
return value * 60 * 1e3;
|
|
2922
|
+
case "h":
|
|
2923
|
+
return value * 60 * 60 * 1e3;
|
|
2924
|
+
case "d":
|
|
2925
|
+
return value * 24 * 60 * 60 * 1e3;
|
|
2926
|
+
case "w":
|
|
2927
|
+
return value * 7 * 24 * 60 * 60 * 1e3;
|
|
2928
|
+
default:
|
|
2929
|
+
throw new Error(`Unknown unit: ${unit}`);
|
|
2930
|
+
}
|
|
2931
|
+
}
|
|
2932
|
+
|
|
2933
|
+
// src/utils/serialization.ts
|
|
2934
|
+
function serialize(data, options = {}) {
|
|
2935
|
+
const { pretty = false, dateFormat = "iso" } = options;
|
|
2936
|
+
const replacer = (key, value) => {
|
|
2937
|
+
if (value instanceof Date) {
|
|
2938
|
+
return dateFormat === "timestamp" ? value.getTime() : value.toISOString();
|
|
2939
|
+
}
|
|
2940
|
+
if (value === void 0 && !options.includeUndefined) {
|
|
2941
|
+
return null;
|
|
2942
|
+
}
|
|
2943
|
+
if (typeof value === "function") {
|
|
2944
|
+
return void 0;
|
|
2945
|
+
}
|
|
2946
|
+
if (typeof value === "bigint") {
|
|
2947
|
+
return value.toString();
|
|
2948
|
+
}
|
|
2949
|
+
return value;
|
|
2950
|
+
};
|
|
2951
|
+
return JSON.stringify(data, replacer, pretty ? 2 : void 0);
|
|
2952
|
+
}
|
|
2953
|
+
function deserialize(json) {
|
|
2954
|
+
return JSON.parse(json, (key, value) => {
|
|
2955
|
+
if (typeof value === "string") {
|
|
2956
|
+
const isoDateRegex = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d{3})?Z?$/;
|
|
2957
|
+
if (isoDateRegex.test(value)) {
|
|
2958
|
+
return new Date(value);
|
|
2959
|
+
}
|
|
2960
|
+
}
|
|
2961
|
+
return value;
|
|
2962
|
+
});
|
|
2963
|
+
}
|
|
2964
|
+
function serializeSafe(data, options = {}) {
|
|
2965
|
+
const seen = /* @__PURE__ */ new WeakSet();
|
|
2966
|
+
const { pretty = false } = options;
|
|
2967
|
+
const replacer = (key, value) => {
|
|
2968
|
+
if (typeof value === "object" && value !== null) {
|
|
2969
|
+
if (seen.has(value)) {
|
|
2970
|
+
return "[Circular]";
|
|
2971
|
+
}
|
|
2972
|
+
seen.add(value);
|
|
2973
|
+
}
|
|
2974
|
+
if (value instanceof Date) {
|
|
2975
|
+
return value.toISOString();
|
|
2976
|
+
}
|
|
2977
|
+
if (typeof value === "function") {
|
|
2978
|
+
return "[Function]";
|
|
2979
|
+
}
|
|
2980
|
+
if (typeof value === "bigint") {
|
|
2981
|
+
return value.toString();
|
|
2982
|
+
}
|
|
2983
|
+
return value;
|
|
2984
|
+
};
|
|
2985
|
+
return JSON.stringify(data, replacer, pretty ? 2 : void 0);
|
|
2986
|
+
}
|
|
2987
|
+
function cloneViaSerialization(obj) {
|
|
2988
|
+
return deserialize(serialize(obj));
|
|
2989
|
+
}
|
|
2990
|
+
function serializeCompressed(data) {
|
|
2991
|
+
const json = serialize(data);
|
|
2992
|
+
return Buffer.from(json).toString("base64");
|
|
2993
|
+
}
|
|
2994
|
+
function deserializeCompressed(compressed) {
|
|
2995
|
+
const json = Buffer.from(compressed, "base64").toString("utf-8");
|
|
2996
|
+
return deserialize(json);
|
|
2997
|
+
}
|
|
2998
|
+
function isSerializable(value) {
|
|
2999
|
+
try {
|
|
3000
|
+
serialize(value);
|
|
3001
|
+
return true;
|
|
3002
|
+
} catch {
|
|
3003
|
+
return false;
|
|
3004
|
+
}
|
|
3005
|
+
}
|
|
3006
|
+
function getSerializedSize(data) {
|
|
3007
|
+
return Buffer.byteLength(serialize(data), "utf-8");
|
|
3008
|
+
}
|
|
3009
|
+
|
|
3010
|
+
// src/storage/job-storage.ts
|
|
3011
|
+
var MemoryJobStorage = class {
|
|
3012
|
+
constructor() {
|
|
3013
|
+
this.jobs = /* @__PURE__ */ new Map();
|
|
3014
|
+
this.deduplicationKeys = /* @__PURE__ */ new Map();
|
|
3015
|
+
}
|
|
3016
|
+
// key -> jobId
|
|
3017
|
+
async save(queue, job) {
|
|
3018
|
+
const key = `${queue}:${job.id}`;
|
|
3019
|
+
this.jobs.set(key, { ...job });
|
|
3020
|
+
if (job.opts.deduplicationKey) {
|
|
3021
|
+
const dedupKey = `${queue}:${job.opts.deduplicationKey}`;
|
|
3022
|
+
this.deduplicationKeys.set(dedupKey, job.id);
|
|
3023
|
+
}
|
|
3024
|
+
}
|
|
3025
|
+
async get(queue, jobId) {
|
|
3026
|
+
const key = `${queue}:${jobId}`;
|
|
3027
|
+
return this.jobs.get(key) || null;
|
|
3028
|
+
}
|
|
3029
|
+
async getByStatus(queue, status) {
|
|
3030
|
+
const result = [];
|
|
3031
|
+
const prefix = `${queue}:`;
|
|
3032
|
+
for (const [key, job] of this.jobs.entries()) {
|
|
3033
|
+
if (key.startsWith(prefix) && job.state === status) {
|
|
3034
|
+
result.push(job);
|
|
3035
|
+
}
|
|
3036
|
+
}
|
|
3037
|
+
return result;
|
|
3038
|
+
}
|
|
3039
|
+
async getAll(queue) {
|
|
3040
|
+
const result = [];
|
|
3041
|
+
const prefix = `${queue}:`;
|
|
3042
|
+
for (const [key, job] of this.jobs.entries()) {
|
|
3043
|
+
if (key.startsWith(prefix)) {
|
|
3044
|
+
result.push(job);
|
|
3045
|
+
}
|
|
3046
|
+
}
|
|
3047
|
+
return result;
|
|
3048
|
+
}
|
|
3049
|
+
async updateStatus(queue, jobId, status) {
|
|
3050
|
+
const key = `${queue}:${jobId}`;
|
|
3051
|
+
const job = this.jobs.get(key);
|
|
3052
|
+
if (job) {
|
|
3053
|
+
job.state = status;
|
|
3054
|
+
if (status === "completed" || status === "failed") {
|
|
3055
|
+
job.finishedOn = Date.now();
|
|
3056
|
+
}
|
|
3057
|
+
}
|
|
3058
|
+
}
|
|
3059
|
+
async update(queue, jobId, updates) {
|
|
3060
|
+
const key = `${queue}:${jobId}`;
|
|
3061
|
+
const job = this.jobs.get(key);
|
|
3062
|
+
if (job) {
|
|
3063
|
+
Object.assign(job, updates);
|
|
3064
|
+
}
|
|
3065
|
+
}
|
|
3066
|
+
async delete(queue, jobId) {
|
|
3067
|
+
const key = `${queue}:${jobId}`;
|
|
3068
|
+
const job = this.jobs.get(key);
|
|
3069
|
+
if (job?.opts.deduplicationKey) {
|
|
3070
|
+
const dedupKey = `${queue}:${job.opts.deduplicationKey}`;
|
|
3071
|
+
this.deduplicationKeys.delete(dedupKey);
|
|
3072
|
+
}
|
|
3073
|
+
this.jobs.delete(key);
|
|
3074
|
+
}
|
|
3075
|
+
async clean(queue, grace, status) {
|
|
3076
|
+
const now2 = Date.now();
|
|
3077
|
+
const prefix = `${queue}:`;
|
|
3078
|
+
const toDelete = [];
|
|
3079
|
+
for (const [key, job] of this.jobs.entries()) {
|
|
3080
|
+
if (key.startsWith(prefix) && job.state === status) {
|
|
3081
|
+
const timestamp2 = job.finishedOn || job.timestamp;
|
|
3082
|
+
if (now2 - timestamp2 > grace) {
|
|
3083
|
+
toDelete.push(key);
|
|
3084
|
+
}
|
|
3085
|
+
}
|
|
3086
|
+
}
|
|
3087
|
+
for (const key of toDelete) {
|
|
3088
|
+
const job = this.jobs.get(key);
|
|
3089
|
+
if (job?.opts.deduplicationKey) {
|
|
3090
|
+
const dedupKey = `${queue}:${job.opts.deduplicationKey}`;
|
|
3091
|
+
this.deduplicationKeys.delete(dedupKey);
|
|
3092
|
+
}
|
|
3093
|
+
this.jobs.delete(key);
|
|
3094
|
+
}
|
|
3095
|
+
return toDelete.length;
|
|
3096
|
+
}
|
|
3097
|
+
async count(queue, status) {
|
|
3098
|
+
const prefix = `${queue}:`;
|
|
3099
|
+
let count = 0;
|
|
3100
|
+
for (const [key, job] of this.jobs.entries()) {
|
|
3101
|
+
if (key.startsWith(prefix)) {
|
|
3102
|
+
if (!status || job.state === status) {
|
|
3103
|
+
count++;
|
|
3104
|
+
}
|
|
3105
|
+
}
|
|
3106
|
+
}
|
|
3107
|
+
return count;
|
|
3108
|
+
}
|
|
3109
|
+
async existsByDeduplicationKey(queue, key) {
|
|
3110
|
+
const dedupKey = `${queue}:${key}`;
|
|
3111
|
+
return this.deduplicationKeys.has(dedupKey);
|
|
3112
|
+
}
|
|
3113
|
+
/**
|
|
3114
|
+
* Clear all jobs
|
|
3115
|
+
*/
|
|
3116
|
+
clear() {
|
|
3117
|
+
this.jobs.clear();
|
|
3118
|
+
this.deduplicationKeys.clear();
|
|
3119
|
+
}
|
|
3120
|
+
};
|
|
3121
|
+
// Annotate the CommonJS export names for ESM import in node:
|
|
3122
|
+
0 && (module.exports = {
|
|
3123
|
+
BatchAccumulator,
|
|
3124
|
+
BatchWriter,
|
|
3125
|
+
FlowFnImpl,
|
|
3126
|
+
HealthCheckerImpl,
|
|
3127
|
+
MemoryAdapter,
|
|
3128
|
+
MemoryDLQManager,
|
|
3129
|
+
MemoryEventLog,
|
|
3130
|
+
MemoryEventTracker,
|
|
3131
|
+
MemoryJobStorage,
|
|
3132
|
+
MemoryWorkflowStorage,
|
|
3133
|
+
MetricsManager,
|
|
3134
|
+
PostgresAdapter,
|
|
3135
|
+
PriorityQueue,
|
|
3136
|
+
RateLimiter,
|
|
3137
|
+
RedisAdapter,
|
|
3138
|
+
Scheduler,
|
|
3139
|
+
SlidingWindowRateLimiter,
|
|
3140
|
+
TokenBucketRateLimiter,
|
|
3141
|
+
Worker,
|
|
3142
|
+
addDuration,
|
|
3143
|
+
areJobsEquivalent,
|
|
3144
|
+
batch,
|
|
3145
|
+
batchByKey,
|
|
3146
|
+
calculateBackoff,
|
|
3147
|
+
chunk,
|
|
3148
|
+
circuitBreaker,
|
|
3149
|
+
cloneViaSerialization,
|
|
3150
|
+
createFlow,
|
|
3151
|
+
createRateLimiter,
|
|
3152
|
+
createWorker,
|
|
3153
|
+
delayUntil,
|
|
3154
|
+
deserialize,
|
|
3155
|
+
deserializeCompressed,
|
|
3156
|
+
formatDuration,
|
|
3157
|
+
fromMilliseconds,
|
|
3158
|
+
generateDeduplicationKey,
|
|
3159
|
+
generateDeterministicId,
|
|
3160
|
+
generateExecutionId,
|
|
3161
|
+
generateId,
|
|
3162
|
+
generateJobId,
|
|
3163
|
+
generateMessageId,
|
|
3164
|
+
getSerializedSize,
|
|
3165
|
+
hashJob,
|
|
3166
|
+
isFuture,
|
|
3167
|
+
isPast,
|
|
3168
|
+
isSerializable,
|
|
3169
|
+
now,
|
|
3170
|
+
parseDuration,
|
|
3171
|
+
processBatches,
|
|
3172
|
+
serialize,
|
|
3173
|
+
serializeCompressed,
|
|
3174
|
+
serializeSafe,
|
|
3175
|
+
sleep,
|
|
3176
|
+
sleepDuration,
|
|
3177
|
+
timeout,
|
|
3178
|
+
toMilliseconds
|
|
3179
|
+
});
|
|
3180
|
+
//# sourceMappingURL=index.js.map
|