@nicnocquee/dataqueue 1.25.0 → 1.26.0-beta.20260223195940
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ai/build-docs-content.ts +96 -0
- package/ai/build-llms-full.ts +42 -0
- package/ai/docs-content.json +278 -0
- package/ai/rules/advanced.md +132 -0
- package/ai/rules/basic.md +159 -0
- package/ai/rules/react-dashboard.md +83 -0
- package/ai/skills/dataqueue-advanced/SKILL.md +320 -0
- package/ai/skills/dataqueue-core/SKILL.md +234 -0
- package/ai/skills/dataqueue-react/SKILL.md +189 -0
- package/dist/cli.cjs +1149 -14
- package/dist/cli.cjs.map +1 -1
- package/dist/cli.d.cts +66 -1
- package/dist/cli.d.ts +66 -1
- package/dist/cli.js +1146 -13
- package/dist/cli.js.map +1 -1
- package/dist/index.cjs +3157 -1237
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +613 -23
- package/dist/index.d.ts +613 -23
- package/dist/index.js +3156 -1238
- package/dist/index.js.map +1 -1
- package/dist/mcp-server.cjs +186 -0
- package/dist/mcp-server.cjs.map +1 -0
- package/dist/mcp-server.d.cts +32 -0
- package/dist/mcp-server.d.ts +32 -0
- package/dist/mcp-server.js +175 -0
- package/dist/mcp-server.js.map +1 -0
- package/migrations/1781200000004_create_cron_schedules_table.sql +33 -0
- package/migrations/1781200000005_add_retry_config_to_job_queue.sql +17 -0
- package/package.json +24 -21
- package/src/backend.ts +170 -5
- package/src/backends/postgres.ts +992 -63
- package/src/backends/redis-scripts.ts +358 -26
- package/src/backends/redis.test.ts +1363 -0
- package/src/backends/redis.ts +993 -35
- package/src/cli.test.ts +82 -6
- package/src/cli.ts +73 -10
- package/src/cron.test.ts +126 -0
- package/src/cron.ts +40 -0
- package/src/db-util.ts +1 -1
- package/src/index.test.ts +682 -0
- package/src/index.ts +209 -34
- package/src/init-command.test.ts +449 -0
- package/src/init-command.ts +709 -0
- package/src/install-mcp-command.test.ts +216 -0
- package/src/install-mcp-command.ts +185 -0
- package/src/install-rules-command.test.ts +218 -0
- package/src/install-rules-command.ts +233 -0
- package/src/install-skills-command.test.ts +176 -0
- package/src/install-skills-command.ts +124 -0
- package/src/mcp-server.test.ts +162 -0
- package/src/mcp-server.ts +231 -0
- package/src/processor.ts +36 -97
- package/src/queue.test.ts +465 -0
- package/src/queue.ts +34 -252
- package/src/supervisor.test.ts +340 -0
- package/src/supervisor.ts +162 -0
- package/src/types.ts +388 -12
- package/LICENSE +0 -21
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
import { QueueBackend } from './backend.js';
|
|
2
|
+
import { setLogContext, log } from './log-context.js';
|
|
3
|
+
import { Supervisor, SupervisorOptions, SupervisorRunResult } from './types.js';
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Creates a background supervisor that periodically runs maintenance tasks:
|
|
7
|
+
* reclaiming stuck jobs, cleaning up old jobs/events, and expiring
|
|
8
|
+
* timed-out waitpoint tokens.
|
|
9
|
+
*
|
|
10
|
+
* @param backend - The queue backend (Postgres or Redis) to run maintenance against.
|
|
11
|
+
* @param options - Configuration for intervals, retention, and feature toggles.
|
|
12
|
+
* @returns A {@link Supervisor} with `start`, `startInBackground`, `stop`,
|
|
13
|
+
* `stopAndDrain`, and `isRunning` methods.
|
|
14
|
+
*/
|
|
15
|
+
export const createSupervisor = (
|
|
16
|
+
backend: QueueBackend,
|
|
17
|
+
options: SupervisorOptions = {},
|
|
18
|
+
): Supervisor => {
|
|
19
|
+
const {
|
|
20
|
+
intervalMs = 60_000,
|
|
21
|
+
stuckJobsTimeoutMinutes = 10,
|
|
22
|
+
cleanupJobsDaysToKeep = 30,
|
|
23
|
+
cleanupEventsDaysToKeep = 30,
|
|
24
|
+
cleanupBatchSize = 1000,
|
|
25
|
+
reclaimStuckJobs = true,
|
|
26
|
+
expireTimedOutTokens = true,
|
|
27
|
+
onError = (error: Error) =>
|
|
28
|
+
console.error('Supervisor maintenance error:', error),
|
|
29
|
+
verbose = false,
|
|
30
|
+
} = options;
|
|
31
|
+
|
|
32
|
+
let running = false;
|
|
33
|
+
let timeoutId: NodeJS.Timeout | null = null;
|
|
34
|
+
let currentRunPromise: Promise<SupervisorRunResult> | null = null;
|
|
35
|
+
|
|
36
|
+
setLogContext(verbose);
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* Executes every maintenance task once, isolating failures so one
|
|
40
|
+
* broken task does not prevent the others from running.
|
|
41
|
+
*/
|
|
42
|
+
const runOnce = async (): Promise<SupervisorRunResult> => {
|
|
43
|
+
setLogContext(verbose);
|
|
44
|
+
|
|
45
|
+
const result: SupervisorRunResult = {
|
|
46
|
+
reclaimedJobs: 0,
|
|
47
|
+
cleanedUpJobs: 0,
|
|
48
|
+
cleanedUpEvents: 0,
|
|
49
|
+
expiredTokens: 0,
|
|
50
|
+
};
|
|
51
|
+
|
|
52
|
+
if (reclaimStuckJobs) {
|
|
53
|
+
try {
|
|
54
|
+
result.reclaimedJobs = await backend.reclaimStuckJobs(
|
|
55
|
+
stuckJobsTimeoutMinutes,
|
|
56
|
+
);
|
|
57
|
+
if (result.reclaimedJobs > 0) {
|
|
58
|
+
log(`Supervisor: reclaimed ${result.reclaimedJobs} stuck jobs`);
|
|
59
|
+
}
|
|
60
|
+
} catch (e) {
|
|
61
|
+
onError(e instanceof Error ? e : new Error(String(e)));
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
if (cleanupJobsDaysToKeep > 0) {
|
|
66
|
+
try {
|
|
67
|
+
result.cleanedUpJobs = await backend.cleanupOldJobs(
|
|
68
|
+
cleanupJobsDaysToKeep,
|
|
69
|
+
cleanupBatchSize,
|
|
70
|
+
);
|
|
71
|
+
if (result.cleanedUpJobs > 0) {
|
|
72
|
+
log(`Supervisor: cleaned up ${result.cleanedUpJobs} old jobs`);
|
|
73
|
+
}
|
|
74
|
+
} catch (e) {
|
|
75
|
+
onError(e instanceof Error ? e : new Error(String(e)));
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
if (cleanupEventsDaysToKeep > 0) {
|
|
80
|
+
try {
|
|
81
|
+
result.cleanedUpEvents = await backend.cleanupOldJobEvents(
|
|
82
|
+
cleanupEventsDaysToKeep,
|
|
83
|
+
cleanupBatchSize,
|
|
84
|
+
);
|
|
85
|
+
if (result.cleanedUpEvents > 0) {
|
|
86
|
+
log(
|
|
87
|
+
`Supervisor: cleaned up ${result.cleanedUpEvents} old job events`,
|
|
88
|
+
);
|
|
89
|
+
}
|
|
90
|
+
} catch (e) {
|
|
91
|
+
onError(e instanceof Error ? e : new Error(String(e)));
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
if (expireTimedOutTokens) {
|
|
96
|
+
try {
|
|
97
|
+
result.expiredTokens = await backend.expireTimedOutWaitpoints();
|
|
98
|
+
if (result.expiredTokens > 0) {
|
|
99
|
+
log(`Supervisor: expired ${result.expiredTokens} timed-out tokens`);
|
|
100
|
+
}
|
|
101
|
+
} catch (e) {
|
|
102
|
+
onError(e instanceof Error ? e : new Error(String(e)));
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
return result;
|
|
107
|
+
};
|
|
108
|
+
|
|
109
|
+
return {
|
|
110
|
+
start: async () => {
|
|
111
|
+
return runOnce();
|
|
112
|
+
},
|
|
113
|
+
|
|
114
|
+
startInBackground: () => {
|
|
115
|
+
if (running) return;
|
|
116
|
+
log('Supervisor: starting background maintenance loop');
|
|
117
|
+
running = true;
|
|
118
|
+
|
|
119
|
+
const loop = async () => {
|
|
120
|
+
if (!running) return;
|
|
121
|
+
currentRunPromise = runOnce();
|
|
122
|
+
await currentRunPromise;
|
|
123
|
+
currentRunPromise = null;
|
|
124
|
+
if (running) {
|
|
125
|
+
timeoutId = setTimeout(loop, intervalMs);
|
|
126
|
+
}
|
|
127
|
+
};
|
|
128
|
+
|
|
129
|
+
loop();
|
|
130
|
+
},
|
|
131
|
+
|
|
132
|
+
stop: () => {
|
|
133
|
+
running = false;
|
|
134
|
+
if (timeoutId !== null) {
|
|
135
|
+
clearTimeout(timeoutId);
|
|
136
|
+
timeoutId = null;
|
|
137
|
+
}
|
|
138
|
+
log('Supervisor: stopped');
|
|
139
|
+
},
|
|
140
|
+
|
|
141
|
+
stopAndDrain: async (timeoutMs = 30_000) => {
|
|
142
|
+
running = false;
|
|
143
|
+
if (timeoutId !== null) {
|
|
144
|
+
clearTimeout(timeoutId);
|
|
145
|
+
timeoutId = null;
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
if (currentRunPromise) {
|
|
149
|
+
log('Supervisor: draining current maintenance run…');
|
|
150
|
+
await Promise.race([
|
|
151
|
+
currentRunPromise,
|
|
152
|
+
new Promise<void>((resolve) => setTimeout(resolve, timeoutMs)),
|
|
153
|
+
]);
|
|
154
|
+
currentRunPromise = null;
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
log('Supervisor: drained and stopped');
|
|
158
|
+
},
|
|
159
|
+
|
|
160
|
+
isRunning: () => running,
|
|
161
|
+
};
|
|
162
|
+
};
|
package/src/types.ts
CHANGED
|
@@ -1,6 +1,33 @@
|
|
|
1
1
|
// Utility type for job type keys
|
|
2
2
|
export type JobType<PayloadMap> = keyof PayloadMap & string;
|
|
3
3
|
|
|
4
|
+
/**
|
|
5
|
+
* Abstract database client interface for transactional job creation.
|
|
6
|
+
* Compatible with `pg.Pool`, `pg.PoolClient`, `pg.Client`, or any object
|
|
7
|
+
* that exposes a `.query()` method matching the `pg` signature.
|
|
8
|
+
*/
|
|
9
|
+
export interface DatabaseClient {
|
|
10
|
+
query(
|
|
11
|
+
text: string,
|
|
12
|
+
values?: any[],
|
|
13
|
+
): Promise<{ rows: any[]; rowCount: number | null }>;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* Options for `addJob()` beyond the job itself.
|
|
18
|
+
* Use `db` to insert the job within an existing database transaction.
|
|
19
|
+
*/
|
|
20
|
+
export interface AddJobOptions {
|
|
21
|
+
/**
|
|
22
|
+
* An external database client (e.g., a `pg.PoolClient` inside a transaction).
|
|
23
|
+
* When provided, the INSERT runs on this client instead of the internal pool,
|
|
24
|
+
* so the job is part of the caller's transaction.
|
|
25
|
+
*
|
|
26
|
+
* **PostgreSQL only.** Throws if used with the Redis backend.
|
|
27
|
+
*/
|
|
28
|
+
db?: DatabaseClient;
|
|
29
|
+
}
|
|
30
|
+
|
|
4
31
|
export interface JobOptions<PayloadMap, T extends JobType<PayloadMap>> {
|
|
5
32
|
jobType: T;
|
|
6
33
|
payload: PayloadMap[T];
|
|
@@ -73,6 +100,26 @@ export interface JobOptions<PayloadMap, T extends JobType<PayloadMap>> {
|
|
|
73
100
|
* Once a key exists, it cannot be reused until the job is cleaned up (via `cleanupOldJobs`).
|
|
74
101
|
*/
|
|
75
102
|
idempotencyKey?: string;
|
|
103
|
+
/**
|
|
104
|
+
* Base delay between retries in seconds. When `retryBackoff` is true (the default),
|
|
105
|
+
* this is the base for exponential backoff: `retryDelay * 2^attempts`.
|
|
106
|
+
* When `retryBackoff` is false, retries use this fixed delay.
|
|
107
|
+
* @default 60
|
|
108
|
+
*/
|
|
109
|
+
retryDelay?: number;
|
|
110
|
+
/**
|
|
111
|
+
* Whether to use exponential backoff for retries. When true, delay doubles
|
|
112
|
+
* with each attempt and includes jitter to prevent thundering herd.
|
|
113
|
+
* When false, a fixed `retryDelay` is used between every retry.
|
|
114
|
+
* @default true
|
|
115
|
+
*/
|
|
116
|
+
retryBackoff?: boolean;
|
|
117
|
+
/**
|
|
118
|
+
* Maximum delay between retries in seconds. Caps the exponential backoff
|
|
119
|
+
* so retries never wait longer than this value. Only meaningful when
|
|
120
|
+
* `retryBackoff` is true. No limit when omitted.
|
|
121
|
+
*/
|
|
122
|
+
retryDelayMax?: number;
|
|
76
123
|
}
|
|
77
124
|
|
|
78
125
|
/**
|
|
@@ -196,6 +243,18 @@ export interface JobRecord<PayloadMap, T extends JobType<PayloadMap>> {
|
|
|
196
243
|
* Updated by the handler via `ctx.setProgress(percent)`.
|
|
197
244
|
*/
|
|
198
245
|
progress?: number | null;
|
|
246
|
+
/**
|
|
247
|
+
* Base delay between retries in seconds, or null if using legacy default.
|
|
248
|
+
*/
|
|
249
|
+
retryDelay?: number | null;
|
|
250
|
+
/**
|
|
251
|
+
* Whether exponential backoff is enabled for retries, or null if using legacy default.
|
|
252
|
+
*/
|
|
253
|
+
retryBackoff?: boolean | null;
|
|
254
|
+
/**
|
|
255
|
+
* Maximum delay cap for retries in seconds, or null if no cap.
|
|
256
|
+
*/
|
|
257
|
+
retryDelayMax?: number | null;
|
|
199
258
|
}
|
|
200
259
|
|
|
201
260
|
/**
|
|
@@ -452,6 +511,91 @@ export interface Processor {
|
|
|
452
511
|
start: () => Promise<number>;
|
|
453
512
|
}
|
|
454
513
|
|
|
514
|
+
export interface SupervisorOptions {
|
|
515
|
+
/**
|
|
516
|
+
* How often the maintenance loop runs, in milliseconds.
|
|
517
|
+
* @default 60000 (1 minute)
|
|
518
|
+
*/
|
|
519
|
+
intervalMs?: number;
|
|
520
|
+
/**
|
|
521
|
+
* Reclaim jobs stuck in `processing` longer than this many minutes.
|
|
522
|
+
* @default 10
|
|
523
|
+
*/
|
|
524
|
+
stuckJobsTimeoutMinutes?: number;
|
|
525
|
+
/**
|
|
526
|
+
* Auto-delete completed jobs older than this many days. Set to 0 to disable.
|
|
527
|
+
* @default 30
|
|
528
|
+
*/
|
|
529
|
+
cleanupJobsDaysToKeep?: number;
|
|
530
|
+
/**
|
|
531
|
+
* Auto-delete job events older than this many days. Set to 0 to disable.
|
|
532
|
+
* @default 30
|
|
533
|
+
*/
|
|
534
|
+
cleanupEventsDaysToKeep?: number;
|
|
535
|
+
/**
|
|
536
|
+
* Batch size for cleanup deletions.
|
|
537
|
+
* @default 1000
|
|
538
|
+
*/
|
|
539
|
+
cleanupBatchSize?: number;
|
|
540
|
+
/**
|
|
541
|
+
* Whether to reclaim stuck jobs each cycle.
|
|
542
|
+
* @default true
|
|
543
|
+
*/
|
|
544
|
+
reclaimStuckJobs?: boolean;
|
|
545
|
+
/**
|
|
546
|
+
* Whether to expire timed-out waitpoint tokens each cycle.
|
|
547
|
+
* @default true
|
|
548
|
+
*/
|
|
549
|
+
expireTimedOutTokens?: boolean;
|
|
550
|
+
/**
|
|
551
|
+
* Called when a maintenance task throws. One failure does not block other tasks.
|
|
552
|
+
* @default console.error
|
|
553
|
+
*/
|
|
554
|
+
onError?: (error: Error) => void;
|
|
555
|
+
/** Enable verbose logging. */
|
|
556
|
+
verbose?: boolean;
|
|
557
|
+
}
|
|
558
|
+
|
|
559
|
+
export interface SupervisorRunResult {
|
|
560
|
+
/** Number of stuck jobs reclaimed back to pending. */
|
|
561
|
+
reclaimedJobs: number;
|
|
562
|
+
/** Number of old completed jobs deleted. */
|
|
563
|
+
cleanedUpJobs: number;
|
|
564
|
+
/** Number of old job events deleted. */
|
|
565
|
+
cleanedUpEvents: number;
|
|
566
|
+
/** Number of timed-out waitpoint tokens expired. */
|
|
567
|
+
expiredTokens: number;
|
|
568
|
+
}
|
|
569
|
+
|
|
570
|
+
export interface Supervisor {
|
|
571
|
+
/**
|
|
572
|
+
* Run all maintenance tasks once and return the results.
|
|
573
|
+
* Ideal for serverless or cron-triggered invocations.
|
|
574
|
+
*/
|
|
575
|
+
start: () => Promise<SupervisorRunResult>;
|
|
576
|
+
/**
|
|
577
|
+
* Start the maintenance loop in the background.
|
|
578
|
+
* Runs every `intervalMs` milliseconds (default: 60 000).
|
|
579
|
+
* Call `stop()` or `stopAndDrain()` to halt the loop.
|
|
580
|
+
*/
|
|
581
|
+
startInBackground: () => void;
|
|
582
|
+
/**
|
|
583
|
+
* Stop the background maintenance loop immediately.
|
|
584
|
+
* Does not wait for an in-flight maintenance run to complete.
|
|
585
|
+
*/
|
|
586
|
+
stop: () => void;
|
|
587
|
+
/**
|
|
588
|
+
* Stop the background loop and wait for the current maintenance run
|
|
589
|
+
* (if any) to finish before resolving.
|
|
590
|
+
*
|
|
591
|
+
* @param timeoutMs - Maximum time to wait (default: 30 000 ms).
|
|
592
|
+
* If the run does not finish within this time the promise resolves anyway.
|
|
593
|
+
*/
|
|
594
|
+
stopAndDrain: (timeoutMs?: number) => Promise<void>;
|
|
595
|
+
/** Whether the background maintenance loop is currently running. */
|
|
596
|
+
isRunning: () => boolean;
|
|
597
|
+
}
|
|
598
|
+
|
|
455
599
|
export interface DatabaseSSLConfig {
|
|
456
600
|
/**
|
|
457
601
|
* CA certificate as PEM string or file path. If the value starts with 'file://', it will be loaded from file, otherwise treated as PEM string.
|
|
@@ -474,10 +618,13 @@ export interface DatabaseSSLConfig {
|
|
|
474
618
|
/**
|
|
475
619
|
* Configuration for PostgreSQL backend (default).
|
|
476
620
|
* Backward-compatible: omitting `backend` defaults to 'postgres'.
|
|
621
|
+
*
|
|
622
|
+
* Provide either `databaseConfig` (the library creates a pool) or `pool`
|
|
623
|
+
* (bring your own `pg.Pool`). At least one must be set.
|
|
477
624
|
*/
|
|
478
625
|
export interface PostgresJobQueueConfig {
|
|
479
626
|
backend?: 'postgres';
|
|
480
|
-
databaseConfig
|
|
627
|
+
databaseConfig?: {
|
|
481
628
|
connectionString?: string;
|
|
482
629
|
host?: string;
|
|
483
630
|
port?: number;
|
|
@@ -485,7 +632,29 @@ export interface PostgresJobQueueConfig {
|
|
|
485
632
|
user?: string;
|
|
486
633
|
password?: string;
|
|
487
634
|
ssl?: DatabaseSSLConfig;
|
|
635
|
+
/**
|
|
636
|
+
* Maximum number of clients in the pool (default: 10).
|
|
637
|
+
* Increase when running multiple processors in the same process.
|
|
638
|
+
*/
|
|
639
|
+
max?: number;
|
|
640
|
+
/**
|
|
641
|
+
* Minimum number of idle clients in the pool (default: 0).
|
|
642
|
+
*/
|
|
643
|
+
min?: number;
|
|
644
|
+
/**
|
|
645
|
+
* Milliseconds a client must sit idle before being closed (default: 10000).
|
|
646
|
+
*/
|
|
647
|
+
idleTimeoutMillis?: number;
|
|
648
|
+
/**
|
|
649
|
+
* Milliseconds to wait for a connection before throwing (default: 0, no timeout).
|
|
650
|
+
*/
|
|
651
|
+
connectionTimeoutMillis?: number;
|
|
488
652
|
};
|
|
653
|
+
/**
|
|
654
|
+
* Bring your own `pg.Pool` instance. When provided, `databaseConfig` is
|
|
655
|
+
* ignored and the library will not close the pool on shutdown.
|
|
656
|
+
*/
|
|
657
|
+
pool?: import('pg').Pool;
|
|
489
658
|
verbose?: boolean;
|
|
490
659
|
}
|
|
491
660
|
|
|
@@ -501,10 +670,13 @@ export interface RedisTLSConfig {
|
|
|
501
670
|
|
|
502
671
|
/**
|
|
503
672
|
* Configuration for Redis backend.
|
|
673
|
+
*
|
|
674
|
+
* Provide either `redisConfig` (the library creates an ioredis client) or
|
|
675
|
+
* `client` (bring your own ioredis instance). At least one must be set.
|
|
504
676
|
*/
|
|
505
677
|
export interface RedisJobQueueConfig {
|
|
506
678
|
backend: 'redis';
|
|
507
|
-
redisConfig
|
|
679
|
+
redisConfig?: {
|
|
508
680
|
/** Redis URL (e.g. redis://localhost:6379) */
|
|
509
681
|
url?: string;
|
|
510
682
|
host?: string;
|
|
@@ -519,6 +691,17 @@ export interface RedisJobQueueConfig {
|
|
|
519
691
|
*/
|
|
520
692
|
keyPrefix?: string;
|
|
521
693
|
};
|
|
694
|
+
/**
|
|
695
|
+
* Bring your own ioredis client instance. When provided, `redisConfig` is
|
|
696
|
+
* ignored and the library will not close the client on shutdown.
|
|
697
|
+
* Use `keyPrefix` to set the key namespace (default: 'dq:').
|
|
698
|
+
*/
|
|
699
|
+
client?: unknown;
|
|
700
|
+
/**
|
|
701
|
+
* Key prefix when using an external `client`. Ignored when `redisConfig` is used
|
|
702
|
+
* (set `redisConfig.keyPrefix` instead). Default: 'dq:'.
|
|
703
|
+
*/
|
|
704
|
+
keyPrefix?: string;
|
|
522
705
|
verbose?: boolean;
|
|
523
706
|
}
|
|
524
707
|
|
|
@@ -533,13 +716,133 @@ export type JobQueueConfigLegacy = PostgresJobQueueConfig;
|
|
|
533
716
|
|
|
534
717
|
export type TagQueryMode = 'exact' | 'all' | 'any' | 'none';
|
|
535
718
|
|
|
719
|
+
// ── Cron schedule types ──────────────────────────────────────────────
|
|
720
|
+
|
|
721
|
+
/**
|
|
722
|
+
* Status of a cron schedule.
|
|
723
|
+
*/
|
|
724
|
+
export type CronScheduleStatus = 'active' | 'paused';
|
|
725
|
+
|
|
726
|
+
/**
|
|
727
|
+
* Options for creating a recurring cron schedule.
|
|
728
|
+
* Each schedule defines a recurring job that is automatically enqueued
|
|
729
|
+
* when its cron expression matches.
|
|
730
|
+
*/
|
|
731
|
+
export interface CronScheduleOptions<
|
|
732
|
+
PayloadMap,
|
|
733
|
+
T extends JobType<PayloadMap>,
|
|
734
|
+
> {
|
|
735
|
+
/** Unique human-readable name for the schedule. */
|
|
736
|
+
scheduleName: string;
|
|
737
|
+
/** Standard cron expression (5 fields, e.g. "0 * * * *"). */
|
|
738
|
+
cronExpression: string;
|
|
739
|
+
/** Job type from the PayloadMap. */
|
|
740
|
+
jobType: T;
|
|
741
|
+
/** Payload for each job instance. */
|
|
742
|
+
payload: PayloadMap[T];
|
|
743
|
+
/** Maximum retry attempts for each job instance (default: 3). */
|
|
744
|
+
maxAttempts?: number;
|
|
745
|
+
/** Priority for each job instance (default: 0). */
|
|
746
|
+
priority?: number;
|
|
747
|
+
/** Timeout in milliseconds for each job instance. */
|
|
748
|
+
timeoutMs?: number;
|
|
749
|
+
/** Whether to force-kill the job on timeout (default: false). */
|
|
750
|
+
forceKillOnTimeout?: boolean;
|
|
751
|
+
/** Tags for each job instance. */
|
|
752
|
+
tags?: string[];
|
|
753
|
+
/** IANA timezone string for cron evaluation (default: "UTC"). */
|
|
754
|
+
timezone?: string;
|
|
755
|
+
/**
|
|
756
|
+
* Whether to allow overlapping job instances (default: false).
|
|
757
|
+
* When false, a new job will not be enqueued if the previous instance
|
|
758
|
+
* is still pending, processing, or waiting.
|
|
759
|
+
*/
|
|
760
|
+
allowOverlap?: boolean;
|
|
761
|
+
/** Base delay between retries in seconds for each job instance (default: 60). */
|
|
762
|
+
retryDelay?: number;
|
|
763
|
+
/** Whether to use exponential backoff for retries (default: true). */
|
|
764
|
+
retryBackoff?: boolean;
|
|
765
|
+
/** Maximum delay cap for retries in seconds. */
|
|
766
|
+
retryDelayMax?: number;
|
|
767
|
+
}
|
|
768
|
+
|
|
769
|
+
/**
|
|
770
|
+
* A persisted cron schedule record.
|
|
771
|
+
*/
|
|
772
|
+
export interface CronScheduleRecord {
|
|
773
|
+
id: number;
|
|
774
|
+
scheduleName: string;
|
|
775
|
+
cronExpression: string;
|
|
776
|
+
jobType: string;
|
|
777
|
+
payload: any;
|
|
778
|
+
maxAttempts: number;
|
|
779
|
+
priority: number;
|
|
780
|
+
timeoutMs: number | null;
|
|
781
|
+
forceKillOnTimeout: boolean;
|
|
782
|
+
tags: string[] | undefined;
|
|
783
|
+
timezone: string;
|
|
784
|
+
allowOverlap: boolean;
|
|
785
|
+
status: CronScheduleStatus;
|
|
786
|
+
lastEnqueuedAt: Date | null;
|
|
787
|
+
lastJobId: number | null;
|
|
788
|
+
nextRunAt: Date | null;
|
|
789
|
+
createdAt: Date;
|
|
790
|
+
updatedAt: Date;
|
|
791
|
+
retryDelay: number | null;
|
|
792
|
+
retryBackoff: boolean | null;
|
|
793
|
+
retryDelayMax: number | null;
|
|
794
|
+
}
|
|
795
|
+
|
|
796
|
+
/**
|
|
797
|
+
* Options for editing an existing cron schedule.
|
|
798
|
+
* All fields are optional; only provided fields are updated.
|
|
799
|
+
*/
|
|
800
|
+
export interface EditCronScheduleOptions {
|
|
801
|
+
cronExpression?: string;
|
|
802
|
+
payload?: any;
|
|
803
|
+
maxAttempts?: number;
|
|
804
|
+
priority?: number;
|
|
805
|
+
timeoutMs?: number | null;
|
|
806
|
+
forceKillOnTimeout?: boolean;
|
|
807
|
+
tags?: string[] | null;
|
|
808
|
+
timezone?: string;
|
|
809
|
+
allowOverlap?: boolean;
|
|
810
|
+
retryDelay?: number | null;
|
|
811
|
+
retryBackoff?: boolean | null;
|
|
812
|
+
retryDelayMax?: number | null;
|
|
813
|
+
}
|
|
814
|
+
|
|
536
815
|
export interface JobQueue<PayloadMap> {
|
|
537
816
|
/**
|
|
538
817
|
* Add a job to the job queue.
|
|
818
|
+
*
|
|
819
|
+
* @param job - The job to enqueue.
|
|
820
|
+
* @param options - Optional. Pass `{ db }` with an external database client
|
|
821
|
+
* to insert the job within an existing transaction (PostgreSQL only).
|
|
539
822
|
*/
|
|
540
823
|
addJob: <T extends JobType<PayloadMap>>(
|
|
541
824
|
job: JobOptions<PayloadMap, T>,
|
|
825
|
+
options?: AddJobOptions,
|
|
542
826
|
) => Promise<number>;
|
|
827
|
+
/**
|
|
828
|
+
* Add multiple jobs to the queue in a single operation.
|
|
829
|
+
*
|
|
830
|
+
* More efficient than calling `addJob` in a loop because it batches the
|
|
831
|
+
* INSERT into a single database round-trip (PostgreSQL) or a single
|
|
832
|
+
* atomic Lua script (Redis).
|
|
833
|
+
*
|
|
834
|
+
* Returns an array of job IDs in the same order as the input array.
|
|
835
|
+
* Each job may independently have an `idempotencyKey`; duplicates
|
|
836
|
+
* resolve to the existing job's ID without creating a new row.
|
|
837
|
+
*
|
|
838
|
+
* @param jobs - Array of jobs to enqueue.
|
|
839
|
+
* @param options - Optional. Pass `{ db }` with an external database client
|
|
840
|
+
* to insert the jobs within an existing transaction (PostgreSQL only).
|
|
841
|
+
*/
|
|
842
|
+
addJobs: <T extends JobType<PayloadMap>>(
|
|
843
|
+
jobs: JobOptions<PayloadMap, T>[],
|
|
844
|
+
options?: AddJobOptions,
|
|
845
|
+
) => Promise<number[]>;
|
|
543
846
|
/**
|
|
544
847
|
* Get a job by its ID.
|
|
545
848
|
*/
|
|
@@ -606,12 +909,21 @@ export interface JobQueue<PayloadMap> {
|
|
|
606
909
|
retryJob: (jobId: number) => Promise<void>;
|
|
607
910
|
/**
|
|
608
911
|
* Cleanup jobs that are older than the specified number of days.
|
|
912
|
+
* Deletes in batches for scale safety.
|
|
913
|
+
* @param daysToKeep - Number of days to retain completed jobs (default 30).
|
|
914
|
+
* @param batchSize - Number of rows to delete per batch (default 1000 for PostgreSQL, 200 for Redis).
|
|
609
915
|
*/
|
|
610
|
-
cleanupOldJobs: (daysToKeep?: number) => Promise<number>;
|
|
916
|
+
cleanupOldJobs: (daysToKeep?: number, batchSize?: number) => Promise<number>;
|
|
611
917
|
/**
|
|
612
918
|
* Cleanup job events that are older than the specified number of days.
|
|
919
|
+
* Deletes in batches for scale safety.
|
|
920
|
+
* @param daysToKeep - Number of days to retain events (default 30).
|
|
921
|
+
* @param batchSize - Number of rows to delete per batch (default 1000).
|
|
613
922
|
*/
|
|
614
|
-
cleanupOldJobEvents: (
|
|
923
|
+
cleanupOldJobEvents: (
|
|
924
|
+
daysToKeep?: number,
|
|
925
|
+
batchSize?: number,
|
|
926
|
+
) => Promise<number>;
|
|
615
927
|
/**
|
|
616
928
|
* Cancel a job given its ID.
|
|
617
929
|
* - This will set the job status to 'cancelled' and clear the locked_at and locked_by.
|
|
@@ -685,6 +997,13 @@ export interface JobQueue<PayloadMap> {
|
|
|
685
997
|
options?: ProcessorOptions,
|
|
686
998
|
) => Processor;
|
|
687
999
|
|
|
1000
|
+
/**
|
|
1001
|
+
* Create a background supervisor that automatically reclaims stuck jobs,
|
|
1002
|
+
* cleans up old completed jobs/events, and expires timed-out waitpoint
|
|
1003
|
+
* tokens on a configurable interval.
|
|
1004
|
+
*/
|
|
1005
|
+
createSupervisor: (options?: SupervisorOptions) => Supervisor;
|
|
1006
|
+
|
|
688
1007
|
/**
|
|
689
1008
|
* Get the job events for a job.
|
|
690
1009
|
*/
|
|
@@ -695,8 +1014,6 @@ export interface JobQueue<PayloadMap> {
|
|
|
695
1014
|
* Tokens can be completed externally to resume a waiting job.
|
|
696
1015
|
* Can be called outside of handlers (e.g., from an API route).
|
|
697
1016
|
*
|
|
698
|
-
* **PostgreSQL backend only.** Throws if the backend is Redis.
|
|
699
|
-
*
|
|
700
1017
|
* @param options - Optional token configuration (timeout, tags).
|
|
701
1018
|
* @returns A token object with `id`.
|
|
702
1019
|
*/
|
|
@@ -706,8 +1023,6 @@ export interface JobQueue<PayloadMap> {
|
|
|
706
1023
|
* Complete a waitpoint token, resuming the associated waiting job.
|
|
707
1024
|
* Can be called from anywhere (API routes, external services, etc.).
|
|
708
1025
|
*
|
|
709
|
-
* **PostgreSQL backend only.** Throws if the backend is Redis.
|
|
710
|
-
*
|
|
711
1026
|
* @param tokenId - The ID of the token to complete.
|
|
712
1027
|
* @param data - Optional data to pass to the waiting handler.
|
|
713
1028
|
*/
|
|
@@ -716,8 +1031,6 @@ export interface JobQueue<PayloadMap> {
|
|
|
716
1031
|
/**
|
|
717
1032
|
* Retrieve a waitpoint token by its ID.
|
|
718
1033
|
*
|
|
719
|
-
* **PostgreSQL backend only.** Throws if the backend is Redis.
|
|
720
|
-
*
|
|
721
1034
|
* @param tokenId - The ID of the token to retrieve.
|
|
722
1035
|
* @returns The token record, or null if not found.
|
|
723
1036
|
*/
|
|
@@ -727,12 +1040,75 @@ export interface JobQueue<PayloadMap> {
|
|
|
727
1040
|
* Expire timed-out waitpoint tokens and resume their associated jobs.
|
|
728
1041
|
* Call this periodically (e.g., alongside `reclaimStuckJobs`).
|
|
729
1042
|
*
|
|
730
|
-
* **PostgreSQL backend only.** Throws if the backend is Redis.
|
|
731
|
-
*
|
|
732
1043
|
* @returns The number of tokens that were expired.
|
|
733
1044
|
*/
|
|
734
1045
|
expireTimedOutTokens: () => Promise<number>;
|
|
735
1046
|
|
|
1047
|
+
// ── Cron schedule operations ────────────────────────────────────────
|
|
1048
|
+
|
|
1049
|
+
/**
|
|
1050
|
+
* Add a recurring cron schedule. The processor automatically enqueues
|
|
1051
|
+
* due cron jobs before each batch, so no manual triggering is needed.
|
|
1052
|
+
*
|
|
1053
|
+
* @returns The ID of the created schedule.
|
|
1054
|
+
* @throws If the cron expression is invalid or the schedule name is already taken.
|
|
1055
|
+
*/
|
|
1056
|
+
addCronJob: <T extends JobType<PayloadMap>>(
|
|
1057
|
+
options: CronScheduleOptions<PayloadMap, T>,
|
|
1058
|
+
) => Promise<number>;
|
|
1059
|
+
|
|
1060
|
+
/**
|
|
1061
|
+
* Get a cron schedule by its ID.
|
|
1062
|
+
*/
|
|
1063
|
+
getCronJob: (id: number) => Promise<CronScheduleRecord | null>;
|
|
1064
|
+
|
|
1065
|
+
/**
|
|
1066
|
+
* Get a cron schedule by its unique name.
|
|
1067
|
+
*/
|
|
1068
|
+
getCronJobByName: (name: string) => Promise<CronScheduleRecord | null>;
|
|
1069
|
+
|
|
1070
|
+
/**
|
|
1071
|
+
* List all cron schedules, optionally filtered by status.
|
|
1072
|
+
*/
|
|
1073
|
+
listCronJobs: (status?: CronScheduleStatus) => Promise<CronScheduleRecord[]>;
|
|
1074
|
+
|
|
1075
|
+
/**
|
|
1076
|
+
* Remove a cron schedule by its ID. Does not cancel any already-enqueued jobs.
|
|
1077
|
+
*/
|
|
1078
|
+
removeCronJob: (id: number) => Promise<void>;
|
|
1079
|
+
|
|
1080
|
+
/**
|
|
1081
|
+
* Pause a cron schedule. Paused schedules are skipped by `enqueueDueCronJobs()`.
|
|
1082
|
+
*/
|
|
1083
|
+
pauseCronJob: (id: number) => Promise<void>;
|
|
1084
|
+
|
|
1085
|
+
/**
|
|
1086
|
+
* Resume a paused cron schedule.
|
|
1087
|
+
*/
|
|
1088
|
+
resumeCronJob: (id: number) => Promise<void>;
|
|
1089
|
+
|
|
1090
|
+
/**
|
|
1091
|
+
* Edit an existing cron schedule. Only provided fields are updated.
|
|
1092
|
+
* If `cronExpression` or `timezone` changes, `nextRunAt` is recalculated.
|
|
1093
|
+
*/
|
|
1094
|
+
editCronJob: (id: number, updates: EditCronScheduleOptions) => Promise<void>;
|
|
1095
|
+
|
|
1096
|
+
/**
|
|
1097
|
+
* Check all active cron schedules and enqueue jobs for any whose
|
|
1098
|
+
* `nextRunAt` has passed. When `allowOverlap` is false (the default),
|
|
1099
|
+
* a new job is not enqueued if the previous instance is still
|
|
1100
|
+
* pending, processing, or waiting.
|
|
1101
|
+
*
|
|
1102
|
+
* **Note:** The processor calls this automatically before each batch,
|
|
1103
|
+
* so you typically do not need to call it yourself. It is exposed for
|
|
1104
|
+
* manual use in tests or one-off scripts.
|
|
1105
|
+
*
|
|
1106
|
+
* @returns The number of jobs that were enqueued.
|
|
1107
|
+
*/
|
|
1108
|
+
enqueueDueCronJobs: () => Promise<number>;
|
|
1109
|
+
|
|
1110
|
+
// ── Advanced access ───────────────────────────────────────────────────
|
|
1111
|
+
|
|
736
1112
|
/**
|
|
737
1113
|
* Get the PostgreSQL database pool.
|
|
738
1114
|
* Throws if the backend is not PostgreSQL.
|
package/LICENSE
DELETED
|
@@ -1,21 +0,0 @@
|
|
|
1
|
-
MIT License
|
|
2
|
-
|
|
3
|
-
Copyright (c) Nico Prananta 2024
|
|
4
|
-
|
|
5
|
-
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
-
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
-
in the Software without restriction, including without limitation the rights
|
|
8
|
-
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
-
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
-
furnished to do so, subject to the following conditions:
|
|
11
|
-
|
|
12
|
-
The above copyright notice and this permission notice shall be included in all
|
|
13
|
-
copies or substantial portions of the Software.
|
|
14
|
-
|
|
15
|
-
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
-
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
-
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
-
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
-
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
-
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
-
SOFTWARE.
|