@bitclaw/jobs 1.3.0 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/events.d.ts +20 -0
- package/dist/events.d.ts.map +1 -0
- package/dist/events.js +33 -0
- package/dist/index.d.ts +3 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -0
- package/dist/queue.d.ts +16 -4
- package/dist/queue.d.ts.map +1 -1
- package/dist/queue.js +341 -45
- package/dist/schema.d.ts.map +1 -1
- package/dist/schema.js +17 -1
- package/dist/types.d.ts +33 -2
- package/dist/types.d.ts.map +1 -1
- package/dist/worker.d.ts +4 -0
- package/dist/worker.d.ts.map +1 -1
- package/dist/worker.js +73 -7
- package/package.json +1 -1
package/dist/events.d.ts
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import type { Job, JobBatch } from './types';
|
|
2
|
+
export type JobQueueEventMap = {
|
|
3
|
+
'job:done': (job: Job) => void;
|
|
4
|
+
'job:failed': (job: Job, error: string) => void;
|
|
5
|
+
'job:dead': (job: Job, error: string) => void;
|
|
6
|
+
'job:progress': (job: Job, progress: number) => void;
|
|
7
|
+
'job:stale': (count: number) => void;
|
|
8
|
+
'batch:complete': (batch: JobBatch) => void;
|
|
9
|
+
'batch:failed': (batch: JobBatch) => void;
|
|
10
|
+
};
|
|
11
|
+
type Handler<K extends keyof JobQueueEventMap> = JobQueueEventMap[K];
|
|
12
|
+
export declare class JobQueueEmitter {
|
|
13
|
+
private readonly listeners;
|
|
14
|
+
on<K extends keyof JobQueueEventMap>(event: K, handler: Handler<K>): () => void;
|
|
15
|
+
off<K extends keyof JobQueueEventMap>(event: K, handler: Handler<K>): void;
|
|
16
|
+
once<K extends keyof JobQueueEventMap>(event: K, handler: Handler<K>): void;
|
|
17
|
+
emit<K extends keyof JobQueueEventMap>(event: K, ...args: Parameters<JobQueueEventMap[K]>): void;
|
|
18
|
+
}
|
|
19
|
+
export {};
|
|
20
|
+
//# sourceMappingURL=events.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"events.d.ts","sourceRoot":"","sources":["../src/events.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,GAAG,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAE7C,MAAM,MAAM,gBAAgB,GAAG;IAC7B,UAAU,EAAE,CAAC,GAAG,EAAE,GAAG,KAAK,IAAI,CAAC;IAC/B,YAAY,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;IAChD,UAAU,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;IAC9C,cAAc,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,QAAQ,EAAE,MAAM,KAAK,IAAI,CAAC;IACrD,WAAW,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;IACrC,gBAAgB,EAAE,CAAC,KAAK,EAAE,QAAQ,KAAK,IAAI,CAAC;IAC5C,cAAc,EAAE,CAAC,KAAK,EAAE,QAAQ,KAAK,IAAI,CAAC;CAC3C,CAAC;AAEF,KAAK,OAAO,CAAC,CAAC,SAAS,MAAM,gBAAgB,IAAI,gBAAgB,CAAC,CAAC,CAAC,CAAC;AAErE,qBAAa,eAAe;IAC1B,OAAO,CAAC,QAAQ,CAAC,SAAS,CAGtB;IAEJ,EAAE,CAAC,CAAC,SAAS,MAAM,gBAAgB,EACjC,KAAK,EAAE,CAAC,EACR,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,GAClB,MAAM,IAAI;IAQb,GAAG,CAAC,CAAC,SAAS,MAAM,gBAAgB,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,GAAG,IAAI;IAI1E,IAAI,CAAC,CAAC,SAAS,MAAM,gBAAgB,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,GAAG,IAAI;IAQ3E,IAAI,CAAC,CAAC,SAAS,MAAM,gBAAgB,EACnC,KAAK,EAAE,CAAC,EACR,GAAG,IAAI,EAAE,UAAU,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,GACvC,IAAI;CAWR"}
|
package/dist/events.js
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
export class JobQueueEmitter {
|
|
2
|
+
listeners = new Map();
|
|
3
|
+
on(event, handler) {
|
|
4
|
+
if (!this.listeners.has(event)) {
|
|
5
|
+
this.listeners.set(event, new Set());
|
|
6
|
+
}
|
|
7
|
+
this.listeners.get(event).add(handler);
|
|
8
|
+
return () => this.off(event, handler);
|
|
9
|
+
}
|
|
10
|
+
off(event, handler) {
|
|
11
|
+
this.listeners.get(event)?.delete(handler);
|
|
12
|
+
}
|
|
13
|
+
once(event, handler) {
|
|
14
|
+
const wrapper = ((...args) => {
|
|
15
|
+
this.off(event, wrapper);
|
|
16
|
+
handler(...args);
|
|
17
|
+
});
|
|
18
|
+
this.on(event, wrapper);
|
|
19
|
+
}
|
|
20
|
+
emit(event, ...args) {
|
|
21
|
+
const handlers = this.listeners.get(event);
|
|
22
|
+
if (!handlers)
|
|
23
|
+
return;
|
|
24
|
+
for (const handler of handlers) {
|
|
25
|
+
try {
|
|
26
|
+
handler(...args);
|
|
27
|
+
}
|
|
28
|
+
catch {
|
|
29
|
+
// silently swallow listener errors
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
}
|
package/dist/index.d.ts
CHANGED
|
@@ -1,10 +1,12 @@
|
|
|
1
1
|
export type { ParsedCron } from './cron';
|
|
2
2
|
export { cronMatches, nextCronOccurrence, parseCron } from './cron';
|
|
3
|
+
export type { JobQueueEventMap } from './events';
|
|
4
|
+
export { JobQueueEmitter } from './events';
|
|
3
5
|
export { JobQueue } from './queue';
|
|
4
6
|
export { SlidingWindowRateLimiter } from './rate-limiter';
|
|
5
7
|
export { Scheduler } from './scheduler';
|
|
6
8
|
export { applyPragmas, initializeSchema } from './schema';
|
|
7
|
-
export type { AddJobOptions, AddScheduleOptions, BackoffConfig, BatchOptions, FailedJob, Job, JobBatch, JobContext, JobMap, JobStats, JobStatus, ListJobsOptions, PaginatedResult, PurgeOptions, RateLimit, Schedule, WorkerOptions } from './types';
|
|
9
|
+
export type { AddJobOptions, AddScheduleOptions, BackoffConfig, BatchOptions, FailedJob, Job, JobBatch, JobContext, JobGraphNode, JobMap, JobStats, JobStatus, ListJobsOptions, MiddlewareFn, PaginatedResult, PurgeOptions, RateLimit, Schedule, WorkerOptions } from './types';
|
|
8
10
|
export { NonRetryableError } from './types';
|
|
9
11
|
export { JobWorker } from './worker';
|
|
10
12
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAGA,YAAY,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACzC,OAAO,EAAE,WAAW,EAAE,kBAAkB,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AACpE,OAAO,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AACnC,OAAO,EAAE,wBAAwB,EAAE,MAAM,gBAAgB,CAAC;AAC1D,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AACxC,OAAO,EAAE,YAAY,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC;AAC1D,YAAY,EACV,aAAa,EACb,kBAAkB,EAClB,aAAa,EACb,YAAY,EACZ,SAAS,EACT,GAAG,EACH,QAAQ,EACR,UAAU,EACV,MAAM,EACN,QAAQ,EACR,SAAS,EACT,eAAe,EACf,eAAe,EACf,YAAY,EACZ,SAAS,EACT,QAAQ,EACR,aAAa,EACd,MAAM,SAAS,CAAC;AACjB,OAAO,EAAE,iBAAiB,EAAE,MAAM,SAAS,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAGA,YAAY,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACzC,OAAO,EAAE,WAAW,EAAE,kBAAkB,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AACpE,YAAY,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC;AACjD,OAAO,EAAE,eAAe,EAAE,MAAM,UAAU,CAAC;AAC3C,OAAO,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AACnC,OAAO,EAAE,wBAAwB,EAAE,MAAM,gBAAgB,CAAC;AAC1D,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AACxC,OAAO,EAAE,YAAY,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC;AAC1D,YAAY,EACV,aAAa,EACb,kBAAkB,EAClB,aAAa,EACb,YAAY,EACZ,SAAS,EACT,GAAG,EACH,QAAQ,EACR,UAAU,EACV,YAAY,EACZ,MAAM,EACN,QAAQ,EACR,SAAS,EACT,eAAe,EACf,YAAY,EACZ,eAAe,EACf,YAAY,EACZ,SAAS,EACT,QAAQ,EACR,aAAa,EACd,MAAM,SAAS,CAAC;AACjB,OAAO,EAAE,iBAAiB,EAAE,MAAM,SAAS,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
// packages/jobs/src/index.ts
|
|
2
2
|
// Barrel export for @bitclaw/jobs
|
|
3
3
|
export { cronMatches, nextCronOccurrence, parseCron } from './cron';
|
|
4
|
+
export { JobQueueEmitter } from './events';
|
|
4
5
|
export { JobQueue } from './queue';
|
|
5
6
|
export { SlidingWindowRateLimiter } from './rate-limiter';
|
|
6
7
|
export { Scheduler } from './scheduler';
|
package/dist/queue.d.ts
CHANGED
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
import { Database } from 'bun:sqlite';
|
|
2
|
-
import
|
|
2
|
+
import { JobQueueEmitter } from './events';
|
|
3
|
+
import type { AddJobOptions, BatchOptions, FailedJob, Job, JobBatch, JobGraphNode, JobMap, JobStats, ListJobsOptions, MiddlewareFn, PaginatedResult, PurgeOptions, WorkerOptions } from './types';
|
|
3
4
|
import { JobWorker } from './worker';
|
|
4
|
-
export declare class JobQueue<TMap extends JobMap = Record<string, unknown>> {
|
|
5
|
+
export declare class JobQueue<TMap extends JobMap = Record<string, unknown>> extends JobQueueEmitter {
|
|
5
6
|
readonly db: Database;
|
|
7
|
+
readonly middlewares: MiddlewareFn[];
|
|
6
8
|
private readonly insertJobStmt;
|
|
7
9
|
private readonly selectDedupedJobStmt;
|
|
8
10
|
private readonly insertDepStmt;
|
|
@@ -20,6 +22,7 @@ export declare class JobQueue<TMap extends JobMap = Record<string, unknown>> {
|
|
|
20
22
|
private readonly countUnmetDepsStmt;
|
|
21
23
|
private readonly unblockJobStmt;
|
|
22
24
|
private readonly lastInsertRowIdStmt;
|
|
25
|
+
private readonly renewLeaseStmt;
|
|
23
26
|
private readonly insertBatchStmt;
|
|
24
27
|
private readonly selectBatchStmt;
|
|
25
28
|
private readonly decrementBatchPendingStmt;
|
|
@@ -44,10 +47,12 @@ export declare class JobQueue<TMap extends JobMap = Record<string, unknown>> {
|
|
|
44
47
|
retryFailedJob(failedJobId: number): number;
|
|
45
48
|
purgeFailedJobs(olderThanMs: number): number;
|
|
46
49
|
purge(options: PurgeOptions): number;
|
|
47
|
-
pollAndClaim(type: string): Job | null;
|
|
48
|
-
|
|
50
|
+
pollAndClaim(type: string, leaseMs?: number): Job | null;
|
|
51
|
+
renewLease(id: number, leaseMs: number): void;
|
|
52
|
+
markJobDone(id: number, result?: unknown): void;
|
|
49
53
|
markJobDead(id: number, error: string): void;
|
|
50
54
|
markJobFailed(id: number, error: string): void;
|
|
55
|
+
private fib;
|
|
51
56
|
updateProgress(id: number, progress: number): void;
|
|
52
57
|
createBatch(name: string, options?: BatchOptions): string;
|
|
53
58
|
addToBatch<K extends string & keyof TMap>(batchId: string, type: K, data: TMap[K], options?: AddJobOptions): number;
|
|
@@ -69,6 +74,13 @@ export declare class JobQueue<TMap extends JobMap = Record<string, unknown>> {
|
|
|
69
74
|
* Returns null if no such job exists (completed, dead-lettered, or never queued).
|
|
70
75
|
*/
|
|
71
76
|
getJobByUniqueKey(type: string, uniqueKey: string): Job | null;
|
|
77
|
+
getJobResult<T>(id: number): T | null;
|
|
78
|
+
cancelByUniqueKey(type: string, uniqueKey: string): boolean;
|
|
79
|
+
retryFailedJobsByType(type: string): number;
|
|
80
|
+
purgeExpiredJobs(): number;
|
|
81
|
+
use(fn: MiddlewareFn): void;
|
|
82
|
+
getJobGraph(rootId: number): JobGraphNode[];
|
|
83
|
+
mountAdminHandler(prefix?: string): (req: Request) => Promise<Response>;
|
|
72
84
|
close(): void;
|
|
73
85
|
private unblockDependents;
|
|
74
86
|
private handleBatchJobComplete;
|
package/dist/queue.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"queue.d.ts","sourceRoot":"","sources":["../src/queue.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"queue.d.ts","sourceRoot":"","sources":["../src/queue.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAGtC,OAAO,EAAE,eAAe,EAAE,MAAM,UAAU,CAAC;AAE3C,OAAO,KAAK,EACV,aAAa,EAEb,YAAY,EACZ,SAAS,EAET,GAAG,EACH,QAAQ,EAER,YAAY,EACZ,MAAM,EAEN,QAAQ,EAER,eAAe,EACf,YAAY,EACZ,eAAe,EACf,YAAY,EAEZ,aAAa,EACd,MAAM,SAAS,CAAC;AAEjB,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AA2DrC,qBAAa,QAAQ,CACnB,IAAI,SAAS,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAC7C,SAAQ,eAAe;IACvB,QAAQ,CAAC,EAAE,EAAE,QAAQ,CAAC;IACtB,QAAQ,CAAC,WAAW,EAAE,YAAY,EAAE,CAAM;IAE1C,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC;IAC/B,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAC;IACtC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC;IAC/B,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC;IAC/B,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAC;IACnC,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAC;IACpC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC;IAC9B,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC;IAChC,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAC;IACpC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC;IACjC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC;IACjC,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAC;IACrC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC;IAC/B,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAC;IACtC,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAC;IACpC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC;IAChC,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAC;IAErC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC;IAGhC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC;IACjC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC;IACjC,OAAO,CAAC,QAAQ,CAAC,yBAAyB,CAAC;IAC3C,OAAO,CAAC,QAAQ,CAAC,wBAAwB,CAAC;IAC1C,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC;IACjC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC;IACjC,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAC;gBAEzB,MAAM,EAAE,MAAM;IAiH1B,GAAG,CAAC,CAAC,SAAS,MAAM,GAAG,MAAM,IAAI,EAC/B,IAAI,EAAE,CAAC,EACP,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,EACb,OAAO,CAAC,EAAE,aAAa,GACtB,MAAM;IAIT,MAAM,CAAC,EAAE,EAAE,MAAM,GAAG,GAAG,GAAG,IAAI;IAK9B,QAAQ,IAAI,QAAQ;IAwBpB,aAAa,CAAC,OAAO,CAAC,EAAE;QACtB,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,MAAM,CAAC,EAAE,MAAM,CAAC;KACjB,GAAG,eAAe,CAAC,SAAS,CAAC;IAkC9B,QAAQ,CAAC,OAAO,CAAC,EAAE,eAAe,GAAG,eAAe,CAAC,GAAG,CAAC;IAkCzD,SAAS,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO;IAS9B,aAAa,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO;IASlC,aAAa,CAAC,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,GAAG,IAAI;IAQxE,WAAW,IAAI,MAAM,EAAE;IAOvB,cAAc,CAAC,WAAW,EAAE,MAAM,GAAG,MAAM;IAiC3C,eAAe,CAAC,WAAW,EAAE,MAAM,GAAG,MAAM;IAQ5C,KAAK,CAAC,OAAO,EAAE,YAAY,GAAG,MAAM;IAQpC,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,SAAU,GAAG,GAAG,GAAG,IAAI;IAqBzD,UAAU,CAAC,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,IAAI;IAS7C,WAAW,CAAC,EAAE,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,OAAO,GAAG,IAAI;IAgD/C,WAAW,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI;IAiC5C,aAAa,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI;IA2E9C,OAAO,CAAC,GAAG;IAUX,cAAc,CAAC,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,IAAI;IAYlD,WAAW,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,YAAY,GAAG,MAAM;IAWzD,UAAU,CAAC,CAAC,SAAS,MAAM,GAAG,MAAM,IAAI,EACtC,OAAO,EAAE,MAAM,EACf,IAAI,EAAE,CAAC,EACP,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,EACb,OAAO,CAAC,EAAE,aAAa,GACtB,MAAM;IAYT,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,QAAQ,GAAG,IAAI;IAO1C,WAAW,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI;IAMlC,YAAY,CAAC,CAAC,SAAS,MAAM,GAAG,MAAM,IAAI,EACxC,OAAO,EAAE,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG;QAAE,IAAI,EAAE,CAAC,CAAA;KAAE,GAC5C,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC;IAIrB,OAAO,CAAC,SAAS;IA8EjB;;;;;OAKG;IACH,kBAAkB,CAAC,WAAW,SAAU,GAAG,MAAM;IAgBjD;;;OAGG;IACH,iBAAiB,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,GAAG,GAAG,IAAI;IAY9D,YAAY,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,GAAG,CAAC,GAAG,IAAI;IAMrC,iBAAiB,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO;IAS3D,qBAAqB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM;IAgC3C,gBAAgB,IAAI,MAAM;IAS1B,GAAG,CAAC,EAAE,EAAE,YAAY,GAAG,IAAI;IAI3B,WAAW,CAAC,MAAM,EAAE,MAAM,GAAG,YAAY,EAAE;IAiD3C,iBAAiB,CAAC,MAAM,SAAK,GAAG,CAAC,GAAG,EAAE,OAAO,KAAK,OAAO,CAAC,QAAQ,CAAC;IAyFnE,KAAK,IAAI,IAAI;IAWb,OAAO,CAAC,iBAAiB;IAiBzB,OAAO,CAAC,sBAAsB;CAgE/B"}
|
package/dist/queue.js
CHANGED
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
import { Database } from 'bun:sqlite';
|
|
4
4
|
import { mkdirSync } from 'node:fs';
|
|
5
5
|
import { dirname } from 'node:path';
|
|
6
|
+
import { JobQueueEmitter } from './events';
|
|
6
7
|
import { applyPragmas, initializeSchema } from './schema';
|
|
7
8
|
import { nowISO } from './utils';
|
|
8
9
|
import { JobWorker } from './worker';
|
|
@@ -24,7 +25,11 @@ function toJob(row) {
|
|
|
24
25
|
error: row.error,
|
|
25
26
|
batchId: row.batch_id,
|
|
26
27
|
requestLog: row.request_log,
|
|
27
|
-
responseLog: row.response_log
|
|
28
|
+
responseLog: row.response_log,
|
|
29
|
+
uniqueKey: row.unique_key,
|
|
30
|
+
claimedUntil: row.claimed_until,
|
|
31
|
+
result: row.result ? JSON.parse(row.result) : null,
|
|
32
|
+
expireAt: row.expire_at
|
|
28
33
|
};
|
|
29
34
|
}
|
|
30
35
|
function toBatch(row) {
|
|
@@ -56,8 +61,9 @@ function toFailedJob(row) {
|
|
|
56
61
|
responseLog: row.response_log
|
|
57
62
|
};
|
|
58
63
|
}
|
|
59
|
-
export class JobQueue {
|
|
64
|
+
export class JobQueue extends JobQueueEmitter {
|
|
60
65
|
db;
|
|
66
|
+
middlewares = [];
|
|
61
67
|
insertJobStmt;
|
|
62
68
|
selectDedupedJobStmt;
|
|
63
69
|
insertDepStmt;
|
|
@@ -75,6 +81,7 @@ export class JobQueue {
|
|
|
75
81
|
countUnmetDepsStmt;
|
|
76
82
|
unblockJobStmt;
|
|
77
83
|
lastInsertRowIdStmt;
|
|
84
|
+
renewLeaseStmt;
|
|
78
85
|
// Batch statements
|
|
79
86
|
insertBatchStmt;
|
|
80
87
|
selectBatchStmt;
|
|
@@ -84,13 +91,14 @@ export class JobQueue {
|
|
|
84
91
|
cancelBatchStmt;
|
|
85
92
|
cancelBatchJobsStmt;
|
|
86
93
|
constructor(dbPath) {
|
|
94
|
+
super();
|
|
87
95
|
mkdirSync(dirname(dbPath), { recursive: true });
|
|
88
96
|
this.db = new Database(dbPath, { create: true });
|
|
89
97
|
applyPragmas(this.db);
|
|
90
98
|
initializeSchema(this.db);
|
|
91
99
|
this.insertJobStmt = this.db.query(`
|
|
92
|
-
INSERT OR IGNORE INTO jobs (type, data, status, priority, max_retries, run_at, batch_id, unique_key, backoff_config)
|
|
93
|
-
VALUES ($type, $data, $status, $priority, $maxRetries, $runAt, $batchId, $uniqueKey, $backoffConfig)
|
|
100
|
+
INSERT OR IGNORE INTO jobs (type, data, status, priority, max_retries, run_at, batch_id, unique_key, backoff_config, expire_at, webhook_config)
|
|
101
|
+
VALUES ($type, $data, $status, $priority, $maxRetries, $runAt, $batchId, $uniqueKey, $backoffConfig, $expireAt, $webhookConfig)
|
|
94
102
|
`);
|
|
95
103
|
this.selectDedupedJobStmt = this.db.query(`
|
|
96
104
|
SELECT id FROM jobs
|
|
@@ -104,16 +112,21 @@ export class JobQueue {
|
|
|
104
112
|
this.selectJobStmt = this.db.query('SELECT * FROM jobs WHERE id = $id');
|
|
105
113
|
this.selectPendingStmt = this.db.query(`
|
|
106
114
|
SELECT * FROM jobs
|
|
107
|
-
WHERE
|
|
115
|
+
WHERE type = $type AND run_at <= $now
|
|
116
|
+
AND (expire_at IS NULL OR expire_at > $now)
|
|
117
|
+
AND (
|
|
118
|
+
(status = 'pending')
|
|
119
|
+
OR (status = 'processing' AND claimed_until IS NOT NULL AND claimed_until < $now)
|
|
120
|
+
)
|
|
108
121
|
ORDER BY priority DESC, created_at ASC
|
|
109
122
|
LIMIT 1
|
|
110
123
|
`);
|
|
111
124
|
this.markProcessingStmt = this.db.query(`
|
|
112
|
-
UPDATE jobs SET status = 'processing', started_at = $now, updated_at = $now
|
|
125
|
+
UPDATE jobs SET status = 'processing', started_at = $now, updated_at = $now, claimed_until = $claimedUntil
|
|
113
126
|
WHERE id = $id
|
|
114
127
|
`);
|
|
115
128
|
this.markDoneStmt = this.db.query(`
|
|
116
|
-
UPDATE jobs SET status = 'done', completed_at = $now, updated_at = $now, progress = 100
|
|
129
|
+
UPDATE jobs SET status = 'done', completed_at = $now, updated_at = $now, progress = 100, result = $result
|
|
117
130
|
WHERE id = $id
|
|
118
131
|
`);
|
|
119
132
|
this.markFailedStmt = this.db.query(`
|
|
@@ -148,6 +161,10 @@ export class JobQueue {
|
|
|
148
161
|
WHERE id = $id AND status = 'blocked'
|
|
149
162
|
`);
|
|
150
163
|
this.lastInsertRowIdStmt = this.db.query('SELECT last_insert_rowid() as id');
|
|
164
|
+
this.renewLeaseStmt = this.db.query(`
|
|
165
|
+
UPDATE jobs SET claimed_until = $claimedUntil, updated_at = $now
|
|
166
|
+
WHERE id = $id AND status = 'processing'
|
|
167
|
+
`);
|
|
151
168
|
// Batch statements
|
|
152
169
|
this.insertBatchStmt = this.db.query(`
|
|
153
170
|
INSERT INTO job_batches (id, name, options, created_at)
|
|
@@ -289,7 +306,9 @@ export class JobQueue {
|
|
|
289
306
|
$runAt: now,
|
|
290
307
|
$batchId: null,
|
|
291
308
|
$uniqueKey: null,
|
|
292
|
-
$backoffConfig: null
|
|
309
|
+
$backoffConfig: null,
|
|
310
|
+
$expireAt: null,
|
|
311
|
+
$webhookConfig: null
|
|
293
312
|
});
|
|
294
313
|
const newJobId = this.lastInsertRowIdStmt.get();
|
|
295
314
|
this.db
|
|
@@ -311,8 +330,9 @@ export class JobQueue {
|
|
|
311
330
|
.run({ $status: options.status, $cutoff: cutoff });
|
|
312
331
|
return result.changes;
|
|
313
332
|
}
|
|
314
|
-
pollAndClaim(type) {
|
|
333
|
+
pollAndClaim(type, leaseMs = 300_000) {
|
|
315
334
|
const now = nowISO();
|
|
335
|
+
const claimedUntil = new Date(Date.now() + leaseMs).toISOString();
|
|
316
336
|
const claimTx = this.db.transaction(() => {
|
|
317
337
|
const row = this.selectPendingStmt.get({
|
|
318
338
|
$type: type,
|
|
@@ -320,28 +340,72 @@ export class JobQueue {
|
|
|
320
340
|
});
|
|
321
341
|
if (!row)
|
|
322
342
|
return null;
|
|
323
|
-
this.markProcessingStmt.run({
|
|
343
|
+
this.markProcessingStmt.run({
|
|
344
|
+
$id: row.id,
|
|
345
|
+
$now: now,
|
|
346
|
+
$claimedUntil: claimedUntil
|
|
347
|
+
});
|
|
324
348
|
return row;
|
|
325
349
|
});
|
|
326
350
|
const row = claimTx.immediate();
|
|
327
351
|
return row ? toJob(row) : null;
|
|
328
352
|
}
|
|
329
|
-
|
|
353
|
+
renewLease(id, leaseMs) {
|
|
354
|
+
const claimedUntil = new Date(Date.now() + leaseMs).toISOString();
|
|
355
|
+
this.renewLeaseStmt.run({
|
|
356
|
+
$id: id,
|
|
357
|
+
$claimedUntil: claimedUntil,
|
|
358
|
+
$now: nowISO()
|
|
359
|
+
});
|
|
360
|
+
}
|
|
361
|
+
markJobDone(id, result) {
|
|
362
|
+
let doneJob = null;
|
|
363
|
+
// Use container object so TypeScript tracks mutation across the closure
|
|
364
|
+
const wh = { config: null };
|
|
330
365
|
this.db.transaction(() => {
|
|
331
366
|
const now = nowISO();
|
|
332
367
|
const row = this.selectJobStmt.get({ $id: id });
|
|
333
|
-
|
|
368
|
+
if (row?.webhook_config) {
|
|
369
|
+
try {
|
|
370
|
+
wh.config = JSON.parse(row.webhook_config);
|
|
371
|
+
}
|
|
372
|
+
catch {
|
|
373
|
+
// ignore malformed config
|
|
374
|
+
}
|
|
375
|
+
}
|
|
376
|
+
this.markDoneStmt.run({
|
|
377
|
+
$id: id,
|
|
378
|
+
$now: now,
|
|
379
|
+
$result: result !== undefined ? JSON.stringify(result) : null
|
|
380
|
+
});
|
|
334
381
|
this.unblockDependents(id);
|
|
335
382
|
if (row?.batch_id) {
|
|
336
383
|
this.handleBatchJobComplete(row.batch_id);
|
|
337
384
|
}
|
|
385
|
+
const updatedRow = this.selectJobStmt.get({ $id: id });
|
|
386
|
+
if (updatedRow)
|
|
387
|
+
doneJob = toJob(updatedRow);
|
|
338
388
|
})();
|
|
389
|
+
if (doneJob)
|
|
390
|
+
this.emit('job:done', doneJob);
|
|
391
|
+
if (wh.config && doneJob) {
|
|
392
|
+
const cfg = wh.config;
|
|
393
|
+
const payload = doneJob;
|
|
394
|
+
void fetch(cfg.url, {
|
|
395
|
+
method: cfg.method ?? 'POST',
|
|
396
|
+
headers: { 'Content-Type': 'application/json', ...(cfg.headers ?? {}) },
|
|
397
|
+
body: JSON.stringify({ job: payload, result })
|
|
398
|
+
}).catch(() => { });
|
|
399
|
+
}
|
|
339
400
|
}
|
|
340
401
|
markJobDead(id, error) {
|
|
402
|
+
let deadJob = null;
|
|
341
403
|
this.db.transaction(() => {
|
|
342
404
|
const row = this.selectJobStmt.get({ $id: id });
|
|
343
405
|
if (!row)
|
|
344
406
|
return;
|
|
407
|
+
// capture before delete
|
|
408
|
+
deadJob = toJob(row);
|
|
345
409
|
this.insertFailedJobStmt.run({
|
|
346
410
|
$originalJobId: row.id,
|
|
347
411
|
$type: row.type,
|
|
@@ -362,8 +426,11 @@ export class JobQueue {
|
|
|
362
426
|
this.handleBatchJobComplete(row.batch_id);
|
|
363
427
|
}
|
|
364
428
|
})();
|
|
429
|
+
if (deadJob)
|
|
430
|
+
this.emit('job:dead', deadJob, error);
|
|
365
431
|
}
|
|
366
432
|
markJobFailed(id, error) {
|
|
433
|
+
let failedJob = null;
|
|
367
434
|
this.db.transaction(() => {
|
|
368
435
|
const now = nowISO();
|
|
369
436
|
const row = this.selectJobStmt.get({ $id: id });
|
|
@@ -397,9 +464,20 @@ export class JobQueue {
|
|
|
397
464
|
: null;
|
|
398
465
|
let retryRunAt = now;
|
|
399
466
|
if (backoff) {
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
:
|
|
467
|
+
let delayMs;
|
|
468
|
+
switch (backoff.type) {
|
|
469
|
+
case 'exponential':
|
|
470
|
+
delayMs = Math.min(backoff.delayMs * 2 ** row.retry_count, 3_600_000);
|
|
471
|
+
break;
|
|
472
|
+
case 'jitter':
|
|
473
|
+
delayMs = Math.min(backoff.delayMs * 2 ** row.retry_count * (0.5 + Math.random()), 3_600_000);
|
|
474
|
+
break;
|
|
475
|
+
case 'fibonacci':
|
|
476
|
+
delayMs = Math.min(backoff.delayMs * this.fib(row.retry_count), 3_600_000);
|
|
477
|
+
break;
|
|
478
|
+
default: // 'fixed'
|
|
479
|
+
delayMs = backoff.delayMs;
|
|
480
|
+
}
|
|
403
481
|
retryRunAt = new Date(Date.now() + delayMs).toISOString();
|
|
404
482
|
}
|
|
405
483
|
this.markFailedStmt.run({
|
|
@@ -408,8 +486,23 @@ export class JobQueue {
|
|
|
408
486
|
$runAt: retryRunAt,
|
|
409
487
|
$now: now
|
|
410
488
|
});
|
|
489
|
+
// capture updated job for post-tx emit
|
|
490
|
+
const updatedRow = this.selectJobStmt.get({ $id: id });
|
|
491
|
+
if (updatedRow)
|
|
492
|
+
failedJob = toJob(updatedRow);
|
|
411
493
|
}
|
|
412
494
|
})();
|
|
495
|
+
if (failedJob)
|
|
496
|
+
this.emit('job:failed', failedJob, error);
|
|
497
|
+
}
|
|
498
|
+
fib(n) {
|
|
499
|
+
if (n <= 1)
|
|
500
|
+
return 1;
|
|
501
|
+
let a = 1, b = 1;
|
|
502
|
+
for (let i = 2; i <= n; i++) {
|
|
503
|
+
[a, b] = [b, a + b];
|
|
504
|
+
}
|
|
505
|
+
return b;
|
|
413
506
|
}
|
|
414
507
|
updateProgress(id, progress) {
|
|
415
508
|
this.updateProgressStmt.run({
|
|
@@ -417,6 +510,9 @@ export class JobQueue {
|
|
|
417
510
|
$progress: progress,
|
|
418
511
|
$now: nowISO()
|
|
419
512
|
});
|
|
513
|
+
const row = this.selectJobStmt.get({ $id: id });
|
|
514
|
+
if (row)
|
|
515
|
+
this.emit('job:progress', toJob(row), progress);
|
|
420
516
|
}
|
|
421
517
|
// --- Batch API ---
|
|
422
518
|
createBatch(name, options) {
|
|
@@ -455,6 +551,28 @@ export class JobQueue {
|
|
|
455
551
|
const runAt = options?.runAt ? options.runAt.toISOString() : now;
|
|
456
552
|
const hasDeps = options?.dependsOn && options.dependsOn.length > 0;
|
|
457
553
|
const status = hasDeps ? 'blocked' : 'pending';
|
|
554
|
+
const expireAt = options?.expireAt ? options.expireAt.toISOString() : null;
|
|
555
|
+
const webhookConfig = options?.onComplete
|
|
556
|
+
? JSON.stringify(options.onComplete)
|
|
557
|
+
: null;
|
|
558
|
+
// dedup='replace': update existing pending job's data + run_at
|
|
559
|
+
if (options?.dedup === 'replace' && options.uniqueKey) {
|
|
560
|
+
const existing = this.selectDedupedJobStmt.get({
|
|
561
|
+
$type: type,
|
|
562
|
+
$uniqueKey: options.uniqueKey
|
|
563
|
+
});
|
|
564
|
+
if (existing) {
|
|
565
|
+
this.db
|
|
566
|
+
.query('UPDATE jobs SET data = $data, run_at = $runAt, updated_at = $now WHERE id = $id')
|
|
567
|
+
.run({
|
|
568
|
+
$id: existing.id,
|
|
569
|
+
$data: JSON.stringify(data),
|
|
570
|
+
$runAt: runAt,
|
|
571
|
+
$now: now
|
|
572
|
+
});
|
|
573
|
+
return existing.id;
|
|
574
|
+
}
|
|
575
|
+
}
|
|
458
576
|
const result = this.insertJobStmt.run({
|
|
459
577
|
$type: type,
|
|
460
578
|
$data: JSON.stringify(data),
|
|
@@ -464,7 +582,9 @@ export class JobQueue {
|
|
|
464
582
|
$runAt: runAt,
|
|
465
583
|
$batchId: batchId,
|
|
466
584
|
$uniqueKey: options?.uniqueKey ?? null,
|
|
467
|
-
$backoffConfig: options?.backoff ? JSON.stringify(options.backoff) : null
|
|
585
|
+
$backoffConfig: options?.backoff ? JSON.stringify(options.backoff) : null,
|
|
586
|
+
$expireAt: expireAt,
|
|
587
|
+
$webhookConfig: webhookConfig
|
|
468
588
|
});
|
|
469
589
|
// INSERT OR IGNORE: if a pending/processing job with same (type, uniqueKey)
|
|
470
590
|
// already exists, the insert is a no-op. Return the existing job id.
|
|
@@ -505,7 +625,10 @@ export class JobQueue {
|
|
|
505
625
|
updated_at = $now
|
|
506
626
|
WHERE status = 'processing' AND updated_at < $cutoff`)
|
|
507
627
|
.run({ $now: nowISO(), $cutoff: cutoff });
|
|
508
|
-
|
|
628
|
+
const count = result.changes;
|
|
629
|
+
if (count > 0)
|
|
630
|
+
this.emit('job:stale', count);
|
|
631
|
+
return count;
|
|
509
632
|
}
|
|
510
633
|
/**
|
|
511
634
|
* Look up a pending or processing job by its uniqueKey.
|
|
@@ -520,6 +643,163 @@ export class JobQueue {
|
|
|
520
643
|
.get({ $type: type, $uniqueKey: uniqueKey });
|
|
521
644
|
return row ? toJob(row) : null;
|
|
522
645
|
}
|
|
646
|
+
getJobResult(id) {
|
|
647
|
+
const row = this.selectJobStmt.get({ $id: id });
|
|
648
|
+
if (!row?.result)
|
|
649
|
+
return null;
|
|
650
|
+
return JSON.parse(row.result);
|
|
651
|
+
}
|
|
652
|
+
cancelByUniqueKey(type, uniqueKey) {
|
|
653
|
+
const result = this.db
|
|
654
|
+
.query("UPDATE jobs SET status = 'cancelled', updated_at = $now WHERE type = $type AND unique_key = $uniqueKey AND status IN ('pending', 'blocked')")
|
|
655
|
+
.run({ $type: type, $uniqueKey: uniqueKey, $now: nowISO() });
|
|
656
|
+
return result.changes > 0;
|
|
657
|
+
}
|
|
658
|
+
retryFailedJobsByType(type) {
|
|
659
|
+
const rows = this.db
|
|
660
|
+
.query('SELECT * FROM failed_jobs WHERE type = $type')
|
|
661
|
+
.all({ $type: type });
|
|
662
|
+
if (rows.length === 0)
|
|
663
|
+
return 0;
|
|
664
|
+
const now = nowISO();
|
|
665
|
+
this.db.transaction(() => {
|
|
666
|
+
for (const row of rows) {
|
|
667
|
+
this.insertJobStmt.run({
|
|
668
|
+
$type: row.type,
|
|
669
|
+
$data: row.data,
|
|
670
|
+
$status: 'pending',
|
|
671
|
+
$priority: 0,
|
|
672
|
+
$maxRetries: row.max_retries,
|
|
673
|
+
$runAt: now,
|
|
674
|
+
$batchId: null,
|
|
675
|
+
$uniqueKey: null,
|
|
676
|
+
$backoffConfig: null,
|
|
677
|
+
$expireAt: null,
|
|
678
|
+
$webhookConfig: null
|
|
679
|
+
});
|
|
680
|
+
this.db
|
|
681
|
+
.query('DELETE FROM failed_jobs WHERE id = $id')
|
|
682
|
+
.run({ $id: row.id });
|
|
683
|
+
}
|
|
684
|
+
})();
|
|
685
|
+
return rows.length;
|
|
686
|
+
}
|
|
687
|
+
purgeExpiredJobs() {
|
|
688
|
+
const result = this.db
|
|
689
|
+
.query("DELETE FROM jobs WHERE expire_at IS NOT NULL AND expire_at <= $now AND status = 'pending'")
|
|
690
|
+
.run({ $now: nowISO() });
|
|
691
|
+
return result.changes;
|
|
692
|
+
}
|
|
693
|
+
use(fn) {
|
|
694
|
+
this.middlewares.push(fn);
|
|
695
|
+
}
|
|
696
|
+
getJobGraph(rootId) {
|
|
697
|
+
const relatedRows = this.db
|
|
698
|
+
.query(`WITH RECURSIVE
|
|
699
|
+
ancestors(id) AS (
|
|
700
|
+
SELECT depends_on_id FROM job_dependencies WHERE job_id = $root
|
|
701
|
+
UNION
|
|
702
|
+
SELECT jd.depends_on_id FROM job_dependencies jd JOIN ancestors a ON jd.job_id = a.id
|
|
703
|
+
),
|
|
704
|
+
descendants(id) AS (
|
|
705
|
+
SELECT job_id FROM job_dependencies WHERE depends_on_id = $root
|
|
706
|
+
UNION
|
|
707
|
+
SELECT jd.job_id FROM job_dependencies jd JOIN descendants d ON jd.depends_on_id = d.id
|
|
708
|
+
)
|
|
709
|
+
SELECT * FROM jobs
|
|
710
|
+
WHERE id IN (SELECT id FROM ancestors UNION SELECT $root UNION SELECT id FROM descendants)`)
|
|
711
|
+
.all({ $root: rootId });
|
|
712
|
+
if (relatedRows.length === 0)
|
|
713
|
+
return [];
|
|
714
|
+
const ids = relatedRows.map(r => r.id);
|
|
715
|
+
const namedParams = {};
|
|
716
|
+
for (let i = 0; i < ids.length; i++)
|
|
717
|
+
namedParams[`$id${i}`] = ids[i];
|
|
718
|
+
const ph = ids.map((_, i) => `$id${i}`).join(',');
|
|
719
|
+
const edges = this.db
|
|
720
|
+
.query(`SELECT job_id, depends_on_id FROM job_dependencies
|
|
721
|
+
WHERE job_id IN (${ph}) OR depends_on_id IN (${ph})`)
|
|
722
|
+
.all(namedParams);
|
|
723
|
+
return relatedRows.map(row => ({
|
|
724
|
+
id: row.id,
|
|
725
|
+
type: row.type,
|
|
726
|
+
status: row.status,
|
|
727
|
+
result: row.result ? JSON.parse(row.result) : null,
|
|
728
|
+
dependsOn: edges
|
|
729
|
+
.filter(e => e.job_id === row.id)
|
|
730
|
+
.map(e => e.depends_on_id),
|
|
731
|
+
dependents: edges
|
|
732
|
+
.filter(e => e.depends_on_id === row.id)
|
|
733
|
+
.map(e => e.job_id)
|
|
734
|
+
}));
|
|
735
|
+
}
|
|
736
|
+
mountAdminHandler(prefix = '') {
|
|
737
|
+
const base = prefix.replace(/\/$/, '');
|
|
738
|
+
return async (req) => {
|
|
739
|
+
const url = new URL(req.url);
|
|
740
|
+
const path = url.pathname.slice(base.length).replace(/^\//, '');
|
|
741
|
+
const parts = path.split('/').filter(Boolean);
|
|
742
|
+
const method = req.method.toUpperCase();
|
|
743
|
+
try {
|
|
744
|
+
if (method === 'GET' && parts[0] === 'stats' && parts.length === 1) {
|
|
745
|
+
return Response.json(this.getStats());
|
|
746
|
+
}
|
|
747
|
+
if (method === 'GET' && parts[0] === 'jobs' && parts[1] === 'types') {
|
|
748
|
+
return Response.json(this.getJobTypes());
|
|
749
|
+
}
|
|
750
|
+
if (method === 'GET' && parts[0] === 'jobs' && parts.length === 1) {
|
|
751
|
+
const status = url.searchParams.get('status');
|
|
752
|
+
const type = url.searchParams.get('type') ?? undefined;
|
|
753
|
+
const limit = Number(url.searchParams.get('limit') ?? 50);
|
|
754
|
+
const offset = Number(url.searchParams.get('offset') ?? 0);
|
|
755
|
+
return Response.json(this.listJobs({ status: status ?? undefined, type, limit, offset }));
|
|
756
|
+
}
|
|
757
|
+
if (method === 'GET' && parts[0] === 'jobs' && parts.length === 2) {
|
|
758
|
+
const id = Number(parts[1]);
|
|
759
|
+
const job = this.getJob(id);
|
|
760
|
+
if (!job)
|
|
761
|
+
return Response.json({ error: 'Not found' }, { status: 404 });
|
|
762
|
+
return Response.json(job);
|
|
763
|
+
}
|
|
764
|
+
if (method === 'GET' && parts[0] === 'jobs' && parts[2] === 'graph') {
|
|
765
|
+
return Response.json(this.getJobGraph(Number(parts[1])));
|
|
766
|
+
}
|
|
767
|
+
if (method === 'POST' && parts[0] === 'jobs' && parts[2] === 'cancel') {
|
|
768
|
+
return Response.json({ ok: this.cancelJob(Number(parts[1])) });
|
|
769
|
+
}
|
|
770
|
+
if (method === 'POST' &&
|
|
771
|
+
parts[0] === 'jobs' &&
|
|
772
|
+
parts[2] === 'force-retry') {
|
|
773
|
+
return Response.json({ ok: this.forceRetryJob(Number(parts[1])) });
|
|
774
|
+
}
|
|
775
|
+
if (method === 'GET' && parts[0] === 'failed' && parts.length === 1) {
|
|
776
|
+
const type = url.searchParams.get('type') ?? undefined;
|
|
777
|
+
const limit = Number(url.searchParams.get('limit') ?? 50);
|
|
778
|
+
const offset = Number(url.searchParams.get('offset') ?? 0);
|
|
779
|
+
return Response.json(this.getFailedJobs({ type, limit, offset }));
|
|
780
|
+
}
|
|
781
|
+
if (method === 'POST' &&
|
|
782
|
+
parts[0] === 'failed' &&
|
|
783
|
+
parts[1] === 'retry-by-type') {
|
|
784
|
+
const body = (await req.json());
|
|
785
|
+
if (!body.type)
|
|
786
|
+
return Response.json({ error: 'type required' }, { status: 400 });
|
|
787
|
+
return Response.json({
|
|
788
|
+
count: this.retryFailedJobsByType(body.type)
|
|
789
|
+
});
|
|
790
|
+
}
|
|
791
|
+
if (method === 'POST' &&
|
|
792
|
+
parts[0] === 'failed' &&
|
|
793
|
+
parts[2] === 'retry') {
|
|
794
|
+
return Response.json({ id: this.retryFailedJob(Number(parts[1])) });
|
|
795
|
+
}
|
|
796
|
+
return Response.json({ error: 'Not found' }, { status: 404 });
|
|
797
|
+
}
|
|
798
|
+
catch (err) {
|
|
799
|
+
return Response.json({ error: err instanceof Error ? err.message : 'Internal error' }, { status: 500 });
|
|
800
|
+
}
|
|
801
|
+
};
|
|
802
|
+
}
|
|
523
803
|
close() {
|
|
524
804
|
try {
|
|
525
805
|
if (this.db.filename !== ':memory:' && this.db.filename !== '') {
|
|
@@ -558,35 +838,51 @@ export class JobQueue {
|
|
|
558
838
|
const options = batch.options
|
|
559
839
|
? JSON.parse(batch.options)
|
|
560
840
|
: null;
|
|
561
|
-
if (
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
841
|
+
if (options) {
|
|
842
|
+
// Enqueue "then" callback job only if zero failures
|
|
843
|
+
if (batch.failed_jobs === 0 && options.thenType) {
|
|
844
|
+
this.insertJobStmt.run({
|
|
845
|
+
$type: options.thenType,
|
|
846
|
+
$data: JSON.stringify(options.thenData ?? {}),
|
|
847
|
+
$status: 'pending',
|
|
848
|
+
$priority: 0,
|
|
849
|
+
$maxRetries: 3,
|
|
850
|
+
$runAt: now,
|
|
851
|
+
$batchId: null,
|
|
852
|
+
$uniqueKey: null,
|
|
853
|
+
$backoffConfig: null,
|
|
854
|
+
$expireAt: null,
|
|
855
|
+
$webhookConfig: null
|
|
856
|
+
});
|
|
857
|
+
}
|
|
858
|
+
// Enqueue "finally" callback job regardless of failures
|
|
859
|
+
if (options.finallyType) {
|
|
860
|
+
this.insertJobStmt.run({
|
|
861
|
+
$type: options.finallyType,
|
|
862
|
+
$data: JSON.stringify(options.finallyData ?? {}),
|
|
863
|
+
$status: 'pending',
|
|
864
|
+
$priority: 0,
|
|
865
|
+
$maxRetries: 3,
|
|
866
|
+
$runAt: now,
|
|
867
|
+
$batchId: null,
|
|
868
|
+
$uniqueKey: null,
|
|
869
|
+
$backoffConfig: null,
|
|
870
|
+
$expireAt: null,
|
|
871
|
+
$webhookConfig: null
|
|
872
|
+
});
|
|
873
|
+
}
|
|
576
874
|
}
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
$backoffConfig: null
|
|
589
|
-
});
|
|
875
|
+
const finishedBatch = this.selectBatchStmt.get({
|
|
876
|
+
$id: batchId
|
|
877
|
+
});
|
|
878
|
+
if (finishedBatch) {
|
|
879
|
+
const b = toBatch(finishedBatch);
|
|
880
|
+
if (b.failedJobs === 0) {
|
|
881
|
+
this.emit('batch:complete', b);
|
|
882
|
+
}
|
|
883
|
+
else {
|
|
884
|
+
this.emit('batch:failed', b);
|
|
885
|
+
}
|
|
590
886
|
}
|
|
591
887
|
}
|
|
592
888
|
}
|
package/dist/schema.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../src/schema.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../src/schema.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAwG3C,wBAAgB,YAAY,CAAC,EAAE,EAAE,QAAQ,GAAG,IAAI,CAY/C;AAED,wBAAgB,gBAAgB,CAAC,EAAE,EAAE,QAAQ,GAAG,IAAI,CAkCnD"}
|
package/dist/schema.js
CHANGED
|
@@ -18,7 +18,11 @@ CREATE TABLE IF NOT EXISTS jobs (
|
|
|
18
18
|
response_log TEXT,
|
|
19
19
|
batch_id TEXT REFERENCES job_batches(id),
|
|
20
20
|
unique_key TEXT,
|
|
21
|
-
backoff_config TEXT
|
|
21
|
+
backoff_config TEXT,
|
|
22
|
+
claimed_until TEXT,
|
|
23
|
+
result TEXT,
|
|
24
|
+
expire_at TEXT,
|
|
25
|
+
webhook_config TEXT
|
|
22
26
|
)`;
|
|
23
27
|
// State-aware dedup: same (type, unique_key) cannot be both pending/processing at once.
|
|
24
28
|
// Once the job completes, the same key can be re-enqueued.
|
|
@@ -116,5 +120,17 @@ export function initializeSchema(db) {
|
|
|
116
120
|
if (!cols.some(c => c.name === 'backoff_config')) {
|
|
117
121
|
db.run('ALTER TABLE jobs ADD COLUMN backoff_config TEXT');
|
|
118
122
|
}
|
|
123
|
+
if (!cols.some(c => c.name === 'claimed_until')) {
|
|
124
|
+
db.run('ALTER TABLE jobs ADD COLUMN claimed_until TEXT');
|
|
125
|
+
}
|
|
126
|
+
if (!cols.some(c => c.name === 'result')) {
|
|
127
|
+
db.run('ALTER TABLE jobs ADD COLUMN result TEXT');
|
|
128
|
+
}
|
|
129
|
+
if (!cols.some(c => c.name === 'expire_at')) {
|
|
130
|
+
db.run('ALTER TABLE jobs ADD COLUMN expire_at TEXT');
|
|
131
|
+
}
|
|
132
|
+
if (!cols.some(c => c.name === 'webhook_config')) {
|
|
133
|
+
db.run('ALTER TABLE jobs ADD COLUMN webhook_config TEXT');
|
|
134
|
+
}
|
|
119
135
|
db.run(JOBS_UNIQUE_KEY_INDEX);
|
|
120
136
|
}
|
package/dist/types.d.ts
CHANGED
|
@@ -27,6 +27,10 @@ export type Job<T = unknown> = {
|
|
|
27
27
|
readonly batchId: string | null;
|
|
28
28
|
readonly requestLog: string | null;
|
|
29
29
|
readonly responseLog: string | null;
|
|
30
|
+
readonly uniqueKey: string | null;
|
|
31
|
+
readonly claimedUntil: string | null;
|
|
32
|
+
readonly result: unknown | null;
|
|
33
|
+
readonly expireAt: string | null;
|
|
30
34
|
};
|
|
31
35
|
export type FailedJob = {
|
|
32
36
|
readonly id: number;
|
|
@@ -42,7 +46,7 @@ export type FailedJob = {
|
|
|
42
46
|
readonly responseLog: string | null;
|
|
43
47
|
};
|
|
44
48
|
export type BackoffConfig = {
|
|
45
|
-
type: 'exponential' | 'fixed';
|
|
49
|
+
type: 'exponential' | 'fixed' | 'jitter' | 'fibonacci';
|
|
46
50
|
/** Base delay in ms. Exponential: delayMs * 2^retryCount. Fixed: always delayMs. Max 1h. */
|
|
47
51
|
delayMs: number;
|
|
48
52
|
};
|
|
@@ -62,18 +66,35 @@ export type AddJobOptions = {
|
|
|
62
66
|
* Fixed: always delayMs between retries. Default: retry immediately.
|
|
63
67
|
*/
|
|
64
68
|
backoff?: BackoffConfig;
|
|
69
|
+
dedup?: 'ignore' | 'replace';
|
|
70
|
+
expireAt?: Date;
|
|
71
|
+
onComplete?: {
|
|
72
|
+
url: string;
|
|
73
|
+
method?: string;
|
|
74
|
+
headers?: Record<string, string>;
|
|
75
|
+
};
|
|
65
76
|
};
|
|
66
77
|
export type JobContext = {
|
|
67
78
|
reportProgress: (percent: number) => void;
|
|
68
79
|
signal: AbortSignal;
|
|
80
|
+
renewLease(): void;
|
|
69
81
|
};
|
|
70
82
|
export type RateLimit = {
|
|
71
83
|
count: number;
|
|
72
84
|
windowMs: number;
|
|
73
85
|
};
|
|
86
|
+
export type MiddlewareFn<T = unknown> = (job: Job<T>, next: () => Promise<unknown>) => Promise<unknown>;
|
|
87
|
+
export type JobGraphNode = {
|
|
88
|
+
readonly id: number;
|
|
89
|
+
readonly type: string;
|
|
90
|
+
readonly status: JobStatus;
|
|
91
|
+
readonly result: unknown | null;
|
|
92
|
+
readonly dependsOn: number[];
|
|
93
|
+
readonly dependents: number[];
|
|
94
|
+
};
|
|
74
95
|
export type WorkerOptions<T = unknown> = {
|
|
75
96
|
type: string;
|
|
76
|
-
handler: (job: Job<T>, ctx: JobContext) => Promise<
|
|
97
|
+
handler: (job: Job<T>, ctx: JobContext) => Promise<unknown>;
|
|
77
98
|
pollIntervalMs?: number;
|
|
78
99
|
maxRate?: RateLimit;
|
|
79
100
|
onError?: (job: Job<T>, error: unknown) => void;
|
|
@@ -81,6 +102,12 @@ export type WorkerOptions<T = unknown> = {
|
|
|
81
102
|
timeoutMs?: number;
|
|
82
103
|
/** Max concurrent jobs this worker runs simultaneously. Default: 1. */
|
|
83
104
|
concurrency?: number;
|
|
105
|
+
leaseMs?: number;
|
|
106
|
+
retryIf?: (error: unknown, job: Job<T>) => boolean;
|
|
107
|
+
aging?: {
|
|
108
|
+
boostPerMinute: number;
|
|
109
|
+
maxBoost: number;
|
|
110
|
+
};
|
|
84
111
|
};
|
|
85
112
|
export type JobStats = {
|
|
86
113
|
pending: number;
|
|
@@ -126,6 +153,10 @@ export type JobRow = {
|
|
|
126
153
|
response_log: string | null;
|
|
127
154
|
unique_key: string | null;
|
|
128
155
|
backoff_config: string | null;
|
|
156
|
+
claimed_until: string | null;
|
|
157
|
+
result: string | null;
|
|
158
|
+
expire_at: string | null;
|
|
159
|
+
webhook_config: string | null;
|
|
129
160
|
};
|
|
130
161
|
export type FailedJobRow = {
|
|
131
162
|
id: number;
|
package/dist/types.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAGA;;;;;GAKG;AACH,qBAAa,iBAAkB,SAAQ,KAAK;IAC1C,QAAQ,CAAC,cAAc,QAAQ;gBAEnB,OAAO,EAAE,MAAM;CAI5B;AAED,MAAM,MAAM,SAAS,GACjB,SAAS,GACT,YAAY,GACZ,MAAM,GACN,QAAQ,GACR,SAAS,GACT,WAAW,CAAC;AAEhB,MAAM,MAAM,GAAG,CAAC,CAAC,GAAG,OAAO,IAAI;IAC7B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;IACjB,QAAQ,CAAC,MAAM,EAAE,SAAS,CAAC;IAC3B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAGA;;;;;GAKG;AACH,qBAAa,iBAAkB,SAAQ,KAAK;IAC1C,QAAQ,CAAC,cAAc,QAAQ;gBAEnB,OAAO,EAAE,MAAM;CAI5B;AAED,MAAM,MAAM,SAAS,GACjB,SAAS,GACT,YAAY,GACZ,MAAM,GACN,QAAQ,GACR,SAAS,GACT,WAAW,CAAC;AAEhB,MAAM,MAAM,GAAG,CAAC,CAAC,GAAG,OAAO,IAAI;IAC7B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;IACjB,QAAQ,CAAC,MAAM,EAAE,SAAS,CAAC;IAC3B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,QAAQ,CAAC,MAAM,EAAE,OAAO,GAAG,IAAI,CAAC;IAChC,QAAQ,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;CAClC,CAAC;AAEF,MAAM,MAAM,SAAS,GAAG;IACtB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC;IACvB,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;CACrC,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG;IAC1B,IAAI,EAAE,aAAa,GAAG,OAAO,GAAG,QAAQ,GAAG,WAAW,CAAC;IACvD,4FAA4F;IAC5F,OAAO,EAAE,MAAM,CAAC;CACjB,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG;IAC1B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,IAAI,CAAC;IACb,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;OAGG;IACH,OAAO,CAAC,EAAE,aAAa,CAAC;IACxB,KAAK,CAAC,EAAE,QAAQ,GAAG,SAAS,CAAC;IAC7B,QAAQ,CAAC,EAAE,IAAI,CAAC;IAChB,UAAU,CAAC,EAAE;QACX,GAAG,EAAE,MAAM,CAAC;QACZ,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;KAClC,CAAC;CACH,CAAC;AAEF,MAAM,MAAM,UAAU,GAAG;IACvB,cAAc,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;IAC1C,MAAM,EAAE,WAAW,CAAC;IACpB,UAAU,IAAI,IAAI,CAAC;CACpB,CAAC;AAEF,MAAM,MAAM,SAAS,GAAG;IACtB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF,MAAM,MAAM,YAAY,CAAC,CAAC,GAAG,OAAO,IAAI,CACtC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,EACX,IAAI,EAAE,MAAM,OAAO,CAAC,OAAO,CAAC,KACzB,OAAO,CAAC,OAAO,CAAC,CAAC;AAEtB,MAAM,MAAM,YAAY,GAAG;IACzB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,MAAM,EAAE,SAAS,CAAC;IAC3B,QAAQ,CAAC,MAAM,EAAE,OAAO,GAAG,IAAI,CAAC;IAChC,QAAQ,CAAC,SAAS,EAAE,MAAM,EAAE,CAAC;IAC7B,QAAQ,CAAC,UAAU,EAAE,MAAM,EAAE,CAAC;CAC/B,CAAC;AAEF,MAAM,MAAM,aAAa,CAAC,CAAC,GAAG,OAAO,IAAI;IACvC,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,UAAU,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;IAC5D,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,OAAO,CAAC,EAAE,SAAS,CAAC;IACpB,OAAO,CAAC,EAAE,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,KAAK,IAAI,CAAC;IAChD,sFAAsF;IACtF,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,uEAAuE;IACvE,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,OAAO,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,KAAK,OAAO,CAAC;IACnD,KAAK,CAAC,EAAE;QAAE,cAAc,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC;CACtD,CAAC;AAEF,MAAM,MAAM,QAAQ,GAAG;IACrB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,MAAM,CAAC;CACd,CAAC;AAEF,MAAM,MAAM,YAAY,GAAG;IACzB,MAAM,EAAE,MAAM,GAAG,QAAQ,CAAC;IAC1B,WAAW,EAAE,MAAM,CAAC;CACrB,CAAC;AAEF,MAAM,MAAM,eAAe,GAAG;IAC5B,MAAM,CAAC,EAAE,SAAS,CAAC;IACnB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB,CAAC;AAEF,MAAM,MAAM,eAAe,CAAC,CAAC,IAAI;IAC/B,KAAK,EAAE,CAAC,EAAE,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;CACf,CAAC;AAEF,MAAM,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;AAG7C,MAAM,MAAM,MAAM,GAAG;IACnB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;CAC/B,CAAC;AAEF,MAAM,MAAM,YAAY,GAAG;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,eAAe,EAAE,MAAM,CAAC;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;CAC7B,CAAC;AAEF,MAAM,MAAM,QAAQ,GAAG;IACrB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;CACf,CAAC;AAIF,MAAM,MAAM,YAAY,GAAG;IACzB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,OAAO,CAAC;CACvB,CAAC;AAEF,MAAM,MAAM,QAAQ,GAAG;IACrB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,YAAY,EAAE,MAAM,EAAE,CAAC;IAChC,QAAQ,CAAC,OAAO,EAAE,YAAY,GAAG,IAAI,CAAC;IACtC,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;CACpC,CAAC;AAEF,MAAM,MAAM,WAAW,GAAG;IACxB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;IACpB,cAAc,EAAE,MAAM,CAAC;IACvB,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;CAC5B,CAAC;AAIF,MAAM,MAAM,QAAQ,GAAG;IACrB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC;IACvB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;CAC5B,CAAC;AAEF,MAAM,MAAM,WAAW,GAAG;IACxB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,MAAM,MAAM,kBAAkB,GAAG;IAC/B,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB,CAAC"}
|
package/dist/worker.d.ts
CHANGED
|
@@ -7,14 +7,18 @@ export declare class JobWorker<TMap extends JobMap = Record<string, unknown>, K
|
|
|
7
7
|
private abortController;
|
|
8
8
|
private timer;
|
|
9
9
|
private running;
|
|
10
|
+
private paused;
|
|
10
11
|
private activeCount;
|
|
11
12
|
private stopResolve;
|
|
12
13
|
constructor(queue: JobQueue<TMap>, options: WorkerOptions<TMap[K]> & {
|
|
13
14
|
type: K;
|
|
14
15
|
});
|
|
15
16
|
get isRunning(): boolean;
|
|
17
|
+
get isPaused(): boolean;
|
|
16
18
|
start(): void;
|
|
17
19
|
stop(): Promise<void>;
|
|
20
|
+
pause(): void;
|
|
21
|
+
resume(): void;
|
|
18
22
|
private scheduleNext;
|
|
19
23
|
private poll;
|
|
20
24
|
private runJob;
|
package/dist/worker.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"worker.d.ts","sourceRoot":"","sources":["../src/worker.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAExC,OAAO,KAAK,
|
|
1
|
+
{"version":3,"file":"worker.d.ts","sourceRoot":"","sources":["../src/worker.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAExC,OAAO,KAAK,EAGV,MAAM,EAEN,aAAa,EACd,MAAM,SAAS,CAAC;AAIjB,qBAAa,SAAS,CACpB,IAAI,SAAS,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAC7C,CAAC,SAAS,MAAM,GAAG,MAAM,IAAI,GAAG,MAAM,GAAG,MAAM,IAAI;IAEnD,OAAO,CAAC,KAAK,CAAiB;IAC9B,OAAO,CAAC,OAAO,CAAuC;IACtD,OAAO,CAAC,WAAW,CAAkC;IACrD,OAAO,CAAC,eAAe,CAAgC;IACvD,OAAO,CAAC,KAAK,CAA8C;IAC3D,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,WAAW,CAAK;IACxB,OAAO,CAAC,WAAW,CAA6B;gBAG9C,KAAK,EAAE,QAAQ,CAAC,IAAI,CAAC,EACrB,OAAO,EAAE,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG;QAAE,IAAI,EAAE,CAAC,CAAA;KAAE;IAY/C,IAAI,SAAS,IAAI,OAAO,CAEvB;IAED,IAAI,QAAQ,IAAI,OAAO,CAEtB;IAED,KAAK,IAAI,IAAI;IAOP,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAkB3B,KAAK,IAAI,IAAI;IAIb,MAAM,IAAI,IAAI;IAYd,OAAO,CAAC,YAAY;YAMN,IAAI;YAoDJ,MAAM;CAwErB"}
|
package/dist/worker.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { SlidingWindowRateLimiter } from './rate-limiter';
|
|
2
2
|
import { NonRetryableError } from './types';
|
|
3
|
+
import { nowISO } from './utils';
|
|
3
4
|
export class JobWorker {
|
|
4
5
|
queue;
|
|
5
6
|
options;
|
|
@@ -7,6 +8,7 @@ export class JobWorker {
|
|
|
7
8
|
abortController = null;
|
|
8
9
|
timer = null;
|
|
9
10
|
running = false;
|
|
11
|
+
paused = false;
|
|
10
12
|
activeCount = 0;
|
|
11
13
|
stopResolve = null;
|
|
12
14
|
constructor(queue, options) {
|
|
@@ -19,6 +21,9 @@ export class JobWorker {
|
|
|
19
21
|
get isRunning() {
|
|
20
22
|
return this.running;
|
|
21
23
|
}
|
|
24
|
+
get isPaused() {
|
|
25
|
+
return this.paused;
|
|
26
|
+
}
|
|
22
27
|
start() {
|
|
23
28
|
if (this.running)
|
|
24
29
|
return;
|
|
@@ -41,6 +46,20 @@ export class JobWorker {
|
|
|
41
46
|
});
|
|
42
47
|
}
|
|
43
48
|
}
|
|
49
|
+
pause() {
|
|
50
|
+
this.paused = true;
|
|
51
|
+
}
|
|
52
|
+
resume() {
|
|
53
|
+
this.paused = false;
|
|
54
|
+
if (this.running) {
|
|
55
|
+
// Cancel existing timer and poll immediately
|
|
56
|
+
if (this.timer) {
|
|
57
|
+
clearTimeout(this.timer);
|
|
58
|
+
this.timer = null;
|
|
59
|
+
}
|
|
60
|
+
void this.poll();
|
|
61
|
+
}
|
|
62
|
+
}
|
|
44
63
|
scheduleNext() {
|
|
45
64
|
if (!this.running)
|
|
46
65
|
return;
|
|
@@ -50,18 +69,46 @@ export class JobWorker {
|
|
|
50
69
|
async poll() {
|
|
51
70
|
if (!this.running)
|
|
52
71
|
return;
|
|
72
|
+
if (this.paused) {
|
|
73
|
+
this.scheduleNext();
|
|
74
|
+
return;
|
|
75
|
+
}
|
|
53
76
|
const concurrency = this.options.concurrency ?? 1;
|
|
77
|
+
const leaseMs = this.options.leaseMs ?? 300_000;
|
|
54
78
|
// Drain available jobs up to available capacity in one poll tick
|
|
55
79
|
while (this.activeCount < concurrency) {
|
|
56
80
|
if (this.rateLimiter && !this.rateLimiter.canProceed())
|
|
57
81
|
break;
|
|
58
|
-
const job = this.queue.pollAndClaim(this.options.type);
|
|
82
|
+
const job = this.queue.pollAndClaim(this.options.type, leaseMs);
|
|
59
83
|
if (!job)
|
|
60
84
|
break;
|
|
61
85
|
this.rateLimiter?.record();
|
|
62
86
|
this.activeCount++;
|
|
63
87
|
void this.runJob(job);
|
|
64
88
|
}
|
|
89
|
+
// Apply priority aging if configured
|
|
90
|
+
if (this.options.aging) {
|
|
91
|
+
const { boostPerMinute, maxBoost } = this.options.aging;
|
|
92
|
+
const pollIntervalMs = this.options.pollIntervalMs ?? 1000;
|
|
93
|
+
const boostPerTick = (boostPerMinute * pollIntervalMs) / 60_000;
|
|
94
|
+
if (boostPerTick > 0) {
|
|
95
|
+
const cutoff = new Date(Date.now() - pollIntervalMs).toISOString();
|
|
96
|
+
this.queue.db.run(`UPDATE jobs
|
|
97
|
+
SET priority = MIN(priority + ?, ?),
|
|
98
|
+
updated_at = ?
|
|
99
|
+
WHERE status = 'pending'
|
|
100
|
+
AND type = ?
|
|
101
|
+
AND created_at < ?
|
|
102
|
+
AND priority < ?`, [
|
|
103
|
+
boostPerTick,
|
|
104
|
+
maxBoost,
|
|
105
|
+
nowISO(),
|
|
106
|
+
this.options.type,
|
|
107
|
+
cutoff,
|
|
108
|
+
maxBoost
|
|
109
|
+
]);
|
|
110
|
+
}
|
|
111
|
+
}
|
|
65
112
|
this.scheduleNext();
|
|
66
113
|
}
|
|
67
114
|
async runJob(job) {
|
|
@@ -70,20 +117,36 @@ export class JobWorker {
|
|
|
70
117
|
reportProgress: (percent) => {
|
|
71
118
|
this.queue.updateProgress(job.id, percent);
|
|
72
119
|
},
|
|
120
|
+
renewLease: () => {
|
|
121
|
+
this.queue.renewLease(job.id, this.options.leaseMs ?? 300_000);
|
|
122
|
+
},
|
|
73
123
|
signal: this.abortController.signal
|
|
74
124
|
};
|
|
75
|
-
const
|
|
125
|
+
const handler = this.options.handler;
|
|
126
|
+
const middlewares = this.queue.middlewares;
|
|
127
|
+
const chain = [...middlewares, (j) => handler(j, ctx)];
|
|
128
|
+
const execute = () => {
|
|
129
|
+
let i = 0;
|
|
130
|
+
const run = () => {
|
|
131
|
+
const mw = chain[i++];
|
|
132
|
+
if (!mw)
|
|
133
|
+
return Promise.resolve(undefined);
|
|
134
|
+
return mw(job, run);
|
|
135
|
+
};
|
|
136
|
+
return run();
|
|
137
|
+
};
|
|
138
|
+
let handlerResult;
|
|
76
139
|
if (this.options.timeoutMs) {
|
|
77
140
|
const timeoutMs = this.options.timeoutMs;
|
|
78
|
-
await Promise.race([
|
|
79
|
-
|
|
141
|
+
handlerResult = await Promise.race([
|
|
142
|
+
execute(),
|
|
80
143
|
new Promise((_, reject) => setTimeout(() => reject(new Error(`Job timed out after ${timeoutMs}ms`)), timeoutMs))
|
|
81
144
|
]);
|
|
82
145
|
}
|
|
83
146
|
else {
|
|
84
|
-
await
|
|
147
|
+
handlerResult = await execute();
|
|
85
148
|
}
|
|
86
|
-
this.queue.markJobDone(job.id);
|
|
149
|
+
this.queue.markJobDone(job.id, handlerResult);
|
|
87
150
|
}
|
|
88
151
|
catch (error) {
|
|
89
152
|
const message = error instanceof Error ? error.message : String(error);
|
|
@@ -93,7 +156,10 @@ export class JobWorker {
|
|
|
93
156
|
(typeof error === 'object' &&
|
|
94
157
|
error !== null &&
|
|
95
158
|
error.isNonRetryable === true);
|
|
96
|
-
|
|
159
|
+
const shouldRetry = this.options.retryIf
|
|
160
|
+
? this.options.retryIf(error, job)
|
|
161
|
+
: true;
|
|
162
|
+
if (isNonRetryable || !shouldRetry) {
|
|
97
163
|
this.queue.markJobDead(job.id, message);
|
|
98
164
|
}
|
|
99
165
|
else {
|