@bitclaw/jobs 1.2.0 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/events.d.ts +20 -0
- package/dist/events.d.ts.map +1 -0
- package/dist/events.js +33 -0
- package/dist/index.d.ts +3 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -0
- package/dist/queue.d.ts +23 -3
- package/dist/queue.d.ts.map +1 -1
- package/dist/queue.js +257 -40
- package/dist/schema.d.ts.map +1 -1
- package/dist/schema.js +23 -5
- package/dist/types.d.ts +36 -1
- package/dist/types.d.ts.map +1 -1
- package/dist/worker.d.ts +6 -1
- package/dist/worker.d.ts.map +1 -1
- package/dist/worker.js +49 -18
- package/package.json +1 -1
package/dist/events.d.ts
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import type { Job, JobBatch } from './types';
|
|
2
|
+
export type JobQueueEventMap = {
|
|
3
|
+
'job:done': (job: Job) => void;
|
|
4
|
+
'job:failed': (job: Job, error: string) => void;
|
|
5
|
+
'job:dead': (job: Job, error: string) => void;
|
|
6
|
+
'job:progress': (job: Job, progress: number) => void;
|
|
7
|
+
'job:stale': (count: number) => void;
|
|
8
|
+
'batch:complete': (batch: JobBatch) => void;
|
|
9
|
+
'batch:failed': (batch: JobBatch) => void;
|
|
10
|
+
};
|
|
11
|
+
type Handler<K extends keyof JobQueueEventMap> = JobQueueEventMap[K];
|
|
12
|
+
export declare class JobQueueEmitter {
|
|
13
|
+
private readonly listeners;
|
|
14
|
+
on<K extends keyof JobQueueEventMap>(event: K, handler: Handler<K>): () => void;
|
|
15
|
+
off<K extends keyof JobQueueEventMap>(event: K, handler: Handler<K>): void;
|
|
16
|
+
once<K extends keyof JobQueueEventMap>(event: K, handler: Handler<K>): void;
|
|
17
|
+
emit<K extends keyof JobQueueEventMap>(event: K, ...args: Parameters<JobQueueEventMap[K]>): void;
|
|
18
|
+
}
|
|
19
|
+
export {};
|
|
20
|
+
//# sourceMappingURL=events.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"events.d.ts","sourceRoot":"","sources":["../src/events.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,GAAG,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAE7C,MAAM,MAAM,gBAAgB,GAAG;IAC7B,UAAU,EAAE,CAAC,GAAG,EAAE,GAAG,KAAK,IAAI,CAAC;IAC/B,YAAY,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;IAChD,UAAU,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;IAC9C,cAAc,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,QAAQ,EAAE,MAAM,KAAK,IAAI,CAAC;IACrD,WAAW,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;IACrC,gBAAgB,EAAE,CAAC,KAAK,EAAE,QAAQ,KAAK,IAAI,CAAC;IAC5C,cAAc,EAAE,CAAC,KAAK,EAAE,QAAQ,KAAK,IAAI,CAAC;CAC3C,CAAC;AAEF,KAAK,OAAO,CAAC,CAAC,SAAS,MAAM,gBAAgB,IAAI,gBAAgB,CAAC,CAAC,CAAC,CAAC;AAErE,qBAAa,eAAe;IAC1B,OAAO,CAAC,QAAQ,CAAC,SAAS,CAGtB;IAEJ,EAAE,CAAC,CAAC,SAAS,MAAM,gBAAgB,EACjC,KAAK,EAAE,CAAC,EACR,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,GAClB,MAAM,IAAI;IAQb,GAAG,CAAC,CAAC,SAAS,MAAM,gBAAgB,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,GAAG,IAAI;IAI1E,IAAI,CAAC,CAAC,SAAS,MAAM,gBAAgB,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,GAAG,IAAI;IAQ3E,IAAI,CAAC,CAAC,SAAS,MAAM,gBAAgB,EACnC,KAAK,EAAE,CAAC,EACR,GAAG,IAAI,EAAE,UAAU,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,GACvC,IAAI;CAWR"}
|
package/dist/events.js
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
export class JobQueueEmitter {
|
|
2
|
+
listeners = new Map();
|
|
3
|
+
on(event, handler) {
|
|
4
|
+
if (!this.listeners.has(event)) {
|
|
5
|
+
this.listeners.set(event, new Set());
|
|
6
|
+
}
|
|
7
|
+
this.listeners.get(event).add(handler);
|
|
8
|
+
return () => this.off(event, handler);
|
|
9
|
+
}
|
|
10
|
+
off(event, handler) {
|
|
11
|
+
this.listeners.get(event)?.delete(handler);
|
|
12
|
+
}
|
|
13
|
+
once(event, handler) {
|
|
14
|
+
const wrapper = ((...args) => {
|
|
15
|
+
this.off(event, wrapper);
|
|
16
|
+
handler(...args);
|
|
17
|
+
});
|
|
18
|
+
this.on(event, wrapper);
|
|
19
|
+
}
|
|
20
|
+
emit(event, ...args) {
|
|
21
|
+
const handlers = this.listeners.get(event);
|
|
22
|
+
if (!handlers)
|
|
23
|
+
return;
|
|
24
|
+
for (const handler of handlers) {
|
|
25
|
+
try {
|
|
26
|
+
handler(...args);
|
|
27
|
+
}
|
|
28
|
+
catch {
|
|
29
|
+
// silently swallow listener errors
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
}
|
package/dist/index.d.ts
CHANGED
|
@@ -1,10 +1,12 @@
|
|
|
1
1
|
export type { ParsedCron } from './cron';
|
|
2
2
|
export { cronMatches, nextCronOccurrence, parseCron } from './cron';
|
|
3
|
+
export type { JobQueueEventMap } from './events';
|
|
4
|
+
export { JobQueueEmitter } from './events';
|
|
3
5
|
export { JobQueue } from './queue';
|
|
4
6
|
export { SlidingWindowRateLimiter } from './rate-limiter';
|
|
5
7
|
export { Scheduler } from './scheduler';
|
|
6
8
|
export { applyPragmas, initializeSchema } from './schema';
|
|
7
|
-
export type { AddJobOptions, AddScheduleOptions, BatchOptions, FailedJob, Job, JobBatch, JobContext, JobMap, JobStats, JobStatus, ListJobsOptions, PaginatedResult, PurgeOptions, RateLimit, Schedule, WorkerOptions } from './types';
|
|
9
|
+
export type { AddJobOptions, AddScheduleOptions, BackoffConfig, BatchOptions, FailedJob, Job, JobBatch, JobContext, JobMap, JobStats, JobStatus, ListJobsOptions, PaginatedResult, PurgeOptions, RateLimit, Schedule, WorkerOptions } from './types';
|
|
8
10
|
export { NonRetryableError } from './types';
|
|
9
11
|
export { JobWorker } from './worker';
|
|
10
12
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAGA,YAAY,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACzC,OAAO,EAAE,WAAW,EAAE,kBAAkB,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AACpE,OAAO,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AACnC,OAAO,EAAE,wBAAwB,EAAE,MAAM,gBAAgB,CAAC;AAC1D,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AACxC,OAAO,EAAE,YAAY,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC;AAC1D,YAAY,EACV,aAAa,EACb,kBAAkB,EAClB,YAAY,EACZ,SAAS,EACT,GAAG,EACH,QAAQ,EACR,UAAU,EACV,MAAM,EACN,QAAQ,EACR,SAAS,EACT,eAAe,EACf,eAAe,EACf,YAAY,EACZ,SAAS,EACT,QAAQ,EACR,aAAa,EACd,MAAM,SAAS,CAAC;AACjB,OAAO,EAAE,iBAAiB,EAAE,MAAM,SAAS,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAGA,YAAY,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACzC,OAAO,EAAE,WAAW,EAAE,kBAAkB,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AACpE,YAAY,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC;AACjD,OAAO,EAAE,eAAe,EAAE,MAAM,UAAU,CAAC;AAC3C,OAAO,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AACnC,OAAO,EAAE,wBAAwB,EAAE,MAAM,gBAAgB,CAAC;AAC1D,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AACxC,OAAO,EAAE,YAAY,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC;AAC1D,YAAY,EACV,aAAa,EACb,kBAAkB,EAClB,aAAa,EACb,YAAY,EACZ,SAAS,EACT,GAAG,EACH,QAAQ,EACR,UAAU,EACV,MAAM,EACN,QAAQ,EACR,SAAS,EACT,eAAe,EACf,eAAe,EACf,YAAY,EACZ,SAAS,EACT,QAAQ,EACR,aAAa,EACd,MAAM,SAAS,CAAC;AACjB,OAAO,EAAE,iBAAiB,EAAE,MAAM,SAAS,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
// packages/jobs/src/index.ts
|
|
2
2
|
// Barrel export for @bitclaw/jobs
|
|
3
3
|
export { cronMatches, nextCronOccurrence, parseCron } from './cron';
|
|
4
|
+
export { JobQueueEmitter } from './events';
|
|
4
5
|
export { JobQueue } from './queue';
|
|
5
6
|
export { SlidingWindowRateLimiter } from './rate-limiter';
|
|
6
7
|
export { Scheduler } from './scheduler';
|
package/dist/queue.d.ts
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import { Database } from 'bun:sqlite';
|
|
2
|
+
import { JobQueueEmitter } from './events';
|
|
2
3
|
import type { AddJobOptions, BatchOptions, FailedJob, Job, JobBatch, JobMap, JobStats, ListJobsOptions, PaginatedResult, PurgeOptions, WorkerOptions } from './types';
|
|
3
4
|
import { JobWorker } from './worker';
|
|
4
|
-
export declare class JobQueue<TMap extends JobMap = Record<string, unknown>> {
|
|
5
|
+
export declare class JobQueue<TMap extends JobMap = Record<string, unknown>> extends JobQueueEmitter {
|
|
5
6
|
readonly db: Database;
|
|
6
7
|
private readonly insertJobStmt;
|
|
7
8
|
private readonly selectDedupedJobStmt;
|
|
@@ -20,6 +21,7 @@ export declare class JobQueue<TMap extends JobMap = Record<string, unknown>> {
|
|
|
20
21
|
private readonly countUnmetDepsStmt;
|
|
21
22
|
private readonly unblockJobStmt;
|
|
22
23
|
private readonly lastInsertRowIdStmt;
|
|
24
|
+
private readonly renewLeaseStmt;
|
|
23
25
|
private readonly insertBatchStmt;
|
|
24
26
|
private readonly selectBatchStmt;
|
|
25
27
|
private readonly decrementBatchPendingStmt;
|
|
@@ -44,10 +46,12 @@ export declare class JobQueue<TMap extends JobMap = Record<string, unknown>> {
|
|
|
44
46
|
retryFailedJob(failedJobId: number): number;
|
|
45
47
|
purgeFailedJobs(olderThanMs: number): number;
|
|
46
48
|
purge(options: PurgeOptions): number;
|
|
47
|
-
pollAndClaim(type: string): Job | null;
|
|
48
|
-
|
|
49
|
+
pollAndClaim(type: string, leaseMs?: number): Job | null;
|
|
50
|
+
renewLease(id: number, leaseMs: number): void;
|
|
51
|
+
markJobDone(id: number, result?: unknown): void;
|
|
49
52
|
markJobDead(id: number, error: string): void;
|
|
50
53
|
markJobFailed(id: number, error: string): void;
|
|
54
|
+
private fib;
|
|
51
55
|
updateProgress(id: number, progress: number): void;
|
|
52
56
|
createBatch(name: string, options?: BatchOptions): string;
|
|
53
57
|
addToBatch<K extends string & keyof TMap>(batchId: string, type: K, data: TMap[K], options?: AddJobOptions): number;
|
|
@@ -57,6 +61,22 @@ export declare class JobQueue<TMap extends JobMap = Record<string, unknown>> {
|
|
|
57
61
|
type: K;
|
|
58
62
|
}): JobWorker<TMap, K>;
|
|
59
63
|
private insertJob;
|
|
64
|
+
/**
|
|
65
|
+
* Reset stuck `processing` jobs back to `pending`. Call at startup to recover
|
|
66
|
+
* from server crashes that left jobs claimed but never completed.
|
|
67
|
+
* @param thresholdMs Jobs processing longer than this (ms) are reset. Default 5min.
|
|
68
|
+
* @returns Number of jobs reset.
|
|
69
|
+
*/
|
|
70
|
+
reconcileStaleJobs(thresholdMs?: number): number;
|
|
71
|
+
/**
|
|
72
|
+
* Look up a pending or processing job by its uniqueKey.
|
|
73
|
+
* Returns null if no such job exists (completed, dead-lettered, or never queued).
|
|
74
|
+
*/
|
|
75
|
+
getJobByUniqueKey(type: string, uniqueKey: string): Job | null;
|
|
76
|
+
getJobResult<T>(id: number): T | null;
|
|
77
|
+
cancelByUniqueKey(type: string, uniqueKey: string): boolean;
|
|
78
|
+
retryFailedJobsByType(type: string): number;
|
|
79
|
+
purgeExpiredJobs(): number;
|
|
60
80
|
close(): void;
|
|
61
81
|
private unblockDependents;
|
|
62
82
|
private handleBatchJobComplete;
|
package/dist/queue.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"queue.d.ts","sourceRoot":"","sources":["../src/queue.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"queue.d.ts","sourceRoot":"","sources":["../src/queue.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAGtC,OAAO,EAAE,eAAe,EAAE,MAAM,UAAU,CAAC;AAE3C,OAAO,KAAK,EACV,aAAa,EAEb,YAAY,EACZ,SAAS,EAET,GAAG,EACH,QAAQ,EAER,MAAM,EAEN,QAAQ,EACR,eAAe,EACf,eAAe,EACf,YAAY,EAEZ,aAAa,EACd,MAAM,SAAS,CAAC;AAEjB,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AA2DrC,qBAAa,QAAQ,CACnB,IAAI,SAAS,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAC7C,SAAQ,eAAe;IACvB,QAAQ,CAAC,EAAE,EAAE,QAAQ,CAAC;IAEtB,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC;IAC/B,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAC;IACtC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC;IAC/B,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC;IAC/B,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAC;IACnC,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAC;IACpC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC;IAC9B,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC;IAChC,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAC;IACpC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC;IACjC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC;IACjC,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAC;IACrC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC;IAC/B,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAC;IACtC,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAC;IACpC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC;IAChC,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAC;IAErC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC;IAGhC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC;IACjC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC;IACjC,OAAO,CAAC,QAAQ,CAAC,yBAAyB,CAAC;IAC3C,OAAO,CAAC,QAAQ,CAAC,wBAAwB,CAAC;IAC1C,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC;IACjC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC;IACjC,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAC;gBAEzB,MAAM,EAAE,MAAM;IAiH1B,GAAG,CAAC,CAAC,SAAS,MAAM,GAAG,MAAM,IAAI,EAC/B,IAAI,EAAE,CAAC,EACP,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,EACb,OAAO,CAAC,EAAE,aAAa,GACtB,MAAM;IAIT,MAAM,CAAC,EAAE,EAAE,MAAM,GAAG,GAAG,GAAG,IAAI;IAK9B,QAAQ,IAAI,QAAQ;IAwBpB,aAAa,CAAC,OAAO,CAAC,EAAE;QACtB,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,MAAM,CAAC,EAAE,MAAM,CAAC;KACjB,GAAG,eAAe,CAAC,SAAS,CAAC;IAkC9B,QAAQ,CAAC,OAAO,CAAC,EAAE,eAAe,GAAG,eAAe,CAAC,GAAG,CAAC;IAkCzD,SAAS,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO;IAS9B,aAAa,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO;IASlC,aAAa,CAAC,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,GAAG,IAAI;IAQxE,WAAW,IAAI,MAAM,EAAE;IAOvB,cAAc,CAAC,WAAW,EAAE,MAAM,GAAG,MAAM;IAiC3C,eAAe,CAAC,WAAW,EAAE,MAAM,GAAG,MAAM;IAQ5C,KAAK,CAAC,OAAO,EAAE,YAAY,GAAG,MAAM;IAQpC,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,SAAU,GAAG,GAAG,GAAG,IAAI;IAqBzD,UAAU,CAAC,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,IAAI;IAS7C,WAAW,CAAC,EAAE,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,OAAO,GAAG,IAAI;IAuB/C,WAAW,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI;IAiC5C,aAAa,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI;IA2E9C,OAAO,CAAC,GAAG;IAUX,cAAc,CAAC,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,IAAI;IAYlD,WAAW,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,YAAY,GAAG,MAAM;IAWzD,UAAU,CAAC,CAAC,SAAS,MAAM,GAAG,MAAM,IAAI,EACtC,OAAO,EAAE,MAAM,EACf,IAAI,EAAE,CAAC,EACP,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,EACb,OAAO,CAAC,EAAE,aAAa,GACtB,MAAM;IAYT,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,QAAQ,GAAG,IAAI;IAO1C,WAAW,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI;IAMlC,YAAY,CAAC,CAAC,SAAS,MAAM,GAAG,MAAM,IAAI,EACxC,OAAO,EAAE,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG;QAAE,IAAI,EAAE,CAAC,CAAA;KAAE,GAC5C,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC;IAIrB,OAAO,CAAC,SAAS;IA8EjB;;;;;OAKG;IACH,kBAAkB,CAAC,WAAW,SAAU,GAAG,MAAM;IAgBjD;;;OAGG;IACH,iBAAiB,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,GAAG,GAAG,IAAI;IAY9D,YAAY,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,GAAG,CAAC,GAAG,IAAI;IAMrC,iBAAiB,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO;IAS3D,qBAAqB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM;IAgC3C,gBAAgB,IAAI,MAAM;IAS1B,KAAK,IAAI,IAAI;IAWb,OAAO,CAAC,iBAAiB;IAiBzB,OAAO,CAAC,sBAAsB;CAgE/B"}
|
package/dist/queue.js
CHANGED
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
import { Database } from 'bun:sqlite';
|
|
4
4
|
import { mkdirSync } from 'node:fs';
|
|
5
5
|
import { dirname } from 'node:path';
|
|
6
|
+
import { JobQueueEmitter } from './events';
|
|
6
7
|
import { applyPragmas, initializeSchema } from './schema';
|
|
7
8
|
import { nowISO } from './utils';
|
|
8
9
|
import { JobWorker } from './worker';
|
|
@@ -24,7 +25,11 @@ function toJob(row) {
|
|
|
24
25
|
error: row.error,
|
|
25
26
|
batchId: row.batch_id,
|
|
26
27
|
requestLog: row.request_log,
|
|
27
|
-
responseLog: row.response_log
|
|
28
|
+
responseLog: row.response_log,
|
|
29
|
+
uniqueKey: row.unique_key,
|
|
30
|
+
claimedUntil: row.claimed_until,
|
|
31
|
+
result: row.result ? JSON.parse(row.result) : null,
|
|
32
|
+
expireAt: row.expire_at
|
|
28
33
|
};
|
|
29
34
|
}
|
|
30
35
|
function toBatch(row) {
|
|
@@ -56,7 +61,7 @@ function toFailedJob(row) {
|
|
|
56
61
|
responseLog: row.response_log
|
|
57
62
|
};
|
|
58
63
|
}
|
|
59
|
-
export class JobQueue {
|
|
64
|
+
export class JobQueue extends JobQueueEmitter {
|
|
60
65
|
db;
|
|
61
66
|
insertJobStmt;
|
|
62
67
|
selectDedupedJobStmt;
|
|
@@ -75,6 +80,7 @@ export class JobQueue {
|
|
|
75
80
|
countUnmetDepsStmt;
|
|
76
81
|
unblockJobStmt;
|
|
77
82
|
lastInsertRowIdStmt;
|
|
83
|
+
renewLeaseStmt;
|
|
78
84
|
// Batch statements
|
|
79
85
|
insertBatchStmt;
|
|
80
86
|
selectBatchStmt;
|
|
@@ -84,13 +90,14 @@ export class JobQueue {
|
|
|
84
90
|
cancelBatchStmt;
|
|
85
91
|
cancelBatchJobsStmt;
|
|
86
92
|
constructor(dbPath) {
|
|
93
|
+
super();
|
|
87
94
|
mkdirSync(dirname(dbPath), { recursive: true });
|
|
88
95
|
this.db = new Database(dbPath, { create: true });
|
|
89
96
|
applyPragmas(this.db);
|
|
90
97
|
initializeSchema(this.db);
|
|
91
98
|
this.insertJobStmt = this.db.query(`
|
|
92
|
-
INSERT OR IGNORE INTO jobs (type, data, status, priority, max_retries, run_at, batch_id, unique_key)
|
|
93
|
-
VALUES ($type, $data, $status, $priority, $maxRetries, $runAt, $batchId, $uniqueKey)
|
|
99
|
+
INSERT OR IGNORE INTO jobs (type, data, status, priority, max_retries, run_at, batch_id, unique_key, backoff_config, expire_at, webhook_config)
|
|
100
|
+
VALUES ($type, $data, $status, $priority, $maxRetries, $runAt, $batchId, $uniqueKey, $backoffConfig, $expireAt, $webhookConfig)
|
|
94
101
|
`);
|
|
95
102
|
this.selectDedupedJobStmt = this.db.query(`
|
|
96
103
|
SELECT id FROM jobs
|
|
@@ -104,16 +111,21 @@ export class JobQueue {
|
|
|
104
111
|
this.selectJobStmt = this.db.query('SELECT * FROM jobs WHERE id = $id');
|
|
105
112
|
this.selectPendingStmt = this.db.query(`
|
|
106
113
|
SELECT * FROM jobs
|
|
107
|
-
WHERE
|
|
114
|
+
WHERE type = $type AND run_at <= $now
|
|
115
|
+
AND (expire_at IS NULL OR expire_at > $now)
|
|
116
|
+
AND (
|
|
117
|
+
(status = 'pending')
|
|
118
|
+
OR (status = 'processing' AND claimed_until IS NOT NULL AND claimed_until < $now)
|
|
119
|
+
)
|
|
108
120
|
ORDER BY priority DESC, created_at ASC
|
|
109
121
|
LIMIT 1
|
|
110
122
|
`);
|
|
111
123
|
this.markProcessingStmt = this.db.query(`
|
|
112
|
-
UPDATE jobs SET status = 'processing', started_at = $now, updated_at = $now
|
|
124
|
+
UPDATE jobs SET status = 'processing', started_at = $now, updated_at = $now, claimed_until = $claimedUntil
|
|
113
125
|
WHERE id = $id
|
|
114
126
|
`);
|
|
115
127
|
this.markDoneStmt = this.db.query(`
|
|
116
|
-
UPDATE jobs SET status = 'done', completed_at = $now, updated_at = $now, progress = 100
|
|
128
|
+
UPDATE jobs SET status = 'done', completed_at = $now, updated_at = $now, progress = 100, result = $result
|
|
117
129
|
WHERE id = $id
|
|
118
130
|
`);
|
|
119
131
|
this.markFailedStmt = this.db.query(`
|
|
@@ -121,6 +133,7 @@ export class JobQueue {
|
|
|
121
133
|
SET status = 'pending',
|
|
122
134
|
retry_count = retry_count + 1,
|
|
123
135
|
error = $error,
|
|
136
|
+
run_at = $runAt,
|
|
124
137
|
updated_at = $now
|
|
125
138
|
WHERE id = $id
|
|
126
139
|
`);
|
|
@@ -147,6 +160,10 @@ export class JobQueue {
|
|
|
147
160
|
WHERE id = $id AND status = 'blocked'
|
|
148
161
|
`);
|
|
149
162
|
this.lastInsertRowIdStmt = this.db.query('SELECT last_insert_rowid() as id');
|
|
163
|
+
this.renewLeaseStmt = this.db.query(`
|
|
164
|
+
UPDATE jobs SET claimed_until = $claimedUntil, updated_at = $now
|
|
165
|
+
WHERE id = $id AND status = 'processing'
|
|
166
|
+
`);
|
|
150
167
|
// Batch statements
|
|
151
168
|
this.insertBatchStmt = this.db.query(`
|
|
152
169
|
INSERT INTO job_batches (id, name, options, created_at)
|
|
@@ -287,7 +304,10 @@ export class JobQueue {
|
|
|
287
304
|
$maxRetries: row.max_retries,
|
|
288
305
|
$runAt: now,
|
|
289
306
|
$batchId: null,
|
|
290
|
-
$uniqueKey: null
|
|
307
|
+
$uniqueKey: null,
|
|
308
|
+
$backoffConfig: null,
|
|
309
|
+
$expireAt: null,
|
|
310
|
+
$webhookConfig: null
|
|
291
311
|
});
|
|
292
312
|
const newJobId = this.lastInsertRowIdStmt.get();
|
|
293
313
|
this.db
|
|
@@ -309,8 +329,9 @@ export class JobQueue {
|
|
|
309
329
|
.run({ $status: options.status, $cutoff: cutoff });
|
|
310
330
|
return result.changes;
|
|
311
331
|
}
|
|
312
|
-
pollAndClaim(type) {
|
|
332
|
+
pollAndClaim(type, leaseMs = 300_000) {
|
|
313
333
|
const now = nowISO();
|
|
334
|
+
const claimedUntil = new Date(Date.now() + leaseMs).toISOString();
|
|
314
335
|
const claimTx = this.db.transaction(() => {
|
|
315
336
|
const row = this.selectPendingStmt.get({
|
|
316
337
|
$type: type,
|
|
@@ -318,28 +339,54 @@ export class JobQueue {
|
|
|
318
339
|
});
|
|
319
340
|
if (!row)
|
|
320
341
|
return null;
|
|
321
|
-
this.markProcessingStmt.run({
|
|
342
|
+
this.markProcessingStmt.run({
|
|
343
|
+
$id: row.id,
|
|
344
|
+
$now: now,
|
|
345
|
+
$claimedUntil: claimedUntil
|
|
346
|
+
});
|
|
322
347
|
return row;
|
|
323
348
|
});
|
|
324
349
|
const row = claimTx.immediate();
|
|
325
350
|
return row ? toJob(row) : null;
|
|
326
351
|
}
|
|
327
|
-
|
|
352
|
+
renewLease(id, leaseMs) {
|
|
353
|
+
const claimedUntil = new Date(Date.now() + leaseMs).toISOString();
|
|
354
|
+
this.renewLeaseStmt.run({
|
|
355
|
+
$id: id,
|
|
356
|
+
$claimedUntil: claimedUntil,
|
|
357
|
+
$now: nowISO()
|
|
358
|
+
});
|
|
359
|
+
}
|
|
360
|
+
markJobDone(id, result) {
|
|
361
|
+
let doneJob = null;
|
|
328
362
|
this.db.transaction(() => {
|
|
329
363
|
const now = nowISO();
|
|
330
364
|
const row = this.selectJobStmt.get({ $id: id });
|
|
331
|
-
this.markDoneStmt.run({
|
|
365
|
+
this.markDoneStmt.run({
|
|
366
|
+
$id: id,
|
|
367
|
+
$now: now,
|
|
368
|
+
$result: result !== undefined ? JSON.stringify(result) : null
|
|
369
|
+
});
|
|
332
370
|
this.unblockDependents(id);
|
|
333
371
|
if (row?.batch_id) {
|
|
334
372
|
this.handleBatchJobComplete(row.batch_id);
|
|
335
373
|
}
|
|
374
|
+
// capture for post-tx emit
|
|
375
|
+
const updatedRow = this.selectJobStmt.get({ $id: id });
|
|
376
|
+
if (updatedRow)
|
|
377
|
+
doneJob = toJob(updatedRow);
|
|
336
378
|
})();
|
|
379
|
+
if (doneJob)
|
|
380
|
+
this.emit('job:done', doneJob);
|
|
337
381
|
}
|
|
338
382
|
markJobDead(id, error) {
|
|
383
|
+
let deadJob = null;
|
|
339
384
|
this.db.transaction(() => {
|
|
340
385
|
const row = this.selectJobStmt.get({ $id: id });
|
|
341
386
|
if (!row)
|
|
342
387
|
return;
|
|
388
|
+
// capture before delete
|
|
389
|
+
deadJob = toJob(row);
|
|
343
390
|
this.insertFailedJobStmt.run({
|
|
344
391
|
$originalJobId: row.id,
|
|
345
392
|
$type: row.type,
|
|
@@ -360,8 +407,11 @@ export class JobQueue {
|
|
|
360
407
|
this.handleBatchJobComplete(row.batch_id);
|
|
361
408
|
}
|
|
362
409
|
})();
|
|
410
|
+
if (deadJob)
|
|
411
|
+
this.emit('job:dead', deadJob, error);
|
|
363
412
|
}
|
|
364
413
|
markJobFailed(id, error) {
|
|
414
|
+
let failedJob = null;
|
|
365
415
|
this.db.transaction(() => {
|
|
366
416
|
const now = nowISO();
|
|
367
417
|
const row = this.selectJobStmt.get({ $id: id });
|
|
@@ -390,9 +440,50 @@ export class JobQueue {
|
|
|
390
440
|
}
|
|
391
441
|
}
|
|
392
442
|
else {
|
|
393
|
-
|
|
443
|
+
const backoff = row.backoff_config
|
|
444
|
+
? JSON.parse(row.backoff_config)
|
|
445
|
+
: null;
|
|
446
|
+
let retryRunAt = now;
|
|
447
|
+
if (backoff) {
|
|
448
|
+
let delayMs;
|
|
449
|
+
switch (backoff.type) {
|
|
450
|
+
case 'exponential':
|
|
451
|
+
delayMs = Math.min(backoff.delayMs * 2 ** row.retry_count, 3_600_000);
|
|
452
|
+
break;
|
|
453
|
+
case 'jitter':
|
|
454
|
+
delayMs = Math.min(backoff.delayMs * 2 ** row.retry_count * (0.5 + Math.random()), 3_600_000);
|
|
455
|
+
break;
|
|
456
|
+
case 'fibonacci':
|
|
457
|
+
delayMs = Math.min(backoff.delayMs * this.fib(row.retry_count), 3_600_000);
|
|
458
|
+
break;
|
|
459
|
+
default: // 'fixed'
|
|
460
|
+
delayMs = backoff.delayMs;
|
|
461
|
+
}
|
|
462
|
+
retryRunAt = new Date(Date.now() + delayMs).toISOString();
|
|
463
|
+
}
|
|
464
|
+
this.markFailedStmt.run({
|
|
465
|
+
$id: id,
|
|
466
|
+
$error: error,
|
|
467
|
+
$runAt: retryRunAt,
|
|
468
|
+
$now: now
|
|
469
|
+
});
|
|
470
|
+
// capture updated job for post-tx emit
|
|
471
|
+
const updatedRow = this.selectJobStmt.get({ $id: id });
|
|
472
|
+
if (updatedRow)
|
|
473
|
+
failedJob = toJob(updatedRow);
|
|
394
474
|
}
|
|
395
475
|
})();
|
|
476
|
+
if (failedJob)
|
|
477
|
+
this.emit('job:failed', failedJob, error);
|
|
478
|
+
}
|
|
479
|
+
fib(n) {
|
|
480
|
+
if (n <= 1)
|
|
481
|
+
return 1;
|
|
482
|
+
let a = 1, b = 1;
|
|
483
|
+
for (let i = 2; i <= n; i++) {
|
|
484
|
+
[a, b] = [b, a + b];
|
|
485
|
+
}
|
|
486
|
+
return b;
|
|
396
487
|
}
|
|
397
488
|
updateProgress(id, progress) {
|
|
398
489
|
this.updateProgressStmt.run({
|
|
@@ -400,6 +491,9 @@ export class JobQueue {
|
|
|
400
491
|
$progress: progress,
|
|
401
492
|
$now: nowISO()
|
|
402
493
|
});
|
|
494
|
+
const row = this.selectJobStmt.get({ $id: id });
|
|
495
|
+
if (row)
|
|
496
|
+
this.emit('job:progress', toJob(row), progress);
|
|
403
497
|
}
|
|
404
498
|
// --- Batch API ---
|
|
405
499
|
createBatch(name, options) {
|
|
@@ -438,6 +532,28 @@ export class JobQueue {
|
|
|
438
532
|
const runAt = options?.runAt ? options.runAt.toISOString() : now;
|
|
439
533
|
const hasDeps = options?.dependsOn && options.dependsOn.length > 0;
|
|
440
534
|
const status = hasDeps ? 'blocked' : 'pending';
|
|
535
|
+
const expireAt = options?.expireAt ? options.expireAt.toISOString() : null;
|
|
536
|
+
const webhookConfig = options?.onComplete
|
|
537
|
+
? JSON.stringify(options.onComplete)
|
|
538
|
+
: null;
|
|
539
|
+
// dedup='replace': update existing pending job's data + run_at
|
|
540
|
+
if (options?.dedup === 'replace' && options.uniqueKey) {
|
|
541
|
+
const existing = this.selectDedupedJobStmt.get({
|
|
542
|
+
$type: type,
|
|
543
|
+
$uniqueKey: options.uniqueKey
|
|
544
|
+
});
|
|
545
|
+
if (existing) {
|
|
546
|
+
this.db
|
|
547
|
+
.query('UPDATE jobs SET data = $data, run_at = $runAt, updated_at = $now WHERE id = $id')
|
|
548
|
+
.run({
|
|
549
|
+
$id: existing.id,
|
|
550
|
+
$data: JSON.stringify(data),
|
|
551
|
+
$runAt: runAt,
|
|
552
|
+
$now: now
|
|
553
|
+
});
|
|
554
|
+
return existing.id;
|
|
555
|
+
}
|
|
556
|
+
}
|
|
441
557
|
const result = this.insertJobStmt.run({
|
|
442
558
|
$type: type,
|
|
443
559
|
$data: JSON.stringify(data),
|
|
@@ -446,7 +562,10 @@ export class JobQueue {
|
|
|
446
562
|
$maxRetries: options?.maxRetries ?? 3,
|
|
447
563
|
$runAt: runAt,
|
|
448
564
|
$batchId: batchId,
|
|
449
|
-
$uniqueKey: options?.uniqueKey ?? null
|
|
565
|
+
$uniqueKey: options?.uniqueKey ?? null,
|
|
566
|
+
$backoffConfig: options?.backoff ? JSON.stringify(options.backoff) : null,
|
|
567
|
+
$expireAt: expireAt,
|
|
568
|
+
$webhookConfig: webhookConfig
|
|
450
569
|
});
|
|
451
570
|
// INSERT OR IGNORE: if a pending/processing job with same (type, uniqueKey)
|
|
452
571
|
// already exists, the insert is a no-op. Return the existing job id.
|
|
@@ -472,6 +591,86 @@ export class JobQueue {
|
|
|
472
591
|
}
|
|
473
592
|
return jobId.id;
|
|
474
593
|
}
|
|
594
|
+
/**
|
|
595
|
+
* Reset stuck `processing` jobs back to `pending`. Call at startup to recover
|
|
596
|
+
* from server crashes that left jobs claimed but never completed.
|
|
597
|
+
* @param thresholdMs Jobs processing longer than this (ms) are reset. Default 5min.
|
|
598
|
+
* @returns Number of jobs reset.
|
|
599
|
+
*/
|
|
600
|
+
reconcileStaleJobs(thresholdMs = 300_000) {
|
|
601
|
+
const cutoff = new Date(Date.now() - thresholdMs).toISOString();
|
|
602
|
+
const result = this.db
|
|
603
|
+
.query(`UPDATE jobs
|
|
604
|
+
SET status = 'pending',
|
|
605
|
+
error = 'stale: worker crash or restart — reset for retry',
|
|
606
|
+
updated_at = $now
|
|
607
|
+
WHERE status = 'processing' AND updated_at < $cutoff`)
|
|
608
|
+
.run({ $now: nowISO(), $cutoff: cutoff });
|
|
609
|
+
const count = result.changes;
|
|
610
|
+
if (count > 0)
|
|
611
|
+
this.emit('job:stale', count);
|
|
612
|
+
return count;
|
|
613
|
+
}
|
|
614
|
+
/**
|
|
615
|
+
* Look up a pending or processing job by its uniqueKey.
|
|
616
|
+
* Returns null if no such job exists (completed, dead-lettered, or never queued).
|
|
617
|
+
*/
|
|
618
|
+
getJobByUniqueKey(type, uniqueKey) {
|
|
619
|
+
const row = this.db
|
|
620
|
+
.query(`SELECT * FROM jobs
|
|
621
|
+
WHERE type = $type AND unique_key = $uniqueKey
|
|
622
|
+
AND status IN ('pending', 'processing')
|
|
623
|
+
LIMIT 1`)
|
|
624
|
+
.get({ $type: type, $uniqueKey: uniqueKey });
|
|
625
|
+
return row ? toJob(row) : null;
|
|
626
|
+
}
|
|
627
|
+
getJobResult(id) {
|
|
628
|
+
const row = this.selectJobStmt.get({ $id: id });
|
|
629
|
+
if (!row?.result)
|
|
630
|
+
return null;
|
|
631
|
+
return JSON.parse(row.result);
|
|
632
|
+
}
|
|
633
|
+
cancelByUniqueKey(type, uniqueKey) {
|
|
634
|
+
const result = this.db
|
|
635
|
+
.query("UPDATE jobs SET status = 'cancelled', updated_at = $now WHERE type = $type AND unique_key = $uniqueKey AND status IN ('pending', 'blocked')")
|
|
636
|
+
.run({ $type: type, $uniqueKey: uniqueKey, $now: nowISO() });
|
|
637
|
+
return result.changes > 0;
|
|
638
|
+
}
|
|
639
|
+
retryFailedJobsByType(type) {
|
|
640
|
+
const rows = this.db
|
|
641
|
+
.query('SELECT * FROM failed_jobs WHERE type = $type')
|
|
642
|
+
.all({ $type: type });
|
|
643
|
+
if (rows.length === 0)
|
|
644
|
+
return 0;
|
|
645
|
+
const now = nowISO();
|
|
646
|
+
this.db.transaction(() => {
|
|
647
|
+
for (const row of rows) {
|
|
648
|
+
this.insertJobStmt.run({
|
|
649
|
+
$type: row.type,
|
|
650
|
+
$data: row.data,
|
|
651
|
+
$status: 'pending',
|
|
652
|
+
$priority: 0,
|
|
653
|
+
$maxRetries: row.max_retries,
|
|
654
|
+
$runAt: now,
|
|
655
|
+
$batchId: null,
|
|
656
|
+
$uniqueKey: null,
|
|
657
|
+
$backoffConfig: null,
|
|
658
|
+
$expireAt: null,
|
|
659
|
+
$webhookConfig: null
|
|
660
|
+
});
|
|
661
|
+
this.db
|
|
662
|
+
.query('DELETE FROM failed_jobs WHERE id = $id')
|
|
663
|
+
.run({ $id: row.id });
|
|
664
|
+
}
|
|
665
|
+
})();
|
|
666
|
+
return rows.length;
|
|
667
|
+
}
|
|
668
|
+
purgeExpiredJobs() {
|
|
669
|
+
const result = this.db
|
|
670
|
+
.query("DELETE FROM jobs WHERE expire_at IS NOT NULL AND expire_at <= $now AND status = 'pending'")
|
|
671
|
+
.run({ $now: nowISO() });
|
|
672
|
+
return result.changes;
|
|
673
|
+
}
|
|
475
674
|
close() {
|
|
476
675
|
try {
|
|
477
676
|
if (this.db.filename !== ':memory:' && this.db.filename !== '') {
|
|
@@ -510,33 +709,51 @@ export class JobQueue {
|
|
|
510
709
|
const options = batch.options
|
|
511
710
|
? JSON.parse(batch.options)
|
|
512
711
|
: null;
|
|
513
|
-
if (
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
712
|
+
if (options) {
|
|
713
|
+
// Enqueue "then" callback job only if zero failures
|
|
714
|
+
if (batch.failed_jobs === 0 && options.thenType) {
|
|
715
|
+
this.insertJobStmt.run({
|
|
716
|
+
$type: options.thenType,
|
|
717
|
+
$data: JSON.stringify(options.thenData ?? {}),
|
|
718
|
+
$status: 'pending',
|
|
719
|
+
$priority: 0,
|
|
720
|
+
$maxRetries: 3,
|
|
721
|
+
$runAt: now,
|
|
722
|
+
$batchId: null,
|
|
723
|
+
$uniqueKey: null,
|
|
724
|
+
$backoffConfig: null,
|
|
725
|
+
$expireAt: null,
|
|
726
|
+
$webhookConfig: null
|
|
727
|
+
});
|
|
728
|
+
}
|
|
729
|
+
// Enqueue "finally" callback job regardless of failures
|
|
730
|
+
if (options.finallyType) {
|
|
731
|
+
this.insertJobStmt.run({
|
|
732
|
+
$type: options.finallyType,
|
|
733
|
+
$data: JSON.stringify(options.finallyData ?? {}),
|
|
734
|
+
$status: 'pending',
|
|
735
|
+
$priority: 0,
|
|
736
|
+
$maxRetries: 3,
|
|
737
|
+
$runAt: now,
|
|
738
|
+
$batchId: null,
|
|
739
|
+
$uniqueKey: null,
|
|
740
|
+
$backoffConfig: null,
|
|
741
|
+
$expireAt: null,
|
|
742
|
+
$webhookConfig: null
|
|
743
|
+
});
|
|
744
|
+
}
|
|
527
745
|
}
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
});
|
|
746
|
+
const finishedBatch = this.selectBatchStmt.get({
|
|
747
|
+
$id: batchId
|
|
748
|
+
});
|
|
749
|
+
if (finishedBatch) {
|
|
750
|
+
const b = toBatch(finishedBatch);
|
|
751
|
+
if (b.failedJobs === 0) {
|
|
752
|
+
this.emit('batch:complete', b);
|
|
753
|
+
}
|
|
754
|
+
else {
|
|
755
|
+
this.emit('batch:failed', b);
|
|
756
|
+
}
|
|
540
757
|
}
|
|
541
758
|
}
|
|
542
759
|
}
|
package/dist/schema.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../src/schema.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../src/schema.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAwG3C,wBAAgB,YAAY,CAAC,EAAE,EAAE,QAAQ,GAAG,IAAI,CAY/C;AAED,wBAAgB,gBAAgB,CAAC,EAAE,EAAE,QAAQ,GAAG,IAAI,CAkCnD"}
|
package/dist/schema.js
CHANGED
|
@@ -17,7 +17,12 @@ CREATE TABLE IF NOT EXISTS jobs (
|
|
|
17
17
|
request_log TEXT,
|
|
18
18
|
response_log TEXT,
|
|
19
19
|
batch_id TEXT REFERENCES job_batches(id),
|
|
20
|
-
unique_key TEXT
|
|
20
|
+
unique_key TEXT,
|
|
21
|
+
backoff_config TEXT,
|
|
22
|
+
claimed_until TEXT,
|
|
23
|
+
result TEXT,
|
|
24
|
+
expire_at TEXT,
|
|
25
|
+
webhook_config TEXT
|
|
21
26
|
)`;
|
|
22
27
|
// State-aware dedup: same (type, unique_key) cannot be both pending/processing at once.
|
|
23
28
|
// Once the job completes, the same key can be re-enqueued.
|
|
@@ -107,12 +112,25 @@ export function initializeSchema(db) {
|
|
|
107
112
|
db.run(FAILED_JOBS_TABLE);
|
|
108
113
|
db.run(SCHEDULES_TABLE);
|
|
109
114
|
db.run(SCHEDULES_NEXT_RUN_INDEX);
|
|
110
|
-
//
|
|
111
|
-
const cols = db
|
|
112
|
-
.prepare("PRAGMA table_info(jobs)")
|
|
113
|
-
.all();
|
|
115
|
+
// Migrations for columns added after initial schema creation
|
|
116
|
+
const cols = db.prepare('PRAGMA table_info(jobs)').all();
|
|
114
117
|
if (!cols.some(c => c.name === 'unique_key')) {
|
|
115
118
|
db.run('ALTER TABLE jobs ADD COLUMN unique_key TEXT');
|
|
116
119
|
}
|
|
120
|
+
if (!cols.some(c => c.name === 'backoff_config')) {
|
|
121
|
+
db.run('ALTER TABLE jobs ADD COLUMN backoff_config TEXT');
|
|
122
|
+
}
|
|
123
|
+
if (!cols.some(c => c.name === 'claimed_until')) {
|
|
124
|
+
db.run('ALTER TABLE jobs ADD COLUMN claimed_until TEXT');
|
|
125
|
+
}
|
|
126
|
+
if (!cols.some(c => c.name === 'result')) {
|
|
127
|
+
db.run('ALTER TABLE jobs ADD COLUMN result TEXT');
|
|
128
|
+
}
|
|
129
|
+
if (!cols.some(c => c.name === 'expire_at')) {
|
|
130
|
+
db.run('ALTER TABLE jobs ADD COLUMN expire_at TEXT');
|
|
131
|
+
}
|
|
132
|
+
if (!cols.some(c => c.name === 'webhook_config')) {
|
|
133
|
+
db.run('ALTER TABLE jobs ADD COLUMN webhook_config TEXT');
|
|
134
|
+
}
|
|
117
135
|
db.run(JOBS_UNIQUE_KEY_INDEX);
|
|
118
136
|
}
|
package/dist/types.d.ts
CHANGED
|
@@ -27,6 +27,10 @@ export type Job<T = unknown> = {
|
|
|
27
27
|
readonly batchId: string | null;
|
|
28
28
|
readonly requestLog: string | null;
|
|
29
29
|
readonly responseLog: string | null;
|
|
30
|
+
readonly uniqueKey: string | null;
|
|
31
|
+
readonly claimedUntil: string | null;
|
|
32
|
+
readonly result: unknown | null;
|
|
33
|
+
readonly expireAt: string | null;
|
|
30
34
|
};
|
|
31
35
|
export type FailedJob = {
|
|
32
36
|
readonly id: number;
|
|
@@ -41,6 +45,11 @@ export type FailedJob = {
|
|
|
41
45
|
readonly requestLog: string | null;
|
|
42
46
|
readonly responseLog: string | null;
|
|
43
47
|
};
|
|
48
|
+
export type BackoffConfig = {
|
|
49
|
+
type: 'exponential' | 'fixed' | 'jitter' | 'fibonacci';
|
|
50
|
+
/** Base delay in ms. Exponential: delayMs * 2^retryCount. Fixed: always delayMs. Max 1h. */
|
|
51
|
+
delayMs: number;
|
|
52
|
+
};
|
|
44
53
|
export type AddJobOptions = {
|
|
45
54
|
priority?: number;
|
|
46
55
|
runAt?: Date;
|
|
@@ -52,10 +61,23 @@ export type AddJobOptions = {
|
|
|
52
61
|
* existing job id is returned. Once the job completes, the same key can be re-used.
|
|
53
62
|
*/
|
|
54
63
|
uniqueKey?: string;
|
|
64
|
+
/**
|
|
65
|
+
* Backoff strategy for retries. Exponential: delayMs * 2^retryCount, capped at 1h.
|
|
66
|
+
* Fixed: always delayMs between retries. Default: retry immediately.
|
|
67
|
+
*/
|
|
68
|
+
backoff?: BackoffConfig;
|
|
69
|
+
dedup?: 'ignore' | 'replace';
|
|
70
|
+
expireAt?: Date;
|
|
71
|
+
onComplete?: {
|
|
72
|
+
url: string;
|
|
73
|
+
method?: string;
|
|
74
|
+
headers?: Record<string, string>;
|
|
75
|
+
};
|
|
55
76
|
};
|
|
56
77
|
export type JobContext = {
|
|
57
78
|
reportProgress: (percent: number) => void;
|
|
58
79
|
signal: AbortSignal;
|
|
80
|
+
renewLease(): void;
|
|
59
81
|
};
|
|
60
82
|
export type RateLimit = {
|
|
61
83
|
count: number;
|
|
@@ -63,12 +85,20 @@ export type RateLimit = {
|
|
|
63
85
|
};
|
|
64
86
|
export type WorkerOptions<T = unknown> = {
|
|
65
87
|
type: string;
|
|
66
|
-
handler: (job: Job<T>, ctx: JobContext) => Promise<
|
|
88
|
+
handler: (job: Job<T>, ctx: JobContext) => Promise<unknown>;
|
|
67
89
|
pollIntervalMs?: number;
|
|
68
90
|
maxRate?: RateLimit;
|
|
69
91
|
onError?: (job: Job<T>, error: unknown) => void;
|
|
70
92
|
/** Hard wall-clock limit per job execution in ms. Job is marked failed on timeout. */
|
|
71
93
|
timeoutMs?: number;
|
|
94
|
+
/** Max concurrent jobs this worker runs simultaneously. Default: 1. */
|
|
95
|
+
concurrency?: number;
|
|
96
|
+
leaseMs?: number;
|
|
97
|
+
retryIf?: (error: unknown, job: Job<T>) => boolean;
|
|
98
|
+
aging?: {
|
|
99
|
+
boostPerMinute: number;
|
|
100
|
+
maxBoost: number;
|
|
101
|
+
};
|
|
72
102
|
};
|
|
73
103
|
export type JobStats = {
|
|
74
104
|
pending: number;
|
|
@@ -113,6 +143,11 @@ export type JobRow = {
|
|
|
113
143
|
request_log: string | null;
|
|
114
144
|
response_log: string | null;
|
|
115
145
|
unique_key: string | null;
|
|
146
|
+
backoff_config: string | null;
|
|
147
|
+
claimed_until: string | null;
|
|
148
|
+
result: string | null;
|
|
149
|
+
expire_at: string | null;
|
|
150
|
+
webhook_config: string | null;
|
|
116
151
|
};
|
|
117
152
|
export type FailedJobRow = {
|
|
118
153
|
id: number;
|
package/dist/types.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAGA;;;;;GAKG;AACH,qBAAa,iBAAkB,SAAQ,KAAK;IAC1C,QAAQ,CAAC,cAAc,QAAQ;gBAEnB,OAAO,EAAE,MAAM;CAI5B;AAED,MAAM,MAAM,SAAS,GACjB,SAAS,GACT,YAAY,GACZ,MAAM,GACN,QAAQ,GACR,SAAS,GACT,WAAW,CAAC;AAEhB,MAAM,MAAM,GAAG,CAAC,CAAC,GAAG,OAAO,IAAI;IAC7B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;IACjB,QAAQ,CAAC,MAAM,EAAE,SAAS,CAAC;IAC3B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAGA;;;;;GAKG;AACH,qBAAa,iBAAkB,SAAQ,KAAK;IAC1C,QAAQ,CAAC,cAAc,QAAQ;gBAEnB,OAAO,EAAE,MAAM;CAI5B;AAED,MAAM,MAAM,SAAS,GACjB,SAAS,GACT,YAAY,GACZ,MAAM,GACN,QAAQ,GACR,SAAS,GACT,WAAW,CAAC;AAEhB,MAAM,MAAM,GAAG,CAAC,CAAC,GAAG,OAAO,IAAI;IAC7B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;IACjB,QAAQ,CAAC,MAAM,EAAE,SAAS,CAAC;IAC3B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,QAAQ,CAAC,MAAM,EAAE,OAAO,GAAG,IAAI,CAAC;IAChC,QAAQ,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;CAClC,CAAC;AAEF,MAAM,MAAM,SAAS,GAAG;IACtB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC;IACvB,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;CACrC,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG;IAC1B,IAAI,EAAE,aAAa,GAAG,OAAO,GAAG,QAAQ,GAAG,WAAW,CAAC;IACvD,4FAA4F;IAC5F,OAAO,EAAE,MAAM,CAAC;CACjB,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG;IAC1B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,IAAI,CAAC;IACb,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;OAGG;IACH,OAAO,CAAC,EAAE,aAAa,CAAC;IACxB,KAAK,CAAC,EAAE,QAAQ,GAAG,SAAS,CAAC;IAC7B,QAAQ,CAAC,EAAE,IAAI,CAAC;IAChB,UAAU,CAAC,EAAE;QACX,GAAG,EAAE,MAAM,CAAC;QACZ,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;KAClC,CAAC;CACH,CAAC;AAEF,MAAM,MAAM,UAAU,GAAG;IACvB,cAAc,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;IAC1C,MAAM,EAAE,WAAW,CAAC;IACpB,UAAU,IAAI,IAAI,CAAC;CACpB,CAAC;AAEF,MAAM,MAAM,SAAS,GAAG;IACtB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF,MAAM,MAAM,aAAa,CAAC,CAAC,GAAG,OAAO,IAAI;IACvC,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,UAAU,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;IAC5D,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,OAAO,CAAC,EAAE,SAAS,CAAC;IACpB,OAAO,CAAC,EAAE,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,KAAK,IAAI,CAAC;IAChD,sFAAsF;IACtF,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,uEAAuE;IACvE,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,OAAO,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,KAAK,OAAO,CAAC;IACnD,KAAK,CAAC,EAAE;QAAE,cAAc,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC;CACtD,CAAC;AAEF,MAAM,MAAM,QAAQ,GAAG;IACrB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,MAAM,CAAC;CACd,CAAC;AAEF,MAAM,MAAM,YAAY,GAAG;IACzB,MAAM,EAAE,MAAM,GAAG,QAAQ,CAAC;IAC1B,WAAW,EAAE,MAAM,CAAC;CACrB,CAAC;AAEF,MAAM,MAAM,eAAe,GAAG;IAC5B,MAAM,CAAC,EAAE,SAAS,CAAC;IACnB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB,CAAC;AAEF,MAAM,MAAM,eAAe,CAAC,CAAC,IAAI;IAC/B,KAAK,EAAE,CAAC,EAAE,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;CACf,CAAC;AAEF,MAAM,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;AAG7C,MAAM,MAAM,MAAM,GAAG;IACnB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;CAC/B,CAAC;AAEF,MAAM,MAAM,YAAY,GAAG;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,eAAe,EAAE,MAAM,CAAC;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;CAC7B,CAAC;AAEF,MAAM,MAAM,QAAQ,GAAG;IACrB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;CACf,CAAC;AAIF,MAAM,MAAM,YAAY,GAAG;IACzB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,OAAO,CAAC;CACvB,CAAC;AAEF,MAAM,MAAM,QAAQ,GAAG;IACrB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,YAAY,EAAE,MAAM,EAAE,CAAC;IAChC,QAAQ,CAAC,OAAO,EAAE,YAAY,GAAG,IAAI,CAAC;IACtC,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;CACpC,CAAC;AAEF,MAAM,MAAM,WAAW,GAAG;IACxB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;IACpB,cAAc,EAAE,MAAM,CAAC;IACvB,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;CAC5B,CAAC;AAIF,MAAM,MAAM,QAAQ,GAAG;IACrB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC;IACvB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;CAC5B,CAAC;AAEF,MAAM,MAAM,WAAW,GAAG;IACxB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,MAAM,MAAM,kBAAkB,GAAG;IAC/B,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB,CAAC"}
|
package/dist/worker.d.ts
CHANGED
|
@@ -7,15 +7,20 @@ export declare class JobWorker<TMap extends JobMap = Record<string, unknown>, K
|
|
|
7
7
|
private abortController;
|
|
8
8
|
private timer;
|
|
9
9
|
private running;
|
|
10
|
-
private
|
|
10
|
+
private paused;
|
|
11
|
+
private activeCount;
|
|
11
12
|
private stopResolve;
|
|
12
13
|
constructor(queue: JobQueue<TMap>, options: WorkerOptions<TMap[K]> & {
|
|
13
14
|
type: K;
|
|
14
15
|
});
|
|
15
16
|
get isRunning(): boolean;
|
|
17
|
+
get isPaused(): boolean;
|
|
16
18
|
start(): void;
|
|
17
19
|
stop(): Promise<void>;
|
|
20
|
+
pause(): void;
|
|
21
|
+
resume(): void;
|
|
18
22
|
private scheduleNext;
|
|
19
23
|
private poll;
|
|
24
|
+
private runJob;
|
|
20
25
|
}
|
|
21
26
|
//# sourceMappingURL=worker.d.ts.map
|
package/dist/worker.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"worker.d.ts","sourceRoot":"","sources":["../src/worker.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAExC,OAAO,KAAK,EAAmB,MAAM,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAGtE,qBAAa,SAAS,CACpB,IAAI,SAAS,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAC7C,CAAC,SAAS,MAAM,GAAG,MAAM,IAAI,GAAG,MAAM,GAAG,MAAM,IAAI;IAEnD,OAAO,CAAC,KAAK,CAAiB;IAC9B,OAAO,CAAC,OAAO,CAAuC;IACtD,OAAO,CAAC,WAAW,CAAkC;IACrD,OAAO,CAAC,eAAe,CAAgC;IACvD,OAAO,CAAC,KAAK,CAA8C;IAC3D,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,
|
|
1
|
+
{"version":3,"file":"worker.d.ts","sourceRoot":"","sources":["../src/worker.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAExC,OAAO,KAAK,EAAmB,MAAM,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAGtE,qBAAa,SAAS,CACpB,IAAI,SAAS,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAC7C,CAAC,SAAS,MAAM,GAAG,MAAM,IAAI,GAAG,MAAM,GAAG,MAAM,IAAI;IAEnD,OAAO,CAAC,KAAK,CAAiB;IAC9B,OAAO,CAAC,OAAO,CAAuC;IACtD,OAAO,CAAC,WAAW,CAAkC;IACrD,OAAO,CAAC,eAAe,CAAgC;IACvD,OAAO,CAAC,KAAK,CAA8C;IAC3D,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,WAAW,CAAK;IACxB,OAAO,CAAC,WAAW,CAA6B;gBAG9C,KAAK,EAAE,QAAQ,CAAC,IAAI,CAAC,EACrB,OAAO,EAAE,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG;QAAE,IAAI,EAAE,CAAC,CAAA;KAAE;IAY/C,IAAI,SAAS,IAAI,OAAO,CAEvB;IAED,IAAI,QAAQ,IAAI,OAAO,CAEtB;IAED,KAAK,IAAI,IAAI;IAOP,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAkB3B,KAAK,IAAI,IAAI;IAIb,MAAM,IAAI,IAAI;IAYd,OAAO,CAAC,YAAY;YAMN,IAAI;YAyBJ,MAAM;CA0DrB"}
|
package/dist/worker.js
CHANGED
|
@@ -7,7 +7,8 @@ export class JobWorker {
|
|
|
7
7
|
abortController = null;
|
|
8
8
|
timer = null;
|
|
9
9
|
running = false;
|
|
10
|
-
|
|
10
|
+
paused = false;
|
|
11
|
+
activeCount = 0;
|
|
11
12
|
stopResolve = null;
|
|
12
13
|
constructor(queue, options) {
|
|
13
14
|
this.queue = queue;
|
|
@@ -19,6 +20,9 @@ export class JobWorker {
|
|
|
19
20
|
get isRunning() {
|
|
20
21
|
return this.running;
|
|
21
22
|
}
|
|
23
|
+
get isPaused() {
|
|
24
|
+
return this.paused;
|
|
25
|
+
}
|
|
22
26
|
start() {
|
|
23
27
|
if (this.running)
|
|
24
28
|
return;
|
|
@@ -35,12 +39,26 @@ export class JobWorker {
|
|
|
35
39
|
this.timer = null;
|
|
36
40
|
}
|
|
37
41
|
this.abortController?.abort();
|
|
38
|
-
if (this.
|
|
42
|
+
if (this.activeCount > 0) {
|
|
39
43
|
return new Promise(resolve => {
|
|
40
44
|
this.stopResolve = resolve;
|
|
41
45
|
});
|
|
42
46
|
}
|
|
43
47
|
}
|
|
48
|
+
pause() {
|
|
49
|
+
this.paused = true;
|
|
50
|
+
}
|
|
51
|
+
resume() {
|
|
52
|
+
this.paused = false;
|
|
53
|
+
if (this.running) {
|
|
54
|
+
// Cancel existing timer and poll immediately
|
|
55
|
+
if (this.timer) {
|
|
56
|
+
clearTimeout(this.timer);
|
|
57
|
+
this.timer = null;
|
|
58
|
+
}
|
|
59
|
+
void this.poll();
|
|
60
|
+
}
|
|
61
|
+
}
|
|
44
62
|
scheduleNext() {
|
|
45
63
|
if (!this.running)
|
|
46
64
|
return;
|
|
@@ -50,36 +68,49 @@ export class JobWorker {
|
|
|
50
68
|
async poll() {
|
|
51
69
|
if (!this.running)
|
|
52
70
|
return;
|
|
53
|
-
if (this.
|
|
71
|
+
if (this.paused) {
|
|
54
72
|
this.scheduleNext();
|
|
55
73
|
return;
|
|
56
74
|
}
|
|
57
|
-
const
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
75
|
+
const concurrency = this.options.concurrency ?? 1;
|
|
76
|
+
const leaseMs = this.options.leaseMs ?? 300_000;
|
|
77
|
+
// Drain available jobs up to available capacity in one poll tick
|
|
78
|
+
while (this.activeCount < concurrency) {
|
|
79
|
+
if (this.rateLimiter && !this.rateLimiter.canProceed())
|
|
80
|
+
break;
|
|
81
|
+
const job = this.queue.pollAndClaim(this.options.type, leaseMs);
|
|
82
|
+
if (!job)
|
|
83
|
+
break;
|
|
84
|
+
this.rateLimiter?.record();
|
|
85
|
+
this.activeCount++;
|
|
86
|
+
void this.runJob(job);
|
|
61
87
|
}
|
|
62
|
-
this.
|
|
88
|
+
this.scheduleNext();
|
|
89
|
+
}
|
|
90
|
+
async runJob(job) {
|
|
63
91
|
try {
|
|
64
92
|
const ctx = {
|
|
65
93
|
reportProgress: (percent) => {
|
|
66
94
|
this.queue.updateProgress(job.id, percent);
|
|
67
95
|
},
|
|
96
|
+
renewLease: () => {
|
|
97
|
+
this.queue.renewLease(job.id, this.options.leaseMs ?? 300_000);
|
|
98
|
+
},
|
|
68
99
|
signal: this.abortController.signal
|
|
69
100
|
};
|
|
70
|
-
this.rateLimiter?.record();
|
|
71
101
|
const handlerPromise = this.options.handler(job, ctx);
|
|
102
|
+
let handlerResult;
|
|
72
103
|
if (this.options.timeoutMs) {
|
|
73
104
|
const timeoutMs = this.options.timeoutMs;
|
|
74
|
-
await Promise.race([
|
|
105
|
+
handlerResult = await Promise.race([
|
|
75
106
|
handlerPromise,
|
|
76
107
|
new Promise((_, reject) => setTimeout(() => reject(new Error(`Job timed out after ${timeoutMs}ms`)), timeoutMs))
|
|
77
108
|
]);
|
|
78
109
|
}
|
|
79
110
|
else {
|
|
80
|
-
await handlerPromise;
|
|
111
|
+
handlerResult = await handlerPromise;
|
|
81
112
|
}
|
|
82
|
-
this.queue.markJobDone(job.id);
|
|
113
|
+
this.queue.markJobDone(job.id, handlerResult);
|
|
83
114
|
}
|
|
84
115
|
catch (error) {
|
|
85
116
|
const message = error instanceof Error ? error.message : String(error);
|
|
@@ -89,7 +120,10 @@ export class JobWorker {
|
|
|
89
120
|
(typeof error === 'object' &&
|
|
90
121
|
error !== null &&
|
|
91
122
|
error.isNonRetryable === true);
|
|
92
|
-
|
|
123
|
+
const shouldRetry = this.options.retryIf
|
|
124
|
+
? this.options.retryIf(error, job)
|
|
125
|
+
: true;
|
|
126
|
+
if (isNonRetryable || !shouldRetry) {
|
|
93
127
|
this.queue.markJobDead(job.id, message);
|
|
94
128
|
}
|
|
95
129
|
else {
|
|
@@ -98,14 +132,11 @@ export class JobWorker {
|
|
|
98
132
|
this.options.onError?.(job, error);
|
|
99
133
|
}
|
|
100
134
|
finally {
|
|
101
|
-
this.
|
|
102
|
-
if (this.stopResolve) {
|
|
135
|
+
this.activeCount--;
|
|
136
|
+
if (!this.running && this.activeCount === 0 && this.stopResolve) {
|
|
103
137
|
this.stopResolve();
|
|
104
138
|
this.stopResolve = null;
|
|
105
139
|
}
|
|
106
|
-
else {
|
|
107
|
-
this.scheduleNext();
|
|
108
|
-
}
|
|
109
140
|
}
|
|
110
141
|
}
|
|
111
142
|
}
|