@nest-native/jobs 0.4.0 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -2
- package/dist/dialects/mysql/job-store.d.ts +5 -4
- package/dist/dialects/mysql/job-store.js +31 -12
- package/dist/dialects/mysql/job-store.js.map +1 -1
- package/dist/dialects/postgres/job-store.d.ts +5 -4
- package/dist/dialects/postgres/job-store.js +41 -12
- package/dist/dialects/postgres/job-store.js.map +1 -1
- package/dist/dialects/sqlite/job-store.d.ts +5 -4
- package/dist/dialects/sqlite/job-store.js +26 -12
- package/dist/dialects/sqlite/job-store.js.map +1 -1
- package/dist/interfaces.d.ts +38 -4
- package/dist/jobs-claimer.service.d.ts +43 -0
- package/dist/jobs-claimer.service.js +147 -20
- package/dist/jobs-claimer.service.js.map +1 -1
- package/dist/jobs-worker.d.ts +5 -1
- package/dist/jobs-worker.js +6 -0
- package/dist/jobs-worker.js.map +1 -1
- package/dist/testing/harness.js +2 -1
- package/dist/testing/harness.js.map +1 -1
- package/package.json +3 -3
package/README.md
CHANGED
|
@@ -104,10 +104,11 @@ export class WelcomeEmailHandler implements JobHandler {
|
|
|
104
104
|
|
|
105
105
|
// worker (same process or a dedicated one)
|
|
106
106
|
const controller = new AbortController();
|
|
107
|
-
|
|
107
|
+
runWorkerLoop(app.get(JobsClaimer), { signal: controller.signal })
|
|
108
|
+
.catch((error) => console.error('jobs worker stopped', error));
|
|
108
109
|
```
|
|
109
110
|
|
|
110
|
-
Delivery is **at-least-once**: a worker crash mid-job means the row is reclaimed after `stuckTimeoutMs` and run again. Make handlers idempotent or key their side effects on `ctx.jobId`.
|
|
111
|
+
Delivery is **at-least-once**: a worker crash mid-job means the row is reclaimed after `stuckTimeoutMs` and run again. Make handlers idempotent or key their side effects on `ctx.jobId`. Outcomes are recorded under the claim: a worker whose claim was taken over records nothing, and the job counts as `lost` in its `TickReport`.
|
|
111
112
|
|
|
112
113
|
### The uniqueKey contract
|
|
113
114
|
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { EnqueueJobInput, JobRow, JobStore, ResolvedRunnerConfig } from '../../interfaces';
|
|
1
|
+
import type { EnqueueJobInput, JobClaim, JobRow, JobStore, ResolvedRunnerConfig } from '../../interfaces';
|
|
2
2
|
/**
|
|
3
3
|
* MySQL surfaces a unique-constraint violation as error code `ER_DUP_ENTRY`
|
|
4
4
|
* (errno `1062`). mysql2 sets both `error.code === 'ER_DUP_ENTRY'` and
|
|
@@ -29,7 +29,8 @@ export declare class MysqlJobStore implements JobStore {
|
|
|
29
29
|
*/
|
|
30
30
|
private findActiveDuplicate;
|
|
31
31
|
claimBatch(db: unknown, cfg: ResolvedRunnerConfig): Promise<JobRow[]>;
|
|
32
|
-
markCompleted(db: unknown,
|
|
33
|
-
retry(db: unknown,
|
|
34
|
-
|
|
32
|
+
markCompleted(db: unknown, claim: JobClaim): Promise<boolean>;
|
|
33
|
+
retry(db: unknown, claim: JobClaim, delayMs: number, lastError?: string): Promise<boolean>;
|
|
34
|
+
release(db: unknown, claim: JobClaim): Promise<boolean>;
|
|
35
|
+
markFailed(db: unknown, claim: JobClaim, reason: string): Promise<boolean>;
|
|
35
36
|
}
|
|
@@ -6,6 +6,12 @@ const node_crypto_1 = require("node:crypto");
|
|
|
6
6
|
const drizzle_orm_1 = require("drizzle-orm");
|
|
7
7
|
const enqueue_input_1 = require("../../enqueue-input");
|
|
8
8
|
const schema_1 = require("./schema");
|
|
9
|
+
/**
|
|
10
|
+
* Matches the job only while it is `processing` under exactly this claim, so a
|
|
11
|
+
* transition from a worker whose claim was taken over after the stuck timeout
|
|
12
|
+
* writes nothing, even when a later claim reused the same `workerInstanceId`.
|
|
13
|
+
*/
|
|
14
|
+
const heldBy = (claim) => (0, drizzle_orm_1.and)((0, drizzle_orm_1.eq)(schema_1.jobs.id, claim.id), (0, drizzle_orm_1.eq)(schema_1.jobs.status, 'processing'), (0, drizzle_orm_1.eq)(schema_1.jobs.claimedBy, claim.claimedBy), (0, drizzle_orm_1.eq)(schema_1.jobs.claimedAt, claim.claimedAt));
|
|
9
15
|
/**
|
|
10
16
|
* MySQL surfaces a unique-constraint violation as error code `ER_DUP_ENTRY`
|
|
11
17
|
* (errno `1062`). mysql2 sets both `error.code === 'ER_DUP_ENTRY'` and
|
|
@@ -81,10 +87,13 @@ class MysqlJobStore {
|
|
|
81
87
|
return existing;
|
|
82
88
|
}
|
|
83
89
|
async claimBatch(db, cfg) {
|
|
84
|
-
const now = new Date();
|
|
85
|
-
const nowIso = now.toISOString();
|
|
86
|
-
const stuckCutoff = new Date(now.getTime() - cfg.stuckTimeoutMs).toISOString();
|
|
87
90
|
return db.transaction(async (tx) => {
|
|
91
|
+
// Stamped once the connection is ours: a checkout that waited on a busy
|
|
92
|
+
// pool must not leave this claim looking older than it is, or another
|
|
93
|
+
// worker would treat its jobs as stuck that much sooner.
|
|
94
|
+
const now = new Date();
|
|
95
|
+
const nowIso = now.toISOString();
|
|
96
|
+
const stuckCutoff = new Date(now.getTime() - cfg.stuckTimeoutMs).toISOString();
|
|
88
97
|
const candidates = await tx
|
|
89
98
|
.select({ id: schema_1.jobs.id })
|
|
90
99
|
.from(schema_1.jobs)
|
|
@@ -111,8 +120,8 @@ class MysqlJobStore {
|
|
|
111
120
|
// would wait for the claim to commit.
|
|
112
121
|
}, { isolationLevel: 'read committed' });
|
|
113
122
|
}
|
|
114
|
-
async markCompleted(db,
|
|
115
|
-
await db
|
|
123
|
+
async markCompleted(db, claim) {
|
|
124
|
+
const [result] = await db
|
|
116
125
|
.update(schema_1.jobs)
|
|
117
126
|
.set({
|
|
118
127
|
status: 'completed',
|
|
@@ -121,11 +130,12 @@ class MysqlJobStore {
|
|
|
121
130
|
// Terminal → release the active-dedup key.
|
|
122
131
|
uniqueKey: null,
|
|
123
132
|
})
|
|
124
|
-
.where((
|
|
133
|
+
.where(heldBy(claim));
|
|
134
|
+
return result.affectedRows > 0;
|
|
125
135
|
}
|
|
126
|
-
async retry(db,
|
|
136
|
+
async retry(db, claim, delayMs, lastError) {
|
|
127
137
|
const nextAvailable = new Date(Date.now() + delayMs).toISOString();
|
|
128
|
-
await db
|
|
138
|
+
const [result] = await db
|
|
129
139
|
.update(schema_1.jobs)
|
|
130
140
|
.set({
|
|
131
141
|
status: 'pending',
|
|
@@ -136,10 +146,18 @@ class MysqlJobStore {
|
|
|
136
146
|
lastError: lastError ?? null,
|
|
137
147
|
// Still active → the uniqueKey stays claimed.
|
|
138
148
|
})
|
|
139
|
-
.where((
|
|
149
|
+
.where(heldBy(claim));
|
|
150
|
+
return result.affectedRows > 0;
|
|
151
|
+
}
|
|
152
|
+
async release(db, claim) {
|
|
153
|
+
const [result] = await db
|
|
154
|
+
.update(schema_1.jobs)
|
|
155
|
+
.set({ status: 'pending', claimedAt: null, claimedBy: null })
|
|
156
|
+
.where(heldBy(claim));
|
|
157
|
+
return result.affectedRows > 0;
|
|
140
158
|
}
|
|
141
|
-
async markFailed(db,
|
|
142
|
-
await db
|
|
159
|
+
async markFailed(db, claim, reason) {
|
|
160
|
+
const [result] = await db
|
|
143
161
|
.update(schema_1.jobs)
|
|
144
162
|
.set({
|
|
145
163
|
status: 'failed',
|
|
@@ -149,7 +167,8 @@ class MysqlJobStore {
|
|
|
149
167
|
// Terminal → release the active-dedup key.
|
|
150
168
|
uniqueKey: null,
|
|
151
169
|
})
|
|
152
|
-
.where((
|
|
170
|
+
.where(heldBy(claim));
|
|
171
|
+
return result.affectedRows > 0;
|
|
153
172
|
}
|
|
154
173
|
}
|
|
155
174
|
exports.MysqlJobStore = MysqlJobStore;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"job-store.js","sourceRoot":"","sources":["../../../dialects/mysql/job-store.ts"],"names":[],"mappings":";;;
|
|
1
|
+
{"version":3,"file":"job-store.js","sourceRoot":"","sources":["../../../dialects/mysql/job-store.ts"],"names":[],"mappings":";;;AAoCA,wDAEC;AAtCD,6CAAyC;AACzC,6CAAwE;AAExE,uDAAyD;AAQzD,qCAAgC;AAIhC;;;;GAIG;AACH,MAAM,MAAM,GAAG,CAAC,KAAe,EAAE,EAAE,CACjC,IAAA,iBAAG,EACD,IAAA,gBAAE,EAAC,aAAI,CAAC,EAAE,EAAE,KAAK,CAAC,EAAE,CAAC,EACrB,IAAA,gBAAE,EAAC,aAAI,CAAC,MAAM,EAAE,YAAY,CAAC,EAC7B,IAAA,gBAAE,EAAC,aAAI,CAAC,SAAS,EAAE,KAAK,CAAC,SAAS,CAAC,EACnC,IAAA,gBAAE,EAAC,aAAI,CAAC,SAAS,EAAE,KAAK,CAAC,SAAS,CAAC,CACpC,CAAC;AAEJ;;;;;;;GAOG;AACH,SAAgB,sBAAsB,CAAC,KAAc;IACnD,OAAO,WAAW,CAAC,KAAK,CAAC,IAAI,WAAW,CAAE,KAA6B,EAAE,KAAK,CAAC,CAAC;AAClF,CAAC;AAED,SAAS,WAAW,CAAC,KAAc;IACjC,OAAO,OAAO,CAAC,KAAK,EAAE,MAAM,EAAE,cAAc,CAAC,IAAI,OAAO,CAAC,KAAK,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC;AACjF,CAAC;AAED,SAAS,OAAO,CAAC,KAAc,EAAE,GAAW,EAAE,KAAc;IAC1D,OAAO,CACL,OAAO,KAAK,KAAK,QAAQ;QACzB,KAAK,KAAK,IAAI;QACd,GAAG,IAAI,KAAK;QACX,KAAiC,CAAC,GAAG,CAAC,KAAK,KAAK,CAClD,CAAC;AACJ,CAAC;AAED;;;;;;;;;GASG;AACH,MAAa,aAAa;IACxB,KAAK,CAAC,OAAO,CAAC,EAAW,EAAE,KAA8B;QACvD,MAAM,EAAE,GAAG,IAAA,wBAAU,GAAE,CAAC;QACxB,MAAM,WAAW,GAAG,IAAA,kCAAkB,EAAC,KAAK,CAAC,CAAC,WAAW,EAAE,CAAC;QAC5D,IAAI,CAAC;YACH,MAAO,EAAS,CAAC,MAAM,CAAC,aAAI,CAAC,CAAC,MAAM,CAAC;gBACnC,EAAE;gBACF,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,yEAAyE;gBACzE,OAAO,EAAE,KAAK,CAAC,OAAkC;gBACjD,MAAM,EAAE,SAAS;gBACjB,WAAW,EAAE,KAAK,CAAC,WAAW,IAAI,EAAE;gBACpC,SAAS,EAAE,KAAK,CAAC,SAAS,IAAI,IAAI;gBAClC,QAAQ,EAAE,KAAK,CAAC,QAAQ,IAAI,CAAC;gBAC7B,WAAW;gBACX,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;aACpC,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,EAAE,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;YAClE,IAAI,CAAC,QAAQ;gBAAE,MAAM,KAAK,CAAC;YAC3B,OAAO,QAAQ,CAAC;QAClB,CAAC;QACD,MAAM,CAAC,GAAG,CAAC,GAAG,MAAO,EAAS,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,aAAI,CAAC,CAAC,KAAK,CAAC,IAAA,gBAAE,EAAC,aAAI,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC;QAC1E,OAAO,GAAG,CAAC;IACb,CAAC;IAED;;;;;;OAMG;IACK,KAAK,CAAC,mBAAmB,CAC/B,EAAW,EACX,KAA8B,EAC9B,KAAc;QAEd,IAAI,CAAC,sBAAsB,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,SAAS,IAAI,IAAI,EAAE,CAAC;YAC9D,OAAO,SAAS,CAAC;QACnB,CAAC;QACD,MAAM,CAAC,QAAQ,CAAC,GAAG,MAAO,EAAS;aAChC,MAAM,EAAE;aACR,IAAI,CAAC,aAAI,CAAC;aACV,KAAK,CAAC,IAAA,iBAAG,EAAC,IAAA,gBAAE,EAAC,aAAI,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,EAAE,IAAA,gBAAE,EAAC,aAAI,CAAC,SAAS,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;QAC9E,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,EAAW,EAAE,GAAyB;QACrD,OAAQ,EAAS,CAAC,WAAW,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE;YACzC,wEAAwE;YACxE,sEAAsE;YACtE,yDAAyD;YACzD,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,MAAM,MAAM,GAAG,GAAG,CAAC,WAAW,EAAE,CAAC;YACjC,MAAM,WAAW,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,GAAG,CAAC,cAAc,CAAC,CAAC,WAAW,EAAE,CAAC;YAC/E,MAAM,UAAU,GAAG,MAAM,EAAE;iBACxB,MAAM,CAAC,EAAE,EAAE,EAAE,aAAI,CAAC,EAAE,EAAE,CAAC;iBACvB,IAAI,CAAC,aAAI,CAAC;iBACV,KAAK,CACJ,IAAA,gBAAE,EACA,IAAA,iBAAG,EAAC,IAAA,gBAAE,EAAC,aAAI,CAAC,MAAM,EAAE,SAAS,CAAC,EAAE,IAAA,iBAAG,EAAC,aAAI,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC,EAC9D,IAAA,iBAAG,EAAC,IAAA,gBAAE,EAAC,aAAI,CAAC,MAAM,EAAE,YAAY,CAAC,EAAE,IAAA,iBAAG,EAAC,aAAI,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC,CACrE,CACF;iBACA,OAAO,CAAC,IAAA,kBAAI,EAAC,aAAI,CAAC,QAAQ,CAAC,EAAE,IAAA,iBAAG,EAAC,aAAI,CAAC,WAAW,CAAC,CAAC;iBACnD,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC;gBACrB,iEAAiE;gBACjE,8DAA8D;iBAC7D,GAAG,CAAC,QAAQ,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC;YACvC,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO,EAAE,CAAC;YACvC,MAAM,GAAG,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YACxC,MAAM,EAAE;iBACL,MAAM,CAAC,aAAI,CAAC;iBACZ,GAAG,CAAC,EAAE,MAAM,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,CAAC,gBAAgB,EAAE,CAAC;iBACjF,KAAK,CAAC,IAAA,qBAAO,EAAC,aAAI,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC,CAAC;YAChC,OAAO,EAAE;iBACN,MAAM,EAAE;iBACR,IAAI,CAAC,aAAI,CAAC;iBACV,KAAK,CAAC,IAAA,qBAAO,EAAC,aAAI,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;iBAC5B,OAAO,CAAC,IAAA,kBAAI,EAAC,aAAI,CAAC,QAAQ,CAAC,EAAE,IAAA,iBAAG,EAAC,aAAI,CAAC,WAAW,CAAC,CAAC,CAAC;YACvD,0EAA0E;YAC1E,wEAAwE;YACxE,sCAAsC;QACxC,CAAC,EAAE,EAAE,cAAc,EAAE,gBAAgB,EAAE,CAAC,CAAC;IAC3C,CAAC;IAED,KAAK,CAAC,aAAa,CAAC,EAAW,EAAE,KAAe;QAC9C,MAAM,CAAC,MAAM,CAAC,GAAG,MAAO,EAAS;aAC9B,MAAM,CAAC,aAAI,CAAC;aACZ,GAAG,CAAC;YACH,MAAM,EAAE,WAAW;YACnB,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACrC,SAAS,EAAE,IAAI;YACf,2CAA2C;YAC3C,SAAS,EAAE,IAAI;SAChB,CAAC;aACD,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;QACxB,OAAO,MAAM,CAAC,YAAY,GAAG,CAAC,CAAC;IACjC,CAAC;IAED,KAAK,CAAC,KAAK,CACT,EAAW,EACX,KAAe,EACf,OAAe,EACf,SAAkB;QAElB,MAAM,aAAa,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,CAAC,WAAW,EAAE,CAAC;QACnE,MAAM,CAAC,MAAM,CAAC,GAAG,MAAO,EAAS;aAC9B,MAAM,CAAC,aAAI,CAAC;aACZ,GAAG,CAAC;YACH,MAAM,EAAE,SAAS;YACjB,QAAQ,EAAE,IAAA,iBAAG,EAAA,GAAG,aAAI,CAAC,QAAQ,MAAM;YACnC,WAAW,EAAE,aAAa;YAC1B,SAAS,EAAE,IAAI;YACf,SAAS,EAAE,IAAI;YACf,SAAS,EAAE,SAAS,IAAI,IAAI;YAC5B,8CAA8C;SAC/C,CAAC;aACD,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;QACxB,OAAO,MAAM,CAAC,YAAY,GAAG,CAAC,CAAC;IACjC,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,EAAW,EAAE,KAAe;QACxC,MAAM,CAAC,MAAM,CAAC,GAAG,MAAO,EAAS;aAC9B,MAAM,CAAC,aAAI,CAAC;aACZ,GAAG,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,SAAS,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;aAC5D,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;QACxB,OAAO,MAAM,CAAC,YAAY,GAAG,CAAC,CAAC;IACjC,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,EAAW,EAAE,KAAe,EAAE,MAAc;QAC3D,MAAM,CAAC,MAAM,CAAC,GAAG,MAAO,EAAS;aAC9B,MAAM,CAAC,aAAI,CAAC;aACZ,GAAG,CAAC;YACH,MAAM,EAAE,QAAQ;YAChB,QAAQ,EAAE,IAAA,iBAAG,EAAA,GAAG,aAAI,CAAC,QAAQ,MAAM;YACnC,SAAS,EAAE,MAAM;YACjB,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACrC,2CAA2C;YAC3C,SAAS,EAAE,IAAI;SAChB,CAAC;aACD,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;QACxB,OAAO,MAAM,CAAC,YAAY,GAAG,CAAC,CAAC;IACjC,CAAC;CACF;AAjJD,sCAiJC"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { EnqueueJobInput, JobRow, JobStore, ResolvedRunnerConfig } from '../../interfaces';
|
|
1
|
+
import type { EnqueueJobInput, JobClaim, JobRow, JobStore, ResolvedRunnerConfig } from '../../interfaces';
|
|
2
2
|
/**
|
|
3
3
|
* Postgres surfaces a unique-constraint violation with SQLSTATE `23505`
|
|
4
4
|
* (`unique_violation`). node-postgres exposes it as `error.code === '23505'`;
|
|
@@ -26,7 +26,8 @@ export declare class PostgresJobStore implements JobStore {
|
|
|
26
26
|
*/
|
|
27
27
|
private findActiveDuplicate;
|
|
28
28
|
claimBatch(db: unknown, cfg: ResolvedRunnerConfig): Promise<JobRow[]>;
|
|
29
|
-
markCompleted(db: unknown,
|
|
30
|
-
retry(db: unknown,
|
|
31
|
-
|
|
29
|
+
markCompleted(db: unknown, claim: JobClaim): Promise<boolean>;
|
|
30
|
+
retry(db: unknown, claim: JobClaim, delayMs: number, lastError?: string): Promise<boolean>;
|
|
31
|
+
release(db: unknown, claim: JobClaim): Promise<boolean>;
|
|
32
|
+
markFailed(db: unknown, claim: JobClaim, reason: string): Promise<boolean>;
|
|
32
33
|
}
|
|
@@ -22,6 +22,22 @@ function hasCode(error, code) {
|
|
|
22
22
|
'code' in error &&
|
|
23
23
|
error.code === code);
|
|
24
24
|
}
|
|
25
|
+
/**
|
|
26
|
+
* Matches the job only while it is `processing` under exactly this claim, so a
|
|
27
|
+
* transition from a worker whose claim was taken over after the stuck timeout
|
|
28
|
+
* writes nothing, even when a later claim reused the same `workerInstanceId`.
|
|
29
|
+
*/
|
|
30
|
+
const heldBy = (claim) => (0, drizzle_orm_1.and)((0, drizzle_orm_1.eq)(schema_1.jobs.id, claim.id), (0, drizzle_orm_1.eq)(schema_1.jobs.status, 'processing'), (0, drizzle_orm_1.eq)(schema_1.jobs.claimedBy, claim.claimedBy), (0, drizzle_orm_1.eq)(schema_1.jobs.claimedAt, claim.claimedAt));
|
|
31
|
+
/**
|
|
32
|
+
* Runs one fenced transition and reports whether it wrote the job. Each runs in
|
|
33
|
+
* its own READ COMMITTED transaction, like the claim: under a SERIALIZABLE
|
|
34
|
+
* server default the fenced UPDATE's scan (on the status index, in steady state)
|
|
35
|
+
* makes two workers' transitions abort each other (40001) while both claims
|
|
36
|
+
* still hold their jobs. Under READ COMMITTED an UPDATE that waited on a
|
|
37
|
+
* reclaim re-checks the fence against the committed row instead, so `false`
|
|
38
|
+
* means the claim really was taken over, and any error is a real one.
|
|
39
|
+
*/
|
|
40
|
+
const fenced = (db, update) => (0, transaction_1.readCommitted)(db, async (tx) => (await update(tx)).length > 0);
|
|
25
41
|
/**
|
|
26
42
|
* Postgres (node-postgres) job store. Every method is **asynchronous** —
|
|
27
43
|
* `enqueue` awaits the insert (call it with `await` inside an async
|
|
@@ -77,12 +93,15 @@ class PostgresJobStore {
|
|
|
77
93
|
return existing;
|
|
78
94
|
}
|
|
79
95
|
async claimBatch(db, cfg) {
|
|
80
|
-
const now = new Date();
|
|
81
|
-
const nowIso = now.toISOString();
|
|
82
|
-
const stuckCutoff = new Date(now.getTime() - cfg.stuckTimeoutMs).toISOString();
|
|
83
96
|
// FOR UPDATE SKIP LOCKED: two workers claiming at once split the backlog
|
|
84
97
|
// instead of both taking (and running) the same jobs.
|
|
85
98
|
return (0, transaction_1.readCommitted)(db, async (tx) => {
|
|
99
|
+
// Stamped once the connection is ours: a checkout that waited on a busy
|
|
100
|
+
// pool must not leave this claim looking older than it is, or another
|
|
101
|
+
// worker would treat its jobs as stuck that much sooner.
|
|
102
|
+
const now = new Date();
|
|
103
|
+
const nowIso = now.toISOString();
|
|
104
|
+
const stuckCutoff = new Date(now.getTime() - cfg.stuckTimeoutMs).toISOString();
|
|
86
105
|
const candidates = await tx
|
|
87
106
|
.select({ id: schema_1.jobs.id })
|
|
88
107
|
.from(schema_1.jobs)
|
|
@@ -104,8 +123,8 @@ class PostgresJobStore {
|
|
|
104
123
|
.orderBy((0, drizzle_orm_1.desc)(schema_1.jobs.priority), (0, drizzle_orm_1.asc)(schema_1.jobs.availableAt));
|
|
105
124
|
});
|
|
106
125
|
}
|
|
107
|
-
async markCompleted(db,
|
|
108
|
-
|
|
126
|
+
async markCompleted(db, claim) {
|
|
127
|
+
return fenced(db, (tx) => tx
|
|
109
128
|
.update(schema_1.jobs)
|
|
110
129
|
.set({
|
|
111
130
|
status: 'completed',
|
|
@@ -114,11 +133,12 @@ class PostgresJobStore {
|
|
|
114
133
|
// Terminal → release the active-dedup key.
|
|
115
134
|
uniqueKey: null,
|
|
116
135
|
})
|
|
117
|
-
.where((
|
|
136
|
+
.where(heldBy(claim))
|
|
137
|
+
.returning({ id: schema_1.jobs.id }));
|
|
118
138
|
}
|
|
119
|
-
async retry(db,
|
|
139
|
+
async retry(db, claim, delayMs, lastError) {
|
|
120
140
|
const nextAvailable = new Date(Date.now() + delayMs).toISOString();
|
|
121
|
-
|
|
141
|
+
return fenced(db, (tx) => tx
|
|
122
142
|
.update(schema_1.jobs)
|
|
123
143
|
.set({
|
|
124
144
|
status: 'pending',
|
|
@@ -129,10 +149,18 @@ class PostgresJobStore {
|
|
|
129
149
|
lastError: lastError ?? null,
|
|
130
150
|
// Still active → the uniqueKey stays claimed.
|
|
131
151
|
})
|
|
132
|
-
.where((
|
|
152
|
+
.where(heldBy(claim))
|
|
153
|
+
.returning({ id: schema_1.jobs.id }));
|
|
154
|
+
}
|
|
155
|
+
async release(db, claim) {
|
|
156
|
+
return fenced(db, (tx) => tx
|
|
157
|
+
.update(schema_1.jobs)
|
|
158
|
+
.set({ status: 'pending', claimedAt: null, claimedBy: null })
|
|
159
|
+
.where(heldBy(claim))
|
|
160
|
+
.returning({ id: schema_1.jobs.id }));
|
|
133
161
|
}
|
|
134
|
-
async markFailed(db,
|
|
135
|
-
|
|
162
|
+
async markFailed(db, claim, reason) {
|
|
163
|
+
return fenced(db, (tx) => tx
|
|
136
164
|
.update(schema_1.jobs)
|
|
137
165
|
.set({
|
|
138
166
|
status: 'failed',
|
|
@@ -142,7 +170,8 @@ class PostgresJobStore {
|
|
|
142
170
|
// Terminal → release the active-dedup key.
|
|
143
171
|
uniqueKey: null,
|
|
144
172
|
})
|
|
145
|
-
.where((
|
|
173
|
+
.where(heldBy(claim))
|
|
174
|
+
.returning({ id: schema_1.jobs.id }));
|
|
146
175
|
}
|
|
147
176
|
}
|
|
148
177
|
exports.PostgresJobStore = PostgresJobStore;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"job-store.js","sourceRoot":"","sources":["../../../dialects/postgres/job-store.ts"],"names":[],"mappings":";;;
|
|
1
|
+
{"version":3,"file":"job-store.js","sourceRoot":"","sources":["../../../dialects/postgres/job-store.ts"],"names":[],"mappings":";;;AAsBA,kDAEC;AAxBD,6CAAyC;AACzC,6CAAwE;AAExE,uDAAyD;AAQzD,qCAAgC;AAChC,+CAA8C;AAI9C;;;;;GAKG;AACH,SAAgB,mBAAmB,CAAC,KAAc;IAChD,OAAO,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,OAAO,CAAE,KAA6B,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;AAC5F,CAAC;AAED,SAAS,OAAO,CAAC,KAAc,EAAE,IAAY;IAC3C,OAAO,CACL,OAAO,KAAK,KAAK,QAAQ;QACzB,KAAK,KAAK,IAAI;QACd,MAAM,IAAI,KAAK;QACd,KAA4B,CAAC,IAAI,KAAK,IAAI,CAC5C,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,MAAM,MAAM,GAAG,CAAC,KAAe,EAAE,EAAE,CACjC,IAAA,iBAAG,EACD,IAAA,gBAAE,EAAC,aAAI,CAAC,EAAE,EAAE,KAAK,CAAC,EAAE,CAAC,EACrB,IAAA,gBAAE,EAAC,aAAI,CAAC,MAAM,EAAE,YAAY,CAAC,EAC7B,IAAA,gBAAE,EAAC,aAAI,CAAC,SAAS,EAAE,KAAK,CAAC,SAAS,CAAC,EACnC,IAAA,gBAAE,EAAC,aAAI,CAAC,SAAS,EAAE,KAAK,CAAC,SAAS,CAAC,CACpC,CAAC;AAEJ;;;;;;;;GAQG;AACH,MAAM,MAAM,GAAG,CACb,EAAW,EACX,MAAiD,EAC/B,EAAE,CAAC,IAAA,2BAAa,EAAC,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC,MAAM,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AAEtF;;;;;;;;GAQG;AACH,MAAa,gBAAgB;IAC3B,KAAK,CAAC,OAAO,CAAC,EAAW,EAAE,KAA8B;QACvD,MAAM,WAAW,GAAG,IAAA,kCAAkB,EAAC,KAAK,CAAC,CAAC,WAAW,EAAE,CAAC;QAC5D,IAAI,CAAC;YACH,MAAM,CAAC,GAAG,CAAC,GAAG,MAAO,EAAS;iBAC3B,MAAM,CAAC,aAAI,CAAC;iBACZ,MAAM,CAAC;gBACN,EAAE,EAAE,IAAA,wBAAU,GAAE;gBAChB,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,yEAAyE;gBACzE,OAAO,EAAE,KAAK,CAAC,OAAkC;gBACjD,MAAM,EAAE,SAAS;gBACjB,WAAW,EAAE,KAAK,CAAC,WAAW,IAAI,EAAE;gBACpC,SAAS,EAAE,KAAK,CAAC,SAAS,IAAI,IAAI;gBAClC,QAAQ,EAAE,KAAK,CAAC,QAAQ,IAAI,CAAC;gBAC7B,WAAW;gBACX,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;aACpC,CAAC;iBACD,SAAS,EAAE,CAAC;YACf,OAAO,GAAG,CAAC;QACb,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,EAAE,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;YAClE,IAAI,CAAC,QAAQ;gBAAE,MAAM,KAAK,CAAC;YAC3B,OAAO,QAAQ,CAAC;QAClB,CAAC;IACH,CAAC;IAED;;;;;;OAMG;IACK,KAAK,CAAC,mBAAmB,CAC/B,EAAW,EACX,KAA8B,EAC9B,KAAc;QAEd,IAAI,CAAC,mBAAmB,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,SAAS,IAAI,IAAI,EAAE,CAAC;YAC3D,OAAO,SAAS,CAAC;QACnB,CAAC;QACD,MAAM,CAAC,QAAQ,CAAC,GAAG,MAAO,EAAS;aAChC,MAAM,EAAE;aACR,IAAI,CAAC,aAAI,CAAC;aACV,KAAK,CAAC,IAAA,iBAAG,EAAC,IAAA,gBAAE,EAAC,aAAI,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,EAAE,IAAA,gBAAE,EAAC,aAAI,CAAC,SAAS,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;QAC9E,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,EAAW,EAAE,GAAyB;QACrD,yEAAyE;QACzE,sDAAsD;QACtD,OAAO,IAAA,2BAAa,EAAC,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,EAAE;YACpC,wEAAwE;YACxE,sEAAsE;YACtE,yDAAyD;YACzD,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,MAAM,MAAM,GAAG,GAAG,CAAC,WAAW,EAAE,CAAC;YACjC,MAAM,WAAW,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,GAAG,CAAC,cAAc,CAAC,CAAC,WAAW,EAAE,CAAC;YAC/E,MAAM,UAAU,GAAG,MAAM,EAAE;iBACxB,MAAM,CAAC,EAAE,EAAE,EAAE,aAAI,CAAC,EAAE,EAAE,CAAC;iBACvB,IAAI,CAAC,aAAI,CAAC;iBACV,KAAK,CACJ,IAAA,gBAAE,EACA,IAAA,iBAAG,EAAC,IAAA,gBAAE,EAAC,aAAI,CAAC,MAAM,EAAE,SAAS,CAAC,EAAE,IAAA,iBAAG,EAAC,aAAI,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC,EAC9D,IAAA,iBAAG,EAAC,IAAA,gBAAE,EAAC,aAAI,CAAC,MAAM,EAAE,YAAY,CAAC,EAAE,IAAA,iBAAG,EAAC,aAAI,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC,CACrE,CACF;iBACA,OAAO,CAAC,IAAA,kBAAI,EAAC,aAAI,CAAC,QAAQ,CAAC,EAAE,IAAA,iBAAG,EAAC,aAAI,CAAC,WAAW,CAAC,CAAC;iBACnD,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC;iBACpB,GAAG,CAAC,QAAQ,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC;YACvC,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO,EAAE,CAAC;YACvC,MAAM,GAAG,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YACxC,MAAM,EAAE;iBACL,MAAM,CAAC,aAAI,CAAC;iBACZ,GAAG,CAAC,EAAE,MAAM,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,CAAC,gBAAgB,EAAE,CAAC;iBACjF,KAAK,CAAC,IAAA,qBAAO,EAAC,aAAI,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC,CAAC;YAChC,OAAO,EAAE;iBACN,MAAM,EAAE;iBACR,IAAI,CAAC,aAAI,CAAC;iBACV,KAAK,CAAC,IAAA,qBAAO,EAAC,aAAI,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;iBAC5B,OAAO,CAAC,IAAA,kBAAI,EAAC,aAAI,CAAC,QAAQ,CAAC,EAAE,IAAA,iBAAG,EAAC,aAAI,CAAC,WAAW,CAAC,CAAC,CAAC;QACzD,CAAC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,aAAa,CAAC,EAAW,EAAE,KAAe;QAC9C,OAAO,MAAM,CAAC,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CACvB,EAAE;aACC,MAAM,CAAC,aAAI,CAAC;aACZ,GAAG,CAAC;YACH,MAAM,EAAE,WAAW;YACnB,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACrC,SAAS,EAAE,IAAI;YACf,2CAA2C;YAC3C,SAAS,EAAE,IAAI;SAChB,CAAC;aACD,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;aACpB,SAAS,CAAC,EAAE,EAAE,EAAE,aAAI,CAAC,EAAE,EAAE,CAAC,CAC9B,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,KAAK,CACT,EAAW,EACX,KAAe,EACf,OAAe,EACf,SAAkB;QAElB,MAAM,aAAa,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,CAAC,WAAW,EAAE,CAAC;QACnE,OAAO,MAAM,CAAC,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CACvB,EAAE;aACC,MAAM,CAAC,aAAI,CAAC;aACZ,GAAG,CAAC;YACH,MAAM,EAAE,SAAS;YACjB,QAAQ,EAAE,IAAA,iBAAG,EAAA,GAAG,aAAI,CAAC,QAAQ,MAAM;YACnC,WAAW,EAAE,aAAa;YAC1B,SAAS,EAAE,IAAI;YACf,SAAS,EAAE,IAAI;YACf,SAAS,EAAE,SAAS,IAAI,IAAI;YAC5B,8CAA8C;SAC/C,CAAC;aACD,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;aACpB,SAAS,CAAC,EAAE,EAAE,EAAE,aAAI,CAAC,EAAE,EAAE,CAAC,CAC9B,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,EAAW,EAAE,KAAe;QACxC,OAAO,MAAM,CAAC,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CACvB,EAAE;aACC,MAAM,CAAC,aAAI,CAAC;aACZ,GAAG,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,SAAS,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;aAC5D,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;aACpB,SAAS,CAAC,EAAE,EAAE,EAAE,aAAI,CAAC,EAAE,EAAE,CAAC,CAC9B,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,EAAW,EAAE,KAAe,EAAE,MAAc;QAC3D,OAAO,MAAM,CAAC,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CACvB,EAAE;aACC,MAAM,CAAC,aAAI,CAAC;aACZ,GAAG,CAAC;YACH,MAAM,EAAE,QAAQ;YAChB,QAAQ,EAAE,IAAA,iBAAG,EAAA,GAAG,aAAI,CAAC,QAAQ,MAAM;YACnC,SAAS,EAAE,MAAM;YACjB,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACrC,2CAA2C;YAC3C,SAAS,EAAE,IAAI;SAChB,CAAC;aACD,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;aACpB,SAAS,CAAC,EAAE,EAAE,EAAE,aAAI,CAAC,EAAE,EAAE,CAAC,CAC9B,CAAC;IACJ,CAAC;CACF;AAvJD,4CAuJC"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { EnqueueJobInput, JobRow, JobStore, ResolvedRunnerConfig } from '../../interfaces';
|
|
1
|
+
import type { EnqueueJobInput, JobClaim, JobRow, JobStore, ResolvedRunnerConfig } from '../../interfaces';
|
|
2
2
|
/**
|
|
3
3
|
* better-sqlite3 surfaces a unique-constraint violation as a `SqliteError` with
|
|
4
4
|
* `code === 'SQLITE_CONSTRAINT_UNIQUE'`. Match on the code (stable across driver
|
|
@@ -26,7 +26,8 @@ export declare class SqliteJobStore implements JobStore {
|
|
|
26
26
|
*/
|
|
27
27
|
private findActiveDuplicate;
|
|
28
28
|
claimBatch(db: unknown, cfg: ResolvedRunnerConfig): Promise<JobRow[]>;
|
|
29
|
-
markCompleted(db: unknown,
|
|
30
|
-
retry(db: unknown,
|
|
31
|
-
|
|
29
|
+
markCompleted(db: unknown, claim: JobClaim): Promise<boolean>;
|
|
30
|
+
retry(db: unknown, claim: JobClaim, delayMs: number, lastError?: string): Promise<boolean>;
|
|
31
|
+
release(db: unknown, claim: JobClaim): Promise<boolean>;
|
|
32
|
+
markFailed(db: unknown, claim: JobClaim, reason: string): Promise<boolean>;
|
|
32
33
|
}
|
|
@@ -22,6 +22,12 @@ function hasCode(error, code) {
|
|
|
22
22
|
'code' in error &&
|
|
23
23
|
error.code === code);
|
|
24
24
|
}
|
|
25
|
+
/**
|
|
26
|
+
* Matches the job only while it is `processing` under exactly this claim, so a
|
|
27
|
+
* transition from a worker whose claim was taken over after the stuck timeout
|
|
28
|
+
* writes nothing, even when a later claim reused the same `workerInstanceId`.
|
|
29
|
+
*/
|
|
30
|
+
const heldBy = (claim) => (0, drizzle_orm_1.and)((0, drizzle_orm_1.eq)(schema_1.jobs.id, claim.id), (0, drizzle_orm_1.eq)(schema_1.jobs.status, 'processing'), (0, drizzle_orm_1.eq)(schema_1.jobs.claimedBy, claim.claimedBy), (0, drizzle_orm_1.eq)(schema_1.jobs.claimedAt, claim.claimedAt));
|
|
25
31
|
/**
|
|
26
32
|
* SQLite (better-sqlite3) job store. Every method runs **synchronously** —
|
|
27
33
|
* `enqueue` returns the row directly so it composes inside a synchronous
|
|
@@ -104,8 +110,8 @@ class SqliteJobStore {
|
|
|
104
110
|
});
|
|
105
111
|
return Promise.resolve(rows);
|
|
106
112
|
}
|
|
107
|
-
markCompleted(db,
|
|
108
|
-
db
|
|
113
|
+
markCompleted(db, claim) {
|
|
114
|
+
const { changes } = db
|
|
109
115
|
.update(schema_1.jobs)
|
|
110
116
|
.set({
|
|
111
117
|
status: 'completed',
|
|
@@ -114,13 +120,13 @@ class SqliteJobStore {
|
|
|
114
120
|
// Terminal → release the active-dedup key.
|
|
115
121
|
uniqueKey: null,
|
|
116
122
|
})
|
|
117
|
-
.where((
|
|
123
|
+
.where(heldBy(claim))
|
|
118
124
|
.run();
|
|
119
|
-
return Promise.resolve();
|
|
125
|
+
return Promise.resolve(changes > 0);
|
|
120
126
|
}
|
|
121
|
-
retry(db,
|
|
127
|
+
retry(db, claim, delayMs, lastError) {
|
|
122
128
|
const nextAvailable = new Date(Date.now() + delayMs).toISOString();
|
|
123
|
-
db
|
|
129
|
+
const { changes } = db
|
|
124
130
|
.update(schema_1.jobs)
|
|
125
131
|
.set({
|
|
126
132
|
status: 'pending',
|
|
@@ -131,12 +137,20 @@ class SqliteJobStore {
|
|
|
131
137
|
lastError: lastError ?? null,
|
|
132
138
|
// Still active → the uniqueKey stays claimed.
|
|
133
139
|
})
|
|
134
|
-
.where((
|
|
140
|
+
.where(heldBy(claim))
|
|
141
|
+
.run();
|
|
142
|
+
return Promise.resolve(changes > 0);
|
|
143
|
+
}
|
|
144
|
+
release(db, claim) {
|
|
145
|
+
const { changes } = db
|
|
146
|
+
.update(schema_1.jobs)
|
|
147
|
+
.set({ status: 'pending', claimedAt: null, claimedBy: null })
|
|
148
|
+
.where(heldBy(claim))
|
|
135
149
|
.run();
|
|
136
|
-
return Promise.resolve();
|
|
150
|
+
return Promise.resolve(changes > 0);
|
|
137
151
|
}
|
|
138
|
-
markFailed(db,
|
|
139
|
-
db
|
|
152
|
+
markFailed(db, claim, reason) {
|
|
153
|
+
const { changes } = db
|
|
140
154
|
.update(schema_1.jobs)
|
|
141
155
|
.set({
|
|
142
156
|
status: 'failed',
|
|
@@ -146,9 +160,9 @@ class SqliteJobStore {
|
|
|
146
160
|
// Terminal → release the active-dedup key.
|
|
147
161
|
uniqueKey: null,
|
|
148
162
|
})
|
|
149
|
-
.where((
|
|
163
|
+
.where(heldBy(claim))
|
|
150
164
|
.run();
|
|
151
|
-
return Promise.resolve();
|
|
165
|
+
return Promise.resolve(changes > 0);
|
|
152
166
|
}
|
|
153
167
|
}
|
|
154
168
|
exports.SqliteJobStore = SqliteJobStore;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"job-store.js","sourceRoot":"","sources":["../../../dialects/sqlite/job-store.ts"],"names":[],"mappings":";;;
|
|
1
|
+
{"version":3,"file":"job-store.js","sourceRoot":"","sources":["../../../dialects/sqlite/job-store.ts"],"names":[],"mappings":";;;AAqBA,0DAGC;AAxBD,6CAAyC;AACzC,6CAAwE;AAExE,uDAAyD;AAQzD,qCAAgC;AAIhC;;;;;GAKG;AACH,SAAgB,uBAAuB,CAAC,KAAc;IACpD,MAAM,IAAI,GAAG,0BAA0B,CAAC;IACxC,OAAO,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,OAAO,CAAE,KAA6B,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC;AACtF,CAAC;AAED,SAAS,OAAO,CAAC,KAAc,EAAE,IAAY;IAC3C,OAAO,CACL,OAAO,KAAK,KAAK,QAAQ;QACzB,KAAK,KAAK,IAAI;QACd,MAAM,IAAI,KAAK;QACd,KAA4B,CAAC,IAAI,KAAK,IAAI,CAC5C,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,MAAM,MAAM,GAAG,CAAC,KAAe,EAAE,EAAE,CACjC,IAAA,iBAAG,EACD,IAAA,gBAAE,EAAC,aAAI,CAAC,EAAE,EAAE,KAAK,CAAC,EAAE,CAAC,EACrB,IAAA,gBAAE,EAAC,aAAI,CAAC,MAAM,EAAE,YAAY,CAAC,EAC7B,IAAA,gBAAE,EAAC,aAAI,CAAC,SAAS,EAAE,KAAK,CAAC,SAAS,CAAC,EACnC,IAAA,gBAAE,EAAC,aAAI,CAAC,SAAS,EAAE,KAAK,CAAC,SAAS,CAAC,CACpC,CAAC;AAEJ;;;;;;;;GAQG;AACH,MAAa,cAAc;IACzB,OAAO,CAAC,EAAW,EAAE,KAA8B;QACjD,MAAM,WAAW,GAAG,IAAA,kCAAkB,EAAC,KAAK,CAAC,CAAC,WAAW,EAAE,CAAC;QAC5D,IAAI,CAAC;YACH,OAAQ,EAAS;iBACd,MAAM,CAAC,aAAI,CAAC;iBACZ,MAAM,CAAC;gBACN,EAAE,EAAE,IAAA,wBAAU,GAAE;gBAChB,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,yEAAyE;gBACzE,OAAO,EAAE,KAAK,CAAC,OAAkC;gBACjD,MAAM,EAAE,SAAS;gBACjB,WAAW,EAAE,KAAK,CAAC,WAAW,IAAI,EAAE;gBACpC,SAAS,EAAE,KAAK,CAAC,SAAS,IAAI,IAAI;gBAClC,QAAQ,EAAE,KAAK,CAAC,QAAQ,IAAI,CAAC;gBAC7B,WAAW;gBACX,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;aACpC,CAAC;iBACD,SAAS,EAAE;iBACX,GAAG,EAAE,CAAC;QACX,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,QAAQ,GAAG,IAAI,CAAC,mBAAmB,CAAC,EAAE,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;YAC5D,IAAI,CAAC,QAAQ;gBAAE,MAAM,KAAK,CAAC;YAC3B,OAAO,QAAQ,CAAC;QAClB,CAAC;IACH,CAAC;IAED;;;;;;OAMG;IACK,mBAAmB,CACzB,EAAW,EACX,KAA8B,EAC9B,KAAc;QAEd,IAAI,CAAC,uBAAuB,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,SAAS,IAAI,IAAI,EAAE,CAAC;YAC/D,OAAO,SAAS,CAAC;QACnB,CAAC;QACD,OAAQ,EAAS;aACd,MAAM,EAAE;aACR,IAAI,CAAC,aAAI,CAAC;aACV,KAAK,CAAC,IAAA,iBAAG,EAAC,IAAA,gBAAE,EAAC,aAAI,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,EAAE,IAAA,gBAAE,EAAC,aAAI,CAAC,SAAS,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC;aAC1E,GAAG,EAAE,CAAC;IACX,CAAC;IAED,UAAU,CAAC,EAAW,EAAE,GAAyB;QAC/C,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,MAAM,GAAG,GAAG,CAAC,WAAW,EAAE,CAAC;QACjC,MAAM,WAAW,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,GAAG,CAAC,cAAc,CAAC,CAAC,WAAW,EAAE,CAAC;QAC/E,MAAM,IAAI,GAAI,EAAS,CAAC,WAAW,CAAC,CAAC,EAAE,EAAE,EAAE;YACzC,MAAM,UAAU,GAAG,EAAE;iBAClB,MAAM,CAAC,EAAE,EAAE,EAAE,aAAI,CAAC,EAAE,EAAE,CAAC;iBACvB,IAAI,CAAC,aAAI,CAAC;iBACV,KAAK,CACJ,IAAA,gBAAE,EACA,IAAA,iBAAG,EAAC,IAAA,gBAAE,EAAC,aAAI,CAAC,MAAM,EAAE,SAAS,CAAC,EAAE,IAAA,iBAAG,EAAC,aAAI,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC,EAC9D,IAAA,iBAAG,EAAC,IAAA,gBAAE,EAAC,aAAI,CAAC,MAAM,EAAE,YAAY,CAAC,EAAE,IAAA,iBAAG,EAAC,aAAI,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC,CACrE,CACF;iBACA,OAAO,CAAC,IAAA,kBAAI,EAAC,aAAI,CAAC,QAAQ,CAAC,EAAE,IAAA,iBAAG,EAAC,aAAI,CAAC,WAAW,CAAC,CAAC;iBACnD,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC;iBACpB,GAAG,EAAE,CAAC;YACT,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO,EAAE,CAAC;YACvC,MAAM,GAAG,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YACxC,EAAE,CAAC,MAAM,CAAC,aAAI,CAAC;iBACZ,GAAG,CAAC,EAAE,MAAM,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,CAAC,gBAAgB,EAAE,CAAC;iBACjF,KAAK,CAAC,IAAA,qBAAO,EAAC,aAAI,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;iBAC5B,GAAG,EAAE,CAAC;YACT,OAAO,EAAE;iBACN,MAAM,EAAE;iBACR,IAAI,CAAC,aAAI,CAAC;iBACV,KAAK,CAAC,IAAA,qBAAO,EAAC,aAAI,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;iBAC5B,OAAO,CAAC,IAAA,kBAAI,EAAC,aAAI,CAAC,QAAQ,CAAC,EAAE,IAAA,iBAAG,EAAC,aAAI,CAAC,WAAW,CAAC,CAAC;iBACnD,GAAG,EAAE,CAAC;QACX,CAAC,CAAC,CAAC;QACH,OAAO,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IAC/B,CAAC;IAED,aAAa,CAAC,EAAW,EAAE,KAAe;QACxC,MAAM,EAAE,OAAO,EAAE,GAAI,EAAS;aAC3B,MAAM,CAAC,aAAI,CAAC;aACZ,GAAG,CAAC;YACH,MAAM,EAAE,WAAW;YACnB,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACrC,SAAS,EAAE,IAAI;YACf,2CAA2C;YAC3C,SAAS,EAAE,IAAI;SAChB,CAAC;aACD,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;aACpB,GAAG,EAAE,CAAC;QACT,OAAO,OAAO,CAAC,OAAO,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC;IACtC,CAAC;IAED,KAAK,CAAC,EAAW,EAAE,KAAe,EAAE,OAAe,EAAE,SAAkB;QACrE,MAAM,aAAa,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,CAAC,WAAW,EAAE,CAAC;QACnE,MAAM,EAAE,OAAO,EAAE,GAAI,EAAS;aAC3B,MAAM,CAAC,aAAI,CAAC;aACZ,GAAG,CAAC;YACH,MAAM,EAAE,SAAS;YACjB,QAAQ,EAAE,IAAA,iBAAG,EAAA,GAAG,aAAI,CAAC,QAAQ,MAAM;YACnC,WAAW,EAAE,aAAa;YAC1B,SAAS,EAAE,IAAI;YACf,SAAS,EAAE,IAAI;YACf,SAAS,EAAE,SAAS,IAAI,IAAI;YAC5B,8CAA8C;SAC/C,CAAC;aACD,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;aACpB,GAAG,EAAE,CAAC;QACT,OAAO,OAAO,CAAC,OAAO,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC;IACtC,CAAC;IAED,OAAO,CAAC,EAAW,EAAE,KAAe;QAClC,MAAM,EAAE,OAAO,EAAE,GAAI,EAAS;aAC3B,MAAM,CAAC,aAAI,CAAC;aACZ,GAAG,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,SAAS,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;aAC5D,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;aACpB,GAAG,EAAE,CAAC;QACT,OAAO,OAAO,CAAC,OAAO,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC;IACtC,CAAC;IAED,UAAU,CAAC,EAAW,EAAE,KAAe,EAAE,MAAc;QACrD,MAAM,EAAE,OAAO,EAAE,GAAI,EAAS;aAC3B,MAAM,CAAC,aAAI,CAAC;aACZ,GAAG,CAAC;YACH,MAAM,EAAE,QAAQ;YAChB,QAAQ,EAAE,IAAA,iBAAG,EAAA,GAAG,aAAI,CAAC,QAAQ,MAAM;YACnC,SAAS,EAAE,MAAM;YACjB,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACrC,2CAA2C;YAC3C,SAAS,EAAE,IAAI;SAChB,CAAC;aACD,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;aACpB,GAAG,EAAE,CAAC;QACT,OAAO,OAAO,CAAC,OAAO,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC;IACtC,CAAC;CACF;AA3ID,wCA2IC"}
|
package/dist/interfaces.d.ts
CHANGED
|
@@ -92,17 +92,51 @@ export interface JobContext {
|
|
|
92
92
|
*
|
|
93
93
|
* `claimBatch` opens its own transaction, claims due `pending` jobs plus
|
|
94
94
|
* `processing` jobs stuck past `stuckTimeoutMs`, ordered by `priority DESC,
|
|
95
|
-
* available_at ASC`, and marks them `processing`.
|
|
95
|
+
* available_at ASC`, and marks them `processing`. It must be atomic across
|
|
96
|
+
* workers: two concurrent calls never return the same job (the shipped
|
|
97
|
+
* Postgres and MySQL stores use `FOR UPDATE SKIP LOCKED`). It stamps every job
|
|
98
|
+
* it returns with `claimedBy` and `claimedAt`, and returns each job as that
|
|
99
|
+
* UPDATE left it. The claimer refuses a job without a string stamp, but it
|
|
100
|
+
* cannot tell a stale stamp from the one the claim wrote: a reclaimed job read
|
|
101
|
+
* before the UPDATE carries the previous claim's stamp, and runs under a claim
|
|
102
|
+
* no transition matches, again after every stuck timeout.
|
|
103
|
+
*
|
|
104
|
+
* The three transitions take that stamp back as a {@link JobClaim} and apply
|
|
105
|
+
* only while the job still holds it, resolving `false` (and writing nothing)
|
|
106
|
+
* once the claim has been taken over. A database error rejects; it never reads
|
|
107
|
+
* as `false`.
|
|
96
108
|
*/
|
|
97
109
|
export interface JobStore {
|
|
98
110
|
enqueue(db: unknown, input: EnqueueJobInput<object>): JobRow | Promise<JobRow>;
|
|
99
111
|
claimBatch(db: unknown, cfg: ResolvedRunnerConfig): Promise<JobRow[]>;
|
|
100
112
|
/** Terminal: sets `completed` and clears `uniqueKey` (releases the key). */
|
|
101
|
-
markCompleted(db: unknown,
|
|
113
|
+
markCompleted(db: unknown, claim: JobClaim): Promise<boolean>;
|
|
102
114
|
/** Re-arms the job: `pending`, attempts+1, due in `delayMs`. Keeps `uniqueKey`. */
|
|
103
|
-
retry(db: unknown,
|
|
115
|
+
retry(db: unknown, claim: JobClaim, delayMs: number, lastError?: string): Promise<boolean>;
|
|
104
116
|
/** Terminal: sets `failed`, attempts+1, and clears `uniqueKey`. */
|
|
105
|
-
markFailed(db: unknown,
|
|
117
|
+
markFailed(db: unknown, claim: JobClaim, reason: string): Promise<boolean>;
|
|
118
|
+
/**
|
|
119
|
+
* Optional. Hands a claimed job back untouched while the claim still holds
|
|
120
|
+
* it: `pending` and unclaimed, with its attempts and due time unchanged.
|
|
121
|
+
* Resolves `false` once the claim has been taken over. The claimer calls it
|
|
122
|
+
* for the batch's jobs that have not run yet when recording an outcome
|
|
123
|
+
* failed, so they are available again at once instead of after
|
|
124
|
+
* `stuckTimeoutMs`; a store without it leaves them to the stuck timeout.
|
|
125
|
+
*/
|
|
126
|
+
release?(db: unknown, claim: JobClaim): Promise<boolean>;
|
|
127
|
+
}
|
|
128
|
+
/**
|
|
129
|
+
* The claim a job transition must still hold: the job's `id` plus the stamp
|
|
130
|
+
* `claimBatch` wrote. A transition matches the job only while it is
|
|
131
|
+
* `processing` under exactly this `claimedBy` and `claimedAt`, so a worker
|
|
132
|
+
* whose claim was taken over after the stuck timeout (by another worker, or by
|
|
133
|
+
* a later claim under the same `workerInstanceId`) can never overwrite the new
|
|
134
|
+
* owner's outcome.
|
|
135
|
+
*/
|
|
136
|
+
export interface JobClaim {
|
|
137
|
+
readonly id: string;
|
|
138
|
+
readonly claimedBy: string;
|
|
139
|
+
readonly claimedAt: string;
|
|
106
140
|
}
|
|
107
141
|
/**
|
|
108
142
|
* The dialect-agnostic shape of a `job_schedules` row. Like {@link JobRow},
|
|
@@ -1,6 +1,14 @@
|
|
|
1
1
|
import type { JobStore, ResolvedRunnerConfig, RunnerConfig, ScheduleStore } from './interfaces';
|
|
2
2
|
import { JobsHandlerExplorer } from './jobs-handler.explorer';
|
|
3
3
|
export declare const DEFAULT_RUNNER_CONFIG: ResolvedRunnerConfig;
|
|
4
|
+
/**
|
|
5
|
+
* Applies `overrides` over {@link DEFAULT_RUNNER_CONFIG}, as `tick()` does. A
|
|
6
|
+
* key set to `undefined` keeps its default: `{ workerInstanceId:
|
|
7
|
+
* process.env.WORKER_ID }` with the variable unset must not claim jobs under no
|
|
8
|
+
* owner, which no transition could then match. A value that would break
|
|
9
|
+
* claiming throws, so a worker can call this at startup to check its config.
|
|
10
|
+
*/
|
|
11
|
+
export declare function resolveRunnerConfig(overrides?: RunnerConfig | null): ResolvedRunnerConfig;
|
|
4
12
|
export interface TickReport {
|
|
5
13
|
/** Schedule occurrences THIS tick enqueued (won claims; dedup-suppressed occurrences still count as claimed). */
|
|
6
14
|
scheduled: number;
|
|
@@ -8,6 +16,16 @@ export interface TickReport {
|
|
|
8
16
|
completed: number;
|
|
9
17
|
retried: number;
|
|
10
18
|
failed: number;
|
|
19
|
+
/**
|
|
20
|
+
* Claimed jobs this worker let go without recording an outcome. Either the
|
|
21
|
+
* batch had been held longer than `stuckTimeoutMs` by the time the job's turn
|
|
22
|
+
* came, so it never ran here, or another claim took the job over before the
|
|
23
|
+
* outcome could be recorded; whoever claims such a job next runs it, and a
|
|
24
|
+
* steady non-zero count means batches take longer than `stuckTimeoutMs`. A
|
|
25
|
+
* job the store returned without its claim's stamp counts here too, logged
|
|
26
|
+
* as an error.
|
|
27
|
+
*/
|
|
28
|
+
lost: number;
|
|
11
29
|
}
|
|
12
30
|
/**
|
|
13
31
|
* Executes committed jobs. `tick()` claims a batch of due jobs (the store opens
|
|
@@ -22,6 +40,10 @@ export interface TickReport {
|
|
|
22
40
|
* until `maxAttempts`, then `markFailed`
|
|
23
41
|
* - no handler registered → `PermanentError` → `markFailed`
|
|
24
42
|
*
|
|
43
|
+
* Each outcome is recorded under the job's claim: once another worker has
|
|
44
|
+
* taken the job over (its claim went stale after `stuckTimeoutMs`), the
|
|
45
|
+
* transition writes nothing, and the job counts as `lost` instead.
|
|
46
|
+
*
|
|
25
47
|
* Runs in a background worker (see `runWorkerLoop`) — never inside a business
|
|
26
48
|
* transaction — so it freely awaits the store and the handlers.
|
|
27
49
|
*/
|
|
@@ -45,6 +67,27 @@ export declare class JobsClaimer {
|
|
|
45
67
|
private drainSchedules;
|
|
46
68
|
private fireSchedule;
|
|
47
69
|
private processOne;
|
|
70
|
+
/**
|
|
71
|
+
* After recording an outcome failed, hands the batch's jobs that have not run
|
|
72
|
+
* back through the store's optional `release`, so they are available again
|
|
73
|
+
* at once rather than after `stuckTimeoutMs`. Best effort: the database just
|
|
74
|
+
* failed, so the first error ends it, and the original error is the one the
|
|
75
|
+
* tick throws.
|
|
76
|
+
*/
|
|
77
|
+
private releaseNotRun;
|
|
78
|
+
/**
|
|
79
|
+
* The claim to run under, or `undefined` when `claimedBy` or `claimedAt` is
|
|
80
|
+
* not a string: run without a usable stamp, the job would execute with no
|
|
81
|
+
* transition able to record it, and again after every stuck timeout. Whether
|
|
82
|
+
* a string stamp is the one the claim wrote cannot be told apart here; that
|
|
83
|
+
* is the store's contract.
|
|
84
|
+
*/
|
|
85
|
+
private claimOf;
|
|
86
|
+
/**
|
|
87
|
+
* `outcome` when the transition applied; otherwise another claim took the
|
|
88
|
+
* job over first. `reason` is the handler's error, for the outcomes it caused.
|
|
89
|
+
*/
|
|
90
|
+
private settle;
|
|
48
91
|
private onHandlerError;
|
|
49
92
|
private fail;
|
|
50
93
|
private backoff;
|
|
@@ -14,6 +14,7 @@ var __param = (this && this.__param) || function (paramIndex, decorator) {
|
|
|
14
14
|
var JobsClaimer_1;
|
|
15
15
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
16
16
|
exports.JobsClaimer = exports.DEFAULT_RUNNER_CONFIG = void 0;
|
|
17
|
+
exports.resolveRunnerConfig = resolveRunnerConfig;
|
|
17
18
|
const node_os_1 = require("node:os");
|
|
18
19
|
const common_1 = require("@nestjs/common");
|
|
19
20
|
const errors_1 = require("./errors");
|
|
@@ -27,6 +28,46 @@ exports.DEFAULT_RUNNER_CONFIG = {
|
|
|
27
28
|
baseBackoffMs: 1_000,
|
|
28
29
|
maxBackoffMs: 60_000,
|
|
29
30
|
};
|
|
31
|
+
// The furthest a Date can sit from the epoch: a stuck cutoff beyond it is an
|
|
32
|
+
// invalid date, and every claim would throw.
|
|
33
|
+
const MAX_DATE_OFFSET_MS = 8.64e15;
|
|
34
|
+
// A stamp value for a log line: a string quoted, anything else by its type, so
|
|
35
|
+
// a Date does not pass for a valid stamp and a BigInt cannot throw.
|
|
36
|
+
const stampShown = (value) => typeof value === 'string' ? JSON.stringify(value) : Object.prototype.toString.call(value).slice(8, -1);
|
|
37
|
+
// A thrown value as a log line can show it.
|
|
38
|
+
const messageOf = (error) => error instanceof Error ? error.message : String(error);
|
|
39
|
+
// A rejected value as the caller wrote it: a string from an env-backed config
|
|
40
|
+
// keeps its quotes, so `"60000" (string)` does not pass for the number 60000.
|
|
41
|
+
const shown = (value) => typeof value === 'number' ? String(value) : `${JSON.stringify(value)} (${typeof value})`;
|
|
42
|
+
/**
|
|
43
|
+
* Applies `overrides` over {@link DEFAULT_RUNNER_CONFIG}, as `tick()` does. A
|
|
44
|
+
* key set to `undefined` keeps its default: `{ workerInstanceId:
|
|
45
|
+
* process.env.WORKER_ID }` with the variable unset must not claim jobs under no
|
|
46
|
+
* owner, which no transition could then match. A value that would break
|
|
47
|
+
* claiming throws, so a worker can call this at startup to check its config.
|
|
48
|
+
*/
|
|
49
|
+
function resolveRunnerConfig(overrides = {}) {
|
|
50
|
+
const defined = Object.fromEntries(Object.entries(overrides ?? {}).filter(([, value]) => value !== undefined));
|
|
51
|
+
const cfg = { ...exports.DEFAULT_RUNNER_CONFIG, ...defined };
|
|
52
|
+
if (typeof cfg.workerInstanceId !== 'string' || cfg.workerInstanceId.trim() === '') {
|
|
53
|
+
throw new TypeError(`runner workerInstanceId must be a non-empty string, got ${shown(cfg.workerInstanceId)}`);
|
|
54
|
+
}
|
|
55
|
+
if (!Number.isInteger(cfg.batchSize) || cfg.batchSize < 1) {
|
|
56
|
+
throw new RangeError(`runner batchSize must be a positive integer, got ${shown(cfg.batchSize)}`);
|
|
57
|
+
}
|
|
58
|
+
// At zero every processing job is stuck the moment it is claimed, so two
|
|
59
|
+
// workers would run the same job side by side.
|
|
60
|
+
const stuck = cfg.stuckTimeoutMs;
|
|
61
|
+
if (!(typeof stuck === 'number' && stuck > 0 && stuck <= MAX_DATE_OFFSET_MS)) {
|
|
62
|
+
throw new RangeError(`runner stuckTimeoutMs must be a positive number of milliseconds within Date's range, got ${shown(stuck)}`);
|
|
63
|
+
}
|
|
64
|
+
for (const key of ['baseBackoffMs', 'maxBackoffMs']) {
|
|
65
|
+
if (!(Number.isFinite(cfg[key]) && cfg[key] >= 0)) {
|
|
66
|
+
throw new RangeError(`runner ${key} must be a non-negative number, got ${shown(cfg[key])}`);
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
return cfg;
|
|
70
|
+
}
|
|
30
71
|
/**
|
|
31
72
|
* Executes committed jobs. `tick()` claims a batch of due jobs (the store opens
|
|
32
73
|
* its own transaction; priority first, oldest due first, reclaiming jobs stuck
|
|
@@ -40,6 +81,10 @@ exports.DEFAULT_RUNNER_CONFIG = {
|
|
|
40
81
|
* until `maxAttempts`, then `markFailed`
|
|
41
82
|
* - no handler registered → `PermanentError` → `markFailed`
|
|
42
83
|
*
|
|
84
|
+
* Each outcome is recorded under the job's claim: once another worker has
|
|
85
|
+
* taken the job over (its claim went stale after `stuckTimeoutMs`), the
|
|
86
|
+
* transition writes nothing, and the job counts as `lost` instead.
|
|
87
|
+
*
|
|
43
88
|
* Runs in a background worker (see `runWorkerLoop`) — never inside a business
|
|
44
89
|
* transaction — so it freely awaits the store and the handlers.
|
|
45
90
|
*/
|
|
@@ -52,21 +97,42 @@ let JobsClaimer = JobsClaimer_1 = class JobsClaimer {
|
|
|
52
97
|
this.logger = new common_1.Logger(JobsClaimer_1.name);
|
|
53
98
|
}
|
|
54
99
|
async tick(overrides = {}) {
|
|
55
|
-
const cfg =
|
|
100
|
+
const cfg = resolveRunnerConfig(overrides);
|
|
56
101
|
// Due schedules fire first, so their occurrences (due immediately) are
|
|
57
102
|
// claimable by this very tick's batch claim.
|
|
58
103
|
const scheduled = this.scheduleStore ? await this.drainSchedules(cfg) : 0;
|
|
59
104
|
const claimed = await this.store.claimBatch(this.db, cfg);
|
|
105
|
+
// Measured from the batch's arrival, so its first job always runs and a
|
|
106
|
+
// claim that itself took longer than stuckTimeoutMs still makes progress.
|
|
107
|
+
const heldSince = Date.now();
|
|
60
108
|
const report = {
|
|
61
109
|
scheduled,
|
|
62
110
|
claimed: claimed.length,
|
|
63
111
|
completed: 0,
|
|
64
112
|
retried: 0,
|
|
65
113
|
failed: 0,
|
|
114
|
+
lost: 0,
|
|
66
115
|
};
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
116
|
+
// Once the batch has been held past stuckTimeoutMs, another worker may
|
|
117
|
+
// already have reclaimed its remaining jobs; running them here as well
|
|
118
|
+
// would only run them twice.
|
|
119
|
+
const expired = [];
|
|
120
|
+
for (const [index, job] of claimed.entries()) {
|
|
121
|
+
if (Date.now() - heldSince >= cfg.stuckTimeoutMs) {
|
|
122
|
+
expired.push(job.id);
|
|
123
|
+
continue;
|
|
124
|
+
}
|
|
125
|
+
try {
|
|
126
|
+
report[await this.processOne(job, cfg)] += 1;
|
|
127
|
+
}
|
|
128
|
+
catch (error) {
|
|
129
|
+
await this.releaseNotRun(claimed.slice(index + 1));
|
|
130
|
+
throw error;
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
if (expired.length > 0) {
|
|
134
|
+
report.lost += expired.length;
|
|
135
|
+
this.logger.warn(`skipped ${expired.length} claimed job(s) [${expired.join(', ')}]: the batch was held longer than stuckTimeoutMs (${cfg.stuckTimeoutMs} ms), so another worker may own them now; raise stuckTimeoutMs or lower batchSize`);
|
|
70
136
|
}
|
|
71
137
|
return report;
|
|
72
138
|
}
|
|
@@ -116,12 +182,15 @@ let JobsClaimer = JobsClaimer_1 = class JobsClaimer {
|
|
|
116
182
|
// A transient store error (connection drop, lock timeout, serialization
|
|
117
183
|
// failure) must NOT kill the schedule: nothing was written, the row is
|
|
118
184
|
// still due, and the next tick retries it naturally.
|
|
119
|
-
const message =
|
|
185
|
+
const message = messageOf(error);
|
|
120
186
|
this.logger.warn(`schedule ${schedule.id} ("${schedule.name}") claim failed, will retry next tick: ${message}`);
|
|
121
187
|
return 0;
|
|
122
188
|
}
|
|
123
189
|
}
|
|
124
190
|
async processOne(job, cfg) {
|
|
191
|
+
const claim = this.claimOf(job);
|
|
192
|
+
if (claim === undefined)
|
|
193
|
+
return 'lost';
|
|
125
194
|
try {
|
|
126
195
|
const handler = this.explorer.get(job.name);
|
|
127
196
|
if (!handler) {
|
|
@@ -131,36 +200,94 @@ let JobsClaimer = JobsClaimer_1 = class JobsClaimer {
|
|
|
131
200
|
jobId: job.id,
|
|
132
201
|
attempt: job.attempts + 1,
|
|
133
202
|
});
|
|
134
|
-
await this.store.markCompleted(this.db, job.id);
|
|
135
|
-
return 'completed';
|
|
136
203
|
}
|
|
137
204
|
catch (error) {
|
|
138
|
-
return this.onHandlerError(job, cfg, error);
|
|
205
|
+
return this.onHandlerError(job, claim, cfg, error);
|
|
139
206
|
}
|
|
207
|
+
// Outside the try: failing to record a completion is not a failed run.
|
|
208
|
+
// Treating it as one spent an attempt and could mark a job that ran failed.
|
|
209
|
+
// The tick throws instead, and the job, still claimed, runs again once its
|
|
210
|
+
// claim goes stale.
|
|
211
|
+
return this.settle(job, await this.store.markCompleted(this.db, claim), 'completed');
|
|
212
|
+
}
|
|
213
|
+
/**
|
|
214
|
+
* After recording an outcome failed, hands the batch's jobs that have not run
|
|
215
|
+
* back through the store's optional `release`, so they are available again
|
|
216
|
+
* at once rather than after `stuckTimeoutMs`. Best effort: the database just
|
|
217
|
+
* failed, so the first error ends it, and the original error is the one the
|
|
218
|
+
* tick throws.
|
|
219
|
+
*/
|
|
220
|
+
async releaseNotRun(jobs) {
|
|
221
|
+
const release = this.store.release?.bind(this.store);
|
|
222
|
+
if (release === undefined || jobs.length === 0)
|
|
223
|
+
return;
|
|
224
|
+
let released = 0;
|
|
225
|
+
try {
|
|
226
|
+
for (const { id, claimedBy, claimedAt } of jobs) {
|
|
227
|
+
if (typeof claimedBy !== 'string' || typeof claimedAt !== 'string')
|
|
228
|
+
continue;
|
|
229
|
+
if (await release(this.db, { id, claimedBy, claimedAt }))
|
|
230
|
+
released += 1;
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
catch (error) {
|
|
234
|
+
this.logger.warn(`could not hand back ${jobs.length - released} job(s) that had not run after a failed transition (${messageOf(error)}); they are reclaimed after stuckTimeoutMs`);
|
|
235
|
+
return;
|
|
236
|
+
}
|
|
237
|
+
if (released > 0) {
|
|
238
|
+
this.logger.warn(`handed back ${released} job(s) that had not run after a failed transition, for the next claim to take`);
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
/**
|
|
242
|
+
* The claim to run under, or `undefined` when `claimedBy` or `claimedAt` is
|
|
243
|
+
* not a string: run without a usable stamp, the job would execute with no
|
|
244
|
+
* transition able to record it, and again after every stuck timeout. Whether
|
|
245
|
+
* a string stamp is the one the claim wrote cannot be told apart here; that
|
|
246
|
+
* is the store's contract.
|
|
247
|
+
*/
|
|
248
|
+
claimOf(job) {
|
|
249
|
+
const { id, claimedBy, claimedAt } = job;
|
|
250
|
+
if (typeof claimedBy === 'string' && typeof claimedAt === 'string') {
|
|
251
|
+
return { id, claimedBy, claimedAt };
|
|
252
|
+
}
|
|
253
|
+
this.logger.error(`job ${id} came back from claimBatch without a string claim stamp (claimedBy ${stampShown(claimedBy)}, claimedAt ${stampShown(claimedAt)}); a store must return every job it claims as its claiming UPDATE left it, with claimedBy and claimedAt as strings`);
|
|
254
|
+
return undefined;
|
|
255
|
+
}
|
|
256
|
+
/**
|
|
257
|
+
* `outcome` when the transition applied; otherwise another claim took the
|
|
258
|
+
* job over first. `reason` is the handler's error, for the outcomes it caused.
|
|
259
|
+
*/
|
|
260
|
+
settle(job, applied, outcome, reason) {
|
|
261
|
+
if (applied)
|
|
262
|
+
return outcome;
|
|
263
|
+
this.logger.warn(reason === undefined
|
|
264
|
+
? `job ${job.id} ("${job.name}") ran, but another claim took it over before it was marked completed; the new owner records its outcome and may run it again`
|
|
265
|
+
: `job ${job.id} ("${job.name}") failed (${reason}), but another claim took it over before it was marked ${outcome}; leaving it to the new owner`);
|
|
266
|
+
return 'lost';
|
|
140
267
|
}
|
|
141
|
-
async onHandlerError(job, cfg, error) {
|
|
142
|
-
const message =
|
|
268
|
+
async onHandlerError(job, claim, cfg, error) {
|
|
269
|
+
const message = messageOf(error);
|
|
143
270
|
// Permanent: retrying can never succeed — fail now instead of burning attempts.
|
|
144
271
|
if (error instanceof errors_1.PermanentError) {
|
|
145
|
-
return this.fail(job, message);
|
|
272
|
+
return this.fail(job, claim, message);
|
|
146
273
|
}
|
|
147
274
|
// Retryable: schedule another attempt, honouring a handler-supplied delay.
|
|
148
275
|
if (error instanceof errors_1.RetryableError) {
|
|
149
276
|
const delay = error.delayMs ?? this.backoff(job.attempts, cfg);
|
|
150
|
-
await this.store.retry(this.db,
|
|
151
|
-
return 'retried';
|
|
277
|
+
return this.settle(job, await this.store.retry(this.db, claim, delay, message), 'retried', message);
|
|
152
278
|
}
|
|
153
279
|
// Anything else: retry with backoff until maxAttempts, then fail.
|
|
154
280
|
if (job.attempts + 1 >= job.maxAttempts) {
|
|
155
|
-
return this.fail(job, message);
|
|
281
|
+
return this.fail(job, claim, message);
|
|
156
282
|
}
|
|
157
|
-
|
|
158
|
-
return 'retried';
|
|
283
|
+
const delay = this.backoff(job.attempts, cfg);
|
|
284
|
+
return this.settle(job, await this.store.retry(this.db, claim, delay, message), 'retried', message);
|
|
159
285
|
}
|
|
160
|
-
async fail(job, reason) {
|
|
161
|
-
this.
|
|
162
|
-
|
|
163
|
-
|
|
286
|
+
async fail(job, claim, reason) {
|
|
287
|
+
const applied = await this.store.markFailed(this.db, claim, reason);
|
|
288
|
+
if (applied)
|
|
289
|
+
this.logger.warn(`job ${job.id} ("${job.name}") failed: ${reason}`);
|
|
290
|
+
return this.settle(job, applied, 'failed', reason);
|
|
164
291
|
}
|
|
165
292
|
backoff(attempts, cfg) {
|
|
166
293
|
const base = cfg.baseBackoffMs * 2 ** attempts;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"jobs-claimer.service.js","sourceRoot":"","sources":["../jobs-claimer.service.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;
|
|
1
|
+
{"version":3,"file":"jobs-claimer.service.js","sourceRoot":"","sources":["../jobs-claimer.service.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAkDA,kDA6BC;AA/ED,qCAAmC;AACnC,2CAAsE;AACtE,qCAA0D;AAW1D,mEAA8D;AAC9D,yDAAoD;AACpD,qCAAyE;AAE5D,QAAA,qBAAqB,GAAyB;IACzD,gBAAgB,EAAE,GAAG,IAAA,kBAAQ,GAAE,IAAI,OAAO,CAAC,GAAG,EAAE;IAChD,cAAc,EAAE,MAAM;IACtB,SAAS,EAAE,EAAE;IACb,aAAa,EAAE,KAAK;IACpB,YAAY,EAAE,MAAM;CACrB,CAAC;AAEF,6EAA6E;AAC7E,6CAA6C;AAC7C,MAAM,kBAAkB,GAAG,OAAO,CAAC;AAEnC,+EAA+E;AAC/E,oEAAoE;AACpE,MAAM,UAAU,GAAG,CAAC,KAAc,EAAU,EAAE,CAC5C,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;AAEzG,4CAA4C;AAC5C,MAAM,SAAS,GAAG,CAAC,KAAc,EAAU,EAAE,CAC3C,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAEzD,8EAA8E;AAC9E,8EAA8E;AAC9E,MAAM,KAAK,GAAG,CAAC,KAAc,EAAU,EAAE,CACvC,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,OAAO,KAAK,GAAG,CAAC;AAE3F;;;;;;GAMG;AACH,SAAgB,mBAAmB,CACjC,YAAiC,EAAE;IAEnC,MAAM,OAAO,GAAG,MAAM,CAAC,WAAW,CAChC,MAAM,CAAC,OAAO,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,KAAK,KAAK,SAAS,CAAC,CAC3D,CAAC;IAClB,MAAM,GAAG,GAAyB,EAAE,GAAG,6BAAqB,EAAE,GAAG,OAAO,EAAE,CAAC;IAC3E,IAAI,OAAO,GAAG,CAAC,gBAAgB,KAAK,QAAQ,IAAI,GAAG,CAAC,gBAAgB,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;QACnF,MAAM,IAAI,SAAS,CACjB,2DAA2D,KAAK,CAAC,GAAG,CAAC,gBAAgB,CAAC,EAAE,CACzF,CAAC;IACJ,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,GAAG,CAAC,SAAS,GAAG,CAAC,EAAE,CAAC;QAC1D,MAAM,IAAI,UAAU,CAAC,oDAAoD,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;IACnG,CAAC;IACD,yEAAyE;IACzE,+CAA+C;IAC/C,MAAM,KAAK,GAAG,GAAG,CAAC,cAAc,CAAC;IACjC,IAAI,CAAC,CAAC,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,GAAG,CAAC,IAAI,KAAK,IAAI,kBAAkB,CAAC,EAAE,CAAC;QAC7E,MAAM,IAAI,UAAU,CAClB,4FAA4F,KAAK,CAAC,KAAK,CAAC,EAAE,CAC3G,CAAC;IACJ,CAAC;IACD,KAAK,MAAM,GAAG,IAAI,CAAC,eAAe,EAAE,cAAc,CAAU,EAAE,CAAC;QAC7D,IAAI,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,IAAI,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;YAClD,MAAM,IAAI,UAAU,CAAC,UAAU,GAAG,uCAAuC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC;QAC9F,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAuBD;;;;;;;;;;;;;;;;;;;GAmBG;AAEI,IAAM,WAAW,mBAAjB,MAAM,WAAW;IAGtB,YACwB,EAA4B,EAC9B,KAAgC,EACnC,QAA6B,EAG9C,gBAAuD,IAAI;QALpB,OAAE,GAAF,EAAE,CAAS;QACb,UAAK,GAAL,KAAK,CAAU;QACnC,aAAQ,GAAR,QAAQ,CAAqB;QAG7B,kBAAa,GAAb,aAAa,CAA6B;QAR5C,WAAM,GAAG,IAAI,eAAM,CAAC,aAAW,CAAC,IAAI,CAAC,CAAC;IASpD,CAAC;IAEJ,KAAK,CAAC,IAAI,CAAC,YAA0B,EAAE;QACrC,MAAM,GAAG,GAAG,mBAAmB,CAAC,SAAS,CAAC,CAAC;QAC3C,uEAAuE;QACvE,6CAA6C;QAC7C,MAAM,SAAS,GAAG,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC1E,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;QAC1D,wEAAwE;QACxE,0EAA0E;QAC1E,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,MAAM,MAAM,GAAe;YACzB,SAAS;YACT,OAAO,EAAE,OAAO,CAAC,MAAM;YACvB,SAAS,EAAE,CAAC;YACZ,OAAO,EAAE,CAAC;YACV,MAAM,EAAE,CAAC;YACT,IAAI,EAAE,CAAC;SACR,CAAC;QACF,uEAAuE;QACvE,uEAAuE;QACvE,6BAA6B;QAC7B,MAAM,OAAO,GAAa,EAAE,CAAC;QAC7B,KAAK,MAAM,CAAC,KAAK,EAAE,GAAG,CAAC,IAAI,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC;YAC7C,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,IAAI,GAAG,CAAC,cAAc,EAAE,CAAC;gBACjD,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBACrB,SAAS;YACX,CAAC;YACD,IAAI,CAAC;gBACH,MAAM,CAAC,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC;YAC/C,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC;gBACnD,MAAM,KAAK,CAAC;YACd,CAAC;QACH,CAAC;QACD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvB,MAAM,CAAC,IAAI,IAAI,OAAO,CAAC,MAAM,CAAC;YAC9B,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,WAAW,OAAO,CAAC,MAAM,oBAAoB,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,qDAAqD,GAAG,CAAC,cAAc,mFAAmF,CAC1N,CAAC;QACJ,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;;;;;;;OAQG;IACK,KAAK,CAAC,cAAc,CAAC,GAAyB;QACpD,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,aAAc,CAAC,OAAO,CAC3C,IAAI,CAAC,EAAE,EACP,GAAG,CAAC,WAAW,EAAE,EACjB,GAAG,CAAC,SAAS,CACd,CAAC;QACF,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,KAAK,MAAM,QAAQ,IAAI,GAAG,EAAE,CAAC;YAC3B,SAAS,IAAI,MAAM,IAAI,CAAC,YAAY,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;QACtD,CAAC;QACD,OAAO,SAAS,CAAC;IACnB,CAAC;IAEO,KAAK,CAAC,YAAY,CAAC,QAAqB,EAAE,GAAS;QACzD,IAAI,SAAwB,CAAC;QAC7B,IAAI,CAAC;YACH,SAAS,GAAG,IAAA,iCAAc,EAAC,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;QACpE,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,yEAAyE;YACzE,+DAA+D;YAC/D,MAAM,OAAO,GAAI,KAAe,CAAC,OAAO,CAAC;YACzC,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,YAAY,QAAQ,CAAC,EAAE,MAAM,QAAQ,CAAC,IAAI,gBAAgB,OAAO,EAAE,CACpE,CAAC;YACF,MAAM,IAAI,CAAC,aAAc,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,QAAQ,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;YACjE,OAAO,CAAC,CAAC;QACX,CAAC;QACD,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,aAAc,CAAC,eAAe,CAAC,IAAI,CAAC,EAAE,EAAE;gBAChE,EAAE,EAAE,QAAQ,CAAC,EAAE;gBACf,uDAAuD;gBACvD,iBAAiB,EAAE,QAAQ,CAAC,SAAmB;gBAC/C,SAAS;gBACT,MAAM,EAAE,GAAG,CAAC,WAAW,EAAE;gBACzB,KAAK,EAAE,eAAe,CAAC,QAAQ,CAAC;aACjC,CAAC,CAAC;YACH,OAAO,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAChC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,wEAAwE;YACxE,uEAAuE;YACvE,qDAAqD;YACrD,MAAM,OAAO,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC;YACjC,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,YAAY,QAAQ,CAAC,EAAE,MAAM,QAAQ,CAAC,IAAI,0CAA0C,OAAO,EAAE,CAC9F,CAAC;YACF,OAAO,CAAC,CAAC;QACX,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,UAAU,CACtB,GAAW,EACX,GAAyB;QAEzB,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAChC,IAAI,KAAK,KAAK,SAAS;YAAE,OAAO,MAAM,CAAC;QACvC,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAC5C,IAAI,CAAC,OAAO,EAAE,CAAC;gBACb,MAAM,IAAI,uBAAc,CACtB,sCAAsC,GAAG,CAAC,IAAI,GAAG,CAClD,CAAC;YACJ,CAAC;YACD,MAAM,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,OAAO,EAAE;gBAChC,KAAK,EAAE,GAAG,CAAC,EAAE;gBACb,OAAO,EAAE,GAAG,CAAC,QAAQ,GAAG,CAAC;aAC1B,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,IAAI,CAAC,cAAc,CAAC,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,CAAC,CAAC;QACrD,CAAC;QACD,uEAAuE;QACvE,4EAA4E;QAC5E,2EAA2E;QAC3E,oBAAoB;QACpB,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,MAAM,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,IAAI,CAAC,EAAE,EAAE,KAAK,CAAC,EAAE,WAAW,CAAC,CAAC;IACvF,CAAC;IAED;;;;;;OAMG;IACK,KAAK,CAAC,aAAa,CAAC,IAAc;QACxC,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACrD,IAAI,OAAO,KAAK,SAAS,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QACvD,IAAI,QAAQ,GAAG,CAAC,CAAC;QACjB,IAAI,CAAC;YACH,KAAK,MAAM,EAAE,EAAE,EAAE,SAAS,EAAE,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC;gBAChD,IAAI,OAAO,SAAS,KAAK,QAAQ,IAAI,OAAO,SAAS,KAAK,QAAQ;oBAAE,SAAS;gBAC7E,IAAI,MAAM,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC;oBAAE,QAAQ,IAAI,CAAC,CAAC;YAC1E,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,uBAAuB,IAAI,CAAC,MAAM,GAAG,QAAQ,uDAAuD,SAAS,CAAC,KAAK,CAAC,4CAA4C,CACjK,CAAC;YACF,OAAO;QACT,CAAC;QACD,IAAI,QAAQ,GAAG,CAAC,EAAE,CAAC;YACjB,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,eAAe,QAAQ,gFAAgF,CACxG,CAAC;QACJ,CAAC;IACH,CAAC;IAED;;;;;;OAMG;IACK,OAAO,CAAC,GAAW;QACzB,MAAM,EAAE,EAAE,EAAE,SAAS,EAAE,SAAS,EAAE,GAAG,GAAG,CAAC;QACzC,IAAI,OAAO,SAAS,KAAK,QAAQ,IAAI,OAAO,SAAS,KAAK,QAAQ,EAAE,CAAC;YACnE,OAAO,EAAE,EAAE,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC;QACtC,CAAC;QACD,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,OAAO,EAAE,sEAAsE,UAAU,CAAC,SAAS,CAAC,eAAe,UAAU,CAAC,SAAS,CAAC,oHAAoH,CAC7P,CAAC;QACF,OAAO,SAAS,CAAC;IACnB,CAAC;IAED;;;OAGG;IACK,MAAM,CACZ,GAAW,EACX,OAAgB,EAChB,OAAwC,EACxC,MAAe;QAEf,IAAI,OAAO;YAAE,OAAO,OAAO,CAAC;QAC5B,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,MAAM,KAAK,SAAS;YAClB,CAAC,CAAC,OAAO,GAAG,CAAC,EAAE,MAAM,GAAG,CAAC,IAAI,+HAA+H;YAC5J,CAAC,CAAC,OAAO,GAAG,CAAC,EAAE,MAAM,GAAG,CAAC,IAAI,cAAc,MAAM,0DAA0D,OAAO,+BAA+B,CACpJ,CAAC;QACF,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,KAAK,CAAC,cAAc,CAC1B,GAAW,EACX,KAAe,EACf,GAAyB,EACzB,KAAc;QAEd,MAAM,OAAO,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC;QACjC,gFAAgF;QAChF,IAAI,KAAK,YAAY,uBAAc,EAAE,CAAC;YACpC,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;QACxC,CAAC;QACD,2EAA2E;QAC3E,IAAI,KAAK,YAAY,uBAAc,EAAE,CAAC;YACpC,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;YAC/D,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;QACtG,CAAC;QACD,kEAAkE;QAClE,IAAI,GAAG,CAAC,QAAQ,GAAG,CAAC,IAAI,GAAG,CAAC,WAAW,EAAE,CAAC;YACxC,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;QACxC,CAAC;QACD,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;QAC9C,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;IACtG,CAAC;IAEO,KAAK,CAAC,IAAI,CAAC,GAAW,EAAE,KAAe,EAAE,MAAc;QAC7D,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;QACpE,IAAI,OAAO;YAAE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,GAAG,CAAC,EAAE,MAAM,GAAG,CAAC,IAAI,cAAc,MAAM,EAAE,CAAC,CAAC;QACjF,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC;IACrD,CAAC;IAEO,OAAO,CAAC,QAAgB,EAAE,GAAyB;QACzD,MAAM,IAAI,GAAG,GAAG,CAAC,aAAa,GAAG,CAAC,IAAI,QAAQ,CAAC;QAC/C,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,YAAY,CAAC,CAAC;QAChD,OAAO,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,GAAG,CAAC,aAAa,CAAC,CAAC;IAChE,CAAC;CACF,CAAA;AAjPY,kCAAW;sBAAX,WAAW;IADvB,IAAA,mBAAU,GAAE;IAKR,WAAA,IAAA,eAAM,EAAC,qBAAY,CAAC,CAAA;IACpB,WAAA,IAAA,eAAM,EAAC,mBAAU,CAAC,CAAA;IAElB,WAAA,IAAA,iBAAQ,GAAE,CAAA;IACV,WAAA,IAAA,eAAM,EAAC,4BAAmB,CAAC,CAAA;qDAFD,2CAAmB;GANrC,WAAW,CAiPvB;AAED,oFAAoF;AACpF,SAAS,eAAe,CAAC,QAAqB;IAC5C,OAAO;QACL,IAAI,EAAE,QAAQ,CAAC,OAAO;QACtB,OAAO,EAAE,QAAQ,CAAC,OAAO;QACzB,GAAG,CAAC,QAAQ,CAAC,WAAW,KAAK,IAAI,IAAI,EAAE,WAAW,EAAE,QAAQ,CAAC,WAAW,EAAE,CAAC;QAC3E,GAAG,CAAC,QAAQ,CAAC,QAAQ,KAAK,IAAI,IAAI,EAAE,QAAQ,EAAE,QAAQ,CAAC,QAAQ,EAAE,CAAC;QAClE,GAAG,CAAC,QAAQ,CAAC,SAAS,KAAK,IAAI,IAAI,EAAE,SAAS,EAAE,QAAQ,CAAC,SAAS,EAAE,CAAC;KACtE,CAAC;AACJ,CAAC"}
|
package/dist/jobs-worker.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { RunnerConfig } from './interfaces';
|
|
2
|
-
import type
|
|
2
|
+
import { type JobsClaimer, type TickReport } from './jobs-claimer.service';
|
|
3
3
|
export interface WorkerLoopOptions {
|
|
4
4
|
/** Delay between ticks when the last tick claimed nothing (default 2000ms). */
|
|
5
5
|
pollIntervalMs?: number;
|
|
@@ -17,5 +17,9 @@ export interface WorkerLoopOptions {
|
|
|
17
17
|
* batch it loops immediately to drain the backlog; when it claims nothing it
|
|
18
18
|
* waits `pollIntervalMs`. A throwing tick is reported via `onError` and the loop
|
|
19
19
|
* continues after the same wait.
|
|
20
|
+
*
|
|
21
|
+
* An invalid `options.runner` rejects at once, before the first tick: it is a
|
|
22
|
+
* wiring mistake, and every tick would otherwise throw into an optional
|
|
23
|
+
* `onError`. Keep the returned promise and handle that rejection.
|
|
20
24
|
*/
|
|
21
25
|
export declare function runWorkerLoop(claimer: JobsClaimer, options?: WorkerLoopOptions): Promise<void>;
|
package/dist/jobs-worker.js
CHANGED
|
@@ -1,13 +1,19 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.runWorkerLoop = runWorkerLoop;
|
|
4
|
+
const jobs_claimer_service_1 = require("./jobs-claimer.service");
|
|
4
5
|
/**
|
|
5
6
|
* Runs `claimer.tick()` in a loop until `signal` aborts. When a tick claims a
|
|
6
7
|
* batch it loops immediately to drain the backlog; when it claims nothing it
|
|
7
8
|
* waits `pollIntervalMs`. A throwing tick is reported via `onError` and the loop
|
|
8
9
|
* continues after the same wait.
|
|
10
|
+
*
|
|
11
|
+
* An invalid `options.runner` rejects at once, before the first tick: it is a
|
|
12
|
+
* wiring mistake, and every tick would otherwise throw into an optional
|
|
13
|
+
* `onError`. Keep the returned promise and handle that rejection.
|
|
9
14
|
*/
|
|
10
15
|
async function runWorkerLoop(claimer, options = {}) {
|
|
16
|
+
(0, jobs_claimer_service_1.resolveRunnerConfig)(options.runner);
|
|
11
17
|
const pollIntervalMs = options.pollIntervalMs ?? 2_000;
|
|
12
18
|
const { signal } = options;
|
|
13
19
|
while (!signal?.aborted) {
|
package/dist/jobs-worker.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"jobs-worker.js","sourceRoot":"","sources":["../jobs-worker.ts"],"names":[],"mappings":";;
|
|
1
|
+
{"version":3,"file":"jobs-worker.js","sourceRoot":"","sources":["../jobs-worker.ts"],"names":[],"mappings":";;AA8BA,sCAmBC;AAhDD,iEAIgC;AAehC;;;;;;;;;GASG;AACI,KAAK,UAAU,aAAa,CACjC,OAAoB,EACpB,UAA6B,EAAE;IAE/B,IAAA,0CAAmB,EAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IACpC,MAAM,cAAc,GAAG,OAAO,CAAC,cAAc,IAAI,KAAK,CAAC;IACvD,MAAM,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;IAC3B,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC;QACxB,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;YAClD,OAAO,CAAC,MAAM,EAAE,CAAC,MAAM,CAAC,CAAC;YACzB,IAAI,MAAM,CAAC,OAAO,KAAK,CAAC,EAAE,CAAC;gBACzB,MAAM,KAAK,CAAC,cAAc,EAAE,MAAM,CAAC,CAAC;YACtC,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,CAAC;YACzB,MAAM,KAAK,CAAC,cAAc,EAAE,MAAM,CAAC,CAAC;QACtC,CAAC;IACH,CAAC;AACH,CAAC;AAED,SAAS,KAAK,CAAC,EAAU,EAAE,MAAoB;IAC7C,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;QAC7B,IAAI,MAAM,EAAE,OAAO,EAAE,CAAC;YACpB,OAAO,EAAE,CAAC;YACV,OAAO;QACT,CAAC;QACD,MAAM,KAAK,GAAG,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;QACtC,MAAM,EAAE,gBAAgB,CACtB,OAAO,EACP,GAAG,EAAE;YACH,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,OAAO,EAAE,CAAC;QACZ,CAAC,EACD,EAAE,IAAI,EAAE,IAAI,EAAE,CACf,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC"}
|
package/dist/testing/harness.js
CHANGED
|
@@ -17,7 +17,7 @@ exports.drainJobs = drainJobs;
|
|
|
17
17
|
*/
|
|
18
18
|
async function drainJobs(claimer, options = {}) {
|
|
19
19
|
const maxTicks = options.maxTicks ?? 100;
|
|
20
|
-
const total = { scheduled: 0, claimed: 0, completed: 0, retried: 0, failed: 0 };
|
|
20
|
+
const total = { scheduled: 0, claimed: 0, completed: 0, retried: 0, failed: 0, lost: 0 };
|
|
21
21
|
for (let tick = 0; tick < maxTicks; tick += 1) {
|
|
22
22
|
const report = await claimer.tick(options.runner);
|
|
23
23
|
total.scheduled += report.scheduled;
|
|
@@ -25,6 +25,7 @@ async function drainJobs(claimer, options = {}) {
|
|
|
25
25
|
total.completed += report.completed;
|
|
26
26
|
total.retried += report.retried;
|
|
27
27
|
total.failed += report.failed;
|
|
28
|
+
total.lost += report.lost;
|
|
28
29
|
if (report.claimed === 0) {
|
|
29
30
|
return total;
|
|
30
31
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"harness.js","sourceRoot":"","sources":["../../testing/harness.ts"],"names":[],"mappings":";;;AA+BA,
|
|
1
|
+
{"version":3,"file":"harness.js","sourceRoot":"","sources":["../../testing/harness.ts"],"names":[],"mappings":";;;AA+BA,8BAsBC;AAnCD;;;;;;;;;;;;GAYG;AACI,KAAK,UAAU,SAAS,CAC7B,OAAoB,EACpB,UAA4B,EAAE;IAE9B,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,GAAG,CAAC;IACzC,MAAM,KAAK,GAAe,EAAE,SAAS,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;IACrG,KAAK,IAAI,IAAI,GAAG,CAAC,EAAE,IAAI,GAAG,QAAQ,EAAE,IAAI,IAAI,CAAC,EAAE,CAAC;QAC9C,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAClD,KAAK,CAAC,SAAS,IAAI,MAAM,CAAC,SAAS,CAAC;QACpC,KAAK,CAAC,OAAO,IAAI,MAAM,CAAC,OAAO,CAAC;QAChC,KAAK,CAAC,SAAS,IAAI,MAAM,CAAC,SAAS,CAAC;QACpC,KAAK,CAAC,OAAO,IAAI,MAAM,CAAC,OAAO,CAAC;QAChC,KAAK,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,CAAC;QAC9B,KAAK,CAAC,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC;QAC1B,IAAI,MAAM,CAAC,OAAO,KAAK,CAAC,EAAE,CAAC;YACzB,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IACD,MAAM,IAAI,KAAK,CACb,kCAAkC,QAAQ,iCAAiC;QACzE,2DAA2D,CAC9D,CAAC;AACJ,CAAC;AAQD;;;;;;;;;;;;GAYG;AACH,MAAa,mBAAmB;IAAhC;QACmB,YAAO,GAA2B,EAAE,CAAC;QACrC,oBAAe,GAAY,EAAE,CAAC;IAwCjD,CAAC;IArCC,MAAM,CAAC,OAAgC,EAAE,GAAe;QACtD,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC;QACpC,MAAM,OAAO,GAAG,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;QAC7C,IAAI,OAAO,EAAE,CAAC;YACZ,MAAM,OAAO,CAAC;QAChB,CAAC;QACD,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,MAAM,IAAI,CAAC,OAAO,CAAC;QACrB,CAAC;IACH,CAAC;IAED,6DAA6D;IAC7D,YAAY,CAAC,KAAY;QACvB,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACnC,CAAC;IAED,mEAAmE;IACnE,QAAQ,CAAC,KAAY;QACnB,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC;IACvB,CAAC;IAED,yDAAyD;IACzD,YAAY;QACV,IAAI,CAAC,OAAO,GAAG,SAAS,CAAC;QACzB,IAAI,CAAC,eAAe,CAAC,MAAM,GAAG,CAAC,CAAC;IAClC,CAAC;IAED,wCAAwC;IACxC,UAAU;QACR,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;IAED,wDAAwD;IACxD,KAAK;QACH,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;QACxB,IAAI,CAAC,YAAY,EAAE,CAAC;IACtB,CAAC;CACF;AA1CD,kDA0CC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@nest-native/jobs",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.5.1",
|
|
4
4
|
"description": "Background jobs for NestJS without Redis \u2014 a Drizzle-backed job queue (SQLite, Postgres + MySQL) with transactional enqueue, retries, delayed and unique jobs",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"nestjs",
|
|
@@ -100,8 +100,8 @@
|
|
|
100
100
|
}
|
|
101
101
|
},
|
|
102
102
|
"devDependencies": {
|
|
103
|
-
"@nestjs/common": "^11.
|
|
104
|
-
"@nestjs/core": "^11.
|
|
103
|
+
"@nestjs/common": "^11.2.7",
|
|
104
|
+
"@nestjs/core": "^11.2.7",
|
|
105
105
|
"reflect-metadata": "^0.2.2",
|
|
106
106
|
"rxjs": "^7.8.1",
|
|
107
107
|
"typescript": "^6.0.3"
|