@gobing-ai/ts-db 0.4.59 → 0.4.60
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -6
- package/dist/embedded-migrations.d.ts.map +1 -1
- package/dist/embedded-migrations.js +5 -0
- package/dist/queue-job-dao.d.ts +54 -17
- package/dist/queue-job-dao.d.ts.map +1 -1
- package/dist/queue-job-dao.js +123 -18
- package/dist/schema/queue-jobs.d.ts +70 -0
- package/dist/schema/queue-jobs.d.ts.map +1 -1
- package/dist/schema/queue-jobs.js +8 -0
- package/package.json +3 -3
- package/src/embedded-migrations.ts +5 -0
- package/src/queue-job-dao.ts +169 -37
- package/src/schema/queue-jobs.ts +8 -0
package/README.md
CHANGED
|
@@ -233,21 +233,26 @@ const queue = new QueueJobDao(adapter);
|
|
|
233
233
|
|
|
234
234
|
// Enqueue — maxRetries is a total attempt budget, not retries after the first:
|
|
235
235
|
// 5 means up to 5 attempts, and `maxRetries: 1` runs the job once with no retry.
|
|
236
|
-
|
|
236
|
+
// timeoutMs is the per-job execution policy: a positive ms deadline, or explicit
|
|
237
|
+
// `null` = unlimited (omitted = inherit the consumer default).
|
|
238
|
+
const jobId = await queue.enqueue('send-email', { to: 'user@test.com' }, { maxRetries: 5, timeoutMs: 60_000 });
|
|
237
239
|
|
|
238
|
-
// Consumer: claim ready jobs atomically
|
|
239
|
-
|
|
240
|
+
// Consumer: claim ready jobs atomically. claimReady leases each job (mints an
|
|
241
|
+
// attempt token); renew the lease while the handler runs, then ack with the
|
|
242
|
+
// token — fenced acks are ignored if ownership was lost mid-run.
|
|
243
|
+
const jobs = await queue.claimReady(10, { leaseMs: 30_000 });
|
|
244
|
+
await queue.renewLease(jobs[0].id, jobs[0].attemptToken, 30_000);
|
|
240
245
|
|
|
241
246
|
for (const job of jobs) {
|
|
242
247
|
try {
|
|
243
248
|
await processJob(job);
|
|
244
|
-
await queue.markCompleted(job.id);
|
|
249
|
+
await queue.markCompleted(job.id, job.attemptToken);
|
|
245
250
|
} catch (error) {
|
|
246
251
|
if (job.attempts >= job.maxRetries) {
|
|
247
|
-
await queue.markFailed(job.id, job.attempts + 1, String(error));
|
|
252
|
+
await queue.markFailed(job.id, job.attempts + 1, String(error), job.attemptToken);
|
|
248
253
|
} else {
|
|
249
254
|
const retryAt = Date.now() + Math.pow(2, job.attempts) * 1000;
|
|
250
|
-
await queue.markForRetry(job.id, job.attempts + 1, String(error), retryAt);
|
|
255
|
+
await queue.markForRetry(job.id, job.attempts + 1, String(error), retryAt, job.attemptToken);
|
|
251
256
|
}
|
|
252
257
|
}
|
|
253
258
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"embedded-migrations.d.ts","sourceRoot":"","sources":["../src/embedded-migrations.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,mFAAmF;AACnF,MAAM,WAAW,iBAAiB;IAC9B,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,MAAM,CAAC;CAChB;AAED,uEAAuE;AACvE,eAAO,MAAM,kBAAkB,EAAE,iBAAiB,
|
|
1
|
+
{"version":3,"file":"embedded-migrations.d.ts","sourceRoot":"","sources":["../src/embedded-migrations.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,mFAAmF;AACnF,MAAM,WAAW,iBAAiB;IAC9B,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,MAAM,CAAC;CAChB;AAED,uEAAuE;AACvE,eAAO,MAAM,kBAAkB,EAAE,iBAAiB,EA+BjD,CAAC"}
|
|
@@ -33,4 +33,9 @@ export const embeddedMigrations = [
|
|
|
33
33
|
sql: 'CREATE INDEX `idx_inbox_messages_to_status` ON `inbox_messages` (`to_id`,`status`);',
|
|
34
34
|
hash: '3c00f1bce4f569e442f3142df31ff478d6c9b3d460824d46575f50a324bfae61',
|
|
35
35
|
},
|
|
36
|
+
{
|
|
37
|
+
tag: '0005_queue_attempt_ownership',
|
|
38
|
+
sql: 'ALTER TABLE `queue_jobs` ADD `timeout_ms` integer;\nALTER TABLE `queue_jobs` ADD `timeout_unlimited` integer DEFAULT 0 NOT NULL;\nALTER TABLE `queue_jobs` ADD `attempt_token` text;\nALTER TABLE `queue_jobs` ADD `lease_expires_at` integer;',
|
|
39
|
+
hash: 'f6a4eaf11eee838f424515e57eca3a46539260bb2fb5b0b112870ca5ed38d040',
|
|
40
|
+
},
|
|
36
41
|
];
|
package/dist/queue-job-dao.d.ts
CHANGED
|
@@ -10,6 +10,25 @@ export interface QueueStats {
|
|
|
10
10
|
completed: number;
|
|
11
11
|
failed: number;
|
|
12
12
|
}
|
|
13
|
+
/** Options for enqueuing a job: retry policy, delay, TTL, and execution policy. */
|
|
14
|
+
export interface QueueEnqueueOptions {
|
|
15
|
+
/** Total attempts allowed (default 3). */
|
|
16
|
+
maxRetries?: number;
|
|
17
|
+
delay?: number;
|
|
18
|
+
ttlMs?: number;
|
|
19
|
+
/**
|
|
20
|
+
* Job execution policy persisted with the row: a positive integer ms
|
|
21
|
+
* deadline, explicit `null` = unlimited (disables this job's deadline),
|
|
22
|
+
* omitted = no job-level decision (consumers apply their own default).
|
|
23
|
+
* Invalid explicit values are rejected before the row is created.
|
|
24
|
+
*/
|
|
25
|
+
timeoutMs?: number | null;
|
|
26
|
+
}
|
|
27
|
+
/** Options for an atomic claim that takes lease ownership. */
|
|
28
|
+
export interface QueueClaimOptions {
|
|
29
|
+
/** Ownership lease length in ms; omitted claims keep the legacy recovery path. */
|
|
30
|
+
leaseMs?: number;
|
|
31
|
+
}
|
|
13
32
|
/**
|
|
14
33
|
* Row type inferred from the queue_jobs Drizzle schema.
|
|
15
34
|
*/
|
|
@@ -25,22 +44,14 @@ export declare class QueueJobDao extends EntityDao<typeof queueJobs, typeof queu
|
|
|
25
44
|
/**
|
|
26
45
|
* Enqueue a new job.
|
|
27
46
|
*/
|
|
28
|
-
enqueue(type: string, payload: unknown, options?:
|
|
29
|
-
maxRetries?: number;
|
|
30
|
-
delay?: number;
|
|
31
|
-
ttlMs?: number;
|
|
32
|
-
}): Promise<string>;
|
|
47
|
+
enqueue(type: string, payload: unknown, options?: QueueEnqueueOptions): Promise<string>;
|
|
33
48
|
/**
|
|
34
49
|
* Enqueue multiple jobs in a single batch.
|
|
35
50
|
*/
|
|
36
51
|
enqueueBatch(jobs: Array<{
|
|
37
52
|
type: string;
|
|
38
53
|
payload: unknown;
|
|
39
|
-
} &
|
|
40
|
-
maxRetries?: number;
|
|
41
|
-
delay?: number;
|
|
42
|
-
ttlMs?: number;
|
|
43
|
-
}>): Promise<string[]>;
|
|
54
|
+
} & QueueEnqueueOptions>): Promise<string[]>;
|
|
44
55
|
/**
|
|
45
56
|
* Get a job by ID.
|
|
46
57
|
*/
|
|
@@ -58,30 +69,56 @@ export declare class QueueJobDao extends EntityDao<typeof queueJobs, typeof queu
|
|
|
58
69
|
*/
|
|
59
70
|
findPending(batchSize: number): Promise<QueueJobRecord[]>;
|
|
60
71
|
/**
|
|
61
|
-
* Atomically claim ready pending jobs
|
|
72
|
+
* Atomically claim ready pending jobs — and recover expired leased
|
|
73
|
+
* attempts — for processing.
|
|
62
74
|
*
|
|
63
75
|
* The update and selection happen in one SQLite statement so competing
|
|
64
76
|
* consumers only receive rows they actually transitioned to processing.
|
|
77
|
+
* When `options.leaseMs` is provided the claimed row takes an ownership
|
|
78
|
+
* lease and a fresh attempt token; the token fences mutations and keeps
|
|
79
|
+
* rival consumers from re-claiming the live attempt until the lease
|
|
80
|
+
* expires. Expired leased rows (worker loss) are recovered here with a
|
|
81
|
+
* fresh token — not by the age-based `resetStuckJobs` sweep, which skips
|
|
82
|
+
* token-carrying rows. Claims WITHOUT `leaseMs` mint no ownership markers
|
|
83
|
+
* (and clear any stale ones on reclaim) so the row keeps the legacy
|
|
84
|
+
* recovery path through the age sweep — a token without a lease would
|
|
85
|
+
* make the row unrecoverable by both paths.
|
|
65
86
|
*/
|
|
66
|
-
claimReady(batchSize: number): Promise<QueueJobRecord[]>;
|
|
87
|
+
claimReady(batchSize: number, options?: QueueClaimOptions): Promise<QueueJobRecord[]>;
|
|
67
88
|
/**
|
|
68
89
|
* Mark jobs as processing.
|
|
69
90
|
*/
|
|
70
91
|
markProcessing(ids: string[]): Promise<void>;
|
|
71
92
|
/**
|
|
72
93
|
* Mark a job as completed.
|
|
94
|
+
*
|
|
95
|
+
* When `attemptToken` is supplied the mutation is fenced: it applies only
|
|
96
|
+
* if the row is still owned by that attempt. Returns whether it applied.
|
|
73
97
|
*/
|
|
74
|
-
markCompleted(id: string): Promise<
|
|
98
|
+
markCompleted(id: string, attemptToken?: string): Promise<boolean>;
|
|
75
99
|
/**
|
|
76
|
-
* Mark a job as failed.
|
|
100
|
+
* Mark a job as failed, fenced by fresh attempt ownership when supplied.
|
|
77
101
|
*/
|
|
78
|
-
markFailed(id: string, attempts: number, error: string): Promise<
|
|
102
|
+
markFailed(id: string, attempts: number, error: string, attemptToken?: string): Promise<boolean>;
|
|
79
103
|
/**
|
|
80
|
-
* Reset a job to pending for retry with backoff
|
|
104
|
+
* Reset a job to pending for retry with backoff, fenced by fresh attempt
|
|
105
|
+
* ownership when supplied.
|
|
81
106
|
*/
|
|
82
|
-
markForRetry(id: string, attempts: number, errorMessage: string, nextRetryAt: number): Promise<
|
|
107
|
+
markForRetry(id: string, attempts: number, errorMessage: string, nextRetryAt: number, attemptToken?: string): Promise<boolean>;
|
|
108
|
+
/**
|
|
109
|
+
* Extend the ownership lease of a claimed attempt.
|
|
110
|
+
*
|
|
111
|
+
* Applies only while the row is still owned by `attemptToken`; a `false`
|
|
112
|
+
* return means ownership was lost (expired and reclaimed by another
|
|
113
|
+
* consumer) and the caller must abort and fence its acknowledgements.
|
|
114
|
+
*/
|
|
115
|
+
renewLease(id: string, attemptToken: string, leaseMs: number): Promise<boolean>;
|
|
83
116
|
/**
|
|
84
117
|
* Reset stuck processing jobs (processing beyond visibility timeout).
|
|
118
|
+
*
|
|
119
|
+
* Age-based legacy recovery only: rows carrying an attempt token are
|
|
120
|
+
* lease-owned and recover exclusively through their expired lease in
|
|
121
|
+
* `claimReady`, never by processing age.
|
|
85
122
|
*/
|
|
86
123
|
resetStuckJobs(visibilityTimeout: number): Promise<number>;
|
|
87
124
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"queue-job-dao.d.ts","sourceRoot":"","sources":["../src/queue-job-dao.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAQ3C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAEhD;;GAEG;AACH,MAAM,WAAW,UAAU;IACvB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,MAAM,cAAc,GAAG,OAAO,SAAS,CAAC,YAAY,CAAC;AAE3D;;;;;GAKG;AACH,qBAAa,WAAY,SAAQ,SAAS,CAAC,OAAO,SAAS,EAAE,OAAO,SAAS,CAAC,EAAE,CAAC;gBACjE,OAAO,EAAE,SAAS;IAI9B;;OAEG;IACG,OAAO,
|
|
1
|
+
{"version":3,"file":"queue-job-dao.d.ts","sourceRoot":"","sources":["../src/queue-job-dao.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAQ3C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAEhD;;GAEG;AACH,MAAM,WAAW,UAAU;IACvB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;CAClB;AAED,mFAAmF;AACnF,MAAM,WAAW,mBAAmB;IAChC,0CAA0C;IAC1C,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;;;OAKG;IACH,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC7B;AAED,8DAA8D;AAC9D,MAAM,WAAW,iBAAiB;IAC9B,kFAAkF;IAClF,OAAO,CAAC,EAAE,MAAM,CAAC;CACpB;AAoBD;;GAEG;AACH,MAAM,MAAM,cAAc,GAAG,OAAO,SAAS,CAAC,YAAY,CAAC;AAE3D;;;;;GAKG;AACH,qBAAa,WAAY,SAAQ,SAAS,CAAC,OAAO,SAAS,EAAE,OAAO,SAAS,CAAC,EAAE,CAAC;gBACjE,OAAO,EAAE,SAAS;IAI9B;;OAEG;IACG,OAAO,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,mBAAmB,GAAG,OAAO,CAAC,MAAM,CAAC;IAoB7F;;OAEG;IACG,YAAY,CAAC,IAAI,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,OAAO,CAAA;KAAE,GAAG,mBAAmB,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAmC5G;;OAEG;IACG,OAAO,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,GAAG,SAAS,CAAC;IAI9D;;OAEG;IACG,QAAQ,IAAI,OAAO,CAAC,UAAU,CAAC;IAoBrC;;OAEG;IACG,aAAa,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IASpD;;OAEG;IACG,WAAW,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC;IAe/D;;;;;;;;;;;;;;;OAeG;IACG,UAAU,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,iBAAiB,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC;IA0C3F;;OAEG;IACG,cAAc,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAWlD;;;;;OAKG;IACG,aAAa,CAAC,EAAE,EAAE,MAAM,EAAE,YAAY,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAkBxE;;OAEG;IACG,UAAU,CAAC,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,YAAY,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAoBtG;;;OAGG;IACG,YAAY,CACd,EAAE,EAAE,MAAM,EACV,QAAQ,EAAE,MAAM,EAChB,YAAY,EAAE,MAAM,EACpB,WAAW,EAAE,MAAM,EACnB,YAAY,CAAC,EAAE,MAAM,GACtB,OAAO,CAAC,OAAO,CAAC;IAqBnB;;;;;;OAMG;IACG,UAAU,CAAC,EAAE,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAWrF;;;;;;OAMG;IACG,cAAc,CAAC,iBAAiB,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAchE;;;;;OAKG;IACG,eAAe,IAAI,OAAO,CAAC,MAAM,CAAC;CAkB3C"}
|
package/dist/queue-job-dao.js
CHANGED
|
@@ -1,6 +1,24 @@
|
|
|
1
1
|
import { and, eq, inArray, sql } from 'drizzle-orm';
|
|
2
2
|
import { EntityDao } from './entity-dao.js';
|
|
3
3
|
import { queueJobs } from './schema/queue-jobs.js';
|
|
4
|
+
/**
|
|
5
|
+
* Validate the persisted execution-policy leaf value. The scope-resolution
|
|
6
|
+
* chain (inherit/unlimited/finite) lives in the consumer packages; the DAO
|
|
7
|
+
* only stores an already-resolved job option and refuses invalid ones.
|
|
8
|
+
*/
|
|
9
|
+
function assertValidTimeoutMs(value) {
|
|
10
|
+
if (value === undefined || value === null)
|
|
11
|
+
return;
|
|
12
|
+
if (typeof value !== 'number' || !Number.isInteger(value) || value <= 0) {
|
|
13
|
+
throw new RangeError(`enqueue timeoutMs must be a positive integer or null; received ${String(value)}`);
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
/** Persisted execution-policy columns derived from a validated `timeoutMs` option. */
|
|
17
|
+
function timeoutColumns(timeoutMs) {
|
|
18
|
+
if (timeoutMs === undefined)
|
|
19
|
+
return {};
|
|
20
|
+
return timeoutMs === null ? { timeoutUnlimited: 1 } : { timeoutMs };
|
|
21
|
+
}
|
|
4
22
|
/**
|
|
5
23
|
* DAO for the queue_jobs table.
|
|
6
24
|
*
|
|
@@ -15,6 +33,7 @@ export class QueueJobDao extends EntityDao {
|
|
|
15
33
|
* Enqueue a new job.
|
|
16
34
|
*/
|
|
17
35
|
async enqueue(type, payload, options) {
|
|
36
|
+
assertValidTimeoutMs(options?.timeoutMs);
|
|
18
37
|
const now = this.now();
|
|
19
38
|
const id = crypto.randomUUID();
|
|
20
39
|
await this.create({
|
|
@@ -26,6 +45,7 @@ export class QueueJobDao extends EntityDao {
|
|
|
26
45
|
maxRetries: options?.maxRetries ?? 3,
|
|
27
46
|
nextRetryAt: options?.delay !== undefined ? now + options.delay : now,
|
|
28
47
|
...(options?.ttlMs !== undefined ? { expiresAt: now + options.ttlMs } : {}),
|
|
48
|
+
...timeoutColumns(options?.timeoutMs),
|
|
29
49
|
});
|
|
30
50
|
return id;
|
|
31
51
|
}
|
|
@@ -33,6 +53,9 @@ export class QueueJobDao extends EntityDao {
|
|
|
33
53
|
* Enqueue multiple jobs in a single batch.
|
|
34
54
|
*/
|
|
35
55
|
async enqueueBatch(jobs) {
|
|
56
|
+
for (const job of jobs) {
|
|
57
|
+
assertValidTimeoutMs(job.timeoutMs);
|
|
58
|
+
}
|
|
36
59
|
const now = this.now();
|
|
37
60
|
const ids = [];
|
|
38
61
|
const rows = jobs.map((job) => {
|
|
@@ -47,6 +70,7 @@ export class QueueJobDao extends EntityDao {
|
|
|
47
70
|
maxRetries: job.maxRetries ?? 3,
|
|
48
71
|
nextRetryAt: job.delay !== undefined ? now + job.delay : now,
|
|
49
72
|
...(job.ttlMs !== undefined ? { expiresAt: now + job.ttlMs } : {}),
|
|
73
|
+
...timeoutColumns(job.timeoutMs),
|
|
50
74
|
};
|
|
51
75
|
});
|
|
52
76
|
if (rows.length > 0) {
|
|
@@ -108,28 +132,55 @@ export class QueueJobDao extends EntityDao {
|
|
|
108
132
|
return result;
|
|
109
133
|
}
|
|
110
134
|
/**
|
|
111
|
-
* Atomically claim ready pending jobs
|
|
135
|
+
* Atomically claim ready pending jobs — and recover expired leased
|
|
136
|
+
* attempts — for processing.
|
|
112
137
|
*
|
|
113
138
|
* The update and selection happen in one SQLite statement so competing
|
|
114
139
|
* consumers only receive rows they actually transitioned to processing.
|
|
140
|
+
* When `options.leaseMs` is provided the claimed row takes an ownership
|
|
141
|
+
* lease and a fresh attempt token; the token fences mutations and keeps
|
|
142
|
+
* rival consumers from re-claiming the live attempt until the lease
|
|
143
|
+
* expires. Expired leased rows (worker loss) are recovered here with a
|
|
144
|
+
* fresh token — not by the age-based `resetStuckJobs` sweep, which skips
|
|
145
|
+
* token-carrying rows. Claims WITHOUT `leaseMs` mint no ownership markers
|
|
146
|
+
* (and clear any stale ones on reclaim) so the row keeps the legacy
|
|
147
|
+
* recovery path through the age sweep — a token without a lease would
|
|
148
|
+
* make the row unrecoverable by both paths.
|
|
115
149
|
*/
|
|
116
|
-
async claimReady(batchSize) {
|
|
150
|
+
async claimReady(batchSize, options) {
|
|
117
151
|
const limit = Math.floor(batchSize);
|
|
118
152
|
if (limit <= 0)
|
|
119
153
|
return [];
|
|
120
154
|
const now = this.now();
|
|
155
|
+
const leaseMs = options?.leaseMs;
|
|
156
|
+
const leased = leaseMs !== undefined;
|
|
157
|
+
const leaseExpiresAt = leased ? now + leaseMs : null;
|
|
158
|
+
const reclaimable = sql `(
|
|
159
|
+
${queueJobs.status} = 'pending'
|
|
160
|
+
AND (${queueJobs.nextRetryAt} IS NULL OR ${queueJobs.nextRetryAt} <= ${now})
|
|
161
|
+
) OR (
|
|
162
|
+
${queueJobs.status} = 'processing'
|
|
163
|
+
AND ${queueJobs.attemptToken} IS NOT NULL
|
|
164
|
+
AND ${queueJobs.leaseExpiresAt} IS NOT NULL
|
|
165
|
+
AND ${queueJobs.leaseExpiresAt} <= ${now}
|
|
166
|
+
)`;
|
|
121
167
|
const result = await this.db
|
|
122
168
|
.update(queueJobs)
|
|
123
|
-
.set({
|
|
169
|
+
.set({
|
|
170
|
+
status: 'processing',
|
|
171
|
+
processingAt: now,
|
|
172
|
+
updatedAt: now,
|
|
173
|
+
attemptToken: leased ? sql `lower(hex(randomblob(16)))` : null,
|
|
174
|
+
leaseExpiresAt,
|
|
175
|
+
})
|
|
124
176
|
.where(sql `${queueJobs.id} IN (
|
|
125
177
|
SELECT id
|
|
126
178
|
FROM ${queueJobs}
|
|
127
|
-
WHERE
|
|
128
|
-
AND (next_retry_at IS NULL OR next_retry_at <= ${now})
|
|
179
|
+
WHERE ${reclaimable}
|
|
129
180
|
ORDER BY created_at
|
|
130
181
|
LIMIT ${limit}
|
|
131
182
|
)
|
|
132
|
-
AND ${
|
|
183
|
+
AND ${reclaimable}`)
|
|
133
184
|
.returning();
|
|
134
185
|
return result;
|
|
135
186
|
}
|
|
@@ -147,45 +198,99 @@ export class QueueJobDao extends EntityDao {
|
|
|
147
198
|
}
|
|
148
199
|
/**
|
|
149
200
|
* Mark a job as completed.
|
|
201
|
+
*
|
|
202
|
+
* When `attemptToken` is supplied the mutation is fenced: it applies only
|
|
203
|
+
* if the row is still owned by that attempt. Returns whether it applied.
|
|
150
204
|
*/
|
|
151
|
-
async markCompleted(id) {
|
|
152
|
-
|
|
205
|
+
async markCompleted(id, attemptToken) {
|
|
206
|
+
const where = attemptToken === undefined
|
|
207
|
+
? eq(queueJobs.id, id)
|
|
208
|
+
: and(eq(queueJobs.id, id), eq(queueJobs.attemptToken, attemptToken));
|
|
209
|
+
const result = await this.db
|
|
210
|
+
.update(queueJobs)
|
|
211
|
+
.set({
|
|
153
212
|
status: 'completed',
|
|
154
213
|
processingAt: null,
|
|
155
|
-
|
|
214
|
+
attemptToken: null,
|
|
215
|
+
leaseExpiresAt: null,
|
|
216
|
+
updatedAt: this.now(),
|
|
217
|
+
})
|
|
218
|
+
.where(where);
|
|
219
|
+
return result.changes === 1;
|
|
156
220
|
}
|
|
157
221
|
/**
|
|
158
|
-
* Mark a job as failed.
|
|
222
|
+
* Mark a job as failed, fenced by fresh attempt ownership when supplied.
|
|
159
223
|
*/
|
|
160
|
-
async markFailed(id, attempts, error) {
|
|
161
|
-
|
|
224
|
+
async markFailed(id, attempts, error, attemptToken) {
|
|
225
|
+
const where = attemptToken === undefined
|
|
226
|
+
? eq(queueJobs.id, id)
|
|
227
|
+
: and(eq(queueJobs.id, id), eq(queueJobs.attemptToken, attemptToken));
|
|
228
|
+
const result = await this.db
|
|
229
|
+
.update(queueJobs)
|
|
230
|
+
.set({
|
|
162
231
|
status: 'failed',
|
|
163
232
|
attempts,
|
|
164
233
|
lastError: error,
|
|
165
234
|
processingAt: null,
|
|
166
|
-
|
|
235
|
+
attemptToken: null,
|
|
236
|
+
leaseExpiresAt: null,
|
|
237
|
+
updatedAt: this.now(),
|
|
238
|
+
})
|
|
239
|
+
.where(where);
|
|
240
|
+
return result.changes === 1;
|
|
167
241
|
}
|
|
168
242
|
/**
|
|
169
|
-
* Reset a job to pending for retry with backoff
|
|
243
|
+
* Reset a job to pending for retry with backoff, fenced by fresh attempt
|
|
244
|
+
* ownership when supplied.
|
|
170
245
|
*/
|
|
171
|
-
async markForRetry(id, attempts, errorMessage, nextRetryAt) {
|
|
172
|
-
|
|
246
|
+
async markForRetry(id, attempts, errorMessage, nextRetryAt, attemptToken) {
|
|
247
|
+
const where = attemptToken === undefined
|
|
248
|
+
? eq(queueJobs.id, id)
|
|
249
|
+
: and(eq(queueJobs.id, id), eq(queueJobs.attemptToken, attemptToken));
|
|
250
|
+
const result = await this.db
|
|
251
|
+
.update(queueJobs)
|
|
252
|
+
.set({
|
|
173
253
|
status: 'pending',
|
|
174
254
|
attempts,
|
|
175
255
|
lastError: errorMessage,
|
|
176
256
|
nextRetryAt,
|
|
177
257
|
processingAt: null,
|
|
178
|
-
|
|
258
|
+
attemptToken: null,
|
|
259
|
+
leaseExpiresAt: null,
|
|
260
|
+
updatedAt: this.now(),
|
|
261
|
+
})
|
|
262
|
+
.where(where);
|
|
263
|
+
return result.changes === 1;
|
|
264
|
+
}
|
|
265
|
+
/**
|
|
266
|
+
* Extend the ownership lease of a claimed attempt.
|
|
267
|
+
*
|
|
268
|
+
* Applies only while the row is still owned by `attemptToken`; a `false`
|
|
269
|
+
* return means ownership was lost (expired and reclaimed by another
|
|
270
|
+
* consumer) and the caller must abort and fence its acknowledgements.
|
|
271
|
+
*/
|
|
272
|
+
async renewLease(id, attemptToken, leaseMs) {
|
|
273
|
+
const now = this.now();
|
|
274
|
+
const result = await this.db
|
|
275
|
+
.update(queueJobs)
|
|
276
|
+
.set({ leaseExpiresAt: now + leaseMs, updatedAt: now })
|
|
277
|
+
.where(and(eq(queueJobs.id, id), eq(queueJobs.attemptToken, attemptToken), eq(queueJobs.status, 'processing')));
|
|
278
|
+
return result.changes === 1;
|
|
179
279
|
}
|
|
180
280
|
/**
|
|
181
281
|
* Reset stuck processing jobs (processing beyond visibility timeout).
|
|
282
|
+
*
|
|
283
|
+
* Age-based legacy recovery only: rows carrying an attempt token are
|
|
284
|
+
* lease-owned and recover exclusively through their expired lease in
|
|
285
|
+
* `claimReady`, never by processing age.
|
|
182
286
|
*/
|
|
183
287
|
async resetStuckJobs(visibilityTimeout) {
|
|
184
288
|
const cutoff = this.now() - visibilityTimeout;
|
|
185
289
|
const result = await this.db
|
|
186
290
|
.update(queueJobs)
|
|
187
291
|
.set({ status: 'pending', processingAt: null, updatedAt: this.now() })
|
|
188
|
-
.where(sql `${queueJobs.status} = 'processing' AND ${queueJobs.processingAt} IS NOT NULL AND ${queueJobs.processingAt} <= ${cutoff}
|
|
292
|
+
.where(sql `${queueJobs.status} = 'processing' AND ${queueJobs.processingAt} IS NOT NULL AND ${queueJobs.processingAt} <= ${cutoff}
|
|
293
|
+
AND ${queueJobs.attemptToken} IS NULL`);
|
|
189
294
|
return result.changes;
|
|
190
295
|
}
|
|
191
296
|
/**
|
|
@@ -75,6 +75,76 @@ export declare const queueJobs: import("drizzle-orm/sqlite-core").SQLiteTableWit
|
|
|
75
75
|
identity: undefined;
|
|
76
76
|
generated: undefined;
|
|
77
77
|
}, {}, {}>;
|
|
78
|
+
timeoutMs: import("drizzle-orm/sqlite-core").SQLiteColumn<{
|
|
79
|
+
name: "timeout_ms";
|
|
80
|
+
tableName: "queue_jobs";
|
|
81
|
+
dataType: "number";
|
|
82
|
+
columnType: "SQLiteInteger";
|
|
83
|
+
data: number;
|
|
84
|
+
driverParam: number;
|
|
85
|
+
notNull: false;
|
|
86
|
+
hasDefault: false;
|
|
87
|
+
isPrimaryKey: false;
|
|
88
|
+
isAutoincrement: false;
|
|
89
|
+
hasRuntimeDefault: false;
|
|
90
|
+
enumValues: undefined;
|
|
91
|
+
baseColumn: never;
|
|
92
|
+
identity: undefined;
|
|
93
|
+
generated: undefined;
|
|
94
|
+
}, {}, {}>;
|
|
95
|
+
timeoutUnlimited: import("drizzle-orm/sqlite-core").SQLiteColumn<{
|
|
96
|
+
name: "timeout_unlimited";
|
|
97
|
+
tableName: "queue_jobs";
|
|
98
|
+
dataType: "number";
|
|
99
|
+
columnType: "SQLiteInteger";
|
|
100
|
+
data: number;
|
|
101
|
+
driverParam: number;
|
|
102
|
+
notNull: true;
|
|
103
|
+
hasDefault: true;
|
|
104
|
+
isPrimaryKey: false;
|
|
105
|
+
isAutoincrement: false;
|
|
106
|
+
hasRuntimeDefault: false;
|
|
107
|
+
enumValues: undefined;
|
|
108
|
+
baseColumn: never;
|
|
109
|
+
identity: undefined;
|
|
110
|
+
generated: undefined;
|
|
111
|
+
}, {}, {}>;
|
|
112
|
+
attemptToken: import("drizzle-orm/sqlite-core").SQLiteColumn<{
|
|
113
|
+
name: "attempt_token";
|
|
114
|
+
tableName: "queue_jobs";
|
|
115
|
+
dataType: "string";
|
|
116
|
+
columnType: "SQLiteText";
|
|
117
|
+
data: string;
|
|
118
|
+
driverParam: string;
|
|
119
|
+
notNull: false;
|
|
120
|
+
hasDefault: false;
|
|
121
|
+
isPrimaryKey: false;
|
|
122
|
+
isAutoincrement: false;
|
|
123
|
+
hasRuntimeDefault: false;
|
|
124
|
+
enumValues: [string, ...string[]];
|
|
125
|
+
baseColumn: never;
|
|
126
|
+
identity: undefined;
|
|
127
|
+
generated: undefined;
|
|
128
|
+
}, {}, {
|
|
129
|
+
length: number | undefined;
|
|
130
|
+
}>;
|
|
131
|
+
leaseExpiresAt: import("drizzle-orm/sqlite-core").SQLiteColumn<{
|
|
132
|
+
name: "lease_expires_at";
|
|
133
|
+
tableName: "queue_jobs";
|
|
134
|
+
dataType: "number";
|
|
135
|
+
columnType: "SQLiteInteger";
|
|
136
|
+
data: number;
|
|
137
|
+
driverParam: number;
|
|
138
|
+
notNull: false;
|
|
139
|
+
hasDefault: false;
|
|
140
|
+
isPrimaryKey: false;
|
|
141
|
+
isAutoincrement: false;
|
|
142
|
+
hasRuntimeDefault: false;
|
|
143
|
+
enumValues: undefined;
|
|
144
|
+
baseColumn: never;
|
|
145
|
+
identity: undefined;
|
|
146
|
+
generated: undefined;
|
|
147
|
+
}, {}, {}>;
|
|
78
148
|
createdAt: import("drizzle-orm/sqlite-core").SQLiteColumn<{
|
|
79
149
|
name: "created_at";
|
|
80
150
|
tableName: "queue_jobs";
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"queue-jobs.d.ts","sourceRoot":"","sources":["../../src/schema/queue-jobs.ts"],"names":[],"mappings":"AAGA;;GAEG;AACH,eAAO,MAAM,SAAS
|
|
1
|
+
{"version":3,"file":"queue-jobs.d.ts","sourceRoot":"","sources":["../../src/schema/queue-jobs.ts"],"names":[],"mappings":"AAGA;;GAEG;AACH,eAAO,MAAM,SAAS;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAwBrB,CAAC"}
|
|
@@ -15,4 +15,12 @@ export const queueJobs = sqliteTable('queue_jobs', {
|
|
|
15
15
|
lastError: text('last_error'),
|
|
16
16
|
processingAt: integer('processing_at'),
|
|
17
17
|
expiresAt: integer('expires_at'),
|
|
18
|
+
/** Persisted job execution deadline (ms). `null` = not finite; see timeoutUnlimited. */
|
|
19
|
+
timeoutMs: integer('timeout_ms'),
|
|
20
|
+
/** Explicit unlimited mode: 1 = producer disabled this job's execution deadline. */
|
|
21
|
+
timeoutUnlimited: integer('timeout_unlimited').notNull().default(0),
|
|
22
|
+
/** Fresh per-claim ownership token; `null` marks legacy (unfenced) claims. */
|
|
23
|
+
attemptToken: text('attempt_token'),
|
|
24
|
+
/** Ownership lease expiry — independent of execution duration; renewed while live. */
|
|
25
|
+
leaseExpiresAt: integer('lease_expires_at'),
|
|
18
26
|
}, (table) => [index('queue_jobs_ready_idx').on(table.status, table.nextRetryAt, table.createdAt)]);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@gobing-ai/ts-db",
|
|
3
|
-
"version": "0.4.
|
|
3
|
+
"version": "0.4.60",
|
|
4
4
|
"description": "@gobing-ai/ts-db — a drizzle-free database facade: typed DAOs over Bun SQLite / Cloudflare D1, a small predicate query spec, single-source-of-truth tables, and migrations. Drizzle stays an internal detail.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"typescript",
|
|
@@ -66,8 +66,8 @@
|
|
|
66
66
|
"release": "echo 'Manual publish is disabled. Releases go through GitHub Actions via Trusted Publishing — push a tag: git tag @gobing-ai/ts-db-v<version> && git push --tags' && exit 1"
|
|
67
67
|
},
|
|
68
68
|
"dependencies": {
|
|
69
|
-
"@gobing-ai/ts-runtime": "^0.4.
|
|
70
|
-
"@gobing-ai/ts-utils": "^0.4.
|
|
69
|
+
"@gobing-ai/ts-runtime": "^0.4.60",
|
|
70
|
+
"@gobing-ai/ts-utils": "^0.4.60"
|
|
71
71
|
},
|
|
72
72
|
"peerDependencies": {
|
|
73
73
|
"drizzle-orm": ">=0.38.0",
|
|
@@ -41,4 +41,9 @@ export const embeddedMigrations: EmbeddedMigration[] = [
|
|
|
41
41
|
sql: 'CREATE INDEX `idx_inbox_messages_to_status` ON `inbox_messages` (`to_id`,`status`);',
|
|
42
42
|
hash: '3c00f1bce4f569e442f3142df31ff478d6c9b3d460824d46575f50a324bfae61',
|
|
43
43
|
},
|
|
44
|
+
{
|
|
45
|
+
tag: '0005_queue_attempt_ownership',
|
|
46
|
+
sql: 'ALTER TABLE `queue_jobs` ADD `timeout_ms` integer;\nALTER TABLE `queue_jobs` ADD `timeout_unlimited` integer DEFAULT 0 NOT NULL;\nALTER TABLE `queue_jobs` ADD `attempt_token` text;\nALTER TABLE `queue_jobs` ADD `lease_expires_at` integer;',
|
|
47
|
+
hash: 'f6a4eaf11eee838f424515e57eca3a46539260bb2fb5b0b112870ca5ed38d040',
|
|
48
|
+
},
|
|
44
49
|
];
|
package/src/queue-job-dao.ts
CHANGED
|
@@ -20,6 +20,45 @@ export interface QueueStats {
|
|
|
20
20
|
failed: number;
|
|
21
21
|
}
|
|
22
22
|
|
|
23
|
+
/** Options for enqueuing a job: retry policy, delay, TTL, and execution policy. */
|
|
24
|
+
export interface QueueEnqueueOptions {
|
|
25
|
+
/** Total attempts allowed (default 3). */
|
|
26
|
+
maxRetries?: number;
|
|
27
|
+
delay?: number;
|
|
28
|
+
ttlMs?: number;
|
|
29
|
+
/**
|
|
30
|
+
* Job execution policy persisted with the row: a positive integer ms
|
|
31
|
+
* deadline, explicit `null` = unlimited (disables this job's deadline),
|
|
32
|
+
* omitted = no job-level decision (consumers apply their own default).
|
|
33
|
+
* Invalid explicit values are rejected before the row is created.
|
|
34
|
+
*/
|
|
35
|
+
timeoutMs?: number | null;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/** Options for an atomic claim that takes lease ownership. */
|
|
39
|
+
export interface QueueClaimOptions {
|
|
40
|
+
/** Ownership lease length in ms; omitted claims keep the legacy recovery path. */
|
|
41
|
+
leaseMs?: number;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* Validate the persisted execution-policy leaf value. The scope-resolution
|
|
46
|
+
* chain (inherit/unlimited/finite) lives in the consumer packages; the DAO
|
|
47
|
+
* only stores an already-resolved job option and refuses invalid ones.
|
|
48
|
+
*/
|
|
49
|
+
function assertValidTimeoutMs(value: number | null | undefined): void {
|
|
50
|
+
if (value === undefined || value === null) return;
|
|
51
|
+
if (typeof value !== 'number' || !Number.isInteger(value) || value <= 0) {
|
|
52
|
+
throw new RangeError(`enqueue timeoutMs must be a positive integer or null; received ${String(value)}`);
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/** Persisted execution-policy columns derived from a validated `timeoutMs` option. */
|
|
57
|
+
function timeoutColumns(timeoutMs: number | null | undefined): { timeoutMs?: number; timeoutUnlimited?: number } {
|
|
58
|
+
if (timeoutMs === undefined) return {};
|
|
59
|
+
return timeoutMs === null ? { timeoutUnlimited: 1 } : { timeoutMs };
|
|
60
|
+
}
|
|
61
|
+
|
|
23
62
|
/**
|
|
24
63
|
* Row type inferred from the queue_jobs Drizzle schema.
|
|
25
64
|
*/
|
|
@@ -39,11 +78,8 @@ export class QueueJobDao extends EntityDao<typeof queueJobs, typeof queueJobs.id
|
|
|
39
78
|
/**
|
|
40
79
|
* Enqueue a new job.
|
|
41
80
|
*/
|
|
42
|
-
async enqueue(
|
|
43
|
-
|
|
44
|
-
payload: unknown,
|
|
45
|
-
options?: { maxRetries?: number; delay?: number; ttlMs?: number },
|
|
46
|
-
): Promise<string> {
|
|
81
|
+
async enqueue(type: string, payload: unknown, options?: QueueEnqueueOptions): Promise<string> {
|
|
82
|
+
assertValidTimeoutMs(options?.timeoutMs);
|
|
47
83
|
const now = this.now();
|
|
48
84
|
const id = crypto.randomUUID();
|
|
49
85
|
|
|
@@ -56,6 +92,7 @@ export class QueueJobDao extends EntityDao<typeof queueJobs, typeof queueJobs.id
|
|
|
56
92
|
maxRetries: options?.maxRetries ?? 3,
|
|
57
93
|
nextRetryAt: options?.delay !== undefined ? now + options.delay : now,
|
|
58
94
|
...(options?.ttlMs !== undefined ? { expiresAt: now + options.ttlMs } : {}),
|
|
95
|
+
...timeoutColumns(options?.timeoutMs),
|
|
59
96
|
});
|
|
60
97
|
|
|
61
98
|
return id;
|
|
@@ -64,9 +101,10 @@ export class QueueJobDao extends EntityDao<typeof queueJobs, typeof queueJobs.id
|
|
|
64
101
|
/**
|
|
65
102
|
* Enqueue multiple jobs in a single batch.
|
|
66
103
|
*/
|
|
67
|
-
async enqueueBatch(
|
|
68
|
-
|
|
69
|
-
|
|
104
|
+
async enqueueBatch(jobs: Array<{ type: string; payload: unknown } & QueueEnqueueOptions>): Promise<string[]> {
|
|
105
|
+
for (const job of jobs) {
|
|
106
|
+
assertValidTimeoutMs(job.timeoutMs);
|
|
107
|
+
}
|
|
70
108
|
const now = this.now();
|
|
71
109
|
const ids: string[] = [];
|
|
72
110
|
|
|
@@ -83,6 +121,7 @@ export class QueueJobDao extends EntityDao<typeof queueJobs, typeof queueJobs.id
|
|
|
83
121
|
maxRetries: job.maxRetries ?? 3,
|
|
84
122
|
nextRetryAt: job.delay !== undefined ? now + job.delay : now,
|
|
85
123
|
...(job.ttlMs !== undefined ? { expiresAt: now + job.ttlMs } : {}),
|
|
124
|
+
...timeoutColumns(job.timeoutMs),
|
|
86
125
|
};
|
|
87
126
|
});
|
|
88
127
|
|
|
@@ -158,30 +197,57 @@ export class QueueJobDao extends EntityDao<typeof queueJobs, typeof queueJobs.id
|
|
|
158
197
|
}
|
|
159
198
|
|
|
160
199
|
/**
|
|
161
|
-
* Atomically claim ready pending jobs
|
|
200
|
+
* Atomically claim ready pending jobs — and recover expired leased
|
|
201
|
+
* attempts — for processing.
|
|
162
202
|
*
|
|
163
203
|
* The update and selection happen in one SQLite statement so competing
|
|
164
204
|
* consumers only receive rows they actually transitioned to processing.
|
|
205
|
+
* When `options.leaseMs` is provided the claimed row takes an ownership
|
|
206
|
+
* lease and a fresh attempt token; the token fences mutations and keeps
|
|
207
|
+
* rival consumers from re-claiming the live attempt until the lease
|
|
208
|
+
* expires. Expired leased rows (worker loss) are recovered here with a
|
|
209
|
+
* fresh token — not by the age-based `resetStuckJobs` sweep, which skips
|
|
210
|
+
* token-carrying rows. Claims WITHOUT `leaseMs` mint no ownership markers
|
|
211
|
+
* (and clear any stale ones on reclaim) so the row keeps the legacy
|
|
212
|
+
* recovery path through the age sweep — a token without a lease would
|
|
213
|
+
* make the row unrecoverable by both paths.
|
|
165
214
|
*/
|
|
166
|
-
async claimReady(batchSize: number): Promise<QueueJobRecord[]> {
|
|
215
|
+
async claimReady(batchSize: number, options?: QueueClaimOptions): Promise<QueueJobRecord[]> {
|
|
167
216
|
const limit = Math.floor(batchSize);
|
|
168
217
|
if (limit <= 0) return [];
|
|
169
218
|
|
|
170
219
|
const now = this.now();
|
|
220
|
+
const leaseMs = options?.leaseMs;
|
|
221
|
+
const leased = leaseMs !== undefined;
|
|
222
|
+
const leaseExpiresAt = leased ? now + leaseMs : null;
|
|
223
|
+
const reclaimable = sql`(
|
|
224
|
+
${queueJobs.status} = 'pending'
|
|
225
|
+
AND (${queueJobs.nextRetryAt} IS NULL OR ${queueJobs.nextRetryAt} <= ${now})
|
|
226
|
+
) OR (
|
|
227
|
+
${queueJobs.status} = 'processing'
|
|
228
|
+
AND ${queueJobs.attemptToken} IS NOT NULL
|
|
229
|
+
AND ${queueJobs.leaseExpiresAt} IS NOT NULL
|
|
230
|
+
AND ${queueJobs.leaseExpiresAt} <= ${now}
|
|
231
|
+
)`;
|
|
171
232
|
|
|
172
233
|
const result = await (this.db as UpdateReturningDb)
|
|
173
234
|
.update(queueJobs)
|
|
174
|
-
.set({
|
|
235
|
+
.set({
|
|
236
|
+
status: 'processing',
|
|
237
|
+
processingAt: now,
|
|
238
|
+
updatedAt: now,
|
|
239
|
+
attemptToken: leased ? sql`lower(hex(randomblob(16)))` : null,
|
|
240
|
+
leaseExpiresAt,
|
|
241
|
+
})
|
|
175
242
|
.where(
|
|
176
243
|
sql`${queueJobs.id} IN (
|
|
177
244
|
SELECT id
|
|
178
245
|
FROM ${queueJobs}
|
|
179
|
-
WHERE
|
|
180
|
-
AND (next_retry_at IS NULL OR next_retry_at <= ${now})
|
|
246
|
+
WHERE ${reclaimable}
|
|
181
247
|
ORDER BY created_at
|
|
182
248
|
LIMIT ${limit}
|
|
183
249
|
)
|
|
184
|
-
AND ${
|
|
250
|
+
AND ${reclaimable}`,
|
|
185
251
|
)
|
|
186
252
|
.returning();
|
|
187
253
|
|
|
@@ -204,41 +270,106 @@ export class QueueJobDao extends EntityDao<typeof queueJobs, typeof queueJobs.id
|
|
|
204
270
|
|
|
205
271
|
/**
|
|
206
272
|
* Mark a job as completed.
|
|
273
|
+
*
|
|
274
|
+
* When `attemptToken` is supplied the mutation is fenced: it applies only
|
|
275
|
+
* if the row is still owned by that attempt. Returns whether it applied.
|
|
207
276
|
*/
|
|
208
|
-
async markCompleted(id: string): Promise<
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
277
|
+
async markCompleted(id: string, attemptToken?: string): Promise<boolean> {
|
|
278
|
+
const where =
|
|
279
|
+
attemptToken === undefined
|
|
280
|
+
? eq(queueJobs.id, id)
|
|
281
|
+
: and(eq(queueJobs.id, id), eq(queueJobs.attemptToken, attemptToken));
|
|
282
|
+
const result = await (this.db as UpdateChangesDb)
|
|
283
|
+
.update(queueJobs)
|
|
284
|
+
.set({
|
|
285
|
+
status: 'completed',
|
|
286
|
+
processingAt: null,
|
|
287
|
+
attemptToken: null,
|
|
288
|
+
leaseExpiresAt: null,
|
|
289
|
+
updatedAt: this.now(),
|
|
290
|
+
})
|
|
291
|
+
.where(where);
|
|
292
|
+
return (result as { changes: number }).changes === 1;
|
|
213
293
|
}
|
|
214
294
|
|
|
215
295
|
/**
|
|
216
|
-
* Mark a job as failed.
|
|
296
|
+
* Mark a job as failed, fenced by fresh attempt ownership when supplied.
|
|
217
297
|
*/
|
|
218
|
-
async markFailed(id: string, attempts: number, error: string): Promise<
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
298
|
+
async markFailed(id: string, attempts: number, error: string, attemptToken?: string): Promise<boolean> {
|
|
299
|
+
const where =
|
|
300
|
+
attemptToken === undefined
|
|
301
|
+
? eq(queueJobs.id, id)
|
|
302
|
+
: and(eq(queueJobs.id, id), eq(queueJobs.attemptToken, attemptToken));
|
|
303
|
+
const result = await (this.db as UpdateChangesDb)
|
|
304
|
+
.update(queueJobs)
|
|
305
|
+
.set({
|
|
306
|
+
status: 'failed',
|
|
307
|
+
attempts,
|
|
308
|
+
lastError: error,
|
|
309
|
+
processingAt: null,
|
|
310
|
+
attemptToken: null,
|
|
311
|
+
leaseExpiresAt: null,
|
|
312
|
+
updatedAt: this.now(),
|
|
313
|
+
})
|
|
314
|
+
.where(where);
|
|
315
|
+
return (result as { changes: number }).changes === 1;
|
|
225
316
|
}
|
|
226
317
|
|
|
227
318
|
/**
|
|
228
|
-
* Reset a job to pending for retry with backoff
|
|
319
|
+
* Reset a job to pending for retry with backoff, fenced by fresh attempt
|
|
320
|
+
* ownership when supplied.
|
|
229
321
|
*/
|
|
230
|
-
async markForRetry(
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
322
|
+
async markForRetry(
|
|
323
|
+
id: string,
|
|
324
|
+
attempts: number,
|
|
325
|
+
errorMessage: string,
|
|
326
|
+
nextRetryAt: number,
|
|
327
|
+
attemptToken?: string,
|
|
328
|
+
): Promise<boolean> {
|
|
329
|
+
const where =
|
|
330
|
+
attemptToken === undefined
|
|
331
|
+
? eq(queueJobs.id, id)
|
|
332
|
+
: and(eq(queueJobs.id, id), eq(queueJobs.attemptToken, attemptToken));
|
|
333
|
+
const result = await (this.db as UpdateChangesDb)
|
|
334
|
+
.update(queueJobs)
|
|
335
|
+
.set({
|
|
336
|
+
status: 'pending',
|
|
337
|
+
attempts,
|
|
338
|
+
lastError: errorMessage,
|
|
339
|
+
nextRetryAt,
|
|
340
|
+
processingAt: null,
|
|
341
|
+
attemptToken: null,
|
|
342
|
+
leaseExpiresAt: null,
|
|
343
|
+
updatedAt: this.now(),
|
|
344
|
+
})
|
|
345
|
+
.where(where);
|
|
346
|
+
return (result as { changes: number }).changes === 1;
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
/**
|
|
350
|
+
* Extend the ownership lease of a claimed attempt.
|
|
351
|
+
*
|
|
352
|
+
* Applies only while the row is still owned by `attemptToken`; a `false`
|
|
353
|
+
* return means ownership was lost (expired and reclaimed by another
|
|
354
|
+
* consumer) and the caller must abort and fence its acknowledgements.
|
|
355
|
+
*/
|
|
356
|
+
async renewLease(id: string, attemptToken: string, leaseMs: number): Promise<boolean> {
|
|
357
|
+
const now = this.now();
|
|
358
|
+
const result = await (this.db as UpdateChangesDb)
|
|
359
|
+
.update(queueJobs)
|
|
360
|
+
.set({ leaseExpiresAt: now + leaseMs, updatedAt: now })
|
|
361
|
+
.where(
|
|
362
|
+
and(eq(queueJobs.id, id), eq(queueJobs.attemptToken, attemptToken), eq(queueJobs.status, 'processing')),
|
|
363
|
+
);
|
|
364
|
+
return (result as { changes: number }).changes === 1;
|
|
238
365
|
}
|
|
239
366
|
|
|
240
367
|
/**
|
|
241
368
|
* Reset stuck processing jobs (processing beyond visibility timeout).
|
|
369
|
+
*
|
|
370
|
+
* Age-based legacy recovery only: rows carrying an attempt token are
|
|
371
|
+
* lease-owned and recover exclusively through their expired lease in
|
|
372
|
+
* `claimReady`, never by processing age.
|
|
242
373
|
*/
|
|
243
374
|
async resetStuckJobs(visibilityTimeout: number): Promise<number> {
|
|
244
375
|
const cutoff = this.now() - visibilityTimeout;
|
|
@@ -247,7 +378,8 @@ export class QueueJobDao extends EntityDao<typeof queueJobs, typeof queueJobs.id
|
|
|
247
378
|
.update(queueJobs)
|
|
248
379
|
.set({ status: 'pending', processingAt: null, updatedAt: this.now() })
|
|
249
380
|
.where(
|
|
250
|
-
sql`${queueJobs.status} = 'processing' AND ${queueJobs.processingAt} IS NOT NULL AND ${queueJobs.processingAt} <= ${cutoff}
|
|
381
|
+
sql`${queueJobs.status} = 'processing' AND ${queueJobs.processingAt} IS NOT NULL AND ${queueJobs.processingAt} <= ${cutoff}
|
|
382
|
+
AND ${queueJobs.attemptToken} IS NULL`,
|
|
251
383
|
);
|
|
252
384
|
|
|
253
385
|
return (result as { changes: number }).changes;
|
package/src/schema/queue-jobs.ts
CHANGED
|
@@ -18,6 +18,14 @@ export const queueJobs = sqliteTable(
|
|
|
18
18
|
lastError: text('last_error'),
|
|
19
19
|
processingAt: integer('processing_at'),
|
|
20
20
|
expiresAt: integer('expires_at'),
|
|
21
|
+
/** Persisted job execution deadline (ms). `null` = not finite; see timeoutUnlimited. */
|
|
22
|
+
timeoutMs: integer('timeout_ms'),
|
|
23
|
+
/** Explicit unlimited mode: 1 = producer disabled this job's execution deadline. */
|
|
24
|
+
timeoutUnlimited: integer('timeout_unlimited').notNull().default(0),
|
|
25
|
+
/** Fresh per-claim ownership token; `null` marks legacy (unfenced) claims. */
|
|
26
|
+
attemptToken: text('attempt_token'),
|
|
27
|
+
/** Ownership lease expiry — independent of execution duration; renewed while live. */
|
|
28
|
+
leaseExpiresAt: integer('lease_expires_at'),
|
|
21
29
|
},
|
|
22
30
|
(table) => [index('queue_jobs_ready_idx').on(table.status, table.nextRetryAt, table.createdAt)],
|
|
23
31
|
);
|