@sqlane/core 0.1.0 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -0
- package/package.json +20 -5
- package/src/create_sqlane.js +1 -0
- package/src/runner.js +137 -22
- package/src/store/mysql_final.js +23 -9
- package/src/store/postgres_final.js +27 -17
- package/src/store/store.js +7 -2
- package/types/adapters/mysql2.d.ts +13 -0
- package/types/adapters/pg.d.ts +16 -0
- package/types/backoff.d.ts +8 -0
- package/types/create_sqlane.d.ts +8 -0
- package/types/defaults.d.ts +18 -0
- package/types/define_job.d.ts +6 -0
- package/types/duration.d.ts +7 -0
- package/types/enqueue.d.ts +50 -0
- package/types/error_json.d.ts +21 -0
- package/types/errors.d.ts +129 -0
- package/types/hint_channel.d.ts +31 -0
- package/types/index.d.ts +72 -0
- package/types/migrations.d.ts +22 -0
- package/types/mutex_keys.d.ts +9 -0
- package/types/new_job.d.ts +40 -0
- package/types/payload.d.ts +10 -0
- package/types/registry.d.ts +72 -0
- package/types/runner.d.ts +92 -0
- package/types/schema/mysql.d.ts +10 -0
- package/types/schema/postgres.d.ts +10 -0
- package/types/store/mysql_enqueue.d.ts +9 -0
- package/types/store/mysql_final.d.ts +135 -0
- package/types/store/mysql_worker.d.ts +51 -0
- package/types/store/postgres_enqueue.d.ts +9 -0
- package/types/store/postgres_final.d.ts +135 -0
- package/types/store/postgres_worker.d.ts +45 -0
- package/types/store/store.d.ts +221 -0
- package/types/transient_error.d.ts +6 -0
- package/types/types.d.ts +651 -0
- package/types/validate.d.ts +76 -0
package/README.md
CHANGED
|
@@ -52,6 +52,15 @@ process.once('SIGTERM', () => sqlane.close({ deadline: '30s' }));
|
|
|
52
52
|
|
|
53
53
|
In tests, run the due jobs without `start()`: `await sqlane.drain()`. More in [`examples/`](examples).
|
|
54
54
|
|
|
55
|
+
## Performance
|
|
56
|
+
|
|
57
|
+
On a laptop (M1 Pro, Docker, default config), details in [benchmarks](docs/benchmarks.md):
|
|
58
|
+
|
|
59
|
+
- 1000 handled job names, 10 with pending jobs: the probe takes under 1 ms. Idle names cost nothing.
|
|
60
|
+
- From commit to handler start: about 25 ms (re-probe or hint), about 5 ms with PostgreSQL NOTIFY.
|
|
61
|
+
- `enqueueMany`: 58,000–110,000 jobs/s. Empty jobs, one process: 1,000–1,800 jobs/s at concurrency 10,
|
|
62
|
+
3,800–5,800 at concurrency 50.
|
|
63
|
+
|
|
55
64
|
## Development
|
|
56
65
|
|
|
57
66
|
Requires [mise](https://mise.jdx.dev). It installs Node.js, pnpm, hk, betterleaks and typos (`mise.toml`).
|
|
@@ -71,6 +80,7 @@ hk install --global --mise # Git hooks, also for GUI Git clients: lint, format
|
|
|
71
80
|
| `pnpm lint:package` | Check the package that npm would publish (publint) |
|
|
72
81
|
| `pnpm typecheck` | Type-check the JSDoc (`tsc`, no emit) |
|
|
73
82
|
| `pnpm check` | Run all hook checks on all files (`hk check --all`) |
|
|
83
|
+
| `pnpm bench` | Run the [benchmarks](docs/benchmarks.md) (Docker) |
|
|
74
84
|
|
|
75
85
|
Commit messages follow [Conventional Commits](https://www.conventionalcommits.org). A change of the package adds a
|
|
76
86
|
changeset (`pnpm changeset`); see [releasing](docs/releasing.md).
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@sqlane/core",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.2",
|
|
4
4
|
"description": "Transactional SQL job queue for Node.js (MySQL, PostgreSQL)",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"background-jobs",
|
|
@@ -22,13 +22,23 @@
|
|
|
22
22
|
},
|
|
23
23
|
"files": [
|
|
24
24
|
"src",
|
|
25
|
-
"!src/**/*.test.js"
|
|
25
|
+
"!src/**/*.test.js",
|
|
26
|
+
"types"
|
|
26
27
|
],
|
|
27
28
|
"type": "module",
|
|
28
29
|
"exports": {
|
|
29
|
-
".":
|
|
30
|
-
|
|
31
|
-
|
|
30
|
+
".": {
|
|
31
|
+
"types": "./types/index.d.ts",
|
|
32
|
+
"default": "./src/index.js"
|
|
33
|
+
},
|
|
34
|
+
"./mysql2": {
|
|
35
|
+
"types": "./types/adapters/mysql2.d.ts",
|
|
36
|
+
"default": "./src/adapters/mysql2.js"
|
|
37
|
+
},
|
|
38
|
+
"./pg": {
|
|
39
|
+
"types": "./types/adapters/pg.d.ts",
|
|
40
|
+
"default": "./src/adapters/pg.js"
|
|
41
|
+
}
|
|
32
42
|
},
|
|
33
43
|
"publishConfig": {
|
|
34
44
|
"access": "public"
|
|
@@ -38,6 +48,7 @@
|
|
|
38
48
|
"ulid": "^3.0.2"
|
|
39
49
|
},
|
|
40
50
|
"devDependencies": {
|
|
51
|
+
"@arethetypeswrong/cli": "0.18.5",
|
|
41
52
|
"@changesets/changelog-github": "1.0.1",
|
|
42
53
|
"@changesets/cli": "3.0.3",
|
|
43
54
|
"@stylistic/eslint-plugin": "5.10.0",
|
|
@@ -74,15 +85,19 @@
|
|
|
74
85
|
"lint": "oxlint --deny-warnings",
|
|
75
86
|
"lint:fix": "oxlint --fix",
|
|
76
87
|
"lint:package": "publint --strict",
|
|
88
|
+
"lint:types": "attw --pack . --profile esm-only",
|
|
77
89
|
"format": "oxfmt",
|
|
78
90
|
"format:check": "oxfmt --check",
|
|
79
91
|
"typecheck": "tsc -p tsconfig.json",
|
|
92
|
+
"build:types": "node --eval \"require('node:fs').rmSync('types', { recursive: true, force: true })\" && tsc -p tsconfig.build.json",
|
|
93
|
+
"check:types": "out=$(mktemp -d) && tsc -p tsconfig.build.json --declarationDir \"$out\" && diff -r types \"$out\"; status=$?; rm -rf \"$out\"; exit $status",
|
|
80
94
|
"test": "vitest run",
|
|
81
95
|
"test:unit": "vitest run --project unit",
|
|
82
96
|
"test:integration": "vitest run --project 'integration:*'",
|
|
83
97
|
"test:watch": "vitest",
|
|
84
98
|
"test:coverage": "vitest run --coverage",
|
|
85
99
|
"check": "hk check --all",
|
|
100
|
+
"bench": "node bench/run.js",
|
|
86
101
|
"changeset": "changeset",
|
|
87
102
|
"changeset:version": "changeset version && oxfmt --no-error-on-unmatched-pattern CHANGELOG.md"
|
|
88
103
|
}
|
package/src/create_sqlane.js
CHANGED
|
@@ -147,6 +147,7 @@ export function createSqlane(options) {
|
|
|
147
147
|
logger: options.logger,
|
|
148
148
|
onEvent: options.onEvent,
|
|
149
149
|
hintChannel,
|
|
150
|
+
hintsAfterCommit: Boolean(adapter.notify),
|
|
150
151
|
timings: { pollIntervalMs, ...testTimings },
|
|
151
152
|
});
|
|
152
153
|
const enqueuer = createEnqueuer({
|
package/src/runner.js
CHANGED
|
@@ -47,11 +47,13 @@ export const DEFAULT_TIMINGS = {
|
|
|
47
47
|
missingPayloadGraceMs: 60_000,
|
|
48
48
|
};
|
|
49
49
|
|
|
50
|
-
/** The wakes after
|
|
50
|
+
/** The wakes after the last enqueue (or hint), until its commit makes the job visible */
|
|
51
51
|
const REPROBE_DELAYS_MS = [20, 50, 100, 200, 500, 1000, 2000];
|
|
52
52
|
const RELEASE_EXTRA_DELAY_MS = 2000;
|
|
53
53
|
const STALLED = { name: 'StalledError', message: 'The lease of the job ended: its process stopped or blocked' };
|
|
54
54
|
const BATCH = 1000;
|
|
55
|
+
/** The completed jobs of one write */
|
|
56
|
+
const MAX_COMPLETE_BATCH = 100;
|
|
55
57
|
|
|
56
58
|
/**
|
|
57
59
|
* @typedef {object} RunningJob
|
|
@@ -78,6 +80,8 @@ const BATCH = 1000;
|
|
|
78
80
|
* @property {import('./types.js').Logger | undefined} logger
|
|
79
81
|
* @property {((event: import('./types.js').SqlaneEvent) => void) | undefined} onEvent
|
|
80
82
|
* @property {import('./hint_channel.js').HintChannel} hintChannel
|
|
83
|
+
* @property {boolean} hintsAfterCommit The hints come after the commit of their enqueue (PostgreSQL NOTIFY): they wake
|
|
84
|
+
* at once, with no re-probes.
|
|
81
85
|
* @property {Partial<Timings>} [timings] For tests.
|
|
82
86
|
*/
|
|
83
87
|
|
|
@@ -93,6 +97,24 @@ const BATCH = 1000;
|
|
|
93
97
|
* @property {(name: string) => boolean} canStartNow This process handles the name and has a free slot for it.
|
|
94
98
|
*/
|
|
95
99
|
|
|
100
|
+
/**
|
|
101
|
+
* Waits for the promise, at most `ms`. The timer ends with the wait: it keeps no process alive after it.
|
|
102
|
+
* @param {Promise<unknown>} promise
|
|
103
|
+
* @param {number} ms
|
|
104
|
+
* @param {{ ref?: boolean }} [options] `ref: false`: the timer alone does not keep the process alive.
|
|
105
|
+
* @returns {Promise<void>}
|
|
106
|
+
*/
|
|
107
|
+
async function waitAtMost(promise, ms, { ref = true } = {}) {
|
|
108
|
+
const controller = new AbortController();
|
|
109
|
+
|
|
110
|
+
try {
|
|
111
|
+
// The race handles the rejection of the aborted timer
|
|
112
|
+
await Promise.race([promise, sleep(Math.max(ms, 0), undefined, { signal: controller.signal, ref })]);
|
|
113
|
+
} finally {
|
|
114
|
+
controller.abort();
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
|
|
96
118
|
/**
|
|
97
119
|
* @param {RunningJob} job
|
|
98
120
|
* @returns {import('./types.js').HandlerContext}
|
|
@@ -127,6 +149,7 @@ export function createRunner({
|
|
|
127
149
|
logger,
|
|
128
150
|
onEvent,
|
|
129
151
|
hintChannel,
|
|
152
|
+
hintsAfterCommit,
|
|
130
153
|
timings = {},
|
|
131
154
|
}) {
|
|
132
155
|
const t = { ...DEFAULT_TIMINGS, ...timings };
|
|
@@ -145,10 +168,15 @@ export function createRunner({
|
|
|
145
168
|
let closeDone = null;
|
|
146
169
|
/** @type {Set<NodeJS.Timeout>} */
|
|
147
170
|
const timers = new Set();
|
|
148
|
-
/**
|
|
149
|
-
|
|
171
|
+
/** The time of the last enqueue (or hint): the re-probes follow it */
|
|
172
|
+
let reprobeFrom = 0;
|
|
173
|
+
/** @type {{ timer: NodeJS.Timeout, at: number } | null} The next re-probe */
|
|
174
|
+
let reprobe = null;
|
|
150
175
|
let contentionDelayMs = 0;
|
|
151
176
|
const counts = { completed: 0, retried: 0, failed: 0 };
|
|
177
|
+
/** @type {{ job: import('./store/store.js').FinishedJob, written: PromiseWithResolvers<void> }[]} */
|
|
178
|
+
let toComplete = [];
|
|
179
|
+
let completing = false;
|
|
152
180
|
/** The jobs that finished: their results are written */
|
|
153
181
|
let finishedCount = 0;
|
|
154
182
|
|
|
@@ -175,6 +203,7 @@ export function createRunner({
|
|
|
175
203
|
/**
|
|
176
204
|
* @param {() => void} func
|
|
177
205
|
* @param {number} delayMs
|
|
206
|
+
* @returns {NodeJS.Timeout}
|
|
178
207
|
*/
|
|
179
208
|
function setTimer(func, delayMs) {
|
|
180
209
|
const timer = setTimeout(() => {
|
|
@@ -184,6 +213,8 @@ export function createRunner({
|
|
|
184
213
|
|
|
185
214
|
timer.unref();
|
|
186
215
|
timers.add(timer);
|
|
216
|
+
|
|
217
|
+
return timer;
|
|
187
218
|
}
|
|
188
219
|
|
|
189
220
|
/**
|
|
@@ -338,6 +369,50 @@ export function createRunner({
|
|
|
338
369
|
}
|
|
339
370
|
}
|
|
340
371
|
|
|
372
|
+
/**
|
|
373
|
+
* Completes the job in the next write. One write at a time: the jobs that finish during a write go together in the
|
|
374
|
+
* next one, in one transaction. A lone job waits for no other.
|
|
375
|
+
* @param {import('./store/store.js').FinishedJob} job
|
|
376
|
+
* @returns {Promise<void>} Rejects with the error of its write.
|
|
377
|
+
*/
|
|
378
|
+
function completeJob(job) {
|
|
379
|
+
const written = Promise.withResolvers();
|
|
380
|
+
|
|
381
|
+
toComplete.push({ job, written });
|
|
382
|
+
|
|
383
|
+
if (!completing) {
|
|
384
|
+
void writeCompletions();
|
|
385
|
+
}
|
|
386
|
+
|
|
387
|
+
return written.promise;
|
|
388
|
+
}
|
|
389
|
+
|
|
390
|
+
async function writeCompletions() {
|
|
391
|
+
completing = true;
|
|
392
|
+
|
|
393
|
+
while (toComplete.length) {
|
|
394
|
+
const batch = toComplete.slice(0, MAX_COMPLETE_BATCH);
|
|
395
|
+
|
|
396
|
+
toComplete = toComplete.slice(MAX_COMPLETE_BATCH);
|
|
397
|
+
|
|
398
|
+
try {
|
|
399
|
+
// In the order of the ids: the row locks of two writes come in one order
|
|
400
|
+
// oxlint-disable-next-line no-await-in-loop -- one write at a time
|
|
401
|
+
await store.completeJobs(batch.map(entry => entry.job).toSorted((a, b) => (a.id < b.id ? -1 : 1)));
|
|
402
|
+
|
|
403
|
+
for (const { written } of batch) {
|
|
404
|
+
written.resolve();
|
|
405
|
+
}
|
|
406
|
+
} catch (error) {
|
|
407
|
+
for (const { written } of batch) {
|
|
408
|
+
written.reject(error);
|
|
409
|
+
}
|
|
410
|
+
}
|
|
411
|
+
}
|
|
412
|
+
|
|
413
|
+
completing = false;
|
|
414
|
+
}
|
|
415
|
+
|
|
341
416
|
/**
|
|
342
417
|
* Runs a write of the result until it succeeds; on errors, it retries with a backoff while the lease is renewed.
|
|
343
418
|
* @param {RunningJob} job
|
|
@@ -378,7 +453,7 @@ export function createRunner({
|
|
|
378
453
|
const durationMs = Date.now() - job.startedAt;
|
|
379
454
|
|
|
380
455
|
if (error === null && !job.timedOut) {
|
|
381
|
-
await writeResult(job, () =>
|
|
456
|
+
await writeResult(job, () => completeJob(claimed));
|
|
382
457
|
counts.completed++;
|
|
383
458
|
emit({ type: 'completed', ...base, durationMs });
|
|
384
459
|
|
|
@@ -505,7 +580,7 @@ export function createRunner({
|
|
|
505
580
|
}
|
|
506
581
|
|
|
507
582
|
// The job keeps its keys and its slot until the handler settles, at most one more lease
|
|
508
|
-
await
|
|
583
|
+
await waitAtMost(handlerDone, t.leaseMs, { ref: false });
|
|
509
584
|
|
|
510
585
|
return timeoutError;
|
|
511
586
|
}
|
|
@@ -757,22 +832,55 @@ export function createRunner({
|
|
|
757
832
|
return;
|
|
758
833
|
}
|
|
759
834
|
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
}
|
|
835
|
+
if (!handlesAny(names)) {
|
|
836
|
+
return;
|
|
837
|
+
}
|
|
764
838
|
|
|
765
|
-
|
|
839
|
+
// A wake probes all the names: one series of re-probes, from the last enqueue
|
|
840
|
+
reprobeFrom = Date.now();
|
|
766
841
|
|
|
767
|
-
|
|
842
|
+
// A re-probe that comes sooner stays: then the series goes on from this enqueue. A burst of enqueues gives a
|
|
843
|
+
// re-probe at most each first step, not one per enqueue.
|
|
844
|
+
if (reprobe && reprobe.at <= reprobeFrom + REPROBE_DELAYS_MS[0]) {
|
|
845
|
+
return;
|
|
846
|
+
}
|
|
768
847
|
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
848
|
+
if (reprobe) {
|
|
849
|
+
clearTimeout(reprobe.timer);
|
|
850
|
+
timers.delete(reprobe.timer);
|
|
851
|
+
}
|
|
773
852
|
|
|
774
|
-
|
|
853
|
+
scheduleReprobe();
|
|
854
|
+
}
|
|
855
|
+
|
|
856
|
+
/**
|
|
857
|
+
* @param {string[]} names
|
|
858
|
+
* @returns {boolean} This process handles one of the names, or one is `*`.
|
|
859
|
+
*/
|
|
860
|
+
function handlesAny(names) {
|
|
861
|
+
return names.some(name => name === ANY_NAME || registry.getEntry(name));
|
|
862
|
+
}
|
|
863
|
+
|
|
864
|
+
/** Sets the timer of the next step of the series, if one is left before the poll. */
|
|
865
|
+
function scheduleReprobe() {
|
|
866
|
+
const now = Date.now();
|
|
867
|
+
const at = REPROBE_DELAYS_MS
|
|
868
|
+
.filter(delay => delay < t.pollIntervalMs)
|
|
869
|
+
.map(delay => reprobeFrom + delay)
|
|
870
|
+
.find(time => time > now);
|
|
871
|
+
|
|
872
|
+
if (at === undefined || state !== 'started') {
|
|
873
|
+
reprobe = null;
|
|
874
|
+
|
|
875
|
+
return;
|
|
775
876
|
}
|
|
877
|
+
|
|
878
|
+
const timer = setTimer(() => {
|
|
879
|
+
wakeUp();
|
|
880
|
+
scheduleReprobe();
|
|
881
|
+
}, at - now);
|
|
882
|
+
|
|
883
|
+
reprobe = { at, timer };
|
|
776
884
|
}
|
|
777
885
|
|
|
778
886
|
async function start() {
|
|
@@ -796,8 +904,15 @@ export function createRunner({
|
|
|
796
904
|
loopDone = loop();
|
|
797
905
|
|
|
798
906
|
try {
|
|
799
|
-
//
|
|
800
|
-
|
|
907
|
+
// NOTIFY comes after the commit: the job is visible. Another hint is sent before the commit of its enqueue:
|
|
908
|
+
// re-probe until the job is visible.
|
|
909
|
+
await hintChannel.listen((name) => {
|
|
910
|
+
if (!hintsAfterCommit) {
|
|
911
|
+
onEnqueued([name]);
|
|
912
|
+
} else if (state === 'started' && handlesAny([name])) {
|
|
913
|
+
wakeUp();
|
|
914
|
+
}
|
|
915
|
+
});
|
|
801
916
|
} catch (error) {
|
|
802
917
|
reportError('hint_listen', error);
|
|
803
918
|
}
|
|
@@ -916,7 +1031,7 @@ export function createRunner({
|
|
|
916
1031
|
const deadline = startedAt + deadlineMs;
|
|
917
1032
|
const allDone = Promise.allSettled([...running.values()].map(job => job.done));
|
|
918
1033
|
|
|
919
|
-
await
|
|
1034
|
+
await waitAtMost(allDone, deadline - Date.now());
|
|
920
1035
|
|
|
921
1036
|
const unfinished = [...running.values()].filter(job => !job.settled);
|
|
922
1037
|
|
|
@@ -944,10 +1059,10 @@ export function createRunner({
|
|
|
944
1059
|
}
|
|
945
1060
|
|
|
946
1061
|
// The writes of the finished handlers, until the deadline
|
|
947
|
-
await
|
|
1062
|
+
await waitAtMost(
|
|
948
1063
|
Promise.allSettled([...running.values()].filter(job => job.settled).map(job => job.done)),
|
|
949
|
-
|
|
950
|
-
|
|
1064
|
+
deadline - Date.now(),
|
|
1065
|
+
);
|
|
951
1066
|
|
|
952
1067
|
for (const timer of timers) {
|
|
953
1068
|
clearTimeout(timer);
|
package/src/store/mysql_final.js
CHANGED
|
@@ -95,28 +95,42 @@ async function lingerHolds(executor, p, job, lingerMs) {
|
|
|
95
95
|
}
|
|
96
96
|
|
|
97
97
|
/**
|
|
98
|
-
* The
|
|
98
|
+
* The jobs are done: completed until their retention ends, without their payloads; their keys are freed. One
|
|
99
|
+
* transaction for many jobs: one commit.
|
|
99
100
|
* @param {Executor} executor
|
|
100
101
|
* @param {string} p
|
|
101
|
-
* @param {
|
|
102
|
-
* @returns {Promise<
|
|
102
|
+
* @param {import('./store.js').FinishedJob[]} jobs
|
|
103
|
+
* @returns {Promise<number>} The jobs that this call completed: the others lost their lease, or an earlier try
|
|
104
|
+
* committed.
|
|
103
105
|
*/
|
|
104
|
-
export async function
|
|
106
|
+
export async function completeJobs(executor, p, jobs) {
|
|
107
|
+
const ids = jobs.map(job => job.id);
|
|
108
|
+
// `id IN`: the range on the primary key; the pairs check the lock tokens
|
|
105
109
|
const changed = await executor.run(`
|
|
106
110
|
UPDATE ${p}jobs
|
|
107
111
|
SET status = 'completed', finished_at = ${NOW},
|
|
108
112
|
expires_at = ${NOW} + INTERVAL completed_retention_ms * 1000 MICROSECOND, ${CLEAR_LOCK},
|
|
109
113
|
updated_at = ${NOW}
|
|
110
|
-
WHERE id
|
|
111
|
-
`, [job.id, job.lockToken]);
|
|
114
|
+
WHERE id IN (${placeholders(ids.length)}) AND (id, lock_token) IN (${jobs.map(() => '(?, ?)').join(', ')})
|
|
115
|
+
`, [...ids, ...jobs.flatMap(job => [job.id, job.lockToken])]);
|
|
112
116
|
|
|
113
117
|
if (changed) {
|
|
114
|
-
await executor.run(`
|
|
118
|
+
await executor.run(`
|
|
119
|
+
DELETE pl FROM ${p}payloads pl JOIN ${p}jobs j ON j.id = pl.id
|
|
120
|
+
WHERE pl.id IN (${placeholders(ids.length)}) AND j.status = 'completed'
|
|
121
|
+
`, ids);
|
|
115
122
|
}
|
|
116
123
|
|
|
117
|
-
|
|
124
|
+
const keyed = jobs.filter(job => job.mutexKeys.length);
|
|
118
125
|
|
|
119
|
-
|
|
126
|
+
if (keyed.length) {
|
|
127
|
+
await executor.run(`
|
|
128
|
+
DELETE FROM ${p}mutexes
|
|
129
|
+
WHERE job_id IN (${placeholders(keyed.length)}) AND (job_id, lock_token) IN (${keyed.map(() => '(?, ?)').join(', ')})
|
|
130
|
+
`, [...keyed.map(job => job.id), ...keyed.flatMap(job => [job.id, job.lockToken])]);
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
return changed;
|
|
120
134
|
}
|
|
121
135
|
|
|
122
136
|
/**
|
|
@@ -92,27 +92,37 @@ async function lingerHolds(executor, p, job, lingerMs) {
|
|
|
92
92
|
}
|
|
93
93
|
|
|
94
94
|
/**
|
|
95
|
-
* The
|
|
95
|
+
* The jobs are done: completed until their retention ends, without their payloads; their keys are freed. One
|
|
96
|
+
* statement for many jobs.
|
|
96
97
|
* @param {Executor} executor
|
|
97
98
|
* @param {string} p
|
|
98
|
-
* @param {
|
|
99
|
-
* @returns {Promise<
|
|
99
|
+
* @param {import('./store.js').FinishedJob[]} jobs
|
|
100
|
+
* @returns {Promise<number>} The jobs that this call completed: the others lost their lease, or an earlier try
|
|
101
|
+
* committed.
|
|
100
102
|
*/
|
|
101
|
-
export async function
|
|
102
|
-
const
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
103
|
+
export async function completeJobs(executor, p, jobs) {
|
|
104
|
+
const [row] = await executor.query(`
|
|
105
|
+
WITH owned AS (
|
|
106
|
+
SELECT * FROM unnest($1::text[], $2::text[]) AS o(id, lock_token)
|
|
107
|
+
),
|
|
108
|
+
done AS (
|
|
109
|
+
UPDATE ${p}jobs j
|
|
110
|
+
SET status = 'completed', finished_at = ${NOW}, expires_at = ${NOW} + completed_retention_ms * ${MS},
|
|
111
|
+
${CLEAR_LOCK}, updated_at = ${NOW}
|
|
112
|
+
FROM owned
|
|
113
|
+
WHERE j.id = owned.id AND j.lock_token = owned.lock_token
|
|
114
|
+
RETURNING j.id
|
|
115
|
+
),
|
|
116
|
+
payloads AS (
|
|
117
|
+
DELETE FROM ${p}payloads WHERE id IN (SELECT id FROM done)
|
|
118
|
+
),
|
|
119
|
+
holds AS (
|
|
120
|
+
DELETE FROM ${p}mutexes m USING owned WHERE m.job_id = owned.id AND m.lock_token = owned.lock_token
|
|
121
|
+
)
|
|
122
|
+
SELECT count(*)::integer AS changed FROM done
|
|
123
|
+
`, [jobs.map(job => job.id), jobs.map(job => job.lockToken)]);
|
|
114
124
|
|
|
115
|
-
return changed
|
|
125
|
+
return Number(row?.changed ?? 0);
|
|
116
126
|
}
|
|
117
127
|
|
|
118
128
|
/**
|
package/src/store/store.js
CHANGED
|
@@ -50,6 +50,11 @@ import * as postgresWorker from './postgres_worker.js';
|
|
|
50
50
|
* @property {string} lockToken
|
|
51
51
|
*/
|
|
52
52
|
|
|
53
|
+
/**
|
|
54
|
+
* A job that this process ran, with its keys.
|
|
55
|
+
* @typedef {OwnedJob & { mutexKeys: readonly string[] }} FinishedJob
|
|
56
|
+
*/
|
|
57
|
+
|
|
53
58
|
/**
|
|
54
59
|
* The job that fails for good, as the failure job gets it.
|
|
55
60
|
* @typedef {object} FailedJob
|
|
@@ -77,7 +82,7 @@ import * as postgresWorker from './postgres_worker.js';
|
|
|
77
82
|
* @property {(ids: string[]) => Promise<Map<string, unknown>>} readPayloads
|
|
78
83
|
* @property {(jobs: OwnedJob[], leaseMs: number) => Promise<Set<string>>} renewLeases
|
|
79
84
|
* @property {(job: OwnedJob) => Promise<boolean>} markTimedOut
|
|
80
|
-
* @property {(
|
|
85
|
+
* @property {(jobs: FinishedJob[]) => Promise<number>} completeJobs In one transaction: the jobs that it completed.
|
|
81
86
|
* @property {(job: OwnedJob, params: { delayMs: number, error: import('../error_json.js').ErrorJson })
|
|
82
87
|
* => Promise<boolean>} retryJob
|
|
83
88
|
* @property {(
|
|
@@ -122,7 +127,7 @@ export function createStore(adapter, tablePrefix) {
|
|
|
122
127
|
readPayloads: ids => sql.readPayloads(adapter.queue, tablePrefix, ids),
|
|
123
128
|
renewLeases: (jobs, leaseMs) => inQueue(executor => final.renewLeases(executor, p, jobs, leaseMs)),
|
|
124
129
|
markTimedOut: job => final.markTimedOut(adapter.queue, p, job),
|
|
125
|
-
|
|
130
|
+
completeJobs: jobs => inQueue(executor => final.completeJobs(executor, p, jobs)),
|
|
126
131
|
retryJob: (job, params) => inQueue(executor => final.retryJob(executor, p, job, params)),
|
|
127
132
|
failJob: (job, params) => inQueue(executor => final.failJob(executor, p, job, params)),
|
|
128
133
|
releaseJob: (job, params) => inQueue(executor => final.releaseJob(executor, p, job, params)),
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @typedef {import('mysql2/promise').Pool
|
|
3
|
+
* | import('mysql2/promise').PoolConnection
|
|
4
|
+
* | import('mysql2/promise').Connection} Mysql2Connection
|
|
5
|
+
*/
|
|
6
|
+
export type Mysql2Connection = import('mysql2/promise').Pool | import('mysql2/promise').PoolConnection | import('mysql2/promise').Connection;
|
|
7
|
+
/**
|
|
8
|
+
* The adapter for mysql2 (its promise API). `enqueue` then takes a `Connection`, a `PoolConnection` (in a
|
|
9
|
+
* transaction) or a `Pool`.
|
|
10
|
+
* @param {import('mysql2/promise').Pool} pool The queue's own pool: claims, leases, results.
|
|
11
|
+
* @returns {import('../types.js').Adapter<Mysql2Connection>}
|
|
12
|
+
*/
|
|
13
|
+
export declare function mysql2Adapter(pool: import('mysql2/promise').Pool): import('../types.js').Adapter<Mysql2Connection>;
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
/** @typedef {import('pg').Pool | import('pg').PoolClient | import('pg').Client} PgConnection */
|
|
2
|
+
export type PgConnection = import('pg').Pool | import('pg').PoolClient | import('pg').Client;
|
|
3
|
+
export type PgAdapterOptions = {
|
|
4
|
+
/**
|
|
5
|
+
* Hints by LISTEN/NOTIFY: `pg_notify` in the enqueue transaction, delivered on commit.
|
|
6
|
+
* Default: false.
|
|
7
|
+
*/
|
|
8
|
+
notify?: boolean;
|
|
9
|
+
};
|
|
10
|
+
/**
|
|
11
|
+
* The adapter for pg. `enqueue` then takes a `Client`, a `PoolClient` (in a transaction) or a `Pool`.
|
|
12
|
+
* @param {import('pg').Pool} pool The queue's own pool: claims, leases, results.
|
|
13
|
+
* @param {PgAdapterOptions} [options]
|
|
14
|
+
* @returns {import('../types.js').Adapter<PgConnection>}
|
|
15
|
+
*/
|
|
16
|
+
export declare function pgAdapter(pool: import('pg').Pool, options?: PgAdapterOptions): import('../types.js').Adapter<PgConnection>;
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The delay before the next attempt.
|
|
3
|
+
* @param {import('./types.js').Backoff} backoff Checked.
|
|
4
|
+
* @param {number} attempt The attempt that failed: 1 for the first.
|
|
5
|
+
* @param {() => number} [random] For tests. Default: `Math.random`.
|
|
6
|
+
* @returns {number} Milliseconds.
|
|
7
|
+
*/
|
|
8
|
+
export declare function getBackoffMs(backoff: import('./types.js').Backoff, attempt: number, random?: () => number): number;
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Creates a sqlane instance. It connects, migrates and starts nothing.
|
|
3
|
+
* @template C The connections that `enqueue` accepts, from the adapter.
|
|
4
|
+
* @template {string} [L=never] The lane names, from `lanes`.
|
|
5
|
+
* @param {import('./types.js').SqlaneOptions<C, L>} options
|
|
6
|
+
* @returns {import('./types.js').Sqlane<C, L>}
|
|
7
|
+
*/
|
|
8
|
+
export declare function createSqlane<C, L extends string = never>(options: import('./types.js').SqlaneOptions<C, L>): import('./types.js').Sqlane<C, L>;
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
export type ResolvedDefaults = {
|
|
2
|
+
maxAttempts: number;
|
|
3
|
+
maxStalls: number;
|
|
4
|
+
backoff: import('./types.js').Backoff;
|
|
5
|
+
retention: {
|
|
6
|
+
completedMs: number;
|
|
7
|
+
failedMs: number;
|
|
8
|
+
};
|
|
9
|
+
/**
|
|
10
|
+
* Null: no timeout.
|
|
11
|
+
*/
|
|
12
|
+
timeoutMs: number | null;
|
|
13
|
+
};
|
|
14
|
+
/**
|
|
15
|
+
* @param {import('./types.js').JobDefaults} [defaults]
|
|
16
|
+
* @returns {ResolvedDefaults}
|
|
17
|
+
*/
|
|
18
|
+
export declare function resolveDefaults(defaults?: import('./types.js').JobDefaults): ResolvedDefaults;
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Defines a job name and its options. Producers and workers import the definition.
|
|
3
|
+
* @param {import('./types.js').JobOptions} options
|
|
4
|
+
* @returns {import('./types.js').JobDefinition} Frozen.
|
|
5
|
+
*/
|
|
6
|
+
export declare function defineJob(options: import('./types.js').JobOptions): import('./types.js').JobDefinition;
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Parses a duration: milliseconds (a non-negative safe integer), or a string that parse-duration reads, such as
|
|
3
|
+
* `'500ms'`, `'5s'`, `'1.5h'`, `'4m30s'`, `'2 days'`.
|
|
4
|
+
* @param {unknown} value
|
|
5
|
+
* @returns {number | null} The milliseconds, rounded. Null: not a duration, or negative.
|
|
6
|
+
*/
|
|
7
|
+
export declare function parseDuration(value: unknown): number | null;
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
export type EnqueuerOptions<C> = {
|
|
2
|
+
adapter: import('./types.js').Adapter<C>;
|
|
3
|
+
/**
|
|
4
|
+
* Checked.
|
|
5
|
+
*/
|
|
6
|
+
tablePrefix: string;
|
|
7
|
+
/**
|
|
8
|
+
* The known definition of
|
|
9
|
+
* a name.
|
|
10
|
+
*/
|
|
11
|
+
findDefinition: (name: string) => import('./types.js').JobDefinition | null;
|
|
12
|
+
unknownJobs: 'allow' | 'reject';
|
|
13
|
+
defaults: import('./defaults.js').ResolvedDefaults;
|
|
14
|
+
maxPayloadBytes: number;
|
|
15
|
+
/**
|
|
16
|
+
* PostgreSQL NOTIFY: `pg_notify` in the enqueue transaction, delivered on
|
|
17
|
+
* commit. Null: no NOTIFY.
|
|
18
|
+
*/
|
|
19
|
+
notifyChannel: string | null;
|
|
20
|
+
};
|
|
21
|
+
export type Enqueuer<C> = {
|
|
22
|
+
enqueue: import('./types.js').Sqlane<C, never>['enqueue'];
|
|
23
|
+
enqueueMany: import('./types.js').Sqlane<C, never>['enqueueMany'];
|
|
24
|
+
};
|
|
25
|
+
/**
|
|
26
|
+
* @template C
|
|
27
|
+
* @typedef {object} EnqueuerOptions
|
|
28
|
+
* @property {import('./types.js').Adapter<C>} adapter
|
|
29
|
+
* @property {string} tablePrefix Checked.
|
|
30
|
+
* @property {(name: string) => import('./types.js').JobDefinition | null} findDefinition The known definition of
|
|
31
|
+
* a name.
|
|
32
|
+
* @property {'allow' | 'reject'} unknownJobs
|
|
33
|
+
* @property {import('./defaults.js').ResolvedDefaults} defaults
|
|
34
|
+
* @property {number} maxPayloadBytes
|
|
35
|
+
* @property {string | null} notifyChannel PostgreSQL NOTIFY: `pg_notify` in the enqueue transaction, delivered on
|
|
36
|
+
* commit. Null: no NOTIFY.
|
|
37
|
+
*/
|
|
38
|
+
/**
|
|
39
|
+
* @template C
|
|
40
|
+
* @typedef {object} Enqueuer
|
|
41
|
+
* @property {import('./types.js').Sqlane<C, never>['enqueue']} enqueue
|
|
42
|
+
* @property {import('./types.js').Sqlane<C, never>['enqueueMany']} enqueueMany
|
|
43
|
+
*/
|
|
44
|
+
/**
|
|
45
|
+
* The enqueue side of an instance.
|
|
46
|
+
* @template C
|
|
47
|
+
* @param {EnqueuerOptions<C>} options
|
|
48
|
+
* @returns {Enqueuer<C>}
|
|
49
|
+
*/
|
|
50
|
+
export declare function createEnqueuer<C>({ adapter, tablePrefix, findDefinition, unknownJobs, defaults, maxPayloadBytes, notifyChannel, }: EnqueuerOptions<C>): Enqueuer<C>;
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
/** The longest stack kept with a job error: the error row and the failure job stay small. */
|
|
2
|
+
export declare const MAX_STACK_LENGTH: number;
|
|
3
|
+
export type ErrorJson = {
|
|
4
|
+
name: string;
|
|
5
|
+
message: string;
|
|
6
|
+
stack?: string;
|
|
7
|
+
code?: string;
|
|
8
|
+
};
|
|
9
|
+
/**
|
|
10
|
+
* @typedef {object} ErrorJson
|
|
11
|
+
* @property {string} name
|
|
12
|
+
* @property {string} message
|
|
13
|
+
* @property {string} [stack]
|
|
14
|
+
* @property {string} [code]
|
|
15
|
+
*/
|
|
16
|
+
/**
|
|
17
|
+
* The JSON of an error, for `last_error` and the failure job. A thrown non-Error becomes its string.
|
|
18
|
+
* @param {unknown} error
|
|
19
|
+
* @returns {ErrorJson}
|
|
20
|
+
*/
|
|
21
|
+
export declare function toErrorJson(error: unknown): ErrorJson;
|