@sqlane/core 0.0.1 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +44 -6
- package/package.json +20 -2
- package/src/adapters/mysql2.js +96 -0
- package/src/adapters/pg.js +164 -0
- package/src/backoff.js +24 -0
- package/src/create_sqlane.js +210 -0
- package/src/defaults.js +38 -0
- package/src/define_job.js +40 -0
- package/src/duration.js +27 -0
- package/src/enqueue.js +132 -0
- package/src/error_json.js +30 -0
- package/src/errors.js +188 -0
- package/src/hint_channel.js +107 -0
- package/src/index.js +64 -2
- package/src/migrations.js +163 -0
- package/src/mutex_keys.js +34 -0
- package/src/new_job.js +128 -0
- package/src/payload.js +54 -0
- package/src/registry.js +169 -0
- package/src/runner.js +974 -0
- package/src/schema/mysql.js +74 -0
- package/src/schema/postgres.js +67 -0
- package/src/store/mysql_enqueue.js +98 -0
- package/src/store/mysql_final.js +424 -0
- package/src/store/mysql_worker.js +250 -0
- package/src/store/postgres_enqueue.js +54 -0
- package/src/store/postgres_final.js +407 -0
- package/src/store/postgres_worker.js +224 -0
- package/src/store/store.js +136 -0
- package/src/transient_error.js +22 -0
- package/src/types.js +300 -0
- package/src/validate.js +181 -0
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
// The store: the SQL of one dialect, bound to an adapter and a table prefix.
|
|
2
|
+
import * as mysqlFinal from './mysql_final.js';
|
|
3
|
+
import * as mysqlWorker from './mysql_worker.js';
|
|
4
|
+
import * as postgresFinal from './postgres_final.js';
|
|
5
|
+
import * as postgresWorker from './postgres_worker.js';
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* @typedef {object} ProbeRow
|
|
9
|
+
* @property {string} name A job name with pending jobs.
|
|
10
|
+
* @property {number} delayMs Until its oldest pending job is due: 0 or less means due now.
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* @typedef {object} Candidate
|
|
15
|
+
* @property {string} id
|
|
16
|
+
* @property {string} name
|
|
17
|
+
* @property {number} runAtMs
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* A job that a claim locked, and then claimed: it runs.
|
|
22
|
+
* @typedef {object} ClaimedJob
|
|
23
|
+
* @property {string} id
|
|
24
|
+
* @property {string} name
|
|
25
|
+
* @property {string[]} mutexKeys
|
|
26
|
+
* @property {number} attempts The failed attempts so far.
|
|
27
|
+
* @property {number} maxAttempts
|
|
28
|
+
* @property {number} stalls
|
|
29
|
+
* @property {number} maxStalls
|
|
30
|
+
* @property {string | null} failureJob
|
|
31
|
+
* @property {Date} enqueuedAt
|
|
32
|
+
* @property {number} waitedMs Since it was due.
|
|
33
|
+
* @property {string} lockToken
|
|
34
|
+
*/
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* @typedef {object} ClaimParams
|
|
38
|
+
* @property {string[]} candidateIds
|
|
39
|
+
* @property {string} lockToken Of this claim.
|
|
40
|
+
* @property {string} lockedBy The process.
|
|
41
|
+
* @property {number} leaseMs
|
|
42
|
+
* @property {(locked: ClaimedJob[], busyKeys: Set<string>) => ClaimedJob[]} pick The jobs to take among the locked
|
|
43
|
+
* ones (in the order of run_at): the free slots, the free keys. The keys of the picked jobs must be distinct.
|
|
44
|
+
*/
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* A job that this process claimed: every write of it checks the lock token.
|
|
48
|
+
* @typedef {object} OwnedJob
|
|
49
|
+
* @property {string} id
|
|
50
|
+
* @property {string} lockToken
|
|
51
|
+
*/
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* The job that fails for good, as the failure job gets it.
|
|
55
|
+
* @typedef {object} FailedJob
|
|
56
|
+
* @property {string} id
|
|
57
|
+
* @property {string} name
|
|
58
|
+
* @property {unknown} payload Null: its payload is missing.
|
|
59
|
+
* @property {number} attempt
|
|
60
|
+
* @property {number} maxAttempts
|
|
61
|
+
* @property {string} failureJob The name of its failure job.
|
|
62
|
+
* @property {import('../error_json.js').ErrorJson} error
|
|
63
|
+
*/
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* Builds the failure job: its options from its definition, else the defaults; no failure job of its own.
|
|
67
|
+
* @typedef {(failed: FailedJob) => import('../new_job.js').NewJob} FailureJobBuilder
|
|
68
|
+
*/
|
|
69
|
+
|
|
70
|
+
/** @typedef {{ added: import('../new_job.js').NewJob } | { notAdded: unknown }} FailureJobResult */
|
|
71
|
+
|
|
72
|
+
/**
|
|
73
|
+
* @typedef {object} Store
|
|
74
|
+
* @property {() => Promise<ProbeRow[]>} probe
|
|
75
|
+
* @property {(limitsByName: Map<string, number>, keyedNames: string[]) => Promise<Candidate[]>} findCandidates
|
|
76
|
+
* @property {(params: ClaimParams) => Promise<ClaimedJob[]>} claim In a queue transaction of its own.
|
|
77
|
+
* @property {(ids: string[]) => Promise<Map<string, unknown>>} readPayloads
|
|
78
|
+
* @property {(jobs: OwnedJob[], leaseMs: number) => Promise<Set<string>>} renewLeases
|
|
79
|
+
* @property {(job: OwnedJob) => Promise<boolean>} markTimedOut
|
|
80
|
+
* @property {(job: OwnedJob) => Promise<boolean>} completeJob
|
|
81
|
+
* @property {(job: OwnedJob, params: { delayMs: number, error: import('../error_json.js').ErrorJson })
|
|
82
|
+
* => Promise<boolean>} retryJob
|
|
83
|
+
* @property {(
|
|
84
|
+
* job: OwnedJob,
|
|
85
|
+
* params: { error: import('../error_json.js').ErrorJson, buildFailureJob: FailureJobBuilder },
|
|
86
|
+
* ) => Promise<{ changed: boolean, failureJob: FailureJobResult | null }>} failJob
|
|
87
|
+
* @property {(job: OwnedJob, params: { delayMs: number, lingerMs: number }) => Promise<boolean>} releaseJob
|
|
88
|
+
* @property {(limit: number) => Promise<{ id: string, name: string }[]>} findStaleJobs
|
|
89
|
+
* @property {(id: string, params: { leaseMs: number, buildFailureJob: FailureJobBuilder })
|
|
90
|
+
* => Promise<{ result: 'released' | 'failed' | null, failureJob: FailureJobResult | null }>} recoverStaleJob
|
|
91
|
+
* @property {(limit: number) => Promise<{ lingers: number, orphans: number }>} sweepMutexes
|
|
92
|
+
* @property {(limit: number) => Promise<number>} deleteExpiredJobs
|
|
93
|
+
* @property {(names: string[]) => Promise<void>} makeDue For drain({ includeDelayed }).
|
|
94
|
+
* @property {() => Promise<import('../types.js').JobStats[]>} getStats
|
|
95
|
+
*/
|
|
96
|
+
|
|
97
|
+
/**
|
|
98
|
+
* @param {import('../types.js').Adapter<any>} adapter
|
|
99
|
+
* @param {string} tablePrefix Checked.
|
|
100
|
+
* @returns {Store}
|
|
101
|
+
*/
|
|
102
|
+
export function createStore(adapter, tablePrefix) {
|
|
103
|
+
const sql = adapter.dialect === 'mysql' ? mysqlWorker : postgresWorker;
|
|
104
|
+
const final = adapter.dialect === 'mysql' ? mysqlFinal : postgresFinal;
|
|
105
|
+
/**
|
|
106
|
+
* @template T
|
|
107
|
+
* @param {(executor: import('../types.js').Executor) => Promise<T>} func
|
|
108
|
+
* @returns {Promise<T>}
|
|
109
|
+
*/
|
|
110
|
+
const inQueue = func => adapter.queueTransaction(func);
|
|
111
|
+
const p = tablePrefix;
|
|
112
|
+
|
|
113
|
+
return {
|
|
114
|
+
probe: () => sql.probe(adapter.queue, tablePrefix),
|
|
115
|
+
findCandidates: (limitsByName, keyedNames) => sql.findCandidates(
|
|
116
|
+
adapter.queue,
|
|
117
|
+
tablePrefix,
|
|
118
|
+
limitsByName,
|
|
119
|
+
keyedNames,
|
|
120
|
+
),
|
|
121
|
+
claim: params => adapter.queueTransaction(executor => sql.claim(executor, tablePrefix, params)),
|
|
122
|
+
readPayloads: ids => sql.readPayloads(adapter.queue, tablePrefix, ids),
|
|
123
|
+
renewLeases: (jobs, leaseMs) => inQueue(executor => final.renewLeases(executor, p, jobs, leaseMs)),
|
|
124
|
+
markTimedOut: job => final.markTimedOut(adapter.queue, p, job),
|
|
125
|
+
completeJob: job => inQueue(executor => final.completeJob(executor, p, job)),
|
|
126
|
+
retryJob: (job, params) => inQueue(executor => final.retryJob(executor, p, job, params)),
|
|
127
|
+
failJob: (job, params) => inQueue(executor => final.failJob(executor, p, job, params)),
|
|
128
|
+
releaseJob: (job, params) => inQueue(executor => final.releaseJob(executor, p, job, params)),
|
|
129
|
+
findStaleJobs: limit => final.findStaleJobs(adapter.queue, p, limit),
|
|
130
|
+
recoverStaleJob: (id, params) => inQueue(executor => final.recoverStaleJob(executor, p, id, params)),
|
|
131
|
+
sweepMutexes: limit => inQueue(executor => final.sweepMutexes(executor, p, limit)),
|
|
132
|
+
deleteExpiredJobs: limit => inQueue(executor => final.deleteExpiredJobs(executor, p, limit)),
|
|
133
|
+
makeDue: names => inQueue(executor => final.makeDue(executor, p, names)),
|
|
134
|
+
getStats: () => final.getStats(adapter.queue, p),
|
|
135
|
+
};
|
|
136
|
+
}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
/** MySQL: deadlock, lock wait timeout. PostgreSQL: serialization failure, deadlock. */
|
|
2
|
+
const TRANSIENT_CODES = new Set(['ER_LOCK_DEADLOCK', 'ER_LOCK_WAIT_TIMEOUT', '40001', '40P01']);
|
|
3
|
+
const TRANSIENT_ERRNOS = new Set([1205, 1213]);
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* A database error that a retry of the same work can pass: the transaction lost a lock race.
|
|
7
|
+
* @param {unknown} error
|
|
8
|
+
* @returns {boolean}
|
|
9
|
+
*/
|
|
10
|
+
export function isTransientDbError(error) {
|
|
11
|
+
for (let current = error, depth = 0; current && depth < 5; depth++) {
|
|
12
|
+
const { code, errno, cause } = /** @type {{ code?: unknown, errno?: unknown, cause?: unknown }} */ (current);
|
|
13
|
+
|
|
14
|
+
if (TRANSIENT_CODES.has(String(code)) || TRANSIENT_ERRNOS.has(Number(errno))) {
|
|
15
|
+
return true;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
current = cause;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
return false;
|
|
22
|
+
}
|
package/src/types.js
ADDED
|
@@ -0,0 +1,300 @@
|
|
|
1
|
+
// The public types of sqlane. See docs/design.md.
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* A duration: milliseconds, or a string of parse-duration (`'500ms'`, `'5s'`, `'4m30s'`, `'1h'`, `'2 days'`).
|
|
5
|
+
* @typedef {string | number} Duration
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* The JSON form of T: T itself if it is JSON, else a type that T does not match. Optional fields and interfaces pass;
|
|
10
|
+
* a `Date`, a `Map`, a `Set`, a function, a `bigint` or a class instance with methods does not.
|
|
11
|
+
* @template T
|
|
12
|
+
* @typedef {T extends string | number | boolean | null ? T
|
|
13
|
+
* : T extends undefined | bigint | symbol | Function | Date | RegExp | Map<any, any> | Set<any> ? never
|
|
14
|
+
* : T extends readonly (infer U)[] ? readonly Jsonable<U>[]
|
|
15
|
+
* : T extends object ? { [K in keyof T]: Jsonable<T[K]> }
|
|
16
|
+
* : never} Jsonable
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* @typedef {object} ExponentialBackoff
|
|
21
|
+
* @property {'exponential'} type `delay × 2^(attempt − 1)`, at most `maxDelay`.
|
|
22
|
+
* @property {Duration} delay
|
|
23
|
+
* @property {Duration} [maxDelay]
|
|
24
|
+
* @property {number} [jitter] 0–1: the delay is `delay × (1 − jitter × random())`. Default: 0.2.
|
|
25
|
+
*/
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* @typedef {object} FixedBackoff
|
|
29
|
+
* @property {'fixed'} type
|
|
30
|
+
* @property {Duration} delay
|
|
31
|
+
* @property {number} [jitter] 0–1. Default: 0.2.
|
|
32
|
+
*/
|
|
33
|
+
|
|
34
|
+
/** @typedef {ExponentialBackoff | FixedBackoff} Backoff */
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* How long a finished job is kept. It is also the dedupe window of its `jobId`.
|
|
38
|
+
* @typedef {object} Retention
|
|
39
|
+
* @property {Duration} [completed] Default: `'1h'`.
|
|
40
|
+
* @property {Duration} [failed] Default: `'30d'`.
|
|
41
|
+
*/
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* The options of a job name, shared by its producers and workers. A definition overrides the instance `defaults`.
|
|
45
|
+
* @typedef {object} JobOptions
|
|
46
|
+
* @property {string} name Matches `^[a-z][a-z0-9_.-]{0,127}$`.
|
|
47
|
+
* @property {number} [maxAttempts] Default: 3.
|
|
48
|
+
* @property {number} [maxStalls] Default: 2.
|
|
49
|
+
* @property {Backoff} [backoff]
|
|
50
|
+
* @property {Retention} [retention]
|
|
51
|
+
* @property {Duration} [timeout] No default. A job with mutex keys needs one.
|
|
52
|
+
* @property {(payload: any) => readonly string[]} [mutexKeys] Sync, from the payload only.
|
|
53
|
+
* @property {string | JobDefinition} [failureJob] Added when a job fails for good.
|
|
54
|
+
*/
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* The result of `defineJob()`. Frozen.
|
|
58
|
+
* @typedef {Readonly<JobOptions>} JobDefinition
|
|
59
|
+
*/
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* @typedef {object} EnqueueBaseOptions
|
|
63
|
+
* @property {string} [jobId] 1–120 printable ASCII characters. Dedupe: a job with this id is added once while its row
|
|
64
|
+
* exists.
|
|
65
|
+
* @property {number} [maxAttempts]
|
|
66
|
+
* @property {number} [maxStalls]
|
|
67
|
+
* @property {Retention} [retention]
|
|
68
|
+
* @property {string | JobDefinition} [failureJob]
|
|
69
|
+
* @property {readonly string[]} [mutexKeys] Replace the keys of the definition; `[]` clears them.
|
|
70
|
+
*/
|
|
71
|
+
|
|
72
|
+
/**
|
|
73
|
+
* The options of an enqueue: `delay` or `runAt`, not both.
|
|
74
|
+
* @typedef {EnqueueBaseOptions & ({ delay?: Duration, runAt?: never } | { runAt: Date, delay?: never })}
|
|
75
|
+
* EnqueueOptions
|
|
76
|
+
*/
|
|
77
|
+
|
|
78
|
+
/**
|
|
79
|
+
* @typedef {object} EnqueueResult
|
|
80
|
+
* @property {string} id `<job_name>:<jobId>`, or a new ULID without a `jobId`.
|
|
81
|
+
* @property {boolean} created False: a job with this `jobId` exists already, nothing was added.
|
|
82
|
+
*/
|
|
83
|
+
|
|
84
|
+
/**
|
|
85
|
+
* @template P
|
|
86
|
+
* @typedef {object} EnqueueItem
|
|
87
|
+
* @property {P} payload
|
|
88
|
+
* @property {EnqueueOptions} [options]
|
|
89
|
+
*/
|
|
90
|
+
|
|
91
|
+
/**
|
|
92
|
+
* @typedef {object} JobInfo
|
|
93
|
+
* @property {string} id
|
|
94
|
+
* @property {string} name
|
|
95
|
+
* @property {number} attempt 1 for the first run.
|
|
96
|
+
* @property {number} maxAttempts
|
|
97
|
+
* @property {Date} enqueuedAt
|
|
98
|
+
* @property {readonly string[]} mutexKeys
|
|
99
|
+
*/
|
|
100
|
+
|
|
101
|
+
/**
|
|
102
|
+
* @typedef {object} HandlerContext
|
|
103
|
+
* @property {any} payload As JSON gives it back.
|
|
104
|
+
* @property {JobInfo} job
|
|
105
|
+
* @property {AbortSignal} signal Aborted with a `JobTimeoutError`, a `LeaseLostError` or a `ShutdownError`.
|
|
106
|
+
*/
|
|
107
|
+
|
|
108
|
+
/**
|
|
109
|
+
* Runs a job. Its return value is ignored.
|
|
110
|
+
* @typedef {(context: HandlerContext) => unknown} Handler
|
|
111
|
+
*/
|
|
112
|
+
|
|
113
|
+
/**
|
|
114
|
+
* A private lane (`concurrency`) or a shared lane of `lanes` (`lane`), not both.
|
|
115
|
+
* @template {string} L
|
|
116
|
+
* @typedef {{ timeout?: Duration } & (
|
|
117
|
+
* { concurrency?: number, lane?: never } | { lane: L, concurrency?: never }
|
|
118
|
+
* )} HandleOptions
|
|
119
|
+
*/
|
|
120
|
+
|
|
121
|
+
/**
|
|
122
|
+
* @typedef {object} LaneOptions
|
|
123
|
+
* @property {number} concurrency The handlers of the lane that run at the same time in one process.
|
|
124
|
+
*/
|
|
125
|
+
|
|
126
|
+
/**
|
|
127
|
+
* The instance defaults; a definition overrides them.
|
|
128
|
+
* @typedef {object} JobDefaults
|
|
129
|
+
* @property {number} [maxAttempts]
|
|
130
|
+
* @property {number} [maxStalls]
|
|
131
|
+
* @property {Backoff} [backoff]
|
|
132
|
+
* @property {Retention} [retention]
|
|
133
|
+
* @property {Duration} [timeout]
|
|
134
|
+
*/
|
|
135
|
+
|
|
136
|
+
/**
|
|
137
|
+
* A pub/sub that wakes the other processes. Only an optimization: a lost message delays a job until the next poll.
|
|
138
|
+
* @typedef {object} Hint
|
|
139
|
+
* @property {(channel: string, message: string) => Promise<void>} publish
|
|
140
|
+
* @property {(channel: string, onMessage: (message: string) => void) => Promise<() => Promise<void>>} subscribe
|
|
141
|
+
* Resolves to the unsubscribe function.
|
|
142
|
+
*/
|
|
143
|
+
|
|
144
|
+
/**
|
|
145
|
+
* @typedef {(object: object, message?: string) => void} LogFn
|
|
146
|
+
*/
|
|
147
|
+
|
|
148
|
+
/**
|
|
149
|
+
* A pino-compatible logger.
|
|
150
|
+
* @typedef {object} Logger
|
|
151
|
+
* @property {LogFn} debug
|
|
152
|
+
* @property {LogFn} info
|
|
153
|
+
* @property {LogFn} warn
|
|
154
|
+
* @property {LogFn} error
|
|
155
|
+
*/
|
|
156
|
+
|
|
157
|
+
/**
|
|
158
|
+
* @typedef {object} JobEventBase
|
|
159
|
+
* @property {string} jobId
|
|
160
|
+
* @property {string} jobName
|
|
161
|
+
* @property {number} attempt
|
|
162
|
+
*/
|
|
163
|
+
|
|
164
|
+
/**
|
|
165
|
+
* An event of the job lifecycle, by `type`.
|
|
166
|
+
* @typedef {(JobEventBase & (
|
|
167
|
+
* | { type: 'claimed' }
|
|
168
|
+
* | { type: 'completed', durationMs: number }
|
|
169
|
+
* | { type: 'retried', durationMs: number, delayMs: number, error: unknown }
|
|
170
|
+
* | { type: 'failed', durationMs: number, error: unknown }
|
|
171
|
+
* | { type: 'timed_out', durationMs: number }
|
|
172
|
+
* | { type: 'stalled' }
|
|
173
|
+
* | { type: 'released' }
|
|
174
|
+
* | { type: 'lease_lost' }
|
|
175
|
+
* | { type: 'failure_job_not_added', failureJob: string, error: unknown }
|
|
176
|
+
* )) | { type: 'error', operation: string, error: unknown }} SqlaneEvent
|
|
177
|
+
*/
|
|
178
|
+
|
|
179
|
+
/**
|
|
180
|
+
* The connections that `enqueue` accepts (C), and the queue's own pool. Made by `mysql2Adapter()` or `pgAdapter()`.
|
|
181
|
+
* @template C
|
|
182
|
+
* @typedef {object} Adapter
|
|
183
|
+
* @property {'mysql' | 'postgres'} dialect
|
|
184
|
+
* @property {(connection: C) => Executor} wrap Used by sqlane: wraps a connection given to `enqueue`.
|
|
185
|
+
* @property {Executor} queue Used by sqlane: the queue's own pool.
|
|
186
|
+
* @property {<T>(func: (executor: Executor) => Promise<T>) => Promise<T>} withConnection Used by sqlane: runs func on
|
|
187
|
+
* one connection of the queue's pool, then releases it.
|
|
188
|
+
* @property {<T>(func: (executor: Executor) => Promise<T>) => Promise<T>} queueTransaction Used by sqlane: runs func
|
|
189
|
+
* in a `READ COMMITTED` transaction on a connection of the queue's pool.
|
|
190
|
+
* @property {<T>(connection: C, func: (executor: Executor) => Promise<T>) => Promise<T>} inTransaction Used by sqlane:
|
|
191
|
+
* a pool → func in a transaction of its own; a connection or a client → func on it, in the caller's transaction.
|
|
192
|
+
* @property {boolean} [notify] PostgreSQL: hints by LISTEN/NOTIFY.
|
|
193
|
+
* @property {(channel: string, onMessage: (message: string) => void) => Promise<() => Promise<void>>} [listen] Used by
|
|
194
|
+
* sqlane, with `notify`: LISTEN on a connection of its own; resolves to the unlisten function.
|
|
195
|
+
*/
|
|
196
|
+
|
|
197
|
+
/**
|
|
198
|
+
* Runs SQL. Used by sqlane, through an adapter.
|
|
199
|
+
* @typedef {object} Executor
|
|
200
|
+
* @property {(sql: string, params: readonly unknown[]) => Promise<readonly Record<string, unknown>[]>} query
|
|
201
|
+
* @property {(sql: string, params: readonly unknown[]) => Promise<number>} run Runs an INSERT, UPDATE or DELETE:
|
|
202
|
+
* the number of rows it changed.
|
|
203
|
+
*/
|
|
204
|
+
|
|
205
|
+
/**
|
|
206
|
+
* @template C
|
|
207
|
+
* @template {string} L
|
|
208
|
+
* @typedef {object} SqlaneOptions
|
|
209
|
+
* @property {Adapter<C>} adapter
|
|
210
|
+
* @property {readonly JobDefinition[]} [jobs] The definitions known by name.
|
|
211
|
+
* @property {number} [concurrency] The handlers that run at the same time in this process. Default: 10.
|
|
212
|
+
* @property {Record<L, LaneOptions>} [lanes] Limits shared by several job names.
|
|
213
|
+
* @property {JobDefaults} [defaults]
|
|
214
|
+
* @property {'allow' | 'reject'} [unknownJobs] Default: `'allow'`. `'reject'`: `enqueue` of a name that is not in
|
|
215
|
+
* `jobs` and not handled throws an `UnknownJobError`.
|
|
216
|
+
* @property {Hint | null} [hint] Default: null: polling only.
|
|
217
|
+
* @property {Logger} [logger] Default: no logs.
|
|
218
|
+
* @property {(event: SqlaneEvent) => void} [onEvent]
|
|
219
|
+
* @property {string} [tablePrefix] Default: `'sqlane_'`.
|
|
220
|
+
* @property {Duration} [pollInterval] Default: `'5s'`.
|
|
221
|
+
* @property {number} [maxPayloadBytes] Default: 256 KiB.
|
|
222
|
+
* @property {boolean} [cleanup] Built-in deletion of the expired jobs. Default: true.
|
|
223
|
+
*/
|
|
224
|
+
|
|
225
|
+
/**
|
|
226
|
+
* @typedef {object} DrainOptions
|
|
227
|
+
* @property {Duration} [timeout] Default: `'10s'`.
|
|
228
|
+
* @property {boolean} [includeDelayed] Also runs the delayed jobs and the retries in backoff at once.
|
|
229
|
+
*/
|
|
230
|
+
|
|
231
|
+
/**
|
|
232
|
+
* @typedef {object} DrainResult
|
|
233
|
+
* @property {number} completed
|
|
234
|
+
* @property {number} retried Failed attempts that were retried.
|
|
235
|
+
* @property {number} failed Jobs that failed for good.
|
|
236
|
+
*/
|
|
237
|
+
|
|
238
|
+
/**
|
|
239
|
+
* @typedef {object} CloseOptions
|
|
240
|
+
* @property {Duration} [deadline] From the call. Default: `'30s'`.
|
|
241
|
+
* @property {'release' | 'abandon'} [mode] Default: `'release'`.
|
|
242
|
+
* @property {Duration} [exitsBy] From the call: the latest time the process exits. Default: the deadline plus one
|
|
243
|
+
* lease.
|
|
244
|
+
*/
|
|
245
|
+
|
|
246
|
+
/**
|
|
247
|
+
* @typedef {object} MigrateOptions
|
|
248
|
+
* @property {Duration} [lockTimeout] Default: `'10m'`.
|
|
249
|
+
*/
|
|
250
|
+
|
|
251
|
+
/**
|
|
252
|
+
* @typedef {object} JobStats
|
|
253
|
+
* @property {string} name
|
|
254
|
+
* @property {number} pending
|
|
255
|
+
* @property {number} due
|
|
256
|
+
* @property {number} running
|
|
257
|
+
* @property {number} stuck Timed out, the handler has not settled.
|
|
258
|
+
* @property {number} failed
|
|
259
|
+
* @property {number | null} oldestDueAgeMs
|
|
260
|
+
*/
|
|
261
|
+
|
|
262
|
+
/**
|
|
263
|
+
* The payload of a failure job.
|
|
264
|
+
* @typedef {object} FailedJobPayload
|
|
265
|
+
* @property {{ id: string, name: string, payload: any, attempt: number, maxAttempts: number }} job
|
|
266
|
+
* @property {{ name: string, message: string, stack?: string }} error
|
|
267
|
+
*/
|
|
268
|
+
|
|
269
|
+
/**
|
|
270
|
+
* @template C The connections that `enqueue` accepts.
|
|
271
|
+
* @template {string} L The lane names.
|
|
272
|
+
* @typedef {object} Sqlane
|
|
273
|
+
* @property {<P>(
|
|
274
|
+
* connection: C, job: JobDefinition | string, payload: P & NoInfer<Jsonable<P>>, options?: EnqueueOptions,
|
|
275
|
+
* ) => Promise<EnqueueResult>} enqueue Adds a job with the caller's connection: it must be in a transaction (or a
|
|
276
|
+
* pool: then sqlane opens its own transaction).
|
|
277
|
+
* @property {<P>(
|
|
278
|
+
* connection: C,
|
|
279
|
+
* job: JobDefinition | string,
|
|
280
|
+
* items: readonly (EnqueueItem<P> & NoInfer<EnqueueItem<Jsonable<P>>>)[],
|
|
281
|
+
* ) => Promise<EnqueueResult[]>} enqueueMany The results are in the order of the items.
|
|
282
|
+
* @property {(job: JobDefinition | string, handler: Handler, options?: HandleOptions<L>) => void} handle
|
|
283
|
+
* @property {() => Promise<void>} start
|
|
284
|
+
* @property {(options?: DrainOptions) => Promise<DrainResult>} drain For tests: runs the due jobs of the handled
|
|
285
|
+
* names until none is left, without `start()`.
|
|
286
|
+
* @property {(options?: CloseOptions) => Promise<void>} close
|
|
287
|
+
* @property {(options?: MigrateOptions) => Promise<{ applied: string[] }>} migrate Applies the missing migrations:
|
|
288
|
+
* their ids.
|
|
289
|
+
* @property {() => Promise<JobStats[]>} getStats
|
|
290
|
+
* @property {(jobName: string) => void} wake Low level: probes this name now.
|
|
291
|
+
*/
|
|
292
|
+
|
|
293
|
+
/**
|
|
294
|
+
* @typedef {object} Migration
|
|
295
|
+
* @property {string} id
|
|
296
|
+
* @property {readonly string[]} statements
|
|
297
|
+
*/
|
|
298
|
+
|
|
299
|
+
// oxlint-disable-next-line unicorn/require-module-specifiers -- makes the file a module: its typedefs are exported
|
|
300
|
+
export {};
|
package/src/validate.js
ADDED
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
import { parseDuration } from './duration.js';
|
|
2
|
+
|
|
3
|
+
/** A job name: no `:` (the job id is `<name>:<jobId>`), lowercase. */
|
|
4
|
+
export const JOB_NAME_REGEXP = /^[a-z][a-z0-9_.-]{0,127}$/;
|
|
5
|
+
|
|
6
|
+
/** A table prefix: the longest table name, `<prefix>migrations`, stays within the 63 characters of PostgreSQL. */
|
|
7
|
+
export const TABLE_PREFIX_REGEXP = /^[a-z][a-z0-9_]{0,31}$/;
|
|
8
|
+
|
|
9
|
+
/** Printable ASCII, no space at the ends: MySQL `ascii_bin` ignores trailing spaces in comparisons. */
|
|
10
|
+
export const PRINTABLE_ID_REGEXP = /^[\x21-\x7E](?:[\x20-\x7E]*[\x21-\x7E])?$/;
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* Where a check runs: for the error message and its class.
|
|
14
|
+
* @typedef {object} Scope
|
|
15
|
+
* @property {string} where E.g. `defineJob("build_report")`.
|
|
16
|
+
* @property {new (message: string) => Error} ErrorClass
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* @param {unknown} value
|
|
21
|
+
* @returns {value is Record<string, unknown>}
|
|
22
|
+
*/
|
|
23
|
+
export function isPlainObject(value) {
|
|
24
|
+
return typeof value === 'object' && value !== null && !Array.isArray(value);
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* Throws if the value is not an object with only the allowed keys.
|
|
29
|
+
* @param {unknown} value
|
|
30
|
+
* @param {readonly string[]} allowedKeys
|
|
31
|
+
* @param {string} label E.g. `"options"`.
|
|
32
|
+
* @param {Scope} scope
|
|
33
|
+
* @returns {Record<string, unknown>}
|
|
34
|
+
*/
|
|
35
|
+
export function checkObject(value, allowedKeys, label, { where, ErrorClass }) {
|
|
36
|
+
if (!isPlainObject(value)) {
|
|
37
|
+
throw new ErrorClass(`${where}: ${label} must be an object`);
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
const unknownKeys = Object.keys(value).filter(key => !allowedKeys.includes(key));
|
|
41
|
+
|
|
42
|
+
if (unknownKeys.length) {
|
|
43
|
+
throw new ErrorClass(`${where}: unknown ${label}: ${unknownKeys.join(', ')}`);
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
return value;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* @param {unknown} value
|
|
51
|
+
* @param {string} label
|
|
52
|
+
* @param {Scope} scope
|
|
53
|
+
* @param {number} [min] Default: 1.
|
|
54
|
+
*/
|
|
55
|
+
export function checkInteger(value, label, { where, ErrorClass }, min = 1) {
|
|
56
|
+
if (!Number.isSafeInteger(value) || /** @type {number} */ (value) < min) {
|
|
57
|
+
throw new ErrorClass(`${where}: ${label} must be an integer of at least ${min}, got ${JSON.stringify(value)}`);
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* @param {unknown} value
|
|
63
|
+
* @param {string} label
|
|
64
|
+
* @param {Scope} scope
|
|
65
|
+
* @param {boolean} [positive] Default: false: 0 is a duration.
|
|
66
|
+
* @returns {number} Milliseconds.
|
|
67
|
+
*/
|
|
68
|
+
export function checkDuration(value, label, { where, ErrorClass }, positive = false) {
|
|
69
|
+
const ms = parseDuration(value);
|
|
70
|
+
|
|
71
|
+
if (ms === null || (positive && ms === 0)) {
|
|
72
|
+
throw new ErrorClass(
|
|
73
|
+
`${where}: ${label} must be a ${positive ? 'positive ' : ''}duration ('5s', '4m30s', or milliseconds), `
|
|
74
|
+
+ `got ${JSON.stringify(value)}`,
|
|
75
|
+
);
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
return ms;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/**
|
|
82
|
+
* @param {unknown} value
|
|
83
|
+
* @param {Scope} scope
|
|
84
|
+
*/
|
|
85
|
+
export function checkBackoff(value, scope) {
|
|
86
|
+
const { where, ErrorClass } = scope;
|
|
87
|
+
const backoff = checkObject(value, ['type', 'delay', 'maxDelay', 'jitter'], '"backoff" options', scope);
|
|
88
|
+
|
|
89
|
+
if (backoff.type !== 'exponential' && backoff.type !== 'fixed') {
|
|
90
|
+
throw new ErrorClass(`${where}: "backoff.type" must be 'exponential' or 'fixed'`);
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
const delayMs = checkDuration(backoff.delay, '"backoff.delay"', scope);
|
|
94
|
+
|
|
95
|
+
if (backoff.maxDelay !== undefined) {
|
|
96
|
+
if (backoff.type === 'fixed') {
|
|
97
|
+
throw new ErrorClass(`${where}: "backoff.maxDelay" is only for the exponential backoff`);
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
if (checkDuration(backoff.maxDelay, '"backoff.maxDelay"', scope) < delayMs) {
|
|
101
|
+
throw new ErrorClass(`${where}: "backoff.maxDelay" must be at least "backoff.delay"`);
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
const { jitter } = backoff;
|
|
106
|
+
|
|
107
|
+
if (jitter !== undefined && (typeof jitter !== 'number' || !(jitter >= 0 && jitter <= 1))) {
|
|
108
|
+
throw new ErrorClass(`${where}: "backoff.jitter" must be a number from 0 to 1`);
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
/**
|
|
113
|
+
* @param {unknown} value
|
|
114
|
+
* @param {Scope} scope
|
|
115
|
+
*/
|
|
116
|
+
export function checkRetention(value, scope) {
|
|
117
|
+
const retention = checkObject(value, ['completed', 'failed'], '"retention" options', scope);
|
|
118
|
+
|
|
119
|
+
for (const key of ['completed', 'failed']) {
|
|
120
|
+
if (retention[key] !== undefined) {
|
|
121
|
+
checkDuration(retention[key], `"retention.${key}"`, scope);
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
/**
|
|
127
|
+
* Checks the options that a definition, the instance defaults and an enqueue share.
|
|
128
|
+
* @param {Record<string, unknown>} options
|
|
129
|
+
* @param {Scope} scope
|
|
130
|
+
*/
|
|
131
|
+
export function checkJobFields(options, scope) {
|
|
132
|
+
if (options.maxAttempts !== undefined) {
|
|
133
|
+
checkInteger(options.maxAttempts, '"maxAttempts"', scope);
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
if (options.maxStalls !== undefined) {
|
|
137
|
+
checkInteger(options.maxStalls, '"maxStalls"', scope, 0);
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
if (options.backoff !== undefined) {
|
|
141
|
+
checkBackoff(options.backoff, scope);
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
if (options.retention !== undefined) {
|
|
145
|
+
checkRetention(options.retention, scope);
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
if (options.timeout !== undefined) {
|
|
149
|
+
checkDuration(options.timeout, '"timeout"', scope, true);
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
/**
|
|
154
|
+
* @param {unknown} value A name, or a definition.
|
|
155
|
+
* @param {string} jobName The job that names it.
|
|
156
|
+
* @param {Scope} scope
|
|
157
|
+
* @returns {string} The name of the failure job.
|
|
158
|
+
*/
|
|
159
|
+
export function checkFailureJob(value, jobName, { where, ErrorClass }) {
|
|
160
|
+
const name = isPlainObject(value) ? value.name : value;
|
|
161
|
+
|
|
162
|
+
if (typeof name !== 'string' || !JOB_NAME_REGEXP.test(name)) {
|
|
163
|
+
throw new ErrorClass(`${where}: "failureJob" must be a job name or a job definition`);
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
if (name === jobName) {
|
|
167
|
+
throw new ErrorClass(`${where}: a job cannot be its own failure job`);
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
return name;
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
/**
|
|
174
|
+
* @param {unknown} tablePrefix
|
|
175
|
+
* @param {Scope} scope
|
|
176
|
+
*/
|
|
177
|
+
export function checkTablePrefix(tablePrefix, { where, ErrorClass }) {
|
|
178
|
+
if (typeof tablePrefix !== 'string' || !TABLE_PREFIX_REGEXP.test(tablePrefix)) {
|
|
179
|
+
throw new ErrorClass(`${where}: "tablePrefix" must match ${TABLE_PREFIX_REGEXP}`);
|
|
180
|
+
}
|
|
181
|
+
}
|