@rvoh/psychic-workers 2.6.0 → 2.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/src/background/BaseBackgroundedModel.js +49 -25
- package/dist/cjs/src/background/BaseBackgroundedService.js +24 -12
- package/dist/cjs/src/background/BaseScheduledService.js +1 -1
- package/dist/cjs/src/background/index.js +383 -98
- package/dist/cjs/src/error/background/DebouncedJobOutpacesRateLimit.js +64 -0
- package/dist/cjs/src/error/background/DebouncedJobRequiresMinimumDelay.js +52 -0
- package/dist/cjs/src/error/background/NamedWorkstreamRateLimitMissingMaxOrDuration.js +31 -0
- package/dist/cjs/src/error/background/RateLimitedPsychicJob.js +82 -0
- package/dist/cjs/src/error/background/RateLimitedPsychicJobThrownFromWorkerWithoutLimiter.js +51 -0
- package/dist/cjs/src/package-exports/errors.js +7 -0
- package/dist/cjs/src/test-utils/WorkerTestUtils.js +78 -13
- package/dist/esm/src/background/BaseBackgroundedModel.js +49 -25
- package/dist/esm/src/background/BaseBackgroundedService.js +24 -12
- package/dist/esm/src/background/BaseScheduledService.js +1 -1
- package/dist/esm/src/background/index.js +383 -98
- package/dist/esm/src/error/background/DebouncedJobOutpacesRateLimit.js +64 -0
- package/dist/esm/src/error/background/DebouncedJobRequiresMinimumDelay.js +52 -0
- package/dist/esm/src/error/background/NamedWorkstreamRateLimitMissingMaxOrDuration.js +31 -0
- package/dist/esm/src/error/background/RateLimitedPsychicJob.js +82 -0
- package/dist/esm/src/error/background/RateLimitedPsychicJobThrownFromWorkerWithoutLimiter.js +51 -0
- package/dist/esm/src/package-exports/errors.js +7 -0
- package/dist/esm/src/test-utils/WorkerTestUtils.js +78 -13
- package/dist/types/src/background/BaseBackgroundedModel.d.ts +49 -25
- package/dist/types/src/background/BaseBackgroundedService.d.ts +24 -12
- package/dist/types/src/background/BaseScheduledService.d.ts +1 -1
- package/dist/types/src/background/index.d.ts +109 -5
- package/dist/types/src/error/background/DebouncedJobOutpacesRateLimit.d.ts +31 -0
- package/dist/types/src/error/background/DebouncedJobRequiresMinimumDelay.d.ts +23 -0
- package/dist/types/src/error/background/NamedWorkstreamRateLimitMissingMaxOrDuration.d.ts +16 -0
- package/dist/types/src/error/background/RateLimitedPsychicJob.d.ts +73 -0
- package/dist/types/src/error/background/RateLimitedPsychicJobThrownFromWorkerWithoutLimiter.d.ts +16 -0
- package/dist/types/src/package-exports/errors.d.ts +1 -0
- package/dist/types/src/psychic-app-workers/index.d.ts +102 -53
- package/dist/types/src/test-utils/WorkerTestUtils.d.ts +32 -1
- package/dist/types/src/types/background.d.ts +88 -6
- package/package.json +1 -1
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Not exported from the package: a delayed background job carried a `jobId`
|
|
3
|
+
* behind a delay short enough that it can enqueue work faster than its own
|
|
4
|
+
* queue's rate limit allows that work to start. Thrown from `_addToQueue`,
|
|
5
|
+
* above the test-mode short circuit, so a consumer's test environment raises it
|
|
6
|
+
* exactly as production does.
|
|
7
|
+
*/
|
|
8
|
+
export default class DebouncedJobOutpacesRateLimit extends Error {
|
|
9
|
+
jobId;
|
|
10
|
+
delayMs;
|
|
11
|
+
keyLifetimeMs;
|
|
12
|
+
limiter;
|
|
13
|
+
queue;
|
|
14
|
+
constructor(jobId, delayMs, keyLifetimeMs, limiter, queue) {
|
|
15
|
+
super();
|
|
16
|
+
this.jobId = jobId;
|
|
17
|
+
this.delayMs = delayMs;
|
|
18
|
+
this.keyLifetimeMs = keyLifetimeMs;
|
|
19
|
+
this.limiter = limiter;
|
|
20
|
+
this.queue = queue;
|
|
21
|
+
}
|
|
22
|
+
get message() {
|
|
23
|
+
return `
|
|
24
|
+
A delayed background job was given the \`jobId\` ${JSON.stringify(this.jobId)} behind a delay of ${this.seconds(this.delayMs)}, but ${this.where}
|
|
25
|
+
is rate limited to ${this.limiter.max} job${this.limiter.max === 1 ? '' : 's'} every ${this.limiter.duration}ms — one job every ${this.seconds(this.msPerJob)}.
|
|
26
|
+
|
|
27
|
+
\`jobId\` is a deduplication key, and the key that does the collapsing lives ${this.seconds(this.keyLifetimeMs)} (the delay minus a
|
|
28
|
+
one-second margin). Calls closer together than that are collapsed; calls further apart than that stop collapsing
|
|
29
|
+
and produce a job each. This \`jobId\` can therefore enqueue a job every ${this.seconds(this.keyLifetimeMs)}, which is faster than
|
|
30
|
+
this ${this.kind} can start them. Under a sustained stream of calls the backlog grows without bound — the rate
|
|
31
|
+
limit keeps the downstream service safe while it happens, so nothing fails and the jobs simply pile up in Redis.
|
|
32
|
+
|
|
33
|
+
Either give the delay at least ${this.seconds(this.minimumDelayMs)}, so that calls which stop collapsing still cannot outrun the rate
|
|
34
|
+
limit, or move the job to a ${this.kind} of its own — one with no rate limit, or one whose limit fits this job's
|
|
35
|
+
endpoint. A burst that stops arriving is collapsed into one run at any delay, but which pattern a caller has is
|
|
36
|
+
not knowable here. Nothing was enqueued.
|
|
37
|
+
`;
|
|
38
|
+
}
|
|
39
|
+
get msPerJob() {
|
|
40
|
+
return this.limiter.duration / this.limiter.max;
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* the shortest delay that would pass: enough that the key outlives the
|
|
44
|
+
* limiter's spacing. Rounded up to a whole second, since the delay is given
|
|
45
|
+
* in whole-ish duration fields and a value that still failed by a fraction
|
|
46
|
+
* of a second would be a hostile suggestion.
|
|
47
|
+
*/
|
|
48
|
+
get minimumDelayMs() {
|
|
49
|
+
return Math.ceil((this.msPerJob + (this.delayMs - this.keyLifetimeMs)) / 1000) * 1000;
|
|
50
|
+
}
|
|
51
|
+
seconds(ms) {
|
|
52
|
+
const secs = ms / 1000;
|
|
53
|
+
const rendered = Number.isInteger(secs) ? String(secs) : secs.toFixed(2).replace(/\.?0+$/, '');
|
|
54
|
+
return `${rendered} second${rendered === '1' ? '' : 's'}`;
|
|
55
|
+
}
|
|
56
|
+
get kind() {
|
|
57
|
+
return this.queue.mode === 'simple' ? 'workstream' : 'queue';
|
|
58
|
+
}
|
|
59
|
+
get where() {
|
|
60
|
+
if (this.queue.isDefaultQueue)
|
|
61
|
+
return `the default ${this.kind}`;
|
|
62
|
+
return `the \`${this.queue.configuredName}\` ${this.queue.transitional ? 'transitional ' : ''}${this.kind}`;
|
|
63
|
+
}
|
|
64
|
+
}
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Not exported from the package, and deliberately so: every branch of the guard
|
|
3
|
+
* that throws it is a call-site mistake — a `jobId` behind a delay under the
|
|
4
|
+
* floor, an empty `jobId`, or a delay that is zero, negative, `Infinity`, `NaN`
|
|
5
|
+
* or past `Number.MAX_SAFE_INTEGER` — so an application can never usefully
|
|
6
|
+
* catch it. The fix is always an edit to the call.
|
|
7
|
+
*
|
|
8
|
+
* `durationToSeconds` catches none of these: it validates nothing, and its
|
|
9
|
+
* falsy-field sum silently drops a `NaN` field rather than propagating it
|
|
10
|
+
* (`{ hours: 1, seconds: NaN }` is `3600`), so `{ seconds: NaN }` reaches this
|
|
11
|
+
* check as `0`.
|
|
12
|
+
*
|
|
13
|
+
* The check runs before the test-mode short circuit in `_addToQueue`, so a
|
|
14
|
+
* consumer's default test environment raises this exactly as production does.
|
|
15
|
+
*/
|
|
16
|
+
export default class DebouncedJobRequiresMinimumDelay extends Error {
|
|
17
|
+
jobId;
|
|
18
|
+
delaySeconds;
|
|
19
|
+
minimumDelayMs;
|
|
20
|
+
constructor(jobId, delaySeconds, minimumDelayMs) {
|
|
21
|
+
super();
|
|
22
|
+
this.jobId = jobId;
|
|
23
|
+
this.delaySeconds = delaySeconds;
|
|
24
|
+
this.minimumDelayMs = minimumDelayMs;
|
|
25
|
+
}
|
|
26
|
+
get message() {
|
|
27
|
+
if (this.jobId === '')
|
|
28
|
+
return this.emptyJobIdMessage;
|
|
29
|
+
return `
|
|
30
|
+
A delayed background job was given the \`jobId\` ${JSON.stringify(this.jobId)}, but a delay too short to debounce with (got \`delaySeconds\`: ${String(this.delaySeconds)}).
|
|
31
|
+
\`jobId\` is a deduplication key: repeat calls arriving within the delay window are collapsed into one run, and each
|
|
32
|
+
call restarts the window, so a burst collapses however long it runs. The delay must be at least ${this.minimumDelayMs / 1000} seconds: the key
|
|
33
|
+
that does the collapsing lives one second less than the delay, and below this floor what is left is too short to
|
|
34
|
+
survive an ordinary pause between two calls. The delay must also be a finite number of milliseconds within
|
|
35
|
+
\`Number.MAX_SAFE_INTEGER\`, since BullMQ passes it to Redis unvalidated.
|
|
36
|
+
|
|
37
|
+
Either give the delay at least ${this.minimumDelayMs / 1000} seconds, e.g. \`{ seconds: ${this.minimumDelayMs / 1000}, jobId: ${JSON.stringify(this.jobId)} }\`, or drop the
|
|
38
|
+
\`jobId\` if you did not want debouncing. Nothing was enqueued.
|
|
39
|
+
`;
|
|
40
|
+
}
|
|
41
|
+
get emptyJobIdMessage() {
|
|
42
|
+
return `
|
|
43
|
+
A delayed background job was given an empty \`jobId\` (got \`delaySeconds\`: ${String(this.delaySeconds)}).
|
|
44
|
+
\`jobId\` is a deduplication key: repeat calls arriving within the delay window are collapsed into one run. An empty
|
|
45
|
+
string is not a usable key — BullMQ refuses it — and it is almost always a key that was built from a value that
|
|
46
|
+
turned out to be missing, e.g. an id read from a nullable column.
|
|
47
|
+
|
|
48
|
+
Either give the \`jobId\` a non-empty value, or drop it if you did not want debouncing (a delay with no \`jobId\`
|
|
49
|
+
is enqueued normally, at any length). Nothing was enqueued.
|
|
50
|
+
`;
|
|
51
|
+
}
|
|
52
|
+
}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Not exported from the package: a named workstream's `rateLimit` reached
|
|
3
|
+
* `connect()` without a positive integer `max` or `duration`. Open-source
|
|
4
|
+
* BullMQ forwards the worker `limiter` it becomes unvalidated: a missing or
|
|
5
|
+
* non-numeric field fails every job fetch on that workstream with a Lua error
|
|
6
|
+
* that never names `rateLimit`, a fractional `duration` floors to 0ms and rate
|
|
7
|
+
* limits nothing, and one past Redis's integer range fails every fetch.
|
|
8
|
+
*/
|
|
9
|
+
export default class NamedWorkstreamRateLimitMissingMaxOrDuration extends Error {
|
|
10
|
+
workstreamName;
|
|
11
|
+
transitional;
|
|
12
|
+
field;
|
|
13
|
+
received;
|
|
14
|
+
constructor(workstreamName, transitional, field, received) {
|
|
15
|
+
super();
|
|
16
|
+
this.workstreamName = workstreamName;
|
|
17
|
+
this.transitional = transitional;
|
|
18
|
+
this.field = field;
|
|
19
|
+
this.received = received;
|
|
20
|
+
}
|
|
21
|
+
get message() {
|
|
22
|
+
const entry = this.transitional ? 'transitionalWorkstreams.namedWorkstreams' : 'namedWorkstreams';
|
|
23
|
+
const received = typeof this.received === 'string' ? JSON.stringify(this.received) : String(this.received);
|
|
24
|
+
return `
|
|
25
|
+
\`rateLimit\` on the \`${this.workstreamName}\` entry in \`${entry}\` is missing a usable \`${this.field}\` (got ${received}).
|
|
26
|
+
Both \`max\` and \`duration\` must be positive integers, e.g. \`rateLimit: { max: 1, duration: 1000 }\`
|
|
27
|
+
(at most \`max\` jobs start in any \`duration\` milliseconds). Open-source BullMQ does not validate the worker
|
|
28
|
+
\`limiter\` this becomes and would fail every job fetch on that workstream or rate limit nothing, so nothing was connected.
|
|
29
|
+
`;
|
|
30
|
+
}
|
|
31
|
+
}
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Throw from inside a backgrounded (or scheduled) method to signal that the
|
|
3
|
+
* outside service it talks to rate-limited the request (an HTTP 429, say) and
|
|
4
|
+
* asked it to stop sending for a while. `pauseQueueForSeconds` is how long that
|
|
5
|
+
* while is, in whole seconds. Importable from `@rvoh/psychic-workers/errors`.
|
|
6
|
+
*
|
|
7
|
+
* ```ts
|
|
8
|
+
* // conf/workers.ts — one named workstream per rate-limited service
|
|
9
|
+
* namedWorkstreams: [{ name: 'slack', rateLimit: { max: 1, duration: 1000 } }]
|
|
10
|
+
*
|
|
11
|
+
* // app/services/SlackNotifier.ts
|
|
12
|
+
* import { RateLimitedPsychicJob } from '@rvoh/psychic-workers/errors'
|
|
13
|
+
*
|
|
14
|
+
* export default class SlackNotifier extends ApplicationBackgroundedService {
|
|
15
|
+
* public static override get backgroundJobConfig() {
|
|
16
|
+
* return { workstream: 'slack' }
|
|
17
|
+
* }
|
|
18
|
+
*
|
|
19
|
+
* public static async post(channel: string, text: string) {
|
|
20
|
+
* const response = await fetch('https://slack.com/api/chat.postMessage', { ... })
|
|
21
|
+
*
|
|
22
|
+
* if (response.status === 429) {
|
|
23
|
+
* // Slack's Retry-After header is in seconds, and so is this field
|
|
24
|
+
* const seconds = Number(response.headers.get('retry-after') ?? 1)
|
|
25
|
+
* throw new RateLimitedPsychicJob({ pauseQueueForSeconds: seconds })
|
|
26
|
+
* }
|
|
27
|
+
* }
|
|
28
|
+
* }
|
|
29
|
+
* ```
|
|
30
|
+
*
|
|
31
|
+
* The worker running the job must carry a BullMQ `limiter` — a named
|
|
32
|
+
* workstream with `rateLimit`, a native `namedQueueWorkers` entry with
|
|
33
|
+
* `limiter`, `nativeBullMQ.defaultWorkerOptions.limiter`, or a global
|
|
34
|
+
* `defaultBullMQWorkerOptions.limiter`. Given one, psychic-workers translates
|
|
35
|
+
* the signal into BullMQ's own rate-limit mechanism: the queue is paused for
|
|
36
|
+
* that many seconds and the job goes back onto the queue with `attemptsMade`
|
|
37
|
+
* unchanged and no backoff, so the burst is not multiplied by the retry
|
|
38
|
+
* schedule. Note that this package writes a BullMQ priority on every job it
|
|
39
|
+
* enqueues, so a re-queued job reports `prioritized` rather than `waiting` —
|
|
40
|
+
* assertions on `getWaitingCount()` do not hold.
|
|
41
|
+
*
|
|
42
|
+
* The pause replaces the queue's current limiter window and counter — the
|
|
43
|
+
* configured `rateLimit` window and any earlier pause alike — so once
|
|
44
|
+
* `pauseQueueForSeconds` elapses up to `max` jobs may start at once, and a
|
|
45
|
+
* `pauseQueueForSeconds` shorter than the window's remaining time shortens the
|
|
46
|
+
* configured limit. Concurrent throws overwrite each other: last writer wins,
|
|
47
|
+
* not longest.
|
|
48
|
+
*
|
|
49
|
+
* `attemptsMade` never grows, but each re-fetch counts in BullMQ's
|
|
50
|
+
* `attemptsStarted`, and the worker option `maxStartedAttempts`
|
|
51
|
+
* (`defaultBullMQWorkerOptions: { maxStartedAttempts: 10 }`, say) is the only
|
|
52
|
+
* bound on that cycle: without it it is unbounded. Set it on any workstream
|
|
53
|
+
* whose jobs throw this signal.
|
|
54
|
+
*
|
|
55
|
+
* Thrown from a job whose worker carries no `limiter`, the signal is a
|
|
56
|
+
* misconfiguration: the job fails with an ordinary error whose message names
|
|
57
|
+
* the fix. The same check runs under test invocation and `WorkerTestUtils`, so
|
|
58
|
+
* a misplaced job goes red in specs.
|
|
59
|
+
*
|
|
60
|
+
* `pauseQueueForSeconds` must be a positive, finite number of safe magnitude;
|
|
61
|
+
* the constructor throws a `RangeError` otherwise, at the throw site, so a
|
|
62
|
+
* nonsense value fails there rather than at the pause. A fractional number of
|
|
63
|
+
* seconds is legal and is rounded **up** — the field is a lower bound on the
|
|
64
|
+
* pause, so overshooting it by under a second cannot break the promise, while
|
|
65
|
+
* rounding down could.
|
|
66
|
+
*/
|
|
67
|
+
export default class RateLimitedPsychicJob extends Error {
|
|
68
|
+
pauseQueueForSeconds;
|
|
69
|
+
constructor({ pauseQueueForSeconds }) {
|
|
70
|
+
super();
|
|
71
|
+
// `Number.isFinite` does not coerce, so a non-numeric value fails it too
|
|
72
|
+
if (!(Number.isFinite(pauseQueueForSeconds) &&
|
|
73
|
+
pauseQueueForSeconds > 0 &&
|
|
74
|
+
pauseQueueForSeconds <= Number.MAX_SAFE_INTEGER)) {
|
|
75
|
+
throw new RangeError(`RateLimitedPsychicJob requires pauseQueueForSeconds to be a positive, finite number of safe magnitude (the number of seconds to pause the queue for); received ${String(pauseQueueForSeconds)}`);
|
|
76
|
+
}
|
|
77
|
+
this.pauseQueueForSeconds = pauseQueueForSeconds;
|
|
78
|
+
}
|
|
79
|
+
get message() {
|
|
80
|
+
return `job was rate limited; pause the queue for ${this.pauseQueueForSeconds} seconds`;
|
|
81
|
+
}
|
|
82
|
+
}
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Not exported from the package: a `RateLimitedPsychicJob` was thrown from a
|
|
3
|
+
* job whose worker carries no BullMQ `limiter`, so the queue cannot be paused.
|
|
4
|
+
* The job runner fails the job with this in the signal's place, so the failure
|
|
5
|
+
* reaches the application's failed-job monitoring.
|
|
6
|
+
*/
|
|
7
|
+
export default class RateLimitedPsychicJobThrownFromWorkerWithoutLimiter extends Error {
|
|
8
|
+
signal;
|
|
9
|
+
queue;
|
|
10
|
+
constructor(signal, queue) {
|
|
11
|
+
super(undefined, { cause: signal });
|
|
12
|
+
this.signal = signal;
|
|
13
|
+
this.queue = queue;
|
|
14
|
+
}
|
|
15
|
+
get message() {
|
|
16
|
+
return `
|
|
17
|
+
RateLimitedPsychicJob (pause the queue for ${this.signal.pauseQueueForSeconds} seconds) was thrown from a job on ${this.where},
|
|
18
|
+
but the workers for that queue carry no BullMQ \`limiter\`, so the queue cannot be paused.
|
|
19
|
+
The job has been failed instead.
|
|
20
|
+
|
|
21
|
+
To fix this, ${this.fix}
|
|
22
|
+
`;
|
|
23
|
+
}
|
|
24
|
+
get where() {
|
|
25
|
+
if (this.queue.isDefaultQueue)
|
|
26
|
+
return this.queue.mode === 'simple' ? 'the default workstream' : 'the default queue';
|
|
27
|
+
const kind = this.queue.mode === 'simple' ? 'workstream' : 'queue';
|
|
28
|
+
return `the \`${this.queue.configuredName}\` ${this.queue.transitional ? 'transitional ' : ''}${kind}`;
|
|
29
|
+
}
|
|
30
|
+
get fix() {
|
|
31
|
+
const name = this.queue.configuredName;
|
|
32
|
+
if (this.queue.mode === 'simple') {
|
|
33
|
+
if (this.queue.isDefaultQueue)
|
|
34
|
+
return `move the job to a named workstream that sets \`rateLimit: { max, duration }\` (a rate limit
|
|
35
|
+
targets one external limit, so the default workstream never carries one, and a service that meters
|
|
36
|
+
its endpoints separately wants a workstream per limit). For example, with
|
|
37
|
+
\`namedWorkstreams: [{ name: 'slack', rateLimit: { max: 1, duration: 1000 } }]\` in your workers config,
|
|
38
|
+
give the backgrounded class \`backgroundJobConfig = { workstream: 'slack' }\`.`;
|
|
39
|
+
const entry = this.queue.transitional
|
|
40
|
+
? `the \`${name}\` entry in \`transitionalWorkstreams.namedWorkstreams\``
|
|
41
|
+
: `the \`${name}\` entry in \`namedWorkstreams\``;
|
|
42
|
+
return `set \`rateLimit: { max, duration }\` on ${entry}, or move the job to a named
|
|
43
|
+
workstream that sets \`rateLimit\`.`;
|
|
44
|
+
}
|
|
45
|
+
if (this.queue.isDefaultQueue)
|
|
46
|
+
return `move the job to a named queue whose \`nativeBullMQ.namedQueueWorkers\` entry sets
|
|
47
|
+
\`limiter: { max, duration }\`.`;
|
|
48
|
+
return `set \`limiter: { max, duration }\` on \`nativeBullMQ.namedQueueWorkers['${name}']\`, or move the job
|
|
49
|
+
to a named queue whose \`namedQueueWorkers\` entry sets \`limiter\`.`;
|
|
50
|
+
}
|
|
51
|
+
}
|
|
@@ -1,3 +1,10 @@
|
|
|
1
1
|
export { default as NoClassForSpecifiedGlobalName } from '../error/background/NoClassForSpecifiedGlobalName.js';
|
|
2
|
+
// exported for backward compatibility only. Thrown by the framework when a job's
|
|
3
|
+
// config names a queue that does not exist — a configuration mistake an
|
|
4
|
+
// application cannot recover from, and not intended to be caught.
|
|
2
5
|
export { default as NoQueueForSpecifiedQueueName } from '../error/background/NoQueueForSpecifiedQueueName.js';
|
|
6
|
+
// exported for backward compatibility only. Thrown by the framework when a job's
|
|
7
|
+
// config names a workstream that does not exist — a configuration mistake an
|
|
8
|
+
// application cannot recover from, and not intended to be caught.
|
|
3
9
|
export { default as NoQueueForSpecifiedWorkstream } from '../error/background/NoQueueForSpecifiedWorkstream.js';
|
|
10
|
+
export { default as RateLimitedPsychicJob } from '../error/background/RateLimitedPsychicJob.js';
|
|
@@ -1,5 +1,19 @@
|
|
|
1
|
+
import nameToRedisQueueName from '../background/helpers/nameToRedisQueueName.js';
|
|
1
2
|
import parallelTestSafeQueueName from '../background/helpers/parallelTestSafeQueueName.js';
|
|
2
3
|
import background, { Background } from '../background/index.js';
|
|
4
|
+
import RateLimitedPsychicJob from '../error/background/RateLimitedPsychicJob.js';
|
|
5
|
+
/**
|
|
6
|
+
* @internal
|
|
7
|
+
*
|
|
8
|
+
* resolves the queue's expected Redis name through `nameToRedisQueueName`,
|
|
9
|
+
* rather than hand-reconstructing a cluster hash tag or parallel-test suffix.
|
|
10
|
+
* `queue.opts.connection` is the same `Redis | Cluster` instance the queue was
|
|
11
|
+
* built with, which is why the cast is safe for every queue this package builds.
|
|
12
|
+
*/
|
|
13
|
+
export function queueNamesMatch(queue, compareQueueName) {
|
|
14
|
+
return (queue.name ===
|
|
15
|
+
nameToRedisQueueName(compareQueueName, queue.opts.connection));
|
|
16
|
+
}
|
|
3
17
|
const LOCK_TOKEN = 'psychic-test-worker';
|
|
4
18
|
export default class WorkerTestUtils {
|
|
5
19
|
/*
|
|
@@ -24,6 +38,12 @@ export default class WorkerTestUtils {
|
|
|
24
38
|
* NOTE: this is only useful if you are running with `testInvocation=manual`.
|
|
25
39
|
* make sure to set this in your workers config, or else in your test,
|
|
26
40
|
* so that jobs are successfully commited to the queue and can be worked off.
|
|
41
|
+
*
|
|
42
|
+
* A job that throws `RateLimitedPsychicJob` stops the loop: on a queue whose
|
|
43
|
+
* workers carry a BullMQ `limiter` the job is moved back to the queue (no
|
|
44
|
+
* attempt counted) and the call rejects with that error; on a queue whose
|
|
45
|
+
* workers carry none, the job fails with the misconfiguration error that
|
|
46
|
+
* production would fail it with, and the call rejects with that instead.
|
|
27
47
|
*/
|
|
28
48
|
static async work(opts = {}) {
|
|
29
49
|
background.connect();
|
|
@@ -32,7 +52,7 @@ export default class WorkerTestUtils {
|
|
|
32
52
|
do {
|
|
33
53
|
workWasDone = false;
|
|
34
54
|
for (const queue of queues) {
|
|
35
|
-
if (opts.queue && !
|
|
55
|
+
if (opts.queue && !queueNamesMatch(queue, opts.queue))
|
|
36
56
|
continue;
|
|
37
57
|
workWasDone ||= await this.workOne(queue);
|
|
38
58
|
}
|
|
@@ -40,8 +60,9 @@ export default class WorkerTestUtils {
|
|
|
40
60
|
}
|
|
41
61
|
static async workScheduled(opts = {}) {
|
|
42
62
|
background.connect();
|
|
43
|
-
const
|
|
44
|
-
|
|
63
|
+
const compareQueueName = opts.queue;
|
|
64
|
+
const queues = compareQueueName
|
|
65
|
+
? background.queues.filter(queue => queueNamesMatch(queue, compareQueueName))
|
|
45
66
|
: background.queues;
|
|
46
67
|
if (opts.queue && !queues.length)
|
|
47
68
|
throw new Error(`Expected to find queue with name: ${opts.queue}, but none were found by that name. The queue names available are: ${background.queues.map(queue => queue.name).join(', ')}`);
|
|
@@ -51,19 +72,33 @@ export default class WorkerTestUtils {
|
|
|
51
72
|
const data = job.data;
|
|
52
73
|
if (opts.for) {
|
|
53
74
|
if (data.globalName === opts.for.globalName) {
|
|
54
|
-
await
|
|
75
|
+
await this.doScheduledWork(job, queue);
|
|
55
76
|
}
|
|
56
77
|
}
|
|
57
78
|
else {
|
|
58
|
-
await
|
|
79
|
+
await this.doScheduledWork(job, queue);
|
|
59
80
|
}
|
|
60
81
|
}
|
|
61
82
|
}
|
|
62
83
|
}
|
|
84
|
+
/**
|
|
85
|
+
* runs a delayed job in place. A `RateLimitedPsychicJob` thrown from a job on
|
|
86
|
+
* a queue whose workers carry no BullMQ `limiter` surfaces as the same
|
|
87
|
+
* misconfiguration error production would fail the job with; on a queue whose
|
|
88
|
+
* workers carry one, it propagates untranslated so the spec can assert on it.
|
|
89
|
+
*/
|
|
90
|
+
static async doScheduledWork(job, queue) {
|
|
91
|
+
try {
|
|
92
|
+
await background.doWork(job);
|
|
93
|
+
}
|
|
94
|
+
catch (err) {
|
|
95
|
+
throw background.misconfiguredRateLimitSignal(err, queue) ?? err;
|
|
96
|
+
}
|
|
97
|
+
}
|
|
63
98
|
/*
|
|
64
99
|
* iterates through each registered queue, and cleans out all
|
|
65
100
|
* jobs, including waiting, paused, prioritized, delayed, completed,
|
|
66
|
-
* failed, and scheduled jobs. This is especially useful before a test
|
|
101
|
+
* failed, active, and scheduled jobs. This is especially useful before a test
|
|
67
102
|
* where you plan to exercise background jobs manually.
|
|
68
103
|
*
|
|
69
104
|
* If your entire app is continuously exercising background jobs
|
|
@@ -84,9 +119,13 @@ export default class WorkerTestUtils {
|
|
|
84
119
|
// belong to a job scheduler are intentionally left alone by BullMQ here, and
|
|
85
120
|
// are cleaned up below when their scheduler is removed.
|
|
86
121
|
await queue.drain(true);
|
|
87
|
-
// clear out completed and
|
|
122
|
+
// clear out completed, failed, and abandoned active jobs. `drain` does not
|
|
123
|
+
// touch `active`, so a job fetched by hand and never moved on would outlive
|
|
124
|
+
// the process and be counted by the next run of the suite. BullMQ leaves a
|
|
125
|
+
// job whose lock is still held alone.
|
|
88
126
|
await queue.clean(0, 10000, 'completed');
|
|
89
127
|
await queue.clean(0, 10000, 'failed');
|
|
128
|
+
await queue.clean(0, 10000, 'active');
|
|
90
129
|
// clear out scheduled jobs
|
|
91
130
|
const schedulers = await queue.getJobSchedulers();
|
|
92
131
|
for (const scheduler of schedulers) {
|
|
@@ -105,19 +144,45 @@ export default class WorkerTestUtils {
|
|
|
105
144
|
const job = await worker.getNextJob(LOCK_TOKEN);
|
|
106
145
|
if (!job)
|
|
107
146
|
return false;
|
|
108
|
-
await this.processJob(job);
|
|
147
|
+
await this.processJob(job, queue);
|
|
109
148
|
return true;
|
|
110
149
|
}
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
150
|
+
/**
|
|
151
|
+
* runs one fetched job to completion or failure, mirroring what a real worker
|
|
152
|
+
* would do with a `RateLimitedPsychicJob`, then surfacing it so the spec goes
|
|
153
|
+
* red rather than looping:
|
|
154
|
+
*
|
|
155
|
+
* - on a queue whose workers carry no BullMQ `limiter`, the job is failed with
|
|
156
|
+
* the misconfiguration error production would fail it with (so, with the
|
|
157
|
+
* queue's `attempts` > 1, it lands in `delayed` for its retry, out of
|
|
158
|
+
* `active`, where `clean()` can reach it), and that error is rethrown
|
|
159
|
+
* - on a queue whose workers carry one, the job is moved back to the queue
|
|
160
|
+
* with no attempt counted (BullMQ's own rate-limit path), and the
|
|
161
|
+
* `RateLimitedPsychicJob` is rethrown
|
|
162
|
+
*
|
|
163
|
+
* Every other error only fails the job, as before.
|
|
164
|
+
*/
|
|
165
|
+
static async processJob(job, queue) {
|
|
116
166
|
try {
|
|
117
167
|
const res = await background.doWork(job);
|
|
118
168
|
await job.moveToCompleted(res, LOCK_TOKEN, false);
|
|
119
169
|
}
|
|
120
170
|
catch (err) {
|
|
171
|
+
const misconfigured = background.misconfiguredRateLimitSignal(err, queue);
|
|
172
|
+
if (misconfigured) {
|
|
173
|
+
await job.moveToFailed(misconfigured, LOCK_TOKEN, false);
|
|
174
|
+
throw misconfigured;
|
|
175
|
+
}
|
|
176
|
+
if (err instanceof RateLimitedPsychicJob) {
|
|
177
|
+
// exactly what a real worker does with this signal: BullMQ's own rate-limit
|
|
178
|
+
// path is `job.moveToWait(token)` (`Worker#moveLimitedBackToWait`). It takes
|
|
179
|
+
// the job off `active` and releases the lock `workOne` took, without counting
|
|
180
|
+
// an attempt. Its Lua reads the job's own priority and puts a prioritized job
|
|
181
|
+
// back in `prioritized` rather than `wait`, so the destination matches
|
|
182
|
+
// production too — this package writes a priority on every job it enqueues.
|
|
183
|
+
await job.moveToWait(LOCK_TOKEN);
|
|
184
|
+
throw err;
|
|
185
|
+
}
|
|
121
186
|
await job.moveToFailed(err, LOCK_TOKEN, false);
|
|
122
187
|
}
|
|
123
188
|
}
|
|
@@ -12,7 +12,7 @@ export default class BaseBackgroundedModel extends Dream {
|
|
|
12
12
|
* @returns {object} config - the background job config
|
|
13
13
|
* @returns {string} config.priority - 'default' | 'urgent' | 'not_urgent' | 'last'
|
|
14
14
|
* @returns {string} config.workstream - a workstream name. This would be the name of a workstream, as defined in conf/workers.ts
|
|
15
|
-
* @returns {string} config.
|
|
15
|
+
* @returns {string} config.queue - the name of the BullMQ queue you wish to connect to. This can only be provided if workstream is not provided.
|
|
16
16
|
* @returns {string} config.groupId - the groupId of the BullMQ queue you wish to connect to. This can only be provided if workstream is not provided.
|
|
17
17
|
*/
|
|
18
18
|
static get backgroundJobConfig(): BackgroundJobConfig<BaseBackgroundedModel>;
|
|
@@ -45,16 +45,22 @@ export default class BaseBackgroundedModel extends Dream {
|
|
|
45
45
|
static background<T, MethodName extends PsychicBackgroundedModelStaticMethods<T & typeof BaseBackgroundedModel>, MethodFunc extends T[MethodName & keyof T], MethodArgs extends BackgroundableMethodArgs<MethodFunc>>(this: T, methodName: MethodName, ...args: MethodArgs): Promise<void>;
|
|
46
46
|
/**
|
|
47
47
|
* runs the specified method in a background queue, driven by BullMQ,
|
|
48
|
-
* sending in the provided args,
|
|
49
|
-
*
|
|
50
|
-
* it is entered into the queue.
|
|
48
|
+
* sending in the provided args, along with a delay object — one or more of
|
|
49
|
+
* `{ seconds, minutes, hours, days }` — which holds the job off for that much
|
|
50
|
+
* time after it is entered into the queue.
|
|
51
|
+
*
|
|
52
|
+
* The delay object also accepts an optional `jobId`, which turns the delay
|
|
53
|
+
* into a **debounce**: repeated calls carrying the same `jobId` collapse into
|
|
54
|
+
* a single execution, which runs once the delay has elapsed without another
|
|
55
|
+
* call arriving. `jobId` is a deduplication key rather than a BullMQ job id.
|
|
56
|
+
*
|
|
57
|
+
* See {@link DelayedJobOpts.jobId}.
|
|
51
58
|
*
|
|
52
59
|
* ```ts
|
|
53
|
-
* await User.backgroundWithDelay('myMethod', 'abc', 123)
|
|
60
|
+
* await User.backgroundWithDelay({ minutes: 5 }, 'myMethod', 'abc', 123)
|
|
54
61
|
* ```
|
|
55
|
-
* though calling
|
|
56
|
-
* is an indication that
|
|
57
|
-
* completed.
|
|
62
|
+
* though calling backgroundWithDelay must be awaited, the resolution of the promise
|
|
63
|
+
* is an indication that a run is pending, not that it has completed.
|
|
58
64
|
*
|
|
59
65
|
* NOTE: in test environments, psychic will immediately invoke the underlying
|
|
60
66
|
* method, preventing you from needing to explicitly wait for queues to flush
|
|
@@ -62,7 +68,7 @@ export default class BaseBackgroundedModel extends Dream {
|
|
|
62
68
|
*
|
|
63
69
|
* @deprecated use `backgroundWith({ delay }, methodName, ...args)` instead. This method will be removed in a future major version.
|
|
64
70
|
*
|
|
65
|
-
* @param
|
|
71
|
+
* @param delay - how long you want to hold off before allowing the job to run, given as at least one of `seconds`, `minutes`, `hours` or `days`, plus an optional `jobId` (a deduplication key) which debounces repeated calls into a single run
|
|
66
72
|
* @param methodName - the name of the static method you wish to run in the background
|
|
67
73
|
* @param args - a variadic list of arguments to be sent to your method
|
|
68
74
|
*/
|
|
@@ -73,19 +79,25 @@ export default class BaseBackgroundedModel extends Dream {
|
|
|
73
79
|
* be used to delay the job and/or override the priority provided by
|
|
74
80
|
* the `backgroundJobConfig` getter on this model.
|
|
75
81
|
*
|
|
82
|
+
* Adding a `jobId` to the delay turns it into a **debounce**: repeated calls
|
|
83
|
+
* carrying the same `jobId` collapse into a single execution, which runs once
|
|
84
|
+
* the delay has elapsed without another call arriving. `jobId` is a
|
|
85
|
+
* deduplication key rather than a BullMQ job id.
|
|
86
|
+
*
|
|
87
|
+
* See {@link DelayedJobOpts.jobId}.
|
|
88
|
+
*
|
|
76
89
|
* ```ts
|
|
77
90
|
* await User.backgroundWith({ delay: { seconds: 30, jobId: 'my-unique-job-id' }, priority: 'urgent' }, 'myMethod', 'abc', 123)
|
|
78
91
|
* ```
|
|
79
92
|
* though calling backgroundWith must be awaited, the resolution of the promise
|
|
80
|
-
* is an indication that
|
|
81
|
-
* completed.
|
|
93
|
+
* is an indication that a run is pending, not that it has completed.
|
|
82
94
|
*
|
|
83
95
|
* NOTE: in test environments, psychic will immediately invoke the underlying
|
|
84
96
|
* method, preventing you from needing to explicitly wait for queues to flush
|
|
85
97
|
* before making assertions.
|
|
86
98
|
*
|
|
87
99
|
* @param opts - options for backgrounding this job
|
|
88
|
-
* @param opts.delay - (optional)
|
|
100
|
+
* @param opts.delay - (optional) how long you want to hold off before allowing the job to run, given as at least one of `seconds`, `minutes`, `hours` or `days`, plus an optional `jobId` (a deduplication key) which debounces repeated calls into a single run
|
|
89
101
|
* @param opts.priority - (optional) a priority which, when provided, overrides the priority provided by `backgroundJobConfig`
|
|
90
102
|
* @param methodName - the name of the static method you wish to run in the background
|
|
91
103
|
* @param args - a variadic list of arguments to be sent to your method
|
|
@@ -114,23 +126,29 @@ export default class BaseBackgroundedModel extends Dream {
|
|
|
114
126
|
* method, preventing you from needing to explicitly wait for queues to flush
|
|
115
127
|
* before making assertions.
|
|
116
128
|
*
|
|
117
|
-
* @param methodName - the name of the
|
|
129
|
+
* @param methodName - the name of the instance method you wish to run in the background
|
|
118
130
|
* @param args - a variadic list of arguments to be sent to your method
|
|
119
131
|
*/
|
|
120
132
|
background<T, MethodName extends PsychicBackgroundedServiceInstanceMethods<T & BaseBackgroundedModel>, MethodFunc extends T[MethodName & keyof T], MethodArgs extends BackgroundableMethodArgs<MethodFunc>>(this: T, methodName: MethodName, ...args: MethodArgs): Promise<void>;
|
|
121
133
|
/**
|
|
122
134
|
* runs the specified method in a background queue, driven by BullMQ,
|
|
123
|
-
* sending in the provided args,
|
|
124
|
-
*
|
|
125
|
-
* it is entered into the queue.
|
|
135
|
+
* sending in the provided args, along with a delay object — one or more of
|
|
136
|
+
* `{ seconds, minutes, hours, days }` — which holds the job off for that much
|
|
137
|
+
* time after it is entered into the queue.
|
|
138
|
+
*
|
|
139
|
+
* The delay object also accepts an optional `jobId`, which turns the delay
|
|
140
|
+
* into a **debounce**: repeated calls carrying the same `jobId` collapse into
|
|
141
|
+
* a single execution, which runs once the delay has elapsed without another
|
|
142
|
+
* call arriving. `jobId` is a deduplication key rather than a BullMQ job id.
|
|
143
|
+
*
|
|
144
|
+
* See {@link DelayedJobOpts.jobId}.
|
|
126
145
|
*
|
|
127
146
|
* ```ts
|
|
128
147
|
* const user = await User.lastOrFail()
|
|
129
|
-
* await user.backgroundWithDelay('myMethod', 'abc', 123)
|
|
148
|
+
* await user.backgroundWithDelay({ minutes: 5 }, 'myMethod', 'abc', 123)
|
|
130
149
|
* ```
|
|
131
|
-
* though calling
|
|
132
|
-
* is an indication that
|
|
133
|
-
* completed.
|
|
150
|
+
* though calling backgroundWithDelay must be awaited, the resolution of the promise
|
|
151
|
+
* is an indication that a run is pending, not that it has completed.
|
|
134
152
|
*
|
|
135
153
|
* NOTE: in test environments, psychic will immediately invoke the underlying
|
|
136
154
|
* method, preventing you from needing to explicitly wait for queues to flush
|
|
@@ -138,8 +156,8 @@ export default class BaseBackgroundedModel extends Dream {
|
|
|
138
156
|
*
|
|
139
157
|
* @deprecated use `backgroundWith({ delay }, methodName, ...args)` instead. This method will be removed in a future major version.
|
|
140
158
|
*
|
|
141
|
-
* @param
|
|
142
|
-
* @param methodName - the name of the
|
|
159
|
+
* @param delay - how long you want to hold off before allowing the job to run, given as at least one of `seconds`, `minutes`, `hours` or `days`, plus an optional `jobId` (a deduplication key) which debounces repeated calls into a single run
|
|
160
|
+
* @param methodName - the name of the instance method you wish to run in the background
|
|
143
161
|
* @param args - a variadic list of arguments to be sent to your method
|
|
144
162
|
*/
|
|
145
163
|
backgroundWithDelay<T, MethodName extends PsychicBackgroundedServiceInstanceMethods<T & BaseBackgroundedModel>, MethodFunc extends T[MethodName & keyof T], MethodArgs extends BackgroundableMethodArgs<MethodFunc>>(this: T, delay: DelayedJobOpts, methodName: MethodName, ...args: MethodArgs): Promise<void>;
|
|
@@ -149,20 +167,26 @@ export default class BaseBackgroundedModel extends Dream {
|
|
|
149
167
|
* be used to delay the job and/or override the priority provided by
|
|
150
168
|
* the `backgroundJobConfig` getter on this model.
|
|
151
169
|
*
|
|
170
|
+
* Adding a `jobId` to the delay turns it into a **debounce**: repeated calls
|
|
171
|
+
* carrying the same `jobId` collapse into a single execution, which runs once
|
|
172
|
+
* the delay has elapsed without another call arriving. `jobId` is a
|
|
173
|
+
* deduplication key rather than a BullMQ job id.
|
|
174
|
+
*
|
|
175
|
+
* See {@link DelayedJobOpts.jobId}.
|
|
176
|
+
*
|
|
152
177
|
* ```ts
|
|
153
178
|
* const user = await User.lastOrFail()
|
|
154
179
|
* await user.backgroundWith({ delay: { seconds: 30, jobId: 'my-unique-job-id' }, priority: 'urgent' }, 'myMethod', 'abc', 123)
|
|
155
180
|
* ```
|
|
156
181
|
* though calling backgroundWith must be awaited, the resolution of the promise
|
|
157
|
-
* is an indication that
|
|
158
|
-
* completed.
|
|
182
|
+
* is an indication that a run is pending, not that it has completed.
|
|
159
183
|
*
|
|
160
184
|
* NOTE: in test environments, psychic will immediately invoke the underlying
|
|
161
185
|
* method, preventing you from needing to explicitly wait for queues to flush
|
|
162
186
|
* before making assertions.
|
|
163
187
|
*
|
|
164
188
|
* @param opts - options for backgrounding this job
|
|
165
|
-
* @param opts.delay - (optional)
|
|
189
|
+
* @param opts.delay - (optional) how long you want to hold off before allowing the job to run, given as at least one of `seconds`, `minutes`, `hours` or `days`, plus an optional `jobId` (a deduplication key) which debounces repeated calls into a single run
|
|
166
190
|
* @param opts.priority - (optional) a priority which, when provided, overrides the priority provided by `backgroundJobConfig`
|
|
167
191
|
* @param methodName - the name of the instance method you wish to run in the background
|
|
168
192
|
* @param args - a variadic list of arguments to be sent to your method
|