@rvoh/psychic-workers 2.4.1 → 2.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/src/background/BaseBackgroundedModel.js +112 -19
- package/dist/cjs/src/background/BaseBackgroundedService.js +73 -9
- package/dist/cjs/src/background/BaseScheduledService.js +59 -1
- package/dist/cjs/src/background/index.js +418 -105
- package/dist/cjs/src/error/background/DebouncedJobOutpacesRateLimit.js +64 -0
- package/dist/cjs/src/error/background/DebouncedJobRequiresMinimumDelay.js +52 -0
- package/dist/cjs/src/error/background/NamedWorkstreamRateLimitMissingMaxOrDuration.js +31 -0
- package/dist/cjs/src/error/background/RateLimitedPsychicJob.js +82 -0
- package/dist/cjs/src/error/background/RateLimitedPsychicJobThrownFromWorkerWithoutLimiter.js +51 -0
- package/dist/cjs/src/package-exports/errors.js +7 -0
- package/dist/cjs/src/test-utils/WorkerTestUtils.js +79 -14
- package/dist/esm/src/background/BaseBackgroundedModel.js +112 -19
- package/dist/esm/src/background/BaseBackgroundedService.js +73 -9
- package/dist/esm/src/background/BaseScheduledService.js +59 -1
- package/dist/esm/src/background/index.js +418 -105
- package/dist/esm/src/error/background/DebouncedJobOutpacesRateLimit.js +64 -0
- package/dist/esm/src/error/background/DebouncedJobRequiresMinimumDelay.js +52 -0
- package/dist/esm/src/error/background/NamedWorkstreamRateLimitMissingMaxOrDuration.js +31 -0
- package/dist/esm/src/error/background/RateLimitedPsychicJob.js +82 -0
- package/dist/esm/src/error/background/RateLimitedPsychicJobThrownFromWorkerWithoutLimiter.js +51 -0
- package/dist/esm/src/package-exports/errors.js +7 -0
- package/dist/esm/src/test-utils/WorkerTestUtils.js +79 -14
- package/dist/types/src/background/BaseBackgroundedModel.d.ts +97 -20
- package/dist/types/src/background/BaseBackgroundedService.d.ts +61 -10
- package/dist/types/src/background/BaseScheduledService.d.ts +54 -1
- package/dist/types/src/background/index.d.ts +137 -5
- package/dist/types/src/error/background/DebouncedJobOutpacesRateLimit.d.ts +31 -0
- package/dist/types/src/error/background/DebouncedJobRequiresMinimumDelay.d.ts +23 -0
- package/dist/types/src/error/background/NamedWorkstreamRateLimitMissingMaxOrDuration.d.ts +16 -0
- package/dist/types/src/error/background/RateLimitedPsychicJob.d.ts +73 -0
- package/dist/types/src/error/background/RateLimitedPsychicJobThrownFromWorkerWithoutLimiter.d.ts +16 -0
- package/dist/types/src/package-exports/errors.d.ts +1 -0
- package/dist/types/src/psychic-app-workers/index.d.ts +102 -53
- package/dist/types/src/test-utils/WorkerTestUtils.d.ts +32 -1
- package/dist/types/src/types/background.d.ts +100 -4
- package/package.json +9 -8
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { Job } from 'bullmq';
|
|
2
|
-
import { BackgroundJobConfig, DelayedJobOpts } from '../types/background.js';
|
|
2
|
+
import { BackgroundJobConfig, BackgroundWithOpts, DelayedJobOpts } from '../types/background.js';
|
|
3
3
|
import { FunctionPropertyNames } from '../types/utils.js';
|
|
4
4
|
export default class BaseBackgroundedService {
|
|
5
5
|
/**
|
|
@@ -11,7 +11,7 @@ export default class BaseBackgroundedService {
|
|
|
11
11
|
* @returns {object} config - the background job config
|
|
12
12
|
* @returns {string} config.priority - 'default' | 'urgent' | 'not_urgent' | 'last'
|
|
13
13
|
* @returns {string} config.workstream - a workstream name. This would be the name of a workstream, as defined in conf/workers.ts
|
|
14
|
-
* @returns {string} config.
|
|
14
|
+
* @returns {string} config.queue - the name of the BullMQ queue you wish to connect to. This can only be provided if workstream is not provided.
|
|
15
15
|
* @returns {string} config.groupId - the groupId of the BullMQ queue you wish to connect to. This can only be provided if workstream is not provided.
|
|
16
16
|
*/
|
|
17
17
|
static get backgroundJobConfig(): BackgroundJobConfig<BaseBackgroundedService>;
|
|
@@ -53,26 +53,69 @@ export default class BaseBackgroundedService {
|
|
|
53
53
|
static background<T, MethodName extends PsychicBackgroundedServiceStaticMethods<T & typeof BaseBackgroundedService>, MethodFunc extends T[MethodName & keyof T], MethodArgs extends BackgroundableMethodArgs<MethodFunc>>(this: T, methodName: MethodName, ...args: MethodArgs): Promise<void>;
|
|
54
54
|
/**
|
|
55
55
|
* runs the specified method in a background queue, driven by BullMQ,
|
|
56
|
-
* sending in the provided args,
|
|
57
|
-
*
|
|
58
|
-
* it is entered into the queue.
|
|
56
|
+
* sending in the provided args, along with a delay object — one or more of
|
|
57
|
+
* `{ seconds, minutes, hours, days }` — which holds the job off for that much
|
|
58
|
+
* time after it is entered into the queue.
|
|
59
|
+
*
|
|
60
|
+
* The delay object also accepts an optional `jobId`, which turns the delay
|
|
61
|
+
* into a **debounce**: repeated calls carrying the same `jobId` collapse into
|
|
62
|
+
* a single execution, which runs once the delay has elapsed without another
|
|
63
|
+
* call arriving. `jobId` is a deduplication key rather than a BullMQ job id.
|
|
64
|
+
*
|
|
65
|
+
* See {@link DelayedJobOpts.jobId}.
|
|
59
66
|
*
|
|
60
67
|
* ```ts
|
|
61
|
-
* await MyBackgroundableClass.backgroundWithDelay('myMethod', 'abc', 123)
|
|
68
|
+
* await MyBackgroundableClass.backgroundWithDelay({ minutes: 5 }, 'myMethod', 'abc', 123)
|
|
62
69
|
* ```
|
|
63
|
-
* though calling
|
|
64
|
-
* is an indication that
|
|
65
|
-
* completed.
|
|
70
|
+
* though calling backgroundWithDelay must be awaited, the resolution of the promise
|
|
71
|
+
* is an indication that a run is pending, not that it has completed.
|
|
66
72
|
*
|
|
67
73
|
* NOTE: in test environments, psychic will immediately invoke the underlying
|
|
68
74
|
* method, preventing you from needing to explicitly wait for queues to flush
|
|
69
75
|
* before making assertions.
|
|
70
76
|
*
|
|
71
|
-
* @
|
|
77
|
+
* @deprecated use `backgroundWith({ delay }, methodName, ...args)` instead. This method will be removed in a future major version.
|
|
78
|
+
*
|
|
79
|
+
* @param delay - how long you want to hold off before allowing the job to run, given as at least one of `seconds`, `minutes`, `hours` or `days`, plus an optional `jobId` (a deduplication key) which debounces repeated calls into a single run
|
|
72
80
|
* @param methodName - the name of the static method you wish to run in the background
|
|
73
81
|
* @param args - a variadic list of arguments to be sent to your method
|
|
74
82
|
*/
|
|
75
83
|
static backgroundWithDelay<T, MethodName extends PsychicBackgroundedServiceStaticMethods<T & typeof BaseBackgroundedService>, MethodFunc extends T[MethodName & keyof T], MethodArgs extends BackgroundableMethodArgs<MethodFunc>>(this: T, delay: DelayedJobOpts, methodName: MethodName, ...args: MethodArgs): Promise<void>;
|
|
84
|
+
/**
|
|
85
|
+
* runs the specified method in a background queue, driven by BullMQ,
|
|
86
|
+
* sending in the provided args, along with an options object which can
|
|
87
|
+
* be used to delay the job and/or override the priority provided by
|
|
88
|
+
* the `backgroundJobConfig` getter on this service.
|
|
89
|
+
*
|
|
90
|
+
* Adding a `jobId` to the delay turns it into a **debounce**: repeated calls
|
|
91
|
+
* carrying the same `jobId` collapse into a single execution, which runs once
|
|
92
|
+
* the delay has elapsed without another call arriving. `jobId` is a
|
|
93
|
+
* deduplication key rather than a BullMQ job id.
|
|
94
|
+
*
|
|
95
|
+
* See {@link DelayedJobOpts.jobId}.
|
|
96
|
+
*
|
|
97
|
+
* ```ts
|
|
98
|
+
* await MyBackgroundableClass.backgroundWith(
|
|
99
|
+
* { delay: { seconds: 30, jobId: 'my-unique-job-id' }, priority: 'urgent' },
|
|
100
|
+
* 'myMethod',
|
|
101
|
+
* 'abc',
|
|
102
|
+
* 123,
|
|
103
|
+
* )
|
|
104
|
+
* ```
|
|
105
|
+
* though calling backgroundWith must be awaited, the resolution of the promise
|
|
106
|
+
* is an indication that a run is pending, not that it has completed.
|
|
107
|
+
*
|
|
108
|
+
* NOTE: in test environments, psychic will immediately invoke the underlying
|
|
109
|
+
* method, preventing you from needing to explicitly wait for queues to flush
|
|
110
|
+
* before making assertions.
|
|
111
|
+
*
|
|
112
|
+
* @param opts - options for backgrounding this job
|
|
113
|
+
* @param opts.delay - (optional) how long you want to hold off before allowing the job to run, given as at least one of `seconds`, `minutes`, `hours` or `days`, plus an optional `jobId` (a deduplication key) which debounces repeated calls into a single run
|
|
114
|
+
* @param opts.priority - (optional) a priority which, when provided, overrides the priority provided by `backgroundJobConfig`
|
|
115
|
+
* @param methodName - the name of the static method you wish to run in the background
|
|
116
|
+
* @param args - a variadic list of arguments to be sent to your method
|
|
117
|
+
*/
|
|
118
|
+
static backgroundWith<T, MethodName extends PsychicBackgroundedServiceStaticMethods<T & typeof BaseBackgroundedService>, MethodFunc extends T[MethodName & keyof T], MethodArgs extends BackgroundableMethodArgs<MethodFunc>>(this: T, opts: BackgroundWithOpts, methodName: MethodName, ...args: MethodArgs): Promise<void>;
|
|
76
119
|
/**
|
|
77
120
|
* types composed by psychic must be provided, since psychic-workers leverages
|
|
78
121
|
* the sync command in psychic to read your backgroundable services and extract
|
|
@@ -81,6 +124,14 @@ export default class BaseBackgroundedService {
|
|
|
81
124
|
*/
|
|
82
125
|
get psychicWorkerTypes(): any;
|
|
83
126
|
}
|
|
127
|
+
/**
|
|
128
|
+
* @internal
|
|
129
|
+
*
|
|
130
|
+
* returns a copy of the provided job config, with the priority
|
|
131
|
+
* replaced by the priority found in the `backgroundWith` opts,
|
|
132
|
+
* if one was provided.
|
|
133
|
+
*/
|
|
134
|
+
export declare function mergeBackgroundWithOptsIntoJobConfig<T extends BackgroundJobConfig<any>>(jobConfig: T, opts: BackgroundWithOpts): T;
|
|
84
135
|
export type PsychicBackgroundedServiceStaticMethods<T extends typeof BaseBackgroundedService> = Exclude<FunctionPropertyNames<Required<T>>, FunctionPropertyNames<typeof BaseBackgroundedService>>;
|
|
85
136
|
export type PsychicBackgroundedServiceInstanceMethods<T extends BaseBackgroundedService> = Exclude<FunctionPropertyNames<Required<T>>, FunctionPropertyNames<BaseBackgroundedService>>;
|
|
86
137
|
type OmitJobFromEndOfArguments<Original extends unknown[]> = Original extends [Job] ? Original extends [string] ? Original : [] : Original extends [...infer Rest, Job] ? Original extends [...unknown[], string] ? Original : Rest : Original;
|
|
@@ -10,7 +10,7 @@ export default class BaseScheduledService {
|
|
|
10
10
|
* @returns {object} config - the background job config
|
|
11
11
|
* @returns {string} config.priority - 'default' | 'urgent' | 'not_urgent' | 'last'
|
|
12
12
|
* @returns {string} config.workstream - a workstream name. This would be the name of a workstream, as defined in conf/workers.ts
|
|
13
|
-
* @returns {string} config.
|
|
13
|
+
* @returns {string} config.queue - the name of the BullMQ queue you wish to connect to. This can only be provided if workstream is not provided.
|
|
14
14
|
* @returns {string} config.groupId - the groupId of the BullMQ queue you wish to connect to. This can only be provided if workstream is not provided.
|
|
15
15
|
*/
|
|
16
16
|
static get backgroundJobConfig(): BackgroundJobConfig<BaseScheduledService>;
|
|
@@ -51,6 +51,59 @@ export default class BaseScheduledService {
|
|
|
51
51
|
* @param args - a variadic list of arguments to be sent to your method
|
|
52
52
|
*/
|
|
53
53
|
static schedule<T, MethodName extends FunctionPropertyNames<Required<T>>, MethodFunc extends T[MethodName & keyof T], MethodArgs extends MethodFunc extends (...args: any) => any ? Parameters<MethodFunc> : never>(this: T, pattern: string, methodName: MethodName, ...args: MethodArgs): Promise<void>;
|
|
54
|
+
/**
|
|
55
|
+
* Returns the id that `schedule` registers a job under, which can be passed
|
|
56
|
+
* to {@link BaseScheduledService.unschedule} to remove that job.
|
|
57
|
+
*
|
|
58
|
+
* The id is derived from the service's global name and the method name. It
|
|
59
|
+
* performs no queue construction and no redis I/O, so it is safe to call
|
|
60
|
+
* anywhere, including from a console in development.
|
|
61
|
+
*
|
|
62
|
+
* The intended workflow for deleting a scheduled service is to read the id
|
|
63
|
+
* off the class before removing it:
|
|
64
|
+
*
|
|
65
|
+
* ```ts
|
|
66
|
+
* MyScheduledService.unscheduleId('myHourlyMethod')
|
|
67
|
+
* // => 'services/MyScheduledService:myHourlyMethod'
|
|
68
|
+
* ```
|
|
69
|
+
*
|
|
70
|
+
* then check that string into a seed or migration and call `unschedule` with
|
|
71
|
+
* it, which allows the class itself to be deleted in the same deploy.
|
|
72
|
+
*
|
|
73
|
+
* ```ts
|
|
74
|
+
* await ApplicationScheduledService.unschedule('services/MyScheduledService:myHourlyMethod')
|
|
75
|
+
* ```
|
|
76
|
+
*
|
|
77
|
+
* @param methodName - the name of the static method that was scheduled
|
|
78
|
+
* @returns the id `schedule` registered this method under
|
|
79
|
+
*/
|
|
80
|
+
static unscheduleId<T, MethodName extends FunctionPropertyNames<Required<T>>>(this: T, methodName: MethodName): string;
|
|
81
|
+
/**
|
|
82
|
+
* Removes a scheduled job, preventing it from being run again.
|
|
83
|
+
*
|
|
84
|
+
* ```ts
|
|
85
|
+
* await MyScheduledService.unschedule(MyScheduledService.unscheduleId('myHourlyMethod'))
|
|
86
|
+
* ```
|
|
87
|
+
*
|
|
88
|
+
* Since the id is just a string, it can also be checked into a seed or
|
|
89
|
+
* migration, which allows the scheduled service class to be deleted in the
|
|
90
|
+
* same deploy that stops its job:
|
|
91
|
+
*
|
|
92
|
+
* ```ts
|
|
93
|
+
* await ApplicationScheduledService.unschedule('services/MyScheduledService:myHourlyMethod')
|
|
94
|
+
* ```
|
|
95
|
+
*
|
|
96
|
+
* Every queue in your application is checked, so this finds the job whether
|
|
97
|
+
* or not the service's workstream has changed since it was scheduled, and
|
|
98
|
+
* whether it landed in a current or a transitional queue.
|
|
99
|
+
*
|
|
100
|
+
* NOTE: unscheduling stops future runs. It does not cancel a run that has
|
|
101
|
+
* already been placed on a queue, so a final invocation may still happen.
|
|
102
|
+
*
|
|
103
|
+
* @param id - the id the job was scheduled under, from {@link BaseScheduledService.unscheduleId}
|
|
104
|
+
* @returns true if a scheduled job was removed, false if none was found
|
|
105
|
+
*/
|
|
106
|
+
static unschedule(id: string): Promise<boolean>;
|
|
54
107
|
/**
|
|
55
108
|
* types composed by psychic must be provided, since psychic-workers leverages
|
|
56
109
|
* the sync command in psychic to read your backgroundable services and extract
|
|
@@ -7,7 +7,13 @@ import { BackgroundJobConfig, BackgroundJobData, BackgroundQueuePriority, JobTyp
|
|
|
7
7
|
*/
|
|
8
8
|
export declare class Background {
|
|
9
9
|
/**
|
|
10
|
-
* returns the default queue
|
|
10
|
+
* returns the **logical** name of your app's default queue, built from your
|
|
11
|
+
* app name (`MyAppBackgroundJobQueue`).
|
|
12
|
+
*
|
|
13
|
+
* This is never the name the queue has in Redis: `connect` rewrites it per
|
|
14
|
+
* connection (brace-stripped, cluster hash-tagged, test-suffixed), and every
|
|
15
|
+
* key BullMQ writes is namespaced by the rewritten name. Do not compose Redis
|
|
16
|
+
* keys from this — read `queue.name` off `background.queues` instead.
|
|
11
17
|
*/
|
|
12
18
|
static get defaultQueueName(): string;
|
|
13
19
|
/**
|
|
@@ -75,17 +81,61 @@ export declare class Background {
|
|
|
75
81
|
*/
|
|
76
82
|
private redisConnections;
|
|
77
83
|
/**
|
|
78
|
-
*
|
|
84
|
+
* @internal
|
|
85
|
+
*
|
|
86
|
+
* For every queue built by `connect`: how it was configured, whether its
|
|
87
|
+
* workers carry a BullMQ `limiter`, and what the workers are built from.
|
|
88
|
+
* Written whether or not workers are activated here, so activation can happen
|
|
89
|
+
* later without re-walking the configuration. Keyed by Queue identity rather
|
|
90
|
+
* than name: a transitional workstream's queue has the same formatted name as
|
|
91
|
+
* its current twin but its own configuration.
|
|
92
|
+
*/
|
|
93
|
+
private queueWorkerRecords;
|
|
94
|
+
/**
|
|
95
|
+
* @internal
|
|
96
|
+
*
|
|
97
|
+
* set immediately before {@link buildRecordedWorkers} runs its build loop, so
|
|
98
|
+
* that a second activation builds nothing
|
|
99
|
+
*/
|
|
100
|
+
private workersActivated;
|
|
101
|
+
/**
|
|
102
|
+
* Establishes connection to BullMQ via redis: builds the `Queue` objects for
|
|
103
|
+
* the default and named workstreams and, only when `activateWorkers` is true,
|
|
104
|
+
* the `Worker` objects that run jobs off them. Synchronous: it returns
|
|
105
|
+
* nothing, so it is called without `await` even from async code.
|
|
106
|
+
*
|
|
107
|
+
* Connecting and activating are separate. `activateWorkers` defaults to
|
|
108
|
+
* `false`, so connecting gets the producer side only. Queues are built once
|
|
109
|
+
* per instance, but a later activation still builds workers, so a process that
|
|
110
|
+
* connects first and calls `work()` afterwards gets its workers.
|
|
111
|
+
*
|
|
112
|
+
* You rarely call this yourself — Psychic connects on the
|
|
113
|
+
* `server:init:after-routes` hook, on every enqueue path, in CLI codegen and
|
|
114
|
+
* in the `WorkerTestUtils` helpers.
|
|
79
115
|
*/
|
|
80
116
|
connect({ activateWorkers, }?: {
|
|
81
117
|
activateWorkers?: boolean;
|
|
82
118
|
}): void;
|
|
83
119
|
/**
|
|
84
|
-
* Returns all the queues in your application
|
|
120
|
+
* Returns all the queues in your application: the default queue, every named
|
|
121
|
+
* queue, and their transitional twins when transitional workstreams are
|
|
122
|
+
* configured.
|
|
123
|
+
*
|
|
124
|
+
* `connect` is what populates this, and before it has run the getter returns
|
|
125
|
+
* an **empty array**, with no error and no warning — indistinguishable from
|
|
126
|
+
* an application with nothing queued. Each `Queue` here carries the rewritten
|
|
127
|
+
* Redis name rather than the name you configured.
|
|
85
128
|
*/
|
|
86
129
|
get queues(): Queue[];
|
|
87
130
|
/**
|
|
88
|
-
* Returns all the workers in your application
|
|
131
|
+
* Returns all the workers in your application: every BullMQ `Worker` this
|
|
132
|
+
* process has built.
|
|
133
|
+
*
|
|
134
|
+
* Only a process that activated workers has any — `work()`, or a direct
|
|
135
|
+
* `connect({ activateWorkers: true })`. Anywhere else the getter returns an
|
|
136
|
+
* **empty array**, with no error and no warning, so a worker entrypoint that
|
|
137
|
+
* attaches failure listeners by looping over it before `work()` runs attaches
|
|
138
|
+
* nothing and logs nothing to say so.
|
|
89
139
|
*/
|
|
90
140
|
get workers(): Worker<any, any, string>[];
|
|
91
141
|
/**
|
|
@@ -141,7 +191,61 @@ export declare class Background {
|
|
|
141
191
|
*/
|
|
142
192
|
private nativeBullMQConnect;
|
|
143
193
|
/**
|
|
144
|
-
*
|
|
194
|
+
* @internal
|
|
195
|
+
*
|
|
196
|
+
* the sole writer of {@link queueWorkerRecords}. `workerOptions` is the very
|
|
197
|
+
* object the workers are built from (minus the connection), so the record and
|
|
198
|
+
* the workers cannot disagree.
|
|
199
|
+
*/
|
|
200
|
+
private recordQueueWorkers;
|
|
201
|
+
/**
|
|
202
|
+
* @internal
|
|
203
|
+
*
|
|
204
|
+
* builds every worker `connect` recorded a queue for, in the order the queues
|
|
205
|
+
* were built, so the default missing-connection throw still pre-empts the
|
|
206
|
+
* named one. The activation flag is set before the loop, not after it, because
|
|
207
|
+
* `_workers` is append-only and a build that throws partway must not be
|
|
208
|
+
* retryable into a second copy of the workers it already constructed.
|
|
209
|
+
*/
|
|
210
|
+
private buildRecordedWorkers;
|
|
211
|
+
/**
|
|
212
|
+
* @internal
|
|
213
|
+
*
|
|
214
|
+
* throws {@link NamedWorkstreamRateLimitMissingMaxOrDuration} for the first
|
|
215
|
+
* workstream whose `rateLimit` lacks a positive integer `max` or `duration`.
|
|
216
|
+
* The type requires both but admits any number, and open-source BullMQ does
|
|
217
|
+
* not validate a `limiter`: a fractional `duration` floors to 0ms and rate
|
|
218
|
+
* limits nothing, and one past Redis's integer range fails every job fetch.
|
|
219
|
+
*/
|
|
220
|
+
private assertNamedWorkstreamRateLimits;
|
|
221
|
+
/**
|
|
222
|
+
* @internal
|
|
223
|
+
*
|
|
224
|
+
* when `err` is a {@link RateLimitedPsychicJob} thrown on a queue whose workers
|
|
225
|
+
* carry no BullMQ `limiter`, returns the misconfiguration error to fail the job
|
|
226
|
+
* with instead; otherwise undefined. Declared as `Error` so the concrete,
|
|
227
|
+
* unexported class stays out of the emitted declarations. Four callers, one
|
|
228
|
+
* across a module boundary in `WorkerTestUtils`.
|
|
229
|
+
*/
|
|
230
|
+
misconfiguredRateLimitSignal(err: unknown, queue: Queue): Error | undefined;
|
|
231
|
+
/**
|
|
232
|
+
* @internal
|
|
233
|
+
*
|
|
234
|
+
* the processor every worker on `queue` runs: {@link doWork}, with a
|
|
235
|
+
* {@link RateLimitedPsychicJob} translated into BullMQ's own rate-limit
|
|
236
|
+
* signal — the queue's limiter key is set, pausing every worker on the queue,
|
|
237
|
+
* then `RateLimitError` moves the job back without counting an attempt or
|
|
238
|
+
* emitting `failed`. Thrown where the workers carry no `limiter`, it is
|
|
239
|
+
* instead a misconfiguration: see {@link misconfiguredRateLimitSignal}.
|
|
240
|
+
*/
|
|
241
|
+
private processorFor;
|
|
242
|
+
/**
|
|
243
|
+
* starts background workers: connects if this process has not connected yet,
|
|
244
|
+
* then builds the `Worker` objects for every queue `connect` recorded, and
|
|
245
|
+
* installs the process-level fatal-error and signal handlers that shut them
|
|
246
|
+
* down. An earlier producer-only `connect()` no longer leaves `work()` with
|
|
247
|
+
* nothing to do. A process with no `defaultWorkerConnection` configured throws
|
|
248
|
+
* here rather than returning quietly.
|
|
145
249
|
*/
|
|
146
250
|
work(): void;
|
|
147
251
|
/**
|
|
@@ -182,6 +286,34 @@ export declare class Background {
|
|
|
182
286
|
jobConfig?: BackgroundJobConfig<any>;
|
|
183
287
|
scheduleOpts?: JobSchedulerTemplateOptions;
|
|
184
288
|
}): Promise<void>;
|
|
289
|
+
/**
|
|
290
|
+
* @internal
|
|
291
|
+
*
|
|
292
|
+
* Returns the id a scheduled job is registered with BullMQ under.
|
|
293
|
+
*
|
|
294
|
+
* `jobId` is used to determine uniqueness along with name and repeat pattern.
|
|
295
|
+
* Since the name is really a job type and never changes, the `jobId` is the only
|
|
296
|
+
* way to allow multiple jobs with the same cron repeat pattern. Uniqueness is
|
|
297
|
+
* enforced by combining the global name and the method name.
|
|
298
|
+
*
|
|
299
|
+
* See: https://docs.bullmq.io/guide/jobs/repeatable
|
|
300
|
+
*
|
|
301
|
+
* @param globalName - the globalName of the class the method belongs to
|
|
302
|
+
* @param method - the name of the scheduled method
|
|
303
|
+
*/
|
|
304
|
+
jobSchedulerId(globalName: string, method: string): string;
|
|
305
|
+
/**
|
|
306
|
+
* removes a scheduled job from BullMQ, preventing it from being run again
|
|
307
|
+
*
|
|
308
|
+
* Every queue is checked, rather than the one the job's current
|
|
309
|
+
* `backgroundJobConfig` routes to, so that a job can still be unscheduled
|
|
310
|
+
* after its workstream has changed, or after the class that scheduled it has
|
|
311
|
+
* been deleted.
|
|
312
|
+
*
|
|
313
|
+
* @param jobSchedulerId - the id the job was scheduled under
|
|
314
|
+
* @returns true if a scheduled job was removed, false if none was found
|
|
315
|
+
*/
|
|
316
|
+
unschedule(jobSchedulerId: string): Promise<boolean>;
|
|
185
317
|
private queueInstance;
|
|
186
318
|
/**
|
|
187
319
|
* adds the instance method of a provided dream model to BullMQ
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import { WorkerQueueDescription } from '../../types/background.js';
|
|
2
|
+
/**
|
|
3
|
+
* Not exported from the package: a delayed background job carried a `jobId`
|
|
4
|
+
* behind a delay short enough that it can enqueue work faster than its own
|
|
5
|
+
* queue's rate limit allows that work to start. Thrown from `_addToQueue`,
|
|
6
|
+
* above the test-mode short circuit, so a consumer's test environment raises it
|
|
7
|
+
* exactly as production does.
|
|
8
|
+
*/
|
|
9
|
+
export default class DebouncedJobOutpacesRateLimit extends Error {
|
|
10
|
+
private jobId;
|
|
11
|
+
private delayMs;
|
|
12
|
+
private keyLifetimeMs;
|
|
13
|
+
private limiter;
|
|
14
|
+
private queue;
|
|
15
|
+
constructor(jobId: string, delayMs: number, keyLifetimeMs: number, limiter: {
|
|
16
|
+
max: number;
|
|
17
|
+
duration: number;
|
|
18
|
+
}, queue: WorkerQueueDescription);
|
|
19
|
+
get message(): string;
|
|
20
|
+
private get msPerJob();
|
|
21
|
+
/**
|
|
22
|
+
* the shortest delay that would pass: enough that the key outlives the
|
|
23
|
+
* limiter's spacing. Rounded up to a whole second, since the delay is given
|
|
24
|
+
* in whole-ish duration fields and a value that still failed by a fraction
|
|
25
|
+
* of a second would be a hostile suggestion.
|
|
26
|
+
*/
|
|
27
|
+
private get minimumDelayMs();
|
|
28
|
+
private seconds;
|
|
29
|
+
private get kind();
|
|
30
|
+
private get where();
|
|
31
|
+
}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Not exported from the package, and deliberately so: every branch of the guard
|
|
3
|
+
* that throws it is a call-site mistake — a `jobId` behind a delay under the
|
|
4
|
+
* floor, an empty `jobId`, or a delay that is zero, negative, `Infinity`, `NaN`
|
|
5
|
+
* or past `Number.MAX_SAFE_INTEGER` — so an application can never usefully
|
|
6
|
+
* catch it. The fix is always an edit to the call.
|
|
7
|
+
*
|
|
8
|
+
* `durationToSeconds` catches none of these: it validates nothing, and its
|
|
9
|
+
* falsy-field sum silently drops a `NaN` field rather than propagating it
|
|
10
|
+
* (`{ hours: 1, seconds: NaN }` is `3600`), so `{ seconds: NaN }` reaches this
|
|
11
|
+
* check as `0`.
|
|
12
|
+
*
|
|
13
|
+
* The check runs before the test-mode short circuit in `_addToQueue`, so a
|
|
14
|
+
* consumer's default test environment raises this exactly as production does.
|
|
15
|
+
*/
|
|
16
|
+
export default class DebouncedJobRequiresMinimumDelay extends Error {
|
|
17
|
+
private jobId;
|
|
18
|
+
private delaySeconds;
|
|
19
|
+
private minimumDelayMs;
|
|
20
|
+
constructor(jobId: string, delaySeconds: number | undefined, minimumDelayMs: number);
|
|
21
|
+
get message(): string;
|
|
22
|
+
private get emptyJobIdMessage();
|
|
23
|
+
}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Not exported from the package: a named workstream's `rateLimit` reached
|
|
3
|
+
* `connect()` without a positive integer `max` or `duration`. Open-source
|
|
4
|
+
* BullMQ forwards the worker `limiter` it becomes unvalidated: a missing or
|
|
5
|
+
* non-numeric field fails every job fetch on that workstream with a Lua error
|
|
6
|
+
* that never names `rateLimit`, a fractional `duration` floors to 0ms and rate
|
|
7
|
+
* limits nothing, and one past Redis's integer range fails every fetch.
|
|
8
|
+
*/
|
|
9
|
+
export default class NamedWorkstreamRateLimitMissingMaxOrDuration extends Error {
|
|
10
|
+
private workstreamName;
|
|
11
|
+
private transitional;
|
|
12
|
+
private field;
|
|
13
|
+
private received;
|
|
14
|
+
constructor(workstreamName: string, transitional: boolean, field: 'max' | 'duration', received: unknown);
|
|
15
|
+
get message(): string;
|
|
16
|
+
}
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Throw from inside a backgrounded (or scheduled) method to signal that the
|
|
3
|
+
* outside service it talks to rate-limited the request (an HTTP 429, say) and
|
|
4
|
+
* asked it to stop sending for a while. `pauseQueueForSeconds` is how long that
|
|
5
|
+
* while is, in whole seconds. Importable from `@rvoh/psychic-workers/errors`.
|
|
6
|
+
*
|
|
7
|
+
* ```ts
|
|
8
|
+
* // conf/workers.ts — one named workstream per rate-limited service
|
|
9
|
+
* namedWorkstreams: [{ name: 'slack', rateLimit: { max: 1, duration: 1000 } }]
|
|
10
|
+
*
|
|
11
|
+
* // app/services/SlackNotifier.ts
|
|
12
|
+
* import { RateLimitedPsychicJob } from '@rvoh/psychic-workers/errors'
|
|
13
|
+
*
|
|
14
|
+
* export default class SlackNotifier extends ApplicationBackgroundedService {
|
|
15
|
+
* public static override get backgroundJobConfig() {
|
|
16
|
+
* return { workstream: 'slack' }
|
|
17
|
+
* }
|
|
18
|
+
*
|
|
19
|
+
* public static async post(channel: string, text: string) {
|
|
20
|
+
* const response = await fetch('https://slack.com/api/chat.postMessage', { ... })
|
|
21
|
+
*
|
|
22
|
+
* if (response.status === 429) {
|
|
23
|
+
* // Slack's Retry-After header is in seconds, and so is this field
|
|
24
|
+
* const seconds = Number(response.headers.get('retry-after') ?? 1)
|
|
25
|
+
* throw new RateLimitedPsychicJob({ pauseQueueForSeconds: seconds })
|
|
26
|
+
* }
|
|
27
|
+
* }
|
|
28
|
+
* }
|
|
29
|
+
* ```
|
|
30
|
+
*
|
|
31
|
+
* The worker running the job must carry a BullMQ `limiter` — a named
|
|
32
|
+
* workstream with `rateLimit`, a native `namedQueueWorkers` entry with
|
|
33
|
+
* `limiter`, `nativeBullMQ.defaultWorkerOptions.limiter`, or a global
|
|
34
|
+
* `defaultBullMQWorkerOptions.limiter`. Given one, psychic-workers translates
|
|
35
|
+
* the signal into BullMQ's own rate-limit mechanism: the queue is paused for
|
|
36
|
+
* that many seconds and the job goes back onto the queue with `attemptsMade`
|
|
37
|
+
* unchanged and no backoff, so the burst is not multiplied by the retry
|
|
38
|
+
* schedule. Note that this package writes a BullMQ priority on every job it
|
|
39
|
+
* enqueues, so a re-queued job reports `prioritized` rather than `waiting` —
|
|
40
|
+
* assertions on `getWaitingCount()` do not hold.
|
|
41
|
+
*
|
|
42
|
+
* The pause replaces the queue's current limiter window and counter — the
|
|
43
|
+
* configured `rateLimit` window and any earlier pause alike — so once
|
|
44
|
+
* `pauseQueueForSeconds` elapses up to `max` jobs may start at once, and a
|
|
45
|
+
* `pauseQueueForSeconds` shorter than the window's remaining time shortens the
|
|
46
|
+
* configured limit. Concurrent throws overwrite each other: last writer wins,
|
|
47
|
+
* not longest.
|
|
48
|
+
*
|
|
49
|
+
* `attemptsMade` never grows, but each re-fetch counts in BullMQ's
|
|
50
|
+
* `attemptsStarted`, and the worker option `maxStartedAttempts`
|
|
51
|
+
* (`defaultBullMQWorkerOptions: { maxStartedAttempts: 10 }`, say) is the only
|
|
52
|
+
* bound on that cycle: without it it is unbounded. Set it on any workstream
|
|
53
|
+
* whose jobs throw this signal.
|
|
54
|
+
*
|
|
55
|
+
* Thrown from a job whose worker carries no `limiter`, the signal is a
|
|
56
|
+
* misconfiguration: the job fails with an ordinary error whose message names
|
|
57
|
+
* the fix. The same check runs under test invocation and `WorkerTestUtils`, so
|
|
58
|
+
* a misplaced job goes red in specs.
|
|
59
|
+
*
|
|
60
|
+
* `pauseQueueForSeconds` must be a positive, finite number of safe magnitude;
|
|
61
|
+
* the constructor throws a `RangeError` otherwise, at the throw site, so a
|
|
62
|
+
* nonsense value fails there rather than at the pause. A fractional number of
|
|
63
|
+
* seconds is legal and is rounded **up** — the field is a lower bound on the
|
|
64
|
+
* pause, so overshooting it by under a second cannot break the promise, while
|
|
65
|
+
* rounding down could.
|
|
66
|
+
*/
|
|
67
|
+
export default class RateLimitedPsychicJob extends Error {
|
|
68
|
+
readonly pauseQueueForSeconds: number;
|
|
69
|
+
constructor({ pauseQueueForSeconds }: {
|
|
70
|
+
pauseQueueForSeconds: number;
|
|
71
|
+
});
|
|
72
|
+
get message(): string;
|
|
73
|
+
}
|
package/dist/types/src/error/background/RateLimitedPsychicJobThrownFromWorkerWithoutLimiter.d.ts
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import { WorkerQueueDescription } from '../../types/background.js';
|
|
2
|
+
import RateLimitedPsychicJob from './RateLimitedPsychicJob.js';
|
|
3
|
+
/**
|
|
4
|
+
* Not exported from the package: a `RateLimitedPsychicJob` was thrown from a
|
|
5
|
+
* job whose worker carries no BullMQ `limiter`, so the queue cannot be paused.
|
|
6
|
+
* The job runner fails the job with this in the signal's place, so the failure
|
|
7
|
+
* reaches the application's failed-job monitoring.
|
|
8
|
+
*/
|
|
9
|
+
export default class RateLimitedPsychicJobThrownFromWorkerWithoutLimiter extends Error {
|
|
10
|
+
private signal;
|
|
11
|
+
private queue;
|
|
12
|
+
constructor(signal: RateLimitedPsychicJob, queue: WorkerQueueDescription);
|
|
13
|
+
get message(): string;
|
|
14
|
+
private get where();
|
|
15
|
+
private get fix();
|
|
16
|
+
}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
1
|
export { default as NoClassForSpecifiedGlobalName } from '../error/background/NoClassForSpecifiedGlobalName.js';
|
|
2
2
|
export { default as NoQueueForSpecifiedQueueName } from '../error/background/NoQueueForSpecifiedQueueName.js';
|
|
3
3
|
export { default as NoQueueForSpecifiedWorkstream } from '../error/background/NoQueueForSpecifiedWorkstream.js';
|
|
4
|
+
export { default as RateLimitedPsychicJob } from '../error/background/RateLimitedPsychicJob.js';
|