@rvoh/psychic-workers 2.4.1 → 2.7.0

package/dist/types/src/psychic-app-workers/index.d.ts

@@ -64,7 +64,25 @@ export interface BullMQNativeWorkerOptions extends WorkerOptions {
         concurrency?: number;
         priority?: number;
     };
+    /**
+     * How many jobs each worker built from this configuration runs at once
+     * (https://docs.bullmq.io/guide/workers/concurrency).
+     *
+     * Native BullMQ mode writes none, so BullMQ's default of **1** applies unless
+     * this or `defaultBullMQWorkerOptions.concurrency` sets it. See
+     * `PsychicBackgroundWorkstreamOptions.concurrency`.
+     */
     concurrency?: number;
+    /**
+     * the number of workers to create with this configuration, in this process.
+     *
+     * Read **only** for `nativeBullMQ.namedQueueWorkers`, where it defaults to 1
+     * (a named queue with no entry in `namedQueueWorkers` gets no workers at
+     * all). It is *not* read for `nativeBullMQ.defaultWorkerOptions`: the default
+     * queue's worker count comes from `nativeBullMQ.defaultWorkerCount`. See
+     * `PsychicBackgroundWorkstreamOptions.workerCount` for the
+     * `workerCount × concurrency` product.
+     */
     workerCount?: number;
 }
 export interface PsychicBackgroundNativeBullMQOptions extends PsychicBackgroundSharedOptions {
@@ -82,26 +100,24 @@ export interface PsychicBackgroundNativeBullMQOptions extends PsychicBackgroundS
          */
         namedQueueOptions?: Record<string, QueueOptionsWithConnectionInstance>;
         /**
-         * Native BullMQ options to provide to configure the default workers
-         * for psychic. By default, Psychic leverages a single-queue system, with
-         * many workers running off the queue. Each worker receives the
-         * same worker configuration, so this configuration is really
-         * only used to supply the number of default workers that you want.
+         * Native BullMQ options for the default workers, spread over
+         * `defaultBullMQWorkerOptions` — where `concurrency`, `limiter` and every
+         * other BullMQ worker option for them comes from. A `workerCount` set here
+         * is never read; the count comes from `defaultWorkerCount` below.
          */
         defaultWorkerOptions?: BullMQNativeWorkerOptions;
         /**
-         * The number of default workers to run against the default Psychic
-         * background queues.
-         *
-         * By default, Psychic leverages a single-queue system, with
-         * many workers running off a single queue. This number determines
-         * the number of those default workers to provide.
+         * The number of default workers to run in this process, and the only source
+         * of that count in native mode. Defaults to 1, and takes effect only in a
+         * process that activates workers. See
+         * `PsychicBackgroundWorkstreamOptions.workerCount`.
          */
         defaultWorkerCount?: number;
         /**
-         * namedQueueWorkers are necessary to work off namedQueues
-         * With BullMQ Pro, namedQueueWorkers can be rate limited (useful
-         * for interacting with external APIs)
+         * namedQueueWorkers are necessary to work off namedQueues.
+         * A named queue's workers can be rate limited with BullMQ's `limiter`
+         * option (https://docs.bullmq.io/guide/rate-limiting; useful for
+         * interacting with external APIs) on open-source BullMQ and BullMQ Pro alike
          */
         namedQueueWorkers?: Record<string, BullMQNativeWorkerOptions>;
     };
@@ -138,24 +154,34 @@ export interface PsychicBackgroundSimpleOptions extends PsychicBackgroundSharedO
      */
     defaultWorkerConnection: RedisOrRedisClusterConnection | undefined;
     /**
-     * Every Psychic application that leverages simple background jobs will have a default
-     * workstream. Set workerCount to set the number of workers that will work through the
-     * default queue
+     * The default workstream, whose queue carries every job not backgrounded to a
+     * named workstream. Both fields mean exactly what they mean on a named
+     * workstream — see `PsychicBackgroundWorkstreamOptions.workerCount` and
+     * `.concurrency`.
      */
     defaultWorkstream?: {
+        /** The number of workers working through the default queue, in this process. Defaults to 1. */
         workerCount?: number;
+        /**
+         * How many jobs each default worker runs at once
+         * (https://docs.bullmq.io/guide/workers/concurrency). Left unset, psychic
+         * falls back to 10, as everywhere in simple mode. See
+         * `PsychicBackgroundWorkstreamOptions.concurrency`.
+         */
         concurrency?: number;
     };
     /**
      * When running background jobs on BullMQ, each named workstream corresponds
-     * to a specific queue and workers created for a named workstream are given
-     * a groupId corresponding to the workstream name
-     *
-     * named workstreams are useful for dispersing queues among nodes in a Redis cluster
-     * and for running queues on different Redis instances
+     * to a specific queue, and jobs backgrounded to a named workstream are added
+     * with a group id equal to the workstream name (a BullMQ Pro concept, ignored
+     * by open-source BullMQ)
      *
-     * With BullMQ Pro, named workstreams can be rate limited (useful
-     * for interacting with external APIs)
+     * named workstreams are useful for dispersing queues among nodes in a Redis cluster,
+     * for running queues on different Redis instances, and for rate limiting, on
+     * open-source BullMQ and BullMQ Pro alike (see
+     * `PsychicBackgroundWorkstreamOptions.rateLimit`). A rate limit targets one external
+     * limit, so give each rate-limited service its own named workstream — and where a
+     * service meters its endpoints separately, one workstream per limit
      */
     namedWorkstreams?: PsychicBackgroundWorkstreamOptions[];
     /**
@@ -171,43 +197,66 @@ export interface PsychicBackgroundWorkstreamOptions {
      */
     name: string;
     /**
-     * The number of workers you want to run on this configuration
+     * The number of workers you want to run on this configuration, in **this
+     * process**. Defaults to 1. These are BullMQ `Worker` objects sharing the
+     * activating process's event loop, not forked processes, so raising this buys
+     * concurrent waiting rather than concurrent CPU.
+     *
+     * `workerCount × concurrency` is the ceiling on this workstream's jobs in
+     * flight in one process, and that product is what to size against a
+     * downstream limit such as a connection pool or an external API's quota.
      */
     workerCount?: number;
-    concurrency?: number;
-    /**
-     * See https://docs.bullmq.io/bullmq-pro/groups/rate-limiting for documentation
-     * on rate limiting in BullMQ Pro (requires paid BullMQ Pro license)
-     */
-    rateLimit?: {
-        max?: number;
-        duration?: number;
-    };
-    /**
-     * Optional redis connection. If not provided, the default background redis connection will be used.
-     * See https://docs.bullmq.io/guide/going-to-production for the different settings to use between
-     * queue and worker connections.
-     */
-    queueConnection?: RedisOrRedisClusterConnection;
-    workerConnection?: RedisOrRedisClusterConnection;
-}
-export interface PsychicBackgroundWorkstreamOptions {
-    /**
-     * This will be the name of the queue (and the group if using BullMQ Pro)
-     */
-    name: string;
     /**
-     * The number of workers you want to run on this configuration
+     * How many jobs each of this workstream's workers runs at once
+     * (https://docs.bullmq.io/guide/workers/concurrency).
+     *
+     * **In simple mode this defaults to 10.** Psychic always writes a
+     * `concurrency` onto a simple-mode worker, falling back to 10 when the
+     * workstream does not set one — which also means it overrides any
+     * `concurrency` placed in `defaultBullMQWorkerOptions`. Native BullMQ mode
+     * writes none, so BullMQ's own default of 1 applies there. Moving a queue
+     * between the two modes therefore changes how many jobs it runs at once by a
+     * factor of ten, with nothing in either configuration saying so.
+     *
+     * See `workerCount` above for the `workerCount × concurrency` product.
      */
-    workerCount?: number;
     concurrency?: number;
     /**
-     * See https://docs.bullmq.io/bullmq-pro/groups/rate-limiting for documentation
-     * on rate limiting in BullMQ Pro (requires paid BullMQ Pro license)
+     * Rate-limit this workstream: at most `max` jobs start in any `duration`
+     * milliseconds. Works on open-source BullMQ and on BullMQ Pro.
+     *
+     * The limit is per queue, not per worker: every one of this workstream's
+     * workers, in every process running it, shares one counter kept in Redis, so
+     * `{ max: 1, duration: 1000 }` means one job per second for the workstream as
+     * a whole. `concurrency` caps how many jobs each worker runs at once;
+     * `rateLimit` caps how many may start per window across the whole workstream.
+     * For this workstream's workers it takes precedence over a global
+     * `defaultBullMQWorkerOptions.limiter`.
+     *
+     * A job on this workstream that is told to slow down (an HTTP 429, say) can
+     * throw `RateLimitedPsychicJob` from `@rvoh/psychic-workers/errors` to pause
+     * the whole workstream without burning a retry attempt; see that class for
+     * the pause semantics and for the `maxStartedAttempts` bound.
+     *
+     * A debounced job on this workstream must carry a delay long enough that it
+     * cannot enqueue faster than this limit starts jobs, or it is refused at
+     * enqueue: see `DelayedJobOpts.jobId`.
+     *
+     * `max` and `duration` are both required positive integers (`duration` in
+     * milliseconds): open-source BullMQ floors a fractional `duration` to 0ms
+     * (rate limiting nothing) and fails every job fetch on one past Redis's
+     * integer range. A `rateLimit` that reaches `connect()` without both fails
+     * `connect()` naming the workstream and the field, before any queue or worker
+     * is built.
+     *
+     * Implemented as BullMQ's worker `limiter` option, and on BullMQ Pro also as
+     * the workstream group's rate limit (unverified against an installed Pro
+     * build).
      */
     rateLimit?: {
-        max?: number;
-        duration?: number;
+        max: number;
+        duration: number;
     };
     /**
      * Optional redis connection. If not provided, the default background redis connection will be used.

package/dist/types/src/test-utils/WorkerTestUtils.d.ts

@@ -1,10 +1,41 @@
+import { Queue } from 'bullmq';
+/**
+ * @internal
+ *
+ * resolves the queue's expected Redis name through `nameToRedisQueueName`,
+ * rather than hand-reconstructing a cluster hash tag or parallel-test suffix.
+ * `queue.opts.connection` is the same `Redis | Cluster` instance the queue was
+ * built with, which is why the cast is safe for every queue this package builds.
+ */
+export declare function queueNamesMatch(queue: Queue, compareQueueName: string): boolean;
 export default class WorkerTestUtils {
     static parallelTestSafeQueueName(queueName: string): string;
     static work(opts?: TestWorkerWorkOffOpts): Promise<void>;
     static workScheduled(opts?: TestWorkerScheduledWorkOffOpts): Promise<void>;
+    /**
+     * runs a delayed job in place. A `RateLimitedPsychicJob` thrown from a job on
+     * a queue whose workers carry no BullMQ `limiter` surfaces as the same
+     * misconfiguration error production would fail the job with; on a queue whose
+     * workers carry one, it propagates untranslated so the spec can assert on it.
+     */
+    private static doScheduledWork;
     static clean(): Promise<void>;
     private static workOne;
-    private static queueNamesMatch;
+    /**
+     * runs one fetched job to completion or failure, mirroring what a real worker
+     * would do with a `RateLimitedPsychicJob`, then surfacing it so the spec goes
+     * red rather than looping:
+     *
+     * - on a queue whose workers carry no BullMQ `limiter`, the job is failed with
+     *   the misconfiguration error production would fail it with (so, with the
+     *   queue's `attempts` > 1, it lands in `delayed` for its retry, out of
+     *   `active`, where `clean()` can reach it), and that error is rethrown
+     * - on a queue whose workers carry one, the job is moved back to the queue
+     *   with no attempt counted (BullMQ's own rate-limit path), and the
+     *   `RateLimitedPsychicJob` is rethrown
+     *
+     * Every other error only fails the job, as before.
+     */
     private static processJob;
 }
 interface TestWorkerWorkOffOpts {

package/dist/types/src/types/background.d.ts

@@ -29,14 +29,73 @@ export interface BackgroundJobData {
      */
     globalName?: string;
 }
-export type DelayedJobOpts = DelayedJobDuration & {
+/**
+ * the delay to hold a job for and, when a `jobId` is provided, the debounce
+ * that delay drives.
+ *
+ * Calling a delayed background method repeatedly with the same `jobId`
+ * collapses those calls into a **single execution**. Each call slides the
+ * pending job's fire time further out, and the job runs once, after the last
+ * call, when the delay has finally elapsed without another call arriving.
+ *
+ * ```ts
+ * // ten calls in quick succession; the report is generated once,
+ * // ten seconds after the last of them
+ * await Report.backgroundWith({ delay: { seconds: 10, jobId: `report-${user.id}` } }, 'generate', user.id)
+ * ```
+ *
+ * A delay object must carry at least one of `seconds`, `minutes`, `hours` or
+ * `days`. `{}` and `{ jobId: 'my-job' }` are compile errors, since a delay with
+ * no duration delays nothing and a `jobId` with no duration debounces
+ * nothing.
+ */
+export type DelayedJobOpts = AtLeastOneDelayedJobDuration & {
     /**
-     * a unique identifier for your job. this identifier will be
-     * used to debounce, leveraging the internal throttling mechanisms
-     * provided by BullMQ
+     * a **deduplication key**, which debounces repeated calls: calls carrying the
+     * same `jobId` collapse into a single execution, which runs once the delay
+     * has elapsed without another call arriving. The window runs from the last
+     * call, not the first, so a burst is collapsed however long it lasts.
+     *
+     * Despite the name, this is not the BullMQ job id of the enqueued job, and
+     * `queue.getJob(jobId)` will not resolve the debounced job through it. BullMQ
+     * stores it as a separate Redis key; the matching read is
+     * `queue.getDeduplicationJobId(jobId)`.
+     *
+     * A delay carrying a `jobId` must be **at least three seconds**, and a
+     * shorter one is refused at enqueue.
+     *
+     * On a rate-limited workstream the delay must also clear the limiter's
+     * spacing: `delay - 1s >= duration / max`, so a workstream limited to one job
+     * a minute needs a 61-second delay. A delay that does not is refused at
+     * enqueue, naming the queue, both limiter numbers and a delay that would
+     * pass.
      */
     jobId?: string;
 };
+/**
+ * `DelayedJobDuration`, but at least one field has to be there. Every field is
+ * optional on its own, so `{}` would otherwise compile and delay nothing.
+ * TypeScript has no "at least one of these" operator, so this spells out the
+ * four ways to satisfy it and joins them with `|`:
+ *
+ * ```ts
+ * | { seconds: number;  minutes?: number; hours?:   number; days?:  number }
+ * | { minutes: number;  seconds?: number; hours?:   number; days?:  number }
+ * | { hours:   number;  seconds?: number; minutes?: number; days?:  number }
+ * | { days:    number;  seconds?: number; minutes?: number; hours?: number }
+ * ```
+ *
+ * A value has to match one of those, which means carrying that line's required
+ * field. `{ seconds: 30 }` matches the first. `{}` matches none, and neither
+ * does `{ jobId: 'my-job' }`.
+ *
+ * Not exported: it exists only to narrow `DelayedJobOpts`. `DelayedJobDuration`
+ * itself stays all-optional because it is also the parameter type of the shared
+ * `durationToSeconds` helper.
+ */
+type AtLeastOneDelayedJobDuration = {
+    [Field in keyof DelayedJobDuration]-?: Required<Pick<DelayedJobDuration, Field>> & Partial<Omit<DelayedJobDuration, Field>>;
+}[keyof DelayedJobDuration];
 export interface DelayedJobDuration {
     seconds?: number;
     minutes?: number;
@@ -45,6 +104,27 @@ export interface DelayedJobDuration {
 }
 export type JobTypes = 'BackgroundJobQueueFunctionJob' | 'BackgroundJobQueueStaticJob' | 'BackgroundJobQueueModelInstanceJob';
 export type BackgroundQueuePriority = 'default' | 'urgent' | 'not_urgent' | 'last';
+export interface BackgroundWithOpts {
+    /**
+     * an optional delay to hold off the job for a certain amount of
+     * time after it is entered into the queue. Accepts the same options
+     * as `backgroundWithDelay`, and must carry at least one of `seconds`,
+     * `minutes`, `hours` or `days`.
+     *
+     * Adding a `jobId` turns the delay into a **debounce**: repeated calls
+     * carrying the same `jobId` collapse into a single execution, which runs
+     * once the delay has elapsed without another call arriving. `jobId` is a
+     * deduplication key rather than a BullMQ job id.
+     *
+     * See {@link DelayedJobOpts.jobId}.
+     */
+    delay?: DelayedJobOpts;
+    /**
+     * an optional priority. When provided, this overrides the priority
+     * set on the `backgroundJobConfig` of the service or model.
+     */
+    priority?: BackgroundQueuePriority;
+}
 interface BaseBackgroundJobConfig {
     priority?: BackgroundQueuePriority;
 }
@@ -57,4 +137,20 @@ export interface QueueBackgroundJobConfig<T extends BaseScheduledService | BaseB
 }
 export type BackgroundJobConfig<T extends BaseScheduledService | BaseBackgroundedService> = Either<WorkstreamBackgroundJobConfig<T>, QueueBackgroundJobConfig<T>>;
 export type PsychicBackgroundOptions = (PsychicBackgroundSimpleOptions & Partial<Record<Exclude<keyof PsychicBackgroundNativeBullMQOptions, keyof PsychicBackgroundSimpleOptions>, never>>) | (PsychicBackgroundNativeBullMQOptions & Partial<Record<Exclude<keyof PsychicBackgroundSimpleOptions, keyof PsychicBackgroundNativeBullMQOptions>, never>>);
+/**
+ * @internal
+ *
+ * how a queue was configured, as far as the misconfiguration message needs to
+ * know in order to name the fix that applies to the worker that threw
+ */
+export interface WorkerQueueDescription {
+    /** simple (workstream) configuration or native BullMQ configuration */
+    mode: 'simple' | 'native';
+    /** the default workstream/queue, as opposed to a named one */
+    isDefaultQueue: boolean;
+    /** the workstream or queue name as configured (not the formatted Redis queue name) */
+    configuredName: string;
+    /** a workstream declared under `transitionalWorkstreams` */
+    transitional: boolean;
+}
 export {};

package/package.json

@@ -2,7 +2,7 @@
   "type": "module",
   "name": "@rvoh/psychic-workers",
   "description": "Background job system for Psychic applications",
-  "version": "2.4.1",
+  "version": "2.7.0",
   "author": "RVO Health",
   "publishConfig": {
     "access": "public"
@@ -61,7 +61,7 @@
     "ioredis": "^5.8.2"
   },
   "devDependencies": {
-    "@eslint/js": "^9.0.0",
+    "@eslint/js": "^10.0.1",
     "@koa/cors": "^5.0.0",
     "@koa/etag": "^5.0.2",
     "@koa/router": "^15.4.0",
@@ -78,15 +78,15 @@
     "@types/node": "^22.12.0",
     "@types/pg": "^8",
     "@types/supertest": "^6.0.3",
-    "@typescript-eslint/parser": "^8.48.1",
+    "@typescript-eslint/parser": "^8.67.0",
     "bullmq": "^5.64.0",
-    "eslint": "^9.39.1",
+    "eslint": "^10.0.1",
     "ioredis": "^5.8.2",
     "koa": "^3.1.2",
     "koa-bodyparser": "^4.4.1",
     "koa-conditional-get": "^3.0.0",
-    "kysely": "^0.28.13",
-    "kysely-codegen": "^0.19.0",
+    "kysely": "^0.28.17",
+    "kysely-codegen": "~0.20.0",
     "luxon-jest-matchers": "^0.1.14",
     "openapi-typescript": "^7.9.1",
     "pg": "^8.13.1",
@@ -96,8 +96,9 @@
     "tsx": "^4.20.6",
     "typedoc": "^0.28.17",
     "typescript": "^5.8.2",
-    "typescript-eslint": "^8.48.1",
-    "vitest": "^4.1.0"
+    "typescript-eslint": "^8.67.0",
+    "vite": "^7.3.6",
+    "vitest": "^4.1.11"
   },
   "packageManager": "pnpm@11.9.0+sha512.bd682d5d03fe525ef7c9fd6780c6884d1e756ac4c9c9fe00c538782824310dcf90e3ddc4f53835f06dfaebd5085e41855e0bcbb3b60de2ac5bbab89e5036f03b",
   "dependencies": {