enqiu 0.1.2 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +298 -0
- package/README.md +132 -235
- package/dist/api.d.ts +16 -236
- package/dist/api.js +383 -478
- package/dist/backend.d.ts +25 -0
- package/dist/backend.js +18 -0
- package/dist/definition.d.ts +13 -0
- package/dist/definition.js +51 -0
- package/dist/errors.d.ts +57 -0
- package/dist/errors.js +83 -0
- package/dist/events.d.ts +30 -0
- package/dist/events.js +53 -0
- package/dist/index.d.ts +4 -5
- package/dist/index.js +3 -3
- package/dist/mapping.d.ts +99 -0
- package/dist/mapping.js +167 -0
- package/dist/markers.d.ts +25 -0
- package/dist/markers.js +51 -0
- package/dist/runner.d.ts +20 -0
- package/dist/runner.js +101 -0
- package/dist/serialize.d.ts +15 -0
- package/dist/serialize.js +90 -0
- package/dist/types.d.ts +326 -0
- package/dist/types.js +9 -0
- package/package.json +29 -13
- package/dist/codec.d.ts +0 -8
- package/dist/codec.js +0 -74
- package/dist/cron.d.ts +0 -19
- package/dist/cron.js +0 -217
- package/dist/memory-scheduler.d.ts +0 -24
- package/dist/memory-scheduler.js +0 -163
- package/dist/memory.d.ts +0 -344
- package/dist/memory.js +0 -1201
- package/dist/redis.d.ts +0 -202
- package/dist/redis.js +0 -2180
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The one place Enqiu reaches past BullMQ's public API.
|
|
3
|
+
*
|
|
4
|
+
* `getBackend().client` is documented by BullMQ as a Redis-specific escape
|
|
5
|
+
* hatch that is deliberately not part of `IQueueBackend`, so using it ties
|
|
6
|
+
* Enqiu to the Redis backend. Two features need it — cancellation markers and
|
|
7
|
+
* reading the tail of the event stream — and confining the cast to this file
|
|
8
|
+
* keeps the coupling one import wide instead of scattered.
|
|
9
|
+
*
|
|
10
|
+
* Going through `RedisClient`, BullMQ's own abstraction, rather than an
|
|
11
|
+
* ioredis-shaped interface is what keeps those two working on node-redis and
|
|
12
|
+
* Bun, where `hset` takes a field map instead of positional arguments.
|
|
13
|
+
* XREVRANGE is genuinely absent from it: nothing in BullMQ reads a stream
|
|
14
|
+
* backwards.
|
|
15
|
+
*/
|
|
16
|
+
import type { ConnectionOptions, Queue, RedisClient } from "bullmq";
|
|
17
|
+
/** The BullMQ options every object Enqiu constructs is built from. */
|
|
18
|
+
export interface BullBase {
|
|
19
|
+
readonly connection: ConnectionOptions;
|
|
20
|
+
readonly prefix?: string;
|
|
21
|
+
}
|
|
22
|
+
export type BackendClient = RedisClient & {
|
|
23
|
+
xrevrange(key: string, end: string, start: string, count: string, limit: string): Promise<unknown>;
|
|
24
|
+
};
|
|
25
|
+
export declare function backendClient(queue: Queue): Promise<BackendClient>;
|
package/dist/backend.js
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The one place Enqiu reaches past BullMQ's public API.
|
|
3
|
+
*
|
|
4
|
+
* `getBackend().client` is documented by BullMQ as a Redis-specific escape
|
|
5
|
+
* hatch that is deliberately not part of `IQueueBackend`, so using it ties
|
|
6
|
+
* Enqiu to the Redis backend. Two features need it — cancellation markers and
|
|
7
|
+
* reading the tail of the event stream — and confining the cast to this file
|
|
8
|
+
* keeps the coupling one import wide instead of scattered.
|
|
9
|
+
*
|
|
10
|
+
* Going through `RedisClient`, BullMQ's own abstraction, rather than an
|
|
11
|
+
* ioredis-shaped interface is what keeps those two working on node-redis and
|
|
12
|
+
* Bun, where `hset` takes a field map instead of positional arguments.
|
|
13
|
+
* XREVRANGE is genuinely absent from it: nothing in BullMQ reads a stream
|
|
14
|
+
* backwards.
|
|
15
|
+
*/
|
|
16
|
+
export async function backendClient(queue) {
|
|
17
|
+
return (await queue.getBackend().client);
|
|
18
|
+
}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/** Turning what a caller wrote into something the worker can run. */
|
|
2
|
+
import { definitionMarker } from "./types.js";
|
|
3
|
+
import type { JobDefinition, JobHandler, JobPolicyOptions, SchemaJobDefinition, StandardSchemaV1 } from "./types.js";
|
|
4
|
+
/** Declare a job with a Standard Schema input and per-job policies. */
|
|
5
|
+
export declare function job<const Schema extends StandardSchemaV1, Output>(definition: Omit<SchemaJobDefinition<Schema, Output>, typeof definitionMarker>): SchemaJobDefinition<Schema, Output>;
|
|
6
|
+
/** A definition reduced to the three things the runtime actually reads. */
|
|
7
|
+
export interface RuntimeDefinition {
|
|
8
|
+
readonly schema: StandardSchemaV1 | undefined;
|
|
9
|
+
readonly run: JobHandler;
|
|
10
|
+
readonly policy: JobPolicyOptions;
|
|
11
|
+
}
|
|
12
|
+
export declare function normalizeDefinition(definition: JobDefinition): RuntimeDefinition;
|
|
13
|
+
export declare function validateInput(name: string, schema: StandardSchemaV1 | undefined, input: unknown): Promise<unknown>;
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
/** Turning what a caller wrote into something the worker can run. */
|
|
2
|
+
import { JobValidationError } from "./errors.js";
|
|
3
|
+
import { definitionMarker } from "./types.js";
|
|
4
|
+
/** Declare a job with a Standard Schema input and per-job policies. */
|
|
5
|
+
export function job(definition) {
|
|
6
|
+
if (!definition || typeof definition !== "object") {
|
|
7
|
+
throw new TypeError("job() requires a definition object");
|
|
8
|
+
}
|
|
9
|
+
if (!isStandardSchema(definition.input)) {
|
|
10
|
+
throw new TypeError("job.input must implement Standard Schema");
|
|
11
|
+
}
|
|
12
|
+
if (typeof definition.run !== "function") {
|
|
13
|
+
throw new TypeError("job.run must be a function");
|
|
14
|
+
}
|
|
15
|
+
return Object.freeze({ ...definition, [definitionMarker]: true });
|
|
16
|
+
}
|
|
17
|
+
function isStandardSchema(value) {
|
|
18
|
+
if (!value || typeof value !== "object")
|
|
19
|
+
return false;
|
|
20
|
+
const standard = value["~standard"];
|
|
21
|
+
return (standard?.version === 1 &&
|
|
22
|
+
typeof standard.vendor === "string" &&
|
|
23
|
+
typeof standard.validate === "function");
|
|
24
|
+
}
|
|
25
|
+
export function normalizeDefinition(definition) {
|
|
26
|
+
if (typeof definition === "function") {
|
|
27
|
+
return { schema: undefined, run: definition, policy: {} };
|
|
28
|
+
}
|
|
29
|
+
if (!definition ||
|
|
30
|
+
typeof definition !== "object" ||
|
|
31
|
+
definition[definitionMarker] !== true) {
|
|
32
|
+
throw new TypeError("Every job must be a handler or a definition created with job()");
|
|
33
|
+
}
|
|
34
|
+
const policy = {};
|
|
35
|
+
if (definition.retry !== undefined)
|
|
36
|
+
policy.retry = definition.retry;
|
|
37
|
+
if (definition.timeout !== undefined)
|
|
38
|
+
policy.timeout = definition.timeout;
|
|
39
|
+
if (definition.expiresIn !== undefined) {
|
|
40
|
+
policy.expiresIn = definition.expiresIn;
|
|
41
|
+
}
|
|
42
|
+
return { schema: definition.input, run: definition.run, policy };
|
|
43
|
+
}
|
|
44
|
+
export async function validateInput(name, schema, input) {
|
|
45
|
+
if (!schema)
|
|
46
|
+
return input;
|
|
47
|
+
const result = await schema["~standard"].validate(input);
|
|
48
|
+
if (result.issues)
|
|
49
|
+
throw new JobValidationError(name, result.issues);
|
|
50
|
+
return result.value;
|
|
51
|
+
}
|
package/dist/errors.d.ts
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
/** Errors a job can settle with, and how a reason survives the queue. */
|
|
2
|
+
export interface SerializedError {
|
|
3
|
+
name: string;
|
|
4
|
+
message: string;
|
|
5
|
+
}
|
|
6
|
+
export declare class JobFailedError extends Error {
|
|
7
|
+
readonly jobId: string;
|
|
8
|
+
constructor(jobId: string, message: string, options?: ErrorOptions);
|
|
9
|
+
}
|
|
10
|
+
export declare class JobCancelledError extends Error {
|
|
11
|
+
readonly jobId: string;
|
|
12
|
+
constructor(jobId: string, message?: string);
|
|
13
|
+
}
|
|
14
|
+
export declare class JobTimeoutError extends Error {
|
|
15
|
+
readonly jobId: string;
|
|
16
|
+
readonly timeout: number;
|
|
17
|
+
constructor(jobId: string, timeout: number);
|
|
18
|
+
}
|
|
19
|
+
export declare class JobExpiredError extends Error {
|
|
20
|
+
readonly jobId: string;
|
|
21
|
+
constructor(jobId: string);
|
|
22
|
+
}
|
|
23
|
+
export declare class QueueClosedError extends Error {
|
|
24
|
+
constructor(name: string);
|
|
25
|
+
}
|
|
26
|
+
export declare class JobValidationError extends TypeError {
|
|
27
|
+
readonly issues: readonly StandardSchemaIssue[];
|
|
28
|
+
constructor(name: string, issues: readonly StandardSchemaIssue[]);
|
|
29
|
+
}
|
|
30
|
+
export interface StandardSchemaIssue {
|
|
31
|
+
readonly message: string;
|
|
32
|
+
readonly path?: ReadonlyArray<PropertyKey | {
|
|
33
|
+
readonly key: PropertyKey;
|
|
34
|
+
}> | undefined;
|
|
35
|
+
}
|
|
36
|
+
export declare function toError(value: unknown): Error;
|
|
37
|
+
/**
|
|
38
|
+
* Why a job failed, in a form that survives the trip through Redis.
|
|
39
|
+
*
|
|
40
|
+
* BullMQ stores one string per failure, `failedReason`, and hands it to other
|
|
41
|
+
* processes as a plain `Error`. So the reason has to travel as text — but not
|
|
42
|
+
* as prose. Reading the kind back out of a human sentence means rewording that
|
|
43
|
+
* sentence silently changes which class callers catch, with no test to notice;
|
|
44
|
+
* this codebase has already been bitten by exactly that once. The kind is
|
|
45
|
+
* therefore written down, and only the message is prose.
|
|
46
|
+
*/
|
|
47
|
+
export type JobFailure = {
|
|
48
|
+
readonly kind: "timeout";
|
|
49
|
+
readonly jobId: string;
|
|
50
|
+
readonly timeout: number;
|
|
51
|
+
} | {
|
|
52
|
+
readonly kind: "expired";
|
|
53
|
+
readonly jobId: string;
|
|
54
|
+
};
|
|
55
|
+
export declare function encodeFailure(failure: JobFailure): string;
|
|
56
|
+
export declare function decodeFailure(reason: string): JobFailure | undefined;
|
|
57
|
+
export declare function failureToError(failure: JobFailure): Error;
|
package/dist/errors.js
ADDED
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
/** Errors a job can settle with, and how a reason survives the queue. */
|
|
2
|
+
export class JobFailedError extends Error {
|
|
3
|
+
jobId;
|
|
4
|
+
constructor(jobId, message, options) {
|
|
5
|
+
super(message, options);
|
|
6
|
+
this.name = "JobFailedError";
|
|
7
|
+
this.jobId = jobId;
|
|
8
|
+
}
|
|
9
|
+
}
|
|
10
|
+
export class JobCancelledError extends Error {
|
|
11
|
+
jobId;
|
|
12
|
+
constructor(jobId, message = "Job was cancelled") {
|
|
13
|
+
super(message);
|
|
14
|
+
this.name = "JobCancelledError";
|
|
15
|
+
this.jobId = jobId;
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
export class JobTimeoutError extends Error {
|
|
19
|
+
jobId;
|
|
20
|
+
timeout;
|
|
21
|
+
constructor(jobId, timeout) {
|
|
22
|
+
super(`Job "${jobId}" timed out after ${timeout}ms`);
|
|
23
|
+
this.name = "JobTimeoutError";
|
|
24
|
+
this.jobId = jobId;
|
|
25
|
+
this.timeout = timeout;
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
export class JobExpiredError extends Error {
|
|
29
|
+
jobId;
|
|
30
|
+
constructor(jobId) {
|
|
31
|
+
super(`Job "${jobId}" expired before it could start`);
|
|
32
|
+
this.name = "JobExpiredError";
|
|
33
|
+
this.jobId = jobId;
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
export class QueueClosedError extends Error {
|
|
37
|
+
constructor(name) {
|
|
38
|
+
super(`Queue "${name}" is closed`);
|
|
39
|
+
this.name = "QueueClosedError";
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
export class JobValidationError extends TypeError {
|
|
43
|
+
issues;
|
|
44
|
+
constructor(name, issues) {
|
|
45
|
+
super(`Invalid input for job "${name}": ${issues[0]?.message ?? "validation failed"}`);
|
|
46
|
+
this.name = "JobValidationError";
|
|
47
|
+
this.issues = issues;
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
export function toError(value) {
|
|
51
|
+
return value instanceof Error ? value : new Error(String(value));
|
|
52
|
+
}
|
|
53
|
+
const FAILURE_PREFIX = "enqiu-failure:";
|
|
54
|
+
export function encodeFailure(failure) {
|
|
55
|
+
return FAILURE_PREFIX + JSON.stringify(failure);
|
|
56
|
+
}
|
|
57
|
+
export function decodeFailure(reason) {
|
|
58
|
+
if (!reason.startsWith(FAILURE_PREFIX))
|
|
59
|
+
return undefined;
|
|
60
|
+
let parsed;
|
|
61
|
+
try {
|
|
62
|
+
parsed = JSON.parse(reason.slice(FAILURE_PREFIX.length));
|
|
63
|
+
}
|
|
64
|
+
catch {
|
|
65
|
+
return undefined;
|
|
66
|
+
}
|
|
67
|
+
if (!parsed || typeof parsed !== "object")
|
|
68
|
+
return undefined;
|
|
69
|
+
const { kind, jobId, timeout } = parsed;
|
|
70
|
+
if (typeof jobId !== "string")
|
|
71
|
+
return undefined;
|
|
72
|
+
if (kind === "expired")
|
|
73
|
+
return { kind, jobId };
|
|
74
|
+
if (kind === "timeout" && typeof timeout === "number") {
|
|
75
|
+
return { kind, jobId, timeout };
|
|
76
|
+
}
|
|
77
|
+
return undefined;
|
|
78
|
+
}
|
|
79
|
+
export function failureToError(failure) {
|
|
80
|
+
return failure.kind === "timeout"
|
|
81
|
+
? new JobTimeoutError(failure.jobId, failure.timeout)
|
|
82
|
+
: new JobExpiredError(failure.jobId);
|
|
83
|
+
}
|
package/dist/events.d.ts
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The queue's event stream, opened only if something asks for it.
|
|
3
|
+
*
|
|
4
|
+
* A `QueueEvents` costs a dedicated blocking Redis connection, which a
|
|
5
|
+
* producer that never subscribes and never awaits a result should not pay for.
|
|
6
|
+
*
|
|
7
|
+
* Opening it from a known point matters. BullMQ's QueueEvents defaults to
|
|
8
|
+
* reading from the present, which it evaluates when its read loop starts
|
|
9
|
+
* rather than when it is constructed — so a caller that subscribed and
|
|
10
|
+
* immediately submitted could miss its own `added` event. Capturing the stream
|
|
11
|
+
* tail first and passing it as `lastEventId` closes that window: nothing
|
|
12
|
+
* appended after this point can fall between the two.
|
|
13
|
+
*/
|
|
14
|
+
import { QueueEvents, type Queue } from "bullmq";
|
|
15
|
+
import { type BullBase } from "./backend.js";
|
|
16
|
+
export declare class QueueEventStream {
|
|
17
|
+
private readonly queue;
|
|
18
|
+
private readonly base;
|
|
19
|
+
private opened;
|
|
20
|
+
constructor(queue: Queue, base: BullBase);
|
|
21
|
+
open(): Promise<QueueEvents>;
|
|
22
|
+
/**
|
|
23
|
+
* Waits only if a subscription is already opening.
|
|
24
|
+
*
|
|
25
|
+
* Submissions call this so that a caller who subscribed and then submitted
|
|
26
|
+
* cannot outrun the stream — without it being the thing that opens one.
|
|
27
|
+
*/
|
|
28
|
+
settle(): Promise<void>;
|
|
29
|
+
close(): Promise<void>;
|
|
30
|
+
}
|
package/dist/events.js
ADDED
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The queue's event stream, opened only if something asks for it.
|
|
3
|
+
*
|
|
4
|
+
* A `QueueEvents` costs a dedicated blocking Redis connection, which a
|
|
5
|
+
* producer that never subscribes and never awaits a result should not pay for.
|
|
6
|
+
*
|
|
7
|
+
* Opening it from a known point matters. BullMQ's QueueEvents defaults to
|
|
8
|
+
* reading from the present, which it evaluates when its read loop starts
|
|
9
|
+
* rather than when it is constructed — so a caller that subscribed and
|
|
10
|
+
* immediately submitted could miss its own `added` event. Capturing the stream
|
|
11
|
+
* tail first and passing it as `lastEventId` closes that window: nothing
|
|
12
|
+
* appended after this point can fall between the two.
|
|
13
|
+
*/
|
|
14
|
+
import { QueueEvents } from "bullmq";
|
|
15
|
+
import { backendClient } from "./backend.js";
|
|
16
|
+
export class QueueEventStream {
|
|
17
|
+
queue;
|
|
18
|
+
base;
|
|
19
|
+
opened;
|
|
20
|
+
constructor(queue, base) {
|
|
21
|
+
this.queue = queue;
|
|
22
|
+
this.base = base;
|
|
23
|
+
}
|
|
24
|
+
open() {
|
|
25
|
+
this.opened ??= (async () => {
|
|
26
|
+
const client = await backendClient(this.queue);
|
|
27
|
+
const tail = await client.xrevrange(this.queue.toKey("events"), "+", "-", "COUNT", "1");
|
|
28
|
+
const first = Array.isArray(tail) ? tail[0] : undefined;
|
|
29
|
+
const events = new QueueEvents(this.queue.name, {
|
|
30
|
+
...this.base,
|
|
31
|
+
lastEventId: first ? String(first[0]) : "0-0",
|
|
32
|
+
});
|
|
33
|
+
await events.waitUntilReady();
|
|
34
|
+
return events;
|
|
35
|
+
})();
|
|
36
|
+
return this.opened;
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* Waits only if a subscription is already opening.
|
|
40
|
+
*
|
|
41
|
+
* Submissions call this so that a caller who subscribed and then submitted
|
|
42
|
+
* cannot outrun the stream — without it being the thing that opens one.
|
|
43
|
+
*/
|
|
44
|
+
async settle() {
|
|
45
|
+
if (this.opened)
|
|
46
|
+
await this.opened;
|
|
47
|
+
}
|
|
48
|
+
async close() {
|
|
49
|
+
if (!this.opened)
|
|
50
|
+
return;
|
|
51
|
+
await (await this.opened).close();
|
|
52
|
+
}
|
|
53
|
+
}
|
package/dist/index.d.ts
CHANGED
|
@@ -1,5 +1,4 @@
|
|
|
1
|
-
export {
|
|
2
|
-
export
|
|
3
|
-
export {
|
|
4
|
-
export type { JobSnapshot, JobStatus,
|
|
5
|
-
export { enqiu as default } from "./api.js";
|
|
1
|
+
export { enqiu, enqiu as default, job } from "./api.js";
|
|
2
|
+
export { JobCancelledError, JobExpiredError, JobFailedError, JobTimeoutError, JobValidationError, QueueClosedError, toError, } from "./errors.js";
|
|
3
|
+
export { JobSerializationError } from "./serialize.js";
|
|
4
|
+
export type { AnyJobSnapshot, BackoffOptions, BulkOptions, CleanupQuery, Enqiu, EnqiuOptions, HandlerJobDefinition, InferSchemaInput, InferSchemaOutput, JobCallable, JobContext, JobDefinition, JobDefinitions, JobHandle, JobHandler, JobListPage, JobListQuery, JobLogger, JobPolicyOptions, JobSnapshot, JobStatus, JobsApi, MaybePromise, Progress, QueueApi, QueueEventMap, QueueStats, RetryPolicy, ScheduleHandle, ScheduleOptions, ScheduleSnapshot, SchemaJobDefinition, SerializedError, StandardSchemaIssue, StandardSchemaV1, SubmitOptions, WorkerApi, WorkerOptions, WorkerStartOptions, } from "./types.js";
|
package/dist/index.js
CHANGED
|
@@ -1,3 +1,3 @@
|
|
|
1
|
-
export {
|
|
2
|
-
export {
|
|
3
|
-
export {
|
|
1
|
+
export { enqiu, enqiu as default, job } from "./api.js";
|
|
2
|
+
export { JobCancelledError, JobExpiredError, JobFailedError, JobTimeoutError, JobValidationError, QueueClosedError, toError, } from "./errors.js";
|
|
3
|
+
export { JobSerializationError } from "./serialize.js";
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Everything that translates between BullMQ's vocabulary and Enqiu's.
|
|
3
|
+
*
|
|
4
|
+
* Pure, and the one part of the library that can be tested without a Redis:
|
|
5
|
+
* every other path goes through BullMQ. It lives in one place because the four
|
|
6
|
+
* directions — reading a state back, listing, cleaning, and counting — used to
|
|
7
|
+
* be four hand-written mappings, and they disagreed with each other.
|
|
8
|
+
*/
|
|
9
|
+
import type { Job as BullJob, FinishedStatus, JobState, JobsOptions } from "bullmq";
|
|
10
|
+
import type { JobPolicyOptions, JobSnapshot, JobStatus, RetryPolicy, SubmitOptions } from "./types.js";
|
|
11
|
+
/**
|
|
12
|
+
* Which BullMQ states make up each Enqiu status.
|
|
13
|
+
*
|
|
14
|
+
* `cancelled` maps to nothing: BullMQ has no state for it, so it lists and
|
|
15
|
+
* cleans as empty and the marker hash is the only record.
|
|
16
|
+
*/
|
|
17
|
+
export declare const bullStates: {
|
|
18
|
+
readonly queued: readonly ["waiting", "prioritized"];
|
|
19
|
+
readonly scheduled: readonly ["delayed"];
|
|
20
|
+
readonly running: readonly ["active"];
|
|
21
|
+
readonly succeeded: readonly ["completed"];
|
|
22
|
+
readonly failed: readonly ["failed"];
|
|
23
|
+
readonly cancelled: readonly [];
|
|
24
|
+
};
|
|
25
|
+
export declare const everyState: readonly JobState[];
|
|
26
|
+
/** BullMQ's states, mapped onto Enqiu's vocabulary. */
|
|
27
|
+
export declare function toStatus(state: string): JobStatus;
|
|
28
|
+
/** The two states a job can no longer leave — BullMQ names the pair too. */
|
|
29
|
+
export declare function isFinished(state: string): state is FinishedStatus;
|
|
30
|
+
/**
|
|
31
|
+
* Enqiu's events, and the BullMQ event that carries each.
|
|
32
|
+
*
|
|
33
|
+
* `state` is what the event itself proves about the job, which saves asking
|
|
34
|
+
* Redis for a state the notification already carried — and is the more
|
|
35
|
+
* faithful answer besides, since a subscriber wants the state at the time of
|
|
36
|
+
* the event rather than whatever it has become since. `added` proves only that
|
|
37
|
+
* the job exists, not whether it was delayed or prioritized.
|
|
38
|
+
*/
|
|
39
|
+
export declare const queueEventMap: {
|
|
40
|
+
readonly added: {
|
|
41
|
+
readonly name: "added";
|
|
42
|
+
readonly state: undefined;
|
|
43
|
+
};
|
|
44
|
+
readonly started: {
|
|
45
|
+
readonly name: "active";
|
|
46
|
+
readonly state: "active";
|
|
47
|
+
};
|
|
48
|
+
readonly progress: {
|
|
49
|
+
readonly name: "progress";
|
|
50
|
+
readonly state: "active";
|
|
51
|
+
};
|
|
52
|
+
readonly succeeded: {
|
|
53
|
+
readonly name: "completed";
|
|
54
|
+
readonly state: "completed";
|
|
55
|
+
};
|
|
56
|
+
readonly failed: {
|
|
57
|
+
readonly name: "failed";
|
|
58
|
+
readonly state: "failed";
|
|
59
|
+
};
|
|
60
|
+
readonly error: {
|
|
61
|
+
readonly name: "error";
|
|
62
|
+
readonly state: undefined;
|
|
63
|
+
};
|
|
64
|
+
};
|
|
65
|
+
export declare function toSnapshot(bull: BullJob, status: JobStatus): JobSnapshot;
|
|
66
|
+
export interface SubmitDefaults {
|
|
67
|
+
readonly logLimit: number;
|
|
68
|
+
readonly retry: number | RetryPolicy | undefined;
|
|
69
|
+
}
|
|
70
|
+
/** Options BullMQ can carry across the queue, resolved submit → job → queue. */
|
|
71
|
+
export declare function toJobsOptions(options: SubmitOptions, policy: JobPolicyOptions, defaults: SubmitDefaults): JobsOptions;
|
|
72
|
+
/**
|
|
73
|
+
* One offset per underlying state, because BullMQ ranges over each separately.
|
|
74
|
+
*
|
|
75
|
+
* A single number would mean "position within each state", which skips and
|
|
76
|
+
* repeats jobs the moment a status spans more than one of them.
|
|
77
|
+
*/
|
|
78
|
+
export declare function decodeCursor(cursor: string | undefined, states: number): number[];
|
|
79
|
+
export declare function encodeCursor(offsets: readonly number[]): string;
|
|
80
|
+
/** What one BullMQ state contributed to a page, and where it was read from. */
|
|
81
|
+
export interface StatePage<T> {
|
|
82
|
+
readonly offset: number;
|
|
83
|
+
readonly items: readonly T[];
|
|
84
|
+
}
|
|
85
|
+
/**
|
|
86
|
+
* Take one page's worth across the states, and say where each got to.
|
|
87
|
+
*
|
|
88
|
+
* Pure on purpose. This is the arithmetic a single-number cursor got wrong, and
|
|
89
|
+
* leaving it inside the async method that fetches the states put it out of
|
|
90
|
+
* reach of every test that runs without a server.
|
|
91
|
+
*
|
|
92
|
+
* Offsets arrive attached to their items rather than as a second array: they
|
|
93
|
+
* are only ever meaningful in pairs, and two arrays could disagree in length
|
|
94
|
+
* with nothing to catch it.
|
|
95
|
+
*/
|
|
96
|
+
export declare function mergePage<T>(pages: readonly StatePage<T>[], limit: number): {
|
|
97
|
+
items: T[];
|
|
98
|
+
next: number[];
|
|
99
|
+
};
|
package/dist/mapping.js
ADDED
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Everything that translates between BullMQ's vocabulary and Enqiu's.
|
|
3
|
+
*
|
|
4
|
+
* Pure, and the one part of the library that can be tested without a Redis:
|
|
5
|
+
* every other path goes through BullMQ. It lives in one place because the four
|
|
6
|
+
* directions — reading a state back, listing, cleaning, and counting — used to
|
|
7
|
+
* be four hand-written mappings, and they disagreed with each other.
|
|
8
|
+
*/
|
|
9
|
+
import { decodeFailure, failureToError } from "./errors.js";
|
|
10
|
+
/**
|
|
11
|
+
* Which BullMQ states make up each Enqiu status.
|
|
12
|
+
*
|
|
13
|
+
* `cancelled` maps to nothing: BullMQ has no state for it, so it lists and
|
|
14
|
+
* cleans as empty and the marker hash is the only record.
|
|
15
|
+
*/
|
|
16
|
+
export const bullStates = {
|
|
17
|
+
queued: ["waiting", "prioritized"],
|
|
18
|
+
scheduled: ["delayed"],
|
|
19
|
+
running: ["active"],
|
|
20
|
+
succeeded: ["completed"],
|
|
21
|
+
failed: ["failed"],
|
|
22
|
+
cancelled: [],
|
|
23
|
+
};
|
|
24
|
+
export const everyState = Object.values(bullStates).flat();
|
|
25
|
+
const statusByState = new Map(Object.entries(bullStates).flatMap(([status, states]) => states.map((state) => [state, status])));
|
|
26
|
+
/** BullMQ's states, mapped onto Enqiu's vocabulary. */
|
|
27
|
+
export function toStatus(state) {
|
|
28
|
+
return statusByState.get(state) ?? "queued";
|
|
29
|
+
}
|
|
30
|
+
/** The two states a job can no longer leave — BullMQ names the pair too. */
|
|
31
|
+
export function isFinished(state) {
|
|
32
|
+
return state === "completed" || state === "failed";
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* Enqiu's events, and the BullMQ event that carries each.
|
|
36
|
+
*
|
|
37
|
+
* `state` is what the event itself proves about the job, which saves asking
|
|
38
|
+
* Redis for a state the notification already carried — and is the more
|
|
39
|
+
* faithful answer besides, since a subscriber wants the state at the time of
|
|
40
|
+
* the event rather than whatever it has become since. `added` proves only that
|
|
41
|
+
* the job exists, not whether it was delayed or prioritized.
|
|
42
|
+
*/
|
|
43
|
+
export const queueEventMap = {
|
|
44
|
+
added: { name: "added", state: undefined },
|
|
45
|
+
started: { name: "active", state: "active" },
|
|
46
|
+
progress: { name: "progress", state: "active" },
|
|
47
|
+
succeeded: { name: "completed", state: "completed" },
|
|
48
|
+
failed: { name: "failed", state: "failed" },
|
|
49
|
+
error: { name: "error", state: undefined },
|
|
50
|
+
};
|
|
51
|
+
export function toSnapshot(bull, status) {
|
|
52
|
+
const snapshot = {
|
|
53
|
+
id: String(bull.id),
|
|
54
|
+
name: bull.name,
|
|
55
|
+
input: bull.data,
|
|
56
|
+
status,
|
|
57
|
+
attempt: bull.attemptsMade,
|
|
58
|
+
createdAt: bull.timestamp,
|
|
59
|
+
};
|
|
60
|
+
if (bull.processedOn)
|
|
61
|
+
snapshot.startedAt = bull.processedOn;
|
|
62
|
+
if (bull.finishedOn)
|
|
63
|
+
snapshot.finishedAt = bull.finishedOn;
|
|
64
|
+
if (bull.progress !== undefined && bull.progress !== 0) {
|
|
65
|
+
snapshot.progress = bull.progress;
|
|
66
|
+
}
|
|
67
|
+
if (bull.returnvalue !== undefined && bull.returnvalue !== null) {
|
|
68
|
+
snapshot.output = bull.returnvalue;
|
|
69
|
+
}
|
|
70
|
+
if (bull.failedReason) {
|
|
71
|
+
// A timeout or an expiry wrote down which it was, so the snapshot can name
|
|
72
|
+
// the error rather than calling everything "Error".
|
|
73
|
+
const failure = decodeFailure(bull.failedReason);
|
|
74
|
+
const error = failure ? failureToError(failure) : undefined;
|
|
75
|
+
snapshot.error = error
|
|
76
|
+
? { name: error.name, message: error.message }
|
|
77
|
+
: { name: "Error", message: bull.failedReason };
|
|
78
|
+
}
|
|
79
|
+
return snapshot;
|
|
80
|
+
}
|
|
81
|
+
// BullMQ orders ascending: a lower number runs sooner.
|
|
82
|
+
const priorities = { high: 1, normal: 2, low: 3 };
|
|
83
|
+
/** Options BullMQ can carry across the queue, resolved submit → job → queue. */
|
|
84
|
+
export function toJobsOptions(options, policy, defaults) {
|
|
85
|
+
const jobsOptions = { keepLogs: defaults.logLimit };
|
|
86
|
+
if (options.id !== undefined)
|
|
87
|
+
jobsOptions.jobId = options.id;
|
|
88
|
+
if (options.delay !== undefined) {
|
|
89
|
+
jobsOptions.delay =
|
|
90
|
+
options.delay instanceof Date
|
|
91
|
+
? Math.max(0, options.delay.getTime() - Date.now())
|
|
92
|
+
: options.delay;
|
|
93
|
+
}
|
|
94
|
+
if (options.priority !== undefined) {
|
|
95
|
+
jobsOptions.priority =
|
|
96
|
+
typeof options.priority === "string"
|
|
97
|
+
? priorities[options.priority]
|
|
98
|
+
: options.priority;
|
|
99
|
+
}
|
|
100
|
+
if (options.idempotencyKey !== undefined) {
|
|
101
|
+
jobsOptions.deduplication = { id: options.idempotencyKey };
|
|
102
|
+
if (options.idempotencyTtl !== undefined) {
|
|
103
|
+
jobsOptions.deduplication.ttl = options.idempotencyTtl;
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
const retry = options.retry ?? policy.retry ?? defaults.retry;
|
|
107
|
+
if (retry !== undefined) {
|
|
108
|
+
const attempts = typeof retry === "number" ? retry + 1 : retry.attempts;
|
|
109
|
+
if (!Number.isInteger(attempts) || attempts < 1) {
|
|
110
|
+
throw new RangeError("retry.attempts must be a positive integer");
|
|
111
|
+
}
|
|
112
|
+
jobsOptions.attempts = attempts;
|
|
113
|
+
const backoff = typeof retry === "number" ? undefined : retry.backoff;
|
|
114
|
+
if (backoff !== undefined) {
|
|
115
|
+
jobsOptions.backoff =
|
|
116
|
+
typeof backoff === "number"
|
|
117
|
+
? backoff
|
|
118
|
+
: { type: backoff.type ?? "fixed", delay: backoff.delay };
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
return jobsOptions;
|
|
122
|
+
}
|
|
123
|
+
/**
|
|
124
|
+
* One offset per underlying state, because BullMQ ranges over each separately.
|
|
125
|
+
*
|
|
126
|
+
* A single number would mean "position within each state", which skips and
|
|
127
|
+
* repeats jobs the moment a status spans more than one of them.
|
|
128
|
+
*/
|
|
129
|
+
export function decodeCursor(cursor, states) {
|
|
130
|
+
if (cursor === undefined)
|
|
131
|
+
return new Array(states).fill(0);
|
|
132
|
+
const offsets = cursor.split(".").map((part) => Number.parseInt(part, 10));
|
|
133
|
+
const usable = offsets.length === states &&
|
|
134
|
+
offsets.every((offset) => Number.isInteger(offset) && offset >= 0);
|
|
135
|
+
if (!usable)
|
|
136
|
+
throw new TypeError("Invalid list cursor");
|
|
137
|
+
return offsets;
|
|
138
|
+
}
|
|
139
|
+
export function encodeCursor(offsets) {
|
|
140
|
+
return offsets.join(".");
|
|
141
|
+
}
|
|
142
|
+
/**
|
|
143
|
+
* Take one page's worth across the states, and say where each got to.
|
|
144
|
+
*
|
|
145
|
+
* Pure on purpose. This is the arithmetic a single-number cursor got wrong, and
|
|
146
|
+
* leaving it inside the async method that fetches the states put it out of
|
|
147
|
+
* reach of every test that runs without a server.
|
|
148
|
+
*
|
|
149
|
+
* Offsets arrive attached to their items rather than as a second array: they
|
|
150
|
+
* are only ever meaningful in pairs, and two arrays could disagree in length
|
|
151
|
+
* with nothing to catch it.
|
|
152
|
+
*/
|
|
153
|
+
export function mergePage(pages, limit) {
|
|
154
|
+
const items = [];
|
|
155
|
+
const next = [];
|
|
156
|
+
for (const page of pages) {
|
|
157
|
+
let taken = 0;
|
|
158
|
+
for (const item of page.items) {
|
|
159
|
+
if (items.length === limit)
|
|
160
|
+
break;
|
|
161
|
+
items.push(item);
|
|
162
|
+
taken += 1;
|
|
163
|
+
}
|
|
164
|
+
next.push(page.offset + taken);
|
|
165
|
+
}
|
|
166
|
+
return { items, next };
|
|
167
|
+
}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Where cancellations are recorded.
|
|
3
|
+
*
|
|
4
|
+
* BullMQ has no cancelled state, and cancelling a job that has not started
|
|
5
|
+
* removes it — so without a marker the only evidence is gone and `refresh()`
|
|
6
|
+
* cannot tell "cancelled" from "never existed". A process-local Set was the
|
|
7
|
+
* obvious first answer and the wrong one: it made a job's fate depend on which
|
|
8
|
+
* process asked, and lost it entirely on restart.
|
|
9
|
+
*
|
|
10
|
+
* What is stored is the finished snapshot itself, not a reason beside a copy of
|
|
11
|
+
* one: it is the only record left once the job is gone, and everything a reader
|
|
12
|
+
* wants is already a field of `JobSnapshot`. Wrapping it meant three records of
|
|
13
|
+
* one event that had to agree, reassembled differently by each reader.
|
|
14
|
+
*/
|
|
15
|
+
import type { Queue } from "bullmq";
|
|
16
|
+
import type { JobSnapshot } from "./types.js";
|
|
17
|
+
export declare class CancellationMarkers {
|
|
18
|
+
private readonly queue;
|
|
19
|
+
private readonly key;
|
|
20
|
+
constructor(queue: Queue);
|
|
21
|
+
write(id: string, snapshot: JobSnapshot): Promise<void>;
|
|
22
|
+
read(id: string): Promise<JobSnapshot | undefined>;
|
|
23
|
+
/** Markers outlive their jobs otherwise, so the hash grows forever. */
|
|
24
|
+
prune(threshold: number): Promise<void>;
|
|
25
|
+
}
|