enqiu 0.1.2 → 0.4.0

package/dist/markers.js

@@ -0,0 +1,51 @@
+/**
+ * Where cancellations are recorded.
+ *
+ * BullMQ has no cancelled state, and cancelling a job that has not started
+ * removes it — so without a marker the only evidence is gone and `refresh()`
+ * cannot tell "cancelled" from "never existed". A process-local Set was the
+ * obvious first answer and the wrong one: it made a job's fate depend on which
+ * process asked, and lost it entirely on restart.
+ *
+ * What is stored is the finished snapshot itself, not a reason beside a copy of
+ * one: it is the only record left once the job is gone, and everything a reader
+ * wants is already a field of `JobSnapshot`. Wrapping it meant three records of
+ * one event that had to agree, reassembled differently by each reader.
+ */
+import { backendClient } from "./backend.js";
+/** Deleted in batches: one HDEL naming every field can exceed Redis' limit. */
+const DELETE_BATCH = 500;
+export class CancellationMarkers {
+    queue;
+    key;
+    constructor(queue) {
+        this.queue = queue;
+        this.key = queue.toKey("enqiu:cancelled");
+    }
+    async write(id, snapshot) {
+        const client = await backendClient(this.queue);
+        await client.hset(this.key, { [id]: JSON.stringify(snapshot) });
+    }
+    async read(id) {
+        const client = await backendClient(this.queue);
+        const raw = await client.hget(this.key, id);
+        return raw ? JSON.parse(raw) : undefined;
+    }
+    /** Markers outlive their jobs otherwise, so the hash grows forever. */
+    async prune(threshold) {
+        const client = await backendClient(this.queue);
+        const stale = Object.entries(await client.hgetall(this.key))
+            .filter(([, raw]) => {
+            try {
+                return (JSON.parse(raw).finishedAt ?? 0) <= threshold;
+            }
+            catch {
+                return true;
+            }
+        })
+            .map(([id]) => id);
+        for (let from = 0; from < stale.length; from += DELETE_BATCH) {
+            await client.hdel(this.key, ...stale.slice(from, from + DELETE_BATCH));
+        }
+    }
+}

package/dist/runner.d.ts

@@ -0,0 +1,20 @@
+/**
+ * Running one job: expiry, the per-attempt deadline, and the handler's context.
+ *
+ * These are the two policies Enqiu enforces itself, because BullMQ has neither
+ * and both cost nothing to add around the handler. Everything else about
+ * execution — claiming, retrying, stalling, locking — is BullMQ's.
+ */
+import { type Job as BullJob } from "bullmq";
+import type { RuntimeDefinition } from "./definition.js";
+export interface RunnerOptions {
+    /** Queue-wide deadline, overridden per job. */
+    readonly timeout: number | undefined;
+}
+export declare class JobRunner {
+    private readonly definitions;
+    private readonly options;
+    constructor(definitions: ReadonlyMap<string, RuntimeDefinition>, options: RunnerOptions);
+    run(bull: BullJob, external?: AbortSignal): Promise<unknown>;
+    private createContext;
+}

package/dist/runner.js

@@ -0,0 +1,101 @@
+/**
+ * Running one job: expiry, the per-attempt deadline, and the handler's context.
+ *
+ * These are the two policies Enqiu enforces itself, because BullMQ has neither
+ * and both cost nothing to add around the handler. Everything else about
+ * execution — claiming, retrying, stalling, locking — is BullMQ's.
+ */
+import { UnrecoverableError } from "bullmq";
+import { JobTimeoutError, encodeFailure } from "./errors.js";
+import { assertJobValue } from "./serialize.js";
+/** For handlers with no deadline, whose only abort source is BullMQ's own. */
+const neverAborts = new AbortController().signal;
+function validateProgress(progress) {
+    if (!Number.isFinite(progress.completed) ||
+        !Number.isFinite(progress.total) ||
+        progress.completed < 0 ||
+        progress.total <= 0 ||
+        progress.completed > progress.total) {
+        throw new RangeError("Progress requires 0 <= completed <= total and total > 0");
+    }
+}
+export class JobRunner {
+    definitions;
+    options;
+    constructor(definitions, options) {
+        this.definitions = definitions;
+        this.options = options;
+    }
+    async run(bull, external) {
+        const id = String(bull.id);
+        const definition = this.definitions.get(bull.name);
+        if (!definition) {
+            throw new UnrecoverableError(`No handler registered for job "${bull.name}"`);
+        }
+        const { expiresIn } = definition.policy;
+        if (expiresIn !== undefined && Date.now() - bull.timestamp > expiresIn) {
+            // Expiry is a property of the job, not of this attempt, so retrying
+            // cannot help — UnrecoverableError stops BullMQ retrying it.
+            throw new UnrecoverableError(encodeFailure({ kind: "expired", jobId: id }));
+        }
+        const timeout = definition.policy.timeout ?? this.options.timeout;
+        if (timeout === undefined) {
+            // BullMQ's signal already aborts on cancellation, and with no deadline to
+            // merge in there is nothing left for a second controller to do.
+            const context = this.createContext(bull, external ?? neverAborts);
+            return assertJobValue(await definition.run(bull.data, context));
+        }
+        // One signal for the handler, whichever reason fires first: this queue's
+        // own deadline, or a cancellation delivered through BullMQ.
+        const controller = new AbortController();
+        const signal = external
+            ? AbortSignal.any([external, controller.signal])
+            : controller.signal;
+        const execution = Promise.resolve(definition.run(bull.data, this.createContext(bull, signal)));
+        let timer;
+        try {
+            return assertJobValue(await Promise.race([
+                execution,
+                new Promise((_, reject) => {
+                    timer = setTimeout(() => {
+                        // The handler is told what happened; BullMQ is told in a form
+                        // that survives being stored as a single string.
+                        controller.abort(new JobTimeoutError(id, timeout));
+                        reject(new Error(encodeFailure({ kind: "timeout", jobId: id, timeout })));
+                    }, timeout);
+                }),
+            ]));
+        }
+        finally {
+            if (timer !== undefined)
+                clearTimeout(timer);
+        }
+    }
+    createContext(bull, signal) {
+        const id = String(bull.id);
+        const write = (level, message, fields) => {
+            if (!message)
+                throw new TypeError("Job log messages must not be empty");
+            void bull
+                .log(JSON.stringify({ level, message, fields, at: Date.now() }))
+                .catch(() => undefined);
+        };
+        const log = {
+            debug: (m, f) => write("debug", m, f),
+            info: (m, f) => write("info", m, f),
+            warn: (m, f) => write("warn", m, f),
+            error: (m, f) => write("error", m, f),
+        };
+        return {
+            id,
+            name: bull.name,
+            attempt: bull.attemptsMade + 1,
+            signal,
+            log,
+            reportProgress: async (progress) => {
+                validateProgress(progress);
+                await bull.updateProgress(assertJobValue(progress));
+            },
+        };
+    }
+}

package/dist/serialize.d.ts

@@ -0,0 +1,15 @@
+/**
+ * The check that a job's value can survive the trip through the queue.
+ *
+ * BullMQ serialises on submit, so copying the value first would be redundant —
+ * nothing the caller does afterwards can reach the stored job. What is worth
+ * paying for is the error: `JSON.stringify` says "Converting circular structure
+ * to JSON" once the value is already on its way to Redis, while this names the
+ * exact path that cannot be stored.
+ */
+export declare class JobSerializationError extends TypeError {
+    readonly path: string;
+    constructor(path: string, reason: string);
+}
+/** Checks a value is storable and returns it unchanged. */
+export declare function assertJobValue<T>(value: T): T;

package/dist/serialize.js

@@ -0,0 +1,90 @@
+/**
+ * The check that a job's value can survive the trip through the queue.
+ *
+ * BullMQ serialises on submit, so copying the value first would be redundant —
+ * nothing the caller does afterwards can reach the stored job. What is worth
+ * paying for is the error: `JSON.stringify` says "Converting circular structure
+ * to JSON" once the value is already on its way to Redis, while this names the
+ * exact path that cannot be stored.
+ */
+export class JobSerializationError extends TypeError {
+    path;
+    constructor(path, reason) {
+        super(`Job data at ${path} is not JSON-safe: ${reason}`);
+        this.name = "JobSerializationError";
+        this.path = path;
+    }
+}
+/** Checks a value is storable and returns it unchanged. */
+export function assertJobValue(value) {
+    assertJsonSafe(value, [], new Set());
+    return value;
+}
+/**
+ * `["a", 0, "b"]` reads back as `$.a[0].b`.
+ *
+ * The path is carried as one array pushed and popped down the walk, and only
+ * rendered when something is about to throw. Building the string at every node
+ * cost more than the rest of the check put together.
+ */
+function fail(path, reason) {
+    let rendered = "$";
+    for (const segment of path) {
+        rendered += typeof segment === "number" ? `[${segment}]` : `.${segment}`;
+    }
+    throw new JobSerializationError(rendered, reason);
+}
+function assertJsonSafe(value, path, ancestors) {
+    if (value === null ||
+        typeof value === "string" ||
+        typeof value === "boolean") {
+        return;
+    }
+    if (value === undefined) {
+        if (path.length > 0) {
+            fail(path, "undefined is only valid as a root value");
+        }
+        return;
+    }
+    if (typeof value === "number") {
+        if (!Number.isFinite(value))
+            fail(path, "numbers must be finite");
+        return;
+    }
+    if (typeof value === "bigint" ||
+        typeof value === "function" ||
+        typeof value === "symbol") {
+        fail(path, `${typeof value} is unsupported`);
+    }
+    if (typeof value !== "object")
+        fail(path, "unsupported value");
+    if (ancestors.has(value))
+        fail(path, "circular reference");
+    ancestors.add(value);
+    if (Array.isArray(value)) {
+        for (let index = 0; index < value.length; index += 1) {
+            path.push(index);
+            if (!(index in value)) {
+                fail(path, "sparse array entries are unsupported");
+            }
+            assertJsonSafe(value[index], path, ancestors);
+            path.pop();
+        }
+        ancestors.delete(value);
+        return;
+    }
+    const prototype = Object.getPrototypeOf(value);
+    if (prototype !== Object.prototype && prototype !== null) {
+        fail(path, "only plain objects and arrays are supported");
+    }
+    const entries = value;
+    for (const key of Object.keys(entries)) {
+        path.push(key);
+        if (entries[key] === undefined) {
+            fail(path, "undefined object fields are unsupported");
+        }
+        assertJsonSafe(entries[key], path, ancestors);
+        path.pop();
+    }
+    ancestors.delete(value);
+}

package/dist/types.d.ts

@@ -0,0 +1,326 @@
+/**
+ * The public vocabulary of `enqiu()`.
+ *
+ * Enqiu is a typed layer over BullMQ: it owns inference and validation, BullMQ
+ * owns storage and execution. Anything BullMQ's open-source tier cannot
+ * express is absent here rather than faked — see the compatibility notes in
+ * the README.
+ */
+import type { ConnectionOptions, Queue, Worker } from "bullmq";
+import type { SerializedError, StandardSchemaIssue } from "./errors.js";
+export type { SerializedError, StandardSchemaIssue } from "./errors.js";
+export type MaybePromise<T> = T | PromiseLike<T>;
+export declare const definitionMarker: unique symbol;
+export type JobStatus = "queued" | "scheduled" | "running" | "succeeded" | "failed" | "cancelled";
+export interface StandardSchemaV1<Input = unknown, Output = Input> {
+    readonly "~standard": {
+        readonly version: 1;
+        readonly vendor: string;
+        readonly validate: (value: unknown) => MaybePromise<{
+            readonly value: Output;
+            readonly issues?: undefined;
+        } | {
+            readonly value?: undefined;
+            readonly issues: readonly StandardSchemaIssue[];
+        }>;
+        readonly types?: {
+            readonly input: Input;
+            readonly output: Output;
+        } | undefined;
+    };
+}
+export type InferSchemaInput<Schema extends StandardSchemaV1> = NonNullable<Schema["~standard"]["types"]>["input"];
+export type InferSchemaOutput<Schema extends StandardSchemaV1> = NonNullable<Schema["~standard"]["types"]>["output"];
+export interface Progress {
+    readonly completed: number;
+    readonly total: number;
+    readonly message?: string;
+    readonly details?: Readonly<Record<string, unknown>>;
+}
+export interface JobLogger {
+    debug(message: string, fields?: Readonly<Record<string, unknown>>): void;
+    info(message: string, fields?: Readonly<Record<string, unknown>>): void;
+    warn(message: string, fields?: Readonly<Record<string, unknown>>): void;
+    error(message: string, fields?: Readonly<Record<string, unknown>>): void;
+}
+export interface JobContext<Name extends string = string> {
+    readonly id: string;
+    readonly name: Name;
+    readonly attempt: number;
+    readonly signal: AbortSignal;
+    reportProgress(progress: Progress): Promise<void>;
+    readonly log: JobLogger;
+}
+export type JobHandler<Input = unknown, Output = unknown, Name extends string = string> = (input: Input, context: JobContext<Name>) => MaybePromise<Output>;
+export interface BackoffOptions {
+    type?: "fixed" | "exponential";
+    delay: number;
+}
+export interface RetryPolicy {
+    /** Total number of attempts, including the first. */
+    attempts: number;
+    backoff?: number | BackoffOptions;
+}
+/**
+ * Per-job policies.
+ *
+ * `timeout` and `expiresIn` live here rather than on a submission because
+ * Enqiu enforces them worker-side, and BullMQ's job options have no field to
+ * carry a per-submission value across the queue. Keyed concurrency, keyed
+ * throttling and debounce are absent: BullMQ's open-source tier offers only a
+ * global per-worker rate limit, and per-key grouping is a Pro feature.
+ */
+export interface JobPolicyOptions {
+    retry?: number | RetryPolicy;
+    /** Per-attempt deadline. Aborts the handler's signal and fails the attempt. */
+    timeout?: number;
+    /** Fail without running if the job has waited longer than this. */
+    expiresIn?: number;
+}
+export interface SchemaJobDefinition<Schema extends StandardSchemaV1 = StandardSchemaV1, Output = unknown> extends JobPolicyOptions {
+    readonly [definitionMarker]: true;
+    readonly input: Schema;
+    readonly run: JobHandler<InferSchemaOutput<Schema>, Output>;
+}
+export type HandlerJobDefinition<Input = unknown, Output = unknown> = JobHandler<Input, Output>;
+/**
+ * `any` is required in the schema position. JobHandler is contravariant in its
+ * input and the generic is invariant in Schema, so pinning it rejects every
+ * real schema and collapses the inferred API to `unknown`.
+ */
+export type JobDefinition = SchemaJobDefinition<any, any> | HandlerJobDefinition<never, unknown>;
+export type JobDefinitions = Record<string, JobDefinition>;
+/**
+ * Everything inferred from one definition, worked out once.
+ *
+ * A schema job and a bare handler differ in four derived facts, and writing
+ * each as its own conditional meant restating "which of the two is this" four
+ * times. They are not independent: a third form of definition would have to be
+ * added to all of them or to none.
+ *
+ * `input` is what a caller passes and `runInput` is what the handler receives.
+ * They differ only for a schema that transforms — `z.string().transform(Number)`
+ * takes a string and hands the handler a number.
+ */
+type DefinitionShape<Definition> = Definition extends SchemaJobDefinition<infer Schema, infer Output> ? {
+    input: InferSchemaInput<Schema>;
+    runInput: InferSchemaOutput<Schema>;
+    output: Awaited<Output>;
+    schema: Schema;
+} : Definition extends JobHandler<infer Input, infer Output, string> ? {
+    input: Input;
+    runInput: Input;
+    output: Awaited<Output>;
+    schema: undefined;
+} : {
+    input: never;
+    runInput: never;
+    output: never;
+    schema: undefined;
+};
+type DefinitionInput<Definition> = DefinitionShape<Definition>["input"];
+type DefinitionRunInput<Definition> = DefinitionShape<Definition>["runInput"];
+type DefinitionOutput<Definition> = DefinitionShape<Definition>["output"];
+type DefinitionSchema<Definition> = DefinitionShape<Definition>["schema"];
+export interface SubmitOptions {
+    id?: string;
+    /** Reuses an existing job with the same key instead of creating a new one. */
+    idempotencyKey?: string;
+    idempotencyTtl?: number;
+    delay?: number | Date;
+    priority?: number | "low" | "normal" | "high";
+    retry?: number | RetryPolicy;
+}
+/** One key across a batch would collapse it into a single job, so it is out. */
+export interface BulkOptions extends Omit<SubmitOptions, "id" | "idempotencyKey" | "idempotencyTtl"> {
+    ids?: readonly string[];
+}
+export interface ScheduleOptions<Input> {
+    id?: string;
+    cron: string;
+    timezone?: string;
+    input: Input;
+}
+export interface ScheduleSnapshot {
+    id: string;
+    jobName: string;
+    cron: string;
+    timezone: string;
+    nextRunAt: number;
+    input: unknown;
+}
+export interface ScheduleHandle {
+    readonly id: string;
+    remove(): Promise<void>;
+    refresh(): Promise<ScheduleSnapshot>;
+}
+export interface JobSnapshot<Input = unknown, Output = unknown, Name extends string = string> {
+    id: string;
+    name: Name;
+    input: Input;
+    status: JobStatus;
+    attempt: number;
+    createdAt: number;
+    startedAt?: number | undefined;
+    finishedAt?: number | undefined;
+    progress?: unknown;
+    output?: Output | undefined;
+    error?: SerializedError | undefined;
+}
+/**
+ * What a submission gives you back.
+ *
+ * There is no `status` field. It could only ever report what was true when the
+ * handle was made, which is a value that goes quietly wrong while you hold it.
+ * `refresh()` answers the question at the moment you ask.
+ */
+export interface JobHandle<Output = unknown, Input = unknown, Name extends string = string> {
+    readonly id: string;
+    readonly name: Name;
+    readonly input: Input;
+    readonly deduplicated: boolean;
+    readonly result: Promise<Output>;
+    cancel(reason?: string): Promise<boolean>;
+    refresh(): Promise<JobSnapshot<Input, Output, Name>>;
+}
+export interface JobCallable<Input, RunInput, Output, Name extends string, Schema extends StandardSchemaV1 | undefined = undefined> {
+    (input: Input, options?: SubmitOptions): Promise<JobHandle<Output, RunInput, Name>>;
+    bulk(inputs: readonly Input[], options?: BulkOptions): Promise<Array<JobHandle<Output, RunInput, Name>>>;
+    schedule(options: ScheduleOptions<Input>): Promise<ScheduleHandle>;
+    readonly input: Schema;
+}
+export interface QueueStats {
+    queued: number;
+    scheduled: number;
+    running: number;
+    succeeded: number;
+    failed: number;
+    total: number;
+}
+/**
+ * `status` is required.
+ *
+ * BullMQ ranges over each underlying state separately, so an offset across a
+ * merged set of states cannot mean "position in the result" — it would skip
+ * and repeat jobs. Naming the status keeps the cursor honest.
+ */
+export interface JobListQuery {
+    status: JobStatus;
+    limit?: number;
+    cursor?: string;
+}
+export interface JobListPage<Job = JobSnapshot> {
+    jobs: Job[];
+    cursor?: string;
+}
+export interface CleanupQuery {
+    status?: JobStatus;
+    olderThan?: number;
+    limit?: number;
+}
+export type AnyJobSnapshot<Definitions extends JobDefinitions> = {
+    [Name in keyof Definitions]: JobSnapshot<DefinitionRunInput<Definitions[Name]>, DefinitionOutput<Definitions[Name]>, Extract<Name, string>>;
+}[keyof Definitions];
+export interface QueueEventMap {
+    added: JobSnapshot;
+    started: JobSnapshot;
+    progress: JobSnapshot;
+    succeeded: JobSnapshot;
+    failed: JobSnapshot;
+    error: Error;
+}
+/**
+ * Only what Enqiu types or computes. Pausing, resuming and setting global
+ * concurrency are `bull.queue.pause()`, `.resume()` and
+ * `.setGlobalConcurrency()`; re-exporting them under new names would be a
+ * second vocabulary for the same call.
+ */
+export interface QueueApi<Definitions extends JobDefinitions> {
+    get(id: string): Promise<AnyJobSnapshot<Definitions> | undefined>;
+    list(query: JobListQuery): Promise<JobListPage<AnyJobSnapshot<Definitions>>>;
+    stats(): Promise<QueueStats>;
+    redrive(id: string): Promise<JobHandle>;
+    cleanup(query?: CleanupQuery): Promise<string[]>;
+    /**
+     * Resolves once nothing is waiting, running, delayed or prioritized.
+     *
+     * A property of the queue, not of any one worker: every process sharing the
+     * queue sees the same answer, and a producer that keeps submitting will keep
+     * this pending.
+     */
+    onIdle(): Promise<void>;
+    on<Event extends keyof QueueEventMap>(event: Event, listener: (payload: QueueEventMap[Event]) => void): () => void;
+}
+export interface WorkerStartOptions {
+    concurrency?: number;
+}
+/**
+ * Only what needs logic. Pausing and resuming a worker is `bull.worker.pause()`
+ * and `bull.worker.resume()` — forwarding those through a second name adds a
+ * thing to learn and nothing else.
+ */
+export interface WorkerApi {
+    readonly running: boolean;
+    start(options?: WorkerStartOptions): Promise<void>;
+}
+export interface WorkerOptions {
+    concurrency?: number;
+    autoStart?: boolean;
+}
+export interface EnqiuOptions {
+    /** Queue name. @default "default" */
+    name?: string;
+    /** Redis connection, passed straight to BullMQ. */
+    connection: ConnectionOptions;
+    /** Redis key namespace. */
+    prefix?: string;
+    /** Run handlers in this process. `false` makes it producer-only. */
+    worker?: false | WorkerOptions;
+    /** Queue-wide defaults, overridden per job and (for `retry`) per submission. */
+    retry?: number | RetryPolicy;
+    timeout?: number;
+    /**
+     * Structured log lines retained per job. Read them back with
+     * `bull.queue.getJobLogs(id)` — an extra round trip Enqiu does not make on
+     * your behalf. @default 100
+     */
+    logLimit?: number;
+}
+/** Your jobs, and nothing else. */
+export type JobsApi<Definitions extends JobDefinitions> = {
+    readonly [Name in keyof Definitions]: JobCallable<DefinitionInput<Definitions[Name]>, DefinitionRunInput<Definitions[Name]>, DefinitionOutput<Definitions[Name]>, Extract<Name, string>, DefinitionSchema<Definitions[Name]>>;
+};
+/**
+ * What `enqiu()` returns, meant to be destructured.
+ *
+ * Keeping the queue and worker controls beside your jobs rather than among
+ * them is why no job name is reserved: `jobs.queue` is a job called `queue`,
+ * and nothing here has to police the difference.
+ */
+export interface Enqiu<Definitions extends JobDefinitions> {
+    readonly jobs: JobsApi<Definitions>;
+    readonly queue: QueueApi<Definitions>;
+    readonly worker: WorkerApi;
+    /**
+     * The BullMQ objects underneath, unwrapped.
+     *
+     * Enqiu's own surface is the typed, validated path. Everything BullMQ can do
+     * that Enqiu does not model — flows, Pro groups, metrics, raw options — is
+     * reachable here without a fork, and calls through it cost nothing extra
+     * because there is no layer in the way.
+     */
+    readonly bull: {
+        readonly queue: Queue;
+        readonly worker: Worker | undefined;
+    };
+    /**
+     * Shuts down the queue, the worker and any open event stream.
+     *
+     * `drain` waits for outstanding work first. It lives here rather than on
+     * `worker` because it ends all three, including on a producer-only queue
+     * that has no worker at all.
+     */
+    close(options?: {
+        drain?: boolean;
+    }): Promise<void>;
+}

package/dist/types.js

@@ -0,0 +1,9 @@
+/**
+ * The public vocabulary of `enqiu()`.
+ *
+ * Enqiu is a typed layer over BullMQ: it owns inference and validation, BullMQ
+ * owns storage and execution. Anything BullMQ's open-source tier cannot
+ * express is absent here rather than faked — see the compatibility notes in
+ * the README.
+ */
+export const definitionMarker = Symbol("enqiu.job");