gemi 0.116.0 → 0.118.0

package/dist/services/queue/DatabaseQueueDriver.d.ts

@@ -1,13 +1,30 @@
 import type { DatabaseConnection } from "../../database/Connection";
 import type { Dialect } from "../../database/dialect";
 import { DatabaseLockStore } from "../lock/DatabaseLockStore";
-import type { ClaimOptions, ClaimedJob, EnqueueJob, JobFailure, JobRelease, QueueDriver } from "./QueueDriver";
+import { type BatchStatus } from "./batch";
+import type { ClaimOptions, ClaimedJob, EnqueueBatch, EnqueueJob, JobFailure, JobRelease, QueueDriver } from "./QueueDriver";
+import { DatabaseWorkflowStore } from "./workflow/DatabaseWorkflowStore";
 export type DatabaseQueueDriverOptions = {
     /**
      * The jobs table. Default `gemi_jobs`. A plain identifier — letters, digits
      * and underscores — because it is spliced into every statement.
      */
     table?: string;
+    /**
+     * The table batches are kept in. Default `gemi_job_batches` beside
+     * `gemi_jobs`, and `<table>_batches` beside a jobs table of another name.
+     * A plain identifier, like `table`. Only touched by `Job.dispatchBatch` and
+     * the batch methods.
+     */
+    batchTable?: string;
+    /**
+     * The tables workflows are kept in. Default `gemi_workflows` and
+     * `gemi_workflow_steps` beside `gemi_jobs`, and `<table>_workflows` and
+     * `<table>_workflow_steps` beside a jobs table of another name. Plain
+     * identifiers, like `table`. Only touched by `Workflow`.
+     */
+    workflowTable?: string;
+    workflowStepTable?: string;
     /**
      * SQLite only: how long, in milliseconds, a statement waits for another
      * process's write lock before failing with `SQLITE_BUSY`. Default `1000`.
@@ -72,6 +89,15 @@ export type DatabaseJobStatus = "pending" | "claimed" | "dead";
  * `transaction` below for why SQLite is left out, and why a driver built from
  * a bare client never joins.
  *
+ * ### Batches
+ *
+ * A batch is a row of `batchTable`, and its jobs are rows of the jobs table
+ * with `batch_id` set. Ending a job of a batch, counting it and enqueueing any
+ * callback it makes due happen in one transaction that locks the batch row
+ * first; see the "Batches" section below. A table from before batches, without
+ * `batch_id` and `progress`, still runs every other job: the claim reads `*`,
+ * and only the batch methods need the new columns.
+ *
  * ### What it does not do
  *
  * It has no `subscribe`: another process's dispatch cannot wake this one, so
@@ -80,12 +106,18 @@ export type DatabaseJobStatus = "pending" | "claimed" | "dead";
  */
 export declare class DatabaseQueueDriver implements QueueDriver {
     readonly table: string;
+    readonly batchTable: string;
+    readonly workflowTable: string;
+    readonly workflowStepTable: string;
+    private workflows;
     private readonly sql;
     private readonly dialect;
     /** The connection's name, when it came with one; see `transaction`. */
     private readonly connection;
     private readonly busyTimeout;
     private configured;
+    /** SQLite: the transaction before the next; see `atomically`. */
+    private sqliteTail;
     constructor(connection: Pick<DatabaseConnection, "sql" | "dialect"> & Partial<Pick<DatabaseConnection, "name">>, options?: DatabaseQueueDriverOptions);
     /**
      * Gives a SQLite connection a busy timeout, once, before the driver's first
@@ -118,14 +150,18 @@ export declare class DatabaseQueueDriver implements QueueDriver {
      */
     private configure;
     /**
-     * Creates the table and its indexes if they do not exist — the same DDL
-     * Prisma generates for the model in the docs, so a table made either way is
-     * the table the other expects. For tests and for an app that does not
-     * manage its schema with Prisma; one that does should add the model
-     * instead, or `prisma migrate` will see a table it does not know and drop
-     * it.
+     * Creates the jobs table, the batches table, the two workflow tables and
+     * their indexes if they do not exist — the same DDL Prisma generates for
+     * the models in the docs, so
+     * a table made either way is the table the other expects — and adds the
+     * batch columns to a jobs table from before batches. For tests and for an
+     * app that does not manage its schema with Prisma; one that does should add
+     * the models instead, or `prisma migrate` will see tables it does not know
+     * and drop them.
      */
     createTable(): Promise<void>;
+    /** Whether the jobs table has `batch_id` and `progress` yet. */
+    private hasBatchColumns;
     enqueue(job: EnqueueJob): Promise<string>;
     private insert;
     /**
@@ -182,6 +218,65 @@ export declare class DatabaseQueueDriver implements QueueDriver {
      * a scheduled job, say.
      */
     prune(olderThanMs: number): Promise<number>;
+    enqueueBatch(batch: EnqueueBatch): Promise<void>;
+    private insertBatch;
+    findBatch(id: string, options?: {
+        progress?: boolean;
+    }): Promise<BatchStatus | null>;
+    cancelBatch(id: string): Promise<boolean>;
+    /** `cancelBatch`, inside a transaction the caller has open. */
+    private cancelBatchOn;
+    /**
+     * Workflows (#846), in `workflowTable` and `workflowStepTable` of this
+     * driver's database. Their writes share this driver's transactions, so a
+     * step's record and the tick or batch it schedules commit together.
+     */
+    workflowStore(): DatabaseWorkflowStore;
+    reportProgress(job: ClaimedJob, progress: number): Promise<void>;
+    /**
+     * Deletes batches that finished more than `olderThanMs` ago, and resolves
+     * to how many. Nothing else removes them; a finished batch is only kept so
+     * `findBatch` can still answer for it.
+     */
+    pruneBatches(olderThanMs: number): Promise<number>;
+    /**
+     * `complete` and `fail` for a job of a batch: the job's row is ended and the
+     * batch counts it, in one transaction.
+     */
+    private endBatchJob;
+    /** Counts one job's end against its locked batch row. */
+    private count;
+    /**
+     * Ends the batch's waiting jobs as cancelled — kept as dead rows that say
+     * why — and resolves to how many there were. Their rows are locked after
+     * the batch row, the order every batch transaction takes.
+     */
+    private cancelWaiting;
+    /** Writes the batch's new counters and enqueues the callbacks they made due. */
+    private saveBatch;
+    /**
+     * Reads the batch row and holds it until the transaction ends. Postgres and
+     * MySQL lock the row; SQLite has no row locks, so a write to the row takes
+     * the file's write lock instead, waiting out another process's for the
+     * busy timeout rather than failing on the upgrade from a read lock.
+     */
+    private lockBatch;
+    /**
+     * Runs `fn` in a transaction of the driver's own.
+     *
+     * On SQLite one at a time per driver. Bun gives a SQLite client a single
+     * connection, and a second `begin` on it while one is open is refused with
+     * "cannot start a transaction within a transaction". One the application
+     * has open on the same client — an ORM transaction, when the queue shares
+     * the default connection — is waited out the same way, a few times.
+     */
+    private atomically;
+    /** The database's clock now, as a number, for values computed here. */
+    private clock;
+    /** An epoch-milliseconds value or `NULL`, typed as the BIGINT columns are. */
+    private timeOrNull;
+    /** A 0..1 progress value as a parameter Postgres can assign to its column. */
+    private fraction;
     /**
      * Postgres and SQLite: one statement. Postgres locks the rows it picks with
      * `SKIP LOCKED` in a CTE, which it evaluates once, so a concurrent claimer
@@ -242,6 +337,9 @@ export declare class DatabaseQueueDriver implements QueueDriver {
      * `@@map`) made, and a reserved word such as `order` is a syntax error.
      */
     private name;
+    /** The batches table, quoted as `name` quotes the jobs table. */
+    private batches;
+    private quoted;
     /** The database's clock, in epoch milliseconds. */
     private now;
     /**
@@ -257,4 +355,16 @@ export declare class DatabaseQueueDriver implements QueueDriver {
  * the table, which is also where Prisma puts them.
  */
 export declare function createTableStatements(dialect: Dialect, table: string): string[];
+/**
+ * The two columns and the index batches add to a jobs table made before them,
+ * as Prisma's migration for the updated model writes them. `createTable` runs
+ * these when the columns are missing; an app on Prisma gets the same from
+ * `prisma migrate dev` after updating the model.
+ */
+export declare function addBatchColumnsStatements(dialect: Dialect, table: string): string[];
+/**
+ * The batches table, as Prisma generates it for the `GemiJobBatch` model in
+ * the docs, with `IF NOT EXISTS` added.
+ */
+export declare function createBatchTableStatements(dialect: Dialect, table: string): string[];
 //# sourceMappingURL=DatabaseQueueDriver.d.ts.map

package/dist/services/queue/DatabaseQueueDriver.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"DatabaseQueueDriver.d.ts","sourceRoot":"","sources":["../../../services/queue/DatabaseQueueDriver.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,2BAA2B,CAAC;AACpE,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,wBAAwB,CAAC;AACtD,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAE9D,OAAO,KAAK,EACV,YAAY,EACZ,UAAU,EACV,UAAU,EACV,UAAU,EACV,UAAU,EACV,WAAW,EACZ,MAAM,eAAe,CAAC;AAEvB,MAAM,MAAM,0BAA0B,GAAG;IACvC;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;;;;OAMG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB,CAAC;AAEF;;;;;GAKG;AACH,MAAM,MAAM,iBAAiB,GAAG,SAAS,GAAG,SAAS,GAAG,MAAM,CAAC;AAW/D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqDG;AACH,qBAAa,mBAAoB,YAAW,WAAW;IACrD,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAM;IAC1B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAU;IAClC,uEAAuE;IACvE,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAqB;IAChD,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,UAAU,CAA4B;gBAG5C,UAAU,EAAE,IAAI,CAAC,kBAAkB,EAAE,KAAK,GAAG,SAAS,CAAC,GACrD,OAAO,CAAC,IAAI,CAAC,kBAAkB,EAAE,MAAM,CAAC,CAAC,EAC3C,OAAO,GAAE,0BAA+B;IAsB1C;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA4BG;IACH,OAAO,CAAC,SAAS;IAejB;;;;;;;OAOG;IACG,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC;IAOlC,OAAO,CAAC,GAAG,EAAE,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC;YAgB3B,MAAM;IAepB;;;;OAIG;IACH,SAAS,IAAI,iBAAiB;IAO9B,gBAAgB,IAAI,OAAO;IAI3B;;;;;;;;;;;;;;;;;;OAkBG;IACH,OAAO,CAAC,WAAW;IAOb,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,UAAU,EAAE,CAAC;IA0BlE,QAAQ,CAAC,GAAG,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;IASxC,IAAI,CAAC,GAAG,EAAE,UAAU,EAAE,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;IAqB/D;;;;OAIG;IACG,OAAO,CAAC,GAAG,EAAE,UAAU,EAAE,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;IAYlE;;;OAGG;IACG,SAAS,CAAC,IAAI,EAAE,UAAU,EAAE,EAAE,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;IAgBzE;;;;OAIG;IACG,SAAS,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAa7C;;;;;OAKG;IACG,KAAK,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAUjD;;;;;OAKG;YACW,aAAa;IAsC3B;;;;;;;;;;;;;;OAcG;YACW,YAAY;YA6BZ,cAAc;IAgD5B;;;;OAIG;IACH,OAAO,CAAC,SAAS;IAUjB;;;;;;;;;;;;;;;;;;;;;OAqBG;IACH,OAAO,CAAC,QAAQ;IAahB;;;;;OAKG;IACH,OAAO,CAAC,IAAI;IAKZ,mDAAmD;IACnD,OAAO,CAAC,GAAG;IAcX;;;OAGG;IACH,OAAO,CAAC,EAAE;CAMX;AAYD;;;;;GAKG;AACH,wBAAgB,qBAAqB,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,EAAE,CA2C/E"}
+{"version":3,"file":"DatabaseQueueDriver.d.ts","sourceRoot":"","sources":["../../../services/queue/DatabaseQueueDriver.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,2BAA2B,CAAC;AACpE,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,wBAAwB,CAAC;AACtD,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAE9D,OAAO,EAIL,KAAK,WAAW,EAQjB,MAAM,SAAS,CAAC;AACjB,OAAO,KAAK,EACV,YAAY,EACZ,UAAU,EACV,YAAY,EACZ,UAAU,EACV,UAAU,EACV,UAAU,EACV,WAAW,EACZ,MAAM,eAAe,CAAC;AACvB,OAAO,EACL,qBAAqB,EAEtB,MAAM,kCAAkC,CAAC;AAE1C,MAAM,MAAM,0BAA0B,GAAG;IACvC;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;;;OAKG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;;;;OAKG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B;;;;;;OAMG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB,CAAC;AAEF;;;;;GAKG;AACH,MAAM,MAAM,iBAAiB,GAAG,SAAS,GAAG,SAAS,GAAG,MAAM,CAAC;AAkC/D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8DG;AACH,qBAAa,mBAAoB,YAAW,WAAW;IACrD,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;IACnC,OAAO,CAAC,SAAS,CAAoC;IACrD,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAM;IAC1B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAU;IAClC,uEAAuE;IACvE,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAqB;IAChD,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,UAAU,CAA4B;IAC9C,iEAAiE;IACjE,OAAO,CAAC,UAAU,CAAuC;gBAGvD,UAAU,EAAE,IAAI,CAAC,kBAAkB,EAAE,KAAK,GAAG,SAAS,CAAC,GACrD,OAAO,CAAC,IAAI,CAAC,kBAAkB,EAAE,MAAM,CAAC,CAAC,EAC3C,OAAO,GAAE,0BAA+B;IAmC1C;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA4BG;IACH,OAAO,CAAC,SAAS;IAejB;;;;;;;;;OASG;IACG,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC;IAwBlC,gEAAgE;YAClD,eAAe;IAU7B,OAAO,CAAC,GAAG,EAAE,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC;YAgB3B,MAAM;IAepB;;;;OAIG;IACH,SAAS,IAAI,iBAAiB;IAO9B,gBAAgB,IAAI,OAAO;IAI3B;;;;;;;;;;;;;;;;;;OAkBG;IACH,OAAO,CAAC,WAAW;IAOb,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,UAAU,EAAE,CAAC;IA+BlE,QAAQ,CAAC,GAAG,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;IAUxC,IAAI,CAAC,GAAG,EAAE,UAAU,EAAE,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;IAsB/D;;;;OAIG;IACG,OAAO,CAAC,GAAG,EAAE,UAAU,EAAE,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;IAYlE;;;OAGG;IACG,SAAS,CAAC,IAAI,EAAE,UAAU,EAAE,EAAE,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;IAgBzE;;;;OAIG;IACG,SAAS,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAoB7C;;;;;OAKG;IACG,KAAK,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IA2BjD,YAAY,CAAC,KAAK,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;YAYlC,WAAW;IAwCnB,SAAS,CAAC,EAAE,EAAE,MAAM,EAAE,OAAO,GAAE;QAAE,QAAQ,CAAC,EAAE,OAAO,CAAA;KAAO,GAAG,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC;IAiBxF,WAAW,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAI/C,+DAA+D;YACjD,aAAa;IAS3B;;;;OAIG;IACH,aAAa,IAAI,qBAAqB;IAwBhC,cAAc,CAAC,GAAG,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAUtE;;;;OAIG;IACG,YAAY,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAUxD;;;OAGG;IACH,OAAO,CAAC,WAAW;IAuCnB,yDAAyD;YAC3C,KAAK;IASnB;;;;OAIG;YACW,aAAa;IAU3B,gFAAgF;YAClE,SAAS;IAavB;;;;;OAKG;YACW,SAAS;IAWvB;;;;;;;;OAQG;IACH,OAAO,CAAC,UAAU;IAwBlB,uEAAuE;YACzD,KAAK;IAKnB,8EAA8E;IAC9E,OAAO,CAAC,UAAU;IAIlB,8EAA8E;IAC9E,OAAO,CAAC,QAAQ;IAIhB;;;;;OAKG;YACW,aAAa;IAsC3B;;;;;;;;;;;;;;OAcG;YACW,YAAY;YA6BZ,cAAc;IAgD5B;;;;OAIG;IACH,OAAO,CAAC,SAAS;IAUjB;;;;;;;;;;;;;;;;;;;;;OAqBG;IACH,OAAO,CAAC,QAAQ;IAahB;;;;;OAKG;IACH,OAAO,CAAC,IAAI;IAIZ,iEAAiE;IACjE,OAAO,CAAC,OAAO;IAIf,OAAO,CAAC,MAAM;IAKd,mDAAmD;IACnD,OAAO,CAAC,GAAG;IAcX;;;OAGG;IACH,OAAO,CAAC,EAAE;CAMX;AAYD;;;;;GAKG;AACH,wBAAgB,qBAAqB,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,EAAE,CAiD/E;AAED;;;;;GAKG;AACH,wBAAgB,yBAAyB,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,EAAE,CAqBnF;AAED;;;GAGG;AACH,wBAAgB,0BAA0B,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,EAAE,CA2CpF"}

package/dist/services/queue/Job.d.ts

@@ -1,3 +1,24 @@
+import { type BatchStatus, type JobCall } from "./batch";
+import type { ClaimedJob } from "./QueueDriver";
+import { type DispatchedBatch, type JobBatchOptions } from "./QueueManager";
+/** `this.batch` inside a job dispatched with `dispatchBatch`. */
+export type JobBatch = {
+    /** The batch's id. */
+    id: string;
+    /**
+     * Whether the batch has been cancelled — by `Job.cancelBatch`, or by a
+     * failed job of a batch without `allowFailures`. Running jobs are not
+     * stopped; a long one checks this between steps and returns early.
+     */
+    cancelled(): Promise<boolean>;
+};
+/**
+ * The arguments `Job.with` takes for a job whose `run` ends with a
+ * `BatchStatus` parameter: the ones before it, because the batch appends its
+ * status when it enqueues the callback. Any other `run` takes its parameters
+ * as they are.
+ */
+export type JobCallArgs<A extends any[]> = A extends [...infer Rest, infer Last] ? BatchStatus extends Last ? Rest : A : A;
 export declare class Job {
     static name: string;
     worker: boolean;
@@ -55,6 +76,22 @@ export declare class Job {
     fail(error: unknown, options?: {
         retry?: boolean;
     }): void;
+    /**
+     * The batch this run belongs to, when the job was dispatched with
+     * `dispatchBatch`; `undefined` otherwise. Set by the queue before `run`.
+     */
+    batch: JobBatch | undefined;
+    /**
+     * Called from `run` in a job of a batch: how far this job is, from 0 to 1,
+     * for the batch's `progress`. A retry starts again from 0, and a job that
+     * ends counts as 1 whatever it last said. Outside a batch it does nothing.
+     * Each call is a write, so report at steps, not per item of a hot loop.
+     */
+    progress(value: number): Promise<void>;
+    /** @internal Where `progress` writes; set by the queue for a job of a batch. */
+    $progress: ((value: number) => Promise<void>) | undefined;
+    /** @internal The claim this run is, set by the queue before `run`. */
+    $claimed: ClaimedJob | undefined;
     /** @internal What `release` or `fail` asked for during this run. */
     $outcome: {
         kind: "release";
@@ -78,6 +115,44 @@ export declare class Job {
      * that can, an ignored rejection is an unhandled one, so await it.
      */
     static dispatch<T extends Job>(this: new () => T, ...args: Parameters<T["run"]>): Promise<string>;
+    /**
+     * Queues one job per argument tuple as a batch, and resolves to the batch's
+     * id and size once every job is recorded. The jobs, the batch and its
+     * callbacks are recorded in one atomic write: all of them or none.
+     *
+     * ```ts
+     * const { id } = await BuildPageJob.dispatchBatch(
+     *   pages.map((page) => [page.id, importId] as const),
+     *   {
+     *     name: `import:${importId}`,
+     *     allowFailures: true,
+     *     then: ImportFinishedJob.with(importId),
+     *     catch: ImportFailedJob.with(importId),
+     *     finally: ImportCleanupJob.with(importId),
+     *   },
+     * );
+     * ```
+     *
+     * Callbacks are jobs, enqueued exactly once, by whichever process ends the
+     * job that makes them due, with the batch's status appended to their
+     * arguments. Each job's own hooks run as usual. A worker job, a unique job
+     * and a driver without batches are refused here, on the caller's stack.
+     */
+    static dispatchBatch<T extends Job>(this: new () => T, args: ReadonlyArray<Readonly<Parameters<T["run"]>>>, options?: JobBatchOptions): Promise<DispatchedBatch>;
+    /**
+     * A job and its arguments, without dispatching it: a batch's `then`,
+     * `catch` or `finally`. When the job runs, the batch's `BatchStatus` is
+     * appended to these arguments.
+     */
+    static with<T extends Job>(this: new () => T, ...args: JobCallArgs<Parameters<T["run"]>>): JobCall;
+    /** A batch's status, or `null` for an id the queue has no batch under. */
+    static findBatch(id: string): Promise<BatchStatus | null>;
+    /**
+     * Cancels a batch that is still running, and resolves to whether it did.
+     * Its waiting jobs never run; running ones finish unless they check
+     * `this.batch.cancelled()`. `catch` and then `finally` run; `then` does not.
+     */
+    static cancelBatch(id: string): Promise<boolean>;
 }
 export type JobThrottle = {
     /** Shared by every job that counts against the same budget. */

package/dist/services/queue/Job.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"Job.d.ts","sourceRoot":"","sources":["../../../services/queue/Job.ts"],"names":[],"mappings":"AAGA,qBAAa,GAAG;IACd,MAAM,CAAC,IAAI,SAAW;IACtB,MAAM,UAAS;IACf,WAAW,SAAK;IAEhB;;;;;OAKG;IACH,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,CAAK;IAE/B;;;;;;;;;OASG;IACH,QAAQ,CAAC,GAAG,KAAK,EAAE,GAAG,EAAE,GAAG,MAAM,GAAG,MAAM,GAAG,SAAS,GAAG,IAAI;IAI7D;;;OAGG;IACH,SAAS,SAAe;IAExB;;;;;;OAMG;IACH,QAAQ,CAAC,GAAG,KAAK,EAAE,GAAG,EAAE,GAAG,WAAW,GAAG,WAAW,EAAE,GAAG,SAAS,GAAG,IAAI;IAIzE;;;;OAIG;IACH,WAAW,CAAC,GAAG,KAAK,EAAE,GAAG,EAAE,GAAG,cAAc,GAAG,SAAS,GAAG,IAAI;IAI/D;;;;OAIG;IACH,OAAO,CAAC,OAAO,SAAI,GAAG,IAAI;IAI1B;;;;;;OAMG;IACH,IAAI,CAAC,KAAK,EAAE,OAAO,EAAE,OAAO,GAAE;QAAE,KAAK,CAAC,EAAE,OAAO,CAAA;KAAO,GAAG,IAAI;IAQ7D,oEAAoE;IACpE,QAAQ,EACJ;QAAE,IAAI,EAAE,SAAS,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,GACpC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,KAAK,CAAC;QAAC,KAAK,EAAE,OAAO,CAAA;KAAE,GAC9C,SAAS,CAAC;IAEd,GAAG,CAAC,GAAG,KAAK,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,GAAG;IAExC,MAAM,CAAC,MAAM,EAAE,KAAK,EAAE,GAAG,KAAK,EAAE,GAAG,EAAE,GAAG,IAAI;IAC5C,SAAS,CAAC,OAAO,EAAE,GAAG,EAAE,GAAG,KAAK,EAAE,GAAG,EAAE,GAAG,IAAI;IAC9C,YAAY,CAAC,MAAM,EAAE,KAAK,EAAE,GAAG,KAAK,EAAE,GAAG,EAAE,GAAG,IAAI;IAElD;;;;;;;;OAQG;IACH,MAAM,CAAC,QAAQ,CAAC,CAAC,SAAS,GAAG,EAC3B,IAAI,EAAE,UAAU,CAAC,EACjB,GAAG,IAAI,EAAE,UAAU,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,GAC5B,OAAO,CAAC,MAAM,CAAC;CAOnB;AAED,MAAM,MAAM,WAAW,GAAG;IACxB,+DAA+D;IAC/D,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,mCAAmC;IACnC,MAAM,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF,MAAM,MAAM,cAAc,GAAG;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC"}
+{"version":3,"file":"Job.d.ts","sourceRoot":"","sources":["../../../services/queue/Job.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,WAAW,EAAE,KAAK,OAAO,EAAiB,MAAM,SAAS,CAAC;AACxE,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAChD,OAAO,EAAE,KAAK,eAAe,EAAE,KAAK,eAAe,EAAgB,MAAM,gBAAgB,CAAC;AAE1F,iEAAiE;AACjE,MAAM,MAAM,QAAQ,GAAG;IACrB,sBAAsB;IACtB,EAAE,EAAE,MAAM,CAAC;IACX;;;;OAIG;IACH,SAAS,IAAI,OAAO,CAAC,OAAO,CAAC,CAAC;CAC/B,CAAC;AAEF;;;;;GAKG;AACH,MAAM,MAAM,WAAW,CAAC,CAAC,SAAS,GAAG,EAAE,IAAI,CAAC,SAAS,CAAC,GAAG,MAAM,IAAI,EAAE,MAAM,IAAI,CAAC,GAC5E,WAAW,SAAS,IAAI,GACtB,IAAI,GACJ,CAAC,GACH,CAAC,CAAC;AAEN,qBAAa,GAAG;IACd,MAAM,CAAC,IAAI,SAAW;IACtB,MAAM,UAAS;IACf,WAAW,SAAK;IAEhB;;;;;OAKG;IACH,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,CAAK;IAE/B;;;;;;;;;OASG;IACH,QAAQ,CAAC,GAAG,KAAK,EAAE,GAAG,EAAE,GAAG,MAAM,GAAG,MAAM,GAAG,SAAS,GAAG,IAAI;IAI7D;;;OAGG;IACH,SAAS,SAAe;IAExB;;;;;;OAMG;IACH,QAAQ,CAAC,GAAG,KAAK,EAAE,GAAG,EAAE,GAAG,WAAW,GAAG,WAAW,EAAE,GAAG,SAAS,GAAG,IAAI;IAIzE;;;;OAIG;IACH,WAAW,CAAC,GAAG,KAAK,EAAE,GAAG,EAAE,GAAG,cAAc,GAAG,SAAS,GAAG,IAAI;IAI/D;;;;OAIG;IACH,OAAO,CAAC,OAAO,SAAI,GAAG,IAAI;IAI1B;;;;;;OAMG;IACH,IAAI,CAAC,KAAK,EAAE,OAAO,EAAE,OAAO,GAAE;QAAE,KAAK,CAAC,EAAE,OAAO,CAAA;KAAO,GAAG,IAAI;IAQ7D;;;OAGG;IACH,KAAK,EAAE,QAAQ,GAAG,SAAS,CAAa;IAExC;;;;;OAKG;IACG,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAK5C,gFAAgF;IAChF,SAAS,EAAE,CAAC,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,SAAS,CAAC;IAE1D,sEAAsE;IACtE,QAAQ,EAAE,UAAU,GAAG,SAAS,CAAC;IAEjC,oEAAoE;IACpE,QAAQ,EACJ;QAAE,IAAI,EAAE,SAAS,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,GACpC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,KAAK,CAAC;QAAC,KAAK,EAAE,OAAO,CAAA;KAAE,GAC9C,SAAS,CAAC;IAEd,GAAG,CAAC,GAAG,KAAK,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,GAAG;IAExC,MAAM,CAAC,MAAM,EAAE,KAAK,EAAE,GAAG,KAAK,EAAE,GAAG,EAAE,GAAG,IAAI;IAC5C,SAAS,CAAC,OAAO,EAAE,GAAG,EAAE,GAAG,KAAK,EAAE,GAAG,EAAE,GAAG,IAAI;IAC9C,YAAY,CAAC,MAAM,EAAE,KAAK,EAAE,GAAG,KAAK,EAAE,GAAG,EAAE,GAAG,IAAI;IAElD;;;;;;;;OAQG;IACH,MAAM,CAAC,QAAQ,CAAC,CAAC,SAAS,GAAG,EAC3B,IAAI,EAAE,UAAU,CAAC,EACjB,GAAG,IAAI,EAAE,UAAU,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,GAC5B,OAAO,CAAC,MAAM,CAAC;IAQlB;;;;;;;;;;;;;;;;;;;;;;OAsBG;IACH,MAAM,CAAC,aAAa,CAAC,CAAC,SAAS,GAAG,EAChC,IAAI,EAAE,UAAU,CAAC,EACjB,IAAI,EAAE,aAAa,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EACnD,OAAO,GAAE,eAAoB,GAC5B,OAAO,CAAC,eAAe,CAAC;IAW3B;;;;OAIG;IACH,MAAM,CAAC,IAAI,CAAC,CAAC,SAAS,GAAG,EACvB,IAAI,EAAE,UAAU,CAAC,EACjB,GAAG,IAAI,EAAE,WAAW,CAAC,UAAU,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,GACzC,OAAO;IAUV,0EAA0E;IAC1E,MAAM,CAAC,SAAS,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC;IAIzD;;;;OAIG;IACH,MAAM,CAAC,WAAW,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;CAGjD;AAED,MAAM,MAAM,WAAW,GAAG;IACxB,+DAA+D;IAC/D,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,mCAAmC;IACnC,MAAM,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF,MAAM,MAAM,cAAc,GAAG;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC"}

package/dist/services/queue/MemoryQueueDriver.d.ts

@@ -1,4 +1,6 @@
-import type { ClaimOptions, ClaimedJob, EnqueueJob, JobFailure, JobRelease, QueueDriver } from "./QueueDriver";
+import { type BatchStatus } from "./batch";
+import type { ClaimOptions, ClaimedJob, EnqueueBatch, EnqueueJob, JobFailure, JobRelease, QueueDriver } from "./QueueDriver";
+import type { WorkflowStore } from "./workflow/WorkflowStore";
 /**
  * The default driver: a Map in this process's memory.
  *
@@ -22,12 +24,18 @@ import type { ClaimOptions, ClaimedJob, EnqueueJob, JobFailure, JobRelease, Queu
  * not be known by anyone, and the manager dead-letters it at once with a line
  * saying so; filtered out, it would wait here unseen until the process exited.
  *
+ * Batches are kept in a Map beside the jobs, and a finished one for a day.
+ * Everything a batch does happens synchronously inside the call that causes
+ * it, so the counters and the callbacks need no further care here.
+ *
  * One timer at most, for the next moment something becomes claimable, and it
  * is unref'd: a queue with nothing due does not hold the process open, and
  * neither does one waiting out a backoff — the same way the queue never has.
  */
 export declare class MemoryQueueDriver implements QueueDriver {
     private entries;
+    private batches;
+    private workflows;
     private listeners;
     private timer;
     private timerAt;
@@ -36,12 +44,34 @@ export declare class MemoryQueueDriver implements QueueDriver {
     get waiting(): number;
     /** Jobs currently claimed and not yet reported on. */
     get leased(): number;
-    enqueue({ name, args, delayMs, id, }: EnqueueJob): Promise<string>;
+    enqueue(job: EnqueueJob): Promise<string>;
+    private add;
+    enqueueBatch(batch: EnqueueBatch): Promise<void>;
+    /** `enqueueBatch`, synchronously, for the workflow store too. */
+    private recordBatch;
+    findBatch(id: string, options?: {
+        progress?: boolean;
+    }): Promise<BatchStatus | null>;
+    cancelBatch(id: string): Promise<boolean>;
+    private cancelBatchNow;
+    /**
+     * Workflows (#846), kept in memory beside the jobs and lost with them. The
+     * store's writes, ticks and batches included, happen synchronously here.
+     */
+    workflowStore(): WorkflowStore;
+    reportProgress(job: ClaimedJob, progress: number): Promise<void>;
     claim(limit: number, options: ClaimOptions): Promise<ClaimedJob[]>;
     complete(job: ClaimedJob): Promise<void>;
     fail(job: ClaimedJob, failure: JobFailure): Promise<void>;
     release(job: ClaimedJob, release: JobRelease): Promise<void>;
     heartbeat(jobs: ClaimedJob[], options: ClaimOptions): Promise<void>;
+    /** Counts a job's end against its batch, if it has one. */
+    private count;
+    /** Stores the batch's new state and enqueues the callbacks it made due. */
+    private save;
+    /** Drops the batch's waiting jobs, and says how many there were. */
+    private cancelWaiting;
+    private forgetFinishedBatches;
     subscribe(wake: () => void): () => void;
     /**
      * The entry for a report, if the report is about the claim that currently

package/dist/services/queue/MemoryQueueDriver.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"MemoryQueueDriver.d.ts","sourceRoot":"","sources":["../../../services/queue/MemoryQueueDriver.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,YAAY,EACZ,UAAU,EACV,UAAU,EACV,UAAU,EACV,UAAU,EACV,WAAW,EACZ,MAAM,eAAe,CAAC;AAWvB;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,qBAAa,iBAAkB,YAAW,WAAW;IACnD,OAAO,CAAC,OAAO,CAA4B;IAC3C,OAAO,CAAC,SAAS,CAAyB;IAC1C,OAAO,CAAC,KAAK,CAA4C;IACzD,OAAO,CAAC,OAAO,CAAY;IAC3B,OAAO,CAAC,GAAG,CAAK;IAEhB,8DAA8D;IAC9D,IAAI,OAAO,IAAI,MAAM,CAIpB;IAED,sDAAsD;IACtD,IAAI,MAAM,IAAI,MAAM,CAEnB;IAEK,OAAO,CAAC,EACZ,IAAI,EACJ,IAAI,EACJ,OAAW,EACX,EAAwB,GACzB,EAAE,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC;IAYzB,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,UAAU,EAAE,CAAC;IAkBlE,QAAQ,CAAC,GAAG,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;IAMxC,IAAI,CAAC,GAAG,EAAE,UAAU,EAAE,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;IAgBzD,OAAO,CAAC,GAAG,EAAE,UAAU,EAAE,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;IAW5D,SAAS,CAAC,IAAI,EAAE,UAAU,EAAE,EAAE,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;IASzE,SAAS,CAAC,IAAI,EAAE,MAAM,IAAI,GAAG,MAAM,IAAI;IAKvC;;;;OAIG;IACH,OAAO,CAAC,OAAO;IAQf,OAAO,CAAC,OAAO;IAWf,OAAO,CAAC,IAAI;IAIZ,2EAA2E;IAC3E,OAAO,CAAC,QAAQ;CAoBjB"}
+{"version":3,"file":"MemoryQueueDriver.d.ts","sourceRoot":"","sources":["../../../services/queue/MemoryQueueDriver.ts"],"names":[],"mappings":"AAAA,OAAO,EAGL,KAAK,WAAW,EAQjB,MAAM,SAAS,CAAC;AACjB,OAAO,KAAK,EACV,YAAY,EACZ,UAAU,EACV,YAAY,EACZ,UAAU,EACV,UAAU,EACV,UAAU,EACV,WAAW,EACZ,MAAM,eAAe,CAAC;AAEvB,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AAoB9D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AACH,qBAAa,iBAAkB,YAAW,WAAW;IACnD,OAAO,CAAC,OAAO,CAA4B;IAC3C,OAAO,CAAC,OAAO,CAAkC;IACjD,OAAO,CAAC,SAAS,CAAkC;IACnD,OAAO,CAAC,SAAS,CAAyB;IAC1C,OAAO,CAAC,KAAK,CAA4C;IACzD,OAAO,CAAC,OAAO,CAAY;IAC3B,OAAO,CAAC,GAAG,CAAK;IAEhB,8DAA8D;IAC9D,IAAI,OAAO,IAAI,MAAM,CAIpB;IAED,sDAAsD;IACtD,IAAI,MAAM,IAAI,MAAM,CAEnB;IAEK,OAAO,CAAC,GAAG,EAAE,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC;IAM/C,OAAO,CAAC,GAAG;IAsBL,YAAY,CAAC,KAAK,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;IAItD,iEAAiE;IACjE,OAAO,CAAC,WAAW;IAYb,SAAS,CAAC,EAAE,EAAE,MAAM,EAAE,OAAO,GAAE;QAAE,QAAQ,CAAC,EAAE,OAAO,CAAA;KAAO,GAAG,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC;IAWxF,WAAW,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAI/C,OAAO,CAAC,cAAc;IAUtB;;;OAGG;IACH,aAAa,IAAI,aAAa;IAYxB,cAAc,CAAC,GAAG,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAKhE,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,UAAU,EAAE,CAAC;IAkBlE,QAAQ,CAAC,GAAG,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;IAQxC,IAAI,CAAC,GAAG,EAAE,UAAU,EAAE,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;IAwBzD,OAAO,CAAC,GAAG,EAAE,UAAU,EAAE,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;IAW5D,SAAS,CAAC,IAAI,EAAE,UAAU,EAAE,EAAE,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;IASzE,2DAA2D;IAC3D,OAAO,CAAC,KAAK;IAcb,2EAA2E;IAC3E,OAAO,CAAC,IAAI;IAMZ,oEAAoE;IACpE,OAAO,CAAC,aAAa;IAWrB,OAAO,CAAC,qBAAqB;IAQ7B,SAAS,CAAC,IAAI,EAAE,MAAM,IAAI,GAAG,MAAM,IAAI;IAKvC;;;;OAIG;IACH,OAAO,CAAC,OAAO;IAQf,OAAO,CAAC,OAAO;IAWf,OAAO,CAAC,IAAI;IAIZ,2EAA2E;IAC3E,OAAO,CAAC,QAAQ;CAoBjB"}

package/dist/services/queue/QueueDriver.d.ts

@@ -41,6 +41,8 @@
  * every other one.
  */
 import type { LockStore } from "../lock/LockStore";
+import type { BatchCallbacks, BatchStatus } from "./batch";
+import type { WorkflowStore } from "./workflow/WorkflowStore";
 export interface QueueDriver {
     /**
      * Records a job and returns its id. The job is claimable once `delayMs` has
@@ -83,13 +85,25 @@ export interface QueueDriver {
      * ahead of everything enqueued while it waited. The contract suite pins it.
      */
     claim(limit: number, options: ClaimOptions): Promise<ClaimedJob[]>;
-    /** Ends a claim successfully. The job is never claimed again. */
+    /**
+     * Ends a claim successfully. The job is never claimed again.
+     *
+     * For a job of a batch (`batchId`), the batch's counters are updated in the
+     * same atomic step as the job is ended, and any batch callback that became
+     * due is enqueued in it too; see `enqueueBatch`. A stale claim's report
+     * changes neither.
+     */
     complete(job: ClaimedJob): Promise<void>;
     /**
      * Ends a claim unsuccessfully. With a `retryInMs`, the job becomes claimable
      * again after that long, keeping its attempt count. With `retryInMs: null`
      * it is dead-lettered: terminal, never claimed again. What a driver keeps of
      * a dead-lettered job is its own business; the memory driver keeps nothing.
+     *
+     * For a job of a batch, a dead-letter is counted as the batch's `complete`
+     * counts a success, and a retry resets the job's reported progress to 0. A
+     * retry of a job whose batch has been cancelled is a dead-letter instead,
+     * counted as `cancelled`, as is a failure with `cancelled: true`.
      */
     fail(job: ClaimedJob, failure: JobFailure): Promise<void>;
     /**
@@ -136,7 +150,68 @@ export interface QueueDriver {
      * once per manager.
      */
     lockStore?(): LockStore;
+    /**
+     * Records a batch and all of its jobs in one atomic step: either every job
+     * and the batch are recorded, or none is. Optional; a driver without it
+     * cannot run `Job.dispatchBatch`.
+     *
+     * The driver then keeps the batch's counters (see `batch.ts`): every
+     * `complete` or dead-lettering `fail` of one of its jobs counts it, in the
+     * same atomic step that ends the job, and the step that counts the last one
+     * finishes the batch and enqueues its due callbacks. A callback's arguments
+     * are its own followed by the batch's status, and it is enqueued in that
+     * same step, so it is enqueued exactly once, by whichever process ended the
+     * job that made it due.
+     *
+     * Like `enqueue`, it writes on the caller's ORM transaction when
+     * `joinsTransaction` says so. A batch of no jobs is finished at once.
+     */
+    enqueueBatch?(batch: EnqueueBatch): Promise<void>;
+    /**
+     * The batch's status, or `null` for an id the driver has no batch under.
+     * `progress: false` skips summing what the running jobs reported, for a
+     * caller that only wants the counters and `cancelledAt`; `progress` is then
+     * the ended jobs over `total`.
+     */
+    findBatch?(id: string, options?: {
+        progress?: boolean;
+    }): Promise<BatchStatus | null>;
+    /**
+     * Cancels a batch that is still running, and resolves to whether it did:
+     * `false` for a batch that is unknown, finished or already cancelled. Its
+     * waiting jobs are ended as `cancelled` and never claimed; running ones
+     * finish, or notice through `this.batch.cancelled()`. `catch` becomes due
+     * unless a failure already made it due, and `finally` once nothing is
+     * pending. `then` never runs for a cancelled batch.
+     */
+    cancelBatch?(id: string): Promise<boolean>;
+    /**
+     * Records how far a claimed job of a batch is, from 0 to 1, for the batch's
+     * `progress`. A stale claim's report is ignored.
+     */
+    reportProgress?(job: ClaimedJob, progress: number): Promise<void>;
+    /**
+     * Where `Workflow` keeps its workflows and steps (#846), over the same
+     * storage, so that a workflow's writes and the jobs and batches they
+     * schedule are recorded in one atomic step. Optional; a driver without it
+     * cannot run workflows. Called once per manager. A driver that has it also
+     * needs the batch methods for `step.batch`.
+     */
+    workflowStore?(): WorkflowStore;
 }
+export type EnqueueBatch = {
+    /** The id to record the batch under. Chosen by the manager. */
+    id: string;
+    /** The `name` option, for finding it in storage. */
+    name: string | null;
+    /** The registered name every job of the batch runs as. */
+    job: string;
+    /** One JSON array of `run` arguments per job. */
+    args: string[];
+    /** Whether a failed job leaves the rest running; see `batch.ts`. */
+    allowFailures: boolean;
+    callbacks: BatchCallbacks;
+};
 export type EnqueueJob = {
     /** The job's registered name — its class's `static name`. */
     name: string;
@@ -194,6 +269,8 @@ export type ClaimedJob = {
     attempt: number;
     /** When `enqueue` recorded it, in epoch milliseconds. */
     createdAt: number;
+    /** The batch the job belongs to. Absent for a job dispatched on its own. */
+    batchId?: string;
 };
 export type JobRelease = {
     /** Milliseconds until the job is claimable again. */
@@ -204,5 +281,10 @@ export type JobFailure = {
     error: string;
     /** Milliseconds until the retry is claimable, or `null` to dead-letter. */
     retryInMs: number | null;
+    /**
+     * The job is dead-lettered because its batch was cancelled, not because it
+     * failed: the batch counts it as `cancelled`. Only with `retryInMs: null`.
+     */
+    cancelled?: boolean;
 };
 //# sourceMappingURL=QueueDriver.d.ts.map

package/dist/services/queue/QueueDriver.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"QueueDriver.d.ts","sourceRoot":"","sources":["../../../services/queue/QueueDriver.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyCG;AACH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAEnD,MAAM,WAAW,WAAW;IAC1B;;;;OAIG;IACH,OAAO,CAAC,GAAG,EAAE,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAE1C;;;;;;;;;;;;;;;;;;OAkBG;IACH,gBAAgB,CAAC,IAAI,OAAO,CAAC;IAE7B;;;;;;;;;;;;;OAaG;IACH,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC;IAEnE,iEAAiE;IACjE,QAAQ,CAAC,GAAG,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAEzC;;;;;OAKG;IACH,IAAI,CAAC,GAAG,EAAE,UAAU,EAAE,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAE1D;;;;;;;;;;;;OAYG;IACH,OAAO,CAAC,GAAG,EAAE,UAAU,EAAE,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAE7D;;;OAGG;IACH,SAAS,CAAC,CAAC,IAAI,EAAE,UAAU,EAAE,EAAE,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAErE;;;;;;;OAOG;IACH,SAAS,CAAC,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IAEzC;;;;;;OAMG;IACH,SAAS,CAAC,CAAC,IAAI,EAAE,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC;IAEzC;;;;;;OAMG;IACH,SAAS,CAAC,IAAI,SAAS,CAAC;CACzB;AAED,MAAM,MAAM,UAAU,GAAG;IACvB,6DAA6D;IAC7D,IAAI,EAAE,MAAM,CAAC;IACb,yEAAyE;IACzE,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;;;;;;OAMG;IACH,EAAE,CAAC,EAAE,MAAM,CAAC;CACb,CAAC;AAEF,MAAM,MAAM,YAAY,GAAG;IACzB,mBAAmB,EAAE,MAAM,CAAC;IAC5B;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACH,UAAU,CAAC,EAAE;QAAE,KAAK,EAAE,SAAS,MAAM,EAAE,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC;CAC5D,CAAC;AAEF,MAAM,MAAM,UAAU,GAAG;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb;;;OAGG;IACH,OAAO,EAAE,MAAM,CAAC;IAChB,yDAAyD;IACzD,SAAS,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,MAAM,MAAM,UAAU,GAAG;IACvB,qDAAqD;IACrD,SAAS,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,MAAM,MAAM,UAAU,GAAG;IACvB,+DAA+D;IAC/D,KAAK,EAAE,MAAM,CAAC;IACd,2EAA2E;IAC3E,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;CAC1B,CAAC"}
+{"version":3,"file":"QueueDriver.d.ts","sourceRoot":"","sources":["../../../services/queue/QueueDriver.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyCG;AACH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAC3D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AAE9D,MAAM,WAAW,WAAW;IAC1B;;;;OAIG;IACH,OAAO,CAAC,GAAG,EAAE,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAE1C;;;;;;;;;;;;;;;;;;OAkBG;IACH,gBAAgB,CAAC,IAAI,OAAO,CAAC;IAE7B;;;;;;;;;;;;;OAaG;IACH,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC;IAEnE;;;;;;;OAOG;IACH,QAAQ,CAAC,GAAG,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAEzC;;;;;;;;;;OAUG;IACH,IAAI,CAAC,GAAG,EAAE,UAAU,EAAE,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAE1D;;;;;;;;;;;;OAYG;IACH,OAAO,CAAC,GAAG,EAAE,UAAU,EAAE,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAE7D;;;OAGG;IACH,SAAS,CAAC,CAAC,IAAI,EAAE,UAAU,EAAE,EAAE,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAErE;;;;;;;OAOG;IACH,SAAS,CAAC,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IAEzC;;;;;;OAMG;IACH,SAAS,CAAC,CAAC,IAAI,EAAE,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC;IAEzC;;;;;;OAMG;IACH,SAAS,CAAC,IAAI,SAAS,CAAC;IAExB;;;;;;;;;;;;;;;OAeG;IACH,YAAY,CAAC,CAAC,KAAK,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAElD;;;;;OAKG;IACH,SAAS,CAAC,CAAC,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE;QAAE,QAAQ,CAAC,EAAE,OAAO,CAAA;KAAE,GAAG,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC,CAAC;IAEtF;;;;;;;OAOG;IACH,WAAW,CAAC,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IAE3C;;;OAGG;IACH,cAAc,CAAC,CAAC,GAAG,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAElE;;;;;;OAMG;IACH,aAAa,CAAC,IAAI,aAAa,CAAC;CACjC;AAED,MAAM,MAAM,YAAY,GAAG;IACzB,+DAA+D;IAC/D,EAAE,EAAE,MAAM,CAAC;IACX,oDAAoD;IACpD,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACpB,0DAA0D;IAC1D,GAAG,EAAE,MAAM,CAAC;IACZ,iDAAiD;IACjD,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,oEAAoE;IACpE,aAAa,EAAE,OAAO,CAAC;IACvB,SAAS,EAAE,cAAc,CAAC;CAC3B,CAAC;AAEF,MAAM,MAAM,UAAU,GAAG;IACvB,6DAA6D;IAC7D,IAAI,EAAE,MAAM,CAAC;IACb,yEAAyE;IACzE,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;;;;;;OAMG;IACH,EAAE,CAAC,EAAE,MAAM,CAAC;CACb,CAAC;AAEF,MAAM,MAAM,YAAY,GAAG;IACzB,mBAAmB,EAAE,MAAM,CAAC;IAC5B;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACH,UAAU,CAAC,EAAE;QAAE,KAAK,EAAE,SAAS,MAAM,EAAE,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC;CAC5D,CAAC;AAEF,MAAM,MAAM,UAAU,GAAG;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb;;;OAGG;IACH,OAAO,EAAE,MAAM,CAAC;IAChB,yDAAyD;IACzD,SAAS,EAAE,MAAM,CAAC;IAClB,4EAA4E;IAC5E,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF,MAAM,MAAM,UAAU,GAAG;IACvB,qDAAqD;IACrD,SAAS,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,MAAM,MAAM,UAAU,GAAG;IACvB,+DAA+D;IAC/D,KAAK,EAAE,MAAM,CAAC;IACd,2EAA2E;IAC3E,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB;;;OAGG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB,CAAC"}

package/dist/services/queue/QueueManager.d.ts

@@ -2,7 +2,12 @@ import type { Application } from "../../foundation/Application";
 import { LockManager } from "../lock/LockManager";
 import { Job } from "./Job";
 import { type QueueConfig } from "./config";
-import type { ClaimedJob, QueueDriver } from "./QueueDriver";
+import type { ClaimedJob, EnqueueBatch, QueueDriver } from "./QueueDriver";
+import type { BatchStatus, JobCall } from "./batch";
+import { backoffFor } from "./backoff";
+import type { WorkflowClass } from "./workflow/Workflow";
+import { WorkflowRuntime } from "./workflow/WorkflowRuntime";
+export { backoffFor };
 declare global {
     /**
      * The queue loop a `gemi dev` server is running over a driver that outlives
@@ -19,6 +24,32 @@ declare global {
         stop(): Promise<unknown>;
     } | undefined;
 }
+/** What `Job.dispatchBatch` resolves to. */
+export type DispatchedBatch = {
+    /** The batch's id, for `Job.findBatch` and `Job.cancelBatch`. */
+    id: string;
+    /** How many jobs it was dispatched with. */
+    total: number;
+};
+/** The options of `Job.dispatchBatch`. */
+export type JobBatchOptions = {
+    /** A label kept with the batch, for finding it in storage and in its status. */
+    name?: string;
+    /**
+     * Whether a dead-lettered job leaves the rest running. Default `false`: the
+     * first one cancels the batch.
+     */
+    allowFailures?: boolean;
+    /**
+     * Enqueued once every job has ended, when the batch was not cancelled and
+     * no job failed (or `allowFailures` is set). Build it with `Job.with(...)`.
+     */
+    then?: JobCall;
+    /** Enqueued at the first failed job, or at `cancelBatch`, whichever is first. */
+    catch?: JobCall;
+    /** Enqueued once every job has ended, whatever happened. */
+    finally?: JobCall;
+};
 /** What `drain` could not wait out. */
 export type DrainResult = {
     /**
@@ -59,6 +90,7 @@ export declare class QueueManager {
     /** Unknown names already reported as left for another replica; see `run`. */
     private readonly releasedNames;
     private lockManager;
+    private workflowRuntime;
     /**
      * `application` is entered around every job, so `app()` inside one resolves
      * to it rather than to whichever application happens to be the static
@@ -88,6 +120,24 @@ export declare class QueueManager {
      * unaffected, because that list is already in place by the end of `register()`.
      */
     useJobs(jobs: Array<new () => Job>): void;
+    /**
+     * Replaces the registered workflows, for the provider to hand over what it
+     * found under `app/workflows`. With at least one, the internal job their
+     * passes run in is registered too. See `Workflow`.
+     */
+    useWorkflows(workflows: WorkflowClass[]): void;
+    /** The registered workflow classes, as handed over. */
+    get registeredWorkflows(): ReadonlyArray<WorkflowClass>;
+    /**
+     * Workflows (#846) over this queue's driver: what `Workflow.start`,
+     * `signal`, `find` and `cancel` call.
+     */
+    get workflows(): WorkflowRuntime;
+    /**
+     * The tick job, beside the app's own, once there is a workflow to tick.
+     * Not in `registeredJobs`, which reports what the app handed over.
+     */
+    private registerTickJob;
     /**
      * Registers one more job beside whatever `useJobs` already took.
      *
@@ -163,6 +213,31 @@ export declare class QueueManager {
          */
         reportFailure?: boolean;
     }): Promise<string>;
+    /**
+     * Queues a batch: every job, the batch and its callbacks in one atomic
+     * write. See `Job.dispatchBatch`.
+     *
+     * Inside an ORM transaction the batch belongs to it, exactly as a single
+     * dispatch does in `push`: written on it by a driver that joins, otherwise
+     * held and recorded at the commit, resolving at once to the id it will have.
+     */
+    pushBatch(job: new () => Job, args: string[], options?: JobBatchOptions): Promise<DispatchedBatch>;
+    /**
+     * @internal The batch `pushBatch` records, checked: for `pushBatch` and for
+     * a workflow's `step.batch`, which records it in a write of its own. Throws
+     * on the caller's stack for a driver without batches, a worker job, a
+     * unique job and a callback that is not a job call.
+     */
+    buildBatch(job: new () => Job, args: string[], options?: JobBatchOptions): EnqueueBatch;
+    /** `pushBatch`, once it is known that the batch is recorded now; see `record`. */
+    private recordBatch;
+    /** The batch's status, or `null` for an unknown id. See `Job.findBatch`. */
+    findBatch(id: string): Promise<BatchStatus | null>;
+    /**
+     * Cancels a running batch, and resolves to whether it did. See
+     * `Job.cancelBatch`.
+     */
+    cancelBatch(id: string): Promise<boolean>;
     /**
      * The locks unique jobs, cron jobs' `withoutOverlapping` and `onOneServer`,
      * and the `Lock` facade use, over the store the `locks` setting names. Built
@@ -224,7 +299,8 @@ export declare class QueueManager {
      * For a job the driver has just made claimable: start the loop, or wake a
      * running one that only polls, as `push` does for a dispatch.
      */
-    private claimSoon;
+    /** @internal */
+    claimSoon(): void;
     /**
      * Starts claiming. Idempotent, and called by the first `push`, so an app
      * does not need to; it is public for a process that should run jobs it
@@ -309,6 +385,8 @@ export declare class QueueManager {
      * escape the queue with the job's slot still counted as taken.
      */
     private run;
+    /** What `this.batch` and `this.progress()` reach in a job of a batch. */
+    private batchHandle;
     /**
      * Takes a slot under the job's `concurrency` key and a hit under each of
      * its `throttle` keys, all in the queue's lock store, so the limits hold
@@ -346,9 +424,4 @@ export declare function claimingTurnedOff(env?: Record<string, string | undefine
 /** Marks this process as a `gemi queue:work` worker; see `claimsInThisProcess`. */
 export declare function markQueueWorker(worker?: boolean): void;
 export declare function isQueueWorker(): boolean;
-/**
- * The delay before the retry that follows attempt `attempt`: the number
- * itself, or the array's entry for that retry with its last entry repeated.
- */
-export declare function backoffFor(backoff: number | number[], attempt: number): number;
 //# sourceMappingURL=QueueManager.d.ts.map