@nestjs/workflows 0.0.1

package/dist/mysql/key-limits.js

@@ -0,0 +1,41 @@
+/**
+ * The longest keys MySqlWorkflowStore keeps, in characters: its key columns' lengths (the migrations use them, and the
+ * store checks every key it writes against them, so they can't drift apart). Every id, name and key is an indexed
+ * `utf8mb4_0900_bin` column on MySQL, and InnoDB caps an index key at 3,072 bytes, 768 four-byte characters, which a
+ * composite index's columns share: a journal entry's `(instance id, name)` takes 255 + 512.
+ */
+export const MYSQL_KEY_LIMITS = {
+    /** An instance's id (`start({ id })`), and so a parent's; a child's id and a schedule run's id are instance ids too. */
+    instanceId: 255,
+    /** A schedule's id: its runs' ids (`<id>@<ISO time>`) add 25 characters and must fit an instance id. */
+    scheduleId: 230,
+    /** A workflow's name. */
+    workflow: 255,
+    /** An instance's concurrency key. */
+    concurrencyKey: 255,
+    /** An instance's rate-limit key. */
+    rateLimitKey: 255,
+    /** A signal's name, and a wait's. */
+    signal: 255,
+    /** A signal's key, and a wait's. */
+    signalKey: 255,
+    /** A signal's id (`signal(..., { id })`), which deduplicates it. */
+    dedupeId: 255,
+    /** A journal entry's name: a step's, a wait's, or the engine's (`$child:<child id>`, `$compensate:<step>`...). */
+    journalName: 512,
+    /** A lease's token (the engine's: a UUID). */
+    leaseToken: 255,
+};
+/** Throws a `RangeError` that names `what` when `value` is longer than `max` characters (code points, as MySQL counts). */
+export function checkKey(what, value, max) {
+    // A string's UTF-16 length is at least its code points: most keys pass on the first test.
+    if (typeof value !== 'string' || value.length <= max) {
+        return;
+    }
+    const characters = [...value].length;
+    if (characters > max) {
+        const preview = characters > 40 ? `${[...value].slice(0, 40).join('')}…` : value;
+        throw new RangeError(`MySqlWorkflowStore: ${what} holds at most ${max} characters on MySQL, and ${JSON.stringify(preview)} has ${characters}. ` +
+            "The store's ids, names and keys are indexed columns of bounded length there.");
+    }
+}

package/dist/mysql/migrations/index.d.ts

@@ -0,0 +1,7 @@
+import { StoreSchema } from '@nestjs/store-kit/mysql';
+/**
+ * MySqlWorkflowStore's schema: every version of it, in order (a new one goes last, and none ever changes once
+ * released), and what the store, its statics and `nest-workflows` do with them. Its tables live in the connection's
+ * database, named `<schema>_<table>`, next to the kit's `<schema>_migrations` and `<schema>_locks`.
+ */
+export declare const mysqlWorkflowStoreSchema: StoreSchema;

package/dist/mysql/migrations/index.js

@@ -0,0 +1,16 @@
+import { StoreSchema } from '@nestjs/store-kit/mysql';
+import { WorkflowSchemaError } from '../../sql/workflow-schema.error.js';
+import { initialMigration } from './initial.migration.js';
+/**
+ * MySqlWorkflowStore's schema: every version of it, in order (a new one goes last, and none ever changes once
+ * released), and what the store, its statics and `nest-workflows` do with them. Its tables live in the connection's
+ * database, named `<schema>_<table>`, next to the kit's `<schema>_migrations` and `<schema>_locks`.
+ */
+export const mysqlWorkflowStoreSchema = new StoreSchema({
+    packageName: '@nestjs/workflows',
+    storeName: 'MySqlWorkflowStore',
+    command: 'nest-workflows',
+    defaultSchema: 'nest_workflows',
+    migrations: [initialMigration],
+    createError: (message, details) => new WorkflowSchemaError(message, details),
+});

package/dist/mysql/migrations/initial.migration.d.ts

@@ -0,0 +1,9 @@
+import { type StoreMigration } from '@nestjs/store-kit/mysql';
+/**
+ * The store's tables on MySQL, in the connection's database (`<schema>_instances`...), each created with its indexes in
+ * one statement. Times are epoch milliseconds from the engine's clock (`bigint`); inputs, outputs, errors, payloads,
+ * journal entries and schedule specs are the engine's JSON (`json`). Every id, name and key is a `utf8mb4_0900_bin`
+ * column (`keyColumn()`), so keys that differ in case or accents stay apart, of a bounded length (`MYSQL_KEY_LIMITS`).
+ * No foreign keys: a store's delete removes an instance's journal and waits in its own transaction.
+ */
+export declare const initialMigration: StoreMigration;

package/dist/mysql/migrations/initial.migration.js

@@ -0,0 +1,104 @@
+import { keyColumn } from '@nestjs/store-kit/mysql';
+import { MYSQL_KEY_LIMITS as L } from '../key-limits.js';
+/**
+ * The store's tables on MySQL, in the connection's database (`<schema>_instances`...), each created with its indexes in
+ * one statement. Times are epoch milliseconds from the engine's clock (`bigint`); inputs, outputs, errors, payloads,
+ * journal entries and schedule specs are the engine's JSON (`json`). Every id, name and key is a `utf8mb4_0900_bin`
+ * column (`keyColumn()`), so keys that differ in case or accents stay apart, of a bounded length (`MYSQL_KEY_LIMITS`).
+ * No foreign keys: a store's delete removes an instance's journal and waits in its own transaction.
+ */
+export const initialMigration = {
+    version: 1,
+    name: 'initial',
+    up: (t) => [
+        `CREATE TABLE ${t('instances')} (
+  id ${keyColumn(L.instanceId)} NOT NULL PRIMARY KEY,
+  workflow ${keyColumn(L.workflow)} NOT NULL,
+  version int NOT NULL,
+  parent_id ${keyColumn(L.instanceId)},
+  parent_close ${keyColumn(16)},
+  concurrency_key ${keyColumn(L.concurrencyKey)},
+  rate_limit_key ${keyColumn(L.rateLimitKey)},
+  priority int NOT NULL DEFAULT 0,
+  schedule_id ${keyColumn(L.scheduleId)},
+  scheduled_at bigint,
+  status ${keyColumn(32)} NOT NULL,
+  input json,
+  output json,
+  error json,
+  wake_at bigint,
+  lease_token ${keyColumn(L.leaseToken)},
+  lease_owner text,
+  lease_until bigint,
+  cancel_requested boolean NOT NULL DEFAULT false,
+  terminate_requested boolean NOT NULL DEFAULT false,
+  cancel_reason text,
+  deadline bigint,
+  custom_status json,
+  signal_cursor bigint NOT NULL,
+  runs int NOT NULL DEFAULT 0,
+  created_at bigint NOT NULL,
+  updated_at bigint NOT NULL,
+  KEY instances_due (wake_at),
+  KEY instances_created (created_at, id),
+  KEY instances_status (status, updated_at),
+  KEY instances_leased (workflow, concurrency_key, lease_until),
+  KEY instances_parent (parent_id, created_at),
+  KEY instances_schedule (schedule_id, created_at)
+)`,
+        // One row per step, sleep, wait or compensation; seq keeps them in first-write order.
+        `CREATE TABLE ${t('journal')} (
+  seq bigint NOT NULL AUTO_INCREMENT PRIMARY KEY,
+  instance_id ${keyColumn(L.instanceId)} NOT NULL,
+  name ${keyColumn(L.journalName)} NOT NULL,
+  entry json NOT NULL,
+  UNIQUE KEY journal_name (instance_id, name)
+)`,
+        // The signals a suspended instance waits for.
+        `CREATE TABLE ${t('waits')} (
+  instance_id ${keyColumn(L.instanceId)} NOT NULL,
+  position int NOT NULL,
+  \`signal\` ${keyColumn(L.signal)} NOT NULL,
+  \`key\` ${keyColumn(L.signalKey)},
+  PRIMARY KEY (instance_id, position),
+  KEY waits_signal (\`signal\`, \`key\`)
+)`,
+        `CREATE TABLE ${t('signals')} (
+  id bigint NOT NULL AUTO_INCREMENT PRIMARY KEY,
+  name ${keyColumn(L.signal)} NOT NULL,
+  \`key\` ${keyColumn(L.signalKey)},
+  dedupe_id ${keyColumn(L.dedupeId)},
+  payload json,
+  created_at bigint NOT NULL,
+  UNIQUE KEY signals_dedupe (name, dedupe_id),
+  KEY signals_lookup (name, \`key\`, id)
+)`,
+        `CREATE TABLE ${t('schedules')} (
+  id ${keyColumn(L.scheduleId)} NOT NULL PRIMARY KEY,
+  workflow ${keyColumn(L.workflow)} NOT NULL,
+  declared boolean NOT NULL,
+  spec json NOT NULL,
+  input json,
+  paused boolean NOT NULL,
+  wake_at bigint,
+  state json NOT NULL,
+  revision int NOT NULL,
+  lease_token ${keyColumn(L.leaseToken)},
+  lease_owner text,
+  lease_until bigint,
+  created_at bigint NOT NULL,
+  updated_at bigint NOT NULL,
+  KEY schedules_due (wake_at),
+  KEY schedules_workflow (workflow, id)
+)`,
+        // The rate-limit windows claims count: the workflow's own (key '', which no instance key is) and one per key.
+        `CREATE TABLE ${t('rate_limits')} (
+  workflow ${keyColumn(L.workflow)} NOT NULL,
+  \`key\` ${keyColumn(L.rateLimitKey)} NOT NULL,
+  window_end bigint NOT NULL,
+  \`count\` int NOT NULL,
+  PRIMARY KEY (workflow, \`key\`),
+  KEY rate_limits_end (window_end)
+)`,
+    ],
+};

package/dist/mysql/mysql-workflow.store.d.ts

@@ -0,0 +1,197 @@
+import { type OnModuleInit } from '@nestjs/common';
+import { type MigrationSqlOptions, type MigrationStatementsOptions } from '@nestjs/store-kit/mysql';
+import type { WorkflowInstance, WorkflowStatus } from '../interfaces/workflow-instance.interface.js';
+import type { NewWorkflowInstance, NewWorkflowSignal, WorkflowCancelRequest, WorkflowClaim, WorkflowClaimRequest, WorkflowInstanceDetails, WorkflowListQuery, WorkflowPurgeQuery, WorkflowPurgeResult, WorkflowReopen, WorkflowScheduleClaimRequest, WorkflowScheduleQuery, WorkflowScheduleRecord, WorkflowScheduleSave, WorkflowScheduleWrite, WorkflowSignalQuery, WorkflowSignalRecord, WorkflowSignalResult, WorkflowStore, WorkflowWrite } from '../interfaces/workflow-store.interface.js';
+import type { WorkflowStorage } from '../storage/workflow.storage.js';
+import type { MySqlWorkflowStoreOptions } from './interfaces/mysql-workflow-store-options.interface.js';
+/**
+ * The first-party `WorkflowStore` on MySQL (8.4 LTS and 9.x), through the client the application already has
+ * (`fromMysql2()`, `fromDrizzle()`, `fromTypeOrm()`, `fromPrisma()`, `fromKysely()`). It keeps its tables in the
+ * connection's database, named `<schema>_<table>` (`nest_workflows_instances`...), which its migrations create and
+ * bring up to date, and it joins the application's transactions for `start()` and `signal()` with `{ transaction }`.
+ *
+ * ```ts
+ * @Module({
+ *   imports: [WorkflowsModule.forRoot()],
+ *   providers: [
+ *     {
+ *       provide: MySqlWorkflowStore,
+ *       inject: [getDrizzleToken(), WorkflowStorage],
+ *       useFactory: (db: Database, storage: WorkflowStorage) => new MySqlWorkflowStore({ executor: fromDrizzle(db) }, storage),
+ *     },
+ *   ],
+ * })
+ * export class AppModule {}
+ * ```
+ *
+ * Given `storage`, it registers itself (`storage.registerSource(this)`), as every store does. It checks the server and
+ * its schema, or migrates it (`migrate`), in `onModuleInit` (so before the worker starts), or at its first call
+ * outside Nest.
+ *
+ * On MySQL:
+ * - Ids, names and keys are case- and accent-sensitive, and bounded: an instance id, a workflow's or a signal's name,
+ *   a signal's key or id, a concurrency or rate-limit key hold at most 255 characters, a schedule's id 230, a
+ *   journal entry's name 512. A longer one fails with a `RangeError` that names it, before any SQL.
+ * - The application's transactions may be REPEATABLE READ, MySQL's default. `signal(..., { transaction })` finds the
+ *   instances to wake with a locking read of their waits, which sees the latest committed ones whatever the level; the
+ *   signal lock is held until the application's transaction ends, as on PostgreSQL. `start(..., { transaction })`
+ *   reads the last signal id from the transaction's snapshot: under REPEATABLE READ, signals committed after the
+ *   transaction's first read can match the new instance's waits.
+ * - The store's own transactions run READ COMMITTED and are run again when MySQL breaks a deadlock (1213). A deadlock in
+ *   the application's transaction rolled the application's own writes back too: the store lets it through, for the
+ *   application to retry its transaction.
+ */
+export declare class MySqlWorkflowStore implements WorkflowStore, OnModuleInit {
+    /**
+     * The SQL of the store's migrations, for teams that apply migrations with their own tool (drizzle-kit, TypeORM,
+     * Prisma Migrate, Flyway...) and run the store with `migrate: false`: from version `from` (default `0`, a new
+     * database) to `to` (default: the version this version of the package needs), with the bookkeeping that tells the
+     * store which versions a schema has (`<schema>_migrations`). MySQL commits each DDL statement on its own, so the
+     * statements don't run in one transaction: apply them in order, each once. Downgrades aren't supported. With
+     * `statementBreakpoints`, drizzle-kit's `--> statement-breakpoint` separates the statements, which its MySQL
+     * migrator needs (it runs one statement per call).
+     *
+     * ```ts
+     * // drizzle/0003_workflows.sql, created empty by `npx drizzle-kit generate --custom --name=workflows`
+     * writeFileSync('drizzle/0003_workflows.sql', MySqlWorkflowStore.migrationSql({ statementBreakpoints: true }));
+     * ```
+     */
+    static migrationSql(options?: MigrationSqlOptions): string;
+    /**
+     * `migrationSql()`'s statements, one per string, for a migration tool that runs one statement per call (TypeORM's
+     * `queryRunner.query()`, mysql2): run them in order, each once.
+     *
+     * ```ts
+     * export class Workflows1790000000000 implements MigrationInterface {
+     *   async up(queryRunner: QueryRunner): Promise<void> {
+     *     for (const statement of MySqlWorkflowStore.migrationStatements()) {
+     *       await queryRunner.query(statement);
+     *     }
+     *   }
+     * }
+     * ```
+     */
+    static migrationStatements(options?: MigrationStatementsOptions): string[];
+    /** The schema version this version of the package needs: its last migration. */
+    static readonly schemaVersion: number;
+    private readonly logger;
+    private readonly executor;
+    private readonly schema;
+    private readonly t;
+    /** The server and the schema checked, and the schema migrated (`migrate`), before the first statement. */
+    private readonly readiness;
+    /** The lock keys whose rows this store created, or found (see `createLockRows()`). */
+    private readonly lockRows;
+    constructor(options: MySqlWorkflowStoreOptions, storage?: WorkflowStorage);
+    /**
+     * Checks the server and migrates the schema (`migrate`) or checks it, before the worker starts: startup fails if it
+     * can't serve. Then creates the signal lock's row (see `createLockRows()`).
+     */
+    onModuleInit(): Promise<void>;
+    /**
+     * Applies the migrations the schema hasn't had yet, whatever `migrate` says, one statement at a time under a lock
+     * (`GET_LOCK()`): of processes that migrate together, one applies them, and a run that stopped halfway resumes where
+     * it stopped. Resolves to the versions it applied (`[]`: none were pending).
+     */
+    migrate(): Promise<number[]>;
+    create(instance: NewWorkflowInstance): Promise<{
+        instance: WorkflowInstance;
+        created: boolean;
+    }>;
+    /** `start(..., { transaction })`: the instance commits or rolls back with the application's rows. */
+    createInTransaction(transaction: unknown, instance: NewWorkflowInstance): Promise<{
+        instance: WorkflowInstance;
+        created: boolean;
+    }>;
+    get(id: string, options?: {
+        journal?: boolean;
+    }): Promise<WorkflowInstanceDetails | null>;
+    list(query: WorkflowListQuery): Promise<WorkflowInstance[]>;
+    requestCancel(id: string, { reason, now, terminate }: WorkflowCancelRequest): Promise<boolean>;
+    /** `WorkflowClient.retry()`: one conditional update, then the journal, in one transaction. */
+    reopen(id: string, reopen: WorkflowReopen): Promise<boolean>;
+    /** `WorkflowClient.delete()`: its journal and waits go with it, in the same transaction. */
+    delete(id: string, statuses: WorkflowStatus[]): Promise<boolean>;
+    signal(signal: NewWorkflowSignal): Promise<WorkflowSignalResult>;
+    /**
+     * `signal(..., { transaction })`: the signal and its wake-ups commit with the application's rows. Any isolation
+     * level: the wake-up reads the latest committed waits with a locking read, even under REPEATABLE READ.
+     */
+    signalInTransaction(transaction: unknown, signal: NewWorkflowSignal): Promise<WorkflowSignalResult>;
+    signals(query: WorkflowSignalQuery): Promise<WorkflowSignalRecord[]>;
+    purge(query: WorkflowPurgeQuery): Promise<WorkflowPurgeResult>;
+    saveSchedule(save: WorkflowScheduleSave): Promise<WorkflowScheduleRecord | null>;
+    getSchedule(id: string): Promise<WorkflowScheduleRecord | null>;
+    listSchedules(query: WorkflowScheduleQuery): Promise<WorkflowScheduleRecord[]>;
+    deleteSchedule(id: string, revision?: number): Promise<boolean>;
+    claimSchedules(request: WorkflowScheduleClaimRequest): Promise<WorkflowScheduleRecord[]>;
+    writeSchedule(id: string, token: string, write: WorkflowScheduleWrite): Promise<boolean>;
+    claim(request: WorkflowClaimRequest): Promise<WorkflowClaim>;
+    /**
+     * A claim under concurrency or rate limits. Claims of a workflow with a concurrency limit take its lock first (the
+     * kit's `lockKeys()`, in one order), so they count the slots live leases hold and lease the instances that fit one
+     * after the other: two never both take the last slot. Rate-limit windows are counted again under their rows' locks,
+     * once the instances are picked (see `takeRoom()`).
+     */
+    private claimWithin;
+    /**
+     * Of `picked` (in claim order), the instances their rate-limit windows have room for, recorded in the windows. Each
+     * window's row is inserted where missing and locked, in one order (`RateWindowClaim.windows`): the count read under
+     * the lock is exact (a concurrent claim of the same window waits for this one), and a purge can't delete the row in
+     * between.
+     */
+    private takeRoom;
+    /**
+     * Locks up to `limit` rows of `table` (as `alias`) that `due` selects, in `order`, passing over the rows other
+     * transactions hold, as PostgreSQL's `FOR UPDATE SKIP LOCKED` under `ORDER BY ... LIMIT` does. InnoDB's own locks
+     * every row its sort reads, not only those it returns (a claim of 7 among 60 due instances held all 60, and a
+     * concurrent claim found none), so each round:
+     *
+     * 1. a consistent read finds the next candidates in `order` (it locks nothing, whatever the plan): as many as the rows
+     *    still wanted, and as many more as other transactions held in the rounds before, so a round reaches past the rows
+     *    the claims beside this one are taking;
+     * 2. a locking read takes those still due that no one holds (`SKIP LOCKED`), by primary key and in its order, up to the
+     *    rows wanted: InnoDB stops there, and locks no candidate it doesn't return. (Among the candidates of a round, the
+     *    ones taken are the first free by id: under contention, a claim may take a less overdue one than another claim.)
+     *
+     * Resolves to the ids locked.
+     */
+    private lockDue;
+    /** Leases the instances `ids` (locked by this transaction), and reads them back in claim order. */
+    private lease;
+    renew(id: string, token: string, leaseUntil: number): Promise<{
+        cancelRequested: boolean;
+        terminateRequested: boolean;
+    } | null>;
+    write(id: string, token: string, write: WorkflowWrite): Promise<boolean>;
+    /** The store's readiness (the server and the schema), then the signal lock's row: before its own first statement. */
+    private ready;
+    /**
+     * Creates the rows of lock keys (the kit's `<schema>_locks`, one per key, never deleted) ahead of time, with the kit's
+     * `ensureLockRows()`: in transactions of their own, which commit before any transaction takes those locks. When the
+     * transaction that created a lock's row rolls back, MySQL makes the transactions waiting for that row deadlock
+     * (1213): so a lock's first use, which may be in the application's transaction (`signal(..., { transaction })`) or in
+     * one that fails, never creates its row. The rows there already are only noted, by a read that takes no lock: a
+     * process's startup never waits behind the transactions holding them (a signal in the application's transaction).
+     * The store's lock keys are few: `signals`, and a concurrency lock per workflow with a limit. (A store first called
+     * inside the application's transaction, before `onModuleInit()` or any call outside one, creates the signal lock's row
+     * there.)
+     */
+    private createLockRows;
+    private insertInstance;
+    private insertSignal;
+    /**
+     * Replaces the instance's waits, and says whether a signal committed after the execution read its cursor matches
+     * one of the new ones: then the instance stays due instead of losing it. (Run under the signal lock, shared: every
+     * signal before it has committed, and none after it can commit before this transaction.)
+     */
+    private replaceWaits;
+    /** Journal entries by name: a new name goes last (`seq`), a known one is replaced in place. */
+    private upsertEntries;
+    /** Deletes the journals and waits of instances deleted in this transaction. */
+    private deleteChildren;
+    private readSchedule;
+    /** Due, unleased instances of the versions the worker runs, as `alias` in a statement of `p`. */
+    private isDue;
+    private lastSignalId;
+}