@webda/workflow 0.0.0-stage → 4.0.0-beta.14

package/lib/workflow.service.js

@@ -0,0 +1,1951 @@
+var __runInitializers = (this && this.__runInitializers) || function (thisArg, initializers, value) {
+    var useValue = arguments.length > 2;
+    for (var i = 0; i < initializers.length; i++) {
+        value = useValue ? initializers[i].call(thisArg, value) : initializers[i].call(thisArg);
+    }
+    return useValue ? value : void 0;
+};
+var __esDecorate = (this && this.__esDecorate) || function (ctor, descriptorIn, decorators, contextIn, initializers, extraInitializers) {
+    function accept(f) { if (f !== void 0 && typeof f !== "function") throw new TypeError("Function expected"); return f; }
+    var kind = contextIn.kind, key = kind === "getter" ? "get" : kind === "setter" ? "set" : "value";
+    var target = !descriptorIn && ctor ? contextIn["static"] ? ctor : ctor.prototype : null;
+    var descriptor = descriptorIn || (target ? Object.getOwnPropertyDescriptor(target, contextIn.name) : {});
+    var _, done = false;
+    for (var i = decorators.length - 1; i >= 0; i--) {
+        var context = {};
+        for (var p in contextIn) context[p] = p === "access" ? {} : contextIn[p];
+        for (var p in contextIn.access) context.access[p] = contextIn.access[p];
+        context.addInitializer = function (f) { if (done) throw new TypeError("Cannot add initializers after decoration has completed"); extraInitializers.push(accept(f || null)); };
+        var result = (0, decorators[i])(kind === "accessor" ? { get: descriptor.get, set: descriptor.set } : descriptor[key], context);
+        if (kind === "accessor") {
+            if (result === void 0) continue;
+            if (result === null || typeof result !== "object") throw new TypeError("Object expected");
+            if (_ = accept(result.get)) descriptor.get = _;
+            if (_ = accept(result.set)) descriptor.set = _;
+            if (_ = accept(result.init)) initializers.unshift(_);
+        }
+        else if (_ = accept(result)) {
+            if (kind === "field") initializers.unshift(_);
+            else descriptor[key] = _;
+        }
+    }
+    if (target) Object.defineProperty(target, contextIn.name, descriptor);
+    done = true;
+};
+import { escape } from "@webda/ql";
+import { callOperation, canCallOperation, CronService, hasSchema, Operation, listFullOperations, registerOperation, registerSchema, runAsSystem, Service, ServiceParameters, Session, SimpleOperationContext, useApplication, useContext, useCore, useDynamicService, useInstanceStorage, useSystem, WebdaError } from "@webda/core";
+import { Duration } from "@webda/utils";
+import { createHash, randomUUID } from "node:crypto";
+import { BUILTINS, flowVersion } from "./graph.js";
+import { keysOf, run, statementIdOf } from "./interpreter.js";
+import { setWorkflowControls } from "./hooks.js";
+import { WorkflowInstance } from "./models/workflowinstance.model.js";
+import { WorkflowTask } from "./models/workflowtask.model.js";
+import { WorkflowSignal } from "./models/workflowsignal.model.js";
+import { Workflow as StoredWorkflow } from "./models/workflow.model.js";
+import { validateExpr } from "./expression.js";
+import { useRepository } from "@webda/models";
+/**
+ * Output schema of every flow launch operation
+ */
+export const LAUNCH_OUTPUT_SCHEMA = "Webda/WorkflowLaunch.output";
+/**
+ * Status of a task error that is never retried: the same call would fail the same way
+ */
+const NOT_RETRYABLE_STATUS = {
+    400: "BadRequest",
+    401: "Unauthorized",
+    403: "Forbidden",
+    404: "NotFound"
+};
+/**
+ * An error of a task, as the engine reports it in the flow
+ */
+class TaskError extends Error {
+    code;
+    retryable;
+    /**
+     * Create a new TaskError instance
+     * @param code - error code
+     * @param message - error message
+     * @param retryable - whether another attempt may succeed
+     */
+    constructor(code, message, retryable = true) {
+        super(message);
+        this.code = code;
+        this.retryable = retryable;
+    }
+}
+/**
+ * Statuses of a task row that will still produce an outcome
+ */
+const LIVE_TASK = ["PENDING", "RUNNING", "WAITING_EVENT", "WAITING_CHILD"];
+/**
+ * `runAt` of a wait without timeout (JSON has no Infinity)
+ */
+export const NO_TIMEOUT = Number.MAX_SAFE_INTEGER;
+/**
+ * Built-ins an instance waits on: when all its pending tasks are of these, it is WAITING
+ */
+const WAITING_OPERATIONS = [BUILTINS.Sleep, BUILTINS.WaitForEvent];
+/**
+ * Statement kinds of a flow graph, with their expression fields
+ */
+const STATEMENT_EXPRESSIONS = {
+    task: ["input"],
+    assign: ["value"],
+    if: ["cond"],
+    forOf: ["items"],
+    while: ["cond"],
+    try: [],
+    parallel: [],
+    map: ["items"],
+    return: ["value"],
+    throw: ["value"]
+};
+/**
+ * Operation id syntax
+ */
+const OPERATION_ID = /^([A-Z][A-Za-z0-9]*\.)*([A-Z][a-zA-Z0-9]*)$/;
+/**
+ * Whether an object is an instance of a class with this name, its own or an ancestor
+ * @param object - the object
+ * @param className - the class name
+ * @returns true when a constructor of its prototype chain has this name
+ */
+function isInstanceOfClass(object, className) {
+    for (let proto = object ? Object.getPrototypeOf(object) : null; proto; proto = Object.getPrototypeOf(proto)) {
+        if (proto.constructor?.name === className)
+            return true;
+    }
+    return false;
+}
+/**
+ * Abort reason of an attempt whose outcome must not be written: its lease was lost, or the service stopped
+ */
+const DROPPED = "WorkflowAttemptDropped";
+/**
+ * Abort signal of the current workflow task attempt
+ *
+ * It is aborted on timeout, when the worker loses the task lease, and when the service stops. An operation run by a
+ * workflow should observe it to stop early: an attempt that keeps running is ignored, and the task may run again
+ * (operations are assumed idempotent).
+ * @returns the signal, undefined outside a workflow task
+ */
+export function useAbortSignal() {
+    try {
+        return useContext()?.getExtension?.("abortSignal");
+    }
+    catch {
+        // A context without extensions (the global context)
+        return undefined;
+    }
+}
+/**
+ * Parameters of the {@link WorkflowService}
+ */
+export class WorkflowServiceParameters extends ServiceParameters {
+    /**
+     * Interval between two polls of the worker (ms)
+     * @default 1000
+     */
+    pollInterval;
+    /**
+     * Maximum number of tasks running at once on this replica
+     * @default 10
+     */
+    concurrency;
+    /**
+     * Lease of a claimed task (ms), renewed while it runs
+     * @default 60000
+     */
+    leaseDuration;
+    /**
+     * Maximum size of the journal and input of an instance (bytes of JSON)
+     * @default 262144
+     */
+    maxContextSize;
+    /**
+     * Run the worker loop on this replica
+     * @default true
+     */
+    worker;
+    /**
+     * Allow `useSystem` tasks in stored workflows
+     * @default false
+     */
+    allowStoredSystem;
+    /**
+     * How long a signal sent before any wait is kept (Duration syntax)
+     * @default "1d"
+     */
+    signalTtl;
+    /**
+     * Interval of the recovery sweep (ms): an active instance not updated for this long and without a live task is
+     * replayed
+     * @default 60000
+     */
+    sweepInterval;
+    /**
+     * Grace period of stop() for the running attempts (ms), then they are aborted and their outcome dropped
+     * @default 30000
+     */
+    stopTimeout;
+    /**
+     * Flows to register
+     */
+    flows;
+    /**
+     * Maximum size of a signal payload (bytes of JSON)
+     * @default 65536
+     */
+    maxSignalPayload;
+    /**
+     * Maximum number of unexpired buffered signals per `(event, key)`: a new one past it is refused
+     * @default 100
+     */
+    maxBufferedSignals;
+    /**
+     * Role needed to create, update or delete a stored workflow
+     * @default "workflow-admin"
+     */
+    storedWorkflowRole;
+    /**
+     * PubSub service announcing the stored workflow changes to the other replicas
+     */
+    syncPubSub;
+    /**
+     * Without `syncPubSub`, interval between two reloads of the stored workflows (ms)
+     * @default 60000
+     */
+    syncInterval;
+    /**
+     * @override
+     * @param params - the input parameters
+     * @returns this
+     */
+    load(params = {}) {
+        super.load(params);
+        this.pollInterval ??= 1000;
+        this.concurrency ??= 10;
+        this.leaseDuration ??= 60000;
+        this.maxContextSize ??= 262144;
+        this.worker ??= true;
+        this.allowStoredSystem ??= false;
+        this.sweepInterval ??= 60000;
+        this.stopTimeout ??= 30000;
+        this.signalTtl ??= "1d";
+        this.storedWorkflowRole ??= "workflow-admin";
+        this.maxSignalPayload ??= 65536;
+        this.maxBufferedSignals ??= 100;
+        this.syncInterval ??= 60000;
+        return this;
+    }
+}
+/**
+ * Whether an instance still runs
+ * @param instance - the instance
+ * @returns true when RUNNING or WAITING
+ */
+function isActive(instance) {
+    return instance?.status === "RUNNING" || instance?.status === "WAITING";
+}
+/**
+ * Whether two flow errors are the same
+ * @param a - an error
+ * @param b - another error
+ * @returns true when code, message and operation match
+ */
+function sameError(a, b) {
+    return !!a && !!b && a.code === b.code && a.message === b.message && a.operation === b.operation;
+}
+/**
+ * Uuid of the n-th row of a task key: the same for every worker, so two advances cannot create it twice
+ * @param instance - instance uuid
+ * @param key - task key
+ * @param index - row index of the key, from 1
+ * @returns a uuid
+ */
+function taskUuid(instance, key, index) {
+    const h = createHash("sha256").update(`${instance}\n${key}\n${index}`).digest("hex");
+    return `${h.substring(0, 8)}-${h.substring(8, 12)}-5${h.substring(13, 16)}-a${h.substring(17, 20)}-${h.substring(20, 32)}`;
+}
+/**
+ * Runs durable workflows: registers each flow as an operation, launches instances and runs their tasks
+ *
+ * Every replica runs the worker loop: a task is claimed with a lease (a conditional patch on `leaseUntil`), so it
+ * runs on one replica at a time. Its outcome goes to the instance journal (a conditional patch on `revision`), then
+ * the flow is replayed to find the next tasks.
+ *
+ * @WebdaModda WorkflowService
+ */
+let WorkflowService = (() => {
+    let _classSuper = Service;
+    let _instanceExtraInitializers = [];
+    let _signal_decorators;
+    return class WorkflowService extends _classSuper {
+        static {
+            const _metadata = typeof Symbol === "function" && Symbol.metadata ? Object.create(_classSuper[Symbol.metadata] ?? null) : void 0;
+            _signal_decorators = [Operation({
+                    id: "Workflow.Signal",
+                    rest: { method: "post", path: "workflows/signal" },
+                    graphql: false,
+                    grpc: false
+                })];
+            __esDecorate(this, null, _signal_decorators, { kind: "method", name: "signal", static: false, private: false, access: { has: obj => "signal" in obj, get: obj => obj.signal }, metadata: _metadata }, null, _instanceExtraInitializers);
+            if (_metadata) Object.defineProperty(this, Symbol.metadata, { enumerable: true, configurable: true, writable: true, value: _metadata });
+        }
+        /**
+         * Flows registered on this service, by flow id
+         */
+        flows = (__runInitializers(this, _instanceExtraInitializers), new Map());
+        /**
+         * Every version registered of each flow, for migrate
+         */
+        versions = new Map();
+        /**
+         * Flows of the modules (compiled `.flow.ts`) registered by this service
+         */
+        moduleFlows = new Set();
+        /**
+         * Worker id, the lease owner of the tasks it claims
+         */
+        workerId = randomUUID();
+        /**
+         * Attempts running on this replica, by task uuid
+         */
+        running = new Map();
+        /**
+         * Tasks reclaimed after too many crashes: they fail without running
+         */
+        crashed = new WeakSet();
+        /**
+         * The service is stopping: nothing more is claimed
+         */
+        stopping = false;
+        /**
+         * Poll in progress
+         */
+        pollPromise;
+        /**
+         * Last recovery sweep (ms)
+         */
+        lastSweep = 0;
+        /**
+         * Executions started by the worker loop
+         */
+        executions = new Set();
+        /**
+         * Worker loop timer
+         */
+        timer;
+        /**
+         * Stored workflows registered here, by workflow uuid
+         */
+        storedFlows = new Map();
+        /**
+         * Stored workflows reload timer
+         */
+        syncTimer;
+        /**
+         * Subscription to the stored workflow changes
+         */
+        syncConsumer;
+        /**
+         * @override
+         */
+        resolve() {
+            super.resolve();
+            // The configured service serves the WorkflowInstance actions
+            setWorkflowControls(this);
+            // Before the transports init: they snapshot the operations
+            for (const def of this.parameters.flows ?? []) {
+                this.registerFlow({ ...def, version: def.version ?? "", source: def.source ?? "code" });
+            }
+            this.registerModuleFlows();
+            return this;
+        }
+        /**
+         * Register the flows compiled from the `.flow.ts` files (`flows` of the modules), with their crons
+         *
+         * The first WorkflowService to resolve registers them; the typed calls are resolved in {@link init}, once every
+         * service registered its operations.
+         */
+        registerModuleFlows() {
+            const flows = useApplication().getModules()?.flows ?? {};
+            for (const [id, entry] of Object.entries(flows)) {
+                const operation = listFullOperations()[id];
+                if (operation?.context?.flowId === id && operation.service !== this.getName())
+                    continue;
+                if (operation && !operation.context?.flowId) {
+                    throw new WebdaError.FatalInitError(`Flow ${id}: its operation id is used by the operation of service ${operation.service}`);
+                }
+                const { crons, ...def } = structuredClone(entry);
+                // The compiled version is kept: the build hashed the graph before its targets are resolved
+                this.registerFlow({ ...def, source: "code" });
+                this.moduleFlows.add(id);
+                for (const cron of crons ?? []) {
+                    CronService.register(id, cron);
+                }
+            }
+        }
+        /**
+         * The operation a typed call (`target: { class, method }`) runs: the operation of the configured service whose
+         * instance is of that class (matched by constructor name, the class or an ancestor)
+         * @param target - the target of the task
+         * @param target.class - the service class name
+         * @param target.method - the method name
+         * @returns the operation id, or why there is none: no service (missing) or several (ambiguous)
+         */
+        resolveTarget(target) {
+            const { class: className, method } = target ?? {};
+            const operations = Object.entries(listFullOperations());
+            const matches = Object.entries(useCore().getServices())
+                .filter(([, service]) => isInstanceOfClass(service, className))
+                .flatMap(([name]) => operations.filter(([, op]) => op.service === name && op.method === method).map(([opId]) => ({ name, opId })));
+            if (matches.length === 1)
+                return { operation: matches[0].opId };
+            return matches.length
+                ? {
+                    ambiguous: `several configured services expose ${className}.${method}: ${matches.map(m => m.name).join(", ")}`
+                }
+                : { missing: `no configured service of class ${className} exposes ${method} as an operation` };
+        }
+        /**
+         * Resolve the typed calls of a flow body in place
+         * @param body - the statements
+         * @returns the problems, by statement id
+         */
+        resolveTargets(body) {
+            const problems = { missing: [], ambiguous: [] };
+            const walk = (stmts) => {
+                for (const stmt of stmts ?? []) {
+                    for (const key of ["then", "else", "body", "catch", "finally"])
+                        walk(stmt[key]);
+                    (stmt.branches ?? []).forEach(walk);
+                    if (stmt.kind !== "task" || !stmt.target)
+                        continue;
+                    const res = this.resolveTarget(stmt.target);
+                    if (res.operation)
+                        stmt.operation = res.operation;
+                    else if (res.ambiguous)
+                        problems.ambiguous.push(`${stmt.id}: ${res.ambiguous}`);
+                    else
+                        problems.missing.push(`${stmt.id}: ${res.missing}`);
+                }
+            };
+            walk(body);
+            return problems;
+        }
+        /**
+         * Point every typed call of the compiled flows at the operation of the configured service of its class
+         *
+         * A flow whose service is not configured (a packaged unit without it) is removed with its crons; a call that several
+         * services serve, or a flow whose operation was taken over, stops the startup.
+         * @throws FatalInitError on an ambiguous call, or a flow id taken by another operation
+         */
+        resolveModuleTargets() {
+            for (const id of [...this.moduleFlows]) {
+                const operation = listFullOperations()[id];
+                if (operation?.context?.flowId !== id || operation.service !== this.getName()) {
+                    throw new WebdaError.FatalInitError(`Flow ${id}: its operation id is used by the operation of service ${operation?.service}`);
+                }
+                const flow = this.flows.get(id);
+                const { missing, ambiguous } = this.resolveTargets(flow.body);
+                if (ambiguous.length) {
+                    throw new WebdaError.FatalInitError(`Flow ${id} calls an ambiguous operation: ${ambiguous.join("; ")}`);
+                }
+                if (missing.length) {
+                    this.log("ERROR", `Flow ${id} is not registered: ${missing.join("; ")}`);
+                    this.unregisterModuleFlow(id);
+                }
+            }
+        }
+        /**
+         * Remove a module flow: its operation and its crons
+         * @param id - the flow id
+         */
+        unregisterModuleFlow(id) {
+            this.moduleFlows.delete(id);
+            this.flows.delete(id);
+            this.versions.delete(id);
+            CronService.unregister(id);
+            const operation = useInstanceStorage().operations?.[id];
+            if (operation?.service === this.getName() && operation.context?.flowId === id) {
+                delete useInstanceStorage().operations[id];
+            }
+        }
+        /**
+         * @override
+         */
+        async init() {
+            await super.init();
+            this.resolveModuleTargets();
+            this.listenStored();
+            await this.reloadStored();
+            this.startSync();
+            if (this.parameters.worker) {
+                this.startWorker();
+            }
+            return this;
+        }
+        /**
+         * Stop the worker loop
+         *
+         * Running attempts get `stopTimeout` to finish, then they are aborted and their outcome dropped: their lease
+         * expires and another worker runs them again.
+         * @override
+         */
+        async stop() {
+            this.stopping = true;
+            if (this.timer) {
+                clearInterval(this.timer);
+                this.timer = undefined;
+            }
+            if (this.syncTimer) {
+                clearInterval(this.syncTimer);
+                this.syncTimer = undefined;
+            }
+            await this.syncConsumer?.cancel();
+            this.syncConsumer = undefined;
+            // A poll in flight may still start executions
+            await this.pollPromise;
+            let timer;
+            const grace = new Promise(resolve => {
+                timer = setTimeout(resolve, this.parameters.stopTimeout);
+            });
+            await Promise.race([Promise.allSettled([...this.executions]), grace]);
+            clearTimeout(timer);
+            for (const attempt of this.running.values()) {
+                attempt.controller.abort(DROPPED);
+            }
+            await super.stop();
+        }
+        /**
+         * Register a schema in the validator and the application
+         * @param name - schema name
+         * @param schema - the JSON schema
+         */
+        registerSchema(name, schema) {
+            if (!hasSchema(name)) {
+                registerSchema(name, schema);
+            }
+            useApplication().getSchemas()[name] = schema;
+        }
+        /**
+         * Register a flow as an operation `<flow id>` served by {@link launch}
+         * @param def - the flow definition
+         */
+        registerFlow(def) {
+            const flow = { ...def, version: def.version || flowVersion(def) };
+            this.flows.set(flow.id, flow);
+            if (!this.versions.has(flow.id))
+                this.versions.set(flow.id, new Map());
+            this.versions.get(flow.id).set(flow.version, flow);
+            // Core cannot remove a schema: a stored flow, which changes at runtime, has one per version
+            const input = flow.source === "stored" ? `${flow.id}.input.${flow.version}` : `${flow.id}.input`;
+            // The parameters are given to launch in this order
+            this.registerSchema(input, {
+                type: "object",
+                ...flow.input,
+                properties: flow.input?.properties ?? {},
+                "x-webda-arguments": flow.params
+            });
+            this.registerSchema(LAUNCH_OUTPUT_SCHEMA, {
+                type: "object",
+                properties: { uuid: { type: "string" } },
+                required: ["uuid"]
+            });
+            registerOperation(flow.id, {
+                service: this.getName(),
+                method: "launch",
+                input,
+                output: LAUNCH_OUTPUT_SCHEMA,
+                context: { flowId: flow.id },
+                graphql: false,
+                grpc: false
+            });
+        }
+        /**
+         * Get a registered flow
+         * @param flowId - the flow id
+         * @returns the definition
+         */
+        getFlow(flowId) {
+            return this.flows.get(flowId);
+        }
+        /**
+         * Find a flow registered on any WorkflowService, by its operation
+         * @param flowId - the flow id
+         * @returns the current definition, undefined when the operation is not a flow
+         */
+        resolveFlow(flowId) {
+            if (!flowId)
+                return undefined;
+            const local = this.flows.get(flowId);
+            if (local)
+                return local;
+            const operation = listFullOperations()[flowId];
+            if (operation?.context?.flowId !== flowId || !operation.service)
+                return undefined;
+            const service = useDynamicService(operation.service);
+            return typeof service?.getFlow === "function" ? service.getFlow(flowId) : undefined;
+        }
+        /**
+         * Find a version of a flow
+         * @param flowId - the flow id
+         * @param version - the version
+         * @returns the definition
+         */
+        getFlowVersion(flowId, version) {
+            const local = this.versions.get(flowId)?.get(version);
+            if (local)
+                return local;
+            const operation = listFullOperations()[flowId];
+            const service = operation?.service ? useDynamicService(operation.service) : undefined;
+            return service !== this && typeof service?.getFlowVersion === "function"
+                ? service.getFlowVersion(flowId, version)
+                : undefined;
+        }
+        /**
+         * Role needed to write a stored workflow
+         * @returns the role
+         */
+        getStoredWorkflowRole() {
+            return this.parameters.storedWorkflowRole;
+        }
+        /**
+         * Validate and stamp stored workflows on save, and register them once saved
+         */
+        listenStored() {
+            const repo = useRepository(StoredWorkflow);
+            repo.on("Create", ({ object }) => this.prepareStored(object, object));
+            repo.on("Update", ({ object }) => this.prepareStored(object, object));
+            repo.on("Patch", async ({ object_id, object }) => {
+                const current = await runAsSystem(() => StoredWorkflow.ref(object_id).get());
+                await this.prepareStored({ ...current, ...object, uuid: object_id }, object);
+            });
+            // Partial updates (increments, collections, attribute removal) would skip the validation
+            repo.on("PartialUpdate", () => {
+                throw new WebdaError.BadRequest("A stored workflow is changed with update or patch, not partial updates");
+            });
+            for (const event of ["Created", "Updated", "Patched", "Deleted", "PartialUpdated"]) {
+                repo.on(event, ({ object_id }) => this.storedChanged(object_id));
+            }
+        }
+        /**
+         * Validate a stored workflow about to be saved and stamp its version and last saver
+         * @param workflow - the workflow as it will be saved
+         * @param target - the data written: receives `version` and `updatedBy`
+         */
+        prepareStored(workflow, target) {
+            this.validateStored(workflow);
+            target.version = flowVersion(workflow);
+            const context = useContext();
+            // A save by the system keeps the last saver: its crons never run as the system
+            if (context && !context.isGlobalContext() && context.getCurrentUserId()) {
+                target.updatedBy = context.getCurrentUserId();
+            }
+        }
+        /**
+         * Validate a stored workflow
+         * @param workflow - the workflow
+         * @throws BadRequest when its name, graph or options are refused
+         */
+        validateStored(workflow) {
+            const fail = (message) => {
+                throw new WebdaError.BadRequest(`Invalid workflow: ${message}`);
+            };
+            const name = workflow?.name;
+            if (typeof name !== "string" || !OPERATION_ID.test(name)) {
+                fail(`name ${name} must be an operation id (Group.Name)`);
+            }
+            // The registered operation decides: a module enabled later wins over a stored flow
+            if (listFullOperations()[name]) {
+                const flow = this.operationFlow(name);
+                if (!flow)
+                    fail(`${name} is already an operation`);
+                if (flow.source !== "stored")
+                    fail(`${name} is already a code flow`);
+            }
+            if ([...this.storedFlows].some(([uuid, f]) => f.flowId === name && uuid !== workflow.uuid)) {
+                fail(`${name} is already a stored workflow`);
+            }
+            if (!Array.isArray(workflow.params) || workflow.params.some((p) => typeof p !== "string")) {
+                fail("params must be an array of names");
+            }
+            if (workflow.input === null || typeof workflow.input !== "object") {
+                fail("input must be a JSON schema");
+            }
+            if (!Array.isArray(workflow.body)) {
+                fail("body must be an array of statements");
+            }
+            if (workflow.options?.useSystem && !this.parameters.allowStoredSystem) {
+                fail("useSystem is not allowed in a stored workflow");
+            }
+            for (const cron of workflow.crons ?? []) {
+                if (typeof cron?.cron !== "string" || (cron.input !== undefined && typeof cron.input !== "object")) {
+                    fail("a cron needs a cron expression and an object input");
+                }
+            }
+            const walk = (stmts, path) => {
+                if (!Array.isArray(stmts))
+                    fail(`${path} must be an array of statements`);
+                stmts.forEach((stmt, i) => {
+                    const where = typeof stmt?.id === "string" ? stmt.id : `${path}[${i}]`;
+                    if (typeof stmt?.id !== "string")
+                        fail(`${where} has no id`);
+                    const fields = STATEMENT_EXPRESSIONS[stmt.kind];
+                    if (!fields)
+                        fail(`${where}: unknown statement kind ${stmt.kind}`);
+                    for (const field of fields) {
+                        if (stmt[field] === undefined && field === "value" && stmt.kind === "return")
+                            continue;
+                        const invalid = validateExpr(stmt[field], `$.${field}`);
+                        if (invalid.length)
+                            fail(`${where}: invalid expression at ${invalid.join(", ")}`);
+                    }
+                    if (stmt.kind === "task") {
+                        if (stmt.target !== undefined) {
+                            // A typed call compiled from a .flow.ts: it runs the operation of the configured service of its class
+                            const res = this.resolveTarget(stmt.target);
+                            if (!res.operation)
+                                fail(`${where}: ${res.missing ?? res.ambiguous}`);
+                        }
+                        else {
+                            const known = Object.values(BUILTINS).includes(stmt.operation) ||
+                                stmt.operation === name ||
+                                !!listFullOperations()[stmt.operation];
+                            if (!known)
+                                fail(`${where}: unknown operation ${stmt.operation}`);
+                        }
+                        if (stmt.options?.useSystem && !this.parameters.allowStoredSystem) {
+                            fail(`${where}: useSystem is not allowed in a stored workflow`);
+                        }
+                    }
+                    for (const key of ["then", "else", "body", "catch", "finally"]) {
+                        if (stmt[key] !== undefined)
+                            walk(stmt[key], `${where}.${key}`);
+                    }
+                    (stmt.branches ?? []).forEach((branch, b) => walk(branch, `${where}.branches[${b}]`));
+                });
+            };
+            walk(workflow.body, "body");
+        }
+        /**
+         * The flow served by an operation, if it is a flow
+         * @param flowId - the operation id
+         * @returns the flow of the WorkflowService serving the operation
+         */
+        operationFlow(flowId) {
+            const operation = listFullOperations()[flowId];
+            if (operation?.context?.flowId !== flowId || !operation.service)
+                return undefined;
+            const service = operation.service === this.getName() ? this : useDynamicService(operation.service);
+            return typeof service?.getFlow === "function" ? service.getFlow(flowId) : undefined;
+        }
+        /**
+         * User a stored flow runs as when the system launches it: its last saver
+         * @param flowId - the flow id
+         * @returns the user id, undefined when unknown
+         */
+        storedRunAs(flowId) {
+            const local = [...this.storedFlows.values()].find(f => f.flowId === flowId);
+            if (local)
+                return local.updatedBy;
+            const operation = listFullOperations()[flowId];
+            const service = operation?.service ? useDynamicService(operation.service) : undefined;
+            return service && service !== this && typeof service.storedRunAs === "function"
+                ? service.storedRunAs(flowId)
+                : undefined;
+        }
+        /**
+         * Identity of a new instance: a stored flow never runs as the system, it runs as its last saver
+         * @param def - the flow
+         * @param launcher - the identity of the caller (launch) or of the parent instance (child flows)
+         * @returns the launcher of the instance
+         * @throws Forbidden when a stored flow launched by the system has no user to run as
+         */
+        instanceLauncher(def, launcher) {
+            if (!launcher?.system || def.source !== "stored")
+                return launcher;
+            const updatedBy = this.storedRunAs(def.id);
+            if (!updatedBy) {
+                throw new WebdaError.Forbidden(`${def.id} has no user to run as`);
+            }
+            return { userId: updatedBy };
+        }
+        /**
+         * Identity of a child flow, which must be allowed to call the flow operation like a launch
+         * @param def - the child flow
+         * @param launcher - the launcher of the parent
+         * @param input - the child input
+         * @returns the launcher of the child
+         * @throws TaskError Forbidden when the child may not be started
+         */
+        async childLauncher(def, launcher, input) {
+            let child;
+            try {
+                child = this.instanceLauncher(def, launcher);
+            }
+            catch (err) {
+                throw new TaskError("Forbidden", err.message, false);
+            }
+            const ctx = child.system ? useSystem(input) : this.launcherContext(child, input);
+            if (!(await canCallOperation(ctx, def.id, { input }))) {
+                throw new TaskError("Forbidden", `${def.id} PermissionDenied`, false);
+            }
+            return child;
+        }
+        /**
+         * A stored workflow changed on this replica: register it here and tell the others
+         * @param uuid - the workflow uuid
+         */
+        async storedChanged(uuid) {
+            await this.syncStored(uuid);
+            if (this.parameters.syncPubSub) {
+                await useDynamicService(this.parameters.syncPubSub).sendMessage({ workflow: uuid });
+            }
+        }
+        /**
+         * Register, re-register or remove a stored workflow from the store
+         * @param uuid - the workflow uuid
+         */
+        async syncStored(uuid) {
+            const workflow = await runAsSystem(async () => (await StoredWorkflow.ref(uuid).exists()) ? (await StoredWorkflow.ref(uuid).get()) : undefined);
+            if (workflow) {
+                this.registerStored(workflow);
+            }
+            else {
+                this.unregisterStored(uuid);
+            }
+        }
+        /**
+         * Validate then register a stored workflow; an invalid one is removed and logged
+         * @param workflow - the stored workflow
+         * @returns true when registered
+         */
+        registerStored(workflow) {
+            try {
+                // Every registration path: the store may hold a workflow written behind the validation
+                this.validateStored(workflow);
+                this.applyStored(workflow);
+                return true;
+            }
+            catch (err) {
+                this.log("ERROR", `Stored workflow ${workflow?.name} (${workflow?.uuid}) is not registered`, err);
+                this.unregisterStored(workflow.uuid);
+                return false;
+            }
+        }
+        /**
+         * Register a stored workflow as a flow, with its crons
+         * @param workflow - the stored workflow
+         */
+        applyStored(workflow) {
+            const known = this.storedFlows.get(workflow.uuid);
+            const crons = JSON.stringify(workflow.crons ?? []);
+            const version = workflow.version || flowVersion(workflow);
+            if (known && known.flowId === workflow.name && known.version === version && known.crons === crons) {
+                known.updatedBy = workflow.updatedBy;
+                return;
+            }
+            if (known && known.flowId !== workflow.name) {
+                this.unregisterStored(workflow.uuid);
+            }
+            // The typed calls of a graph compiled from a .flow.ts run the configured operation, as for the code flow
+            const body = structuredClone(workflow.body);
+            this.resolveTargets(body);
+            this.registerFlow({
+                id: workflow.name,
+                version,
+                params: workflow.params,
+                input: workflow.input,
+                body,
+                options: workflow.options,
+                source: "stored"
+            });
+            for (const id of known?.cronIds ?? [])
+                CronService.unregister(id);
+            const cronIds = [];
+            for (const cron of workflow.crons ?? []) {
+                try {
+                    cronIds.push(CronService.getExportId(CronService.register(workflow.name, cron)));
+                }
+                catch (err) {
+                    this.log("ERROR", `Cannot register the cron ${cron.cron} of ${workflow.name}`, err);
+                }
+            }
+            this.storedFlows.set(workflow.uuid, {
+                flowId: workflow.name,
+                version,
+                updatedBy: workflow.updatedBy,
+                crons,
+                cronIds
+            });
+        }
+        /**
+         * Remove a stored workflow: its operation and its crons (running instances keep their pinned definition)
+         * @param uuid - the workflow uuid
+         */
+        unregisterStored(uuid) {
+            const known = this.storedFlows.get(uuid);
+            if (!known)
+                return;
+            this.storedFlows.delete(uuid);
+            for (const id of known.cronIds)
+                CronService.unregister(id);
+            // Only what this stored flow registered: a code flow or another operation of the same name stays
+            if (this.flows.get(known.flowId)?.source !== "stored")
+                return;
+            this.flows.delete(known.flowId);
+            const operation = useInstanceStorage().operations?.[known.flowId];
+            if (operation?.service === this.getName() && operation.context?.flowId === known.flowId) {
+                delete useInstanceStorage().operations[known.flowId];
+            }
+        }
+        /**
+         * Load every stored workflow: register the new and changed ones, remove the deleted ones
+         */
+        async reloadStored() {
+            const workflows = await runAsSystem(async () => {
+                const all = [];
+                for await (const workflow of StoredWorkflow.iterate(""))
+                    all.push(workflow);
+                return all;
+            });
+            const seen = new Set();
+            for (const workflow of workflows) {
+                seen.add(workflow.uuid);
+                this.registerStored(workflow);
+            }
+            for (const uuid of [...this.storedFlows.keys()]) {
+                if (!seen.has(uuid))
+                    this.unregisterStored(uuid);
+            }
+        }
+        /**
+         * Follow the stored workflow changes of the other replicas: through `syncPubSub`, or a reload every
+         * `syncInterval`
+         */
+        startSync() {
+            if (this.parameters.syncPubSub) {
+                this.syncConsumer ??= useDynamicService(this.parameters.syncPubSub).consume(async (event) => {
+                    if (event?.workflow)
+                        await this.syncStored(String(event.workflow));
+                });
+                return;
+            }
+            if (this.syncTimer)
+                return;
+            this.syncTimer = setInterval(() => {
+                this.reloadStored().catch(err => this.log("ERROR", "Cannot reload the stored workflows", err));
+            }, this.parameters.syncInterval);
+            this.syncTimer.unref?.();
+        }
+        /**
+         * Launch the flow of the current operation, as the caller
+         * @param args - the flow parameters, in order
+         * @returns the instance uuid
+         */
+        async launch(...args) {
+            const context = useContext();
+            const flowId = context.getExtension("operationContext")?.flowId;
+            const def = flowId ? this.flows.get(flowId) : undefined;
+            if (!def) {
+                throw new WebdaError.NotFound(`Unknown flow ${flowId}`);
+            }
+            const input = {};
+            def.params.forEach((name, i) => {
+                if (args[i] !== undefined)
+                    input[name] = args[i];
+            });
+            // A stored workflow never runs as the system: a cron tick runs it as the user who saved it last
+            const session = context.getSession();
+            const launcher = this.instanceLauncher(def, context.isGlobalContext()
+                ? { system: true }
+                : {
+                    userId: context.getCurrentUserId() || undefined,
+                    identId: session.identUsed,
+                    scopes: session.scopes,
+                    clientId: session.clientId
+                });
+            const instance = await runAsSystem(() => this.createInstance(def, input, launcher));
+            return { uuid: instance.uuid };
+        }
+        /**
+         * Create an instance and run it to its first tasks
+         * @param def - the flow definition, pinned in the instance
+         * @param input - the flow input
+         * @param launcher - the identity of the steps
+         * @param options - the parent of an awaited child, a fixed uuid (created once)
+         * @param options.parent - parent instance and task key
+         * @param options.parent.instance - parent instance uuid
+         * @param options.parent.key - task key in the parent
+         * @param options.uuid - uuid of the instance: an existing one is not created again
+         * @returns the instance
+         */
+        async createInstance(def, input, launcher, options = {}) {
+            const now = Date.now();
+            if (options.uuid && (await WorkflowInstance.ref(options.uuid).exists())) {
+                return (await WorkflowInstance.ref(options.uuid).get());
+            }
+            const instance = await WorkflowInstance.create({
+                ...(options.uuid ? { uuid: options.uuid } : {}),
+                ...(options.parent ? { parent: options.parent } : {}),
+                flowId: def.id,
+                version: def.version,
+                definition: def,
+                input,
+                journal: {},
+                status: "RUNNING",
+                launcher: JSON.parse(JSON.stringify(launcher)),
+                revision: 0,
+                createdAt: now,
+                updatedAt: now
+            });
+            await this.advance(instance.uuid);
+            return instance;
+        }
+        /**
+         * Start the worker loop
+         */
+        startWorker() {
+            if (this.timer)
+                return;
+            this.timer = setInterval(() => this.poll(), this.parameters.pollInterval);
+            this.timer.unref?.();
+        }
+        /**
+         * One iteration of the worker loop: claim due tasks and run them in the background
+         * @returns the poll in progress, it ends when the tasks are claimed and started
+         */
+        poll() {
+            if (this.pollPromise)
+                return this.pollPromise;
+            if (this.stopping)
+                return Promise.resolve();
+            this.pollPromise = this.claimDue()
+                .then(tasks => {
+                for (const task of tasks) {
+                    const execution = this.execute(task)
+                        .catch(err => this.log("ERROR", "Workflow task failed", task.uuid, err))
+                        .finally(() => this.executions.delete(execution));
+                    this.executions.add(execution);
+                }
+            })
+                .then(() => this.maybeSweep())
+                .catch(err => this.log("ERROR", "Workflow poll failed", err))
+                .finally(() => {
+                this.pollPromise = undefined;
+            });
+            return this.pollPromise;
+        }
+        /**
+         * Run the recovery sweep when `sweepInterval` passed since the last one
+         */
+        async maybeSweep() {
+            if (this.stopping || Date.now() - this.lastSweep < this.parameters.sweepInterval)
+                return;
+            this.lastSweep = Date.now();
+            await this.sweep();
+        }
+        /**
+         * Replay the stranded instances: active, not updated for `sweepInterval`, without a PENDING or RUNNING task
+         *
+         * A crash between two steps leaves such an instance: replay is idempotent, and the revision CAS of
+         * {@link advance} guards against a concurrent writer.
+         * @returns the number of instances replayed
+         */
+        async sweep() {
+            return runAsSystem(async () => {
+                const before = Date.now() - this.parameters.sweepInterval;
+                let count = 0;
+                for (const status of ["RUNNING", "WAITING"]) {
+                    const { results } = await WorkflowInstance.query(`status = ? AND updatedAt < ? LIMIT 100`, [status, before]);
+                    for (const instance of results) {
+                        const live = (await WorkflowTask.query(`instance = ? AND (status = 'PENDING' OR status = 'RUNNING' OR status = 'WAITING_EVENT' OR status = 'WAITING_CHILD') LIMIT 1`, [instance.uuid])).results;
+                        if (live.length)
+                            continue;
+                        await this.advance(instance.uuid);
+                        count++;
+                    }
+                }
+                // Buffered signals nobody consumed in time
+                const expired = (await WorkflowSignal.query(`expiresAt <= ? LIMIT 100`, [Date.now()])).results;
+                for (const signal of expired) {
+                    await WorkflowSignal.ref(signal.uuid)
+                        .delete()
+                        .catch(() => { });
+                }
+                return count;
+            });
+        }
+        /**
+         * One poll: claim the due tasks and run them
+         * @returns the number of tasks claimed
+         */
+        async tick() {
+            const tasks = await this.claimDue();
+            await Promise.all(tasks.map(task => this.execute(task)));
+            return tasks.length;
+        }
+        /**
+         * Claim the due tasks, up to the free slots of this replica
+         *
+         * The PENDING tasks due now, and the RUNNING ones whose lease expired (their worker crashed)
+         * @returns the tasks claimed
+         */
+        async claimDue() {
+            const slots = this.parameters.concurrency - this.running.size;
+            if (slots <= 0)
+                return [];
+            return runAsSystem(async () => {
+                const now = Date.now();
+                const candidates = [
+                    ...(await WorkflowTask.query(escape(["status = 'PENDING' AND runAt <= ", " LIMIT ", ""], [now, slots]))).results,
+                    ...(await WorkflowTask.query(escape(["status = 'RUNNING' AND leaseUntil < ", " LIMIT ", ""], [now, slots]))).results,
+                    // Waits whose timeout passed
+                    ...(await WorkflowTask.query(escape(["status = 'WAITING_EVENT' AND runAt <= ", " LIMIT ", ""], [now, slots]))).results
+                ];
+                const claimed = [];
+                for (const task of candidates) {
+                    if (claimed.length >= slots || this.stopping)
+                        break;
+                    if (await this.claim(task)) {
+                        claimed.push(task);
+                    }
+                }
+                return claimed;
+            });
+        }
+        /**
+         * Claim a task with a lease: a conditional patch on its `leaseUntil`
+         *
+         * Reclaiming an expired lease counts as a new attempt.
+         * @param task - the task, as queried
+         * @param extra - other fields to set with the claim
+         * @returns true when this worker holds the task
+         */
+        async claim(task, extra = {}) {
+            const now = Date.now();
+            const patch = {
+                ...extra,
+                status: "RUNNING",
+                owner: this.workerId,
+                leaseUntil: now + this.parameters.leaseDuration,
+                startedAt: now
+            };
+            const reclaim = task.status === "RUNNING";
+            if (reclaim) {
+                patch.attempt = task.attempt + 1;
+            }
+            const previous = task.leaseUntil;
+            try {
+                await WorkflowTask.ref(task.uuid).patch(patch, "leaseUntil", previous);
+            }
+            catch (err) {
+                const current = (await WorkflowTask.ref(task.uuid).get());
+                if (current && current.leaseUntil === previous)
+                    throw err;
+                // Another worker has it
+                return false;
+            }
+            Object.assign(task, patch);
+            // A crash recovery gets one more attempt than the retries: past it the task keeps killing its workers
+            if (reclaim && task.attempt - 1 > (task.options?.retries ?? 0) + 1) {
+                this.crashed.add(task);
+            }
+            return true;
+        }
+        /**
+         * Run a claimed task, write its outcome and advance its instance
+         *
+         * Every write of the attempt is conditional on the lease this worker holds: an attempt whose lease was lost (its
+         * worker was too slow and another one reclaimed it) drops its outcome.
+         * @param task - the task
+         * @param advance - advance the instance after the outcome is journaled
+         */
+        async execute(task, advance = true) {
+            const instance = await runAsSystem(() => WorkflowInstance.ref(task.instance).get());
+            if (!isActive(instance) || instance.journal?.[task.key]) {
+                // Cancelled, or its outcome is already journaled (a crash before the row was finished): never run it again
+                await this.finishTask(task, { status: "CANCELLED" });
+                return;
+            }
+            const controller = new AbortController();
+            this.running.set(task.uuid, { task, controller });
+            const renew = setInterval(() => this.renewLease(task), Math.max(this.parameters.leaseDuration / 2, 1));
+            renew.unref?.();
+            let entry;
+            let retryable = false;
+            try {
+                if (this.crashed.has(task)) {
+                    throw new TaskError("Crashed", `${task.operation} was abandoned by its worker ${task.attempt - 1} times`, false);
+                }
+                if (task.signaled) {
+                    // Resolved by a signal or a child: the outcome is on the row
+                    entry = task.error ? { error: task.error } : { result: task.output };
+                }
+                else {
+                    entry = { result: await this.runTask(task, instance, controller) };
+                }
+            }
+            catch (err) {
+                const error = this.toFlowError(err, task.operation);
+                retryable =
+                    !(err instanceof TaskError && !err.retryable) && !Object.values(NOT_RETRYABLE_STATUS).includes(error.code);
+                entry = { error };
+            }
+            finally {
+                clearInterval(renew);
+                this.running.delete(task.uuid);
+            }
+            if (controller.signal.reason === DROPPED || !(await this.holdsLease(task))) {
+                this.log("DEBUG", "Workflow task attempt dropped", task.uuid);
+                return;
+            }
+            if ("error" in entry) {
+                const retries = task.options?.retries ?? 0;
+                if (retryable && task.attempt <= retries) {
+                    await this.scheduleRetry(task);
+                    await this.finishTask(task, { status: "ERROR", error: entry.error });
+                    return;
+                }
+            }
+            await this.writeJournal(task.instance, task.key, entry);
+            await this.finishTask(task, "error" in entry ? { status: "ERROR", error: entry.error } : { status: "SUCCESS", output: entry.result });
+            if (advance) {
+                await this.advance(task.instance);
+            }
+        }
+        /**
+         * Conditional patch of a task row on the lease this worker holds
+         * @param task - the task, with the lease this worker holds
+         * @param data - the changes
+         * @returns false when another worker holds the lease now
+         */
+        async patchLeased(task, data) {
+            return runAsSystem(async () => {
+                try {
+                    await WorkflowTask.ref(task.uuid).patch(data, "leaseUntil", task.leaseUntil);
+                }
+                catch (err) {
+                    const current = (await WorkflowTask.ref(task.uuid).get());
+                    if (current && current.leaseUntil === task.leaseUntil)
+                        throw err;
+                    return false;
+                }
+                Object.assign(task, data);
+                return true;
+            });
+        }
+        /**
+         * Check this worker still holds the lease of a task, and extend it
+         * @param task - the task
+         * @returns false when the lease was lost
+         */
+        holdsLease(task) {
+            return this.patchLeased(task, { leaseUntil: Date.now() + this.parameters.leaseDuration });
+        }
+        /**
+         * Extend the lease of a running task, abort the attempt when the lease was lost
+         * @param task - the task
+         */
+        async renewLease(task) {
+            let held = false;
+            try {
+                held = await this.holdsLease(task);
+            }
+            catch (err) {
+                this.log("WARN", "Cannot renew the lease of workflow task", task.uuid, err);
+            }
+            if (!held) {
+                this.running.get(task.uuid)?.controller.abort(DROPPED);
+            }
+        }
+        /**
+         * Record the end of an attempt, if this worker still holds its lease
+         *
+         * `leaseUntil` becomes the end time, never 0: a worker that queried the task before cannot claim it again.
+         * @param task - the task
+         * @param data - status and outcome
+         * @returns false when the lease was lost
+         */
+        finishTask(task, data) {
+            const endedAt = Date.now();
+            return this.patchLeased(task, { ...data, endedAt, leaseUntil: endedAt });
+        }
+        /**
+         * Create the next attempt of a failed task, after the backoff delay
+         * @param task - the failed attempt
+         */
+        async scheduleRetry(task) {
+            const delay = new Duration(task.options?.delay ?? "1s").toMs();
+            const factor = (task.options?.backoff ?? "exponential") === "exponential" ? 2 ** (task.attempt - 1) : 1;
+            await this.createTask(task.instance, { key: task.key, operation: task.operation, input: task.input, options: task.options }, task.launcherId, Date.now() + delay * factor, task.attempt + 1);
+        }
+        /**
+         * Run the operation (or built-in) of a task
+         * @param task - the task
+         * @param instance - its instance
+         * @param controller - aborted on timeout
+         * @returns the result
+         */
+        async runTask(task, instance, controller) {
+            switch (task.operation) {
+                case BUILTINS.DateNow:
+                    return Date.now();
+                case BUILTINS.RandomNumber:
+                    return Math.random();
+                case BUILTINS.RandomUuid:
+                    return randomUUID();
+                case BUILTINS.Sleep:
+                    // The timer is the runAt of the task
+                    return null;
+                case BUILTINS.WaitForEvent:
+                    throw new TaskError("Timeout", `No ${task.event} event for ${task.eventKey} before the timeout`, false);
+                case BUILTINS.Start: {
+                    const def = this.resolveFlow(task.input?.flow);
+                    if (!def) {
+                        throw new TaskError("NotFound", `Unknown flow ${task.input?.flow}`, false);
+                    }
+                    const input = task.input?.input ?? {};
+                    const launcher = await this.childLauncher(def, instance.launcher, input);
+                    // The same uuid for every attempt: a child started twice is created once
+                    const uuid = taskUuid(task.uuid, "start", 1);
+                    await runAsSystem(() => this.createInstance(def, input, launcher, { uuid }));
+                    return { uuid };
+                }
+            }
+            if (this.resolveFlow(task.operation)) {
+                throw new TaskError("InvalidTask", `Child flow ${task.operation} runs as a WAITING_CHILD task`, false);
+            }
+            const options = task.options;
+            if (options.useSystem && instance.definition?.source === "stored" && !this.parameters.allowStoredSystem) {
+                throw new TaskError("SystemNotAllowed", "A stored workflow cannot run a task as the system", false);
+            }
+            const ctx = options.useSystem || instance.launcher?.system
+                ? useSystem(task.input ?? null)
+                : this.launcherContext(instance.launcher, task.input);
+            ctx.setExtension("abortSignal", controller.signal);
+            await ctx.init();
+            const call = callOperation(ctx, task.operation);
+            if (options.timeout) {
+                const timeoutMs = new Duration(options.timeout).toMs();
+                let timer;
+                const timeout = new Promise((_resolve, reject) => {
+                    timer = setTimeout(() => {
+                        controller.abort();
+                        reject(new TaskError("Timeout", `${task.operation} timed out after ${options.timeout}`));
+                    }, timeoutMs);
+                });
+                // An attempt still running after its timeout is ignored
+                call.catch(() => { });
+                try {
+                    await Promise.race([call, timeout]);
+                }
+                finally {
+                    clearTimeout(timer);
+                }
+            }
+            else {
+                await call;
+            }
+            const output = ctx.getOutput();
+            if (!output)
+                return null;
+            try {
+                return JSON.parse(output);
+            }
+            catch {
+                // A string result is written as is
+                return output;
+            }
+        }
+        /**
+         * Build the context of the launcher: the session is rebuilt from the user id, so the permissions are fresh
+         * @param launcher - the launcher
+         * @param input - the operation input
+         * @returns the context
+         */
+        launcherContext(launcher, input) {
+            const ctx = new SimpleOperationContext();
+            const session = new Session();
+            if (launcher?.userId) {
+                session.login(launcher.userId, launcher.identId ?? "");
+            }
+            session.scopes = launcher.scopes;
+            session.clientId = launcher.clientId;
+            ctx.setSession(session);
+            ctx.setInput(Buffer.from(JSON.stringify(input ?? null)));
+            return ctx;
+        }
+        /**
+         * Convert an error to the data the flow sees
+         * @param err - the error
+         * @param operation - the operation of the task
+         * @returns the flow error
+         */
+        toFlowError(err, operation) {
+            let code;
+            const status = typeof err?.getResponseCode === "function" ? err.getResponseCode() : undefined;
+            if (err instanceof TaskError) {
+                code = err.code;
+            }
+            else if (NOT_RETRYABLE_STATUS[status]) {
+                code = NOT_RETRYABLE_STATUS[status];
+            }
+            else if (err instanceof WebdaError.HttpError) {
+                code = err.constructor.name;
+            }
+            else {
+                code = typeof err?.code === "string" && err.code ? err.code : err?.name || "Error";
+            }
+            return { code, message: String(err?.message ?? err), operation };
+        }
+        /**
+         * Write a task outcome to the instance journal: a conditional patch on `revision`, retried on conflict
+         *
+         * The first outcome of a key wins. A journal over `maxContextSize` fails the instance.
+         * @param instanceId - the instance uuid
+         * @param key - the task key
+         * @param entry - the outcome
+         * @returns true when the outcome is in the journal
+         */
+        async writeJournal(instanceId, key, entry) {
+            return runAsSystem(async () => {
+                for (;;) {
+                    const instance = (await WorkflowInstance.ref(instanceId).get());
+                    if (!isActive(instance))
+                        return false;
+                    if (instance.journal?.[key])
+                        return true;
+                    const journal = { ...instance.journal, [key]: { ...entry, at: Date.now(), seq: instance.revision + 1 } };
+                    const size = Buffer.byteLength(JSON.stringify(journal)) + Buffer.byteLength(JSON.stringify(instance.input));
+                    if (size > this.parameters.maxContextSize) {
+                        const error = {
+                            code: "ContextTooLarge",
+                            message: `The context of the workflow is ${size} bytes, over ${this.parameters.maxContextSize}`,
+                            operation: instance.flowId
+                        };
+                        if (await this.patchInstance(instance, { status: "ERROR", error })) {
+                            await this.deliverToParent(instance.uuid);
+                            return false;
+                        }
+                        continue;
+                    }
+                    if (await this.patchInstance(instance, { journal }))
+                        return true;
+                }
+            });
+        }
+        /**
+         * Patch an instance if its revision did not change
+         * @param instance - the instance as read
+         * @param data - the changes
+         * @returns false when another writer changed it first
+         */
+        async patchInstance(instance, data) {
+            try {
+                await WorkflowInstance.ref(instance.uuid).patch({ ...data, revision: instance.revision + 1, updatedAt: Date.now() }, "revision", instance.revision);
+                return true;
+            }
+            catch (err) {
+                const current = (await WorkflowInstance.ref(instance.uuid).get());
+                if (current?.revision === instance.revision)
+                    throw err;
+                return false;
+            }
+        }
+        /**
+         * Replay an instance and act on the result: finish it, or create its missing tasks
+         * @param instanceId - the instance uuid
+         */
+        async advance(instanceId) {
+            await runAsSystem(async () => {
+                for (;;) {
+                    const instance = (await WorkflowInstance.ref(instanceId).get());
+                    if (!isActive(instance))
+                        return;
+                    const res = run(instance.definition, instance.input, instance.journal);
+                    if (res.status === "done" || res.status === "error") {
+                        const end = res.status === "done" ? { status: "SUCCESS", output: res.output } : { status: "ERROR", error: res.error };
+                        if (await this.patchInstance(instance, end)) {
+                            await this.deliverToParent(instanceId);
+                            return;
+                        }
+                        continue;
+                    }
+                    let resolved = false;
+                    for (const scheduled of res.schedule) {
+                        resolved = (await this.scheduleTask(instance, scheduled)) || resolved;
+                    }
+                    // A buffered signal resolved a new wait: replay again
+                    if (resolved)
+                        continue;
+                    const status = res.schedule.every(t => WAITING_OPERATIONS.includes(t.operation) || this.resolveFlow(t.operation))
+                        ? "WAITING"
+                        : "RUNNING";
+                    if (status === instance.status || (await this.patchInstance(instance, { status })))
+                        return;
+                }
+            });
+        }
+        /**
+         * Create the task of a scheduled key, unless a PENDING or RUNNING one exists
+         * @param instance - the instance
+         * @param scheduled - the task to run
+         * @returns true when the task was resolved at once (a buffered signal): the journal changed
+         */
+        async scheduleTask(instance, scheduled) {
+            const existing = (await WorkflowTask.query(`instance = ? AND key = ?`, [instance.uuid, scheduled.key]))
+                .results;
+            // A SUCCESS row is journaled before it is finished: the replay read the instance before it
+            if (existing.some(t => LIVE_TASK.includes(t.status) || t.status === "SUCCESS"))
+                return false;
+            const now = Date.now();
+            let runAt = now;
+            let extra = {};
+            if (scheduled.operation === BUILTINS.Sleep) {
+                runAt = this.sleepUntil(scheduled.input, runAt);
+            }
+            else if (this.resolveFlow(scheduled.operation)) {
+                return this.scheduleChild(instance, scheduled, existing.length + 1);
+            }
+            else if (scheduled.operation === BUILTINS.WaitForEvent) {
+                const input = scheduled.input ?? {};
+                extra = {
+                    status: "WAITING_EVENT",
+                    event: String(input.name),
+                    eventKey: String(input.key),
+                    allow: Array.isArray(input.allow) ? input.allow.map(String) : undefined
+                };
+                runAt = input.timeout ? this.sleepUntil({ duration: input.timeout }, now) : NO_TIMEOUT;
+            }
+            const task = await this.createTask(instance.uuid, scheduled, instance.launcher?.userId, runAt, 1, existing.length + 1, extra);
+            return task?.status === "WAITING_EVENT" ? this.consumeBuffered(instance, task) : false;
+        }
+        /**
+         * Start an awaited child flow: the task waits for the child instance, which runs as the same launcher
+         * @param instance - the parent instance
+         * @param scheduled - the task, its operation is a flow id
+         * @param index - row index of the key
+         * @returns true when the child finished at once: the parent journal changed
+         */
+        async scheduleChild(instance, scheduled, index) {
+            const def = this.resolveFlow(scheduled.operation);
+            // Created with the row: a worker resuming after a crash finds the same child
+            const child = taskUuid(instance.uuid, `${scheduled.key}#child`, index);
+            const task = await this.createTask(instance.uuid, scheduled, instance.launcher?.userId, NO_TIMEOUT, 1, index, {
+                status: "WAITING_CHILD",
+                child
+            });
+            if (!task)
+                return false;
+            let launcher;
+            try {
+                launcher = await this.childLauncher(def, instance.launcher, scheduled.input ?? {});
+            }
+            catch (err) {
+                // Refused: the task fails like any task, the flow can catch it
+                if (await this.claim(task, { signaled: true, error: this.toFlowError(err, scheduled.operation) })) {
+                    await this.execute(task, false);
+                    return true;
+                }
+                return false;
+            }
+            await this.createInstance(def, scheduled.input ?? {}, launcher, {
+                parent: { instance: instance.uuid, key: scheduled.key },
+                uuid: child
+            });
+            return (await WorkflowTask.ref(task.uuid).get()).status !== "WAITING_CHILD";
+        }
+        /**
+         * Journal the end of a child into its parent: its output, its error, or `Cancelled`
+         * @param childId - the child instance uuid
+         */
+        async deliverToParent(childId) {
+            await runAsSystem(async () => {
+                const child = (await WorkflowInstance.ref(childId).get());
+                if (!child?.parent || isActive(child))
+                    return;
+                const rows = (await WorkflowTask.query(`instance = ? AND key = ? AND status = 'WAITING_CHILD'`, [
+                    child.parent.instance,
+                    child.parent.key
+                ])).results;
+                const row = rows.find(r => r.child === child.uuid);
+                if (!row)
+                    return;
+                const outcome = child.status === "SUCCESS"
+                    ? { signaled: true, output: child.output }
+                    : {
+                        signaled: true,
+                        error: child.error ?? {
+                            code: "Cancelled",
+                            message: "The child flow was cancelled",
+                            operation: child.flowId
+                        }
+                    };
+                if (await this.claim(row, outcome)) {
+                    await this.execute(row);
+                }
+            });
+        }
+        /**
+         * Cancel an instance, its live tasks and the children it waits on
+         * @param uuid - the instance uuid
+         */
+        async cancelInstance(uuid) {
+            await runAsSystem(async () => {
+                if (!(await this.cancelTree(uuid))) {
+                    throw new WebdaError.BadRequest("cancel: the instance is not running");
+                }
+            });
+        }
+        /**
+         * Cancel an instance and, recursively, the children it waits on
+         * @param uuid - the instance uuid
+         * @returns false when the instance was not running
+         */
+        async cancelTree(uuid) {
+            for (;;) {
+                const instance = (await WorkflowInstance.ref(uuid).get());
+                if (!isActive(instance))
+                    return false;
+                if (await this.patchInstance(instance, { status: "CANCELLED" }))
+                    break;
+            }
+            for (const row of (await WorkflowTask.query(`instance = ?`, [uuid])).results) {
+                if (!LIVE_TASK.includes(row.status))
+                    continue;
+                // A local attempt stops now, a remote one loses its lease and drops its outcome
+                this.running.get(row.uuid)?.controller.abort(DROPPED);
+                await this.supersede(row);
+                if (row.status === "WAITING_CHILD" && row.child) {
+                    await this.cancelTree(row.child);
+                }
+            }
+            // A child cancelled on its own throws Cancelled in its parent
+            await this.deliverToParent(uuid);
+            return true;
+        }
+        /**
+         * Mark a task row CANCELLED, conditionally on its lease: whoever holds it can no longer write its outcome
+         * @param row - the task row, as read
+         */
+        async supersede(row) {
+            const status = row.status;
+            for (let current = row; current.status !== "CANCELLED";) {
+                const now = Date.now();
+                if (await this.patchLeased(current, { status: "CANCELLED", endedAt: now, leaseUntil: now }))
+                    break;
+                current = (await WorkflowTask.ref(row.uuid).get());
+            }
+            // Keep the status seen by the caller
+            row.status = status;
+        }
+        /**
+         * Restart a failed instance: drop the outcome that failed it and every outcome journaled after it, then replay
+         * @param uuid - the instance uuid
+         */
+        async retryInstance(uuid) {
+            await runAsSystem(async () => {
+                let kept;
+                for (;;) {
+                    const instance = (await WorkflowInstance.ref(uuid).get());
+                    if (instance?.status !== "ERROR") {
+                        throw new WebdaError.BadRequest("retry: only a failed instance can be retried");
+                    }
+                    const order = (entry) => entry.seq ?? entry.at ?? 0;
+                    const entries = Object.entries(instance.journal);
+                    const failing = entries
+                        .filter(([, entry]) => "error" in entry && sameError(entry.error, instance.error))
+                        .sort(([, a], [, b]) => order(b) - order(a))[0];
+                    if (!failing) {
+                        throw new WebdaError.BadRequest("retry: the error does not come from a task");
+                    }
+                    kept = Object.fromEntries(entries.filter(([key, entry]) => key !== failing[0] && order(entry) < order(failing[1])));
+                    if (await this.patchInstance(instance, { status: "RUNNING", journal: kept, error: null }))
+                        break;
+                }
+                // Finished rows of the dropped keys would stop them from being scheduled again
+                for (const row of (await WorkflowTask.query(`instance = ?`, [uuid])).results) {
+                    if ((row.status === "SUCCESS" || row.status === "ERROR") && !kept[row.key]) {
+                        await this.supersede(row);
+                    }
+                }
+                await this.advance(uuid);
+            });
+        }
+        /**
+         * Move a running instance to another version of its flow
+         *
+         * Refused when a journaled or live task key has no statement in the target graph.
+         * @param uuid - the instance uuid
+         * @param version - the target version, the latest registered when undefined
+         */
+        async migrateInstance(uuid, version) {
+            await runAsSystem(async () => {
+                for (;;) {
+                    const instance = (await WorkflowInstance.ref(uuid).get());
+                    if (!isActive(instance)) {
+                        throw new WebdaError.BadRequest("migrate: the instance is not running");
+                    }
+                    const target = version ? this.getFlowVersion(instance.flowId, version) : this.resolveFlow(instance.flowId);
+                    if (!target) {
+                        throw new WebdaError.BadRequest(`migrate: unknown version ${version ?? "(latest)"} of ${instance.flowId}`);
+                    }
+                    const ids = keysOf(target);
+                    const live = (await WorkflowTask.query(`instance = ?`, [uuid])).results
+                        .filter(row => LIVE_TASK.includes(row.status))
+                        .map(row => row.key);
+                    const missing = [...new Set([...Object.keys(instance.journal), ...live])].filter(key => !ids.has(statementIdOf(key)));
+                    if (missing.length) {
+                        throw new WebdaError.BadRequest(`migrate: ${missing.join(", ")} not in version ${target.version}`);
+                    }
+                    if (await this.patchInstance(instance, { definition: target, version: target.version }))
+                        break;
+                }
+                await this.advance(uuid);
+            });
+        }
+        /**
+         * Deliver a signal: resolve the waits on `(event, key)` whose instance accepts the caller, or buffer it when
+         * nothing waits
+         * @param event - event name
+         * @param key - business key
+         * @param payload - the result of the waits
+         * @returns the number of waits resolved
+         */
+        async signal(event, key, payload) {
+            const context = useContext();
+            if (!context.isGlobalContext() && !context.getCurrentUserId()) {
+                throw new WebdaError.Unauthorized("Workflow.Signal needs an authenticated caller");
+            }
+            if (Buffer.byteLength(JSON.stringify(payload ?? null)) > this.parameters.maxSignalPayload) {
+                throw new WebdaError.BadRequest(`Workflow.Signal payload is over ${this.parameters.maxSignalPayload} bytes`);
+            }
+            const waits = await runAsSystem(() => this.waitsFor(event, key));
+            if (waits.length === 0) {
+                const buffered = await this.bufferSignal(context, event, key, payload);
+                // A wait created meanwhile looked for buffered signals before this one was saved
+                return { delivered: (await this.deliverBuffered(buffered)) ? 1 : 0 };
+            }
+            let delivered = 0;
+            let refused = 0;
+            for (const { instance, task } of waits) {
+                if (!(await this.canSignal(context, instance, task))) {
+                    refused++;
+                    continue;
+                }
+                if (await runAsSystem(() => this.resolveWait(task, payload))) {
+                    delivered++;
+                }
+            }
+            // Refused by every wait, or waits resolved by a concurrent signal: not delivered, not buffered. A refused sender
+            // gets the answer of "nothing waits", so the pending waits cannot be probed
+            return { delivered };
+        }
+        /**
+         * The waits on `(event, key)` of active instances
+         * @param event - event name
+         * @param key - business key
+         * @returns the waits and their instance
+         */
+        async waitsFor(event, key) {
+            const tasks = (await WorkflowTask.query(`status = 'WAITING_EVENT' AND event = ? AND eventKey = ?`, [event, key]))
+                .results;
+            const waits = [];
+            for (const task of tasks) {
+                const instance = (await WorkflowInstance.ref(task.instance).get());
+                if (isActive(instance))
+                    waits.push({ instance, task });
+            }
+            return waits;
+        }
+        /**
+         * Whether a sender may signal a wait: the launcher, the system, and the users in the `allow` of the wait
+         * @param context - the sender
+         * @param instance - the instance waiting
+         * @param task - the wait
+         * @returns true when allowed
+         */
+        async canSignal(context, instance, task) {
+            if ((await instance.canAct(context, "signal")) === true)
+                return true;
+            const userId = context.getCurrentUserId();
+            return !!userId && (task.allow ?? []).includes(userId);
+        }
+        /**
+         * Save a signal nothing waits for yet
+         * @param context - the sender
+         * @param event - event name
+         * @param key - business key
+         * @param payload - the payload
+         * @returns the buffered signal
+         */
+        async bufferSignal(context, event, key, payload) {
+            const now = Date.now();
+            const system = context.isGlobalContext();
+            const pending = await runAsSystem(async () => (await WorkflowSignal.query(escape(["event = ? AND key = ? AND expiresAt > ? LIMIT ", ""], [this.parameters.maxBufferedSignals]), [event, key, now])).results.length);
+            if (pending >= this.parameters.maxBufferedSignals) {
+                // Refused rather than dropping an older one: an accepted signal is never lost
+                throw new WebdaError.TooManyRequests(`Too many buffered signals for ${event}/${key}`);
+            }
+            return runAsSystem(() => WorkflowSignal.create({
+                event,
+                key,
+                payload,
+                senderId: system ? undefined : context.getCurrentUserId(),
+                senderIdent: system ? undefined : context.getSession()?.identUsed,
+                system: system || undefined,
+                createdAt: now,
+                expiresAt: now + new Duration(this.parameters.signalTtl).toMs()
+            }));
+        }
+        /**
+         * Deliver a just buffered signal to a wait created while it was being saved
+         * @param signal - the buffered signal
+         * @returns true when a wait consumed it
+         */
+        async deliverBuffered(signal) {
+            return runAsSystem(async () => {
+                for (const { instance, task } of await this.waitsFor(signal.event, signal.key)) {
+                    if (await this.consumeSignal(signal, instance, task, true))
+                        return true;
+                }
+                return false;
+            });
+        }
+        /**
+         * Resolve a new wait with the oldest unexpired buffered signal its instance accepts
+         * @param instance - the instance
+         * @param task - the wait
+         * @returns true when a signal was consumed
+         */
+        async consumeBuffered(instance, task) {
+            const signals = (await WorkflowSignal.query(`event = ? AND key = ? AND expiresAt > ?`, [task.event, task.eventKey, Date.now()]))
+                .results.sort((a, b) => a.createdAt - b.createdAt);
+            for (const signal of signals) {
+                // Advanced by the caller (advance)
+                if (await this.consumeSignal(signal, instance, task, false))
+                    return true;
+            }
+            return false;
+        }
+        /**
+         * Consume a buffered signal for a wait: its sender is checked now, against this instance
+         * @param signal - the buffered signal
+         * @param instance - the instance waiting
+         * @param task - the wait
+         * @param advance - advance the instance once the payload is journaled
+         * @returns true when consumed and delivered
+         */
+        async consumeSignal(signal, instance, task, advance) {
+            if (signal.expiresAt <= Date.now())
+                return false;
+            const sender = signal.system
+                ? useSystem()
+                : this.launcherContext({ userId: signal.senderId, identId: signal.senderIdent }, null);
+            if (!(await this.canSignal(sender, instance, task)))
+                return false;
+            try {
+                // Consumed once: a concurrent consumer fails the condition
+                await WorkflowSignal.ref(signal.uuid).delete("expiresAt", signal.expiresAt);
+            }
+            catch {
+                return false;
+            }
+            if (await this.resolveWait(task, signal.payload, advance))
+                return true;
+            // The wait timed out or another signal resolved it first: the signal stays buffered
+            const { uuid, event, key, payload, senderId, senderIdent, system, createdAt, expiresAt } = signal;
+            await WorkflowSignal.create({
+                uuid,
+                event,
+                key,
+                payload,
+                senderId,
+                senderIdent,
+                system,
+                createdAt,
+                expiresAt
+            });
+            return false;
+        }
+        /**
+         * Resolve a wait with a payload: claimed like a task, so it resolves once
+         * @param task - the wait
+         * @param payload - its result
+         * @param advance - advance the instance once the payload is journaled
+         * @returns false when the wait was already resolved or timed out
+         */
+        async resolveWait(task, payload, advance = true) {
+            if (!(await this.claim(task, { signaled: true, output: payload })))
+                return false;
+            await this.execute(task, advance);
+            return true;
+        }
+        /**
+         * End of a `Workflow.Sleep`
+         * @param input - `{ duration?: string, until?: number }`
+         * @param now - the current time
+         * @returns when the timer fires
+         */
+        sleepUntil(input, now) {
+            if (typeof input?.until === "number") {
+                return input.until;
+            }
+            try {
+                return now + new Duration(input?.duration ?? 0).toMs();
+            }
+            catch {
+                // An invalid duration fires at once
+                return now;
+            }
+        }
+        /**
+         * Create a task row, unless another worker created the same row first
+         * @param instance - the instance uuid
+         * @param scheduled - the task
+         * @param launcherId - user id of the launcher
+         * @param runAt - when it becomes runnable
+         * @param attempt - attempt number
+         * @param index - row index of the key, when known
+         * @param extra - other fields of the row
+         * @returns the task created, undefined when another worker created it
+         */
+        async createTask(instance, scheduled, launcherId, runAt, attempt, index, extra = {}) {
+            return runAsSystem(async () => {
+                index ??= (await WorkflowTask.query(`instance = ? AND key = ?`, [instance, scheduled.key])).results.length + 1;
+                const uuid = taskUuid(instance, scheduled.key, index);
+                try {
+                    return (await WorkflowTask.create({
+                        uuid,
+                        instance,
+                        key: scheduled.key,
+                        operation: scheduled.operation,
+                        input: scheduled.input,
+                        options: scheduled.options,
+                        attempt,
+                        status: "PENDING",
+                        runAt,
+                        leaseUntil: 0,
+                        launcherId,
+                        ...extra
+                    }));
+                }
+                catch (err) {
+                    if (!(await WorkflowTask.ref(uuid).exists()))
+                        throw err;
+                    // Created by another worker
+                    return undefined;
+                }
+            });
+        }
+    };
+})();
+export { WorkflowService };
+export default WorkflowService;
+//# sourceMappingURL=workflow.service.js.map