@adonis-agora/durable 0.6.0 → 0.7.0

package/dist/src/engine.js

@@ -7,7 +7,9 @@ import { ContinueAsNew, NonDeterminismError, RemoteStepTimeout, WorkflowSuspende
 import { EventAccumulators } from './event-accumulators.js';
 import { eventMatchOf, eventMatches, eventPrefix } from './events.js';
 import { breakpointToken, stepId } from './protocol.js';
+import { RemoteWorkflowExecutor } from './remote-workflow-executor.js';
 import { SingletonGate } from './singleton-gate.js';
+import { sanitizeQueueToken, tenantGroup } from './tenant-group.js';
 import { TransportPool } from './transport-pool.js';
 import { createWorkflowCtx, } from './workflow-ctx.js';
 import { workflowName, } from './workflow-ref.js';
@@ -52,6 +54,11 @@ export class WorkflowEngine {
     store;
     /** Ordered transport pool (dispatch + failover). Empty = no remote steps. */
     pool;
+    /** The primary task transport (first of the pool), used to build an on-the-fly remote-workflow
+     *  executor for {@link WorkflowEngineDeps.remoteByConvention}. Undefined when no transport is wired. */
+    primaryTransport;
+    /** Convention-dispatch opt-in — see {@link WorkflowEngineDeps.remoteByConvention}. */
+    remoteByConvention;
     controlPlane;
     clock;
     instanceId;
@@ -110,6 +117,8 @@ export class WorkflowEngine {
     constructor(deps) {
         this.store = deps.store;
         this.pool = new TransportPool(deps.transports ?? (deps.transport ? [{ id: 'default', transport: deps.transport }] : []));
+        this.primaryTransport = deps.transports?.[0]?.transport ?? deps.transport;
+        this.remoteByConvention = deps.remoteByConvention ?? false;
         this.controlPlane = deps.controlPlane;
         this.clock = deps.clock ?? Date.now;
         this.admission = deps.admission ?? new InMemoryAdmissionBackend(this.clock);
@@ -294,7 +303,28 @@ export class WorkflowEngine {
             this.latest.set(name, registered);
     }
     /**
-     * Register a flow-control queue referenced by `ctx.call(step, input, { queue })`. Caps concurrent
+     * Convention-dispatch resolution (opt-in via {@link WorkflowEngineDeps.remoteByConvention}): when
+     * `name` isn't locally registered but a LIVE worker group of the same name exists, auto-register a
+     * remote workflow routed to that group (bare group, no partition) so its runs advance over the
+     * broker like any `registerRemote`'d one. Returns whether `name` is registered afterwards. A no-op
+     * (returns `false`) when the opt-in is off, no transport is wired, or no matching group is live.
+     */
+    async ensureConventionWorkflow(name) {
+        if (!this.remoteByConvention || !this.primaryTransport)
+            return false;
+        if (this.latest.get(name))
+            return true;
+        const groups = await this.pool.listWorkerGroups();
+        if (!groups.includes(name))
+            return false;
+        this.registerRemote(name, '1', {
+            group: name,
+            executor: new RemoteWorkflowExecutor(this.primaryTransport, name),
+        });
+        return true;
+    }
+    /**
+     * Register a flow-control queue referenced by `ctx.step(name, input, { queue })`. Caps concurrent
      * in-flight steps and/or the admission rate; blocked calls re-suspend and retry, so the limit is
      * durable. Per engine instance (see {@link QueueConfig}). Registering the same name replaces it.
      */
@@ -394,7 +424,12 @@ export class WorkflowEngine {
     }
     async start(workflow, input, runId, opts) {
         const name = workflowName(workflow);
-        const registered = this.latest.get(name);
+        let registered = this.latest.get(name);
+        // remoteByConvention: an unregistered workflow whose name matches a LIVE worker group is routed to
+        // it as a remote workflow (bare group). Off by default → the original "not registered" throw.
+        if (!registered && (await this.ensureConventionWorkflow(name))) {
+            registered = this.latest.get(name);
+        }
         if (!registered)
             throw new Error(`workflow ${name} is not registered`);
         // Validate the input up front — a bad payload is rejected before any run is created.
@@ -421,7 +456,7 @@ export class WorkflowEngine {
             workflow: name,
             workflowVersion: registered.version,
             status: 'pending',
-            namespace: this.namespace,
+            namespace: opts?.namespace ?? this.namespace,
             input,
             tags,
             searchAttributes: opts?.searchAttributes,
@@ -496,7 +531,14 @@ export class WorkflowEngine {
         }
         // Pin to the version the run STARTED on — replay is positional, so running a changed
         // workflow body against old checkpoints would corrupt the run.
-        const registered = this.workflows.get(versionKey(run.workflow, run.workflowVersion));
+        let registered = this.workflows.get(versionKey(run.workflow, run.workflowVersion));
+        // remoteByConvention: a run recovered on an instance that never registered this workflow (e.g. a
+        // crash-recovery pickup) is re-routed to a live worker group of the same name, if one exists.
+        if (!registered && (await this.ensureConventionWorkflow(run.workflow))) {
+            registered =
+                this.workflows.get(versionKey(run.workflow, run.workflowVersion)) ??
+                    this.latest.get(run.workflow);
+        }
         if (!registered) {
             throw new Error(`workflow ${run.workflow}@${run.workflowVersion} is not registered — keep the prior version deployed so in-flight runs can drain (skew protection)`);
         }
@@ -550,7 +592,7 @@ export class WorkflowEngine {
                     continue;
                 const error = { message: 'execution timeout', code: 'execution_timeout' };
                 await this.store.updateRun(run.id, { status: 'cancelled', error, updatedAt: new Date() });
-                this.emit({ type: 'run.failed', runId: run.id, workflow: run.workflow, error });
+                this.emit({ type: 'run.failed', runId: run.id, workflow: run.workflow, namespace: run.namespace, error });
             }
         }
     }
@@ -597,7 +639,7 @@ export class WorkflowEngine {
             };
             await this.store.updateRun(run.id, { status: 'dead', error, updatedAt: new Date() });
             await this.store.releaseRunLock(run.id);
-            this.emit({ type: 'run.failed', runId: run.id, workflow: run.workflow, error });
+            this.emit({ type: 'run.failed', runId: run.id, workflow: run.workflow, namespace: run.namespace, error });
             this.notifyDead({ ...run, status: 'dead', error, recoveryAttempts: attempts });
             return { runId: run.id, status: 'dead', error };
         }
@@ -659,7 +701,9 @@ export class WorkflowEngine {
         if (!run)
             return null;
         const id = newRunId ?? `${runId}~retry~${globalThis.crypto.randomUUID().slice(0, 8)}`;
-        await this.start(run.workflow, input, id, { tags: run.tags });
+        // Inherit the original run's namespace so a retry lands in the SAME worker-pool partition (an
+        // operator-routed run keeps its tenant on retry, rather than falling back to the engine's own).
+        await this.start(run.workflow, input, id, { tags: run.tags, namespace: run.namespace });
         return { runId: id };
     }
     /**
@@ -885,7 +929,7 @@ export class WorkflowEngine {
         }
         const error = { message: 'cancelled' };
         await this.store.updateRun(runId, { status: 'cancelled', error, updatedAt: new Date() });
-        this.emit({ type: 'run.failed', runId, workflow: run.workflow, error });
+        this.emit({ type: 'run.failed', runId, workflow: run.workflow, namespace: run.namespace, error });
         await this.cancelChildren(runId, opts);
         // Notify local cancel listeners now (a worker on this pod), and broadcast so the instance/worker
         // actually running this run learns of it and can abort cooperatively (the store already records
@@ -1204,6 +1248,7 @@ export class WorkflowEngine {
                 type: 'run.completed',
                 runId: run.id,
                 workflow: run.workflow,
+                namespace: run.namespace,
                 output: outcome.output,
             });
             void this.notifyParent(run.id, { ok: true, value: outcome.output });
@@ -1215,13 +1260,14 @@ export class WorkflowEngine {
                 type: 'run.failed',
                 runId: run.id,
                 workflow: run.workflow,
+                namespace: run.namespace,
                 error: outcome.error,
             });
             void this.notifyParent(run.id, { ok: false, error: outcome.error.message });
             return { runId: run.id, status: 'failed', error: outcome.error };
         }
         await this.store.updateRun(run.id, { status: 'suspended', wakeAt: outcome.wakeAt, updatedAt });
-        this.emit({ type: 'run.suspended', runId: run.id, workflow: run.workflow });
+        this.emit({ type: 'run.suspended', runId: run.id, workflow: run.workflow, namespace: run.namespace });
         return { runId: run.id, status: 'suspended' };
     }
     async runRemoteExecution(run, registered) {
@@ -1229,12 +1275,12 @@ export class WorkflowEngine {
         if (run.status === 'pending') {
             await this.store.updateRun(run.id, { status: 'running', updatedAt: new Date() });
             run.status = 'running';
-            this.emit({ type: 'run.started', runId: run.id, workflow: run.workflow });
+            this.emit({ type: 'run.started', runId: run.id, workflow: run.workflow, namespace: run.namespace });
         }
         if (registered.singleton && !(await this.singletons.admit(run, registered.singleton))) {
             const wakeAt = this.singletons.retryWakeAt();
             await this.store.updateRun(run.id, { status: 'suspended', wakeAt, updatedAt: new Date() });
-            this.emit({ type: 'run.suspended', runId: run.id, workflow: run.workflow });
+            this.emit({ type: 'run.suspended', runId: run.id, workflow: run.workflow, namespace: run.namespace });
             await this.store.releaseRunLock(run.id);
             return { runId: run.id, status: 'suspended' };
         }
@@ -1249,7 +1295,7 @@ export class WorkflowEngine {
                 stack: err instanceof Error ? err.stack : undefined,
             };
             await this.store.updateRun(run.id, { status: 'failed', error, updatedAt: new Date() });
-            this.emit({ type: 'run.failed', runId: run.id, workflow: run.workflow, error });
+            this.emit({ type: 'run.failed', runId: run.id, workflow: run.workflow, namespace: run.namespace, error });
             return { runId: run.id, status: 'failed', error };
         }
         if (decision.status === 'completed') {
@@ -1375,6 +1421,10 @@ export class WorkflowEngine {
                 // Mirrors the `startChild` `getRun(childId)` guard below.
                 if (await this.store.getCheckpoint(run.id, cmd.seq))
                     continue;
+                // Route this cross-SDK-worker `call` by the SAME name-based token a native `ctx.step`
+                // dispatches with (and a worker subscribes to per handler name), so a decision-driven remote
+                // step and an in-process one land on the identical queue. The command carries no partition.
+                const callToken = tenantGroup(sanitizeQueueToken(cmd.name), undefined);
                 await this.store.saveCheckpoint(stepCheckpoint({
                     runId: run.id,
                     seq: cmd.seq,
@@ -1383,13 +1433,13 @@ export class WorkflowEngine {
                     status: 'pending',
                     input: cmd.input,
                     attempts: 1,
-                    workerGroup: cmd.group,
+                    workerGroup: callToken,
                     enqueuedAt: at,
                     startedAt: at,
                     finishedAt: at,
                     // A `ctx.gather_calls([...])` fan-out stamps every dispatched `call` in the fan with the
                     // same group, so the dashboard renders the remote steps as one parallel fan (parity with
-                    // the gathered `recordStep`/`startChild` tags). Undefined for a lone sequential `ctx.call`.
+                    // the gathered `recordStep`/`startChild` tags). Undefined for a lone sequential `ctx.step`.
                     parallelGroup: cmd.parallelGroup,
                 }));
                 await this.pool.dispatch({
@@ -1397,7 +1447,7 @@ export class WorkflowEngine {
                     seq: cmd.seq,
                     name: cmd.name,
                     stepId: id,
-                    group: cmd.group,
+                    group: callToken,
                     input: cmd.input,
                     traceparent: this.traceparent?.(),
                     context: this.context?.(),
@@ -1483,14 +1533,14 @@ export class WorkflowEngine {
         if (run.status === 'pending') {
             await this.store.updateRun(run.id, { status: 'running', updatedAt: new Date() });
             run.status = 'running';
-            this.emit({ type: 'run.started', runId: run.id, workflow: run.workflow });
+            this.emit({ type: 'run.started', runId: run.id, workflow: run.workflow, namespace: run.namespace });
         }
         // Singleton admission gate: if this run shares its key with `limit` older in-flight runs, wait
         // (suspend on a short timer) until a slot frees instead of running now. Re-checked on each resume.
         if (registered?.singleton && !(await this.singletons.admit(run, registered.singleton))) {
             const wakeAt = this.singletons.retryWakeAt();
             await this.store.updateRun(run.id, { status: 'suspended', wakeAt, updatedAt: new Date() });
-            this.emit({ type: 'run.suspended', runId: run.id, workflow: run.workflow });
+            this.emit({ type: 'run.suspended', runId: run.id, workflow: run.workflow, namespace: run.namespace });
             await this.store.releaseRunLock(run.id);
             return { runId: run.id, status: 'suspended' };
         }
@@ -1521,7 +1571,7 @@ export class WorkflowEngine {
                     error: undefined,
                     updatedAt: new Date(),
                 });
-                this.emit({ type: 'run.completed', runId: run.id, workflow: run.workflow });
+                this.emit({ type: 'run.completed', runId: run.id, workflow: run.workflow, namespace: run.namespace });
                 void this.notifyParent(run.id, { ok: true, value: undefined });
                 const nextId = nextContinuationId(run.id);
                 queueMicrotask(() => void this.start(run.workflow, err.input, nextId).catch(() => undefined));
@@ -1539,7 +1589,7 @@ export class WorkflowEngine {
                     }
                     const error = { message: 'cancelled' };
                     await this.store.updateRun(run.id, { status: 'cancelled', error, updatedAt: new Date() });
-                    this.emit({ type: 'run.failed', runId: run.id, workflow: run.workflow, error });
+                    this.emit({ type: 'run.failed', runId: run.id, workflow: run.workflow, namespace: run.namespace, error });
                     return { runId: run.id, status: 'cancelled', error };
                 }
                 return this.settleRun(run, { kind: 'suspended', wakeAt: err.wakeAt });
@@ -1569,14 +1619,21 @@ export class WorkflowEngine {
     /**
      * Run one saga compensation, retried up to `compensationRetries`, emitting a `compensate:<step>`
      * step event for its outcome so a stranded undo is visible. Never throws — a permanently-failing
-     * compensation is skipped so it can't mask the original failure.
+     * compensation is skipped so it can't mask the original failure. Handles both compensation shapes:
+     * a LOCAL `fn` (run in-process) and a DISPATCHED undo (an ordinary step def sent to a worker and
+     * awaited inline — the worker serving its name runs it with the {@link StepUndo} envelope).
      */
     async runCompensation(run, comp) {
         const name = `compensate:${comp.name}`;
         for (let attempt = 1; attempt <= this.compensationRetries; attempt += 1) {
             const startedAt = Date.now();
             try {
-                await comp.fn();
+                if ('fn' in comp) {
+                    await comp.fn();
+                }
+                else {
+                    await this.dispatchCompensation(run, comp.dispatch.def, comp.dispatch.args, attempt);
+                }
                 this.emit({
                     type: 'step.completed',
                     runId: run.id,
@@ -1602,6 +1659,43 @@ export class WorkflowEngine {
             }
         }
     }
+    /**
+     * Dispatch a saga undo step to a worker and await its result INLINE (the saga unwind already runs
+     * outside the positional replay, so this is not checkpointed — it's a one-shot dispatch + await).
+     * Routes by the undo def's own name/partition token, applies its own optional input schema, and
+     * honours its own liveness `timeoutMs` via the heartbeat window. Throws on a worker-reported failure
+     * or timeout so `runCompensation`'s retry/skip logic applies uniformly to both compensation shapes.
+     */
+    async dispatchCompensation(run, def, args, attempt) {
+        if (this.pool.size === 0)
+            throw new Error('dispatched compensation requires a Transport');
+        const token = tenantGroup(sanitizeQueueToken(def.name), def.partition);
+        const validInput = def.input ? def.input.parse(args) : args;
+        const id = `${run.id}:compensate:${def.name}:${attempt}`;
+        const resultPromise = new Promise((resolve, reject) => {
+            this.pending.set(id, { resolve, reject });
+        });
+        try {
+            await this.pool.dispatch({
+                runId: run.id,
+                seq: -1,
+                name: def.name,
+                stepId: id,
+                group: token,
+                input: validInput,
+                traceparent: this.traceparent?.(),
+                context: this.context?.(),
+                attempt: 1,
+            });
+            await (def.timeoutMs
+                ? this.awaitWithHeartbeat(id, resultPromise, def.timeoutMs)
+                : resultPromise);
+        }
+        catch (err) {
+            this.pending.delete(id);
+            throw err;
+        }
+    }
     /** The seam handed to {@link createWorkflowCtx}: the authoring API reaches durability + lifecycle
      *  (checkpointing, dispatch, child start) through this, so the ctx primitives live in their own
      *  module and the engine stays the orchestrator. */
@@ -1760,7 +1854,13 @@ export class WorkflowEngine {
             this.queueWaiters.get(queue)?.delete(runId);
             this.stepQueue.set(id, queue);
         }
-        const validInput = step.input.parse(input);
+        // Optional runtime schema: validate only when the def carries one (a bare `@Step()` / string-name
+        // call carries none — the engine passes the input through untouched then).
+        const validInput = step.input ? step.input.parse(input) : input;
+        // Routing token: BY NAME (sanitized for brokers), optionally partition-suffixed. Computed once and
+        // used at BOTH the checkpoint's workerGroup and the dispatched task's routing `group`, so the same
+        // token the worker subscribes to per handler name serves this step.
+        const token = tenantGroup(sanitizeQueueToken(step.name), step.partition);
         const enqueuedAt = new Date();
         // Persist the pending checkpoint BEFORE dispatching, so a fast result always finds it to complete.
         await this.store.saveCheckpoint({
@@ -1772,7 +1872,7 @@ export class WorkflowEngine {
             status: 'pending',
             input: validInput,
             attempts: attempt,
-            workerGroup: step.group,
+            workerGroup: token,
             enqueuedAt,
             startedAt: enqueuedAt, // placeholders until the worker result lands
             finishedAt: enqueuedAt,
@@ -1782,7 +1882,7 @@ export class WorkflowEngine {
             seq,
             name: step.name,
             stepId: id,
-            group: step.group,
+            group: token,
             input: validInput,
             traceparent: this.traceparent?.(),
             context: this.context?.(),
@@ -1859,7 +1959,8 @@ export class WorkflowEngine {
     async callRemoteInMemory(runId, seq, step, input, transport) {
         if (this.pool.size === 0)
             throw new Error('remote steps require a Transport');
-        const validInput = step.input.parse(input);
+        const validInput = step.input ? step.input.parse(input) : input;
+        const token = tenantGroup(sanitizeQueueToken(step.name), step.partition);
         const id = stepId(runId, seq);
         const enqueuedAt = new Date();
         this.emit({ type: 'step.started', runId, seq, name: step.name, kind: 'remote' });
@@ -1878,7 +1979,7 @@ export class WorkflowEngine {
                 seq,
                 name: step.name,
                 stepId: id,
-                group: step.group,
+                group: token,
                 input: validInput,
                 traceparent: this.traceparent?.(),
                 context: this.context?.(),
@@ -1888,7 +1989,7 @@ export class WorkflowEngine {
                 const resolution = step.timeoutMs
                     ? await this.awaitWithHeartbeat(id, resultPromise, step.timeoutMs)
                     : await resultPromise;
-                const output = step.output.parse(resolution.output);
+                const output = (step.output ? step.output.parse(resolution.output) : resolution.output);
                 // The worker reports when it actually picked the task up; fall back to dispatch time if a
                 // transport doesn't carry it (queue-wait then reads as zero rather than going negative).
                 const startedAt = resolution.startedAt ? new Date(resolution.startedAt) : enqueuedAt;
@@ -1901,7 +2002,7 @@ export class WorkflowEngine {
                     output,
                     events: resolution.events,
                     attempts: attempt,
-                    workerGroup: step.group,
+                    workerGroup: token,
                     enqueuedAt,
                     startedAt,
                 });