@adonis-agora/durable 1.0.0-rc.2 → 1.0.0-rc.20

package/dist/src/engine.js

@@ -36,6 +36,7 @@ import { SINGLETON_RESERVED_TAGS, withoutAdmission, withSingletonTracked, } from
 import { SingletonGate } from './singleton-gate.js';
 import { validateStandard, validateStandardSync, } from './standard-schema.js';
 import { StickyCache, StickyContinuation } from './sticky-continuation.js';
+import { intersectTenantIds, runTenantOf, runVisible, tenantScopeIds, } from './tenancy.js';
 import { sanitizeQueueToken, tenantGroup } from './tenant-group.js';
 import { TransportPool } from './transport-pool.js';
 import { workflowAls } from './workflow-als.js';
@@ -72,6 +73,13 @@ const isBreakpoint = (cp) => cp.status === 'pending' && cp.name.startsWith(BREAK
  * through some other path (an operator's cancel, a timeout sweep, a dead-letter) while this turn's
  * outcome was being computed from a possibly-stale snapshot.
  */
+/** Statuses a wake refused by a held lease may stamp a request for another turn on. */
+const WAKEABLE_RUN_STATUSES = ['running', 'suspended', 'blocked', 'cancelling'];
+/**
+ * Statuses a wake that finds the lease free re-drives: parked runs, and a `cancelling` run whose
+ * undo is not executing (its turn unwinds the saga).
+ */
+const PARKED_RUN_STATUSES = ['suspended', 'blocked', 'cancelling'];
 const SETTLE_ELIGIBLE_STATUSES = [
     'pending',
     'running',
@@ -247,7 +255,13 @@ function isNewerVersion(a, b) {
     return a.localeCompare(b, undefined, { numeric: true }) > 0;
 }
 /** The statuses a run can be skipped in for an unregistered workflow — every non-terminal one. */
-const UNREGISTERED_DEAD_LETTER_FROM = ['pending', 'running', 'suspended', 'blocked'];
+const UNREGISTERED_DEAD_LETTER_FROM = [
+    'pending',
+    'running',
+    'suspended',
+    'blocked',
+    'cancelling',
+];
 /** Thrown by {@link WorkflowEngine.resume} when the run belongs to a different namespace. */
 export class NamespaceMismatch extends Error {
     constructor() {
@@ -325,6 +339,7 @@ export class WorkflowEngine {
     context;
     /** Establish the originating context ambiently around a local step body (see {@link WorkflowEngineDeps.rehydrate}). Default passthrough. */
     rehydrate;
+    tenancy;
     compensationRetries;
     compensationTimeoutMs;
     retention;
@@ -402,6 +417,8 @@ export class WorkflowEngine {
     #allSchedules = [];
     /** Runtime pause overrides by schedule key (console-issued; win over the config's `paused`). */
     #schedulePauseOverrides = new Map();
+    /** Schedule-pause broadcasts still being handed to the control plane (see {@link flushControl}). */
+    #controlInFlight = new Set();
     /** Flow-control admission backend for remote steps (see {@link registerQueue}). */
     admission;
     /** Runs on THIS instance blocked on admission, by queue — woken early on a freed-slot signal. */
@@ -602,6 +619,7 @@ export class WorkflowEngine {
         this.context = deps.context;
         // Default passthrough: with no bridge supplied, a local step body runs exactly as before.
         this.rehydrate = deps.rehydrate ?? ((_carrier, fn) => fn());
+        this.tenancy = deps.tenancy;
         this.compensationRetries = Math.max(1, deps.compensationRetries ?? 1);
         this.compensationTimeoutMs = Math.max(1_000, deps.compensationTimeoutMs ?? 300_000);
         this.retention = deps.retention;
@@ -613,6 +631,8 @@ export class WorkflowEngine {
             start: (workflow, input, runId, opts) => this.start(workflow, input, runId, opts),
             getRun: (runId) => this.store.getRun(runId),
             isDraining: () => this.draining,
+            tenant: () => this.startTenant(undefined),
+            visible: (record) => this.runVisibleNow({ tenantId: record.tenantId }),
         }, { namespace: this.namespace, clock: this.clock });
         this.stalledAfterMs = Math.max(60_000, deps.stalledAfterMs ?? 900_000);
         this.trackStepStart = deps.trackStepStart ?? true;
@@ -1007,12 +1027,23 @@ export class WorkflowEngine {
             return false;
         this.#schedulePauseOverrides.set(key, paused);
         if (this.controlPlane) {
-            void this.controlPlane
+            const sent = this.controlPlane
                 .publishControl({ kind: 'schedulePause', key, paused, from: this.instanceId })
-                .catch(() => undefined);
+                .catch(() => undefined)
+                .finally(() => this.#controlInFlight.delete(sent));
+            this.#controlInFlight.add(sent);
         }
         return true;
     }
+    /**
+     * Wait until the schedule pauses this instance broadcast have been handed to the control plane. A
+     * short-lived process (an ace command) calls it before it exits. Resolves `false` when the engine
+     * has no control plane, so a pause reached this process only.
+     */
+    async flushControl() {
+        await Promise.allSettled([...this.#controlInFlight]);
+        return this.controlPlane !== undefined;
+    }
     /**
      * Fire a schedule's CURRENT window now (console "run now"). Idempotent by the window's
      * deterministic run id — triggering a window that already fired returns the existing run instead
@@ -1452,6 +1483,8 @@ export class WorkflowEngine {
         };
         this.trackEffect(this.start(target, input, `dlq:${run.id}`, {
             ...(run.namespace !== undefined ? { namespace: run.namespace } : {}),
+            tenantId: run.tenantId ?? null,
+            ...(run.tenantKind ? { tenantKind: run.tenantKind } : {}),
         }).catch((err) => {
             console.warn(`[adonis-durable] could not start dead-letter workflow ${target} for run ${run.id}: ${err instanceof Error ? err.message : String(err)}`);
         }));
@@ -2535,6 +2568,11 @@ export class WorkflowEngine {
      */
     async republishLifecycle(target) {
         const result = { runs: 0, started: 0, failed: [] };
+        if ('runId' in target &&
+            typeof target.runId === 'string' &&
+            (await this.runHidden(target.runId))) {
+            return result;
+        }
         // A backfill usually follows a deploy: read the fleet's subscriptions now, not from the cache.
         await this.refreshAnnouncedSubscriptions();
         const publish = async (run) => {
@@ -2779,6 +2817,9 @@ export class WorkflowEngine {
         // Idempotent by run id: a redelivered trigger (at-least-once queues) or a scheduler re-tick for
         // the same id is a no-op, returning the existing run's state instead of starting a duplicate.
         const existing = async (prior) => {
+            // Another tenant's run id: answer like a clash, never with that run's state.
+            if (!this.runVisibleNow(prior))
+                throw new Error(`run ${runId} already exists`);
             // A PENDING prior still needs an executor: re-nudge the dispatcher so a redelivered trigger —
             // or a continuation persisted pre-settle (see the ContinueAsNew handoff) picked up by a custom
             // dispatcher — doesn't sit `pending` until the next poll. Idempotent: execution is leased, so
@@ -2827,6 +2868,8 @@ export class WorkflowEngine {
         // Compared against the ENGINE clock (injectable), not wall time — the same clock the timer
         // poller will use to decide the run is due.
         const delayed = startAtMs !== undefined && startAtMs > this.clock();
+        const tenant = this.startTenant(opts);
+        const tenantNamespace = tenant && opts?.namespace === undefined ? this.tenancy?.namespace?.(tenant) : undefined;
         const run = {
             id: runId,
             workflow: name,
@@ -2834,7 +2877,10 @@ export class WorkflowEngine {
             status: delayed ? 'suspended' : 'pending',
             ...(delayed ? { wakeAt: startAtMs } : {}),
             origin: opts?.origin ?? registered.origin,
-            namespace: opts?.namespace ?? this.namespace,
+            namespace: opts?.namespace ?? tenantNamespace ?? this.namespace,
+            ...(tenant
+                ? { tenantId: tenant.id, ...(tenant.kind ? { tenantKind: tenant.kind } : {}) }
+                : {}),
             input,
             tags,
             searchAttributes: opts?.searchAttributes,
@@ -2897,6 +2943,16 @@ export class WorkflowEngine {
      * `limiter` bounds how many of these run at once (a fan's children wait in it as ids).
      */
     handoffRun(workflow, input, runId, opts, onStartError, onAlreadySettled, limiter) {
+        // The tenant is read NOW, inside the parent's turn: a bounded handoff may run later, elsewhere.
+        if (opts?.tenantId === undefined) {
+            const tenant = this.startTenant(opts);
+            if (tenant)
+                opts = {
+                    ...opts,
+                    tenantId: tenant.id,
+                    ...(tenant.kind ? { tenantKind: tenant.kind } : {}),
+                };
+        }
         const handoff = async () => {
             // Defer one microtask: never reentrantly resume the parent that is still suspending/settling.
             await Promise.resolve();
@@ -2936,8 +2992,54 @@ export class WorkflowEngine {
         this.trackEffect(limiter ? limiter.submit(runId, handoff) : handoff());
     }
     /** Read a run's current persisted state (or null if unknown). A thin pass-through to the store. */
-    getRun(runId) {
-        return this.store.getRun(runId);
+    async getRun(runId) {
+        const run = await this.store.getRun(runId);
+        return run && this.runVisibleNow(run) ? run : null;
+    }
+    /** The tenant a new run gets: the explicit option, else the active tenant (tenancy on), else none. */
+    startTenant(opts) {
+        if (opts?.tenantId === null)
+            return undefined;
+        if (typeof opts?.tenantId === 'string' && opts.tenantId !== '') {
+            return opts.tenantKind ? { id: opts.tenantId, kind: opts.tenantKind } : { id: opts.tenantId };
+        }
+        const filter = this.tenantFilterNow();
+        return filter.mode === 'tenant' ? filter.tenant : undefined;
+    }
+    /** How the current app-facing call is scoped (`system` inside the engine's own work). */
+    tenantFilterNow() {
+        if (!this.tenancy || engineInternal.getStore() === true)
+            return { mode: 'system' };
+        return this.tenancy.filter();
+    }
+    /** Whether the current call may see `run` (always, with tenancy off). */
+    runVisibleNow(run) {
+        if (!this.tenancy)
+            return true;
+        return runVisible(this.tenantFilterNow(), run, this.tenancy.nullRows);
+    }
+    /** Whether `runId` exists but the current call may not see it (an app-facing guard: answer not-found). */
+    async runHidden(runId) {
+        if (!this.tenancy)
+            return false;
+        const filter = this.tenantFilterNow();
+        if (filter.mode === 'system' || filter.mode === 'all')
+            return false;
+        const run = await this.store.getRun(runId);
+        return run !== null && !runVisible(filter, run, this.tenancy.nullRows);
+    }
+    /** `query` confined to the runs the current call may see. */
+    tenantScoped(query) {
+        if (!this.tenancy)
+            return query;
+        const ids = tenantScopeIds(this.tenantFilterNow(), this.tenancy.nullRows);
+        if (ids === undefined)
+            return query;
+        return { ...query, tenantIds: intersectTenantIds(query.tenantIds, ids) };
+    }
+    /** Run `fn` as the engine's own work: app-facing tenant guards do not apply inside it. */
+    asEngine(fn) {
+        return this.tenancy ? engineInternal.run(true, fn) : fn();
     }
     /**
      * List persisted runs matching `query` (newest activity first, per the store's own ordering). The
@@ -2945,7 +3047,7 @@ export class WorkflowEngine {
      * through the engine rather than reaching for its private store.
      */
     async listRuns(query) {
-        const scoped = withoutBookkeepingRuns(query);
+        const scoped = withoutBookkeepingRuns(this.tenantScoped(query));
         const runs = await this.store.listRuns(scoped);
         const hidden = scoped.excludeWorkflows;
         return hidden?.length ? runs.filter((run) => !hidden.includes(run.workflow)) : runs;
@@ -2957,14 +3059,16 @@ export class WorkflowEngine {
      * {@link listRuns}.
      */
     runValueFacets(axis, query, opts) {
-        const scoped = withoutBookkeepingRuns(query);
+        const scoped = withoutBookkeepingRuns(this.tenantScoped(query));
         if (this.store.runValueFacets) {
             return this.store.runValueFacets(axis, scoped, opts);
         }
         return scanRunValueFacets(this.store, axis, scoped, opts);
     }
     /** List a run's step checkpoints (its timeline). Part of the engine's read API; see {@link listRuns}. */
-    listCheckpoints(runId) {
+    async listCheckpoints(runId) {
+        if (await this.runHidden(runId))
+            return [];
         return this.store.listCheckpoints(runId);
     }
     /**
@@ -2972,7 +3076,7 @@ export class WorkflowEngine {
      * has {@link StateStore.runFacets} (else counted over a listing). Part of the engine's read API.
      */
     runFacets(query) {
-        return runFacetsOf(this.store, withoutBookkeepingRuns(query));
+        return runFacetsOf(this.store, withoutBookkeepingRuns(this.tenantScoped(query)));
     }
     /**
      * A run's detail (see `loadRunDetail`), built from the store's checkpoint aggregates when it has
@@ -2980,6 +3084,8 @@ export class WorkflowEngine {
      * instead of the whole history, and carries an `etag` that `options.ifNoneMatch` can match.
      */
     async getRunDetail(runId, options) {
+        if (await this.runHidden(runId))
+            return null;
         const source = this.detailSource();
         const withExtras = async (detail) => {
             const extras = await this.lifecycleDetail(detail.run);
@@ -3117,7 +3223,9 @@ export class WorkflowEngine {
         return out;
     }
     /** One page of a summarized fan's members (see `loadFanMembers`), versioned like {@link getRunDetail}. */
-    getFanMembers(runId, group, query) {
+    async getFanMembers(runId, group, query) {
+        if (await this.runHidden(runId))
+            return null;
         return loadFanMembers(this.detailSource(), runId, group, query);
     }
     detailSource() {
@@ -3225,7 +3333,7 @@ export class WorkflowEngine {
             return [];
         this.#lastStalledSweepAt = now;
         const candidates = await this.store.listRuns({
-            statuses: ['suspended', 'running'],
+            statuses: ['suspended', 'running', 'cancelling'],
             namespace: this.namespace,
             updatedBefore: now - this.stalledAfterMs,
             size: 100,
@@ -3455,7 +3563,7 @@ export class WorkflowEngine {
     }
     async deleteRun(runId) {
         const run = await this.store.getRun(runId);
-        if (!run)
+        if (!run || !this.runVisibleNow(run))
             return 0;
         // Collect children BEFORE deleting this run's checkpoints (getRunChildren reads them).
         let deleted = 0;
@@ -3469,11 +3577,10 @@ export class WorkflowEngine {
     /**
      * Sweep a deleted run's token-keyed buffered signals — they are not run-scoped rows, so the store's
      * deleteRun can't cascade them. Without this every fire-and-forget child that was never joined leaks
-     * its buffered `child:<id>` completion FOREVER (one row per spawn), and a raced compensate-cancel
-     * can leave a `cancel:<id>` marker behind.
+     * its buffered `child:<id>` completion FOREVER (one row per spawn).
      */
     async sweepRunBufferedSignals(runId) {
-        for (const token of [`child:${runId}`, `cancel:${runId}`]) {
+        for (const token of [`child:${runId}`]) {
             for (;;) {
                 const buffered = await this.store.takeBufferedSignal(token).catch(() => null);
                 if (!buffered)
@@ -3541,7 +3648,7 @@ export class WorkflowEngine {
                 if (!TERMINAL_RUN_STATUSES.includes(status)) {
                     if (!cancelLive)
                         continue;
-                    await this.cancel(id).catch(() => undefined);
+                    await this.asEngine(() => this.cancel(id)).catch(() => undefined);
                 }
                 doomed.push(id);
             }
@@ -3706,7 +3813,7 @@ export class WorkflowEngine {
         // Piggybacked on the existing store.getRun above — no extra async step on the happy path. An
         // undefined namespace (a store row created before the field existed) is treated as "belongs to
         // everyone" for back-compat: don't skip it. An operator (this.namespace undefined) belongs to
-        // every namespace too, so the guard never applies to it — that is the whole point of the role.
+        // every namespace too, so the guard never applies to it — that is the whole point of the topology.
         if (this.namespace !== undefined &&
             run.namespace !== undefined &&
             run.namespace !== this.namespace) {
@@ -3725,7 +3832,7 @@ export class WorkflowEngine {
         }
         if (!registered)
             throw new WorkflowNotRegistered(run.workflow, run.workflowVersion);
-        return this.track(runId, this.execute(run, registered.fn));
+        return this.track(runId, this.inRunTenant(run, () => this.execute(run, registered.fn)));
     }
     /**
      * Deferred re-drive after a suspended settle whose waited checkpoint(s) already settled — the
@@ -4069,6 +4176,27 @@ export class WorkflowEngine {
             // finished run) and re-enqueues it for a second execution. Only a run that still reads
             // `running` under the lease is an orphan; release the lease taken just to look at anything else.
             const run = await this.store.getRun(listed.id);
+            // A compensating cancel whose undo died with its pod (or never started: the pod that took the
+            // cancel crashed before its resume ran). Re-drive it rather than re-pend it, which would drop
+            // the cancel: any turn of a `cancelling` run unwinds the saga and settles it `cancelled`.
+            if (run?.status === 'cancelling') {
+                // Only a pod that has the workflow can run its compensations; past the grace, one that is
+                // registered nowhere is dead-lettered like any orphan.
+                if (!this.isRegisteredHere(run) && !(await this.canResolveWorkflow(run))) {
+                    const settled = await this.settleUnregistered(run, nowMs);
+                    if (settled)
+                        results.push(settled);
+                    else if (settled === undefined)
+                        this.logUnregisteredSkip(run);
+                    continue;
+                }
+                await this.store.releaseRunLock(listed.id, this.instanceId);
+                this.trackEffect(Promise.resolve(this.wake(run.id)).catch((error) => {
+                    console.error(`[adonis-durable] could not re-drive cancelling run ${run.id}; retrying on a later tick:`, error);
+                }));
+                results.push({ runId: run.id, status: 'cancelling' });
+                continue;
+            }
             if (run?.status !== 'running') {
                 await this.store.releaseRunLock(listed.id, this.instanceId);
                 continue;
@@ -4547,10 +4675,16 @@ export class WorkflowEngine {
         const seen = _seen ?? new Set();
         if (seen.has(runId))
             return null;
+        if (_seen === undefined && (await this.runHidden(runId)))
+            return null;
         seen.add(runId);
         const run = await this.store.getRun(runId);
         if (!run)
             return null;
+        // A run undoing its saga for a compensating cancel is not retried over: re-pending it would drop
+        // the cancel. It ends `cancelled`, and can be requeued from there.
+        if (run.status === 'cancelling')
+            return { runId, status: 'cancelling' };
         // A FAILED/DEAD run's replay deterministically re-throws its recorded failures — an exhausted
         // failed checkpoint rethrows, and an awaited child's failure completion rethrows through
         // unwrapCompletion — so "retry" without resetting that state is a no-op that re-fails in
@@ -4691,7 +4825,7 @@ export class WorkflowEngine {
      */
     async redispatchPending(runId) {
         const run = await this.store.getRun(runId);
-        if (!run)
+        if (!run || !this.runVisibleNow(run))
             return null;
         const pending = (await this.store.listCheckpoints(runId)).filter((cp) => cp.kind === 'remote' && cp.status === 'pending');
         let redispatched = 0;
@@ -4727,12 +4861,18 @@ export class WorkflowEngine {
      */
     async retryWithInput(runId, input, newRunId) {
         const run = await this.store.getRun(runId);
-        if (!run)
+        if (!run || !this.runVisibleNow(run))
             return null;
         const id = newRunId ?? `${runId}~retry~${globalThis.crypto.randomUUID().slice(0, 8)}`;
         // Inherit the original run's namespace so a retry lands in the SAME worker-pool partition (an
-        // operator-routed run keeps its tenant on retry, rather than falling back to the engine's own).
-        await this.start(run.workflow, input, id, { tags: run.tags, namespace: run.namespace });
+        // operator-routed run keeps its pool on retry, rather than falling back to the engine's own),
+        // and its tenant (or none).
+        await this.start(run.workflow, input, id, {
+            tags: run.tags,
+            namespace: run.namespace,
+            tenantId: run.tenantId ?? null,
+            ...(run.tenantKind ? { tenantKind: run.tenantKind } : {}),
+        });
         return { runId: id };
     }
     /**
@@ -4821,8 +4961,10 @@ export class WorkflowEngine {
         // (`awaitingDecisionTaskId` set) is mid-turn, not settled: that suspend is the engine's re-drive
         // marker, not a sleep/signal the body asked for. Keep waiting until the decision lands. A `blocked`
         // run is parked on the fleet, not on anything the body asked for, and is re-driven on its own.
-        const isSettled = (run) => isTerminal(run.status) ||
-            (run.status === 'suspended' && run.awaitingDecisionTaskId === undefined);
+        // A `cancelling` run is still undoing its saga: it settles once it is `cancelled`.
+        const isSettled = (run) => run.status !== 'cancelling' &&
+            (isTerminal(run.status) ||
+                (run.status === 'suspended' && run.awaitingDecisionTaskId === undefined));
         const shouldResolve = (run) => opts?.terminal ? isTerminal(run.status) : isSettled(run);
         const toResult = (run) => ({
             runId,
@@ -4997,7 +5139,10 @@ export class WorkflowEngine {
         const waiters = await this.store.listSignalWaiters(eventPrefix(name));
         for (const w of waiters) {
             if (eventMatches(payload, eventMatchOf(w.token))) {
-                await this.signal(w.token, payload);
+                // With tenancy on, a tenant's event reaches its own runs only.
+                if (await this.runHidden(w.runId))
+                    continue;
+                await this.asEngine(() => this.signal(w.token, payload));
                 touched += 1;
             }
         }
@@ -5015,6 +5160,13 @@ export class WorkflowEngine {
         // unless the caller opted out. `touched > 0` covers BOTH the waiter fan-out and the subscriber loop.
         if (touched > 0 || opts?.buffer === false)
             return touched;
+        // A buffered copy carries no tenant and would reach whichever run waits next, so a publisher
+        // scoped to one tenant (or to none) does not buffer: its event reaches live waiters only.
+        if (this.tenancy) {
+            const mode = this.tenantFilterNow().mode;
+            if (mode === 'tenant' || mode === 'tenantless')
+                return touched;
+        }
         const bufferedId = globalThis.crypto.randomUUID();
         await this.store.bufferEvent({ name, payload, id: bufferedId, publishedAt: this.clock() });
         // Re-check: a waiter may have registered in the window between the miss above and this buffer
@@ -5125,6 +5277,8 @@ export class WorkflowEngine {
      * side's write, and the destructive `take*` ops arbitrate so a signal is never delivered twice.
      */
     async signal(token, payload) {
+        if (await this.signalHidden(token))
+            return null;
         const waiter = await this.store.takeSignalWaiter(token);
         if (waiter)
             return this.deliverSignal(waiter, payload);
@@ -5146,6 +5300,23 @@ export class WorkflowEngine {
         }
         return this.deliverSignal(lateWaiter, reclaimed.payload);
     }
+    /** Whether a run another tenant owns is the one waiting on `token` (an app-facing signal is dropped). */
+    async signalHidden(token) {
+        if (!this.tenancy)
+            return false;
+        const filter = this.tenantFilterNow();
+        if (filter.mode === 'system' || filter.mode === 'all')
+            return false;
+        const waiters = await this.store.listSignalWaiters(token);
+        for (const waiter of waiters) {
+            if (waiter.token !== token)
+                continue;
+            const run = await this.store.getRun(waiter.runId);
+            if (run && !runVisible(filter, run, this.tenancy.nullRows))
+                return true;
+        }
+        return false;
+    }
     /**
      * Wake the run parked on `token` with `payload`, if there is one; otherwise do nothing — unlike
      * {@link signal}, it never buffers. For a wake-up whose real payload is ALREADY buffered on the
@@ -5177,14 +5348,18 @@ export class WorkflowEngine {
      * null if no run is waiting on the task (e.g. a duplicate/late delivery) — a safe no-op.
      */
     async completeTask(runId, name, result) {
-        return this.signal(`task:${runId}:${name}`, {
+        if (await this.runHidden(runId))
+            return null;
+        return this.asEngine(() => this.signal(`task:${runId}:${name}`, {
             ok: true,
             value: result,
-        });
+        }));
     }
     /** Report that a `ctx.task` failed — the run resumes and throws a FatalError at the task. */
     async failTask(runId, name, error) {
-        return this.signal(`task:${runId}:${name}`, { ok: false, error });
+        if (await this.runHidden(runId))
+            return null;
+        return this.asEngine(() => this.signal(`task:${runId}:${name}`, { ok: false, error }));
     }
     /**
      * Notify a parent that's waiting on `runId` as a child of its terminal outcome (the `ctx.child`
@@ -5237,7 +5412,7 @@ export class WorkflowEngine {
                 if (!(await childCompletionNeedsBuffer(this.store, token.slice(CHILD_TOKEN_PREFIX.length), completion))) {
                     return;
                 }
-                const resumed = await this.signal(token, completion);
+                const resumed = await this.asEngine(() => this.signal(token, completion));
                 // The parent may settle the child from its run row between the look above and the buffer
                 // write (a join that registers its waiter only now): it then stamped the child first, and the
                 // copy just buffered would never be read.
@@ -5302,7 +5477,7 @@ export class WorkflowEngine {
             : settled.status === 'completed'
                 ? { ok: true, value: settled.output }
                 : failedCompletion(settled.error ?? { message: settled.status });
-        await this.signal(token, completion);
+        await this.asEngine(() => this.signal(token, completion));
     }
     /**
      * Cancel a run (e.g. from the dashboard). Returns null if the run does not exist. Pass
@@ -5311,6 +5486,11 @@ export class WorkflowEngine {
      * cancelled. Without it, cancellation is immediate (no undo).
      */
     async cancel(runId, opts) {
+        if (await this.runHidden(runId))
+            return null;
+        return this.asEngine(() => this.cancelOnce(runId, opts, true));
+    }
+    async cancelOnce(runId, opts, mayRetry) {
         const run = await this.store.getRun(runId);
         if (!run)
             return null;
@@ -5323,18 +5503,35 @@ export class WorkflowEngine {
         if (TERMINAL_RUN_STATUSES.includes(run.status)) {
             return { runId, status: run.status, output: run.output, error: run.error };
         }
+        // A compensating cancel is already undoing the saga: a repeat cancel, with or without
+        // `compensate`, echoes it. Taking the instant path below would mark it `cancelled` and skip the
+        // undo still in flight. Recovery re-drives a `cancelling` run whose undo died.
+        if (run.status === 'cancelling')
+            return { runId, status: 'cancelling' };
         // Compensating cancel: resume the run with a cancellation pending — replay re-registers the saga,
         // and at the suspension point execute() runs the undo and marks it cancelled. Run that resume in
         // the BACKGROUND so the caller (e.g. an HTTP request) never blocks on replaying the workflow +
         // compensations. `execute` holds the run's lease, so this can't double-run one a live worker owns
         // (its lease acquire fails and it no-ops); the broadcast tells that worker to abort cooperatively.
         if (opts?.compensate && (run.status === 'suspended' || run.status === 'running')) {
+            // DURABLE cancel intent: the persisted `cancelling` status. It survives a crash or a pod
+            // handoff (any turn of a `cancelling` run unwinds it, a turn already executing on another pod
+            // finds it when its settle loses the status race, and crash recovery re-drives one whose undo
+            // died), and shows the dashboard/API that an undo is in progress. Conditional: a run that
+            // moved meanwhile (settled, parked) is re-read and cancelled for what it is now.
+            const marked = await this.store.updateRunIf(runId, ['suspended', 'running'], {
+                status: 'cancelling',
+                updatedAt: new Date(),
+            });
+            if (!marked) {
+                if (mayRetry)
+                    return this.cancelOnce(runId, opts, false);
+                const current = await this.store.getRun(runId);
+                return current
+                    ? { runId, status: current.status, output: current.output, error: current.error }
+                    : null;
+            }
             this.cancelRequested.add(runId);
-            // DURABLE cancel intent: the in-memory flag above only survives on this pod, this boot. The
-            // buffered `cancel:<id>` marker is what lets the compensating unwind survive a crash or be
-            // picked up by ANOTHER pod (its resume replays into the WorkflowSuspended catch, which checks
-            // the marker when the flag is absent). Consumed there once `cancelled` is durably recorded.
-            await this.store.bufferSignal(`cancel:${runId}`, true).catch(() => undefined);
             if (this.controlPlane) {
                 void this.controlPlane
                     .publishControl({ kind: 'cancel', runId, from: this.instanceId }, runTarget(run.namespace))
@@ -5352,7 +5549,7 @@ export class WorkflowEngine {
             // needed covering. The one-microtask defer is preserved; the promise is registered synchronously.
             this.trackEffect(Promise.resolve().then(() => this.resumeImplicit(runId).then(() => this.notifyCancelled(runId))));
             await this.cancelChildren(runId, opts);
-            return { runId, status: run.status };
+            return { runId, status: 'cancelling' };
         }
         const error = { message: 'cancelled' };
         // Clears a remote turn's decision marker too: a cancelled run awaits no decision.
@@ -5412,7 +5609,7 @@ export class WorkflowEngine {
      */
     async cancelChildren(parentRunId, opts) {
         for (const id of await this.getRunChildren(parentRunId)) {
-            await this.cancel(id, opts).catch(() => undefined);
+            await this.asEngine(() => this.cancel(id, opts)).catch(() => undefined);
         }
     }
     /**
@@ -5468,7 +5665,7 @@ export class WorkflowEngine {
      * the full listing for a store without one.
      */
     countRuns(query) {
-        return countRuns(this.store, withoutBookkeepingRuns(query));
+        return countRuns(this.store, withoutBookkeepingRuns(this.tenantScoped(query)));
     }
     /**
      * Every workflow the live fleet can execute in this engine's partition: what workers DECLARE in
@@ -5689,7 +5886,7 @@ export class WorkflowEngine {
                 publishedAt: this.clock(),
             })
                 .catch(() => undefined);
-            const marked = await this.store.updateRunIf(runId, ['running', 'suspended', 'blocked'], {
+            const marked = await this.store.updateRunIf(runId, WAKEABLE_RUN_STATUSES, {
                 wakeRequestedAt: this.clock(),
             });
             return marked ? 'stamped' : 'gone';
@@ -5728,7 +5925,7 @@ export class WorkflowEngine {
         if (!(await this.store.tryLockRun(runId, this.instanceId, nowMs + this.leaseMs, nowMs)))
             return;
         const fresh = await this.store.getRun(runId);
-        if (fresh?.status !== 'suspended' && fresh?.status !== 'blocked') {
+        if (!fresh || !PARKED_RUN_STATUSES.includes(fresh.status)) {
             await this.store.releaseRunLock(runId, this.instanceId);
             return;
         }
@@ -5828,7 +6025,7 @@ export class WorkflowEngine {
         }
         // Free, and `pending` (its dispatch or the pending-run poll runs it) or `running` with no lease
         // (a holder that died mid-turn: crash recovery takes it): nothing for a wake to do.
-        if (run.status !== 'suspended' && run.status !== 'blocked')
+        if (!PARKED_RUN_STATUSES.includes(run.status))
             return;
         if (this.runQueue) {
             this.enqueueRun({
@@ -5841,7 +6038,7 @@ export class WorkflowEngine {
         if (!(await this.store.tryLockRun(runId, this.instanceId, nowMs + this.leaseMs, nowMs)))
             return;
         const fresh = await this.store.getRun(runId);
-        if (fresh?.status !== 'suspended' && fresh?.status !== 'blocked') {
+        if (!fresh || !PARKED_RUN_STATUSES.includes(fresh.status)) {
             await this.store.releaseRunLock(runId, this.instanceId);
             return;
         }
@@ -5850,6 +6047,18 @@ export class WorkflowEngine {
                 throw err;
         });
     }
+    /**
+     * Run a turn with the run's own tenant active, so a resume driven by a timer, a signal or recovery
+     * (no request around it) sees the tenant the run was started under, and what the turn starts or
+     * dispatches carries it. A run with no tenant runs as before.
+     */
+    inRunTenant(run, fn) {
+        const tenant = runTenantOf(run);
+        if (!this.tenancy || !tenant)
+            return fn();
+        const tenancy = this.tenancy;
+        return engineInternal.run(false, () => tenancy.run(tenant, fn));
+    }
     async execute(run, fn) {
         const registered = this.workflows.get(versionKey(run.workflow, run.workflowVersion));
         // Hold the lease for the WHOLE execution — whatever path got us here (leased sweep, a signal, a
@@ -5916,6 +6125,11 @@ export class WorkflowEngine {
             // A remote (e.g. Python) workflow is advanced by dispatching workflow tasks, not by running an
             // in-process body — but everything around it (lease, recovery, timers, the resume that lands us
             // here on a step result) is identical, so it branches here under the same lease.
+            // A `cancelling` run is a compensating cancel in flight, maybe re-driven by crash recovery or
+            // by another pod that never saw the in-memory flag: re-derive the intent from the status, so
+            // this turn unwinds the saga and settles `cancelled` at whichever exit it reaches.
+            if (run.status === 'cancelling')
+                this.cancelRequested.add(run.id);
             const result = registered?.remote
                 ? await this.runRemoteExecution(run, registered)
                 : await this.runExecution(run, fn, registered);
@@ -6306,7 +6520,7 @@ export class WorkflowEngine {
         // settle — and it must happen even while a dispatched turn is still out (the hold below would
         // otherwise swallow the cancel's resume, and the in-flight turn's decision would carry the run on
         // as if it was never cancelled). The late decision then finds the run cancelled and is dropped.
-        if (await this.takeCancelIntent(run.id))
+        if (this.takeCancelIntent(run.id))
             return this.compensateAndCancel(run, []);
         // A dispatched turn is still out and within its liveness window: whatever woke the run (a step
         // result, a signal, a wait recheck) must not dispatch a second turn over it — that would supersede
@@ -6399,13 +6613,17 @@ export class WorkflowEngine {
             // with its lease released, so crash recovery re-drives it (replaying completed steps; bounded by
             // `maxRecoveryAttempts`).
             if (err instanceof RemoteWorkflowTimeout) {
-                await this.store
+                const reset = await this.store
                     .updateRunIf(run.id, ['running', 'suspended'], {
                     status: 'running',
                     updatedAt: new Date(),
                 })
                     .catch(() => false);
                 await this.releaseTurnLease(run.id);
+                // Not reset: the run moved on meanwhile (a compensating cancel made it `cancelling`, or it
+                // settled). Report what it is now, not a `running` it no longer reads.
+                if (!reset)
+                    return this.echoCurrentStatus(run.id);
                 return { runId: run.id, status: 'running' };
             }
             const error = {
@@ -7317,21 +7535,25 @@ export class WorkflowEngine {
         return wakeAt;
     }
     /**
-     * Whether a compensating cancel is pending for this run, consuming the in-memory flag. The intent
-     * lives in TWO places: the in-memory `cancelRequested` fast path (same pod, same boot), and the
-     * DURABLE buffered `cancel:<id>` marker `cancel()` persisted — so a crash or a cross-pod handoff
-     * mid-cancellation can't silently forget the cancel. The marker is peeked (take + re-buffer), not
-     * consumed: it must survive a crash mid-unwind; {@link compensateAndCancel} drains it once
-     * `cancelled` is durably recorded.
+     * Whether a compensating cancel is pending for this run, consuming the in-memory flag. The flag is
+     * set by `cancel()` on this pod, and at turn start for a run that reads `cancelling` (the durable
+     * intent: a cancel from another pod, or one re-driven by recovery). A cancel another pod persists
+     * while this turn runs is found at the settle instead: the run no longer reads a settle-eligible
+     * status, and {@link settleOrUnwind} unwinds it.
      */
-    async takeCancelIntent(runId) {
-        if (this.cancelRequested.delete(runId))
-            return true;
-        const marker = await this.store.takeBufferedSignal(`cancel:${runId}`);
-        if (!marker)
-            return false;
-        await this.store.bufferSignal(`cancel:${runId}`, marker.payload).catch(() => undefined);
-        return true;
+    takeCancelIntent(runId) {
+        return this.cancelRequested.delete(runId);
+    }
+    /**
+     * Settle this turn's `outcome` — unless a compensating cancel was persisted on another pod while the
+     * turn ran: the settle's status check then reads `cancelling`, and this turn (which still holds the
+     * lease and the saga it registered) undoes it and settles `cancelled` itself.
+     */
+    async settleOrUnwind(run, outcome, compensations) {
+        const settled = await this.settleRun(run, outcome);
+        if (settled.status !== 'cancelling' || this.leaseLostRuns.has(run.id))
+            return settled;
+        return this.compensateAndCancel(run, compensations);
     }
     /**
      * Undo the registered saga compensations in reverse and settle the run `cancelled`. Shared by EVERY
@@ -7341,8 +7563,8 @@ export class WorkflowEngine {
      */
     async compensateAndCancel(run, compensations) {
         this.cancelRequested.delete(run.id);
-        // Zombie fence (as in settleRun): another instance owns the run now; the durable marker is still
-        // there for it, so this stale turn must neither undo nor write.
+        // Zombie fence (as in settleRun): another instance owns the run now; the persisted `cancelling`
+        // status is still there for it, so this stale turn must neither undo nor write.
         if (this.leaseLostRuns.has(run.id))
             return this.echoCurrentStatus(run.id);
         for (let i = compensations.length - 1; i >= 0; i -= 1) {
@@ -7357,13 +7579,6 @@ export class WorkflowEngine {
             awaitingDecisionTaskId: undefined,
             updatedAt: new Date(),
         }));
-        // The cancellation is durably recorded — consume the marker(s) so a later retry of this run id
-        // doesn't get spuriously re-cancelled.
-        for (;;) {
-            const m = await this.store.takeBufferedSignal(`cancel:${run.id}`).catch(() => null);
-            if (!m)
-                break;
-        }
         this.emit({
             type: 'run.failed',
             runId: run.id,
@@ -7408,7 +7623,11 @@ export class WorkflowEngine {
         }
         // Singleton admission gate: if this run shares its key with `limit` older in-flight runs, wait
         // (suspend on a short timer) until a slot frees instead of running now. Re-checked on each resume.
-        if (registered?.singleton && !(await this.singletons.admit(run, registered.singleton))) {
+        // A `cancelling` run skips it: it is tearing down, not competing for a slot, and parking it here
+        // would overwrite `cancelling` with `suspended`.
+        if (run.status !== 'cancelling' &&
+            registered?.singleton &&
+            !(await this.singletons.admit(run, registered.singleton))) {
             return this.singletonWaitResult(run);
         }
         // A body this instance parked on dispatched steps in an earlier turn continues from there (see
@@ -7487,10 +7706,10 @@ export class WorkflowEngine {
             }
             // A compensating cancel requested while this turn ran (or one whose resume replays straight to
             // `return`, with no suspension point left) still wins: undo + settle cancelled, never completed.
-            if (await this.takeCancelIntent(run.id)) {
+            if (this.takeCancelIntent(run.id)) {
                 return await this.compensateAndCancel(run, compensations);
             }
-            return await this.settleRun(run, { kind: 'completed', output });
+            return await this.settleOrUnwind(run, { kind: 'completed', output }, compensations);
         }
         catch (err) {
             if (err instanceof ContinueAsNew) {
@@ -7499,7 +7718,11 @@ export class WorkflowEngine {
                 if (this.leaseLostRuns.has(run.id))
                     return await this.echoCurrentStatus(run.id);
                 // A cancel in flight wins over continue-as-new: undo + cancel instead of forking the next run.
-                if (await this.takeCancelIntent(run.id)) {
+                // The run is re-read here (continue-as-new is rare) so a cancel another pod persisted during
+                // this turn usually stops the fork; one that lands after the read loses the conditional
+                // completion below instead.
+                if (this.takeCancelIntent(run.id) ||
+                    (await this.store.getRun(run.id))?.status === 'cancelling') {
                     return await this.compensateAndCancel(run, compensations);
                 }
                 const nextId = nextContinuationId(run.id);
@@ -7513,13 +7736,28 @@ export class WorkflowEngine {
                 // converges on the already-persisted continuation (idempotent by the continuation id).
                 // `dispatch: false`: the tracked handoff below owns the pickup. The continuation inherits
                 // the parent RUN's namespace so a tenant's chain stays in its worker-pool partition.
-                await this.startCore(run.workflow, err.input, nextId, { namespace: run.namespace }, false);
-                await this.store.updateRun(run.id, {
+                await this.startCore(run.workflow, err.input, nextId, {
+                    namespace: run.namespace,
+                    tenantId: run.tenantId ?? null,
+                    ...(run.tenantKind ? { tenantKind: run.tenantKind } : {}),
+                }, false);
+                // Conditional, like every settle: a cancel persisted after the re-read above (`cancelling`, or a
+                // plain `cancelled`) must win over this completion. The continuation already exists then, and
+                // nothing has handed it off: cancel it, and unwind this run if the cancel compensates.
+                const completed = await this.store.updateRunIf(run.id, SETTLE_ELIGIBLE_STATUSES, {
                     status: 'completed',
                     output: undefined,
                     error: undefined,
                     updatedAt: new Date(),
                 });
+                if (!completed) {
+                    await this.asEngine(() => this.cancel(nextId));
+                    const current = await this.store.getRun(run.id);
+                    if (current?.status === 'cancelling' && !this.leaseLostRuns.has(run.id)) {
+                        return await this.compensateAndCancel(run, compensations);
+                    }
+                    return await this.echoCurrentStatus(run.id);
+                }
                 this.emit({
                     type: 'run.completed',
                     runId: run.id,
@@ -7531,7 +7769,11 @@ export class WorkflowEngine {
                 // continuation has entered `inflight` and settled, so `drain()` can't return before the next
                 // run's store writes land. Still deferred; its startCore call converges on the run persisted
                 // above (prior-run branch) and drives the pickup.
-                this.handoffRun(run.workflow, err.input, nextId, { namespace: run.namespace });
+                this.handoffRun(run.workflow, err.input, nextId, {
+                    namespace: run.namespace,
+                    tenantId: run.tenantId ?? null,
+                    ...(run.tenantKind ? { tenantKind: run.tenantKind } : {}),
+                });
                 return { runId: run.id, status: 'completed' };
             }
             if (err instanceof WorkflowBlocked) {
@@ -7543,10 +7785,10 @@ export class WorkflowEngine {
             if (err instanceof WorkflowSuspended) {
                 // A compensating cancel resumed this run to reach here: the replay re-registered the saga,
                 // so undo the completed steps in reverse and mark it cancelled instead of re-suspending.
-                if (await this.takeCancelIntent(run.id)) {
+                if (this.takeCancelIntent(run.id)) {
                     return await this.compensateAndCancel(run, compensations);
                 }
-                const settled = await this.settleRun(run, { kind: 'suspended', wakeAt: err.wakeAt });
+                const settled = await this.settleOrUnwind(run, { kind: 'suspended', wakeAt: err.wakeAt }, compensations);
                 const stamp = this.suspendStamps.get(run.id);
                 this.suspendStamps.delete(run.id);
                 if (parkedTurn && cont && settled.status === 'suspended' && stamp !== undefined) {
@@ -7571,7 +7813,7 @@ export class WorkflowEngine {
             }
             // A cancel in flight that surfaced as a thrown error settles `cancelled`, not `failed`: the saga
             // undo runs either way; this keeps the terminal status faithful to the cancel request.
-            if (await this.takeCancelIntent(run.id)) {
+            if (this.takeCancelIntent(run.id)) {
                 return await this.compensateAndCancel(run, compensations);
             }
             const error = {
@@ -7589,7 +7831,8 @@ export class WorkflowEngine {
                     continue;
                 await this.runCompensation(run, comp, i);
             }
-            return await this.settleRun(run, { kind: 'failed', error });
+            // The saga is undone already: a cancel persisted meanwhile only needs the `cancelled` settle.
+            return await this.settleOrUnwind(run, { kind: 'failed', error }, []);
         }
         finally {
             // Release the recovery lease once the run reaches a terminal/suspended state, so the
@@ -7903,7 +8146,7 @@ export class WorkflowEngine {
             // escape `drain()` onto a torn-down connection ("Transaction query already complete"). The
             // one-microtask defer is preserved; the promise is registered synchronously.
             cancelChild: (childId) => {
-                this.trackEffect(Promise.resolve().then(() => this.cancel(childId)));
+                this.trackEffect(Promise.resolve().then(() => this.asEngine(() => this.cancel(childId))));
             },
             // Shallow-merge into the run's searchAttributes (the ctx primitive makes this exactly-once).
             upsertSearchAttributes: async (runId, attrs) => {
@@ -7951,11 +8194,13 @@ export class WorkflowEngine {
      * paused at a breakpoint.
      */
     async continue(runId) {
+        if (await this.runHidden(runId))
+            return null;
         const checkpoints = await this.store.listCheckpoints(runId);
         const bp = checkpoints.find(isBreakpoint);
         if (!bp)
             return null;
-        return this.signal(breakpointToken(runId, bp.seq), undefined);
+        return this.asEngine(() => this.signal(breakpointToken(runId, bp.seq), undefined));
     }
     /**
      * Read the latest value a run published for `key` via {@link WorkflowCtx.setEvent} — a
@@ -7963,6 +8208,8 @@ export class WorkflowEngine {
      * never published that key. The suspend-model counterpart of a Temporal query.
      */
     async getEvent(runId, key) {
+        if (await this.runHidden(runId))
+            return undefined;
         const name = `event:${key}`;
         // Targeted read: the highest-seq checkpoint for this name is the most recent value (a re-published
         // key overwrites at a higher seq), matching the old "last in seq order wins" scan. Falls back to a
@@ -7994,7 +8241,7 @@ export class WorkflowEngine {
      */
     async update(runId, name, arg) {
         const run = await this.store.getRun(runId);
-        if (!run)
+        if (!run || !this.runVisibleNow(run))
             return { accepted: false, reason: `run ${runId} not found` };
         const validate = this.updateValidators.get(`${run.workflow}:${name}`);
         if (validate) {
@@ -8007,7 +8254,7 @@ export class WorkflowEngine {
                 return { accepted: false, reason: err instanceof Error ? err.message : String(err) };
             }
         }
-        const result = await this.signal(`update:${runId}:${name}`, arg);
+        const result = await this.asEngine(() => this.signal(`update:${runId}:${name}`, arg));
         return { accepted: true, run: result };
     }
     // ---------------------------------------------------------------------------
@@ -9365,7 +9612,7 @@ export class WorkflowEngine {
         // batch result resurrected a `failed` run into `suspended`, which then continued executing as if
         // the failure never happened, racing whatever the operator did about it, and after any saga
         // compensations already ran). Terminal is terminal — recovery of a failed run belongs to an
-        // explicit `requeue`/`durable:retry`, never to a stray result.
+        // explicit `requeue`/`durable:runs:retry`, never to a stray result.
         if (cp.status !== 'pending') {
             if (!terminal)
                 return { cp, run };
@@ -9731,6 +9978,8 @@ function turnBeatKey(runId) {
 /** The run whose remote turn is being enqueued in the current async context — lets the decision
  *  handler spot a transport delivering the decision from inside its own `dispatchWorkflowTask`. */
 const turnDispatchAls = new AsyncLocalStorage();
+/** `true` while the engine does its own work: app-facing tenant guards do not apply inside it. */
+const engineInternal = new AsyncLocalStorage();
 /** Tolerance (ms) between the worker's clock, which stamps a persisted heartbeat, and this engine's,
  *  when deciding whether a beat belongs to the current attempt (see `awaitWithHeartbeat`). */
 const HEARTBEAT_CLOCK_SKEW_MS = 1_000;