@adonis-agora/durable 1.0.0-rc.12 → 1.0.0-rc.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +20 -0
- package/dist/assets/spa/assets/index-DlaQG-kn.js +16 -0
- package/dist/assets/spa/assets/index-DlaQG-kn.js.map +1 -0
- package/dist/assets/spa/assets/index-oUo7_8ab.css +1 -0
- package/dist/assets/spa/index.html +2 -2
- package/dist/commands/agora_doctor.d.ts +23 -0
- package/dist/commands/agora_doctor.d.ts.map +1 -0
- package/dist/commands/agora_doctor.js +110 -0
- package/dist/commands/agora_doctor.js.map +1 -0
- package/dist/commands/durable_doctor.d.ts +8 -0
- package/dist/commands/durable_doctor.d.ts.map +1 -0
- package/dist/commands/durable_doctor.js +8 -0
- package/dist/commands/durable_doctor.js.map +1 -0
- package/dist/commands/main.d.ts +4 -3
- package/dist/commands/main.d.ts.map +1 -1
- package/dist/commands/main.js +5 -1
- package/dist/commands/main.js.map +1 -1
- package/dist/providers/dashboard_provider.d.ts +31 -0
- package/dist/providers/dashboard_provider.d.ts.map +1 -1
- package/dist/providers/dashboard_provider.js +304 -14
- package/dist/providers/dashboard_provider.js.map +1 -1
- package/dist/providers/durable_provider.d.ts.map +1 -1
- package/dist/providers/durable_provider.js +16 -1
- package/dist/providers/durable_provider.js.map +1 -1
- package/dist/src/dashboard/console_authorize.d.ts +22 -0
- package/dist/src/dashboard/console_authorize.d.ts.map +1 -1
- package/dist/src/dashboard/console_authorize.js +63 -1
- package/dist/src/dashboard/console_authorize.js.map +1 -1
- package/dist/src/dashboard/console_switcher.d.ts +121 -0
- package/dist/src/dashboard/console_switcher.d.ts.map +1 -0
- package/dist/src/dashboard/console_switcher.js +201 -0
- package/dist/src/dashboard/console_switcher.js.map +1 -0
- package/dist/src/dashboard/define_config.d.ts +4 -2
- package/dist/src/dashboard/define_config.d.ts.map +1 -1
- package/dist/src/dashboard/define_config.js.map +1 -1
- package/dist/src/dashboard/dev_tools.d.ts +102 -0
- package/dist/src/dashboard/dev_tools.d.ts.map +1 -0
- package/dist/src/dashboard/dev_tools.js +491 -0
- package/dist/src/dashboard/dev_tools.js.map +1 -0
- package/dist/src/dashboard/handlers.d.ts.map +1 -1
- package/dist/src/dashboard/handlers.js +2 -0
- package/dist/src/dashboard/handlers.js.map +1 -1
- package/dist/src/dashboard/index.d.ts +6 -2
- package/dist/src/dashboard/index.d.ts.map +1 -1
- package/dist/src/dashboard/index.js +6 -2
- package/dist/src/dashboard/index.js.map +1 -1
- package/dist/src/dashboard/spa.d.ts +12 -1
- package/dist/src/dashboard/spa.d.ts.map +1 -1
- package/dist/src/dashboard/spa.js +6 -2
- package/dist/src/dashboard/spa.js.map +1 -1
- package/dist/src/dashboard/tenant_scope.d.ts +38 -0
- package/dist/src/dashboard/tenant_scope.d.ts.map +1 -0
- package/dist/src/dashboard/tenant_scope.js +166 -0
- package/dist/src/dashboard/tenant_scope.js.map +1 -0
- package/dist/src/define_config.d.ts +9 -0
- package/dist/src/define_config.d.ts.map +1 -1
- package/dist/src/define_config.js.map +1 -1
- package/dist/src/doctor/agora_doctor.d.ts +116 -0
- package/dist/src/doctor/agora_doctor.d.ts.map +1 -0
- package/dist/src/doctor/agora_doctor.js +685 -0
- package/dist/src/doctor/agora_doctor.js.map +1 -0
- package/dist/src/doctor/checks.d.ts +22 -0
- package/dist/src/doctor/checks.d.ts.map +1 -0
- package/dist/src/doctor/checks.js +197 -0
- package/dist/src/doctor/checks.js.map +1 -0
- package/dist/src/doctor/index.d.ts +9 -0
- package/dist/src/doctor/index.d.ts.map +1 -0
- package/dist/src/doctor/index.js +16 -0
- package/dist/src/doctor/index.js.map +1 -0
- package/dist/src/engine.d.ts +35 -0
- package/dist/src/engine.d.ts.map +1 -1
- package/dist/src/engine.js +182 -31
- package/dist/src/engine.js.map +1 -1
- package/dist/src/index.d.ts +5 -2
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +4 -2
- package/dist/src/index.js.map +1 -1
- package/dist/src/interfaces.d.ts +29 -1
- package/dist/src/interfaces.d.ts.map +1 -1
- package/dist/src/interfaces.js.map +1 -1
- package/dist/src/run-gateway/proxy-run-gateway.d.ts.map +1 -1
- package/dist/src/run-gateway/proxy-run-gateway.js +3 -0
- package/dist/src/run-gateway/proxy-run-gateway.js.map +1 -1
- package/dist/src/run-gateway/run-request-responder.d.ts +5 -0
- package/dist/src/run-gateway/run-request-responder.d.ts.map +1 -1
- package/dist/src/run-gateway/run-request-responder.js +28 -3
- package/dist/src/run-gateway/run-request-responder.js.map +1 -1
- package/dist/src/run-gateway/tenant-scoped-gateway.d.ts +10 -0
- package/dist/src/run-gateway/tenant-scoped-gateway.d.ts.map +1 -0
- package/dist/src/run-gateway/tenant-scoped-gateway.js +107 -0
- package/dist/src/run-gateway/tenant-scoped-gateway.js.map +1 -0
- package/dist/src/run-value-facets.d.ts +1 -1
- package/dist/src/run-value-facets.d.ts.map +1 -1
- package/dist/src/run-value-facets.js +5 -1
- package/dist/src/run-value-facets.js.map +1 -1
- package/dist/src/schedules.d.ts +16 -1
- package/dist/src/schedules.d.ts.map +1 -1
- package/dist/src/schedules.js +29 -4
- package/dist/src/schedules.js.map +1 -1
- package/dist/src/stores/lucid-mappers.d.ts +3 -0
- package/dist/src/stores/lucid-mappers.d.ts.map +1 -1
- package/dist/src/stores/lucid-mappers.js +12 -0
- package/dist/src/stores/lucid-mappers.js.map +1 -1
- package/dist/src/stores/lucid-schema.d.ts +1 -1
- package/dist/src/stores/lucid-schema.d.ts.map +1 -1
- package/dist/src/stores/lucid-schema.js +24 -1
- package/dist/src/stores/lucid-schema.js.map +1 -1
- package/dist/src/stores/lucid.d.ts.map +1 -1
- package/dist/src/stores/lucid.js +31 -2
- package/dist/src/stores/lucid.js.map +1 -1
- package/dist/src/tenancy.d.ts +104 -0
- package/dist/src/tenancy.d.ts.map +1 -0
- package/dist/src/tenancy.js +91 -0
- package/dist/src/tenancy.js.map +1 -0
- package/dist/src/testing/in-memory-state-store.d.ts.map +1 -1
- package/dist/src/testing/in-memory-state-store.js +6 -0
- package/dist/src/testing/in-memory-state-store.js.map +1 -1
- package/dist/src/testing-kit/state-store-conformance.d.ts.map +1 -1
- package/dist/src/testing-kit/state-store-conformance.js +46 -0
- package/dist/src/testing-kit/state-store-conformance.js.map +1 -1
- package/package.json +10 -4
- package/skills/durable-cluster/SKILL.md +2 -1
- package/dist/assets/spa/assets/index-B2kIfzsc.css +0 -1
- package/dist/assets/spa/assets/index-Bgf_8IJ0.js +0 -16
- package/dist/assets/spa/assets/index-Bgf_8IJ0.js.map +0 -1
package/dist/src/engine.js
CHANGED
|
@@ -36,6 +36,7 @@ import { SINGLETON_RESERVED_TAGS, withoutAdmission, withSingletonTracked, } from
|
|
|
36
36
|
import { SingletonGate } from './singleton-gate.js';
|
|
37
37
|
import { validateStandard, validateStandardSync, } from './standard-schema.js';
|
|
38
38
|
import { StickyCache, StickyContinuation } from './sticky-continuation.js';
|
|
39
|
+
import { intersectTenantIds, runTenantOf, runVisible, tenantScopeIds, } from './tenancy.js';
|
|
39
40
|
import { sanitizeQueueToken, tenantGroup } from './tenant-group.js';
|
|
40
41
|
import { TransportPool } from './transport-pool.js';
|
|
41
42
|
import { workflowAls } from './workflow-als.js';
|
|
@@ -338,6 +339,7 @@ export class WorkflowEngine {
|
|
|
338
339
|
context;
|
|
339
340
|
/** Establish the originating context ambiently around a local step body (see {@link WorkflowEngineDeps.rehydrate}). Default passthrough. */
|
|
340
341
|
rehydrate;
|
|
342
|
+
tenancy;
|
|
341
343
|
compensationRetries;
|
|
342
344
|
compensationTimeoutMs;
|
|
343
345
|
retention;
|
|
@@ -615,6 +617,7 @@ export class WorkflowEngine {
|
|
|
615
617
|
this.context = deps.context;
|
|
616
618
|
// Default passthrough: with no bridge supplied, a local step body runs exactly as before.
|
|
617
619
|
this.rehydrate = deps.rehydrate ?? ((_carrier, fn) => fn());
|
|
620
|
+
this.tenancy = deps.tenancy;
|
|
618
621
|
this.compensationRetries = Math.max(1, deps.compensationRetries ?? 1);
|
|
619
622
|
this.compensationTimeoutMs = Math.max(1_000, deps.compensationTimeoutMs ?? 300_000);
|
|
620
623
|
this.retention = deps.retention;
|
|
@@ -626,6 +629,8 @@ export class WorkflowEngine {
|
|
|
626
629
|
start: (workflow, input, runId, opts) => this.start(workflow, input, runId, opts),
|
|
627
630
|
getRun: (runId) => this.store.getRun(runId),
|
|
628
631
|
isDraining: () => this.draining,
|
|
632
|
+
tenant: () => this.startTenant(undefined),
|
|
633
|
+
visible: (record) => this.runVisibleNow({ tenantId: record.tenantId }),
|
|
629
634
|
}, { namespace: this.namespace, clock: this.clock });
|
|
630
635
|
this.stalledAfterMs = Math.max(60_000, deps.stalledAfterMs ?? 900_000);
|
|
631
636
|
this.trackStepStart = deps.trackStepStart ?? true;
|
|
@@ -1465,6 +1470,8 @@ export class WorkflowEngine {
|
|
|
1465
1470
|
};
|
|
1466
1471
|
this.trackEffect(this.start(target, input, `dlq:${run.id}`, {
|
|
1467
1472
|
...(run.namespace !== undefined ? { namespace: run.namespace } : {}),
|
|
1473
|
+
tenantId: run.tenantId ?? null,
|
|
1474
|
+
...(run.tenantKind ? { tenantKind: run.tenantKind } : {}),
|
|
1468
1475
|
}).catch((err) => {
|
|
1469
1476
|
console.warn(`[adonis-durable] could not start dead-letter workflow ${target} for run ${run.id}: ${err instanceof Error ? err.message : String(err)}`);
|
|
1470
1477
|
}));
|
|
@@ -2548,6 +2555,11 @@ export class WorkflowEngine {
|
|
|
2548
2555
|
*/
|
|
2549
2556
|
async republishLifecycle(target) {
|
|
2550
2557
|
const result = { runs: 0, started: 0, failed: [] };
|
|
2558
|
+
if ('runId' in target &&
|
|
2559
|
+
typeof target.runId === 'string' &&
|
|
2560
|
+
(await this.runHidden(target.runId))) {
|
|
2561
|
+
return result;
|
|
2562
|
+
}
|
|
2551
2563
|
// A backfill usually follows a deploy: read the fleet's subscriptions now, not from the cache.
|
|
2552
2564
|
await this.refreshAnnouncedSubscriptions();
|
|
2553
2565
|
const publish = async (run) => {
|
|
@@ -2792,6 +2804,9 @@ export class WorkflowEngine {
|
|
|
2792
2804
|
// Idempotent by run id: a redelivered trigger (at-least-once queues) or a scheduler re-tick for
|
|
2793
2805
|
// the same id is a no-op, returning the existing run's state instead of starting a duplicate.
|
|
2794
2806
|
const existing = async (prior) => {
|
|
2807
|
+
// Another tenant's run id: answer like a clash, never with that run's state.
|
|
2808
|
+
if (!this.runVisibleNow(prior))
|
|
2809
|
+
throw new Error(`run ${runId} already exists`);
|
|
2795
2810
|
// A PENDING prior still needs an executor: re-nudge the dispatcher so a redelivered trigger —
|
|
2796
2811
|
// or a continuation persisted pre-settle (see the ContinueAsNew handoff) picked up by a custom
|
|
2797
2812
|
// dispatcher — doesn't sit `pending` until the next poll. Idempotent: execution is leased, so
|
|
@@ -2840,6 +2855,8 @@ export class WorkflowEngine {
|
|
|
2840
2855
|
// Compared against the ENGINE clock (injectable), not wall time — the same clock the timer
|
|
2841
2856
|
// poller will use to decide the run is due.
|
|
2842
2857
|
const delayed = startAtMs !== undefined && startAtMs > this.clock();
|
|
2858
|
+
const tenant = this.startTenant(opts);
|
|
2859
|
+
const tenantNamespace = tenant && opts?.namespace === undefined ? this.tenancy?.namespace?.(tenant) : undefined;
|
|
2843
2860
|
const run = {
|
|
2844
2861
|
id: runId,
|
|
2845
2862
|
workflow: name,
|
|
@@ -2847,7 +2864,10 @@ export class WorkflowEngine {
|
|
|
2847
2864
|
status: delayed ? 'suspended' : 'pending',
|
|
2848
2865
|
...(delayed ? { wakeAt: startAtMs } : {}),
|
|
2849
2866
|
origin: opts?.origin ?? registered.origin,
|
|
2850
|
-
namespace: opts?.namespace ?? this.namespace,
|
|
2867
|
+
namespace: opts?.namespace ?? tenantNamespace ?? this.namespace,
|
|
2868
|
+
...(tenant
|
|
2869
|
+
? { tenantId: tenant.id, ...(tenant.kind ? { tenantKind: tenant.kind } : {}) }
|
|
2870
|
+
: {}),
|
|
2851
2871
|
input,
|
|
2852
2872
|
tags,
|
|
2853
2873
|
searchAttributes: opts?.searchAttributes,
|
|
@@ -2910,6 +2930,16 @@ export class WorkflowEngine {
|
|
|
2910
2930
|
* `limiter` bounds how many of these run at once (a fan's children wait in it as ids).
|
|
2911
2931
|
*/
|
|
2912
2932
|
handoffRun(workflow, input, runId, opts, onStartError, onAlreadySettled, limiter) {
|
|
2933
|
+
// The tenant is read NOW, inside the parent's turn: a bounded handoff may run later, elsewhere.
|
|
2934
|
+
if (opts?.tenantId === undefined) {
|
|
2935
|
+
const tenant = this.startTenant(opts);
|
|
2936
|
+
if (tenant)
|
|
2937
|
+
opts = {
|
|
2938
|
+
...opts,
|
|
2939
|
+
tenantId: tenant.id,
|
|
2940
|
+
...(tenant.kind ? { tenantKind: tenant.kind } : {}),
|
|
2941
|
+
};
|
|
2942
|
+
}
|
|
2913
2943
|
const handoff = async () => {
|
|
2914
2944
|
// Defer one microtask: never reentrantly resume the parent that is still suspending/settling.
|
|
2915
2945
|
await Promise.resolve();
|
|
@@ -2949,8 +2979,54 @@ export class WorkflowEngine {
|
|
|
2949
2979
|
this.trackEffect(limiter ? limiter.submit(runId, handoff) : handoff());
|
|
2950
2980
|
}
|
|
2951
2981
|
/** Read a run's current persisted state (or null if unknown). A thin pass-through to the store. */
|
|
2952
|
-
getRun(runId) {
|
|
2953
|
-
|
|
2982
|
+
async getRun(runId) {
|
|
2983
|
+
const run = await this.store.getRun(runId);
|
|
2984
|
+
return run && this.runVisibleNow(run) ? run : null;
|
|
2985
|
+
}
|
|
2986
|
+
/** The tenant a new run gets: the explicit option, else the active tenant (tenancy on), else none. */
|
|
2987
|
+
startTenant(opts) {
|
|
2988
|
+
if (opts?.tenantId === null)
|
|
2989
|
+
return undefined;
|
|
2990
|
+
if (typeof opts?.tenantId === 'string' && opts.tenantId !== '') {
|
|
2991
|
+
return opts.tenantKind ? { id: opts.tenantId, kind: opts.tenantKind } : { id: opts.tenantId };
|
|
2992
|
+
}
|
|
2993
|
+
const filter = this.tenantFilterNow();
|
|
2994
|
+
return filter.mode === 'tenant' ? filter.tenant : undefined;
|
|
2995
|
+
}
|
|
2996
|
+
/** How the current app-facing call is scoped (`system` inside the engine's own work). */
|
|
2997
|
+
tenantFilterNow() {
|
|
2998
|
+
if (!this.tenancy || engineInternal.getStore() === true)
|
|
2999
|
+
return { mode: 'system' };
|
|
3000
|
+
return this.tenancy.filter();
|
|
3001
|
+
}
|
|
3002
|
+
/** Whether the current call may see `run` (always, with tenancy off). */
|
|
3003
|
+
runVisibleNow(run) {
|
|
3004
|
+
if (!this.tenancy)
|
|
3005
|
+
return true;
|
|
3006
|
+
return runVisible(this.tenantFilterNow(), run, this.tenancy.nullRows);
|
|
3007
|
+
}
|
|
3008
|
+
/** Whether `runId` exists but the current call may not see it (an app-facing guard: answer not-found). */
|
|
3009
|
+
async runHidden(runId) {
|
|
3010
|
+
if (!this.tenancy)
|
|
3011
|
+
return false;
|
|
3012
|
+
const filter = this.tenantFilterNow();
|
|
3013
|
+
if (filter.mode === 'system' || filter.mode === 'all')
|
|
3014
|
+
return false;
|
|
3015
|
+
const run = await this.store.getRun(runId);
|
|
3016
|
+
return run !== null && !runVisible(filter, run, this.tenancy.nullRows);
|
|
3017
|
+
}
|
|
3018
|
+
/** `query` confined to the runs the current call may see. */
|
|
3019
|
+
tenantScoped(query) {
|
|
3020
|
+
if (!this.tenancy)
|
|
3021
|
+
return query;
|
|
3022
|
+
const ids = tenantScopeIds(this.tenantFilterNow(), this.tenancy.nullRows);
|
|
3023
|
+
if (ids === undefined)
|
|
3024
|
+
return query;
|
|
3025
|
+
return { ...query, tenantIds: intersectTenantIds(query.tenantIds, ids) };
|
|
3026
|
+
}
|
|
3027
|
+
/** Run `fn` as the engine's own work: app-facing tenant guards do not apply inside it. */
|
|
3028
|
+
asEngine(fn) {
|
|
3029
|
+
return this.tenancy ? engineInternal.run(true, fn) : fn();
|
|
2954
3030
|
}
|
|
2955
3031
|
/**
|
|
2956
3032
|
* List persisted runs matching `query` (newest activity first, per the store's own ordering). The
|
|
@@ -2958,7 +3034,7 @@ export class WorkflowEngine {
|
|
|
2958
3034
|
* through the engine rather than reaching for its private store.
|
|
2959
3035
|
*/
|
|
2960
3036
|
async listRuns(query) {
|
|
2961
|
-
const scoped = withoutBookkeepingRuns(query);
|
|
3037
|
+
const scoped = withoutBookkeepingRuns(this.tenantScoped(query));
|
|
2962
3038
|
const runs = await this.store.listRuns(scoped);
|
|
2963
3039
|
const hidden = scoped.excludeWorkflows;
|
|
2964
3040
|
return hidden?.length ? runs.filter((run) => !hidden.includes(run.workflow)) : runs;
|
|
@@ -2970,14 +3046,16 @@ export class WorkflowEngine {
|
|
|
2970
3046
|
* {@link listRuns}.
|
|
2971
3047
|
*/
|
|
2972
3048
|
runValueFacets(axis, query, opts) {
|
|
2973
|
-
const scoped = withoutBookkeepingRuns(query);
|
|
3049
|
+
const scoped = withoutBookkeepingRuns(this.tenantScoped(query));
|
|
2974
3050
|
if (this.store.runValueFacets) {
|
|
2975
3051
|
return this.store.runValueFacets(axis, scoped, opts);
|
|
2976
3052
|
}
|
|
2977
3053
|
return scanRunValueFacets(this.store, axis, scoped, opts);
|
|
2978
3054
|
}
|
|
2979
3055
|
/** List a run's step checkpoints (its timeline). Part of the engine's read API; see {@link listRuns}. */
|
|
2980
|
-
listCheckpoints(runId) {
|
|
3056
|
+
async listCheckpoints(runId) {
|
|
3057
|
+
if (await this.runHidden(runId))
|
|
3058
|
+
return [];
|
|
2981
3059
|
return this.store.listCheckpoints(runId);
|
|
2982
3060
|
}
|
|
2983
3061
|
/**
|
|
@@ -2985,7 +3063,7 @@ export class WorkflowEngine {
|
|
|
2985
3063
|
* has {@link StateStore.runFacets} (else counted over a listing). Part of the engine's read API.
|
|
2986
3064
|
*/
|
|
2987
3065
|
runFacets(query) {
|
|
2988
|
-
return runFacetsOf(this.store, withoutBookkeepingRuns(query));
|
|
3066
|
+
return runFacetsOf(this.store, withoutBookkeepingRuns(this.tenantScoped(query)));
|
|
2989
3067
|
}
|
|
2990
3068
|
/**
|
|
2991
3069
|
* A run's detail (see `loadRunDetail`), built from the store's checkpoint aggregates when it has
|
|
@@ -2993,6 +3071,8 @@ export class WorkflowEngine {
|
|
|
2993
3071
|
* instead of the whole history, and carries an `etag` that `options.ifNoneMatch` can match.
|
|
2994
3072
|
*/
|
|
2995
3073
|
async getRunDetail(runId, options) {
|
|
3074
|
+
if (await this.runHidden(runId))
|
|
3075
|
+
return null;
|
|
2996
3076
|
const source = this.detailSource();
|
|
2997
3077
|
const withExtras = async (detail) => {
|
|
2998
3078
|
const extras = await this.lifecycleDetail(detail.run);
|
|
@@ -3130,7 +3210,9 @@ export class WorkflowEngine {
|
|
|
3130
3210
|
return out;
|
|
3131
3211
|
}
|
|
3132
3212
|
/** One page of a summarized fan's members (see `loadFanMembers`), versioned like {@link getRunDetail}. */
|
|
3133
|
-
getFanMembers(runId, group, query) {
|
|
3213
|
+
async getFanMembers(runId, group, query) {
|
|
3214
|
+
if (await this.runHidden(runId))
|
|
3215
|
+
return null;
|
|
3134
3216
|
return loadFanMembers(this.detailSource(), runId, group, query);
|
|
3135
3217
|
}
|
|
3136
3218
|
detailSource() {
|
|
@@ -3468,7 +3550,7 @@ export class WorkflowEngine {
|
|
|
3468
3550
|
}
|
|
3469
3551
|
async deleteRun(runId) {
|
|
3470
3552
|
const run = await this.store.getRun(runId);
|
|
3471
|
-
if (!run)
|
|
3553
|
+
if (!run || !this.runVisibleNow(run))
|
|
3472
3554
|
return 0;
|
|
3473
3555
|
// Collect children BEFORE deleting this run's checkpoints (getRunChildren reads them).
|
|
3474
3556
|
let deleted = 0;
|
|
@@ -3553,7 +3635,7 @@ export class WorkflowEngine {
|
|
|
3553
3635
|
if (!TERMINAL_RUN_STATUSES.includes(status)) {
|
|
3554
3636
|
if (!cancelLive)
|
|
3555
3637
|
continue;
|
|
3556
|
-
await this.cancel(id).catch(() => undefined);
|
|
3638
|
+
await this.asEngine(() => this.cancel(id)).catch(() => undefined);
|
|
3557
3639
|
}
|
|
3558
3640
|
doomed.push(id);
|
|
3559
3641
|
}
|
|
@@ -3737,7 +3819,7 @@ export class WorkflowEngine {
|
|
|
3737
3819
|
}
|
|
3738
3820
|
if (!registered)
|
|
3739
3821
|
throw new WorkflowNotRegistered(run.workflow, run.workflowVersion);
|
|
3740
|
-
return this.track(runId, this.execute(run, registered.fn));
|
|
3822
|
+
return this.track(runId, this.inRunTenant(run, () => this.execute(run, registered.fn)));
|
|
3741
3823
|
}
|
|
3742
3824
|
/**
|
|
3743
3825
|
* Deferred re-drive after a suspended settle whose waited checkpoint(s) already settled — the
|
|
@@ -4580,6 +4662,8 @@ export class WorkflowEngine {
|
|
|
4580
4662
|
const seen = _seen ?? new Set();
|
|
4581
4663
|
if (seen.has(runId))
|
|
4582
4664
|
return null;
|
|
4665
|
+
if (_seen === undefined && (await this.runHidden(runId)))
|
|
4666
|
+
return null;
|
|
4583
4667
|
seen.add(runId);
|
|
4584
4668
|
const run = await this.store.getRun(runId);
|
|
4585
4669
|
if (!run)
|
|
@@ -4728,7 +4812,7 @@ export class WorkflowEngine {
|
|
|
4728
4812
|
*/
|
|
4729
4813
|
async redispatchPending(runId) {
|
|
4730
4814
|
const run = await this.store.getRun(runId);
|
|
4731
|
-
if (!run)
|
|
4815
|
+
if (!run || !this.runVisibleNow(run))
|
|
4732
4816
|
return null;
|
|
4733
4817
|
const pending = (await this.store.listCheckpoints(runId)).filter((cp) => cp.kind === 'remote' && cp.status === 'pending');
|
|
4734
4818
|
let redispatched = 0;
|
|
@@ -4764,12 +4848,18 @@ export class WorkflowEngine {
|
|
|
4764
4848
|
*/
|
|
4765
4849
|
async retryWithInput(runId, input, newRunId) {
|
|
4766
4850
|
const run = await this.store.getRun(runId);
|
|
4767
|
-
if (!run)
|
|
4851
|
+
if (!run || !this.runVisibleNow(run))
|
|
4768
4852
|
return null;
|
|
4769
4853
|
const id = newRunId ?? `${runId}~retry~${globalThis.crypto.randomUUID().slice(0, 8)}`;
|
|
4770
4854
|
// Inherit the original run's namespace so a retry lands in the SAME worker-pool partition (an
|
|
4771
|
-
// operator-routed run keeps its
|
|
4772
|
-
|
|
4855
|
+
// operator-routed run keeps its pool on retry, rather than falling back to the engine's own),
|
|
4856
|
+
// and its tenant (or none).
|
|
4857
|
+
await this.start(run.workflow, input, id, {
|
|
4858
|
+
tags: run.tags,
|
|
4859
|
+
namespace: run.namespace,
|
|
4860
|
+
tenantId: run.tenantId ?? null,
|
|
4861
|
+
...(run.tenantKind ? { tenantKind: run.tenantKind } : {}),
|
|
4862
|
+
});
|
|
4773
4863
|
return { runId: id };
|
|
4774
4864
|
}
|
|
4775
4865
|
/**
|
|
@@ -5036,7 +5126,10 @@ export class WorkflowEngine {
|
|
|
5036
5126
|
const waiters = await this.store.listSignalWaiters(eventPrefix(name));
|
|
5037
5127
|
for (const w of waiters) {
|
|
5038
5128
|
if (eventMatches(payload, eventMatchOf(w.token))) {
|
|
5039
|
-
|
|
5129
|
+
// With tenancy on, a tenant's event reaches its own runs only.
|
|
5130
|
+
if (await this.runHidden(w.runId))
|
|
5131
|
+
continue;
|
|
5132
|
+
await this.asEngine(() => this.signal(w.token, payload));
|
|
5040
5133
|
touched += 1;
|
|
5041
5134
|
}
|
|
5042
5135
|
}
|
|
@@ -5054,6 +5147,13 @@ export class WorkflowEngine {
|
|
|
5054
5147
|
// unless the caller opted out. `touched > 0` covers BOTH the waiter fan-out and the subscriber loop.
|
|
5055
5148
|
if (touched > 0 || opts?.buffer === false)
|
|
5056
5149
|
return touched;
|
|
5150
|
+
// A buffered copy carries no tenant and would reach whichever run waits next, so a publisher
|
|
5151
|
+
// scoped to one tenant (or to none) does not buffer: its event reaches live waiters only.
|
|
5152
|
+
if (this.tenancy) {
|
|
5153
|
+
const mode = this.tenantFilterNow().mode;
|
|
5154
|
+
if (mode === 'tenant' || mode === 'tenantless')
|
|
5155
|
+
return touched;
|
|
5156
|
+
}
|
|
5057
5157
|
const bufferedId = globalThis.crypto.randomUUID();
|
|
5058
5158
|
await this.store.bufferEvent({ name, payload, id: bufferedId, publishedAt: this.clock() });
|
|
5059
5159
|
// Re-check: a waiter may have registered in the window between the miss above and this buffer
|
|
@@ -5164,6 +5264,8 @@ export class WorkflowEngine {
|
|
|
5164
5264
|
* side's write, and the destructive `take*` ops arbitrate so a signal is never delivered twice.
|
|
5165
5265
|
*/
|
|
5166
5266
|
async signal(token, payload) {
|
|
5267
|
+
if (await this.signalHidden(token))
|
|
5268
|
+
return null;
|
|
5167
5269
|
const waiter = await this.store.takeSignalWaiter(token);
|
|
5168
5270
|
if (waiter)
|
|
5169
5271
|
return this.deliverSignal(waiter, payload);
|
|
@@ -5185,6 +5287,23 @@ export class WorkflowEngine {
|
|
|
5185
5287
|
}
|
|
5186
5288
|
return this.deliverSignal(lateWaiter, reclaimed.payload);
|
|
5187
5289
|
}
|
|
5290
|
+
/** Whether a run another tenant owns is the one waiting on `token` (an app-facing signal is dropped). */
|
|
5291
|
+
async signalHidden(token) {
|
|
5292
|
+
if (!this.tenancy)
|
|
5293
|
+
return false;
|
|
5294
|
+
const filter = this.tenantFilterNow();
|
|
5295
|
+
if (filter.mode === 'system' || filter.mode === 'all')
|
|
5296
|
+
return false;
|
|
5297
|
+
const waiters = await this.store.listSignalWaiters(token);
|
|
5298
|
+
for (const waiter of waiters) {
|
|
5299
|
+
if (waiter.token !== token)
|
|
5300
|
+
continue;
|
|
5301
|
+
const run = await this.store.getRun(waiter.runId);
|
|
5302
|
+
if (run && !runVisible(filter, run, this.tenancy.nullRows))
|
|
5303
|
+
return true;
|
|
5304
|
+
}
|
|
5305
|
+
return false;
|
|
5306
|
+
}
|
|
5188
5307
|
/**
|
|
5189
5308
|
* Wake the run parked on `token` with `payload`, if there is one; otherwise do nothing — unlike
|
|
5190
5309
|
* {@link signal}, it never buffers. For a wake-up whose real payload is ALREADY buffered on the
|
|
@@ -5216,14 +5335,18 @@ export class WorkflowEngine {
|
|
|
5216
5335
|
* null if no run is waiting on the task (e.g. a duplicate/late delivery) — a safe no-op.
|
|
5217
5336
|
*/
|
|
5218
5337
|
async completeTask(runId, name, result) {
|
|
5219
|
-
|
|
5338
|
+
if (await this.runHidden(runId))
|
|
5339
|
+
return null;
|
|
5340
|
+
return this.asEngine(() => this.signal(`task:${runId}:${name}`, {
|
|
5220
5341
|
ok: true,
|
|
5221
5342
|
value: result,
|
|
5222
|
-
});
|
|
5343
|
+
}));
|
|
5223
5344
|
}
|
|
5224
5345
|
/** Report that a `ctx.task` failed — the run resumes and throws a FatalError at the task. */
|
|
5225
5346
|
async failTask(runId, name, error) {
|
|
5226
|
-
|
|
5347
|
+
if (await this.runHidden(runId))
|
|
5348
|
+
return null;
|
|
5349
|
+
return this.asEngine(() => this.signal(`task:${runId}:${name}`, { ok: false, error }));
|
|
5227
5350
|
}
|
|
5228
5351
|
/**
|
|
5229
5352
|
* Notify a parent that's waiting on `runId` as a child of its terminal outcome (the `ctx.child`
|
|
@@ -5276,7 +5399,7 @@ export class WorkflowEngine {
|
|
|
5276
5399
|
if (!(await childCompletionNeedsBuffer(this.store, token.slice(CHILD_TOKEN_PREFIX.length), completion))) {
|
|
5277
5400
|
return;
|
|
5278
5401
|
}
|
|
5279
|
-
const resumed = await this.signal(token, completion);
|
|
5402
|
+
const resumed = await this.asEngine(() => this.signal(token, completion));
|
|
5280
5403
|
// The parent may settle the child from its run row between the look above and the buffer
|
|
5281
5404
|
// write (a join that registers its waiter only now): it then stamped the child first, and the
|
|
5282
5405
|
// copy just buffered would never be read.
|
|
@@ -5341,7 +5464,7 @@ export class WorkflowEngine {
|
|
|
5341
5464
|
: settled.status === 'completed'
|
|
5342
5465
|
? { ok: true, value: settled.output }
|
|
5343
5466
|
: failedCompletion(settled.error ?? { message: settled.status });
|
|
5344
|
-
await this.signal(token, completion);
|
|
5467
|
+
await this.asEngine(() => this.signal(token, completion));
|
|
5345
5468
|
}
|
|
5346
5469
|
/**
|
|
5347
5470
|
* Cancel a run (e.g. from the dashboard). Returns null if the run does not exist. Pass
|
|
@@ -5350,7 +5473,9 @@ export class WorkflowEngine {
|
|
|
5350
5473
|
* cancelled. Without it, cancellation is immediate (no undo).
|
|
5351
5474
|
*/
|
|
5352
5475
|
async cancel(runId, opts) {
|
|
5353
|
-
|
|
5476
|
+
if (await this.runHidden(runId))
|
|
5477
|
+
return null;
|
|
5478
|
+
return this.asEngine(() => this.cancelOnce(runId, opts, true));
|
|
5354
5479
|
}
|
|
5355
5480
|
async cancelOnce(runId, opts, mayRetry) {
|
|
5356
5481
|
const run = await this.store.getRun(runId);
|
|
@@ -5471,7 +5596,7 @@ export class WorkflowEngine {
|
|
|
5471
5596
|
*/
|
|
5472
5597
|
async cancelChildren(parentRunId, opts) {
|
|
5473
5598
|
for (const id of await this.getRunChildren(parentRunId)) {
|
|
5474
|
-
await this.cancel(id, opts).catch(() => undefined);
|
|
5599
|
+
await this.asEngine(() => this.cancel(id, opts)).catch(() => undefined);
|
|
5475
5600
|
}
|
|
5476
5601
|
}
|
|
5477
5602
|
/**
|
|
@@ -5527,7 +5652,7 @@ export class WorkflowEngine {
|
|
|
5527
5652
|
* the full listing for a store without one.
|
|
5528
5653
|
*/
|
|
5529
5654
|
countRuns(query) {
|
|
5530
|
-
return countRuns(this.store, withoutBookkeepingRuns(query));
|
|
5655
|
+
return countRuns(this.store, withoutBookkeepingRuns(this.tenantScoped(query)));
|
|
5531
5656
|
}
|
|
5532
5657
|
/**
|
|
5533
5658
|
* Every workflow the live fleet can execute in this engine's partition: what workers DECLARE in
|
|
@@ -5909,6 +6034,18 @@ export class WorkflowEngine {
|
|
|
5909
6034
|
throw err;
|
|
5910
6035
|
});
|
|
5911
6036
|
}
|
|
6037
|
+
/**
|
|
6038
|
+
* Run a turn with the run's own tenant active, so a resume driven by a timer, a signal or recovery
|
|
6039
|
+
* (no request around it) sees the tenant the run was started under, and what the turn starts or
|
|
6040
|
+
* dispatches carries it. A run with no tenant runs as before.
|
|
6041
|
+
*/
|
|
6042
|
+
inRunTenant(run, fn) {
|
|
6043
|
+
const tenant = runTenantOf(run);
|
|
6044
|
+
if (!this.tenancy || !tenant)
|
|
6045
|
+
return fn();
|
|
6046
|
+
const tenancy = this.tenancy;
|
|
6047
|
+
return engineInternal.run(false, () => tenancy.run(tenant, fn));
|
|
6048
|
+
}
|
|
5912
6049
|
async execute(run, fn) {
|
|
5913
6050
|
const registered = this.workflows.get(versionKey(run.workflow, run.workflowVersion));
|
|
5914
6051
|
// Hold the lease for the WHOLE execution — whatever path got us here (leased sweep, a signal, a
|
|
@@ -7586,7 +7723,11 @@ export class WorkflowEngine {
|
|
|
7586
7723
|
// converges on the already-persisted continuation (idempotent by the continuation id).
|
|
7587
7724
|
// `dispatch: false`: the tracked handoff below owns the pickup. The continuation inherits
|
|
7588
7725
|
// the parent RUN's namespace so a tenant's chain stays in its worker-pool partition.
|
|
7589
|
-
await this.startCore(run.workflow, err.input, nextId, {
|
|
7726
|
+
await this.startCore(run.workflow, err.input, nextId, {
|
|
7727
|
+
namespace: run.namespace,
|
|
7728
|
+
tenantId: run.tenantId ?? null,
|
|
7729
|
+
...(run.tenantKind ? { tenantKind: run.tenantKind } : {}),
|
|
7730
|
+
}, false);
|
|
7590
7731
|
// Conditional, like every settle: a cancel persisted after the re-read above (`cancelling`, or a
|
|
7591
7732
|
// plain `cancelled`) must win over this completion. The continuation already exists then, and
|
|
7592
7733
|
// nothing has handed it off: cancel it, and unwind this run if the cancel compensates.
|
|
@@ -7597,7 +7738,7 @@ export class WorkflowEngine {
|
|
|
7597
7738
|
updatedAt: new Date(),
|
|
7598
7739
|
});
|
|
7599
7740
|
if (!completed) {
|
|
7600
|
-
await this.cancel(nextId);
|
|
7741
|
+
await this.asEngine(() => this.cancel(nextId));
|
|
7601
7742
|
const current = await this.store.getRun(run.id);
|
|
7602
7743
|
if (current?.status === 'cancelling' && !this.leaseLostRuns.has(run.id)) {
|
|
7603
7744
|
return await this.compensateAndCancel(run, compensations);
|
|
@@ -7615,7 +7756,11 @@ export class WorkflowEngine {
|
|
|
7615
7756
|
// continuation has entered `inflight` and settled, so `drain()` can't return before the next
|
|
7616
7757
|
// run's store writes land. Still deferred; its startCore call converges on the run persisted
|
|
7617
7758
|
// above (prior-run branch) and drives the pickup.
|
|
7618
|
-
this.handoffRun(run.workflow, err.input, nextId, {
|
|
7759
|
+
this.handoffRun(run.workflow, err.input, nextId, {
|
|
7760
|
+
namespace: run.namespace,
|
|
7761
|
+
tenantId: run.tenantId ?? null,
|
|
7762
|
+
...(run.tenantKind ? { tenantKind: run.tenantKind } : {}),
|
|
7763
|
+
});
|
|
7619
7764
|
return { runId: run.id, status: 'completed' };
|
|
7620
7765
|
}
|
|
7621
7766
|
if (err instanceof WorkflowBlocked) {
|
|
@@ -7988,7 +8133,7 @@ export class WorkflowEngine {
|
|
|
7988
8133
|
// escape `drain()` onto a torn-down connection ("Transaction query already complete"). The
|
|
7989
8134
|
// one-microtask defer is preserved; the promise is registered synchronously.
|
|
7990
8135
|
cancelChild: (childId) => {
|
|
7991
|
-
this.trackEffect(Promise.resolve().then(() => this.cancel(childId)));
|
|
8136
|
+
this.trackEffect(Promise.resolve().then(() => this.asEngine(() => this.cancel(childId))));
|
|
7992
8137
|
},
|
|
7993
8138
|
// Shallow-merge into the run's searchAttributes (the ctx primitive makes this exactly-once).
|
|
7994
8139
|
upsertSearchAttributes: async (runId, attrs) => {
|
|
@@ -8036,11 +8181,13 @@ export class WorkflowEngine {
|
|
|
8036
8181
|
* paused at a breakpoint.
|
|
8037
8182
|
*/
|
|
8038
8183
|
async continue(runId) {
|
|
8184
|
+
if (await this.runHidden(runId))
|
|
8185
|
+
return null;
|
|
8039
8186
|
const checkpoints = await this.store.listCheckpoints(runId);
|
|
8040
8187
|
const bp = checkpoints.find(isBreakpoint);
|
|
8041
8188
|
if (!bp)
|
|
8042
8189
|
return null;
|
|
8043
|
-
return this.signal(breakpointToken(runId, bp.seq), undefined);
|
|
8190
|
+
return this.asEngine(() => this.signal(breakpointToken(runId, bp.seq), undefined));
|
|
8044
8191
|
}
|
|
8045
8192
|
/**
|
|
8046
8193
|
* Read the latest value a run published for `key` via {@link WorkflowCtx.setEvent} — a
|
|
@@ -8048,6 +8195,8 @@ export class WorkflowEngine {
|
|
|
8048
8195
|
* never published that key. The suspend-model counterpart of a Temporal query.
|
|
8049
8196
|
*/
|
|
8050
8197
|
async getEvent(runId, key) {
|
|
8198
|
+
if (await this.runHidden(runId))
|
|
8199
|
+
return undefined;
|
|
8051
8200
|
const name = `event:${key}`;
|
|
8052
8201
|
// Targeted read: the highest-seq checkpoint for this name is the most recent value (a re-published
|
|
8053
8202
|
// key overwrites at a higher seq), matching the old "last in seq order wins" scan. Falls back to a
|
|
@@ -8079,7 +8228,7 @@ export class WorkflowEngine {
|
|
|
8079
8228
|
*/
|
|
8080
8229
|
async update(runId, name, arg) {
|
|
8081
8230
|
const run = await this.store.getRun(runId);
|
|
8082
|
-
if (!run)
|
|
8231
|
+
if (!run || !this.runVisibleNow(run))
|
|
8083
8232
|
return { accepted: false, reason: `run ${runId} not found` };
|
|
8084
8233
|
const validate = this.updateValidators.get(`${run.workflow}:${name}`);
|
|
8085
8234
|
if (validate) {
|
|
@@ -8092,7 +8241,7 @@ export class WorkflowEngine {
|
|
|
8092
8241
|
return { accepted: false, reason: err instanceof Error ? err.message : String(err) };
|
|
8093
8242
|
}
|
|
8094
8243
|
}
|
|
8095
|
-
const result = await this.signal(`update:${runId}:${name}`, arg);
|
|
8244
|
+
const result = await this.asEngine(() => this.signal(`update:${runId}:${name}`, arg));
|
|
8096
8245
|
return { accepted: true, run: result };
|
|
8097
8246
|
}
|
|
8098
8247
|
// ---------------------------------------------------------------------------
|
|
@@ -9816,6 +9965,8 @@ function turnBeatKey(runId) {
|
|
|
9816
9965
|
/** The run whose remote turn is being enqueued in the current async context — lets the decision
|
|
9817
9966
|
* handler spot a transport delivering the decision from inside its own `dispatchWorkflowTask`. */
|
|
9818
9967
|
const turnDispatchAls = new AsyncLocalStorage();
|
|
9968
|
+
/** `true` while the engine does its own work: app-facing tenant guards do not apply inside it. */
|
|
9969
|
+
const engineInternal = new AsyncLocalStorage();
|
|
9819
9970
|
/** Tolerance (ms) between the worker's clock, which stamps a persisted heartbeat, and this engine's,
|
|
9820
9971
|
* when deciding whether a beat belongs to the current attempt (see `awaitWithHeartbeat`). */
|
|
9821
9972
|
const HEARTBEAT_CLOCK_SKEW_MS = 1_000;
|