@atolis-hq/wake 0.4.42 → 0.4.44
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/bootstrap/config/root-schema.d.ts +2 -0
- package/dist/src/bootstrap/self-update-application.d.ts +2 -0
- package/dist/src/bootstrap/self-update-application.js +53 -6
- package/dist/src/bootstrap/surface-cli-applications.js +16 -14
- package/dist/src/bootstrap/version.js +1 -1
- package/dist/src/control-plane/application/activation-scheduler-ports.d.ts +1 -0
- package/dist/src/control-plane/application/activation-scheduler.js +17 -2
- package/dist/src/orchestration/application/accept-signal.js +9 -1
- package/dist/src/orchestration/application/advance-workflow.d.ts +1 -0
- package/dist/src/orchestration/application/advance-workflow.js +4 -0
- package/dist/src/orchestration/application/expire-timed-out-waits.d.ts +3 -0
- package/dist/src/orchestration/application/expire-timed-out-waits.js +17 -0
- package/dist/src/orchestration/application/orchestration-service.d.ts +1 -0
- package/dist/src/orchestration/application/orchestration-service.js +3 -0
- package/dist/src/orchestration/contracts/config.d.ts +6 -0
- package/dist/src/orchestration/contracts/config.js +1 -0
- package/dist/src/orchestration/contracts/event-decoder.d.ts +2 -0
- package/dist/src/orchestration/contracts/event-decoder.js +2 -0
- package/dist/src/orchestration/contracts/events.d.ts +4 -0
- package/dist/src/orchestration/contracts/views.d.ts +2 -0
- package/dist/src/orchestration/domain/compiler.js +1 -0
- package/dist/src/orchestration/domain/interpreter.d.ts +1 -1
- package/dist/src/orchestration/domain/interpreter.js +1 -1
- package/dist/src/orchestration/domain/operator-retry-policy.js +4 -2
- package/dist/src/orchestration/domain/signal-policy.d.ts +4 -0
- package/dist/src/orchestration/domain/signal-policy.js +26 -0
- package/dist/src/orchestration/domain/transition.js +3 -0
- package/dist/src/orchestration/domain/workflow-instance-events.js +4 -0
- package/package.json +1 -1
|
@@ -69,6 +69,7 @@ declare const orchestrationConfigSchema: z.ZodDefault<z.ZodObject<{
|
|
|
69
69
|
kind: z.ZodLiteral<"watch">;
|
|
70
70
|
id: z.ZodString;
|
|
71
71
|
}, z.core.$strict>]>>>;
|
|
72
|
+
timeoutMs: z.ZodOptional<z.ZodNumber>;
|
|
72
73
|
}, z.core.$strict>>;
|
|
73
74
|
watchGates: z.ZodOptional<z.ZodArray<z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
|
|
74
75
|
watch: z.ZodString;
|
|
@@ -215,6 +216,7 @@ export declare const rootConfigSchema: z.ZodObject<{
|
|
|
215
216
|
kind: z.ZodLiteral<"watch">;
|
|
216
217
|
id: z.ZodString;
|
|
217
218
|
}, z.core.$strict>]>>>;
|
|
219
|
+
timeoutMs: z.ZodOptional<z.ZodNumber>;
|
|
218
220
|
}, z.core.$strict>>;
|
|
219
221
|
watchGates: z.ZodOptional<z.ZodArray<z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
|
|
220
222
|
watch: z.ZodString;
|
|
@@ -16,6 +16,7 @@ export interface SourceUpdatePort {
|
|
|
16
16
|
* non-sandbox self-updates.
|
|
17
17
|
*/
|
|
18
18
|
export interface SelfUpdateRolloutPort {
|
|
19
|
+
prepare(tag: string): Promise<void>;
|
|
19
20
|
deploy(tag: string): Promise<void>;
|
|
20
21
|
rollback(tag: string): Promise<void>;
|
|
21
22
|
recordFailure(tag: string, error: unknown): Promise<void>;
|
|
@@ -27,6 +28,7 @@ export interface SelfUpdateRolloutPort {
|
|
|
27
28
|
* durably record every supplied Run's maintenance request.
|
|
28
29
|
*/
|
|
29
30
|
export interface SelfUpdateQuiescePort {
|
|
31
|
+
read?(): Promise<UpdateMaintenanceState | null>;
|
|
30
32
|
acquire(tag: string, retryFailed?: boolean): Promise<UpdateMaintenanceState>;
|
|
31
33
|
exclusive?<Value>(tag: string, retryFailed: boolean, operation: (state: UpdateMaintenanceState) => Promise<Value>): Promise<Value>;
|
|
32
34
|
activeRuns(): Promise<readonly {
|
|
@@ -59,6 +59,11 @@ async function currentTag(input) {
|
|
|
59
59
|
return (await input.source.currentTag?.()) ?? (await input.ledger.read());
|
|
60
60
|
}
|
|
61
61
|
async function startUpdateAttempt(input, tag, force) {
|
|
62
|
+
const existing = await input.quiesce?.read?.();
|
|
63
|
+
if (existing === null ||
|
|
64
|
+
existing === undefined ||
|
|
65
|
+
existing.phase === UpdateMaintenancePhase.Failed)
|
|
66
|
+
await prepareCandidate(input, tag);
|
|
62
67
|
if (input.quiesce?.exclusive !== undefined)
|
|
63
68
|
return input.quiesce.exclusive(tag, force, (lease) => updateAttempt(input, tag, force, lease));
|
|
64
69
|
return updateAttempt(input, tag, force, await input.quiesce?.acquire(tag, force));
|
|
@@ -66,24 +71,30 @@ async function startUpdateAttempt(input, tag, force) {
|
|
|
66
71
|
async function updateAttempt(input, tag, force, lease) {
|
|
67
72
|
const maintenanceTag = lease?.tag ?? tag;
|
|
68
73
|
const attemptId = lease?.attemptId;
|
|
69
|
-
const context = {
|
|
74
|
+
const context = {
|
|
75
|
+
activationStarted: false,
|
|
76
|
+
rollbackSucceeded: false,
|
|
77
|
+
recoveryRequired: false,
|
|
78
|
+
recoveryVerified: false,
|
|
79
|
+
};
|
|
70
80
|
try {
|
|
71
81
|
return await performUpdateAttempt(input, tag, force, lease, context);
|
|
72
82
|
}
|
|
73
83
|
catch (error) {
|
|
74
|
-
await recordUpdateFailure(input, maintenanceTag, attemptId, context.deployError ?? error, error);
|
|
84
|
+
await recordUpdateFailure(input, maintenanceTag, attemptId, context.deployError ?? error, error, context);
|
|
75
85
|
throw error;
|
|
76
86
|
}
|
|
77
87
|
}
|
|
78
88
|
async function performUpdateAttempt(input, tag, force, lease, context) {
|
|
79
89
|
if (lease !== undefined && requiresMaintenanceRecovery(lease)) {
|
|
90
|
+
context.recoveryRequired = true;
|
|
80
91
|
await resumePendingMaintenance(input, lease);
|
|
81
92
|
return false;
|
|
82
93
|
}
|
|
83
94
|
if (!isStartableMaintenanceLease(lease, tag))
|
|
84
95
|
return false;
|
|
85
96
|
await quiesceForUpdate(input);
|
|
86
|
-
await recoverPendingUpdate(input.ledger, input.source);
|
|
97
|
+
await recoverPendingUpdate(input.ledger, input.source, context);
|
|
87
98
|
if (!force && (await input.ledger.read()) === tag) {
|
|
88
99
|
await input.quiesce?.clear(lease?.attemptId);
|
|
89
100
|
return false;
|
|
@@ -116,13 +127,18 @@ async function runForwardUpdate(input, tag, attemptId, context) {
|
|
|
116
127
|
readLedger: input.ledger.read,
|
|
117
128
|
writeLedger: input.ledger.write,
|
|
118
129
|
update: async (nextTag) => {
|
|
119
|
-
|
|
130
|
+
if (input.rollout === undefined)
|
|
131
|
+
await input.source.checkout(nextTag);
|
|
132
|
+
context.activationStarted = true;
|
|
120
133
|
context.deployError = await deployRollout(input.rollout, nextTag);
|
|
121
134
|
},
|
|
122
135
|
health: async () => context.deployError === undefined && input.source.healthy(),
|
|
123
136
|
rollback: async (priorTag) => {
|
|
124
137
|
await rollbackUpdate(input, priorTag, attemptId);
|
|
138
|
+
if (!(await input.source.healthy()))
|
|
139
|
+
throw new Error(`Self-update rollback could not verify the prior tag ${priorTag}`);
|
|
125
140
|
rollbackSucceeded = true;
|
|
141
|
+
context.rollbackSucceeded = true;
|
|
126
142
|
},
|
|
127
143
|
});
|
|
128
144
|
}
|
|
@@ -162,7 +178,18 @@ async function rollbackUpdate(input, priorTag, attemptId) {
|
|
|
162
178
|
await input.source.checkout(priorTag);
|
|
163
179
|
await input.rollout?.rollback(priorTag);
|
|
164
180
|
}
|
|
165
|
-
async function recordUpdateFailure(input, tag, attemptId, rolloutError, originalError) {
|
|
181
|
+
async function recordUpdateFailure(input, tag, attemptId, rolloutError, originalError, context) {
|
|
182
|
+
if (context.rollbackSucceeded ||
|
|
183
|
+
(!context.activationStarted && (!context.recoveryRequired || context.recoveryVerified))) {
|
|
184
|
+
try {
|
|
185
|
+
await recordRolloutFailure(input.rollout, tag, rolloutError);
|
|
186
|
+
await input.ledger.recordBad(tag);
|
|
187
|
+
}
|
|
188
|
+
finally {
|
|
189
|
+
await input.quiesce?.clear(attemptId);
|
|
190
|
+
}
|
|
191
|
+
return;
|
|
192
|
+
}
|
|
166
193
|
try {
|
|
167
194
|
await input.quiesce?.fail(originalError, attemptId);
|
|
168
195
|
}
|
|
@@ -184,6 +211,24 @@ async function recordRolloutFailure(rollout, tag, error) {
|
|
|
184
211
|
// Best-effort: a failure-log write must never mask the original update failure.
|
|
185
212
|
}
|
|
186
213
|
}
|
|
214
|
+
async function prepareCandidate(input, tag) {
|
|
215
|
+
if (!(await input.source.isClean()))
|
|
216
|
+
throw new Error('Self-update requires a clean source checkout');
|
|
217
|
+
if (input.rollout === undefined)
|
|
218
|
+
return;
|
|
219
|
+
const priorTag = await input.ledger.read();
|
|
220
|
+
try {
|
|
221
|
+
await input.source.checkout(tag);
|
|
222
|
+
await input.rollout.prepare(tag);
|
|
223
|
+
}
|
|
224
|
+
catch (error) {
|
|
225
|
+
if (priorTag !== null)
|
|
226
|
+
await input.source.checkout(priorTag);
|
|
227
|
+
await recordRolloutFailure(input.rollout, tag, error);
|
|
228
|
+
await input.ledger.recordBad(tag);
|
|
229
|
+
throw new UpdateCandidateFailure(error);
|
|
230
|
+
}
|
|
231
|
+
}
|
|
187
232
|
/**
|
|
188
233
|
* A restart must never repeat a forward deploy. The persisted phase tells us
|
|
189
234
|
* that source/rollout work may already have begun, so recover the prior
|
|
@@ -210,11 +255,13 @@ async function quiesceActiveRuns(quiesce) {
|
|
|
210
255
|
function formatError(error) {
|
|
211
256
|
return error instanceof Error ? error.message : String(error);
|
|
212
257
|
}
|
|
213
|
-
async function recoverPendingUpdate(ledger, source) {
|
|
258
|
+
async function recoverPendingUpdate(ledger, source, context) {
|
|
214
259
|
const priorTag = await ledger.recover();
|
|
215
260
|
if (priorTag === null)
|
|
216
261
|
return;
|
|
262
|
+
context.recoveryRequired = true;
|
|
217
263
|
await source.checkout(priorTag);
|
|
218
264
|
if (!(await source.healthy()))
|
|
219
265
|
throw new Error(`Self-update recovery could not verify the prior tag ${priorTag}`);
|
|
266
|
+
context.recoveryVerified = true;
|
|
220
267
|
}
|
|
@@ -400,6 +400,7 @@ async function activeExecutionRunIds(root) {
|
|
|
400
400
|
/** The production maintenance boundary shared by the CLI update application. */
|
|
401
401
|
export function createSelfUpdateQuiescePort(root) {
|
|
402
402
|
return {
|
|
403
|
+
read: () => root.maintenance.read(),
|
|
403
404
|
acquire: (tag, retryFailed) => root.maintenance.acquire(tag, retryFailed),
|
|
404
405
|
exclusive: (tag, retryFailed, operation) => root.maintenance.runExclusive(tag, retryFailed, operation),
|
|
405
406
|
activeRuns: () => activeExecutionRuns(root),
|
|
@@ -433,18 +434,20 @@ function createSandboxRuntimeApplications(root) {
|
|
|
433
434
|
};
|
|
434
435
|
}
|
|
435
436
|
/**
|
|
436
|
-
* Builds a
|
|
437
|
-
*
|
|
438
|
-
* resident `wake start` process actually came back up. Reused for both the
|
|
439
|
-
* forward deploy and a same-shaped rollback to a previously-built tag.
|
|
437
|
+
* Builds a candidate image before the maintenance window. A failed build
|
|
438
|
+
* cannot affect the running sandbox.
|
|
440
439
|
*/
|
|
441
|
-
async function
|
|
440
|
+
async function buildSandboxTag(root, docker, image, overrides) {
|
|
442
441
|
const port = createSandboxDockerPort(docker, sandboxDockerOptions(root, { image, ...overrides }));
|
|
443
|
-
|
|
442
|
+
await port.build();
|
|
443
|
+
}
|
|
444
|
+
/** Swaps the already-built candidate image and verifies the resident host. */
|
|
445
|
+
async function deploySandboxTag(root, docker, tag, image, development) {
|
|
446
|
+
const port = createSandboxDockerPort(docker, sandboxDockerOptions(root, { image, development }));
|
|
447
|
+
const wakeInvocation = development?.mode === 'source'
|
|
444
448
|
? ['node', '/app/dist/src/main.js']
|
|
445
449
|
: sandboxWakeInvocation(root);
|
|
446
450
|
const healthcheckRoot = `/tmp/wake-self-update-healthcheck-${tag}`;
|
|
447
|
-
await port.build();
|
|
448
451
|
try {
|
|
449
452
|
await port.update();
|
|
450
453
|
}
|
|
@@ -541,15 +544,14 @@ function createOperationalApplications(root) {
|
|
|
541
544
|
ledger: createUpdateLedger(root.paths.wakeRoot),
|
|
542
545
|
source: updatePort,
|
|
543
546
|
rollout: {
|
|
544
|
-
async
|
|
545
|
-
await
|
|
546
|
-
activeRunIds: () => activeExecutionRunIds(root),
|
|
547
|
-
sleep: async (milliseconds) => new Promise((resolve) => setTimeout(resolve, milliseconds)),
|
|
548
|
-
});
|
|
549
|
-
await deploySandboxTag(root, dockerCli, deployTag, `${imageRepository}:${deployTag}`, {
|
|
547
|
+
async prepare(prepareTag) {
|
|
548
|
+
await buildSandboxTag(root, dockerCli, `${imageRepository}:${prepareTag}`, {
|
|
550
549
|
development: updateDevelopment,
|
|
551
|
-
buildVersion:
|
|
550
|
+
buildVersion: prepareTag,
|
|
552
551
|
});
|
|
552
|
+
},
|
|
553
|
+
async deploy(deployTag) {
|
|
554
|
+
await deploySandboxTag(root, dockerCli, deployTag, `${imageRepository}:${deployTag}`, updateDevelopment);
|
|
553
555
|
// A later deploy succeeding after a prior rollback clears the stale warning.
|
|
554
556
|
await failureLog.clear();
|
|
555
557
|
},
|
|
@@ -124,6 +124,6 @@ export function resolveWakeVersion(options = {}) {
|
|
|
124
124
|
return `g${headHash.slice(0, 7)}`;
|
|
125
125
|
return '0.1.0-dev';
|
|
126
126
|
}
|
|
127
|
-
export const wakeVersion = "v0.4.
|
|
127
|
+
export const wakeVersion = "v0.4.44";
|
|
128
128
|
/** The registry-installable version used by packaged sandbox images. */
|
|
129
129
|
export const wakePackageVersion = packageVersion(defaultRepoRoot) ?? wakeVersion;
|
|
@@ -12,6 +12,7 @@ export interface OrchestrationPort {
|
|
|
12
12
|
activation: ActivityActivationView;
|
|
13
13
|
}[]>;
|
|
14
14
|
listWaiting(): Promise<readonly (WorkflowInstanceView | null)[]>;
|
|
15
|
+
expireTimedOutWaits?(context: CommandContext): Promise<readonly WorkflowInstanceView[]>;
|
|
15
16
|
listAll?(): Promise<readonly WorkflowInstanceView[]>;
|
|
16
17
|
validateActivationDispatch?(workflowInstanceId: string, context: CommandContext): Promise<boolean>;
|
|
17
18
|
acceptOutcome(command: {
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
/* eslint-disable complexity, max-lines-per-function */
|
|
2
2
|
import { correlationId, EventActorKind } from '@atolis-hq/eventing';
|
|
3
3
|
import { RunStatus } from '../../execution/index.js';
|
|
4
|
-
import { isAmbiguityResolutionBlock, WorkflowStatus } from '../../orchestration/index.js';
|
|
4
|
+
import { isAmbiguityResolutionBlock, WorkflowStatus, } from '../../orchestration/index.js';
|
|
5
5
|
import { WorkStatus } from '../../work/index.js';
|
|
6
6
|
import { ControlStreamKind } from '../contracts/streams.js';
|
|
7
7
|
import { DispatchPolicy } from '../domain/dispatch-policy.js';
|
|
@@ -83,6 +83,12 @@ export function createActivationScheduler(orchestration, execution, resources, c
|
|
|
83
83
|
return { kind: 'exhausted', progressCount: 0 };
|
|
84
84
|
if (await isDispatchPaused())
|
|
85
85
|
return { kind: 'paused' };
|
|
86
|
+
let expired = [];
|
|
87
|
+
if (orchestration.expireTimedOutWaits !== undefined) {
|
|
88
|
+
expired = await orchestration.expireTimedOutWaits(context('await-timeout-expiry'));
|
|
89
|
+
if (await isDispatchPaused())
|
|
90
|
+
return { kind: 'paused' };
|
|
91
|
+
}
|
|
86
92
|
const rawPending = await orchestration.listPendingActivations(options.workItemId);
|
|
87
93
|
const pending = (await Promise.all(rawPending.map(async (candidate) => {
|
|
88
94
|
const work = await dependencies.work?.get(candidate.workflow.workItemId);
|
|
@@ -138,7 +144,7 @@ export function createActivationScheduler(orchestration, execution, resources, c
|
|
|
138
144
|
reason: recovery.run.failure?.message ?? 'execution failed',
|
|
139
145
|
};
|
|
140
146
|
}
|
|
141
|
-
|
|
147
|
+
const result = await runDispatchLoop(pending, {
|
|
142
148
|
orchestration,
|
|
143
149
|
execution,
|
|
144
150
|
resources,
|
|
@@ -149,6 +155,15 @@ export function createActivationScheduler(orchestration, execution, resources, c
|
|
|
149
155
|
isDispatchPaused,
|
|
150
156
|
commandContext: context,
|
|
151
157
|
});
|
|
158
|
+
if (expired.length === 0 ||
|
|
159
|
+
(result.kind !== 'no-work' && result.kind !== WorkflowStatus.Waiting))
|
|
160
|
+
return result;
|
|
161
|
+
const workflow = expired[0];
|
|
162
|
+
return {
|
|
163
|
+
kind: WorkflowStatus.Blocked,
|
|
164
|
+
workflowInstanceId: workflow.workflowInstanceId,
|
|
165
|
+
reason: workflow.blockReason ?? 'await.timeout-exceeded',
|
|
166
|
+
};
|
|
152
167
|
};
|
|
153
168
|
const serialise = dependencies.schedulerSerialiser ?? createInProcessSerialiser();
|
|
154
169
|
return {
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { acceptSignal as decideSignal, waitForSignal as decideSignalWait, } from '../domain/interpreter.js';
|
|
1
|
+
import { acceptSignal as decideSignal, waitForSignal as decideSignalWait, expireTimedOutAwait, } from '../domain/interpreter.js';
|
|
2
2
|
export class AcceptSignal {
|
|
3
3
|
repository;
|
|
4
4
|
workflows;
|
|
@@ -21,6 +21,14 @@ export class AcceptSignal {
|
|
|
21
21
|
}
|
|
22
22
|
async execute(workflowInstanceId, signal, context) {
|
|
23
23
|
const loaded = await this.repository.loadRequired(workflowInstanceId);
|
|
24
|
+
const expiry = expireTimedOutAwait(loaded.view, {
|
|
25
|
+
occurredAt: context.occurredAt,
|
|
26
|
+
causationId: context.commandId,
|
|
27
|
+
});
|
|
28
|
+
if (expiry.kind === 'append') {
|
|
29
|
+
await this.repository.append(workflowInstanceId, loaded.sequence, expiry.events);
|
|
30
|
+
return (await this.repository.loadRequired(workflowInstanceId)).view;
|
|
31
|
+
}
|
|
24
32
|
const item = await this.work.get(loaded.view.workItemId);
|
|
25
33
|
const definition = await this.workflows.definitionForOperation(loaded.view, loaded.sequence, context);
|
|
26
34
|
if (definition === null)
|
|
@@ -16,6 +16,7 @@ export declare class AdvanceWorkflow {
|
|
|
16
16
|
requestSupplementalActivity(id: WorkflowInstanceId, request: SupplementalActivityRequest, context: CommandContext): Promise<import("../index.js").WorkflowInstanceView>;
|
|
17
17
|
markActivationStarted(id: WorkflowInstanceId, activationId: ActivationId, context: CommandContext): Promise<import("../index.js").WorkflowInstanceView | null>;
|
|
18
18
|
block(id: WorkflowInstanceId, reason: string, context: CommandContext): Promise<import("../index.js").WorkflowInstanceView | null>;
|
|
19
|
+
expireTimedOutWaits(context: CommandContext): Promise<import("../index.js").WorkflowInstanceView[]>;
|
|
19
20
|
resolveExecutionFailure(id: WorkflowInstanceId, input: {
|
|
20
21
|
readonly activationId: ActivationId;
|
|
21
22
|
readonly runId: string;
|
|
@@ -6,6 +6,7 @@ import { workflowEventData } from '../domain/decision-events.js';
|
|
|
6
6
|
import { requestChangesResume as decideChangesResume, requestFreshOperatorRetry as decideFreshOperatorRetry, requestOperatorRetry as decideOperatorRetry, requestSupplementalActivity as decideSupplementalActivity, isNeedsClarificationResumeEligible, requestRunnerQuotaRetry, } from '../domain/interpreter.js';
|
|
7
7
|
import { isAuthorisedActor } from '../domain/supplemental-policy.js';
|
|
8
8
|
import { appendWithIntentRecovery } from './durable-append.js';
|
|
9
|
+
import { expireTimedOutWaits } from './expire-timed-out-waits.js';
|
|
9
10
|
import { acceptResourceTransition, matchResourceTransitions, } from './resource-transition-matching.js';
|
|
10
11
|
import { matchWatches } from './watch-matching.js';
|
|
11
12
|
export class OperatorRetryIneligibleError extends Error {
|
|
@@ -75,6 +76,9 @@ export class AdvanceWorkflow {
|
|
|
75
76
|
await this.repository.append(id, loaded.sequence, [event]);
|
|
76
77
|
return (await this.repository.load(id)).view;
|
|
77
78
|
}
|
|
79
|
+
async expireTimedOutWaits(context) {
|
|
80
|
+
return expireTimedOutWaits(this.repository, context);
|
|
81
|
+
}
|
|
78
82
|
async resolveExecutionFailure(id, input, context) {
|
|
79
83
|
const loaded = await this.repository.load(id);
|
|
80
84
|
if (loaded.view === null ||
|
|
@@ -0,0 +1,3 @@
|
|
|
1
|
+
import type { CommandContext } from '@atolis-hq/eventing';
|
|
2
|
+
import type { OrchestrationRepository } from './orchestration-repository.js';
|
|
3
|
+
export declare function expireTimedOutWaits(repository: OrchestrationRepository, context: CommandContext): Promise<import("../index.js").WorkflowInstanceView[]>;
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import { expireTimedOutAwait } from '../domain/interpreter.js';
|
|
2
|
+
export async function expireTimedOutWaits(repository, context) {
|
|
3
|
+
const expired = [];
|
|
4
|
+
for (const loaded of await repository.list()) {
|
|
5
|
+
if (loaded.view === null)
|
|
6
|
+
continue;
|
|
7
|
+
const decision = expireTimedOutAwait(loaded.view, {
|
|
8
|
+
occurredAt: context.occurredAt,
|
|
9
|
+
causationId: context.commandId,
|
|
10
|
+
});
|
|
11
|
+
if (decision.kind === 'ignored')
|
|
12
|
+
continue;
|
|
13
|
+
await repository.append(loaded.view.workflowInstanceId, loaded.sequence, decision.events);
|
|
14
|
+
expired.push((await repository.loadRequired(loaded.view.workflowInstanceId)).view);
|
|
15
|
+
}
|
|
16
|
+
return expired;
|
|
17
|
+
}
|
|
@@ -72,6 +72,7 @@ export declare class OrchestrationService {
|
|
|
72
72
|
activation: import("../contracts/views.js").ActivityActivationView;
|
|
73
73
|
}[]>;
|
|
74
74
|
listWaiting(signalKind?: SignalName): Promise<WorkflowInstanceView[]>;
|
|
75
|
+
expireTimedOutWaits(context: CommandContext): Promise<WorkflowInstanceView[]>;
|
|
75
76
|
listAll(): Promise<WorkflowInstanceView[]>;
|
|
76
77
|
listCurrentDefinitions(): readonly {
|
|
77
78
|
readonly definition: CompiledWorkflow;
|
|
@@ -177,6 +177,9 @@ export class OrchestrationService {
|
|
|
177
177
|
listWaiting(signalKind) {
|
|
178
178
|
return this.advanceWorkflow.listWaiting(signalKind);
|
|
179
179
|
}
|
|
180
|
+
expireTimedOutWaits(context) {
|
|
181
|
+
return this.advanceWorkflow.expireTimedOutWaits(context);
|
|
182
|
+
}
|
|
180
183
|
listAll() {
|
|
181
184
|
return this.advanceWorkflow.listAll();
|
|
182
185
|
}
|
|
@@ -14,6 +14,7 @@ export declare const awaitConfigSchema: z.ZodObject<{
|
|
|
14
14
|
kind: z.ZodLiteral<"watch">;
|
|
15
15
|
id: z.ZodString;
|
|
16
16
|
}, z.core.$strict>]>>>;
|
|
17
|
+
timeoutMs: z.ZodOptional<z.ZodNumber>;
|
|
17
18
|
}, z.core.$strict>;
|
|
18
19
|
declare const watchGateConfigSchema: z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
|
|
19
20
|
watch: z.ZodString;
|
|
@@ -86,6 +87,7 @@ export declare const outcomeRouteConfigSchema: z.ZodObject<{
|
|
|
86
87
|
kind: z.ZodLiteral<"watch">;
|
|
87
88
|
id: z.ZodString;
|
|
88
89
|
}, z.core.$strict>]>>>;
|
|
90
|
+
timeoutMs: z.ZodOptional<z.ZodNumber>;
|
|
89
91
|
}, z.core.$strict>>;
|
|
90
92
|
watchGates: z.ZodOptional<z.ZodArray<z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
|
|
91
93
|
watch: z.ZodString;
|
|
@@ -139,6 +141,7 @@ export declare const stageConfigSchema: z.ZodObject<{
|
|
|
139
141
|
kind: z.ZodLiteral<"watch">;
|
|
140
142
|
id: z.ZodString;
|
|
141
143
|
}, z.core.$strict>]>>>;
|
|
144
|
+
timeoutMs: z.ZodOptional<z.ZodNumber>;
|
|
142
145
|
}, z.core.$strict>>;
|
|
143
146
|
watchGates: z.ZodOptional<z.ZodArray<z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
|
|
144
147
|
watch: z.ZodString;
|
|
@@ -236,6 +239,7 @@ export declare const workflowDefinitionConfigSchema: z.ZodObject<{
|
|
|
236
239
|
kind: z.ZodLiteral<"watch">;
|
|
237
240
|
id: z.ZodString;
|
|
238
241
|
}, z.core.$strict>]>>>;
|
|
242
|
+
timeoutMs: z.ZodOptional<z.ZodNumber>;
|
|
239
243
|
}, z.core.$strict>>;
|
|
240
244
|
watchGates: z.ZodOptional<z.ZodArray<z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
|
|
241
245
|
watch: z.ZodString;
|
|
@@ -308,6 +312,8 @@ export interface CompiledAwait {
|
|
|
308
312
|
readonly signal: SignalName;
|
|
309
313
|
readonly from: readonly ApprovalAuthority[];
|
|
310
314
|
readonly resume: TransitionTarget;
|
|
315
|
+
/** Opt-in bound for this explicit await; omitted waits indefinitely. */
|
|
316
|
+
readonly timeoutMs?: number;
|
|
311
317
|
}
|
|
312
318
|
export interface CompiledWatchGate {
|
|
313
319
|
readonly watch: WatchId;
|
|
@@ -71,6 +71,8 @@ export declare const activityRequestedSchema: z.ZodObject<{
|
|
|
71
71
|
}, z.core.$strict>;
|
|
72
72
|
export declare const expectationSchema: z.ZodObject<{
|
|
73
73
|
signalKind: z.ZodPipe<z.ZodString, z.ZodTransform<import("./identifiers.js").SignalName, string>>;
|
|
74
|
+
timeoutMs: z.ZodOptional<z.ZodNumber>;
|
|
75
|
+
waitStartedAt: z.ZodOptional<z.ZodString>;
|
|
74
76
|
suppressWatchDispatch: z.ZodOptional<z.ZodLiteral<true>>;
|
|
75
77
|
resourceId: z.ZodOptional<z.ZodString>;
|
|
76
78
|
revision: z.ZodOptional<z.ZodString>;
|
|
@@ -121,6 +121,8 @@ const resourceTransitionSchema = z
|
|
|
121
121
|
export const expectationSchema = z
|
|
122
122
|
.object({
|
|
123
123
|
signalKind: brandedStringSchema(signalName),
|
|
124
|
+
timeoutMs: z.number().int().positive().optional(),
|
|
125
|
+
waitStartedAt: z.string().datetime().optional(),
|
|
124
126
|
suppressWatchDispatch: z.literal(true).optional(),
|
|
125
127
|
resourceId: z.string().optional(),
|
|
126
128
|
revision: z.string().optional(),
|
|
@@ -187,6 +187,10 @@ export interface ActivityRequestedPayload {
|
|
|
187
187
|
}
|
|
188
188
|
export interface SignalExpectation {
|
|
189
189
|
readonly signalKind: SignalName;
|
|
190
|
+
/** Present only for an explicit workflow await with a configured deadline. */
|
|
191
|
+
readonly timeoutMs?: number | undefined;
|
|
192
|
+
/** The control-plane time recorded by the durable wait-start event. */
|
|
193
|
+
readonly waitStartedAt?: string | undefined;
|
|
190
194
|
/** Re-arm a blocked gate without treating its wait event as a new watch trigger. */
|
|
191
195
|
readonly suppressWatchDispatch?: true | undefined;
|
|
192
196
|
readonly resourceId?: string | undefined;
|
|
@@ -17,6 +17,8 @@ export interface ActivityActivationView {
|
|
|
17
17
|
}
|
|
18
18
|
export interface SignalExpectationView {
|
|
19
19
|
readonly signalKind: SignalName;
|
|
20
|
+
readonly timeoutMs?: number;
|
|
21
|
+
readonly waitStartedAt?: string;
|
|
20
22
|
readonly suppressWatchDispatch?: true;
|
|
21
23
|
readonly intentEventId?: string;
|
|
22
24
|
readonly resourceId?: string;
|
|
@@ -159,6 +159,7 @@ function compileAwait(workflow, rawStageName, resume, config, declaredWatchIds)
|
|
|
159
159
|
signal: signalName(config.signal),
|
|
160
160
|
from: Object.freeze(config.from.map((entry) => compileAuthority(workflow, rawStageName, entry, declaredWatchIds))),
|
|
161
161
|
resume,
|
|
162
|
+
...(config.timeoutMs === undefined ? {} : { timeoutMs: config.timeoutMs }),
|
|
162
163
|
});
|
|
163
164
|
}
|
|
164
165
|
function compileAuthority(workflow, rawStageName, entry, declaredWatchIds) {
|
|
@@ -3,7 +3,7 @@ import type { WorkflowInstanceView } from '../contracts/views.js';
|
|
|
3
3
|
import { type AcceptActivityOutcome, type OrchestrationDecision } from './activation-policy.js';
|
|
4
4
|
export { startInstance } from './activation-policy.js';
|
|
5
5
|
export type { AcceptActivityOutcome, DecisionContext, OrchestrationDecision, QueueSupplementalActivity, StartInstanceInput, } from './activation-policy.js';
|
|
6
|
-
export { acceptSignal, waitForSignal } from './signal-policy.js';
|
|
6
|
+
export { acceptSignal, AwaitTimeoutBlockReason, expireTimedOutAwait, hasTimedOutAwait, waitForSignal, } from './signal-policy.js';
|
|
7
7
|
export type { AcceptSignal } from './signal-policy.js';
|
|
8
8
|
export { requestSupplementalActivity } from './supplemental-policy.js';
|
|
9
9
|
export type { SupplementalActivityRequest } from './supplemental-policy.js';
|
|
@@ -8,7 +8,7 @@ import { acceptWaitingOutcome } from './signal-policy.js';
|
|
|
8
8
|
import { finishSupplemental, requestNextSupplemental } from './supplemental-policy.js';
|
|
9
9
|
import { finishRoute } from './transition.js';
|
|
10
10
|
export { startInstance } from './activation-policy.js';
|
|
11
|
-
export { acceptSignal, waitForSignal } from './signal-policy.js';
|
|
11
|
+
export { acceptSignal, AwaitTimeoutBlockReason, expireTimedOutAwait, hasTimedOutAwait, waitForSignal, } from './signal-policy.js';
|
|
12
12
|
export { requestSupplementalActivity } from './supplemental-policy.js';
|
|
13
13
|
export { isChangesResumeEligible, isGroupBudgetExtensionEligible, isNeedsClarificationResumeEligible, isOperatorRetryEligible, requestChangesResume, requestFreshOperatorRetry, requestOperatorRetry, selectOperatorRetryTarget, } from './operator-retry-policy.js';
|
|
14
14
|
export { isRunnerQuotaRetryEligible, requestRunnerQuotaRetry, } from './runner-quota-retry-policy.js';
|
|
@@ -3,6 +3,7 @@ import { OrchestrationEventType, WatchGateVerdictSignal } from '../contracts/eve
|
|
|
3
3
|
import { signalName, stageName, watchId } from '../contracts/identifiers.js';
|
|
4
4
|
import { ActivityActivationStatus, ApprovalAuthorityKind, WorkflowStatus, } from '../contracts/vocabulary.js';
|
|
5
5
|
import { activation, nextOrdinal, stateDraft } from './decision-events.js';
|
|
6
|
+
import { AwaitTimeoutBlockReason } from './signal-policy.js';
|
|
6
7
|
// `then: await-human` on a `failed` route compiles its wait to this literal
|
|
7
8
|
// signal name (see compileTarget); a workflow parked there has exhausted
|
|
8
9
|
// whatever automatic retries it had and is durably equivalent to a blocked
|
|
@@ -22,8 +23,9 @@ export function isOperatorRetryEligible(view) {
|
|
|
22
23
|
return isRetryEligibleFailedWait(view);
|
|
23
24
|
}
|
|
24
25
|
function isRetryEligibleBlock(view, pending) {
|
|
25
|
-
return (
|
|
26
|
-
view.
|
|
26
|
+
return (view.blockReason === AwaitTimeoutBlockReason ||
|
|
27
|
+
(view.blockReason === 'unconfigured outcome failed' &&
|
|
28
|
+
view.lastOutcome?.kind === ActivityOutcomeKind.Failed) ||
|
|
27
29
|
view.executionFailure?.activationId === pending.activationId);
|
|
28
30
|
}
|
|
29
31
|
function isRetryEligibleFailedWait(view) {
|
|
@@ -2,10 +2,14 @@ import type { CompiledWorkflow } from '../contracts/config.js';
|
|
|
2
2
|
import type { OrchestrationSignal, SignalExpectation } from '../contracts/events.js';
|
|
3
3
|
import type { WorkflowInstanceView } from '../contracts/views.js';
|
|
4
4
|
import type { AcceptActivityOutcome, DecisionContext, OrchestrationDecision } from './activation-policy.js';
|
|
5
|
+
/** Stable reason used whenever an explicit workflow await reaches its deadline. */
|
|
6
|
+
export declare const AwaitTimeoutBlockReason = "await.timeout-exceeded";
|
|
5
7
|
export interface AcceptSignal extends DecisionContext {
|
|
6
8
|
readonly signal: OrchestrationSignal;
|
|
7
9
|
readonly consent?: boolean;
|
|
8
10
|
}
|
|
9
11
|
export declare function waitForSignal(state: WorkflowInstanceView, expectation: SignalExpectation, input: DecisionContext): OrchestrationDecision;
|
|
10
12
|
export declare function acceptSignal(definition: CompiledWorkflow, state: WorkflowInstanceView, input: AcceptSignal): OrchestrationDecision;
|
|
13
|
+
export declare function hasTimedOutAwait(state: WorkflowInstanceView, occurredAt: string): boolean;
|
|
14
|
+
export declare function expireTimedOutAwait(state: WorkflowInstanceView, input: DecisionContext): OrchestrationDecision;
|
|
11
15
|
export declare function acceptWaitingOutcome(state: WorkflowInstanceView, input: AcceptActivityOutcome): OrchestrationDecision;
|
|
@@ -4,6 +4,8 @@ import { stageName } from '../contracts/identifiers.js';
|
|
|
4
4
|
import { ActivityActivationStatus, ApprovalAuthorityKind, WorkflowStatus, } from '../contracts/vocabulary.js';
|
|
5
5
|
import { activation, nextOrdinal, stateDraft } from './decision-events.js';
|
|
6
6
|
import { resumeToTarget } from './transition.js';
|
|
7
|
+
/** Stable reason used whenever an explicit workflow await reaches its deadline. */
|
|
8
|
+
export const AwaitTimeoutBlockReason = 'await.timeout-exceeded';
|
|
7
9
|
export function waitForSignal(state, expectation, input) {
|
|
8
10
|
if (state.status !== WorkflowStatus.Waiting && state.status !== WorkflowStatus.Blocked)
|
|
9
11
|
return { kind: 'ignored', reason: 'WorkflowInstance is not waiting or blocked' };
|
|
@@ -36,6 +38,29 @@ export function acceptSignal(definition, state, input) {
|
|
|
36
38
|
requestLegacyReentry(events, definition, state, input);
|
|
37
39
|
return { kind: 'append', events };
|
|
38
40
|
}
|
|
41
|
+
export function hasTimedOutAwait(state, occurredAt) {
|
|
42
|
+
const wait = state.waitingFor;
|
|
43
|
+
if (state.status !== WorkflowStatus.Waiting || wait?.timeoutMs === undefined)
|
|
44
|
+
return false;
|
|
45
|
+
if (wait.waitStartedAt === undefined)
|
|
46
|
+
return false;
|
|
47
|
+
const startedAt = Date.parse(wait.waitStartedAt);
|
|
48
|
+
const now = Date.parse(occurredAt);
|
|
49
|
+
return Number.isFinite(startedAt) && Number.isFinite(now) && now >= startedAt + wait.timeoutMs;
|
|
50
|
+
}
|
|
51
|
+
export function expireTimedOutAwait(state, input) {
|
|
52
|
+
if (!hasTimedOutAwait(state, input.occurredAt))
|
|
53
|
+
return { kind: 'ignored', reason: 'WorkflowInstance await has not timed out' };
|
|
54
|
+
return {
|
|
55
|
+
kind: 'append',
|
|
56
|
+
events: [
|
|
57
|
+
stateDraft(state, input, {
|
|
58
|
+
eventType: OrchestrationEventType.InstanceBlocked,
|
|
59
|
+
payload: { reason: AwaitTimeoutBlockReason },
|
|
60
|
+
}, 1),
|
|
61
|
+
],
|
|
62
|
+
};
|
|
63
|
+
}
|
|
39
64
|
function requestLegacyReentry(events, definition, state, input) {
|
|
40
65
|
const stage = definition.stages[stageName(state.currentStage)];
|
|
41
66
|
events.push(stateDraft(state, input, {
|
|
@@ -70,6 +95,7 @@ function signalRejectionReason(state, input) {
|
|
|
70
95
|
}
|
|
71
96
|
function isBlockedHumanAuthorizedWait(state, signal, expected) {
|
|
72
97
|
return (state.status === WorkflowStatus.Blocked &&
|
|
98
|
+
state.blockReason !== AwaitTimeoutBlockReason &&
|
|
73
99
|
signal.authority?.kind === ApprovalAuthorityKind.Human &&
|
|
74
100
|
expected.from?.some((authority) => authority.kind === ApprovalAuthorityKind.Human) === true);
|
|
75
101
|
}
|
|
@@ -35,6 +35,9 @@ export function finishRoute(events, definition, state, input, route) {
|
|
|
35
35
|
from: route.await.from,
|
|
36
36
|
resume: route.await.resume,
|
|
37
37
|
onRejectResume: route.reentryTarget,
|
|
38
|
+
...(route.await.timeoutMs === undefined
|
|
39
|
+
? {}
|
|
40
|
+
: { timeoutMs: route.await.timeoutMs, waitStartedAt: input.occurredAt }),
|
|
38
41
|
},
|
|
39
42
|
}, events.length + 1));
|
|
40
43
|
return;
|
|
@@ -177,6 +177,10 @@ function applySignalWaitStarted(state, event) {
|
|
|
177
177
|
state.status = orchestrationStatusTransitions[OrchestrationEventType.SignalWaitStarted];
|
|
178
178
|
state.waitingFor = {
|
|
179
179
|
signalKind: event.payload.signalKind,
|
|
180
|
+
...(event.payload.timeoutMs === undefined ? {} : { timeoutMs: event.payload.timeoutMs }),
|
|
181
|
+
...(event.payload.waitStartedAt === undefined
|
|
182
|
+
? {}
|
|
183
|
+
: { waitStartedAt: event.payload.waitStartedAt }),
|
|
180
184
|
...(event.payload.suppressWatchDispatch === true ? { suppressWatchDispatch: true } : {}),
|
|
181
185
|
...(event.payload.resourceId === undefined ? {} : { resourceId: event.payload.resourceId }),
|
|
182
186
|
...(event.payload.revision === undefined ? {} : { revision: event.payload.revision }),
|