@astrale-os/sdk 0.6.0-beta.40 → 0.6.0-beta.41
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/application/workflow/durable.d.ts +25 -13
- package/dist/execution/workflows/attempts.d.ts +87 -0
- package/dist/execution/workflows/attempts.js +205 -0
- package/dist/execution/workflows/engine.d.ts +6 -12
- package/dist/execution/workflows/failure.d.ts +6 -13
- package/dist/execution/workflows/failure.js +9 -26
- package/dist/execution/workflows/index.d.ts +1 -1
- package/dist/execution/workflows/options.d.ts +21 -6
- package/dist/execution/workflows/options.js +28 -18
- package/dist/execution/workflows/run.d.ts +2 -0
- package/dist/execution/workflows/run.js +120 -76
- package/dist/execution/workflows/steps.d.ts +7 -1
- package/dist/execution/workflows/steps.js +32 -6
- package/dist/testing/durable/engine.d.ts +18 -11
- package/dist/testing/durable/engine.js +43 -38
- package/dist/testing/durable/index.d.ts +2 -2
- package/dist/testing/durable/index.js +1 -1
- package/dist/testing/durable/steps.js +9 -4
- package/dist/testing/index.d.ts +2 -2
- package/dist/testing/index.js +1 -1
- package/dist/tooling/linter/implementations/source/workflows.js +0 -1
- package/docs/durable-workflows.md +69 -42
- package/package.json +1 -1
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import type { IssuerId } from '../../platform/auth/index.js';
|
|
2
|
-
import type { ExecutionOutcome, ExecutionRef } from '../../platform/client/session/index.js';
|
|
3
2
|
import type { NodeId } from '../../platform/graph/node/index.js';
|
|
4
3
|
import type { CallableInputOf, Domain, ResolvedMethod } from '../../platform/schema/index.js';
|
|
5
4
|
import type { Value } from '../../platform/value/index.js';
|
|
@@ -65,18 +64,29 @@ export interface DurableStepAuthority<DomainValue extends Domain = Domain, Defin
|
|
|
65
64
|
* How long a failing Step is retried. Its limits, `maxAttempts` and `maxDurationMs`, go together: a
|
|
66
65
|
* Step that declares either replaces both limits of the deployment, and the one it omits does not
|
|
67
66
|
* bound it. The budget is spent at the first limit reached.
|
|
67
|
+
*
|
|
68
|
+
* Only a failure of the Step's own spends an attempt. A Kernel that refuses with a prescribed delay
|
|
69
|
+
* (capacity, draining, maintenance), or that does not answer a Query or a keyed call, is waited out
|
|
70
|
+
* without spending one; a request the Kernel can never accept fails the Step at once.
|
|
68
71
|
*/
|
|
69
72
|
export interface DurableStepRetry {
|
|
70
|
-
/**
|
|
73
|
+
/**
|
|
74
|
+
* Attempts that may fail on the Step's own account, the first one included: 1 to 1,000. A wait for
|
|
75
|
+
* the platform spends none.
|
|
76
|
+
*/
|
|
71
77
|
readonly maxAttempts?: number;
|
|
72
78
|
/**
|
|
73
|
-
* Time from the first attempt after which a failed attempt is not retried,
|
|
74
|
-
* attempt already scheduled, such as after a long prescribed delay, still
|
|
79
|
+
* Time from the first attempt after which a failed attempt is not retried, whatever it failed on,
|
|
80
|
+
* 1 ms to 365 days; an attempt already scheduled, such as after a long prescribed delay, still
|
|
81
|
+
* runs. The only bound on waiting for the platform.
|
|
75
82
|
*/
|
|
76
83
|
readonly maxDurationMs?: number;
|
|
77
84
|
/** Delay before the first retry, doubled on each failure: 1 ms to one hour. */
|
|
78
85
|
readonly initialIntervalMs?: number;
|
|
79
|
-
/**
|
|
86
|
+
/**
|
|
87
|
+
* Longest backoff between two attempts: `initialIntervalMs` to one day. A longer delay the Kernel
|
|
88
|
+
* prescribes is still waited.
|
|
89
|
+
*/
|
|
80
90
|
readonly maxIntervalMs?: number;
|
|
81
91
|
}
|
|
82
92
|
/** A Step: its id, and the retry budget and timeout it declares instead of the deployment's. */
|
|
@@ -88,8 +98,8 @@ export interface DurableStepDeclaration {
|
|
|
88
98
|
}
|
|
89
99
|
/**
|
|
90
100
|
* A Step's id alone, under the deployment's retry budget and Step timeout, or a declaration of its
|
|
91
|
-
* own. A Step that spends its budget
|
|
92
|
-
* ends `failed` with code `STEP_EXHAUSTED`.
|
|
101
|
+
* own. A Step that spends its budget, or that the Kernel refuses for good, throws an error
|
|
102
|
+
* `exhausted` recognizes; uncaught, the Execution ends `failed` with code `STEP_EXHAUSTED`.
|
|
93
103
|
*/
|
|
94
104
|
export type DurableStepId = string | DurableStepDeclaration;
|
|
95
105
|
export interface DurableStep<DomainValue extends Domain = Domain, Definitions extends Integrations = Readonly<Record<never, never>>> {
|
|
@@ -171,15 +181,17 @@ export type DurableWorkflowContext<Callable, DomainValue extends Domain = Domain
|
|
|
171
181
|
}): Promise<SignalOutcome<Signal>>;
|
|
172
182
|
/** Whether `error` is the engine's cooperative cancellation, which the Workflow may compensate. */
|
|
173
183
|
cancelled(error: unknown): boolean;
|
|
174
|
-
/**
|
|
184
|
+
/**
|
|
185
|
+
* Whether `error` is a Step that spent its retry budget, or that the Kernel refused for good,
|
|
186
|
+
* which the Workflow may handle.
|
|
187
|
+
*/
|
|
175
188
|
exhausted(error: unknown): boolean;
|
|
176
189
|
/**
|
|
177
|
-
* The failure of a Step attempt, to throw inside the Step, that the engine attempts again
|
|
178
|
-
*
|
|
179
|
-
*
|
|
190
|
+
* The failure of a Step attempt, to throw inside the Step, that the engine attempts again after
|
|
191
|
+
* exactly `delayMs` (at most one hour), as an HTTP `Retry-After` asks. The attempt spends the
|
|
192
|
+
* Step's budget, since only the Step knows whether its cause clears. A Kernel refusal that
|
|
193
|
+
* prescribes a delay is waited out already, without spending it.
|
|
180
194
|
*/
|
|
181
195
|
retryAfter(delayMs: number, cause?: unknown): Error;
|
|
182
|
-
/** Wait durably for another Execution to end, by journaled polling of its `result`. */
|
|
183
|
-
awaitExecution<Output>(execution: ExecutionRef<Output>): Promise<ExecutionOutcome<Output>>;
|
|
184
196
|
} & DurableSelfOf<Callable>;
|
|
185
197
|
export {};
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
import type { DurableEngine } from './engine.js';
|
|
2
|
+
import type { KernelRefusal } from './failure.js';
|
|
3
|
+
import type { StepRetryBudget } from './options.js';
|
|
4
|
+
/**
|
|
5
|
+
* What the failure of one Step attempt means for its Step, read from protocol facts alone:
|
|
6
|
+
*
|
|
7
|
+
* - `permanent`: the Kernel refused a request that cannot succeed sent again unchanged. The Step
|
|
8
|
+
* fails at once.
|
|
9
|
+
* - `platform`: the Kernel or the way to it was unavailable, and sending the request again repeats
|
|
10
|
+
* no effect. The Step waits and tries again without spending an attempt, bounded only by its
|
|
11
|
+
* `maxDurationMs` and by cancellation. An attempt after an unkeyed `invoke` that may have run,
|
|
12
|
+
* which the next attempt sends again, is never one: it spends an attempt.
|
|
13
|
+
* - `own`: anything else, the Step's code, a Provider, or a refusal whose retry meaning the Kernel
|
|
14
|
+
* leaves open. The attempt spends one of the Step's attempts.
|
|
15
|
+
*/
|
|
16
|
+
export type AttemptFailureKind = 'permanent' | 'platform' | 'own';
|
|
17
|
+
/** One failed Step attempt, as its journal entry records it. */
|
|
18
|
+
export interface FailedAttempt {
|
|
19
|
+
readonly kind: AttemptFailureKind;
|
|
20
|
+
/** The delay the failure prescribes: a Kernel `retryAfterMs`, or the Step's own `retryAfter`. */
|
|
21
|
+
readonly delayMs?: number;
|
|
22
|
+
readonly refusal?: KernelRefusal;
|
|
23
|
+
/** When the attempt started and ended, on the run's clock. */
|
|
24
|
+
readonly startedAt: number;
|
|
25
|
+
readonly endedAt: number;
|
|
26
|
+
}
|
|
27
|
+
/** What the failed attempts of one Step spent so far. */
|
|
28
|
+
export interface StepSpending {
|
|
29
|
+
/** The start of the Step's first attempt, from which `maxDurationMs` runs. */
|
|
30
|
+
readonly startedAt: number;
|
|
31
|
+
/** Attempts that failed for the Step's own reason. */
|
|
32
|
+
readonly attempts: number;
|
|
33
|
+
/** Platform failures since the Step's last own failure, which space the next platform wait. */
|
|
34
|
+
readonly unavailable: number;
|
|
35
|
+
}
|
|
36
|
+
/** How a Step goes on after a failed attempt. */
|
|
37
|
+
export type NextAttempt = {
|
|
38
|
+
readonly kind: 'retry';
|
|
39
|
+
readonly delayMs: number;
|
|
40
|
+
/** A wait for the platform, or the backoff of an own failure. */
|
|
41
|
+
readonly wait: typeof UNAVAILABLE | typeof BACKOFF;
|
|
42
|
+
readonly spending: StepSpending;
|
|
43
|
+
} | {
|
|
44
|
+
readonly kind: 'exhausted';
|
|
45
|
+
readonly refusal?: KernelRefusal;
|
|
46
|
+
};
|
|
47
|
+
/** Name prefix of the durable wait before the attempt that follows an own failure. */
|
|
48
|
+
export declare const BACKOFF = "retry";
|
|
49
|
+
/** Name prefix of the durable wait before the attempt that follows a platform failure. */
|
|
50
|
+
export declare const UNAVAILABLE = "unavailable";
|
|
51
|
+
/**
|
|
52
|
+
* The outcome of a Kernel call that repeats no effect when sent again, a read or a call carrying
|
|
53
|
+
* its idempotency key: its failure lets the Step tell an unreachable Kernel from its own failure.
|
|
54
|
+
*/
|
|
55
|
+
export declare function resendableCall<Output>(call: () => Output): Output;
|
|
56
|
+
/**
|
|
57
|
+
* The outcome of a Kernel call that runs its effect again when sent again, an `invoke` without a
|
|
58
|
+
* key: `ran` hears that the call may have run, unless its failure proves it did not.
|
|
59
|
+
*/
|
|
60
|
+
export declare function unkeyedCall<Output>(call: () => Promise<Output>, ran: () => void): Promise<Output>;
|
|
61
|
+
/**
|
|
62
|
+
* Classify the failure of one Step attempt, a declared error excepted, by the first Kernel answer or
|
|
63
|
+
* transport failure among its causes, and name the Kernel refusal behind it, if any.
|
|
64
|
+
*/
|
|
65
|
+
export declare function classifyFailure(cause: unknown): Pick<FailedAttempt, 'kind' | 'delayMs' | 'refusal'>;
|
|
66
|
+
/**
|
|
67
|
+
* The terminal failure a Step attempt hands the engine for `cause`: the engine journals it as the
|
|
68
|
+
* attempt's outcome, so the Step decides whether to go on from the journal alone, live and on
|
|
69
|
+
* replay. Its message starts with the Kernel refusal's codes, if any, and is bounded.
|
|
70
|
+
* `repeatsEffects` says whether the attempt ran a request the next attempt runs again with its
|
|
71
|
+
* effect, so an outage never repeats that effect beyond the Step's budget.
|
|
72
|
+
*/
|
|
73
|
+
export declare function failedAttemptError(engine: DurableEngine, cause: unknown, startedAt: number, endedAt: number, repeatsEffects: boolean): Error;
|
|
74
|
+
/** The failed attempt a journaled `run` outcome records, if it records one. */
|
|
75
|
+
export declare function failedAttemptOf(engine: DurableEngine, outcome: unknown): FailedAttempt | undefined;
|
|
76
|
+
/**
|
|
77
|
+
* How a Step goes on after `failed`, given what its earlier attempts spent:
|
|
78
|
+
*
|
|
79
|
+
* - A permanent failure exhausts the Step at once.
|
|
80
|
+
* - An own failure spends an attempt; the next one waits the delay the failure prescribes, or the
|
|
81
|
+
* Step's backoff over its own failures.
|
|
82
|
+
* - A platform failure spends no attempt; the next one waits the longer of the delay the Kernel
|
|
83
|
+
* prescribes and the Step's backoff over its platform failures since its last own one, so a long
|
|
84
|
+
* outage costs one attempt per `maxIntervalMs`.
|
|
85
|
+
* - Either way, a failure at or past `maxDurationMs` from the Step's first attempt exhausts it.
|
|
86
|
+
*/
|
|
87
|
+
export declare function nextAttempt(spent: StepSpending | undefined, failed: FailedAttempt, budget: StepRetryBudget): NextAttempt;
|
|
@@ -0,0 +1,205 @@
|
|
|
1
|
+
import { ClientError, ResponseError, TransportError, isExpectationMismatch, isMaintenanceSessionStale, isSnapshotStale, retryAfterOf, } from '../../platform/client/index.js';
|
|
2
|
+
import { attemptMessage, kernelRefusalOf, StepRetryAfter } from './failure.js';
|
|
3
|
+
/** Name prefix of the durable wait before the attempt that follows an own failure. */
|
|
4
|
+
export const BACKOFF = 'retry';
|
|
5
|
+
/** Name prefix of the durable wait before the attempt that follows a platform failure. */
|
|
6
|
+
export const UNAVAILABLE = 'unavailable';
|
|
7
|
+
/** Metadata key under which a failed attempt's journal entry records its `FailedAttempt`. */
|
|
8
|
+
const ATTEMPT = 'astrale.attempt';
|
|
9
|
+
/** The longest message a failed attempt's journal entry keeps, so an outage journals little. */
|
|
10
|
+
const MAXIMUM_MESSAGE = 1_024;
|
|
11
|
+
/** How deep a failure's causes are searched for the call that failed. */
|
|
12
|
+
const MAXIMUM_CAUSE_DEPTH = 8;
|
|
13
|
+
/**
|
|
14
|
+
* Kernel refusals that name a request that cannot succeed sent again unchanged: a malformed or
|
|
15
|
+
* unsupported request, or a Domain meaning this Instance does not serve.
|
|
16
|
+
*/
|
|
17
|
+
const PERMANENT_CODES = new Set([
|
|
18
|
+
1001, // PARSE_ERROR
|
|
19
|
+
1002, // INVALID_REQUEST
|
|
20
|
+
3003, // METHOD_NOT_ALLOWED
|
|
21
|
+
4002, // UNSUPPORTED
|
|
22
|
+
]);
|
|
23
|
+
const OUTCOME_UNKNOWN = 5002;
|
|
24
|
+
/** HTTP answers without an invocation response that say the server took no work. */
|
|
25
|
+
const UNAVAILABLE_STATUS = /^(?:429|502|503)$/u;
|
|
26
|
+
const STATUS_OF_ANSWER = /^server returned HTTP (\d{3}) without an invocation response\.$/u;
|
|
27
|
+
const STATUS_OF_DISCOVERY = /^Release discovery returned HTTP (\d{3})\.$/u;
|
|
28
|
+
/** Failures of Kernel calls that send again without repeating an effect: reads and keyed calls. */
|
|
29
|
+
const resendable = new WeakSet();
|
|
30
|
+
/**
|
|
31
|
+
* The outcome of a Kernel call that repeats no effect when sent again, a read or a call carrying
|
|
32
|
+
* its idempotency key: its failure lets the Step tell an unreachable Kernel from its own failure.
|
|
33
|
+
*/
|
|
34
|
+
export function resendableCall(call) {
|
|
35
|
+
let output;
|
|
36
|
+
try {
|
|
37
|
+
output = call();
|
|
38
|
+
}
|
|
39
|
+
catch (cause) {
|
|
40
|
+
markResendable(cause);
|
|
41
|
+
throw cause;
|
|
42
|
+
}
|
|
43
|
+
if (!(output instanceof Promise))
|
|
44
|
+
return output;
|
|
45
|
+
return output.catch((cause) => {
|
|
46
|
+
markResendable(cause);
|
|
47
|
+
throw cause;
|
|
48
|
+
});
|
|
49
|
+
}
|
|
50
|
+
function markResendable(cause) {
|
|
51
|
+
if (cause !== null && typeof cause === 'object')
|
|
52
|
+
resendable.add(cause);
|
|
53
|
+
}
|
|
54
|
+
/**
|
|
55
|
+
* The outcome of a Kernel call that runs its effect again when sent again, an `invoke` without a
|
|
56
|
+
* key: `ran` hears that the call may have run, unless its failure proves it did not.
|
|
57
|
+
*/
|
|
58
|
+
export function unkeyedCall(call, ran) {
|
|
59
|
+
const failed = (cause) => {
|
|
60
|
+
// For a call sent without a key, only a failure that proves nothing ran is the platform's.
|
|
61
|
+
if (classifyFailure(cause).kind !== 'platform')
|
|
62
|
+
ran();
|
|
63
|
+
throw cause;
|
|
64
|
+
};
|
|
65
|
+
let output;
|
|
66
|
+
try {
|
|
67
|
+
output = call();
|
|
68
|
+
}
|
|
69
|
+
catch (cause) {
|
|
70
|
+
return failed(cause);
|
|
71
|
+
}
|
|
72
|
+
return Promise.resolve(output).then((value) => {
|
|
73
|
+
ran();
|
|
74
|
+
return value;
|
|
75
|
+
}, failed);
|
|
76
|
+
}
|
|
77
|
+
/**
|
|
78
|
+
* Classify the failure of one Step attempt, a declared error excepted, by the first Kernel answer or
|
|
79
|
+
* transport failure among its causes, and name the Kernel refusal behind it, if any.
|
|
80
|
+
*/
|
|
81
|
+
export function classifyFailure(cause) {
|
|
82
|
+
const refusal = kernelRefusalOf(cause);
|
|
83
|
+
const named = refusal === undefined ? {} : { refusal };
|
|
84
|
+
if (cause instanceof StepRetryAfter)
|
|
85
|
+
return { kind: 'own', delayMs: cause.delayMs, ...named };
|
|
86
|
+
let resent = false;
|
|
87
|
+
let current = cause;
|
|
88
|
+
for (let depth = 0; depth < MAXIMUM_CAUSE_DEPTH; depth += 1) {
|
|
89
|
+
if (current === null || typeof current !== 'object')
|
|
90
|
+
break;
|
|
91
|
+
resent ||= resendable.has(current);
|
|
92
|
+
if (current instanceof ResponseError)
|
|
93
|
+
return { ...refusalKind(current, resent), ...named };
|
|
94
|
+
if (current instanceof TransportError)
|
|
95
|
+
return { kind: transportKind(current, resent), ...named };
|
|
96
|
+
if (current instanceof ClientError && unavailableStatus(STATUS_OF_DISCOVERY, current.message)) {
|
|
97
|
+
// Release discovery is a read the Kernel's own server failed to answer.
|
|
98
|
+
return { kind: 'platform', ...named };
|
|
99
|
+
}
|
|
100
|
+
current = current.cause;
|
|
101
|
+
}
|
|
102
|
+
return { kind: 'own', ...named };
|
|
103
|
+
}
|
|
104
|
+
function refusalKind(error, resent) {
|
|
105
|
+
// Read before the refusal guards, which narrow a `ResponseError` they reject to nothing.
|
|
106
|
+
const { code } = error;
|
|
107
|
+
// Capacity, draining and maintenance: refused before any work was admitted.
|
|
108
|
+
const delayMs = retryAfterOf(error);
|
|
109
|
+
if (delayMs !== undefined)
|
|
110
|
+
return { kind: 'platform', delayMs };
|
|
111
|
+
// A sealed conflict whose snapshot advanced ran nothing, and may be sent again at once.
|
|
112
|
+
if (isSnapshotStale(error))
|
|
113
|
+
return { kind: 'platform' };
|
|
114
|
+
if (PERMANENT_CODES.has(code) ||
|
|
115
|
+
isExpectationMismatch(error) ||
|
|
116
|
+
isMaintenanceSessionStale(error)) {
|
|
117
|
+
return { kind: 'permanent' };
|
|
118
|
+
}
|
|
119
|
+
// The Kernel asks not to send again an unknown outcome automatically, unless its key replays it.
|
|
120
|
+
if (code === OUTCOME_UNKNOWN && resent)
|
|
121
|
+
return { kind: 'platform' };
|
|
122
|
+
return { kind: 'own' };
|
|
123
|
+
}
|
|
124
|
+
/**
|
|
125
|
+
* A transport failure is the platform's when it is no local bound or deadline, and its request
|
|
126
|
+
* provably did not reach the Kernel or repeats no effect when sent again.
|
|
127
|
+
*/
|
|
128
|
+
function transportKind(error, resent) {
|
|
129
|
+
// A deadline may be the request's own cost: it spends an attempt, as any slow attempt does.
|
|
130
|
+
if (error.phase === 'timeout')
|
|
131
|
+
return 'own';
|
|
132
|
+
// A failure with neither a network cause nor an unavailable answer is a local bound.
|
|
133
|
+
const network = error.cause !== undefined || unavailableStatus(STATUS_OF_ANSWER, error.message);
|
|
134
|
+
if (!network)
|
|
135
|
+
return 'own';
|
|
136
|
+
if (error.context.kind === 'acquisition')
|
|
137
|
+
return 'platform';
|
|
138
|
+
return error.context.delivery === 'not-sent' || resent ? 'platform' : 'own';
|
|
139
|
+
}
|
|
140
|
+
function unavailableStatus(pattern, message) {
|
|
141
|
+
const status = pattern.exec(message)?.[1];
|
|
142
|
+
return status !== undefined && UNAVAILABLE_STATUS.test(status);
|
|
143
|
+
}
|
|
144
|
+
/**
|
|
145
|
+
* The terminal failure a Step attempt hands the engine for `cause`: the engine journals it as the
|
|
146
|
+
* attempt's outcome, so the Step decides whether to go on from the journal alone, live and on
|
|
147
|
+
* replay. Its message starts with the Kernel refusal's codes, if any, and is bounded.
|
|
148
|
+
* `repeatsEffects` says whether the attempt ran a request the next attempt runs again with its
|
|
149
|
+
* effect, so an outage never repeats that effect beyond the Step's budget.
|
|
150
|
+
*/
|
|
151
|
+
export function failedAttemptError(engine, cause, startedAt, endedAt, repeatsEffects) {
|
|
152
|
+
const classified = classifyFailure(cause);
|
|
153
|
+
const kind = repeatsEffects && classified.kind === 'platform' ? 'own' : classified.kind;
|
|
154
|
+
const failed = { ...classified, kind, startedAt, endedAt };
|
|
155
|
+
const message = attemptMessage(cause);
|
|
156
|
+
const error = new engine.TerminalError(message.length > MAXIMUM_MESSAGE ? `${message.slice(0, MAXIMUM_MESSAGE - 1)}…` : message, { metadata: { [ATTEMPT]: JSON.stringify(failed) } });
|
|
157
|
+
// Kept on the live error only: what the attempt threw, for a test engine to show.
|
|
158
|
+
Object.defineProperty(error, 'cause', { value: cause, configurable: true, writable: true });
|
|
159
|
+
return error;
|
|
160
|
+
}
|
|
161
|
+
/** The failed attempt a journaled `run` outcome records, if it records one. */
|
|
162
|
+
export function failedAttemptOf(engine, outcome) {
|
|
163
|
+
if (!(outcome instanceof engine.TerminalError) || outcome instanceof engine.CancelledError) {
|
|
164
|
+
return undefined;
|
|
165
|
+
}
|
|
166
|
+
const encoded = outcome.metadata?.[ATTEMPT];
|
|
167
|
+
return encoded === undefined ? undefined : JSON.parse(encoded);
|
|
168
|
+
}
|
|
169
|
+
/**
|
|
170
|
+
* How a Step goes on after `failed`, given what its earlier attempts spent:
|
|
171
|
+
*
|
|
172
|
+
* - A permanent failure exhausts the Step at once.
|
|
173
|
+
* - An own failure spends an attempt; the next one waits the delay the failure prescribes, or the
|
|
174
|
+
* Step's backoff over its own failures.
|
|
175
|
+
* - A platform failure spends no attempt; the next one waits the longer of the delay the Kernel
|
|
176
|
+
* prescribes and the Step's backoff over its platform failures since its last own one, so a long
|
|
177
|
+
* outage costs one attempt per `maxIntervalMs`.
|
|
178
|
+
* - Either way, a failure at or past `maxDurationMs` from the Step's first attempt exhausts it.
|
|
179
|
+
*/
|
|
180
|
+
export function nextAttempt(spent, failed, budget) {
|
|
181
|
+
const refusal = failed.refusal === undefined ? {} : { refusal: failed.refusal };
|
|
182
|
+
if (failed.kind === 'permanent')
|
|
183
|
+
return { kind: 'exhausted', ...refusal };
|
|
184
|
+
const startedAt = spent?.startedAt ?? failed.startedAt;
|
|
185
|
+
const own = failed.kind === 'own';
|
|
186
|
+
const attempts = (spent?.attempts ?? 0) + (own ? 1 : 0);
|
|
187
|
+
const unavailable = own ? 0 : (spent?.unavailable ?? 0) + 1;
|
|
188
|
+
if ((budget.maxAttempts !== undefined && attempts >= budget.maxAttempts) ||
|
|
189
|
+
(budget.maxDurationMs !== undefined && failed.endedAt - startedAt >= budget.maxDurationMs)) {
|
|
190
|
+
return { kind: 'exhausted', ...refusal };
|
|
191
|
+
}
|
|
192
|
+
const delayMs = own
|
|
193
|
+
? (failed.delayMs ?? backoff(budget, attempts))
|
|
194
|
+
: Math.max(failed.delayMs ?? 0, backoff(budget, unavailable));
|
|
195
|
+
return {
|
|
196
|
+
kind: 'retry',
|
|
197
|
+
delayMs,
|
|
198
|
+
wait: own ? BACKOFF : UNAVAILABLE,
|
|
199
|
+
spending: { startedAt, attempts, unavailable },
|
|
200
|
+
};
|
|
201
|
+
}
|
|
202
|
+
/** The `failures`-th wait of a capped exponential backoff, the first one `initialIntervalMs`. */
|
|
203
|
+
function backoff(budget, failures) {
|
|
204
|
+
return Math.min(budget.maxIntervalMs, budget.initialIntervalMs * budget.intervalFactor ** (failures - 1));
|
|
205
|
+
}
|
|
@@ -52,17 +52,6 @@ export interface EngineRetryPolicy {
|
|
|
52
52
|
readonly maxInterval: number;
|
|
53
53
|
readonly exponentiationFactor: number;
|
|
54
54
|
}
|
|
55
|
-
/**
|
|
56
|
-
* The retry budget of one journaled `run`. Spent, the `run` rejects with a `TerminalError` and the
|
|
57
|
-
* engine attempts it no more; an absent limit does not bound it.
|
|
58
|
-
*/
|
|
59
|
-
export interface EngineRunOptions {
|
|
60
|
-
readonly maxRetryAttempts?: number;
|
|
61
|
-
readonly maxRetryDuration?: number;
|
|
62
|
-
readonly initialRetryInterval: number;
|
|
63
|
-
readonly maxRetryInterval: number;
|
|
64
|
-
readonly retryIntervalFactor: number;
|
|
65
|
-
}
|
|
66
55
|
/**
|
|
67
56
|
* A promise the engine journals. Its combinators are journaled too, so a replay settles them as the
|
|
68
57
|
* first run did; racing it against another promise with `Promise.race` would not be.
|
|
@@ -73,7 +62,12 @@ export interface EnginePromise<Output> extends PromiseLike<Output> {
|
|
|
73
62
|
}
|
|
74
63
|
/** Journaled operations of one workflow run. */
|
|
75
64
|
export interface EngineWorkflowContext {
|
|
76
|
-
|
|
65
|
+
/**
|
|
66
|
+
* Run `action` once and journal its outcome: its value, or the `TerminalError` it throws. Any
|
|
67
|
+
* other failure fails the run's attempt, which the engine attempts again under its retry policy.
|
|
68
|
+
*/
|
|
69
|
+
run<Output>(name: string, action: () => Promise<Output>): PromiseLike<Output>;
|
|
70
|
+
/** A durable timer; `name` labels its journal entry. */
|
|
77
71
|
sleep(durationMs: number, name?: string): PromiseLike<void>;
|
|
78
72
|
promise(name: string): {
|
|
79
73
|
get(): EnginePromise<unknown>;
|
|
@@ -7,8 +7,9 @@ export interface KernelRefusal {
|
|
|
7
7
|
readonly reason?: string;
|
|
8
8
|
}
|
|
9
9
|
/**
|
|
10
|
-
* A Step that spent its retry budget
|
|
11
|
-
* the Kernel refused that attempt, `refusal` names
|
|
10
|
+
* A Step that spent its retry budget, or whose attempt the Kernel refused for good. Its message
|
|
11
|
+
* never carries what the last attempt threw; when the Kernel refused that attempt, `refusal` names
|
|
12
|
+
* the refusal by its codes alone.
|
|
12
13
|
*/
|
|
13
14
|
export declare class StepExhausted extends Error {
|
|
14
15
|
readonly step: string;
|
|
@@ -17,21 +18,13 @@ export declare class StepExhausted extends Error {
|
|
|
17
18
|
}
|
|
18
19
|
/** The Kernel refusal behind `cause`, itself or one of its causes. */
|
|
19
20
|
export declare function kernelRefusalOf(cause: unknown): KernelRefusal | undefined;
|
|
20
|
-
/**
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
*/
|
|
24
|
-
export declare function attemptFailure(cause: unknown): unknown;
|
|
25
|
-
/** A Step attempt's failure that asks the engine to wait `delayMs` before the next attempt. */
|
|
21
|
+
/** The message of a failed attempt: what it threw, after the Kernel refusal's codes, if any. */
|
|
22
|
+
export declare function attemptMessage(cause: unknown, refusal?: KernelRefusal | undefined): string;
|
|
23
|
+
/** A Step attempt's failure that asks to wait `delayMs` before the next attempt. */
|
|
26
24
|
export declare class StepRetryAfter extends Error {
|
|
27
25
|
readonly delayMs: number;
|
|
28
26
|
constructor(delayMs: number, cause?: unknown);
|
|
29
27
|
}
|
|
30
|
-
/**
|
|
31
|
-
* The failure of a Step attempt that waits out a prescribed delay before the next attempt, instead
|
|
32
|
-
* of the Step's backoff: a `retryAfter` failure, or a Kernel refusal that prescribes `retryAfterMs`.
|
|
33
|
-
*/
|
|
34
|
-
export declare function delayedRetry(engine: DurableEngine, cause: unknown): Error | undefined;
|
|
35
28
|
/**
|
|
36
29
|
* Encode a declared business error of `definition`, or a Step that spent its retry budget, as a
|
|
37
30
|
* terminal engine failure, or return undefined for any other error, which the engine retries.
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { declaredErrorFactory, declaredErrorOf } from '../../application/error/index.js';
|
|
2
|
-
import { ResponseError
|
|
2
|
+
import { ResponseError } from '../../platform/client/index.js';
|
|
3
3
|
/** Metadata key the Kernel reads a declared failure from (`backend/restate` outcome decoding). */
|
|
4
4
|
const FAILURE = 'astrale.failure';
|
|
5
5
|
/** Failure code of an Execution whose Workflow did not catch a Step that spent its retry budget. */
|
|
@@ -15,8 +15,9 @@ const REASON_CODE = /^[A-Z][A-Z0-9_]{0,63}$/u;
|
|
|
15
15
|
*/
|
|
16
16
|
const REFUSAL_MARK = /\[kernel refusal ([1-9][0-9]{3})(?: ([A-Z][A-Z0-9_]{0,63}))?\] /gu;
|
|
17
17
|
/**
|
|
18
|
-
* A Step that spent its retry budget
|
|
19
|
-
* the Kernel refused that attempt, `refusal` names
|
|
18
|
+
* A Step that spent its retry budget, or whose attempt the Kernel refused for good. Its message
|
|
19
|
+
* never carries what the last attempt threw; when the Kernel refused that attempt, `refusal` names
|
|
20
|
+
* the refusal by its codes alone.
|
|
20
21
|
*/
|
|
21
22
|
export class StepExhausted extends Error {
|
|
22
23
|
step;
|
|
@@ -46,17 +47,8 @@ export function kernelRefusalOf(cause) {
|
|
|
46
47
|
}
|
|
47
48
|
return undefined;
|
|
48
49
|
}
|
|
49
|
-
/**
|
|
50
|
-
|
|
51
|
-
* attempt, an error whose message starts with the refusal's codes, so the exhaustion can name it.
|
|
52
|
-
*/
|
|
53
|
-
export function attemptFailure(cause) {
|
|
54
|
-
const refusal = kernelRefusalOf(cause);
|
|
55
|
-
if (refusal === undefined)
|
|
56
|
-
return cause;
|
|
57
|
-
return new Error(attemptMessage(cause, refusal), { cause });
|
|
58
|
-
}
|
|
59
|
-
function attemptMessage(cause, refusal = kernelRefusalOf(cause)) {
|
|
50
|
+
/** The message of a failed attempt: what it threw, after the Kernel refusal's codes, if any. */
|
|
51
|
+
export function attemptMessage(cause, refusal = kernelRefusalOf(cause)) {
|
|
60
52
|
const message = cause instanceof Error ? cause.message : String(cause);
|
|
61
53
|
if (refusal === undefined)
|
|
62
54
|
return message;
|
|
@@ -74,7 +66,7 @@ function refusalIn(message) {
|
|
|
74
66
|
...(last[2] === undefined ? {} : { reason: last[2] }),
|
|
75
67
|
});
|
|
76
68
|
}
|
|
77
|
-
/** A Step attempt's failure that asks
|
|
69
|
+
/** A Step attempt's failure that asks to wait `delayMs` before the next attempt. */
|
|
78
70
|
export class StepRetryAfter extends Error {
|
|
79
71
|
delayMs;
|
|
80
72
|
constructor(delayMs, cause) {
|
|
@@ -86,16 +78,6 @@ export class StepRetryAfter extends Error {
|
|
|
86
78
|
this.delayMs = delayMs;
|
|
87
79
|
}
|
|
88
80
|
}
|
|
89
|
-
/**
|
|
90
|
-
* The failure of a Step attempt that waits out a prescribed delay before the next attempt, instead
|
|
91
|
-
* of the Step's backoff: a `retryAfter` failure, or a Kernel refusal that prescribes `retryAfterMs`.
|
|
92
|
-
*/
|
|
93
|
-
export function delayedRetry(engine, cause) {
|
|
94
|
-
const delayMs = cause instanceof StepRetryAfter ? cause.delayMs : retryAfterOf(cause);
|
|
95
|
-
if (delayMs === undefined)
|
|
96
|
-
return undefined;
|
|
97
|
-
return new engine.RetryableError(attemptMessage(cause), { retryAfter: delayMs, cause });
|
|
98
|
-
}
|
|
99
81
|
const STATUS = {
|
|
100
82
|
VALIDATION_ERROR: 422,
|
|
101
83
|
NOT_FOUND: 404,
|
|
@@ -147,7 +129,8 @@ export function replayedFailure(engine, cause, definition, step) {
|
|
|
147
129
|
return undefined;
|
|
148
130
|
}
|
|
149
131
|
const encoded = cause.metadata?.[FAILURE];
|
|
150
|
-
// The Step's own code turns every terminal failure into a declared one
|
|
132
|
+
// The Step's own code turns every terminal failure into a declared one or a failed attempt; a
|
|
133
|
+
// bare one is an engine's budget.
|
|
151
134
|
if (encoded === undefined)
|
|
152
135
|
return new StepExhausted(step, refusalIn(cause.message));
|
|
153
136
|
const failure = JSON.parse(encoded);
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
export type { DurableEngine, EnginePromise, EngineRetryPolicy, EngineCombinators,
|
|
1
|
+
export type { DurableEngine, EnginePromise, EngineRetryPolicy, EngineCombinators, EngineSharedContext, EngineWorkflowContext, EngineWorkflowDefinition, } from './engine.js';
|
|
2
2
|
export { DURABLE_ENGINE_EXPORTS, durableServing, isDurableEngine, type AdmittedDurableServing, type DurableServing, type DurableServingProvider, } from './options.js';
|
|
3
3
|
export { createDurableReceiver, type DurableReceiver } from './receiver.js';
|
|
4
4
|
export { isDurableRequest, isCanonicalDurableRequest } from './route.js';
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { Fetch } from '@astrale-os/kernel-client';
|
|
2
2
|
import type { DurableStepRetry } from '../../application/workflow/index.js';
|
|
3
|
-
import type { DurableEngine, EngineRetryPolicy
|
|
3
|
+
import type { DurableEngine, EngineRetryPolicy } from './engine.js';
|
|
4
4
|
/** Serving configuration of the durable Workflows of one deployment. */
|
|
5
5
|
export interface DurableServing {
|
|
6
6
|
/** The Restate SDK Fetch build of this platform. */
|
|
@@ -14,9 +14,11 @@ export interface DurableServing {
|
|
|
14
14
|
*/
|
|
15
15
|
readonly retentionMs?: number;
|
|
16
16
|
/**
|
|
17
|
-
* Retry budget of every Step that declares none:
|
|
18
|
-
*
|
|
19
|
-
*
|
|
17
|
+
* Retry budget of every Step that declares none: 10 attempts from 1 s to 60 s by default, and no
|
|
18
|
+
* duration limit. Only a Step's own failures spend its attempts; it waits out an unavailable Kernel
|
|
19
|
+
* within its `maxDurationMs`. A Step that spends it fails into its Workflow. `maxAttempts` also
|
|
20
|
+
* bounds the attempts of a run that fails outside any Step, an unreachable deployment included,
|
|
21
|
+
* which then pauses for `resume` or `cancel`; 20 when it declares none. The Host caps them.
|
|
20
22
|
*/
|
|
21
23
|
readonly retry?: DurableStepRetry;
|
|
22
24
|
/** Deadline of one Step attempt that declares none, including its Kernel calls. One minute by default. */
|
|
@@ -33,11 +35,24 @@ export interface AdmittedDurableServing {
|
|
|
33
35
|
/** Retry budget and attempt deadline of a Step that declares none. */
|
|
34
36
|
readonly step: StepServing;
|
|
35
37
|
}
|
|
36
|
-
/** What a Step is held to: its retry budget
|
|
38
|
+
/** What a Step is held to: its retry budget and its attempt deadline. */
|
|
37
39
|
export interface StepServing {
|
|
38
|
-
readonly retry:
|
|
40
|
+
readonly retry: StepRetryBudget;
|
|
39
41
|
readonly timeoutMs: number;
|
|
40
42
|
}
|
|
43
|
+
/**
|
|
44
|
+
* The retry budget of one Step, which the Step spends attempt by attempt (see `attempts.ts`). An
|
|
45
|
+
* absent limit does not bound it.
|
|
46
|
+
*/
|
|
47
|
+
export interface StepRetryBudget {
|
|
48
|
+
/** Attempts whose failure is the Step's own, the first one included. */
|
|
49
|
+
readonly maxAttempts?: number;
|
|
50
|
+
/** Time from the start of the Step's first attempt after which a failed attempt is not retried. */
|
|
51
|
+
readonly maxDurationMs?: number;
|
|
52
|
+
readonly initialIntervalMs: number;
|
|
53
|
+
readonly maxIntervalMs: number;
|
|
54
|
+
readonly intervalFactor: number;
|
|
55
|
+
}
|
|
41
56
|
/**
|
|
42
57
|
* The members of the Restate SDK Fetch entrypoint the durable receiver uses, which a platform's
|
|
43
58
|
* engine module must export.
|
|
@@ -29,7 +29,17 @@ export function isDurableEngine(module) {
|
|
|
29
29
|
}
|
|
30
30
|
const HOUR_MS = 60 * 60_000;
|
|
31
31
|
const DAY_MS = 24 * HOUR_MS;
|
|
32
|
-
|
|
32
|
+
/**
|
|
33
|
+
* Attempts a Step's own failures may spend when the deployment declares none. An unavailable Kernel
|
|
34
|
+
* spends none of them, so they bound only failures that sending again may not cure.
|
|
35
|
+
*/
|
|
36
|
+
const DEFAULT_STEP_ATTEMPTS = 10;
|
|
37
|
+
/**
|
|
38
|
+
* Attempts of a run failing outside any Step when the deployment declares none. The engine counts an
|
|
39
|
+
* unreachable deployment as such a failure, which it cannot tell apart from the code's own, so these
|
|
40
|
+
* also span an unavailability of the deployment, about fourteen minutes at the default intervals.
|
|
41
|
+
*/
|
|
42
|
+
const DEFAULT_RUN_ATTEMPTS = 20;
|
|
33
43
|
/** The longest attempt a Step may declare, which the engine's abort timeout leaves to end first. */
|
|
34
44
|
const MAXIMUM_STEP_TIMEOUT_MS = 10 * 60_000;
|
|
35
45
|
/**
|
|
@@ -39,10 +49,10 @@ const MAXIMUM_STEP_TIMEOUT_MS = 10 * 60_000;
|
|
|
39
49
|
*/
|
|
40
50
|
export const ENGINE_ABORT_TIMEOUT_MS = MAXIMUM_STEP_TIMEOUT_MS + 5 * 60_000;
|
|
41
51
|
const DEFAULT_STEP_RETRY = Object.freeze({
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
52
|
+
maxAttempts: DEFAULT_STEP_ATTEMPTS,
|
|
53
|
+
initialIntervalMs: 1_000,
|
|
54
|
+
maxIntervalMs: 60_000,
|
|
55
|
+
intervalFactor: 2,
|
|
46
56
|
});
|
|
47
57
|
/** Capture one durable serving configuration, failing closed on any gap. */
|
|
48
58
|
export function durableServing(input) {
|
|
@@ -79,11 +89,11 @@ export function durableServing(input) {
|
|
|
79
89
|
fetch,
|
|
80
90
|
retentionMs: bounded(input.retentionMs, 7 * DAY_MS, HOUR_MS, 365 * DAY_MS, 'retentionMs'),
|
|
81
91
|
retryPolicy: Object.freeze({
|
|
82
|
-
maxAttempts: input.retry?.maxAttempts ??
|
|
92
|
+
maxAttempts: input.retry?.maxAttempts ?? DEFAULT_RUN_ATTEMPTS,
|
|
83
93
|
onMaxAttempts: 'pause',
|
|
84
|
-
initialInterval: step.retry.
|
|
85
|
-
maxInterval: step.retry.
|
|
86
|
-
exponentiationFactor: step.retry.
|
|
94
|
+
initialInterval: step.retry.initialIntervalMs,
|
|
95
|
+
maxInterval: step.retry.maxIntervalMs,
|
|
96
|
+
exponentiationFactor: step.retry.intervalFactor,
|
|
87
97
|
}),
|
|
88
98
|
step,
|
|
89
99
|
});
|
|
@@ -112,19 +122,19 @@ export function stepServing(declared, fallback) {
|
|
|
112
122
|
const limited = retry.maxAttempts !== undefined || retry.maxDurationMs !== undefined;
|
|
113
123
|
const maxAttempts = limited
|
|
114
124
|
? optionalBounded(retry.maxAttempts, 1, 1_000, 'maxAttempts')
|
|
115
|
-
: fallback.retry.
|
|
125
|
+
: fallback.retry.maxAttempts;
|
|
116
126
|
const maxDuration = limited
|
|
117
127
|
? optionalBounded(retry.maxDurationMs, 1, 365 * DAY_MS, 'maxDurationMs')
|
|
118
|
-
: fallback.retry.
|
|
119
|
-
const initialInterval = bounded(retry.initialIntervalMs, fallback.retry.
|
|
120
|
-
const maxInterval = bounded(retry.maxIntervalMs, Math.max(fallback.retry.
|
|
128
|
+
: fallback.retry.maxDurationMs;
|
|
129
|
+
const initialInterval = bounded(retry.initialIntervalMs, fallback.retry.initialIntervalMs, 1, HOUR_MS, 'initialIntervalMs');
|
|
130
|
+
const maxInterval = bounded(retry.maxIntervalMs, Math.max(fallback.retry.maxIntervalMs, initialInterval), initialInterval, DAY_MS, 'maxIntervalMs');
|
|
121
131
|
return Object.freeze({
|
|
122
132
|
retry: Object.freeze({
|
|
123
|
-
...(maxAttempts === undefined ? {} : {
|
|
124
|
-
...(maxDuration === undefined ? {} : {
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
133
|
+
...(maxAttempts === undefined ? {} : { maxAttempts }),
|
|
134
|
+
...(maxDuration === undefined ? {} : { maxDurationMs: maxDuration }),
|
|
135
|
+
initialIntervalMs: initialInterval,
|
|
136
|
+
maxIntervalMs: maxInterval,
|
|
137
|
+
intervalFactor: fallback.retry.intervalFactor,
|
|
128
138
|
}),
|
|
129
139
|
timeoutMs: stepTimeout(declared.timeoutMs, fallback.timeoutMs),
|
|
130
140
|
});
|