@telorun/sdk 0.77.0 → 0.79.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cancellation.d.ts +50 -1
- package/dist/cancellation.d.ts.map +1 -1
- package/dist/contract-errors.d.ts +8 -1
- package/dist/contract-errors.d.ts.map +1 -1
- package/dist/contract-errors.js +8 -0
- package/dist/durable-run.d.ts +310 -0
- package/dist/durable-run.d.ts.map +1 -0
- package/dist/durable-run.js +223 -0
- package/dist/durable-suspension.d.ts +143 -0
- package/dist/durable-suspension.d.ts.map +1 -0
- package/dist/durable-suspension.js +153 -0
- package/dist/durable-target-encoding.d.ts +49 -0
- package/dist/durable-target-encoding.d.ts.map +1 -0
- package/dist/durable-target-encoding.js +121 -0
- package/dist/duration.d.ts +1 -1
- package/dist/duration.js +5 -5
- package/dist/evaluation-context.d.ts +16 -0
- package/dist/evaluation-context.d.ts.map +1 -1
- package/dist/index.d.ts +5 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +5 -0
- package/dist/invoke-step.d.ts +86 -1
- package/dist/invoke-step.d.ts.map +1 -1
- package/dist/invoke-step.js +261 -18
- package/dist/resource-context.d.ts +23 -0
- package/dist/resource-context.d.ts.map +1 -1
- package/dist/resource-instance.d.ts +21 -1
- package/dist/resource-instance.d.ts.map +1 -1
- package/dist/resource-instance.js +6 -2
- package/dist/step-engine.d.ts +170 -0
- package/dist/step-engine.d.ts.map +1 -0
- package/dist/step-engine.js +365 -0
- package/dist/zone-attribute.d.ts +101 -0
- package/dist/zone-attribute.d.ts.map +1 -0
- package/dist/zone-attribute.js +130 -0
- package/dist/zone-attributes/entries/atomic.json +7 -0
- package/dist/zone-attributes/entries/idempotent.json +6 -0
- package/dist/zone-attributes/entries/index.d.ts +3 -0
- package/dist/zone-attributes/entries/index.d.ts.map +1 -0
- package/dist/zone-attributes/entries/index.js +13 -0
- package/dist/zone-attributes/entries/no-suspend.json +6 -0
- package/dist/zone-attributes/entries/replayed.json +6 -0
- package/package.json +1 -1
- package/src/cancellation.ts +50 -1
- package/src/contract-errors.ts +9 -0
- package/src/durable-run.ts +450 -0
- package/src/durable-suspension.ts +188 -0
- package/src/durable-target-encoding.ts +181 -0
- package/src/duration.ts +5 -5
- package/src/evaluation-context.ts +17 -0
- package/src/index.ts +5 -1
- package/src/invoke-step.ts +378 -24
- package/src/resource-context.ts +23 -0
- package/src/resource-instance.ts +32 -2
- package/src/step-engine.ts +627 -0
- package/src/zone-attribute.ts +208 -0
- package/src/zone-attributes/entries/atomic.json +7 -0
- package/src/zone-attributes/entries/idempotent.json +6 -0
- package/src/zone-attributes/entries/index.ts +14 -0
- package/src/zone-attributes/entries/no-suspend.json +6 -0
- package/src/zone-attributes/entries/replayed.json +6 -0
package/src/invoke-step.ts
CHANGED
|
@@ -3,9 +3,26 @@ import {
|
|
|
3
3
|
type CancellationToken,
|
|
4
4
|
ERR_INVOKE_CANCELLED,
|
|
5
5
|
type InvokeContext,
|
|
6
|
+
UNCANCELLABLE_CONTEXT,
|
|
7
|
+
createCancellationSource,
|
|
8
|
+
deriveContext,
|
|
6
9
|
isCancellationError,
|
|
7
10
|
} from "./cancellation.js";
|
|
8
11
|
import { isAmbientContractErrorCode } from "./contract-errors.js";
|
|
12
|
+
import {
|
|
13
|
+
durableHandleOf,
|
|
14
|
+
journalingSuppressed,
|
|
15
|
+
stepPath,
|
|
16
|
+
type DurableRunHandle,
|
|
17
|
+
type DurableTarget,
|
|
18
|
+
} from "./durable-run.js";
|
|
19
|
+
import {
|
|
20
|
+
SUSPENDING_BACKOFF_MS,
|
|
21
|
+
assertMaySuspend,
|
|
22
|
+
isSuspension,
|
|
23
|
+
parkRun,
|
|
24
|
+
} from "./durable-suspension.js";
|
|
25
|
+
import type { OpenZoneAttributes } from "./zone-attribute.js";
|
|
9
26
|
import { tryParseDurationMs } from "./duration.js";
|
|
10
27
|
import { InvokeError } from "./invoke-error.js";
|
|
11
28
|
import type { KindRef, ScopeContext } from "./ref.js";
|
|
@@ -40,6 +57,26 @@ export interface InvokeStepRetry {
|
|
|
40
57
|
jitter?: "none" | "full";
|
|
41
58
|
/** DEPRECATED duration string (`"250ms"`, `"1s"`) — read as `initialDelay`. */
|
|
42
59
|
delay?: string;
|
|
60
|
+
/**
|
|
61
|
+
* Error codes that end the loop immediately instead of consuming the budget.
|
|
62
|
+
*
|
|
63
|
+
* The leaf's built-in exclusions are the ones decidable WITHOUT judgement —
|
|
64
|
+
* cancellation, and the kernel's verdicts on the shape of the call. Whether a
|
|
65
|
+
* DOMAIN failure is worth re-attempting is not decidable here at all: an
|
|
66
|
+
* `ERR_PAYMENT_DECLINED` and an `ERR_UPSTREAM_TIMEOUT` are the same shape to
|
|
67
|
+
* this loop, and only the author knows that re-presenting a declined card
|
|
68
|
+
* changes nothing. Without this, every terminal domain failure is retried to
|
|
69
|
+
* exhaustion — which is not merely wasted time but, for a non-idempotent
|
|
70
|
+
* target, N extra attempts at a side effect.
|
|
71
|
+
*
|
|
72
|
+
* Named by CODE rather than by a predicate because a code is what crosses
|
|
73
|
+
* every boundary this has to survive: a manifest declares it, `catches:`
|
|
74
|
+
* already matches on it, and a relocating backend ships it as data. Both
|
|
75
|
+
* hosted engines express the same knob — Temporal's non-retryable error types,
|
|
76
|
+
* Restate's terminal-versus-retryable split — so a policy written here means
|
|
77
|
+
* the same thing wherever the step ends up executing.
|
|
78
|
+
*/
|
|
79
|
+
nonRetryable?: string[];
|
|
43
80
|
}
|
|
44
81
|
|
|
45
82
|
/**
|
|
@@ -53,6 +90,29 @@ export interface InvokeStep {
|
|
|
53
90
|
invoke: KindRef<Invocable> | Invocable;
|
|
54
91
|
inputs?: Record<string, unknown>;
|
|
55
92
|
retry?: InvokeStepRetry;
|
|
93
|
+
/**
|
|
94
|
+
* How long ONE attempt may take, in milliseconds. On elapse the dispatch is
|
|
95
|
+
* cancelled and the step fails `ERR_STEP_TIMEOUT`.
|
|
96
|
+
*
|
|
97
|
+
* Per attempt rather than for the whole loop, matching Temporal's
|
|
98
|
+
* start-to-close: a budget spanning the retries would make the last attempt's
|
|
99
|
+
* allowance depend on how slow the earlier ones were, so an author could not
|
|
100
|
+
* state what any single call is allowed to take. A whole-operation bound is a
|
|
101
|
+
* different thing and belongs to whatever owns the operation.
|
|
102
|
+
*
|
|
103
|
+
* It bounds the step rather than the target because the target does not know
|
|
104
|
+
* who is waiting: the same `Http.Request` is a 30-second batch call from one
|
|
105
|
+
* step and a 500ms call on a request path from another. It is also what a
|
|
106
|
+
* backend that chooses WHERE a step executes needs in order to choose —
|
|
107
|
+
* Temporal runs a short step as a local activity and a long one as an
|
|
108
|
+
* activity, which is not a decision a manifest should have to make.
|
|
109
|
+
*
|
|
110
|
+
* Enforced by cancellation, never by abandoning the call: the timeout source
|
|
111
|
+
* is linked to the caller's token and threaded into the dispatch, so a target
|
|
112
|
+
* that honours cancellation stops working rather than continuing unobserved
|
|
113
|
+
* past a deadline nobody is waiting on any more.
|
|
114
|
+
*/
|
|
115
|
+
timeout?: number;
|
|
56
116
|
}
|
|
57
117
|
|
|
58
118
|
/** An inline flat invoke step on an Application's `targets`. Same as an
|
|
@@ -87,16 +147,30 @@ export type BootTarget =
|
|
|
87
147
|
export interface InvokeStepContext {
|
|
88
148
|
expandValue(value: any, context: Record<string, any>): any;
|
|
89
149
|
invoke<TInputs>(kind: string, name: string, inputs: TInputs, options?: any): Promise<any>;
|
|
150
|
+
/** Open zones with what each declares about its contents — how the leaf and
|
|
151
|
+
* the engine read the collapse rule. Optional so the structural context stays
|
|
152
|
+
* satisfiable by a host with no zone machinery; ABSENT MEANS "no zone is
|
|
153
|
+
* open", which is the direction that journals MORE rather than less, and so
|
|
154
|
+
* the safe one. */
|
|
155
|
+
zoneAttributes?(ctx?: InvokeContext): readonly OpenZoneAttributes[];
|
|
90
156
|
invokeResolved<TInputs>(
|
|
91
157
|
kind: string,
|
|
92
158
|
name: string,
|
|
93
159
|
instance: ResourceInstance,
|
|
94
160
|
inputs: TInputs,
|
|
161
|
+
/** Seeds the dispatch's invocation context, replacing the ambient — how a
|
|
162
|
+
* step `timeout:` reaches the target it bounds. Optional so a leaf caller
|
|
163
|
+
* that never sets one is unchanged. */
|
|
164
|
+
ctx?: InvokeContext,
|
|
95
165
|
): Promise<any>;
|
|
96
166
|
/** Resolve a cross-module exported instance (`!ref Alias.name`) to its live instance.
|
|
97
167
|
* Optional — providers that pre-resolve cross-module refs before reaching the leaf
|
|
98
168
|
* (e.g. the boot-target runner) may omit it. */
|
|
99
169
|
resolveImportedInstance?(alias: string, name: string): ResourceInstance | undefined;
|
|
170
|
+
/** Resolve a module-level sibling by name. Optional for the same reason the
|
|
171
|
+
* one above is, and read here ONLY to recover a target's declaration site —
|
|
172
|
+
* the dispatch itself still goes by name through the chokepoint. */
|
|
173
|
+
resolveLocalInstance?(name: string): ResourceInstance | undefined;
|
|
100
174
|
}
|
|
101
175
|
|
|
102
176
|
/**
|
|
@@ -127,6 +201,16 @@ export interface InvokeStepState {
|
|
|
127
201
|
* invocation to forward.
|
|
128
202
|
*/
|
|
129
203
|
invokeCtx?: InvokeContext;
|
|
204
|
+
/**
|
|
205
|
+
* This step's journal key, when the composer tracks one — see
|
|
206
|
+
* {@link stepPath}. Absent for a caller that assembled a step in code, and for
|
|
207
|
+
* the boot runner, whose targets are not a durable body.
|
|
208
|
+
*
|
|
209
|
+
* Supplied by the composer rather than derived here because a path is a
|
|
210
|
+
* property of the enclosing STRUCTURE (which branch, which loop turn, which
|
|
211
|
+
* fan-out element), and the leaf sees one step.
|
|
212
|
+
*/
|
|
213
|
+
journalPath?: string;
|
|
130
214
|
}
|
|
131
215
|
|
|
132
216
|
/**
|
|
@@ -143,15 +227,148 @@ export async function executeInvokeStep(
|
|
|
143
227
|
const cel = { steps: state.steps, ...state.cel };
|
|
144
228
|
if (step.when !== undefined && !ctx.expandValue(step.when, cel)) return;
|
|
145
229
|
|
|
146
|
-
const
|
|
230
|
+
const rawHandle = durableHandleOf(state.invokeCtx);
|
|
231
|
+
// Inside a collapsed region the engine records NOTHING of its own: the region
|
|
232
|
+
// re-runs whole on resume, so per-step entries would describe work that is
|
|
233
|
+
// about to happen again. A resource inside it may still journal directly —
|
|
234
|
+
// which is what lets `Durable.Value` pin an impure evaluation there rather
|
|
235
|
+
// than be a prescription with nowhere to write.
|
|
236
|
+
const handle =
|
|
237
|
+
rawHandle && journalingSuppressed(ctx, state.invokeCtx, rawHandle) ? undefined : rawHandle;
|
|
238
|
+
const path = state.journalPath ?? step.name;
|
|
239
|
+
|
|
240
|
+
// The RESOLVED inputs are journaled, not re-derived. They are read from a CEL
|
|
241
|
+
// scope carrying live readings — `resources.<name>.status` is republished on
|
|
242
|
+
// every dispatch by design — so a fresh process can compute different
|
|
243
|
+
// arguments for the same step and hand them to a target the journal will then
|
|
244
|
+
// answer for, with no mismatch to detect.
|
|
245
|
+
const inputs = (handle
|
|
246
|
+
? await handle.decide(stepPath(path, "inputs"), "inputs", () =>
|
|
247
|
+
ctx.expandValue(step.inputs ?? {}, cel),
|
|
248
|
+
)
|
|
249
|
+
: ctx.expandValue(step.inputs ?? {}, cel)) as Record<string, unknown>;
|
|
250
|
+
|
|
147
251
|
const raw = step.invoke as unknown;
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
252
|
+
// The dispatch carries THIS step's path, so anything it reaches that runs a
|
|
253
|
+
// step body of its own hangs those paths under this one rather than starting
|
|
254
|
+
// over at the root. Set only inside a durable run: outside one there is no
|
|
255
|
+
// path to carry, and deriving a context would be a rebuild for nothing.
|
|
256
|
+
// Derived from the RAW handle, not the collapse-suppressed one: collapse
|
|
257
|
+
// suppresses the engine's own per-step entries, never the journal. A
|
|
258
|
+
// `Durable.Value` or a parking kind inside a collapsed region still records
|
|
259
|
+
// directly, and it keys off this path — so dropping it here would give every
|
|
260
|
+
// such resource in the region ONE key inherited from an enclosing level.
|
|
261
|
+
const stepCtx =
|
|
262
|
+
rawHandle && state.invokeCtx
|
|
263
|
+
? deriveContext(state.invokeCtx, { durablePath: path })
|
|
264
|
+
: state.invokeCtx;
|
|
265
|
+
const execute = () =>
|
|
266
|
+
withStepRetry(step, stepCtx, ctx, rawHandle, path, (attemptCtx) =>
|
|
267
|
+
withStepTimeout(step, attemptCtx, (dispatchCtx) =>
|
|
268
|
+
dispatch(raw, inputs, ctx, { ...state, invokeCtx: dispatchCtx }),
|
|
269
|
+
),
|
|
270
|
+
);
|
|
271
|
+
|
|
272
|
+
// Retry and the timeout sit INSIDE the handed-over effect, not around it: a
|
|
273
|
+
// re-attempt is part of performing this step once, so a backend that ships the
|
|
274
|
+
// step elsewhere ships its policy with it rather than re-attempting a remote
|
|
275
|
+
// dispatch it does not own. This is also what keeps the attempt loop's own
|
|
276
|
+
// rule intact — the journal records the OUTCOME of the step, and a step whose
|
|
277
|
+
// third attempt succeeded completed once.
|
|
278
|
+
const result = handle
|
|
279
|
+
? await handle.step(path, targetIdentityOf(raw, ctx, state), inputs, execute)
|
|
280
|
+
: await execute();
|
|
151
281
|
|
|
152
282
|
state.steps[step.name] = { result };
|
|
153
283
|
}
|
|
154
284
|
|
|
285
|
+
/**
|
|
286
|
+
* The target's DECLARATION-SITE identity, for a backend that may execute the
|
|
287
|
+
* step somewhere the instance does not exist.
|
|
288
|
+
*
|
|
289
|
+
* Derived from the `!ref` identity the kernel stamps at Phase-5 injection, which
|
|
290
|
+
* is the only declaration-site fact a resolved target carries — instance
|
|
291
|
+
* identity is process-local by construction. Undefined when the step dispatches
|
|
292
|
+
* something with no stamp (a truly anonymous instance), which a local backend
|
|
293
|
+
* handles by simply running `execute` and a relocating one must refuse rather
|
|
294
|
+
* than guess.
|
|
295
|
+
*/
|
|
296
|
+
function targetIdentityOf(
|
|
297
|
+
raw: unknown,
|
|
298
|
+
ctx: InvokeStepContext,
|
|
299
|
+
state: InvokeStepState,
|
|
300
|
+
): DurableTarget | undefined {
|
|
301
|
+
if (!raw || typeof raw !== "object") return undefined;
|
|
302
|
+
// A pre-injected instance carries the stamp already.
|
|
303
|
+
const behind = getRefIdentity(raw as object) ? undefined : instanceBehind(raw, ctx, state);
|
|
304
|
+
const stamped = getRefIdentity(raw as object) ?? (behind && getRefIdentity(behind.instance));
|
|
305
|
+
const ref = raw as Partial<KindRef>;
|
|
306
|
+
const kind = stamped?.kind ?? (typeof ref.kind === "string" ? ref.kind : undefined);
|
|
307
|
+
const name = stamped?.name ?? (typeof ref.name === "string" ? ref.name : undefined);
|
|
308
|
+
if (!kind || !name) return undefined;
|
|
309
|
+
return {
|
|
310
|
+
kind,
|
|
311
|
+
name,
|
|
312
|
+
// Whether the name resolved inside a `with:` scope, which is knowable HERE
|
|
313
|
+
// and only here — the resolution is what answers it. The tuple identifying
|
|
314
|
+
// which scope RUN is not derivable yet, so this is what stops such a target
|
|
315
|
+
// being encoded as the module-level resource that shares its name.
|
|
316
|
+
...(behind?.scoped ? { scoped: true as const } : {}),
|
|
317
|
+
// Carried through verbatim: the kernel derived it at the declaration site,
|
|
318
|
+
// which is the only place it is knowable, and a consumer that must name this
|
|
319
|
+
// target elsewhere has nothing else to name it with.
|
|
320
|
+
...(stamped?.origin?.module === undefined ? {} : { module: stamped.origin.module }),
|
|
321
|
+
...(stamped?.origin?.pointer === undefined ? {} : { pointer: stamped.origin.pointer }),
|
|
322
|
+
};
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
/**
|
|
326
|
+
* The live instance behind a step's `invoke:` REF, when this host can name one.
|
|
327
|
+
*
|
|
328
|
+
* A step's slot is not a Phase-5 injection site — it resolves at dispatch — so a
|
|
329
|
+
* step target arrives as a `{kind, name, alias?}` reference and carries no stamp
|
|
330
|
+
* of its own. Reading the instance's is what recovers the declaration site, and
|
|
331
|
+
* the resolution mirrors `dispatch`'s branches exactly so the identity describes
|
|
332
|
+
* the resource the step will actually reach.
|
|
333
|
+
*
|
|
334
|
+
* Reports whether the name resolved inside a `with:` SCOPE as well as what it
|
|
335
|
+
* resolved to, because that is a fact only the resolution knows and it changes
|
|
336
|
+
* what the identity means: a scoped name and a module-level one can be the same
|
|
337
|
+
* string and different resources.
|
|
338
|
+
*
|
|
339
|
+
* Best-effort by design: every resolver here is optional, and a host that
|
|
340
|
+
* supplies none yields an identity of kind and name alone — enough for a local
|
|
341
|
+
* backend, and refused by a relocating one rather than guessed at.
|
|
342
|
+
*/
|
|
343
|
+
function instanceBehind(
|
|
344
|
+
raw: object,
|
|
345
|
+
ctx: InvokeStepContext,
|
|
346
|
+
state: InvokeStepState,
|
|
347
|
+
): { instance: object; scoped: boolean } | undefined {
|
|
348
|
+
const ref = raw as Partial<KindRef>;
|
|
349
|
+
if (typeof ref.name !== "string") return undefined;
|
|
350
|
+
try {
|
|
351
|
+
if (ref.alias && ref.alias !== "Self") {
|
|
352
|
+
const imported = ctx.resolveImportedInstance?.(ref.alias, ref.name) as object | undefined;
|
|
353
|
+
return imported && { instance: imported, scoped: false };
|
|
354
|
+
}
|
|
355
|
+
if (state.scope) {
|
|
356
|
+
// Scope-local FIRST, enclosing module as the fallback — the order
|
|
357
|
+
// `ScopeContext.getInstance` and the CEL `resources` layering already use,
|
|
358
|
+
// so what this reports a name to mean is what the dispatch will reach.
|
|
359
|
+
const scoped = state.scope.getInstance(ref.name) as unknown as object | undefined;
|
|
360
|
+
if (scoped) return { instance: scoped, scoped: true };
|
|
361
|
+
}
|
|
362
|
+
const local = ctx.resolveLocalInstance?.(ref.name) as object | undefined;
|
|
363
|
+
return local && { instance: local, scoped: false };
|
|
364
|
+
} catch {
|
|
365
|
+
// A resolver that throws is answering "not here", and the dispatch below is
|
|
366
|
+
// where that becomes an error with a message about the dispatch. Recovering
|
|
367
|
+
// provenance must not be the thing that reports it.
|
|
368
|
+
return undefined;
|
|
369
|
+
}
|
|
370
|
+
}
|
|
371
|
+
|
|
155
372
|
/**
|
|
156
373
|
* Re-attempt a step's dispatch while its policy allows.
|
|
157
374
|
*
|
|
@@ -185,45 +402,169 @@ export async function executeInvokeStep(
|
|
|
185
402
|
async function withStepRetry<T>(
|
|
186
403
|
step: InvokeStep,
|
|
187
404
|
invokeCtx: InvokeContext | undefined,
|
|
188
|
-
|
|
405
|
+
ctx: InvokeStepContext,
|
|
406
|
+
handle: DurableRunHandle | undefined,
|
|
407
|
+
path: string,
|
|
408
|
+
dispatch: (ctx: InvokeContext | undefined) => Promise<T>,
|
|
189
409
|
): Promise<T> {
|
|
190
410
|
const policy = step.retry;
|
|
191
411
|
const attempts = policy?.attempts ?? 0;
|
|
192
|
-
if (!policy || attempts <= 0) return dispatch();
|
|
412
|
+
if (!policy || attempts <= 0) return dispatch(invokeCtx);
|
|
193
413
|
|
|
194
|
-
const initial = policy.initialDelay ?? parseDuration(policy.delay) ?? 250;
|
|
195
|
-
const factor = policy.factor ?? 2;
|
|
196
|
-
const maxDelay = policy.maxDelay ?? 32_000;
|
|
197
414
|
const jitter = policy.jitter ?? "full";
|
|
198
415
|
|
|
199
416
|
for (let resend = 0; ; resend++) {
|
|
200
417
|
try {
|
|
201
|
-
return await dispatch();
|
|
418
|
+
return await dispatch(invokeCtx);
|
|
202
419
|
} catch (err) {
|
|
203
|
-
if (resend >= attempts || !isRetryable(err)) throw err;
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
420
|
+
if (resend >= attempts || !isRetryable(err, policy.nonRetryable)) throw err;
|
|
421
|
+
// The UN-JITTERED backoff, which is a pure function of the declared policy
|
|
422
|
+
// and the attempt index — and which is therefore what the park decision
|
|
423
|
+
// below turns on. Deciding on the jittered value instead would put
|
|
424
|
+
// `Math.random()` on a control-flow branch inside a determinism contract:
|
|
425
|
+
// the same attempt could sleep on one pass and park on the next, for no
|
|
426
|
+
// reason a reader of the manifest could see. It would also make the static
|
|
427
|
+
// check unstateable, since the analyzer cannot know which way a coin
|
|
428
|
+
// landed. Jitter still does its whole job — spreading re-attempts — on the
|
|
429
|
+
// duration itself, in both branches.
|
|
430
|
+
const backoff = retryBackoffMs(policy, resend);
|
|
431
|
+
const delay = jitter === "full" ? Math.random() * backoff : backoff;
|
|
432
|
+
|
|
433
|
+
// A LONG backoff inside a durable run suspends rather than sleeps, and the
|
|
434
|
+
// attempt state is journaled with it. The obvious reading — only the
|
|
435
|
+
// outcome matters, so journal once — is wrong the moment a backoff
|
|
436
|
+
// suspends: a run that parks mid-retry and resumes in another process must
|
|
437
|
+
// know which attempt it was on, or it restarts the policy from zero and a
|
|
438
|
+
// three-attempt cap becomes unbounded.
|
|
439
|
+
//
|
|
440
|
+
// ONE DECISION PER ATTEMPT, holding when that attempt was due, is the
|
|
441
|
+
// whole mechanism. It needs nothing beyond `decide`: on resume the loop
|
|
442
|
+
// re-runs from attempt zero, each recorded attempt hands back a due time
|
|
443
|
+
// already in the past and is therefore consumed without waiting, and the
|
|
444
|
+
// first UNRECORDED attempt computes a fresh one and parks. The budget is
|
|
445
|
+
// preserved because the replayed attempts still count against it.
|
|
446
|
+
if (handle && backoff >= SUSPENDING_BACKOFF_MS) {
|
|
447
|
+
const attemptPath = stepPath(path, "retry", resend);
|
|
448
|
+
const dueAt = (await handle.decide(attemptPath, "value", () => Date.now() + delay)) as number;
|
|
449
|
+
if (Date.now() < dueAt) {
|
|
450
|
+
// Refused inside a region that promised nothing in it suspends. The
|
|
451
|
+
// check is here rather than at the policy, because a short backoff
|
|
452
|
+
// never suspends and rejecting one would forbid a retry that is
|
|
453
|
+
// perfectly safe in a lease.
|
|
454
|
+
assertMaySuspend(ctx, invokeCtx, { resource: step.name ?? path });
|
|
455
|
+
await parkRun(handle, { path: attemptPath, resource: step.name ?? path }, { at: dueAt });
|
|
456
|
+
}
|
|
457
|
+
continue;
|
|
458
|
+
}
|
|
459
|
+
|
|
460
|
+
await waitBeforeResend(delay, invokeCtx?.cancellation, step, err);
|
|
211
461
|
}
|
|
212
462
|
}
|
|
213
463
|
}
|
|
214
464
|
|
|
465
|
+
/**
|
|
466
|
+
* The backoff before attempt `resend + 1`, before jitter.
|
|
467
|
+
*
|
|
468
|
+
* ONE formula, exported because the analyzer needs the same number: it decides
|
|
469
|
+
* statically whether a step's declared policy would park inside a region that
|
|
470
|
+
* cannot be held open, and a second copy of exponential-backoff arithmetic in
|
|
471
|
+
* the analyzer would be a rule that drifts from the behaviour it describes the
|
|
472
|
+
* first time either side gains a knob.
|
|
473
|
+
*
|
|
474
|
+
* Jitter is deliberately NOT applied here. It belongs to the duration, not to
|
|
475
|
+
* the shape of the policy, and the two consumers want the shape: the runtime
|
|
476
|
+
* branches on it (see `withStepRetry`) and the analyzer reports on it.
|
|
477
|
+
*
|
|
478
|
+
* The `??` fallbacks are the floor for a caller that assembled a policy in code;
|
|
479
|
+
* a manifest gets its defaults from the schema.
|
|
480
|
+
*/
|
|
481
|
+
export function retryBackoffMs(policy: InvokeStepRetry, resend: number): number {
|
|
482
|
+
const initial = policy.initialDelay ?? parseDuration(policy.delay) ?? 250;
|
|
483
|
+
const factor = policy.factor ?? 2;
|
|
484
|
+
const maxDelay = policy.maxDelay ?? 32_000;
|
|
485
|
+
return Math.min(maxDelay, initial * Math.pow(factor, resend));
|
|
486
|
+
}
|
|
487
|
+
|
|
215
488
|
/** Kernel verdicts on the CALL rather than on the work, beyond the ambient
|
|
216
489
|
* contract set. A dispatch that cannot resolve its target is a manifest defect;
|
|
217
490
|
* re-issuing it re-resolves the same name against the same registry. */
|
|
218
491
|
const UNRETRYABLE_CODES = new Set(["ERR_RESOURCE_NOT_FOUND", "ERR_RESOURCE_NOT_INVOKABLE"]);
|
|
219
492
|
|
|
220
|
-
function isRetryable(err: unknown): boolean {
|
|
493
|
+
function isRetryable(err: unknown, nonRetryable?: string[]): boolean {
|
|
221
494
|
if (isCancellationError(err)) return false;
|
|
495
|
+
// A suspension is not a failure — it is the run leaving. Re-attempting it
|
|
496
|
+
// would park again under the same policy until the budget ran out, and the
|
|
497
|
+
// last attempt would propagate a park the earlier ones had already recorded.
|
|
498
|
+
if (isSuspension(err)) return false;
|
|
222
499
|
const code = (err as { code?: unknown } | null | undefined)?.code;
|
|
223
500
|
if (typeof code !== "string") return true;
|
|
501
|
+
// The author's own exclusions, checked beside the built-in ones rather than
|
|
502
|
+
// before or after them: they are the same question — is re-issuing this call
|
|
503
|
+
// capable of a different answer — asked about a domain failure the leaf has no
|
|
504
|
+
// way to classify. A step timeout is never in this set by default; whether a
|
|
505
|
+
// slow call is worth re-attempting is exactly the judgement an author makes.
|
|
506
|
+
if (nonRetryable?.includes(code)) return false;
|
|
224
507
|
return !isAmbientContractErrorCode(code) && !UNRETRYABLE_CODES.has(code);
|
|
225
508
|
}
|
|
226
509
|
|
|
510
|
+
/**
|
|
511
|
+
* Bound ONE attempt, by cancellation rather than by abandonment.
|
|
512
|
+
*
|
|
513
|
+
* A `Promise.race` that simply rejects would leave the call running, holding its
|
|
514
|
+
* connection and eventually completing a side effect nobody is waiting on — the
|
|
515
|
+
* failure mode a timeout is usually adopted to prevent. So a timeout mints a
|
|
516
|
+
* cancellation source, LINKS it to the caller's token (or the caller's
|
|
517
|
+
* cancellation would stop propagating the moment a step declared a bound), and
|
|
518
|
+
* threads its context into the dispatch. A target that honours cancellation
|
|
519
|
+
* stops; one that does not is no worse off than before.
|
|
520
|
+
*
|
|
521
|
+
* The elapse is reported as `ERR_STEP_TIMEOUT` rather than as a cancellation,
|
|
522
|
+
* because the two want opposite follow-ups: a cancelled run was asked to stop,
|
|
523
|
+
* while a timed-out step is a target that is too slow for this call site — and
|
|
524
|
+
* `catches:` can only tell them apart if they carry different codes.
|
|
525
|
+
*/
|
|
526
|
+
async function withStepTimeout<T>(
|
|
527
|
+
step: InvokeStep,
|
|
528
|
+
invokeCtx: InvokeContext | undefined,
|
|
529
|
+
dispatch: (ctx: InvokeContext | undefined) => Promise<T>,
|
|
530
|
+
): Promise<T> {
|
|
531
|
+
const ms = step.timeout;
|
|
532
|
+
if (ms === undefined || !(ms > 0)) return dispatch(invokeCtx);
|
|
533
|
+
|
|
534
|
+
const source = createCancellationSource();
|
|
535
|
+
const base = invokeCtx ?? UNCANCELLABLE_CONTEXT;
|
|
536
|
+
// Everything else on the context — zones, tracing, and whatever a later
|
|
537
|
+
// member adds — rides across unchanged. Rebuilding it as a literal here is
|
|
538
|
+
// the drop `deriveContext` exists to prevent.
|
|
539
|
+
const scoped = deriveContext(base, { cancellation: source.token });
|
|
540
|
+
const unlink = invokeCtx?.cancellation.onCancelled((reason) => source.cancel(reason));
|
|
541
|
+
|
|
542
|
+
let elapsed = false;
|
|
543
|
+
source.cancelAfter(ms);
|
|
544
|
+
const timedOut = source.token.onCancelled(() => {
|
|
545
|
+
if (!invokeCtx?.cancellation.isCancelled) elapsed = true;
|
|
546
|
+
});
|
|
547
|
+
|
|
548
|
+
try {
|
|
549
|
+
return await dispatch(scoped);
|
|
550
|
+
} catch (err) {
|
|
551
|
+
if (elapsed && isCancellationError(err)) {
|
|
552
|
+
throw new InvokeError(
|
|
553
|
+
"ERR_STEP_TIMEOUT",
|
|
554
|
+
`Step '${step.name}' exceeded its timeout of ${ms}ms and was cancelled`,
|
|
555
|
+
{ step: step.name, timeout: ms },
|
|
556
|
+
);
|
|
557
|
+
}
|
|
558
|
+
throw err;
|
|
559
|
+
} finally {
|
|
560
|
+
timedOut();
|
|
561
|
+
unlink?.();
|
|
562
|
+
// Releases the pending deadline timer, so a step that finished early does
|
|
563
|
+
// not pin a timer alive until its bound elapses.
|
|
564
|
+
source.dispose();
|
|
565
|
+
}
|
|
566
|
+
}
|
|
567
|
+
|
|
227
568
|
/**
|
|
228
569
|
* Wait out the backoff, or give up the moment the invocation is cancelled.
|
|
229
570
|
*
|
|
@@ -299,6 +640,13 @@ async function dispatch(
|
|
|
299
640
|
): Promise<unknown> {
|
|
300
641
|
let result: unknown;
|
|
301
642
|
|
|
643
|
+
// The context this attempt runs under: the step's timeout scope when it
|
|
644
|
+
// declares one, else whatever the composer forwarded. Threaded explicitly
|
|
645
|
+
// rather than installed as ambient because the SDK leaf has no ambient store
|
|
646
|
+
// to install into — that is one runtime's mechanism, and a second-language
|
|
647
|
+
// leaf has no `AsyncLocalStorage`.
|
|
648
|
+
const attemptCtx = state.invokeCtx;
|
|
649
|
+
|
|
302
650
|
if (raw && typeof (raw as Invocable).invoke === "function") {
|
|
303
651
|
// A pre-injected live instance (a `!ref` resolved at Phase 5). Route it
|
|
304
652
|
// through the traced chokepoint using the identity the kernel stamped at
|
|
@@ -306,8 +654,14 @@ async function dispatch(
|
|
|
306
654
|
// A truly anonymous instance (no stamp) falls back to a direct call.
|
|
307
655
|
const identity = getRefIdentity(raw as object);
|
|
308
656
|
result = identity
|
|
309
|
-
? await ctx.invokeResolved(
|
|
310
|
-
|
|
657
|
+
? await ctx.invokeResolved(
|
|
658
|
+
identity.kind,
|
|
659
|
+
identity.name,
|
|
660
|
+
raw as ResourceInstance,
|
|
661
|
+
inputs,
|
|
662
|
+
attemptCtx,
|
|
663
|
+
)
|
|
664
|
+
: await (raw as Invocable).invoke(inputs, attemptCtx);
|
|
311
665
|
} else {
|
|
312
666
|
const ref = raw as KindRef<Invocable>;
|
|
313
667
|
if (ref.alias && ref.alias !== "Self") {
|
|
@@ -320,12 +674,12 @@ async function dispatch(
|
|
|
320
674
|
`Cross-module reference '${ref.alias}.${ref.name}' did not resolve to an exported instance.`,
|
|
321
675
|
);
|
|
322
676
|
}
|
|
323
|
-
result = await ctx.invokeResolved(ref.kind, ref.name, instance, inputs);
|
|
677
|
+
result = await ctx.invokeResolved(ref.kind, ref.name, instance, inputs, attemptCtx);
|
|
324
678
|
} else if (state.scope) {
|
|
325
679
|
const instance = state.scope.getInstance(ref.name) as unknown as ResourceInstance;
|
|
326
|
-
result = await ctx.invokeResolved(ref.kind, ref.name, instance, inputs);
|
|
680
|
+
result = await ctx.invokeResolved(ref.kind, ref.name, instance, inputs, attemptCtx);
|
|
327
681
|
} else {
|
|
328
|
-
result = await ctx.invoke(ref.kind, ref.name, inputs);
|
|
682
|
+
result = await ctx.invoke(ref.kind, ref.name, inputs, { ctx: attemptCtx });
|
|
329
683
|
}
|
|
330
684
|
}
|
|
331
685
|
return result;
|
package/src/resource-context.ts
CHANGED
|
@@ -17,6 +17,7 @@ import { ResourceInstance } from "./resource-instance.js";
|
|
|
17
17
|
import { ResourceManifest } from "./resource-manifest.js";
|
|
18
18
|
import { RuntimeResource } from "./runtime-resource.js";
|
|
19
19
|
import type { RuntimeSeam } from "./runtime-seam.js";
|
|
20
|
+
import type { OpenZoneAttributes } from "./zone-attribute.js";
|
|
20
21
|
|
|
21
22
|
export interface LoadOptions {
|
|
22
23
|
/** When true, `${{ }}` templates are replaced with CompiledValue wrappers
|
|
@@ -150,6 +151,28 @@ export interface ResourceContext extends ControllerContext {
|
|
|
150
151
|
* one). No kind parameter: the provider's own per-instance map already
|
|
151
152
|
* discriminates — a zone this instance's owner did not open simply misses. */
|
|
152
153
|
zonesFor(instance: ResourceInstance, ctx?: InvokeContext): readonly ZoneEntry[];
|
|
154
|
+
/**
|
|
155
|
+
* Every open zone with what it DECLARES about its contents, innermost first —
|
|
156
|
+
* the runtime half of `x-telo-provides-zone`'s attributes.
|
|
157
|
+
*
|
|
158
|
+
* Read off the declaring kind's schema, never off the entry: a
|
|
159
|
+
* {@link ZoneEntry} is three identities *because* that keeps it
|
|
160
|
+
* ABI-serializable and stops any module reading another's private state off
|
|
161
|
+
* the stack, and hanging attributes on it would trade that away for every
|
|
162
|
+
* zone. The kernel resolves the schema instead — the one place that lookup is
|
|
163
|
+
* already available — and hands the attributes over WITHOUT branching on a
|
|
164
|
+
* name, exactly as `readRefSlot` returns `use` without acting on it.
|
|
165
|
+
*
|
|
166
|
+
* The vocabulary is closed (`sdk/zone-attributes/`), which is what lets this
|
|
167
|
+
* be a typed record rather than a string-keyed bag. That is a readability
|
|
168
|
+
* gain and not a semantic one: interpreting an attribute is entirely the
|
|
169
|
+
* caller's — the step engine reads `atomic` to decide collapse, a parking kind
|
|
170
|
+
* reads `noSuspend` to refuse, and the kernel reads neither.
|
|
171
|
+
*
|
|
172
|
+
* Each value is the author's REASON, so a controller refusing on an attribute
|
|
173
|
+
* quotes the manifest's own sentence instead of inventing a generic message.
|
|
174
|
+
*/
|
|
175
|
+
zoneAttributes(ctx?: InvokeContext): readonly OpenZoneAttributes[];
|
|
153
176
|
/** The root context for runtime-driven inbound work (request, timer, queue
|
|
154
177
|
* message): inherits nothing from whatever ambient happens to be live at the
|
|
155
178
|
* registration site — no zones, no trace parent, no caller token. An inbound
|
package/src/resource-instance.ts
CHANGED
|
@@ -35,6 +35,27 @@ export const TEARDOWN_LAST = 1000;
|
|
|
35
35
|
export interface RefIdentity {
|
|
36
36
|
kind: string;
|
|
37
37
|
name: string;
|
|
38
|
+
/**
|
|
39
|
+
* Where the instance was DECLARED, for a consumer that must name it somewhere
|
|
40
|
+
* the instance does not exist — a durable step shipped to another process.
|
|
41
|
+
*
|
|
42
|
+
* Optional because it is derived from the declaration and an instance stamped
|
|
43
|
+
* by a path that has none is still dispatchable; a consumer that genuinely
|
|
44
|
+
* needs it refuses rather than guessing (see `encodeDurableTarget`). Kept ON
|
|
45
|
+
* the identity rather than in a second table because it answers the same
|
|
46
|
+
* question the identity does — *which declaration is this* — one level more
|
|
47
|
+
* precisely.
|
|
48
|
+
*/
|
|
49
|
+
origin?: RefOrigin;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/** The declaration site behind a live instance. `module` is the source of the
|
|
53
|
+
* module that declared it, which is what tells two libraries' same-named
|
|
54
|
+
* resources apart; `pointer` is set when the declaration is inline, and is a
|
|
55
|
+
* JSON pointer into its declaring resource. */
|
|
56
|
+
export interface RefOrigin {
|
|
57
|
+
module?: string;
|
|
58
|
+
pointer?: string;
|
|
38
59
|
}
|
|
39
60
|
|
|
40
61
|
/**
|
|
@@ -69,10 +90,19 @@ export const REF_IDENTITY: unique symbol = Symbol.for("telo.refIdentity");
|
|
|
69
90
|
|
|
70
91
|
/** Stamp the resolved kind+name onto an injected instance. Idempotent — an
|
|
71
92
|
* instance has exactly one identity, so re-injection into other slots is a no-op. */
|
|
72
|
-
export function stampRefIdentity(
|
|
93
|
+
export function stampRefIdentity(
|
|
94
|
+
instance: object,
|
|
95
|
+
kind: string,
|
|
96
|
+
name: string,
|
|
97
|
+
origin?: RefOrigin,
|
|
98
|
+
): void {
|
|
73
99
|
if (!(REF_IDENTITY in instance)) {
|
|
74
100
|
Object.defineProperty(instance, REF_IDENTITY, {
|
|
75
|
-
value: {
|
|
101
|
+
value: {
|
|
102
|
+
kind,
|
|
103
|
+
name,
|
|
104
|
+
...(origin && (origin.module || origin.pointer) ? { origin } : {}),
|
|
105
|
+
} satisfies RefIdentity,
|
|
76
106
|
enumerable: false,
|
|
77
107
|
configurable: true,
|
|
78
108
|
writable: false,
|