@swirls/looms 0.4.0 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (241) hide show
  1. package/CHANGELOG.md +55 -0
  2. package/dist/actor/cell.d.ts +21 -2
  3. package/dist/actor/cell.js +15 -3
  4. package/dist/actor/host.d.ts +10 -2
  5. package/dist/actor/host.js +27 -4
  6. package/dist/actor/index.d.ts +2 -1
  7. package/dist/actor/index.js +1 -1
  8. package/dist/actor/routing.d.ts +3 -0
  9. package/dist/actor/routing.js +13 -2
  10. package/dist/agent/definitions-store.d.ts +1 -1
  11. package/dist/agent/definitions-store.js +1 -1
  12. package/dist/agent/definitions.d.ts +48 -0
  13. package/dist/agent/definitions.js +44 -0
  14. package/dist/agent/effects.d.ts +184 -0
  15. package/dist/agent/effects.js +172 -40
  16. package/dist/agent/events.d.ts +89 -0
  17. package/dist/agent/events.js +25 -0
  18. package/dist/agent/module.d.ts +412 -0
  19. package/dist/agent/module.js +5 -1
  20. package/dist/agent/scope.d.ts +88 -0
  21. package/dist/agent/threads.js +39 -3
  22. package/dist/agent/types.d.ts +4 -0
  23. package/dist/approval/effects.d.ts +40 -0
  24. package/dist/approval/module.d.ts +80 -0
  25. package/dist/client/client.d.ts +468 -119
  26. package/dist/client/client.js +214 -21
  27. package/dist/client/index.d.ts +1 -1
  28. package/dist/client/sse.d.ts +1 -1
  29. package/dist/client/sse.js +2 -2
  30. package/dist/cloudflare/durable-object.d.ts +66 -2
  31. package/dist/cloudflare/durable-object.js +96 -1
  32. package/dist/cloudflare/event-store.d.ts +7 -3
  33. package/dist/cloudflare/event-store.js +16 -3
  34. package/dist/cloudflare/index.d.ts +7 -1
  35. package/dist/cloudflare/index.js +7 -1
  36. package/dist/cloudflare/memory-queue.d.ts +18 -0
  37. package/dist/cloudflare/memory-queue.js +53 -0
  38. package/dist/cloudflare/queue-consumer.d.ts +52 -0
  39. package/dist/cloudflare/queue-consumer.js +102 -0
  40. package/dist/cloudflare/queue-message.d.ts +31 -0
  41. package/dist/cloudflare/queue-message.js +33 -0
  42. package/dist/cloudflare/queue-pull.d.ts +62 -0
  43. package/dist/cloudflare/queue-pull.js +130 -0
  44. package/dist/cloudflare/queue-worker.d.ts +39 -0
  45. package/dist/cloudflare/queue-worker.js +62 -0
  46. package/dist/cloudflare/r2-blob-store.d.ts +33 -0
  47. package/dist/cloudflare/r2-blob-store.js +60 -0
  48. package/dist/cloudflare/router.d.ts +4 -1
  49. package/dist/cloudflare/router.js +8 -3
  50. package/dist/cloudflare/websocket.d.ts +27 -0
  51. package/dist/cloudflare/websocket.js +215 -0
  52. package/dist/core/blob-offload.d.ts +21 -0
  53. package/dist/core/blob-offload.js +143 -0
  54. package/dist/core/blob-store.d.ts +45 -0
  55. package/dist/core/blob-store.js +164 -0
  56. package/dist/core/blobs.d.ts +37 -0
  57. package/dist/core/blobs.js +160 -0
  58. package/dist/core/definition-resolver.d.ts +40 -0
  59. package/dist/core/definition-resolver.js +129 -0
  60. package/dist/core/definition-store.d.ts +6 -0
  61. package/dist/core/definition-store.js +37 -1
  62. package/dist/core/effects.d.ts +86 -3
  63. package/dist/core/effects.js +99 -3
  64. package/dist/core/envelope.d.ts +9 -0
  65. package/dist/core/envelope.js +4 -0
  66. package/dist/core/event-codec.js +3 -0
  67. package/dist/core/fold.js +83 -1
  68. package/dist/core/ids.d.ts +2 -0
  69. package/dist/core/ids.js +4 -0
  70. package/dist/core/index.d.ts +8 -0
  71. package/dist/core/index.js +8 -0
  72. package/dist/core/launcher.d.ts +97 -0
  73. package/dist/core/launcher.js +67 -0
  74. package/dist/core/module-scope.d.ts +7 -3
  75. package/dist/core/module-scope.js +3 -0
  76. package/dist/core/module.d.ts +10 -0
  77. package/dist/core/projection.js +4 -0
  78. package/dist/core/protocol.d.ts +36 -1
  79. package/dist/core/protocol.js +35 -0
  80. package/dist/core/published.d.ts +2 -0
  81. package/dist/core/published.js +4 -1
  82. package/dist/core/replay.d.ts +96 -0
  83. package/dist/core/run-context.d.ts +33 -0
  84. package/dist/core/run-context.js +23 -0
  85. package/dist/core/sqlite-store.js +57 -5
  86. package/dist/core/state.d.ts +60 -0
  87. package/dist/core/state.js +19 -0
  88. package/dist/core/store.d.ts +6 -0
  89. package/dist/core/store.js +7 -0
  90. package/dist/core/subscription.d.ts +97 -0
  91. package/dist/core/subscription.js +224 -0
  92. package/dist/core/thread.d.ts +7 -0
  93. package/dist/core/thread.js +1 -0
  94. package/dist/core/trace.d.ts +21 -0
  95. package/dist/core/trace.js +45 -0
  96. package/dist/core/worker-callback.d.ts +25 -1
  97. package/dist/debugger/app/assets/{index-CgDzWJg7.css → index-B7GdT1UA.css} +1 -1
  98. package/dist/debugger/app/assets/index-C31SnCuN.js +22 -0
  99. package/dist/debugger/app/assets/routes-DPA3O5AF.js +47 -0
  100. package/dist/debugger/app/index.html +2 -2
  101. package/dist/debugger/components/event-inspector.d.ts +3 -1
  102. package/dist/debugger/components/event-inspector.js +25 -4
  103. package/dist/debugger/components/event-stream.d.ts +4 -1
  104. package/dist/debugger/components/event-stream.js +2 -2
  105. package/dist/debugger/components/run-tree.d.ts +5 -2
  106. package/dist/debugger/components/run-tree.js +10 -4
  107. package/dist/debugger/context.d.ts +8 -1
  108. package/dist/debugger/context.js +15 -3
  109. package/dist/debugger/events/protocol-summarize.js +18 -3
  110. package/dist/debugger/index.d.ts +1 -1
  111. package/dist/debugger/index.js +1 -1
  112. package/dist/debugger/shell/debugger-shell.js +1 -1
  113. package/dist/debugger/shell/events-pane.js +4 -3
  114. package/dist/debugger/styles.css +1 -0
  115. package/dist/evaluator/definitions-store.d.ts +3 -4
  116. package/dist/evaluator/definitions-store.js +3 -3
  117. package/dist/evaluator/definitions.d.ts +4 -1
  118. package/dist/evaluator/definitions.js +5 -2
  119. package/dist/evaluator/effects.d.ts +74 -33
  120. package/dist/evaluator/effects.js +6 -2
  121. package/dist/evaluator/module.d.ts +150 -66
  122. package/dist/evaluator/module.js +7 -3
  123. package/dist/evaluator/threads.d.ts +2 -0
  124. package/dist/evaluator/threads.js +4 -0
  125. package/dist/projectors/event-log.d.ts +87 -0
  126. package/dist/projectors/event-log.js +396 -0
  127. package/dist/projectors/index-model.d.ts +12 -0
  128. package/dist/projectors/index-model.js +12 -3
  129. package/dist/projectors/memory.js +7 -0
  130. package/dist/projectors/postgres.d.ts +1 -0
  131. package/dist/projectors/postgres.js +32 -15
  132. package/dist/projectors/schema.d.ts +5 -0
  133. package/dist/projectors/schema.js +26 -1
  134. package/dist/projectors/sqlite.js +47 -19
  135. package/dist/projectors/with-projectors.d.ts +7 -1
  136. package/dist/projectors/with-projectors.js +13 -3
  137. package/dist/react/materialize.js +5 -1
  138. package/dist/react/react.d.ts +10 -2
  139. package/dist/react/react.js +43 -19
  140. package/dist/react/store.d.ts +11 -2
  141. package/dist/react/store.js +104 -12
  142. package/dist/react/tables.d.ts +4 -1
  143. package/dist/runtime/cancellation.d.ts +2 -1
  144. package/dist/runtime/cancellation.js +35 -2
  145. package/dist/runtime/child-run.d.ts +10 -0
  146. package/dist/runtime/child-run.js +58 -0
  147. package/dist/runtime/deadline-transition-planner.js +11 -2
  148. package/dist/runtime/dispatch-effect.d.ts +2 -2
  149. package/dist/runtime/dispatch-effect.js +43 -19
  150. package/dist/runtime/errors.d.ts +14 -1
  151. package/dist/runtime/errors.js +10 -0
  152. package/dist/runtime/filesystem-blob-store.d.ts +6 -0
  153. package/dist/runtime/filesystem-blob-store.js +59 -0
  154. package/dist/runtime/helpers.d.ts +15 -6
  155. package/dist/runtime/helpers.js +46 -6
  156. package/dist/runtime/index.d.ts +4 -2
  157. package/dist/runtime/index.js +3 -1
  158. package/dist/runtime/local-effect-executor.d.ts +11 -2
  159. package/dist/runtime/local-effect-executor.js +80 -4
  160. package/dist/runtime/looms.d.ts +38 -4
  161. package/dist/runtime/looms.js +17 -3
  162. package/dist/runtime/repository.d.ts +5 -4
  163. package/dist/runtime/repository.js +5 -2
  164. package/dist/runtime/retry-policy.d.ts +5 -0
  165. package/dist/runtime/retry-policy.js +10 -2
  166. package/dist/runtime/run-start.d.ts +1 -1
  167. package/dist/runtime/run-start.js +10 -0
  168. package/dist/runtime/runtime.d.ts +2 -2
  169. package/dist/runtime/runtime.js +191 -68
  170. package/dist/runtime/server/auth.d.ts +3 -0
  171. package/dist/runtime/server/fetch-handler.d.ts +2 -2
  172. package/dist/runtime/server/fetch-handler.js +35 -4
  173. package/dist/runtime/server/grant.d.ts +49 -0
  174. package/dist/runtime/server/grant.js +177 -0
  175. package/dist/runtime/server/routes/effects.js +47 -5
  176. package/dist/runtime/server/routes/runs.js +6 -1
  177. package/dist/runtime/server/run-route.d.ts +18 -0
  178. package/dist/runtime/server/run-route.js +26 -0
  179. package/dist/runtime/server.d.ts +3 -1
  180. package/dist/runtime/server.js +3 -1
  181. package/dist/runtime/sse.js +26 -4
  182. package/dist/runtime/subscription.d.ts +8 -0
  183. package/dist/runtime/subscription.js +23 -0
  184. package/dist/runtime/types.d.ts +87 -8
  185. package/dist/runtime/wake-engine.d.ts +4 -2
  186. package/dist/runtime/wake-engine.js +57 -6
  187. package/dist/runtime/worker-callbacks.d.ts +12 -2
  188. package/dist/runtime/worker-callbacks.js +102 -72
  189. package/dist/runtime/worker-effect-dispatch.d.ts +14 -5
  190. package/dist/runtime/worker-effect-dispatch.js +84 -18
  191. package/dist/schedule/debugger/index.d.ts +4 -0
  192. package/dist/schedule/debugger/index.js +11 -0
  193. package/dist/schedule/debugger/schedule-view.d.ts +3 -0
  194. package/dist/schedule/debugger/schedule-view.js +68 -0
  195. package/dist/schedule/debugger/summarize.d.ts +2 -0
  196. package/dist/schedule/debugger/summarize.js +43 -0
  197. package/dist/schedule/definitions-store.d.ts +10 -0
  198. package/dist/schedule/definitions-store.js +8 -0
  199. package/dist/schedule/definitions.d.ts +57 -0
  200. package/dist/schedule/definitions.js +30 -0
  201. package/dist/schedule/effects.d.ts +756 -0
  202. package/dist/schedule/effects.js +244 -0
  203. package/dist/schedule/ensure.d.ts +55 -0
  204. package/dist/schedule/ensure.js +109 -0
  205. package/dist/schedule/events.d.ts +116 -0
  206. package/dist/schedule/events.js +49 -0
  207. package/dist/schedule/index.d.ts +11 -0
  208. package/dist/schedule/index.js +11 -0
  209. package/dist/schedule/module.d.ts +1474 -0
  210. package/dist/schedule/module.js +22 -0
  211. package/dist/schedule/projections.d.ts +55 -0
  212. package/dist/schedule/projections.js +88 -0
  213. package/dist/schedule/scope.d.ts +121 -0
  214. package/dist/schedule/scope.js +7 -0
  215. package/dist/schedule/signals.d.ts +25 -0
  216. package/dist/schedule/signals.js +33 -0
  217. package/dist/schedule/specs.d.ts +59 -0
  218. package/dist/schedule/specs.js +231 -0
  219. package/dist/schedule/threads.d.ts +25 -0
  220. package/dist/schedule/threads.js +259 -0
  221. package/dist/testing/corpus-module.d.ts +34 -0
  222. package/dist/testing/corpus-module.js +87 -0
  223. package/dist/testing/index.d.ts +2 -0
  224. package/dist/testing/index.js +5 -6
  225. package/dist/testing/replay-corpus.d.ts +22 -0
  226. package/dist/testing/replay-corpus.js +50 -0
  227. package/dist/workflow/definitions-store.d.ts +1 -1
  228. package/dist/workflow/definitions-store.js +1 -1
  229. package/dist/workflow/definitions.d.ts +14 -0
  230. package/dist/workflow/effects.d.ts +122 -0
  231. package/dist/workflow/effects.js +16 -6
  232. package/dist/workflow/events.d.ts +42 -0
  233. package/dist/workflow/events.js +4 -0
  234. package/dist/workflow/module.d.ts +345 -32
  235. package/dist/workflow/module.js +7 -3
  236. package/dist/workflow/scope.d.ts +42 -0
  237. package/dist/workflow/threads.d.ts +72 -0
  238. package/dist/workflow/threads.js +106 -38
  239. package/package.json +10 -1
  240. package/dist/debugger/app/assets/index-D--9esm9.js +0 -22
  241. package/dist/debugger/app/assets/routes-Dakrdusf.js +0 -47
@@ -0,0 +1,58 @@
1
+ import { Effect } from 'effect';
2
+ import { descendantThreadIds, isTerminalStatus, } from '@swirls/looms/core';
3
+ /** Tell the launching run that this child has reached a terminal status. */
4
+ export function notifyLaunchingParent(launcher, runId, state) {
5
+ const launchedBy = state.startIdentity?.launchedBy;
6
+ const status = state.status;
7
+ if (!launchedBy || status === 'running') {
8
+ return Effect.void;
9
+ }
10
+ const root = state.rootThreadId ? state.threads[state.rootThreadId] : undefined;
11
+ const error = status === 'completed' ? null : (root?.error ?? status);
12
+ return launcher.signal(launchedBy.runId, [
13
+ {
14
+ type: 'runtime.child.settled',
15
+ payload: {
16
+ runId,
17
+ status,
18
+ output: root?.output ?? null,
19
+ error,
20
+ },
21
+ threadId: launchedBy.threadId,
22
+ idempotencyKey: `child-settled:${runId}`,
23
+ origin: { type: 'run', runId },
24
+ },
25
+ ], { idempotencyKey: `child-settled:${runId}` });
26
+ }
27
+ /**
28
+ * When a run asked to wait for child cancellation and those children are gone, cancel the
29
+ * run's own threads. The caller appends these events from the parent wake, which must not
30
+ * call `runtime.cancel` on itself.
31
+ */
32
+ export function childCloseCancellations(state) {
33
+ if (!state.closeWaiting || state.launchedRuns.some((run) => run.close === 'wait')) {
34
+ return [];
35
+ }
36
+ const rootId = state.rootThreadId;
37
+ if (!rootId) {
38
+ return [];
39
+ }
40
+ const root = state.threads[rootId];
41
+ if (!root || isTerminalStatus(root.status)) {
42
+ return [];
43
+ }
44
+ const ids = [...descendantThreadIds(state, rootId).toReversed(), rootId];
45
+ return ids.flatMap((threadId) => {
46
+ const record = state.threads[threadId];
47
+ if (!record || isTerminalStatus(record.status)) {
48
+ return [];
49
+ }
50
+ return [
51
+ {
52
+ type: 'runtime.thread.cancelled',
53
+ payload: { threadId, reason: 'cancelled' },
54
+ threadId,
55
+ },
56
+ ];
57
+ });
58
+ }
@@ -1,4 +1,4 @@
1
- import { createEvent, foldRun, isRunTerminal, isWaitOnTimer, } from '@swirls/looms/core';
1
+ import { createEvent, deferredUntil, foldRun, isRunTerminal, isWaitOnTimer, } from '@swirls/looms/core';
2
2
  import { waitSatisfiedEvents } from './helpers.js';
3
3
  import { planAttemptFailure } from './retry-policy.js';
4
4
  const EMPTY_PLAN = {
@@ -122,7 +122,16 @@ export function nextRuntimeDeadline(state, after = Number.NEGATIVE_INFINITY) {
122
122
  execution.deadlineAt > after
123
123
  ? [execution.deadlineAt]
124
124
  : []);
125
- const deadlines = [...timers, ...executions];
125
+ const deferred = terminal
126
+ ? []
127
+ : state.outstandingEffects.flatMap((item) => {
128
+ if (state.effectExecutions[item.effectId]) {
129
+ return [];
130
+ }
131
+ const at = deferredUntil(item.effect);
132
+ return at !== undefined && at > after ? [at] : [];
133
+ });
134
+ const deadlines = [...timers, ...executions, ...deferred];
126
135
  return deadlines.length > 0 ? Math.min(...deadlines) : undefined;
127
136
  }
128
137
  function cancellationAmbiguous(runId, effectId, attempt) {
@@ -1,4 +1,4 @@
1
1
  import type { Context, ManagedRuntime } from 'effect';
2
2
  import { Effect } from 'effect';
3
- import { type ComposedRegistry, type EffectContext, type EventInput, type RegisteredDefinition, type RuntimeEffect } from '@swirls/looms/core';
4
- export declare function dispatchEffect(registry: ComposedRegistry, services: ManagedRuntime.ManagedRuntime<Context.Service.Any, never>, definitions: ReadonlyMap<string, RegisteredDefinition>, effect: RuntimeEffect, ctx: EffectContext): Effect.Effect<ReadonlyArray<EventInput>, Error>;
3
+ import { type ComposedRegistry, type EffectContext, type EventInput, type DefinitionResolver, type RunLauncher, type RuntimeEffect } from '@swirls/looms/core';
4
+ export declare function dispatchEffect(registry: ComposedRegistry, services: ManagedRuntime.ManagedRuntime<Context.Service.Any, never>, definitions: DefinitionResolver, launcher: RunLauncher, effect: RuntimeEffect, ctx: EffectContext): Effect.Effect<ReadonlyArray<EventInput>, Error>;
@@ -1,7 +1,28 @@
1
- import { Effect } from 'effect';
2
- import { asJson, isPrimitiveEffect, isWaitOnTimer, } from '@swirls/looms/core';
1
+ import { Effect, Predicate } from 'effect';
2
+ import { asJson, isPrimitiveEffect, isWaitOnTimer, readEffectFailure, RunLauncherTag, } from '@swirls/looms/core';
3
3
  import { threadStartedEvents } from './helpers.js';
4
- export function dispatchEffect(registry, services, definitions, effect, ctx) {
4
+ function causeMessage(cause) {
5
+ if (Predicate.isString(cause)) {
6
+ return cause;
7
+ }
8
+ if (cause instanceof Error) {
9
+ return cause.message;
10
+ }
11
+ return String(cause);
12
+ }
13
+ function failedAttempt(effectId, threadId, cause) {
14
+ const failure = readEffectFailure(cause);
15
+ const error = failure?.error ?? causeMessage(cause);
16
+ const payload = { effectId, error };
17
+ if (failure?.nonRetryable) {
18
+ payload.nonRetryable = true;
19
+ }
20
+ if (failure?.retryAfterMs !== undefined) {
21
+ payload.retryAfterMs = failure.retryAfterMs;
22
+ }
23
+ return { type: 'runtime.effect.failed', payload, threadId };
24
+ }
25
+ export function dispatchEffect(registry, services, definitions, launcher, effect, ctx) {
5
26
  const currentThreadId = ctx.threadId;
6
27
  const baseDispatch = (eff, effCtx) => {
7
28
  if (isPrimitiveEffect(eff)) {
@@ -67,6 +88,20 @@ export function dispatchEffect(registry, services, definitions, effect, ctx) {
67
88
  },
68
89
  ]);
69
90
  }
91
+ case 'runtime.launch':
92
+ return launcher
93
+ .launch({
94
+ runId: eff.runId,
95
+ kind: eff.kind,
96
+ definitionName: eff.definitionName,
97
+ definitionVersion: eff.definitionVersion,
98
+ input: eff.input,
99
+ idempotencyKey: eff.runId,
100
+ close: eff.close,
101
+ })
102
+ .pipe(Effect.map(() => []));
103
+ case 'runtime.cancelRun':
104
+ return launcher.cancel(eff.runId).pipe(Effect.map(() => []));
70
105
  default: {
71
106
  const exhaustiveCheck = eff;
72
107
  return exhaustiveCheck;
@@ -76,28 +111,17 @@ export function dispatchEffect(registry, services, definitions, effect, ctx) {
76
111
  const handler = registry.effects.get(eff.type);
77
112
  if (!handler) {
78
113
  return Effect.succeed([
79
- {
80
- type: 'runtime.effect.failed',
81
- payload: { effectId: effCtx.effectId, error: `No handler for ${eff.type}` },
82
- threadId: currentThreadId,
83
- },
114
+ failedAttempt(effCtx.effectId, currentThreadId, `No handler for ${eff.type}`),
84
115
  ]);
85
116
  }
86
117
  const input = 'input' in eff ? eff.input : {};
87
118
  const execution = Effect.gen(function* () {
88
119
  const serviceContext = yield* services.contextEffect;
89
- return yield* handler.execute(input, effCtx).pipe(Effect.provide(serviceContext));
120
+ return yield* handler
121
+ .execute(input, effCtx)
122
+ .pipe(Effect.provideService(RunLauncherTag, launcher), Effect.provide(serviceContext));
90
123
  });
91
- return execution.pipe(Effect.catch((err) => Effect.succeed([
92
- {
93
- type: 'runtime.effect.failed',
94
- payload: {
95
- effectId: effCtx.effectId,
96
- error: err instanceof Error ? err.message : String(err),
97
- },
98
- threadId: currentThreadId,
99
- },
100
- ])));
124
+ return execution.pipe(Effect.catch((err) => Effect.succeed([failedAttempt(effCtx.effectId, currentThreadId, err)])));
101
125
  };
102
126
  if (registry.middleware && registry.middleware.length > 0) {
103
127
  const pipeline = registry.middleware.reduceRight((next, mw) => (e, c) => mw(e, c, next), baseDispatch);
@@ -1,4 +1,4 @@
1
- import type { JsonValue, RunStartIdentity as DurableRunStartIdentity } from '@swirls/looms/core';
1
+ import type { JsonValue, RunAttributes, RunLaunchSource, RunStartIdentity as DurableRunStartIdentity } from '@swirls/looms/core';
2
2
  export interface RunStartIdentity {
3
3
  readonly kind: string;
4
4
  readonly definitionName: string;
@@ -6,6 +6,19 @@ export interface RunStartIdentity {
6
6
  readonly input: JsonValue;
7
7
  readonly requestedThreadId: string | null;
8
8
  readonly idempotencyKey: string | null;
9
+ readonly launchedBy: RunLaunchSource | null;
10
+ readonly correlationId: string | null;
11
+ readonly attributes: RunAttributes;
12
+ }
13
+ declare const SecretContextConflictError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").VoidIfEmpty<{ readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }>) => import("effect/Cause").YieldableError & {
14
+ readonly _tag: "SecretContextConflictError";
15
+ } & Readonly<A>;
16
+ /** The stored secret context differs. The message never includes the secret values. */
17
+ export declare class SecretContextConflictError extends SecretContextConflictError_base<{
18
+ readonly runId: string;
19
+ readonly message: string;
20
+ }> {
21
+ constructor(runId: string);
9
22
  }
10
23
  declare const StartRunConflictError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").VoidIfEmpty<{ readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }>) => import("effect/Cause").YieldableError & {
11
24
  readonly _tag: "StartRunConflictError";
@@ -1,4 +1,14 @@
1
1
  import { Data } from 'effect';
2
+ /** The stored secret context differs. The message never includes the secret values. */
3
+ export class SecretContextConflictError extends Data.TaggedError('SecretContextConflictError') {
4
+ constructor(runId) {
5
+ super({
6
+ runId,
7
+ message: `Run "${runId}" already has a different secret context`,
8
+ });
9
+ this.name = 'SecretContextConflictError';
10
+ }
11
+ }
2
12
  export class StartRunConflictError extends Data.TaggedError('StartRunConflictError') {
3
13
  constructor(runId, requested, existing, mismatches) {
4
14
  super({
@@ -0,0 +1,6 @@
1
+ import { type BlobStore } from '@swirls/looms/core';
2
+ /**
3
+ * Content-addressed blob store rooted at `directory`. For Bun and Node hosts.
4
+ * A Durable Object should use `r2BlobStore` instead.
5
+ */
6
+ export declare function filesystemBlobStore(directory: string): BlobStore;
@@ -0,0 +1,59 @@
1
+ import { mkdir, readdir, readFile, rm, stat, writeFile } from 'node:fs/promises';
2
+ import { Data, Effect, Predicate } from 'effect';
3
+ import { blobStoreFromBytes, BlobStoreError } from '@swirls/looms/core';
4
+ class MissingBlobFile extends Data.TaggedError('MissingBlobFile') {
5
+ }
6
+ /**
7
+ * Content-addressed blob store rooted at `directory`. For Bun and Node hosts.
8
+ * A Durable Object should use `r2BlobStore` instead.
9
+ */
10
+ export function filesystemBlobStore(directory) {
11
+ return blobStoreFromBytes(filesystemBlobBytes(directory));
12
+ }
13
+ function filesystemBlobBytes(directory) {
14
+ return {
15
+ read: (key) => readOptional(`${directory}/${key}`, key),
16
+ write: (key, body) => Effect.tryPromise({
17
+ try: () => mkdir(`${directory}/${folderOf(key)}`, { recursive: true }),
18
+ catch: (cause) => new BlobStoreError(`Unable to create a directory for blob key ${key}`, cause),
19
+ }).pipe(Effect.andThen(Effect.tryPromise({
20
+ try: () => writeFile(`${directory}/${key}`, body),
21
+ catch: (cause) => new BlobStoreError(`Unable to write blob key ${key}`, cause),
22
+ }))),
23
+ remove: (key) => Effect.tryPromise({
24
+ try: () => rm(`${directory}/${key}`, { force: true }),
25
+ catch: (cause) => new BlobStoreError(`Unable to delete blob key ${key}`, cause),
26
+ }),
27
+ exists: (key) => Effect.tryPromise({
28
+ try: () => stat(`${directory}/${key}`),
29
+ catch: (cause) => isMissingFile(cause)
30
+ ? new MissingBlobFile()
31
+ : new BlobStoreError(`Unable to check blob key ${key}`, cause),
32
+ }).pipe(Effect.as(true), Effect.catchTag('MissingBlobFile', () => Effect.succeed(false))),
33
+ list: (prefix) => listFolder(directory, prefix),
34
+ };
35
+ }
36
+ function readOptional(path, key) {
37
+ return Effect.tryPromise({
38
+ try: () => readFile(path),
39
+ catch: (cause) => isMissingFile(cause)
40
+ ? new MissingBlobFile()
41
+ : new BlobStoreError(`Unable to read blob key ${key}`, cause),
42
+ }).pipe(Effect.catchTag('MissingBlobFile', () => Effect.succeed(null)), Effect.map((body) => (body === null ? null : new Uint8Array(body))));
43
+ }
44
+ function listFolder(directory, prefix) {
45
+ const folder = prefix.endsWith('/') ? prefix.slice(0, -1) : prefix;
46
+ return Effect.tryPromise({
47
+ try: () => readdir(`${directory}/${folder}`),
48
+ catch: (cause) => isMissingFile(cause)
49
+ ? new MissingBlobFile()
50
+ : new BlobStoreError(`Unable to list blob keys ${prefix}`, cause),
51
+ }).pipe(Effect.catchTag('MissingBlobFile', () => Effect.succeed([])), Effect.map((names) => names.map((name) => `${folder}/${name}`)));
52
+ }
53
+ function folderOf(key) {
54
+ const slash = key.lastIndexOf('/');
55
+ return slash < 0 ? '' : key.slice(0, slash);
56
+ }
57
+ function isMissingFile(cause) {
58
+ return Predicate.isObject(cause) && 'code' in cause && cause.code === 'ENOENT';
59
+ }
@@ -1,8 +1,17 @@
1
- import { Effect, Layer, type Context } from 'effect';
2
- import { type AnyRuntimeModule, type AppendableEvent, type EventCatalog, type EventEnvelope, type EventInput, type EventOrigin, type EventStore, InvalidEventError, type JsonValue, type OutstandingEffect, type RegisteredDefinition, type RunCursor, type RunState, type SnapshotStore } from '@swirls/looms/core';
1
+ import { Effect, Layer, type Context, type Types } from 'effect';
2
+ import { type AnyRuntimeModule, type AppendableEvent, type EventCatalog, type EventEnvelope, type EventInput, type EventOrigin, type EventStore, InvalidEventError, type DefinitionResolveError, type DefinitionResolver, type JsonValue, type OutstandingEffect, type RetryPolicy, type SecretContext, type TraceContext, type RunCursor, type RunState, type SnapshotStore } from '@swirls/looms/core';
3
3
  import type { EffectWorkerTask } from './types';
4
- export declare function workerTask(runId: string, item: OutstandingEffect, attempt: number): EffectWorkerTask;
5
- declare const UnknownDefinitionError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").VoidIfEmpty<{ readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }>) => import("effect/Cause").YieldableError & {
4
+ /** What a dispatch carries beyond the effect itself. */
5
+ export interface WorkerTaskExtras {
6
+ /** Progress from the previous attempt's last heartbeat. */
7
+ readonly details?: JsonValue | null;
8
+ readonly secrets?: SecretContext;
9
+ readonly trace?: TraceContext | null;
10
+ readonly retry?: RetryPolicy;
11
+ readonly scheduleToStartAt?: number | null;
12
+ }
13
+ export declare function workerTask(runId: string, item: OutstandingEffect, attempt: number, extras?: WorkerTaskExtras): EffectWorkerTask;
14
+ declare const UnknownDefinitionError_base: new <A extends Record<string, any> = {}>(args: Types.VoidIfEmpty<{ readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }>) => import("effect/Cause").YieldableError & {
6
15
  readonly _tag: "UnknownDefinitionError";
7
16
  } & Readonly<A>;
8
17
  export declare class UnknownDefinitionError extends UnknownDefinitionError_base<{
@@ -34,14 +43,14 @@ export declare function groupOutstanding(items: readonly {
34
43
  * A future third producer of thread.started must call this; do not emit the
35
44
  * event with raw input.
36
45
  */
37
- export declare function threadStartedEvents(definitions: ReadonlyMap<string, RegisteredDefinition>, args: {
46
+ export declare function threadStartedEvents(definitions: DefinitionResolver, args: {
38
47
  kind: string;
39
48
  definitionName: string;
40
49
  definitionVersion: string;
41
50
  input: JsonValue;
42
51
  threadId: string;
43
52
  parentThreadId: string | null;
44
- }): Effect.Effect<ReadonlyArray<EventInput>, UnknownDefinitionError>;
53
+ }): Effect.Effect<ReadonlyArray<EventInput>, UnknownDefinitionError | DefinitionResolveError>;
45
54
  export declare function synthesizedThreadFailed(before: RunState, after: RunState, written: readonly EventEnvelope[]): EventInput[];
46
55
  /** Cancel whatever is still live below each thread `written` cancels, deepest first. */
47
56
  export declare function cascadedCancellations(state: RunState, written: readonly EventEnvelope[]): EventInput[];
@@ -1,13 +1,40 @@
1
1
  import { Data, Effect, Layer, Predicate } from 'effect';
2
- import { asJson, createEvent, definitionKey, descendantThreadIds, isTerminalStatus, makeMemorySnapshotStore, matchingWaits, snapshotStoreOf, validateEventInput, validateEventsEffect, validateInputEffect, withSnapshotStore, InvalidEventError, } from '@swirls/looms/core';
3
- export function workerTask(runId, item, attempt) {
4
- return {
2
+ import { asJson, createEvent, definitionKey, descendantThreadIds, isTerminalStatus, makeMemorySnapshotStore, matchingWaits, snapshotStoreOf, validateEventInput, validateEventsEffect, validateInputEffect, withSnapshotStore, hasEntries, InvalidEventError, } from '@swirls/looms/core';
3
+ export function workerTask(runId, item, attempt, extras = {}) {
4
+ const task = {
5
5
  runId,
6
6
  effectId: item.effectId,
7
7
  attempt,
8
8
  type: item.effect.type,
9
9
  input: 'input' in item.effect ? item.effect.input : null,
10
10
  };
11
+ if (extras.details != null) {
12
+ task.details = extras.details;
13
+ }
14
+ if (extras.secrets && hasEntries(extras.secrets)) {
15
+ task.secrets = extras.secrets;
16
+ }
17
+ if (extras.trace) {
18
+ task.trace = extras.trace;
19
+ }
20
+ const deadlines = workerDeadlines(extras);
21
+ if (deadlines) {
22
+ task.deadlines = deadlines;
23
+ }
24
+ return task;
25
+ }
26
+ function workerDeadlines(extras) {
27
+ const deadlines = {};
28
+ if (extras.scheduleToStartAt != null) {
29
+ deadlines.scheduleToStartAt = extras.scheduleToStartAt;
30
+ }
31
+ if (extras.retry?.startToCloseTimeoutMs) {
32
+ deadlines.startToCloseTimeoutMs = extras.retry.startToCloseTimeoutMs;
33
+ }
34
+ if (extras.retry?.heartbeatTimeoutMs) {
35
+ deadlines.heartbeatTimeoutMs = extras.retry.heartbeatTimeoutMs;
36
+ }
37
+ return Object.keys(deadlines).length > 0 ? deadlines : undefined;
11
38
  }
12
39
  export class UnknownDefinitionError extends Data.TaggedError('UnknownDefinitionError') {
13
40
  constructor(kind, definitionName, definitionVersion) {
@@ -105,15 +132,16 @@ export function groupOutstanding(items) {
105
132
  */
106
133
  export function threadStartedEvents(definitions, args) {
107
134
  return Effect.gen(function* () {
108
- const def = definitions.get(definitionKey(args.kind, args.definitionName, args.definitionVersion));
135
+ const def = yield* definitions.resolve(args.kind, args.definitionName, args.definitionVersion);
109
136
  if (!def) {
110
137
  return yield* new UnknownDefinitionError(args.kind, args.definitionName, args.definitionVersion);
111
138
  }
112
139
  const raw = args.input ?? null;
113
140
  let startedInput = raw;
114
141
  let validationError;
115
- if (def?.input) {
116
- const validated = yield* validateInputEffect(def.input, raw).pipe(Effect.match({
142
+ const input = schemaInputOf(def);
143
+ if (input) {
144
+ const validated = yield* validateInputEffect(input, raw).pipe(Effect.match({
117
145
  onFailure: (error) => ({ ok: false, error: error.message }),
118
146
  onSuccess: (value) => ({ ok: true, value }),
119
147
  }));
@@ -150,6 +178,18 @@ export function threadStartedEvents(definitions, args) {
150
178
  ];
151
179
  });
152
180
  }
181
+ function schemaInputOf(definition) {
182
+ if (!('input' in definition)) {
183
+ return undefined;
184
+ }
185
+ return isSchemaInput(definition.input) ? definition.input : undefined;
186
+ }
187
+ function isSchemaInput(value) {
188
+ if (value === null || (!Predicate.isObject(value) && !Predicate.isFunction(value))) {
189
+ return false;
190
+ }
191
+ return '~standard' in value || 'ast' in value;
192
+ }
153
193
  export function synthesizedThreadFailed(before, after, written) {
154
194
  const out = [];
155
195
  for (const [threadId, thread] of Object.entries(after.threads)) {
@@ -1,5 +1,7 @@
1
- export { createRuntime, createTimeoutScheduler, DuplicateEffectDispatchError, ForkConflictError, MaxWakeIterationsError, RunNotFinishedError, RunNotFoundError, RunOverloadedError, RunTerminalError, StartRunConflictError, type CreateRuntimeOptions, type EffectWorker, type EffectWorkerTask, type LoomsRuntime, type RunOperationalStatus, type RegisteredDefinition, type StartRunArgs, type WakeError, type WakeScheduler, type WorkerCallback, type WorkerCallbackInput, } from './runtime';
1
+ export { createRuntime, createTimeoutScheduler, DuplicateEffectDispatchError, ForkConflictError, MaxWakeIterationsError, RunNotFinishedError, RunNotFoundError, RunOverloadedError, RunTerminalError, StartRunConflictError, type CreateRuntimeOptions, type EffectWorker, type EffectWorkerDeadlines, type EffectWorkerTask, type LoomsRuntime, type RunOperationalStatus, type RegisteredDefinition, type StartRunArgs, type WakeError, type WakeScheduler, type WorkerCallback, type WorkerCallbackInput, type WorkerCallbackResult, } from './runtime';
2
2
  export { createLooms, type Looms, type CreateLoomsOptions, type StartResult } from './looms';
3
- export { serveHttp, createFetchHandler, isLoomsApiPath, type FetchHandlerOptions, authAllowed, authDenied, bearerAuth, defaultAuthorize, type AuthDecision, type AuthorizedAccess, type Authorize, type BearerTokens, type RouteAccess, type RouteInfo, type RunningServer, type LoomsFetchResult, } from './server';
3
+ export { loadSubscriptionOpening, readSubscriptionBounds } from './subscription';
4
+ export { serveHttp, createFetchHandler, isLoomsApiPath, type FetchHandlerOptions, authAllowed, authDenied, authResponse, bearerAuth, defaultAuthorize, grantAuth, issueRunGrant, grantSubprotocol, runGrantToken, verifyRunGrant, GrantError, type GrantAuthOptions, type GrantFailure, type RunGrant, runRouteInfo, type AuthDecision, type AuthorizedAccess, type Authorize, type BearerTokens, type RouteAccess, type RouteInfo, type RunningServer, type LoomsFetchResult, } from './server';
5
+ export { filesystemBlobStore } from './filesystem-blob-store';
4
6
  export { createEventStreamResponse, type EventStreamOptions } from './sse';
5
7
  export { notifyObserver, type RuntimeObservation, type RuntimeObserver } from './observer';
@@ -1,5 +1,7 @@
1
1
  export { createRuntime, createTimeoutScheduler, DuplicateEffectDispatchError, ForkConflictError, MaxWakeIterationsError, RunNotFinishedError, RunNotFoundError, RunOverloadedError, RunTerminalError, StartRunConflictError, } from './runtime.js';
2
2
  export { createLooms } from './looms.js';
3
- export { serveHttp, createFetchHandler, isLoomsApiPath, authAllowed, authDenied, bearerAuth, defaultAuthorize, } from './server.js';
3
+ export { loadSubscriptionOpening, readSubscriptionBounds } from './subscription.js';
4
+ export { serveHttp, createFetchHandler, isLoomsApiPath, authAllowed, authDenied, authResponse, bearerAuth, defaultAuthorize, grantAuth, issueRunGrant, grantSubprotocol, runGrantToken, verifyRunGrant, GrantError, runRouteInfo, } from './server.js';
5
+ export { filesystemBlobStore } from './filesystem-blob-store.js';
4
6
  export { createEventStreamResponse } from './sse.js';
5
7
  export { notifyObserver } from './observer.js';
@@ -1,19 +1,28 @@
1
1
  import { Effect, type Context, type ManagedRuntime } from 'effect';
2
- import { type ComposedRegistry, type EventEnvelope, type OutstandingEffect, type RegisteredDefinition } from '@swirls/looms/core';
2
+ import { type ComposedRegistry, type EventEnvelope, type DefinitionResolver, type OutstandingEffect, type RunAttributes, type RunLauncher, type SecretContext, type TraceContext } from '@swirls/looms/core';
3
3
  import type { RuntimeObserver } from './observer';
4
4
  import type { RuntimeRepository } from './repository';
5
5
  import type { RunExecutionContext } from './run-execution-context';
6
6
  export interface LocalEffectExecutorOptions {
7
7
  readonly registry: ComposedRegistry;
8
8
  readonly services: ManagedRuntime.ManagedRuntime<Context.Service.Any, never>;
9
- readonly definitions: ReadonlyMap<string, RegisteredDefinition>;
9
+ readonly definitionResolver: DefinitionResolver;
10
10
  readonly executionContext: RunExecutionContext;
11
11
  readonly liveAppends: RuntimeRepository['liveAppends'];
12
12
  readonly runInContext: <A, E>(effect: Effect.Effect<A, E>) => Promise<A>;
13
13
  readonly observe: (event: Parameters<RuntimeObserver['observe']>[0]) => void;
14
+ readonly launcher: RunLauncher;
15
+ }
16
+ /** Run-level values every effect sees. Runs the effects launch inherit them unless the launch overrides them. */
17
+ export interface RunContext {
18
+ readonly correlationId: string;
19
+ readonly attributes: RunAttributes;
20
+ readonly secrets: SecretContext;
21
+ readonly trace: TraceContext | null;
14
22
  }
15
23
  export interface ExecuteLocalEffectsInput {
16
24
  readonly runId: string;
25
+ readonly context: RunContext;
17
26
  readonly outstanding: ReadonlyArray<OutstandingEffect>;
18
27
  readonly attempts: ReadonlyMap<string, number>;
19
28
  readonly now: number;
@@ -1,9 +1,64 @@
1
1
  import { Effect, Option, Predicate } from 'effect';
2
- import { createEvent, validateEventInput, withdrawnError, } from '@swirls/looms/core';
2
+ import { createEvent, createRunId, DEFAULT_DEFINITION_VERSION, validateEventInput, withdrawnError, } from '@swirls/looms/core';
3
3
  import { dispatchEffect } from './dispatch-effect.js';
4
4
  import { RuntimeExecutionError } from './errors.js';
5
5
  import { createEffectFailedEvent, createLiveEvent, groupOutstanding, materializeEffectOutcome, } from './helpers.js';
6
6
  import { planAttemptFailure } from './retry-policy.js';
7
+ function runLaunchedPayload(args, targetRunId, effectId) {
8
+ const definitionVersion = args.definitionVersion ?? DEFAULT_DEFINITION_VERSION;
9
+ const base = {
10
+ runId: targetRunId,
11
+ kind: args.kind,
12
+ definitionName: args.definitionName,
13
+ definitionVersion,
14
+ effectId,
15
+ };
16
+ if (args.close === undefined) {
17
+ return base;
18
+ }
19
+ return { ...base, close: args.close };
20
+ }
21
+ /**
22
+ * Stamps every launch with the effect that made it, replacing anything the handler passed, and
23
+ * records a `runtime.run.launched` event for each launch the target accepts.
24
+ */
25
+ function launcherFor(launcher, runId, context, item, launched) {
26
+ const launchedBy = {
27
+ runId,
28
+ threadId: item.threadId,
29
+ effectId: item.effectId,
30
+ causationId: item.causingEventId,
31
+ causationSeq: item.causingSeq,
32
+ };
33
+ return {
34
+ launch: (args) => {
35
+ const targetRunId = args.runId ?? createRunId();
36
+ return launcher
37
+ .launch({
38
+ ...args,
39
+ runId: targetRunId,
40
+ launchedBy,
41
+ correlationId: context.correlationId,
42
+ attributes: args.attributes ?? context.attributes,
43
+ secrets: args.secrets ?? context.secrets,
44
+ trace: args.trace ?? context.trace ?? undefined,
45
+ })
46
+ .pipe(Effect.tap(() => Effect.sync(() => {
47
+ launched.push(createEvent(runId, {
48
+ type: 'runtime.run.launched',
49
+ payload: runLaunchedPayload(args, targetRunId, item.effectId),
50
+ threadId: item.threadId,
51
+ causationId: item.causingEventId,
52
+ correlationId: context.correlationId,
53
+ origin: { type: 'thread', threadId: item.threadId },
54
+ }));
55
+ })));
56
+ },
57
+ status: launcher.status,
58
+ signal: (targetRunId, events, options) => launcher.signal(targetRunId, events, options),
59
+ cancel: (targetRunId) => launcher.cancel(targetRunId),
60
+ };
61
+ }
7
62
  function makeAbortController() {
8
63
  return new AbortController();
9
64
  }
@@ -20,7 +75,7 @@ function abortion(signal) {
20
75
  });
21
76
  }
22
77
  export function executeLocalEffects(options, input) {
23
- const { registry, services, definitions, executionContext, liveAppends, runInContext, observe } = options;
78
+ const { registry, services, definitionResolver, executionContext, liveAppends, runInContext, observe, launcher, } = options;
24
79
  return Effect.forEach(groupOutstanding(input.outstanding), (groupKey) => Effect.gen(function* () {
25
80
  const group = input.outstanding.filter((item) => `${item.threadId}:${item.causingSeq}` === groupKey);
26
81
  const produced = [];
@@ -43,11 +98,15 @@ export function executeLocalEffects(options, input) {
43
98
  threadId: item.threadId,
44
99
  });
45
100
  const retry = registry.effects.get(item.effect.type)?.retry;
46
- const dispatched = dispatchEffect(registry, services, definitions, item.effect, {
101
+ const launched = [];
102
+ const dispatched = dispatchEffect(registry, services, definitionResolver, launcherFor(launcher, input.runId, input.context, item, launched), item.effect, {
47
103
  effectId: item.effectId,
48
104
  runId: input.runId,
49
105
  threadId: item.threadId,
50
106
  causingEventId: item.causingEventId,
107
+ attributes: input.context.attributes,
108
+ secrets: input.context.secrets,
109
+ trace: input.context.trace,
51
110
  signal: controller.signal,
52
111
  emit: appendLive,
53
112
  }).pipe(Effect.map((outcomes) => ({ _tag: 'completed', outcomes })), Effect.raceFirst(abortion(controller.signal)));
@@ -56,6 +115,8 @@ export function executeLocalEffects(options, input) {
56
115
  : dispatched).pipe(Effect.ensuring(Effect.sync(() => {
57
116
  executionContext.activeEffects.delete(item.effectId);
58
117
  })));
118
+ // A launch the target accepted happened even if the attempt was later aborted or failed.
119
+ produced.push(...launched);
59
120
  if (settled._tag === 'aborted') {
60
121
  continue;
61
122
  }
@@ -94,7 +155,19 @@ export function executeLocalEffects(options, input) {
94
155
  continue;
95
156
  }
96
157
  const { outcomes } = settled;
97
- if (outcomes.length === 0) {
158
+ const sideEffect = item.effect.type === 'runtime.launch' || item.effect.type === 'runtime.cancelRun';
159
+ if (outcomes.length === 0 && sideEffect) {
160
+ const attempt = input.attempts.get(item.effectId) ?? 1;
161
+ produced.push(createEvent(input.runId, {
162
+ type: 'runtime.effect.completed',
163
+ payload: { effectId: item.effectId, attempt },
164
+ threadId: item.threadId,
165
+ effectId: item.effectId,
166
+ causationId: item.causingEventId,
167
+ origin: { type: 'system' },
168
+ }));
169
+ }
170
+ else if (outcomes.length === 0) {
98
171
  produced.push(createEffectFailedEvent(input.runId, item, 'empty-outcome'));
99
172
  }
100
173
  for (const outcome of outcomes) {
@@ -102,6 +175,7 @@ export function executeLocalEffects(options, input) {
102
175
  const attempt = input.attempts.get(item.effectId) ?? 1;
103
176
  const payload = Predicate.isObject(outcome.payload) ? outcome.payload : {};
104
177
  const error = Predicate.isString(payload.error) ? payload.error : 'effect failed';
178
+ const retryAfterMs = payload.retryAfterMs;
105
179
  const failure = planAttemptFailure({
106
180
  runId: input.runId,
107
181
  item,
@@ -110,6 +184,8 @@ export function executeLocalEffects(options, input) {
110
184
  now: input.now,
111
185
  retry,
112
186
  origin: { type: 'system' },
187
+ nonRetryable: payload.nonRetryable === true,
188
+ retryAfterMs: Predicate.isNumber(retryAfterMs) ? retryAfterMs : undefined,
113
189
  });
114
190
  if (failure.retrying) {
115
191
  observe({