deepline 0.1.227 → 0.1.229
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/apps/play-runner-workers/src/coordinator-entry.ts +76 -1842
- package/dist/bundling-sources/apps/play-runner-workers/src/dedup-do.ts +0 -113
- package/dist/bundling-sources/apps/play-runner-workers/src/entry.ts +144 -1000
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/ledger-event-batches.ts +81 -0
- package/dist/bundling-sources/apps/play-runner-workers/src/workflow-retry-state.ts +1 -50
- package/dist/bundling-sources/sdk/src/client.ts +24 -2
- package/dist/bundling-sources/sdk/src/play.ts +2 -2
- package/dist/bundling-sources/sdk/src/release.ts +2 -2
- package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +382 -31
- package/dist/bundling-sources/shared_libs/play-runtime/child-execution-strategy.ts +84 -79
- package/dist/bundling-sources/shared_libs/play-runtime/context.ts +201 -705
- package/dist/bundling-sources/shared_libs/play-runtime/coordinator-headers.ts +3 -10
- package/dist/bundling-sources/shared_libs/play-runtime/ctx-contract.ts +2 -2
- package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +12 -47
- package/dist/bundling-sources/shared_libs/play-runtime/governor/block-reserving-budget-state-backend.ts +0 -2
- package/dist/bundling-sources/shared_libs/play-runtime/governor/governor.ts +12 -200
- package/dist/bundling-sources/shared_libs/play-runtime/governor/policy.ts +2 -17
- package/dist/bundling-sources/shared_libs/play-runtime/live-state-contract.ts +3 -9
- package/dist/bundling-sources/shared_libs/play-runtime/play-call-execution.ts +6 -1
- package/dist/bundling-sources/shared_libs/play-runtime/play-latency-trace.ts +28 -0
- package/dist/bundling-sources/shared_libs/play-runtime/run-execution-scope.ts +16 -64
- package/dist/bundling-sources/shared_libs/play-runtime/run-ledger.ts +26 -0
- package/dist/bundling-sources/shared_libs/play-runtime/run-lifecycle-policy.ts +5 -2
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-api-paths.ts +0 -4
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +81 -18
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backend.ts +0 -36
- package/dist/bundling-sources/shared_libs/play-runtime/suspension.ts +3 -21
- package/dist/bundling-sources/shared_libs/play-runtime/test-runtime-seams.ts +9 -1
- package/dist/bundling-sources/shared_libs/play-runtime/transport-error-diagnostics.ts +63 -0
- package/dist/bundling-sources/shared_libs/plays/static-pipeline.ts +2 -0
- package/dist/cli/index.js +78 -11
- package/dist/cli/index.mjs +78 -11
- package/dist/{compiler-manifest-BhgZ23A4.d.ts → compiler-manifest-CYcwzSOJ.d.mts} +2 -0
- package/dist/{compiler-manifest-BhgZ23A4.d.mts → compiler-manifest-CYcwzSOJ.d.ts} +2 -0
- package/dist/index.d.mts +3 -3
- package/dist/index.d.ts +3 -3
- package/dist/index.js +14 -5
- package/dist/index.mjs +14 -5
- package/dist/plays/bundle-play-file.d.mts +2 -2
- package/dist/plays/bundle-play-file.d.ts +2 -2
- package/package.json +1 -1
- package/dist/bundling-sources/apps/play-runner-workers/src/child-manifest-resolver.ts +0 -108
- package/dist/bundling-sources/apps/play-runner-workers/src/child-play-await.ts +0 -374
- package/dist/bundling-sources/apps/play-runner-workers/src/child-play-submit.ts +0 -203
- package/dist/bundling-sources/shared_libs/play-runtime/child-execution-placement.ts +0 -14
- package/dist/bundling-sources/shared_libs/play-runtime/child-run-id.ts +0 -121
|
@@ -1,108 +0,0 @@
|
|
|
1
|
-
import type {
|
|
2
|
-
PlayRuntimeManifest,
|
|
3
|
-
PlayRuntimeManifestMap,
|
|
4
|
-
} from '../../../shared_libs/plays/compiler-manifest';
|
|
5
|
-
import { createSingleFlight } from '../../../shared_libs/play-runtime/single-flight';
|
|
6
|
-
|
|
7
|
-
type ChildManifestResolverInput = {
|
|
8
|
-
preloaded?: PlayRuntimeManifestMap | null;
|
|
9
|
-
resolveManifest: (playRef: string) => Promise<PlayRuntimeManifest | null>;
|
|
10
|
-
};
|
|
11
|
-
|
|
12
|
-
type ChildManifestResolutionState = {
|
|
13
|
-
resolved: Map<string, PlayRuntimeManifest>;
|
|
14
|
-
flights: ReturnType<
|
|
15
|
-
typeof createSingleFlight<string, PlayRuntimeManifest | null>
|
|
16
|
-
>;
|
|
17
|
-
lastAccessedAt: number;
|
|
18
|
-
};
|
|
19
|
-
|
|
20
|
-
const RUN_RESOLUTION_CACHE_TTL_MS = 10 * 60_000;
|
|
21
|
-
const RUN_RESOLUTION_CACHE_MAX_ENTRIES = 128;
|
|
22
|
-
const runResolutionStates = new Map<string, ChildManifestResolutionState>();
|
|
23
|
-
|
|
24
|
-
function seedPreloadedManifests(
|
|
25
|
-
resolved: Map<string, PlayRuntimeManifest>,
|
|
26
|
-
preloaded?: PlayRuntimeManifestMap | null,
|
|
27
|
-
): void {
|
|
28
|
-
for (const [playRef, manifest] of Object.entries(preloaded ?? {})) {
|
|
29
|
-
resolved.set(playRef, manifest);
|
|
30
|
-
}
|
|
31
|
-
}
|
|
32
|
-
|
|
33
|
-
function createResolutionState(
|
|
34
|
-
preloaded?: PlayRuntimeManifestMap | null,
|
|
35
|
-
): ChildManifestResolutionState {
|
|
36
|
-
const resolved = new Map<string, PlayRuntimeManifest>(
|
|
37
|
-
Object.entries(preloaded ?? {}),
|
|
38
|
-
);
|
|
39
|
-
return {
|
|
40
|
-
resolved,
|
|
41
|
-
flights: createSingleFlight<string, PlayRuntimeManifest | null>(),
|
|
42
|
-
lastAccessedAt: Date.now(),
|
|
43
|
-
};
|
|
44
|
-
}
|
|
45
|
-
|
|
46
|
-
function resolveWithState(
|
|
47
|
-
state: ChildManifestResolutionState,
|
|
48
|
-
input: ChildManifestResolverInput,
|
|
49
|
-
playRef: string,
|
|
50
|
-
): Promise<PlayRuntimeManifest | null> {
|
|
51
|
-
state.lastAccessedAt = Date.now();
|
|
52
|
-
seedPreloadedManifests(state.resolved, input.preloaded);
|
|
53
|
-
const cached = state.resolved.get(playRef);
|
|
54
|
-
if (cached) return Promise.resolve(cached);
|
|
55
|
-
|
|
56
|
-
return state.flights.run(playRef, async () => {
|
|
57
|
-
const cachedAfterAdmission = state.resolved.get(playRef);
|
|
58
|
-
if (cachedAfterAdmission) return cachedAfterAdmission;
|
|
59
|
-
|
|
60
|
-
const manifest = await input.resolveManifest(playRef);
|
|
61
|
-
if (manifest) state.resolved.set(playRef, manifest);
|
|
62
|
-
return manifest;
|
|
63
|
-
});
|
|
64
|
-
}
|
|
65
|
-
|
|
66
|
-
function pruneRunResolutionStates(now: number): void {
|
|
67
|
-
for (const [scopeKey, state] of runResolutionStates) {
|
|
68
|
-
if (
|
|
69
|
-
state.flights.activeCount() === 0 &&
|
|
70
|
-
now - state.lastAccessedAt >= RUN_RESOLUTION_CACHE_TTL_MS
|
|
71
|
-
) {
|
|
72
|
-
runResolutionStates.delete(scopeKey);
|
|
73
|
-
}
|
|
74
|
-
}
|
|
75
|
-
if (runResolutionStates.size <= RUN_RESOLUTION_CACHE_MAX_ENTRIES) return;
|
|
76
|
-
|
|
77
|
-
const evictable = [...runResolutionStates.entries()]
|
|
78
|
-
.filter(([, state]) => state.flights.activeCount() === 0)
|
|
79
|
-
.sort((a, b) => a[1].lastAccessedAt - b[1].lastAccessedAt);
|
|
80
|
-
for (const [scopeKey] of evictable) {
|
|
81
|
-
if (runResolutionStates.size <= RUN_RESOLUTION_CACHE_MAX_ENTRIES) break;
|
|
82
|
-
runResolutionStates.delete(scopeKey);
|
|
83
|
-
}
|
|
84
|
-
}
|
|
85
|
-
|
|
86
|
-
export function createChildManifestResolver(
|
|
87
|
-
input: ChildManifestResolverInput,
|
|
88
|
-
): (playRef: string) => Promise<PlayRuntimeManifest | null> {
|
|
89
|
-
const state = createResolutionState(input.preloaded);
|
|
90
|
-
|
|
91
|
-
return (playRef) => resolveWithState(state, input, playRef);
|
|
92
|
-
}
|
|
93
|
-
|
|
94
|
-
export function createRunScopedChildManifestResolver(
|
|
95
|
-
input: ChildManifestResolverInput & { scopeKey: string },
|
|
96
|
-
): (playRef: string) => Promise<PlayRuntimeManifest | null> {
|
|
97
|
-
const now = Date.now();
|
|
98
|
-
pruneRunResolutionStates(now);
|
|
99
|
-
let state = runResolutionStates.get(input.scopeKey);
|
|
100
|
-
if (!state) {
|
|
101
|
-
state = createResolutionState(input.preloaded);
|
|
102
|
-
runResolutionStates.set(input.scopeKey, state);
|
|
103
|
-
pruneRunResolutionStates(now);
|
|
104
|
-
}
|
|
105
|
-
state.lastAccessedAt = now;
|
|
106
|
-
|
|
107
|
-
return (playRef) => resolveWithState(state, input, playRef);
|
|
108
|
-
}
|
|
@@ -1,374 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Child-play terminal await — the race between a Cloudflare Workflow
|
|
3
|
-
* `waitForEvent('child_play_terminal:...')` and a coordinator terminal-state
|
|
4
|
-
* poll, extracted from entry.ts so the runtime keeps a small, testable surface
|
|
5
|
-
* for "block until this child run reaches a terminal state".
|
|
6
|
-
*
|
|
7
|
-
* Behavior is identical to the inline implementation it replaces: arm the
|
|
8
|
-
* workflow event wait, race it against `pollParentChildTerminalState`, and
|
|
9
|
-
* resolve on whichever reports the child's terminal status first. The poll
|
|
10
|
-
* never resolves on a non-terminal/absent state (it returns a never-settling
|
|
11
|
-
* promise on timeout) so the event wait remains the primary path and the poll
|
|
12
|
-
* is a coordinator-side safety net for missed signals.
|
|
13
|
-
*/
|
|
14
|
-
|
|
15
|
-
import { isRetryableWorkerRuntimeApiError } from './runtime-api-retry';
|
|
16
|
-
|
|
17
|
-
export type WorkflowStepLike = {
|
|
18
|
-
waitForEvent: (
|
|
19
|
-
name: string,
|
|
20
|
-
options: { type: string; timeout: string },
|
|
21
|
-
) => Promise<{ payload: unknown }>;
|
|
22
|
-
};
|
|
23
|
-
|
|
24
|
-
export type ChildTerminalCoordinator = {
|
|
25
|
-
readChildTerminalState?(
|
|
26
|
-
parentRunId: string,
|
|
27
|
-
eventKey: string,
|
|
28
|
-
timeoutMs?: number,
|
|
29
|
-
): Promise<{ data?: unknown } | null>;
|
|
30
|
-
readRunTerminalState(
|
|
31
|
-
runId: string,
|
|
32
|
-
timeoutMs?: number,
|
|
33
|
-
): Promise<ChildRunTerminalSnapshotState | null>;
|
|
34
|
-
};
|
|
35
|
-
|
|
36
|
-
export type ChildPlayTerminalWaitResult = {
|
|
37
|
-
output: unknown;
|
|
38
|
-
source: 'workflow_event' | 'parent_child_terminal_cache';
|
|
39
|
-
attempts?: number;
|
|
40
|
-
waitMs: number;
|
|
41
|
-
};
|
|
42
|
-
|
|
43
|
-
export type ChildRunTerminalSnapshotState = {
|
|
44
|
-
data?: unknown;
|
|
45
|
-
reason?: string;
|
|
46
|
-
status?: string;
|
|
47
|
-
runId?: string;
|
|
48
|
-
playName?: string;
|
|
49
|
-
parentRunId?: string | null;
|
|
50
|
-
};
|
|
51
|
-
|
|
52
|
-
export class ChildPlayTerminalVerificationError extends Error {
|
|
53
|
-
readonly parentRunId: string;
|
|
54
|
-
readonly eventKey: string;
|
|
55
|
-
readonly cause: unknown;
|
|
56
|
-
|
|
57
|
-
constructor(input: {
|
|
58
|
-
parentRunId: string;
|
|
59
|
-
eventKey: string;
|
|
60
|
-
cause: unknown;
|
|
61
|
-
}) {
|
|
62
|
-
const causeMessage =
|
|
63
|
-
input.cause instanceof Error
|
|
64
|
-
? input.cause.message
|
|
65
|
-
: String(input.cause ?? '');
|
|
66
|
-
super(
|
|
67
|
-
`Child play terminal verification failed for parent run ${input.parentRunId} and event ${input.eventKey}.${
|
|
68
|
-
causeMessage ? ` Cause: ${causeMessage}` : ''
|
|
69
|
-
}`,
|
|
70
|
-
);
|
|
71
|
-
this.name = 'ChildPlayTerminalVerificationError';
|
|
72
|
-
this.parentRunId = input.parentRunId;
|
|
73
|
-
this.eventKey = input.eventKey;
|
|
74
|
-
this.cause = input.cause;
|
|
75
|
-
}
|
|
76
|
-
}
|
|
77
|
-
|
|
78
|
-
const CHILD_TERMINAL_DURABLE_VERIFY_GRACE_MS = 5_000;
|
|
79
|
-
const CHILD_TERMINAL_DURABLE_VERIFY_POLL_MS = 250;
|
|
80
|
-
const CHILD_TERMINAL_DURABLE_VERIFY_READ_TIMEOUT_MS = 3_000;
|
|
81
|
-
const CHILD_TERMINAL_DURABLE_VERIFY_MIN_READ_TIMEOUT_MS = 2_000;
|
|
82
|
-
|
|
83
|
-
export interface AwaitChildTerminalInput {
|
|
84
|
-
parentRunId: string;
|
|
85
|
-
workflowStep: WorkflowStepLike | undefined;
|
|
86
|
-
workflowId: string;
|
|
87
|
-
playName: string;
|
|
88
|
-
key: string;
|
|
89
|
-
timeoutMs: number;
|
|
90
|
-
/** Coordinator binding for the terminal-state poll and durable run verification. */
|
|
91
|
-
coordinator: ChildTerminalCoordinator | null;
|
|
92
|
-
now: () => number;
|
|
93
|
-
/** SHA-256 hex digest helper (same canonical hash entry.ts uses). */
|
|
94
|
-
hashJson: (value: unknown) => Promise<string>;
|
|
95
|
-
}
|
|
96
|
-
|
|
97
|
-
function isRecord(value: unknown): value is Record<string, unknown> {
|
|
98
|
-
return typeof value === 'object' && value !== null && !Array.isArray(value);
|
|
99
|
-
}
|
|
100
|
-
|
|
101
|
-
function workflowEventType(name: string): string {
|
|
102
|
-
const normalized = name
|
|
103
|
-
.trim()
|
|
104
|
-
.replace(/[^A-Za-z0-9_-]+/g, '_')
|
|
105
|
-
.replace(/^_+|_+$/g, '')
|
|
106
|
-
.slice(0, 100);
|
|
107
|
-
return normalized || 'deepline_event';
|
|
108
|
-
}
|
|
109
|
-
|
|
110
|
-
function integrationEventType(eventKey: string): string {
|
|
111
|
-
return workflowEventType(`integration_event_${eventKey}`);
|
|
112
|
-
}
|
|
113
|
-
|
|
114
|
-
function workflowTimeoutFromMs(timeoutMs: number): string {
|
|
115
|
-
const seconds = Math.max(1, Math.ceil(timeoutMs / 1000));
|
|
116
|
-
return `${seconds} second${seconds === 1 ? '' : 's'}`;
|
|
117
|
-
}
|
|
118
|
-
|
|
119
|
-
function readChildTerminalPayload(value: unknown): Record<string, unknown> {
|
|
120
|
-
if (!isRecord(value)) return {};
|
|
121
|
-
const data = value.data;
|
|
122
|
-
return isRecord(data) ? data : value;
|
|
123
|
-
}
|
|
124
|
-
|
|
125
|
-
function extractChildPlayOutput(status: Record<string, unknown>): unknown {
|
|
126
|
-
const result = status.result;
|
|
127
|
-
if (isRecord(result) && 'output' in result) {
|
|
128
|
-
return result.output;
|
|
129
|
-
}
|
|
130
|
-
return result ?? null;
|
|
131
|
-
}
|
|
132
|
-
|
|
133
|
-
function terminalPayloadFromState(
|
|
134
|
-
state: { data?: unknown } | null | undefined,
|
|
135
|
-
): Record<string, unknown> | null {
|
|
136
|
-
return isRecord(state?.data) ? state.data : null;
|
|
137
|
-
}
|
|
138
|
-
|
|
139
|
-
function sleep(ms: number): Promise<void> {
|
|
140
|
-
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
141
|
-
}
|
|
142
|
-
|
|
143
|
-
async function childPlayEventKey(input: {
|
|
144
|
-
key: string;
|
|
145
|
-
workflowId: string;
|
|
146
|
-
hashJson: (value: unknown) => Promise<string>;
|
|
147
|
-
}): Promise<string> {
|
|
148
|
-
const readableKey = workflowEventType(input.key).slice(0, 40);
|
|
149
|
-
const digest = (
|
|
150
|
-
await input.hashJson({ key: input.key, workflowId: input.workflowId })
|
|
151
|
-
).slice(0, 32);
|
|
152
|
-
return `child_play_${digest}_${readableKey}`;
|
|
153
|
-
}
|
|
154
|
-
|
|
155
|
-
async function pollParentChildTerminalState(input: {
|
|
156
|
-
coordinator: ChildTerminalCoordinator | null;
|
|
157
|
-
parentRunId: string;
|
|
158
|
-
eventKey: string;
|
|
159
|
-
timeoutMs: number;
|
|
160
|
-
now: () => number;
|
|
161
|
-
}): Promise<{
|
|
162
|
-
source: 'parent_child_terminal_cache';
|
|
163
|
-
payload: Record<string, unknown>;
|
|
164
|
-
attempts: number;
|
|
165
|
-
}> {
|
|
166
|
-
const { coordinator, parentRunId } = input;
|
|
167
|
-
if (!coordinator?.readChildTerminalState || !parentRunId) {
|
|
168
|
-
return await new Promise(() => undefined);
|
|
169
|
-
}
|
|
170
|
-
const startedAt = input.now();
|
|
171
|
-
let attempts = 0;
|
|
172
|
-
while (input.now() - startedAt < input.timeoutMs) {
|
|
173
|
-
attempts += 1;
|
|
174
|
-
const remainingMs = Math.max(
|
|
175
|
-
0,
|
|
176
|
-
input.timeoutMs - (input.now() - startedAt),
|
|
177
|
-
);
|
|
178
|
-
const waitMs = Math.min(remainingMs, 30_000);
|
|
179
|
-
let state: { data?: unknown } | null;
|
|
180
|
-
try {
|
|
181
|
-
// prettier-ignore
|
|
182
|
-
state = await coordinator.readChildTerminalState(parentRunId, input.eventKey, waitMs);
|
|
183
|
-
} catch {
|
|
184
|
-
// The parent-child cache is a safety net for missed workflow events.
|
|
185
|
-
// Transient cache read failures must not beat the event wait plus durable
|
|
186
|
-
// child run verification path in the race.
|
|
187
|
-
if (remainingMs > 0) {
|
|
188
|
-
await sleep(Math.min(remainingMs, CHILD_TERMINAL_DURABLE_VERIFY_POLL_MS));
|
|
189
|
-
}
|
|
190
|
-
continue;
|
|
191
|
-
}
|
|
192
|
-
if (isRecord(state?.data)) {
|
|
193
|
-
return {
|
|
194
|
-
source: 'parent_child_terminal_cache',
|
|
195
|
-
payload: state.data,
|
|
196
|
-
attempts,
|
|
197
|
-
};
|
|
198
|
-
}
|
|
199
|
-
}
|
|
200
|
-
return await new Promise(() => undefined);
|
|
201
|
-
}
|
|
202
|
-
|
|
203
|
-
async function readVerifiedChildTerminalPayload(input: {
|
|
204
|
-
coordinator: ChildTerminalCoordinator;
|
|
205
|
-
workflowId: string;
|
|
206
|
-
timeoutMs: number;
|
|
207
|
-
}): Promise<{
|
|
208
|
-
payload: Record<string, unknown> | null;
|
|
209
|
-
lastState: ChildRunTerminalSnapshotState | null;
|
|
210
|
-
lastError: unknown;
|
|
211
|
-
}> {
|
|
212
|
-
const graceMs = Math.max(
|
|
213
|
-
1,
|
|
214
|
-
Math.min(input.timeoutMs, CHILD_TERMINAL_DURABLE_VERIFY_GRACE_MS),
|
|
215
|
-
);
|
|
216
|
-
const pollMs = Math.max(
|
|
217
|
-
1,
|
|
218
|
-
Math.min(
|
|
219
|
-
CHILD_TERMINAL_DURABLE_VERIFY_POLL_MS,
|
|
220
|
-
Math.max(1, Math.floor(graceMs / 2)),
|
|
221
|
-
),
|
|
222
|
-
);
|
|
223
|
-
const deadlineMs = Date.now() + graceMs;
|
|
224
|
-
let lastState: ChildRunTerminalSnapshotState | null = null;
|
|
225
|
-
let lastError: unknown = null;
|
|
226
|
-
const minReadTimeoutMs =
|
|
227
|
-
graceMs >= CHILD_TERMINAL_DURABLE_VERIFY_MIN_READ_TIMEOUT_MS
|
|
228
|
-
? CHILD_TERMINAL_DURABLE_VERIFY_MIN_READ_TIMEOUT_MS
|
|
229
|
-
: 1;
|
|
230
|
-
while (Date.now() <= deadlineMs) {
|
|
231
|
-
const remainingMs = Math.max(0, deadlineMs - Date.now());
|
|
232
|
-
if (remainingMs < minReadTimeoutMs) break;
|
|
233
|
-
const readTimeoutMs = Math.max(
|
|
234
|
-
minReadTimeoutMs,
|
|
235
|
-
Math.min(remainingMs, CHILD_TERMINAL_DURABLE_VERIFY_READ_TIMEOUT_MS),
|
|
236
|
-
);
|
|
237
|
-
try {
|
|
238
|
-
const state = await input.coordinator.readRunTerminalState(
|
|
239
|
-
input.workflowId,
|
|
240
|
-
readTimeoutMs,
|
|
241
|
-
);
|
|
242
|
-
lastState = state ?? null;
|
|
243
|
-
const payload = terminalPayloadFromState(state);
|
|
244
|
-
if (payload) return { payload, lastState, lastError };
|
|
245
|
-
} catch (error) {
|
|
246
|
-
lastError = error;
|
|
247
|
-
if (!isRetryableWorkerRuntimeApiError(error)) {
|
|
248
|
-
throw error;
|
|
249
|
-
}
|
|
250
|
-
}
|
|
251
|
-
const sleepMs = Math.min(
|
|
252
|
-
pollMs,
|
|
253
|
-
Math.max(0, deadlineMs - Date.now() - minReadTimeoutMs),
|
|
254
|
-
);
|
|
255
|
-
if (sleepMs <= 0) break;
|
|
256
|
-
await sleep(sleepMs);
|
|
257
|
-
}
|
|
258
|
-
return { payload: null, lastState, lastError };
|
|
259
|
-
}
|
|
260
|
-
|
|
261
|
-
function describeSnapshotState(
|
|
262
|
-
state: ChildRunTerminalSnapshotState | null,
|
|
263
|
-
): string {
|
|
264
|
-
if (!state) return 'no snapshot response';
|
|
265
|
-
const parts = [
|
|
266
|
-
state.reason ? `reason=${state.reason}` : null,
|
|
267
|
-
state.status ? `status=${state.status}` : null,
|
|
268
|
-
state.runId ? `runId=${state.runId}` : null,
|
|
269
|
-
state.playName ? `playName=${state.playName}` : null,
|
|
270
|
-
state.parentRunId ? `parentRunId=${state.parentRunId}` : null,
|
|
271
|
-
].filter((part): part is string => Boolean(part));
|
|
272
|
-
return parts.length > 0 ? parts.join(' ') : 'empty snapshot response';
|
|
273
|
-
}
|
|
274
|
-
|
|
275
|
-
/**
|
|
276
|
-
* Block until the child run reaches a terminal state, via the workflow event
|
|
277
|
-
* wait raced against the coordinator terminal-state poll. Throws on a non-
|
|
278
|
-
* completed terminal status.
|
|
279
|
-
*/
|
|
280
|
-
export async function awaitChildTerminal(
|
|
281
|
-
input: AwaitChildTerminalInput,
|
|
282
|
-
): Promise<ChildPlayTerminalWaitResult> {
|
|
283
|
-
if (!input.workflowStep) {
|
|
284
|
-
throw new Error(
|
|
285
|
-
'ctx.runPlay child waits require the cf-workflows runtime event scheduler.',
|
|
286
|
-
);
|
|
287
|
-
}
|
|
288
|
-
const waitStartedAt = input.now();
|
|
289
|
-
const eventKey = await childPlayEventKey({
|
|
290
|
-
key: input.key,
|
|
291
|
-
workflowId: input.workflowId,
|
|
292
|
-
hashJson: input.hashJson,
|
|
293
|
-
});
|
|
294
|
-
if (!input.coordinator?.readRunTerminalState) {
|
|
295
|
-
throw new ChildPlayTerminalVerificationError({
|
|
296
|
-
parentRunId: input.parentRunId,
|
|
297
|
-
eventKey,
|
|
298
|
-
cause: new Error(
|
|
299
|
-
`Child run ${input.workflowId} terminal snapshot reader was not available.`,
|
|
300
|
-
),
|
|
301
|
-
});
|
|
302
|
-
}
|
|
303
|
-
const eventPromise = input.workflowStep
|
|
304
|
-
.waitForEvent(`child_play_terminal:${eventKey}`, {
|
|
305
|
-
type: integrationEventType(eventKey),
|
|
306
|
-
timeout: workflowTimeoutFromMs(input.timeoutMs),
|
|
307
|
-
})
|
|
308
|
-
.then((event) => ({
|
|
309
|
-
source: 'workflow_event' as const,
|
|
310
|
-
payload: readChildTerminalPayload(event.payload),
|
|
311
|
-
attempts: undefined as number | undefined,
|
|
312
|
-
}));
|
|
313
|
-
const terminal = await Promise.race([
|
|
314
|
-
eventPromise,
|
|
315
|
-
pollParentChildTerminalState({
|
|
316
|
-
coordinator: input.coordinator,
|
|
317
|
-
parentRunId: input.parentRunId,
|
|
318
|
-
eventKey,
|
|
319
|
-
timeoutMs: Math.min(input.timeoutMs, 30_000),
|
|
320
|
-
now: input.now,
|
|
321
|
-
}),
|
|
322
|
-
]);
|
|
323
|
-
let verified:
|
|
324
|
-
| {
|
|
325
|
-
payload: Record<string, unknown> | null;
|
|
326
|
-
lastState: ChildRunTerminalSnapshotState | null;
|
|
327
|
-
lastError: unknown;
|
|
328
|
-
}
|
|
329
|
-
| null = null;
|
|
330
|
-
try {
|
|
331
|
-
verified = await readVerifiedChildTerminalPayload({
|
|
332
|
-
coordinator: input.coordinator,
|
|
333
|
-
workflowId: input.workflowId,
|
|
334
|
-
timeoutMs: input.timeoutMs,
|
|
335
|
-
});
|
|
336
|
-
} catch (error) {
|
|
337
|
-
throw new ChildPlayTerminalVerificationError({
|
|
338
|
-
parentRunId: input.parentRunId,
|
|
339
|
-
eventKey,
|
|
340
|
-
cause: error,
|
|
341
|
-
});
|
|
342
|
-
}
|
|
343
|
-
if (!verified?.payload) {
|
|
344
|
-
const lastErrorMessage =
|
|
345
|
-
verified?.lastError instanceof Error
|
|
346
|
-
? ` lastError=${verified.lastError.message}`
|
|
347
|
-
: verified?.lastError
|
|
348
|
-
? ` lastError=${String(verified.lastError)}`
|
|
349
|
-
: '';
|
|
350
|
-
throw new ChildPlayTerminalVerificationError({
|
|
351
|
-
parentRunId: input.parentRunId,
|
|
352
|
-
eventKey,
|
|
353
|
-
cause: new Error(
|
|
354
|
-
`Child run ${input.workflowId} terminal snapshot was not available after durable verification grace; ${describeSnapshotState(verified?.lastState ?? null)}.${lastErrorMessage}`,
|
|
355
|
-
),
|
|
356
|
-
});
|
|
357
|
-
}
|
|
358
|
-
const payload = verified.payload;
|
|
359
|
-
const status = String(payload.status ?? '').toLowerCase();
|
|
360
|
-
if (status === 'completed') {
|
|
361
|
-
return {
|
|
362
|
-
output: extractChildPlayOutput(payload),
|
|
363
|
-
source: terminal.source,
|
|
364
|
-
attempts: terminal.attempts,
|
|
365
|
-
waitMs: input.now() - waitStartedAt,
|
|
366
|
-
};
|
|
367
|
-
}
|
|
368
|
-
const error = isRecord(payload.error) ? payload.error : null;
|
|
369
|
-
const message =
|
|
370
|
-
(typeof error?.message === 'string' && error.message.trim()) ||
|
|
371
|
-
(typeof payload.error === 'string' && payload.error.trim()) ||
|
|
372
|
-
`Child play ${input.playName} (${input.workflowId}) finished with status ${status || 'unknown'}.`;
|
|
373
|
-
throw new Error(message);
|
|
374
|
-
}
|
|
@@ -1,203 +0,0 @@
|
|
|
1
|
-
export type ChildSubmitResponse = {
|
|
2
|
-
workflowId?: string;
|
|
3
|
-
runId?: string;
|
|
4
|
-
status?: string;
|
|
5
|
-
mode?: string;
|
|
6
|
-
output?: unknown;
|
|
7
|
-
result?: unknown;
|
|
8
|
-
error?: unknown;
|
|
9
|
-
logs?: string[];
|
|
10
|
-
timings?: Array<{ phase: string; ms: number }>;
|
|
11
|
-
};
|
|
12
|
-
|
|
13
|
-
export type ChildCoordinatorBinding = {
|
|
14
|
-
submitChild(
|
|
15
|
-
parentRunId: string,
|
|
16
|
-
body: Record<string, unknown>,
|
|
17
|
-
): Promise<ChildSubmitResponse>;
|
|
18
|
-
submitWorkflowChild?(
|
|
19
|
-
parentRunId: string,
|
|
20
|
-
body: Record<string, unknown>,
|
|
21
|
-
): Promise<ChildSubmitResponse>;
|
|
22
|
-
};
|
|
23
|
-
|
|
24
|
-
export type ChildSubmitRequest = {
|
|
25
|
-
runId: string;
|
|
26
|
-
coordinatorUrl?: string | null;
|
|
27
|
-
coordinatorInternalToken?: string | null;
|
|
28
|
-
};
|
|
29
|
-
|
|
30
|
-
export type CoordinatorRequestHeadersInput = {
|
|
31
|
-
runId: string;
|
|
32
|
-
contentType?: string;
|
|
33
|
-
internalToken?: string | null;
|
|
34
|
-
};
|
|
35
|
-
|
|
36
|
-
export type SubmitChildPlayThroughCoordinatorInput = {
|
|
37
|
-
req: ChildSubmitRequest;
|
|
38
|
-
body: unknown;
|
|
39
|
-
coordinatorBinding?: ChildCoordinatorBinding | null;
|
|
40
|
-
allowInline?: boolean;
|
|
41
|
-
makeRequestId: () => string;
|
|
42
|
-
coordinatorRequestHeaders: (
|
|
43
|
-
input: CoordinatorRequestHeadersInput,
|
|
44
|
-
) => Record<string, string>;
|
|
45
|
-
fetchFn?: typeof fetch;
|
|
46
|
-
retryDelaysMs?: readonly number[];
|
|
47
|
-
sleep?: (ms: number) => Promise<void>;
|
|
48
|
-
};
|
|
49
|
-
|
|
50
|
-
function isRecord(value: unknown): value is Record<string, unknown> {
|
|
51
|
-
return typeof value === 'object' && value !== null && !Array.isArray(value);
|
|
52
|
-
}
|
|
53
|
-
|
|
54
|
-
function requireChildSubmitBody(body: unknown): Record<string, unknown> {
|
|
55
|
-
if (!isRecord(body)) {
|
|
56
|
-
throw new Error('ctx.runPlay child submit requires an object body.');
|
|
57
|
-
}
|
|
58
|
-
return body;
|
|
59
|
-
}
|
|
60
|
-
|
|
61
|
-
function isRetryableChildSubmitError(error: unknown): boolean {
|
|
62
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
63
|
-
return /timed out|timeout|fetch failed|ECONNRESET|ECONNREFUSED|UND_ERR_CONNECT_TIMEOUT/i.test(
|
|
64
|
-
message,
|
|
65
|
-
);
|
|
66
|
-
}
|
|
67
|
-
|
|
68
|
-
function isRetryableChildSubmitResponse(status: number, body: string): boolean {
|
|
69
|
-
if (
|
|
70
|
-
status === 408 ||
|
|
71
|
-
status === 429 ||
|
|
72
|
-
status === 502 ||
|
|
73
|
-
status === 503 ||
|
|
74
|
-
status === 504 ||
|
|
75
|
-
status === 530
|
|
76
|
-
) {
|
|
77
|
-
return true;
|
|
78
|
-
}
|
|
79
|
-
return (
|
|
80
|
-
status === 500 &&
|
|
81
|
-
/timeout exceeded when trying to connect|timed out|fetch failed|ECONNRESET|UND_ERR_CONNECT_TIMEOUT/i.test(
|
|
82
|
-
body,
|
|
83
|
-
)
|
|
84
|
-
);
|
|
85
|
-
}
|
|
86
|
-
|
|
87
|
-
function parseChildSubmitResponse(text: string): ChildSubmitResponse {
|
|
88
|
-
if (!text) {
|
|
89
|
-
return {};
|
|
90
|
-
}
|
|
91
|
-
try {
|
|
92
|
-
const parsed = JSON.parse(text) as unknown;
|
|
93
|
-
return isRecord(parsed) ? parsed : { result: parsed };
|
|
94
|
-
} catch {
|
|
95
|
-
return { error: text };
|
|
96
|
-
}
|
|
97
|
-
}
|
|
98
|
-
|
|
99
|
-
function childSubmitErrorMessage(input: {
|
|
100
|
-
parsed: ChildSubmitResponse;
|
|
101
|
-
text: string;
|
|
102
|
-
status: number;
|
|
103
|
-
}): string {
|
|
104
|
-
const error = isRecord(input.parsed.error) ? input.parsed.error : null;
|
|
105
|
-
return (
|
|
106
|
-
(typeof error?.message === 'string' && error.message.trim()) ||
|
|
107
|
-
(typeof input.parsed.error === 'string' && input.parsed.error.trim()) ||
|
|
108
|
-
input.text.slice(0, 800) ||
|
|
109
|
-
`Coordinator child submit failed with ${input.status}.`
|
|
110
|
-
);
|
|
111
|
-
}
|
|
112
|
-
|
|
113
|
-
async function defaultSleep(ms: number): Promise<void> {
|
|
114
|
-
await new Promise((resolve) => setTimeout(resolve, ms));
|
|
115
|
-
}
|
|
116
|
-
|
|
117
|
-
export async function submitChildPlayThroughCoordinator(
|
|
118
|
-
input: SubmitChildPlayThroughCoordinatorInput,
|
|
119
|
-
): Promise<ChildSubmitResponse> {
|
|
120
|
-
const body = requireChildSubmitBody(input.body);
|
|
121
|
-
const binding = input.coordinatorBinding;
|
|
122
|
-
if (binding && input.allowInline !== false) {
|
|
123
|
-
return await binding.submitChild(input.req.runId, body);
|
|
124
|
-
}
|
|
125
|
-
if (binding?.submitWorkflowChild) {
|
|
126
|
-
return await binding.submitWorkflowChild(input.req.runId, body);
|
|
127
|
-
}
|
|
128
|
-
|
|
129
|
-
const coordinatorUrl = input.req.coordinatorUrl?.trim();
|
|
130
|
-
if (!coordinatorUrl) {
|
|
131
|
-
throw new Error(
|
|
132
|
-
'ctx.runPlay child submit requires a coordinator binding in the cf-workflows runtime.',
|
|
133
|
-
);
|
|
134
|
-
}
|
|
135
|
-
|
|
136
|
-
const fetchFn = input.fetchFn ?? fetch;
|
|
137
|
-
const retryDelaysMs = input.retryDelaysMs ?? [
|
|
138
|
-
250, 750, 1500, 3000, 5000, 10000,
|
|
139
|
-
];
|
|
140
|
-
const sleep = input.sleep ?? defaultSleep;
|
|
141
|
-
const url = `${coordinatorUrl.replace(/\/$/, '')}/workflow/${encodeURIComponent(
|
|
142
|
-
input.req.runId,
|
|
143
|
-
)}/submit-child`;
|
|
144
|
-
const serializedBody = JSON.stringify(body);
|
|
145
|
-
const requestId = input.makeRequestId();
|
|
146
|
-
const idempotencyKey =
|
|
147
|
-
typeof body.childIdempotencyKey === 'string' &&
|
|
148
|
-
body.childIdempotencyKey.trim()
|
|
149
|
-
? body.childIdempotencyKey.trim()
|
|
150
|
-
: requestId;
|
|
151
|
-
let lastError: unknown = null;
|
|
152
|
-
|
|
153
|
-
for (let attempt = 0; attempt <= retryDelaysMs.length; attempt += 1) {
|
|
154
|
-
let response: Response;
|
|
155
|
-
try {
|
|
156
|
-
response = await fetchFn(url, {
|
|
157
|
-
method: 'POST',
|
|
158
|
-
headers: {
|
|
159
|
-
'x-deepline-request-id': requestId,
|
|
160
|
-
'x-deepline-idempotency-key': idempotencyKey,
|
|
161
|
-
...input.coordinatorRequestHeaders({
|
|
162
|
-
runId: input.req.runId,
|
|
163
|
-
contentType: 'application/json',
|
|
164
|
-
internalToken: input.req.coordinatorInternalToken,
|
|
165
|
-
}),
|
|
166
|
-
},
|
|
167
|
-
body: serializedBody,
|
|
168
|
-
});
|
|
169
|
-
} catch (error) {
|
|
170
|
-
lastError = error;
|
|
171
|
-
if (
|
|
172
|
-
attempt >= retryDelaysMs.length ||
|
|
173
|
-
!isRetryableChildSubmitError(error)
|
|
174
|
-
) {
|
|
175
|
-
throw error;
|
|
176
|
-
}
|
|
177
|
-
await sleep(retryDelaysMs[attempt] ?? 0);
|
|
178
|
-
continue;
|
|
179
|
-
}
|
|
180
|
-
|
|
181
|
-
const text = await response.text().catch(() => '');
|
|
182
|
-
const parsed = parseChildSubmitResponse(text);
|
|
183
|
-
if (response.ok) {
|
|
184
|
-
return parsed;
|
|
185
|
-
}
|
|
186
|
-
|
|
187
|
-
const message = childSubmitErrorMessage({
|
|
188
|
-
parsed,
|
|
189
|
-
text,
|
|
190
|
-
status: response.status,
|
|
191
|
-
});
|
|
192
|
-
lastError = new Error(message);
|
|
193
|
-
if (
|
|
194
|
-
attempt >= retryDelaysMs.length ||
|
|
195
|
-
!isRetryableChildSubmitResponse(response.status, text)
|
|
196
|
-
) {
|
|
197
|
-
throw lastError;
|
|
198
|
-
}
|
|
199
|
-
await sleep(retryDelaysMs[attempt] ?? 0);
|
|
200
|
-
}
|
|
201
|
-
|
|
202
|
-
throw lastError instanceof Error ? lastError : new Error(String(lastError));
|
|
203
|
-
}
|
|
@@ -1,14 +0,0 @@
|
|
|
1
|
-
import type { PlayCallExecution } from './play-call-execution';
|
|
2
|
-
|
|
3
|
-
export function resolveChildExecutionPlacement(input: {
|
|
4
|
-
requiresWorkflow: boolean;
|
|
5
|
-
execution?: PlayCallExecution | null;
|
|
6
|
-
}): { strategy: 'inline' | 'scheduled' } {
|
|
7
|
-
return {
|
|
8
|
-
strategy:
|
|
9
|
-
input.execution === 'child-workflow' ||
|
|
10
|
-
(input.execution !== 'inline' && input.requiresWorkflow)
|
|
11
|
-
? 'scheduled'
|
|
12
|
-
: 'inline',
|
|
13
|
-
};
|
|
14
|
-
}
|