deepline 0.1.47 → 0.1.49
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -14
- package/dist/cli/index.js +863 -260
- package/dist/cli/index.mjs +863 -260
- package/dist/index.d.mts +44 -34
- package/dist/index.d.ts +44 -34
- package/dist/index.js +57 -16
- package/dist/index.mjs +57 -16
- package/dist/repo/apps/play-runner-workers/src/coordinator-entry.ts +5 -2
- package/dist/repo/apps/play-runner-workers/src/entry.ts +383 -305
- package/dist/repo/apps/play-runner-workers/src/runtime/receipts.ts +51 -21
- package/dist/repo/sdk/src/client.ts +29 -4
- package/dist/repo/sdk/src/play.ts +35 -42
- package/dist/repo/sdk/src/plays/harness-stub.ts +1 -1
- package/dist/repo/sdk/src/version.ts +1 -1
- package/dist/repo/sdk/src/worker-play-entry.ts +17 -67
- package/dist/repo/shared_libs/play-runtime/tool-result.ts +34 -3
- package/dist/repo/shared_libs/plays/row-identity.ts +5 -59
- package/package.json +1 -1
|
@@ -18,12 +18,12 @@
|
|
|
18
18
|
* bundled into the Worker script.
|
|
19
19
|
* - Workers don't have node:fs / source-map-support. Stack traces are raw.
|
|
20
20
|
* - Direct postgres (`pg` library) won't bundle for Workers. This harness
|
|
21
|
-
* uses HTTP-only ctx — every ctx.csv / ctx.
|
|
21
|
+
* uses HTTP-only ctx — every ctx.csv / ctx.tools.execute / row write goes through
|
|
22
22
|
* the runtime API endpoint, not direct DB. That keeps the Worker bundle
|
|
23
23
|
* compatible with the V8 isolate runtime.
|
|
24
24
|
*
|
|
25
25
|
* Status: experimental. First cut targets tool-basic (ctx.csv + ctx.map +
|
|
26
|
-
* ctx.
|
|
26
|
+
* ctx.tools.execute). Plays that depend on the full ctx surface (durable sleep,
|
|
27
27
|
* checkpoints, batched waterfalls, etc.) will fall back to "not implemented"
|
|
28
28
|
* rather than producing wrong results — opt-in via DEEPLINE_PLAY_RUNNER_BACKEND.
|
|
29
29
|
*/
|
|
@@ -217,7 +217,10 @@ function getStringField(value: unknown, key: string): string | null {
|
|
|
217
217
|
return typeof field === 'string' && field.trim() ? field : null;
|
|
218
218
|
}
|
|
219
219
|
|
|
220
|
-
function getObjectField(
|
|
220
|
+
function getObjectField(
|
|
221
|
+
value: unknown,
|
|
222
|
+
key: string,
|
|
223
|
+
): Record<string, unknown> | null {
|
|
221
224
|
if (!isRecord(value)) return null;
|
|
222
225
|
const field = value[key];
|
|
223
226
|
return isRecord(field) ? field : null;
|
|
@@ -269,10 +272,12 @@ function normalizeToolHttpErrorMessage(input: {
|
|
|
269
272
|
const billing = getObjectField(parsed, 'billing');
|
|
270
273
|
if (isInsufficientCreditsBilling(billing)) {
|
|
271
274
|
return new ToolHttpError(
|
|
272
|
-
`tool ${input.toolId} ${input.status} attempt ${input.attempt}/${input.maxAttempts}: ${formatInsufficientCreditsMessage(
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
275
|
+
`tool ${input.toolId} ${input.status} attempt ${input.attempt}/${input.maxAttempts}: ${formatInsufficientCreditsMessage(
|
|
276
|
+
{
|
|
277
|
+
billing,
|
|
278
|
+
toolId: input.toolId,
|
|
279
|
+
},
|
|
280
|
+
)}`,
|
|
276
281
|
billing,
|
|
277
282
|
);
|
|
278
283
|
}
|
|
@@ -451,7 +456,10 @@ async function probeHarnessOnce(
|
|
|
451
456
|
*/
|
|
452
457
|
const RUNTIME_API_TIMEOUT_MS = 30_000;
|
|
453
458
|
const RUNTIME_API_PLAY_RUN_TIMEOUT_MS = 75_000;
|
|
454
|
-
const
|
|
459
|
+
const RUNTIME_API_INTEGRATION_EXECUTE_TIMEOUT_MS = 180_000;
|
|
460
|
+
const RUNTIME_API_RETRY_DELAYS_MS = [
|
|
461
|
+
250, 750, 1500, 3000, 5000, 10000,
|
|
462
|
+
] as const;
|
|
455
463
|
let loggedMissingRuntimeApiBinding = false;
|
|
456
464
|
|
|
457
465
|
async function fetchRuntimeApi(
|
|
@@ -462,6 +470,8 @@ async function fetchRuntimeApi(
|
|
|
462
470
|
const timeoutMs =
|
|
463
471
|
path === '/api/v2/plays/run'
|
|
464
472
|
? RUNTIME_API_PLAY_RUN_TIMEOUT_MS
|
|
473
|
+
: /^\/api\/v2\/integrations\/[^/]+\/execute$/.test(path)
|
|
474
|
+
? RUNTIME_API_INTEGRATION_EXECUTE_TIMEOUT_MS
|
|
465
475
|
: RUNTIME_API_TIMEOUT_MS;
|
|
466
476
|
const controller = new AbortController();
|
|
467
477
|
let timeout: ReturnType<typeof setTimeout> | null = null;
|
|
@@ -496,7 +506,21 @@ async function fetchRuntimeApi(
|
|
|
496
506
|
const responsePromise = cachedRuntimeApiBinding.fetch(
|
|
497
507
|
new Request(`${baseUrl.replace(/\/$/, '')}${path}`, mergedInit),
|
|
498
508
|
);
|
|
499
|
-
|
|
509
|
+
const response = await Promise.race([responsePromise, timeoutPromise]);
|
|
510
|
+
if (await shouldFallbackRuntimeApiBindingResponse(response)) {
|
|
511
|
+
console.warn(
|
|
512
|
+
`[play-harness] RUNTIME_API binding returned coordinator not found; using public runtime API transport. path=${path}`,
|
|
513
|
+
);
|
|
514
|
+
return await Promise.race([
|
|
515
|
+
fetch(`${baseUrl.replace(/\/$/, '')}${path}`, {
|
|
516
|
+
...init,
|
|
517
|
+
headers: runtimeApiHeaders(init.headers, true),
|
|
518
|
+
signal: controller.signal,
|
|
519
|
+
}),
|
|
520
|
+
timeoutPromise,
|
|
521
|
+
]);
|
|
522
|
+
}
|
|
523
|
+
return response;
|
|
500
524
|
} catch (err) {
|
|
501
525
|
if (err instanceof Error && err.name === 'AbortError') {
|
|
502
526
|
throw new Error(
|
|
@@ -509,6 +533,19 @@ async function fetchRuntimeApi(
|
|
|
509
533
|
}
|
|
510
534
|
}
|
|
511
535
|
|
|
536
|
+
async function shouldFallbackRuntimeApiBindingResponse(
|
|
537
|
+
response: Response,
|
|
538
|
+
): Promise<boolean> {
|
|
539
|
+
if (response.status !== 404) {
|
|
540
|
+
return false;
|
|
541
|
+
}
|
|
542
|
+
const body = await response
|
|
543
|
+
.clone()
|
|
544
|
+
.text()
|
|
545
|
+
.catch(() => '');
|
|
546
|
+
return body.trim().toLowerCase() === 'not found';
|
|
547
|
+
}
|
|
548
|
+
|
|
512
549
|
function runtimeApiHeaders(
|
|
513
550
|
headers: HeadersInit | undefined,
|
|
514
551
|
includeVercelBypass: boolean,
|
|
@@ -922,18 +959,20 @@ function extractChildPlayOutput(status: Record<string, unknown>): unknown {
|
|
|
922
959
|
return result ?? null;
|
|
923
960
|
}
|
|
924
961
|
|
|
925
|
-
function hashChildPlayEventKey(input:
|
|
926
|
-
|
|
927
|
-
for (let index = 0; index < input.length; index += 1) {
|
|
928
|
-
hash ^= input.charCodeAt(index);
|
|
929
|
-
hash = Math.imul(hash, 16777619);
|
|
930
|
-
}
|
|
931
|
-
return (hash >>> 0).toString(36);
|
|
962
|
+
async function hashChildPlayEventKey(input: unknown): Promise<string> {
|
|
963
|
+
return (await hashJson(input)).slice(0, 32);
|
|
932
964
|
}
|
|
933
965
|
|
|
934
|
-
function childPlayEventKey(input: {
|
|
966
|
+
async function childPlayEventKey(input: {
|
|
967
|
+
key: string;
|
|
968
|
+
workflowId: string;
|
|
969
|
+
}): Promise<string> {
|
|
935
970
|
const readableKey = workflowEventType(input.key).slice(0, 40);
|
|
936
|
-
|
|
971
|
+
const digest = await hashChildPlayEventKey({
|
|
972
|
+
key: input.key,
|
|
973
|
+
workflowId: input.workflowId,
|
|
974
|
+
});
|
|
975
|
+
return `child_play_${digest}_${readableKey}`;
|
|
937
976
|
}
|
|
938
977
|
|
|
939
978
|
function workflowTimeoutFromMs(timeoutMs: number): string {
|
|
@@ -954,7 +993,7 @@ async function waitForChildPlayTerminalEvent(input: {
|
|
|
954
993
|
'ctx.runPlay child waits require the cf-workflows runtime event scheduler.',
|
|
955
994
|
);
|
|
956
995
|
}
|
|
957
|
-
const eventKey = childPlayEventKey({
|
|
996
|
+
const eventKey = await childPlayEventKey({
|
|
958
997
|
key: input.key,
|
|
959
998
|
workflowId: input.workflowId,
|
|
960
999
|
});
|
|
@@ -989,7 +1028,7 @@ async function signalParentPlayTerminal(input: {
|
|
|
989
1028
|
}): Promise<void> {
|
|
990
1029
|
const governance = input.req.playCallGovernance;
|
|
991
1030
|
if (!governance?.parentRunId || !governance.key) return;
|
|
992
|
-
const eventKey = childPlayEventKey({
|
|
1031
|
+
const eventKey = await childPlayEventKey({
|
|
993
1032
|
key: governance.key,
|
|
994
1033
|
workflowId: input.req.runId,
|
|
995
1034
|
});
|
|
@@ -1098,9 +1137,12 @@ function isToolExecuteRecord(value: unknown): value is Record<string, unknown> {
|
|
|
1098
1137
|
return typeof value === 'object' && value !== null && !Array.isArray(value);
|
|
1099
1138
|
}
|
|
1100
1139
|
|
|
1101
|
-
function normalizeToolExecuteArgs(
|
|
1102
|
-
|
|
1103
|
-
|
|
1140
|
+
function normalizeToolExecuteArgs(request: unknown): {
|
|
1141
|
+
id: string;
|
|
1142
|
+
toolId: string;
|
|
1143
|
+
input: Record<string, unknown>;
|
|
1144
|
+
staleAfterSeconds?: number;
|
|
1145
|
+
} {
|
|
1104
1146
|
if (!isToolExecuteRecord(request)) {
|
|
1105
1147
|
throw new Error(
|
|
1106
1148
|
'ctx.tools.execute requires a request object: ctx.tools.execute({ id, tool, input, description }).',
|
|
@@ -1117,7 +1159,14 @@ function normalizeToolExecuteArgs(
|
|
|
1117
1159
|
'ctx.tools.execute({ id, tool, input }) requires a non-empty id, tool string, and input object.',
|
|
1118
1160
|
);
|
|
1119
1161
|
}
|
|
1120
|
-
return {
|
|
1162
|
+
return {
|
|
1163
|
+
id: request.id.trim(),
|
|
1164
|
+
toolId: request.tool,
|
|
1165
|
+
input: request.input,
|
|
1166
|
+
...(typeof request.staleAfterSeconds === 'number'
|
|
1167
|
+
? { staleAfterSeconds: request.staleAfterSeconds }
|
|
1168
|
+
: {}),
|
|
1169
|
+
};
|
|
1121
1170
|
}
|
|
1122
1171
|
|
|
1123
1172
|
function integrationEventType(eventKey: string): string {
|
|
@@ -1329,7 +1378,11 @@ function parseExtractorMetadata(
|
|
|
1329
1378
|
}
|
|
1330
1379
|
const entries = Object.entries(value as Record<string, unknown>).flatMap(
|
|
1331
1380
|
([key, descriptor]) => {
|
|
1332
|
-
if (
|
|
1381
|
+
if (
|
|
1382
|
+
!descriptor ||
|
|
1383
|
+
typeof descriptor !== 'object' ||
|
|
1384
|
+
Array.isArray(descriptor)
|
|
1385
|
+
) {
|
|
1333
1386
|
return [];
|
|
1334
1387
|
}
|
|
1335
1388
|
const record = descriptor as Record<string, unknown>;
|
|
@@ -1630,17 +1683,14 @@ type WorkerInlineWaterfallSpec = {
|
|
|
1630
1683
|
input: Record<string, unknown>,
|
|
1631
1684
|
ctx: {
|
|
1632
1685
|
tools: {
|
|
1633
|
-
execute(
|
|
1634
|
-
|
|
1635
|
-
|
|
1636
|
-
input: Record<string, unknown
|
|
1637
|
-
|
|
1686
|
+
execute(request: {
|
|
1687
|
+
id: string;
|
|
1688
|
+
tool: string;
|
|
1689
|
+
input: Record<string, unknown>;
|
|
1690
|
+
description?: string;
|
|
1691
|
+
staleAfterSeconds?: number;
|
|
1692
|
+
}): Promise<unknown>;
|
|
1638
1693
|
};
|
|
1639
|
-
tool(
|
|
1640
|
-
key: string,
|
|
1641
|
-
toolId: string,
|
|
1642
|
-
input: Record<string, unknown>,
|
|
1643
|
-
): Promise<unknown>;
|
|
1644
1694
|
},
|
|
1645
1695
|
) => unknown | Promise<unknown>;
|
|
1646
1696
|
}
|
|
@@ -1907,6 +1957,14 @@ function extractWorkerInlineCodeStepValue(
|
|
|
1907
1957
|
return result ?? null;
|
|
1908
1958
|
}
|
|
1909
1959
|
|
|
1960
|
+
function isCompletedWorkerFieldValue(value: unknown): boolean {
|
|
1961
|
+
return (
|
|
1962
|
+
value !== null &&
|
|
1963
|
+
value !== undefined &&
|
|
1964
|
+
!(typeof value === 'string' && value.length === 0)
|
|
1965
|
+
);
|
|
1966
|
+
}
|
|
1967
|
+
|
|
1910
1968
|
type WorkerMapChunkSummary<T extends Record<string, unknown>> = {
|
|
1911
1969
|
chunkIndex: number;
|
|
1912
1970
|
rangeStart: number;
|
|
@@ -1929,7 +1987,9 @@ function serializeDurableStepValue<T>(value: T, depth = 0): T {
|
|
|
1929
1987
|
if (isToolExecuteResult(value)) return serializeToolExecuteResult(value) as T;
|
|
1930
1988
|
if (isDatasetHandle(value)) return serializeValue(value, depth) as T;
|
|
1931
1989
|
if (Array.isArray(value)) {
|
|
1932
|
-
return value.map((entry) =>
|
|
1990
|
+
return value.map((entry) =>
|
|
1991
|
+
serializeDurableStepValue(entry, depth + 1),
|
|
1992
|
+
) as T;
|
|
1933
1993
|
}
|
|
1934
1994
|
if (typeof value !== 'object') return value;
|
|
1935
1995
|
return Object.fromEntries(
|
|
@@ -2087,9 +2147,9 @@ async function executeWorkerStepProgram(
|
|
|
2087
2147
|
recorder
|
|
2088
2148
|
? {
|
|
2089
2149
|
...recorder,
|
|
2090
|
-
|
|
2091
|
-
|
|
2092
|
-
|
|
2150
|
+
path: stepPath,
|
|
2151
|
+
}
|
|
2152
|
+
: undefined,
|
|
2093
2153
|
);
|
|
2094
2154
|
return {
|
|
2095
2155
|
value: serializeDurableStepValue(resolution.value),
|
|
@@ -2160,13 +2220,6 @@ async function executeWorkerWaterfall(
|
|
|
2160
2220
|
callbacks,
|
|
2161
2221
|
),
|
|
2162
2222
|
},
|
|
2163
|
-
tool: async (key, toolId, toolInput) =>
|
|
2164
|
-
await executeToolWithLifecycle(
|
|
2165
|
-
req,
|
|
2166
|
-
{ id: key, toolId, input: toolInput },
|
|
2167
|
-
workflowStep,
|
|
2168
|
-
callbacks,
|
|
2169
|
-
),
|
|
2170
2223
|
});
|
|
2171
2224
|
} else {
|
|
2172
2225
|
result = await executeToolWithLifecycle(
|
|
@@ -2884,25 +2937,32 @@ function augmentSheetContractWithDatasetFields(input: {
|
|
|
2884
2937
|
input.contract.columns.map((column) => column.sqlName),
|
|
2885
2938
|
);
|
|
2886
2939
|
const columns = [...input.contract.columns];
|
|
2940
|
+
const candidateFields = new Set<string>();
|
|
2887
2941
|
for (const row of input.rows) {
|
|
2888
2942
|
for (const field of Object.keys(row)) {
|
|
2889
|
-
|
|
2890
|
-
continue;
|
|
2891
|
-
}
|
|
2892
|
-
const sqlName = sqlSafePlayColumnName(field);
|
|
2893
|
-
if (existingSqlNames.has(sqlName)) {
|
|
2894
|
-
continue;
|
|
2895
|
-
}
|
|
2896
|
-
existingFields.add(field);
|
|
2897
|
-
existingSqlNames.add(sqlName);
|
|
2898
|
-
columns.push({
|
|
2899
|
-
id: `runtime:${input.contract.tableNamespace}:${field}`,
|
|
2900
|
-
sqlName,
|
|
2901
|
-
source: outputFields.has(field) ? 'mapField' : 'input',
|
|
2902
|
-
field,
|
|
2903
|
-
});
|
|
2943
|
+
candidateFields.add(field);
|
|
2904
2944
|
}
|
|
2905
2945
|
}
|
|
2946
|
+
for (const field of outputFields) {
|
|
2947
|
+
candidateFields.add(field);
|
|
2948
|
+
}
|
|
2949
|
+
for (const field of candidateFields) {
|
|
2950
|
+
if (!isDatasetPayloadField(field) || existingFields.has(field)) {
|
|
2951
|
+
continue;
|
|
2952
|
+
}
|
|
2953
|
+
const sqlName = sqlSafePlayColumnName(field);
|
|
2954
|
+
if (existingSqlNames.has(sqlName)) {
|
|
2955
|
+
continue;
|
|
2956
|
+
}
|
|
2957
|
+
existingFields.add(field);
|
|
2958
|
+
existingSqlNames.add(sqlName);
|
|
2959
|
+
columns.push({
|
|
2960
|
+
id: `runtime:${input.contract.tableNamespace}:${field}`,
|
|
2961
|
+
sqlName,
|
|
2962
|
+
source: outputFields.has(field) ? 'mapField' : 'input',
|
|
2963
|
+
field,
|
|
2964
|
+
});
|
|
2965
|
+
}
|
|
2906
2966
|
return { ...input.contract, columns };
|
|
2907
2967
|
}
|
|
2908
2968
|
|
|
@@ -2942,6 +3002,7 @@ async function prepareMapRows(input: {
|
|
|
2942
3002
|
req: RunRequest;
|
|
2943
3003
|
tableNamespace: string;
|
|
2944
3004
|
rows: Record<string, unknown>[];
|
|
3005
|
+
outputFields: string[];
|
|
2945
3006
|
}): Promise<{
|
|
2946
3007
|
inserted: number;
|
|
2947
3008
|
skipped: number;
|
|
@@ -2960,6 +3021,7 @@ async function prepareMapRows(input: {
|
|
|
2960
3021
|
sheetContract: augmentSheetContractWithDatasetFields({
|
|
2961
3022
|
contract: requireSheetContract(input.req, input.tableNamespace),
|
|
2962
3023
|
rows: input.rows,
|
|
3024
|
+
outputFields: input.outputFields,
|
|
2963
3025
|
}),
|
|
2964
3026
|
rows: input.rows.map((row) => ({ ...row })),
|
|
2965
3027
|
runId: input.req.runId,
|
|
@@ -3003,7 +3065,7 @@ async function prepareMapRows(input: {
|
|
|
3003
3065
|
* - ctx.log(msg)
|
|
3004
3066
|
* - ctx.csv(filename | inline rows) (calls runtime API for file resolve)
|
|
3005
3067
|
* - ctx.map(name, rows, fields, opts)
|
|
3006
|
-
* - ctx.tools.execute(
|
|
3068
|
+
* - ctx.tools.execute({ id, tool, input, ... })
|
|
3007
3069
|
* - ctx.runPlay(key, playRef, input, opts)
|
|
3008
3070
|
*
|
|
3009
3071
|
* Not supported (will throw):
|
|
@@ -3128,16 +3190,32 @@ function createMinimalWorkerCtx(
|
|
|
3128
3190
|
const executeWithRuntimeReceipt = async <T>(
|
|
3129
3191
|
key: string,
|
|
3130
3192
|
execute: () => Promise<T> | T,
|
|
3131
|
-
): Promise<T> =>
|
|
3132
|
-
runWorkerRuntimeReceiptBoundary({
|
|
3193
|
+
): Promise<T> => {
|
|
3194
|
+
const serialized = await runWorkerRuntimeReceiptBoundary<unknown>({
|
|
3133
3195
|
baseUrl: req.baseUrl,
|
|
3134
3196
|
executorToken: req.executorToken,
|
|
3197
|
+
orgId: req.orgId,
|
|
3135
3198
|
playName: req.playName,
|
|
3136
3199
|
runId: req.runId,
|
|
3137
3200
|
key,
|
|
3138
3201
|
postRuntimeApi,
|
|
3139
|
-
execute,
|
|
3202
|
+
execute: async () => serializeDurableStepValue(await execute()),
|
|
3140
3203
|
});
|
|
3204
|
+
return deserializeDurableStepValue(serialized) as T;
|
|
3205
|
+
};
|
|
3206
|
+
const staleRuntimeSuffix = (staleAfterSeconds?: number): string => {
|
|
3207
|
+
if (staleAfterSeconds === undefined) return '';
|
|
3208
|
+
if (
|
|
3209
|
+
!Number.isFinite(staleAfterSeconds) ||
|
|
3210
|
+
!Number.isInteger(staleAfterSeconds) ||
|
|
3211
|
+
staleAfterSeconds <= 0
|
|
3212
|
+
) {
|
|
3213
|
+
throw new Error(
|
|
3214
|
+
'staleAfterSeconds must be a positive whole number of seconds.',
|
|
3215
|
+
);
|
|
3216
|
+
}
|
|
3217
|
+
return `:stale:${staleAfterSeconds}:${Math.floor(nowMs() / (staleAfterSeconds * 1000))}`;
|
|
3218
|
+
};
|
|
3141
3219
|
// Local ancestry chain that always ENDS with the currently-executing play
|
|
3142
3220
|
// (req.playName). The /api/v2/plays/run lineage validator requires the
|
|
3143
3221
|
// submitted ancestry's tail to equal the executor token's play name (i.e.
|
|
@@ -3253,7 +3331,6 @@ function createMinimalWorkerCtx(
|
|
|
3253
3331
|
: JSON.stringify(normalizedParts);
|
|
3254
3332
|
return keyValue;
|
|
3255
3333
|
};
|
|
3256
|
-
const mapLogicFingerprint = req.graphHash ?? null;
|
|
3257
3334
|
const resolveRowKey = (
|
|
3258
3335
|
row: Record<string, unknown>,
|
|
3259
3336
|
index: number,
|
|
@@ -3261,12 +3338,8 @@ function createMinimalWorkerCtx(
|
|
|
3261
3338
|
const inputRow = publicCsvInputRow(row);
|
|
3262
3339
|
const explicitKeyValue = resolveExplicitKeyValue(row, index);
|
|
3263
3340
|
return explicitKeyValue == null
|
|
3264
|
-
? derivePlayRowIdentity(inputRow, name
|
|
3265
|
-
: derivePlayRowIdentityFromKey(
|
|
3266
|
-
explicitKeyValue,
|
|
3267
|
-
name,
|
|
3268
|
-
mapLogicFingerprint,
|
|
3269
|
-
);
|
|
3341
|
+
? derivePlayRowIdentity(inputRow, name)
|
|
3342
|
+
: derivePlayRowIdentityFromKey(explicitKeyValue, name);
|
|
3270
3343
|
};
|
|
3271
3344
|
const assertUniqueExplicitRowKeys = (
|
|
3272
3345
|
chunkRows: readonly Record<string, unknown>[],
|
|
@@ -3311,6 +3384,7 @@ function createMinimalWorkerCtx(
|
|
|
3311
3384
|
const prepared = await prepareMapRows({
|
|
3312
3385
|
req,
|
|
3313
3386
|
tableNamespace: name,
|
|
3387
|
+
outputFields,
|
|
3314
3388
|
rows: chunkEntries.map(({ row, rowKey }) => ({
|
|
3315
3389
|
...row,
|
|
3316
3390
|
__deeplineRowKey: rowKey,
|
|
@@ -3331,19 +3405,17 @@ function createMinimalWorkerCtx(
|
|
|
3331
3405
|
},
|
|
3332
3406
|
});
|
|
3333
3407
|
const pendingKeys = new Set<string>();
|
|
3408
|
+
const pendingRowsByKey = new Map<string, Record<string, unknown>>();
|
|
3334
3409
|
const completedKeys = new Set<string>();
|
|
3335
3410
|
const preparedKeys = new Set<string>();
|
|
3336
3411
|
for (const row of prepared.pendingRows) {
|
|
3337
3412
|
const key =
|
|
3338
3413
|
typeof row.__deeplineRowKey === 'string'
|
|
3339
3414
|
? row.__deeplineRowKey
|
|
3340
|
-
: derivePlayRowIdentity(
|
|
3341
|
-
publicCsvInputRow(row),
|
|
3342
|
-
name,
|
|
3343
|
-
mapLogicFingerprint,
|
|
3344
|
-
);
|
|
3415
|
+
: derivePlayRowIdentity(publicCsvInputRow(row), name);
|
|
3345
3416
|
if (key) {
|
|
3346
3417
|
pendingKeys.add(key);
|
|
3418
|
+
pendingRowsByKey.set(key, row);
|
|
3347
3419
|
preparedKeys.add(key);
|
|
3348
3420
|
}
|
|
3349
3421
|
}
|
|
@@ -3351,11 +3423,7 @@ function createMinimalWorkerCtx(
|
|
|
3351
3423
|
const key =
|
|
3352
3424
|
typeof row.__deeplineRowKey === 'string'
|
|
3353
3425
|
? row.__deeplineRowKey
|
|
3354
|
-
: derivePlayRowIdentity(
|
|
3355
|
-
publicCsvInputRow(row),
|
|
3356
|
-
name,
|
|
3357
|
-
mapLogicFingerprint,
|
|
3358
|
-
);
|
|
3426
|
+
: derivePlayRowIdentity(publicCsvInputRow(row), name);
|
|
3359
3427
|
if (key) {
|
|
3360
3428
|
completedKeys.add(key);
|
|
3361
3429
|
preparedKeys.add(key);
|
|
@@ -3388,7 +3456,16 @@ function createMinimalWorkerCtx(
|
|
|
3388
3456
|
rowsToExecute.length,
|
|
3389
3457
|
);
|
|
3390
3458
|
const executedCellMetaPatches: Array<
|
|
3391
|
-
Record<
|
|
3459
|
+
| Record<
|
|
3460
|
+
string,
|
|
3461
|
+
{
|
|
3462
|
+
status: 'cached' | 'skipped';
|
|
3463
|
+
stage?: string | null;
|
|
3464
|
+
reused?: boolean;
|
|
3465
|
+
runId?: string;
|
|
3466
|
+
}
|
|
3467
|
+
>
|
|
3468
|
+
| undefined
|
|
3392
3469
|
> = new Array(rowsToExecute.length);
|
|
3393
3470
|
const toolBatchScheduler = new WorkerToolBatchScheduler(req);
|
|
3394
3471
|
const generatedOutputFields = new Set<string>();
|
|
@@ -3402,31 +3479,28 @@ function createMinimalWorkerCtx(
|
|
|
3402
3479
|
const myIndex = idx++;
|
|
3403
3480
|
if (myIndex >= rowsToExecute.length) return;
|
|
3404
3481
|
const entry = uniqueRowsToExecuteEntries[myIndex]!;
|
|
3405
|
-
const row = entry.
|
|
3482
|
+
const row = pendingRowsByKey.has(entry.rowKey)
|
|
3483
|
+
? ({
|
|
3484
|
+
...entry.row,
|
|
3485
|
+
...publicCsvInputRow(pendingRowsByKey.get(entry.rowKey)!),
|
|
3486
|
+
} as T & Record<string, unknown>)
|
|
3487
|
+
: entry.row;
|
|
3406
3488
|
const absoluteIndex = entry.absoluteIndex;
|
|
3407
3489
|
const enriched: Record<string, unknown> = cloneCsvAliasedRow(row);
|
|
3408
3490
|
const fieldOutputs: Record<string, unknown> = {};
|
|
3409
3491
|
const cellMetaPatch: Record<
|
|
3410
3492
|
string,
|
|
3411
|
-
{
|
|
3493
|
+
{
|
|
3494
|
+
status: 'cached' | 'skipped';
|
|
3495
|
+
stage?: string | null;
|
|
3496
|
+
reused?: boolean;
|
|
3497
|
+
runId?: string;
|
|
3498
|
+
}
|
|
3412
3499
|
> = {};
|
|
3413
3500
|
const waterfallOutputs: RecordedWaterfallOutput[] = [];
|
|
3414
3501
|
const stepProgramOutputs: RecordedStepProgramOutput[] = [];
|
|
3415
3502
|
const rowCtx = {
|
|
3416
3503
|
...(ctx as Record<string, unknown>),
|
|
3417
|
-
tool: async (
|
|
3418
|
-
key: string,
|
|
3419
|
-
toolId: string,
|
|
3420
|
-
input: Record<string, unknown>,
|
|
3421
|
-
): Promise<unknown> => {
|
|
3422
|
-
assertNotAborted(abortSignal);
|
|
3423
|
-
return await toolBatchScheduler.execute(
|
|
3424
|
-
key,
|
|
3425
|
-
toolId,
|
|
3426
|
-
input,
|
|
3427
|
-
workflowStep,
|
|
3428
|
-
);
|
|
3429
|
-
},
|
|
3430
3504
|
tools: {
|
|
3431
3505
|
...((ctx as { tools?: Record<string, unknown> }).tools ?? {}),
|
|
3432
3506
|
execute: async (requestArg: unknown): Promise<unknown> => {
|
|
@@ -3456,6 +3530,15 @@ function createMinimalWorkerCtx(
|
|
|
3456
3530
|
),
|
|
3457
3531
|
};
|
|
3458
3532
|
for (const [key, value] of fieldEntries) {
|
|
3533
|
+
if (isCompletedWorkerFieldValue(enriched[key])) {
|
|
3534
|
+
cellMetaPatch[key] = {
|
|
3535
|
+
status: 'cached',
|
|
3536
|
+
stage: key,
|
|
3537
|
+
reused: true,
|
|
3538
|
+
runId: req.runId,
|
|
3539
|
+
};
|
|
3540
|
+
continue;
|
|
3541
|
+
}
|
|
3459
3542
|
const resolved = await executeWorkerStepResolver(
|
|
3460
3543
|
value,
|
|
3461
3544
|
enriched,
|
|
@@ -3472,7 +3555,11 @@ function createMinimalWorkerCtx(
|
|
|
3472
3555
|
enriched[key] = resolved.value;
|
|
3473
3556
|
fieldOutputs[key] = resolved.value;
|
|
3474
3557
|
if (resolved.status === 'skipped') {
|
|
3475
|
-
cellMetaPatch[key] = {
|
|
3558
|
+
cellMetaPatch[key] = {
|
|
3559
|
+
status: 'skipped',
|
|
3560
|
+
stage: key,
|
|
3561
|
+
runId: req.runId,
|
|
3562
|
+
};
|
|
3476
3563
|
}
|
|
3477
3564
|
}
|
|
3478
3565
|
for (const stepOutput of stepProgramOutputs) {
|
|
@@ -3483,6 +3570,7 @@ function createMinimalWorkerCtx(
|
|
|
3483
3570
|
cellMetaPatch[stepOutput.columnName] = {
|
|
3484
3571
|
status: 'skipped',
|
|
3485
3572
|
stage: stepOutput.stepId,
|
|
3573
|
+
runId: req.runId,
|
|
3486
3574
|
};
|
|
3487
3575
|
}
|
|
3488
3576
|
}
|
|
@@ -3602,11 +3690,7 @@ function createMinimalWorkerCtx(
|
|
|
3602
3690
|
const key =
|
|
3603
3691
|
typeof completedRow.__deeplineRowKey === 'string'
|
|
3604
3692
|
? completedRow.__deeplineRowKey
|
|
3605
|
-
: derivePlayRowIdentity(
|
|
3606
|
-
publicCsvInputRow(completedRow),
|
|
3607
|
-
name,
|
|
3608
|
-
mapLogicFingerprint,
|
|
3609
|
-
);
|
|
3693
|
+
: derivePlayRowIdentity(publicCsvInputRow(completedRow), name);
|
|
3610
3694
|
if (key) {
|
|
3611
3695
|
const { __deeplineRowKey: _rowKey, ...cleanedRow } =
|
|
3612
3696
|
publicCsvInputRow(completedRow);
|
|
@@ -3866,7 +3950,11 @@ function createMinimalWorkerCtx(
|
|
|
3866
3950
|
ts: nowMs(),
|
|
3867
3951
|
});
|
|
3868
3952
|
},
|
|
3869
|
-
async step<T>(
|
|
3953
|
+
async step<T>(
|
|
3954
|
+
name: string,
|
|
3955
|
+
callback: () => Promise<T> | T,
|
|
3956
|
+
options?: { staleAfterSeconds?: number },
|
|
3957
|
+
): Promise<T> {
|
|
3870
3958
|
assertNotAborted(abortSignal);
|
|
3871
3959
|
const normalizedName = name.trim();
|
|
3872
3960
|
if (!normalizedName) {
|
|
@@ -3875,7 +3963,10 @@ function createMinimalWorkerCtx(
|
|
|
3875
3963
|
// Static pipeline JS blocks are already Workflow steps in the Workers
|
|
3876
3964
|
// backend. Nesting another `step.do` here can leave preview runs parked
|
|
3877
3965
|
// inside the JS stage before they reach subsequent event waits.
|
|
3878
|
-
return await executeWithRuntimeReceipt(
|
|
3966
|
+
return await executeWithRuntimeReceipt(
|
|
3967
|
+
`step:${normalizedName}${staleRuntimeSuffix(options?.staleAfterSeconds)}`,
|
|
3968
|
+
callback,
|
|
3969
|
+
);
|
|
3879
3970
|
},
|
|
3880
3971
|
async runSteps<T>(
|
|
3881
3972
|
program: WorkerStepProgram,
|
|
@@ -4032,39 +4123,16 @@ function createMinimalWorkerCtx(
|
|
|
4032
4123
|
'ctx.map(key, rows, fields, options) was removed. Use ctx.map(key, rows).step(...).run(options).',
|
|
4033
4124
|
);
|
|
4034
4125
|
},
|
|
4035
|
-
tool: async (
|
|
4036
|
-
key: string,
|
|
4037
|
-
toolId: string,
|
|
4038
|
-
input: Record<string, unknown>,
|
|
4039
|
-
): Promise<unknown> => {
|
|
4040
|
-
assertNotAborted(abortSignal);
|
|
4041
|
-
return await executeWithRuntimeReceipt(
|
|
4042
|
-
deriveToolRequestIdentity({ toolId, requestInput: input }),
|
|
4043
|
-
() =>
|
|
4044
|
-
executeToolWithLifecycle(
|
|
4045
|
-
req,
|
|
4046
|
-
{ id: key, toolId, input },
|
|
4047
|
-
workflowStep,
|
|
4048
|
-
callbacks,
|
|
4049
|
-
),
|
|
4050
|
-
);
|
|
4051
|
-
},
|
|
4052
4126
|
tools: {
|
|
4053
4127
|
async execute(requestArg: unknown): Promise<unknown> {
|
|
4054
4128
|
assertNotAborted(abortSignal);
|
|
4055
4129
|
const request = normalizeToolExecuteArgs(requestArg);
|
|
4056
4130
|
return await executeWithRuntimeReceipt(
|
|
4057
|
-
deriveToolRequestIdentity({
|
|
4131
|
+
`tool:${request.id}:${deriveToolRequestIdentity({
|
|
4058
4132
|
toolId: request.toolId,
|
|
4059
4133
|
requestInput: request.input,
|
|
4060
|
-
})
|
|
4061
|
-
() =>
|
|
4062
|
-
executeToolWithLifecycle(
|
|
4063
|
-
req,
|
|
4064
|
-
request,
|
|
4065
|
-
workflowStep,
|
|
4066
|
-
callbacks,
|
|
4067
|
-
),
|
|
4134
|
+
})}${staleRuntimeSuffix(request.staleAfterSeconds)}`,
|
|
4135
|
+
() => executeToolWithLifecycle(req, request, workflowStep, callbacks),
|
|
4068
4136
|
);
|
|
4069
4137
|
},
|
|
4070
4138
|
},
|
|
@@ -4123,13 +4191,21 @@ function createMinimalWorkerCtx(
|
|
|
4123
4191
|
key: string,
|
|
4124
4192
|
playRef: string | { playName?: string; name?: string },
|
|
4125
4193
|
input: Record<string, unknown>,
|
|
4126
|
-
options?: {
|
|
4194
|
+
options?: {
|
|
4195
|
+
description?: string;
|
|
4196
|
+
timeoutMs?: number;
|
|
4197
|
+
staleAfterSeconds?: number;
|
|
4198
|
+
},
|
|
4127
4199
|
): Promise<unknown> {
|
|
4128
4200
|
const normalizedKey = normalizeContextKey(key, 'runPlay');
|
|
4129
4201
|
const resolvedName = resolvePlayRefName(playRef);
|
|
4130
4202
|
if (!resolvedName) {
|
|
4131
4203
|
throw new Error('ctx.runPlay(...) requires a resolvable play name.');
|
|
4132
4204
|
}
|
|
4205
|
+
const receiptKey = `runPlay:${normalizedKey}:${await hashJson({
|
|
4206
|
+
childPlayName: resolvedName,
|
|
4207
|
+
input,
|
|
4208
|
+
})}${staleRuntimeSuffix(options?.staleAfterSeconds)}`;
|
|
4133
4209
|
if (ancestryPlayIds.includes(resolvedName)) {
|
|
4134
4210
|
const chain = [...ancestryPlayIds, resolvedName].join(' -> ');
|
|
4135
4211
|
throw new Error(`Recursive play graph detected: ${chain}`);
|
|
@@ -4154,163 +4230,180 @@ function createMinimalWorkerCtx(
|
|
|
4154
4230
|
`Child play-call cap exceeded for ${req.playName} (${nextParentCalls}/${WORKER_PLAY_CALL_LIMITS.maxChildPlayCallsPerParent}).`,
|
|
4155
4231
|
);
|
|
4156
4232
|
}
|
|
4157
|
-
|
|
4158
|
-
|
|
4233
|
+
return await executeWithRuntimeReceipt(receiptKey, async () => {
|
|
4234
|
+
playCallCount = nextPlayCallCount;
|
|
4235
|
+
parentChildCalls[req.playName] = nextParentCalls;
|
|
4159
4236
|
|
|
4160
|
-
|
|
4161
|
-
|
|
4162
|
-
|
|
4163
|
-
|
|
4164
|
-
|
|
4165
|
-
|
|
4166
|
-
|
|
4167
|
-
|
|
4168
|
-
throw new Error(
|
|
4169
|
-
`ctx.runPlay(${normalizedKey}) cannot start ${resolvedName}: missing trusted Cloudflare child manifest from top-level submit.`,
|
|
4170
|
-
);
|
|
4171
|
-
}
|
|
4172
|
-
const childIsMapBacked = childPipelineUsesCtxMap(
|
|
4173
|
-
childManifest.staticPipeline,
|
|
4174
|
-
);
|
|
4175
|
-
const childNeedsWorkflowScheduler = childPipelineNeedsWorkflowScheduler(
|
|
4176
|
-
childManifest.staticPipeline,
|
|
4177
|
-
);
|
|
4178
|
-
let childConcurrencyAcquired = false;
|
|
4179
|
-
let releaseChildPlaySlot: (() => void) | null = null;
|
|
4180
|
-
if (childIsMapBacked) {
|
|
4181
|
-
const nextInFlight =
|
|
4182
|
-
(inFlightChildCallsByPlayName[resolvedName] ?? 0) + 1;
|
|
4183
|
-
if (nextInFlight > 1) {
|
|
4237
|
+
emitEvent({
|
|
4238
|
+
type: 'log',
|
|
4239
|
+
level: 'info',
|
|
4240
|
+
message: `Starting child play ${resolvedName} (${normalizedKey})`,
|
|
4241
|
+
ts: nowMs(),
|
|
4242
|
+
});
|
|
4243
|
+
const childManifest = req.childPlayManifests?.[resolvedName];
|
|
4244
|
+
if (!childManifest) {
|
|
4184
4245
|
throw new Error(
|
|
4185
|
-
`
|
|
4186
|
-
'A child play that uses ctx.map() cannot run more than once at the same time because its map tables share durable row identity. ' +
|
|
4187
|
-
'Run these child play calls sequentially, or give each concurrent branch a different child play/table contract.',
|
|
4246
|
+
`ctx.runPlay(${normalizedKey}) cannot start ${resolvedName}: missing trusted Cloudflare child manifest from top-level submit.`,
|
|
4188
4247
|
);
|
|
4189
4248
|
}
|
|
4190
|
-
|
|
4191
|
-
|
|
4192
|
-
|
|
4193
|
-
|
|
4194
|
-
|
|
4195
|
-
|
|
4196
|
-
let
|
|
4197
|
-
|
|
4198
|
-
|
|
4199
|
-
|
|
4200
|
-
|
|
4201
|
-
|
|
4202
|
-
|
|
4203
|
-
|
|
4249
|
+
const childIsMapBacked = childPipelineUsesCtxMap(
|
|
4250
|
+
childManifest.staticPipeline,
|
|
4251
|
+
);
|
|
4252
|
+
const childNeedsWorkflowScheduler = childPipelineNeedsWorkflowScheduler(
|
|
4253
|
+
childManifest.staticPipeline,
|
|
4254
|
+
);
|
|
4255
|
+
let childConcurrencyAcquired = false;
|
|
4256
|
+
let releaseChildPlaySlot: (() => void) | null = null;
|
|
4257
|
+
if (childIsMapBacked) {
|
|
4258
|
+
const nextInFlight =
|
|
4259
|
+
(inFlightChildCallsByPlayName[resolvedName] ?? 0) + 1;
|
|
4260
|
+
if (nextInFlight > 1) {
|
|
4261
|
+
throw new Error(
|
|
4262
|
+
`Concurrent map-backed play call blocked for ${resolvedName}. ` +
|
|
4263
|
+
'A child play that uses ctx.map() cannot run more than once at the same time because its map tables share durable row identity. ' +
|
|
4264
|
+
'Run these child play calls sequentially, or give each concurrent branch a different child play/table contract.',
|
|
4265
|
+
);
|
|
4266
|
+
}
|
|
4267
|
+
inFlightChildCallsByPlayName[resolvedName] = nextInFlight;
|
|
4268
|
+
childConcurrencyAcquired = true;
|
|
4269
|
+
}
|
|
4204
4270
|
try {
|
|
4205
|
-
|
|
4206
|
-
|
|
4207
|
-
|
|
4208
|
-
|
|
4209
|
-
|
|
4210
|
-
|
|
4211
|
-
|
|
4212
|
-
|
|
4213
|
-
|
|
4214
|
-
|
|
4215
|
-
|
|
4216
|
-
|
|
4217
|
-
|
|
4218
|
-
|
|
4219
|
-
|
|
4220
|
-
|
|
4221
|
-
|
|
4222
|
-
|
|
4223
|
-
|
|
4224
|
-
|
|
4225
|
-
|
|
4226
|
-
|
|
4227
|
-
|
|
4228
|
-
|
|
4229
|
-
|
|
4230
|
-
|
|
4271
|
+
releaseChildPlaySlot = await acquireChildPlaySlot();
|
|
4272
|
+
const childSubmitStartedAt = nowMs();
|
|
4273
|
+
let started: {
|
|
4274
|
+
workflowId?: string;
|
|
4275
|
+
runId?: string;
|
|
4276
|
+
status?: string;
|
|
4277
|
+
output?: unknown;
|
|
4278
|
+
result?: unknown;
|
|
4279
|
+
error?: unknown;
|
|
4280
|
+
};
|
|
4281
|
+
try {
|
|
4282
|
+
started = await submitChildPlayThroughCoordinator({
|
|
4283
|
+
req,
|
|
4284
|
+
allowInline:
|
|
4285
|
+
options?.timeoutMs == null && !childNeedsWorkflowScheduler,
|
|
4286
|
+
body: {
|
|
4287
|
+
name: resolvedName,
|
|
4288
|
+
input: isRecord(input) ? input : {},
|
|
4289
|
+
orgId: req.orgId,
|
|
4290
|
+
parentExecutorToken: req.executorToken,
|
|
4291
|
+
userEmail: req.userEmail ?? '',
|
|
4292
|
+
profile: 'workers_edge',
|
|
4293
|
+
manifest: childManifest,
|
|
4294
|
+
childPlayManifests: req.childPlayManifests ?? null,
|
|
4295
|
+
internalRunPlay: {
|
|
4296
|
+
rootRunId,
|
|
4297
|
+
parentRunId: req.runId,
|
|
4298
|
+
parentPlayName: req.playName,
|
|
4299
|
+
key: normalizedKey,
|
|
4300
|
+
// Per the lineage validator: ancestry tail must equal the
|
|
4301
|
+
// executor token's play name (the parent making this call).
|
|
4302
|
+
ancestryPlayIds,
|
|
4303
|
+
callDepth: nextDepth,
|
|
4304
|
+
description:
|
|
4305
|
+
typeof options?.description === 'string'
|
|
4306
|
+
? options.description
|
|
4307
|
+
: null,
|
|
4308
|
+
},
|
|
4231
4309
|
},
|
|
4232
|
-
}
|
|
4233
|
-
})
|
|
4234
|
-
|
|
4310
|
+
});
|
|
4311
|
+
} catch (error) {
|
|
4312
|
+
console.info('[play.runtime.span]', {
|
|
4313
|
+
event: 'play.runtime.span',
|
|
4314
|
+
phase: 'child_submit',
|
|
4315
|
+
runId: req.runId,
|
|
4316
|
+
parentRunId: req.runId,
|
|
4317
|
+
playName: resolvedName,
|
|
4318
|
+
graphHash: req.graphHash ?? null,
|
|
4319
|
+
depth: nextDepth,
|
|
4320
|
+
fanoutIndex: nextParentCalls - 1,
|
|
4321
|
+
ms: nowMs() - childSubmitStartedAt,
|
|
4322
|
+
status: 'failed',
|
|
4323
|
+
errorCode: 'CHILD_SUBMIT_FAILED',
|
|
4324
|
+
});
|
|
4325
|
+
throw error;
|
|
4326
|
+
}
|
|
4327
|
+
const workflowId = started.workflowId ?? started.runId;
|
|
4328
|
+
if (!workflowId) {
|
|
4329
|
+
const startedError = isRecord(started.error)
|
|
4330
|
+
? started.error
|
|
4331
|
+
: { message: started.error };
|
|
4332
|
+
const startedErrorMessage =
|
|
4333
|
+
typeof startedError.message === 'string' &&
|
|
4334
|
+
startedError.message.trim()
|
|
4335
|
+
? startedError.message.trim()
|
|
4336
|
+
: null;
|
|
4337
|
+
throw new Error(
|
|
4338
|
+
startedErrorMessage ??
|
|
4339
|
+
`ctx.runPlay(${normalizedKey}) did not receive a child workflow id.`,
|
|
4340
|
+
);
|
|
4341
|
+
}
|
|
4235
4342
|
console.info('[play.runtime.span]', {
|
|
4236
4343
|
event: 'play.runtime.span',
|
|
4237
4344
|
phase: 'child_submit',
|
|
4238
4345
|
runId: req.runId,
|
|
4239
4346
|
parentRunId: req.runId,
|
|
4347
|
+
childRunId: workflowId,
|
|
4240
4348
|
playName: resolvedName,
|
|
4241
4349
|
graphHash: req.graphHash ?? null,
|
|
4242
4350
|
depth: nextDepth,
|
|
4243
4351
|
fanoutIndex: nextParentCalls - 1,
|
|
4244
4352
|
ms: nowMs() - childSubmitStartedAt,
|
|
4245
|
-
status: '
|
|
4246
|
-
errorCode: 'CHILD_SUBMIT_FAILED',
|
|
4353
|
+
status: 'ok',
|
|
4247
4354
|
});
|
|
4248
|
-
|
|
4249
|
-
|
|
4250
|
-
|
|
4251
|
-
|
|
4252
|
-
|
|
4253
|
-
|
|
4254
|
-
|
|
4255
|
-
|
|
4256
|
-
|
|
4257
|
-
|
|
4258
|
-
|
|
4259
|
-
|
|
4260
|
-
|
|
4261
|
-
|
|
4262
|
-
|
|
4263
|
-
|
|
4264
|
-
|
|
4265
|
-
|
|
4266
|
-
|
|
4267
|
-
|
|
4268
|
-
|
|
4269
|
-
|
|
4270
|
-
|
|
4271
|
-
|
|
4272
|
-
|
|
4273
|
-
|
|
4274
|
-
|
|
4275
|
-
|
|
4276
|
-
|
|
4277
|
-
|
|
4278
|
-
|
|
4279
|
-
|
|
4280
|
-
|
|
4281
|
-
|
|
4282
|
-
|
|
4283
|
-
|
|
4284
|
-
|
|
4285
|
-
|
|
4286
|
-
|
|
4287
|
-
|
|
4288
|
-
|
|
4289
|
-
|
|
4290
|
-
|
|
4291
|
-
|
|
4292
|
-
|
|
4293
|
-
|
|
4294
|
-
|
|
4295
|
-
|
|
4296
|
-
:
|
|
4297
|
-
|
|
4298
|
-
|
|
4299
|
-
|
|
4300
|
-
let result: unknown;
|
|
4301
|
-
try {
|
|
4302
|
-
result = await waitForChildPlayTerminalEvent({
|
|
4303
|
-
req,
|
|
4304
|
-
workflowStep,
|
|
4305
|
-
workflowId,
|
|
4306
|
-
playName: resolvedName,
|
|
4307
|
-
key: normalizedKey,
|
|
4308
|
-
timeoutMs: Math.max(
|
|
4309
|
-
1_000,
|
|
4310
|
-
Math.min(options?.timeoutMs ?? 5 * 60_000, 30 * 60_000),
|
|
4311
|
-
),
|
|
4312
|
-
});
|
|
4313
|
-
} catch (error) {
|
|
4355
|
+
const startedStatus = String(started.status ?? '').toLowerCase();
|
|
4356
|
+
if (startedStatus === 'completed') {
|
|
4357
|
+
emitEvent({
|
|
4358
|
+
type: 'log',
|
|
4359
|
+
level: 'info',
|
|
4360
|
+
message: `Completed child play ${resolvedName} (${normalizedKey})`,
|
|
4361
|
+
ts: nowMs(),
|
|
4362
|
+
});
|
|
4363
|
+
return started.output ?? extractChildPlayOutput(started);
|
|
4364
|
+
}
|
|
4365
|
+
if (startedStatus === 'failed') {
|
|
4366
|
+
const startedError = isRecord(started.error)
|
|
4367
|
+
? started.error
|
|
4368
|
+
: { message: started.error };
|
|
4369
|
+
const startedErrorMessage =
|
|
4370
|
+
typeof startedError.message === 'string' &&
|
|
4371
|
+
startedError.message.trim()
|
|
4372
|
+
? startedError.message.trim()
|
|
4373
|
+
: `Child play ${resolvedName} (${workflowId}) failed.`;
|
|
4374
|
+
throw new Error(startedErrorMessage);
|
|
4375
|
+
}
|
|
4376
|
+
const childWaitStartedAt = nowMs();
|
|
4377
|
+
let result: unknown;
|
|
4378
|
+
try {
|
|
4379
|
+
result = await waitForChildPlayTerminalEvent({
|
|
4380
|
+
req,
|
|
4381
|
+
workflowStep,
|
|
4382
|
+
workflowId,
|
|
4383
|
+
playName: resolvedName,
|
|
4384
|
+
key: normalizedKey,
|
|
4385
|
+
timeoutMs: Math.max(
|
|
4386
|
+
1_000,
|
|
4387
|
+
Math.min(options?.timeoutMs ?? 5 * 60_000, 30 * 60_000),
|
|
4388
|
+
),
|
|
4389
|
+
});
|
|
4390
|
+
} catch (error) {
|
|
4391
|
+
console.info('[play.runtime.span]', {
|
|
4392
|
+
event: 'play.runtime.span',
|
|
4393
|
+
phase: 'child_wait',
|
|
4394
|
+
runId: req.runId,
|
|
4395
|
+
parentRunId: req.runId,
|
|
4396
|
+
childRunId: workflowId,
|
|
4397
|
+
playName: resolvedName,
|
|
4398
|
+
graphHash: req.graphHash ?? null,
|
|
4399
|
+
depth: nextDepth,
|
|
4400
|
+
fanoutIndex: nextParentCalls - 1,
|
|
4401
|
+
ms: nowMs() - childWaitStartedAt,
|
|
4402
|
+
status: 'failed',
|
|
4403
|
+
errorCode: 'CHILD_WAIT_FAILED',
|
|
4404
|
+
});
|
|
4405
|
+
throw error;
|
|
4406
|
+
}
|
|
4314
4407
|
console.info('[play.runtime.span]', {
|
|
4315
4408
|
event: 'play.runtime.span',
|
|
4316
4409
|
phase: 'child_wait',
|
|
@@ -4322,40 +4415,25 @@ function createMinimalWorkerCtx(
|
|
|
4322
4415
|
depth: nextDepth,
|
|
4323
4416
|
fanoutIndex: nextParentCalls - 1,
|
|
4324
4417
|
ms: nowMs() - childWaitStartedAt,
|
|
4325
|
-
status: '
|
|
4326
|
-
errorCode: 'CHILD_WAIT_FAILED',
|
|
4418
|
+
status: 'ok',
|
|
4327
4419
|
});
|
|
4328
|
-
|
|
4329
|
-
|
|
4330
|
-
|
|
4331
|
-
|
|
4332
|
-
|
|
4333
|
-
|
|
4334
|
-
|
|
4335
|
-
|
|
4336
|
-
|
|
4337
|
-
|
|
4338
|
-
|
|
4339
|
-
|
|
4340
|
-
|
|
4341
|
-
|
|
4342
|
-
|
|
4343
|
-
emitEvent({
|
|
4344
|
-
type: 'log',
|
|
4345
|
-
level: 'info',
|
|
4346
|
-
message: `Completed child play ${resolvedName} (${normalizedKey})`,
|
|
4347
|
-
ts: nowMs(),
|
|
4348
|
-
});
|
|
4349
|
-
return result;
|
|
4350
|
-
} finally {
|
|
4351
|
-
releaseChildPlaySlot?.();
|
|
4352
|
-
if (childConcurrencyAcquired) {
|
|
4353
|
-
releaseChildPlayConcurrency(
|
|
4354
|
-
inFlightChildCallsByPlayName,
|
|
4355
|
-
resolvedName,
|
|
4356
|
-
);
|
|
4420
|
+
emitEvent({
|
|
4421
|
+
type: 'log',
|
|
4422
|
+
level: 'info',
|
|
4423
|
+
message: `Completed child play ${resolvedName} (${normalizedKey})`,
|
|
4424
|
+
ts: nowMs(),
|
|
4425
|
+
});
|
|
4426
|
+
return result;
|
|
4427
|
+
} finally {
|
|
4428
|
+
releaseChildPlaySlot?.();
|
|
4429
|
+
if (childConcurrencyAcquired) {
|
|
4430
|
+
releaseChildPlayConcurrency(
|
|
4431
|
+
inFlightChildCallsByPlayName,
|
|
4432
|
+
resolvedName,
|
|
4433
|
+
);
|
|
4434
|
+
}
|
|
4357
4435
|
}
|
|
4358
|
-
}
|
|
4436
|
+
});
|
|
4359
4437
|
},
|
|
4360
4438
|
fetch(): never {
|
|
4361
4439
|
throw new Error(
|