deepline 0.1.46 → 0.1.48
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -14
- package/dist/cli/index.js +85 -25
- package/dist/cli/index.mjs +85 -25
- package/dist/index.d.mts +59 -35
- package/dist/index.d.ts +59 -35
- package/dist/index.js +5 -9
- package/dist/index.mjs +5 -9
- package/dist/repo/apps/play-runner-workers/src/entry.ts +380 -305
- package/dist/repo/apps/play-runner-workers/src/runtime/receipts.ts +51 -21
- package/dist/repo/sdk/src/play.ts +35 -42
- package/dist/repo/sdk/src/plays/harness-stub.ts +1 -1
- package/dist/repo/sdk/src/types.ts +26 -3
- package/dist/repo/sdk/src/version.ts +1 -1
- package/dist/repo/sdk/src/worker-play-entry.ts +17 -67
- package/dist/repo/shared_libs/plays/row-identity.ts +5 -59
- package/package.json +1 -1
|
@@ -18,12 +18,12 @@
|
|
|
18
18
|
* bundled into the Worker script.
|
|
19
19
|
* - Workers don't have node:fs / source-map-support. Stack traces are raw.
|
|
20
20
|
* - Direct postgres (`pg` library) won't bundle for Workers. This harness
|
|
21
|
-
* uses HTTP-only ctx — every ctx.csv / ctx.
|
|
21
|
+
* uses HTTP-only ctx — every ctx.csv / ctx.tools.execute / row write goes through
|
|
22
22
|
* the runtime API endpoint, not direct DB. That keeps the Worker bundle
|
|
23
23
|
* compatible with the V8 isolate runtime.
|
|
24
24
|
*
|
|
25
25
|
* Status: experimental. First cut targets tool-basic (ctx.csv + ctx.map +
|
|
26
|
-
* ctx.
|
|
26
|
+
* ctx.tools.execute). Plays that depend on the full ctx surface (durable sleep,
|
|
27
27
|
* checkpoints, batched waterfalls, etc.) will fall back to "not implemented"
|
|
28
28
|
* rather than producing wrong results — opt-in via DEEPLINE_PLAY_RUNNER_BACKEND.
|
|
29
29
|
*/
|
|
@@ -217,7 +217,10 @@ function getStringField(value: unknown, key: string): string | null {
|
|
|
217
217
|
return typeof field === 'string' && field.trim() ? field : null;
|
|
218
218
|
}
|
|
219
219
|
|
|
220
|
-
function getObjectField(
|
|
220
|
+
function getObjectField(
|
|
221
|
+
value: unknown,
|
|
222
|
+
key: string,
|
|
223
|
+
): Record<string, unknown> | null {
|
|
221
224
|
if (!isRecord(value)) return null;
|
|
222
225
|
const field = value[key];
|
|
223
226
|
return isRecord(field) ? field : null;
|
|
@@ -269,10 +272,12 @@ function normalizeToolHttpErrorMessage(input: {
|
|
|
269
272
|
const billing = getObjectField(parsed, 'billing');
|
|
270
273
|
if (isInsufficientCreditsBilling(billing)) {
|
|
271
274
|
return new ToolHttpError(
|
|
272
|
-
`tool ${input.toolId} ${input.status} attempt ${input.attempt}/${input.maxAttempts}: ${formatInsufficientCreditsMessage(
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
275
|
+
`tool ${input.toolId} ${input.status} attempt ${input.attempt}/${input.maxAttempts}: ${formatInsufficientCreditsMessage(
|
|
276
|
+
{
|
|
277
|
+
billing,
|
|
278
|
+
toolId: input.toolId,
|
|
279
|
+
},
|
|
280
|
+
)}`,
|
|
276
281
|
billing,
|
|
277
282
|
);
|
|
278
283
|
}
|
|
@@ -451,7 +456,9 @@ async function probeHarnessOnce(
|
|
|
451
456
|
*/
|
|
452
457
|
const RUNTIME_API_TIMEOUT_MS = 30_000;
|
|
453
458
|
const RUNTIME_API_PLAY_RUN_TIMEOUT_MS = 75_000;
|
|
454
|
-
const RUNTIME_API_RETRY_DELAYS_MS = [
|
|
459
|
+
const RUNTIME_API_RETRY_DELAYS_MS = [
|
|
460
|
+
250, 750, 1500, 3000, 5000, 10000,
|
|
461
|
+
] as const;
|
|
455
462
|
let loggedMissingRuntimeApiBinding = false;
|
|
456
463
|
|
|
457
464
|
async function fetchRuntimeApi(
|
|
@@ -496,7 +503,21 @@ async function fetchRuntimeApi(
|
|
|
496
503
|
const responsePromise = cachedRuntimeApiBinding.fetch(
|
|
497
504
|
new Request(`${baseUrl.replace(/\/$/, '')}${path}`, mergedInit),
|
|
498
505
|
);
|
|
499
|
-
|
|
506
|
+
const response = await Promise.race([responsePromise, timeoutPromise]);
|
|
507
|
+
if (await shouldFallbackRuntimeApiBindingResponse(response)) {
|
|
508
|
+
console.warn(
|
|
509
|
+
`[play-harness] RUNTIME_API binding returned coordinator not found; using public runtime API transport. path=${path}`,
|
|
510
|
+
);
|
|
511
|
+
return await Promise.race([
|
|
512
|
+
fetch(`${baseUrl.replace(/\/$/, '')}${path}`, {
|
|
513
|
+
...init,
|
|
514
|
+
headers: runtimeApiHeaders(init.headers, true),
|
|
515
|
+
signal: controller.signal,
|
|
516
|
+
}),
|
|
517
|
+
timeoutPromise,
|
|
518
|
+
]);
|
|
519
|
+
}
|
|
520
|
+
return response;
|
|
500
521
|
} catch (err) {
|
|
501
522
|
if (err instanceof Error && err.name === 'AbortError') {
|
|
502
523
|
throw new Error(
|
|
@@ -509,6 +530,19 @@ async function fetchRuntimeApi(
|
|
|
509
530
|
}
|
|
510
531
|
}
|
|
511
532
|
|
|
533
|
+
async function shouldFallbackRuntimeApiBindingResponse(
|
|
534
|
+
response: Response,
|
|
535
|
+
): Promise<boolean> {
|
|
536
|
+
if (response.status !== 404) {
|
|
537
|
+
return false;
|
|
538
|
+
}
|
|
539
|
+
const body = await response
|
|
540
|
+
.clone()
|
|
541
|
+
.text()
|
|
542
|
+
.catch(() => '');
|
|
543
|
+
return body.trim().toLowerCase() === 'not found';
|
|
544
|
+
}
|
|
545
|
+
|
|
512
546
|
function runtimeApiHeaders(
|
|
513
547
|
headers: HeadersInit | undefined,
|
|
514
548
|
includeVercelBypass: boolean,
|
|
@@ -922,18 +956,20 @@ function extractChildPlayOutput(status: Record<string, unknown>): unknown {
|
|
|
922
956
|
return result ?? null;
|
|
923
957
|
}
|
|
924
958
|
|
|
925
|
-
function hashChildPlayEventKey(input:
|
|
926
|
-
|
|
927
|
-
for (let index = 0; index < input.length; index += 1) {
|
|
928
|
-
hash ^= input.charCodeAt(index);
|
|
929
|
-
hash = Math.imul(hash, 16777619);
|
|
930
|
-
}
|
|
931
|
-
return (hash >>> 0).toString(36);
|
|
959
|
+
async function hashChildPlayEventKey(input: unknown): Promise<string> {
|
|
960
|
+
return (await hashJson(input)).slice(0, 32);
|
|
932
961
|
}
|
|
933
962
|
|
|
934
|
-
function childPlayEventKey(input: {
|
|
963
|
+
async function childPlayEventKey(input: {
|
|
964
|
+
key: string;
|
|
965
|
+
workflowId: string;
|
|
966
|
+
}): Promise<string> {
|
|
935
967
|
const readableKey = workflowEventType(input.key).slice(0, 40);
|
|
936
|
-
|
|
968
|
+
const digest = await hashChildPlayEventKey({
|
|
969
|
+
key: input.key,
|
|
970
|
+
workflowId: input.workflowId,
|
|
971
|
+
});
|
|
972
|
+
return `child_play_${digest}_${readableKey}`;
|
|
937
973
|
}
|
|
938
974
|
|
|
939
975
|
function workflowTimeoutFromMs(timeoutMs: number): string {
|
|
@@ -954,7 +990,7 @@ async function waitForChildPlayTerminalEvent(input: {
|
|
|
954
990
|
'ctx.runPlay child waits require the cf-workflows runtime event scheduler.',
|
|
955
991
|
);
|
|
956
992
|
}
|
|
957
|
-
const eventKey = childPlayEventKey({
|
|
993
|
+
const eventKey = await childPlayEventKey({
|
|
958
994
|
key: input.key,
|
|
959
995
|
workflowId: input.workflowId,
|
|
960
996
|
});
|
|
@@ -989,7 +1025,7 @@ async function signalParentPlayTerminal(input: {
|
|
|
989
1025
|
}): Promise<void> {
|
|
990
1026
|
const governance = input.req.playCallGovernance;
|
|
991
1027
|
if (!governance?.parentRunId || !governance.key) return;
|
|
992
|
-
const eventKey = childPlayEventKey({
|
|
1028
|
+
const eventKey = await childPlayEventKey({
|
|
993
1029
|
key: governance.key,
|
|
994
1030
|
workflowId: input.req.runId,
|
|
995
1031
|
});
|
|
@@ -1098,9 +1134,12 @@ function isToolExecuteRecord(value: unknown): value is Record<string, unknown> {
|
|
|
1098
1134
|
return typeof value === 'object' && value !== null && !Array.isArray(value);
|
|
1099
1135
|
}
|
|
1100
1136
|
|
|
1101
|
-
function normalizeToolExecuteArgs(
|
|
1102
|
-
|
|
1103
|
-
|
|
1137
|
+
function normalizeToolExecuteArgs(request: unknown): {
|
|
1138
|
+
id: string;
|
|
1139
|
+
toolId: string;
|
|
1140
|
+
input: Record<string, unknown>;
|
|
1141
|
+
staleAfterSeconds?: number;
|
|
1142
|
+
} {
|
|
1104
1143
|
if (!isToolExecuteRecord(request)) {
|
|
1105
1144
|
throw new Error(
|
|
1106
1145
|
'ctx.tools.execute requires a request object: ctx.tools.execute({ id, tool, input, description }).',
|
|
@@ -1117,7 +1156,14 @@ function normalizeToolExecuteArgs(
|
|
|
1117
1156
|
'ctx.tools.execute({ id, tool, input }) requires a non-empty id, tool string, and input object.',
|
|
1118
1157
|
);
|
|
1119
1158
|
}
|
|
1120
|
-
return {
|
|
1159
|
+
return {
|
|
1160
|
+
id: request.id.trim(),
|
|
1161
|
+
toolId: request.tool,
|
|
1162
|
+
input: request.input,
|
|
1163
|
+
...(typeof request.staleAfterSeconds === 'number'
|
|
1164
|
+
? { staleAfterSeconds: request.staleAfterSeconds }
|
|
1165
|
+
: {}),
|
|
1166
|
+
};
|
|
1121
1167
|
}
|
|
1122
1168
|
|
|
1123
1169
|
function integrationEventType(eventKey: string): string {
|
|
@@ -1329,7 +1375,11 @@ function parseExtractorMetadata(
|
|
|
1329
1375
|
}
|
|
1330
1376
|
const entries = Object.entries(value as Record<string, unknown>).flatMap(
|
|
1331
1377
|
([key, descriptor]) => {
|
|
1332
|
-
if (
|
|
1378
|
+
if (
|
|
1379
|
+
!descriptor ||
|
|
1380
|
+
typeof descriptor !== 'object' ||
|
|
1381
|
+
Array.isArray(descriptor)
|
|
1382
|
+
) {
|
|
1333
1383
|
return [];
|
|
1334
1384
|
}
|
|
1335
1385
|
const record = descriptor as Record<string, unknown>;
|
|
@@ -1630,17 +1680,14 @@ type WorkerInlineWaterfallSpec = {
|
|
|
1630
1680
|
input: Record<string, unknown>,
|
|
1631
1681
|
ctx: {
|
|
1632
1682
|
tools: {
|
|
1633
|
-
execute(
|
|
1634
|
-
|
|
1635
|
-
|
|
1636
|
-
input: Record<string, unknown
|
|
1637
|
-
|
|
1683
|
+
execute(request: {
|
|
1684
|
+
id: string;
|
|
1685
|
+
tool: string;
|
|
1686
|
+
input: Record<string, unknown>;
|
|
1687
|
+
description?: string;
|
|
1688
|
+
staleAfterSeconds?: number;
|
|
1689
|
+
}): Promise<unknown>;
|
|
1638
1690
|
};
|
|
1639
|
-
tool(
|
|
1640
|
-
key: string,
|
|
1641
|
-
toolId: string,
|
|
1642
|
-
input: Record<string, unknown>,
|
|
1643
|
-
): Promise<unknown>;
|
|
1644
1691
|
},
|
|
1645
1692
|
) => unknown | Promise<unknown>;
|
|
1646
1693
|
}
|
|
@@ -1907,6 +1954,14 @@ function extractWorkerInlineCodeStepValue(
|
|
|
1907
1954
|
return result ?? null;
|
|
1908
1955
|
}
|
|
1909
1956
|
|
|
1957
|
+
function isCompletedWorkerFieldValue(value: unknown): boolean {
|
|
1958
|
+
return (
|
|
1959
|
+
value !== null &&
|
|
1960
|
+
value !== undefined &&
|
|
1961
|
+
!(typeof value === 'string' && value.length === 0)
|
|
1962
|
+
);
|
|
1963
|
+
}
|
|
1964
|
+
|
|
1910
1965
|
type WorkerMapChunkSummary<T extends Record<string, unknown>> = {
|
|
1911
1966
|
chunkIndex: number;
|
|
1912
1967
|
rangeStart: number;
|
|
@@ -1929,7 +1984,9 @@ function serializeDurableStepValue<T>(value: T, depth = 0): T {
|
|
|
1929
1984
|
if (isToolExecuteResult(value)) return serializeToolExecuteResult(value) as T;
|
|
1930
1985
|
if (isDatasetHandle(value)) return serializeValue(value, depth) as T;
|
|
1931
1986
|
if (Array.isArray(value)) {
|
|
1932
|
-
return value.map((entry) =>
|
|
1987
|
+
return value.map((entry) =>
|
|
1988
|
+
serializeDurableStepValue(entry, depth + 1),
|
|
1989
|
+
) as T;
|
|
1933
1990
|
}
|
|
1934
1991
|
if (typeof value !== 'object') return value;
|
|
1935
1992
|
return Object.fromEntries(
|
|
@@ -2087,9 +2144,9 @@ async function executeWorkerStepProgram(
|
|
|
2087
2144
|
recorder
|
|
2088
2145
|
? {
|
|
2089
2146
|
...recorder,
|
|
2090
|
-
|
|
2091
|
-
|
|
2092
|
-
|
|
2147
|
+
path: stepPath,
|
|
2148
|
+
}
|
|
2149
|
+
: undefined,
|
|
2093
2150
|
);
|
|
2094
2151
|
return {
|
|
2095
2152
|
value: serializeDurableStepValue(resolution.value),
|
|
@@ -2160,13 +2217,6 @@ async function executeWorkerWaterfall(
|
|
|
2160
2217
|
callbacks,
|
|
2161
2218
|
),
|
|
2162
2219
|
},
|
|
2163
|
-
tool: async (key, toolId, toolInput) =>
|
|
2164
|
-
await executeToolWithLifecycle(
|
|
2165
|
-
req,
|
|
2166
|
-
{ id: key, toolId, input: toolInput },
|
|
2167
|
-
workflowStep,
|
|
2168
|
-
callbacks,
|
|
2169
|
-
),
|
|
2170
2220
|
});
|
|
2171
2221
|
} else {
|
|
2172
2222
|
result = await executeToolWithLifecycle(
|
|
@@ -2884,25 +2934,32 @@ function augmentSheetContractWithDatasetFields(input: {
|
|
|
2884
2934
|
input.contract.columns.map((column) => column.sqlName),
|
|
2885
2935
|
);
|
|
2886
2936
|
const columns = [...input.contract.columns];
|
|
2937
|
+
const candidateFields = new Set<string>();
|
|
2887
2938
|
for (const row of input.rows) {
|
|
2888
2939
|
for (const field of Object.keys(row)) {
|
|
2889
|
-
|
|
2890
|
-
continue;
|
|
2891
|
-
}
|
|
2892
|
-
const sqlName = sqlSafePlayColumnName(field);
|
|
2893
|
-
if (existingSqlNames.has(sqlName)) {
|
|
2894
|
-
continue;
|
|
2895
|
-
}
|
|
2896
|
-
existingFields.add(field);
|
|
2897
|
-
existingSqlNames.add(sqlName);
|
|
2898
|
-
columns.push({
|
|
2899
|
-
id: `runtime:${input.contract.tableNamespace}:${field}`,
|
|
2900
|
-
sqlName,
|
|
2901
|
-
source: outputFields.has(field) ? 'mapField' : 'input',
|
|
2902
|
-
field,
|
|
2903
|
-
});
|
|
2940
|
+
candidateFields.add(field);
|
|
2904
2941
|
}
|
|
2905
2942
|
}
|
|
2943
|
+
for (const field of outputFields) {
|
|
2944
|
+
candidateFields.add(field);
|
|
2945
|
+
}
|
|
2946
|
+
for (const field of candidateFields) {
|
|
2947
|
+
if (!isDatasetPayloadField(field) || existingFields.has(field)) {
|
|
2948
|
+
continue;
|
|
2949
|
+
}
|
|
2950
|
+
const sqlName = sqlSafePlayColumnName(field);
|
|
2951
|
+
if (existingSqlNames.has(sqlName)) {
|
|
2952
|
+
continue;
|
|
2953
|
+
}
|
|
2954
|
+
existingFields.add(field);
|
|
2955
|
+
existingSqlNames.add(sqlName);
|
|
2956
|
+
columns.push({
|
|
2957
|
+
id: `runtime:${input.contract.tableNamespace}:${field}`,
|
|
2958
|
+
sqlName,
|
|
2959
|
+
source: outputFields.has(field) ? 'mapField' : 'input',
|
|
2960
|
+
field,
|
|
2961
|
+
});
|
|
2962
|
+
}
|
|
2906
2963
|
return { ...input.contract, columns };
|
|
2907
2964
|
}
|
|
2908
2965
|
|
|
@@ -2942,6 +2999,7 @@ async function prepareMapRows(input: {
|
|
|
2942
2999
|
req: RunRequest;
|
|
2943
3000
|
tableNamespace: string;
|
|
2944
3001
|
rows: Record<string, unknown>[];
|
|
3002
|
+
outputFields: string[];
|
|
2945
3003
|
}): Promise<{
|
|
2946
3004
|
inserted: number;
|
|
2947
3005
|
skipped: number;
|
|
@@ -2960,6 +3018,7 @@ async function prepareMapRows(input: {
|
|
|
2960
3018
|
sheetContract: augmentSheetContractWithDatasetFields({
|
|
2961
3019
|
contract: requireSheetContract(input.req, input.tableNamespace),
|
|
2962
3020
|
rows: input.rows,
|
|
3021
|
+
outputFields: input.outputFields,
|
|
2963
3022
|
}),
|
|
2964
3023
|
rows: input.rows.map((row) => ({ ...row })),
|
|
2965
3024
|
runId: input.req.runId,
|
|
@@ -3003,7 +3062,7 @@ async function prepareMapRows(input: {
|
|
|
3003
3062
|
* - ctx.log(msg)
|
|
3004
3063
|
* - ctx.csv(filename | inline rows) (calls runtime API for file resolve)
|
|
3005
3064
|
* - ctx.map(name, rows, fields, opts)
|
|
3006
|
-
* - ctx.tools.execute(
|
|
3065
|
+
* - ctx.tools.execute({ id, tool, input, ... })
|
|
3007
3066
|
* - ctx.runPlay(key, playRef, input, opts)
|
|
3008
3067
|
*
|
|
3009
3068
|
* Not supported (will throw):
|
|
@@ -3128,16 +3187,32 @@ function createMinimalWorkerCtx(
|
|
|
3128
3187
|
const executeWithRuntimeReceipt = async <T>(
|
|
3129
3188
|
key: string,
|
|
3130
3189
|
execute: () => Promise<T> | T,
|
|
3131
|
-
): Promise<T> =>
|
|
3132
|
-
runWorkerRuntimeReceiptBoundary({
|
|
3190
|
+
): Promise<T> => {
|
|
3191
|
+
const serialized = await runWorkerRuntimeReceiptBoundary<unknown>({
|
|
3133
3192
|
baseUrl: req.baseUrl,
|
|
3134
3193
|
executorToken: req.executorToken,
|
|
3194
|
+
orgId: req.orgId,
|
|
3135
3195
|
playName: req.playName,
|
|
3136
3196
|
runId: req.runId,
|
|
3137
3197
|
key,
|
|
3138
3198
|
postRuntimeApi,
|
|
3139
|
-
execute,
|
|
3199
|
+
execute: async () => serializeDurableStepValue(await execute()),
|
|
3140
3200
|
});
|
|
3201
|
+
return deserializeDurableStepValue(serialized) as T;
|
|
3202
|
+
};
|
|
3203
|
+
const staleRuntimeSuffix = (staleAfterSeconds?: number): string => {
|
|
3204
|
+
if (staleAfterSeconds === undefined) return '';
|
|
3205
|
+
if (
|
|
3206
|
+
!Number.isFinite(staleAfterSeconds) ||
|
|
3207
|
+
!Number.isInteger(staleAfterSeconds) ||
|
|
3208
|
+
staleAfterSeconds <= 0
|
|
3209
|
+
) {
|
|
3210
|
+
throw new Error(
|
|
3211
|
+
'staleAfterSeconds must be a positive whole number of seconds.',
|
|
3212
|
+
);
|
|
3213
|
+
}
|
|
3214
|
+
return `:stale:${staleAfterSeconds}:${Math.floor(nowMs() / (staleAfterSeconds * 1000))}`;
|
|
3215
|
+
};
|
|
3141
3216
|
// Local ancestry chain that always ENDS with the currently-executing play
|
|
3142
3217
|
// (req.playName). The /api/v2/plays/run lineage validator requires the
|
|
3143
3218
|
// submitted ancestry's tail to equal the executor token's play name (i.e.
|
|
@@ -3253,7 +3328,6 @@ function createMinimalWorkerCtx(
|
|
|
3253
3328
|
: JSON.stringify(normalizedParts);
|
|
3254
3329
|
return keyValue;
|
|
3255
3330
|
};
|
|
3256
|
-
const mapLogicFingerprint = req.graphHash ?? null;
|
|
3257
3331
|
const resolveRowKey = (
|
|
3258
3332
|
row: Record<string, unknown>,
|
|
3259
3333
|
index: number,
|
|
@@ -3261,12 +3335,8 @@ function createMinimalWorkerCtx(
|
|
|
3261
3335
|
const inputRow = publicCsvInputRow(row);
|
|
3262
3336
|
const explicitKeyValue = resolveExplicitKeyValue(row, index);
|
|
3263
3337
|
return explicitKeyValue == null
|
|
3264
|
-
? derivePlayRowIdentity(inputRow, name
|
|
3265
|
-
: derivePlayRowIdentityFromKey(
|
|
3266
|
-
explicitKeyValue,
|
|
3267
|
-
name,
|
|
3268
|
-
mapLogicFingerprint,
|
|
3269
|
-
);
|
|
3338
|
+
? derivePlayRowIdentity(inputRow, name)
|
|
3339
|
+
: derivePlayRowIdentityFromKey(explicitKeyValue, name);
|
|
3270
3340
|
};
|
|
3271
3341
|
const assertUniqueExplicitRowKeys = (
|
|
3272
3342
|
chunkRows: readonly Record<string, unknown>[],
|
|
@@ -3311,6 +3381,7 @@ function createMinimalWorkerCtx(
|
|
|
3311
3381
|
const prepared = await prepareMapRows({
|
|
3312
3382
|
req,
|
|
3313
3383
|
tableNamespace: name,
|
|
3384
|
+
outputFields,
|
|
3314
3385
|
rows: chunkEntries.map(({ row, rowKey }) => ({
|
|
3315
3386
|
...row,
|
|
3316
3387
|
__deeplineRowKey: rowKey,
|
|
@@ -3331,19 +3402,17 @@ function createMinimalWorkerCtx(
|
|
|
3331
3402
|
},
|
|
3332
3403
|
});
|
|
3333
3404
|
const pendingKeys = new Set<string>();
|
|
3405
|
+
const pendingRowsByKey = new Map<string, Record<string, unknown>>();
|
|
3334
3406
|
const completedKeys = new Set<string>();
|
|
3335
3407
|
const preparedKeys = new Set<string>();
|
|
3336
3408
|
for (const row of prepared.pendingRows) {
|
|
3337
3409
|
const key =
|
|
3338
3410
|
typeof row.__deeplineRowKey === 'string'
|
|
3339
3411
|
? row.__deeplineRowKey
|
|
3340
|
-
: derivePlayRowIdentity(
|
|
3341
|
-
publicCsvInputRow(row),
|
|
3342
|
-
name,
|
|
3343
|
-
mapLogicFingerprint,
|
|
3344
|
-
);
|
|
3412
|
+
: derivePlayRowIdentity(publicCsvInputRow(row), name);
|
|
3345
3413
|
if (key) {
|
|
3346
3414
|
pendingKeys.add(key);
|
|
3415
|
+
pendingRowsByKey.set(key, row);
|
|
3347
3416
|
preparedKeys.add(key);
|
|
3348
3417
|
}
|
|
3349
3418
|
}
|
|
@@ -3351,11 +3420,7 @@ function createMinimalWorkerCtx(
|
|
|
3351
3420
|
const key =
|
|
3352
3421
|
typeof row.__deeplineRowKey === 'string'
|
|
3353
3422
|
? row.__deeplineRowKey
|
|
3354
|
-
: derivePlayRowIdentity(
|
|
3355
|
-
publicCsvInputRow(row),
|
|
3356
|
-
name,
|
|
3357
|
-
mapLogicFingerprint,
|
|
3358
|
-
);
|
|
3423
|
+
: derivePlayRowIdentity(publicCsvInputRow(row), name);
|
|
3359
3424
|
if (key) {
|
|
3360
3425
|
completedKeys.add(key);
|
|
3361
3426
|
preparedKeys.add(key);
|
|
@@ -3388,7 +3453,16 @@ function createMinimalWorkerCtx(
|
|
|
3388
3453
|
rowsToExecute.length,
|
|
3389
3454
|
);
|
|
3390
3455
|
const executedCellMetaPatches: Array<
|
|
3391
|
-
Record<
|
|
3456
|
+
| Record<
|
|
3457
|
+
string,
|
|
3458
|
+
{
|
|
3459
|
+
status: 'cached' | 'skipped';
|
|
3460
|
+
stage?: string | null;
|
|
3461
|
+
reused?: boolean;
|
|
3462
|
+
runId?: string;
|
|
3463
|
+
}
|
|
3464
|
+
>
|
|
3465
|
+
| undefined
|
|
3392
3466
|
> = new Array(rowsToExecute.length);
|
|
3393
3467
|
const toolBatchScheduler = new WorkerToolBatchScheduler(req);
|
|
3394
3468
|
const generatedOutputFields = new Set<string>();
|
|
@@ -3402,31 +3476,28 @@ function createMinimalWorkerCtx(
|
|
|
3402
3476
|
const myIndex = idx++;
|
|
3403
3477
|
if (myIndex >= rowsToExecute.length) return;
|
|
3404
3478
|
const entry = uniqueRowsToExecuteEntries[myIndex]!;
|
|
3405
|
-
const row = entry.
|
|
3479
|
+
const row = pendingRowsByKey.has(entry.rowKey)
|
|
3480
|
+
? ({
|
|
3481
|
+
...entry.row,
|
|
3482
|
+
...publicCsvInputRow(pendingRowsByKey.get(entry.rowKey)!),
|
|
3483
|
+
} as T & Record<string, unknown>)
|
|
3484
|
+
: entry.row;
|
|
3406
3485
|
const absoluteIndex = entry.absoluteIndex;
|
|
3407
3486
|
const enriched: Record<string, unknown> = cloneCsvAliasedRow(row);
|
|
3408
3487
|
const fieldOutputs: Record<string, unknown> = {};
|
|
3409
3488
|
const cellMetaPatch: Record<
|
|
3410
3489
|
string,
|
|
3411
|
-
{
|
|
3490
|
+
{
|
|
3491
|
+
status: 'cached' | 'skipped';
|
|
3492
|
+
stage?: string | null;
|
|
3493
|
+
reused?: boolean;
|
|
3494
|
+
runId?: string;
|
|
3495
|
+
}
|
|
3412
3496
|
> = {};
|
|
3413
3497
|
const waterfallOutputs: RecordedWaterfallOutput[] = [];
|
|
3414
3498
|
const stepProgramOutputs: RecordedStepProgramOutput[] = [];
|
|
3415
3499
|
const rowCtx = {
|
|
3416
3500
|
...(ctx as Record<string, unknown>),
|
|
3417
|
-
tool: async (
|
|
3418
|
-
key: string,
|
|
3419
|
-
toolId: string,
|
|
3420
|
-
input: Record<string, unknown>,
|
|
3421
|
-
): Promise<unknown> => {
|
|
3422
|
-
assertNotAborted(abortSignal);
|
|
3423
|
-
return await toolBatchScheduler.execute(
|
|
3424
|
-
key,
|
|
3425
|
-
toolId,
|
|
3426
|
-
input,
|
|
3427
|
-
workflowStep,
|
|
3428
|
-
);
|
|
3429
|
-
},
|
|
3430
3501
|
tools: {
|
|
3431
3502
|
...((ctx as { tools?: Record<string, unknown> }).tools ?? {}),
|
|
3432
3503
|
execute: async (requestArg: unknown): Promise<unknown> => {
|
|
@@ -3456,6 +3527,15 @@ function createMinimalWorkerCtx(
|
|
|
3456
3527
|
),
|
|
3457
3528
|
};
|
|
3458
3529
|
for (const [key, value] of fieldEntries) {
|
|
3530
|
+
if (isCompletedWorkerFieldValue(enriched[key])) {
|
|
3531
|
+
cellMetaPatch[key] = {
|
|
3532
|
+
status: 'cached',
|
|
3533
|
+
stage: key,
|
|
3534
|
+
reused: true,
|
|
3535
|
+
runId: req.runId,
|
|
3536
|
+
};
|
|
3537
|
+
continue;
|
|
3538
|
+
}
|
|
3459
3539
|
const resolved = await executeWorkerStepResolver(
|
|
3460
3540
|
value,
|
|
3461
3541
|
enriched,
|
|
@@ -3472,7 +3552,11 @@ function createMinimalWorkerCtx(
|
|
|
3472
3552
|
enriched[key] = resolved.value;
|
|
3473
3553
|
fieldOutputs[key] = resolved.value;
|
|
3474
3554
|
if (resolved.status === 'skipped') {
|
|
3475
|
-
cellMetaPatch[key] = {
|
|
3555
|
+
cellMetaPatch[key] = {
|
|
3556
|
+
status: 'skipped',
|
|
3557
|
+
stage: key,
|
|
3558
|
+
runId: req.runId,
|
|
3559
|
+
};
|
|
3476
3560
|
}
|
|
3477
3561
|
}
|
|
3478
3562
|
for (const stepOutput of stepProgramOutputs) {
|
|
@@ -3483,6 +3567,7 @@ function createMinimalWorkerCtx(
|
|
|
3483
3567
|
cellMetaPatch[stepOutput.columnName] = {
|
|
3484
3568
|
status: 'skipped',
|
|
3485
3569
|
stage: stepOutput.stepId,
|
|
3570
|
+
runId: req.runId,
|
|
3486
3571
|
};
|
|
3487
3572
|
}
|
|
3488
3573
|
}
|
|
@@ -3602,11 +3687,7 @@ function createMinimalWorkerCtx(
|
|
|
3602
3687
|
const key =
|
|
3603
3688
|
typeof completedRow.__deeplineRowKey === 'string'
|
|
3604
3689
|
? completedRow.__deeplineRowKey
|
|
3605
|
-
: derivePlayRowIdentity(
|
|
3606
|
-
publicCsvInputRow(completedRow),
|
|
3607
|
-
name,
|
|
3608
|
-
mapLogicFingerprint,
|
|
3609
|
-
);
|
|
3690
|
+
: derivePlayRowIdentity(publicCsvInputRow(completedRow), name);
|
|
3610
3691
|
if (key) {
|
|
3611
3692
|
const { __deeplineRowKey: _rowKey, ...cleanedRow } =
|
|
3612
3693
|
publicCsvInputRow(completedRow);
|
|
@@ -3866,7 +3947,11 @@ function createMinimalWorkerCtx(
|
|
|
3866
3947
|
ts: nowMs(),
|
|
3867
3948
|
});
|
|
3868
3949
|
},
|
|
3869
|
-
async step<T>(
|
|
3950
|
+
async step<T>(
|
|
3951
|
+
name: string,
|
|
3952
|
+
callback: () => Promise<T> | T,
|
|
3953
|
+
options?: { staleAfterSeconds?: number },
|
|
3954
|
+
): Promise<T> {
|
|
3870
3955
|
assertNotAborted(abortSignal);
|
|
3871
3956
|
const normalizedName = name.trim();
|
|
3872
3957
|
if (!normalizedName) {
|
|
@@ -3875,7 +3960,10 @@ function createMinimalWorkerCtx(
|
|
|
3875
3960
|
// Static pipeline JS blocks are already Workflow steps in the Workers
|
|
3876
3961
|
// backend. Nesting another `step.do` here can leave preview runs parked
|
|
3877
3962
|
// inside the JS stage before they reach subsequent event waits.
|
|
3878
|
-
return await executeWithRuntimeReceipt(
|
|
3963
|
+
return await executeWithRuntimeReceipt(
|
|
3964
|
+
`step:${normalizedName}${staleRuntimeSuffix(options?.staleAfterSeconds)}`,
|
|
3965
|
+
callback,
|
|
3966
|
+
);
|
|
3879
3967
|
},
|
|
3880
3968
|
async runSteps<T>(
|
|
3881
3969
|
program: WorkerStepProgram,
|
|
@@ -4032,39 +4120,16 @@ function createMinimalWorkerCtx(
|
|
|
4032
4120
|
'ctx.map(key, rows, fields, options) was removed. Use ctx.map(key, rows).step(...).run(options).',
|
|
4033
4121
|
);
|
|
4034
4122
|
},
|
|
4035
|
-
tool: async (
|
|
4036
|
-
key: string,
|
|
4037
|
-
toolId: string,
|
|
4038
|
-
input: Record<string, unknown>,
|
|
4039
|
-
): Promise<unknown> => {
|
|
4040
|
-
assertNotAborted(abortSignal);
|
|
4041
|
-
return await executeWithRuntimeReceipt(
|
|
4042
|
-
deriveToolRequestIdentity({ toolId, requestInput: input }),
|
|
4043
|
-
() =>
|
|
4044
|
-
executeToolWithLifecycle(
|
|
4045
|
-
req,
|
|
4046
|
-
{ id: key, toolId, input },
|
|
4047
|
-
workflowStep,
|
|
4048
|
-
callbacks,
|
|
4049
|
-
),
|
|
4050
|
-
);
|
|
4051
|
-
},
|
|
4052
4123
|
tools: {
|
|
4053
4124
|
async execute(requestArg: unknown): Promise<unknown> {
|
|
4054
4125
|
assertNotAborted(abortSignal);
|
|
4055
4126
|
const request = normalizeToolExecuteArgs(requestArg);
|
|
4056
4127
|
return await executeWithRuntimeReceipt(
|
|
4057
|
-
deriveToolRequestIdentity({
|
|
4128
|
+
`tool:${request.id}:${deriveToolRequestIdentity({
|
|
4058
4129
|
toolId: request.toolId,
|
|
4059
4130
|
requestInput: request.input,
|
|
4060
|
-
})
|
|
4061
|
-
() =>
|
|
4062
|
-
executeToolWithLifecycle(
|
|
4063
|
-
req,
|
|
4064
|
-
request,
|
|
4065
|
-
workflowStep,
|
|
4066
|
-
callbacks,
|
|
4067
|
-
),
|
|
4131
|
+
})}${staleRuntimeSuffix(request.staleAfterSeconds)}`,
|
|
4132
|
+
() => executeToolWithLifecycle(req, request, workflowStep, callbacks),
|
|
4068
4133
|
);
|
|
4069
4134
|
},
|
|
4070
4135
|
},
|
|
@@ -4123,13 +4188,21 @@ function createMinimalWorkerCtx(
|
|
|
4123
4188
|
key: string,
|
|
4124
4189
|
playRef: string | { playName?: string; name?: string },
|
|
4125
4190
|
input: Record<string, unknown>,
|
|
4126
|
-
options?: {
|
|
4191
|
+
options?: {
|
|
4192
|
+
description?: string;
|
|
4193
|
+
timeoutMs?: number;
|
|
4194
|
+
staleAfterSeconds?: number;
|
|
4195
|
+
},
|
|
4127
4196
|
): Promise<unknown> {
|
|
4128
4197
|
const normalizedKey = normalizeContextKey(key, 'runPlay');
|
|
4129
4198
|
const resolvedName = resolvePlayRefName(playRef);
|
|
4130
4199
|
if (!resolvedName) {
|
|
4131
4200
|
throw new Error('ctx.runPlay(...) requires a resolvable play name.');
|
|
4132
4201
|
}
|
|
4202
|
+
const receiptKey = `runPlay:${normalizedKey}:${await hashJson({
|
|
4203
|
+
childPlayName: resolvedName,
|
|
4204
|
+
input,
|
|
4205
|
+
})}${staleRuntimeSuffix(options?.staleAfterSeconds)}`;
|
|
4133
4206
|
if (ancestryPlayIds.includes(resolvedName)) {
|
|
4134
4207
|
const chain = [...ancestryPlayIds, resolvedName].join(' -> ');
|
|
4135
4208
|
throw new Error(`Recursive play graph detected: ${chain}`);
|
|
@@ -4154,163 +4227,180 @@ function createMinimalWorkerCtx(
|
|
|
4154
4227
|
`Child play-call cap exceeded for ${req.playName} (${nextParentCalls}/${WORKER_PLAY_CALL_LIMITS.maxChildPlayCallsPerParent}).`,
|
|
4155
4228
|
);
|
|
4156
4229
|
}
|
|
4157
|
-
|
|
4158
|
-
|
|
4230
|
+
return await executeWithRuntimeReceipt(receiptKey, async () => {
|
|
4231
|
+
playCallCount = nextPlayCallCount;
|
|
4232
|
+
parentChildCalls[req.playName] = nextParentCalls;
|
|
4159
4233
|
|
|
4160
|
-
|
|
4161
|
-
|
|
4162
|
-
|
|
4163
|
-
|
|
4164
|
-
|
|
4165
|
-
|
|
4166
|
-
|
|
4167
|
-
|
|
4168
|
-
throw new Error(
|
|
4169
|
-
`ctx.runPlay(${normalizedKey}) cannot start ${resolvedName}: missing trusted Cloudflare child manifest from top-level submit.`,
|
|
4170
|
-
);
|
|
4171
|
-
}
|
|
4172
|
-
const childIsMapBacked = childPipelineUsesCtxMap(
|
|
4173
|
-
childManifest.staticPipeline,
|
|
4174
|
-
);
|
|
4175
|
-
const childNeedsWorkflowScheduler = childPipelineNeedsWorkflowScheduler(
|
|
4176
|
-
childManifest.staticPipeline,
|
|
4177
|
-
);
|
|
4178
|
-
let childConcurrencyAcquired = false;
|
|
4179
|
-
let releaseChildPlaySlot: (() => void) | null = null;
|
|
4180
|
-
if (childIsMapBacked) {
|
|
4181
|
-
const nextInFlight =
|
|
4182
|
-
(inFlightChildCallsByPlayName[resolvedName] ?? 0) + 1;
|
|
4183
|
-
if (nextInFlight > 1) {
|
|
4234
|
+
emitEvent({
|
|
4235
|
+
type: 'log',
|
|
4236
|
+
level: 'info',
|
|
4237
|
+
message: `Starting child play ${resolvedName} (${normalizedKey})`,
|
|
4238
|
+
ts: nowMs(),
|
|
4239
|
+
});
|
|
4240
|
+
const childManifest = req.childPlayManifests?.[resolvedName];
|
|
4241
|
+
if (!childManifest) {
|
|
4184
4242
|
throw new Error(
|
|
4185
|
-
`
|
|
4186
|
-
'A child play that uses ctx.map() cannot run more than once at the same time because its map tables share durable row identity. ' +
|
|
4187
|
-
'Run these child play calls sequentially, or give each concurrent branch a different child play/table contract.',
|
|
4243
|
+
`ctx.runPlay(${normalizedKey}) cannot start ${resolvedName}: missing trusted Cloudflare child manifest from top-level submit.`,
|
|
4188
4244
|
);
|
|
4189
4245
|
}
|
|
4190
|
-
|
|
4191
|
-
|
|
4192
|
-
|
|
4193
|
-
|
|
4194
|
-
|
|
4195
|
-
|
|
4196
|
-
let
|
|
4197
|
-
|
|
4198
|
-
|
|
4199
|
-
|
|
4200
|
-
|
|
4201
|
-
|
|
4202
|
-
|
|
4203
|
-
|
|
4246
|
+
const childIsMapBacked = childPipelineUsesCtxMap(
|
|
4247
|
+
childManifest.staticPipeline,
|
|
4248
|
+
);
|
|
4249
|
+
const childNeedsWorkflowScheduler = childPipelineNeedsWorkflowScheduler(
|
|
4250
|
+
childManifest.staticPipeline,
|
|
4251
|
+
);
|
|
4252
|
+
let childConcurrencyAcquired = false;
|
|
4253
|
+
let releaseChildPlaySlot: (() => void) | null = null;
|
|
4254
|
+
if (childIsMapBacked) {
|
|
4255
|
+
const nextInFlight =
|
|
4256
|
+
(inFlightChildCallsByPlayName[resolvedName] ?? 0) + 1;
|
|
4257
|
+
if (nextInFlight > 1) {
|
|
4258
|
+
throw new Error(
|
|
4259
|
+
`Concurrent map-backed play call blocked for ${resolvedName}. ` +
|
|
4260
|
+
'A child play that uses ctx.map() cannot run more than once at the same time because its map tables share durable row identity. ' +
|
|
4261
|
+
'Run these child play calls sequentially, or give each concurrent branch a different child play/table contract.',
|
|
4262
|
+
);
|
|
4263
|
+
}
|
|
4264
|
+
inFlightChildCallsByPlayName[resolvedName] = nextInFlight;
|
|
4265
|
+
childConcurrencyAcquired = true;
|
|
4266
|
+
}
|
|
4204
4267
|
try {
|
|
4205
|
-
|
|
4206
|
-
|
|
4207
|
-
|
|
4208
|
-
|
|
4209
|
-
|
|
4210
|
-
|
|
4211
|
-
|
|
4212
|
-
|
|
4213
|
-
|
|
4214
|
-
|
|
4215
|
-
|
|
4216
|
-
|
|
4217
|
-
|
|
4218
|
-
|
|
4219
|
-
|
|
4220
|
-
|
|
4221
|
-
|
|
4222
|
-
|
|
4223
|
-
|
|
4224
|
-
|
|
4225
|
-
|
|
4226
|
-
|
|
4227
|
-
|
|
4228
|
-
|
|
4229
|
-
|
|
4230
|
-
|
|
4268
|
+
releaseChildPlaySlot = await acquireChildPlaySlot();
|
|
4269
|
+
const childSubmitStartedAt = nowMs();
|
|
4270
|
+
let started: {
|
|
4271
|
+
workflowId?: string;
|
|
4272
|
+
runId?: string;
|
|
4273
|
+
status?: string;
|
|
4274
|
+
output?: unknown;
|
|
4275
|
+
result?: unknown;
|
|
4276
|
+
error?: unknown;
|
|
4277
|
+
};
|
|
4278
|
+
try {
|
|
4279
|
+
started = await submitChildPlayThroughCoordinator({
|
|
4280
|
+
req,
|
|
4281
|
+
allowInline:
|
|
4282
|
+
options?.timeoutMs == null && !childNeedsWorkflowScheduler,
|
|
4283
|
+
body: {
|
|
4284
|
+
name: resolvedName,
|
|
4285
|
+
input: isRecord(input) ? input : {},
|
|
4286
|
+
orgId: req.orgId,
|
|
4287
|
+
parentExecutorToken: req.executorToken,
|
|
4288
|
+
userEmail: req.userEmail ?? '',
|
|
4289
|
+
profile: 'workers_edge',
|
|
4290
|
+
manifest: childManifest,
|
|
4291
|
+
childPlayManifests: req.childPlayManifests ?? null,
|
|
4292
|
+
internalRunPlay: {
|
|
4293
|
+
rootRunId,
|
|
4294
|
+
parentRunId: req.runId,
|
|
4295
|
+
parentPlayName: req.playName,
|
|
4296
|
+
key: normalizedKey,
|
|
4297
|
+
// Per the lineage validator: ancestry tail must equal the
|
|
4298
|
+
// executor token's play name (the parent making this call).
|
|
4299
|
+
ancestryPlayIds,
|
|
4300
|
+
callDepth: nextDepth,
|
|
4301
|
+
description:
|
|
4302
|
+
typeof options?.description === 'string'
|
|
4303
|
+
? options.description
|
|
4304
|
+
: null,
|
|
4305
|
+
},
|
|
4231
4306
|
},
|
|
4232
|
-
}
|
|
4233
|
-
})
|
|
4234
|
-
|
|
4307
|
+
});
|
|
4308
|
+
} catch (error) {
|
|
4309
|
+
console.info('[play.runtime.span]', {
|
|
4310
|
+
event: 'play.runtime.span',
|
|
4311
|
+
phase: 'child_submit',
|
|
4312
|
+
runId: req.runId,
|
|
4313
|
+
parentRunId: req.runId,
|
|
4314
|
+
playName: resolvedName,
|
|
4315
|
+
graphHash: req.graphHash ?? null,
|
|
4316
|
+
depth: nextDepth,
|
|
4317
|
+
fanoutIndex: nextParentCalls - 1,
|
|
4318
|
+
ms: nowMs() - childSubmitStartedAt,
|
|
4319
|
+
status: 'failed',
|
|
4320
|
+
errorCode: 'CHILD_SUBMIT_FAILED',
|
|
4321
|
+
});
|
|
4322
|
+
throw error;
|
|
4323
|
+
}
|
|
4324
|
+
const workflowId = started.workflowId ?? started.runId;
|
|
4325
|
+
if (!workflowId) {
|
|
4326
|
+
const startedError = isRecord(started.error)
|
|
4327
|
+
? started.error
|
|
4328
|
+
: { message: started.error };
|
|
4329
|
+
const startedErrorMessage =
|
|
4330
|
+
typeof startedError.message === 'string' &&
|
|
4331
|
+
startedError.message.trim()
|
|
4332
|
+
? startedError.message.trim()
|
|
4333
|
+
: null;
|
|
4334
|
+
throw new Error(
|
|
4335
|
+
startedErrorMessage ??
|
|
4336
|
+
`ctx.runPlay(${normalizedKey}) did not receive a child workflow id.`,
|
|
4337
|
+
);
|
|
4338
|
+
}
|
|
4235
4339
|
console.info('[play.runtime.span]', {
|
|
4236
4340
|
event: 'play.runtime.span',
|
|
4237
4341
|
phase: 'child_submit',
|
|
4238
4342
|
runId: req.runId,
|
|
4239
4343
|
parentRunId: req.runId,
|
|
4344
|
+
childRunId: workflowId,
|
|
4240
4345
|
playName: resolvedName,
|
|
4241
4346
|
graphHash: req.graphHash ?? null,
|
|
4242
4347
|
depth: nextDepth,
|
|
4243
4348
|
fanoutIndex: nextParentCalls - 1,
|
|
4244
4349
|
ms: nowMs() - childSubmitStartedAt,
|
|
4245
|
-
status: '
|
|
4246
|
-
errorCode: 'CHILD_SUBMIT_FAILED',
|
|
4350
|
+
status: 'ok',
|
|
4247
4351
|
});
|
|
4248
|
-
|
|
4249
|
-
|
|
4250
|
-
|
|
4251
|
-
|
|
4252
|
-
|
|
4253
|
-
|
|
4254
|
-
|
|
4255
|
-
|
|
4256
|
-
|
|
4257
|
-
|
|
4258
|
-
|
|
4259
|
-
|
|
4260
|
-
|
|
4261
|
-
|
|
4262
|
-
|
|
4263
|
-
|
|
4264
|
-
|
|
4265
|
-
|
|
4266
|
-
|
|
4267
|
-
|
|
4268
|
-
|
|
4269
|
-
|
|
4270
|
-
|
|
4271
|
-
|
|
4272
|
-
|
|
4273
|
-
|
|
4274
|
-
|
|
4275
|
-
|
|
4276
|
-
|
|
4277
|
-
|
|
4278
|
-
|
|
4279
|
-
|
|
4280
|
-
|
|
4281
|
-
|
|
4282
|
-
|
|
4283
|
-
|
|
4284
|
-
|
|
4285
|
-
|
|
4286
|
-
|
|
4287
|
-
|
|
4288
|
-
|
|
4289
|
-
|
|
4290
|
-
|
|
4291
|
-
|
|
4292
|
-
|
|
4293
|
-
|
|
4294
|
-
|
|
4295
|
-
|
|
4296
|
-
:
|
|
4297
|
-
|
|
4298
|
-
|
|
4299
|
-
|
|
4300
|
-
let result: unknown;
|
|
4301
|
-
try {
|
|
4302
|
-
result = await waitForChildPlayTerminalEvent({
|
|
4303
|
-
req,
|
|
4304
|
-
workflowStep,
|
|
4305
|
-
workflowId,
|
|
4306
|
-
playName: resolvedName,
|
|
4307
|
-
key: normalizedKey,
|
|
4308
|
-
timeoutMs: Math.max(
|
|
4309
|
-
1_000,
|
|
4310
|
-
Math.min(options?.timeoutMs ?? 5 * 60_000, 30 * 60_000),
|
|
4311
|
-
),
|
|
4312
|
-
});
|
|
4313
|
-
} catch (error) {
|
|
4352
|
+
const startedStatus = String(started.status ?? '').toLowerCase();
|
|
4353
|
+
if (startedStatus === 'completed') {
|
|
4354
|
+
emitEvent({
|
|
4355
|
+
type: 'log',
|
|
4356
|
+
level: 'info',
|
|
4357
|
+
message: `Completed child play ${resolvedName} (${normalizedKey})`,
|
|
4358
|
+
ts: nowMs(),
|
|
4359
|
+
});
|
|
4360
|
+
return started.output ?? extractChildPlayOutput(started);
|
|
4361
|
+
}
|
|
4362
|
+
if (startedStatus === 'failed') {
|
|
4363
|
+
const startedError = isRecord(started.error)
|
|
4364
|
+
? started.error
|
|
4365
|
+
: { message: started.error };
|
|
4366
|
+
const startedErrorMessage =
|
|
4367
|
+
typeof startedError.message === 'string' &&
|
|
4368
|
+
startedError.message.trim()
|
|
4369
|
+
? startedError.message.trim()
|
|
4370
|
+
: `Child play ${resolvedName} (${workflowId}) failed.`;
|
|
4371
|
+
throw new Error(startedErrorMessage);
|
|
4372
|
+
}
|
|
4373
|
+
const childWaitStartedAt = nowMs();
|
|
4374
|
+
let result: unknown;
|
|
4375
|
+
try {
|
|
4376
|
+
result = await waitForChildPlayTerminalEvent({
|
|
4377
|
+
req,
|
|
4378
|
+
workflowStep,
|
|
4379
|
+
workflowId,
|
|
4380
|
+
playName: resolvedName,
|
|
4381
|
+
key: normalizedKey,
|
|
4382
|
+
timeoutMs: Math.max(
|
|
4383
|
+
1_000,
|
|
4384
|
+
Math.min(options?.timeoutMs ?? 5 * 60_000, 30 * 60_000),
|
|
4385
|
+
),
|
|
4386
|
+
});
|
|
4387
|
+
} catch (error) {
|
|
4388
|
+
console.info('[play.runtime.span]', {
|
|
4389
|
+
event: 'play.runtime.span',
|
|
4390
|
+
phase: 'child_wait',
|
|
4391
|
+
runId: req.runId,
|
|
4392
|
+
parentRunId: req.runId,
|
|
4393
|
+
childRunId: workflowId,
|
|
4394
|
+
playName: resolvedName,
|
|
4395
|
+
graphHash: req.graphHash ?? null,
|
|
4396
|
+
depth: nextDepth,
|
|
4397
|
+
fanoutIndex: nextParentCalls - 1,
|
|
4398
|
+
ms: nowMs() - childWaitStartedAt,
|
|
4399
|
+
status: 'failed',
|
|
4400
|
+
errorCode: 'CHILD_WAIT_FAILED',
|
|
4401
|
+
});
|
|
4402
|
+
throw error;
|
|
4403
|
+
}
|
|
4314
4404
|
console.info('[play.runtime.span]', {
|
|
4315
4405
|
event: 'play.runtime.span',
|
|
4316
4406
|
phase: 'child_wait',
|
|
@@ -4322,40 +4412,25 @@ function createMinimalWorkerCtx(
|
|
|
4322
4412
|
depth: nextDepth,
|
|
4323
4413
|
fanoutIndex: nextParentCalls - 1,
|
|
4324
4414
|
ms: nowMs() - childWaitStartedAt,
|
|
4325
|
-
status: '
|
|
4326
|
-
errorCode: 'CHILD_WAIT_FAILED',
|
|
4415
|
+
status: 'ok',
|
|
4327
4416
|
});
|
|
4328
|
-
|
|
4329
|
-
|
|
4330
|
-
|
|
4331
|
-
|
|
4332
|
-
|
|
4333
|
-
|
|
4334
|
-
|
|
4335
|
-
|
|
4336
|
-
|
|
4337
|
-
|
|
4338
|
-
|
|
4339
|
-
|
|
4340
|
-
|
|
4341
|
-
|
|
4342
|
-
|
|
4343
|
-
emitEvent({
|
|
4344
|
-
type: 'log',
|
|
4345
|
-
level: 'info',
|
|
4346
|
-
message: `Completed child play ${resolvedName} (${normalizedKey})`,
|
|
4347
|
-
ts: nowMs(),
|
|
4348
|
-
});
|
|
4349
|
-
return result;
|
|
4350
|
-
} finally {
|
|
4351
|
-
releaseChildPlaySlot?.();
|
|
4352
|
-
if (childConcurrencyAcquired) {
|
|
4353
|
-
releaseChildPlayConcurrency(
|
|
4354
|
-
inFlightChildCallsByPlayName,
|
|
4355
|
-
resolvedName,
|
|
4356
|
-
);
|
|
4417
|
+
emitEvent({
|
|
4418
|
+
type: 'log',
|
|
4419
|
+
level: 'info',
|
|
4420
|
+
message: `Completed child play ${resolvedName} (${normalizedKey})`,
|
|
4421
|
+
ts: nowMs(),
|
|
4422
|
+
});
|
|
4423
|
+
return result;
|
|
4424
|
+
} finally {
|
|
4425
|
+
releaseChildPlaySlot?.();
|
|
4426
|
+
if (childConcurrencyAcquired) {
|
|
4427
|
+
releaseChildPlayConcurrency(
|
|
4428
|
+
inFlightChildCallsByPlayName,
|
|
4429
|
+
resolvedName,
|
|
4430
|
+
);
|
|
4431
|
+
}
|
|
4357
4432
|
}
|
|
4358
|
-
}
|
|
4433
|
+
});
|
|
4359
4434
|
},
|
|
4360
4435
|
fetch(): never {
|
|
4361
4436
|
throw new Error(
|