deepline 0.1.21 → 0.1.23
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/index.js +552 -237
- package/dist/cli/index.mjs +609 -289
- package/dist/index.d.mts +21 -58
- package/dist/index.d.ts +21 -58
- package/dist/index.js +177 -92
- package/dist/index.mjs +177 -92
- package/dist/repo/apps/play-runner-workers/src/coordinator-entry.ts +3 -1
- package/dist/repo/apps/play-runner-workers/src/entry.ts +153 -0
- package/dist/repo/sdk/src/client.ts +243 -124
- package/dist/repo/sdk/src/types.ts +8 -14
- package/dist/repo/sdk/src/version.ts +1 -1
- package/dist/repo/shared_libs/play-runtime/execution-plan.ts +27 -2
- package/dist/repo/shared_libs/play-runtime/profiles.ts +4 -14
- package/dist/repo/shared_libs/play-runtime/runtime-actions.ts +1 -1
- package/package.json +1 -1
package/dist/index.mjs
CHANGED
|
@@ -195,7 +195,7 @@ function resolveConfig(options) {
|
|
|
195
195
|
}
|
|
196
196
|
|
|
197
197
|
// src/version.ts
|
|
198
|
-
var SDK_VERSION = "0.1.
|
|
198
|
+
var SDK_VERSION = "0.1.23";
|
|
199
199
|
var SDK_API_CONTRACT = "2026-05-runs-v2";
|
|
200
200
|
|
|
201
201
|
// ../shared_libs/play-runtime/coordinator-headers.ts
|
|
@@ -483,11 +483,24 @@ function sleep(ms) {
|
|
|
483
483
|
// src/client.ts
|
|
484
484
|
var TERMINAL_PLAY_STATUSES = /* @__PURE__ */ new Set(["completed", "failed", "cancelled"]);
|
|
485
485
|
var INCLUDE_TOOL_METADATA_HEADER = "x-deepline-include-tool-metadata";
|
|
486
|
+
var COMPILE_MANIFEST_RETRY_DELAYS_MS = [250, 1e3];
|
|
487
|
+
function sleep2(ms) {
|
|
488
|
+
return new Promise((resolve2) => setTimeout(resolve2, ms));
|
|
489
|
+
}
|
|
490
|
+
function isTransientCompileManifestError(error) {
|
|
491
|
+
if (error instanceof DeeplineError && typeof error.statusCode === "number") {
|
|
492
|
+
return error.statusCode === 408 || error.statusCode === 425 || error.statusCode === 499 || error.statusCode >= 500 && error.statusCode < 600;
|
|
493
|
+
}
|
|
494
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
495
|
+
return /fetch failed|connection (?:closed|reset|terminated)|socket hang up|econnreset|etimedout|eai_again|abort/i.test(
|
|
496
|
+
message
|
|
497
|
+
);
|
|
498
|
+
}
|
|
486
499
|
function isRecord(value) {
|
|
487
500
|
return Boolean(value && typeof value === "object" && !Array.isArray(value));
|
|
488
501
|
}
|
|
489
502
|
function normalizePlayStatus(raw) {
|
|
490
|
-
const status = typeof raw.status === "string" ? raw.status :
|
|
503
|
+
const status = typeof raw.status === "string" ? raw.status : "running";
|
|
491
504
|
const runId = typeof raw.runId === "string" ? raw.runId : typeof raw.workflowId === "string" ? raw.workflowId : "";
|
|
492
505
|
return {
|
|
493
506
|
...raw,
|
|
@@ -495,23 +508,6 @@ function normalizePlayStatus(raw) {
|
|
|
495
508
|
status
|
|
496
509
|
};
|
|
497
510
|
}
|
|
498
|
-
function mapLegacyTemporalStatus(status) {
|
|
499
|
-
switch (status.trim().toUpperCase()) {
|
|
500
|
-
case "PENDING":
|
|
501
|
-
return "queued";
|
|
502
|
-
case "COMPLETED":
|
|
503
|
-
return "completed";
|
|
504
|
-
case "FAILED":
|
|
505
|
-
return "failed";
|
|
506
|
-
case "CANCELLED":
|
|
507
|
-
case "TERMINATED":
|
|
508
|
-
case "TIMED_OUT":
|
|
509
|
-
return "cancelled";
|
|
510
|
-
case "RUNNING":
|
|
511
|
-
default:
|
|
512
|
-
return "running";
|
|
513
|
-
}
|
|
514
|
-
}
|
|
515
511
|
function decodeBase64Bytes(value) {
|
|
516
512
|
const binary = atob(value);
|
|
517
513
|
const bytes = new Uint8Array(binary.length);
|
|
@@ -520,6 +516,79 @@ function decodeBase64Bytes(value) {
|
|
|
520
516
|
}
|
|
521
517
|
return bytes;
|
|
522
518
|
}
|
|
519
|
+
function readStringArray(value) {
|
|
520
|
+
return Array.isArray(value) ? value.filter((line) => typeof line === "string") : [];
|
|
521
|
+
}
|
|
522
|
+
function getPlayLiveEventPayload(event) {
|
|
523
|
+
return event.payload && typeof event.payload === "object" ? event.payload : {};
|
|
524
|
+
}
|
|
525
|
+
function normalizeLiveStatus(value) {
|
|
526
|
+
if (value === "queued" || value === "running" || value === "waiting" || value === "completed" || value === "failed" || value === "cancelled") {
|
|
527
|
+
return value;
|
|
528
|
+
}
|
|
529
|
+
return null;
|
|
530
|
+
}
|
|
531
|
+
function updatePlayLiveStatusState(state, event) {
|
|
532
|
+
const payload = getPlayLiveEventPayload(event);
|
|
533
|
+
if (event.type === "play.run.log") {
|
|
534
|
+
state.logs.push(...readStringArray(payload.lines));
|
|
535
|
+
return null;
|
|
536
|
+
}
|
|
537
|
+
if (event.type !== "play.run.snapshot" && event.type !== "play.run.status" && event.type !== "play.run.final_status") {
|
|
538
|
+
return null;
|
|
539
|
+
}
|
|
540
|
+
const runId = typeof payload.runId === "string" && payload.runId ? payload.runId : state.runId;
|
|
541
|
+
const status = normalizeLiveStatus(payload.status) ?? state.status;
|
|
542
|
+
const logs = readStringArray(payload.logs);
|
|
543
|
+
if (logs.length > 0 || event.type === "play.run.snapshot") {
|
|
544
|
+
state.logs = logs;
|
|
545
|
+
}
|
|
546
|
+
if ("result" in payload) {
|
|
547
|
+
state.result = payload.result;
|
|
548
|
+
}
|
|
549
|
+
if (typeof payload.error === "string" && payload.error.trim()) {
|
|
550
|
+
state.error = payload.error;
|
|
551
|
+
}
|
|
552
|
+
state.runId = runId;
|
|
553
|
+
state.status = status;
|
|
554
|
+
const progressRecord = payload.progress && typeof payload.progress === "object" && !Array.isArray(payload.progress) ? payload.progress : {};
|
|
555
|
+
const next = {
|
|
556
|
+
...payload,
|
|
557
|
+
runId,
|
|
558
|
+
status,
|
|
559
|
+
progress: {
|
|
560
|
+
...progressRecord,
|
|
561
|
+
status: typeof progressRecord.status === "string" ? progressRecord.status : status,
|
|
562
|
+
logs: state.logs,
|
|
563
|
+
...state.error ? { error: state.error } : {}
|
|
564
|
+
},
|
|
565
|
+
..."result" in state ? { result: state.result } : {}
|
|
566
|
+
};
|
|
567
|
+
state.latest = next;
|
|
568
|
+
return next;
|
|
569
|
+
}
|
|
570
|
+
function playRunResultFromStatus(status, startedAt, fallbackRunId) {
|
|
571
|
+
return {
|
|
572
|
+
success: status.status === "completed",
|
|
573
|
+
runId: status.runId || fallbackRunId,
|
|
574
|
+
result: status.result,
|
|
575
|
+
logs: status.progress?.logs ?? [],
|
|
576
|
+
durationMs: Date.now() - startedAt,
|
|
577
|
+
error: status.progress?.error ?? (status.status !== "completed" ? status.status : void 0)
|
|
578
|
+
};
|
|
579
|
+
}
|
|
580
|
+
function playRunStatusFromState(state) {
|
|
581
|
+
return {
|
|
582
|
+
runId: state.runId,
|
|
583
|
+
status: state.status,
|
|
584
|
+
progress: {
|
|
585
|
+
status: state.status,
|
|
586
|
+
logs: state.logs,
|
|
587
|
+
...state.error ? { error: state.error } : {}
|
|
588
|
+
},
|
|
589
|
+
..."result" in state ? { result: state.result } : {}
|
|
590
|
+
};
|
|
591
|
+
}
|
|
523
592
|
var DeeplineClient = class {
|
|
524
593
|
http;
|
|
525
594
|
config;
|
|
@@ -629,7 +698,7 @@ var DeeplineClient = class {
|
|
|
629
698
|
/**
|
|
630
699
|
* Search available tools using Deepline's ranked backend search.
|
|
631
700
|
*
|
|
632
|
-
* This is the same discovery surface used by the
|
|
701
|
+
* This is the same discovery surface used by the CLI: it ranks across
|
|
633
702
|
* tool metadata, categories, agent guidance, and input schema fields.
|
|
634
703
|
*/
|
|
635
704
|
async searchTools(options = {}) {
|
|
@@ -722,7 +791,7 @@ var DeeplineClient = class {
|
|
|
722
791
|
* `progress.logs`; they are not part of the user output object.
|
|
723
792
|
*
|
|
724
793
|
* @param request - Play run configuration (name, code, input, etc.)
|
|
725
|
-
* @returns
|
|
794
|
+
* @returns Run metadata including the public `workflowId`
|
|
726
795
|
*
|
|
727
796
|
* @example
|
|
728
797
|
* ```typescript
|
|
@@ -832,8 +901,22 @@ var DeeplineClient = class {
|
|
|
832
901
|
});
|
|
833
902
|
}
|
|
834
903
|
async compilePlayManifest(input) {
|
|
835
|
-
const
|
|
836
|
-
|
|
904
|
+
const retryDelays = COMPILE_MANIFEST_RETRY_DELAYS_MS.slice(
|
|
905
|
+
0,
|
|
906
|
+
Math.max(0, this.config.maxRetries)
|
|
907
|
+
);
|
|
908
|
+
for (let attempt = 0; ; attempt += 1) {
|
|
909
|
+
try {
|
|
910
|
+
const response = await this.http.post("/api/v2/plays/compile-manifest", input);
|
|
911
|
+
return response.compilerManifest;
|
|
912
|
+
} catch (error) {
|
|
913
|
+
const delayMs = retryDelays[attempt];
|
|
914
|
+
if (delayMs === void 0 || !isTransientCompileManifestError(error)) {
|
|
915
|
+
throw error;
|
|
916
|
+
}
|
|
917
|
+
await sleep2(delayMs);
|
|
918
|
+
}
|
|
919
|
+
}
|
|
837
920
|
}
|
|
838
921
|
/**
|
|
839
922
|
* Check a bundled play artifact against the server's current play compiler.
|
|
@@ -1014,9 +1097,6 @@ var DeeplineClient = class {
|
|
|
1014
1097
|
* Internal/advanced primitive. Public callers should usually prefer
|
|
1015
1098
|
* {@link runPlay}, {@link PlayJob.get}, or `deepline play run --watch`.
|
|
1016
1099
|
*
|
|
1017
|
-
* Poll this method until `status` reaches a terminal state:
|
|
1018
|
-
* `'completed'`, `'failed'`, or `'cancelled'`.
|
|
1019
|
-
*
|
|
1020
1100
|
* @param workflowId - Play-run id from {@link startPlayRun}
|
|
1021
1101
|
* @returns Current status with progress logs and partial results
|
|
1022
1102
|
*
|
|
@@ -1038,35 +1118,11 @@ var DeeplineClient = class {
|
|
|
1038
1118
|
);
|
|
1039
1119
|
return normalizePlayStatus(response);
|
|
1040
1120
|
}
|
|
1041
|
-
/**
|
|
1042
|
-
* Get the lightweight tail-polling status for a play execution.
|
|
1043
|
-
*
|
|
1044
|
-
* This is intentionally smaller than {@link getPlayStatus}: it returns the
|
|
1045
|
-
* fields needed for CLI log tailing while the run is in flight, without
|
|
1046
|
-
* forcing the API to rebuild final result views on every poll. Call
|
|
1047
|
-
* {@link getPlayStatus} once after a terminal state for the full result.
|
|
1048
|
-
*/
|
|
1049
|
-
async getPlayTailStatus(workflowId, options) {
|
|
1050
|
-
const params = new URLSearchParams({ mode: "tail" });
|
|
1051
|
-
if (typeof options?.afterLogIndex === "number") {
|
|
1052
|
-
params.set("afterLogIndex", String(options.afterLogIndex));
|
|
1053
|
-
}
|
|
1054
|
-
if (typeof options?.waitMs === "number") {
|
|
1055
|
-
params.set("waitMs", String(options.waitMs));
|
|
1056
|
-
}
|
|
1057
|
-
if (options?.terminalOnly) {
|
|
1058
|
-
params.set("terminalOnly", "true");
|
|
1059
|
-
}
|
|
1060
|
-
const response = await this.http.get(
|
|
1061
|
-
`/api/v2/plays/run/${encodeURIComponent(workflowId)}?${params.toString()}`
|
|
1062
|
-
);
|
|
1063
|
-
return normalizePlayStatus(response);
|
|
1064
|
-
}
|
|
1065
1121
|
/**
|
|
1066
1122
|
* Stream semantic play-run events using the same SSE feed as the dashboard.
|
|
1067
1123
|
*
|
|
1068
|
-
*
|
|
1069
|
-
*
|
|
1124
|
+
* The server emits a canonical `play.run.snapshot` event first for every
|
|
1125
|
+
* connection, then incremental live events until terminal state or reconnect.
|
|
1070
1126
|
*/
|
|
1071
1127
|
async *streamPlayRunEvents(workflowId, options) {
|
|
1072
1128
|
const headers = options?.lastEventId && options.lastEventId.trim() ? { "Last-Event-ID": options.lastEventId.trim() } : void 0;
|
|
@@ -1086,7 +1142,7 @@ var DeeplineClient = class {
|
|
|
1086
1142
|
*
|
|
1087
1143
|
* Sends a stop request for the run.
|
|
1088
1144
|
*
|
|
1089
|
-
* @param workflowId -
|
|
1145
|
+
* @param workflowId - Public Deepline play-run id to cancel
|
|
1090
1146
|
*
|
|
1091
1147
|
* @example
|
|
1092
1148
|
* ```typescript
|
|
@@ -1102,7 +1158,7 @@ var DeeplineClient = class {
|
|
|
1102
1158
|
/**
|
|
1103
1159
|
* Stop a running play execution, including open HITL waits.
|
|
1104
1160
|
*
|
|
1105
|
-
* @param workflowId -
|
|
1161
|
+
* @param workflowId - Public Deepline play-run id to stop
|
|
1106
1162
|
* @param options.reason - Optional audit/debug reason
|
|
1107
1163
|
*/
|
|
1108
1164
|
async stopPlay(workflowId, options) {
|
|
@@ -1174,32 +1230,42 @@ var DeeplineClient = class {
|
|
|
1174
1230
|
);
|
|
1175
1231
|
return response.runs ?? [];
|
|
1176
1232
|
}
|
|
1177
|
-
/**
|
|
1178
|
-
* Fetch the lightweight tail status for a run using the public runs resource model.
|
|
1179
|
-
*
|
|
1180
|
-
* This is the SDK equivalent of:
|
|
1181
|
-
*
|
|
1182
|
-
* ```bash
|
|
1183
|
-
* deepline runs tail <run-id> --json
|
|
1184
|
-
* ```
|
|
1185
|
-
*/
|
|
1233
|
+
/** Read the canonical run stream and return the latest run snapshot. */
|
|
1186
1234
|
async tailRun(runId, options) {
|
|
1187
|
-
const
|
|
1188
|
-
|
|
1189
|
-
|
|
1190
|
-
|
|
1235
|
+
const state = {
|
|
1236
|
+
runId,
|
|
1237
|
+
status: "running",
|
|
1238
|
+
logs: [],
|
|
1239
|
+
latest: null
|
|
1240
|
+
};
|
|
1241
|
+
let terminal = false;
|
|
1242
|
+
for await (const event of this.streamPlayRunEvents(runId, {
|
|
1243
|
+
mode: "cli",
|
|
1244
|
+
signal: options?.signal
|
|
1245
|
+
})) {
|
|
1246
|
+
const status = updatePlayLiveStatusState(state, event);
|
|
1247
|
+
if (!status) {
|
|
1248
|
+
continue;
|
|
1249
|
+
}
|
|
1250
|
+
terminal = TERMINAL_PLAY_STATUSES.has(status.status);
|
|
1251
|
+
if (terminal) {
|
|
1252
|
+
break;
|
|
1253
|
+
}
|
|
1191
1254
|
}
|
|
1192
|
-
if (
|
|
1193
|
-
|
|
1255
|
+
if (terminal && state.latest) {
|
|
1256
|
+
return await this.getRunStatus(state.latest.runId || runId).catch(
|
|
1257
|
+
() => state.latest ?? playRunStatusFromState(state)
|
|
1258
|
+
);
|
|
1194
1259
|
}
|
|
1195
|
-
if (
|
|
1196
|
-
|
|
1260
|
+
if (state.latest) {
|
|
1261
|
+
return state.latest;
|
|
1197
1262
|
}
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
|
|
1263
|
+
throw new DeeplineError(
|
|
1264
|
+
`Run stream for ${runId} ended before the initial snapshot.`,
|
|
1265
|
+
void 0,
|
|
1266
|
+
"PLAY_RUN_STREAM_EMPTY",
|
|
1267
|
+
{ runId }
|
|
1201
1268
|
);
|
|
1202
|
-
return normalizePlayStatus(response);
|
|
1203
1269
|
}
|
|
1204
1270
|
/**
|
|
1205
1271
|
* Fetch persisted logs for a run using the public runs resource model.
|
|
@@ -1356,11 +1422,11 @@ var DeeplineClient = class {
|
|
|
1356
1422
|
// Plays — high-level orchestration
|
|
1357
1423
|
// ——————————————————————————————————————————————————————————
|
|
1358
1424
|
/**
|
|
1359
|
-
* Run a play end-to-end: submit,
|
|
1425
|
+
* Run a play end-to-end: submit, stream until terminal, return result.
|
|
1360
1426
|
*
|
|
1361
1427
|
* This is the highest-level play execution method. It submits the play,
|
|
1362
|
-
*
|
|
1363
|
-
* and timing. Supports cancellation via `AbortSignal`.
|
|
1428
|
+
* reads the canonical run stream for status updates, and returns a structured
|
|
1429
|
+
* result with logs and timing. Supports cancellation via `AbortSignal`.
|
|
1364
1430
|
*
|
|
1365
1431
|
* @param code - Source string fallback; pass the bundled artifact in `options.artifact`
|
|
1366
1432
|
* @param csvPath - Input CSV path, or `null`
|
|
@@ -1376,7 +1442,6 @@ var DeeplineClient = class {
|
|
|
1376
1442
|
* const logs = status.progress?.logs ?? [];
|
|
1377
1443
|
* console.log(`[${status.status}] ${logs.length} log lines`);
|
|
1378
1444
|
* },
|
|
1379
|
-
* pollIntervalMs: 1000,
|
|
1380
1445
|
* });
|
|
1381
1446
|
*
|
|
1382
1447
|
* if (result.success) {
|
|
@@ -1407,33 +1472,53 @@ var DeeplineClient = class {
|
|
|
1407
1472
|
packagedFiles: options?.packagedFiles,
|
|
1408
1473
|
force: options?.force
|
|
1409
1474
|
});
|
|
1410
|
-
const pollInterval = options?.pollIntervalMs ?? 500;
|
|
1411
1475
|
const start = Date.now();
|
|
1412
|
-
|
|
1476
|
+
const state = {
|
|
1477
|
+
runId: workflowId,
|
|
1478
|
+
status: "running",
|
|
1479
|
+
logs: [],
|
|
1480
|
+
latest: null
|
|
1481
|
+
};
|
|
1482
|
+
if (options?.signal?.aborted) {
|
|
1483
|
+
await this.cancelPlay(workflowId);
|
|
1484
|
+
return {
|
|
1485
|
+
success: false,
|
|
1486
|
+
runId: workflowId,
|
|
1487
|
+
logs: [],
|
|
1488
|
+
durationMs: Date.now() - start,
|
|
1489
|
+
error: "Cancelled by user"
|
|
1490
|
+
};
|
|
1491
|
+
}
|
|
1492
|
+
for await (const event of this.streamPlayRunEvents(workflowId, {
|
|
1493
|
+
mode: "cli",
|
|
1494
|
+
signal: options?.signal
|
|
1495
|
+
})) {
|
|
1413
1496
|
if (options?.signal?.aborted) {
|
|
1414
1497
|
await this.cancelPlay(workflowId);
|
|
1415
1498
|
return {
|
|
1416
1499
|
success: false,
|
|
1417
1500
|
runId: workflowId,
|
|
1418
|
-
logs:
|
|
1501
|
+
logs: state.logs,
|
|
1419
1502
|
durationMs: Date.now() - start,
|
|
1420
1503
|
error: "Cancelled by user"
|
|
1421
1504
|
};
|
|
1422
1505
|
}
|
|
1423
|
-
const status =
|
|
1506
|
+
const status = updatePlayLiveStatusState(state, event);
|
|
1507
|
+
if (!status) {
|
|
1508
|
+
continue;
|
|
1509
|
+
}
|
|
1424
1510
|
options?.onProgress?.(status);
|
|
1425
1511
|
if (TERMINAL_PLAY_STATUSES.has(status.status)) {
|
|
1426
|
-
|
|
1427
|
-
|
|
1428
|
-
runId: status.runId || workflowId,
|
|
1429
|
-
result: status.result,
|
|
1430
|
-
logs: status.progress?.logs ?? [],
|
|
1431
|
-
durationMs: Date.now() - start,
|
|
1432
|
-
error: status.progress?.error ?? (status.status !== "completed" ? status.status : void 0)
|
|
1433
|
-
};
|
|
1512
|
+
const finalStatus = await this.getPlayStatus(status.runId || workflowId).catch(() => status);
|
|
1513
|
+
return playRunResultFromStatus(finalStatus, start, workflowId);
|
|
1434
1514
|
}
|
|
1435
|
-
await new Promise((resolve2) => setTimeout(resolve2, pollInterval));
|
|
1436
1515
|
}
|
|
1516
|
+
throw new DeeplineError(
|
|
1517
|
+
`Run stream for ${workflowId} ended before the run reached a terminal state.`,
|
|
1518
|
+
void 0,
|
|
1519
|
+
"PLAY_RUN_STREAM_ENDED",
|
|
1520
|
+
{ runId: workflowId, workflowId }
|
|
1521
|
+
);
|
|
1437
1522
|
}
|
|
1438
1523
|
// ——————————————————————————————————————————————————————————
|
|
1439
1524
|
// Health
|
|
@@ -3017,6 +3017,7 @@ async function handleWorkflowRoute(input: {
|
|
|
3017
3017
|
error: terminalEvent.error ?? null,
|
|
3018
3018
|
totalRows: terminalEvent.totalRows ?? null,
|
|
3019
3019
|
durationMs: terminalEvent.durationMs ?? null,
|
|
3020
|
+
completedAt: terminalEvent.ts,
|
|
3020
3021
|
events: eventResult?.events ?? [],
|
|
3021
3022
|
latestSeq: eventResult?.latestSeq ?? afterSeq,
|
|
3022
3023
|
wait: null,
|
|
@@ -3147,6 +3148,7 @@ async function handleWorkflowRoute(input: {
|
|
|
3147
3148
|
error: terminalState.error ?? null,
|
|
3148
3149
|
totalRows: terminalState.totalRows ?? null,
|
|
3149
3150
|
durationMs: terminalState.durationMs ?? null,
|
|
3151
|
+
completedAt: terminalState.completedAt ?? null,
|
|
3150
3152
|
wait: null,
|
|
3151
3153
|
coordinatorObserve: {
|
|
3152
3154
|
ms: Date.now() - statusStartedAt,
|
|
@@ -3229,7 +3231,7 @@ function stableHash(value: string): string {
|
|
|
3229
3231
|
}
|
|
3230
3232
|
|
|
3231
3233
|
const DYNAMIC_PLAY_WORKER_HARNESS_VERSION =
|
|
3232
|
-
'
|
|
3234
|
+
'h7-skip-high-volume-tool-traces';
|
|
3233
3235
|
const DYNAMIC_WORKER_BUNDLED_CODE_CACHE_MAX_ENTRIES = 64;
|
|
3234
3236
|
const dynamicWorkerBundledCodeCache = new Map<string, string>();
|
|
3235
3237
|
|
|
@@ -524,6 +524,26 @@ type WorkflowRunOutput = {
|
|
|
524
524
|
durationMs: number;
|
|
525
525
|
};
|
|
526
526
|
|
|
527
|
+
type LiveNodeProgressSnapshot = {
|
|
528
|
+
completed?: number;
|
|
529
|
+
total?: number;
|
|
530
|
+
failed?: number;
|
|
531
|
+
message?: string;
|
|
532
|
+
updatedAt?: number;
|
|
533
|
+
startedAt?: number;
|
|
534
|
+
completedAt?: number;
|
|
535
|
+
artifactTableNamespace?: string | null;
|
|
536
|
+
};
|
|
537
|
+
|
|
538
|
+
type LiveNodeProgressMap = Record<string, LiveNodeProgressSnapshot>;
|
|
539
|
+
|
|
540
|
+
type WorkerCtxCallbacks = {
|
|
541
|
+
onNodeProgress?: (input: {
|
|
542
|
+
nodeId: string;
|
|
543
|
+
progress: LiveNodeProgressSnapshot;
|
|
544
|
+
}) => void;
|
|
545
|
+
};
|
|
546
|
+
|
|
527
547
|
function nowMs(): number {
|
|
528
548
|
return Date.now();
|
|
529
549
|
}
|
|
@@ -535,6 +555,12 @@ function recordRunnerPerfTrace(input: {
|
|
|
535
555
|
extra?: Record<string, unknown>;
|
|
536
556
|
}): void {
|
|
537
557
|
if (!input.req.runId || !input.phase) return;
|
|
558
|
+
// Tool-level traces can fire once per row/provider step. Forwarding each one
|
|
559
|
+
// through the coordinator binding can consume Cloudflare's subrequest budget
|
|
560
|
+
// before large batched maps finish.
|
|
561
|
+
if (input.phase.startsWith('runner.tool.')) {
|
|
562
|
+
return;
|
|
563
|
+
}
|
|
538
564
|
const payload = {
|
|
539
565
|
ts: Date.now(),
|
|
540
566
|
source: 'dynamic_worker' as const,
|
|
@@ -1092,6 +1118,29 @@ async function callToolDirect(
|
|
|
1092
1118
|
let lastError: Error | null = null;
|
|
1093
1119
|
|
|
1094
1120
|
for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
|
|
1121
|
+
if (toolId === 'test_transient_500' || toolId === 'test_transient_429') {
|
|
1122
|
+
const syntheticResult = executeSyntheticTransientRetry(
|
|
1123
|
+
toolId,
|
|
1124
|
+
input,
|
|
1125
|
+
attempt,
|
|
1126
|
+
);
|
|
1127
|
+
if (syntheticResult.ok) {
|
|
1128
|
+
return wrapWorkerToolResult(
|
|
1129
|
+
toolId,
|
|
1130
|
+
syntheticResult.result,
|
|
1131
|
+
syntheticToolMetadata(toolId),
|
|
1132
|
+
);
|
|
1133
|
+
}
|
|
1134
|
+
lastError = new Error(
|
|
1135
|
+
`tool ${toolId} ${syntheticResult.status} attempt ${attempt}/${maxAttempts}: ${syntheticResult.message}`,
|
|
1136
|
+
);
|
|
1137
|
+
if (attempt >= maxAttempts) {
|
|
1138
|
+
throw lastError;
|
|
1139
|
+
}
|
|
1140
|
+
await new Promise((resolve) => setTimeout(resolve, 1_000));
|
|
1141
|
+
continue;
|
|
1142
|
+
}
|
|
1143
|
+
|
|
1095
1144
|
const res = await fetchRuntimeApi(req.baseUrl, path, {
|
|
1096
1145
|
method: 'POST',
|
|
1097
1146
|
headers: {
|
|
@@ -1277,6 +1326,41 @@ async function executeSyntheticTestRateLimitBatch(
|
|
|
1277
1326
|
};
|
|
1278
1327
|
}
|
|
1279
1328
|
|
|
1329
|
+
type SyntheticTransientRetryResult =
|
|
1330
|
+
| { ok: true; result: Record<string, unknown> }
|
|
1331
|
+
| { ok: false; status: number; message: string };
|
|
1332
|
+
|
|
1333
|
+
function executeSyntheticTransientRetry(
|
|
1334
|
+
toolId: string,
|
|
1335
|
+
input: Record<string, unknown>,
|
|
1336
|
+
attempt: number,
|
|
1337
|
+
): SyntheticTransientRetryResult {
|
|
1338
|
+
const failuresBeforeSuccess =
|
|
1339
|
+
typeof input.failures_before_success === 'number' &&
|
|
1340
|
+
Number.isInteger(input.failures_before_success) &&
|
|
1341
|
+
input.failures_before_success >= 0
|
|
1342
|
+
? input.failures_before_success
|
|
1343
|
+
: 1;
|
|
1344
|
+
if (attempt <= failuresBeforeSuccess) {
|
|
1345
|
+
const status = toolId === 'test_transient_429' ? 429 : 502;
|
|
1346
|
+
return {
|
|
1347
|
+
ok: false,
|
|
1348
|
+
status,
|
|
1349
|
+
message: `Synthetic transient ${status} for attempt ${attempt}`,
|
|
1350
|
+
};
|
|
1351
|
+
}
|
|
1352
|
+
return {
|
|
1353
|
+
ok: true,
|
|
1354
|
+
result: {
|
|
1355
|
+
status: 'completed',
|
|
1356
|
+
provider: 'test',
|
|
1357
|
+
key: String(input.key ?? 'transient'),
|
|
1358
|
+
attempts: attempt,
|
|
1359
|
+
recovered: attempt > 1,
|
|
1360
|
+
},
|
|
1361
|
+
};
|
|
1362
|
+
}
|
|
1363
|
+
|
|
1280
1364
|
function executeSyntheticTestRateLimit(
|
|
1281
1365
|
input: Record<string, unknown>,
|
|
1282
1366
|
): Record<string, unknown> {
|
|
@@ -2561,6 +2645,7 @@ function createMinimalWorkerCtx(
|
|
|
2561
2645
|
env: WorkerEnv,
|
|
2562
2646
|
workflowStep?: WorkflowStep,
|
|
2563
2647
|
abortSignal?: AbortSignal,
|
|
2648
|
+
callbacks?: WorkerCtxCallbacks,
|
|
2564
2649
|
): unknown {
|
|
2565
2650
|
let playCallCount = 0;
|
|
2566
2651
|
const parentChildCalls: Record<string, number> = {};
|
|
@@ -2635,6 +2720,7 @@ function createMinimalWorkerCtx(
|
|
|
2635
2720
|
opts?: WorkerMapOptions,
|
|
2636
2721
|
): Promise<unknown> => {
|
|
2637
2722
|
const mapStartedAt = nowMs();
|
|
2723
|
+
const mapNodeId = `map:${name}`;
|
|
2638
2724
|
const sliced = rows;
|
|
2639
2725
|
const baseOffset = 0;
|
|
2640
2726
|
const fieldEntries = Object.entries(fieldsDef);
|
|
@@ -2655,6 +2741,30 @@ function createMinimalWorkerCtx(
|
|
|
2655
2741
|
softWorkflowStepBudget: plan?.chunkPlan.softWorkflowStepBudget,
|
|
2656
2742
|
});
|
|
2657
2743
|
const outputFields = fieldEntries.map(([field]) => field);
|
|
2744
|
+
const updateMapProgress = (progress: LiveNodeProgressSnapshot) => {
|
|
2745
|
+
callbacks?.onNodeProgress?.({
|
|
2746
|
+
nodeId: mapNodeId,
|
|
2747
|
+
progress: {
|
|
2748
|
+
artifactTableNamespace: name,
|
|
2749
|
+
failed: 0,
|
|
2750
|
+
...progress,
|
|
2751
|
+
updatedAt: progress.updatedAt ?? nowMs(),
|
|
2752
|
+
},
|
|
2753
|
+
});
|
|
2754
|
+
};
|
|
2755
|
+
const formatMapProgressMessage = (completed: number, total?: number) =>
|
|
2756
|
+
typeof total === 'number' && Number.isFinite(total) && total > 0
|
|
2757
|
+
? `${completed.toLocaleString()} / ${total.toLocaleString()} rows processed`
|
|
2758
|
+
: `${completed.toLocaleString()} rows processed`;
|
|
2759
|
+
updateMapProgress({
|
|
2760
|
+
completed: 0,
|
|
2761
|
+
total: streaming ? undefined : sliced.length,
|
|
2762
|
+
startedAt: mapStartedAt,
|
|
2763
|
+
message: formatMapProgressMessage(
|
|
2764
|
+
0,
|
|
2765
|
+
streaming ? undefined : sliced.length,
|
|
2766
|
+
),
|
|
2767
|
+
});
|
|
2658
2768
|
const explicitRowKeysSeen =
|
|
2659
2769
|
opts?.key === undefined ? null : new Map<string, number>();
|
|
2660
2770
|
const resolveExplicitKeyValue = (
|
|
@@ -3156,6 +3266,14 @@ function createMinimalWorkerCtx(
|
|
|
3156
3266
|
`Map completed: ${totalRowsWritten} results ` +
|
|
3157
3267
|
`(${totalRowsExecuted} executed, ${totalRowsCached} already satisfied) ` +
|
|
3158
3268
|
`inserted=${totalRowsInserted} skipped=${totalRowsSkipped}`;
|
|
3269
|
+
const completedAt = nowMs();
|
|
3270
|
+
updateMapProgress({
|
|
3271
|
+
completed: totalRowsWritten,
|
|
3272
|
+
total: totalRowsWritten,
|
|
3273
|
+
completedAt,
|
|
3274
|
+
updatedAt: completedAt,
|
|
3275
|
+
message: formatMapProgressMessage(totalRowsWritten, totalRowsWritten),
|
|
3276
|
+
});
|
|
3159
3277
|
emitEvent({
|
|
3160
3278
|
type: 'log',
|
|
3161
3279
|
level: 'info',
|
|
@@ -3199,6 +3317,10 @@ function createMinimalWorkerCtx(
|
|
|
3199
3317
|
totalRowsDuplicateReused += chunkResult.rowsDuplicateReused;
|
|
3200
3318
|
totalRowsInserted += chunkResult.rowsInserted;
|
|
3201
3319
|
totalRowsSkipped += chunkResult.rowsSkipped;
|
|
3320
|
+
updateMapProgress({
|
|
3321
|
+
completed: totalRowsWritten,
|
|
3322
|
+
message: formatMapProgressMessage(totalRowsWritten),
|
|
3323
|
+
});
|
|
3202
3324
|
if (out.length < 10) {
|
|
3203
3325
|
out.push(...chunkResult.preview.slice(0, 10 - out.length));
|
|
3204
3326
|
}
|
|
@@ -3230,6 +3352,11 @@ function createMinimalWorkerCtx(
|
|
|
3230
3352
|
totalRowsDuplicateReused += chunkResult.rowsDuplicateReused;
|
|
3231
3353
|
totalRowsInserted += chunkResult.rowsInserted;
|
|
3232
3354
|
totalRowsSkipped += chunkResult.rowsSkipped;
|
|
3355
|
+
updateMapProgress({
|
|
3356
|
+
completed: totalRowsWritten,
|
|
3357
|
+
total: sliced.length,
|
|
3358
|
+
message: formatMapProgressMessage(totalRowsWritten, sliced.length),
|
|
3359
|
+
});
|
|
3233
3360
|
if (out.length < 10) {
|
|
3234
3361
|
out.push(...chunkResult.preview.slice(0, 10 - out.length));
|
|
3235
3362
|
}
|
|
@@ -3252,6 +3379,11 @@ function createMinimalWorkerCtx(
|
|
|
3252
3379
|
totalRowsInserted = chunkResult.rowsInserted;
|
|
3253
3380
|
totalRowsSkipped = chunkResult.rowsSkipped;
|
|
3254
3381
|
out.push(...chunkResult.preview);
|
|
3382
|
+
updateMapProgress({
|
|
3383
|
+
completed: chunkResult.rowsWritten,
|
|
3384
|
+
total: sliced.length,
|
|
3385
|
+
message: formatMapProgressMessage(chunkResult.rowsWritten, sliced.length),
|
|
3386
|
+
});
|
|
3255
3387
|
const dataset = finalize(chunkResult.rowsWritten);
|
|
3256
3388
|
recordRunnerPerfTrace({
|
|
3257
3389
|
req,
|
|
@@ -3919,6 +4051,7 @@ async function executeRunRequest(
|
|
|
3919
4051
|
// sees the final terminal status with no intermediate logs/progress.
|
|
3920
4052
|
let liveLogs: string[] = [];
|
|
3921
4053
|
let liveLogsDirty = false;
|
|
4054
|
+
let liveNodeProgress: LiveNodeProgressMap = {};
|
|
3922
4055
|
let lastLiveLogFlushAt =
|
|
3923
4056
|
nowMs() - LIVE_LOG_FLUSH_INTERVAL_MS + LIVE_LOG_FIRST_FLUSH_DELAY_MS;
|
|
3924
4057
|
let liveLogFlushInFlight: Promise<void> = Promise.resolve();
|
|
@@ -3928,6 +4061,21 @@ async function executeRunRequest(
|
|
|
3928
4061
|
liveLogs = [...liveLogs, trimmed].slice(-LIVE_LOG_BUFFER_LIMIT);
|
|
3929
4062
|
liveLogsDirty = true;
|
|
3930
4063
|
};
|
|
4064
|
+
const updateLiveNodeProgress = (input: {
|
|
4065
|
+
nodeId: string;
|
|
4066
|
+
progress: LiveNodeProgressSnapshot;
|
|
4067
|
+
}) => {
|
|
4068
|
+
const nodeId = input.nodeId.trim();
|
|
4069
|
+
if (!nodeId) return;
|
|
4070
|
+
liveNodeProgress = {
|
|
4071
|
+
...liveNodeProgress,
|
|
4072
|
+
[nodeId]: {
|
|
4073
|
+
...(liveNodeProgress[nodeId] ?? {}),
|
|
4074
|
+
...input.progress,
|
|
4075
|
+
},
|
|
4076
|
+
};
|
|
4077
|
+
};
|
|
4078
|
+
const liveNodeProgressSnapshot = () => ({ ...liveNodeProgress });
|
|
3931
4079
|
const flushLiveLogs = (force: boolean): void => {
|
|
3932
4080
|
if (!options?.persistResultDatasets) return;
|
|
3933
4081
|
if (!liveLogsDirty && !force) return;
|
|
@@ -3946,6 +4094,7 @@ async function executeRunRequest(
|
|
|
3946
4094
|
status: 'running',
|
|
3947
4095
|
runtimeBackend: 'cf_workflows_dynamic_worker',
|
|
3948
4096
|
liveLogs: snapshot,
|
|
4097
|
+
liveNodeProgress: liveNodeProgressSnapshot(),
|
|
3949
4098
|
lastCheckpointAt: now,
|
|
3950
4099
|
});
|
|
3951
4100
|
} catch {
|
|
@@ -3975,6 +4124,7 @@ async function executeRunRequest(
|
|
|
3975
4124
|
env,
|
|
3976
4125
|
workflowStep,
|
|
3977
4126
|
abortSignal,
|
|
4127
|
+
{ onNodeProgress: updateLiveNodeProgress },
|
|
3978
4128
|
);
|
|
3979
4129
|
try {
|
|
3980
4130
|
const playStartedAt = nowMs();
|
|
@@ -4017,12 +4167,14 @@ async function executeRunRequest(
|
|
|
4017
4167
|
action: 'update_run_status',
|
|
4018
4168
|
playId: req.runId,
|
|
4019
4169
|
status: 'completed',
|
|
4170
|
+
error: null,
|
|
4020
4171
|
result: terminalResult,
|
|
4021
4172
|
runtimeBackend: 'cf_workflows_dynamic_worker',
|
|
4022
4173
|
waitKind: null,
|
|
4023
4174
|
waitUntil: null,
|
|
4024
4175
|
activeBoundaryId: null,
|
|
4025
4176
|
liveLogs,
|
|
4177
|
+
liveNodeProgress: liveNodeProgressSnapshot(),
|
|
4026
4178
|
lastCheckpointAt: nowMs(),
|
|
4027
4179
|
});
|
|
4028
4180
|
recordRunnerPerfTrace({
|
|
@@ -4092,6 +4244,7 @@ async function executeRunRequest(
|
|
|
4092
4244
|
waitUntil: null,
|
|
4093
4245
|
activeBoundaryId: null,
|
|
4094
4246
|
liveLogs,
|
|
4247
|
+
liveNodeProgress: liveNodeProgressSnapshot(),
|
|
4095
4248
|
lastCheckpointAt: nowMs(),
|
|
4096
4249
|
});
|
|
4097
4250
|
await finalizeWorkerComputeBilling({
|