deepline 0.1.10 → 0.1.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -4
- package/dist/cli/index.js +509 -353
- package/dist/cli/index.js.map +1 -1
- package/dist/cli/index.mjs +513 -358
- package/dist/cli/index.mjs.map +1 -1
- package/dist/index.d.mts +250 -305
- package/dist/index.d.ts +250 -305
- package/dist/index.js +174 -286
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +174 -285
- package/dist/index.mjs.map +1 -1
- package/dist/repo/apps/play-runner-workers/src/coordinator-entry.ts +23 -13
- package/dist/repo/apps/play-runner-workers/src/entry.ts +581 -1220
- package/dist/repo/sdk/src/cli/commands/play.ts +381 -247
- package/dist/repo/sdk/src/cli/commands/tools.ts +1 -1
- package/dist/repo/sdk/src/cli/dataset-stats.ts +86 -12
- package/dist/repo/sdk/src/client.ts +54 -51
- package/dist/repo/sdk/src/index.ts +7 -16
- package/dist/repo/sdk/src/play.ts +122 -135
- package/dist/repo/sdk/src/plays/bundle-play-file.ts +6 -3
- package/dist/repo/sdk/src/tool-output.ts +0 -111
- package/dist/repo/sdk/src/types.ts +2 -0
- package/dist/repo/sdk/src/version.ts +1 -1
- package/dist/repo/sdk/src/worker-play-entry.ts +3 -0
- package/dist/repo/shared_libs/play-runtime/context.ts +510 -267
- package/dist/repo/shared_libs/play-runtime/csv-rename.ts +180 -0
- package/dist/repo/shared_libs/play-runtime/ctx-types.ts +13 -1
- package/dist/repo/shared_libs/play-runtime/tool-result.ts +139 -114
- package/dist/repo/shared_libs/plays/bundling/index.ts +68 -5
- package/dist/repo/shared_libs/plays/compiler-manifest.ts +1 -1
- package/dist/repo/shared_libs/plays/dataset.ts +1 -1
- package/dist/repo/shared_libs/plays/runtime-validation.ts +8 -28
- package/package.json +1 -1
- package/dist/repo/apps/play-runner-workers/src/runtime/tool-result.ts +0 -184
|
@@ -17,10 +17,10 @@
|
|
|
17
17
|
* to load CJS at runtime). The play here is statically imported and
|
|
18
18
|
* bundled into the Worker script.
|
|
19
19
|
* - Workers don't have node:fs / source-map-support. Stack traces are raw.
|
|
20
|
-
* - Direct postgres (`pg` library) won't bundle for
|
|
21
|
-
*
|
|
22
|
-
*
|
|
23
|
-
*
|
|
20
|
+
* - Direct postgres (`pg` library) won't bundle for Workers. This harness
|
|
21
|
+
* uses HTTP-only ctx — every ctx.csv / ctx.tool / row write goes through
|
|
22
|
+
* the runtime API endpoint, not direct DB. That keeps the Worker bundle
|
|
23
|
+
* compatible with the V8 isolate runtime.
|
|
24
24
|
*
|
|
25
25
|
* Status: experimental. First cut targets tool-basic (ctx.csv + ctx.map +
|
|
26
26
|
* ctx.tool). Plays that depend on the full ctx surface (durable sleep,
|
|
@@ -33,13 +33,28 @@ import {
|
|
|
33
33
|
type WorkflowEvent,
|
|
34
34
|
type WorkflowStep,
|
|
35
35
|
} from 'cloudflare:workers';
|
|
36
|
-
import {
|
|
36
|
+
import {
|
|
37
|
+
chooseMapChunkSize,
|
|
38
|
+
deterministicMapChunkStepName,
|
|
39
|
+
type ExecutionPlan,
|
|
40
|
+
} from '../../../shared_libs/play-runtime/execution-plan';
|
|
41
|
+
import {
|
|
42
|
+
compileRequestsWithStrategy,
|
|
43
|
+
executeChunkedRequests,
|
|
44
|
+
type ChunkExecutionResult,
|
|
45
|
+
} from '../../../shared_libs/play-runtime/batch-runtime';
|
|
46
|
+
import { getDefaultPlayRuntimeBatchStrategy } from '../../../shared_libs/play-runtime/default-batch-strategies';
|
|
37
47
|
import type { AnyBatchOperationStrategy } from '../../../shared_libs/play-runtime/batching-types';
|
|
38
48
|
import {
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
49
|
+
createToolBatchExecutor,
|
|
50
|
+
type ToolBatchRequest,
|
|
51
|
+
} from '../../../shared_libs/play-runtime/tool-batch-executor';
|
|
52
|
+
import {
|
|
53
|
+
createToolExecuteResult,
|
|
54
|
+
isToolExecuteResult,
|
|
55
|
+
type ToolExecuteResult,
|
|
56
|
+
type ToolResultMetadataInput,
|
|
57
|
+
} from '../../../shared_libs/play-runtime/tool-result';
|
|
43
58
|
import type { PlayCallGovernanceSnapshot } from '../../../shared_libs/play-runtime/scheduler-backend';
|
|
44
59
|
import type { PlayRuntimeManifestMap } from '../../../shared_libs/plays/compiler-manifest';
|
|
45
60
|
import {
|
|
@@ -64,31 +79,15 @@ import {
|
|
|
64
79
|
// re-bundle harness internals into per-play. Keep that in mind.
|
|
65
80
|
import {
|
|
66
81
|
harnessFetchStagedFile,
|
|
67
|
-
harnessPersistCompletedSheetRows,
|
|
68
|
-
harnessRuntimeApiCall,
|
|
69
82
|
harnessStartSheetDataset,
|
|
70
83
|
setHarnessBinding,
|
|
71
84
|
} from '../../../sdk/src/plays/harness-stub';
|
|
72
85
|
import {
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
executeChunkedRequests,
|
|
79
|
-
getDefaultPlayRuntimeBatchStrategy,
|
|
80
|
-
type ChunkExecutionResult,
|
|
81
|
-
} from './runtime/batching';
|
|
82
|
-
import {
|
|
83
|
-
createToolBatchExecutor,
|
|
84
|
-
type ToolBatchRequest,
|
|
85
|
-
} from './runtime/tool-batch';
|
|
86
|
-
import {
|
|
87
|
-
createToolExecuteResult,
|
|
88
|
-
isToolExecuteResult,
|
|
89
|
-
type ToolExecuteResult,
|
|
90
|
-
type ToolResultMetadataInput,
|
|
91
|
-
} from './runtime/tool-result';
|
|
86
|
+
applyCsvRenameProjection,
|
|
87
|
+
stripCsvProjectedFields,
|
|
88
|
+
cloneCsvAliasedRow,
|
|
89
|
+
type CsvRenameOptions,
|
|
90
|
+
} from '../../../shared_libs/play-runtime/csv-rename';
|
|
92
91
|
import { coordinatorRequestHeaders } from '../../../shared_libs/play-runtime/coordinator-headers';
|
|
93
92
|
|
|
94
93
|
// The play's default export. The bundler injects this — see bundle-play-file.ts.
|
|
@@ -149,6 +148,19 @@ type WorkerEnv = {
|
|
|
149
148
|
PLAY_ASSETS?: {
|
|
150
149
|
readText(logicalPath: string): Promise<string>;
|
|
151
150
|
};
|
|
151
|
+
/**
|
|
152
|
+
* In-process Fetcher constructed by the coordinator and handed to the
|
|
153
|
+
* per-graphHash play Worker. When present, runtime callbacks
|
|
154
|
+
* (`/api/v2/plays/internal/runtime`, `/api/v2/plays/internal/*`,
|
|
155
|
+
* `/api/v2/plays/runtime-tools/*`) skip the public callback URL and route
|
|
156
|
+
* directly through the coordinator's process to the configured app — saves
|
|
157
|
+
* the *.workers.dev → CF edge → cloudflared → localhost chain on every
|
|
158
|
+
* runtime callback. Absent on legacy coordinator deploys; the fetch
|
|
159
|
+
* helpers fall back to `globalThis.fetch(req.baseUrl + path)`.
|
|
160
|
+
*/
|
|
161
|
+
RUNTIME_API?: {
|
|
162
|
+
fetch(input: Request): Promise<Response>;
|
|
163
|
+
};
|
|
152
164
|
/**
|
|
153
165
|
* Loopback RPC binding into the coordinator Worker. Used for CF-to-CF
|
|
154
166
|
* child orchestration so nested plays do not bounce through a public
|
|
@@ -176,7 +188,7 @@ type WorkerEnv = {
|
|
|
176
188
|
): Promise<Record<string, unknown>>;
|
|
177
189
|
recordPerfTrace(
|
|
178
190
|
runId: string,
|
|
179
|
-
payload:
|
|
191
|
+
payload: Record<string, unknown>,
|
|
180
192
|
): Promise<void>;
|
|
181
193
|
};
|
|
182
194
|
/**
|
|
@@ -194,6 +206,11 @@ type WorkerEnv = {
|
|
|
194
206
|
HARNESS?: import('../../play-harness-worker/src/rpc-types').PlayHarnessRpc;
|
|
195
207
|
};
|
|
196
208
|
|
|
209
|
+
let cachedRuntimeApiBinding: WorkerEnv['RUNTIME_API'] | null = null;
|
|
210
|
+
function captureRuntimeApiBinding(env: WorkerEnv): void {
|
|
211
|
+
cachedRuntimeApiBinding = env.RUNTIME_API ?? null;
|
|
212
|
+
}
|
|
213
|
+
|
|
197
214
|
let cachedCoordinatorBinding: WorkerEnv['COORDINATOR'] | null = null;
|
|
198
215
|
function captureCoordinatorBinding(env: WorkerEnv): void {
|
|
199
216
|
cachedCoordinatorBinding = env.COORDINATOR ?? null;
|
|
@@ -215,8 +232,10 @@ function captureHarnessBinding(env: WorkerEnv): void {
|
|
|
215
232
|
|
|
216
233
|
/**
|
|
217
234
|
* One-shot per-isolate harness wiring probe. Logs a single line that
|
|
218
|
-
*
|
|
219
|
-
*
|
|
235
|
+
* confirms whether `env.HARNESS` resolved to a live Worker on this
|
|
236
|
+
* isolate's first run. We use this as a low-noise diagnostic operators
|
|
237
|
+
* can grep for in any env — local dev, preview, prod — without having
|
|
238
|
+
* to plumb a separate health route or instrument SDK call sites.
|
|
220
239
|
*
|
|
221
240
|
* Behavior on fail:
|
|
222
241
|
* - Binding missing entirely → log clearly that HARNESS is unwired so
|
|
@@ -239,9 +258,8 @@ async function probeHarnessOnce(
|
|
|
239
258
|
if (harnessProbeFiredForIsolate) return;
|
|
240
259
|
harnessProbeFiredForIsolate = true;
|
|
241
260
|
if (!env.HARNESS) {
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
`[harness-probe] env.HARNESS unwired — coordinator did not pass the binding. ` +
|
|
261
|
+
console.log(
|
|
262
|
+
`${runPrefix} [harness-probe] env.HARNESS unwired — coordinator did not pass the binding. ` +
|
|
245
263
|
`Per-play SDK call sites that reach into the harness will throw clearly. ` +
|
|
246
264
|
`See apps/play-harness-worker/README.md.`,
|
|
247
265
|
);
|
|
@@ -251,158 +269,66 @@ async function probeHarnessOnce(
|
|
|
251
269
|
typeof (env.HARNESS as { ping?: () => Promise<{ ok: true; ts: number }> })
|
|
252
270
|
.ping !== 'function'
|
|
253
271
|
) {
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
`[harness-probe] env.HARNESS is present but does not expose ping(); ` +
|
|
272
|
+
console.log(
|
|
273
|
+
`${runPrefix} [harness-probe] env.HARNESS is present but does not expose ping(); ` +
|
|
257
274
|
`continuing and relying on the first real call to fail if the contract changed.`,
|
|
258
275
|
);
|
|
259
276
|
return;
|
|
260
277
|
}
|
|
261
278
|
try {
|
|
262
|
-
await env.HARNESS.ping();
|
|
279
|
+
const result = await env.HARNESS.ping();
|
|
280
|
+
console.log(
|
|
281
|
+
`${runPrefix} [harness-probe] env.HARNESS connected ts=${result.ts}`,
|
|
282
|
+
);
|
|
263
283
|
} catch (error) {
|
|
264
284
|
const message = error instanceof Error ? error.message : String(error);
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
`[harness-probe] env.HARNESS resolved but ping failed: ${message}`,
|
|
285
|
+
console.log(
|
|
286
|
+
`${runPrefix} [harness-probe] env.HARNESS resolved but ping failed: ${message}`,
|
|
268
287
|
);
|
|
269
288
|
}
|
|
270
289
|
}
|
|
271
290
|
/**
|
|
272
|
-
* Routes
|
|
273
|
-
*
|
|
274
|
-
* quirks across WorkerEntrypoint boundaries.
|
|
291
|
+
* Routes runtime API requests through the in-process RUNTIME_API binding.
|
|
292
|
+
* The coordinator MUST provide this binding — there is no public-fetch fallback.
|
|
275
293
|
*/
|
|
276
294
|
const RUNTIME_API_TIMEOUT_MS = 30_000;
|
|
277
295
|
const RUNTIME_API_PLAY_RUN_TIMEOUT_MS = 75_000;
|
|
278
|
-
const HARNESS_RUNTIME_FORWARD_HEADERS = [
|
|
279
|
-
'x-deepline-request-id',
|
|
280
|
-
EXECUTE_TOOL_METADATA_HEADER,
|
|
281
|
-
] as const;
|
|
282
|
-
const TOOL_HTTP_MAX_ATTEMPTS = 3;
|
|
283
|
-
const TOOL_RETRY_AFTER_DEFAULT_MS = 1_000;
|
|
284
|
-
const TOOL_RETRY_AFTER_MAX_MS = 30 * 60 * 1_000;
|
|
285
|
-
const TOOL_IN_MEMORY_RETRY_SLEEP_MAX_MS = 30_000;
|
|
286
|
-
const TRANSIENT_HTTP_RETRY_SAFE_TOOL_IDS = new Set(['test_transient_500']);
|
|
287
|
-
|
|
288
|
-
function parseRetryAfterMs(header: string | null): number {
|
|
289
|
-
if (!header) return TOOL_RETRY_AFTER_DEFAULT_MS;
|
|
290
|
-
const numeric = Number(header);
|
|
291
|
-
if (Number.isFinite(numeric) && numeric >= 0) {
|
|
292
|
-
return Math.max(1, Math.ceil(numeric * 1_000));
|
|
293
|
-
}
|
|
294
|
-
const retryAt = Date.parse(header);
|
|
295
|
-
if (Number.isFinite(retryAt)) {
|
|
296
|
-
return Math.max(1, retryAt - Date.now());
|
|
297
|
-
}
|
|
298
|
-
return TOOL_RETRY_AFTER_DEFAULT_MS;
|
|
299
|
-
}
|
|
300
|
-
|
|
301
|
-
function isRetryableToolHttpStatus(toolId: string, status: number): boolean {
|
|
302
|
-
if (status === 429) return true;
|
|
303
|
-
return (
|
|
304
|
-
status >= 500 &&
|
|
305
|
-
status < 600 &&
|
|
306
|
-
TRANSIENT_HTTP_RETRY_SAFE_TOOL_IDS.has(toolId)
|
|
307
|
-
);
|
|
308
|
-
}
|
|
309
|
-
|
|
310
|
-
async function sleepForToolRetry(input: {
|
|
311
|
-
workflowStep?: WorkflowStep;
|
|
312
|
-
toolId: string;
|
|
313
|
-
status: number;
|
|
314
|
-
attempt: number;
|
|
315
|
-
retryAfterMs: number;
|
|
316
|
-
retryKey?: string | null;
|
|
317
|
-
toolInput: Record<string, unknown>;
|
|
318
|
-
}): Promise<void> {
|
|
319
|
-
const retryAfterMs = Math.max(1, Math.ceil(input.retryAfterMs));
|
|
320
|
-
if (retryAfterMs > TOOL_RETRY_AFTER_MAX_MS) {
|
|
321
|
-
throw new Error(
|
|
322
|
-
`tool ${input.toolId} returned ${input.status} with retry-after ${retryAfterMs}ms, above max supported retry wait ${TOOL_RETRY_AFTER_MAX_MS}ms.`,
|
|
323
|
-
);
|
|
324
|
-
}
|
|
325
|
-
|
|
326
|
-
if (input.workflowStep) {
|
|
327
|
-
const inputHash = (
|
|
328
|
-
await hashJson({
|
|
329
|
-
key: input.retryKey ?? '',
|
|
330
|
-
input: input.toolInput,
|
|
331
|
-
})
|
|
332
|
-
).slice(0, 16);
|
|
333
|
-
await (
|
|
334
|
-
input.workflowStep.sleep as unknown as (
|
|
335
|
-
name: string,
|
|
336
|
-
duration: number,
|
|
337
|
-
) => Promise<void>
|
|
338
|
-
)(
|
|
339
|
-
`tool-retry:${input.toolId}:${input.status}:attempt-${input.attempt}:${inputHash}`,
|
|
340
|
-
retryAfterMs,
|
|
341
|
-
);
|
|
342
|
-
return;
|
|
343
|
-
}
|
|
344
|
-
|
|
345
|
-
if (retryAfterMs > TOOL_IN_MEMORY_RETRY_SLEEP_MAX_MS) {
|
|
346
|
-
throw new Error(
|
|
347
|
-
`tool ${input.toolId} returned ${input.status} with retry-after ${retryAfterMs}ms, but no durable workflow step was available.`,
|
|
348
|
-
);
|
|
349
|
-
}
|
|
350
|
-
await new Promise((resolve) => setTimeout(resolve, retryAfterMs));
|
|
351
|
-
}
|
|
352
296
|
|
|
353
297
|
async function fetchRuntimeApi(
|
|
354
298
|
baseUrl: string,
|
|
355
299
|
path: string,
|
|
356
300
|
init: RequestInit,
|
|
357
301
|
): Promise<Response> {
|
|
302
|
+
if (!cachedRuntimeApiBinding) {
|
|
303
|
+
throw new Error(
|
|
304
|
+
`[play-harness] RUNTIME_API binding missing — coordinator did not wire it before invoking the play. path=${path}`,
|
|
305
|
+
);
|
|
306
|
+
}
|
|
358
307
|
const timeoutMs =
|
|
359
308
|
path === '/api/v2/plays/run'
|
|
360
309
|
? RUNTIME_API_PLAY_RUN_TIMEOUT_MS
|
|
361
310
|
: RUNTIME_API_TIMEOUT_MS;
|
|
362
|
-
const
|
|
363
|
-
const
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
311
|
+
const controller = new AbortController();
|
|
312
|
+
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
|
313
|
+
try {
|
|
314
|
+
const mergedInit: RequestInit = {
|
|
315
|
+
...init,
|
|
316
|
+
signal: controller.signal,
|
|
317
|
+
};
|
|
318
|
+
const res = await cachedRuntimeApiBinding.fetch(
|
|
319
|
+
new Request(`${baseUrl}${path}`, mergedInit),
|
|
368
320
|
);
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
for (const name of HARNESS_RUNTIME_FORWARD_HEADERS) {
|
|
373
|
-
const value = headers.get(name);
|
|
374
|
-
if (value) forwardedHeaders[name] = value;
|
|
375
|
-
}
|
|
376
|
-
|
|
377
|
-
let body: unknown = {};
|
|
378
|
-
if (typeof init.body === 'string' && init.body.trim()) {
|
|
379
|
-
try {
|
|
380
|
-
body = JSON.parse(init.body) as unknown;
|
|
381
|
-
} catch (error) {
|
|
321
|
+
return res;
|
|
322
|
+
} catch (err) {
|
|
323
|
+
if (err instanceof Error && err.name === 'AbortError') {
|
|
382
324
|
throw new Error(
|
|
383
|
-
`[play-harness]
|
|
384
|
-
error instanceof Error ? error.message : String(error)
|
|
385
|
-
}`,
|
|
325
|
+
`[play-harness] RUNTIME_API call timed out after ${timeoutMs}ms. path=${path} baseUrl=${baseUrl}`,
|
|
386
326
|
);
|
|
387
327
|
}
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
);
|
|
328
|
+
throw err;
|
|
329
|
+
} finally {
|
|
330
|
+
clearTimeout(timer);
|
|
392
331
|
}
|
|
393
|
-
|
|
394
|
-
void baseUrl;
|
|
395
|
-
const result = await harnessRuntimeApiCall({
|
|
396
|
-
executorToken: authorization.slice(bearerPrefix.length).trim(),
|
|
397
|
-
path,
|
|
398
|
-
body,
|
|
399
|
-
headers: Object.keys(forwardedHeaders).length ? forwardedHeaders : undefined,
|
|
400
|
-
timeoutMs,
|
|
401
|
-
});
|
|
402
|
-
return new Response(result.body, {
|
|
403
|
-
status: result.status,
|
|
404
|
-
headers: result.headers,
|
|
405
|
-
});
|
|
406
332
|
}
|
|
407
333
|
|
|
408
334
|
const WORKER_PLAY_CALL_LIMITS = {
|
|
@@ -448,7 +374,7 @@ function makeWorkerDataset<T extends Record<string, unknown>>(
|
|
|
448
374
|
datasetId: string;
|
|
449
375
|
tableNamespace: string;
|
|
450
376
|
};
|
|
451
|
-
const previewLimit =
|
|
377
|
+
const previewLimit = 5;
|
|
452
378
|
const inferredColumns = (() => {
|
|
453
379
|
const cols = new Set<string>();
|
|
454
380
|
for (const r of rows) {
|
|
@@ -533,60 +459,19 @@ type WorkflowRunOutput = {
|
|
|
533
459
|
durationMs: number;
|
|
534
460
|
};
|
|
535
461
|
|
|
536
|
-
type DynamicWorkerPerfTracePayload = {
|
|
537
|
-
ts: number;
|
|
538
|
-
source: 'dynamic_worker';
|
|
539
|
-
runId: string;
|
|
540
|
-
phase: string;
|
|
541
|
-
ms: number;
|
|
542
|
-
graphHash?: string | null;
|
|
543
|
-
[key: string]: unknown;
|
|
544
|
-
};
|
|
545
|
-
|
|
546
462
|
function nowMs(): number {
|
|
547
463
|
return Date.now();
|
|
548
464
|
}
|
|
549
465
|
|
|
550
|
-
function recordDynamicWorkerPerfTrace(input: {
|
|
551
|
-
env: WorkerEnv;
|
|
552
|
-
runId: string;
|
|
553
|
-
phase: string;
|
|
554
|
-
ms: number;
|
|
555
|
-
graphHash?: string | null;
|
|
556
|
-
extra?: Record<string, unknown>;
|
|
557
|
-
waitUntil?: (promise: Promise<unknown>) => void;
|
|
558
|
-
}): void {
|
|
559
|
-
if (!input.runId || !input.phase) return;
|
|
560
|
-
const payload: DynamicWorkerPerfTracePayload = {
|
|
561
|
-
ts: Date.now(),
|
|
562
|
-
source: 'dynamic_worker',
|
|
563
|
-
runId: input.runId,
|
|
564
|
-
phase: input.phase,
|
|
565
|
-
ms: input.ms,
|
|
566
|
-
...(input.graphHash ? { graphHash: input.graphHash } : {}),
|
|
567
|
-
...(input.extra ?? {}),
|
|
568
|
-
};
|
|
569
|
-
console.log(`[perf-trace] ${JSON.stringify(payload)}`);
|
|
570
|
-
const send = input.env.COORDINATOR?.recordPerfTrace(
|
|
571
|
-
input.runId,
|
|
572
|
-
payload,
|
|
573
|
-
).catch((error) => {
|
|
574
|
-
console.error(
|
|
575
|
-
`[play-harness] non-fatal dynamic trace append failed runId=${input.runId}: ${
|
|
576
|
-
error instanceof Error ? error.message : String(error)
|
|
577
|
-
}`,
|
|
578
|
-
);
|
|
579
|
-
});
|
|
580
|
-
if (send && input.waitUntil) {
|
|
581
|
-
input.waitUntil(send);
|
|
582
|
-
}
|
|
583
|
-
}
|
|
584
|
-
|
|
585
466
|
function makeRequestId(): string {
|
|
586
467
|
// Workers crypto.randomUUID is available without nodejs_compat.
|
|
587
468
|
return crypto.randomUUID();
|
|
588
469
|
}
|
|
589
470
|
|
|
471
|
+
function publicCsvInputRow<T extends Record<string, unknown>>(row: T): T {
|
|
472
|
+
return stripCsvProjectedFields(row) as T;
|
|
473
|
+
}
|
|
474
|
+
|
|
590
475
|
/**
|
|
591
476
|
* Strip credentials and JWT-shaped tokens from any string before it lands in
|
|
592
477
|
* a log buffer or upstream error message. The harness routinely echoes
|
|
@@ -609,11 +494,11 @@ function redactSecretsFromLogString(value: string): string {
|
|
|
609
494
|
async function postRuntimeApi<T>(
|
|
610
495
|
baseUrl: string,
|
|
611
496
|
executorToken: string,
|
|
612
|
-
body:
|
|
497
|
+
body: unknown,
|
|
613
498
|
): Promise<T> {
|
|
614
|
-
// Routes through the
|
|
615
|
-
//
|
|
616
|
-
//
|
|
499
|
+
// Routes through the in-process RUNTIME_API binding when present; otherwise
|
|
500
|
+
// falls back to a public fetch against `${baseUrl}${path}`. Either path
|
|
501
|
+
// hits the same handler with the same auth — only the transport changes.
|
|
617
502
|
const res = await fetchRuntimeApi(baseUrl, '/api/v2/plays/internal/runtime', {
|
|
618
503
|
method: 'POST',
|
|
619
504
|
headers: {
|
|
@@ -674,7 +559,7 @@ function describeRuntimeApiBody(body: unknown): string {
|
|
|
674
559
|
async function postRuntimeApiBestEffort(
|
|
675
560
|
baseUrl: string,
|
|
676
561
|
executorToken: string,
|
|
677
|
-
body:
|
|
562
|
+
body: unknown,
|
|
678
563
|
): Promise<boolean> {
|
|
679
564
|
try {
|
|
680
565
|
await postRuntimeApi(baseUrl, executorToken, body);
|
|
@@ -689,67 +574,6 @@ async function postRuntimeApiBestEffort(
|
|
|
689
574
|
}
|
|
690
575
|
}
|
|
691
576
|
|
|
692
|
-
type WorkerRuntimeTransport = {
|
|
693
|
-
postRuntimeApi<T>(body: RuntimeApiAction): Promise<T>;
|
|
694
|
-
postRuntimeApiBestEffort(body: RuntimeApiAction): Promise<boolean>;
|
|
695
|
-
submitChild(body: Record<string, unknown>): Promise<{
|
|
696
|
-
workflowId?: string;
|
|
697
|
-
runId?: string;
|
|
698
|
-
status?: string;
|
|
699
|
-
mode?: string;
|
|
700
|
-
output?: unknown;
|
|
701
|
-
result?: unknown;
|
|
702
|
-
error?: unknown;
|
|
703
|
-
logs?: string[];
|
|
704
|
-
timings?: Array<{ phase: string; ms: number }>;
|
|
705
|
-
}>;
|
|
706
|
-
signalParentTerminal(input: {
|
|
707
|
-
status: 'completed' | 'failed' | 'cancelled';
|
|
708
|
-
result?: Record<string, unknown> | null;
|
|
709
|
-
error?: string | null;
|
|
710
|
-
}): Promise<void>;
|
|
711
|
-
};
|
|
712
|
-
|
|
713
|
-
function createWorkerRuntimeTransport(req: RunRequest): WorkerRuntimeTransport {
|
|
714
|
-
return {
|
|
715
|
-
postRuntimeApi: <T>(body: RuntimeApiAction) =>
|
|
716
|
-
postRuntimeApi<T>(req.baseUrl, req.executorToken, body),
|
|
717
|
-
postRuntimeApiBestEffort: (body: RuntimeApiAction) =>
|
|
718
|
-
postRuntimeApiBestEffort(req.baseUrl, req.executorToken, body),
|
|
719
|
-
submitChild: (body: Record<string, unknown>) =>
|
|
720
|
-
submitChildPlayThroughCoordinator({ req, body }),
|
|
721
|
-
signalParentTerminal: (input) =>
|
|
722
|
-
signalParentPlayTerminal({ req, ...input }),
|
|
723
|
-
};
|
|
724
|
-
}
|
|
725
|
-
|
|
726
|
-
type WorkerChildPlayExecutor = {
|
|
727
|
-
submit(
|
|
728
|
-
body: Record<string, unknown>,
|
|
729
|
-
): ReturnType<WorkerRuntimeTransport['submitChild']>;
|
|
730
|
-
waitTerminal(input: {
|
|
731
|
-
workflowStep?: WorkflowStep;
|
|
732
|
-
workflowId: string;
|
|
733
|
-
playName: string;
|
|
734
|
-
key: string;
|
|
735
|
-
timeoutMs: number;
|
|
736
|
-
}): Promise<unknown>;
|
|
737
|
-
};
|
|
738
|
-
|
|
739
|
-
function createWorkerChildPlayExecutor(input: {
|
|
740
|
-
req: RunRequest;
|
|
741
|
-
transport: WorkerRuntimeTransport;
|
|
742
|
-
}): WorkerChildPlayExecutor {
|
|
743
|
-
return {
|
|
744
|
-
submit: (body) => input.transport.submitChild(body),
|
|
745
|
-
waitTerminal: (waitInput) =>
|
|
746
|
-
waitForChildPlayTerminalEvent({
|
|
747
|
-
req: input.req,
|
|
748
|
-
...waitInput,
|
|
749
|
-
}),
|
|
750
|
-
};
|
|
751
|
-
}
|
|
752
|
-
|
|
753
577
|
async function submitChildPlayThroughCoordinator(input: {
|
|
754
578
|
req: RunRequest;
|
|
755
579
|
body: unknown;
|
|
@@ -792,7 +616,17 @@ async function submitChildPlayThroughCoordinator(input: {
|
|
|
792
616
|
},
|
|
793
617
|
);
|
|
794
618
|
const text = await res.text().catch(() => '');
|
|
795
|
-
let parsed: {
|
|
619
|
+
let parsed: {
|
|
620
|
+
workflowId?: string;
|
|
621
|
+
runId?: string;
|
|
622
|
+
status?: string;
|
|
623
|
+
mode?: string;
|
|
624
|
+
output?: unknown;
|
|
625
|
+
result?: unknown;
|
|
626
|
+
error?: unknown;
|
|
627
|
+
logs?: string[];
|
|
628
|
+
timings?: Array<{ phase: string; ms: number }>;
|
|
629
|
+
} = {};
|
|
796
630
|
try {
|
|
797
631
|
parsed = text ? JSON.parse(text) : {};
|
|
798
632
|
} catch {
|
|
@@ -998,11 +832,7 @@ async function signalParentPlayTerminal(input: {
|
|
|
998
832
|
|
|
999
833
|
async function executeTool(
|
|
1000
834
|
req: RunRequest,
|
|
1001
|
-
args: {
|
|
1002
|
-
toolId: string;
|
|
1003
|
-
input: Record<string, unknown>;
|
|
1004
|
-
retryKey?: string | null;
|
|
1005
|
-
},
|
|
835
|
+
args: { id: string; toolId: string; input: Record<string, unknown> },
|
|
1006
836
|
workflowStep?: WorkflowStep,
|
|
1007
837
|
): Promise<ToolExecuteResult> {
|
|
1008
838
|
if (args.toolId === 'test_wait_for_event' && workflowStep) {
|
|
@@ -1018,76 +848,52 @@ async function executeTool(
|
|
|
1018
848
|
// service bindings, NOT through HTTP from this worker. Removing the
|
|
1019
849
|
// dispatcher-side coordinatorUrl plumbing intentionally turns the old
|
|
1020
850
|
// HTTP-based dedup helpers into dead code.
|
|
1021
|
-
return callToolDirect(
|
|
1022
|
-
req,
|
|
1023
|
-
args.toolId,
|
|
1024
|
-
args.input,
|
|
1025
|
-
workflowStep,
|
|
1026
|
-
args.retryKey,
|
|
1027
|
-
);
|
|
851
|
+
return callToolDirect(req, args);
|
|
1028
852
|
}
|
|
1029
853
|
|
|
1030
|
-
function
|
|
1031
|
-
return
|
|
854
|
+
function isToolExecuteRecord(value: unknown): value is Record<string, unknown> {
|
|
855
|
+
return typeof value === 'object' && value !== null && !Array.isArray(value);
|
|
1032
856
|
}
|
|
1033
857
|
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
|
|
858
|
+
function normalizeToolExecuteArgs(
|
|
859
|
+
requestOrKey: unknown,
|
|
860
|
+
toolId?: unknown,
|
|
861
|
+
input?: unknown,
|
|
862
|
+
): { id: string; toolId: string; input: Record<string, unknown> } {
|
|
863
|
+
if (isToolExecuteRecord(requestOrKey)) {
|
|
864
|
+
const id = requestOrKey.id;
|
|
865
|
+
const tool = requestOrKey.tool;
|
|
866
|
+
const requestInput = requestOrKey.input;
|
|
867
|
+
if (
|
|
868
|
+
typeof id !== 'string' ||
|
|
869
|
+
!id.trim() ||
|
|
870
|
+
typeof tool !== 'string' ||
|
|
871
|
+
!tool ||
|
|
872
|
+
!isToolExecuteRecord(requestInput)
|
|
873
|
+
) {
|
|
874
|
+
throw new Error(
|
|
875
|
+
'ctx.tools.execute({ id, tool, input }) requires a non-empty id, tool string, and input object.',
|
|
876
|
+
);
|
|
877
|
+
}
|
|
878
|
+
return { id: id.trim(), toolId: tool, input: requestInput };
|
|
1052
879
|
}
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
|
|
880
|
+
|
|
881
|
+
if (
|
|
882
|
+
typeof requestOrKey !== 'string' ||
|
|
883
|
+
!requestOrKey.trim() ||
|
|
884
|
+
typeof toolId !== 'string' ||
|
|
885
|
+
!toolId ||
|
|
886
|
+
!isToolExecuteRecord(input)
|
|
887
|
+
) {
|
|
888
|
+
throw new Error(
|
|
889
|
+
'ctx.tools.execute(key, toolId, input) requires a tool ID and input object.',
|
|
890
|
+
);
|
|
1056
891
|
}
|
|
1057
|
-
|
|
1058
|
-
const firstEntry = waits.entries().next().value;
|
|
1059
|
-
if (!firstEntry) return null;
|
|
1060
|
-
const [eventKey, waitUntil] = firstEntry;
|
|
1061
|
-
return { eventKey, waitUntil };
|
|
892
|
+
return { id: requestOrKey.trim(), toolId, input };
|
|
1062
893
|
}
|
|
1063
894
|
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
eventKey: string;
|
|
1067
|
-
timeoutMs: number;
|
|
1068
|
-
waiting: boolean;
|
|
1069
|
-
}): Promise<void> {
|
|
1070
|
-
const activeWait = updateActiveSyntheticIntegrationWait({
|
|
1071
|
-
runId: input.req.runId,
|
|
1072
|
-
eventKey: input.eventKey,
|
|
1073
|
-
timeoutMs: input.timeoutMs,
|
|
1074
|
-
waiting: input.waiting,
|
|
1075
|
-
});
|
|
1076
|
-
await postRuntimeApiBestEffort(
|
|
1077
|
-
input.req.baseUrl,
|
|
1078
|
-
input.req.executorToken,
|
|
1079
|
-
runtimeRunActions.updateStatus({
|
|
1080
|
-
playId: input.req.runId,
|
|
1081
|
-
status: 'running',
|
|
1082
|
-
runtimeBackend: 'cf_workflows_dynamic_worker',
|
|
1083
|
-
waitKind: activeWait ? 'integration_event_batch' : null,
|
|
1084
|
-
waitUntil: activeWait?.waitUntil ?? null,
|
|
1085
|
-
activeBoundaryId: activeWait
|
|
1086
|
-
? `integration_event:${activeWait.eventKey}`
|
|
1087
|
-
: null,
|
|
1088
|
-
lastCheckpointAt: Date.now(),
|
|
1089
|
-
}),
|
|
1090
|
-
);
|
|
895
|
+
function integrationEventType(eventKey: string): string {
|
|
896
|
+
return workflowEventType(`integration_event_${eventKey}`);
|
|
1091
897
|
}
|
|
1092
898
|
|
|
1093
899
|
async function waitForSyntheticIntegrationEvent(
|
|
@@ -1104,7 +910,7 @@ async function waitForSyntheticIntegrationEvent(
|
|
|
1104
910
|
? Math.max(1, Math.round(input.timeout_ms))
|
|
1105
911
|
: 30_000;
|
|
1106
912
|
try {
|
|
1107
|
-
const
|
|
913
|
+
const event = (await (
|
|
1108
914
|
workflowStep.waitForEvent as unknown as (
|
|
1109
915
|
name: string,
|
|
1110
916
|
options: { type: string; timeout: number },
|
|
@@ -1112,14 +918,7 @@ async function waitForSyntheticIntegrationEvent(
|
|
|
1112
918
|
)(`integration_event:${eventKey}`, {
|
|
1113
919
|
type: integrationEventType(eventKey),
|
|
1114
920
|
timeout: timeoutMs,
|
|
1115
|
-
});
|
|
1116
|
-
await updateSyntheticIntegrationWaitStatus({
|
|
1117
|
-
req,
|
|
1118
|
-
eventKey,
|
|
1119
|
-
timeoutMs,
|
|
1120
|
-
waiting: true,
|
|
1121
|
-
});
|
|
1122
|
-
const event = (await eventPromise) as { payload: unknown };
|
|
921
|
+
})) as { payload: unknown };
|
|
1123
922
|
const payload =
|
|
1124
923
|
event.payload &&
|
|
1125
924
|
typeof event.payload === 'object' &&
|
|
@@ -1152,23 +951,14 @@ async function waitForSyntheticIntegrationEvent(
|
|
|
1152
951
|
resumed: false,
|
|
1153
952
|
timed_out: true,
|
|
1154
953
|
};
|
|
1155
|
-
} finally {
|
|
1156
|
-
await updateSyntheticIntegrationWaitStatus({
|
|
1157
|
-
req,
|
|
1158
|
-
eventKey,
|
|
1159
|
-
timeoutMs,
|
|
1160
|
-
waiting: false,
|
|
1161
|
-
});
|
|
1162
954
|
}
|
|
1163
955
|
}
|
|
1164
956
|
|
|
1165
957
|
async function callToolDirect(
|
|
1166
958
|
req: RunRequest,
|
|
1167
|
-
toolId: string,
|
|
1168
|
-
input: Record<string, unknown>,
|
|
1169
|
-
workflowStep?: WorkflowStep,
|
|
1170
|
-
retryKey?: string | null,
|
|
959
|
+
args: { id: string; toolId: string; input: Record<string, unknown> },
|
|
1171
960
|
): Promise<ToolExecuteResult> {
|
|
961
|
+
const { id, toolId, input } = args;
|
|
1172
962
|
if (toolId === 'test_rate_limit') {
|
|
1173
963
|
return wrapWorkerToolResult(
|
|
1174
964
|
toolId,
|
|
@@ -1184,60 +974,56 @@ async function callToolDirect(
|
|
|
1184
974
|
);
|
|
1185
975
|
}
|
|
1186
976
|
const path = `/api/v2/integrations/${encodeURIComponent(toolId)}/execute`;
|
|
1187
|
-
|
|
1188
|
-
|
|
1189
|
-
|
|
977
|
+
const maxAttempts = 3;
|
|
978
|
+
let lastError: Error | null = null;
|
|
979
|
+
|
|
980
|
+
for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
|
|
981
|
+
const res = await fetchRuntimeApi(req.baseUrl, path, {
|
|
1190
982
|
method: 'POST',
|
|
1191
983
|
headers: {
|
|
1192
984
|
'content-type': 'application/json',
|
|
1193
985
|
authorization: `Bearer ${req.executorToken}`,
|
|
1194
|
-
'x-deepline-request-id':
|
|
986
|
+
'x-deepline-request-id': `${req.runId}:${toolId}:${id}`,
|
|
1195
987
|
[EXECUTE_TOOL_METADATA_HEADER]: 'true',
|
|
1196
988
|
},
|
|
1197
989
|
body: JSON.stringify({ payload: input }),
|
|
1198
990
|
});
|
|
1199
|
-
|
|
1200
|
-
|
|
1201
|
-
|
|
991
|
+
if (res.ok) {
|
|
992
|
+
const body = (await res.json()) as Record<string, unknown>;
|
|
993
|
+
const result = (body.result ?? body) as unknown;
|
|
994
|
+
const status =
|
|
995
|
+
typeof body.status === 'string'
|
|
996
|
+
? body.status
|
|
997
|
+
: result == null
|
|
998
|
+
? 'no_result'
|
|
999
|
+
: 'completed';
|
|
1000
|
+
return wrapWorkerToolResult(
|
|
1001
|
+
toolId,
|
|
1002
|
+
result,
|
|
1003
|
+
parseExecuteToolMetadata(toolId, body),
|
|
1004
|
+
status,
|
|
1005
|
+
);
|
|
1202
1006
|
}
|
|
1203
|
-
|
|
1204
|
-
console.log(
|
|
1205
|
-
`[play-harness] tool ${toolId} returned ${res.status}; retrying after ${retryAfterMs}ms attempt=${attempt}/${TOOL_HTTP_MAX_ATTEMPTS}`,
|
|
1206
|
-
);
|
|
1207
|
-
await res.body?.cancel().catch(() => undefined);
|
|
1208
|
-
await sleepForToolRetry({
|
|
1209
|
-
workflowStep,
|
|
1210
|
-
toolId,
|
|
1211
|
-
status: res.status,
|
|
1212
|
-
attempt,
|
|
1213
|
-
retryAfterMs,
|
|
1214
|
-
retryKey,
|
|
1215
|
-
toolInput: input,
|
|
1216
|
-
});
|
|
1217
|
-
}
|
|
1218
|
-
if (!res.ok) {
|
|
1007
|
+
|
|
1219
1008
|
const text = await res.text().catch(() => '');
|
|
1220
|
-
|
|
1009
|
+
lastError = new Error(
|
|
1010
|
+
`tool ${toolId} ${res.status} attempt ${attempt}/${maxAttempts}: ${text.slice(0, 500)}`,
|
|
1011
|
+
);
|
|
1012
|
+
const retryable =
|
|
1013
|
+
res.status === 429 ||
|
|
1014
|
+
(res.status >= 500 && WORKER_RETRY_SAFE_5XX_TOOLS.has(toolId));
|
|
1015
|
+
if (!retryable || attempt >= maxAttempts) {
|
|
1016
|
+
throw lastError;
|
|
1017
|
+
}
|
|
1018
|
+
const retryAfterSeconds = Number(res.headers.get('retry-after'));
|
|
1019
|
+
const delayMs =
|
|
1020
|
+
Number.isFinite(retryAfterSeconds) && retryAfterSeconds > 0
|
|
1021
|
+
? Math.min(5_000, Math.ceil(retryAfterSeconds * 1000))
|
|
1022
|
+
: 1_000;
|
|
1023
|
+
await new Promise((resolve) => setTimeout(resolve, delayMs));
|
|
1221
1024
|
}
|
|
1222
|
-
|
|
1223
|
-
|
|
1224
|
-
// normalized = normalizePlayToolResult(data.result ?? data)
|
|
1225
|
-
// where normalizePlayToolResult recursively unwraps `{data: X}` → `X`.
|
|
1226
|
-
const body = (await res.json()) as Record<string, unknown>;
|
|
1227
|
-
const picked = (body.result ?? body) as unknown;
|
|
1228
|
-
const result = normalizePlayToolResult(picked);
|
|
1229
|
-
const status =
|
|
1230
|
-
typeof body.status === 'string'
|
|
1231
|
-
? body.status
|
|
1232
|
-
: result == null
|
|
1233
|
-
? 'no_result'
|
|
1234
|
-
: 'completed';
|
|
1235
|
-
return wrapWorkerToolResult(
|
|
1236
|
-
toolId,
|
|
1237
|
-
result,
|
|
1238
|
-
parseExecuteToolMetadata(toolId, body),
|
|
1239
|
-
status,
|
|
1240
|
-
);
|
|
1025
|
+
|
|
1026
|
+
throw lastError ?? new Error(`tool ${toolId} failed before execution.`);
|
|
1241
1027
|
}
|
|
1242
1028
|
|
|
1243
1029
|
function parseExecuteToolMetadata(
|
|
@@ -1482,22 +1268,23 @@ type WorkerInlineWaterfallSpec = {
|
|
|
1482
1268
|
input: Record<string, unknown>,
|
|
1483
1269
|
ctx: {
|
|
1484
1270
|
tools: {
|
|
1485
|
-
execute(
|
|
1271
|
+
execute(
|
|
1272
|
+
key: string,
|
|
1273
|
+
toolId: string,
|
|
1274
|
+
input: Record<string, unknown>,
|
|
1275
|
+
): Promise<unknown>;
|
|
1486
1276
|
};
|
|
1487
|
-
tool(
|
|
1277
|
+
tool(
|
|
1278
|
+
key: string,
|
|
1279
|
+
toolId: string,
|
|
1280
|
+
input: Record<string, unknown>,
|
|
1281
|
+
): Promise<unknown>;
|
|
1488
1282
|
},
|
|
1489
1283
|
) => unknown | Promise<unknown>;
|
|
1490
1284
|
}
|
|
1491
1285
|
>;
|
|
1492
1286
|
};
|
|
1493
1287
|
|
|
1494
|
-
type WorkerToolExecutionRequest = {
|
|
1495
|
-
id: string;
|
|
1496
|
-
tool: string;
|
|
1497
|
-
input: Record<string, unknown>;
|
|
1498
|
-
description?: string;
|
|
1499
|
-
};
|
|
1500
|
-
|
|
1501
1288
|
type WorkerWaterfallOptions = {
|
|
1502
1289
|
providers?: string[];
|
|
1503
1290
|
min_results?: number;
|
|
@@ -1524,8 +1311,8 @@ type WorkerStepResolution = {
|
|
|
1524
1311
|
};
|
|
1525
1312
|
|
|
1526
1313
|
type WorkerToolBatchRequest = {
|
|
1314
|
+
id: string;
|
|
1527
1315
|
toolId: string;
|
|
1528
|
-
retryKey?: string | null;
|
|
1529
1316
|
input: Record<string, unknown>;
|
|
1530
1317
|
workflowStep?: WorkflowStep;
|
|
1531
1318
|
resolve: (value: unknown) => void;
|
|
@@ -1533,6 +1320,7 @@ type WorkerToolBatchRequest = {
|
|
|
1533
1320
|
};
|
|
1534
1321
|
|
|
1535
1322
|
const WORKER_TOOL_BATCH_GRACE_MS = 15;
|
|
1323
|
+
const WORKER_RETRY_SAFE_5XX_TOOLS = new Set(['test_transient_500']);
|
|
1536
1324
|
|
|
1537
1325
|
function stepProgramColumnName(parentField: string, stepId: string): string {
|
|
1538
1326
|
return sqlSafePlayColumnName(`${parentField}.${stepId}`);
|
|
@@ -1545,20 +1333,13 @@ class WorkerToolBatchScheduler {
|
|
|
1545
1333
|
constructor(private readonly req: RunRequest) {}
|
|
1546
1334
|
|
|
1547
1335
|
execute(
|
|
1548
|
-
|
|
1336
|
+
id: string,
|
|
1549
1337
|
toolId: string,
|
|
1550
1338
|
input: Record<string, unknown>,
|
|
1551
1339
|
workflowStep?: WorkflowStep,
|
|
1552
1340
|
): Promise<unknown> {
|
|
1553
1341
|
return new Promise((resolve, reject) => {
|
|
1554
|
-
this.queue.push({
|
|
1555
|
-
toolId,
|
|
1556
|
-
retryKey,
|
|
1557
|
-
input,
|
|
1558
|
-
workflowStep,
|
|
1559
|
-
resolve,
|
|
1560
|
-
reject,
|
|
1561
|
-
});
|
|
1342
|
+
this.queue.push({ id, toolId, input, workflowStep, resolve, reject });
|
|
1562
1343
|
this.scheduleDrain();
|
|
1563
1344
|
});
|
|
1564
1345
|
}
|
|
@@ -1612,7 +1393,7 @@ class WorkerToolBatchScheduler {
|
|
|
1612
1393
|
request.resolve(
|
|
1613
1394
|
await executeTool(
|
|
1614
1395
|
this.req,
|
|
1615
|
-
{
|
|
1396
|
+
{ id: request.id, toolId, input: request.input },
|
|
1616
1397
|
request.workflowStep,
|
|
1617
1398
|
),
|
|
1618
1399
|
);
|
|
@@ -1662,17 +1443,11 @@ async function executeBatchedWorkerToolGroup(input: {
|
|
|
1662
1443
|
requests: compiledBatches,
|
|
1663
1444
|
batchSize: Math.max(1, Math.min(4, compiledBatches.length || 1)),
|
|
1664
1445
|
execute: async (batch) =>
|
|
1665
|
-
await executeTool(
|
|
1666
|
-
|
|
1667
|
-
|
|
1668
|
-
|
|
1669
|
-
|
|
1670
|
-
retryKey: batch.memberRequests
|
|
1671
|
-
.map((request) => request.retryKey ?? '')
|
|
1672
|
-
.join('|'),
|
|
1673
|
-
},
|
|
1674
|
-
batch.memberRequests[0]?.workflowStep,
|
|
1675
|
-
),
|
|
1446
|
+
await executeTool(input.req, {
|
|
1447
|
+
id: `batch:${batch.memberRequests.map((request) => request.id).join('|')}`,
|
|
1448
|
+
toolId: batch.batchOperation,
|
|
1449
|
+
input: batch.batchPayload,
|
|
1450
|
+
}),
|
|
1676
1451
|
onChunkComplete: async (
|
|
1677
1452
|
chunkResults: Array<
|
|
1678
1453
|
ChunkExecutionResult<(typeof compiledBatches)[number], unknown>
|
|
@@ -1680,7 +1455,7 @@ async function executeBatchedWorkerToolGroup(input: {
|
|
|
1680
1455
|
) => {
|
|
1681
1456
|
for (const entry of chunkResults) {
|
|
1682
1457
|
const batchResult = isToolExecuteResult(entry.result)
|
|
1683
|
-
? entry.result.result
|
|
1458
|
+
? entry.result.result.data
|
|
1684
1459
|
: entry.result;
|
|
1685
1460
|
const splitResults =
|
|
1686
1461
|
batchResult != null
|
|
@@ -1785,7 +1560,6 @@ type WorkerStepProgram = {
|
|
|
1785
1560
|
type WorkerMapOptions = {
|
|
1786
1561
|
description?: string;
|
|
1787
1562
|
concurrency?: number;
|
|
1788
|
-
staleAfterSeconds?: number;
|
|
1789
1563
|
key?:
|
|
1790
1564
|
| string
|
|
1791
1565
|
| readonly string[]
|
|
@@ -1795,46 +1569,6 @@ type WorkerMapOptions = {
|
|
|
1795
1569
|
) => string | number | readonly unknown[]);
|
|
1796
1570
|
};
|
|
1797
1571
|
|
|
1798
|
-
function workerMapRowIdentity(
|
|
1799
|
-
row: Record<string, unknown>,
|
|
1800
|
-
tableNamespace: string,
|
|
1801
|
-
opts: WorkerMapOptions | undefined,
|
|
1802
|
-
index = 0,
|
|
1803
|
-
): string {
|
|
1804
|
-
const key = opts?.key;
|
|
1805
|
-
if (!key) return derivePlayRowIdentity(row, tableNamespace);
|
|
1806
|
-
const raw =
|
|
1807
|
-
typeof key === 'function'
|
|
1808
|
-
? key(row, index)
|
|
1809
|
-
: typeof key === 'string'
|
|
1810
|
-
? row[key]
|
|
1811
|
-
: key.map((fieldName) => row[fieldName]);
|
|
1812
|
-
const normalized = normalizeWorkerExplicitMapKey(raw);
|
|
1813
|
-
if (!normalized) {
|
|
1814
|
-
throw new Error(
|
|
1815
|
-
`ctx.map("${tableNamespace}") key produced an empty value for row ${index}. ` +
|
|
1816
|
-
'Use non-empty stable input columns or return a non-empty string, number, or tuple.',
|
|
1817
|
-
);
|
|
1818
|
-
}
|
|
1819
|
-
return derivePlayRowIdentityFromKey(normalized, tableNamespace);
|
|
1820
|
-
}
|
|
1821
|
-
|
|
1822
|
-
function normalizeWorkerExplicitMapKey(value: unknown): string {
|
|
1823
|
-
if (Array.isArray(value)) {
|
|
1824
|
-
const parts = value.map((entry) =>
|
|
1825
|
-
normalizeWorkerExplicitMapKeyPart(entry),
|
|
1826
|
-
);
|
|
1827
|
-
return parts.every(Boolean) ? JSON.stringify(parts) : '';
|
|
1828
|
-
}
|
|
1829
|
-
return normalizeWorkerExplicitMapKeyPart(value);
|
|
1830
|
-
}
|
|
1831
|
-
|
|
1832
|
-
function normalizeWorkerExplicitMapKeyPart(value: unknown): string {
|
|
1833
|
-
if (typeof value === 'number')
|
|
1834
|
-
return Number.isFinite(value) ? String(value) : '';
|
|
1835
|
-
return String(value ?? '').trim();
|
|
1836
|
-
}
|
|
1837
|
-
|
|
1838
1572
|
function isWorkerStepProgram(value: unknown): value is WorkerStepProgram {
|
|
1839
1573
|
return (
|
|
1840
1574
|
!!value &&
|
|
@@ -1844,16 +1578,6 @@ function isWorkerStepProgram(value: unknown): value is WorkerStepProgram {
|
|
|
1844
1578
|
);
|
|
1845
1579
|
}
|
|
1846
1580
|
|
|
1847
|
-
function isWorkerMapDefinitionOptions(
|
|
1848
|
-
value: unknown,
|
|
1849
|
-
): value is Omit<WorkerMapOptions, 'description' | 'concurrency'> {
|
|
1850
|
-
if (!value || typeof value !== 'object' || Array.isArray(value)) {
|
|
1851
|
-
return false;
|
|
1852
|
-
}
|
|
1853
|
-
const keys = Object.keys(value);
|
|
1854
|
-
return keys.every((key) => key === 'key' || key === 'staleAfterSeconds');
|
|
1855
|
-
}
|
|
1856
|
-
|
|
1857
1581
|
function isWorkerConditionalStepResolver(
|
|
1858
1582
|
value: unknown,
|
|
1859
1583
|
): value is WorkerConditionalStepResolver {
|
|
@@ -1918,7 +1642,7 @@ async function executeWorkerStepProgram(
|
|
|
1918
1642
|
outputs: RecordedStepProgramOutput[];
|
|
1919
1643
|
},
|
|
1920
1644
|
): Promise<unknown> {
|
|
1921
|
-
|
|
1645
|
+
let currentRow: Record<string, unknown> = cloneCsvAliasedRow(inputRow);
|
|
1922
1646
|
for (const step of program.steps) {
|
|
1923
1647
|
const stepPath = [...(recorder?.path ?? []), step.name];
|
|
1924
1648
|
const resolution = await executeWorkerStepResolver(
|
|
@@ -1934,7 +1658,7 @@ async function executeWorkerStepProgram(
|
|
|
1934
1658
|
: undefined,
|
|
1935
1659
|
);
|
|
1936
1660
|
const value = resolution.value;
|
|
1937
|
-
currentRow[step.name]
|
|
1661
|
+
currentRow = cloneCsvAliasedRow(currentRow, { [step.name]: value });
|
|
1938
1662
|
if (recorder) {
|
|
1939
1663
|
const stepId = stepPath.join('.');
|
|
1940
1664
|
recorder.outputs.push({
|
|
@@ -1968,7 +1692,6 @@ async function executeWorkerWaterfall(
|
|
|
1968
1692
|
toolNameOrSpec: string | WorkerInlineWaterfallSpec,
|
|
1969
1693
|
input: Record<string, unknown>,
|
|
1970
1694
|
opts?: WorkerWaterfallOptions,
|
|
1971
|
-
workflowStep?: WorkflowStep,
|
|
1972
1695
|
): Promise<unknown | null> {
|
|
1973
1696
|
// Inline-spec form
|
|
1974
1697
|
if (typeof toolNameOrSpec === 'object' && toolNameOrSpec) {
|
|
@@ -1980,38 +1703,25 @@ async function executeWorkerWaterfall(
|
|
|
1980
1703
|
if (isWorkerInlineCodeStep(step)) {
|
|
1981
1704
|
result = await step.run(input, {
|
|
1982
1705
|
tools: {
|
|
1983
|
-
execute: async (
|
|
1706
|
+
execute: async (
|
|
1707
|
+
requestOrKey: unknown,
|
|
1708
|
+
toolId?: unknown,
|
|
1709
|
+
toolInput?: unknown,
|
|
1710
|
+
) =>
|
|
1984
1711
|
await executeTool(
|
|
1985
1712
|
req,
|
|
1986
|
-
|
|
1987
|
-
toolId: request.tool,
|
|
1988
|
-
input: request.input,
|
|
1989
|
-
retryKey: request.id,
|
|
1990
|
-
},
|
|
1991
|
-
workflowStep,
|
|
1713
|
+
normalizeToolExecuteArgs(requestOrKey, toolId, toolInput),
|
|
1992
1714
|
),
|
|
1993
1715
|
},
|
|
1994
|
-
tool: async (
|
|
1995
|
-
await executeTool(
|
|
1996
|
-
req,
|
|
1997
|
-
{
|
|
1998
|
-
toolId: request.tool,
|
|
1999
|
-
input: request.input,
|
|
2000
|
-
retryKey: request.id,
|
|
2001
|
-
},
|
|
2002
|
-
workflowStep,
|
|
2003
|
-
),
|
|
1716
|
+
tool: async (key, toolId, toolInput) =>
|
|
1717
|
+
await executeTool(req, { id: key, toolId, input: toolInput }),
|
|
2004
1718
|
});
|
|
2005
1719
|
} else {
|
|
2006
|
-
result = await executeTool(
|
|
2007
|
-
|
|
2008
|
-
|
|
2009
|
-
|
|
2010
|
-
|
|
2011
|
-
retryKey: step.id,
|
|
2012
|
-
},
|
|
2013
|
-
workflowStep,
|
|
2014
|
-
);
|
|
1720
|
+
result = await executeTool(req, {
|
|
1721
|
+
id: step.id,
|
|
1722
|
+
toolId: step.toolId,
|
|
1723
|
+
input: step.mapInput(input),
|
|
1724
|
+
});
|
|
2015
1725
|
}
|
|
2016
1726
|
} catch {
|
|
2017
1727
|
continue;
|
|
@@ -2097,11 +1807,7 @@ async function executeWorkerWaterfall(
|
|
|
2097
1807
|
const providers = opts?.providers ?? [];
|
|
2098
1808
|
if (providers.length === 0) {
|
|
2099
1809
|
try {
|
|
2100
|
-
return await executeTool(
|
|
2101
|
-
req,
|
|
2102
|
-
{ toolId: toolName, input, retryKey: toolName },
|
|
2103
|
-
workflowStep,
|
|
2104
|
-
);
|
|
1810
|
+
return await executeTool(req, { id: toolName, toolId: toolName, input });
|
|
2105
1811
|
} catch {
|
|
2106
1812
|
return null;
|
|
2107
1813
|
}
|
|
@@ -2109,15 +1815,11 @@ async function executeWorkerWaterfall(
|
|
|
2109
1815
|
let lastError: Error | null = null;
|
|
2110
1816
|
for (const provider of providers) {
|
|
2111
1817
|
try {
|
|
2112
|
-
const result = await executeTool(
|
|
2113
|
-
|
|
2114
|
-
|
|
2115
|
-
|
|
2116
|
-
|
|
2117
|
-
retryKey: `${toolName}:${provider}`,
|
|
2118
|
-
},
|
|
2119
|
-
workflowStep,
|
|
2120
|
-
);
|
|
1818
|
+
const result = await executeTool(req, {
|
|
1819
|
+
id: `${toolName}:${provider}`,
|
|
1820
|
+
toolId: toolName,
|
|
1821
|
+
input: { ...input, provider },
|
|
1822
|
+
});
|
|
2121
1823
|
if (resultHasContent(result)) {
|
|
2122
1824
|
recorder.push({
|
|
2123
1825
|
waterfallId: toolName,
|
|
@@ -2134,18 +1836,6 @@ async function executeWorkerWaterfall(
|
|
|
2134
1836
|
return null;
|
|
2135
1837
|
}
|
|
2136
1838
|
|
|
2137
|
-
function normalizePlayToolResult(value: unknown): unknown {
|
|
2138
|
-
if (!isRecordLike(value)) return value;
|
|
2139
|
-
if ('data' in value) return normalizePlayToolResult(value.data);
|
|
2140
|
-
if ('result' in value) {
|
|
2141
|
-
const normalizedResult = normalizePlayToolResult(value.result);
|
|
2142
|
-
if (normalizedResult !== value.result) {
|
|
2143
|
-
return { ...value, result: normalizedResult };
|
|
2144
|
-
}
|
|
2145
|
-
}
|
|
2146
|
-
return value;
|
|
2147
|
-
}
|
|
2148
|
-
|
|
2149
1839
|
async function hashJson(value: unknown): Promise<string> {
|
|
2150
1840
|
const bytes = new TextEncoder().encode(canonicalizeJson(value));
|
|
2151
1841
|
const digest = await crypto.subtle.digest('SHA-256', bytes);
|
|
@@ -2266,18 +1956,12 @@ async function* streamCsvRowsFromBody<T extends Record<string, unknown>>(
|
|
|
2266
1956
|
const flushPhysicalRowsAsObjects = (terminal: boolean): T[][] => {
|
|
2267
1957
|
const yielded: T[][] = [];
|
|
2268
1958
|
if (physicalRowBuffer.length === 0) return yielded;
|
|
2269
|
-
let startIndex = 0;
|
|
2270
1959
|
if (!headers) {
|
|
2271
|
-
headers = physicalRowBuffer
|
|
1960
|
+
headers = physicalRowBuffer.shift() ?? null;
|
|
2272
1961
|
if (!headers) return yielded;
|
|
2273
|
-
startIndex = 1;
|
|
2274
1962
|
}
|
|
2275
|
-
|
|
2276
|
-
|
|
2277
|
-
rowIndex < physicalRowBuffer.length;
|
|
2278
|
-
rowIndex += 1
|
|
2279
|
-
) {
|
|
2280
|
-
const cells = physicalRowBuffer[rowIndex]!;
|
|
1963
|
+
while (physicalRowBuffer.length > 0) {
|
|
1964
|
+
const cells = physicalRowBuffer.shift()!;
|
|
2281
1965
|
const obj: Record<string, unknown> = {};
|
|
2282
1966
|
for (let c = 0; c < headers.length; c += 1) {
|
|
2283
1967
|
obj[headers[c]!] = cells[c] ?? '';
|
|
@@ -2288,7 +1972,6 @@ async function* streamCsvRowsFromBody<T extends Record<string, unknown>>(
|
|
|
2288
1972
|
pendingChunk = [];
|
|
2289
1973
|
}
|
|
2290
1974
|
}
|
|
2291
|
-
physicalRowBuffer.length = 0;
|
|
2292
1975
|
if (terminal && pendingChunk.length > 0) {
|
|
2293
1976
|
yielded.push(pendingChunk);
|
|
2294
1977
|
pendingChunk = [];
|
|
@@ -2333,12 +2016,7 @@ async function openR2BodyStream(input: {
|
|
|
2333
2016
|
return object.body;
|
|
2334
2017
|
}
|
|
2335
2018
|
}
|
|
2336
|
-
|
|
2337
|
-
(file) =>
|
|
2338
|
-
file.playPath === input.logicalPath ||
|
|
2339
|
-
file.playPath === input.logicalPath.replace(/^\.\//, ''),
|
|
2340
|
-
);
|
|
2341
|
-
if (input.env.PLAY_ASSETS && packagedAsset) {
|
|
2019
|
+
if (input.env.PLAY_ASSETS) {
|
|
2342
2020
|
try {
|
|
2343
2021
|
const text = await input.env.PLAY_ASSETS.readText(input.logicalPath);
|
|
2344
2022
|
const bytes = new TextEncoder().encode(text);
|
|
@@ -2355,80 +2033,28 @@ async function openR2BodyStream(input: {
|
|
|
2355
2033
|
}
|
|
2356
2034
|
}
|
|
2357
2035
|
|
|
2358
|
-
|
|
2359
|
-
|
|
2360
|
-
|
|
2361
|
-
|
|
2362
|
-
|
|
2363
|
-
logicalPath: string;
|
|
2364
|
-
storageKey: string;
|
|
2365
|
-
}): Promise<ReadableStream<Uint8Array>> {
|
|
2366
|
-
const headResponse = await harnessFetchStagedFile({
|
|
2036
|
+
// The harness fetch path returns a real Response body backed by R2.
|
|
2037
|
+
// Errors are loud: we want CI / regression failures to surface the real
|
|
2038
|
+
// cause (auth, missing object, network) rather than getting squashed into a
|
|
2039
|
+
// generic "R2 asset is not reachable".
|
|
2040
|
+
const response = await harnessFetchStagedFile({
|
|
2367
2041
|
executorToken: input.req.executorToken,
|
|
2368
2042
|
storageKey: input.storageKey,
|
|
2369
|
-
method: 'HEAD',
|
|
2370
2043
|
});
|
|
2371
|
-
if (
|
|
2044
|
+
if (response.status === 404) {
|
|
2372
2045
|
throw new Error(
|
|
2373
2046
|
`ctx.csv("${input.logicalPath}"): harness R2 fetch returned 404 for storageKey=${input.storageKey}. ` +
|
|
2374
2047
|
`The staged file is missing from R2; the upload either failed silently before the run started, ` +
|
|
2375
2048
|
`or the storageKey threaded through the workflow params no longer matches what the harness resolves.`,
|
|
2376
2049
|
);
|
|
2377
2050
|
}
|
|
2378
|
-
if (!
|
|
2379
|
-
const body = await
|
|
2051
|
+
if (!response.ok || !response.body) {
|
|
2052
|
+
const body = await response.text().catch(() => '');
|
|
2380
2053
|
throw new Error(
|
|
2381
|
-
`ctx.csv("${input.logicalPath}"): harness R2
|
|
2054
|
+
`ctx.csv("${input.logicalPath}"): harness R2 fetch failed ${response.status}: ${body.slice(0, 400)}`,
|
|
2382
2055
|
);
|
|
2383
2056
|
}
|
|
2384
|
-
|
|
2385
|
-
headResponse.headers.get('x-deepline-object-size') ??
|
|
2386
|
-
headResponse.headers.get('content-length') ??
|
|
2387
|
-
'';
|
|
2388
|
-
const objectSize = Number(rawSize);
|
|
2389
|
-
if (!Number.isSafeInteger(objectSize) || objectSize < 0) {
|
|
2390
|
-
throw new Error(
|
|
2391
|
-
`ctx.csv("${input.logicalPath}"): harness R2 metadata missing a valid object size for storageKey=${input.storageKey}.`,
|
|
2392
|
-
);
|
|
2393
|
-
}
|
|
2394
|
-
|
|
2395
|
-
let offset = 0;
|
|
2396
|
-
return new ReadableStream<Uint8Array>({
|
|
2397
|
-
async pull(controller) {
|
|
2398
|
-
if (offset >= objectSize) {
|
|
2399
|
-
controller.close();
|
|
2400
|
-
return;
|
|
2401
|
-
}
|
|
2402
|
-
const length = Math.min(
|
|
2403
|
-
TARGET_CSV_DECODE_CHUNK_BYTES,
|
|
2404
|
-
objectSize - offset,
|
|
2405
|
-
);
|
|
2406
|
-
const response = await harnessFetchStagedFile({
|
|
2407
|
-
executorToken: input.req.executorToken,
|
|
2408
|
-
storageKey: input.storageKey,
|
|
2409
|
-
range: { offset, length },
|
|
2410
|
-
});
|
|
2411
|
-
if (response.status === 404) {
|
|
2412
|
-
throw new Error(
|
|
2413
|
-
`ctx.csv("${input.logicalPath}"): harness R2 range fetch returned 404 for storageKey=${input.storageKey}.`,
|
|
2414
|
-
);
|
|
2415
|
-
}
|
|
2416
|
-
if (!response.ok || response.status !== 206) {
|
|
2417
|
-
const body = await response.text().catch(() => '');
|
|
2418
|
-
throw new Error(
|
|
2419
|
-
`ctx.csv("${input.logicalPath}"): harness R2 range fetch failed ${response.status}: ${body.slice(0, 400)}`,
|
|
2420
|
-
);
|
|
2421
|
-
}
|
|
2422
|
-
const bytes = new Uint8Array(await response.arrayBuffer());
|
|
2423
|
-
if (bytes.length === 0 && length > 0) {
|
|
2424
|
-
throw new Error(
|
|
2425
|
-
`ctx.csv("${input.logicalPath}"): harness R2 range fetch returned an empty body before EOF at offset=${offset}.`,
|
|
2426
|
-
);
|
|
2427
|
-
}
|
|
2428
|
-
offset += bytes.length;
|
|
2429
|
-
controller.enqueue(bytes);
|
|
2430
|
-
},
|
|
2431
|
-
});
|
|
2057
|
+
return response.body;
|
|
2432
2058
|
}
|
|
2433
2059
|
|
|
2434
2060
|
/**
|
|
@@ -2454,11 +2080,11 @@ type StreamingCsvDataset<T extends Record<string, unknown>> = T[] & {
|
|
|
2454
2080
|
};
|
|
2455
2081
|
|
|
2456
2082
|
const MAX_MATERIALIZE_ROWS_DEFAULT = 50_000;
|
|
2457
|
-
const STREAMING_MAP_DEFAULT_CHUNK_SIZE = 5_000;
|
|
2458
2083
|
|
|
2459
2084
|
function makeStreamingCsvDataset<T extends Record<string, unknown>>(input: {
|
|
2460
2085
|
name: string;
|
|
2461
2086
|
logicalPath: string;
|
|
2087
|
+
renameOptions?: CsvRenameOptions;
|
|
2462
2088
|
open: () => Promise<ReadableStream<Uint8Array> | null>;
|
|
2463
2089
|
}): StreamingCsvDataset<T> {
|
|
2464
2090
|
const datasetId = `csv:${input.name}`;
|
|
@@ -2472,7 +2098,12 @@ function makeStreamingCsvDataset<T extends Record<string, unknown>>(input: {
|
|
|
2472
2098
|
`ctx.csv("${input.logicalPath}"): R2 asset is not reachable (no PLAYS_BUCKET binding and signed URL unavailable).`,
|
|
2473
2099
|
);
|
|
2474
2100
|
}
|
|
2475
|
-
|
|
2101
|
+
for await (const chunk of streamCsvRowsFromBody<T>(
|
|
2102
|
+
body,
|
|
2103
|
+
Math.max(1, Math.floor(chunkSize)),
|
|
2104
|
+
)) {
|
|
2105
|
+
yield applyCsvRenameProjection(chunk, input.renameOptions) as T[];
|
|
2106
|
+
}
|
|
2476
2107
|
}
|
|
2477
2108
|
|
|
2478
2109
|
Object.defineProperty(arr, 'iterChunks', {
|
|
@@ -2629,9 +2260,12 @@ async function persistCompletedMapRows(input: {
|
|
|
2629
2260
|
extraOutputFields?: string[];
|
|
2630
2261
|
}): Promise<void> {
|
|
2631
2262
|
if (input.rows.length === 0) return;
|
|
2632
|
-
await
|
|
2633
|
-
|
|
2634
|
-
|
|
2263
|
+
await postRuntimeApi<{
|
|
2264
|
+
ok: true;
|
|
2265
|
+
rowsWritten: number;
|
|
2266
|
+
tableNamespace: string;
|
|
2267
|
+
}>(input.req.baseUrl, input.req.executorToken, {
|
|
2268
|
+
action: 'persist_completed_sheet_rows',
|
|
2635
2269
|
playName: input.req.playName,
|
|
2636
2270
|
tableNamespace: input.tableNamespace,
|
|
2637
2271
|
sheetContract: requireSheetContract(input.req, input.tableNamespace),
|
|
@@ -2643,7 +2277,6 @@ async function persistCompletedMapRows(input: {
|
|
|
2643
2277
|
),
|
|
2644
2278
|
],
|
|
2645
2279
|
runId: input.req.runId,
|
|
2646
|
-
userEmail: input.req.userEmail,
|
|
2647
2280
|
});
|
|
2648
2281
|
}
|
|
2649
2282
|
|
|
@@ -2660,15 +2293,19 @@ async function prepareMapRows(input: {
|
|
|
2660
2293
|
if (input.rows.length === 0) {
|
|
2661
2294
|
return { inserted: 0, skipped: 0, pendingRows: [], completedRows: [] };
|
|
2662
2295
|
}
|
|
2663
|
-
const result = await
|
|
2664
|
-
|
|
2665
|
-
|
|
2296
|
+
const result = await postRuntimeApi<{
|
|
2297
|
+
inserted: number;
|
|
2298
|
+
skipped: number;
|
|
2299
|
+
pendingRows: Record<string, unknown>[];
|
|
2300
|
+
completedRows: Record<string, unknown>[];
|
|
2301
|
+
tableNamespace: string;
|
|
2302
|
+
}>(input.req.baseUrl, input.req.executorToken, {
|
|
2303
|
+
action: 'start_sheet_dataset',
|
|
2666
2304
|
playName: input.req.playName,
|
|
2667
2305
|
tableNamespace: input.tableNamespace,
|
|
2668
2306
|
sheetContract: requireSheetContract(input.req, input.tableNamespace),
|
|
2669
2307
|
rows: input.rows.map((row) => ({ ...row })),
|
|
2670
2308
|
runId: input.req.runId,
|
|
2671
|
-
userEmail: input.req.userEmail,
|
|
2672
2309
|
});
|
|
2673
2310
|
return {
|
|
2674
2311
|
inserted: result.inserted,
|
|
@@ -2678,26 +2315,6 @@ async function prepareMapRows(input: {
|
|
|
2678
2315
|
};
|
|
2679
2316
|
}
|
|
2680
2317
|
|
|
2681
|
-
type WorkerMapExecutor = {
|
|
2682
|
-
prepareRows(input: {
|
|
2683
|
-
tableNamespace: string;
|
|
2684
|
-
rows: Record<string, unknown>[];
|
|
2685
|
-
}): ReturnType<typeof prepareMapRows>;
|
|
2686
|
-
persistCompletedRows(input: {
|
|
2687
|
-
tableNamespace: string;
|
|
2688
|
-
rows: Record<string, unknown>[];
|
|
2689
|
-
outputFields: string[];
|
|
2690
|
-
extraOutputFields?: string[];
|
|
2691
|
-
}): ReturnType<typeof persistCompletedMapRows>;
|
|
2692
|
-
};
|
|
2693
|
-
|
|
2694
|
-
function createWorkerMapExecutor(req: RunRequest): WorkerMapExecutor {
|
|
2695
|
-
return {
|
|
2696
|
-
prepareRows: (input) => prepareMapRows({ req, ...input }),
|
|
2697
|
-
persistCompletedRows: (input) => persistCompletedMapRows({ req, ...input }),
|
|
2698
|
-
};
|
|
2699
|
-
}
|
|
2700
|
-
|
|
2701
2318
|
/**
|
|
2702
2319
|
* Builds the minimal HTTP-backed ctx surface needed to run tool-basic-shaped
|
|
2703
2320
|
* plays. NOT a full implementation of shared_libs/play-runtime/context.ts.
|
|
@@ -2705,13 +2322,8 @@ function createWorkerMapExecutor(req: RunRequest): WorkerMapExecutor {
|
|
|
2705
2322
|
* Supported:
|
|
2706
2323
|
* - ctx.log(msg)
|
|
2707
2324
|
* - ctx.csv(filename | inline rows) (calls runtime API for file resolve)
|
|
2708
|
-
* - ctx.map(name, rows, opts)
|
|
2709
|
-
* - ctx.tools.execute(
|
|
2710
|
-
id: namespace,
|
|
2711
|
-
tool: op,
|
|
2712
|
-
input: input,
|
|
2713
|
-
...(opts),
|
|
2714
|
-
})
|
|
2325
|
+
* - ctx.map(name, rows, fields, opts)
|
|
2326
|
+
* - ctx.tools.execute(namespace, op, input, opts)
|
|
2715
2327
|
* - ctx.runPlay(key, playRef, input, opts)
|
|
2716
2328
|
*
|
|
2717
2329
|
* Not supported (will throw):
|
|
@@ -2773,7 +2385,7 @@ function releaseChildPlayConcurrency(
|
|
|
2773
2385
|
inFlightByPlayName[playName] = next;
|
|
2774
2386
|
}
|
|
2775
2387
|
|
|
2776
|
-
function
|
|
2388
|
+
function createMinimalWorkerCtx(
|
|
2777
2389
|
req: RunRequest,
|
|
2778
2390
|
emitEvent: (event: RunnerEvent) => void,
|
|
2779
2391
|
env: WorkerEnv,
|
|
@@ -2783,9 +2395,6 @@ function createWorkerCtxFactory(
|
|
|
2783
2395
|
let playCallCount = 0;
|
|
2784
2396
|
const parentChildCalls: Record<string, number> = {};
|
|
2785
2397
|
const inFlightChildCallsByPlayName: Record<string, number> = {};
|
|
2786
|
-
const mapExecutor = createWorkerMapExecutor(req);
|
|
2787
|
-
const transport = createWorkerRuntimeTransport(req);
|
|
2788
|
-
const childPlayExecutor = createWorkerChildPlayExecutor({ req, transport });
|
|
2789
2398
|
const rootGovernance = req.playCallGovernance;
|
|
2790
2399
|
const rootRunId = rootGovernance?.rootRunId ?? req.runId;
|
|
2791
2400
|
// Local ancestry chain that always ENDS with the currently-executing play
|
|
@@ -2829,16 +2438,88 @@ function createWorkerCtxFactory(
|
|
|
2829
2438
|
candidate.mapName === name || candidate.tableNamespace === name,
|
|
2830
2439
|
);
|
|
2831
2440
|
const streaming = isStreamingDataset<T>(sliced);
|
|
2832
|
-
|
|
2833
|
-
|
|
2441
|
+
// For streaming inputs we don't know the row count upfront — pass
|
|
2442
|
+
// `totalRows: 0` so chooseMapChunkSize falls back to the preferred /
|
|
2443
|
+
// default chunk size rather than trying to budget against an unknown.
|
|
2834
2444
|
const rowsPerChunk = chooseMapChunkSize({
|
|
2835
|
-
totalRows: streaming ?
|
|
2445
|
+
totalRows: streaming ? 0 : sliced.length,
|
|
2836
2446
|
mapCount: Math.max(1, plan?.maps.length ?? 1),
|
|
2837
2447
|
stepsPerChunk: planMap?.stepsPerChunk ?? 1,
|
|
2838
|
-
preferredChunkSize,
|
|
2448
|
+
preferredChunkSize: planMap?.defaultChunkSize,
|
|
2839
2449
|
softWorkflowStepBudget: plan?.chunkPlan.softWorkflowStepBudget,
|
|
2840
2450
|
});
|
|
2841
2451
|
const outputFields = fieldEntries.map(([field]) => field);
|
|
2452
|
+
const explicitRowKeysSeen =
|
|
2453
|
+
opts?.key === undefined ? null : new Map<string, number>();
|
|
2454
|
+
const resolveExplicitKeyValue = (
|
|
2455
|
+
row: Record<string, unknown>,
|
|
2456
|
+
index: number,
|
|
2457
|
+
): string | null => {
|
|
2458
|
+
const inputRow = publicCsvInputRow(row);
|
|
2459
|
+
const keyOption = opts?.key;
|
|
2460
|
+
if (keyOption === undefined) {
|
|
2461
|
+
return null;
|
|
2462
|
+
}
|
|
2463
|
+
const raw =
|
|
2464
|
+
typeof keyOption === 'function'
|
|
2465
|
+
? keyOption(inputRow, index)
|
|
2466
|
+
: typeof keyOption === 'string'
|
|
2467
|
+
? inputRow[keyOption]
|
|
2468
|
+
: keyOption.map((fieldName) => inputRow[fieldName]);
|
|
2469
|
+
const parts = Array.isArray(raw) ? raw : [raw];
|
|
2470
|
+
if (parts.some((part) => part === null || part === undefined)) {
|
|
2471
|
+
throw new Error(
|
|
2472
|
+
`ctx.map("${name}") key returned null or undefined for row ${index}. ` +
|
|
2473
|
+
'Return a non-empty string or number derived from a stable input column.',
|
|
2474
|
+
);
|
|
2475
|
+
}
|
|
2476
|
+
const normalizedParts = parts.map((part) => {
|
|
2477
|
+
if (typeof part === 'number') {
|
|
2478
|
+
return Number.isFinite(part) ? String(part) : '';
|
|
2479
|
+
}
|
|
2480
|
+
return String(part).trim();
|
|
2481
|
+
});
|
|
2482
|
+
if (normalizedParts.some((part) => !part)) {
|
|
2483
|
+
throw new Error(
|
|
2484
|
+
`ctx.map("${name}") key returned an empty value for row ${index}. ` +
|
|
2485
|
+
'Return a non-empty string or finite number derived from a stable input column.',
|
|
2486
|
+
);
|
|
2487
|
+
}
|
|
2488
|
+
const keyValue =
|
|
2489
|
+
normalizedParts.length === 1
|
|
2490
|
+
? normalizedParts[0]!
|
|
2491
|
+
: JSON.stringify(normalizedParts);
|
|
2492
|
+
return keyValue;
|
|
2493
|
+
};
|
|
2494
|
+
const resolveRowKey = (
|
|
2495
|
+
row: Record<string, unknown>,
|
|
2496
|
+
index: number,
|
|
2497
|
+
): string => {
|
|
2498
|
+
const inputRow = publicCsvInputRow(row);
|
|
2499
|
+
const explicitKeyValue = resolveExplicitKeyValue(row, index);
|
|
2500
|
+
return explicitKeyValue == null
|
|
2501
|
+
? derivePlayRowIdentity(inputRow, name)
|
|
2502
|
+
: derivePlayRowIdentityFromKey(explicitKeyValue, name);
|
|
2503
|
+
};
|
|
2504
|
+
const assertUniqueExplicitRowKeys = (
|
|
2505
|
+
chunkRows: readonly Record<string, unknown>[],
|
|
2506
|
+
chunkStart: number,
|
|
2507
|
+
) => {
|
|
2508
|
+
if (!explicitRowKeysSeen) return;
|
|
2509
|
+
for (let localIndex = 0; localIndex < chunkRows.length; localIndex += 1) {
|
|
2510
|
+
const index = chunkStart + localIndex;
|
|
2511
|
+
const keyValue = resolveExplicitKeyValue(chunkRows[localIndex]!, index);
|
|
2512
|
+
if (keyValue == null) continue;
|
|
2513
|
+
const previousIndex = explicitRowKeysSeen?.get(keyValue);
|
|
2514
|
+
if (previousIndex !== undefined) {
|
|
2515
|
+
throw new Error(
|
|
2516
|
+
`ctx.map("${name}") key function produced duplicate value "${keyValue}" for rows ${previousIndex} and ${index}. ` +
|
|
2517
|
+
'Each row must produce a unique key. Combine columns (e.g. `${row.email}|${row.company}`) or pick a column that is unique per row.',
|
|
2518
|
+
);
|
|
2519
|
+
}
|
|
2520
|
+
explicitRowKeysSeen?.set(keyValue, index);
|
|
2521
|
+
}
|
|
2522
|
+
};
|
|
2842
2523
|
|
|
2843
2524
|
const processChunk = async (
|
|
2844
2525
|
chunkRows: T[],
|
|
@@ -2846,16 +2527,17 @@ function createWorkerCtxFactory(
|
|
|
2846
2527
|
chunkIndex: number,
|
|
2847
2528
|
): Promise<WorkerMapChunkSummary<T & Record<string, unknown>>> => {
|
|
2848
2529
|
assertNotAborted(abortSignal);
|
|
2849
|
-
const
|
|
2530
|
+
const chunkEntries = chunkRows.map((row, localIndex) => {
|
|
2531
|
+
const absoluteIndex = baseOffset + chunkStart + localIndex;
|
|
2532
|
+
const rowKey = resolveRowKey(row, absoluteIndex);
|
|
2533
|
+
return { row, absoluteIndex, rowKey };
|
|
2534
|
+
});
|
|
2535
|
+
const prepared = await prepareMapRows({
|
|
2536
|
+
req,
|
|
2850
2537
|
tableNamespace: name,
|
|
2851
|
-
rows:
|
|
2538
|
+
rows: chunkEntries.map(({ row, rowKey }) => ({
|
|
2852
2539
|
...row,
|
|
2853
|
-
__deeplineRowKey:
|
|
2854
|
-
row,
|
|
2855
|
-
name,
|
|
2856
|
-
opts,
|
|
2857
|
-
baseOffset + chunkStart + index,
|
|
2858
|
-
),
|
|
2540
|
+
__deeplineRowKey: rowKey,
|
|
2859
2541
|
})),
|
|
2860
2542
|
});
|
|
2861
2543
|
const pendingKeys = new Set<string>();
|
|
@@ -2865,7 +2547,7 @@ function createWorkerCtxFactory(
|
|
|
2865
2547
|
const key =
|
|
2866
2548
|
typeof row.__deeplineRowKey === 'string'
|
|
2867
2549
|
? row.__deeplineRowKey
|
|
2868
|
-
:
|
|
2550
|
+
: derivePlayRowIdentity(publicCsvInputRow(row), name);
|
|
2869
2551
|
if (key) {
|
|
2870
2552
|
pendingKeys.add(key);
|
|
2871
2553
|
preparedKeys.add(key);
|
|
@@ -2875,20 +2557,19 @@ function createWorkerCtxFactory(
|
|
|
2875
2557
|
const key =
|
|
2876
2558
|
typeof row.__deeplineRowKey === 'string'
|
|
2877
2559
|
? row.__deeplineRowKey
|
|
2878
|
-
:
|
|
2560
|
+
: derivePlayRowIdentity(publicCsvInputRow(row), name);
|
|
2879
2561
|
if (key) {
|
|
2880
2562
|
completedKeys.add(key);
|
|
2881
2563
|
preparedKeys.add(key);
|
|
2882
2564
|
}
|
|
2883
2565
|
}
|
|
2884
|
-
const missingPreparedRows =
|
|
2885
|
-
|
|
2886
|
-
|
|
2887
|
-
|
|
2888
|
-
|
|
2889
|
-
|
|
2890
|
-
|
|
2891
|
-
});
|
|
2566
|
+
const missingPreparedRows = chunkEntries.filter(
|
|
2567
|
+
({ rowKey }) => !preparedKeys.has(rowKey),
|
|
2568
|
+
);
|
|
2569
|
+
const rowsToExecuteEntries = chunkEntries.filter(
|
|
2570
|
+
({ rowKey }) => pendingKeys.has(rowKey) || !completedKeys.has(rowKey),
|
|
2571
|
+
);
|
|
2572
|
+
const rowsToExecute = rowsToExecuteEntries.map(({ row }) => row);
|
|
2892
2573
|
const rowsInserted = prepared.inserted + missingPreparedRows.length;
|
|
2893
2574
|
const rowsSkipped = Math.max(
|
|
2894
2575
|
0,
|
|
@@ -2912,9 +2593,10 @@ function createWorkerCtxFactory(
|
|
|
2912
2593
|
if (abortSignal?.aborted) return;
|
|
2913
2594
|
const myIndex = idx++;
|
|
2914
2595
|
if (myIndex >= rowsToExecute.length) return;
|
|
2915
|
-
const
|
|
2916
|
-
const
|
|
2917
|
-
const
|
|
2596
|
+
const entry = rowsToExecuteEntries[myIndex]!;
|
|
2597
|
+
const row = entry.row;
|
|
2598
|
+
const absoluteIndex = entry.absoluteIndex;
|
|
2599
|
+
const enriched: Record<string, unknown> = cloneCsvAliasedRow(row);
|
|
2918
2600
|
const fieldOutputs: Record<string, unknown> = {};
|
|
2919
2601
|
const cellMetaPatch: Record<
|
|
2920
2602
|
string,
|
|
@@ -2925,25 +2607,35 @@ function createWorkerCtxFactory(
|
|
|
2925
2607
|
const rowCtx = {
|
|
2926
2608
|
...(ctx as Record<string, unknown>),
|
|
2927
2609
|
tool: async (
|
|
2928
|
-
|
|
2610
|
+
key: string,
|
|
2611
|
+
toolId: string,
|
|
2612
|
+
input: Record<string, unknown>,
|
|
2929
2613
|
): Promise<unknown> => {
|
|
2930
2614
|
assertNotAborted(abortSignal);
|
|
2931
2615
|
return await toolBatchScheduler.execute(
|
|
2932
|
-
|
|
2933
|
-
|
|
2934
|
-
|
|
2616
|
+
key,
|
|
2617
|
+
toolId,
|
|
2618
|
+
input,
|
|
2935
2619
|
workflowStep,
|
|
2936
2620
|
);
|
|
2937
2621
|
},
|
|
2938
2622
|
tools: {
|
|
2939
2623
|
...((ctx as { tools?: Record<string, unknown> }).tools ?? {}),
|
|
2940
2624
|
execute: async (
|
|
2941
|
-
|
|
2625
|
+
requestOrKey: unknown,
|
|
2626
|
+
toolId?: unknown,
|
|
2627
|
+
input?: unknown,
|
|
2628
|
+
_opts?: { description?: string },
|
|
2942
2629
|
): Promise<unknown> => {
|
|
2943
2630
|
assertNotAborted(abortSignal);
|
|
2631
|
+
const request = normalizeToolExecuteArgs(
|
|
2632
|
+
requestOrKey,
|
|
2633
|
+
toolId,
|
|
2634
|
+
input,
|
|
2635
|
+
);
|
|
2944
2636
|
return await toolBatchScheduler.execute(
|
|
2945
2637
|
request.id,
|
|
2946
|
-
request.
|
|
2638
|
+
request.toolId,
|
|
2947
2639
|
request.input,
|
|
2948
2640
|
workflowStep,
|
|
2949
2641
|
);
|
|
@@ -2960,7 +2652,6 @@ function createWorkerCtxFactory(
|
|
|
2960
2652
|
toolNameOrSpec,
|
|
2961
2653
|
waterfallInput,
|
|
2962
2654
|
waterfallOpts,
|
|
2963
|
-
workflowStep,
|
|
2964
2655
|
),
|
|
2965
2656
|
};
|
|
2966
2657
|
for (const [key, value] of fieldEntries) {
|
|
@@ -3012,7 +2703,8 @@ function createWorkerCtxFactory(
|
|
|
3012
2703
|
}
|
|
3013
2704
|
await Promise.all(workers);
|
|
3014
2705
|
if (executedRows.length > 0) {
|
|
3015
|
-
await
|
|
2706
|
+
await persistCompletedMapRows({
|
|
2707
|
+
req,
|
|
3016
2708
|
tableNamespace: name,
|
|
3017
2709
|
outputFields,
|
|
3018
2710
|
extraOutputFields: Array.from(generatedOutputFields),
|
|
@@ -3024,11 +2716,7 @@ function createWorkerCtxFactory(
|
|
|
3024
2716
|
executedCellMetaPatches[executedIndex],
|
|
3025
2717
|
}
|
|
3026
2718
|
: {}),
|
|
3027
|
-
__deeplineRowKey:
|
|
3028
|
-
rowsToExecute[executedIndex]!,
|
|
3029
|
-
name,
|
|
3030
|
-
opts,
|
|
3031
|
-
),
|
|
2719
|
+
__deeplineRowKey: rowsToExecuteEntries[executedIndex]!.rowKey,
|
|
3032
2720
|
})),
|
|
3033
2721
|
});
|
|
3034
2722
|
}
|
|
@@ -3037,9 +2725,10 @@ function createWorkerCtxFactory(
|
|
|
3037
2725
|
const key =
|
|
3038
2726
|
typeof completedRow.__deeplineRowKey === 'string'
|
|
3039
2727
|
? completedRow.__deeplineRowKey
|
|
3040
|
-
:
|
|
2728
|
+
: derivePlayRowIdentity(publicCsvInputRow(completedRow), name);
|
|
3041
2729
|
if (key) {
|
|
3042
|
-
const { __deeplineRowKey: _rowKey, ...cleanedRow } =
|
|
2730
|
+
const { __deeplineRowKey: _rowKey, ...cleanedRow } =
|
|
2731
|
+
publicCsvInputRow(completedRow);
|
|
3043
2732
|
void _rowKey;
|
|
3044
2733
|
resultByKey.set(key, cleanedRow as T & Record<string, unknown>);
|
|
3045
2734
|
}
|
|
@@ -3050,17 +2739,13 @@ function createWorkerCtxFactory(
|
|
|
3050
2739
|
executedIndex += 1
|
|
3051
2740
|
) {
|
|
3052
2741
|
const executedRow = executedRows[executedIndex]!;
|
|
3053
|
-
const key =
|
|
3054
|
-
rowsToExecute[executedIndex]!,
|
|
3055
|
-
name,
|
|
3056
|
-
opts,
|
|
3057
|
-
);
|
|
2742
|
+
const key = rowsToExecuteEntries[executedIndex]!.rowKey;
|
|
3058
2743
|
if (key) resultByKey.set(key, executedRow);
|
|
3059
2744
|
}
|
|
3060
2745
|
const out = chunkRows
|
|
3061
|
-
.map((
|
|
3062
|
-
const key =
|
|
3063
|
-
return
|
|
2746
|
+
.map((_row, index) => {
|
|
2747
|
+
const key = chunkEntries[index]!.rowKey;
|
|
2748
|
+
return resultByKey.get(key);
|
|
3064
2749
|
})
|
|
3065
2750
|
.filter((row): row is T & Record<string, unknown> => Boolean(row));
|
|
3066
2751
|
return {
|
|
@@ -3075,7 +2760,7 @@ function createWorkerCtxFactory(
|
|
|
3075
2760
|
rowsSkipped,
|
|
3076
2761
|
outputDatasetId: `map:${name}`,
|
|
3077
2762
|
hash: await hashJson(out),
|
|
3078
|
-
preview: toWorkflowSerializableValue(out.slice(0,
|
|
2763
|
+
preview: toWorkflowSerializableValue(out.slice(0, 5)),
|
|
3079
2764
|
};
|
|
3080
2765
|
};
|
|
3081
2766
|
|
|
@@ -3135,6 +2820,7 @@ function createWorkerCtxFactory(
|
|
|
3135
2820
|
for await (const chunkRows of streamingDataset.iterChunks(rowsPerChunk)) {
|
|
3136
2821
|
assertNotAborted(abortSignal);
|
|
3137
2822
|
if (chunkRows.length === 0) continue;
|
|
2823
|
+
assertUniqueExplicitRowKeys(chunkRows, chunkStart);
|
|
3138
2824
|
const chunkResult = await runChunkStep(
|
|
3139
2825
|
chunkRows,
|
|
3140
2826
|
chunkStart,
|
|
@@ -3161,6 +2847,7 @@ function createWorkerCtxFactory(
|
|
|
3161
2847
|
const end = Math.min(sliced.length, start + rowsPerChunk);
|
|
3162
2848
|
const chunkRows = sliced.slice(start, end);
|
|
3163
2849
|
const chunkIndex = Math.floor(start / rowsPerChunk);
|
|
2850
|
+
assertUniqueExplicitRowKeys(chunkRows, start);
|
|
3164
2851
|
const chunkResult = await runChunkStep(chunkRows, start, chunkIndex);
|
|
3165
2852
|
totalRowsWritten += chunkResult.rowsWritten;
|
|
3166
2853
|
totalRowsExecuted += chunkResult.rowsExecuted;
|
|
@@ -3174,6 +2861,7 @@ function createWorkerCtxFactory(
|
|
|
3174
2861
|
return finalize(totalRowsWritten);
|
|
3175
2862
|
}
|
|
3176
2863
|
|
|
2864
|
+
assertUniqueExplicitRowKeys(sliced, 0);
|
|
3177
2865
|
const chunkResult = await runChunkStep(sliced, 0, 0);
|
|
3178
2866
|
totalRowsExecuted = chunkResult.rowsExecuted;
|
|
3179
2867
|
totalRowsCached = chunkResult.rowsCached;
|
|
@@ -3192,10 +2880,6 @@ function createWorkerCtxFactory(
|
|
|
3192
2880
|
constructor(
|
|
3193
2881
|
private readonly name: string,
|
|
3194
2882
|
private readonly rows: T[],
|
|
3195
|
-
private readonly mapOptions?: Omit<
|
|
3196
|
-
WorkerMapOptions,
|
|
3197
|
-
'description' | 'concurrency'
|
|
3198
|
-
>,
|
|
3199
2883
|
) {}
|
|
3200
2884
|
|
|
3201
2885
|
step(name: string, resolver: WorkerStepProgramStep['resolver']): this {
|
|
@@ -3207,23 +2891,10 @@ function createWorkerCtxFactory(
|
|
|
3207
2891
|
}
|
|
3208
2892
|
|
|
3209
2893
|
run(opts?: WorkerMapOptions): Promise<unknown> {
|
|
3210
|
-
if (
|
|
3211
|
-
opts &&
|
|
3212
|
-
Object.keys(opts).some(
|
|
3213
|
-
(optionKey) => optionKey !== 'description' && optionKey !== 'concurrency',
|
|
3214
|
-
)
|
|
3215
|
-
) {
|
|
3216
|
-
throw new Error(
|
|
3217
|
-
'ctx.map(...).run() only accepts description and concurrency.',
|
|
3218
|
-
);
|
|
3219
|
-
}
|
|
3220
2894
|
const fields = Object.fromEntries(
|
|
3221
2895
|
this.program.steps.map((step) => [step.name, step.resolver]),
|
|
3222
2896
|
);
|
|
3223
|
-
return runMap(this.name, this.rows, fields,
|
|
3224
|
-
...this.mapOptions,
|
|
3225
|
-
...opts,
|
|
3226
|
-
});
|
|
2897
|
+
return runMap(this.name, this.rows, fields, opts);
|
|
3227
2898
|
}
|
|
3228
2899
|
}
|
|
3229
2900
|
|
|
@@ -3261,21 +2932,30 @@ function createWorkerCtxFactory(
|
|
|
3261
2932
|
},
|
|
3262
2933
|
async csv<T extends Record<string, unknown> = Record<string, unknown>>(
|
|
3263
2934
|
arg: unknown,
|
|
2935
|
+
options?: CsvRenameOptions,
|
|
3264
2936
|
): Promise<T[]> {
|
|
3265
2937
|
if (Array.isArray(arg)) {
|
|
3266
2938
|
// Inline rows passed at call site — already in memory, keep the
|
|
3267
2939
|
// legacy array-backed dataset shape.
|
|
3268
|
-
return makeWorkerDataset(
|
|
3269
|
-
|
|
3270
|
-
|
|
2940
|
+
return makeWorkerDataset(
|
|
2941
|
+
'csv',
|
|
2942
|
+
applyCsvRenameProjection(arg as T[], options),
|
|
2943
|
+
{
|
|
2944
|
+
datasetKind: 'csv',
|
|
2945
|
+
},
|
|
2946
|
+
) as unknown as T[];
|
|
3271
2947
|
}
|
|
3272
2948
|
const filename = String(arg ?? '');
|
|
3273
2949
|
if (req.inlineCsv && filename === req.inlineCsv.name) {
|
|
3274
2950
|
// Inline CSV pre-staged by the dispatcher (small files <1 MiB). Already
|
|
3275
2951
|
// in memory; no streaming needed.
|
|
3276
|
-
return makeWorkerDataset(
|
|
3277
|
-
|
|
3278
|
-
|
|
2952
|
+
return makeWorkerDataset(
|
|
2953
|
+
'csv',
|
|
2954
|
+
applyCsvRenameProjection(req.inlineCsv.rows as T[], options),
|
|
2955
|
+
{
|
|
2956
|
+
datasetKind: 'csv',
|
|
2957
|
+
},
|
|
2958
|
+
) as unknown as T[];
|
|
3279
2959
|
}
|
|
3280
2960
|
// Resolution order: explicit inputR2Keys (runtime input) → packaged
|
|
3281
2961
|
// files (relative-path imports bundled with the play artifact).
|
|
@@ -3303,6 +2983,7 @@ function createWorkerCtxFactory(
|
|
|
3303
2983
|
return makeStreamingCsvDataset<T>({
|
|
3304
2984
|
name: filename,
|
|
3305
2985
|
logicalPath: filename,
|
|
2986
|
+
renameOptions: options,
|
|
3306
2987
|
open: () =>
|
|
3307
2988
|
openR2BodyStream({
|
|
3308
2989
|
req,
|
|
@@ -3327,18 +3008,10 @@ function createWorkerCtxFactory(
|
|
|
3327
3008
|
) => Promise<unknown> | unknown)
|
|
3328
3009
|
>
|
|
3329
3010
|
| WorkerStepProgram,
|
|
3330
|
-
opts?:
|
|
3011
|
+
opts?: WorkerMapOptions,
|
|
3331
3012
|
): unknown {
|
|
3332
|
-
if (
|
|
3333
|
-
|
|
3334
|
-
fieldsDef === undefined ||
|
|
3335
|
-
isWorkerMapDefinitionOptions(fieldsDef)
|
|
3336
|
-
) {
|
|
3337
|
-
return new WorkerMapBuilder(
|
|
3338
|
-
name,
|
|
3339
|
-
rows,
|
|
3340
|
-
fieldsDef as Omit<WorkerMapOptions, 'description' | 'concurrency'>,
|
|
3341
|
-
);
|
|
3013
|
+
if (arguments.length <= 2 || fieldsDef === undefined) {
|
|
3014
|
+
return new WorkerMapBuilder(name, rows);
|
|
3342
3015
|
}
|
|
3343
3016
|
if (isWorkerStepProgram(fieldsDef)) {
|
|
3344
3017
|
const fields = Object.fromEntries(
|
|
@@ -3346,7 +3019,9 @@ function createWorkerCtxFactory(
|
|
|
3346
3019
|
);
|
|
3347
3020
|
return runMap(name, rows, fields, opts);
|
|
3348
3021
|
}
|
|
3349
|
-
throw new Error(
|
|
3022
|
+
throw new Error(
|
|
3023
|
+
'ctx.map(key, rows, fields, options) was removed. Use ctx.map(key, rows).step(...).run(options).',
|
|
3024
|
+
);
|
|
3350
3025
|
/*
|
|
3351
3026
|
const sliced = rows;
|
|
3352
3027
|
const baseOffset = 0;
|
|
@@ -3357,13 +3032,14 @@ function createWorkerCtxFactory(
|
|
|
3357
3032
|
candidate.mapName === name || candidate.tableNamespace === name,
|
|
3358
3033
|
);
|
|
3359
3034
|
const streaming = isStreamingDataset<T>(sliced);
|
|
3360
|
-
|
|
3361
|
-
|
|
3035
|
+
// For streaming inputs we don't know the row count upfront — pass
|
|
3036
|
+
// `totalRows: 0` so chooseMapChunkSize falls back to the preferred /
|
|
3037
|
+
// default chunk size rather than trying to budget against an unknown.
|
|
3362
3038
|
const rowsPerChunk = chooseMapChunkSize({
|
|
3363
|
-
totalRows: streaming ?
|
|
3039
|
+
totalRows: streaming ? 0 : sliced.length,
|
|
3364
3040
|
mapCount: Math.max(1, plan?.maps.length ?? 1),
|
|
3365
3041
|
stepsPerChunk: planMap?.stepsPerChunk ?? 1,
|
|
3366
|
-
preferredChunkSize,
|
|
3042
|
+
preferredChunkSize: planMap?.defaultChunkSize,
|
|
3367
3043
|
softWorkflowStepBudget: plan?.chunkPlan.softWorkflowStepBudget,
|
|
3368
3044
|
});
|
|
3369
3045
|
const outputFields = fieldEntries.map(([field]) => field);
|
|
@@ -3387,7 +3063,7 @@ function createWorkerCtxFactory(
|
|
|
3387
3063
|
const completedKeys = new Set<string>();
|
|
3388
3064
|
const preparedKeys = new Set<string>();
|
|
3389
3065
|
for (const row of prepared.pendingRows) {
|
|
3390
|
-
const key = derivePlayRowIdentity(row, name);
|
|
3066
|
+
const key = derivePlayRowIdentity(publicCsvInputRow(row), name);
|
|
3391
3067
|
if (key) {
|
|
3392
3068
|
pendingKeys.add(key);
|
|
3393
3069
|
preparedKeys.add(key);
|
|
@@ -3397,18 +3073,18 @@ function createWorkerCtxFactory(
|
|
|
3397
3073
|
const key =
|
|
3398
3074
|
typeof row.__deeplineRowKey === 'string'
|
|
3399
3075
|
? row.__deeplineRowKey
|
|
3400
|
-
: derivePlayRowIdentity(row, name);
|
|
3076
|
+
: derivePlayRowIdentity(publicCsvInputRow(row), name);
|
|
3401
3077
|
if (key) {
|
|
3402
3078
|
completedKeys.add(key);
|
|
3403
3079
|
preparedKeys.add(key);
|
|
3404
3080
|
}
|
|
3405
3081
|
}
|
|
3406
3082
|
const missingPreparedRows = chunkRows.filter((row) => {
|
|
3407
|
-
const key = derivePlayRowIdentity(row, name);
|
|
3083
|
+
const key = derivePlayRowIdentity(publicCsvInputRow(row), name);
|
|
3408
3084
|
return !key || !preparedKeys.has(key);
|
|
3409
3085
|
});
|
|
3410
3086
|
const rowsToExecute = chunkRows.filter((row) => {
|
|
3411
|
-
const key = derivePlayRowIdentity(row, name);
|
|
3087
|
+
const key = derivePlayRowIdentity(publicCsvInputRow(row), name);
|
|
3412
3088
|
return !key || pendingKeys.has(key) || !completedKeys.has(key);
|
|
3413
3089
|
});
|
|
3414
3090
|
const rowsInserted = prepared.inserted + missingPreparedRows.length;
|
|
@@ -3432,7 +3108,7 @@ function createWorkerCtxFactory(
|
|
|
3432
3108
|
if (myIndex >= rowsToExecute.length) return;
|
|
3433
3109
|
const row = rowsToExecute[myIndex]!;
|
|
3434
3110
|
const absoluteIndex = baseOffset + chunkStart + myIndex;
|
|
3435
|
-
const enriched: Record<string, unknown> =
|
|
3111
|
+
const enriched: Record<string, unknown> = cloneCsvAliasedRow(row);
|
|
3436
3112
|
const fieldOutputs: Record<string, unknown> = {};
|
|
3437
3113
|
const waterfallOutputs: RecordedWaterfallOutput[] = [];
|
|
3438
3114
|
const rowCtx = {
|
|
@@ -3448,7 +3124,6 @@ function createWorkerCtxFactory(
|
|
|
3448
3124
|
toolNameOrSpec,
|
|
3449
3125
|
waterfallInput,
|
|
3450
3126
|
waterfallOpts,
|
|
3451
|
-
workflowStep,
|
|
3452
3127
|
),
|
|
3453
3128
|
};
|
|
3454
3129
|
for (const [key, value] of fieldEntries) {
|
|
@@ -3490,7 +3165,7 @@ function createWorkerCtxFactory(
|
|
|
3490
3165
|
rows: executedRows.map((row, executedIndex) => ({
|
|
3491
3166
|
...row,
|
|
3492
3167
|
__deeplineRowKey: derivePlayRowIdentity(
|
|
3493
|
-
rowsToExecute[executedIndex]
|
|
3168
|
+
publicCsvInputRow(rowsToExecute[executedIndex]!),
|
|
3494
3169
|
name,
|
|
3495
3170
|
),
|
|
3496
3171
|
})),
|
|
@@ -3501,9 +3176,10 @@ function createWorkerCtxFactory(
|
|
|
3501
3176
|
const key =
|
|
3502
3177
|
typeof completedRow.__deeplineRowKey === 'string'
|
|
3503
3178
|
? completedRow.__deeplineRowKey
|
|
3504
|
-
: derivePlayRowIdentity(completedRow, name);
|
|
3179
|
+
: derivePlayRowIdentity(publicCsvInputRow(completedRow), name);
|
|
3505
3180
|
if (key) {
|
|
3506
|
-
const { __deeplineRowKey: _rowKey, ...cleanedRow } =
|
|
3181
|
+
const { __deeplineRowKey: _rowKey, ...cleanedRow } =
|
|
3182
|
+
publicCsvInputRow(completedRow);
|
|
3507
3183
|
void _rowKey;
|
|
3508
3184
|
resultByKey.set(key, cleanedRow as T & Record<string, unknown>);
|
|
3509
3185
|
}
|
|
@@ -3515,14 +3191,14 @@ function createWorkerCtxFactory(
|
|
|
3515
3191
|
) {
|
|
3516
3192
|
const executedRow = executedRows[executedIndex]!;
|
|
3517
3193
|
const key = derivePlayRowIdentity(
|
|
3518
|
-
rowsToExecute[executedIndex]
|
|
3194
|
+
publicCsvInputRow(rowsToExecute[executedIndex]!),
|
|
3519
3195
|
name,
|
|
3520
3196
|
);
|
|
3521
3197
|
if (key) resultByKey.set(key, executedRow);
|
|
3522
3198
|
}
|
|
3523
3199
|
const out = chunkRows
|
|
3524
3200
|
.map((row) => {
|
|
3525
|
-
const key = derivePlayRowIdentity(row, name);
|
|
3201
|
+
const key = derivePlayRowIdentity(publicCsvInputRow(row), name);
|
|
3526
3202
|
return key ? resultByKey.get(key) : undefined;
|
|
3527
3203
|
})
|
|
3528
3204
|
.filter((row): row is T & Record<string, unknown> => Boolean(row));
|
|
@@ -3538,7 +3214,7 @@ function createWorkerCtxFactory(
|
|
|
3538
3214
|
rowsSkipped,
|
|
3539
3215
|
outputDatasetId: `map:${name}`,
|
|
3540
3216
|
hash: await hashJson(out),
|
|
3541
|
-
preview: toWorkflowSerializableValue(out.slice(0,
|
|
3217
|
+
preview: toWorkflowSerializableValue(out.slice(0, 5)),
|
|
3542
3218
|
};
|
|
3543
3219
|
};
|
|
3544
3220
|
|
|
@@ -3653,28 +3329,25 @@ function createWorkerCtxFactory(
|
|
|
3653
3329
|
return finalize(chunkResult.rowsWritten);
|
|
3654
3330
|
*/
|
|
3655
3331
|
},
|
|
3656
|
-
tool: async (
|
|
3332
|
+
tool: async (
|
|
3333
|
+
key: string,
|
|
3334
|
+
toolId: string,
|
|
3335
|
+
input: Record<string, unknown>,
|
|
3336
|
+
): Promise<unknown> => {
|
|
3657
3337
|
assertNotAborted(abortSignal);
|
|
3658
|
-
return executeTool(
|
|
3659
|
-
req,
|
|
3660
|
-
{
|
|
3661
|
-
toolId: request.tool,
|
|
3662
|
-
input: request.input,
|
|
3663
|
-
retryKey: request.id,
|
|
3664
|
-
},
|
|
3665
|
-
workflowStep,
|
|
3666
|
-
);
|
|
3338
|
+
return executeTool(req, { id: key, toolId, input }, workflowStep);
|
|
3667
3339
|
},
|
|
3668
3340
|
tools: {
|
|
3669
|
-
async execute(
|
|
3341
|
+
async execute(
|
|
3342
|
+
requestOrKey: unknown,
|
|
3343
|
+
toolId?: unknown,
|
|
3344
|
+
input?: unknown,
|
|
3345
|
+
_opts?: { description?: string },
|
|
3346
|
+
): Promise<unknown> {
|
|
3670
3347
|
assertNotAborted(abortSignal);
|
|
3671
3348
|
return executeTool(
|
|
3672
3349
|
req,
|
|
3673
|
-
|
|
3674
|
-
toolId: request.tool,
|
|
3675
|
-
input: request.input,
|
|
3676
|
-
retryKey: request.id,
|
|
3677
|
-
},
|
|
3350
|
+
normalizeToolExecuteArgs(requestOrKey, toolId, input),
|
|
3678
3351
|
workflowStep,
|
|
3679
3352
|
);
|
|
3680
3353
|
},
|
|
@@ -3702,14 +3375,7 @@ function createWorkerCtxFactory(
|
|
|
3702
3375
|
input: Record<string, unknown>,
|
|
3703
3376
|
opts?: WorkerWaterfallOptions,
|
|
3704
3377
|
): Promise<unknown | null> {
|
|
3705
|
-
return executeWorkerWaterfall(
|
|
3706
|
-
req,
|
|
3707
|
-
[],
|
|
3708
|
-
toolNameOrSpec,
|
|
3709
|
-
input,
|
|
3710
|
-
opts,
|
|
3711
|
-
workflowStep,
|
|
3712
|
-
);
|
|
3378
|
+
return executeWorkerWaterfall(req, [], toolNameOrSpec, input, opts);
|
|
3713
3379
|
},
|
|
3714
3380
|
async sleep(ms: number): Promise<void> {
|
|
3715
3381
|
assertNotAborted(abortSignal);
|
|
@@ -3798,33 +3464,33 @@ function createWorkerCtxFactory(
|
|
|
3798
3464
|
}
|
|
3799
3465
|
try {
|
|
3800
3466
|
const childSubmitStartedAt = nowMs();
|
|
3801
|
-
let started:
|
|
3802
|
-
ReturnType<typeof submitChildPlayThroughCoordinator>
|
|
3803
|
-
>;
|
|
3467
|
+
let started: { workflowId?: string; runId?: string; error?: unknown };
|
|
3804
3468
|
try {
|
|
3805
|
-
started = await
|
|
3806
|
-
|
|
3807
|
-
|
|
3808
|
-
|
|
3809
|
-
|
|
3810
|
-
|
|
3811
|
-
|
|
3812
|
-
|
|
3813
|
-
|
|
3814
|
-
|
|
3815
|
-
|
|
3816
|
-
|
|
3817
|
-
|
|
3818
|
-
|
|
3819
|
-
|
|
3820
|
-
|
|
3821
|
-
|
|
3822
|
-
|
|
3823
|
-
|
|
3824
|
-
|
|
3825
|
-
|
|
3826
|
-
|
|
3827
|
-
|
|
3469
|
+
started = await submitChildPlayThroughCoordinator({
|
|
3470
|
+
req,
|
|
3471
|
+
body: {
|
|
3472
|
+
name: resolvedName,
|
|
3473
|
+
input: isRecord(input) ? input : {},
|
|
3474
|
+
orgId: req.orgId,
|
|
3475
|
+
parentExecutorToken: req.executorToken,
|
|
3476
|
+
userEmail: req.userEmail ?? '',
|
|
3477
|
+
profile: 'workers_edge',
|
|
3478
|
+
manifest: childManifest,
|
|
3479
|
+
childPlayManifests: req.childPlayManifests ?? null,
|
|
3480
|
+
internalRunPlay: {
|
|
3481
|
+
rootRunId,
|
|
3482
|
+
parentRunId: req.runId,
|
|
3483
|
+
parentPlayName: req.playName,
|
|
3484
|
+
key: normalizedKey,
|
|
3485
|
+
// Per the lineage validator: ancestry tail must equal the
|
|
3486
|
+
// executor token's play name (the parent making this call).
|
|
3487
|
+
ancestryPlayIds,
|
|
3488
|
+
callDepth: nextDepth,
|
|
3489
|
+
description:
|
|
3490
|
+
typeof options?.description === 'string'
|
|
3491
|
+
? options.description
|
|
3492
|
+
: null,
|
|
3493
|
+
},
|
|
3828
3494
|
},
|
|
3829
3495
|
});
|
|
3830
3496
|
} catch (error) {
|
|
@@ -3870,112 +3536,12 @@ function createWorkerCtxFactory(
|
|
|
3870
3536
|
fanoutIndex: nextParentCalls - 1,
|
|
3871
3537
|
ms: nowMs() - childSubmitStartedAt,
|
|
3872
3538
|
status: 'ok',
|
|
3873
|
-
mode:
|
|
3874
|
-
typeof started.mode === 'string' ? started.mode : 'workflow_child',
|
|
3875
3539
|
});
|
|
3876
|
-
const terminalStatus =
|
|
3877
|
-
typeof started.status === 'string'
|
|
3878
|
-
? started.status.toLowerCase()
|
|
3879
|
-
: '';
|
|
3880
|
-
if (started.mode === 'inline_dynamic_worker') {
|
|
3881
|
-
const timingSummary =
|
|
3882
|
-
Array.isArray(started.timings) && started.timings.length > 0
|
|
3883
|
-
? started.timings
|
|
3884
|
-
.map(
|
|
3885
|
-
(timing) =>
|
|
3886
|
-
`${timing.phase.replace('coordinator.inline_child_', '')}:${timing.ms}ms`,
|
|
3887
|
-
)
|
|
3888
|
-
.join(' ')
|
|
3889
|
-
: 'timings:none';
|
|
3890
|
-
emitEvent({
|
|
3891
|
-
type: 'log',
|
|
3892
|
-
level: terminalStatus === 'failed' ? 'error' : 'info',
|
|
3893
|
-
message: `Inline child ${resolvedName} (${workflowId}) boundary=${terminalStatus || 'submitted'} ${timingSummary}`,
|
|
3894
|
-
ts: nowMs(),
|
|
3895
|
-
});
|
|
3896
|
-
const childLogs = Array.isArray(started.logs)
|
|
3897
|
-
? started.logs.filter((line) => typeof line === 'string')
|
|
3898
|
-
: [];
|
|
3899
|
-
for (const line of childLogs.slice(0, 24)) {
|
|
3900
|
-
emitEvent({
|
|
3901
|
-
type: 'log',
|
|
3902
|
-
level: 'info',
|
|
3903
|
-
message: ` ${resolvedName}> ${line}`,
|
|
3904
|
-
ts: nowMs(),
|
|
3905
|
-
});
|
|
3906
|
-
}
|
|
3907
|
-
if (childLogs.length > 24) {
|
|
3908
|
-
emitEvent({
|
|
3909
|
-
type: 'log',
|
|
3910
|
-
level: 'info',
|
|
3911
|
-
message: ` ${resolvedName}> ... ${childLogs.length - 24} more inline child log lines omitted`,
|
|
3912
|
-
ts: nowMs(),
|
|
3913
|
-
});
|
|
3914
|
-
}
|
|
3915
|
-
}
|
|
3916
|
-
if (terminalStatus === 'completed') {
|
|
3917
|
-
console.info('[play.runtime.span]', {
|
|
3918
|
-
event: 'play.runtime.span',
|
|
3919
|
-
phase: 'child_wait',
|
|
3920
|
-
runId: req.runId,
|
|
3921
|
-
parentRunId: req.runId,
|
|
3922
|
-
childRunId: workflowId,
|
|
3923
|
-
playName: resolvedName,
|
|
3924
|
-
graphHash: req.graphHash ?? null,
|
|
3925
|
-
depth: nextDepth,
|
|
3926
|
-
fanoutIndex: nextParentCalls - 1,
|
|
3927
|
-
ms: 0,
|
|
3928
|
-
status: 'ok',
|
|
3929
|
-
mode:
|
|
3930
|
-
typeof started.mode === 'string'
|
|
3931
|
-
? started.mode
|
|
3932
|
-
: 'inline_terminal',
|
|
3933
|
-
});
|
|
3934
|
-
emitEvent({
|
|
3935
|
-
type: 'log',
|
|
3936
|
-
level: 'info',
|
|
3937
|
-
message: `Completed child play ${resolvedName} (${normalizedKey})`,
|
|
3938
|
-
ts: nowMs(),
|
|
3939
|
-
});
|
|
3940
|
-
return 'output' in started ? started.output : started.result;
|
|
3941
|
-
}
|
|
3942
|
-
if (terminalStatus === 'failed') {
|
|
3943
|
-
const inlineError = isRecord(started.error) ? started.error : null;
|
|
3944
|
-
const message =
|
|
3945
|
-
(typeof inlineError?.message === 'string' &&
|
|
3946
|
-
inlineError.message.trim()) ||
|
|
3947
|
-
(typeof started.error === 'string' && started.error.trim()) ||
|
|
3948
|
-
`Child play ${resolvedName} (${workflowId}) failed.`;
|
|
3949
|
-
console.info('[play.runtime.span]', {
|
|
3950
|
-
event: 'play.runtime.span',
|
|
3951
|
-
phase: 'child_wait',
|
|
3952
|
-
runId: req.runId,
|
|
3953
|
-
parentRunId: req.runId,
|
|
3954
|
-
childRunId: workflowId,
|
|
3955
|
-
playName: resolvedName,
|
|
3956
|
-
graphHash: req.graphHash ?? null,
|
|
3957
|
-
depth: nextDepth,
|
|
3958
|
-
fanoutIndex: nextParentCalls - 1,
|
|
3959
|
-
ms: 0,
|
|
3960
|
-
status: 'failed',
|
|
3961
|
-
mode:
|
|
3962
|
-
typeof started.mode === 'string'
|
|
3963
|
-
? started.mode
|
|
3964
|
-
: 'inline_terminal',
|
|
3965
|
-
errorCode: 'CHILD_INLINE_FAILED',
|
|
3966
|
-
});
|
|
3967
|
-
emitEvent({
|
|
3968
|
-
type: 'log',
|
|
3969
|
-
level: 'error',
|
|
3970
|
-
message: `Inline child ${resolvedName} (${workflowId}) failed: ${message}`,
|
|
3971
|
-
ts: nowMs(),
|
|
3972
|
-
});
|
|
3973
|
-
throw new Error(message);
|
|
3974
|
-
}
|
|
3975
3540
|
const childWaitStartedAt = nowMs();
|
|
3976
3541
|
let result: unknown;
|
|
3977
3542
|
try {
|
|
3978
|
-
result = await
|
|
3543
|
+
result = await waitForChildPlayTerminalEvent({
|
|
3544
|
+
req,
|
|
3979
3545
|
workflowStep,
|
|
3980
3546
|
workflowId,
|
|
3981
3547
|
playName: resolvedName,
|
|
@@ -4086,9 +3652,12 @@ async function handleRun(request: Request, env: WorkerEnv): Promise<Response> {
|
|
|
4086
3652
|
(async () => {
|
|
4087
3653
|
try {
|
|
4088
3654
|
installProcessExitTrap();
|
|
3655
|
+
const runPrefix = `[deepline-run:${req.runId}]`;
|
|
4089
3656
|
captureCoordinatorBinding(env);
|
|
3657
|
+
captureRuntimeApiBinding(env);
|
|
4090
3658
|
captureHarnessBinding(env);
|
|
4091
|
-
|
|
3659
|
+
await probeHarnessOnce(env, runPrefix);
|
|
3660
|
+
const ctx = createMinimalWorkerCtx(req, emit, env);
|
|
4092
3661
|
const result = await (
|
|
4093
3662
|
playFn as (
|
|
4094
3663
|
ctx: unknown,
|
|
@@ -4142,8 +3711,6 @@ async function executeRunRequest(
|
|
|
4142
3711
|
workflowStep?: WorkflowStep,
|
|
4143
3712
|
options?: {
|
|
4144
3713
|
persistResultDatasets?: boolean;
|
|
4145
|
-
signalParentTerminal?: boolean;
|
|
4146
|
-
waitUntil?: (promise: Promise<unknown>) => void;
|
|
4147
3714
|
/**
|
|
4148
3715
|
* Cooperative cancellation token. CF Workflows surfaces termination as a
|
|
4149
3716
|
* thrown error from any in-progress step; the harness catches that, flips
|
|
@@ -4156,7 +3723,6 @@ async function executeRunRequest(
|
|
|
4156
3723
|
const startedAt = nowMs();
|
|
4157
3724
|
const abortController = options?.abortController ?? new AbortController();
|
|
4158
3725
|
const abortSignal = abortController.signal;
|
|
4159
|
-
const transport = createWorkerRuntimeTransport(req);
|
|
4160
3726
|
// Maintain a rolling buffer of log lines emitted during the run. This is
|
|
4161
3727
|
// what the play-page UI consumes via Convex polling + diffPlayRunStreamEvents
|
|
4162
3728
|
// → play.run.log SSE events. Without periodic flushing, the play page only
|
|
@@ -4184,15 +3750,14 @@ async function executeRunRequest(
|
|
|
4184
3750
|
.catch(() => undefined)
|
|
4185
3751
|
.then(async () => {
|
|
4186
3752
|
try {
|
|
4187
|
-
await
|
|
4188
|
-
|
|
4189
|
-
|
|
4190
|
-
|
|
4191
|
-
|
|
4192
|
-
|
|
4193
|
-
|
|
4194
|
-
|
|
4195
|
-
);
|
|
3753
|
+
await postRuntimeApi(req.baseUrl, req.executorToken, {
|
|
3754
|
+
action: 'update_run_status',
|
|
3755
|
+
playId: req.runId,
|
|
3756
|
+
status: 'running',
|
|
3757
|
+
runtimeBackend: 'cf_workflows_dynamic_worker',
|
|
3758
|
+
liveLogs: snapshot,
|
|
3759
|
+
lastCheckpointAt: now,
|
|
3760
|
+
});
|
|
4196
3761
|
} catch {
|
|
4197
3762
|
// Best-effort; the terminal update still carries the final logs.
|
|
4198
3763
|
}
|
|
@@ -4214,7 +3779,7 @@ async function executeRunRequest(
|
|
|
4214
3779
|
emit(event);
|
|
4215
3780
|
};
|
|
4216
3781
|
|
|
4217
|
-
const ctx =
|
|
3782
|
+
const ctx = createMinimalWorkerCtx(
|
|
4218
3783
|
req,
|
|
4219
3784
|
wrappedEmit,
|
|
4220
3785
|
env,
|
|
@@ -4222,104 +3787,43 @@ async function executeRunRequest(
|
|
|
4222
3787
|
abortSignal,
|
|
4223
3788
|
);
|
|
4224
3789
|
try {
|
|
4225
|
-
const playStartedAt = nowMs();
|
|
4226
3790
|
const result = await (
|
|
4227
3791
|
playFn as (
|
|
4228
3792
|
ctx: unknown,
|
|
4229
3793
|
input: Record<string, unknown>,
|
|
4230
3794
|
) => Promise<unknown>
|
|
4231
3795
|
)(ctx, req.runtimeInput);
|
|
4232
|
-
recordDynamicWorkerPerfTrace({
|
|
4233
|
-
env,
|
|
4234
|
-
runId: req.runId,
|
|
4235
|
-
phase: 'dynamic_worker.play_fn',
|
|
4236
|
-
ms: nowMs() - playStartedAt,
|
|
4237
|
-
graphHash: req.graphHash ?? null,
|
|
4238
|
-
waitUntil: options?.waitUntil,
|
|
4239
|
-
});
|
|
4240
3796
|
const serializedResult = serializePlayReturnValue(result);
|
|
4241
3797
|
if (options?.persistResultDatasets) {
|
|
4242
3798
|
await liveLogFlushInFlight.catch(() => undefined);
|
|
4243
|
-
const persistStartedAt = nowMs();
|
|
4244
3799
|
await persistResultDatasets(req, serializedResult);
|
|
4245
|
-
|
|
4246
|
-
|
|
4247
|
-
|
|
4248
|
-
|
|
4249
|
-
|
|
4250
|
-
|
|
4251
|
-
|
|
3800
|
+
const terminalResult = trimResultForStatus(serializedResult);
|
|
3801
|
+
await postRuntimeApiBestEffort(req.baseUrl, req.executorToken, {
|
|
3802
|
+
action: 'update_run_status',
|
|
3803
|
+
playId: req.runId,
|
|
3804
|
+
status: 'completed',
|
|
3805
|
+
result: terminalResult,
|
|
3806
|
+
runtimeBackend: 'cf_workflows_dynamic_worker',
|
|
3807
|
+
liveLogs,
|
|
3808
|
+
lastCheckpointAt: nowMs(),
|
|
4252
3809
|
});
|
|
4253
|
-
|
|
4254
|
-
|
|
4255
|
-
|
|
4256
|
-
|
|
4257
|
-
req,
|
|
4258
|
-
transport,
|
|
4259
|
-
success: true,
|
|
4260
|
-
actionEstimate: 4,
|
|
4261
|
-
});
|
|
4262
|
-
if (billingCapMustBlockTerminal) {
|
|
4263
|
-
await finalizeBilling();
|
|
4264
|
-
}
|
|
4265
|
-
const terminalResult = trimResultForStatus(serializedResult) as
|
|
4266
|
-
| Record<string, unknown>
|
|
4267
|
-
| null;
|
|
4268
|
-
const terminalStartedAt = nowMs();
|
|
4269
|
-
await transport.postRuntimeApiBestEffort(
|
|
4270
|
-
runtimeRunActions.updateStatus({
|
|
4271
|
-
playId: req.runId,
|
|
4272
|
-
status: 'completed',
|
|
4273
|
-
result: terminalResult,
|
|
4274
|
-
runtimeBackend: 'cf_workflows_dynamic_worker',
|
|
4275
|
-
liveLogs,
|
|
4276
|
-
lastCheckpointAt: nowMs(),
|
|
4277
|
-
}),
|
|
4278
|
-
);
|
|
4279
|
-
recordDynamicWorkerPerfTrace({
|
|
4280
|
-
env,
|
|
4281
|
-
runId: req.runId,
|
|
4282
|
-
phase: 'dynamic_worker.terminal_status_update',
|
|
4283
|
-
ms: nowMs() - terminalStartedAt,
|
|
4284
|
-
graphHash: req.graphHash ?? null,
|
|
4285
|
-
waitUntil: options?.waitUntil,
|
|
3810
|
+
await finalizeWorkerComputeBilling({
|
|
3811
|
+
req,
|
|
3812
|
+
success: true,
|
|
3813
|
+
actionEstimate: 4,
|
|
4286
3814
|
});
|
|
4287
|
-
if (!billingCapMustBlockTerminal) {
|
|
4288
|
-
const finalizeBillingPromise = finalizeBilling();
|
|
4289
|
-
if (options?.waitUntil) {
|
|
4290
|
-
options.waitUntil(
|
|
4291
|
-
finalizeBillingPromise.catch((finalizeError) => {
|
|
4292
|
-
console.error(
|
|
4293
|
-
`[play-harness] non-fatal compute billing finalize failed runId=${req.runId}: ${
|
|
4294
|
-
finalizeError instanceof Error
|
|
4295
|
-
? finalizeError.message
|
|
4296
|
-
: String(finalizeError)
|
|
4297
|
-
}`,
|
|
4298
|
-
);
|
|
4299
|
-
}),
|
|
4300
|
-
);
|
|
4301
|
-
} else {
|
|
4302
|
-
await finalizeBillingPromise;
|
|
4303
|
-
}
|
|
4304
|
-
}
|
|
4305
|
-
}
|
|
4306
|
-
if (options?.signalParentTerminal !== false) {
|
|
4307
|
-
await transport
|
|
4308
|
-
.signalParentTerminal({
|
|
4309
|
-
status: 'completed',
|
|
4310
|
-
result: trimResultForStatus(serializedResult) as Record<
|
|
4311
|
-
string,
|
|
4312
|
-
unknown
|
|
4313
|
-
>,
|
|
4314
|
-
})
|
|
4315
|
-
.catch((error) => {
|
|
4316
|
-
console.error(
|
|
4317
|
-
`[play-harness] non-fatal parent completion signal failed runId=${req.runId}: ${
|
|
4318
|
-
error instanceof Error ? error.message : String(error)
|
|
4319
|
-
}`,
|
|
4320
|
-
);
|
|
4321
|
-
});
|
|
4322
3815
|
}
|
|
3816
|
+
await signalParentPlayTerminal({
|
|
3817
|
+
req,
|
|
3818
|
+
status: 'completed',
|
|
3819
|
+
result: trimResultForStatus(serializedResult) as Record<string, unknown>,
|
|
3820
|
+
}).catch((error) => {
|
|
3821
|
+
console.error(
|
|
3822
|
+
`[play-harness] non-fatal parent completion signal failed runId=${req.runId}: ${
|
|
3823
|
+
error instanceof Error ? error.message : String(error)
|
|
3824
|
+
}`,
|
|
3825
|
+
);
|
|
3826
|
+
});
|
|
4323
3827
|
return {
|
|
4324
3828
|
result: serializedResult,
|
|
4325
3829
|
outputRows: inferOutputRows(serializedResult),
|
|
@@ -4337,23 +3841,20 @@ async function executeRunRequest(
|
|
|
4337
3841
|
const message = error instanceof Error ? error.message : String(error);
|
|
4338
3842
|
if (options?.persistResultDatasets) {
|
|
4339
3843
|
await liveLogFlushInFlight.catch(() => undefined);
|
|
4340
|
-
await
|
|
4341
|
-
|
|
4342
|
-
|
|
4343
|
-
|
|
4344
|
-
|
|
4345
|
-
|
|
4346
|
-
|
|
4347
|
-
|
|
4348
|
-
|
|
4349
|
-
|
|
4350
|
-
const finalizeBillingPromise = finalizeWorkerComputeBilling({
|
|
3844
|
+
await postRuntimeApiBestEffort(req.baseUrl, req.executorToken, {
|
|
3845
|
+
action: 'update_run_status',
|
|
3846
|
+
playId: req.runId,
|
|
3847
|
+
status: aborted ? 'cancelled' : 'failed',
|
|
3848
|
+
error: message,
|
|
3849
|
+
runtimeBackend: 'cf_workflows_dynamic_worker',
|
|
3850
|
+
liveLogs,
|
|
3851
|
+
lastCheckpointAt: nowMs(),
|
|
3852
|
+
});
|
|
3853
|
+
await finalizeWorkerComputeBilling({
|
|
4351
3854
|
req,
|
|
4352
|
-
transport,
|
|
4353
3855
|
success: false,
|
|
4354
3856
|
actionEstimate: 4,
|
|
4355
|
-
})
|
|
4356
|
-
const logFinalizeError = (finalizeError: unknown) => {
|
|
3857
|
+
}).catch((finalizeError) => {
|
|
4357
3858
|
console.error(
|
|
4358
3859
|
`[play-harness] non-fatal compute billing finalize failed runId=${req.runId}: ${
|
|
4359
3860
|
finalizeError instanceof Error
|
|
@@ -4361,21 +3862,13 @@ async function executeRunRequest(
|
|
|
4361
3862
|
: String(finalizeError)
|
|
4362
3863
|
}`,
|
|
4363
3864
|
);
|
|
4364
|
-
};
|
|
4365
|
-
if (options?.waitUntil) {
|
|
4366
|
-
options.waitUntil(finalizeBillingPromise.catch(logFinalizeError));
|
|
4367
|
-
} else {
|
|
4368
|
-
await finalizeBillingPromise.catch(logFinalizeError);
|
|
4369
|
-
}
|
|
4370
|
-
}
|
|
4371
|
-
if (options?.signalParentTerminal !== false) {
|
|
4372
|
-
await transport
|
|
4373
|
-
.signalParentTerminal({
|
|
4374
|
-
status: aborted ? 'cancelled' : 'failed',
|
|
4375
|
-
error: message,
|
|
4376
|
-
})
|
|
4377
|
-
.catch(() => null);
|
|
3865
|
+
});
|
|
4378
3866
|
}
|
|
3867
|
+
await signalParentPlayTerminal({
|
|
3868
|
+
req,
|
|
3869
|
+
status: aborted ? 'cancelled' : 'failed',
|
|
3870
|
+
error: message,
|
|
3871
|
+
}).catch(() => null);
|
|
4379
3872
|
throw error;
|
|
4380
3873
|
}
|
|
4381
3874
|
}
|
|
@@ -4397,39 +3890,36 @@ function extractMaxCreditsPerRun(contractSnapshot: unknown): number | null {
|
|
|
4397
3890
|
|
|
4398
3891
|
async function finalizeWorkerComputeBilling(input: {
|
|
4399
3892
|
req: RunRequest;
|
|
4400
|
-
transport?: WorkerRuntimeTransport;
|
|
4401
3893
|
success: boolean;
|
|
4402
3894
|
actionEstimate: number;
|
|
4403
3895
|
}): Promise<void> {
|
|
4404
3896
|
const maxCreditsPerRun = extractMaxCreditsPerRun(input.req.contractSnapshot);
|
|
4405
|
-
|
|
4406
|
-
|
|
4407
|
-
|
|
4408
|
-
|
|
4409
|
-
|
|
4410
|
-
|
|
4411
|
-
|
|
4412
|
-
|
|
4413
|
-
|
|
4414
|
-
|
|
4415
|
-
|
|
4416
|
-
|
|
4417
|
-
|
|
4418
|
-
|
|
4419
|
-
|
|
4420
|
-
|
|
4421
|
-
(
|
|
4422
|
-
|
|
4423
|
-
|
|
4424
|
-
|
|
4425
|
-
|
|
4426
|
-
|
|
4427
|
-
|
|
4428
|
-
actionEstimate: input.actionEstimate,
|
|
4429
|
-
},
|
|
3897
|
+
await postRuntimeApi(input.req.baseUrl, input.req.executorToken, {
|
|
3898
|
+
action: 'compute_billing_finalize',
|
|
3899
|
+
sessionId: input.req.runId,
|
|
3900
|
+
orgId: input.req.orgId,
|
|
3901
|
+
operation: 'workflow_run',
|
|
3902
|
+
status: input.success ? 'completed' : 'error',
|
|
3903
|
+
workflowId: input.req.runId,
|
|
3904
|
+
runId: input.req.runId,
|
|
3905
|
+
...(maxCreditsPerRun !== null ? { maxCreditsPerRun } : {}),
|
|
3906
|
+
finalItem: {
|
|
3907
|
+
itemId: `cloudflare-workflows:${input.req.runId}`,
|
|
3908
|
+
source: 'cloudflare_workflows',
|
|
3909
|
+
unit: 'action',
|
|
3910
|
+
units: Math.max(1, Math.ceil(input.actionEstimate)),
|
|
3911
|
+
providerCostUsd: Number(
|
|
3912
|
+
(
|
|
3913
|
+
Math.max(1, Math.ceil(input.actionEstimate)) *
|
|
3914
|
+
(50 / 1_000_000)
|
|
3915
|
+
).toFixed(12),
|
|
3916
|
+
),
|
|
3917
|
+
metadata: {
|
|
3918
|
+
workflowId: input.req.runId,
|
|
3919
|
+
actionEstimate: input.actionEstimate,
|
|
4430
3920
|
},
|
|
4431
|
-
}
|
|
4432
|
-
);
|
|
3921
|
+
},
|
|
3922
|
+
});
|
|
4433
3923
|
}
|
|
4434
3924
|
|
|
4435
3925
|
function isInlineCsv(
|
|
@@ -4593,7 +4083,7 @@ function trimResultShape(value: unknown): unknown {
|
|
|
4593
4083
|
const out: Record<string, unknown> = {};
|
|
4594
4084
|
for (const [key, child] of Object.entries(value)) {
|
|
4595
4085
|
if (key === 'preview' && Array.isArray(child) && value.kind === 'dataset') {
|
|
4596
|
-
out[key] = child.slice(0,
|
|
4086
|
+
out[key] = child.slice(0, 5).map(trimResultShape);
|
|
4597
4087
|
} else {
|
|
4598
4088
|
out[key] = trimResultShape(child);
|
|
4599
4089
|
}
|
|
@@ -4631,18 +4121,21 @@ function serializeValue(value: unknown, depth: number): unknown {
|
|
|
4631
4121
|
? (value as unknown as { __deeplineCacheSummary: string })
|
|
4632
4122
|
.__deeplineCacheSummary
|
|
4633
4123
|
: null;
|
|
4634
|
-
const
|
|
4124
|
+
const previewRows = value
|
|
4125
|
+
.slice(0, 5)
|
|
4635
4126
|
.map((row) => serializeValue(row, depth + 1))
|
|
4636
4127
|
.filter(isRecord);
|
|
4637
4128
|
if (tableNamespace && datasetId) {
|
|
4638
|
-
const columns = inferColumns(
|
|
4129
|
+
const columns = inferColumns(
|
|
4130
|
+
value.map((row) => serializeValue(row, depth + 1)).filter(isRecord),
|
|
4131
|
+
);
|
|
4639
4132
|
return {
|
|
4640
4133
|
kind: 'dataset' as const,
|
|
4641
4134
|
datasetKind,
|
|
4642
4135
|
datasetId,
|
|
4643
4136
|
count: datasetCount,
|
|
4644
4137
|
columns,
|
|
4645
|
-
preview:
|
|
4138
|
+
preview: previewRows,
|
|
4646
4139
|
tableNamespace,
|
|
4647
4140
|
...(cacheSummary ? { cacheSummary } : {}),
|
|
4648
4141
|
};
|
|
@@ -4729,18 +4222,16 @@ export class TenantWorkflow extends WorkflowEntrypoint<
|
|
|
4729
4222
|
): Promise<unknown> {
|
|
4730
4223
|
const req = runRequestFromWorkflowParams(event.payload);
|
|
4731
4224
|
const runPrefix = `[deepline-run:${req.runId}]`;
|
|
4732
|
-
|
|
4733
|
-
|
|
4734
|
-
|
|
4735
|
-
|
|
4736
|
-
|
|
4737
|
-
|
|
4738
|
-
|
|
4739
|
-
|
|
4740
|
-
? (promise) => this.ctx.waitUntil(promise)
|
|
4741
|
-
: undefined,
|
|
4742
|
-
});
|
|
4225
|
+
// DEBUG: confirm TenantWorkflow.run was invoked at all. If this log
|
|
4226
|
+
// appears in tail (parent's tail consumer captures harness logs by
|
|
4227
|
+
// the deepline-run prefix in flushTailRunLogs), the throw is
|
|
4228
|
+
// somewhere inside executeRunRequest. If it doesn't appear, the
|
|
4229
|
+
// throw is in the framework wrapper between the loader and run().
|
|
4230
|
+
console.log(
|
|
4231
|
+
`${runPrefix} TenantWorkflow.run entered baseUrl=${req.baseUrl}`,
|
|
4232
|
+
);
|
|
4743
4233
|
captureCoordinatorBinding(this.env);
|
|
4234
|
+
captureRuntimeApiBinding(this.env);
|
|
4744
4235
|
// Hand the harness service binding (if wired) to the SDK-side stub.
|
|
4745
4236
|
// Must run BEFORE any SDK call site that would reach into HARNESS,
|
|
4746
4237
|
// i.e. before user play code is invoked. Idempotent within a run.
|
|
@@ -4749,23 +4240,10 @@ export class TenantWorkflow extends WorkflowEntrypoint<
|
|
|
4749
4240
|
// same isolate). Awaited so the result is in the log before user code
|
|
4750
4241
|
// begins, but never throws — broken HARNESS at probe time doesn't
|
|
4751
4242
|
// block the play; real call-site errors do.
|
|
4752
|
-
const harnessProbeStartedAt = nowMs();
|
|
4753
4243
|
await probeHarnessOnce(this.env, runPrefix);
|
|
4754
|
-
recordDynamicWorkerPerfTrace({
|
|
4755
|
-
env: this.env,
|
|
4756
|
-
runId: req.runId,
|
|
4757
|
-
phase: 'dynamic_worker.harness_probe',
|
|
4758
|
-
ms: nowMs() - harnessProbeStartedAt,
|
|
4759
|
-
graphHash: req.graphHash ?? null,
|
|
4760
|
-
waitUntil:
|
|
4761
|
-
typeof this.ctx.waitUntil === 'function'
|
|
4762
|
-
? (promise) => this.ctx.waitUntil(promise)
|
|
4763
|
-
: undefined,
|
|
4764
|
-
});
|
|
4765
4244
|
const abortController = new AbortController();
|
|
4766
|
-
const executeStartedAt = nowMs();
|
|
4767
4245
|
try {
|
|
4768
|
-
|
|
4246
|
+
return (await executeRunRequest(
|
|
4769
4247
|
req,
|
|
4770
4248
|
this.env,
|
|
4771
4249
|
(runnerEvent) => {
|
|
@@ -4778,47 +4256,9 @@ export class TenantWorkflow extends WorkflowEntrypoint<
|
|
|
4778
4256
|
}
|
|
4779
4257
|
},
|
|
4780
4258
|
step,
|
|
4781
|
-
{
|
|
4782
|
-
persistResultDatasets: !req.playCallGovernance,
|
|
4783
|
-
abortController,
|
|
4784
|
-
waitUntil:
|
|
4785
|
-
typeof this.ctx.waitUntil === 'function'
|
|
4786
|
-
? (promise) => this.ctx.waitUntil(promise)
|
|
4787
|
-
: undefined,
|
|
4788
|
-
},
|
|
4259
|
+
{ persistResultDatasets: !req.playCallGovernance, abortController },
|
|
4789
4260
|
)) as Record<string, unknown>;
|
|
4790
|
-
recordDynamicWorkerPerfTrace({
|
|
4791
|
-
env: this.env,
|
|
4792
|
-
runId: req.runId,
|
|
4793
|
-
phase: 'dynamic_worker.execute_run_request',
|
|
4794
|
-
ms: nowMs() - executeStartedAt,
|
|
4795
|
-
graphHash: req.graphHash ?? null,
|
|
4796
|
-
extra: { ok: true },
|
|
4797
|
-
waitUntil:
|
|
4798
|
-
typeof this.ctx.waitUntil === 'function'
|
|
4799
|
-
? (promise) => this.ctx.waitUntil(promise)
|
|
4800
|
-
: undefined,
|
|
4801
|
-
});
|
|
4802
|
-
return output;
|
|
4803
4261
|
} catch (error) {
|
|
4804
|
-
recordDynamicWorkerPerfTrace({
|
|
4805
|
-
env: this.env,
|
|
4806
|
-
runId: req.runId,
|
|
4807
|
-
phase: 'dynamic_worker.execute_run_request',
|
|
4808
|
-
ms: nowMs() - executeStartedAt,
|
|
4809
|
-
graphHash: req.graphHash ?? null,
|
|
4810
|
-
extra: {
|
|
4811
|
-
ok: false,
|
|
4812
|
-
error:
|
|
4813
|
-
error instanceof Error
|
|
4814
|
-
? error.message.slice(0, 300)
|
|
4815
|
-
: String(error),
|
|
4816
|
-
},
|
|
4817
|
-
waitUntil:
|
|
4818
|
-
typeof this.ctx.waitUntil === 'function'
|
|
4819
|
-
? (promise) => this.ctx.waitUntil(promise)
|
|
4820
|
-
: undefined,
|
|
4821
|
-
});
|
|
4822
4262
|
// CF Workflows + the dynamic-workflows framework swallow the error
|
|
4823
4263
|
// message and surface only "internal error; reference = <id>" via
|
|
4824
4264
|
// instance.status(). The per-play Worker's console.error doesn't
|
|
@@ -4840,13 +4280,12 @@ export class TenantWorkflow extends WorkflowEntrypoint<
|
|
|
4840
4280
|
// so this callback is the ONLY way the real error message reaches the
|
|
4841
4281
|
// user via tail/SSE. Retry with backoff before giving up; if we drop
|
|
4842
4282
|
// it, the user is stuck staring at the opaque CF reference id.
|
|
4843
|
-
const errorPayload = JSON.stringify(
|
|
4844
|
-
|
|
4845
|
-
|
|
4846
|
-
|
|
4847
|
-
|
|
4848
|
-
|
|
4849
|
-
);
|
|
4283
|
+
const errorPayload = JSON.stringify({
|
|
4284
|
+
action: 'update_run_status',
|
|
4285
|
+
playId: req.runId,
|
|
4286
|
+
status: 'failed',
|
|
4287
|
+
error: `TenantWorkflow.run threw: ${detail.name ?? 'Error'}: ${detail.message}\n${detail.stack ?? ''}`,
|
|
4288
|
+
});
|
|
4850
4289
|
const backoffMs = [200, 500, 1500];
|
|
4851
4290
|
let lastCallbackError: unknown = null;
|
|
4852
4291
|
let delivered = false;
|
|
@@ -4903,80 +4342,17 @@ export class TenantWorkflow extends WorkflowEntrypoint<
|
|
|
4903
4342
|
}
|
|
4904
4343
|
}
|
|
4905
4344
|
|
|
4906
|
-
async function handleInlineRun(
|
|
4907
|
-
request: Request,
|
|
4908
|
-
env: WorkerEnv,
|
|
4909
|
-
): Promise<Response> {
|
|
4910
|
-
const parseStartedAt = nowMs();
|
|
4911
|
-
const req = (await request.json()) as RunRequest;
|
|
4912
|
-
const runPrefix = `[deepline-run:${req.runId}]`;
|
|
4913
|
-
const events: RunnerEvent[] = [];
|
|
4914
|
-
captureCoordinatorBinding(env);
|
|
4915
|
-
captureHarnessBinding(env);
|
|
4916
|
-
await probeHarnessOnce(env, runPrefix);
|
|
4917
|
-
try {
|
|
4918
|
-
const output = await executeRunRequest(
|
|
4919
|
-
req,
|
|
4920
|
-
env,
|
|
4921
|
-
(runnerEvent) => {
|
|
4922
|
-
events.push(runnerEvent);
|
|
4923
|
-
if (runnerEvent.type === 'log') {
|
|
4924
|
-
console.log(`${runPrefix} ${runnerEvent.message}`);
|
|
4925
|
-
} else if (runnerEvent.type === 'error') {
|
|
4926
|
-
console.error(
|
|
4927
|
-
`${runPrefix} ${runnerEvent.message}${runnerEvent.stack ? `\n${runnerEvent.stack}` : ''}`,
|
|
4928
|
-
);
|
|
4929
|
-
}
|
|
4930
|
-
},
|
|
4931
|
-
undefined,
|
|
4932
|
-
{
|
|
4933
|
-
persistResultDatasets: false,
|
|
4934
|
-
signalParentTerminal: false,
|
|
4935
|
-
},
|
|
4936
|
-
);
|
|
4937
|
-
return Response.json({
|
|
4938
|
-
status: 'completed',
|
|
4939
|
-
result: output.result,
|
|
4940
|
-
outputRows: output.outputRows,
|
|
4941
|
-
durationMs: output.durationMs,
|
|
4942
|
-
parseMs: nowMs() - parseStartedAt,
|
|
4943
|
-
events,
|
|
4944
|
-
});
|
|
4945
|
-
} catch (error) {
|
|
4946
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
4947
|
-
const stack = error instanceof Error ? error.stack : undefined;
|
|
4948
|
-
events.push({
|
|
4949
|
-
type: 'error',
|
|
4950
|
-
message,
|
|
4951
|
-
...(stack ? { stack } : {}),
|
|
4952
|
-
ts: nowMs(),
|
|
4953
|
-
});
|
|
4954
|
-
return Response.json(
|
|
4955
|
-
{
|
|
4956
|
-
status: 'failed',
|
|
4957
|
-
error: { message, ...(stack ? { stack } : {}) },
|
|
4958
|
-
parseMs: nowMs() - parseStartedAt,
|
|
4959
|
-
events,
|
|
4960
|
-
},
|
|
4961
|
-
{ status: 200 },
|
|
4962
|
-
);
|
|
4963
|
-
}
|
|
4964
|
-
}
|
|
4965
|
-
|
|
4966
4345
|
const workerEntrypoint = {
|
|
4967
4346
|
async fetch(request: Request, env: WorkerEnv): Promise<Response> {
|
|
4968
4347
|
const url = new URL(request.url);
|
|
4969
4348
|
if (request.method === 'POST' && url.pathname === '/start') {
|
|
4970
|
-
const startTotalStartedAt = Date.now();
|
|
4971
4349
|
if (!env.WORKFLOWS) {
|
|
4972
4350
|
return new Response('missing WORKFLOWS binding', { status: 500 });
|
|
4973
4351
|
}
|
|
4974
|
-
const parseStartedAt = Date.now();
|
|
4975
4352
|
const body = (await request.json().catch(() => null)) as {
|
|
4976
4353
|
id?: string;
|
|
4977
4354
|
payload?: Record<string, unknown>;
|
|
4978
4355
|
} | null;
|
|
4979
|
-
const parseMs = Date.now() - parseStartedAt;
|
|
4980
4356
|
if (!body?.id || !isRecord(body.payload)) {
|
|
4981
4357
|
return new Response('invalid workflow start body', { status: 400 });
|
|
4982
4358
|
}
|
|
@@ -4992,16 +4368,6 @@ const workerEntrypoint = {
|
|
|
4992
4368
|
typeof body.payload.graphHash === 'string'
|
|
4993
4369
|
? body.payload.graphHash
|
|
4994
4370
|
: null;
|
|
4995
|
-
console.log(
|
|
4996
|
-
`[perf-trace] ${JSON.stringify({
|
|
4997
|
-
ts: Date.now(),
|
|
4998
|
-
source: 'dynamic_worker',
|
|
4999
|
-
runId,
|
|
5000
|
-
phase: 'dynamic_worker.start_request_parse',
|
|
5001
|
-
ms: parseMs,
|
|
5002
|
-
...(graphHash ? { graphHash } : {}),
|
|
5003
|
-
})}`,
|
|
5004
|
-
);
|
|
5005
4371
|
console.log(
|
|
5006
4372
|
`[perf-trace] ${JSON.stringify({
|
|
5007
4373
|
ts: Date.now(),
|
|
@@ -5016,18 +4382,13 @@ const workerEntrypoint = {
|
|
|
5016
4382
|
id: instance.id,
|
|
5017
4383
|
status: 'submitted',
|
|
5018
4384
|
timingsMs: {
|
|
5019
|
-
startRequestParse: parseMs,
|
|
5020
4385
|
workflowCreate: workflowCreateMs,
|
|
5021
|
-
startTotal: Date.now() - startTotalStartedAt,
|
|
5022
4386
|
},
|
|
5023
4387
|
});
|
|
5024
4388
|
}
|
|
5025
4389
|
if (request.method === 'POST' && url.pathname === '/run') {
|
|
5026
4390
|
return handleRun(request, env);
|
|
5027
4391
|
}
|
|
5028
|
-
if (request.method === 'POST' && url.pathname === '/run-inline') {
|
|
5029
|
-
return handleInlineRun(request, env);
|
|
5030
|
-
}
|
|
5031
4392
|
if (request.method === 'GET' && url.pathname === '/health') {
|
|
5032
4393
|
return new Response('ok', { status: 200 });
|
|
5033
4394
|
}
|