deepline 0.1.212 → 0.1.213
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/apps/play-runner-workers/src/entry.ts +167 -329
- package/dist/bundling-sources/sdk/src/index.ts +2 -0
- package/dist/bundling-sources/sdk/src/play.ts +8 -1
- package/dist/bundling-sources/sdk/src/plays/harness-stub.ts +11 -1
- package/dist/bundling-sources/sdk/src/release.ts +2 -2
- package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +49 -0
- package/dist/bundling-sources/shared_libs/play-runtime/child-execution-placement.ts +14 -0
- package/dist/bundling-sources/shared_libs/play-runtime/child-execution-strategy.ts +57 -23
- package/dist/bundling-sources/shared_libs/play-runtime/completed-receipt-cache.ts +166 -0
- package/dist/bundling-sources/shared_libs/play-runtime/context.ts +142 -101
- package/dist/bundling-sources/shared_libs/play-runtime/ctx-contract.ts +2 -0
- package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +30 -1
- package/dist/bundling-sources/shared_libs/play-runtime/durable-receipt-execution.ts +160 -2
- package/dist/bundling-sources/shared_libs/play-runtime/gateway-auth-session.ts +93 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-call-execution.ts +1 -0
- package/dist/bundling-sources/shared_libs/play-runtime/receipt-completion-sink.ts +16 -1
- package/dist/bundling-sources/shared_libs/play-runtime/receipt-state-sink.ts +10 -0
- package/dist/bundling-sources/shared_libs/play-runtime/resource-governor.ts +80 -3
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-session-execution.ts +94 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts +54 -1
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +477 -85
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-postgres-admission.ts +162 -0
- package/dist/bundling-sources/shared_libs/play-runtime/sheet-attempt-sql.ts +16 -17
- package/dist/bundling-sources/shared_libs/play-runtime/work-receipts.ts +1 -0
- package/dist/bundling-sources/shared_libs/plays/static-pipeline.ts +2 -4
- package/dist/cli/index.js +2 -2
- package/dist/cli/index.mjs +2 -2
- package/dist/{compiler-manifest-j6z8qzpd.d.mts → compiler-manifest-BhgZ23A4.d.mts} +1 -0
- package/dist/{compiler-manifest-j6z8qzpd.d.ts → compiler-manifest-BhgZ23A4.d.ts} +1 -0
- package/dist/index.d.mts +8 -5
- package/dist/index.d.ts +8 -5
- package/dist/index.js +2 -2
- package/dist/index.mjs +2 -2
- package/dist/plays/bundle-play-file.d.mts +2 -2
- package/dist/plays/bundle-play-file.d.ts +2 -2
- package/package.json +1 -1
|
@@ -198,12 +198,12 @@ import {
|
|
|
198
198
|
PLAY_RUNTIME_WORK_RECEIPT_LEASE_TTL_MS,
|
|
199
199
|
runtimeLeaseHeartbeatIntervalMs,
|
|
200
200
|
} from '../../../shared_libs/play-runtime/lease-policy';
|
|
201
|
-
import { RuntimeSheetRowsBlockedError } from '../../../shared_libs/play-runtime/runtime-sheet-errors';
|
|
202
201
|
import {
|
|
203
202
|
resolveRuntimeMapRowAdmission,
|
|
204
203
|
runtimeMapJsonByteLength,
|
|
205
204
|
} from '../../../shared_libs/play-runtime/map-memory-limits';
|
|
206
205
|
import { createSerialTaskQueue } from '../../../shared_libs/play-runtime/serial-task-queue';
|
|
206
|
+
import { resolveChildExecutionPlacement as resolveChildExecutionStrategy } from '../../../shared_libs/play-runtime/child-execution-placement';
|
|
207
207
|
// The harness stub forwards leaf calls (validation, runtime-api HTTP) into
|
|
208
208
|
// the long-lived Play Harness Worker via env.HARNESS. We import the
|
|
209
209
|
// `setHarnessBinding` setter eagerly so it's available the moment
|
|
@@ -216,12 +216,10 @@ import { createSerialTaskQueue } from '../../../shared_libs/play-runtime/serial-
|
|
|
216
216
|
// re-bundle harness internals into per-play. Keep that in mind.
|
|
217
217
|
import {
|
|
218
218
|
harnessPersistCompletedSheetRows,
|
|
219
|
-
harnessHeartbeatSheetAttempt,
|
|
220
219
|
harnessReadSheetDatasetRowKeys,
|
|
221
220
|
harnessReadSheetDatasetRows,
|
|
222
221
|
harnessReadStagedFileChunk,
|
|
223
222
|
harnessReleaseRuntimeReceipts,
|
|
224
|
-
harnessReleaseSheetAttempt,
|
|
225
223
|
harnessStartSheetDataset,
|
|
226
224
|
setHarnessBinding,
|
|
227
225
|
} from '../../../sdk/src/plays/harness-stub';
|
|
@@ -1263,136 +1261,6 @@ function makeRuntimeSheetAttempt(input: {
|
|
|
1263
1261
|
};
|
|
1264
1262
|
}
|
|
1265
1263
|
|
|
1266
|
-
class RuntimeSheetAttemptLeaseLostError extends Error {
|
|
1267
|
-
readonly rowKeys: string[];
|
|
1268
|
-
readonly runId: string;
|
|
1269
|
-
readonly attemptId: string;
|
|
1270
|
-
readonly tableNamespace: string;
|
|
1271
|
-
|
|
1272
|
-
constructor(input: {
|
|
1273
|
-
rowKeys: string[];
|
|
1274
|
-
runId: string;
|
|
1275
|
-
attemptId: string;
|
|
1276
|
-
tableNamespace: string;
|
|
1277
|
-
cause?: unknown;
|
|
1278
|
-
}) {
|
|
1279
|
-
const rowSummary =
|
|
1280
|
-
input.rowKeys.length === 1
|
|
1281
|
-
? input.rowKeys[0]!
|
|
1282
|
-
: `${input.rowKeys.length} rows`;
|
|
1283
|
-
super(
|
|
1284
|
-
`Runtime sheet attempt ${input.attemptId} lost ownership of ${rowSummary} in ${input.tableNamespace}.`,
|
|
1285
|
-
{ cause: input.cause },
|
|
1286
|
-
);
|
|
1287
|
-
this.name = 'RuntimeSheetAttemptLeaseLostError';
|
|
1288
|
-
this.rowKeys = input.rowKeys;
|
|
1289
|
-
this.runId = input.runId;
|
|
1290
|
-
this.attemptId = input.attemptId;
|
|
1291
|
-
this.tableNamespace = input.tableNamespace;
|
|
1292
|
-
}
|
|
1293
|
-
}
|
|
1294
|
-
|
|
1295
|
-
async function executeWithRuntimeSheetAttemptHeartbeat<T>(input: {
|
|
1296
|
-
req: RunRequest;
|
|
1297
|
-
tableNamespace: string;
|
|
1298
|
-
sheetContract: PlaySheetContract;
|
|
1299
|
-
attemptId?: string | null;
|
|
1300
|
-
attemptOwnerRunId?: string | null;
|
|
1301
|
-
attemptSeq?: number | null;
|
|
1302
|
-
rowKeys: string[];
|
|
1303
|
-
execute: () => Promise<T> | T;
|
|
1304
|
-
}): Promise<T> {
|
|
1305
|
-
const attemptId = input.attemptId?.trim();
|
|
1306
|
-
const rowKeys = [
|
|
1307
|
-
...new Set(input.rowKeys.map((key) => key.trim()).filter(Boolean)),
|
|
1308
|
-
];
|
|
1309
|
-
if (!attemptId || rowKeys.length === 0) {
|
|
1310
|
-
return await input.execute();
|
|
1311
|
-
}
|
|
1312
|
-
const heartbeatIntervalMs = runtimePolicyHeartbeatIntervalMs(
|
|
1313
|
-
input.req,
|
|
1314
|
-
runtimeSheetAttemptLeaseTtlMs(input.req) ??
|
|
1315
|
-
PLAY_RUNTIME_SHEET_ATTEMPT_LEASE_TTL_MS,
|
|
1316
|
-
);
|
|
1317
|
-
|
|
1318
|
-
let stopped = false;
|
|
1319
|
-
let timeout: ReturnType<typeof setTimeout> | null = null;
|
|
1320
|
-
let rejectLeaseLost!: (error: RuntimeSheetAttemptLeaseLostError) => void;
|
|
1321
|
-
const leaseLost = new Promise<never>((_, reject) => {
|
|
1322
|
-
rejectLeaseLost = reject;
|
|
1323
|
-
});
|
|
1324
|
-
|
|
1325
|
-
const stop = () => {
|
|
1326
|
-
stopped = true;
|
|
1327
|
-
if (timeout !== null) {
|
|
1328
|
-
clearTimeout(timeout);
|
|
1329
|
-
timeout = null;
|
|
1330
|
-
}
|
|
1331
|
-
};
|
|
1332
|
-
|
|
1333
|
-
const heartbeatOnce = async () => {
|
|
1334
|
-
const result = await harnessHeartbeatSheetAttempt({
|
|
1335
|
-
...runtimeSheetSessionScope(input.req),
|
|
1336
|
-
tableNamespace: input.tableNamespace,
|
|
1337
|
-
sheetContract: input.sheetContract,
|
|
1338
|
-
runId: input.req.runId,
|
|
1339
|
-
attemptId,
|
|
1340
|
-
attemptOwnerRunId: input.attemptOwnerRunId ?? input.req.runId,
|
|
1341
|
-
attemptSeq: input.attemptSeq ?? input.req.runAttempt ?? 0,
|
|
1342
|
-
attemptLeaseTtlMs: runtimeSheetAttemptLeaseTtlMs(input.req),
|
|
1343
|
-
keys: rowKeys,
|
|
1344
|
-
});
|
|
1345
|
-
const renewed = new Set(result.renewedKeys);
|
|
1346
|
-
const lostKeys = rowKeys.filter((key) => !renewed.has(key));
|
|
1347
|
-
if (lostKeys.length > 0) {
|
|
1348
|
-
throw new RuntimeSheetAttemptLeaseLostError({
|
|
1349
|
-
rowKeys: lostKeys,
|
|
1350
|
-
runId: input.req.runId,
|
|
1351
|
-
attemptId,
|
|
1352
|
-
tableNamespace: input.tableNamespace,
|
|
1353
|
-
});
|
|
1354
|
-
}
|
|
1355
|
-
};
|
|
1356
|
-
|
|
1357
|
-
const schedule = () => {
|
|
1358
|
-
timeout = setTimeout(() => {
|
|
1359
|
-
void (async () => {
|
|
1360
|
-
try {
|
|
1361
|
-
await heartbeatOnce();
|
|
1362
|
-
} catch (error) {
|
|
1363
|
-
stop();
|
|
1364
|
-
if (error instanceof RuntimeSheetAttemptLeaseLostError) {
|
|
1365
|
-
rejectLeaseLost(error);
|
|
1366
|
-
} else {
|
|
1367
|
-
rejectLeaseLost(
|
|
1368
|
-
new RuntimeSheetAttemptLeaseLostError({
|
|
1369
|
-
rowKeys,
|
|
1370
|
-
runId: input.req.runId,
|
|
1371
|
-
attemptId,
|
|
1372
|
-
tableNamespace: input.tableNamespace,
|
|
1373
|
-
cause: error,
|
|
1374
|
-
}),
|
|
1375
|
-
);
|
|
1376
|
-
}
|
|
1377
|
-
return;
|
|
1378
|
-
}
|
|
1379
|
-
if (!stopped) {
|
|
1380
|
-
schedule();
|
|
1381
|
-
}
|
|
1382
|
-
})();
|
|
1383
|
-
}, heartbeatIntervalMs);
|
|
1384
|
-
};
|
|
1385
|
-
|
|
1386
|
-
schedule();
|
|
1387
|
-
const execution = Promise.resolve().then(input.execute);
|
|
1388
|
-
execution.catch(() => {});
|
|
1389
|
-
try {
|
|
1390
|
-
return await Promise.race([execution, leaseLost]);
|
|
1391
|
-
} finally {
|
|
1392
|
-
stop();
|
|
1393
|
-
}
|
|
1394
|
-
}
|
|
1395
|
-
|
|
1396
1264
|
/**
|
|
1397
1265
|
* Strip credentials and JWT-shaped tokens from any string before it lands in
|
|
1398
1266
|
* a log buffer or upstream error message. The harness routinely echoes
|
|
@@ -2525,10 +2393,7 @@ function isRunFatalWorkerRowError(error: unknown): boolean {
|
|
|
2525
2393
|
isRowIsolationExemptError(error) ||
|
|
2526
2394
|
isRuntimeReceiptPersistenceError(error) ||
|
|
2527
2395
|
isRuntimePersistenceCircuitOpenError(error) ||
|
|
2528
|
-
error instanceof
|
|
2529
|
-
error instanceof RuntimeLeaseLostError ||
|
|
2530
|
-
(error instanceof Error &&
|
|
2531
|
-
error.name === 'RuntimeSheetAttemptLeaseLostError')
|
|
2396
|
+
error instanceof RuntimeLeaseLostError
|
|
2532
2397
|
);
|
|
2533
2398
|
}
|
|
2534
2399
|
|
|
@@ -3608,53 +3473,18 @@ function resolveSheetContractFromReq(
|
|
|
3608
3473
|
}
|
|
3609
3474
|
}
|
|
3610
3475
|
|
|
3611
|
-
/**
|
|
3612
|
-
* Release attempt-scoped leases after terminal settlement. Completed and failed
|
|
3613
|
-
* rows are durable; retaining their live fences only delays the next repair run.
|
|
3614
|
-
*/
|
|
3476
|
+
/** Release temporary work-receipt ownership after terminal settlement. */
|
|
3615
3477
|
async function releaseRuntimeLeasesOnSettlement(input: {
|
|
3616
3478
|
req: RunRequest;
|
|
3617
3479
|
leasedSheetNamespaces: Set<string>;
|
|
3618
3480
|
emit: (event: RunnerEvent) => void;
|
|
3619
3481
|
reason: 'completed' | 'run-fatal';
|
|
3620
3482
|
}): Promise<void> {
|
|
3621
|
-
const { req,
|
|
3483
|
+
const { req, emit, reason } = input;
|
|
3484
|
+
void input.leasedSheetNamespaces;
|
|
3622
3485
|
const reasonLabel =
|
|
3623
3486
|
reason === 'completed' ? 'completed settlement' : 'run-fatal teardown';
|
|
3624
3487
|
const scope = runtimeSheetSessionScope(req);
|
|
3625
|
-
for (const tableNamespace of leasedSheetNamespaces) {
|
|
3626
|
-
const sheetContract = resolveSheetContractFromReq(req, tableNamespace);
|
|
3627
|
-
if (!sheetContract) continue;
|
|
3628
|
-
try {
|
|
3629
|
-
const released = await harnessReleaseSheetAttempt({
|
|
3630
|
-
...scope,
|
|
3631
|
-
tableNamespace,
|
|
3632
|
-
sheetContract,
|
|
3633
|
-
runId: req.runId,
|
|
3634
|
-
attemptOwnerRunId: req.runId,
|
|
3635
|
-
attemptSeq: req.runAttempt ?? 0,
|
|
3636
|
-
});
|
|
3637
|
-
if (released.released > 0) {
|
|
3638
|
-
emit({
|
|
3639
|
-
type: 'log',
|
|
3640
|
-
level: 'info',
|
|
3641
|
-
message: `Released ${released.released} runtime sheet row lease(s) for ctx.dataset("${tableNamespace}") on ${reasonLabel}.`,
|
|
3642
|
-
ts: nowMs(),
|
|
3643
|
-
});
|
|
3644
|
-
}
|
|
3645
|
-
} catch (releaseError) {
|
|
3646
|
-
emit({
|
|
3647
|
-
type: 'log',
|
|
3648
|
-
level: 'warn',
|
|
3649
|
-
message: `Runtime sheet attempt release failed for ctx.dataset("${tableNamespace}") on ${reasonLabel} (rerun will fall back to lease-TTL expiry): ${
|
|
3650
|
-
releaseError instanceof Error
|
|
3651
|
-
? releaseError.message
|
|
3652
|
-
: String(releaseError)
|
|
3653
|
-
}`,
|
|
3654
|
-
ts: nowMs(),
|
|
3655
|
-
});
|
|
3656
|
-
}
|
|
3657
|
-
}
|
|
3658
3488
|
try {
|
|
3659
3489
|
const released = await harnessReleaseRuntimeReceipts({
|
|
3660
3490
|
...scope,
|
|
@@ -3729,6 +3559,7 @@ async function persistCompletedMapRows(input: {
|
|
|
3729
3559
|
attemptOwnerRunId?: string | null;
|
|
3730
3560
|
attemptExpiresAt?: string | null;
|
|
3731
3561
|
attemptSeq?: number | null;
|
|
3562
|
+
writeVersion?: number | null;
|
|
3732
3563
|
budgetMeter?: WorkerWorkBudgetMeter;
|
|
3733
3564
|
force?: boolean;
|
|
3734
3565
|
forceFailedRows?: boolean;
|
|
@@ -3738,6 +3569,8 @@ async function persistCompletedMapRows(input: {
|
|
|
3738
3569
|
visible: number;
|
|
3739
3570
|
retryWritten: number | null;
|
|
3740
3571
|
retryVisible: number | null;
|
|
3572
|
+
staleDropped: number;
|
|
3573
|
+
staleDroppedKeys: string[];
|
|
3741
3574
|
}> {
|
|
3742
3575
|
if (input.rows.length === 0) {
|
|
3743
3576
|
return {
|
|
@@ -3746,6 +3579,8 @@ async function persistCompletedMapRows(input: {
|
|
|
3746
3579
|
visible: 0,
|
|
3747
3580
|
retryWritten: null,
|
|
3748
3581
|
retryVisible: null,
|
|
3582
|
+
staleDropped: 0,
|
|
3583
|
+
staleDroppedKeys: [],
|
|
3749
3584
|
};
|
|
3750
3585
|
}
|
|
3751
3586
|
const { req, tableNamespace } = input;
|
|
@@ -3781,6 +3616,7 @@ async function persistCompletedMapRows(input: {
|
|
|
3781
3616
|
attemptOwnerRunId: input.attemptOwnerRunId ?? null,
|
|
3782
3617
|
attemptExpiresAt: input.attemptExpiresAt ?? null,
|
|
3783
3618
|
attemptSeq: input.attemptSeq ?? req.runAttempt ?? 0,
|
|
3619
|
+
writeVersion: input.writeVersion ?? null,
|
|
3784
3620
|
forceFailedRows: input.forceFailedRows === true,
|
|
3785
3621
|
};
|
|
3786
3622
|
input.budgetMeter?.count('sheet');
|
|
@@ -3814,24 +3650,8 @@ async function persistCompletedMapRows(input: {
|
|
|
3814
3650
|
return result.rows.length;
|
|
3815
3651
|
};
|
|
3816
3652
|
const result = await harnessPersistCompletedSheetRows(persistRequest);
|
|
3817
|
-
const
|
|
3818
|
-
const
|
|
3819
|
-
written: number,
|
|
3820
|
-
visible: number | null,
|
|
3821
|
-
) => {
|
|
3822
|
-
// Recovery finalization may include rows completed by an earlier attempt.
|
|
3823
|
-
// A complete terminal readback proves they were not lost.
|
|
3824
|
-
if (
|
|
3825
|
-
!hasAttemptFence ||
|
|
3826
|
-
written === rows.length ||
|
|
3827
|
-
(visible !== null && visible >= expectedVisibleRows)
|
|
3828
|
-
) {
|
|
3829
|
-
return;
|
|
3830
|
-
}
|
|
3831
|
-
throw new Error(
|
|
3832
|
-
`Runtime sheet attempt lost ownership for ${tableNamespace}: wrote ${written}/${rows.length}; run ${req.runId}; attempt ${input.attemptId}.`,
|
|
3833
|
-
);
|
|
3834
|
-
};
|
|
3653
|
+
const staleDropped = result.staleDropped ?? 0;
|
|
3654
|
+
const staleDroppedKeys = result.staleDroppedKeys ?? result.fencedKeys ?? [];
|
|
3835
3655
|
console.warn('[play-runner.persist_completed_map_rows.result]', {
|
|
3836
3656
|
runId: req.runId,
|
|
3837
3657
|
playName: req.playName,
|
|
@@ -3860,6 +3680,7 @@ async function persistCompletedMapRows(input: {
|
|
|
3860
3680
|
attemptOwnerRunId: input.attemptOwnerRunId ?? null,
|
|
3861
3681
|
attemptExpiresAt: input.attemptExpiresAt ?? null,
|
|
3862
3682
|
attemptSeq: input.attemptSeq ?? req.runAttempt ?? 0,
|
|
3683
|
+
writeVersion: input.writeVersion ?? null,
|
|
3863
3684
|
attemptLeaseTtlMs: runtimeSheetAttemptLeaseTtlMs(req),
|
|
3864
3685
|
inputOffset: 0,
|
|
3865
3686
|
});
|
|
@@ -3868,7 +3689,6 @@ async function persistCompletedMapRows(input: {
|
|
|
3868
3689
|
input.budgetMeter?.count('sheet');
|
|
3869
3690
|
retryWritten = retry.rowsWritten;
|
|
3870
3691
|
retryVisible = await readVisibleRowCount();
|
|
3871
|
-
assertAttemptStillOwnsTerminalWrites(retryWritten, retryVisible);
|
|
3872
3692
|
if (retryVisible >= expectedVisibleRows) {
|
|
3873
3693
|
return {
|
|
3874
3694
|
rows: rows.length,
|
|
@@ -3876,13 +3696,19 @@ async function persistCompletedMapRows(input: {
|
|
|
3876
3696
|
visible: visibleRows,
|
|
3877
3697
|
retryWritten,
|
|
3878
3698
|
retryVisible,
|
|
3699
|
+
staleDropped: staleDropped + (retry.staleDropped ?? 0),
|
|
3700
|
+
staleDroppedKeys: [
|
|
3701
|
+
...new Set([
|
|
3702
|
+
...staleDroppedKeys,
|
|
3703
|
+
...(retry.staleDroppedKeys ?? retry.fencedKeys ?? []),
|
|
3704
|
+
]),
|
|
3705
|
+
],
|
|
3879
3706
|
};
|
|
3880
3707
|
}
|
|
3881
3708
|
throw new Error(
|
|
3882
3709
|
`Runtime sheet persistence mismatch for ${tableNamespace}: wrote ${result.rowsWritten}/${rows.length}; visible ${visibleRows}/${expectedVisibleRows}; retry wrote ${retryWritten}/${rows.length}; retry visible ${retryVisible}/${expectedVisibleRows}; run ${req.runId}.`,
|
|
3883
3710
|
);
|
|
3884
3711
|
}
|
|
3885
|
-
assertAttemptStillOwnsTerminalWrites(result.rowsWritten, visibleRows);
|
|
3886
3712
|
}
|
|
3887
3713
|
if (result.rowsWritten !== rows.length && visibleRows < expectedVisibleRows) {
|
|
3888
3714
|
throw new Error(
|
|
@@ -3895,6 +3721,8 @@ async function persistCompletedMapRows(input: {
|
|
|
3895
3721
|
visible: visibleRows,
|
|
3896
3722
|
retryWritten,
|
|
3897
3723
|
retryVisible,
|
|
3724
|
+
staleDropped,
|
|
3725
|
+
staleDroppedKeys,
|
|
3898
3726
|
};
|
|
3899
3727
|
}
|
|
3900
3728
|
|
|
@@ -3908,6 +3736,7 @@ async function applyLiveMapRowUpdates(input: {
|
|
|
3908
3736
|
attemptOwnerRunId?: string | null;
|
|
3909
3737
|
attemptExpiresAt?: string | null;
|
|
3910
3738
|
attemptSeq?: number | null;
|
|
3739
|
+
writeVersion?: number | null;
|
|
3911
3740
|
budgetMeter?: WorkerWorkBudgetMeter;
|
|
3912
3741
|
}): Promise<void> {
|
|
3913
3742
|
if (input.updates.length === 0) return;
|
|
@@ -3958,6 +3787,7 @@ async function prepareMapRows(input: {
|
|
|
3958
3787
|
attemptOwnerRunId?: string | null;
|
|
3959
3788
|
attemptExpiresAt?: string | null;
|
|
3960
3789
|
attemptSeq?: number | null;
|
|
3790
|
+
writeVersion?: number | null;
|
|
3961
3791
|
budgetMeter?: WorkerWorkBudgetMeter;
|
|
3962
3792
|
force?: boolean;
|
|
3963
3793
|
}): Promise<{
|
|
@@ -3970,6 +3800,7 @@ async function prepareMapRows(input: {
|
|
|
3970
3800
|
attemptOwnerRunId?: string | null;
|
|
3971
3801
|
attemptExpiresAt?: string | null;
|
|
3972
3802
|
attemptSeq?: number | null;
|
|
3803
|
+
writeVersion?: number | null;
|
|
3973
3804
|
}> {
|
|
3974
3805
|
if (input.rows.length === 0) {
|
|
3975
3806
|
return {
|
|
@@ -3982,6 +3813,7 @@ async function prepareMapRows(input: {
|
|
|
3982
3813
|
attemptOwnerRunId: input.attemptOwnerRunId ?? null,
|
|
3983
3814
|
attemptExpiresAt: input.attemptExpiresAt ?? null,
|
|
3984
3815
|
attemptSeq: input.attemptSeq ?? input.req.runAttempt ?? 0,
|
|
3816
|
+
writeVersion: input.writeVersion ?? null,
|
|
3985
3817
|
};
|
|
3986
3818
|
}
|
|
3987
3819
|
const sessionScope = runtimeSheetSessionScope(input.req);
|
|
@@ -4001,6 +3833,7 @@ async function prepareMapRows(input: {
|
|
|
4001
3833
|
attemptOwnerRunId: input.attemptOwnerRunId ?? null,
|
|
4002
3834
|
attemptExpiresAt: input.attemptExpiresAt ?? null,
|
|
4003
3835
|
attemptSeq: input.attemptSeq ?? input.req.runAttempt ?? 0,
|
|
3836
|
+
writeVersion: input.writeVersion ?? null,
|
|
4004
3837
|
attemptLeaseTtlMs: runtimeSheetAttemptLeaseTtlMs(input.req),
|
|
4005
3838
|
...(input.force ? { force: true } : {}),
|
|
4006
3839
|
});
|
|
@@ -4038,6 +3871,7 @@ async function prepareMapRows(input: {
|
|
|
4038
3871
|
attemptExpiresAt: result.attemptExpiresAt ?? input.attemptExpiresAt ?? null,
|
|
4039
3872
|
attemptSeq:
|
|
4040
3873
|
result.attemptSeq ?? input.attemptSeq ?? input.req.runAttempt ?? 0,
|
|
3874
|
+
writeVersion: result.writeVersion ?? input.writeVersion ?? null,
|
|
4041
3875
|
};
|
|
4042
3876
|
}
|
|
4043
3877
|
|
|
@@ -4091,18 +3925,6 @@ function childPipelineUsesCtxDataset(
|
|
|
4091
3925
|
});
|
|
4092
3926
|
}
|
|
4093
3927
|
|
|
4094
|
-
function childPipelineNeedsWorkflowScheduler(
|
|
4095
|
-
pipeline: PlayStaticPipeline | null | undefined,
|
|
4096
|
-
): boolean {
|
|
4097
|
-
if (!pipeline) return false;
|
|
4098
|
-
return flattenStaticPipeline(pipeline).some(
|
|
4099
|
-
(substep) =>
|
|
4100
|
-
substep.type === 'tool' &&
|
|
4101
|
-
(substep.isEventWait === true ||
|
|
4102
|
-
substep.toolId === 'test_wait_for_event'),
|
|
4103
|
-
);
|
|
4104
|
-
}
|
|
4105
|
-
|
|
4106
3928
|
/**
|
|
4107
3929
|
* Build the per-(org,provider) rate port the distributed Rate State Backend
|
|
4108
3930
|
* RPCs through. When the coordinator binding (or its rate RPCs) is absent we
|
|
@@ -4931,6 +4753,73 @@ function createMinimalWorkerCtx(
|
|
|
4931
4753
|
return kept;
|
|
4932
4754
|
};
|
|
4933
4755
|
|
|
4756
|
+
// Reserve the whole logical map before dispatching any provider work. One
|
|
4757
|
+
// schedule-time version represents the user's intent across every chunk;
|
|
4758
|
+
// a later run can supersede it without waiting, and an older chunk can
|
|
4759
|
+
// never allocate a newer version halfway through the map.
|
|
4760
|
+
const inputCapabilities = datasetHandleCapabilities(inputRows);
|
|
4761
|
+
if (!inputCapabilities.replayable) {
|
|
4762
|
+
throw new Error(
|
|
4763
|
+
`ctx.dataset("${name}") requires a replayable input so all row write intents can be reserved before execution.`,
|
|
4764
|
+
);
|
|
4765
|
+
}
|
|
4766
|
+
const reservationRows: Record<string, unknown>[] = [];
|
|
4767
|
+
const reservationExplicitKeys =
|
|
4768
|
+
opts?.key === undefined ? null : new Set<string>();
|
|
4769
|
+
let reservationOffset = 0;
|
|
4770
|
+
let reservationChunkIndex = 0;
|
|
4771
|
+
for await (const chunk of iterDatasetChunks(inputRows, rowsPerChunk)) {
|
|
4772
|
+
assertNotAborted(abortSignal);
|
|
4773
|
+
for (let localIndex = 0; localIndex < chunk.length; localIndex += 1) {
|
|
4774
|
+
const row = chunk[localIndex]!;
|
|
4775
|
+
const absoluteIndex = baseOffset + reservationOffset + localIndex;
|
|
4776
|
+
const explicitKeyValue = resolveExplicitKeyValue(row, absoluteIndex);
|
|
4777
|
+
if (explicitKeyValue !== null && reservationExplicitKeys) {
|
|
4778
|
+
if (reservationExplicitKeys.has(explicitKeyValue)) continue;
|
|
4779
|
+
reservationExplicitKeys.add(explicitKeyValue);
|
|
4780
|
+
}
|
|
4781
|
+
const rowKey = resolveRowKey(
|
|
4782
|
+
row,
|
|
4783
|
+
absoluteIndex,
|
|
4784
|
+
reservationChunkIndex,
|
|
4785
|
+
localIndex,
|
|
4786
|
+
);
|
|
4787
|
+
reservationRows.push({
|
|
4788
|
+
...runtimeCsvStorageRow(row),
|
|
4789
|
+
...mapRowOutcomeRuntimeFields({
|
|
4790
|
+
key: rowKey,
|
|
4791
|
+
inputIndex: absoluteIndex,
|
|
4792
|
+
}),
|
|
4793
|
+
});
|
|
4794
|
+
}
|
|
4795
|
+
reservationOffset += chunk.length;
|
|
4796
|
+
reservationChunkIndex += 1;
|
|
4797
|
+
}
|
|
4798
|
+
const mapWriteAttempt = makeRuntimeSheetAttempt({
|
|
4799
|
+
runId: req.runId,
|
|
4800
|
+
runAttempt: req.runAttempt,
|
|
4801
|
+
mapName: name,
|
|
4802
|
+
chunkIndex: 'finalize',
|
|
4803
|
+
});
|
|
4804
|
+
const mapReservation =
|
|
4805
|
+
reservationRows.length === 0
|
|
4806
|
+
? null
|
|
4807
|
+
: await harnessStartSheetDataset({
|
|
4808
|
+
...runtimeSheetSessionScope(req),
|
|
4809
|
+
tableNamespace: name,
|
|
4810
|
+
sheetContract: augmentSheetContractWithDatasetFields({
|
|
4811
|
+
contract: requireSheetContract(req, name),
|
|
4812
|
+
rows: reservationRows,
|
|
4813
|
+
outputFields,
|
|
4814
|
+
}),
|
|
4815
|
+
rows: reservationRows,
|
|
4816
|
+
runId: req.runId,
|
|
4817
|
+
inputOffset: 0,
|
|
4818
|
+
attemptLeaseTtlMs: runtimeSheetAttemptLeaseTtlMs(req),
|
|
4819
|
+
...mapWriteAttempt,
|
|
4820
|
+
});
|
|
4821
|
+
const mapWriteVersion = mapReservation?.writeVersion ?? null;
|
|
4822
|
+
|
|
4934
4823
|
let totalRowsWritten = 0;
|
|
4935
4824
|
let datasetLifecycleRegistered = false;
|
|
4936
4825
|
|
|
@@ -4980,6 +4869,7 @@ function createMinimalWorkerCtx(
|
|
|
4980
4869
|
})),
|
|
4981
4870
|
inputOffset: baseOffset + chunkStart,
|
|
4982
4871
|
...rowAttempt,
|
|
4872
|
+
writeVersion: mapWriteVersion,
|
|
4983
4873
|
budgetMeter: workBudgetMeter,
|
|
4984
4874
|
force: !!req.force,
|
|
4985
4875
|
});
|
|
@@ -5006,10 +4896,8 @@ function createMinimalWorkerCtx(
|
|
|
5006
4896
|
attemptExpiresAt:
|
|
5007
4897
|
prepared.attemptExpiresAt ?? rowAttempt.attemptExpiresAt,
|
|
5008
4898
|
attemptSeq: prepared.attemptSeq ?? rowAttempt.attemptSeq,
|
|
4899
|
+
writeVersion: prepared.writeVersion ?? null,
|
|
5009
4900
|
};
|
|
5010
|
-
// Record that this run holds attempt leases in `name`'s sheet so a
|
|
5011
|
-
// run-fatal teardown releases them (the table now exists).
|
|
5012
|
-
leasedSheetNamespaces?.add(name);
|
|
5013
4901
|
recordRunnerPerfTrace({
|
|
5014
4902
|
req,
|
|
5015
4903
|
phase: 'runner.map_chunk.prepare_rows',
|
|
@@ -5045,8 +4933,6 @@ function createMinimalWorkerCtx(
|
|
|
5045
4933
|
const pendingRowsByKey = new Map<string, Record<string, unknown>>();
|
|
5046
4934
|
const completedKeys = new Set<string>();
|
|
5047
4935
|
const completedRowsByKey = new Map<string, Record<string, unknown>>();
|
|
5048
|
-
const blockedKeys = new Set<string>();
|
|
5049
|
-
const blockedRowsByKey = new Map<string, Record<string, unknown>>();
|
|
5050
4936
|
const preparedKeys = new Set<string>();
|
|
5051
4937
|
for (const row of prepared.pendingRows) {
|
|
5052
4938
|
const key =
|
|
@@ -5073,66 +4959,12 @@ function createMinimalWorkerCtx(
|
|
|
5073
4959
|
resolveMapRowOutcomeKey(row) ??
|
|
5074
4960
|
deriveDefaultMapRowIdentity(row, resolveRuntimeInputIndex(row) ?? 0);
|
|
5075
4961
|
if (key) {
|
|
5076
|
-
blockedKeys.add(key);
|
|
5077
|
-
blockedRowsByKey.set(key, row);
|
|
5078
4962
|
preparedKeys.add(key);
|
|
5079
4963
|
}
|
|
5080
4964
|
}
|
|
5081
|
-
|
|
5082
|
-
|
|
5083
|
-
|
|
5084
|
-
MAP_ROW_FAILURE_SAMPLE_LIMIT,
|
|
5085
|
-
);
|
|
5086
|
-
const message =
|
|
5087
|
-
`ctx.dataset("${name}") chunk ${chunkIndex} is blocked by ` +
|
|
5088
|
-
`${blockedKeys.size} active runtime sheet row lease(s) owned by another attempt. ` +
|
|
5089
|
-
`Retry after the owning attempt finishes or the row leases expire.` +
|
|
5090
|
-
(rowKeySamples.length > 0
|
|
5091
|
-
? ` Blocked row keys: ${rowKeySamples.join(', ')}`
|
|
5092
|
-
: '');
|
|
5093
|
-
recordRunnerPerfTrace({
|
|
5094
|
-
req,
|
|
5095
|
-
phase: 'runner.map_chunk.blocked_rows',
|
|
5096
|
-
ms: nowMs() - prepareStartedAt,
|
|
5097
|
-
extra: {
|
|
5098
|
-
mapName: name,
|
|
5099
|
-
chunkIndex,
|
|
5100
|
-
rowsBlocked: blockedKeys.size,
|
|
5101
|
-
rowKeySamples,
|
|
5102
|
-
},
|
|
5103
|
-
});
|
|
5104
|
-
enqueueMapProgressUpdate({
|
|
5105
|
-
completed: preparedCompletedRows,
|
|
5106
|
-
total: progressTotalRows,
|
|
5107
|
-
waitingRows: blockedKeys.size,
|
|
5108
|
-
startedAt: mapStartedAt,
|
|
5109
|
-
message,
|
|
5110
|
-
});
|
|
5111
|
-
return {
|
|
5112
|
-
status: 'blocked',
|
|
5113
|
-
chunkIndex,
|
|
5114
|
-
rangeStart: baseOffset + chunkStart,
|
|
5115
|
-
rangeEnd: baseOffset + chunkStart + chunkRows.length,
|
|
5116
|
-
rowsRead: chunkRows.length,
|
|
5117
|
-
rowsBlocked: blockedKeys.size,
|
|
5118
|
-
rowKeySamples,
|
|
5119
|
-
blockedRowSamples: rowKeySamples.map((rowKey) => {
|
|
5120
|
-
const blockedRow = blockedRowsByKey.get(rowKey);
|
|
5121
|
-
const attemptId =
|
|
5122
|
-
typeof blockedRow?.__deeplineAttemptId === 'string'
|
|
5123
|
-
? blockedRow.__deeplineAttemptId
|
|
5124
|
-
: typeof blockedRow?._attempt_id === 'string'
|
|
5125
|
-
? blockedRow._attempt_id
|
|
5126
|
-
: activeRowAttempt.attemptId;
|
|
5127
|
-
return {
|
|
5128
|
-
__deeplineRowKey: rowKey,
|
|
5129
|
-
__deeplineAttemptId: attemptId,
|
|
5130
|
-
};
|
|
5131
|
-
}),
|
|
5132
|
-
outputDatasetId: `map:${name}`,
|
|
5133
|
-
message,
|
|
5134
|
-
};
|
|
5135
|
-
}
|
|
4965
|
+
// Versioned scheduling never waits for row ownership. Compatibility
|
|
4966
|
+
// adapters may still report an old `blockedRows` disposition; treat it
|
|
4967
|
+
// as a stale/non-executable row and keep it out of this run's output.
|
|
5136
4968
|
const missingPreparedRows = chunkEntries.filter(
|
|
5137
4969
|
({ rowKey }) => !preparedKeys.has(rowKey),
|
|
5138
4970
|
);
|
|
@@ -5459,12 +5291,29 @@ function createMinimalWorkerCtx(
|
|
|
5459
5291
|
ts: nowMs(),
|
|
5460
5292
|
});
|
|
5461
5293
|
}
|
|
5294
|
+
const staleKeys = new Set(persistStats.staleDroppedKeys);
|
|
5462
5295
|
for (const { executedIndex } of rowsToPersist) {
|
|
5463
|
-
|
|
5296
|
+
const rowKey = uniqueRowsToExecuteEntries[executedIndex]!.rowKey;
|
|
5297
|
+
if (staleKeys.has(rowKey)) continue;
|
|
5298
|
+
if (!persistedExecutedIndexes.has(executedIndex)) {
|
|
5299
|
+
persistedExecutedIndexes.add(executedIndex);
|
|
5300
|
+
completedExecutedRows += 1;
|
|
5301
|
+
}
|
|
5464
5302
|
}
|
|
5465
5303
|
for (const { executedIndex } of failedRowsToPersist) {
|
|
5304
|
+
const rowKey = uniqueRowsToExecuteEntries[executedIndex]!.rowKey;
|
|
5305
|
+
if (staleKeys.has(rowKey)) continue;
|
|
5466
5306
|
persistedFailedIndexes.add(executedIndex);
|
|
5467
5307
|
}
|
|
5308
|
+
if (persistStats.staleDropped > 0) {
|
|
5309
|
+
emitEvent({
|
|
5310
|
+
type: 'log',
|
|
5311
|
+
level: 'info',
|
|
5312
|
+
message: `Dropped ${persistStats.staleDropped} stale runtime sheet write(s) for ctx.dataset("${name}"); a newer scheduled row version is visible.`,
|
|
5313
|
+
ts: nowMs(),
|
|
5314
|
+
});
|
|
5315
|
+
}
|
|
5316
|
+
reportChunkProgress(false);
|
|
5468
5317
|
};
|
|
5469
5318
|
|
|
5470
5319
|
const enqueuePersistExecutedRows = (): Promise<void> => {
|
|
@@ -5762,7 +5611,6 @@ function createMinimalWorkerCtx(
|
|
|
5762
5611
|
: undefined;
|
|
5763
5612
|
executedRows[myIndex] = enriched as T &
|
|
5764
5613
|
Record<string, unknown>;
|
|
5765
|
-
completedExecutedRows += 1;
|
|
5766
5614
|
await notePersistableRow(enriched);
|
|
5767
5615
|
reportChunkProgress(false);
|
|
5768
5616
|
} catch (rowError) {
|
|
@@ -5872,25 +5720,14 @@ function createMinimalWorkerCtx(
|
|
|
5872
5720
|
return results;
|
|
5873
5721
|
},
|
|
5874
5722
|
);
|
|
5875
|
-
|
|
5876
|
-
|
|
5877
|
-
|
|
5878
|
-
|
|
5879
|
-
|
|
5880
|
-
|
|
5881
|
-
|
|
5882
|
-
|
|
5883
|
-
await Promise.race([
|
|
5884
|
-
workerResultsPromise,
|
|
5885
|
-
sleepWorkerMs(MAP_EXECUTION_HEARTBEAT_INTERVAL_MS),
|
|
5886
|
-
]);
|
|
5887
|
-
if (!workerSettled) {
|
|
5888
|
-
reportExecutionHeartbeat(false);
|
|
5889
|
-
}
|
|
5890
|
-
}
|
|
5891
|
-
return await workerResultsPromise;
|
|
5892
|
-
},
|
|
5893
|
-
});
|
|
5723
|
+
while (!workerSettled) {
|
|
5724
|
+
await Promise.race([
|
|
5725
|
+
workerResultsPromise,
|
|
5726
|
+
sleepWorkerMs(MAP_EXECUTION_HEARTBEAT_INTERVAL_MS),
|
|
5727
|
+
]);
|
|
5728
|
+
if (!workerSettled) reportExecutionHeartbeat(false);
|
|
5729
|
+
}
|
|
5730
|
+
const workerResults = await workerResultsPromise;
|
|
5894
5731
|
recordRunnerPerfTrace({
|
|
5895
5732
|
req,
|
|
5896
5733
|
phase: 'runner.map_chunk.execute_workers',
|
|
@@ -6088,6 +5925,7 @@ function createMinimalWorkerCtx(
|
|
|
6088
5925
|
executedIndex < executedRows.length;
|
|
6089
5926
|
executedIndex += 1
|
|
6090
5927
|
) {
|
|
5928
|
+
if (!persistedExecutedIndexes.has(executedIndex)) continue;
|
|
6091
5929
|
const executedRow = executedRows[executedIndex];
|
|
6092
5930
|
const key = uniqueRowsToExecuteEntries[executedIndex]!.rowKey;
|
|
6093
5931
|
// Failed rows have no executed result; they stay out of the map output
|
|
@@ -6123,6 +5961,8 @@ function createMinimalWorkerCtx(
|
|
|
6123
5961
|
...mapRowOutcomeRuntimeFields({ key, inputIndex }),
|
|
6124
5962
|
}));
|
|
6125
5963
|
const completedRowsReused = completedChunkEntries.length;
|
|
5964
|
+
const committedExecutedRows =
|
|
5965
|
+
persistedExecutedIndexes.size + persistedFailedIndexes.size;
|
|
6126
5966
|
const hashStartedAt = nowMs();
|
|
6127
5967
|
const hash = await hashJson(publicOut);
|
|
6128
5968
|
const includeCachedRowsInChunkResult = !workflowStep;
|
|
@@ -6152,8 +5992,8 @@ function createMinimalWorkerCtx(
|
|
|
6152
5992
|
chunkIndex,
|
|
6153
5993
|
rowsRead: chunkRows.length,
|
|
6154
5994
|
rowsWritten: out.length,
|
|
6155
|
-
rowsExecuted:
|
|
6156
|
-
rowsFailed:
|
|
5995
|
+
rowsExecuted: committedExecutedRows,
|
|
5996
|
+
rowsFailed: persistedFailedIndexes.size,
|
|
6157
5997
|
rowsCached: completedRowsReused,
|
|
6158
5998
|
},
|
|
6159
5999
|
});
|
|
@@ -6164,12 +6004,12 @@ function createMinimalWorkerCtx(
|
|
|
6164
6004
|
rangeEnd: baseOffset + chunkStart + out.length,
|
|
6165
6005
|
rowsRead: chunkRows.length,
|
|
6166
6006
|
rowsWritten: out.length,
|
|
6167
|
-
rowsExecuted:
|
|
6007
|
+
rowsExecuted: committedExecutedRows,
|
|
6168
6008
|
rowsCached: completedRowsReused,
|
|
6169
6009
|
rowsDuplicateReused: duplicateInputReuseCount,
|
|
6170
6010
|
rowsInserted,
|
|
6171
6011
|
rowsSkipped,
|
|
6172
|
-
rowsFailed:
|
|
6012
|
+
rowsFailed: persistedFailedIndexes.size,
|
|
6173
6013
|
rowFailureSamples,
|
|
6174
6014
|
stepCellsCompleted,
|
|
6175
6015
|
stepCellsSkipped,
|
|
@@ -6201,12 +6041,9 @@ function createMinimalWorkerCtx(
|
|
|
6201
6041
|
let totalRowsInserted = 0;
|
|
6202
6042
|
let totalRowsSkipped = 0;
|
|
6203
6043
|
let totalRowsFailed = 0;
|
|
6204
|
-
let totalRowsBlocked = 0;
|
|
6205
6044
|
let totalStepCellsCompleted = 0;
|
|
6206
6045
|
let totalStepCellsSkipped = 0;
|
|
6207
6046
|
const totalRowFailureSamples: Array<{ rowKey: string; error: string }> = [];
|
|
6208
|
-
const blockedRowKeySamples: string[] = [];
|
|
6209
|
-
const blockedRowSamples: Record<string, unknown>[] = [];
|
|
6210
6047
|
|
|
6211
6048
|
const runChunkStep = async (
|
|
6212
6049
|
chunkRows: T[],
|
|
@@ -6400,6 +6237,7 @@ function createMinimalWorkerCtx(
|
|
|
6400
6237
|
runId: req.runId,
|
|
6401
6238
|
inputOffset: 0,
|
|
6402
6239
|
attemptLeaseTtlMs: runtimeSheetAttemptLeaseTtlMs(req),
|
|
6240
|
+
writeVersion: mapWriteVersion,
|
|
6403
6241
|
...repairAttempt,
|
|
6404
6242
|
});
|
|
6405
6243
|
const activeRepairAttempt = {
|
|
@@ -6409,6 +6247,7 @@ function createMinimalWorkerCtx(
|
|
|
6409
6247
|
attemptExpiresAt:
|
|
6410
6248
|
repairStart.attemptExpiresAt ?? repairAttempt.attemptExpiresAt,
|
|
6411
6249
|
attemptSeq: repairStart.attemptSeq ?? repairAttempt.attemptSeq,
|
|
6250
|
+
writeVersion: repairStart.writeVersion ?? null,
|
|
6412
6251
|
};
|
|
6413
6252
|
const repair = await persistCompletedMapRows({
|
|
6414
6253
|
req,
|
|
@@ -6610,28 +6449,7 @@ function createMinimalWorkerCtx(
|
|
|
6610
6449
|
);
|
|
6611
6450
|
},
|
|
6612
6451
|
onBatchComplete: async (chunkResult) => {
|
|
6613
|
-
if (chunkResult.status === 'blocked')
|
|
6614
|
-
totalRowsBlocked += chunkResult.rowsBlocked;
|
|
6615
|
-
for (const key of chunkResult.rowKeySamples) {
|
|
6616
|
-
if (blockedRowKeySamples.length >= MAP_ROW_FAILURE_SAMPLE_LIMIT) {
|
|
6617
|
-
break;
|
|
6618
|
-
}
|
|
6619
|
-
blockedRowKeySamples.push(key);
|
|
6620
|
-
}
|
|
6621
|
-
for (const row of chunkResult.blockedRowSamples) {
|
|
6622
|
-
if (blockedRowSamples.length >= MAP_ROW_FAILURE_SAMPLE_LIMIT) {
|
|
6623
|
-
break;
|
|
6624
|
-
}
|
|
6625
|
-
blockedRowSamples.push(row);
|
|
6626
|
-
}
|
|
6627
|
-
emitEvent({
|
|
6628
|
-
type: 'log',
|
|
6629
|
-
level: 'warn',
|
|
6630
|
-
message: chunkResult.message,
|
|
6631
|
-
ts: nowMs(),
|
|
6632
|
-
});
|
|
6633
|
-
return true;
|
|
6634
|
-
}
|
|
6452
|
+
if (chunkResult.status === 'blocked') return true;
|
|
6635
6453
|
if (chunkResult.fatalError) {
|
|
6636
6454
|
throw new Error(
|
|
6637
6455
|
`ctx.dataset("${name}") stopped after a runtime persistence failure ` +
|
|
@@ -6723,13 +6541,6 @@ function createMinimalWorkerCtx(
|
|
|
6723
6541
|
ts: nowMs(),
|
|
6724
6542
|
});
|
|
6725
6543
|
}
|
|
6726
|
-
if (totalRowsBlocked > 0) {
|
|
6727
|
-
await flushQueuedMapProgressUpdates();
|
|
6728
|
-
throw new RuntimeSheetRowsBlockedError({
|
|
6729
|
-
tableNamespace: name,
|
|
6730
|
-
blockedRows: blockedRowSamples,
|
|
6731
|
-
});
|
|
6732
|
-
}
|
|
6733
6544
|
if (failFastRowErrors && totalRowsFailed > 0 && totalRowsWritten > 0) {
|
|
6734
6545
|
// onRowError:'fail', PARTIAL failure (some rows committed): fail the run
|
|
6735
6546
|
// without finalizing the dataset. The committed rows already persisted
|
|
@@ -7094,6 +6905,7 @@ function createMinimalWorkerCtx(
|
|
|
7094
6905
|
options?: {
|
|
7095
6906
|
description?: string;
|
|
7096
6907
|
timeoutMs?: number;
|
|
6908
|
+
execution?: 'inline' | 'child-workflow';
|
|
7097
6909
|
},
|
|
7098
6910
|
): Promise<unknown> {
|
|
7099
6911
|
const normalizedKey = normalizeContextKey(key, 'runPlay');
|
|
@@ -7168,8 +6980,35 @@ function createMinimalWorkerCtx(
|
|
|
7168
6980
|
const childIsDatasetBacked = childPipelineUsesCtxDataset(
|
|
7169
6981
|
childManifest.staticPipeline,
|
|
7170
6982
|
);
|
|
6983
|
+
const childWaitsForEvent = (
|
|
6984
|
+
childManifest.staticPipeline
|
|
6985
|
+
? flattenStaticPipeline(childManifest.staticPipeline)
|
|
6986
|
+
: []
|
|
6987
|
+
).some(
|
|
6988
|
+
(step) =>
|
|
6989
|
+
step.type === 'tool' &&
|
|
6990
|
+
(step.isEventWait || step.toolId === 'test_wait_for_event'),
|
|
6991
|
+
);
|
|
6992
|
+
const authoredWorkflow =
|
|
6993
|
+
options?.timeoutMs != null ||
|
|
6994
|
+
childIsDatasetBacked ||
|
|
6995
|
+
childWaitsForEvent;
|
|
6996
|
+
if (options?.execution === 'inline' && authoredWorkflow) {
|
|
6997
|
+
throw new Error(
|
|
6998
|
+
`ctx.runPlay("${resolvedName}") requested inline execution for a child that requires a workflow.`,
|
|
6999
|
+
);
|
|
7000
|
+
}
|
|
7001
|
+
const childExecutionDecision = resolveChildExecutionStrategy({
|
|
7002
|
+
requiresWorkflow: authoredWorkflow,
|
|
7003
|
+
execution: options?.execution,
|
|
7004
|
+
});
|
|
7171
7005
|
const childNeedsWorkflowScheduler =
|
|
7172
|
-
|
|
7006
|
+
childExecutionDecision.strategy === 'scheduled';
|
|
7007
|
+
if (rowScope && childNeedsWorkflowScheduler) {
|
|
7008
|
+
throw new Error(
|
|
7009
|
+
`ctx.runPlay("${resolvedName}") cannot start a child workflow per dataset row.`,
|
|
7010
|
+
);
|
|
7011
|
+
}
|
|
7173
7012
|
console.info('[play.runtime.span]', {
|
|
7174
7013
|
event: 'play.runtime.span',
|
|
7175
7014
|
phase: 'child_route',
|
|
@@ -7218,8 +7057,7 @@ function createMinimalWorkerCtx(
|
|
|
7218
7057
|
coordinatorBinding: cachedCoordinatorBinding,
|
|
7219
7058
|
makeRequestId,
|
|
7220
7059
|
coordinatorRequestHeaders,
|
|
7221
|
-
allowInline:
|
|
7222
|
-
options?.timeoutMs == null && !childNeedsWorkflowScheduler,
|
|
7060
|
+
allowInline: childExecutionDecision.strategy === 'inline',
|
|
7223
7061
|
body: {
|
|
7224
7062
|
name: resolvedName,
|
|
7225
7063
|
childIdempotencyKey: childInvocationId,
|