deepline 0.1.211 → 0.1.213
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/apps/play-runner-workers/src/entry.ts +317 -340
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/customer-console.ts +23 -0
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/tool-dispatch.ts +1 -1
- package/dist/bundling-sources/sdk/src/client.ts +118 -3
- package/dist/bundling-sources/sdk/src/index.ts +2 -0
- package/dist/bundling-sources/sdk/src/play.ts +8 -1
- package/dist/bundling-sources/sdk/src/plays/bundle-play-file.ts +1 -1
- package/dist/bundling-sources/sdk/src/plays/harness-stub.ts +11 -1
- package/dist/bundling-sources/sdk/src/release.ts +2 -2
- package/dist/bundling-sources/sdk/src/types.ts +47 -0
- package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +302 -30
- package/dist/bundling-sources/shared_libs/play-runtime/child-execution-placement.ts +14 -0
- package/dist/bundling-sources/shared_libs/play-runtime/child-execution-strategy.ts +57 -23
- package/dist/bundling-sources/shared_libs/play-runtime/completed-receipt-cache.ts +166 -0
- package/dist/bundling-sources/shared_libs/play-runtime/context.ts +164 -121
- package/dist/bundling-sources/shared_libs/play-runtime/ctx-contract.ts +2 -0
- package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +42 -1
- package/dist/bundling-sources/shared_libs/play-runtime/dataset-id.ts +10 -4
- package/dist/bundling-sources/shared_libs/play-runtime/db-session.ts +9 -0
- package/dist/bundling-sources/shared_libs/play-runtime/durable-receipt-execution.ts +163 -7
- package/dist/bundling-sources/shared_libs/play-runtime/gateway-auth-session.ts +93 -0
- package/dist/bundling-sources/shared_libs/play-runtime/ledger-safe-payload.ts +14 -2
- package/dist/bundling-sources/shared_libs/play-runtime/output-size-limits.ts +4 -2
- package/dist/bundling-sources/shared_libs/play-runtime/play-call-execution.ts +1 -0
- package/dist/bundling-sources/shared_libs/play-runtime/receipt-completion-sink.ts +16 -1
- package/dist/bundling-sources/shared_libs/play-runtime/receipt-sql.ts +21 -1
- package/dist/bundling-sources/shared_libs/play-runtime/receipt-state-sink.ts +10 -0
- package/dist/bundling-sources/shared_libs/play-runtime/resource-governor.ts +80 -3
- package/dist/bundling-sources/shared_libs/play-runtime/run-ledger.ts +128 -6
- package/dist/bundling-sources/shared_libs/play-runtime/run-snapshot-stream.ts +7 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-session-execution.ts +94 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts +91 -6
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/types.ts +19 -10
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +540 -92
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-postgres-admission.ts +162 -0
- package/dist/bundling-sources/shared_libs/play-runtime/secret-capability.ts +18 -4
- package/dist/bundling-sources/shared_libs/play-runtime/sheet-attempt-sql.ts +16 -17
- package/dist/bundling-sources/shared_libs/play-runtime/tool-execute-retry-policy.ts +29 -9
- package/dist/bundling-sources/shared_libs/play-runtime/work-receipt-state-machine.ts +22 -1
- package/dist/bundling-sources/shared_libs/play-runtime/work-receipts.ts +1 -0
- package/dist/bundling-sources/shared_libs/plays/bundling/index.ts +79 -1
- package/dist/bundling-sources/shared_libs/plays/static-pipeline.ts +2 -4
- package/dist/cli/index.js +637 -229
- package/dist/cli/index.mjs +637 -229
- package/dist/{compiler-manifest-j6z8qzpd.d.mts → compiler-manifest-BhgZ23A4.d.mts} +1 -0
- package/dist/{compiler-manifest-j6z8qzpd.d.ts → compiler-manifest-BhgZ23A4.d.ts} +1 -0
- package/dist/index.d.mts +84 -11
- package/dist/index.d.ts +84 -11
- package/dist/index.js +225 -43
- package/dist/index.mjs +225 -43
- package/dist/plays/bundle-play-file.d.mts +2 -2
- package/dist/plays/bundle-play-file.d.ts +2 -2
- package/dist/plays/bundle-play-file.mjs +65 -4
- package/package.json +1 -1
|
@@ -40,7 +40,6 @@ import {
|
|
|
40
40
|
mapRowOutcomeRuntimeFields,
|
|
41
41
|
resolveMapRowOutcomeKey,
|
|
42
42
|
} from './map-row-outcome';
|
|
43
|
-
import { RuntimeSheetRowsBlockedError } from './runtime-sheet-errors';
|
|
44
43
|
import { buildChildRunId } from './child-run-id';
|
|
45
44
|
import type { PlayCallGovernanceSnapshot } from './scheduler-backend';
|
|
46
45
|
import {
|
|
@@ -106,6 +105,7 @@ import {
|
|
|
106
105
|
parseToolExecuteAuthScopeChangedError,
|
|
107
106
|
ToolExecuteAuthScopeChangedError,
|
|
108
107
|
} from './tool-execute-retry-policy';
|
|
108
|
+
import { ToolHttpError } from './tool-http-errors';
|
|
109
109
|
import {
|
|
110
110
|
buildDurableCtxCallCacheKey,
|
|
111
111
|
buildDurableRunPlayInvocationScope,
|
|
@@ -1520,12 +1520,14 @@ export class PlayContextImpl {
|
|
|
1520
1520
|
|
|
1521
1521
|
private recordPlayCallStep(input: {
|
|
1522
1522
|
playId: string;
|
|
1523
|
+
execution?: 'inline' | 'child-workflow';
|
|
1523
1524
|
description?: string | null;
|
|
1524
1525
|
nestedSteps?: PlayStep[];
|
|
1525
1526
|
}): void {
|
|
1526
1527
|
const step = {
|
|
1527
1528
|
type: 'play_call' as const,
|
|
1528
1529
|
playId: input.playId,
|
|
1530
|
+
...(input.execution ? { execution: input.execution } : {}),
|
|
1529
1531
|
nestedSteps: input.nestedSteps ?? [],
|
|
1530
1532
|
description: normalizeStepDescription(input.description ?? undefined),
|
|
1531
1533
|
};
|
|
@@ -1902,6 +1904,7 @@ export class PlayContextImpl {
|
|
|
1902
1904
|
}
|
|
1903
1905
|
const claimed = await this.#options.claimRuntimeStepReceipt({
|
|
1904
1906
|
key,
|
|
1907
|
+
leaseId: `receipt-lease:${crypto.randomUUID()}`,
|
|
1905
1908
|
runId: this.currentReceiptOwnerRunId,
|
|
1906
1909
|
runAttempt: this.currentRunAttempt,
|
|
1907
1910
|
leaseAware: true,
|
|
@@ -1978,30 +1981,6 @@ export class PlayContextImpl {
|
|
|
1978
1981
|
};
|
|
1979
1982
|
}
|
|
1980
1983
|
|
|
1981
|
-
private async skipRuntimeStepReceipt(
|
|
1982
|
-
key: string,
|
|
1983
|
-
runId: string,
|
|
1984
|
-
leaseId?: string | null,
|
|
1985
|
-
): Promise<RuntimeStepReceipt | null> {
|
|
1986
|
-
if (!this.#options.skipRuntimeStepReceipt) {
|
|
1987
|
-
return null;
|
|
1988
|
-
}
|
|
1989
|
-
const skipped = await this.#options.skipRuntimeStepReceipt({
|
|
1990
|
-
key,
|
|
1991
|
-
runId,
|
|
1992
|
-
runAttempt: this.currentRunAttempt,
|
|
1993
|
-
...(leaseId ? { leaseId } : {}),
|
|
1994
|
-
});
|
|
1995
|
-
if (!skipped || typeof skipped.key !== 'string' || !skipped.key.trim()) {
|
|
1996
|
-
return null;
|
|
1997
|
-
}
|
|
1998
|
-
return {
|
|
1999
|
-
...skipped,
|
|
2000
|
-
key: skipped.key.trim(),
|
|
2001
|
-
runId: skipped.runId ?? null,
|
|
2002
|
-
};
|
|
2003
|
-
}
|
|
2004
|
-
|
|
2005
1984
|
private async heartbeatRuntimeStepReceipt(
|
|
2006
1985
|
key: string,
|
|
2007
1986
|
_runId: string,
|
|
@@ -2309,6 +2288,11 @@ export class PlayContextImpl {
|
|
|
2309
2288
|
? await this.dispatchChunkedRuntimeReceiptRequest(uniqueKeys, (chunk) =>
|
|
2310
2289
|
claimReceipts({
|
|
2311
2290
|
keys: chunk,
|
|
2291
|
+
// A transport retry can replay this mutation after the store
|
|
2292
|
+
// committed but before the response body was consumed. Stable
|
|
2293
|
+
// caller-owned tokens distinguish that replay from a concurrent
|
|
2294
|
+
// claimant without weakening the provider-call execution fence.
|
|
2295
|
+
leaseIds: chunk.map(() => `receipt-lease:${crypto.randomUUID()}`),
|
|
2312
2296
|
runId: this.currentReceiptOwnerRunId,
|
|
2313
2297
|
runAttempt: this.currentRunAttempt,
|
|
2314
2298
|
leaseAware: true,
|
|
@@ -2776,6 +2760,24 @@ export class PlayContextImpl {
|
|
|
2776
2760
|
this.#options.completeRuntimeStepReceipt ||
|
|
2777
2761
|
this.#options.completeRuntimeStepReceipts,
|
|
2778
2762
|
),
|
|
2763
|
+
...(this.#options.acquireRuntimeReceiptExecutionLock &&
|
|
2764
|
+
this.#options.releaseRuntimeReceiptExecutionLock
|
|
2765
|
+
? {
|
|
2766
|
+
acquireExecutionLock: ({ receiptKey, ownerExecutionId, ttlMs }) =>
|
|
2767
|
+
this.#options.acquireRuntimeReceiptExecutionLock!({
|
|
2768
|
+
key: receiptKey,
|
|
2769
|
+
runId: this.currentReceiptOwnerRunId,
|
|
2770
|
+
ownerExecutionId,
|
|
2771
|
+
ttlMs,
|
|
2772
|
+
}),
|
|
2773
|
+
releaseExecutionLock: ({ receiptKey, ownerExecutionId }) =>
|
|
2774
|
+
this.#options.releaseRuntimeReceiptExecutionLock!({
|
|
2775
|
+
key: receiptKey,
|
|
2776
|
+
runId: this.currentReceiptOwnerRunId,
|
|
2777
|
+
ownerExecutionId,
|
|
2778
|
+
}),
|
|
2779
|
+
}
|
|
2780
|
+
: {}),
|
|
2779
2781
|
};
|
|
2780
2782
|
}
|
|
2781
2783
|
|
|
@@ -2833,6 +2835,8 @@ export class PlayContextImpl {
|
|
|
2833
2835
|
onClaimedResult?: (output: T, receiptKey: string) => T;
|
|
2834
2836
|
shouldPersistFailure?: (error: unknown) => boolean;
|
|
2835
2837
|
markRunningBeforeExecute?: boolean;
|
|
2838
|
+
requiresExecutionLock?: boolean;
|
|
2839
|
+
executionLockTtlMs?: number;
|
|
2836
2840
|
execute: (context: { leaseId: string | null }) => Promise<T>;
|
|
2837
2841
|
},
|
|
2838
2842
|
): Promise<T> {
|
|
@@ -2864,6 +2868,9 @@ export class PlayContextImpl {
|
|
|
2864
2868
|
onClaimedResult: opts.onClaimedResult,
|
|
2865
2869
|
shouldPersistFailure: opts.shouldPersistFailure,
|
|
2866
2870
|
markRunningBeforeExecute: opts.markRunningBeforeExecute,
|
|
2871
|
+
completedCacheOnly: operation === 'tool',
|
|
2872
|
+
requiresExecutionLock: opts.requiresExecutionLock,
|
|
2873
|
+
executionLockTtlMs: opts.executionLockTtlMs,
|
|
2867
2874
|
formatError: (error) => this.formatRuntimeError(error),
|
|
2868
2875
|
log: (message) => this.log(message),
|
|
2869
2876
|
execute: opts.execute,
|
|
@@ -3486,7 +3493,8 @@ export class PlayContextImpl {
|
|
|
3486
3493
|
);
|
|
3487
3494
|
}
|
|
3488
3495
|
const finalWrapped =
|
|
3489
|
-
completed?.status === 'completed' || completed?.status === 'skipped'
|
|
3496
|
+
(completed?.status === 'completed' || completed?.status === 'skipped') &&
|
|
3497
|
+
completed.output !== undefined
|
|
3490
3498
|
? await this.wrapToolExecutionResult({
|
|
3491
3499
|
toolId,
|
|
3492
3500
|
status:
|
|
@@ -4014,12 +4022,6 @@ export class PlayContextImpl {
|
|
|
4014
4022
|
resolvedTableNamespace = normalizeTableNamespace(
|
|
4015
4023
|
mapStartResult.tableNamespace,
|
|
4016
4024
|
);
|
|
4017
|
-
if ((mapStartResult.blockedRows?.length ?? 0) > 0) {
|
|
4018
|
-
throw new RuntimeSheetRowsBlockedError({
|
|
4019
|
-
tableNamespace: resolvedTableNamespace,
|
|
4020
|
-
blockedRows: mapStartResult.blockedRows ?? [],
|
|
4021
|
-
});
|
|
4022
|
-
}
|
|
4023
4025
|
|
|
4024
4026
|
const persistedRowIdentity = (
|
|
4025
4027
|
row: Record<string, unknown>,
|
|
@@ -4454,12 +4456,6 @@ export class PlayContextImpl {
|
|
|
4454
4456
|
resolvedTableNamespace = normalizeTableNamespace(
|
|
4455
4457
|
mapStartResult.tableNamespace,
|
|
4456
4458
|
);
|
|
4457
|
-
if ((mapStartResult.blockedRows?.length ?? 0) > 0) {
|
|
4458
|
-
throw new RuntimeSheetRowsBlockedError({
|
|
4459
|
-
tableNamespace: resolvedTableNamespace,
|
|
4460
|
-
blockedRows: mapStartResult.blockedRows ?? [],
|
|
4461
|
-
});
|
|
4462
|
-
}
|
|
4463
4459
|
const persistedRowIdentity = (row: Record<string, unknown>, index = 0) =>
|
|
4464
4460
|
resolveMapRowOutcomeKey(row) ?? rowIdentity(row, index);
|
|
4465
4461
|
const pendingRowsByKey = new Map<string, Record<string, unknown>>();
|
|
@@ -4572,6 +4568,7 @@ export class PlayContextImpl {
|
|
|
4572
4568
|
);
|
|
4573
4569
|
|
|
4574
4570
|
this.activeMapCellMeta = new Map();
|
|
4571
|
+
const staleCompletionKeys = new Set<string>();
|
|
4575
4572
|
const persistMapRows = async (rows: PersistableMapRow[]) => {
|
|
4576
4573
|
if (!this.#options.onMapRowsCompleted || rows.length === 0) {
|
|
4577
4574
|
return;
|
|
@@ -4595,7 +4592,7 @@ export class PlayContextImpl {
|
|
|
4595
4592
|
);
|
|
4596
4593
|
const flushStartedAt = Date.now();
|
|
4597
4594
|
try {
|
|
4598
|
-
await this.#options.onMapRowsCompleted!({
|
|
4595
|
+
const writeResult = await this.#options.onMapRowsCompleted!({
|
|
4599
4596
|
playName: this.#options.playName,
|
|
4600
4597
|
playId: this.#options.playId,
|
|
4601
4598
|
runId: this.#options.runId,
|
|
@@ -4607,6 +4604,11 @@ export class PlayContextImpl {
|
|
|
4607
4604
|
),
|
|
4608
4605
|
staticPipeline: this.#options.staticPipeline ?? null,
|
|
4609
4606
|
});
|
|
4607
|
+
if (writeResult) {
|
|
4608
|
+
for (const key of writeResult.staleDroppedKeys ?? []) {
|
|
4609
|
+
staleCompletionKeys.add(key);
|
|
4610
|
+
}
|
|
4611
|
+
}
|
|
4610
4612
|
this.resourceGovernor.observe({
|
|
4611
4613
|
sheetFlushBytes: chunkBytes,
|
|
4612
4614
|
sheetFlushLatencyMs: Date.now() - flushStartedAt,
|
|
@@ -4652,6 +4654,7 @@ export class PlayContextImpl {
|
|
|
4652
4654
|
completedRows: duplicateReuseCount,
|
|
4653
4655
|
},
|
|
4654
4656
|
{
|
|
4657
|
+
emitTerminalEvent: false,
|
|
4655
4658
|
onRowError: options?.onRowError,
|
|
4656
4659
|
executionRowKeys: rowsToExecuteEntries.map((entry) => entry.rowKey),
|
|
4657
4660
|
executionRowIndexes: rowsToExecuteEntries.map(
|
|
@@ -4749,7 +4752,49 @@ export class PlayContextImpl {
|
|
|
4749
4752
|
}
|
|
4750
4753
|
this.activeMapCellMeta = null;
|
|
4751
4754
|
|
|
4752
|
-
if (
|
|
4755
|
+
if (staleCompletionKeys.size > 0) {
|
|
4756
|
+
results = results.filter((row, index) => {
|
|
4757
|
+
const key = rowIdentity(row, itemOriginalIndexes[index] ?? index);
|
|
4758
|
+
return !staleCompletionKeys.has(key);
|
|
4759
|
+
});
|
|
4760
|
+
}
|
|
4761
|
+
|
|
4762
|
+
const durableCompletedRows = Math.max(
|
|
4763
|
+
0,
|
|
4764
|
+
reusedCount + mapResult.completedRows.length - staleCompletionKeys.size,
|
|
4765
|
+
);
|
|
4766
|
+
const durableFailedRows = mapResult.failedRows.length;
|
|
4767
|
+
const terminalFrame = this.checkpoint.mapFrames?.[mapScope.mapInvocationId];
|
|
4768
|
+
if (terminalFrame) {
|
|
4769
|
+
this.setMapFrame({
|
|
4770
|
+
...terminalFrame,
|
|
4771
|
+
status: 'completed',
|
|
4772
|
+
// Incremental rows add keys after their commit. Pure maps use the
|
|
4773
|
+
// durable aggregate count below instead of materializing up to millions
|
|
4774
|
+
// of keys only to mark the terminal frame.
|
|
4775
|
+
completedRowKeys: terminalFrame.completedRowKeys,
|
|
4776
|
+
pendingRowKeys: [],
|
|
4777
|
+
completedRowsCount: durableCompletedRows,
|
|
4778
|
+
pendingRowsCount: 0,
|
|
4779
|
+
failedRowsCount: durableFailedRows,
|
|
4780
|
+
activeBoundaryId: null,
|
|
4781
|
+
updatedAt: Date.now(),
|
|
4782
|
+
});
|
|
4783
|
+
}
|
|
4784
|
+
this.emitExecutionEvent({
|
|
4785
|
+
type: 'map.completed',
|
|
4786
|
+
mapInvocationId: mapScope.mapInvocationId,
|
|
4787
|
+
mapNodeId: mapScope.mapNodeId ?? null,
|
|
4788
|
+
logicalNamespace: mapScope.logicalNamespace,
|
|
4789
|
+
artifactTableNamespace: resolvedTableNamespace,
|
|
4790
|
+
completedRows: durableCompletedRows,
|
|
4791
|
+
failedRows: durableFailedRows,
|
|
4792
|
+
totalRows: totalInputCount,
|
|
4793
|
+
...this.inlineChildAggregateEventFields(),
|
|
4794
|
+
at: Date.now(),
|
|
4795
|
+
});
|
|
4796
|
+
|
|
4797
|
+
if (this.#options.onMapRowsCompleted && staleCompletionKeys.size === 0) {
|
|
4753
4798
|
results = await reconcileNodeRuntimeMapResultsWithPersistedSheet({
|
|
4754
4799
|
mapName: normalizedMapNamespace,
|
|
4755
4800
|
tableNamespace: resolvedTableNamespace,
|
|
@@ -4986,11 +5031,21 @@ export class PlayContextImpl {
|
|
|
4986
5031
|
mapName: normalizedTableNamespace,
|
|
4987
5032
|
budgetBytes: runtimeOptions?.retainedRowsMemoryBudgetBytes,
|
|
4988
5033
|
});
|
|
4989
|
-
const enqueueIncrementalPersist = (
|
|
4990
|
-
|
|
4991
|
-
|
|
4992
|
-
|
|
4993
|
-
|
|
5034
|
+
const enqueueIncrementalPersist = (
|
|
5035
|
+
row: PersistableMapRow,
|
|
5036
|
+
onCommitted: () => void,
|
|
5037
|
+
): void => {
|
|
5038
|
+
if (!incrementalPersistence) {
|
|
5039
|
+
onCommitted();
|
|
5040
|
+
return;
|
|
5041
|
+
}
|
|
5042
|
+
void incrementalPersistence
|
|
5043
|
+
.persistRows([row])
|
|
5044
|
+
.then(onCommitted)
|
|
5045
|
+
.catch(() => {
|
|
5046
|
+
// The final map-level flush awaits the same chain and surfaces the
|
|
5047
|
+
// persistence failure loudly without holding a completed row slot open.
|
|
5048
|
+
});
|
|
4994
5049
|
};
|
|
4995
5050
|
|
|
4996
5051
|
if (completedRows > 0 || pendingRows !== totalRows) {
|
|
@@ -5156,14 +5211,16 @@ export class PlayContextImpl {
|
|
|
5156
5211
|
// this replaces ran AFTER every row had already computed, emitting 150k
|
|
5157
5212
|
// post-hoc map.progress events and re-copying the completed-keys array
|
|
5158
5213
|
// per row — all theater, no live progress value.
|
|
5159
|
-
|
|
5160
|
-
|
|
5161
|
-
|
|
5162
|
-
|
|
5163
|
-
|
|
5214
|
+
if (!incrementalPersistence) {
|
|
5215
|
+
updateMapFrameProgress({
|
|
5216
|
+
completedRowKeys: results.map((_row, index) =>
|
|
5217
|
+
executionRowKey(
|
|
5218
|
+
this.toOutputRow(items[index] as Record<string, unknown>),
|
|
5219
|
+
index,
|
|
5220
|
+
),
|
|
5164
5221
|
),
|
|
5165
|
-
)
|
|
5166
|
-
}
|
|
5222
|
+
});
|
|
5223
|
+
}
|
|
5167
5224
|
if (emitTerminalEvent) {
|
|
5168
5225
|
updateMapFrameProgress({
|
|
5169
5226
|
status: 'completed',
|
|
@@ -5344,9 +5401,6 @@ export class PlayContextImpl {
|
|
|
5344
5401
|
});
|
|
5345
5402
|
retainedRowsMemoryTracker.track(failedRow);
|
|
5346
5403
|
failedRowsToPersist.push(failedRow);
|
|
5347
|
-
updateMapFrameProgress({
|
|
5348
|
-
failedRowKey: rowKey,
|
|
5349
|
-
});
|
|
5350
5404
|
this.emitScopedRowUpdate(rowKey, normalizedTableNamespace, {
|
|
5351
5405
|
rowId: idx,
|
|
5352
5406
|
status: 'failed',
|
|
@@ -5355,7 +5409,9 @@ export class PlayContextImpl {
|
|
|
5355
5409
|
error: formattedError,
|
|
5356
5410
|
dataPatch: {},
|
|
5357
5411
|
});
|
|
5358
|
-
enqueueIncrementalPersist(failedRow)
|
|
5412
|
+
enqueueIncrementalPersist(failedRow, () => {
|
|
5413
|
+
updateMapFrameProgress({ failedRowKey: rowKey });
|
|
5414
|
+
});
|
|
5359
5415
|
return FAILED_ROW;
|
|
5360
5416
|
}
|
|
5361
5417
|
const cellValue = this.serializeCellValue(value);
|
|
@@ -5402,9 +5458,6 @@ export class PlayContextImpl {
|
|
|
5402
5458
|
|
|
5403
5459
|
const merged = cloneCsvAliasedRow(baseRow, computedFields);
|
|
5404
5460
|
activeFieldName = null;
|
|
5405
|
-
updateMapFrameProgress({
|
|
5406
|
-
completedRowKey: rowKey,
|
|
5407
|
-
});
|
|
5408
5461
|
this.emitScopedRowUpdate(rowKey, normalizedTableNamespace, {
|
|
5409
5462
|
rowId: idx,
|
|
5410
5463
|
status: 'completed',
|
|
@@ -5424,7 +5477,9 @@ export class PlayContextImpl {
|
|
|
5424
5477
|
});
|
|
5425
5478
|
retainedRowsMemoryTracker.track(completedRow);
|
|
5426
5479
|
completedRowsToPersist.push(completedRow);
|
|
5427
|
-
enqueueIncrementalPersist(completedRow)
|
|
5480
|
+
enqueueIncrementalPersist(completedRow, () => {
|
|
5481
|
+
updateMapFrameProgress({ completedRowKey: rowKey });
|
|
5482
|
+
});
|
|
5428
5483
|
return publicRow;
|
|
5429
5484
|
} catch (error) {
|
|
5430
5485
|
if (isPlayRowExecutionSuspendedError(error)) {
|
|
@@ -5595,6 +5650,11 @@ export class PlayContextImpl {
|
|
|
5595
5650
|
(result): result is Record<string, unknown> =>
|
|
5596
5651
|
result !== WAITING_ROW && result !== FAILED_ROW,
|
|
5597
5652
|
);
|
|
5653
|
+
// A row is acknowledged only after its terminal sheet batch commits. The
|
|
5654
|
+
// incremental promises publish their frame updates from the commit
|
|
5655
|
+
// continuation, so this barrier also makes the terminal map event derive
|
|
5656
|
+
// from durable row truth instead of resolver completion.
|
|
5657
|
+
await incrementalPersistence?.flush();
|
|
5598
5658
|
if (emitTerminalEvent) {
|
|
5599
5659
|
updateMapFrameProgress({
|
|
5600
5660
|
status: 'completed',
|
|
@@ -6079,7 +6139,10 @@ export class PlayContextImpl {
|
|
|
6079
6139
|
});
|
|
6080
6140
|
let executionAuthScopeDigest =
|
|
6081
6141
|
(await this.resolveToolAuthScopeDigest(toolId))?.trim() ?? null;
|
|
6082
|
-
const
|
|
6142
|
+
const eventWaitHandler =
|
|
6143
|
+
(await this.#options.getIntegrationEventWaitHandler?.(toolId)) ?? null;
|
|
6144
|
+
const cacheableToolResult =
|
|
6145
|
+
!eventWaitHandler && !isQueryResultDatasetReadRequest(toolId, input);
|
|
6083
6146
|
let durableCacheKey = await this.durableToolCallCacheKey({
|
|
6084
6147
|
toolId,
|
|
6085
6148
|
requestInput: input,
|
|
@@ -6087,8 +6150,6 @@ export class PlayContextImpl {
|
|
|
6087
6150
|
staleAfterSeconds: toolCachePolicy.staleAfterSeconds,
|
|
6088
6151
|
});
|
|
6089
6152
|
const checkpointCacheKeys = [durableCacheKey];
|
|
6090
|
-
const eventWaitHandler =
|
|
6091
|
-
(await this.#options.getIntegrationEventWaitHandler?.(toolId)) ?? null;
|
|
6092
6153
|
const store = rowContext.getStore();
|
|
6093
6154
|
|
|
6094
6155
|
const executeTool = async (context?: {
|
|
@@ -6304,12 +6365,10 @@ export class PlayContextImpl {
|
|
|
6304
6365
|
authScopeAttempt < 2;
|
|
6305
6366
|
authScopeAttempt += 1
|
|
6306
6367
|
) {
|
|
6307
|
-
// Capture the lease the receipt store hands to `execute` for THIS
|
|
6308
|
-
// attempt's key. On an auth-scope re-claim we mark that old-scope receipt
|
|
6309
|
-
// `skipped` (below), and the skip is fenced on this run+attempt+lease so
|
|
6310
|
-
// it can only supersede the receipt this attempt owns.
|
|
6311
|
-
let claimedReceiptLeaseId: string | null = null;
|
|
6312
6368
|
try {
|
|
6369
|
+
const toolRetryPolicy = await this.#options
|
|
6370
|
+
.getToolRetryPolicy?.(toolId)
|
|
6371
|
+
.catch(() => null);
|
|
6313
6372
|
return await this.executeWithRuntimeReceipt(
|
|
6314
6373
|
'tool',
|
|
6315
6374
|
normalizedKey,
|
|
@@ -6318,12 +6377,16 @@ export class PlayContextImpl {
|
|
|
6318
6377
|
receiptKey: durableCacheKey,
|
|
6319
6378
|
semanticKey: toolRequestIdentity,
|
|
6320
6379
|
force: toolCachePolicy.force,
|
|
6380
|
+
// Missing metadata is intentionally lock-free. Only explicitly
|
|
6381
|
+
// dangerous non-idempotent side effects enter the fence path.
|
|
6382
|
+
requiresExecutionLock:
|
|
6383
|
+
toolRetryPolicy?.requiresExecutionFence === true,
|
|
6384
|
+
executionLockTtlMs: Math.min(
|
|
6385
|
+
600_000,
|
|
6386
|
+
resolveToolRuntimeTimeoutMs(toolId, options?.timeoutMs) ??
|
|
6387
|
+
300_000,
|
|
6388
|
+
),
|
|
6321
6389
|
staleAfterSeconds: toolCachePolicy.staleAfterSeconds,
|
|
6322
|
-
markSkipped: () => {
|
|
6323
|
-
this.log(
|
|
6324
|
-
`ctx.tools.execute(${toolId}): no-op due completed receipt ${toolRequestIdentity} (label: ${normalizedKey})`,
|
|
6325
|
-
);
|
|
6326
|
-
},
|
|
6327
6390
|
onClaimedResult: (output, receiptKey) =>
|
|
6328
6391
|
markToolExecuteResultExecutionOutcome(output, {
|
|
6329
6392
|
kind: 'live',
|
|
@@ -6340,10 +6403,7 @@ export class PlayContextImpl {
|
|
|
6340
6403
|
shouldPersistFailure: (error) =>
|
|
6341
6404
|
!(error instanceof ToolExecuteAuthScopeChangedError),
|
|
6342
6405
|
markRunningBeforeExecute: false,
|
|
6343
|
-
execute: ({ leaseId }) => {
|
|
6344
|
-
claimedReceiptLeaseId = leaseId;
|
|
6345
|
-
return executeTool({ leaseId });
|
|
6346
|
-
},
|
|
6406
|
+
execute: ({ leaseId }) => executeTool({ leaseId }),
|
|
6347
6407
|
runningReceiptWaitMaxAttempts:
|
|
6348
6408
|
resolveRuntimeToolReceiptWaitMaxAttempts(
|
|
6349
6409
|
typeof options?.receiptWaitMs === 'number'
|
|
@@ -6359,36 +6419,9 @@ export class PlayContextImpl {
|
|
|
6359
6419
|
) {
|
|
6360
6420
|
throw error;
|
|
6361
6421
|
}
|
|
6362
|
-
//
|
|
6363
|
-
//
|
|
6364
|
-
//
|
|
6365
|
-
// the OLD cache key; without an explicit terminal it projects as a failed
|
|
6366
|
-
// node even though the work is about to be re-done under the new scope.
|
|
6367
|
-
// `skipped` is the terminal for superseded work; the skip is fenced on
|
|
6368
|
-
// this run+attempt+lease so it can only touch the receipt this attempt
|
|
6369
|
-
// owns. Best-effort: a skip that cannot fence (store returns null) must
|
|
6370
|
-
// not mask the reclaim itself.
|
|
6371
|
-
const supersededReceiptKey = durableCacheKey;
|
|
6372
|
-
try {
|
|
6373
|
-
const skipped = await this.skipRuntimeStepReceipt(
|
|
6374
|
-
supersededReceiptKey,
|
|
6375
|
-
this.currentRunId,
|
|
6376
|
-
claimedReceiptLeaseId,
|
|
6377
|
-
);
|
|
6378
|
-
if (skipped) {
|
|
6379
|
-
this.log(
|
|
6380
|
-
`ctx.tools.execute(${toolId}): auth_scope_changed_superseded ` +
|
|
6381
|
-
`old-scope receipt ${toolRequestIdentity} (label: ${normalizedKey}) ` +
|
|
6382
|
-
`marked skipped so the reclaim does not project as a failed node.`,
|
|
6383
|
-
);
|
|
6384
|
-
}
|
|
6385
|
-
} catch (skipError) {
|
|
6386
|
-
this.log(
|
|
6387
|
-
`ctx.tools.execute(${toolId}): auth_scope_changed_supersede_skip_failed ` +
|
|
6388
|
-
`(label: ${normalizedKey}); could not mark the old-scope receipt ` +
|
|
6389
|
-
`skipped: ${this.formatRuntimeError(skipError)}`,
|
|
6390
|
-
);
|
|
6391
|
-
}
|
|
6422
|
+
// Completed-receipt cache misses never publish a running receipt. The
|
|
6423
|
+
// old auth-scope attempt therefore has no ownership metadata to clean
|
|
6424
|
+
// up; re-key under the refreshed credential scope and retry once.
|
|
6392
6425
|
// Make the bounded auth-scope re-claim observable in run logs instead
|
|
6393
6426
|
// of a silent continuation: the credential identity changed after the
|
|
6394
6427
|
// receipt key was prepared, so we evict the cached digest, re-resolve,
|
|
@@ -6555,6 +6588,8 @@ export class PlayContextImpl {
|
|
|
6555
6588
|
}
|
|
6556
6589
|
const childExecutionDecision = resolveChildExecutionStrategy({
|
|
6557
6590
|
pipeline: resolvedPlay.staticPipeline,
|
|
6591
|
+
execution: options?.execution,
|
|
6592
|
+
childPlayName: resolvedName,
|
|
6558
6593
|
});
|
|
6559
6594
|
const launchThroughScheduler =
|
|
6560
6595
|
this.shouldLaunchChildPlaysThroughScheduler() &&
|
|
@@ -6699,6 +6734,7 @@ export class PlayContextImpl {
|
|
|
6699
6734
|
}
|
|
6700
6735
|
this.recordPlayCallStep({
|
|
6701
6736
|
playId: resolvedName,
|
|
6737
|
+
execution: options?.execution,
|
|
6702
6738
|
description: options?.description,
|
|
6703
6739
|
});
|
|
6704
6740
|
return checkpointResult.output;
|
|
@@ -6746,6 +6782,7 @@ export class PlayContextImpl {
|
|
|
6746
6782
|
}
|
|
6747
6783
|
this.recordPlayCallStep({
|
|
6748
6784
|
playId: resolvedName,
|
|
6785
|
+
execution: options?.execution,
|
|
6749
6786
|
description: options?.description,
|
|
6750
6787
|
});
|
|
6751
6788
|
return terminal.output;
|
|
@@ -6846,6 +6883,7 @@ export class PlayContextImpl {
|
|
|
6846
6883
|
}
|
|
6847
6884
|
this.recordPlayCallStep({
|
|
6848
6885
|
playId: resolvedName,
|
|
6886
|
+
execution: options?.execution,
|
|
6849
6887
|
description: options?.description,
|
|
6850
6888
|
nestedSteps: childContext.getSteps(),
|
|
6851
6889
|
});
|
|
@@ -7277,11 +7315,6 @@ export class PlayContextImpl {
|
|
|
7277
7315
|
egressSlot.release();
|
|
7278
7316
|
}
|
|
7279
7317
|
},
|
|
7280
|
-
markSkipped: (output) => {
|
|
7281
|
-
this.log(
|
|
7282
|
-
`ctx.fetch(${output?.url ?? ''}): no-op due completed receipt`,
|
|
7283
|
-
);
|
|
7284
|
-
},
|
|
7285
7318
|
},
|
|
7286
7319
|
);
|
|
7287
7320
|
}
|
|
@@ -7348,7 +7381,6 @@ export class PlayContextImpl {
|
|
|
7348
7381
|
: options?.semanticKey,
|
|
7349
7382
|
staleAfterSeconds: options?.staleAfterSeconds,
|
|
7350
7383
|
markSkipped: (output) => {
|
|
7351
|
-
this.log(`ctx.step(${normalizedKey}): no-op due completed receipt`);
|
|
7352
7384
|
assertJsonSerializableStepOutput(normalizedKey, output);
|
|
7353
7385
|
},
|
|
7354
7386
|
execute: executeStep,
|
|
@@ -8500,6 +8532,8 @@ export class PlayContextImpl {
|
|
|
8500
8532
|
|
|
8501
8533
|
while (true) {
|
|
8502
8534
|
let response: Response;
|
|
8535
|
+
let providerCallStartedAt: number | null = null;
|
|
8536
|
+
let providerCallElapsedMs: number | null = null;
|
|
8503
8537
|
const durableCallReceiptKey =
|
|
8504
8538
|
options?.durableCallReceiptKey?.trim() || null;
|
|
8505
8539
|
const executionAuthScopeDigest =
|
|
@@ -8533,7 +8567,6 @@ export class PlayContextImpl {
|
|
|
8533
8567
|
}, timeoutMs)
|
|
8534
8568
|
: null;
|
|
8535
8569
|
try {
|
|
8536
|
-
const providerCallStartedAt = Date.now();
|
|
8537
8570
|
const ownershipStartedAt = Date.now();
|
|
8538
8571
|
await options?.beforeProviderCall?.();
|
|
8539
8572
|
if (runtimeReceiptReadTraceEnabled) {
|
|
@@ -8541,6 +8574,7 @@ export class PlayContextImpl {
|
|
|
8541
8574
|
`[perf] tool call id=${toolId} phase=before_provider_ownership elapsed_ms=${Date.now() - ownershipStartedAt}`,
|
|
8542
8575
|
);
|
|
8543
8576
|
}
|
|
8577
|
+
providerCallStartedAt = Date.now();
|
|
8544
8578
|
const heartbeatIntervalMs = hasReceiptHeartbeat
|
|
8545
8579
|
? runtimeLeaseHeartbeatIntervalFromExpiry({
|
|
8546
8580
|
leaseExpiresAt: options?.receiptLeaseExpiresAt,
|
|
@@ -8640,10 +8674,11 @@ export class PlayContextImpl {
|
|
|
8640
8674
|
`[perf] tool call id=${toolId} phase=integration_fetch_headers elapsed_ms=${Date.now() - integrationFetchStartedAt} status=${response.status}`,
|
|
8641
8675
|
);
|
|
8642
8676
|
}
|
|
8677
|
+
providerCallElapsedMs = Date.now() - providerCallStartedAt;
|
|
8643
8678
|
this.resourceGovernor.observe({
|
|
8644
8679
|
toolId,
|
|
8645
8680
|
providerResourceKey: `tool:${toolId}`,
|
|
8646
|
-
providerLatencyMs:
|
|
8681
|
+
providerLatencyMs: providerCallElapsedMs,
|
|
8647
8682
|
providerSuccess: response.ok,
|
|
8648
8683
|
provider429: response.status === 429,
|
|
8649
8684
|
});
|
|
@@ -8654,16 +8689,18 @@ export class PlayContextImpl {
|
|
|
8654
8689
|
if (heartbeatFailure) {
|
|
8655
8690
|
throw heartbeatFailure;
|
|
8656
8691
|
}
|
|
8657
|
-
|
|
8658
|
-
|
|
8692
|
+
const transportError = abortController?.signal.aborted
|
|
8693
|
+
? abortController.signal.reason instanceof Error
|
|
8659
8694
|
? abortController.signal.reason
|
|
8660
8695
|
: new Error(
|
|
8661
8696
|
`Tool ${toolId} runtime API call timed out after ${timeoutMs}ms.`,
|
|
8662
|
-
)
|
|
8663
|
-
|
|
8697
|
+
)
|
|
8698
|
+
: error;
|
|
8664
8699
|
transportAttempt += 1;
|
|
8665
8700
|
const message =
|
|
8666
|
-
|
|
8701
|
+
transportError instanceof Error
|
|
8702
|
+
? transportError.message
|
|
8703
|
+
: String(transportError);
|
|
8667
8704
|
if (transportAttempt < TOOL_EXECUTE_TRANSPORT_MAX_ATTEMPTS) {
|
|
8668
8705
|
await this.governor.chargeBudget('retry');
|
|
8669
8706
|
const retryAfterMs =
|
|
@@ -8679,8 +8716,11 @@ export class PlayContextImpl {
|
|
|
8679
8716
|
await this.sleepWithCheckpointHeartbeat(retryAfterMs);
|
|
8680
8717
|
continue;
|
|
8681
8718
|
}
|
|
8682
|
-
throw new
|
|
8719
|
+
throw new ToolHttpError(
|
|
8683
8720
|
`Tool ${toolId} transport failed calling ${url} after ${transportAttempt}/${TOOL_EXECUTE_TRANSPORT_MAX_ATTEMPTS} attempts: ${message}`,
|
|
8721
|
+
null,
|
|
8722
|
+
0,
|
|
8723
|
+
'repairable',
|
|
8684
8724
|
);
|
|
8685
8725
|
} finally {
|
|
8686
8726
|
if (timeoutHandle) {
|
|
@@ -8711,6 +8751,9 @@ export class PlayContextImpl {
|
|
|
8711
8751
|
bodyText: text,
|
|
8712
8752
|
retryAfterHeader: response.headers.get('retry-after'),
|
|
8713
8753
|
transientHttpRetrySafe: retrySafeTransientHttp,
|
|
8754
|
+
...(providerCallElapsedMs !== null
|
|
8755
|
+
? { providerLatencyMs: providerCallElapsedMs }
|
|
8756
|
+
: {}),
|
|
8714
8757
|
});
|
|
8715
8758
|
if (failure.backpressureDelayMs !== null) {
|
|
8716
8759
|
// Feed the server-observed Retry-After back into the shared
|
|
@@ -214,6 +214,7 @@ export type PlayStep =
|
|
|
214
214
|
| {
|
|
215
215
|
type: 'play_call';
|
|
216
216
|
playId: string;
|
|
217
|
+
execution?: 'inline' | 'child-workflow';
|
|
217
218
|
results?: PlayStepRowResult[];
|
|
218
219
|
nestedSteps: PlayStep[];
|
|
219
220
|
description?: string;
|
|
@@ -252,6 +253,7 @@ export type PlayDatasetSubstep =
|
|
|
252
253
|
| {
|
|
253
254
|
type: 'play_call';
|
|
254
255
|
playId: string;
|
|
256
|
+
execution?: 'inline' | 'child-workflow';
|
|
255
257
|
results?: PlayStepRowResult[];
|
|
256
258
|
nestedSteps: PlayStep[];
|
|
257
259
|
description?: string;
|