deepline 0.1.212 → 0.1.214
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/apps/play-runner-workers/src/entry.ts +167 -329
- package/dist/bundling-sources/sdk/src/client.ts +2 -2
- package/dist/bundling-sources/sdk/src/index.ts +2 -0
- package/dist/bundling-sources/sdk/src/play.ts +8 -1
- package/dist/bundling-sources/sdk/src/plays/harness-stub.ts +11 -1
- package/dist/bundling-sources/sdk/src/release.ts +2 -2
- package/dist/bundling-sources/sdk/src/types.ts +3 -3
- package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +49 -0
- package/dist/bundling-sources/shared_libs/play-runtime/child-execution-placement.ts +14 -0
- package/dist/bundling-sources/shared_libs/play-runtime/child-execution-strategy.ts +57 -23
- package/dist/bundling-sources/shared_libs/play-runtime/completed-receipt-cache.ts +166 -0
- package/dist/bundling-sources/shared_libs/play-runtime/context.ts +209 -123
- package/dist/bundling-sources/shared_libs/play-runtime/ctx-contract.ts +2 -0
- package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +30 -1
- package/dist/bundling-sources/shared_libs/play-runtime/durable-receipt-execution.ts +160 -2
- package/dist/bundling-sources/shared_libs/play-runtime/gateway-auth-session.ts +93 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-call-execution.ts +1 -0
- package/dist/bundling-sources/shared_libs/play-runtime/providers.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/receipt-completion-sink.ts +16 -1
- package/dist/bundling-sources/shared_libs/play-runtime/receipt-state-sink.ts +10 -0
- package/dist/bundling-sources/shared_libs/play-runtime/resource-governor.ts +80 -3
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-session-execution.ts +94 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts +54 -1
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +477 -85
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-postgres-admission.ts +162 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backend.ts +2 -2
- package/dist/bundling-sources/shared_libs/play-runtime/sheet-attempt-sql.ts +16 -17
- package/dist/bundling-sources/shared_libs/play-runtime/work-receipts.ts +1 -0
- package/dist/bundling-sources/shared_libs/plays/static-pipeline.ts +2 -4
- package/dist/cli/index.js +50 -436
- package/dist/cli/index.mjs +50 -436
- package/dist/{compiler-manifest-j6z8qzpd.d.mts → compiler-manifest-BhgZ23A4.d.mts} +1 -0
- package/dist/{compiler-manifest-j6z8qzpd.d.ts → compiler-manifest-BhgZ23A4.d.ts} +1 -0
- package/dist/index.d.mts +11 -8
- package/dist/index.d.ts +11 -8
- package/dist/index.js +4 -4
- package/dist/index.mjs +4 -4
- package/dist/plays/bundle-play-file.d.mts +2 -2
- package/dist/plays/bundle-play-file.d.ts +2 -2
- package/dist/plays/bundle-play-file.mjs +1 -1
- package/package.json +1 -1
|
@@ -40,7 +40,6 @@ import {
|
|
|
40
40
|
mapRowOutcomeRuntimeFields,
|
|
41
41
|
resolveMapRowOutcomeKey,
|
|
42
42
|
} from './map-row-outcome';
|
|
43
|
-
import { RuntimeSheetRowsBlockedError } from './runtime-sheet-errors';
|
|
44
43
|
import { buildChildRunId } from './child-run-id';
|
|
45
44
|
import type { PlayCallGovernanceSnapshot } from './scheduler-backend';
|
|
46
45
|
import {
|
|
@@ -285,6 +284,11 @@ const TOOL_RETRY_HEARTBEAT_INTERVAL_MS = 30_000;
|
|
|
285
284
|
const DEEPLINEAGENT_TOOL_RUNTIME_TIMEOUT_MS = 15 * 60 * 1000;
|
|
286
285
|
const FETCH_TRANSPORT_MAX_ATTEMPTS = 3;
|
|
287
286
|
const FETCH_TRANSPORT_RETRY_DELAY_MS = 100;
|
|
287
|
+
// cloudflared returns this branded HTML when its connection to the local app
|
|
288
|
+
// resets. It is a transport failure before the app can reach a provider, not a
|
|
289
|
+
// provider 502; require both the branded page and our development tunnel host.
|
|
290
|
+
const DEEPLINE_DEVELOPER_TUNNEL_ORIGIN_502_PATTERN =
|
|
291
|
+
/\bdeeplinedeveloper\.com\s*\|\s*502\s*:\s*bad gateway\b/i;
|
|
288
292
|
const CHILD_PLAY_LAUNCH_WRITE_CONFLICT_RETRY_DELAYS_MS = [
|
|
289
293
|
50, 100, 200, 400, 800,
|
|
290
294
|
] as const;
|
|
@@ -519,6 +523,28 @@ export function resolveToolRuntimeTimeoutMs(
|
|
|
519
523
|
: undefined;
|
|
520
524
|
}
|
|
521
525
|
|
|
526
|
+
function isDeeplineDeveloperTunnelOrigin502(input: {
|
|
527
|
+
url: string;
|
|
528
|
+
status: number;
|
|
529
|
+
bodyText: string;
|
|
530
|
+
}): boolean {
|
|
531
|
+
if (
|
|
532
|
+
input.status !== 502 ||
|
|
533
|
+
!DEEPLINE_DEVELOPER_TUNNEL_ORIGIN_502_PATTERN.test(input.bodyText)
|
|
534
|
+
) {
|
|
535
|
+
return false;
|
|
536
|
+
}
|
|
537
|
+
try {
|
|
538
|
+
const hostname = new URL(input.url).hostname.toLowerCase();
|
|
539
|
+
return (
|
|
540
|
+
hostname === 'deeplinedeveloper.com' ||
|
|
541
|
+
hostname.endsWith('.deeplinedeveloper.com')
|
|
542
|
+
);
|
|
543
|
+
} catch {
|
|
544
|
+
return false;
|
|
545
|
+
}
|
|
546
|
+
}
|
|
547
|
+
|
|
522
548
|
function loadSafeFetch(): Promise<SafeFetchModule> {
|
|
523
549
|
safeFetchModule ??= import('@shared_libs/security/safe-fetch');
|
|
524
550
|
return safeFetchModule;
|
|
@@ -1521,12 +1547,14 @@ export class PlayContextImpl {
|
|
|
1521
1547
|
|
|
1522
1548
|
private recordPlayCallStep(input: {
|
|
1523
1549
|
playId: string;
|
|
1550
|
+
execution?: 'inline' | 'child-workflow';
|
|
1524
1551
|
description?: string | null;
|
|
1525
1552
|
nestedSteps?: PlayStep[];
|
|
1526
1553
|
}): void {
|
|
1527
1554
|
const step = {
|
|
1528
1555
|
type: 'play_call' as const,
|
|
1529
1556
|
playId: input.playId,
|
|
1557
|
+
...(input.execution ? { execution: input.execution } : {}),
|
|
1530
1558
|
nestedSteps: input.nestedSteps ?? [],
|
|
1531
1559
|
description: normalizeStepDescription(input.description ?? undefined),
|
|
1532
1560
|
};
|
|
@@ -1903,6 +1931,7 @@ export class PlayContextImpl {
|
|
|
1903
1931
|
}
|
|
1904
1932
|
const claimed = await this.#options.claimRuntimeStepReceipt({
|
|
1905
1933
|
key,
|
|
1934
|
+
leaseId: `receipt-lease:${crypto.randomUUID()}`,
|
|
1906
1935
|
runId: this.currentReceiptOwnerRunId,
|
|
1907
1936
|
runAttempt: this.currentRunAttempt,
|
|
1908
1937
|
leaseAware: true,
|
|
@@ -1979,30 +2008,6 @@ export class PlayContextImpl {
|
|
|
1979
2008
|
};
|
|
1980
2009
|
}
|
|
1981
2010
|
|
|
1982
|
-
private async skipRuntimeStepReceipt(
|
|
1983
|
-
key: string,
|
|
1984
|
-
runId: string,
|
|
1985
|
-
leaseId?: string | null,
|
|
1986
|
-
): Promise<RuntimeStepReceipt | null> {
|
|
1987
|
-
if (!this.#options.skipRuntimeStepReceipt) {
|
|
1988
|
-
return null;
|
|
1989
|
-
}
|
|
1990
|
-
const skipped = await this.#options.skipRuntimeStepReceipt({
|
|
1991
|
-
key,
|
|
1992
|
-
runId,
|
|
1993
|
-
runAttempt: this.currentRunAttempt,
|
|
1994
|
-
...(leaseId ? { leaseId } : {}),
|
|
1995
|
-
});
|
|
1996
|
-
if (!skipped || typeof skipped.key !== 'string' || !skipped.key.trim()) {
|
|
1997
|
-
return null;
|
|
1998
|
-
}
|
|
1999
|
-
return {
|
|
2000
|
-
...skipped,
|
|
2001
|
-
key: skipped.key.trim(),
|
|
2002
|
-
runId: skipped.runId ?? null,
|
|
2003
|
-
};
|
|
2004
|
-
}
|
|
2005
|
-
|
|
2006
2011
|
private async heartbeatRuntimeStepReceipt(
|
|
2007
2012
|
key: string,
|
|
2008
2013
|
_runId: string,
|
|
@@ -2310,6 +2315,11 @@ export class PlayContextImpl {
|
|
|
2310
2315
|
? await this.dispatchChunkedRuntimeReceiptRequest(uniqueKeys, (chunk) =>
|
|
2311
2316
|
claimReceipts({
|
|
2312
2317
|
keys: chunk,
|
|
2318
|
+
// A transport retry can replay this mutation after the store
|
|
2319
|
+
// committed but before the response body was consumed. Stable
|
|
2320
|
+
// caller-owned tokens distinguish that replay from a concurrent
|
|
2321
|
+
// claimant without weakening the provider-call execution fence.
|
|
2322
|
+
leaseIds: chunk.map(() => `receipt-lease:${crypto.randomUUID()}`),
|
|
2313
2323
|
runId: this.currentReceiptOwnerRunId,
|
|
2314
2324
|
runAttempt: this.currentRunAttempt,
|
|
2315
2325
|
leaseAware: true,
|
|
@@ -2777,6 +2787,24 @@ export class PlayContextImpl {
|
|
|
2777
2787
|
this.#options.completeRuntimeStepReceipt ||
|
|
2778
2788
|
this.#options.completeRuntimeStepReceipts,
|
|
2779
2789
|
),
|
|
2790
|
+
...(this.#options.acquireRuntimeReceiptExecutionLock &&
|
|
2791
|
+
this.#options.releaseRuntimeReceiptExecutionLock
|
|
2792
|
+
? {
|
|
2793
|
+
acquireExecutionLock: ({ receiptKey, ownerExecutionId, ttlMs }) =>
|
|
2794
|
+
this.#options.acquireRuntimeReceiptExecutionLock!({
|
|
2795
|
+
key: receiptKey,
|
|
2796
|
+
runId: this.currentReceiptOwnerRunId,
|
|
2797
|
+
ownerExecutionId,
|
|
2798
|
+
ttlMs,
|
|
2799
|
+
}),
|
|
2800
|
+
releaseExecutionLock: ({ receiptKey, ownerExecutionId }) =>
|
|
2801
|
+
this.#options.releaseRuntimeReceiptExecutionLock!({
|
|
2802
|
+
key: receiptKey,
|
|
2803
|
+
runId: this.currentReceiptOwnerRunId,
|
|
2804
|
+
ownerExecutionId,
|
|
2805
|
+
}),
|
|
2806
|
+
}
|
|
2807
|
+
: {}),
|
|
2780
2808
|
};
|
|
2781
2809
|
}
|
|
2782
2810
|
|
|
@@ -2834,6 +2862,8 @@ export class PlayContextImpl {
|
|
|
2834
2862
|
onClaimedResult?: (output: T, receiptKey: string) => T;
|
|
2835
2863
|
shouldPersistFailure?: (error: unknown) => boolean;
|
|
2836
2864
|
markRunningBeforeExecute?: boolean;
|
|
2865
|
+
requiresExecutionLock?: boolean;
|
|
2866
|
+
executionLockTtlMs?: number;
|
|
2837
2867
|
execute: (context: { leaseId: string | null }) => Promise<T>;
|
|
2838
2868
|
},
|
|
2839
2869
|
): Promise<T> {
|
|
@@ -2865,6 +2895,9 @@ export class PlayContextImpl {
|
|
|
2865
2895
|
onClaimedResult: opts.onClaimedResult,
|
|
2866
2896
|
shouldPersistFailure: opts.shouldPersistFailure,
|
|
2867
2897
|
markRunningBeforeExecute: opts.markRunningBeforeExecute,
|
|
2898
|
+
completedCacheOnly: operation === 'tool',
|
|
2899
|
+
requiresExecutionLock: opts.requiresExecutionLock,
|
|
2900
|
+
executionLockTtlMs: opts.executionLockTtlMs,
|
|
2868
2901
|
formatError: (error) => this.formatRuntimeError(error),
|
|
2869
2902
|
log: (message) => this.log(message),
|
|
2870
2903
|
execute: opts.execute,
|
|
@@ -4016,12 +4049,6 @@ export class PlayContextImpl {
|
|
|
4016
4049
|
resolvedTableNamespace = normalizeTableNamespace(
|
|
4017
4050
|
mapStartResult.tableNamespace,
|
|
4018
4051
|
);
|
|
4019
|
-
if ((mapStartResult.blockedRows?.length ?? 0) > 0) {
|
|
4020
|
-
throw new RuntimeSheetRowsBlockedError({
|
|
4021
|
-
tableNamespace: resolvedTableNamespace,
|
|
4022
|
-
blockedRows: mapStartResult.blockedRows ?? [],
|
|
4023
|
-
});
|
|
4024
|
-
}
|
|
4025
4052
|
|
|
4026
4053
|
const persistedRowIdentity = (
|
|
4027
4054
|
row: Record<string, unknown>,
|
|
@@ -4456,12 +4483,6 @@ export class PlayContextImpl {
|
|
|
4456
4483
|
resolvedTableNamespace = normalizeTableNamespace(
|
|
4457
4484
|
mapStartResult.tableNamespace,
|
|
4458
4485
|
);
|
|
4459
|
-
if ((mapStartResult.blockedRows?.length ?? 0) > 0) {
|
|
4460
|
-
throw new RuntimeSheetRowsBlockedError({
|
|
4461
|
-
tableNamespace: resolvedTableNamespace,
|
|
4462
|
-
blockedRows: mapStartResult.blockedRows ?? [],
|
|
4463
|
-
});
|
|
4464
|
-
}
|
|
4465
4486
|
const persistedRowIdentity = (row: Record<string, unknown>, index = 0) =>
|
|
4466
4487
|
resolveMapRowOutcomeKey(row) ?? rowIdentity(row, index);
|
|
4467
4488
|
const pendingRowsByKey = new Map<string, Record<string, unknown>>();
|
|
@@ -4574,6 +4595,7 @@ export class PlayContextImpl {
|
|
|
4574
4595
|
);
|
|
4575
4596
|
|
|
4576
4597
|
this.activeMapCellMeta = new Map();
|
|
4598
|
+
const staleCompletionKeys = new Set<string>();
|
|
4577
4599
|
const persistMapRows = async (rows: PersistableMapRow[]) => {
|
|
4578
4600
|
if (!this.#options.onMapRowsCompleted || rows.length === 0) {
|
|
4579
4601
|
return;
|
|
@@ -4597,7 +4619,7 @@ export class PlayContextImpl {
|
|
|
4597
4619
|
);
|
|
4598
4620
|
const flushStartedAt = Date.now();
|
|
4599
4621
|
try {
|
|
4600
|
-
await this.#options.onMapRowsCompleted!({
|
|
4622
|
+
const writeResult = await this.#options.onMapRowsCompleted!({
|
|
4601
4623
|
playName: this.#options.playName,
|
|
4602
4624
|
playId: this.#options.playId,
|
|
4603
4625
|
runId: this.#options.runId,
|
|
@@ -4609,6 +4631,11 @@ export class PlayContextImpl {
|
|
|
4609
4631
|
),
|
|
4610
4632
|
staticPipeline: this.#options.staticPipeline ?? null,
|
|
4611
4633
|
});
|
|
4634
|
+
if (writeResult) {
|
|
4635
|
+
for (const key of writeResult.staleDroppedKeys ?? []) {
|
|
4636
|
+
staleCompletionKeys.add(key);
|
|
4637
|
+
}
|
|
4638
|
+
}
|
|
4612
4639
|
this.resourceGovernor.observe({
|
|
4613
4640
|
sheetFlushBytes: chunkBytes,
|
|
4614
4641
|
sheetFlushLatencyMs: Date.now() - flushStartedAt,
|
|
@@ -4654,6 +4681,7 @@ export class PlayContextImpl {
|
|
|
4654
4681
|
completedRows: duplicateReuseCount,
|
|
4655
4682
|
},
|
|
4656
4683
|
{
|
|
4684
|
+
emitTerminalEvent: false,
|
|
4657
4685
|
onRowError: options?.onRowError,
|
|
4658
4686
|
executionRowKeys: rowsToExecuteEntries.map((entry) => entry.rowKey),
|
|
4659
4687
|
executionRowIndexes: rowsToExecuteEntries.map(
|
|
@@ -4751,7 +4779,49 @@ export class PlayContextImpl {
|
|
|
4751
4779
|
}
|
|
4752
4780
|
this.activeMapCellMeta = null;
|
|
4753
4781
|
|
|
4754
|
-
if (
|
|
4782
|
+
if (staleCompletionKeys.size > 0) {
|
|
4783
|
+
results = results.filter((row, index) => {
|
|
4784
|
+
const key = rowIdentity(row, itemOriginalIndexes[index] ?? index);
|
|
4785
|
+
return !staleCompletionKeys.has(key);
|
|
4786
|
+
});
|
|
4787
|
+
}
|
|
4788
|
+
|
|
4789
|
+
const durableCompletedRows = Math.max(
|
|
4790
|
+
0,
|
|
4791
|
+
reusedCount + mapResult.completedRows.length - staleCompletionKeys.size,
|
|
4792
|
+
);
|
|
4793
|
+
const durableFailedRows = mapResult.failedRows.length;
|
|
4794
|
+
const terminalFrame = this.checkpoint.mapFrames?.[mapScope.mapInvocationId];
|
|
4795
|
+
if (terminalFrame) {
|
|
4796
|
+
this.setMapFrame({
|
|
4797
|
+
...terminalFrame,
|
|
4798
|
+
status: 'completed',
|
|
4799
|
+
// Incremental rows add keys after their commit. Pure maps use the
|
|
4800
|
+
// durable aggregate count below instead of materializing up to millions
|
|
4801
|
+
// of keys only to mark the terminal frame.
|
|
4802
|
+
completedRowKeys: terminalFrame.completedRowKeys,
|
|
4803
|
+
pendingRowKeys: [],
|
|
4804
|
+
completedRowsCount: durableCompletedRows,
|
|
4805
|
+
pendingRowsCount: 0,
|
|
4806
|
+
failedRowsCount: durableFailedRows,
|
|
4807
|
+
activeBoundaryId: null,
|
|
4808
|
+
updatedAt: Date.now(),
|
|
4809
|
+
});
|
|
4810
|
+
}
|
|
4811
|
+
this.emitExecutionEvent({
|
|
4812
|
+
type: 'map.completed',
|
|
4813
|
+
mapInvocationId: mapScope.mapInvocationId,
|
|
4814
|
+
mapNodeId: mapScope.mapNodeId ?? null,
|
|
4815
|
+
logicalNamespace: mapScope.logicalNamespace,
|
|
4816
|
+
artifactTableNamespace: resolvedTableNamespace,
|
|
4817
|
+
completedRows: durableCompletedRows,
|
|
4818
|
+
failedRows: durableFailedRows,
|
|
4819
|
+
totalRows: totalInputCount,
|
|
4820
|
+
...this.inlineChildAggregateEventFields(),
|
|
4821
|
+
at: Date.now(),
|
|
4822
|
+
});
|
|
4823
|
+
|
|
4824
|
+
if (this.#options.onMapRowsCompleted && staleCompletionKeys.size === 0) {
|
|
4755
4825
|
results = await reconcileNodeRuntimeMapResultsWithPersistedSheet({
|
|
4756
4826
|
mapName: normalizedMapNamespace,
|
|
4757
4827
|
tableNamespace: resolvedTableNamespace,
|
|
@@ -4988,11 +5058,21 @@ export class PlayContextImpl {
|
|
|
4988
5058
|
mapName: normalizedTableNamespace,
|
|
4989
5059
|
budgetBytes: runtimeOptions?.retainedRowsMemoryBudgetBytes,
|
|
4990
5060
|
});
|
|
4991
|
-
const enqueueIncrementalPersist = (
|
|
4992
|
-
|
|
4993
|
-
|
|
4994
|
-
|
|
4995
|
-
|
|
5061
|
+
const enqueueIncrementalPersist = (
|
|
5062
|
+
row: PersistableMapRow,
|
|
5063
|
+
onCommitted: () => void,
|
|
5064
|
+
): void => {
|
|
5065
|
+
if (!incrementalPersistence) {
|
|
5066
|
+
onCommitted();
|
|
5067
|
+
return;
|
|
5068
|
+
}
|
|
5069
|
+
void incrementalPersistence
|
|
5070
|
+
.persistRows([row])
|
|
5071
|
+
.then(onCommitted)
|
|
5072
|
+
.catch(() => {
|
|
5073
|
+
// The final map-level flush awaits the same chain and surfaces the
|
|
5074
|
+
// persistence failure loudly without holding a completed row slot open.
|
|
5075
|
+
});
|
|
4996
5076
|
};
|
|
4997
5077
|
|
|
4998
5078
|
if (completedRows > 0 || pendingRows !== totalRows) {
|
|
@@ -5158,14 +5238,16 @@ export class PlayContextImpl {
|
|
|
5158
5238
|
// this replaces ran AFTER every row had already computed, emitting 150k
|
|
5159
5239
|
// post-hoc map.progress events and re-copying the completed-keys array
|
|
5160
5240
|
// per row — all theater, no live progress value.
|
|
5161
|
-
|
|
5162
|
-
|
|
5163
|
-
|
|
5164
|
-
|
|
5165
|
-
|
|
5241
|
+
if (!incrementalPersistence) {
|
|
5242
|
+
updateMapFrameProgress({
|
|
5243
|
+
completedRowKeys: results.map((_row, index) =>
|
|
5244
|
+
executionRowKey(
|
|
5245
|
+
this.toOutputRow(items[index] as Record<string, unknown>),
|
|
5246
|
+
index,
|
|
5247
|
+
),
|
|
5166
5248
|
),
|
|
5167
|
-
)
|
|
5168
|
-
}
|
|
5249
|
+
});
|
|
5250
|
+
}
|
|
5169
5251
|
if (emitTerminalEvent) {
|
|
5170
5252
|
updateMapFrameProgress({
|
|
5171
5253
|
status: 'completed',
|
|
@@ -5346,9 +5428,6 @@ export class PlayContextImpl {
|
|
|
5346
5428
|
});
|
|
5347
5429
|
retainedRowsMemoryTracker.track(failedRow);
|
|
5348
5430
|
failedRowsToPersist.push(failedRow);
|
|
5349
|
-
updateMapFrameProgress({
|
|
5350
|
-
failedRowKey: rowKey,
|
|
5351
|
-
});
|
|
5352
5431
|
this.emitScopedRowUpdate(rowKey, normalizedTableNamespace, {
|
|
5353
5432
|
rowId: idx,
|
|
5354
5433
|
status: 'failed',
|
|
@@ -5357,7 +5436,9 @@ export class PlayContextImpl {
|
|
|
5357
5436
|
error: formattedError,
|
|
5358
5437
|
dataPatch: {},
|
|
5359
5438
|
});
|
|
5360
|
-
enqueueIncrementalPersist(failedRow)
|
|
5439
|
+
enqueueIncrementalPersist(failedRow, () => {
|
|
5440
|
+
updateMapFrameProgress({ failedRowKey: rowKey });
|
|
5441
|
+
});
|
|
5361
5442
|
return FAILED_ROW;
|
|
5362
5443
|
}
|
|
5363
5444
|
const cellValue = this.serializeCellValue(value);
|
|
@@ -5404,9 +5485,6 @@ export class PlayContextImpl {
|
|
|
5404
5485
|
|
|
5405
5486
|
const merged = cloneCsvAliasedRow(baseRow, computedFields);
|
|
5406
5487
|
activeFieldName = null;
|
|
5407
|
-
updateMapFrameProgress({
|
|
5408
|
-
completedRowKey: rowKey,
|
|
5409
|
-
});
|
|
5410
5488
|
this.emitScopedRowUpdate(rowKey, normalizedTableNamespace, {
|
|
5411
5489
|
rowId: idx,
|
|
5412
5490
|
status: 'completed',
|
|
@@ -5426,7 +5504,9 @@ export class PlayContextImpl {
|
|
|
5426
5504
|
});
|
|
5427
5505
|
retainedRowsMemoryTracker.track(completedRow);
|
|
5428
5506
|
completedRowsToPersist.push(completedRow);
|
|
5429
|
-
enqueueIncrementalPersist(completedRow)
|
|
5507
|
+
enqueueIncrementalPersist(completedRow, () => {
|
|
5508
|
+
updateMapFrameProgress({ completedRowKey: rowKey });
|
|
5509
|
+
});
|
|
5430
5510
|
return publicRow;
|
|
5431
5511
|
} catch (error) {
|
|
5432
5512
|
if (isPlayRowExecutionSuspendedError(error)) {
|
|
@@ -5597,6 +5677,11 @@ export class PlayContextImpl {
|
|
|
5597
5677
|
(result): result is Record<string, unknown> =>
|
|
5598
5678
|
result !== WAITING_ROW && result !== FAILED_ROW,
|
|
5599
5679
|
);
|
|
5680
|
+
// A row is acknowledged only after its terminal sheet batch commits. The
|
|
5681
|
+
// incremental promises publish their frame updates from the commit
|
|
5682
|
+
// continuation, so this barrier also makes the terminal map event derive
|
|
5683
|
+
// from durable row truth instead of resolver completion.
|
|
5684
|
+
await incrementalPersistence?.flush();
|
|
5600
5685
|
if (emitTerminalEvent) {
|
|
5601
5686
|
updateMapFrameProgress({
|
|
5602
5687
|
status: 'completed',
|
|
@@ -6081,7 +6166,10 @@ export class PlayContextImpl {
|
|
|
6081
6166
|
});
|
|
6082
6167
|
let executionAuthScopeDigest =
|
|
6083
6168
|
(await this.resolveToolAuthScopeDigest(toolId))?.trim() ?? null;
|
|
6084
|
-
const
|
|
6169
|
+
const eventWaitHandler =
|
|
6170
|
+
(await this.#options.getIntegrationEventWaitHandler?.(toolId)) ?? null;
|
|
6171
|
+
const cacheableToolResult =
|
|
6172
|
+
!eventWaitHandler && !isQueryResultDatasetReadRequest(toolId, input);
|
|
6085
6173
|
let durableCacheKey = await this.durableToolCallCacheKey({
|
|
6086
6174
|
toolId,
|
|
6087
6175
|
requestInput: input,
|
|
@@ -6089,8 +6177,6 @@ export class PlayContextImpl {
|
|
|
6089
6177
|
staleAfterSeconds: toolCachePolicy.staleAfterSeconds,
|
|
6090
6178
|
});
|
|
6091
6179
|
const checkpointCacheKeys = [durableCacheKey];
|
|
6092
|
-
const eventWaitHandler =
|
|
6093
|
-
(await this.#options.getIntegrationEventWaitHandler?.(toolId)) ?? null;
|
|
6094
6180
|
const store = rowContext.getStore();
|
|
6095
6181
|
|
|
6096
6182
|
const executeTool = async (context?: {
|
|
@@ -6306,12 +6392,10 @@ export class PlayContextImpl {
|
|
|
6306
6392
|
authScopeAttempt < 2;
|
|
6307
6393
|
authScopeAttempt += 1
|
|
6308
6394
|
) {
|
|
6309
|
-
// Capture the lease the receipt store hands to `execute` for THIS
|
|
6310
|
-
// attempt's key. On an auth-scope re-claim we mark that old-scope receipt
|
|
6311
|
-
// `skipped` (below), and the skip is fenced on this run+attempt+lease so
|
|
6312
|
-
// it can only supersede the receipt this attempt owns.
|
|
6313
|
-
let claimedReceiptLeaseId: string | null = null;
|
|
6314
6395
|
try {
|
|
6396
|
+
const toolRetryPolicy = await this.#options
|
|
6397
|
+
.getToolRetryPolicy?.(toolId)
|
|
6398
|
+
.catch(() => null);
|
|
6315
6399
|
return await this.executeWithRuntimeReceipt(
|
|
6316
6400
|
'tool',
|
|
6317
6401
|
normalizedKey,
|
|
@@ -6320,6 +6404,15 @@ export class PlayContextImpl {
|
|
|
6320
6404
|
receiptKey: durableCacheKey,
|
|
6321
6405
|
semanticKey: toolRequestIdentity,
|
|
6322
6406
|
force: toolCachePolicy.force,
|
|
6407
|
+
// Missing metadata is intentionally lock-free. Only explicitly
|
|
6408
|
+
// dangerous non-idempotent side effects enter the fence path.
|
|
6409
|
+
requiresExecutionLock:
|
|
6410
|
+
toolRetryPolicy?.requiresExecutionFence === true,
|
|
6411
|
+
executionLockTtlMs: Math.min(
|
|
6412
|
+
600_000,
|
|
6413
|
+
resolveToolRuntimeTimeoutMs(toolId, options?.timeoutMs) ??
|
|
6414
|
+
300_000,
|
|
6415
|
+
),
|
|
6323
6416
|
staleAfterSeconds: toolCachePolicy.staleAfterSeconds,
|
|
6324
6417
|
onClaimedResult: (output, receiptKey) =>
|
|
6325
6418
|
markToolExecuteResultExecutionOutcome(output, {
|
|
@@ -6337,10 +6430,7 @@ export class PlayContextImpl {
|
|
|
6337
6430
|
shouldPersistFailure: (error) =>
|
|
6338
6431
|
!(error instanceof ToolExecuteAuthScopeChangedError),
|
|
6339
6432
|
markRunningBeforeExecute: false,
|
|
6340
|
-
execute: ({ leaseId }) => {
|
|
6341
|
-
claimedReceiptLeaseId = leaseId;
|
|
6342
|
-
return executeTool({ leaseId });
|
|
6343
|
-
},
|
|
6433
|
+
execute: ({ leaseId }) => executeTool({ leaseId }),
|
|
6344
6434
|
runningReceiptWaitMaxAttempts:
|
|
6345
6435
|
resolveRuntimeToolReceiptWaitMaxAttempts(
|
|
6346
6436
|
typeof options?.receiptWaitMs === 'number'
|
|
@@ -6356,36 +6446,9 @@ export class PlayContextImpl {
|
|
|
6356
6446
|
) {
|
|
6357
6447
|
throw error;
|
|
6358
6448
|
}
|
|
6359
|
-
//
|
|
6360
|
-
//
|
|
6361
|
-
//
|
|
6362
|
-
// the OLD cache key; without an explicit terminal it projects as a failed
|
|
6363
|
-
// node even though the work is about to be re-done under the new scope.
|
|
6364
|
-
// `skipped` is the terminal for superseded work; the skip is fenced on
|
|
6365
|
-
// this run+attempt+lease so it can only touch the receipt this attempt
|
|
6366
|
-
// owns. Best-effort: a skip that cannot fence (store returns null) must
|
|
6367
|
-
// not mask the reclaim itself.
|
|
6368
|
-
const supersededReceiptKey = durableCacheKey;
|
|
6369
|
-
try {
|
|
6370
|
-
const skipped = await this.skipRuntimeStepReceipt(
|
|
6371
|
-
supersededReceiptKey,
|
|
6372
|
-
this.currentRunId,
|
|
6373
|
-
claimedReceiptLeaseId,
|
|
6374
|
-
);
|
|
6375
|
-
if (skipped) {
|
|
6376
|
-
this.log(
|
|
6377
|
-
`ctx.tools.execute(${toolId}): auth_scope_changed_superseded ` +
|
|
6378
|
-
`old-scope receipt ${toolRequestIdentity} (label: ${normalizedKey}) ` +
|
|
6379
|
-
`marked skipped so the reclaim does not project as a failed node.`,
|
|
6380
|
-
);
|
|
6381
|
-
}
|
|
6382
|
-
} catch (skipError) {
|
|
6383
|
-
this.log(
|
|
6384
|
-
`ctx.tools.execute(${toolId}): auth_scope_changed_supersede_skip_failed ` +
|
|
6385
|
-
`(label: ${normalizedKey}); could not mark the old-scope receipt ` +
|
|
6386
|
-
`skipped: ${this.formatRuntimeError(skipError)}`,
|
|
6387
|
-
);
|
|
6388
|
-
}
|
|
6449
|
+
// Completed-receipt cache misses never publish a running receipt. The
|
|
6450
|
+
// old auth-scope attempt therefore has no ownership metadata to clean
|
|
6451
|
+
// up; re-key under the refreshed credential scope and retry once.
|
|
6389
6452
|
// Make the bounded auth-scope re-claim observable in run logs instead
|
|
6390
6453
|
// of a silent continuation: the credential identity changed after the
|
|
6391
6454
|
// receipt key was prepared, so we evict the cached digest, re-resolve,
|
|
@@ -6552,6 +6615,8 @@ export class PlayContextImpl {
|
|
|
6552
6615
|
}
|
|
6553
6616
|
const childExecutionDecision = resolveChildExecutionStrategy({
|
|
6554
6617
|
pipeline: resolvedPlay.staticPipeline,
|
|
6618
|
+
execution: options?.execution,
|
|
6619
|
+
childPlayName: resolvedName,
|
|
6555
6620
|
});
|
|
6556
6621
|
const launchThroughScheduler =
|
|
6557
6622
|
this.shouldLaunchChildPlaysThroughScheduler() &&
|
|
@@ -6696,6 +6761,7 @@ export class PlayContextImpl {
|
|
|
6696
6761
|
}
|
|
6697
6762
|
this.recordPlayCallStep({
|
|
6698
6763
|
playId: resolvedName,
|
|
6764
|
+
execution: options?.execution,
|
|
6699
6765
|
description: options?.description,
|
|
6700
6766
|
});
|
|
6701
6767
|
return checkpointResult.output;
|
|
@@ -6743,6 +6809,7 @@ export class PlayContextImpl {
|
|
|
6743
6809
|
}
|
|
6744
6810
|
this.recordPlayCallStep({
|
|
6745
6811
|
playId: resolvedName,
|
|
6812
|
+
execution: options?.execution,
|
|
6746
6813
|
description: options?.description,
|
|
6747
6814
|
});
|
|
6748
6815
|
return terminal.output;
|
|
@@ -6843,6 +6910,7 @@ export class PlayContextImpl {
|
|
|
6843
6910
|
}
|
|
6844
6911
|
this.recordPlayCallStep({
|
|
6845
6912
|
playId: resolvedName,
|
|
6913
|
+
execution: options?.execution,
|
|
6846
6914
|
description: options?.description,
|
|
6847
6915
|
nestedSteps: childContext.getSteps(),
|
|
6848
6916
|
});
|
|
@@ -8488,6 +8556,32 @@ export class PlayContextImpl {
|
|
|
8488
8556
|
const retrySafeTransientHttp =
|
|
8489
8557
|
retryPolicy?.retrySafeTransientHttp === true;
|
|
8490
8558
|
let transportAttempt = 0;
|
|
8559
|
+
const retryToolTransportFailure = async (
|
|
8560
|
+
message: string,
|
|
8561
|
+
): Promise<void> => {
|
|
8562
|
+
transportAttempt += 1;
|
|
8563
|
+
if (transportAttempt < TOOL_EXECUTE_TRANSPORT_MAX_ATTEMPTS) {
|
|
8564
|
+
await this.governor.chargeBudget('retry');
|
|
8565
|
+
const retryAfterMs =
|
|
8566
|
+
TOOL_RETRY_AFTER_FALLBACK_MS * transportAttempt;
|
|
8567
|
+
span.setAttribute(
|
|
8568
|
+
'plays.transport_retry_attempt',
|
|
8569
|
+
transportAttempt,
|
|
8570
|
+
);
|
|
8571
|
+
this.log(
|
|
8572
|
+
`Tool ${toolId} transport failed calling ${url} on attempt ${transportAttempt}/${TOOL_EXECUTE_TRANSPORT_MAX_ATTEMPTS}; retrying after ${retryAfterMs}ms: ${message}`,
|
|
8573
|
+
);
|
|
8574
|
+
await options?.parkProviderCall?.();
|
|
8575
|
+
await this.sleepWithCheckpointHeartbeat(retryAfterMs);
|
|
8576
|
+
return;
|
|
8577
|
+
}
|
|
8578
|
+
throw new ToolHttpError(
|
|
8579
|
+
`Tool ${toolId} transport failed calling ${url} after ${transportAttempt}/${TOOL_EXECUTE_TRANSPORT_MAX_ATTEMPTS} attempts: ${message}`,
|
|
8580
|
+
null,
|
|
8581
|
+
0,
|
|
8582
|
+
'repairable',
|
|
8583
|
+
);
|
|
8584
|
+
};
|
|
8491
8585
|
|
|
8492
8586
|
while (true) {
|
|
8493
8587
|
let response: Response;
|
|
@@ -8655,32 +8749,12 @@ export class PlayContextImpl {
|
|
|
8655
8749
|
`Tool ${toolId} runtime API call timed out after ${timeoutMs}ms.`,
|
|
8656
8750
|
)
|
|
8657
8751
|
: error;
|
|
8658
|
-
transportAttempt += 1;
|
|
8659
8752
|
const message =
|
|
8660
8753
|
transportError instanceof Error
|
|
8661
8754
|
? transportError.message
|
|
8662
8755
|
: String(transportError);
|
|
8663
|
-
|
|
8664
|
-
|
|
8665
|
-
const retryAfterMs =
|
|
8666
|
-
TOOL_RETRY_AFTER_FALLBACK_MS * transportAttempt;
|
|
8667
|
-
span.setAttribute(
|
|
8668
|
-
'plays.transport_retry_attempt',
|
|
8669
|
-
transportAttempt,
|
|
8670
|
-
);
|
|
8671
|
-
this.log(
|
|
8672
|
-
`Tool ${toolId} transport failed calling ${url} on attempt ${transportAttempt}/${TOOL_EXECUTE_TRANSPORT_MAX_ATTEMPTS}; retrying after ${retryAfterMs}ms: ${message}`,
|
|
8673
|
-
);
|
|
8674
|
-
await options?.parkProviderCall?.();
|
|
8675
|
-
await this.sleepWithCheckpointHeartbeat(retryAfterMs);
|
|
8676
|
-
continue;
|
|
8677
|
-
}
|
|
8678
|
-
throw new ToolHttpError(
|
|
8679
|
-
`Tool ${toolId} transport failed calling ${url} after ${transportAttempt}/${TOOL_EXECUTE_TRANSPORT_MAX_ATTEMPTS} attempts: ${message}`,
|
|
8680
|
-
null,
|
|
8681
|
-
0,
|
|
8682
|
-
'repairable',
|
|
8683
|
-
);
|
|
8756
|
+
await retryToolTransportFailure(message);
|
|
8757
|
+
continue;
|
|
8684
8758
|
} finally {
|
|
8685
8759
|
if (timeoutHandle) {
|
|
8686
8760
|
clearTimeout(timeoutHandle);
|
|
@@ -8691,6 +8765,18 @@ export class PlayContextImpl {
|
|
|
8691
8765
|
|
|
8692
8766
|
if (!response.ok) {
|
|
8693
8767
|
const text = await response.text();
|
|
8768
|
+
if (
|
|
8769
|
+
isDeeplineDeveloperTunnelOrigin502({
|
|
8770
|
+
url,
|
|
8771
|
+
status: response.status,
|
|
8772
|
+
bodyText: text,
|
|
8773
|
+
})
|
|
8774
|
+
) {
|
|
8775
|
+
await retryToolTransportFailure(
|
|
8776
|
+
'the Deepline development tunnel returned a branded 502 before reaching the app origin',
|
|
8777
|
+
);
|
|
8778
|
+
continue;
|
|
8779
|
+
}
|
|
8694
8780
|
const authScopeChanged = parseToolExecuteAuthScopeChangedError({
|
|
8695
8781
|
status: response.status,
|
|
8696
8782
|
bodyText: text,
|
|
@@ -214,6 +214,7 @@ export type PlayStep =
|
|
|
214
214
|
| {
|
|
215
215
|
type: 'play_call';
|
|
216
216
|
playId: string;
|
|
217
|
+
execution?: 'inline' | 'child-workflow';
|
|
217
218
|
results?: PlayStepRowResult[];
|
|
218
219
|
nestedSteps: PlayStep[];
|
|
219
220
|
description?: string;
|
|
@@ -252,6 +253,7 @@ export type PlayDatasetSubstep =
|
|
|
252
253
|
| {
|
|
253
254
|
type: 'play_call';
|
|
254
255
|
playId: string;
|
|
256
|
+
execution?: 'inline' | 'child-workflow';
|
|
255
257
|
results?: PlayStepRowResult[];
|
|
256
258
|
nestedSteps: PlayStep[];
|
|
257
259
|
description?: string;
|