deepline 0.1.150 → 0.1.152
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/apps/play-runner-workers/src/entry.ts +170 -168
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/csv-rows.ts +2 -19
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/row-isolation.ts +5 -53
- package/dist/bundling-sources/sdk/src/config.ts +2 -2
- package/dist/bundling-sources/sdk/src/release.ts +2 -2
- package/dist/bundling-sources/shared_libs/play-runtime/context.ts +101 -162
- package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +3 -0
- package/dist/bundling-sources/shared_libs/play-runtime/durability-store.ts +54 -0
- package/dist/bundling-sources/shared_libs/play-runtime/map-row-outcome.ts +167 -0
- package/dist/bundling-sources/shared_libs/play-runtime/pacing.ts +79 -0
- package/dist/bundling-sources/shared_libs/play-runtime/row-isolation.ts +39 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +36 -86
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-sheet-row-transition.ts +90 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-sheet-session.ts +43 -0
- package/dist/bundling-sources/shared_libs/play-runtime/tool-execute-retry-policy.ts +142 -11
- package/dist/bundling-sources/shared_libs/play-runtime/tool-http-errors.ts +3 -2
- package/dist/bundling-sources/shared_libs/plays/bundling/index.ts +20 -23
- package/dist/cli/index.js +35 -3
- package/dist/cli/index.mjs +35 -3
- package/dist/index.js +3 -3
- package/dist/index.mjs +3 -3
- package/dist/plays/bundle-play-file.mjs +22 -19
- package/package.json +1 -1
|
@@ -2,14 +2,9 @@ import {
|
|
|
2
2
|
cloneCsvAliasedRow,
|
|
3
3
|
stripCsvProjectionMetadata,
|
|
4
4
|
} from '../../../../shared_libs/play-runtime/csv-rename';
|
|
5
|
+
import { copyMapRowOutcomeRuntimeFields } from '../../../../shared_libs/play-runtime/map-row-outcome';
|
|
5
6
|
|
|
6
7
|
const CSV_TEMPLATE_CONTEXT = '__deeplineCsvProjectedValues';
|
|
7
|
-
const RUNTIME_STORAGE_FIELDS = [
|
|
8
|
-
'__deeplineRowKey',
|
|
9
|
-
'__deeplineCellMetaPatch',
|
|
10
|
-
'__deeplineRowStatus',
|
|
11
|
-
'__deeplineRowError',
|
|
12
|
-
] as const;
|
|
13
8
|
|
|
14
9
|
export type CsvRuntimeRow = Record<string, unknown>;
|
|
15
10
|
|
|
@@ -74,18 +69,6 @@ export function publicCsvOutputRow<T extends CsvRuntimeRow>(row: T): T {
|
|
|
74
69
|
return publicCsvInputRow(row);
|
|
75
70
|
}
|
|
76
71
|
|
|
77
|
-
function copyRuntimeStorageFields(
|
|
78
|
-
target: CsvRuntimeRow,
|
|
79
|
-
row: CsvRuntimeRow,
|
|
80
|
-
): CsvRuntimeRow {
|
|
81
|
-
for (const runtimeField of RUNTIME_STORAGE_FIELDS) {
|
|
82
|
-
if (runtimeField in row) {
|
|
83
|
-
target[runtimeField] = row[runtimeField];
|
|
84
|
-
}
|
|
85
|
-
}
|
|
86
|
-
return target;
|
|
87
|
-
}
|
|
88
|
-
|
|
89
72
|
export function publicCsvStorageRow<T extends CsvRuntimeRow>(row: T): T {
|
|
90
73
|
const publicRow = publicCsvInputRow(row) as CsvRuntimeRow;
|
|
91
74
|
const storageRow: CsvRuntimeRow = {};
|
|
@@ -93,7 +76,7 @@ export function publicCsvStorageRow<T extends CsvRuntimeRow>(row: T): T {
|
|
|
93
76
|
if (typeof fieldName !== 'string') continue;
|
|
94
77
|
storageRow[fieldName] = publicRow[fieldName];
|
|
95
78
|
}
|
|
96
|
-
return
|
|
79
|
+
return copyMapRowOutcomeRuntimeFields(storageRow, row) as T;
|
|
97
80
|
}
|
|
98
81
|
|
|
99
82
|
export function runtimeCsvStorageRow<T extends CsvRuntimeRow>(row: T): T {
|
|
@@ -1,53 +1,5 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
/**
|
|
7
|
-
* Thrown by `assertNotAborted` and surfaced through ctx.step / ctx.sleep / map
|
|
8
|
-
* processing when the workflow has been terminated externally. Cooperatively
|
|
9
|
-
* cancels in-flight user code: the play must check `ctx.signal.aborted` (or
|
|
10
|
-
* await one of the abort-aware ctx methods) before doing more work.
|
|
11
|
-
*/
|
|
12
|
-
export class WorkflowAbortError extends Error {
|
|
13
|
-
override readonly name = 'WorkflowAbort';
|
|
14
|
-
constructor(message = 'Play run cancelled.') {
|
|
15
|
-
super(message);
|
|
16
|
-
}
|
|
17
|
-
}
|
|
18
|
-
|
|
19
|
-
export function isAbortLikeError(error: unknown): boolean {
|
|
20
|
-
if (!error) return false;
|
|
21
|
-
if (error instanceof WorkflowAbortError) return true;
|
|
22
|
-
if (error instanceof Error) {
|
|
23
|
-
if (error.name === 'WorkflowAbort' || error.name === 'AbortError')
|
|
24
|
-
return true;
|
|
25
|
-
return /\b(cancell?ed|aborted|terminate[d]?)\b/i.test(error.message);
|
|
26
|
-
}
|
|
27
|
-
return false;
|
|
28
|
-
}
|
|
29
|
-
|
|
30
|
-
/**
|
|
31
|
-
* Errors that must stay run-fatal even under the default map row failure
|
|
32
|
-
* isolation:
|
|
33
|
-
*
|
|
34
|
-
* - Cancellation/abort must stop the run.
|
|
35
|
-
* - Governor budget exhaustion is a run-level invariant — isolating it per
|
|
36
|
-
* row would silently convert "this run exceeded its execution budget" into
|
|
37
|
-
* thousands of identical row failures.
|
|
38
|
-
* - Rate-limit pushback (a tool call that still got HTTP 429 after the
|
|
39
|
-
* in-process retry budget) is run-level throughput pressure that applies to
|
|
40
|
-
* every row equally, not a row defect. Isolating it silently drops healthy
|
|
41
|
-
* rows from the output dataset whenever a provider throttles — the durable
|
|
42
|
-
* chunk step's retries (and, if the storm persists, a loud run failure with
|
|
43
|
-
* recoverable persisted rows) are the correct response.
|
|
44
|
-
* - Hard billing failures (billing cap / insufficient credits) promise "run
|
|
45
|
-
* halted before marking remaining rows processed"; isolating them would
|
|
46
|
-
* complete the run while silently failing every remaining row.
|
|
47
|
-
*/
|
|
48
|
-
export function isRowIsolationExemptError(error: unknown): boolean {
|
|
49
|
-
if (isAbortLikeError(error)) return true;
|
|
50
|
-
if (error instanceof Error && error.name === 'GovernorBudgetError')
|
|
51
|
-
return true;
|
|
52
|
-
return isRateLimitToolHttpError(error) || isHardBillingToolHttpError(error);
|
|
53
|
-
}
|
|
1
|
+
export {
|
|
2
|
+
WorkflowAbortError,
|
|
3
|
+
isAbortLikeError,
|
|
4
|
+
isRowIsolationExemptError,
|
|
5
|
+
} from '../../../../shared_libs/play-runtime/row-isolation';
|
|
@@ -321,8 +321,8 @@ function loadProjectDeeplineEnv(startDir = process.cwd()): EnvValues {
|
|
|
321
321
|
return loadProjectEnvCandidates(startDir)[0]?.env ?? {};
|
|
322
322
|
}
|
|
323
323
|
|
|
324
|
-
function normalizeBaseUrl(baseUrl: string): string {
|
|
325
|
-
const trimmed = baseUrl
|
|
324
|
+
function normalizeBaseUrl(baseUrl: string | null | undefined): string {
|
|
325
|
+
const trimmed = baseUrl?.trim().replace(/\/+$/, '') ?? '';
|
|
326
326
|
if (!trimmed) return '';
|
|
327
327
|
try {
|
|
328
328
|
const parsed = new URL(trimmed);
|
|
@@ -102,10 +102,10 @@ export const SDK_RELEASE = {
|
|
|
102
102
|
// the SDK enrich generator's one-second stale policy.
|
|
103
103
|
// 0.1.110 ships authored V2 prebuilts and required top-level play descriptions.
|
|
104
104
|
// 0.1.111 ships dataset-native tool list getters and result row datasets.
|
|
105
|
-
version: '0.1.
|
|
105
|
+
version: '0.1.152',
|
|
106
106
|
apiContract: '2026-06-dataset-handle-results-hard-cutover',
|
|
107
107
|
supportPolicy: {
|
|
108
|
-
latest: '0.1.
|
|
108
|
+
latest: '0.1.152',
|
|
109
109
|
minimumSupported: '0.1.53',
|
|
110
110
|
deprecatedBelow: '0.1.53',
|
|
111
111
|
commandMinimumSupported: [
|
|
@@ -28,6 +28,12 @@ import {
|
|
|
28
28
|
} from './batch-runtime';
|
|
29
29
|
import type { PlayQueueHint } from './governor/rate-state-backend';
|
|
30
30
|
import type { MapRowOutcome } from './durability-store';
|
|
31
|
+
import {
|
|
32
|
+
completedMapRowOutcome,
|
|
33
|
+
failedMapRowOutcome,
|
|
34
|
+
mapRowOutcomeRuntimeFields,
|
|
35
|
+
resolveMapRowOutcomeKey,
|
|
36
|
+
} from './map-row-outcome';
|
|
31
37
|
import {
|
|
32
38
|
createDefaultGovernanceSnapshot,
|
|
33
39
|
createPlayExecutionGovernor,
|
|
@@ -35,12 +41,9 @@ import {
|
|
|
35
41
|
type PacingResolver,
|
|
36
42
|
type PlayExecutionGovernor,
|
|
37
43
|
} from './governor/governor';
|
|
38
|
-
import {
|
|
39
|
-
CTX_FETCH_EGRESS_TOOL_ID,
|
|
40
|
-
resolveBuiltinPacing,
|
|
41
|
-
} from './builtin-pacing';
|
|
44
|
+
import { CTX_FETCH_EGRESS_TOOL_ID } from './builtin-pacing';
|
|
42
45
|
import { InMemoryRateStateBackend } from './governor/in-memory-rate-state-backend';
|
|
43
|
-
import
|
|
46
|
+
import { pacingPolicyForTool } from './pacing';
|
|
44
47
|
import {
|
|
45
48
|
cloneToolExecuteResultWithExecution,
|
|
46
49
|
createToolExecuteResult,
|
|
@@ -54,13 +57,11 @@ import {
|
|
|
54
57
|
type ToolResultMetadataInput,
|
|
55
58
|
} from './tool-result';
|
|
56
59
|
import {
|
|
57
|
-
|
|
58
|
-
|
|
60
|
+
TOOL_EXECUTE_TRANSPORT_MAX_ATTEMPTS,
|
|
61
|
+
classifyToolExecuteHttpFailure,
|
|
62
|
+
createToolExecuteHttpFailureAttemptTracker,
|
|
59
63
|
} from './tool-execute-retry-policy';
|
|
60
|
-
import {
|
|
61
|
-
isHardBillingToolHttpError,
|
|
62
|
-
normalizeToolHttpErrorMessage,
|
|
63
|
-
} from './tool-http-errors';
|
|
64
|
+
import { isRowIsolationExemptError } from './row-isolation';
|
|
64
65
|
import { sqlSafePlayColumnName } from '@shared_libs/plays/static-pipeline';
|
|
65
66
|
import { createRuntimeDatasetId } from './dataset-id';
|
|
66
67
|
import { dedupeExplicitMapKeyRows } from './map-row-identity';
|
|
@@ -577,24 +578,6 @@ function normalizeFetchHeaders(
|
|
|
577
578
|
);
|
|
578
579
|
}
|
|
579
580
|
|
|
580
|
-
function parseRetryAfterMs(header: string | null): number {
|
|
581
|
-
if (!header) {
|
|
582
|
-
return TOOL_RETRY_AFTER_FALLBACK_MS;
|
|
583
|
-
}
|
|
584
|
-
|
|
585
|
-
const seconds = Number(header);
|
|
586
|
-
if (Number.isFinite(seconds) && seconds > 0) {
|
|
587
|
-
return Math.ceil(seconds * 1000);
|
|
588
|
-
}
|
|
589
|
-
|
|
590
|
-
const retryAt = Date.parse(header);
|
|
591
|
-
if (Number.isFinite(retryAt)) {
|
|
592
|
-
return Math.max(1, retryAt - Date.now());
|
|
593
|
-
}
|
|
594
|
-
|
|
595
|
-
return TOOL_RETRY_AFTER_FALLBACK_MS;
|
|
596
|
-
}
|
|
597
|
-
|
|
598
581
|
function parseJsonOrNull(bodyText: string): unknown | null {
|
|
599
582
|
if (!bodyText.trim()) return null;
|
|
600
583
|
try {
|
|
@@ -652,20 +635,10 @@ function createPacingResolver(
|
|
|
652
635
|
getToolQueueHints: ContextOptions['getToolQueueHints'],
|
|
653
636
|
): PacingResolver {
|
|
654
637
|
return async (toolId: string) => {
|
|
655
|
-
const builtin =
|
|
638
|
+
const builtin = pacingPolicyForTool(toolId, []);
|
|
656
639
|
if (builtin) return builtin;
|
|
657
640
|
const hints = getToolQueueHints ? await getToolQueueHints(toolId) : [];
|
|
658
|
-
|
|
659
|
-
return null;
|
|
660
|
-
}
|
|
661
|
-
const provider = hints[0]!.provider;
|
|
662
|
-
const rules: PacingRule[] = hints.map((hint) => ({
|
|
663
|
-
ruleId: hint.ruleId,
|
|
664
|
-
requestsPerWindow: hint.requestsPerWindow,
|
|
665
|
-
windowMs: hint.windowMs,
|
|
666
|
-
maxConcurrency: hint.maxConcurrency,
|
|
667
|
-
}));
|
|
668
|
-
return { provider, rules };
|
|
641
|
+
return pacingPolicyForTool(toolId, hints);
|
|
669
642
|
};
|
|
670
643
|
}
|
|
671
644
|
|
|
@@ -2030,9 +2003,8 @@ export class PlayContextImpl {
|
|
|
2030
2003
|
}
|
|
2031
2004
|
|
|
2032
2005
|
const rowIdentity = (row: Record<string, unknown>, index = 0) => {
|
|
2033
|
-
|
|
2034
|
-
|
|
2035
|
-
}
|
|
2006
|
+
const runtimeKey = resolveMapRowOutcomeKey(row);
|
|
2007
|
+
if (runtimeKey) return runtimeKey;
|
|
2036
2008
|
return explicitKeyResolver
|
|
2037
2009
|
? derivePlayRowIdentityFromKey(
|
|
2038
2010
|
explicitKeyResolver(row, index),
|
|
@@ -2081,7 +2053,9 @@ export class PlayContextImpl {
|
|
|
2081
2053
|
const mapStartRows = shouldPassRowKey
|
|
2082
2054
|
? rawItems.map((row, index) => ({
|
|
2083
2055
|
...toSerializableCsvAliasedRow(row),
|
|
2084
|
-
|
|
2056
|
+
...mapRowOutcomeRuntimeFields({
|
|
2057
|
+
key: rowIdentity(row, index),
|
|
2058
|
+
}),
|
|
2085
2059
|
}))
|
|
2086
2060
|
: rawItems.map((row) => toSerializableCsvAliasedRow(row));
|
|
2087
2061
|
const mapStartResult = await this.#options.onMapStart(
|
|
@@ -2099,9 +2073,7 @@ export class PlayContextImpl {
|
|
|
2099
2073
|
mapStartResult.tableNamespace,
|
|
2100
2074
|
);
|
|
2101
2075
|
const persistedRowIdentity = (row: Record<string, unknown>, index = 0) =>
|
|
2102
|
-
|
|
2103
|
-
? row.__deeplineRowKey
|
|
2104
|
-
: rowIdentity(row, index);
|
|
2076
|
+
resolveMapRowOutcomeKey(row) ?? rowIdentity(row, index);
|
|
2105
2077
|
const pendingKeys = new Set(
|
|
2106
2078
|
mapStartResult.pendingRows.map((row, index) =>
|
|
2107
2079
|
persistedRowIdentity(row, index),
|
|
@@ -2271,10 +2243,7 @@ export class PlayContextImpl {
|
|
|
2271
2243
|
);
|
|
2272
2244
|
} catch (error) {
|
|
2273
2245
|
if (error instanceof FailFastMapRowsError) {
|
|
2274
|
-
const rowsToPersist =
|
|
2275
|
-
error.completedRows.length > 0
|
|
2276
|
-
? error.completedRows
|
|
2277
|
-
: error.failedRows;
|
|
2246
|
+
const rowsToPersist = [...error.completedRows, ...error.failedRows];
|
|
2278
2247
|
await incrementalPersistence?.flush();
|
|
2279
2248
|
const unpersistedRows = incrementalPersistence
|
|
2280
2249
|
? rowsToPersist.filter(
|
|
@@ -2336,14 +2305,16 @@ export class PlayContextImpl {
|
|
|
2336
2305
|
if (incrementalPersistence?.isPersisted(row)) continue;
|
|
2337
2306
|
const rowKey = row.key;
|
|
2338
2307
|
const meta = mapCellMeta?.get(rowKey);
|
|
2339
|
-
persistRows.push(
|
|
2340
|
-
|
|
2341
|
-
|
|
2342
|
-
|
|
2343
|
-
|
|
2344
|
-
|
|
2345
|
-
|
|
2346
|
-
|
|
2308
|
+
persistRows.push(
|
|
2309
|
+
completedMapRowOutcome({
|
|
2310
|
+
key: rowKey,
|
|
2311
|
+
data: row.data,
|
|
2312
|
+
cellMetaPatch: {
|
|
2313
|
+
...(row.cellMetaPatch ?? {}),
|
|
2314
|
+
...(meta ?? {}),
|
|
2315
|
+
},
|
|
2316
|
+
}),
|
|
2317
|
+
);
|
|
2347
2318
|
}
|
|
2348
2319
|
persistRows.push(
|
|
2349
2320
|
...mapResult.failedRows.filter(
|
|
@@ -2441,20 +2412,19 @@ export class PlayContextImpl {
|
|
|
2441
2412
|
.filter((fieldName) => shouldPersistMapCellField(fieldName));
|
|
2442
2413
|
const normalizedTableNamespace = mapScope.artifactTableNamespace;
|
|
2443
2414
|
const rowIdentity = (row: Record<string, unknown>, index = 0) =>
|
|
2444
|
-
|
|
2445
|
-
|
|
2446
|
-
|
|
2447
|
-
|
|
2448
|
-
|
|
2449
|
-
|
|
2450
|
-
|
|
2451
|
-
|
|
2452
|
-
!fieldName.startsWith('__deepline'),
|
|
2453
|
-
),
|
|
2454
|
-
),
|
|
2415
|
+
resolveMapRowOutcomeKey(row) ??
|
|
2416
|
+
mapScope.rowIdentity(
|
|
2417
|
+
stripCsvProjectedFields(
|
|
2418
|
+
Object.fromEntries(
|
|
2419
|
+
Object.entries(row).filter(
|
|
2420
|
+
([fieldName]) =>
|
|
2421
|
+
!datasetColumnNames.includes(fieldName) &&
|
|
2422
|
+
!fieldName.startsWith('__deepline'),
|
|
2455
2423
|
),
|
|
2456
|
-
|
|
2457
|
-
|
|
2424
|
+
),
|
|
2425
|
+
),
|
|
2426
|
+
index,
|
|
2427
|
+
);
|
|
2458
2428
|
const executionRowKey = (row: Record<string, unknown>, index: number) =>
|
|
2459
2429
|
runtimeOptions?.executionRowKeys?.[index] ?? rowIdentity(row, index);
|
|
2460
2430
|
const executionRowIndex = (index: number) =>
|
|
@@ -2645,14 +2615,16 @@ export class PlayContextImpl {
|
|
|
2645
2615
|
`Map completed: ${results.length + completedRows} results (${results.length} executed, ${completedRows} already satisfied)`,
|
|
2646
2616
|
);
|
|
2647
2617
|
return {
|
|
2648
|
-
completedRows: results.map((row, index) =>
|
|
2649
|
-
|
|
2650
|
-
|
|
2651
|
-
|
|
2652
|
-
|
|
2653
|
-
|
|
2654
|
-
|
|
2655
|
-
|
|
2618
|
+
completedRows: results.map((row, index) =>
|
|
2619
|
+
completedMapRowOutcome({
|
|
2620
|
+
key: executionRowKey(
|
|
2621
|
+
this.toOutputRow(items[index] as Record<string, unknown>),
|
|
2622
|
+
index,
|
|
2623
|
+
),
|
|
2624
|
+
inputIndex: executionRowIndex(index),
|
|
2625
|
+
data: this.toPersistedOutputRow(row),
|
|
2626
|
+
}),
|
|
2627
|
+
),
|
|
2656
2628
|
failedRows: [],
|
|
2657
2629
|
};
|
|
2658
2630
|
}
|
|
@@ -2767,6 +2739,9 @@ export class PlayContextImpl {
|
|
|
2767
2739
|
) {
|
|
2768
2740
|
throw error;
|
|
2769
2741
|
}
|
|
2742
|
+
if (isRowIsolationExemptError(error)) {
|
|
2743
|
+
throw error;
|
|
2744
|
+
}
|
|
2770
2745
|
|
|
2771
2746
|
value = null;
|
|
2772
2747
|
computedFields[fieldName] = value;
|
|
@@ -2787,7 +2762,7 @@ export class PlayContextImpl {
|
|
|
2787
2762
|
: {},
|
|
2788
2763
|
});
|
|
2789
2764
|
if (failFastOnRowError) {
|
|
2790
|
-
const failedRow: PersistableMapRow = {
|
|
2765
|
+
const failedRow: PersistableMapRow = failedMapRowOutcome({
|
|
2791
2766
|
key: rowKey,
|
|
2792
2767
|
inputIndex: rowIndex,
|
|
2793
2768
|
data: this.toPersistedOutputRow(
|
|
@@ -2796,9 +2771,8 @@ export class PlayContextImpl {
|
|
|
2796
2771
|
...(this.activeMapCellMeta?.get(rowKey)
|
|
2797
2772
|
? { cellMetaPatch: this.activeMapCellMeta.get(rowKey) }
|
|
2798
2773
|
: {}),
|
|
2799
|
-
status: 'failed',
|
|
2800
2774
|
error: formattedError,
|
|
2801
|
-
};
|
|
2775
|
+
});
|
|
2802
2776
|
failedRowsToPersist.push(failedRow);
|
|
2803
2777
|
throw error;
|
|
2804
2778
|
}
|
|
@@ -2806,14 +2780,13 @@ export class PlayContextImpl {
|
|
|
2806
2780
|
cloneCsvAliasedRow(baseRow, computedFields),
|
|
2807
2781
|
);
|
|
2808
2782
|
const cellMetaPatch = this.activeMapCellMeta?.get(rowKey);
|
|
2809
|
-
const failedRow: PersistableMapRow = {
|
|
2783
|
+
const failedRow: PersistableMapRow = failedMapRowOutcome({
|
|
2810
2784
|
key: rowKey,
|
|
2811
2785
|
inputIndex: rowIndex,
|
|
2812
2786
|
data: failedData,
|
|
2813
2787
|
...(cellMetaPatch ? { cellMetaPatch } : {}),
|
|
2814
|
-
status: 'failed',
|
|
2815
2788
|
error: formattedError,
|
|
2816
|
-
};
|
|
2789
|
+
});
|
|
2817
2790
|
failedRowsToPersist.push(failedRow);
|
|
2818
2791
|
updateMapFrameProgress({
|
|
2819
2792
|
failedRowKey: rowKey,
|
|
@@ -2867,14 +2840,14 @@ export class PlayContextImpl {
|
|
|
2867
2840
|
dataPatch: {},
|
|
2868
2841
|
});
|
|
2869
2842
|
const publicRow = this.toPublicOutputRow(merged);
|
|
2870
|
-
const completedRow: PersistableMapRow = {
|
|
2843
|
+
const completedRow: PersistableMapRow = completedMapRowOutcome({
|
|
2871
2844
|
key: rowKey,
|
|
2872
2845
|
inputIndex: rowIndex,
|
|
2873
2846
|
data: this.toPersistedOutputRow(merged),
|
|
2874
2847
|
...(this.activeMapCellMeta?.get(rowKey)
|
|
2875
2848
|
? { cellMetaPatch: this.activeMapCellMeta.get(rowKey) }
|
|
2876
2849
|
: {}),
|
|
2877
|
-
};
|
|
2850
|
+
});
|
|
2878
2851
|
completedRowsToPersist.push(completedRow);
|
|
2879
2852
|
enqueueIncrementalPersist(completedRow);
|
|
2880
2853
|
return publicRow;
|
|
@@ -5538,7 +5511,13 @@ export class PlayContextImpl {
|
|
|
5538
5511
|
},
|
|
5539
5512
|
},
|
|
5540
5513
|
async (span) => {
|
|
5541
|
-
|
|
5514
|
+
const httpFailureAttempts =
|
|
5515
|
+
createToolExecuteHttpFailureAttemptTracker();
|
|
5516
|
+
const retryPolicy = await this.#options
|
|
5517
|
+
.getToolRetryPolicy?.(toolId)
|
|
5518
|
+
.catch(() => null);
|
|
5519
|
+
const retrySafeTransientHttp =
|
|
5520
|
+
retryPolicy?.retrySafeTransientHttp === true;
|
|
5542
5521
|
let transportAttempt = 0;
|
|
5543
5522
|
|
|
5544
5523
|
while (true) {
|
|
@@ -5568,7 +5547,7 @@ export class PlayContextImpl {
|
|
|
5568
5547
|
transportAttempt += 1;
|
|
5569
5548
|
const message =
|
|
5570
5549
|
error instanceof Error ? error.message : String(error);
|
|
5571
|
-
if (transportAttempt <
|
|
5550
|
+
if (transportAttempt < TOOL_EXECUTE_TRANSPORT_MAX_ATTEMPTS) {
|
|
5572
5551
|
this.governor.chargeBudget('retry');
|
|
5573
5552
|
const retryAfterMs =
|
|
5574
5553
|
TOOL_RETRY_AFTER_FALLBACK_MS * transportAttempt;
|
|
@@ -5577,105 +5556,65 @@ export class PlayContextImpl {
|
|
|
5577
5556
|
transportAttempt,
|
|
5578
5557
|
);
|
|
5579
5558
|
this.log(
|
|
5580
|
-
`Tool ${toolId} transport failed calling ${url} on attempt ${transportAttempt}/${
|
|
5559
|
+
`Tool ${toolId} transport failed calling ${url} on attempt ${transportAttempt}/${TOOL_EXECUTE_TRANSPORT_MAX_ATTEMPTS}; retrying after ${retryAfterMs}ms: ${message}`,
|
|
5581
5560
|
);
|
|
5582
5561
|
await this.sleepWithCheckpointHeartbeat(retryAfterMs);
|
|
5583
5562
|
continue;
|
|
5584
5563
|
}
|
|
5585
5564
|
throw new Error(
|
|
5586
|
-
`Tool ${toolId} transport failed calling ${url} after ${transportAttempt}/${
|
|
5565
|
+
`Tool ${toolId} transport failed calling ${url} after ${transportAttempt}/${TOOL_EXECUTE_TRANSPORT_MAX_ATTEMPTS} attempts: ${message}`,
|
|
5587
5566
|
);
|
|
5588
5567
|
}
|
|
5589
5568
|
|
|
5590
5569
|
span.setAttribute('plays.http_status_code', response.status);
|
|
5591
5570
|
|
|
5592
|
-
if (response.status === 429) {
|
|
5593
|
-
rateLimitAttempt += 1;
|
|
5594
|
-
const text = await response.text();
|
|
5595
|
-
const initialRetryDecision = decideToolExecuteHttpRetry({
|
|
5596
|
-
toolId,
|
|
5597
|
-
status: response.status,
|
|
5598
|
-
});
|
|
5599
|
-
const error = normalizeToolHttpErrorMessage({
|
|
5600
|
-
toolId,
|
|
5601
|
-
status: response.status,
|
|
5602
|
-
attempt: rateLimitAttempt,
|
|
5603
|
-
maxAttempts: initialRetryDecision.attemptCap,
|
|
5604
|
-
bodyText: text,
|
|
5605
|
-
});
|
|
5606
|
-
const retryDecision = decideToolExecuteHttpRetry({
|
|
5607
|
-
toolId,
|
|
5608
|
-
status: response.status,
|
|
5609
|
-
hardBillingFailure: isHardBillingToolHttpError(error),
|
|
5610
|
-
});
|
|
5611
|
-
if (
|
|
5612
|
-
!retryDecision.retryable ||
|
|
5613
|
-
rateLimitAttempt >= retryDecision.attemptCap
|
|
5614
|
-
) {
|
|
5615
|
-
throw error;
|
|
5616
|
-
}
|
|
5617
|
-
this.governor.chargeBudget('retry');
|
|
5618
|
-
const retryAfterMs = parseRetryAfterMs(
|
|
5619
|
-
response.headers.get('retry-after'),
|
|
5620
|
-
);
|
|
5621
|
-
// Feed the server-observed Retry-After back into the shared pacer
|
|
5622
|
-
// so subsequent acquires for this provider back off.
|
|
5623
|
-
await this.reportToolBackpressure(toolId, retryAfterMs);
|
|
5624
|
-
span.setAttribute(
|
|
5625
|
-
'plays.rate_limit_retry_after_ms',
|
|
5626
|
-
retryAfterMs,
|
|
5627
|
-
);
|
|
5628
|
-
span.setAttribute('plays.rate_limit_attempt', rateLimitAttempt);
|
|
5629
|
-
this.log(
|
|
5630
|
-
`Tool ${toolId} rate limited; retrying after ${retryAfterMs}ms`,
|
|
5631
|
-
);
|
|
5632
|
-
await this.sleepWithCheckpointHeartbeat(retryAfterMs);
|
|
5633
|
-
continue;
|
|
5634
|
-
}
|
|
5635
|
-
|
|
5636
5571
|
if (!response.ok) {
|
|
5637
5572
|
const text = await response.text();
|
|
5638
|
-
const
|
|
5573
|
+
const httpFailureAttempt = httpFailureAttempts.next({
|
|
5639
5574
|
toolId,
|
|
5640
5575
|
status: response.status,
|
|
5576
|
+
transientHttpRetrySafe: retrySafeTransientHttp,
|
|
5641
5577
|
});
|
|
5642
|
-
const
|
|
5578
|
+
const failure = classifyToolExecuteHttpFailure({
|
|
5643
5579
|
toolId,
|
|
5644
5580
|
status: response.status,
|
|
5645
|
-
attempt:
|
|
5646
|
-
maxAttempts: initialRetryDecision.attemptCap,
|
|
5581
|
+
attempt: httpFailureAttempt,
|
|
5647
5582
|
bodyText: text,
|
|
5583
|
+
retryAfterHeader: response.headers.get('retry-after'),
|
|
5584
|
+
transientHttpRetrySafe: retrySafeTransientHttp,
|
|
5648
5585
|
});
|
|
5649
|
-
|
|
5650
|
-
|
|
5651
|
-
|
|
5652
|
-
|
|
5653
|
-
|
|
5654
|
-
|
|
5655
|
-
|
|
5656
|
-
rateLimitAttempt + 1 < retryDecision.attemptCap
|
|
5657
|
-
) {
|
|
5658
|
-
rateLimitAttempt += 1;
|
|
5659
|
-
this.governor.chargeBudget('retry');
|
|
5660
|
-
const retryAfterMs = parseRetryAfterMs(
|
|
5661
|
-
response.headers.get('retry-after'),
|
|
5586
|
+
if (failure.backpressureDelayMs !== null) {
|
|
5587
|
+
// Feed the server-observed Retry-After back into the shared
|
|
5588
|
+
// pacer even on the final attempt so later provider calls back
|
|
5589
|
+
// off instead of retrying the whole map chunk.
|
|
5590
|
+
await this.reportToolBackpressure(
|
|
5591
|
+
toolId,
|
|
5592
|
+
failure.backpressureDelayMs,
|
|
5662
5593
|
);
|
|
5594
|
+
}
|
|
5595
|
+
if (failure.shouldRetry) {
|
|
5596
|
+
if (failure.chargeRetryBudget) {
|
|
5597
|
+
this.governor.chargeBudget('retry');
|
|
5598
|
+
}
|
|
5599
|
+
const retryAttributePrefix = failure.isRateLimit
|
|
5600
|
+
? 'rate_limit'
|
|
5601
|
+
: 'transient_http';
|
|
5663
5602
|
span.setAttribute(
|
|
5664
|
-
|
|
5665
|
-
|
|
5603
|
+
`plays.${retryAttributePrefix}_retry_after_ms`,
|
|
5604
|
+
failure.retryDelayMs,
|
|
5666
5605
|
);
|
|
5667
5606
|
span.setAttribute(
|
|
5668
|
-
|
|
5669
|
-
|
|
5607
|
+
`plays.${retryAttributePrefix}_attempt`,
|
|
5608
|
+
httpFailureAttempt,
|
|
5670
5609
|
);
|
|
5671
5610
|
this.log(
|
|
5672
|
-
`Tool ${toolId} returned ${response.status}; retrying after ${
|
|
5611
|
+
`Tool ${toolId} returned ${response.status}; retrying after ${failure.retryDelayMs}ms`,
|
|
5673
5612
|
);
|
|
5674
|
-
await this.sleepWithCheckpointHeartbeat(
|
|
5613
|
+
await this.sleepWithCheckpointHeartbeat(failure.retryDelayMs);
|
|
5675
5614
|
continue;
|
|
5676
5615
|
}
|
|
5677
|
-
this.log(error.message);
|
|
5678
|
-
throw error;
|
|
5616
|
+
this.log(failure.error.message);
|
|
5617
|
+
throw failure.error;
|
|
5679
5618
|
}
|
|
5680
5619
|
|
|
5681
5620
|
const data = (await response.json()) as Record<string, unknown>;
|
|
@@ -487,6 +487,9 @@ export interface ContextOptions {
|
|
|
487
487
|
runId?: string;
|
|
488
488
|
resolvePlay?: (playRef: string) => Promise<ResolvedPlayExecution | null>;
|
|
489
489
|
getToolQueueHints?: (toolId: string) => Promise<readonly PlayQueueHint[]>;
|
|
490
|
+
getToolRetryPolicy?: (toolId: string) => Promise<{
|
|
491
|
+
retrySafeTransientHttp?: boolean;
|
|
492
|
+
} | null>;
|
|
490
493
|
getToolTargetGetters?: (
|
|
491
494
|
toolId: string,
|
|
492
495
|
output: string,
|
|
@@ -1,3 +1,13 @@
|
|
|
1
|
+
import type {
|
|
2
|
+
ClaimRuntimeStepReceiptInput,
|
|
3
|
+
CompleteRuntimeStepReceiptInput,
|
|
4
|
+
FailRuntimeStepReceiptInput,
|
|
5
|
+
GetRuntimeStepReceiptInput,
|
|
6
|
+
RuntimeStepReceipt,
|
|
7
|
+
SkipRuntimeStepReceiptInput,
|
|
8
|
+
} from './ctx-types';
|
|
9
|
+
import type { PlayRunLedgerEvent } from './run-ledger';
|
|
10
|
+
|
|
1
11
|
export type MapRowOutcomeStatus = 'completed' | 'failed';
|
|
2
12
|
|
|
3
13
|
export type MapRowOutcome = {
|
|
@@ -18,3 +28,47 @@ export type MapRowOutcome = {
|
|
|
18
28
|
/** Row-level error message persisted when `status` is failed. */
|
|
19
29
|
error?: string | null;
|
|
20
30
|
};
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* Runner-facing Interface for hot-path durable writes.
|
|
34
|
+
*
|
|
35
|
+
* Runtime API and Play Harness Worker Adapters can satisfy this seam while
|
|
36
|
+
* keeping their transport/SQL details private to their Implementations.
|
|
37
|
+
*/
|
|
38
|
+
export interface PlayDurabilityStore {
|
|
39
|
+
appendRunEvents(input: {
|
|
40
|
+
runId: string;
|
|
41
|
+
events: readonly PlayRunLedgerEvent[];
|
|
42
|
+
}): Promise<void>;
|
|
43
|
+
|
|
44
|
+
getRuntimeStepReceipt(
|
|
45
|
+
input: GetRuntimeStepReceiptInput,
|
|
46
|
+
): Promise<RuntimeStepReceipt | null>;
|
|
47
|
+
claimRuntimeStepReceipt(
|
|
48
|
+
input: ClaimRuntimeStepReceiptInput,
|
|
49
|
+
): Promise<RuntimeStepReceipt | null>;
|
|
50
|
+
completeRuntimeStepReceipt(
|
|
51
|
+
input: CompleteRuntimeStepReceiptInput,
|
|
52
|
+
): Promise<RuntimeStepReceipt | null>;
|
|
53
|
+
failRuntimeStepReceipt(
|
|
54
|
+
input: FailRuntimeStepReceiptInput,
|
|
55
|
+
): Promise<RuntimeStepReceipt | null>;
|
|
56
|
+
skipRuntimeStepReceipt(
|
|
57
|
+
input: SkipRuntimeStepReceiptInput,
|
|
58
|
+
): Promise<RuntimeStepReceipt | null>;
|
|
59
|
+
|
|
60
|
+
prepareMapRows(input: {
|
|
61
|
+
tableNamespace: string;
|
|
62
|
+
rows: readonly Record<string, unknown>[];
|
|
63
|
+
outputFields: readonly string[];
|
|
64
|
+
}): Promise<{
|
|
65
|
+
pendingRows: Record<string, unknown>[];
|
|
66
|
+
completedRows: Record<string, unknown>[];
|
|
67
|
+
}>;
|
|
68
|
+
|
|
69
|
+
completeMapRows(input: {
|
|
70
|
+
tableNamespace: string;
|
|
71
|
+
rows: readonly MapRowOutcome[];
|
|
72
|
+
outputFields: readonly string[];
|
|
73
|
+
}): Promise<{ updated: number }>;
|
|
74
|
+
}
|