deepline 0.1.149 → 0.1.151

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (29) hide show
  1. package/dist/bundling-sources/apps/play-runner-workers/src/entry.ts +157 -140
  2. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/csv-rows.ts +2 -19
  3. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/row-isolation.ts +5 -53
  4. package/dist/bundling-sources/sdk/src/client.ts +5 -0
  5. package/dist/bundling-sources/sdk/src/config.ts +2 -2
  6. package/dist/bundling-sources/sdk/src/release.ts +2 -2
  7. package/dist/bundling-sources/sdk/src/tool-output.ts +63 -17
  8. package/dist/bundling-sources/shared_libs/play-runtime/context.ts +100 -158
  9. package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +3 -0
  10. package/dist/bundling-sources/shared_libs/play-runtime/durability-store.ts +54 -0
  11. package/dist/bundling-sources/shared_libs/play-runtime/map-row-outcome.ts +167 -0
  12. package/dist/bundling-sources/shared_libs/play-runtime/pacing.ts +79 -0
  13. package/dist/bundling-sources/shared_libs/play-runtime/row-isolation.ts +39 -0
  14. package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +19 -86
  15. package/dist/bundling-sources/shared_libs/play-runtime/runtime-sheet-row-transition.ts +90 -0
  16. package/dist/bundling-sources/shared_libs/play-runtime/runtime-sheet-session.ts +43 -0
  17. package/dist/bundling-sources/shared_libs/play-runtime/tool-execute-retry-policy.ts +142 -11
  18. package/dist/bundling-sources/shared_libs/play-runtime/tool-http-errors.ts +3 -2
  19. package/dist/bundling-sources/shared_libs/play-runtime/tool-result-types.ts +17 -4
  20. package/dist/bundling-sources/shared_libs/play-runtime/tool-result.ts +343 -26
  21. package/dist/bundling-sources/shared_libs/plays/bundling/index.ts +20 -23
  22. package/dist/cli/index.js +186 -105
  23. package/dist/cli/index.mjs +193 -106
  24. package/dist/index.d.mts +12 -9
  25. package/dist/index.d.ts +12 -9
  26. package/dist/index.js +33 -20
  27. package/dist/index.mjs +40 -21
  28. package/dist/plays/bundle-play-file.mjs +22 -19
  29. package/package.json +1 -1
@@ -1,53 +1,5 @@
1
- import {
2
- isHardBillingToolHttpError,
3
- isRateLimitToolHttpError,
4
- } from './tool-http-errors';
5
-
6
- /**
7
- * Thrown by `assertNotAborted` and surfaced through ctx.step / ctx.sleep / map
8
- * processing when the workflow has been terminated externally. Cooperatively
9
- * cancels in-flight user code: the play must check `ctx.signal.aborted` (or
10
- * await one of the abort-aware ctx methods) before doing more work.
11
- */
12
- export class WorkflowAbortError extends Error {
13
- override readonly name = 'WorkflowAbort';
14
- constructor(message = 'Play run cancelled.') {
15
- super(message);
16
- }
17
- }
18
-
19
- export function isAbortLikeError(error: unknown): boolean {
20
- if (!error) return false;
21
- if (error instanceof WorkflowAbortError) return true;
22
- if (error instanceof Error) {
23
- if (error.name === 'WorkflowAbort' || error.name === 'AbortError')
24
- return true;
25
- return /\b(cancell?ed|aborted|terminate[d]?)\b/i.test(error.message);
26
- }
27
- return false;
28
- }
29
-
30
- /**
31
- * Errors that must stay run-fatal even under the default map row failure
32
- * isolation:
33
- *
34
- * - Cancellation/abort must stop the run.
35
- * - Governor budget exhaustion is a run-level invariant — isolating it per
36
- * row would silently convert "this run exceeded its execution budget" into
37
- * thousands of identical row failures.
38
- * - Rate-limit pushback (a tool call that still got HTTP 429 after the
39
- * in-process retry budget) is run-level throughput pressure that applies to
40
- * every row equally, not a row defect. Isolating it silently drops healthy
41
- * rows from the output dataset whenever a provider throttles — the durable
42
- * chunk step's retries (and, if the storm persists, a loud run failure with
43
- * recoverable persisted rows) are the correct response.
44
- * - Hard billing failures (billing cap / insufficient credits) promise "run
45
- * halted before marking remaining rows processed"; isolating them would
46
- * complete the run while silently failing every remaining row.
47
- */
48
- export function isRowIsolationExemptError(error: unknown): boolean {
49
- if (isAbortLikeError(error)) return true;
50
- if (error instanceof Error && error.name === 'GovernorBudgetError')
51
- return true;
52
- return isRateLimitToolHttpError(error) || isHardBillingToolHttpError(error);
53
- }
1
+ export {
2
+ WorkflowAbortError,
3
+ isAbortLikeError,
4
+ isRowIsolationExemptError,
5
+ } from '../../../../shared_libs/play-runtime/row-isolation';
@@ -84,6 +84,7 @@ import type { EnrichCompiledConfig } from './cli/enrich-play-compiler.js';
84
84
  const TERMINAL_PLAY_STATUSES = new Set(['completed', 'failed', 'cancelled']);
85
85
  const INCLUDE_TOOL_METADATA_HEADER = 'x-deepline-include-tool-metadata';
86
86
  const EXECUTE_RESPONSE_CONTRACT_HEADER = 'x-deepline-execute-response-contract';
87
+ const EXECUTE_RESPONSE_INTENT_HEADER = 'x-deepline-execute-response-intent';
87
88
  const V2_EXECUTE_RESPONSE_CONTRACT = 'v2-tool-response';
88
89
  const COMPILE_MANIFEST_RETRY_DELAYS_MS = [250, 1_000];
89
90
  const REGISTER_PLAY_ARTIFACTS_COMPILE_CONCURRENCY = 3;
@@ -163,6 +164,7 @@ function chunkRegisterPlayArtifacts<T>(artifacts: T[]): T[][] {
163
164
 
164
165
  type ExecuteToolRawOptions = {
165
166
  includeToolMetadata?: boolean;
167
+ responseIntent?: 'raw' | 'row_artifact';
166
168
  };
167
169
 
168
170
  /**
@@ -1156,6 +1158,9 @@ export class DeeplineClient {
1156
1158
  ...(options?.includeToolMetadata
1157
1159
  ? { [INCLUDE_TOOL_METADATA_HEADER]: 'true' }
1158
1160
  : {}),
1161
+ ...(options?.responseIntent
1162
+ ? { [EXECUTE_RESPONSE_INTENT_HEADER]: options.responseIntent }
1163
+ : {}),
1159
1164
  };
1160
1165
  return this.http.post<ToolExecution<TData, TMeta>>(
1161
1166
  `/api/v2/integrations/${encodeURIComponent(toolId)}/execute`,
@@ -321,8 +321,8 @@ function loadProjectDeeplineEnv(startDir = process.cwd()): EnvValues {
321
321
  return loadProjectEnvCandidates(startDir)[0]?.env ?? {};
322
322
  }
323
323
 
324
- function normalizeBaseUrl(baseUrl: string): string {
325
- const trimmed = baseUrl.trim().replace(/\/+$/, '');
324
+ function normalizeBaseUrl(baseUrl: string | null | undefined): string {
325
+ const trimmed = baseUrl?.trim().replace(/\/+$/, '') ?? '';
326
326
  if (!trimmed) return '';
327
327
  try {
328
328
  const parsed = new URL(trimmed);
@@ -102,10 +102,10 @@ export const SDK_RELEASE = {
102
102
  // the SDK enrich generator's one-second stale policy.
103
103
  // 0.1.110 ships authored V2 prebuilts and required top-level play descriptions.
104
104
  // 0.1.111 ships dataset-native tool list getters and result row datasets.
105
- version: '0.1.149',
105
+ version: '0.1.151',
106
106
  apiContract: '2026-06-dataset-handle-results-hard-cutover',
107
107
  supportPolicy: {
108
- latest: '0.1.149',
108
+ latest: '0.1.151',
109
109
  minimumSupported: '0.1.53',
110
110
  deprecatedBelow: '0.1.53',
111
111
  commandMinimumSupported: [
@@ -23,7 +23,13 @@
23
23
  *
24
24
  * @module
25
25
  */
26
- import { mkdirSync, writeFileSync } from 'node:fs';
26
+ import {
27
+ closeSync,
28
+ mkdirSync,
29
+ openSync,
30
+ writeFileSync,
31
+ writeSync,
32
+ } from 'node:fs';
27
33
  import { homedir } from 'node:os';
28
34
  import { dirname, join } from 'node:path';
29
35
 
@@ -54,6 +60,22 @@ export type ListConversionResult = {
54
60
  sourcePath: string | null;
55
61
  };
56
62
 
63
+ export type RowOutputProjection = {
64
+ rows: Array<Record<string, unknown>>;
65
+ rowCount: number;
66
+ columns: string[];
67
+ previewRows: Array<Record<string, unknown>>;
68
+ strategy: ListConversionResult['strategy'];
69
+ sourcePath: string | null;
70
+ };
71
+
72
+ export type CsvOutputArtifact = {
73
+ path: string;
74
+ rowCount: number;
75
+ columns: string[];
76
+ preview: string;
77
+ };
78
+
57
79
  type Scalar = string | number | boolean | null;
58
80
 
59
81
  function isPlainObject(value: unknown): value is Record<string, unknown> {
@@ -109,6 +131,20 @@ function normalizeRows(value: unknown): Array<Record<string, unknown>> | null {
109
131
  });
110
132
  }
111
133
 
134
+ function columnsForRows(rows: readonly Record<string, unknown>[]): string[] {
135
+ const seen = new Set<string>();
136
+ const columns: string[] = [];
137
+ for (const row of rows) {
138
+ for (const key of Object.keys(row)) {
139
+ if (!seen.has(key)) {
140
+ seen.add(key);
141
+ columns.push(key);
142
+ }
143
+ }
144
+ }
145
+ return columns;
146
+ }
147
+
112
148
  /**
113
149
  * Generate candidate root objects to search for lists.
114
150
  * Tries: raw payload → V2 toolResponse.raw → legacy payload.output.body → legacy payload.result → legacy payload.result.data.
@@ -271,6 +307,19 @@ export function tryConvertToList(
271
307
  return null;
272
308
  }
273
309
 
310
+ export function projectRowOutput(
311
+ conversion: ListConversionResult,
312
+ ): RowOutputProjection {
313
+ return {
314
+ rows: conversion.rows,
315
+ rowCount: conversion.rows.length,
316
+ columns: columnsForRows(conversion.rows),
317
+ previewRows: conversion.rows.slice(0, 5),
318
+ strategy: conversion.strategy,
319
+ sourcePath: conversion.sourcePath,
320
+ };
321
+ }
322
+
274
323
  /** Ensure the shared output directory exists. Returns its path. */
275
324
  function ensureOutputDir(): string {
276
325
  const outputDir = join(homedir(), '.local', 'share', 'deepline', 'data');
@@ -330,21 +379,12 @@ export function writeCsvOutputFile(
330
379
  rows: Array<Record<string, unknown>>,
331
380
  stem: string,
332
381
  options?: { outPath?: string },
333
- ): { path: string; rowCount: number; columns: string[]; preview: string } {
382
+ ): CsvOutputArtifact {
334
383
  const outputPath = options?.outPath
335
384
  ? options.outPath
336
385
  : join(ensureOutputDir(), `${stem}_${Date.now()}.csv`);
337
386
  mkdirSync(dirname(outputPath), { recursive: true });
338
- const seen = new Set<string>();
339
- const columns: string[] = [];
340
- for (const row of rows) {
341
- for (const key of Object.keys(row)) {
342
- if (!seen.has(key)) {
343
- seen.add(key);
344
- columns.push(key);
345
- }
346
- }
347
- }
387
+ const columns = columnsForRows(rows);
348
388
 
349
389
  const escapeCell = (value: unknown): string => {
350
390
  const normalized =
@@ -361,12 +401,18 @@ export function writeCsvOutputFile(
361
401
  return normalized;
362
402
  };
363
403
 
364
- const lines: string[] = [];
365
- lines.push(columns.map(escapeCell).join(','));
366
- for (const row of rows) {
367
- lines.push(columns.map((column) => escapeCell(row[column])).join(','));
404
+ const fd = openSync(outputPath, 'w');
405
+ try {
406
+ writeSync(fd, `${columns.map(escapeCell).join(',')}\n`);
407
+ for (const row of rows) {
408
+ writeSync(
409
+ fd,
410
+ `${columns.map((column) => escapeCell(row[column])).join(',')}\n`,
411
+ );
412
+ }
413
+ } finally {
414
+ closeSync(fd);
368
415
  }
369
- writeFileSync(outputPath, `${lines.join('\n')}\n`, 'utf-8');
370
416
 
371
417
  const previewRows = rows.slice(0, 5);
372
418
  const previewColumns = columns.slice(0, 5);
@@ -28,6 +28,12 @@ import {
28
28
  } from './batch-runtime';
29
29
  import type { PlayQueueHint } from './governor/rate-state-backend';
30
30
  import type { MapRowOutcome } from './durability-store';
31
+ import {
32
+ completedMapRowOutcome,
33
+ failedMapRowOutcome,
34
+ mapRowOutcomeRuntimeFields,
35
+ resolveMapRowOutcomeKey,
36
+ } from './map-row-outcome';
31
37
  import {
32
38
  createDefaultGovernanceSnapshot,
33
39
  createPlayExecutionGovernor,
@@ -35,12 +41,9 @@ import {
35
41
  type PacingResolver,
36
42
  type PlayExecutionGovernor,
37
43
  } from './governor/governor';
38
- import {
39
- CTX_FETCH_EGRESS_TOOL_ID,
40
- resolveBuiltinPacing,
41
- } from './builtin-pacing';
44
+ import { CTX_FETCH_EGRESS_TOOL_ID } from './builtin-pacing';
42
45
  import { InMemoryRateStateBackend } from './governor/in-memory-rate-state-backend';
43
- import type { PacingRule } from './governor/rate-state-backend';
46
+ import { pacingPolicyForTool } from './pacing';
44
47
  import {
45
48
  cloneToolExecuteResultWithExecution,
46
49
  createToolExecuteResult,
@@ -54,13 +57,11 @@ import {
54
57
  type ToolResultMetadataInput,
55
58
  } from './tool-result';
56
59
  import {
57
- TOOL_EXECUTE_TRANSIENT_HTTP_MAX_ATTEMPTS,
58
- decideToolExecuteHttpRetry,
60
+ TOOL_EXECUTE_TRANSPORT_MAX_ATTEMPTS,
61
+ classifyToolExecuteHttpFailure,
62
+ createToolExecuteHttpFailureAttemptTracker,
59
63
  } from './tool-execute-retry-policy';
60
- import {
61
- isHardBillingToolHttpError,
62
- normalizeToolHttpErrorMessage,
63
- } from './tool-http-errors';
64
+ import { isRowIsolationExemptError } from './row-isolation';
64
65
  import { sqlSafePlayColumnName } from '@shared_libs/plays/static-pipeline';
65
66
  import { createRuntimeDatasetId } from './dataset-id';
66
67
  import { dedupeExplicitMapKeyRows } from './map-row-identity';
@@ -577,24 +578,6 @@ function normalizeFetchHeaders(
577
578
  );
578
579
  }
579
580
 
580
- function parseRetryAfterMs(header: string | null): number {
581
- if (!header) {
582
- return TOOL_RETRY_AFTER_FALLBACK_MS;
583
- }
584
-
585
- const seconds = Number(header);
586
- if (Number.isFinite(seconds) && seconds > 0) {
587
- return Math.ceil(seconds * 1000);
588
- }
589
-
590
- const retryAt = Date.parse(header);
591
- if (Number.isFinite(retryAt)) {
592
- return Math.max(1, retryAt - Date.now());
593
- }
594
-
595
- return TOOL_RETRY_AFTER_FALLBACK_MS;
596
- }
597
-
598
581
  function parseJsonOrNull(bodyText: string): unknown | null {
599
582
  if (!bodyText.trim()) return null;
600
583
  try {
@@ -652,20 +635,10 @@ function createPacingResolver(
652
635
  getToolQueueHints: ContextOptions['getToolQueueHints'],
653
636
  ): PacingResolver {
654
637
  return async (toolId: string) => {
655
- const builtin = resolveBuiltinPacing(toolId);
638
+ const builtin = pacingPolicyForTool(toolId, []);
656
639
  if (builtin) return builtin;
657
640
  const hints = getToolQueueHints ? await getToolQueueHints(toolId) : [];
658
- if (hints.length === 0) {
659
- return null;
660
- }
661
- const provider = hints[0]!.provider;
662
- const rules: PacingRule[] = hints.map((hint) => ({
663
- ruleId: hint.ruleId,
664
- requestsPerWindow: hint.requestsPerWindow,
665
- windowMs: hint.windowMs,
666
- maxConcurrency: hint.maxConcurrency,
667
- }));
668
- return { provider, rules };
641
+ return pacingPolicyForTool(toolId, hints);
669
642
  };
670
643
  }
671
644
 
@@ -2030,9 +2003,8 @@ export class PlayContextImpl {
2030
2003
  }
2031
2004
 
2032
2005
  const rowIdentity = (row: Record<string, unknown>, index = 0) => {
2033
- if (typeof row.__deeplineRowKey === 'string') {
2034
- return row.__deeplineRowKey;
2035
- }
2006
+ const runtimeKey = resolveMapRowOutcomeKey(row);
2007
+ if (runtimeKey) return runtimeKey;
2036
2008
  return explicitKeyResolver
2037
2009
  ? derivePlayRowIdentityFromKey(
2038
2010
  explicitKeyResolver(row, index),
@@ -2081,7 +2053,9 @@ export class PlayContextImpl {
2081
2053
  const mapStartRows = shouldPassRowKey
2082
2054
  ? rawItems.map((row, index) => ({
2083
2055
  ...toSerializableCsvAliasedRow(row),
2084
- __deeplineRowKey: rowIdentity(row, index),
2056
+ ...mapRowOutcomeRuntimeFields({
2057
+ key: rowIdentity(row, index),
2058
+ }),
2085
2059
  }))
2086
2060
  : rawItems.map((row) => toSerializableCsvAliasedRow(row));
2087
2061
  const mapStartResult = await this.#options.onMapStart(
@@ -2099,9 +2073,7 @@ export class PlayContextImpl {
2099
2073
  mapStartResult.tableNamespace,
2100
2074
  );
2101
2075
  const persistedRowIdentity = (row: Record<string, unknown>, index = 0) =>
2102
- typeof row.__deeplineRowKey === 'string'
2103
- ? row.__deeplineRowKey
2104
- : rowIdentity(row, index);
2076
+ resolveMapRowOutcomeKey(row) ?? rowIdentity(row, index);
2105
2077
  const pendingKeys = new Set(
2106
2078
  mapStartResult.pendingRows.map((row, index) =>
2107
2079
  persistedRowIdentity(row, index),
@@ -2336,14 +2308,16 @@ export class PlayContextImpl {
2336
2308
  if (incrementalPersistence?.isPersisted(row)) continue;
2337
2309
  const rowKey = row.key;
2338
2310
  const meta = mapCellMeta?.get(rowKey);
2339
- persistRows.push({
2340
- key: rowKey,
2341
- data: row.data,
2342
- cellMetaPatch: {
2343
- ...(row.cellMetaPatch ?? {}),
2344
- ...(meta ?? {}),
2345
- },
2346
- });
2311
+ persistRows.push(
2312
+ completedMapRowOutcome({
2313
+ key: rowKey,
2314
+ data: row.data,
2315
+ cellMetaPatch: {
2316
+ ...(row.cellMetaPatch ?? {}),
2317
+ ...(meta ?? {}),
2318
+ },
2319
+ }),
2320
+ );
2347
2321
  }
2348
2322
  persistRows.push(
2349
2323
  ...mapResult.failedRows.filter(
@@ -2441,20 +2415,19 @@ export class PlayContextImpl {
2441
2415
  .filter((fieldName) => shouldPersistMapCellField(fieldName));
2442
2416
  const normalizedTableNamespace = mapScope.artifactTableNamespace;
2443
2417
  const rowIdentity = (row: Record<string, unknown>, index = 0) =>
2444
- typeof row.__deeplineRowKey === 'string'
2445
- ? row.__deeplineRowKey
2446
- : mapScope.rowIdentity(
2447
- stripCsvProjectedFields(
2448
- Object.fromEntries(
2449
- Object.entries(row).filter(
2450
- ([fieldName]) =>
2451
- !datasetColumnNames.includes(fieldName) &&
2452
- !fieldName.startsWith('__deepline'),
2453
- ),
2454
- ),
2418
+ resolveMapRowOutcomeKey(row) ??
2419
+ mapScope.rowIdentity(
2420
+ stripCsvProjectedFields(
2421
+ Object.fromEntries(
2422
+ Object.entries(row).filter(
2423
+ ([fieldName]) =>
2424
+ !datasetColumnNames.includes(fieldName) &&
2425
+ !fieldName.startsWith('__deepline'),
2455
2426
  ),
2456
- index,
2457
- );
2427
+ ),
2428
+ ),
2429
+ index,
2430
+ );
2458
2431
  const executionRowKey = (row: Record<string, unknown>, index: number) =>
2459
2432
  runtimeOptions?.executionRowKeys?.[index] ?? rowIdentity(row, index);
2460
2433
  const executionRowIndex = (index: number) =>
@@ -2645,14 +2618,16 @@ export class PlayContextImpl {
2645
2618
  `Map completed: ${results.length + completedRows} results (${results.length} executed, ${completedRows} already satisfied)`,
2646
2619
  );
2647
2620
  return {
2648
- completedRows: results.map((row, index) => ({
2649
- key: executionRowKey(
2650
- this.toOutputRow(items[index] as Record<string, unknown>),
2651
- index,
2652
- ),
2653
- inputIndex: executionRowIndex(index),
2654
- data: this.toPersistedOutputRow(row),
2655
- })),
2621
+ completedRows: results.map((row, index) =>
2622
+ completedMapRowOutcome({
2623
+ key: executionRowKey(
2624
+ this.toOutputRow(items[index] as Record<string, unknown>),
2625
+ index,
2626
+ ),
2627
+ inputIndex: executionRowIndex(index),
2628
+ data: this.toPersistedOutputRow(row),
2629
+ }),
2630
+ ),
2656
2631
  failedRows: [],
2657
2632
  };
2658
2633
  }
@@ -2767,6 +2742,9 @@ export class PlayContextImpl {
2767
2742
  ) {
2768
2743
  throw error;
2769
2744
  }
2745
+ if (isRowIsolationExemptError(error)) {
2746
+ throw error;
2747
+ }
2770
2748
 
2771
2749
  value = null;
2772
2750
  computedFields[fieldName] = value;
@@ -2787,7 +2765,7 @@ export class PlayContextImpl {
2787
2765
  : {},
2788
2766
  });
2789
2767
  if (failFastOnRowError) {
2790
- const failedRow: PersistableMapRow = {
2768
+ const failedRow: PersistableMapRow = failedMapRowOutcome({
2791
2769
  key: rowKey,
2792
2770
  inputIndex: rowIndex,
2793
2771
  data: this.toPersistedOutputRow(
@@ -2796,9 +2774,8 @@ export class PlayContextImpl {
2796
2774
  ...(this.activeMapCellMeta?.get(rowKey)
2797
2775
  ? { cellMetaPatch: this.activeMapCellMeta.get(rowKey) }
2798
2776
  : {}),
2799
- status: 'failed',
2800
2777
  error: formattedError,
2801
- };
2778
+ });
2802
2779
  failedRowsToPersist.push(failedRow);
2803
2780
  throw error;
2804
2781
  }
@@ -2806,14 +2783,13 @@ export class PlayContextImpl {
2806
2783
  cloneCsvAliasedRow(baseRow, computedFields),
2807
2784
  );
2808
2785
  const cellMetaPatch = this.activeMapCellMeta?.get(rowKey);
2809
- const failedRow: PersistableMapRow = {
2786
+ const failedRow: PersistableMapRow = failedMapRowOutcome({
2810
2787
  key: rowKey,
2811
2788
  inputIndex: rowIndex,
2812
2789
  data: failedData,
2813
2790
  ...(cellMetaPatch ? { cellMetaPatch } : {}),
2814
- status: 'failed',
2815
2791
  error: formattedError,
2816
- };
2792
+ });
2817
2793
  failedRowsToPersist.push(failedRow);
2818
2794
  updateMapFrameProgress({
2819
2795
  failedRowKey: rowKey,
@@ -2867,14 +2843,14 @@ export class PlayContextImpl {
2867
2843
  dataPatch: {},
2868
2844
  });
2869
2845
  const publicRow = this.toPublicOutputRow(merged);
2870
- const completedRow: PersistableMapRow = {
2846
+ const completedRow: PersistableMapRow = completedMapRowOutcome({
2871
2847
  key: rowKey,
2872
2848
  inputIndex: rowIndex,
2873
2849
  data: this.toPersistedOutputRow(merged),
2874
2850
  ...(this.activeMapCellMeta?.get(rowKey)
2875
2851
  ? { cellMetaPatch: this.activeMapCellMeta.get(rowKey) }
2876
2852
  : {}),
2877
- };
2853
+ });
2878
2854
  completedRowsToPersist.push(completedRow);
2879
2855
  enqueueIncrementalPersist(completedRow);
2880
2856
  return publicRow;
@@ -5538,7 +5514,13 @@ export class PlayContextImpl {
5538
5514
  },
5539
5515
  },
5540
5516
  async (span) => {
5541
- let rateLimitAttempt = 0;
5517
+ const httpFailureAttempts =
5518
+ createToolExecuteHttpFailureAttemptTracker();
5519
+ const retryPolicy = await this.#options
5520
+ .getToolRetryPolicy?.(toolId)
5521
+ .catch(() => null);
5522
+ const retrySafeTransientHttp =
5523
+ retryPolicy?.retrySafeTransientHttp === true;
5542
5524
  let transportAttempt = 0;
5543
5525
 
5544
5526
  while (true) {
@@ -5568,7 +5550,7 @@ export class PlayContextImpl {
5568
5550
  transportAttempt += 1;
5569
5551
  const message =
5570
5552
  error instanceof Error ? error.message : String(error);
5571
- if (transportAttempt < TOOL_EXECUTE_TRANSIENT_HTTP_MAX_ATTEMPTS) {
5553
+ if (transportAttempt < TOOL_EXECUTE_TRANSPORT_MAX_ATTEMPTS) {
5572
5554
  this.governor.chargeBudget('retry');
5573
5555
  const retryAfterMs =
5574
5556
  TOOL_RETRY_AFTER_FALLBACK_MS * transportAttempt;
@@ -5577,105 +5559,65 @@ export class PlayContextImpl {
5577
5559
  transportAttempt,
5578
5560
  );
5579
5561
  this.log(
5580
- `Tool ${toolId} transport failed calling ${url} on attempt ${transportAttempt}/${TOOL_EXECUTE_TRANSIENT_HTTP_MAX_ATTEMPTS}; retrying after ${retryAfterMs}ms: ${message}`,
5562
+ `Tool ${toolId} transport failed calling ${url} on attempt ${transportAttempt}/${TOOL_EXECUTE_TRANSPORT_MAX_ATTEMPTS}; retrying after ${retryAfterMs}ms: ${message}`,
5581
5563
  );
5582
5564
  await this.sleepWithCheckpointHeartbeat(retryAfterMs);
5583
5565
  continue;
5584
5566
  }
5585
5567
  throw new Error(
5586
- `Tool ${toolId} transport failed calling ${url} after ${transportAttempt}/${TOOL_EXECUTE_TRANSIENT_HTTP_MAX_ATTEMPTS} attempts: ${message}`,
5568
+ `Tool ${toolId} transport failed calling ${url} after ${transportAttempt}/${TOOL_EXECUTE_TRANSPORT_MAX_ATTEMPTS} attempts: ${message}`,
5587
5569
  );
5588
5570
  }
5589
5571
 
5590
5572
  span.setAttribute('plays.http_status_code', response.status);
5591
5573
 
5592
- if (response.status === 429) {
5593
- rateLimitAttempt += 1;
5594
- const text = await response.text();
5595
- const initialRetryDecision = decideToolExecuteHttpRetry({
5596
- toolId,
5597
- status: response.status,
5598
- });
5599
- const error = normalizeToolHttpErrorMessage({
5600
- toolId,
5601
- status: response.status,
5602
- attempt: rateLimitAttempt,
5603
- maxAttempts: initialRetryDecision.attemptCap,
5604
- bodyText: text,
5605
- });
5606
- const retryDecision = decideToolExecuteHttpRetry({
5607
- toolId,
5608
- status: response.status,
5609
- hardBillingFailure: isHardBillingToolHttpError(error),
5610
- });
5611
- if (
5612
- !retryDecision.retryable ||
5613
- rateLimitAttempt >= retryDecision.attemptCap
5614
- ) {
5615
- throw error;
5616
- }
5617
- this.governor.chargeBudget('retry');
5618
- const retryAfterMs = parseRetryAfterMs(
5619
- response.headers.get('retry-after'),
5620
- );
5621
- // Feed the server-observed Retry-After back into the shared pacer
5622
- // so subsequent acquires for this provider back off.
5623
- await this.reportToolBackpressure(toolId, retryAfterMs);
5624
- span.setAttribute(
5625
- 'plays.rate_limit_retry_after_ms',
5626
- retryAfterMs,
5627
- );
5628
- span.setAttribute('plays.rate_limit_attempt', rateLimitAttempt);
5629
- this.log(
5630
- `Tool ${toolId} rate limited; retrying after ${retryAfterMs}ms`,
5631
- );
5632
- await this.sleepWithCheckpointHeartbeat(retryAfterMs);
5633
- continue;
5634
- }
5635
-
5636
5574
  if (!response.ok) {
5637
5575
  const text = await response.text();
5638
- const initialRetryDecision = decideToolExecuteHttpRetry({
5576
+ const httpFailureAttempt = httpFailureAttempts.next({
5639
5577
  toolId,
5640
5578
  status: response.status,
5579
+ transientHttpRetrySafe: retrySafeTransientHttp,
5641
5580
  });
5642
- const error = normalizeToolHttpErrorMessage({
5581
+ const failure = classifyToolExecuteHttpFailure({
5643
5582
  toolId,
5644
5583
  status: response.status,
5645
- attempt: rateLimitAttempt + 1,
5646
- maxAttempts: initialRetryDecision.attemptCap,
5584
+ attempt: httpFailureAttempt,
5647
5585
  bodyText: text,
5586
+ retryAfterHeader: response.headers.get('retry-after'),
5587
+ transientHttpRetrySafe: retrySafeTransientHttp,
5648
5588
  });
5649
- const retryDecision = decideToolExecuteHttpRetry({
5650
- toolId,
5651
- status: response.status,
5652
- hardBillingFailure: isHardBillingToolHttpError(error),
5653
- });
5654
- if (
5655
- retryDecision.retryable &&
5656
- rateLimitAttempt + 1 < retryDecision.attemptCap
5657
- ) {
5658
- rateLimitAttempt += 1;
5659
- this.governor.chargeBudget('retry');
5660
- const retryAfterMs = parseRetryAfterMs(
5661
- response.headers.get('retry-after'),
5589
+ if (failure.backpressureDelayMs !== null) {
5590
+ // Feed the server-observed Retry-After back into the shared
5591
+ // pacer even on the final attempt so later provider calls back
5592
+ // off instead of retrying the whole map chunk.
5593
+ await this.reportToolBackpressure(
5594
+ toolId,
5595
+ failure.backpressureDelayMs,
5662
5596
  );
5597
+ }
5598
+ if (failure.shouldRetry) {
5599
+ if (failure.chargeRetryBudget) {
5600
+ this.governor.chargeBudget('retry');
5601
+ }
5602
+ const retryAttributePrefix = failure.isRateLimit
5603
+ ? 'rate_limit'
5604
+ : 'transient_http';
5663
5605
  span.setAttribute(
5664
- 'plays.transient_http_retry_after_ms',
5665
- retryAfterMs,
5606
+ `plays.${retryAttributePrefix}_retry_after_ms`,
5607
+ failure.retryDelayMs,
5666
5608
  );
5667
5609
  span.setAttribute(
5668
- 'plays.transient_http_attempt',
5669
- rateLimitAttempt,
5610
+ `plays.${retryAttributePrefix}_attempt`,
5611
+ httpFailureAttempt,
5670
5612
  );
5671
5613
  this.log(
5672
- `Tool ${toolId} returned ${response.status}; retrying after ${retryAfterMs}ms`,
5614
+ `Tool ${toolId} returned ${response.status}; retrying after ${failure.retryDelayMs}ms`,
5673
5615
  );
5674
- await this.sleepWithCheckpointHeartbeat(retryAfterMs);
5616
+ await this.sleepWithCheckpointHeartbeat(failure.retryDelayMs);
5675
5617
  continue;
5676
5618
  }
5677
- this.log(error.message);
5678
- throw error;
5619
+ this.log(failure.error.message);
5620
+ throw failure.error;
5679
5621
  }
5680
5622
 
5681
5623
  const data = (await response.json()) as Record<string, unknown>;
@@ -487,6 +487,9 @@ export interface ContextOptions {
487
487
  runId?: string;
488
488
  resolvePlay?: (playRef: string) => Promise<ResolvedPlayExecution | null>;
489
489
  getToolQueueHints?: (toolId: string) => Promise<readonly PlayQueueHint[]>;
490
+ getToolRetryPolicy?: (toolId: string) => Promise<{
491
+ retrySafeTransientHttp?: boolean;
492
+ } | null>;
490
493
  getToolTargetGetters?: (
491
494
  toolId: string,
492
495
  output: string,