deepline 0.1.150 → 0.1.152

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (23) hide show
  1. package/dist/bundling-sources/apps/play-runner-workers/src/entry.ts +170 -168
  2. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/csv-rows.ts +2 -19
  3. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/row-isolation.ts +5 -53
  4. package/dist/bundling-sources/sdk/src/config.ts +2 -2
  5. package/dist/bundling-sources/sdk/src/release.ts +2 -2
  6. package/dist/bundling-sources/shared_libs/play-runtime/context.ts +101 -162
  7. package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +3 -0
  8. package/dist/bundling-sources/shared_libs/play-runtime/durability-store.ts +54 -0
  9. package/dist/bundling-sources/shared_libs/play-runtime/map-row-outcome.ts +167 -0
  10. package/dist/bundling-sources/shared_libs/play-runtime/pacing.ts +79 -0
  11. package/dist/bundling-sources/shared_libs/play-runtime/row-isolation.ts +39 -0
  12. package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +36 -86
  13. package/dist/bundling-sources/shared_libs/play-runtime/runtime-sheet-row-transition.ts +90 -0
  14. package/dist/bundling-sources/shared_libs/play-runtime/runtime-sheet-session.ts +43 -0
  15. package/dist/bundling-sources/shared_libs/play-runtime/tool-execute-retry-policy.ts +142 -11
  16. package/dist/bundling-sources/shared_libs/play-runtime/tool-http-errors.ts +3 -2
  17. package/dist/bundling-sources/shared_libs/plays/bundling/index.ts +20 -23
  18. package/dist/cli/index.js +35 -3
  19. package/dist/cli/index.mjs +35 -3
  20. package/dist/index.js +3 -3
  21. package/dist/index.mjs +3 -3
  22. package/dist/plays/bundle-play-file.mjs +22 -19
  23. package/package.json +1 -1
@@ -2,14 +2,9 @@ import {
2
2
  cloneCsvAliasedRow,
3
3
  stripCsvProjectionMetadata,
4
4
  } from '../../../../shared_libs/play-runtime/csv-rename';
5
+ import { copyMapRowOutcomeRuntimeFields } from '../../../../shared_libs/play-runtime/map-row-outcome';
5
6
 
6
7
  const CSV_TEMPLATE_CONTEXT = '__deeplineCsvProjectedValues';
7
- const RUNTIME_STORAGE_FIELDS = [
8
- '__deeplineRowKey',
9
- '__deeplineCellMetaPatch',
10
- '__deeplineRowStatus',
11
- '__deeplineRowError',
12
- ] as const;
13
8
 
14
9
  export type CsvRuntimeRow = Record<string, unknown>;
15
10
 
@@ -74,18 +69,6 @@ export function publicCsvOutputRow<T extends CsvRuntimeRow>(row: T): T {
74
69
  return publicCsvInputRow(row);
75
70
  }
76
71
 
77
- function copyRuntimeStorageFields(
78
- target: CsvRuntimeRow,
79
- row: CsvRuntimeRow,
80
- ): CsvRuntimeRow {
81
- for (const runtimeField of RUNTIME_STORAGE_FIELDS) {
82
- if (runtimeField in row) {
83
- target[runtimeField] = row[runtimeField];
84
- }
85
- }
86
- return target;
87
- }
88
-
89
72
  export function publicCsvStorageRow<T extends CsvRuntimeRow>(row: T): T {
90
73
  const publicRow = publicCsvInputRow(row) as CsvRuntimeRow;
91
74
  const storageRow: CsvRuntimeRow = {};
@@ -93,7 +76,7 @@ export function publicCsvStorageRow<T extends CsvRuntimeRow>(row: T): T {
93
76
  if (typeof fieldName !== 'string') continue;
94
77
  storageRow[fieldName] = publicRow[fieldName];
95
78
  }
96
- return copyRuntimeStorageFields(storageRow, row) as T;
79
+ return copyMapRowOutcomeRuntimeFields(storageRow, row) as T;
97
80
  }
98
81
 
99
82
  export function runtimeCsvStorageRow<T extends CsvRuntimeRow>(row: T): T {
@@ -1,53 +1,5 @@
1
- import {
2
- isHardBillingToolHttpError,
3
- isRateLimitToolHttpError,
4
- } from './tool-http-errors';
5
-
6
- /**
7
- * Thrown by `assertNotAborted` and surfaced through ctx.step / ctx.sleep / map
8
- * processing when the workflow has been terminated externally. Cooperatively
9
- * cancels in-flight user code: the play must check `ctx.signal.aborted` (or
10
- * await one of the abort-aware ctx methods) before doing more work.
11
- */
12
- export class WorkflowAbortError extends Error {
13
- override readonly name = 'WorkflowAbort';
14
- constructor(message = 'Play run cancelled.') {
15
- super(message);
16
- }
17
- }
18
-
19
- export function isAbortLikeError(error: unknown): boolean {
20
- if (!error) return false;
21
- if (error instanceof WorkflowAbortError) return true;
22
- if (error instanceof Error) {
23
- if (error.name === 'WorkflowAbort' || error.name === 'AbortError')
24
- return true;
25
- return /\b(cancell?ed|aborted|terminate[d]?)\b/i.test(error.message);
26
- }
27
- return false;
28
- }
29
-
30
- /**
31
- * Errors that must stay run-fatal even under the default map row failure
32
- * isolation:
33
- *
34
- * - Cancellation/abort must stop the run.
35
- * - Governor budget exhaustion is a run-level invariant — isolating it per
36
- * row would silently convert "this run exceeded its execution budget" into
37
- * thousands of identical row failures.
38
- * - Rate-limit pushback (a tool call that still got HTTP 429 after the
39
- * in-process retry budget) is run-level throughput pressure that applies to
40
- * every row equally, not a row defect. Isolating it silently drops healthy
41
- * rows from the output dataset whenever a provider throttles — the durable
42
- * chunk step's retries (and, if the storm persists, a loud run failure with
43
- * recoverable persisted rows) are the correct response.
44
- * - Hard billing failures (billing cap / insufficient credits) promise "run
45
- * halted before marking remaining rows processed"; isolating them would
46
- * complete the run while silently failing every remaining row.
47
- */
48
- export function isRowIsolationExemptError(error: unknown): boolean {
49
- if (isAbortLikeError(error)) return true;
50
- if (error instanceof Error && error.name === 'GovernorBudgetError')
51
- return true;
52
- return isRateLimitToolHttpError(error) || isHardBillingToolHttpError(error);
53
- }
1
+ export {
2
+ WorkflowAbortError,
3
+ isAbortLikeError,
4
+ isRowIsolationExemptError,
5
+ } from '../../../../shared_libs/play-runtime/row-isolation';
@@ -321,8 +321,8 @@ function loadProjectDeeplineEnv(startDir = process.cwd()): EnvValues {
321
321
  return loadProjectEnvCandidates(startDir)[0]?.env ?? {};
322
322
  }
323
323
 
324
- function normalizeBaseUrl(baseUrl: string): string {
325
- const trimmed = baseUrl.trim().replace(/\/+$/, '');
324
+ function normalizeBaseUrl(baseUrl: string | null | undefined): string {
325
+ const trimmed = baseUrl?.trim().replace(/\/+$/, '') ?? '';
326
326
  if (!trimmed) return '';
327
327
  try {
328
328
  const parsed = new URL(trimmed);
@@ -102,10 +102,10 @@ export const SDK_RELEASE = {
102
102
  // the SDK enrich generator's one-second stale policy.
103
103
  // 0.1.110 ships authored V2 prebuilts and required top-level play descriptions.
104
104
  // 0.1.111 ships dataset-native tool list getters and result row datasets.
105
- version: '0.1.150',
105
+ version: '0.1.152',
106
106
  apiContract: '2026-06-dataset-handle-results-hard-cutover',
107
107
  supportPolicy: {
108
- latest: '0.1.150',
108
+ latest: '0.1.152',
109
109
  minimumSupported: '0.1.53',
110
110
  deprecatedBelow: '0.1.53',
111
111
  commandMinimumSupported: [
@@ -28,6 +28,12 @@ import {
28
28
  } from './batch-runtime';
29
29
  import type { PlayQueueHint } from './governor/rate-state-backend';
30
30
  import type { MapRowOutcome } from './durability-store';
31
+ import {
32
+ completedMapRowOutcome,
33
+ failedMapRowOutcome,
34
+ mapRowOutcomeRuntimeFields,
35
+ resolveMapRowOutcomeKey,
36
+ } from './map-row-outcome';
31
37
  import {
32
38
  createDefaultGovernanceSnapshot,
33
39
  createPlayExecutionGovernor,
@@ -35,12 +41,9 @@ import {
35
41
  type PacingResolver,
36
42
  type PlayExecutionGovernor,
37
43
  } from './governor/governor';
38
- import {
39
- CTX_FETCH_EGRESS_TOOL_ID,
40
- resolveBuiltinPacing,
41
- } from './builtin-pacing';
44
+ import { CTX_FETCH_EGRESS_TOOL_ID } from './builtin-pacing';
42
45
  import { InMemoryRateStateBackend } from './governor/in-memory-rate-state-backend';
43
- import type { PacingRule } from './governor/rate-state-backend';
46
+ import { pacingPolicyForTool } from './pacing';
44
47
  import {
45
48
  cloneToolExecuteResultWithExecution,
46
49
  createToolExecuteResult,
@@ -54,13 +57,11 @@ import {
54
57
  type ToolResultMetadataInput,
55
58
  } from './tool-result';
56
59
  import {
57
- TOOL_EXECUTE_TRANSIENT_HTTP_MAX_ATTEMPTS,
58
- decideToolExecuteHttpRetry,
60
+ TOOL_EXECUTE_TRANSPORT_MAX_ATTEMPTS,
61
+ classifyToolExecuteHttpFailure,
62
+ createToolExecuteHttpFailureAttemptTracker,
59
63
  } from './tool-execute-retry-policy';
60
- import {
61
- isHardBillingToolHttpError,
62
- normalizeToolHttpErrorMessage,
63
- } from './tool-http-errors';
64
+ import { isRowIsolationExemptError } from './row-isolation';
64
65
  import { sqlSafePlayColumnName } from '@shared_libs/plays/static-pipeline';
65
66
  import { createRuntimeDatasetId } from './dataset-id';
66
67
  import { dedupeExplicitMapKeyRows } from './map-row-identity';
@@ -577,24 +578,6 @@ function normalizeFetchHeaders(
577
578
  );
578
579
  }
579
580
 
580
- function parseRetryAfterMs(header: string | null): number {
581
- if (!header) {
582
- return TOOL_RETRY_AFTER_FALLBACK_MS;
583
- }
584
-
585
- const seconds = Number(header);
586
- if (Number.isFinite(seconds) && seconds > 0) {
587
- return Math.ceil(seconds * 1000);
588
- }
589
-
590
- const retryAt = Date.parse(header);
591
- if (Number.isFinite(retryAt)) {
592
- return Math.max(1, retryAt - Date.now());
593
- }
594
-
595
- return TOOL_RETRY_AFTER_FALLBACK_MS;
596
- }
597
-
598
581
  function parseJsonOrNull(bodyText: string): unknown | null {
599
582
  if (!bodyText.trim()) return null;
600
583
  try {
@@ -652,20 +635,10 @@ function createPacingResolver(
652
635
  getToolQueueHints: ContextOptions['getToolQueueHints'],
653
636
  ): PacingResolver {
654
637
  return async (toolId: string) => {
655
- const builtin = resolveBuiltinPacing(toolId);
638
+ const builtin = pacingPolicyForTool(toolId, []);
656
639
  if (builtin) return builtin;
657
640
  const hints = getToolQueueHints ? await getToolQueueHints(toolId) : [];
658
- if (hints.length === 0) {
659
- return null;
660
- }
661
- const provider = hints[0]!.provider;
662
- const rules: PacingRule[] = hints.map((hint) => ({
663
- ruleId: hint.ruleId,
664
- requestsPerWindow: hint.requestsPerWindow,
665
- windowMs: hint.windowMs,
666
- maxConcurrency: hint.maxConcurrency,
667
- }));
668
- return { provider, rules };
641
+ return pacingPolicyForTool(toolId, hints);
669
642
  };
670
643
  }
671
644
 
@@ -2030,9 +2003,8 @@ export class PlayContextImpl {
2030
2003
  }
2031
2004
 
2032
2005
  const rowIdentity = (row: Record<string, unknown>, index = 0) => {
2033
- if (typeof row.__deeplineRowKey === 'string') {
2034
- return row.__deeplineRowKey;
2035
- }
2006
+ const runtimeKey = resolveMapRowOutcomeKey(row);
2007
+ if (runtimeKey) return runtimeKey;
2036
2008
  return explicitKeyResolver
2037
2009
  ? derivePlayRowIdentityFromKey(
2038
2010
  explicitKeyResolver(row, index),
@@ -2081,7 +2053,9 @@ export class PlayContextImpl {
2081
2053
  const mapStartRows = shouldPassRowKey
2082
2054
  ? rawItems.map((row, index) => ({
2083
2055
  ...toSerializableCsvAliasedRow(row),
2084
- __deeplineRowKey: rowIdentity(row, index),
2056
+ ...mapRowOutcomeRuntimeFields({
2057
+ key: rowIdentity(row, index),
2058
+ }),
2085
2059
  }))
2086
2060
  : rawItems.map((row) => toSerializableCsvAliasedRow(row));
2087
2061
  const mapStartResult = await this.#options.onMapStart(
@@ -2099,9 +2073,7 @@ export class PlayContextImpl {
2099
2073
  mapStartResult.tableNamespace,
2100
2074
  );
2101
2075
  const persistedRowIdentity = (row: Record<string, unknown>, index = 0) =>
2102
- typeof row.__deeplineRowKey === 'string'
2103
- ? row.__deeplineRowKey
2104
- : rowIdentity(row, index);
2076
+ resolveMapRowOutcomeKey(row) ?? rowIdentity(row, index);
2105
2077
  const pendingKeys = new Set(
2106
2078
  mapStartResult.pendingRows.map((row, index) =>
2107
2079
  persistedRowIdentity(row, index),
@@ -2271,10 +2243,7 @@ export class PlayContextImpl {
2271
2243
  );
2272
2244
  } catch (error) {
2273
2245
  if (error instanceof FailFastMapRowsError) {
2274
- const rowsToPersist =
2275
- error.completedRows.length > 0
2276
- ? error.completedRows
2277
- : error.failedRows;
2246
+ const rowsToPersist = [...error.completedRows, ...error.failedRows];
2278
2247
  await incrementalPersistence?.flush();
2279
2248
  const unpersistedRows = incrementalPersistence
2280
2249
  ? rowsToPersist.filter(
@@ -2336,14 +2305,16 @@ export class PlayContextImpl {
2336
2305
  if (incrementalPersistence?.isPersisted(row)) continue;
2337
2306
  const rowKey = row.key;
2338
2307
  const meta = mapCellMeta?.get(rowKey);
2339
- persistRows.push({
2340
- key: rowKey,
2341
- data: row.data,
2342
- cellMetaPatch: {
2343
- ...(row.cellMetaPatch ?? {}),
2344
- ...(meta ?? {}),
2345
- },
2346
- });
2308
+ persistRows.push(
2309
+ completedMapRowOutcome({
2310
+ key: rowKey,
2311
+ data: row.data,
2312
+ cellMetaPatch: {
2313
+ ...(row.cellMetaPatch ?? {}),
2314
+ ...(meta ?? {}),
2315
+ },
2316
+ }),
2317
+ );
2347
2318
  }
2348
2319
  persistRows.push(
2349
2320
  ...mapResult.failedRows.filter(
@@ -2441,20 +2412,19 @@ export class PlayContextImpl {
2441
2412
  .filter((fieldName) => shouldPersistMapCellField(fieldName));
2442
2413
  const normalizedTableNamespace = mapScope.artifactTableNamespace;
2443
2414
  const rowIdentity = (row: Record<string, unknown>, index = 0) =>
2444
- typeof row.__deeplineRowKey === 'string'
2445
- ? row.__deeplineRowKey
2446
- : mapScope.rowIdentity(
2447
- stripCsvProjectedFields(
2448
- Object.fromEntries(
2449
- Object.entries(row).filter(
2450
- ([fieldName]) =>
2451
- !datasetColumnNames.includes(fieldName) &&
2452
- !fieldName.startsWith('__deepline'),
2453
- ),
2454
- ),
2415
+ resolveMapRowOutcomeKey(row) ??
2416
+ mapScope.rowIdentity(
2417
+ stripCsvProjectedFields(
2418
+ Object.fromEntries(
2419
+ Object.entries(row).filter(
2420
+ ([fieldName]) =>
2421
+ !datasetColumnNames.includes(fieldName) &&
2422
+ !fieldName.startsWith('__deepline'),
2455
2423
  ),
2456
- index,
2457
- );
2424
+ ),
2425
+ ),
2426
+ index,
2427
+ );
2458
2428
  const executionRowKey = (row: Record<string, unknown>, index: number) =>
2459
2429
  runtimeOptions?.executionRowKeys?.[index] ?? rowIdentity(row, index);
2460
2430
  const executionRowIndex = (index: number) =>
@@ -2645,14 +2615,16 @@ export class PlayContextImpl {
2645
2615
  `Map completed: ${results.length + completedRows} results (${results.length} executed, ${completedRows} already satisfied)`,
2646
2616
  );
2647
2617
  return {
2648
- completedRows: results.map((row, index) => ({
2649
- key: executionRowKey(
2650
- this.toOutputRow(items[index] as Record<string, unknown>),
2651
- index,
2652
- ),
2653
- inputIndex: executionRowIndex(index),
2654
- data: this.toPersistedOutputRow(row),
2655
- })),
2618
+ completedRows: results.map((row, index) =>
2619
+ completedMapRowOutcome({
2620
+ key: executionRowKey(
2621
+ this.toOutputRow(items[index] as Record<string, unknown>),
2622
+ index,
2623
+ ),
2624
+ inputIndex: executionRowIndex(index),
2625
+ data: this.toPersistedOutputRow(row),
2626
+ }),
2627
+ ),
2656
2628
  failedRows: [],
2657
2629
  };
2658
2630
  }
@@ -2767,6 +2739,9 @@ export class PlayContextImpl {
2767
2739
  ) {
2768
2740
  throw error;
2769
2741
  }
2742
+ if (isRowIsolationExemptError(error)) {
2743
+ throw error;
2744
+ }
2770
2745
 
2771
2746
  value = null;
2772
2747
  computedFields[fieldName] = value;
@@ -2787,7 +2762,7 @@ export class PlayContextImpl {
2787
2762
  : {},
2788
2763
  });
2789
2764
  if (failFastOnRowError) {
2790
- const failedRow: PersistableMapRow = {
2765
+ const failedRow: PersistableMapRow = failedMapRowOutcome({
2791
2766
  key: rowKey,
2792
2767
  inputIndex: rowIndex,
2793
2768
  data: this.toPersistedOutputRow(
@@ -2796,9 +2771,8 @@ export class PlayContextImpl {
2796
2771
  ...(this.activeMapCellMeta?.get(rowKey)
2797
2772
  ? { cellMetaPatch: this.activeMapCellMeta.get(rowKey) }
2798
2773
  : {}),
2799
- status: 'failed',
2800
2774
  error: formattedError,
2801
- };
2775
+ });
2802
2776
  failedRowsToPersist.push(failedRow);
2803
2777
  throw error;
2804
2778
  }
@@ -2806,14 +2780,13 @@ export class PlayContextImpl {
2806
2780
  cloneCsvAliasedRow(baseRow, computedFields),
2807
2781
  );
2808
2782
  const cellMetaPatch = this.activeMapCellMeta?.get(rowKey);
2809
- const failedRow: PersistableMapRow = {
2783
+ const failedRow: PersistableMapRow = failedMapRowOutcome({
2810
2784
  key: rowKey,
2811
2785
  inputIndex: rowIndex,
2812
2786
  data: failedData,
2813
2787
  ...(cellMetaPatch ? { cellMetaPatch } : {}),
2814
- status: 'failed',
2815
2788
  error: formattedError,
2816
- };
2789
+ });
2817
2790
  failedRowsToPersist.push(failedRow);
2818
2791
  updateMapFrameProgress({
2819
2792
  failedRowKey: rowKey,
@@ -2867,14 +2840,14 @@ export class PlayContextImpl {
2867
2840
  dataPatch: {},
2868
2841
  });
2869
2842
  const publicRow = this.toPublicOutputRow(merged);
2870
- const completedRow: PersistableMapRow = {
2843
+ const completedRow: PersistableMapRow = completedMapRowOutcome({
2871
2844
  key: rowKey,
2872
2845
  inputIndex: rowIndex,
2873
2846
  data: this.toPersistedOutputRow(merged),
2874
2847
  ...(this.activeMapCellMeta?.get(rowKey)
2875
2848
  ? { cellMetaPatch: this.activeMapCellMeta.get(rowKey) }
2876
2849
  : {}),
2877
- };
2850
+ });
2878
2851
  completedRowsToPersist.push(completedRow);
2879
2852
  enqueueIncrementalPersist(completedRow);
2880
2853
  return publicRow;
@@ -5538,7 +5511,13 @@ export class PlayContextImpl {
5538
5511
  },
5539
5512
  },
5540
5513
  async (span) => {
5541
- let rateLimitAttempt = 0;
5514
+ const httpFailureAttempts =
5515
+ createToolExecuteHttpFailureAttemptTracker();
5516
+ const retryPolicy = await this.#options
5517
+ .getToolRetryPolicy?.(toolId)
5518
+ .catch(() => null);
5519
+ const retrySafeTransientHttp =
5520
+ retryPolicy?.retrySafeTransientHttp === true;
5542
5521
  let transportAttempt = 0;
5543
5522
 
5544
5523
  while (true) {
@@ -5568,7 +5547,7 @@ export class PlayContextImpl {
5568
5547
  transportAttempt += 1;
5569
5548
  const message =
5570
5549
  error instanceof Error ? error.message : String(error);
5571
- if (transportAttempt < TOOL_EXECUTE_TRANSIENT_HTTP_MAX_ATTEMPTS) {
5550
+ if (transportAttempt < TOOL_EXECUTE_TRANSPORT_MAX_ATTEMPTS) {
5572
5551
  this.governor.chargeBudget('retry');
5573
5552
  const retryAfterMs =
5574
5553
  TOOL_RETRY_AFTER_FALLBACK_MS * transportAttempt;
@@ -5577,105 +5556,65 @@ export class PlayContextImpl {
5577
5556
  transportAttempt,
5578
5557
  );
5579
5558
  this.log(
5580
- `Tool ${toolId} transport failed calling ${url} on attempt ${transportAttempt}/${TOOL_EXECUTE_TRANSIENT_HTTP_MAX_ATTEMPTS}; retrying after ${retryAfterMs}ms: ${message}`,
5559
+ `Tool ${toolId} transport failed calling ${url} on attempt ${transportAttempt}/${TOOL_EXECUTE_TRANSPORT_MAX_ATTEMPTS}; retrying after ${retryAfterMs}ms: ${message}`,
5581
5560
  );
5582
5561
  await this.sleepWithCheckpointHeartbeat(retryAfterMs);
5583
5562
  continue;
5584
5563
  }
5585
5564
  throw new Error(
5586
- `Tool ${toolId} transport failed calling ${url} after ${transportAttempt}/${TOOL_EXECUTE_TRANSIENT_HTTP_MAX_ATTEMPTS} attempts: ${message}`,
5565
+ `Tool ${toolId} transport failed calling ${url} after ${transportAttempt}/${TOOL_EXECUTE_TRANSPORT_MAX_ATTEMPTS} attempts: ${message}`,
5587
5566
  );
5588
5567
  }
5589
5568
 
5590
5569
  span.setAttribute('plays.http_status_code', response.status);
5591
5570
 
5592
- if (response.status === 429) {
5593
- rateLimitAttempt += 1;
5594
- const text = await response.text();
5595
- const initialRetryDecision = decideToolExecuteHttpRetry({
5596
- toolId,
5597
- status: response.status,
5598
- });
5599
- const error = normalizeToolHttpErrorMessage({
5600
- toolId,
5601
- status: response.status,
5602
- attempt: rateLimitAttempt,
5603
- maxAttempts: initialRetryDecision.attemptCap,
5604
- bodyText: text,
5605
- });
5606
- const retryDecision = decideToolExecuteHttpRetry({
5607
- toolId,
5608
- status: response.status,
5609
- hardBillingFailure: isHardBillingToolHttpError(error),
5610
- });
5611
- if (
5612
- !retryDecision.retryable ||
5613
- rateLimitAttempt >= retryDecision.attemptCap
5614
- ) {
5615
- throw error;
5616
- }
5617
- this.governor.chargeBudget('retry');
5618
- const retryAfterMs = parseRetryAfterMs(
5619
- response.headers.get('retry-after'),
5620
- );
5621
- // Feed the server-observed Retry-After back into the shared pacer
5622
- // so subsequent acquires for this provider back off.
5623
- await this.reportToolBackpressure(toolId, retryAfterMs);
5624
- span.setAttribute(
5625
- 'plays.rate_limit_retry_after_ms',
5626
- retryAfterMs,
5627
- );
5628
- span.setAttribute('plays.rate_limit_attempt', rateLimitAttempt);
5629
- this.log(
5630
- `Tool ${toolId} rate limited; retrying after ${retryAfterMs}ms`,
5631
- );
5632
- await this.sleepWithCheckpointHeartbeat(retryAfterMs);
5633
- continue;
5634
- }
5635
-
5636
5571
  if (!response.ok) {
5637
5572
  const text = await response.text();
5638
- const initialRetryDecision = decideToolExecuteHttpRetry({
5573
+ const httpFailureAttempt = httpFailureAttempts.next({
5639
5574
  toolId,
5640
5575
  status: response.status,
5576
+ transientHttpRetrySafe: retrySafeTransientHttp,
5641
5577
  });
5642
- const error = normalizeToolHttpErrorMessage({
5578
+ const failure = classifyToolExecuteHttpFailure({
5643
5579
  toolId,
5644
5580
  status: response.status,
5645
- attempt: rateLimitAttempt + 1,
5646
- maxAttempts: initialRetryDecision.attemptCap,
5581
+ attempt: httpFailureAttempt,
5647
5582
  bodyText: text,
5583
+ retryAfterHeader: response.headers.get('retry-after'),
5584
+ transientHttpRetrySafe: retrySafeTransientHttp,
5648
5585
  });
5649
- const retryDecision = decideToolExecuteHttpRetry({
5650
- toolId,
5651
- status: response.status,
5652
- hardBillingFailure: isHardBillingToolHttpError(error),
5653
- });
5654
- if (
5655
- retryDecision.retryable &&
5656
- rateLimitAttempt + 1 < retryDecision.attemptCap
5657
- ) {
5658
- rateLimitAttempt += 1;
5659
- this.governor.chargeBudget('retry');
5660
- const retryAfterMs = parseRetryAfterMs(
5661
- response.headers.get('retry-after'),
5586
+ if (failure.backpressureDelayMs !== null) {
5587
+ // Feed the server-observed Retry-After back into the shared
5588
+ // pacer even on the final attempt so later provider calls back
5589
+ // off instead of retrying the whole map chunk.
5590
+ await this.reportToolBackpressure(
5591
+ toolId,
5592
+ failure.backpressureDelayMs,
5662
5593
  );
5594
+ }
5595
+ if (failure.shouldRetry) {
5596
+ if (failure.chargeRetryBudget) {
5597
+ this.governor.chargeBudget('retry');
5598
+ }
5599
+ const retryAttributePrefix = failure.isRateLimit
5600
+ ? 'rate_limit'
5601
+ : 'transient_http';
5663
5602
  span.setAttribute(
5664
- 'plays.transient_http_retry_after_ms',
5665
- retryAfterMs,
5603
+ `plays.${retryAttributePrefix}_retry_after_ms`,
5604
+ failure.retryDelayMs,
5666
5605
  );
5667
5606
  span.setAttribute(
5668
- 'plays.transient_http_attempt',
5669
- rateLimitAttempt,
5607
+ `plays.${retryAttributePrefix}_attempt`,
5608
+ httpFailureAttempt,
5670
5609
  );
5671
5610
  this.log(
5672
- `Tool ${toolId} returned ${response.status}; retrying after ${retryAfterMs}ms`,
5611
+ `Tool ${toolId} returned ${response.status}; retrying after ${failure.retryDelayMs}ms`,
5673
5612
  );
5674
- await this.sleepWithCheckpointHeartbeat(retryAfterMs);
5613
+ await this.sleepWithCheckpointHeartbeat(failure.retryDelayMs);
5675
5614
  continue;
5676
5615
  }
5677
- this.log(error.message);
5678
- throw error;
5616
+ this.log(failure.error.message);
5617
+ throw failure.error;
5679
5618
  }
5680
5619
 
5681
5620
  const data = (await response.json()) as Record<string, unknown>;
@@ -487,6 +487,9 @@ export interface ContextOptions {
487
487
  runId?: string;
488
488
  resolvePlay?: (playRef: string) => Promise<ResolvedPlayExecution | null>;
489
489
  getToolQueueHints?: (toolId: string) => Promise<readonly PlayQueueHint[]>;
490
+ getToolRetryPolicy?: (toolId: string) => Promise<{
491
+ retrySafeTransientHttp?: boolean;
492
+ } | null>;
490
493
  getToolTargetGetters?: (
491
494
  toolId: string,
492
495
  output: string,
@@ -1,3 +1,13 @@
1
+ import type {
2
+ ClaimRuntimeStepReceiptInput,
3
+ CompleteRuntimeStepReceiptInput,
4
+ FailRuntimeStepReceiptInput,
5
+ GetRuntimeStepReceiptInput,
6
+ RuntimeStepReceipt,
7
+ SkipRuntimeStepReceiptInput,
8
+ } from './ctx-types';
9
+ import type { PlayRunLedgerEvent } from './run-ledger';
10
+
1
11
  export type MapRowOutcomeStatus = 'completed' | 'failed';
2
12
 
3
13
  export type MapRowOutcome = {
@@ -18,3 +28,47 @@ export type MapRowOutcome = {
18
28
  /** Row-level error message persisted when `status` is failed. */
19
29
  error?: string | null;
20
30
  };
31
+
32
+ /**
33
+ * Runner-facing Interface for hot-path durable writes.
34
+ *
35
+ * Runtime API and Play Harness Worker Adapters can satisfy this seam while
36
+ * keeping their transport/SQL details private to their Implementations.
37
+ */
38
+ export interface PlayDurabilityStore {
39
+ appendRunEvents(input: {
40
+ runId: string;
41
+ events: readonly PlayRunLedgerEvent[];
42
+ }): Promise<void>;
43
+
44
+ getRuntimeStepReceipt(
45
+ input: GetRuntimeStepReceiptInput,
46
+ ): Promise<RuntimeStepReceipt | null>;
47
+ claimRuntimeStepReceipt(
48
+ input: ClaimRuntimeStepReceiptInput,
49
+ ): Promise<RuntimeStepReceipt | null>;
50
+ completeRuntimeStepReceipt(
51
+ input: CompleteRuntimeStepReceiptInput,
52
+ ): Promise<RuntimeStepReceipt | null>;
53
+ failRuntimeStepReceipt(
54
+ input: FailRuntimeStepReceiptInput,
55
+ ): Promise<RuntimeStepReceipt | null>;
56
+ skipRuntimeStepReceipt(
57
+ input: SkipRuntimeStepReceiptInput,
58
+ ): Promise<RuntimeStepReceipt | null>;
59
+
60
+ prepareMapRows(input: {
61
+ tableNamespace: string;
62
+ rows: readonly Record<string, unknown>[];
63
+ outputFields: readonly string[];
64
+ }): Promise<{
65
+ pendingRows: Record<string, unknown>[];
66
+ completedRows: Record<string, unknown>[];
67
+ }>;
68
+
69
+ completeMapRows(input: {
70
+ tableNamespace: string;
71
+ rows: readonly MapRowOutcome[];
72
+ outputFields: readonly string[];
73
+ }): Promise<{ updated: number }>;
74
+ }