deepline 0.3.183 → 0.3.185

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (34) hide show
  1. package/dist/bundling-sources/sdk/src/release.ts +1 -1
  2. package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +53 -0
  3. package/dist/bundling-sources/shared_libs/play-runtime/context.ts +751 -193
  4. package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +4 -0
  5. package/dist/bundling-sources/shared_libs/play-runtime/durable-receipt-execution.ts +3 -1
  6. package/dist/bundling-sources/shared_libs/play-runtime/fixture-behavior.ts +1 -0
  7. package/dist/bundling-sources/shared_libs/play-runtime/governor/app-runtime-rate-state-backend.ts +437 -150
  8. package/dist/bundling-sources/shared_libs/play-runtime/governor/governor.ts +47 -4
  9. package/dist/bundling-sources/shared_libs/play-runtime/governor/policy.ts +3 -1
  10. package/dist/bundling-sources/shared_libs/play-runtime/governor/rate-state-backend.ts +18 -0
  11. package/dist/bundling-sources/shared_libs/play-runtime/map-execution-frame.ts +3 -0
  12. package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts +22 -2
  13. package/dist/bundling-sources/shared_libs/play-runtime/resource-governor.ts +6 -1
  14. package/dist/bundling-sources/shared_libs/play-runtime/run-failure.ts +11 -5
  15. package/dist/bundling-sources/shared_libs/play-runtime/runtime-flow.ts +102 -12
  16. package/dist/bundling-sources/shared_libs/play-runtime/runtime-operation-contract.ts +8 -0
  17. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-runtime-worker.ts +136 -28
  18. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-rate-scope.ts +80 -0
  19. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-rate-state.ts +897 -207
  20. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts +33 -1
  21. package/dist/bundling-sources/shared_libs/play-runtime/suspension.ts +21 -0
  22. package/dist/bundling-sources/shared_libs/play-runtime/tool-execute-retry-policy.ts +120 -34
  23. package/dist/bundling-sources/shared_libs/play-runtime/tool-result.ts +82 -2
  24. package/dist/bundling-sources/shared_libs/plays/tool-result-types.ts +7 -2
  25. package/dist/cli/index.js +28 -3
  26. package/dist/cli/index.mjs +28 -3
  27. package/dist/index.js +1 -1
  28. package/dist/index.mjs +1 -1
  29. package/dist/install-integrity.json +1 -0
  30. package/dist/release.d.mts +1 -1
  31. package/dist/release.d.ts +1 -1
  32. package/dist/release.js +1 -1
  33. package/dist/release.mjs +1 -1
  34. package/package.json +1 -1
@@ -269,7 +269,10 @@ const POSTGRES_SCHEDULER_SCHEMA_INIT_STATEMENT_TIMEOUT_MS = 60_000;
269
269
  // deployment retries do not re-enter the old cross-table DDL transaction.
270
270
  // v45 adds the bucket-leading expiration index used by rate-reservation
271
271
  // cleanup. Shared schemas build it concurrently through the deploy migrator.
272
- export const POSTGRES_SCHEDULER_SCHEMA_VERSION = 45;
272
+ // v46 adds a bounded provider-scope Retry-After hold with shared adaptive-rate
273
+ // state. The scope owns learning and dispatch across runs and tool rules;
274
+ // existing deadline rows expand additively and initialize on their first seed.
275
+ export const POSTGRES_SCHEDULER_SCHEMA_VERSION = 46;
273
276
  const POSTGRES_SCHEDULER_SCHEMA_MIGRATIONS_TABLE =
274
277
  'scheduler_schema_migrations';
275
278
  export const POSTGRES_SCHEDULER_OPEN_RECOVERY_INDEX =
@@ -1065,6 +1068,8 @@ export type PostgresSchedulerSchemaShapeRequirements = Readonly<{
1065
1068
  cronScheduleSchema?: boolean;
1066
1069
  /** v43 added per-run fair-share tracking for rate buckets; v42 ignores it. */
1067
1070
  rateRequesterFairShare?: boolean;
1071
+ /** v45 added durable Retry-After holds shared across rules and runs. */
1072
+ rateScopeRetryAfter?: boolean;
1068
1073
  }>;
1069
1074
 
1070
1075
  export async function hasCurrentPostgresSchedulerSchemaShape(
@@ -1371,6 +1376,31 @@ export async function hasCurrentPostgresSchedulerSchemaShape(
1371
1376
  )
1372
1377
  )
1373
1378
  )
1379
+ AND (
1380
+ $36::boolean = false
1381
+ OR (
1382
+ to_regclass($37) IS NOT NULL
1383
+ AND (
1384
+ SELECT count(DISTINCT column_name)
1385
+ FROM information_schema.columns
1386
+ WHERE table_schema = $1
1387
+ AND table_name = 'rate_scope_holds'
1388
+ AND column_name IN (
1389
+ 'bucket_id', 'retry_at_ms', 'refill_per_sec', 'capacity', 'max_rps',
1390
+ 'tokens', 'refilled_at_ms', 'next_dispatch_at_ms', 'seen_backpressure',
1391
+ 'cool_until_ms', 'last_ramp_at_ms', 'rate_observation_started_at_ms',
1392
+ 'rate_successes', 'rate_recovery_interval_ms',
1393
+ 'rate_throttle_ceiling_rps'
1394
+ )
1395
+ ) = 15
1396
+ AND (
1397
+ SELECT count(DISTINCT column_name)
1398
+ FROM information_schema.columns
1399
+ WHERE table_schema = $1 AND table_name = 'rate_bucket'
1400
+ AND column_name IN ('rate_observation_started_at_ms', 'rate_successes', 'rate_recovery_interval_ms')
1401
+ ) = 3
1402
+ )
1403
+ )
1374
1404
  AND (
1375
1405
  $32::boolean = false
1376
1406
  OR (
@@ -1695,6 +1725,8 @@ export async function hasCurrentPostgresSchedulerSchemaShape(
1695
1725
  `${quotedSchema}.${quoteIdent(POSTGRES_SCHEDULER_CRON_SCHEDULES_TABLE)}`,
1696
1726
  requirements.rateRequesterFairShare !== false,
1697
1727
  `${quotedSchema}.${quoteIdent('rate_requesters')}`,
1728
+ requirements.rateScopeRetryAfter !== false,
1729
+ `${quotedSchema}.${quoteIdent('rate_scope_holds')}`,
1698
1730
  ],
1699
1731
  );
1700
1732
  if (currentShape.rows[0]?.ready !== true) return false;
@@ -14,6 +14,9 @@ export type PlayExecutionSuspension =
14
14
  boundaryId: string;
15
15
  eventKey: string;
16
16
  timeoutMs: number;
17
+ eventExpiresAtMs?: number;
18
+ /** A terminal timeout's deliberate long park, not its original event wait. */
19
+ postTimeoutPark?: boolean;
17
20
  messageRef?: {
18
21
  channel: string;
19
22
  ts: string;
@@ -21,10 +24,13 @@ export type PlayExecutionSuspension =
21
24
  }
22
25
  | {
23
26
  kind: 'integration_event_batch';
27
+ /** Earliest pending provider row; absence preserves older event-only waits. */
28
+ providerWakeAtMs?: number;
24
29
  boundaries: Array<{
25
30
  boundaryId: string;
26
31
  eventKey: string;
27
32
  timeoutMs: number;
33
+ eventExpiresAtMs?: number;
28
34
  messageRef?: {
29
35
  channel: string;
30
36
  ts: string;
@@ -123,6 +129,7 @@ export class PlayRowExecutionSuspendedError extends Error {
123
129
  boundaryId: string;
124
130
  eventKey: string;
125
131
  timeoutMs: number;
132
+ eventExpiresAtMs?: number;
126
133
  messageRef?: {
127
134
  channel: string;
128
135
  ts: string;
@@ -133,6 +140,7 @@ export class PlayRowExecutionSuspendedError extends Error {
133
140
  boundaryId: string;
134
141
  eventKey: string;
135
142
  timeoutMs: number;
143
+ eventExpiresAtMs?: number;
136
144
  messageRef?: {
137
145
  channel: string;
138
146
  ts: string;
@@ -146,6 +154,19 @@ export class PlayRowExecutionSuspendedError extends Error {
146
154
  }
147
155
  }
148
156
 
157
+ /** A provider wait is a control handoff, never a failed cell or receipt. */
158
+ export class PlayRowProviderBackpressureError extends Error {
159
+ constructor(
160
+ readonly provider: string,
161
+ readonly retryAtMs: number,
162
+ ) {
163
+ super(
164
+ `Play row is waiting for ${provider} provider backpressure to clear.`,
165
+ );
166
+ this.name = 'PlayRowProviderBackpressureError';
167
+ }
168
+ }
169
+
149
170
  export function isPlayRowExecutionSuspendedError(
150
171
  value: unknown,
151
172
  ): value is PlayRowExecutionSuspendedError {
@@ -29,6 +29,8 @@ export const TOOL_EXECUTE_CONCURRENCY_BACKPRESSURE_MAX_ATTEMPTS = 65;
29
29
  // for an operation that has not opted into ordinary transient-5xx retries.
30
30
  // Keep its window aligned with other local control-plane queue signals.
31
31
  export const TOOL_EXECUTE_RUNTIME_SCHEDULER_CAPACITY_MAX_ATTEMPTS = 65;
32
+ /** Re-admission after a stale shared provider permit is safely replayable. */
33
+ export const TOOL_EXECUTE_PROVIDER_PACING_WAIT_MAX_ATTEMPTS = 65;
32
34
  export const TOOL_EXECUTE_TRANSPORT_MAX_ATTEMPTS = 3;
33
35
  export const TOOL_EXECUTE_TRANSPORT_RETRY_DELAY_MS = 1_000;
34
36
  export const TOOL_EXECUTE_RETRY_DELAY_FALLBACK_MS = 1_000;
@@ -69,6 +71,15 @@ export const TOOL_EXECUTE_RUNNER_ACCEPTS_CONCURRENCY_BACKPRESSURE =
69
71
  /** The capability tokens this runner build advertises on provider execute. */
70
72
  export const TOOL_EXECUTE_RUNNER_ACCEPTS_VALUE =
71
73
  TOOL_EXECUTE_RUNNER_ACCEPTS_CONCURRENCY_BACKPRESSURE;
74
+ /** Runner proof headers consumed only by the execution gateway. */
75
+ export const TOOL_EXECUTE_PROVIDER_RATE_SCOPE_HEADER =
76
+ 'x-deepline-provider-rate-scope';
77
+ export const TOOL_EXECUTE_PROVIDER_RATE_RETRY_SNAPSHOT_HEADER =
78
+ 'x-deepline-provider-rate-retry-snapshot';
79
+ /** Server-minted only for an internal executor's provider-owned 429. */
80
+ export const TOOL_EXECUTE_PROVIDER_THROTTLE_HEADER =
81
+ 'x-deepline-provider-throttle';
82
+ export const TOOL_EXECUTE_PROVIDER_PACING_WAIT_CODE = 'PROVIDER_PACING_WAIT';
72
83
 
73
84
  /**
74
85
  * Parse the advertisement as a comma-separated token list so a later runner can
@@ -106,6 +117,7 @@ export type ToolExecuteHttpRetryDecision = {
106
117
  | 'rate_limit'
107
118
  | 'concurrency_backpressure'
108
119
  | 'runtime_scheduler_capacity'
120
+ | 'provider_pacing_wait'
109
121
  | 'idempotency_in_progress'
110
122
  | 'gateway_invocation_in_progress'
111
123
  | 'customer_db_storage_unavailable'
@@ -122,6 +134,8 @@ export type ToolExecuteHttpFailureOutcome = ToolExecuteHttpRetryDecision & {
122
134
  fate: 'retry' | 'settle_row_failure' | 'fail_run';
123
135
  retryDelayMs: number;
124
136
  backpressureDelayMs: number | null;
137
+ /** Retry-After for known-not-dispatched gateway pacing; not provider feedback. */
138
+ providerPacingWaitMs: number | null;
125
139
  chargeRetryBudget: boolean;
126
140
  /** The response proves the provider request was never dispatched. */
127
141
  knownNotDispatched: boolean;
@@ -214,6 +228,19 @@ function isRuntimeSchedulerCapacityResponse(input: {
214
228
  );
215
229
  }
216
230
 
231
+ function isProviderPacingWaitResponse(input: {
232
+ status: number;
233
+ bodyText: string;
234
+ }): boolean {
235
+ const body = parseJsonObject(input.bodyText);
236
+ return (
237
+ input.status === 503 &&
238
+ body?.code === TOOL_EXECUTE_PROVIDER_PACING_WAIT_CODE &&
239
+ body.failure_origin === 'deepline_rate_limit' &&
240
+ body.provider_request_dispatched === false
241
+ );
242
+ }
243
+
217
244
  function idempotencyInProgressRetryDelayMs(attempt: number): number {
218
245
  return (
219
246
  TOOL_EXECUTE_IDEMPOTENCY_IN_PROGRESS_RETRY_DELAYS_MS[
@@ -249,6 +276,7 @@ function decideToolExecuteHttpRetry(input: {
249
276
  customerDbStorageUnavailable?: boolean;
250
277
  concurrencyBackpressure?: boolean;
251
278
  runtimeSchedulerCapacity?: boolean;
279
+ providerPacingWait?: boolean;
252
280
  hardBillingFailure?: boolean;
253
281
  hasRetryAfterHeader?: boolean;
254
282
  transientHttpRetrySafe?: boolean;
@@ -289,6 +317,13 @@ function decideToolExecuteHttpRetry(input: {
289
317
  reason: 'runtime_scheduler_capacity',
290
318
  };
291
319
  }
320
+ if (input.providerPacingWait) {
321
+ return {
322
+ retryable: true,
323
+ attemptCap: TOOL_EXECUTE_PROVIDER_PACING_WAIT_MAX_ATTEMPTS,
324
+ reason: 'provider_pacing_wait',
325
+ };
326
+ }
292
327
  if (input.gatewayInvocationInProgress) {
293
328
  return {
294
329
  retryable: true,
@@ -342,6 +377,7 @@ export function createToolExecuteHttpFailureAttemptTracker(): ToolExecuteHttpFai
342
377
  rate_limit: 0,
343
378
  concurrency_backpressure: 0,
344
379
  runtime_scheduler_capacity: 0,
380
+ provider_pacing_wait: 0,
345
381
  idempotency_in_progress: 0,
346
382
  gateway_invocation_in_progress: 0,
347
383
  customer_db_storage_unavailable: 0,
@@ -376,6 +412,10 @@ export function createToolExecuteHttpFailureAttemptTracker(): ToolExecuteHttpFai
376
412
  status: input.status,
377
413
  bodyText: input.bodyText ?? '',
378
414
  }),
415
+ providerPacingWait: isProviderPacingWaitResponse({
416
+ status: input.status,
417
+ bodyText: input.bodyText ?? '',
418
+ }),
379
419
  hasRetryAfterHeader: true,
380
420
  transientHttpRetrySafe: input.transientHttpRetrySafe === true,
381
421
  });
@@ -391,12 +431,12 @@ export function parseToolExecuteRetryAfterMs(
391
431
  ): number {
392
432
  if (!header) return TOOL_EXECUTE_RETRY_DELAY_FALLBACK_MS;
393
433
  const seconds = Number(header);
394
- if (Number.isFinite(seconds) && seconds > 0) {
434
+ if (Number.isFinite(seconds) && seconds >= 0) {
395
435
  return Math.ceil(seconds * 1000);
396
436
  }
397
437
  const retryAt = Date.parse(header);
398
438
  if (Number.isFinite(retryAt)) {
399
- return Math.max(1, retryAt - nowMs);
439
+ return Math.max(0, retryAt - nowMs);
400
440
  }
401
441
  return TOOL_EXECUTE_RETRY_DELAY_FALLBACK_MS;
402
442
  }
@@ -423,6 +463,7 @@ export function classifyToolExecuteHttpFailure(input: {
423
463
  isCustomerDbStorageUnavailableResponse(input);
424
464
  const concurrencyBackpressure = isConcurrencyBackpressureResponse(input);
425
465
  const runtimeSchedulerCapacity = isRuntimeSchedulerCapacityResponse(input);
466
+ const providerPacingWait = isProviderPacingWaitResponse(input);
426
467
  const initialRetryDecision = decideToolExecuteHttpRetry({
427
468
  status: input.status,
428
469
  idempotencyInProgress,
@@ -430,13 +471,33 @@ export function classifyToolExecuteHttpFailure(input: {
430
471
  customerDbStorageUnavailable,
431
472
  concurrencyBackpressure,
432
473
  runtimeSchedulerCapacity,
474
+ providerPacingWait,
433
475
  hasRetryAfterHeader,
434
476
  transientHttpRetrySafe,
435
477
  });
436
- const retryAfterMs = parseToolExecuteRetryAfterMs(
437
- input.retryAfterHeader,
438
- input.nowMs,
439
- );
478
+ const providerPacingWaitBody = providerPacingWait
479
+ ? parseJsonObject(input.bodyText)
480
+ : null;
481
+ const rawProviderPacingWaitMs = providerPacingWaitBody?.retry_after_ms;
482
+ const exactProviderPacingWaitMs =
483
+ typeof rawProviderPacingWaitMs === 'number' &&
484
+ Number.isFinite(rawProviderPacingWaitMs) &&
485
+ rawProviderPacingWaitMs >= 0
486
+ ? rawProviderPacingWaitMs
487
+ : null;
488
+ const attestedProviderDeadline =
489
+ providerPacingWait &&
490
+ (providerPacingWaitBody?.reason === 'retry_after' ||
491
+ providerPacingWaitBody?.reason === 'stale_permit') &&
492
+ typeof providerPacingWaitBody?.retry_at_ms === 'number' &&
493
+ Number.isFinite(providerPacingWaitBody.retry_at_ms) &&
494
+ providerPacingWaitBody.retry_at_ms > (input.nowMs ?? Date.now()) &&
495
+ exactProviderPacingWaitMs !== null &&
496
+ exactProviderPacingWaitMs > 0;
497
+ const retryAfterMs =
498
+ providerPacingWait && exactProviderPacingWaitMs !== null
499
+ ? exactProviderPacingWaitMs
500
+ : parseToolExecuteRetryAfterMs(input.retryAfterHeader, input.nowMs);
440
501
  const error = normalizeToolHttpErrorMessage({
441
502
  toolId: input.toolId,
442
503
  status: input.status,
@@ -446,7 +507,9 @@ export function classifyToolExecuteHttpFailure(input: {
446
507
  schemaVersion: input.schemaVersion,
447
508
  retryable: initialRetryDecision.retryable,
448
509
  retryAfterMs:
449
- (input.status === 429 && hasRetryAfterHeader) || runtimeSchedulerCapacity
510
+ (input.status === 429 && hasRetryAfterHeader) ||
511
+ runtimeSchedulerCapacity ||
512
+ providerPacingWait
450
513
  ? retryAfterMs
451
514
  : null,
452
515
  });
@@ -458,6 +521,11 @@ export function classifyToolExecuteHttpFailure(input: {
458
521
  error.message = `${error.message} (provider call ${Math.round(input.providerLatencyMs)}ms)`;
459
522
  }
460
523
  const hardBillingFailure = isHardBillingToolHttpError(error);
524
+ const providerOwnedRateLimit =
525
+ input.status === 429 &&
526
+ !hardBillingFailure &&
527
+ error instanceof ToolExecutionError &&
528
+ error.origin === 'provider';
461
529
  const retryDecision = decideToolExecuteHttpRetry({
462
530
  status: input.status,
463
531
  idempotencyInProgress,
@@ -465,12 +533,15 @@ export function classifyToolExecuteHttpFailure(input: {
465
533
  customerDbStorageUnavailable,
466
534
  concurrencyBackpressure,
467
535
  runtimeSchedulerCapacity,
536
+ providerPacingWait,
468
537
  hardBillingFailure,
469
538
  hasRetryAfterHeader,
470
539
  transientHttpRetrySafe,
471
540
  });
472
541
  const shouldRetry =
473
- retryDecision.retryable && input.attempt < retryDecision.attemptCap;
542
+ providerOwnedRateLimit ||
543
+ attestedProviderDeadline ||
544
+ (retryDecision.retryable && input.attempt < retryDecision.attemptCap);
474
545
  // In-attempt retry safety and later-run repairability are different
475
546
  // decisions. A non-idempotent 5xx must not be repeated automatically inside
476
547
  // this execution attempt, but an explicit later `plays run` must be able to
@@ -496,23 +567,37 @@ export function classifyToolExecuteHttpFailure(input: {
496
567
  TOOL_EXECUTE_RETRY_DELAY_MAX_MS,
497
568
  Math.max(retryAfterMs, TOOL_EXECUTE_RETRY_DELAY_FALLBACK_MS),
498
569
  )
499
- : input.status === 429
500
- ? Math.min(
501
- TOOL_EXECUTE_RATE_LIMIT_RETRY_DELAY_MAX_MS,
502
- Math.max(
503
- retryAfterMs,
504
- TOOL_EXECUTE_RETRY_DELAY_FALLBACK_MS * input.attempt,
505
- ),
506
- )
507
- : retryDecision.reason === 'idempotency_in_progress' &&
508
- !hasRetryAfterHeader
509
- ? idempotencyInProgressRetryDelayMs(input.attempt)
510
- : retryDecision.reason === 'gateway_invocation_in_progress' &&
570
+ : retryDecision.reason === 'provider_pacing_wait'
571
+ ? retryAfterMs
572
+ : input.status === 429
573
+ ? hasRetryAfterHeader
574
+ ? providerOwnedRateLimit
575
+ ? // The provider's explicit deadline is authoritative. Attempt-
576
+ // scaled fallback backoff is only for a bare 429; extending a
577
+ // valid Retry-After by the attempt number needlessly idles this
578
+ // row after the shared provider pacer has already enforced the
579
+ // deadline for every caller using the credential.
580
+ retryAfterMs
581
+ : Math.max(
582
+ retryAfterMs,
583
+ TOOL_EXECUTE_RETRY_DELAY_FALLBACK_MS * input.attempt,
584
+ )
585
+ : Math.min(
586
+ TOOL_EXECUTE_RATE_LIMIT_RETRY_DELAY_MAX_MS,
587
+ Math.max(
588
+ retryAfterMs,
589
+ TOOL_EXECUTE_RETRY_DELAY_FALLBACK_MS * input.attempt,
590
+ ),
591
+ )
592
+ : retryDecision.reason === 'idempotency_in_progress' &&
511
593
  !hasRetryAfterHeader
512
594
  ? idempotencyInProgressRetryDelayMs(input.attempt)
513
- : retryAfterMs > 0
514
- ? Math.min(TOOL_EXECUTE_RETRY_DELAY_MAX_MS, retryAfterMs)
515
- : TOOL_EXECUTE_RETRY_DELAY_FALLBACK_MS;
595
+ : retryDecision.reason === 'gateway_invocation_in_progress' &&
596
+ !hasRetryAfterHeader
597
+ ? idempotencyInProgressRetryDelayMs(input.attempt)
598
+ : retryAfterMs > 0
599
+ ? Math.min(TOOL_EXECUTE_RETRY_DELAY_MAX_MS, retryAfterMs)
600
+ : TOOL_EXECUTE_RETRY_DELAY_FALLBACK_MS;
516
601
  return {
517
602
  ...retryDecision,
518
603
  error,
@@ -528,24 +613,25 @@ export function classifyToolExecuteHttpFailure(input: {
528
613
  // caller, and malformed responses are never evidence that a provider is
529
614
  // throttling; feeding any of them into the pacer can incorrectly exhaust
530
615
  // every customer sharing that provider credential.
531
- backpressureDelayMs:
532
- input.status === 429 &&
533
- retryDecision.reason !== 'hard_billing_error' &&
534
- error instanceof ToolExecutionError &&
535
- error.origin === 'provider'
536
- ? hasRetryAfterHeader && retryAfterMs > 0
537
- ? retryAfterMs
538
- : TOOL_EXECUTE_BARE_RATE_LIMIT_BACKPRESSURE_MS
539
- : null,
616
+ backpressureDelayMs: providerOwnedRateLimit
617
+ ? hasRetryAfterHeader
618
+ ? retryAfterMs
619
+ : TOOL_EXECUTE_BARE_RATE_LIMIT_BACKPRESSURE_MS
620
+ : null,
621
+ providerPacingWaitMs:
622
+ attestedProviderDeadline && shouldRetry ? retryAfterMs : null,
540
623
  chargeRetryBudget:
541
624
  shouldRetry &&
625
+ !providerOwnedRateLimit &&
542
626
  retryDecision.reason !== 'gateway_invocation_in_progress' &&
543
627
  retryDecision.reason !== 'concurrency_backpressure' &&
544
- retryDecision.reason !== 'runtime_scheduler_capacity',
628
+ retryDecision.reason !== 'runtime_scheduler_capacity' &&
629
+ retryDecision.reason !== 'provider_pacing_wait',
545
630
  // Scheduler capacity and the relay's explicit concurrency shed both occur
546
631
  // before a provider request is dispatched. Carry that proof in the shared
547
632
  // classification so the retry boundary can safely pace either response
548
633
  // without asking a non-idempotent provider call to be replayed.
549
- knownNotDispatched: runtimeSchedulerCapacity || concurrencyBackpressure,
634
+ knownNotDispatched:
635
+ runtimeSchedulerCapacity || concurrencyBackpressure || providerPacingWait,
550
636
  };
551
637
  }
@@ -1593,6 +1593,22 @@ function serializedListRowsFromRaw(input: {
1593
1593
  return entries.length > 0 ? Object.fromEntries(entries) : undefined;
1594
1594
  }
1595
1595
 
1596
+ /**
1597
+ * V2 receipts keep the complete provider body in rawV2. Use that body as the
1598
+ * durable source for list handles instead of writing the same rows to the
1599
+ * compatibility-only listRows field.
1600
+ */
1601
+ function serializedListRowsFromRawV2(input: {
1602
+ rawV2: unknown;
1603
+ view: 'data' | 'rawV2';
1604
+ metadata: ToolResultMetadataInput;
1605
+ }): Record<string, Array<Record<string, unknown>>> | undefined {
1606
+ return serializedListRowsFromRaw({
1607
+ raw: legacyRawFromToolResponseRawV2(input.rawV2, input.view),
1608
+ metadata: input.metadata,
1609
+ });
1610
+ }
1611
+
1596
1612
  function preservedSerializedListRows(
1597
1613
  value: ToolExecuteResult,
1598
1614
  ): Record<string, Array<Record<string, unknown>>> | undefined {
@@ -1864,6 +1880,52 @@ function sameJsonValue(left: unknown, right: unknown): boolean {
1864
1880
  }
1865
1881
  }
1866
1882
 
1883
+ /**
1884
+ * New V2 receipts must not carry a second full copy of list rows. Keep the
1885
+ * optional field when the rows cannot be proven identical to the canonical
1886
+ * rawV2 body; this makes the compaction conservative for unusual or historical
1887
+ * envelopes while allowing ordinary V2 receipts to store rows exactly once.
1888
+ */
1889
+ function listRowsAreInCanonicalRawV2(input: {
1890
+ rawV2: unknown;
1891
+ view: 'data' | 'rawV2';
1892
+ metadata: ToolResultMetadataInput;
1893
+ listRows: Record<string, Array<Record<string, unknown>>> | undefined;
1894
+ }): boolean {
1895
+ if (!input.listRows || Object.keys(input.listRows).length === 0) {
1896
+ return false;
1897
+ }
1898
+ const canonicalRows = serializedListRowsFromRawV2({
1899
+ rawV2: input.rawV2,
1900
+ view: input.view,
1901
+ metadata: input.metadata,
1902
+ });
1903
+ if (!canonicalRows) return false;
1904
+ return Object.entries(input.listRows).every(([name, rows]) =>
1905
+ sameJsonValue(canonicalRows[name], rows),
1906
+ );
1907
+ }
1908
+
1909
+ function completeListRowsForDatasets(input: {
1910
+ listRows: Record<string, Array<Record<string, unknown>>> | undefined;
1911
+ listDatasets:
1912
+ | Record<string, SerializedPlayDataset<Record<string, unknown>>>
1913
+ | undefined;
1914
+ }): Record<string, Array<Record<string, unknown>>> | undefined {
1915
+ if (!input.listRows || !input.listDatasets) return undefined;
1916
+ const datasets = Object.entries(input.listDatasets);
1917
+ if (datasets.length === 0) return undefined;
1918
+ if (
1919
+ !datasets.every(([name, dataset]) => {
1920
+ const rows = input.listRows?.[name];
1921
+ return rows !== undefined && rows.length >= dataset.count;
1922
+ })
1923
+ ) {
1924
+ return undefined;
1925
+ }
1926
+ return input.listRows;
1927
+ }
1928
+
1867
1929
  function responseMetaOutsideRawV2(input: {
1868
1930
  rawV2: unknown;
1869
1931
  view: 'data' | 'rawV2';
@@ -1931,6 +1993,14 @@ export function serializeToolExecuteResult(
1931
1993
  });
1932
1994
  if (Object.prototype.hasOwnProperty.call(value.toolResponse, 'rawV2')) {
1933
1995
  const view = value.toolResponse.view ?? 'rawV2';
1996
+ const persistedListRows = listRowsAreInCanonicalRawV2({
1997
+ rawV2: value.toolResponse.rawV2,
1998
+ view,
1999
+ metadata,
2000
+ listRows,
2001
+ })
2002
+ ? undefined
2003
+ : listRows;
1934
2004
  const responseMeta = responseMetaOutsideRawV2({
1935
2005
  rawV2: value.toolResponse.rawV2,
1936
2006
  view,
@@ -1960,7 +2030,7 @@ export function serializeToolExecuteResult(
1960
2030
  ...(responseMeta ? { responseMeta } : {}),
1961
2031
  },
1962
2032
  ...(listDatasets ? { listDatasets } : {}),
1963
- ...(listRows ? { listRows } : {}),
2033
+ ...(persistedListRows ? { listRows: persistedListRows } : {}),
1964
2034
  ...(targetValues ? { targetValues } : {}),
1965
2035
  metadata,
1966
2036
  execution: value._metadata.execution,
@@ -2023,6 +2093,16 @@ export function deserializeToolExecuteResult(
2023
2093
  )
2024
2094
  ? value.toolResponse.legacyRaw
2025
2095
  : raw;
2096
+ const listRows =
2097
+ value.listRows ??
2098
+ completeListRowsForDatasets({
2099
+ listRows: serializedListRowsFromRawV2({
2100
+ rawV2,
2101
+ view,
2102
+ metadata: value.metadata,
2103
+ }),
2104
+ listDatasets: value.listDatasets,
2105
+ });
2026
2106
  return applySerializedListDatasets(
2027
2107
  applySerializedTargetValues(
2028
2108
  createToolExecuteResult({
@@ -2045,7 +2125,7 @@ export function deserializeToolExecuteResult(
2045
2125
  value.targetValues,
2046
2126
  ),
2047
2127
  value.listDatasets,
2048
- value.listRows,
2128
+ listRows,
2049
2129
  );
2050
2130
  }
2051
2131
  return applySerializedListDatasets(
@@ -119,7 +119,7 @@ export type SerializedToolExecuteResultV1 = {
119
119
  meta?: Record<string, unknown>;
120
120
  };
121
121
  listDatasets?: Record<string, SerializedPlayDataset<Record<string, unknown>>>;
122
- /** Full declared list rows used to rehydrate list Dataset Handles. */
122
+ /** Full declared list rows retained by historical V1 receipts. */
123
123
  listRows?: Record<string, Array<Record<string, unknown>>>;
124
124
  /** Computed scalar/object getters preserved before raw list previews are clipped. */
125
125
  targetValues?: Record<string, ToolResultTargetMetadata>;
@@ -133,7 +133,7 @@ export type SerializedToolExecuteResultV1 = {
133
133
  */
134
134
  export type SerializedToolExecuteResultV2 = Omit<
135
135
  SerializedToolExecuteResultV1,
136
- '__kind' | 'toolResponse'
136
+ '__kind' | 'toolResponse' | 'listRows'
137
137
  > & {
138
138
  __kind: 'deepline.tool_execute_result.v2';
139
139
  toolResponse: {
@@ -144,6 +144,11 @@ export type SerializedToolExecuteResultV2 = Omit<
144
144
  /** Deepline-owned additions not already represented in `rawV2.meta`. */
145
145
  responseMeta?: Record<string, unknown>;
146
146
  };
147
+ /**
148
+ * Compatibility fallback for older V2 receipts whose rawV2 body was not
149
+ * complete. New V2 receipts derive list rows from rawV2 and omit this copy.
150
+ */
151
+ listRows?: Record<string, Array<Record<string, unknown>>>;
147
152
  };
148
153
 
149
154
  export type SerializedToolExecuteResult =
package/dist/cli/index.js CHANGED
@@ -3051,7 +3051,7 @@ var SDK_RELEASE = {
3051
3051
  // getters keep their established compatibility behavior.
3052
3052
  // 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
3053
3053
  // 0.3.90 is the first deliberately versioned SDK release for API v3.
3054
- version: "0.3.183",
3054
+ version: "0.3.185",
3055
3055
  updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.",
3056
3056
  packageCapabilities: {
3057
3057
  updatePreferences: 1
@@ -14141,6 +14141,7 @@ function mergeCanonicalRowsInfo(left, right) {
14141
14141
  source: preferred.source ?? fallback.source,
14142
14142
  datasetId: preferred.datasetId ?? fallback.datasetId,
14143
14143
  tableNamespace: preferred.tableNamespace ?? fallback.tableNamespace,
14144
+ currentTableQueryCommand: preferred.currentTableQueryCommand ?? fallback.currentTableQueryCommand,
14144
14145
  ...left.recovered || right.recovered ? { recovered: true } : {},
14145
14146
  ...recoveredReturnPaths2.length ? { recoveredReturnPaths: recoveredReturnPaths2 } : {},
14146
14147
  ...currentTable ? { currentTable: true } : {},
@@ -14209,6 +14210,12 @@ function hasCurrentTableExport(value) {
14209
14210
  const exportCsv = value.actions.exportCsv;
14210
14211
  return isRecord8(exportCsv) && isRecord8(exportCsv.scope) && exportCsv.scope.kind === "database" && exportCsv.scope.mutability === "current";
14211
14212
  }
14213
+ function currentTableQueryCommand(value) {
14214
+ if (!isRecord8(value) || !isRecord8(value.actions)) return void 0;
14215
+ const query = value.actions.queryCurrentTable;
14216
+ const command = isRecord8(query) ? query.command : void 0;
14217
+ return typeof command === "string" && command.trim() ? command.trim() : void 0;
14218
+ }
14212
14219
  function recoveredReturnPaths(value) {
14213
14220
  if (!Array.isArray(value)) return [];
14214
14221
  return [
@@ -14316,6 +14323,9 @@ function canonicalRowsInfoFromCandidate(input2) {
14316
14323
  tableNamespace: typeof candidate.value.tableNamespace === "string" ? candidate.value.tableNamespace : null,
14317
14324
  ...candidate.value.recovered === true ? { recovered: true } : {},
14318
14325
  ...hasCurrentTableExport(candidate.value) ? { currentTable: true } : {},
14326
+ ...currentTableQueryCommand(candidate.value) ? {
14327
+ currentTableQueryCommand: currentTableQueryCommand(candidate.value)
14328
+ } : {},
14319
14329
  ...recoveredReturnPaths(candidate.value.recoveredReturnPaths).length ? {
14320
14330
  recoveredReturnPaths: recoveredReturnPaths(
14321
14331
  candidate.value.recoveredReturnPaths
@@ -26317,7 +26327,20 @@ async function fetchBackingDatasetRows(input2) {
26317
26327
  offset += page.rows.length;
26318
26328
  }
26319
26329
  const rows = sheetRows.map(exportableSheetRow).filter((row) => Boolean(row));
26320
- const hasCompleteSettledPrefix = !currentTable && acceptsSettledRowsAsCompleteExport(input2.status) && settledTotal !== null && rows.length >= settledTotal;
26330
+ const hasCompleteSettledPrefix = !currentTable && acceptsSettledRowsAsCompleteExport(input2.status) && settledTotal !== null && // A zero-row run-scoped read is not a settled export when the returned
26331
+ // Dataset Handle advertised rows. That shape is the replay/reuse case:
26332
+ // the rows belong to another writer (or the projection has not caught up)
26333
+ // and must not erase the run package's advertised count.
26334
+ (settledTotal > 0 || input2.rowsInfo.totalRows === 0) && rows.length >= settledTotal;
26335
+ if (!currentTable && acceptsSettledRowsAsCompleteExport(input2.status) && settledTotal === 0 && rows.length === 0 && input2.rowsInfo.totalRows > 0) {
26336
+ return {
26337
+ ...input2.rowsInfo,
26338
+ rows: [],
26339
+ totalRows: input2.rowsInfo.totalRows,
26340
+ complete: true,
26341
+ source: `${input2.rowsInfo.source ?? "result.rows"} -> /api/v2/plays/${playName}/sheet?tableNamespace=${tableNamespace}`
26342
+ };
26343
+ }
26321
26344
  if (!currentTable && rows.length < input2.rowsInfo.totalRows && !hasCompleteSettledPrefix || currentTable && rows.length === 0 && input2.rowsInfo.totalRows > 0) {
26322
26345
  return null;
26323
26346
  }
@@ -26373,7 +26396,8 @@ function assertDatasetHasExportableRows(input2) {
26373
26396
  "output.csv",
26374
26397
  datasetLabel
26375
26398
  );
26376
- const explanation = terminalRunWithMissingAdvertisedRows ? ` The terminal result advertised ${advertisedRows} row(s), but none are currently attributed to this run's durable sheet. Those rows may have been reused from an earlier run or the projection may still be lagging; export refuses to guess another run's rows. Retry the same export after projection catches up (do not rerun the Play): ${retryCommand}. If it remains empty, inspect the full package with ${inspectCommand} and export the run that owns the durable dataset; this command will not use a database-wide table query that could mix runs.` : ` The run status is '${runStatus || "unknown"}', so it likely failed before persisting rows to this dataset. That means provider calls may have been billed without a durable row; inspect ${inspectCommand}, fix the failure cause, then rerun the Play so completed receipts can be reused.`;
26399
+ const currentTableQueryCommand2 = input2.rowsInfo.currentTableQueryCommand;
26400
+ const explanation = terminalRunWithMissingAdvertisedRows ? ` The terminal result advertised ${advertisedRows} row(s), but none are currently attributed to this run's durable sheet. Those rows may have been reused from an earlier run or the projection may still be lagging; export refuses to guess another run's rows. Retry the same export after projection catches up (do not rerun the Play): ${retryCommand}. If it remains empty, inspect the full package with ${inspectCommand}` + (currentTableQueryCommand2 ? `, then inspect the current table's _run_id/_writer_run_id with ${currentTableQueryCommand2} and export the run that owns those rows` : " and export the run that owns the durable dataset") + "; this command will not use a database-wide table query that could mix runs." : ` The run status is '${runStatus || "unknown"}', so it likely failed before persisting rows to this dataset. That means provider calls may have been billed without a durable row; inspect ${inspectCommand}, fix the failure cause, then rerun the Play so completed receipts can be reused.`;
26377
26401
  throw new DeeplineError(
26378
26402
  `Run ${input2.status.runId} has zero rows to export from dataset '${datasetLabel}'.` + explanation + (available.length > 0 ? ` Available datasets: ${available.join(", ")}.` : " No datasets were materialized for this run."),
26379
26403
  void 0,
@@ -26386,6 +26410,7 @@ function assertDatasetHasExportableRows(input2) {
26386
26410
  guidance: terminalRunWithMissingAdvertisedRows ? {
26387
26411
  inspect_command: inspectCommand,
26388
26412
  retry_command: retryCommand,
26413
+ ...currentTableQueryCommand2 ? { lookup_writer_command: currentTableQueryCommand2 } : {},
26389
26414
  do_not_rerun: true,
26390
26415
  reason: "Run output is terminal but its rows are not attributed to this run; export refuses to guess another writer."
26391
26416
  } : {