deepline 0.3.476 → 0.3.477

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (125) hide show
  1. package/dist/bundling-sources/sdk/src/release.ts +1 -1
  2. package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +33 -0
  3. package/dist/bundling-sources/shared_libs/play-runtime/governor/app-runtime-rate-state-backend.ts +66 -0
  4. package/dist/bundling-sources/shared_libs/play-runtime/governor/governor.ts +1 -0
  5. package/dist/bundling-sources/shared_libs/play-runtime/governor/rate-state-backend.ts +3 -0
  6. package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts +6 -0
  7. package/dist/bundling-sources/shared_libs/play-runtime/runner-app/index.ts +13 -0
  8. package/dist/bundling-sources/shared_libs/play-runtime/sandbox-compute-usage.ts +6 -0
  9. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-rate-state.ts +167 -7
  10. package/dist/bundling-sources/shared_libs/play-runtime/worker-api-types.ts +42 -0
  11. package/dist/cli/auth-QITP34OX.js +41 -0
  12. package/dist/cli/{auth-A22724UX.mjs → auth-XVXRRWIQ.mjs} +5 -5
  13. package/dist/cli/{billing-T5Q6OO6M.js → billing-2H7GFJDT.js} +79 -79
  14. package/dist/cli/{billing-IIBEZ6J7.mjs → billing-EOPUJ4JI.mjs} +6 -6
  15. package/dist/cli/{chunk-5FBCMLJW.js → chunk-23EEVP5W.js} +6 -6
  16. package/dist/cli/{chunk-ULKGYVYP.js → chunk-255O6APS.js} +2 -2
  17. package/dist/cli/{chunk-2YHDNHRR.mjs → chunk-2GZFIWSA.mjs} +4 -4
  18. package/dist/cli/{chunk-PGNLD6L6.js → chunk-2QT6KN54.js} +10 -10
  19. package/dist/cli/{chunk-H3GOAZHV.mjs → chunk-2YNLW43I.mjs} +2 -2
  20. package/dist/cli/{chunk-IC3BZJR7.js → chunk-347EE4YE.js} +8 -8
  21. package/dist/cli/{chunk-L45IECBZ.js → chunk-3UKKM6OS.js} +6 -6
  22. package/dist/cli/{chunk-VLYUFZUT.js → chunk-3W6E3WVU.js} +9 -9
  23. package/dist/cli/{chunk-B7PEXXRW.js → chunk-4RAENEAX.js} +53 -53
  24. package/dist/cli/{chunk-CFSCHLNQ.mjs → chunk-6IJJCRE5.mjs} +3 -3
  25. package/dist/cli/{chunk-A6BETJ2A.js → chunk-6KKDIISC.js} +1 -1
  26. package/dist/cli/{chunk-MYYOCJZ2.js → chunk-75NLWBYQ.js} +93 -93
  27. package/dist/cli/{chunk-EBWYNNTJ.js → chunk-7CILZKLE.js} +5 -5
  28. package/dist/cli/{chunk-5AEEINTU.js → chunk-ACDXGOXE.js} +4 -4
  29. package/dist/cli/{chunk-VPAVNEDQ.mjs → chunk-C2AN2HTB.mjs} +4 -4
  30. package/dist/cli/{chunk-ZV4OWYLZ.mjs → chunk-C2V2HQDW.mjs} +2 -2
  31. package/dist/cli/{chunk-YQOWMQEW.mjs → chunk-C46DWB56.mjs} +1 -1
  32. package/dist/cli/{chunk-6YZYCKWD.js → chunk-CH4VMDB5.js} +51 -51
  33. package/dist/cli/{chunk-J4FCTFQS.mjs → chunk-DFKHXZQ7.mjs} +10 -10
  34. package/dist/cli/{chunk-HSANVBVL.js → chunk-DMWHEYZM.js} +2 -2
  35. package/dist/cli/{chunk-NREO7TT4.mjs → chunk-DS3DZUES.mjs} +2 -2
  36. package/dist/cli/{chunk-YEJDFZJP.js → chunk-E2CUSH56.js} +16 -16
  37. package/dist/cli/{chunk-JU2KBLOR.mjs → chunk-ISEUOYCB.mjs} +4 -4
  38. package/dist/cli/{chunk-F5PYBZ2H.mjs → chunk-JS4UO6G4.mjs} +1 -1
  39. package/dist/cli/{chunk-LJFIUT5R.mjs → chunk-KMXBZIKO.mjs} +2 -2
  40. package/dist/cli/{chunk-W3BHZUPL.js → chunk-KWQNNPMH.js} +3 -3
  41. package/dist/cli/{chunk-FVPT44IJ.mjs → chunk-LJYTYNQZ.mjs} +1 -1
  42. package/dist/cli/{chunk-5PYSYW63.js → chunk-LXKGZKTL.js} +26 -26
  43. package/dist/cli/{chunk-AS3VECCY.js → chunk-M2UL4ZS5.js} +7 -7
  44. package/dist/cli/{chunk-7HAUVMRF.js → chunk-NC52JG66.js} +3 -3
  45. package/dist/cli/{chunk-4YWYLHED.mjs → chunk-NJMYX2TP.mjs} +3 -3
  46. package/dist/cli/{chunk-2WEPQSA2.mjs → chunk-NO7ZC2GV.mjs} +5 -5
  47. package/dist/cli/{chunk-UHTJWDYB.mjs → chunk-PQQNMMWM.mjs} +1 -1
  48. package/dist/cli/{chunk-NTJDZURV.mjs → chunk-QODW5ILA.mjs} +2 -2
  49. package/dist/cli/{chunk-FK7SBMSZ.js → chunk-R5FNCFEK.js} +3 -3
  50. package/dist/cli/{chunk-WTPRRKZA.js → chunk-RGNMMDSJ.js} +37 -37
  51. package/dist/cli/{chunk-VKBZLV2K.mjs → chunk-RHIFH3GD.mjs} +1 -1
  52. package/dist/cli/{chunk-4WMIMSSI.mjs → chunk-RLCAECZH.mjs} +5 -5
  53. package/dist/cli/{chunk-NPNQCPFP.mjs → chunk-SG2SZQ4K.mjs} +2 -2
  54. package/dist/cli/{chunk-LZCYDU6Q.mjs → chunk-SVU2VR6Q.mjs} +2 -2
  55. package/dist/cli/{chunk-TEJSRJNH.js → chunk-THMGJOR4.js} +8 -8
  56. package/dist/cli/{chunk-CBVCCXIY.js → chunk-UBNS6GSV.js} +21 -21
  57. package/dist/cli/{chunk-D7EQ4PI5.mjs → chunk-WRHLAAGN.mjs} +1 -1
  58. package/dist/cli/{chunk-KCP6XNRR.mjs → chunk-XQLYPHYW.mjs} +1 -1
  59. package/dist/cli/{chunk-ZOXV6WFI.mjs → chunk-YZA5UJXM.mjs} +1 -1
  60. package/dist/cli/{chunk-KGLIDLJ2.js → chunk-ZDPL5AIK.js} +5 -5
  61. package/dist/cli/{csv-FXOQ2PBE.js → csv-ECB7TYR2.js} +13 -13
  62. package/dist/cli/{csv-GPVCETS2.mjs → csv-IDRPTSSC.mjs} +4 -4
  63. package/dist/cli/{db-F5UJTBGP.mjs → db-FOBZD44N.mjs} +6 -6
  64. package/dist/cli/{db-LXYV4GPZ.js → db-S74XZZ77.js} +27 -27
  65. package/dist/cli/{describe-GYWCI7I6.mjs → describe-KMTPPI2J.mjs} +17 -17
  66. package/dist/cli/describe-PI7HCT7K.js +48 -0
  67. package/dist/cli/{executions-6NL2YW76.mjs → executions-4JAIVE2E.mjs} +6 -6
  68. package/dist/cli/{executions-6SJKTY2L.js → executions-ZE3JMNA4.js} +8 -8
  69. package/dist/cli/{feedback-7AMQCNTB.mjs → feedback-EUX5PDBA.mjs} +4 -4
  70. package/dist/cli/{feedback-E3HULSQD.js → feedback-W6VA47A3.js} +8 -8
  71. package/dist/cli/git-NDOGCQZY.js +25 -0
  72. package/dist/cli/{git-74IS2EFH.mjs → git-UOVG4PLH.mjs} +5 -5
  73. package/dist/cli/index.js +135 -135
  74. package/dist/cli/index.mjs +40 -40
  75. package/dist/cli/{legacy-noops-ALWUQPKS.js → legacy-noops-3H35EH5Y.js} +8 -8
  76. package/dist/cli/{legacy-noops-52FFRTRZ.mjs → legacy-noops-ZBWUNFWV.mjs} +5 -5
  77. package/dist/cli/{monitors-VTJIWWW2.mjs → monitors-AYSGSUAK.mjs} +7 -7
  78. package/dist/cli/{monitors-Z4ADOGTY.js → monitors-PI3KPLR2.js} +91 -91
  79. package/dist/cli/{org-7GU6H7JL.mjs → org-2BIG7U4V.mjs} +6 -6
  80. package/dist/cli/{org-2BTEKDF5.js → org-MVS3Y6BK.js} +72 -72
  81. package/dist/cli/{pages-I6PFQGEU.mjs → pages-4TRRL24G.mjs} +6 -6
  82. package/dist/cli/{pages-MG36E7RY.js → pages-VB6FOVDE.js} +17 -17
  83. package/dist/cli/{play-VN52S3RV.js → play-BLT6IT2O.js} +325 -308
  84. package/dist/cli/{play-7KFSLJX2.mjs → play-JR3VUOZM.mjs} +36 -19
  85. package/dist/cli/{providers-D46JDEL2.mjs → providers-547DPK4S.mjs} +6 -6
  86. package/dist/cli/{providers-W5XLII66.js → providers-BBXIAGW4.js} +9 -9
  87. package/dist/cli/{quickstart-4GTXBZHY.mjs → quickstart-NTLXD7HS.mjs} +4 -4
  88. package/dist/cli/{quickstart-TIZAC63S.js → quickstart-RNPI5B6J.js} +6 -6
  89. package/dist/cli/{run-FOVEMYYX.js → run-BCCEXBOG.js} +36 -36
  90. package/dist/cli/{run-LE3D4LVU.mjs → run-QZ4SFECD.mjs} +22 -22
  91. package/dist/cli/{search-6YL6WJMQ.mjs → search-PYNELH4C.mjs} +7 -7
  92. package/dist/cli/search-YHEICYQL.js +32 -0
  93. package/dist/cli/{secrets-D66U6RTT.mjs → secrets-O5WGREQ7.mjs} +6 -6
  94. package/dist/cli/{secrets-CSTTTMZH.js → secrets-VD35LMYJ.js} +15 -15
  95. package/dist/cli/{sessions-XIO3OUER.mjs → sessions-2TL6Z5J5.mjs} +5 -5
  96. package/dist/cli/{sessions-45M2NRAA.js → sessions-NGGIBYXF.js} +6 -6
  97. package/dist/cli/{settings-R33X2D5T.mjs → settings-6KM74AJT.mjs} +11 -11
  98. package/dist/cli/{settings-N42CVKSK.js → settings-ERFNZRHO.js} +78 -78
  99. package/dist/cli/{setup-HREDVNF2.js → setup-6X7EJ6RG.js} +44 -44
  100. package/dist/cli/{setup-CBBC4FQP.mjs → setup-Y3BN6XRS.mjs} +9 -9
  101. package/dist/cli/skills-F7MPYC4P.js +33 -0
  102. package/dist/cli/{skills-4X6WS2PT.mjs → skills-OUBQASHI.mjs} +7 -7
  103. package/dist/cli/{tools-YR46YFR2.mjs → tools-EELZ5FF5.mjs} +16 -16
  104. package/dist/cli/tools-MCWDAK53.js +55 -0
  105. package/dist/cli/update-G5BRJOVT.js +33 -0
  106. package/dist/cli/{update-5CYAZX5X.mjs → update-TX7SW32Y.mjs} +9 -9
  107. package/dist/cli/{workflow-M7AA722W.js → workflow-5AI2BJA7.js} +17 -17
  108. package/dist/cli/{workflow-NKQ3CWPK.mjs → workflow-PUNQG7LG.mjs} +7 -7
  109. package/dist/index.js +9 -9
  110. package/dist/index.mjs +6 -6
  111. package/dist/install-integrity.json +96 -96
  112. package/dist/release.d.mts +1 -1
  113. package/dist/release.d.ts +1 -1
  114. package/dist/release.js +2 -2
  115. package/dist/release.mjs +1 -1
  116. package/dist/server.js +5 -5
  117. package/dist/server.mjs +3 -3
  118. package/package.json +1 -1
  119. package/dist/cli/auth-2ZY57Q7W.js +0 -41
  120. package/dist/cli/describe-IB7SUIUT.js +0 -48
  121. package/dist/cli/git-A2QIIMW5.js +0 -25
  122. package/dist/cli/search-HN6SVDQU.js +0 -32
  123. package/dist/cli/skills-QZ6HZX2T.js +0 -33
  124. package/dist/cli/tools-KFIH7UWB.js +0 -55
  125. package/dist/cli/update-VZLEJLWJ.js +0 -33
@@ -205,7 +205,7 @@ export const SDK_RELEASE = {
205
205
  // getters keep their established compatibility behavior.
206
206
  // 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
207
207
  // 0.3.90 is the first deliberately versioned SDK release for API v3.
208
- version: '0.3.476',
208
+ version: '0.3.477',
209
209
  updateSummary: {
210
210
  introducedIn: '0.3.90',
211
211
  summary:
@@ -2281,6 +2281,39 @@ export async function releaseRateStateViaAppRuntime(
2281
2281
  });
2282
2282
  }
2283
2283
 
2284
+ /** Persist the final success batch without acquiring another provider permit. */
2285
+ export async function flushRateStateFeedbackViaAppRuntime(
2286
+ context: WorkerRuntimeApiContext,
2287
+ input: PlayRunnerRateStateReleaseInput,
2288
+ signal: AbortSignal,
2289
+ ): Promise<void> {
2290
+ const batch = input.concurrencyObservationBatch;
2291
+ if (!batch)
2292
+ throw new Error('Rate-state feedback is missing its batch identity.');
2293
+ const result = await postAppRuntimeApi<{
2294
+ acknowledgedObservationBatch?: typeof batch;
2295
+ }>(
2296
+ {
2297
+ ...context,
2298
+ connectionRetryBudgetMs: 0,
2299
+ signal: context.signal
2300
+ ? AbortSignal.any([context.signal, signal])
2301
+ : signal,
2302
+ },
2303
+ { action: 'rate_state_release', ...input },
2304
+ );
2305
+ const acknowledged = result.acknowledgedObservationBatch;
2306
+ if (
2307
+ acknowledged?.observerId !== batch.observerId ||
2308
+ acknowledged.laneId !== batch.laneId ||
2309
+ acknowledged.sequence !== batch.sequence
2310
+ ) {
2311
+ throw new Error(
2312
+ 'Rate-state receiver did not acknowledge the success batch.',
2313
+ );
2314
+ }
2315
+ }
2316
+
2284
2317
  export async function penalizeRateStateViaAppRuntime(
2285
2318
  context: WorkerRuntimeApiContext,
2286
2319
  input: PlayRunnerRateStatePenalizeInput,
@@ -2,6 +2,7 @@ import {
2
2
  acquireRateStateViaAppRuntime,
3
3
  AppRuntimeApiTransportError,
4
4
  capacityShedRetryAfterMs,
5
+ flushRateStateFeedbackViaAppRuntime,
5
6
  penalizeRateStateViaAppRuntime,
6
7
  releaseRateStateViaAppRuntime,
7
8
  RateStateValidationUnavailableError,
@@ -54,6 +55,7 @@ export const RATE_ACQUIRE_MAX_WAIT_MS = 300_000;
54
55
  */
55
56
  export const PROVIDER_ADMISSION_WAIT_BUDGET_MS = 120_000;
56
57
  const PROVIDER_NO_PROGRESS_WARNING_MS = 180_000;
58
+ const SUCCESS_FEEDBACK_DRAIN_TIMEOUT_MS = 10_000;
57
59
 
58
60
  /**
59
61
  * Permits leased per app-runtime round trip for the direct Node substrate.
@@ -77,6 +79,10 @@ export interface AppRuntimeRatePort {
77
79
  penalize(
78
80
  input: Parameters<typeof penalizeRateStateViaAppRuntime>[1],
79
81
  ): ReturnType<typeof penalizeRateStateViaAppRuntime>;
82
+ flushFeedback?(
83
+ input: Parameters<typeof flushRateStateFeedbackViaAppRuntime>[1],
84
+ signal: AbortSignal,
85
+ ): Promise<void>;
80
86
  validate?(
81
87
  input: Parameters<typeof validateRateStateViaAppRuntime>[1],
82
88
  ): ReturnType<typeof validateRateStateViaAppRuntime>;
@@ -87,6 +93,8 @@ function contextRatePort(context: WorkerRuntimeApiContext): AppRuntimeRatePort {
87
93
  acquire: (input) => acquireRateStateViaAppRuntime(context, input),
88
94
  release: (input) => releaseRateStateViaAppRuntime(context, input),
89
95
  penalize: (input) => penalizeRateStateViaAppRuntime(context, input),
96
+ flushFeedback: (input, signal) =>
97
+ flushRateStateFeedbackViaAppRuntime(context, input, signal),
90
98
  validate: (input) => validateRateStateViaAppRuntime(context, input),
91
99
  };
92
100
  }
@@ -296,6 +304,16 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
296
304
  private readonly effectiveRps = new Map<string, number>();
297
305
  private readonly pendingSuccesses = new Map<string, number>();
298
306
  private readonly successFeedbackFlights = new Map<string, SuccessFeedback>();
307
+ private readonly successFeedbackScopes = new Map<
308
+ string,
309
+ {
310
+ bucketId: string;
311
+ rateScopeToken?: string | null;
312
+ rules: readonly PacingRule[];
313
+ }
314
+ >();
315
+ private readonly successFeedbackSupported = new Set<string>();
316
+ private successFeedbackDrain: Promise<void> | null = null;
299
317
  /**
300
318
  * Completion samples travel with the next lease refill. This keeps the
301
319
  * Gradient2 checkpoint current without adding one gateway request per row.
@@ -1073,6 +1091,11 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
1073
1091
  }
1074
1092
  : rawResponse;
1075
1093
  this.acknowledgeSuccessFeedback(successKey, feedback);
1094
+ if (response.successFeedbackSupported === true) {
1095
+ this.successFeedbackSupported.add(bucketId);
1096
+ } else {
1097
+ this.successFeedbackSupported.delete(bucketId);
1098
+ }
1076
1099
  this.recordCooldown(bucketId, response);
1077
1100
  if (response.scopeValidationSupported === true)
1078
1101
  this.validationScopes.add(bucketId);
@@ -1559,11 +1582,17 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
1559
1582
 
1560
1583
  observeSuccess(input: {
1561
1584
  bucketId: string;
1585
+ rateScopeToken?: string | null;
1562
1586
  rules: readonly PacingRule[];
1563
1587
  latencyMs?: number | null;
1564
1588
  }): void {
1565
1589
  this.providerStalls.delete(input.bucketId);
1566
1590
  const key = pendingSuccessKey(input.bucketId, input.rules);
1591
+ this.successFeedbackScopes.set(key, {
1592
+ bucketId: input.bucketId,
1593
+ rateScopeToken: input.rateScopeToken,
1594
+ rules: input.rules,
1595
+ });
1567
1596
  this.pendingSuccesses.set(
1568
1597
  key,
1569
1598
  Math.min(1_000_000, (this.pendingSuccesses.get(key) ?? 0) + 1),
@@ -1593,6 +1622,43 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
1593
1622
  this.pendingConcurrencyObservations.set(key, pending);
1594
1623
  }
1595
1624
 
1625
+ async flushSuccessFeedback(): Promise<void> {
1626
+ if (!this.port.flushFeedback) return;
1627
+ if (this.successFeedbackDrain) return await this.successFeedbackDrain;
1628
+ const signal = AbortSignal.timeout(SUCCESS_FEEDBACK_DRAIN_TIMEOUT_MS);
1629
+ const drain = async () => {
1630
+ for (const [key, scope] of this.successFeedbackScopes) {
1631
+ // Older receivers retain their existing piggyback behavior. An explicit
1632
+ // capability and matching batch acknowledgement protect rolling deploys.
1633
+ if (!this.successFeedbackSupported.has(scope.bucketId)) continue;
1634
+ for (;;) {
1635
+ if (signal.aborted) throw abortReason(signal);
1636
+ const feedback = this.snapshotSuccessFeedback(key);
1637
+ if (!feedback.concurrencyObservationBatch) break;
1638
+ await this.port.flushFeedback!(
1639
+ {
1640
+ ...scope,
1641
+ rules: [...scope.rules],
1642
+ leaseIds: [],
1643
+ ...feedback,
1644
+ schedulerSchema: this.schedulerSchema,
1645
+ },
1646
+ signal,
1647
+ );
1648
+ // A lost response leaves this exact frozen payload available to retry.
1649
+ this.acknowledgeSuccessFeedback(key, feedback);
1650
+ }
1651
+ this.successFeedbackScopes.delete(key);
1652
+ }
1653
+ };
1654
+ this.successFeedbackDrain = drain();
1655
+ try {
1656
+ await this.successFeedbackDrain;
1657
+ } finally {
1658
+ this.successFeedbackDrain = null;
1659
+ }
1660
+ }
1661
+
1596
1662
  private newPendingConcurrencyObservations(
1597
1663
  key: string,
1598
1664
  observations: Array<{ kind: 'success'; latencyMs: number }> = [],
@@ -868,6 +868,7 @@ export function createPlayExecutionGovernor(
868
868
  if (!rules) return;
869
869
  input.rateState.observeSuccess?.({
870
870
  bucketId: rateScope[0],
871
+ ...(rateScope[1] ? { rateScopeToken: rateScope[1] } : {}),
871
872
  rules,
872
873
  latencyMs: success.latencyMs,
873
874
  });
@@ -197,6 +197,9 @@ export interface RateStateBackend {
197
197
  /** Provider residence measured at the HTTP boundary. */
198
198
  latencyMs?: number | null;
199
199
  }): void;
200
+
201
+ /** Bounded final feedback write after the invocation's provider work settles. */
202
+ flushSuccessFeedback?(): Promise<void>;
200
203
  }
201
204
 
202
205
  const NOOP_PERMIT: PacingPermit = {
@@ -90,6 +90,8 @@ export type PlayRunnerRateStateAcquireResult = {
90
90
  scopeValidationSupported?: boolean;
91
91
  /** This gateway rechecks signed provider permits on integration execute. */
92
92
  dispatchValidationSupported?: boolean;
93
+ /** This receiver acknowledges replay-safe success batches on release. */
94
+ successFeedbackSupported?: boolean;
93
95
  /**
94
96
  * Gateway clock used to produce `scheduledAtMs`. Runners translate the
95
97
  * schedule into relative delays so cross-machine clock skew cannot compress
@@ -138,6 +140,10 @@ export type PlayRunnerRateStateReleaseInput = {
138
140
  leaseIds: string[];
139
141
  /** Idempotently refund rate reservations that never reached provider fetch. */
140
142
  refundReservationIds?: string[];
143
+ /** Flush confirmed outcomes without reserving another provider permit. */
144
+ observedSuccesses?: number;
145
+ concurrencyObservations?: PlayRunnerRateStateConcurrencyObservation[];
146
+ concurrencyObservationBatch?: PlayRunnerRateStateConcurrencyObservationBatch;
141
147
  schedulerSchema?: string | null;
142
148
  };
143
149
 
@@ -3788,6 +3788,19 @@ async function run(
3788
3788
  } finally {
3789
3789
  clearInterval(workCounterTimer);
3790
3790
  emitWorkCounters();
3791
+ try {
3792
+ await rateStateBackend.rateState?.flushSuccessFeedback?.();
3793
+ } catch (feedbackError) {
3794
+ // Feedback is advisory: preserve the Play's result or original failure.
3795
+ emitWorkerLog(
3796
+ `[warn] runtime.provider_success_feedback_flush_failed: ${
3797
+ feedbackError instanceof Error
3798
+ ? feedbackError.message
3799
+ : String(feedbackError)
3800
+ }`,
3801
+ options,
3802
+ );
3803
+ }
3791
3804
  }
3792
3805
  }
3793
3806
 
@@ -88,6 +88,9 @@ export function resolveRuntimeSandboxComputeItems(input: {
88
88
  sandboxId,
89
89
  startedAt: resource.startedAt,
90
90
  endedAt,
91
+ maxBillingDurationSeconds: resource.maxBillingDurationSeconds,
92
+ observedWallTimeSeconds:
93
+ Math.max(0, observedEndedAt - resource.startedAt) / 1_000,
91
94
  }),
92
95
  );
93
96
  }
@@ -201,6 +204,9 @@ export function resolveDaytonaRuntimeComputeItems(input: {
201
204
  sandboxId,
202
205
  startedAt: resource.startedAt,
203
206
  endedAt,
207
+ maxBillingDurationSeconds: resource.maxBillingDurationSeconds,
208
+ observedWallTimeSeconds:
209
+ Math.max(0, observedEndedAt - resource.startedAt) / 1_000,
204
210
  }),
205
211
  );
206
212
  }
@@ -38,6 +38,26 @@ const RATE_SCOPE_HOLD_TABLE = 'rate_scope_holds';
38
38
  const PHYSICAL_DISPATCH_RULE_ID = 'runtime:physical-dispatch';
39
39
 
40
40
  const MIN_CONCURRENCY_WAIT_MS = 10;
41
+ /**
42
+ * Longest re-ask when every concurrency slot is taken. A full scope re-asks
43
+ * about when its next slot should free: the average age of its live leases
44
+ * divided by their count. A fast provider stays at the 10 ms floor; one whose
45
+ * calls hold a slot for minutes no longer re-locks its scope row every 10 ms.
46
+ * On 2026-11-06 five such runs re-asked about 160 times a second for hours.
47
+ */
48
+ const MAX_CONCURRENCY_WAIT_MS = 1_000;
49
+ /**
50
+ * A healthy scope lock is held for milliseconds. An acquire that cannot get it
51
+ * within this budget stops waiting on the server instead of leaving a backend
52
+ * queued behind a stuck holder after its caller gave up. On 2026-11-06 one
53
+ * stuck holder queued such abandoned waits for 16 minutes until they exhausted
54
+ * the shared scheduler pool.
55
+ */
56
+ const RATE_STATE_LOCK_TIMEOUT_MS = 3_000;
57
+ /** Ends a bounded acquire statement before the gateway's 30 s caller budget. */
58
+ const RATE_STATE_STATEMENT_TIMEOUT_MS = 15_000;
59
+ /** Re-ask after a lock timeout, as for any other busy scope. */
60
+ const RATE_STATE_LOCK_BUSY_WAIT_MS = 1_000;
41
61
  /**
42
62
  * Longest a requester held back by the fair-share cap sleeps before it re-asks.
43
63
  * The natural wait is the rest of the current fair-share window (a sibling's
@@ -771,6 +791,32 @@ async function seedBucketRows(
771
791
  },
772
792
  seeds: RateBucketSeed[],
773
793
  nowMs: number | null,
794
+ ): Promise<void> {
795
+ // Cold path only, so its own bounded transaction costs two round trips once
796
+ // per bucket per process. Its insert waits on the same scope row as a grant.
797
+ await client.query('BEGIN');
798
+ try {
799
+ await boundRateStateTransaction(client);
800
+ await seedBucketRowsStatement(client, input, seeds, nowMs);
801
+ await client.query('COMMIT');
802
+ } catch (error) {
803
+ try {
804
+ await client.query('ROLLBACK');
805
+ } catch {
806
+ // Preserve the original seed/storage error.
807
+ }
808
+ throw rateStateLockBusyOr(error);
809
+ }
810
+ }
811
+
812
+ async function seedBucketRowsStatement(
813
+ client: PostgresSchedulerQueryClient,
814
+ input: { bucketId: string } & {
815
+ schema?: string | null;
816
+ schedulerSchema?: string | null;
817
+ },
818
+ seeds: RateBucketSeed[],
819
+ nowMs: number | null,
774
820
  ): Promise<void> {
775
821
  await client.query(
776
822
  `
@@ -1797,6 +1843,7 @@ async function acquireWithConcurrency(
1797
1843
  while (true) {
1798
1844
  await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
1799
1845
  try {
1846
+ await boundRateStateTransaction(client);
1800
1847
  await client.query(
1801
1848
  `SELECT bucket_id FROM ${scopeHoldTableName(input)}
1802
1849
  WHERE bucket_id = $1 FOR UPDATE`,
@@ -1824,11 +1871,53 @@ async function acquireWithConcurrency(
1824
1871
  probeBaselineSupported = false;
1825
1872
  continue;
1826
1873
  }
1827
- throw error;
1874
+ throw rateStateLockBusyOr(error);
1828
1875
  }
1829
1876
  }
1830
1877
  }
1831
1878
 
1879
+ /**
1880
+ * Bound the current rate-state transaction. `SET LOCAL` keeps the limits inside
1881
+ * this transaction: the scheduler pool runs behind PgBouncer transaction
1882
+ * pooling, where a session setting would leak to another caller.
1883
+ */
1884
+ async function boundRateStateTransaction(
1885
+ client: PostgresSchedulerQueryClient,
1886
+ ): Promise<void> {
1887
+ await client.query(
1888
+ `SELECT set_config('lock_timeout', $1, true),
1889
+ set_config('statement_timeout', $2, true)`,
1890
+ [`${RATE_STATE_LOCK_TIMEOUT_MS}ms`, `${RATE_STATE_STATEMENT_TIMEOUT_MS}ms`],
1891
+ );
1892
+ }
1893
+
1894
+ /** A scope row another transaction still holds after the lock budget. */
1895
+ class RateStateLockBusyError extends Error {
1896
+ constructor(cause: unknown) {
1897
+ super('Rate-state scope lock was not available within its budget.', {
1898
+ cause,
1899
+ });
1900
+ this.name = 'RateStateLockBusyError';
1901
+ }
1902
+ }
1903
+
1904
+ /**
1905
+ * A bounded rate-state transaction that ran out of its lock or statement
1906
+ * budget (57014 is statement_timeout).
1907
+ */
1908
+ function isRateStateBusy(error: unknown): boolean {
1909
+ return (
1910
+ error instanceof RateStateLockBusyError ||
1911
+ (error as { code?: unknown } | null)?.code === '57014'
1912
+ );
1913
+ }
1914
+
1915
+ function rateStateLockBusyOr(error: unknown): unknown {
1916
+ return (error as { code?: unknown } | null)?.code === '55P03'
1917
+ ? new RateStateLockBusyError(error)
1918
+ : error;
1919
+ }
1920
+
1832
1921
  async function grantWithConcurrency(
1833
1922
  client: PostgresSchedulerQueryClient,
1834
1923
  input: RateStateAcquireInput,
@@ -2315,7 +2404,29 @@ async function grantWithConcurrency(
2315
2404
  (SELECT cool_until_ms FROM cooling),
2316
2405
  (SELECT retry_at_ms FROM cooling)
2317
2406
  ) - (SELECT now_ms FROM refreshed LIMIT 1)${fair?.waitBranch ?? ''}
2318
- WHEN (SELECT free FROM free_slots) <= 0 THEN $7::bigint
2407
+ WHEN (SELECT free FROM free_slots) <= 0 THEN (
2408
+ -- Re-ask about when the next slot should free. Leases are never
2409
+ -- renewed, so a lease's age is its TTL minus its remaining life.
2410
+ SELECT LEAST(
2411
+ ${MAX_CONCURRENCY_WAIT_MS}::bigint,
2412
+ GREATEST(
2413
+ $7::bigint,
2414
+ CEIL(
2415
+ COALESCE(
2416
+ AVG(
2417
+ (SELECT now_ms FROM now_value)
2418
+ - (cl.expires_at_ms - $8::bigint)
2419
+ ),
2420
+ 0
2421
+ ) / GREATEST((SELECT active_count FROM concurrency_context), 1)
2422
+ )::bigint
2423
+ )
2424
+ )
2425
+ FROM ${concurrencyLeaseTableName(input)} cl
2426
+ WHERE cl.bucket_id = $1
2427
+ AND cl.rule_id = ANY($5::text[])
2428
+ AND cl.expires_at_ms > (SELECT now_ms FROM now_value)
2429
+ )
2319
2430
  ELSE LEAST((SELECT pending_recovery_wait_ms FROM provider_hold), (
2320
2431
  SELECT CEIL(
2321
2432
  MAX(
@@ -2417,9 +2528,24 @@ export async function acquirePostgresRateState(
2417
2528
  client: PostgresSchedulerQueryClient,
2418
2529
  input: RateStateAcquireInput,
2419
2530
  ): Promise<PlayRunnerRateStateAcquireResult> {
2420
- return await withRateStateAvailabilityGuard(input, () =>
2421
- acquireRateStateStatements(client, input),
2422
- );
2531
+ return await withRateStateAvailabilityGuard(input, async () => {
2532
+ try {
2533
+ return await acquireRateStateStatements(client, input);
2534
+ } catch (error) {
2535
+ // A store error spends the runner's fail-closed retry ladder, so a stuck
2536
+ // holder elsewhere would fail this run. A busy scope is not a broken
2537
+ // store: answer as for any other wait. Feedback rolled back with a busy
2538
+ // feedback step is not re-sent, because the runner acknowledges any
2539
+ // answer. A lost success sample only slows the next ramp, and its batch
2540
+ // identity still prevents a double apply.
2541
+ if (!(error instanceof RateStateLockBusyError)) throw error;
2542
+ return {
2543
+ granted: 0,
2544
+ waitMs: RATE_STATE_LOCK_BUSY_WAIT_MS,
2545
+ leaseIds: [],
2546
+ };
2547
+ }
2548
+ });
2423
2549
  }
2424
2550
 
2425
2551
  /**
@@ -2838,6 +2964,8 @@ async function applyConcurrencyObservations(
2838
2964
  rules: readonly AcquirePacingRule[],
2839
2965
  observations: readonly ConcurrencyObservation[],
2840
2966
  batch: ConcurrencyObservationBatch | null,
2967
+ /** Acquire bounds its lock waits; release and penalty keep their behaviour. */
2968
+ options: { bounded?: boolean } = {},
2841
2969
  ): Promise<{
2842
2970
  disposition: 'unbatched' | 'applied' | 'already_applied';
2843
2971
  ruleCount: number;
@@ -2895,6 +3023,7 @@ async function applyConcurrencyObservations(
2895
3023
 
2896
3024
  await client.query('BEGIN');
2897
3025
  try {
3026
+ if (options.bounded) await boundRateStateTransaction(client);
2898
3027
  const accepted = await client.query<{ last_sequence: number }>(
2899
3028
  `
2900
3029
  WITH now_value AS (
@@ -3103,7 +3232,7 @@ async function applyConcurrencyObservations(
3103
3232
  } catch {
3104
3233
  // Preserve the original controller/storage error.
3105
3234
  }
3106
- throw error;
3235
+ throw options.bounded ? rateStateLockBusyOr(error) : error;
3107
3236
  }
3108
3237
  }
3109
3238
 
@@ -3143,6 +3272,7 @@ async function acquireRateStateStatements(
3143
3272
  rules,
3144
3273
  concurrencyObservationsFrom(input.concurrencyObservations),
3145
3274
  concurrencyObservationBatchFrom(input.concurrencyObservationBatch),
3275
+ { bounded: true },
3146
3276
  );
3147
3277
  // The buffered success count and RTT observations share one identity. If a
3148
3278
  // runner lost the prior acquire response, its retry still needs a fresh
@@ -3179,7 +3309,16 @@ async function acquireRateStateStatements(
3179
3309
  // so a row widened out of band is repaired here instead of waiting for a
3180
3310
  // process restart. Repairing after the grant keeps the acquire itself to a
3181
3311
  // single granting statement, so no permit is ever issued twice.
3182
- await seedBucketRows(client, input, seeds, nowMs);
3312
+ try {
3313
+ await seedBucketRows(client, input, seeds, nowMs);
3314
+ } catch (error) {
3315
+ // The grant above already committed. A busy scope must not turn its
3316
+ // leases into a zero answer the runner never releases. Leave the bucket
3317
+ // cold so the next acquire repairs before it grants.
3318
+ if (!isRateStateBusy(error)) throw error;
3319
+ warmRateBuckets.delete(warmKey);
3320
+ return acquireResultOf(outcome);
3321
+ }
3183
3322
  }
3184
3323
  markRateBucketWarm(warmKey);
3185
3324
  return acquireResultOf(outcome);
@@ -3199,6 +3338,27 @@ async function releaseRateStateStatements(
3199
3338
  input: PlayRunnerRateStateReleaseInput & { nowMs?: number },
3200
3339
  ): Promise<void> {
3201
3340
  const allRules = normalizeRules(input.rules as AcquirePacingRule[]);
3341
+ if (
3342
+ input.observedSuccesses !== undefined ||
3343
+ input.concurrencyObservations !== undefined ||
3344
+ input.concurrencyObservationBatch !== undefined
3345
+ ) {
3346
+ const batch = concurrencyObservationBatchFrom(
3347
+ input.concurrencyObservationBatch,
3348
+ );
3349
+ if (!batch) {
3350
+ throw new Error(
3351
+ 'Rate-state success feedback requires a replay-safe observation batch.',
3352
+ );
3353
+ }
3354
+ await applyConcurrencyObservations(
3355
+ client,
3356
+ input,
3357
+ allRules,
3358
+ concurrencyObservationsFrom(input.concurrencyObservations),
3359
+ batch,
3360
+ );
3361
+ }
3202
3362
  if (allRules.length === 0) return;
3203
3363
  const refundReservationIds = [
3204
3364
  ...new Set(
@@ -98,6 +98,32 @@ function sandboxSizeMetadata(cpu: number, memoryGiB: number) {
98
98
  return entry ? { sandboxSize: entry[0] } : {};
99
99
  }
100
100
 
101
+ /**
102
+ * Lets settlement tell a run that used its declared runtime from a duration no
103
+ * runtime limit explains. Billed time stops at the ceiling, so the unclamped
104
+ * duration travels beside it: without it an overrun looks like a run that
105
+ * stayed inside its limit. Items without a ceiling keep their historical shape.
106
+ */
107
+ function runtimeLimitMetadata(
108
+ maxBillingDurationSeconds: number | null | undefined,
109
+ observedWallTimeSeconds: number | undefined,
110
+ ) {
111
+ if (
112
+ typeof maxBillingDurationSeconds !== 'number' ||
113
+ !Number.isFinite(maxBillingDurationSeconds) ||
114
+ maxBillingDurationSeconds <= 0
115
+ ) {
116
+ return {};
117
+ }
118
+ return {
119
+ maxBillingDurationSeconds,
120
+ ...(typeof observedWallTimeSeconds === 'number' &&
121
+ Number.isFinite(observedWallTimeSeconds)
122
+ ? { observedWallTimeSeconds: Math.max(0, observedWallTimeSeconds) }
123
+ : {}),
124
+ };
125
+ }
126
+
101
127
  export function resolveDaytonaSandboxComputeItem(input: {
102
128
  itemId: string;
103
129
  source?: string;
@@ -108,6 +134,10 @@ export function resolveDaytonaSandboxComputeItem(input: {
108
134
  sandboxId?: string | null;
109
135
  startedAt?: number;
110
136
  endedAt?: number;
137
+ /** The resource's own billing ceiling: its timeout plus terminal grace. */
138
+ maxBillingDurationSeconds?: number | null;
139
+ /** Wall time before the ceiling clamped it; recorded only with a ceiling. */
140
+ observedWallTimeSeconds?: number;
111
141
  }): ComputeBillingItem {
112
142
  const billableSeconds = Math.max(1, Math.ceil(input.wallTimeSeconds));
113
143
  const cpu = Math.max(0, input.cpu);
@@ -138,6 +168,10 @@ export function resolveDaytonaSandboxComputeItem(input: {
138
168
  billableDiskGiB,
139
169
  startedAt: input.startedAt,
140
170
  endedAt: input.endedAt,
171
+ ...runtimeLimitMetadata(
172
+ input.maxBillingDurationSeconds,
173
+ input.observedWallTimeSeconds,
174
+ ),
141
175
  },
142
176
  };
143
177
  }
@@ -152,6 +186,10 @@ export function resolveModalSandboxComputeItem(input: {
152
186
  sandboxId?: string | null;
153
187
  startedAt?: number;
154
188
  endedAt?: number;
189
+ /** The resource's own billing ceiling: its timeout plus terminal grace. */
190
+ maxBillingDurationSeconds?: number | null;
191
+ /** Wall time before the ceiling clamped it; recorded only with a ceiling. */
192
+ observedWallTimeSeconds?: number;
155
193
  }): ComputeBillingItem {
156
194
  const billableSeconds = Math.max(1, Math.ceil(input.wallTimeSeconds));
157
195
  const physicalCpuCores = Math.max(0, input.physicalCpuCores);
@@ -186,6 +224,10 @@ export function resolveModalSandboxComputeItem(input: {
186
224
  }),
187
225
  startedAt: input.startedAt,
188
226
  endedAt: input.endedAt,
227
+ ...runtimeLimitMetadata(
228
+ input.maxBillingDurationSeconds,
229
+ input.observedWallTimeSeconds,
230
+ ),
189
231
  },
190
232
  };
191
233
  }
@@ -0,0 +1,41 @@
1
+ "use strict";Object.defineProperty(exports, "__esModule", {value: true});
2
+
3
+
4
+
5
+
6
+
7
+
8
+
9
+
10
+
11
+
12
+
13
+
14
+
15
+
16
+ var _chunkCH4VMDB5js = require('./chunk-CH4VMDB5.js');
17
+ require('./chunk-M2UL4ZS5.js');
18
+ require('./chunk-NC52JG66.js');
19
+ require('./chunk-ACDXGOXE.js');
20
+ require('./chunk-ALJJRZNJ.js');
21
+ require('./chunk-XNYIZMNQ.js');
22
+ require('./chunk-DIAWYWT5.js');
23
+ require('./chunk-6KKDIISC.js');
24
+ require('./chunk-OG64RBAC.js');
25
+ require('./chunk-COGTRVWW.js');
26
+
27
+
28
+
29
+
30
+
31
+
32
+
33
+
34
+
35
+
36
+
37
+
38
+
39
+
40
+
41
+ exports.EXIT_WORKSPACE_MISMATCH = _chunkCH4VMDB5js.EXIT_WORKSPACE_MISMATCH; exports.authStatusLine = _chunkCH4VMDB5js.authStatusLine; exports.clearPendingClaim = _chunkCH4VMDB5js.clearPendingClaim; exports.clearSatisfiedPendingClaim = _chunkCH4VMDB5js.clearSatisfiedPendingClaim; exports.handleRegister = _chunkCH4VMDB5js.handleRegister; exports.handleStatus = _chunkCH4VMDB5js.handleStatus; exports.handleWait = _chunkCH4VMDB5js.handleWait; exports.httpJson = _chunkCH4VMDB5js.httpJson; exports.readPendingAuthClaim = _chunkCH4VMDB5js.readPendingAuthClaim; exports.readStatusBanner = _chunkCH4VMDB5js.readStatusBanner; exports.registerAuthCommands = _chunkCH4VMDB5js.registerAuthCommands; exports.reportSetupCompleted = _chunkCH4VMDB5js.reportSetupCompleted; exports.resolvePendingAuthClaimForResume = _chunkCH4VMDB5js.resolvePendingAuthClaimForResume; exports.statusBannerLine = _chunkCH4VMDB5js.statusBannerLine;
@@ -13,14 +13,14 @@ import {
13
13
  reportSetupCompleted,
14
14
  resolvePendingAuthClaimForResume,
15
15
  statusBannerLine
16
- } from "./chunk-CFSCHLNQ.mjs";
17
- import "./chunk-NTJDZURV.mjs";
18
- import "./chunk-ZOXV6WFI.mjs";
19
- import "./chunk-FVPT44IJ.mjs";
16
+ } from "./chunk-6IJJCRE5.mjs";
17
+ import "./chunk-QODW5ILA.mjs";
18
+ import "./chunk-YZA5UJXM.mjs";
19
+ import "./chunk-LJYTYNQZ.mjs";
20
20
  import "./chunk-4QF2K7TM.mjs";
21
21
  import "./chunk-DLIRSHV6.mjs";
22
22
  import "./chunk-HBL4PM5Q.mjs";
23
- import "./chunk-UHTJWDYB.mjs";
23
+ import "./chunk-PQQNMMWM.mjs";
24
24
  import "./chunk-WZYKRD7T.mjs";
25
25
  import "./chunk-KRD6H754.mjs";
26
26
  export {