deepline 0.3.474 → 0.3.476

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (155) hide show
  1. package/dist/bundling-sources/sdk/src/play.ts +2 -2
  2. package/dist/bundling-sources/sdk/src/release.ts +3 -3
  3. package/dist/bundling-sources/shared_libs/monitors/provider-monitor-upstream-identity.ts +19 -6
  4. package/dist/bundling-sources/shared_libs/observability/scheduled-jobs.json +4 -4
  5. package/dist/bundling-sources/shared_libs/play-runtime/governor/app-runtime-rate-state-backend.ts +24 -1
  6. package/dist/bundling-sources/shared_libs/play-runtime/play-run-recovery-policy.ts +21 -4
  7. package/dist/bundling-sources/shared_libs/play-runtime/runtime-capacity-policy.ts +12 -0
  8. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-rate-scope.ts +47 -21
  9. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-rate-state.ts +18 -8
  10. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts +163 -40
  11. package/dist/bundling-sources/shared_libs/plays/authoring-contract.ts +1 -1
  12. package/dist/cli/auth-2ZY57Q7W.js +41 -0
  13. package/dist/cli/{auth-QMANU5PV.mjs → auth-A22724UX.mjs} +5 -5
  14. package/dist/cli/{billing-YJAP4VZN.mjs → billing-IIBEZ6J7.mjs} +6 -6
  15. package/dist/cli/{billing-WZ5POBZX.js → billing-T5Q6OO6M.js} +79 -79
  16. package/dist/cli/{chunk-35SZHTRW.mjs → chunk-2WEPQSA2.mjs} +5 -5
  17. package/dist/cli/{chunk-MXUJVCKV.mjs → chunk-2YHDNHRR.mjs} +4 -4
  18. package/dist/cli/{chunk-V4ILQ2DS.mjs → chunk-3A7TDUZV.mjs} +1 -1
  19. package/dist/cli/{chunk-BB4RZMWA.mjs → chunk-4WMIMSSI.mjs} +5 -5
  20. package/dist/cli/{chunk-ZRWQKTSX.mjs → chunk-4YWYLHED.mjs} +3 -3
  21. package/dist/cli/{chunk-ORQYKA4I.js → chunk-5AEEINTU.js} +4 -4
  22. package/dist/cli/{chunk-3NRZFMX4.js → chunk-5FBCMLJW.js} +6 -6
  23. package/dist/cli/{chunk-XNP7XTBN.js → chunk-5PYSYW63.js} +26 -26
  24. package/dist/cli/{chunk-PGKELBIY.js → chunk-6YZYCKWD.js} +51 -51
  25. package/dist/cli/{chunk-RXITRC37.js → chunk-7HAUVMRF.js} +3 -3
  26. package/dist/cli/{chunk-JCNJQH6X.js → chunk-A6BETJ2A.js} +3 -3
  27. package/dist/cli/{chunk-STLJYWUC.js → chunk-AS3VECCY.js} +7 -7
  28. package/dist/cli/{chunk-LHOEWAVD.js → chunk-B7PEXXRW.js} +53 -53
  29. package/dist/cli/{chunk-IK3VOGDR.js → chunk-CBVCCXIY.js} +21 -21
  30. package/dist/cli/{chunk-JEYX3345.mjs → chunk-CFSCHLNQ.mjs} +3 -3
  31. package/dist/cli/{chunk-3MZPWIOE.mjs → chunk-D7EQ4PI5.mjs} +1 -1
  32. package/dist/cli/{chunk-XP7FAYLN.js → chunk-EBWYNNTJ.js} +5 -5
  33. package/dist/cli/{chunk-DQZBERDL.mjs → chunk-F5PYBZ2H.mjs} +1 -1
  34. package/dist/cli/{chunk-AIPTW3EK.js → chunk-FK7SBMSZ.js} +3 -3
  35. package/dist/cli/{chunk-Z27NSLPC.mjs → chunk-FVPT44IJ.mjs} +1 -1
  36. package/dist/cli/{chunk-RGIQ5ZJX.mjs → chunk-H3GOAZHV.mjs} +2 -2
  37. package/dist/cli/{chunk-AWJKYS25.js → chunk-HSANVBVL.js} +2 -2
  38. package/dist/cli/{chunk-JSXMJSBE.js → chunk-IC3BZJR7.js} +8 -8
  39. package/dist/cli/{chunk-MNK4ZK2C.mjs → chunk-J4FCTFQS.mjs} +10 -10
  40. package/dist/cli/{chunk-MKSB2FTU.mjs → chunk-JU2KBLOR.mjs} +97 -12
  41. package/dist/cli/{chunk-EIKONWW7.mjs → chunk-KCP6XNRR.mjs} +1 -1
  42. package/dist/cli/{chunk-YKNULB7V.js → chunk-KGLIDLJ2.js} +5 -5
  43. package/dist/cli/{chunk-N62MS3XJ.js → chunk-L45IECBZ.js} +6 -6
  44. package/dist/cli/{chunk-7IYOWIEJ.mjs → chunk-LJFIUT5R.mjs} +2 -2
  45. package/dist/cli/{chunk-ZGC2SUAA.mjs → chunk-LZCYDU6Q.mjs} +2 -2
  46. package/dist/cli/{chunk-3ODBCZLL.js → chunk-MYYOCJZ2.js} +93 -93
  47. package/dist/cli/{chunk-DKR7QPEO.js → chunk-MZSLFWOT.js} +1 -1
  48. package/dist/cli/{chunk-26CZVDQN.mjs → chunk-NPNQCPFP.mjs} +2 -2
  49. package/dist/cli/{chunk-UKX4FESF.mjs → chunk-NREO7TT4.mjs} +2 -2
  50. package/dist/cli/{chunk-KKWQ24VM.mjs → chunk-NTJDZURV.mjs} +2 -2
  51. package/dist/cli/{chunk-4C34LGI4.js → chunk-PGNLD6L6.js} +10 -10
  52. package/dist/cli/{chunk-G3UHAYRX.js → chunk-TEJSRJNH.js} +8 -8
  53. package/dist/cli/{chunk-UGYLGT7B.mjs → chunk-UHTJWDYB.mjs} +3 -3
  54. package/dist/cli/{chunk-EK2TAZJZ.js → chunk-ULKGYVYP.js} +2 -2
  55. package/dist/cli/{chunk-DUGZKPRK.mjs → chunk-UYA7LJGC.mjs} +2 -2
  56. package/dist/cli/{chunk-YVJ6IWVD.mjs → chunk-VKBZLV2K.mjs} +1 -1
  57. package/dist/cli/{chunk-YFDTF2XC.js → chunk-VLYUFZUT.js} +9 -9
  58. package/dist/cli/{chunk-THDXVSNL.mjs → chunk-VPAVNEDQ.mjs} +4 -4
  59. package/dist/cli/{chunk-RLQIGBFK.js → chunk-W3BHZUPL.js} +3 -3
  60. package/dist/cli/{chunk-5QS4T4KR.js → chunk-WFVEMDQL.js} +2 -2
  61. package/dist/cli/{chunk-475V572K.js → chunk-WTPRRKZA.js} +131 -46
  62. package/dist/cli/{chunk-CXBEZAPU.js → chunk-YEJDFZJP.js} +16 -16
  63. package/dist/cli/{chunk-JJVYUKLY.mjs → chunk-YQOWMQEW.mjs} +1 -1
  64. package/dist/cli/{chunk-YK64UZXN.mjs → chunk-ZOXV6WFI.mjs} +1 -1
  65. package/dist/cli/{chunk-YX762HFK.mjs → chunk-ZV4OWYLZ.mjs} +2 -2
  66. package/dist/cli/command-docs.js +2 -2
  67. package/dist/cli/command-docs.mjs +1 -1
  68. package/dist/cli/{csv-ENKCFCSM.js → csv-FXOQ2PBE.js} +13 -13
  69. package/dist/cli/{csv-QCLQOIEA.mjs → csv-GPVCETS2.mjs} +4 -4
  70. package/dist/cli/{db-P25LGP33.mjs → db-F5UJTBGP.mjs} +6 -6
  71. package/dist/cli/{db-5TLBZDT3.js → db-LXYV4GPZ.js} +27 -27
  72. package/dist/cli/{describe-W2H5DHR6.mjs → describe-GYWCI7I6.mjs} +17 -17
  73. package/dist/cli/describe-IB7SUIUT.js +48 -0
  74. package/dist/cli/describe-core.d.mts +3 -3
  75. package/dist/cli/describe-core.d.ts +3 -3
  76. package/dist/cli/{executions-VYZCYUBR.mjs → executions-6NL2YW76.mjs} +6 -6
  77. package/dist/cli/{executions-YHFAWJOJ.js → executions-6SJKTY2L.js} +8 -8
  78. package/dist/cli/{feedback-K2FWYYLZ.mjs → feedback-7AMQCNTB.mjs} +4 -4
  79. package/dist/cli/{feedback-RJF3QSBB.js → feedback-E3HULSQD.js} +8 -8
  80. package/dist/cli/{git-S3APUPLL.mjs → git-74IS2EFH.mjs} +5 -5
  81. package/dist/cli/git-A2QIIMW5.js +25 -0
  82. package/dist/cli/index.js +139 -139
  83. package/dist/cli/index.mjs +41 -41
  84. package/dist/cli/{legacy-noops-ICYZACTW.mjs → legacy-noops-52FFRTRZ.mjs} +5 -5
  85. package/dist/cli/{legacy-noops-3SWNRE7Z.js → legacy-noops-ALWUQPKS.js} +8 -8
  86. package/dist/cli/{monitors-DEZCEQLB.mjs → monitors-VTJIWWW2.mjs} +7 -7
  87. package/dist/cli/{monitors-VRIWTRPG.js → monitors-Z4ADOGTY.js} +91 -91
  88. package/dist/cli/{org-NX752DTE.js → org-2BTEKDF5.js} +72 -72
  89. package/dist/cli/{org-IQW5ZIAO.mjs → org-7GU6H7JL.mjs} +6 -6
  90. package/dist/cli/{pages-5WQRVVKX.mjs → pages-I6PFQGEU.mjs} +6 -6
  91. package/dist/cli/{pages-YIBWYGVS.js → pages-MG36E7RY.js} +17 -17
  92. package/dist/cli/{play-WOMJXS4O.mjs → play-7KFSLJX2.mjs} +18 -18
  93. package/dist/cli/{play-3EXNH3HD.js → play-VN52S3RV.js} +321 -321
  94. package/dist/cli/{providers-XDVPYRTS.mjs → providers-D46JDEL2.mjs} +6 -6
  95. package/dist/cli/{providers-C2WYSR2Y.js → providers-W5XLII66.js} +9 -9
  96. package/dist/cli/{quickstart-YMSIL2TK.mjs → quickstart-4GTXBZHY.mjs} +4 -4
  97. package/dist/cli/{quickstart-7RABZF66.js → quickstart-TIZAC63S.js} +6 -6
  98. package/dist/cli/{run-5QVVWFGL.js → run-FOVEMYYX.js} +36 -36
  99. package/dist/cli/{run-A4JD6TAR.mjs → run-LE3D4LVU.mjs} +22 -22
  100. package/dist/cli/{search-2TOSF2G3.mjs → search-6YL6WJMQ.mjs} +7 -7
  101. package/dist/cli/search-HN6SVDQU.js +32 -0
  102. package/dist/cli/{secrets-Q7UVOSNM.js → secrets-CSTTTMZH.js} +15 -15
  103. package/dist/cli/{secrets-BUN4ORE7.mjs → secrets-D66U6RTT.mjs} +6 -6
  104. package/dist/cli/{sessions-NXQOQ4HX.js → sessions-45M2NRAA.js} +6 -6
  105. package/dist/cli/{sessions-IAOORI6X.mjs → sessions-XIO3OUER.mjs} +5 -5
  106. package/dist/cli/{settings-WD4KSY5S.js → settings-N42CVKSK.js} +78 -78
  107. package/dist/cli/{settings-BMI373AI.mjs → settings-R33X2D5T.mjs} +11 -11
  108. package/dist/cli/{setup-55VIWAWR.mjs → setup-CBBC4FQP.mjs} +9 -9
  109. package/dist/cli/{setup-3IA7BVPE.js → setup-HREDVNF2.js} +44 -44
  110. package/dist/cli/{skills-GKITMBTM.mjs → skills-4X6WS2PT.mjs} +9 -7
  111. package/dist/cli/skills-QZ6HZX2T.js +33 -0
  112. package/dist/cli/tools-KFIH7UWB.js +55 -0
  113. package/dist/cli/{tools-XHCFC7GP.mjs → tools-YR46YFR2.mjs} +16 -16
  114. package/dist/cli/{update-SWB24YKE.mjs → update-5CYAZX5X.mjs} +9 -9
  115. package/dist/cli/update-VZLEJLWJ.js +33 -0
  116. package/dist/cli/{workflow-5QPMFI3Z.js → workflow-M7AA722W.js} +17 -17
  117. package/dist/cli/{workflow-XO4CKUA7.mjs → workflow-NKQ3CWPK.mjs} +7 -7
  118. package/dist/{client-Bgl0_zXi.d.mts → client-AjmfA5nR.d.mts} +1 -1
  119. package/dist/{client-_ah6IuhK.d.ts → client-DMyY-QL8.d.ts} +1 -1
  120. package/dist/{client-core-C-ZF5vh9.d.ts → client-core-C705vFyV.d.ts} +1 -1
  121. package/dist/{client-core-Bfsedm0_.d.mts → client-core-CpBbvLEX.d.mts} +1 -1
  122. package/dist/{compiler-manifest-CCOqDGjU.d.mts → compiler-manifest-hqIAukR0.d.mts} +1 -1
  123. package/dist/{compiler-manifest-CCOqDGjU.d.ts → compiler-manifest-hqIAukR0.d.ts} +1 -1
  124. package/dist/{errors-Cy3kPnsi.d.ts → errors-CRO2qQ6k.d.ts} +1 -1
  125. package/dist/{errors-BLccD7tG.d.mts → errors-DIHVetNY.d.mts} +1 -1
  126. package/dist/{index-D5e4h1-H.d.ts → index-B8DVoeYt.d.ts} +1 -1
  127. package/dist/{index-4-j44HuT.d.mts → index-DVZ_X4Gb.d.mts} +1 -1
  128. package/dist/index.d.mts +9 -9
  129. package/dist/index.d.ts +9 -9
  130. package/dist/index.js +9 -9
  131. package/dist/index.mjs +6 -6
  132. package/dist/install-integrity.json +110 -110
  133. package/dist/plays/bundle-play-file.d.mts +3 -3
  134. package/dist/plays/bundle-play-file.d.ts +3 -3
  135. package/dist/plays/bundle-play-file.mjs +1 -1
  136. package/dist/plays/play-description.d.mts +2 -2
  137. package/dist/plays/play-description.d.ts +2 -2
  138. package/dist/plays/run-request.d.mts +2 -2
  139. package/dist/plays/run-request.d.ts +2 -2
  140. package/dist/release.d.mts +3 -3
  141. package/dist/release.d.ts +3 -3
  142. package/dist/release.js +2 -2
  143. package/dist/release.mjs +1 -1
  144. package/dist/server.d.mts +4 -4
  145. package/dist/server.d.ts +4 -4
  146. package/dist/server.js +5 -5
  147. package/dist/server.mjs +3 -3
  148. package/package.json +1 -1
  149. package/dist/cli/auth-O6KAH543.js +0 -41
  150. package/dist/cli/describe-44L7OBGM.js +0 -48
  151. package/dist/cli/git-F3EI5DY5.js +0 -25
  152. package/dist/cli/search-772R7YHI.js +0 -32
  153. package/dist/cli/skills-PUUTFMP3.js +0 -31
  154. package/dist/cli/tools-3WX7TUUG.js +0 -55
  155. package/dist/cli/update-PC3CLODV.js +0 -33
@@ -185,8 +185,8 @@ export type PlayFetchResponse = PlayAuthoringFetchResponse;
185
185
  * The changed row is delivered as the listener event's `after` in the play function's input.
186
186
  *
187
187
  * The default Play runtime is 30 minutes with the `standard` 1 GiB / 1 CPU
188
- * profile. Preview deployments also support `large` (4 GiB / 2 CPU) and
189
- * `xlarge` (8 GiB / 4 CPU) through `runtime.size`. Production uses `standard`.
188
+ * profile. Select `large` (4 GiB / 2 CPU) or `xlarge` (8 GiB / 4 CPU)
189
+ * through `runtime.size` when a step needs more memory.
190
190
  * Compute pricing follows the workspace plan; use `deepline billing quote`
191
191
  * for the session ceiling.
192
192
  * Set `runtime.timeout` for longer bounded runs; duration values are whole
@@ -205,7 +205,7 @@ export const SDK_RELEASE = {
205
205
  // getters keep their established compatibility behavior.
206
206
  // 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
207
207
  // 0.3.90 is the first deliberately versioned SDK release for API v3.
208
- version: '0.3.474',
208
+ version: '0.3.476',
209
209
  updateSummary: {
210
210
  introducedIn: '0.3.90',
211
211
  summary:
@@ -252,9 +252,9 @@ export const SDK_RELEASE = {
252
252
  },
253
253
  {
254
254
  command: 'search',
255
- introducedIn: '0.3.91',
255
+ introducedIn: '0.3.287',
256
256
  reason:
257
- 'deepline search was added in SDK CLI 0.3.91 as the one search across tools, monitors, and plays. On an older version run deepline tools search or deepline plays search, or update.',
257
+ 'deepline search was added in SDK CLI 0.3.287 as the one search across tools, monitors, and plays. On an older version run deepline tools search or deepline plays search, or update.',
258
258
  },
259
259
  {
260
260
  command: 'notifications',
@@ -46,8 +46,9 @@ type ProviderMonitorUpstreamIdentityDefinition = {
46
46
  * Payload fields that never change which findings the provider resource
47
47
  * delivers. `updates_since` only sets how far back the first delivery
48
48
  * replays, so two monitors that differ only there watch the same feed.
49
- * Claim keys written before this list existed still carry these fields;
50
- * readers resolve them through `canonicalProviderMonitorIdentityClaimKey`.
49
+ * New claim keys omit them. Keys stored before that still carry them until
50
+ * the `normalizeProviderMonitorIdentityClaimKeys` migration rewrites them;
51
+ * readers resolve both through `canonicalProviderMonitorIdentityClaimKey`.
51
52
  */
52
53
  nonConfigurationFields?: readonly string[];
53
54
  };
@@ -171,9 +172,11 @@ export function providerMonitorUpstreamIdentitySignature(
171
172
  /**
172
173
  * Exact claim identity used by the Convex conflict index. The provider's
173
174
  * declared scope remains visible in `scopeKey`; providers that allow
174
- * configuration variants add every non-scope payload field to the claim key.
175
- * Keeping the old scope JSON as the prefix lets readers recognize claims
176
- * created before this distinction was introduced.
175
+ * configuration variants add every non-scope payload field to the claim key,
176
+ * except `nonConfigurationFields`. Keeping the old scope JSON as the prefix
177
+ * lets readers recognize claims created before this distinction was
178
+ * introduced. The result is always canonical
179
+ * (`canonicalProviderMonitorIdentityClaimKey` returns it unchanged).
177
180
  */
178
181
  export function providerMonitorUpstreamIdentityClaimKey(
179
182
  tool: string,
@@ -206,9 +209,10 @@ export function providerMonitorUpstreamIdentityClaimKey(
206
209
  : []),
207
210
  ]),
208
211
  );
212
+ const ignored = new Set(definition.nonConfigurationFields ?? []);
209
213
  const configuration = Object.fromEntries(
210
214
  Object.entries(payload ?? {}).filter(
211
- ([field]) => !scopeAndAliases.has(field),
215
+ ([field]) => !scopeAndAliases.has(field) && !ignored.has(field),
212
216
  ),
213
217
  );
214
218
  if (Object.keys(configuration).length === 0) return scopeKey;
@@ -275,6 +279,15 @@ export function canonicalProviderMonitorIdentityClaimKey(
275
279
  return `${parts.scopeKey}${CONFIGURATION_SEPARATOR}${stableJson(configuration)}`;
276
280
  }
277
281
 
282
+ /**
283
+ * The scope part of a stored claim key: the whole key when it is scope-only.
284
+ * Every key for one provider scope starts with this exact JSON object, so it
285
+ * bounds an index range over that scope's claims.
286
+ */
287
+ export function providerMonitorIdentityClaimScopeKey(identityKey: string) {
288
+ return splitClaimKey(identityKey)?.scopeKey ?? identityKey;
289
+ }
290
+
278
291
  /** Whether a claim key carries configuration beyond its scope. */
279
292
  export function providerMonitorIdentityClaimKeyHasConfiguration(
280
293
  identityKey: string,
@@ -241,7 +241,7 @@
241
241
  "id": "github.eval-agent-cli-matrix",
242
242
  "name": "Eval Agent CLI Matrix",
243
243
  "scheduler": "github_actions",
244
- "cadence": "0 7 * * *",
244
+ "cadence": "0 7 * * *; 0 22 * * 4; 0 3 * * 1,3,5",
245
245
  "timezone": "America/New_York",
246
246
  "source": ".github/workflows/eval-agent-cli-matrix.yml",
247
247
  "locator": "schedule:",
@@ -265,9 +265,9 @@
265
265
  "locator": "schedule:",
266
266
  "owner": "platform",
267
267
  "criticality": "maintenance",
268
- "purpose": "Reconcile completed PR Preview Machines and release capacity only after the per-app lease fence allows it. Its named-estate lane plans every named Preview estate (eval reuse keys, runtime up preview branch lanes, one-off namespaces) with `bun scripts/fly-preview-estate-lifecycle.ts --report --named-reclaim-plan` and, only on a workflow_dispatch with reclaim_named_estates=true, scales at most 8 planned estates per pass (interrupted reclaims first, then reclaimable estates idle 24 h by lease, release or reclaim evidence) to zero through the claimed runtime-preview-namespace.yml scale-to-zero, recording the start and lane app before the seam, retiring the release lane and recording reclaimed history; scheduled passes only report it.",
269
- "successDefinition": "Fly inventory is reconciled and eligible terminal/closed PR capacity is released; every named estate is classified without a read failure, and a dispatched named reclaim leaves each target at zero Machines with its release lane retired and reclaimed history written. No eligible app or estate is a valid no-op.",
270
- "failureDefinition": "Inventory, lease resolution, scale-down, or empty-app cleanup fails or is ambiguous; a named-estate Fly or authority read fails; a named reclaim is refused inside its claim (active owner, used since discovery, ownership or lane-app refusal) or any role keeps a Machine; or any named reclaim is incomplete (started but its release lane was never retired), which fails every pass until a dispatch finishes it.",
268
+ "purpose": "Reconcile completed PR Preview Machines and release capacity only after the per-app lease fence allows it. Its named-estate lane plans every named Preview estate (eval reuse keys, runtime up preview branch lanes, one-off namespaces) with `bun scripts/fly-preview-estate-lifecycle.ts --report --named-reclaim-plan` and, only on a workflow_dispatch with reclaim_named_estates=true, scales at most 8 planned estates per pass (interrupted reclaims first, then reclaimable estates idle 24 h by lease, release or reclaim evidence) to zero through the claimed runtime-preview-namespace.yml scale-to-zero, recording the start and lane app before the seam, retiring the release lane and recording reclaimed history; scheduled passes only report it. Each pass also returns the deeplinev1/load PlanetScale branch to its parked size when no Customer Workload Suite run is active on load-tests/framework, in case a load test's own teardown never ran.",
269
+ "successDefinition": "Fly inventory is reconciled and eligible terminal/closed PR capacity is released; every named estate is classified without a read failure, and a dispatched named reclaim leaves each target at zero Machines with its release lane retired and reclaimed history written. No eligible app or estate is a valid no-op. The load database is parked, a change toward that size is in flight, or an active Customer Workload Suite run on load-tests/framework may be holding it.",
270
+ "failureDefinition": "Inventory, lease resolution, scale-down, or empty-app cleanup fails or is ambiguous; a named-estate Fly or authority read fails; a named reclaim is refused inside its claim (active owner, used since discovery, ownership or lane-app refusal) or any role keeps a Machine; or any named reclaim is incomplete (started but its release lane was never retired), which fails every pass until a dispatch finishes it; or reading or parking the load database fails.",
271
271
  "startGraceMs": 600000,
272
272
  "runTimeoutMs": 3600000,
273
273
  "failureAlert": "github_workflow_run",
@@ -790,6 +790,13 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
790
790
  ): void {
791
791
  if (retryAtMs <= (this.localProviderRetryAtMs.get(bucketId) ?? 0)) return;
792
792
  this.localProviderRetryAtMs.set(bucketId, retryAtMs);
793
+ const cadence = this.validatedDispatchCadences.get(bucketId);
794
+ const serverOffsetMs = this.serverClockOffsetsMs.get(bucketId) ?? 0;
795
+ // A new hold fences old tickets and their dispatch debt. Keep debt from
796
+ // sends after an expired hold; their physical spacing is still current.
797
+ if (cadence && cadence.lastAtMs + serverOffsetMs <= retryAtMs) {
798
+ this.validatedDispatchCadences.delete(bucketId);
799
+ }
793
800
  for (const waiting of this.scheduledPermitWaits.get(bucketId) ?? []) {
794
801
  if (waiting.retryAtMsAtGrant < retryAtMs) waiting.invalidate();
795
802
  }
@@ -1626,6 +1633,14 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
1626
1633
  scopeRetryAtMsAtGrant,
1627
1634
  dispatchValidationSupported,
1628
1635
  ] = scheduled;
1636
+ // Legacy grants have no shared hold snapshot. Capture their local fence
1637
+ // when assigned, while keeping the signed gateway proof unchanged.
1638
+ const localRetryAtMsAtGrant = this.validationScopes.has(bucketId)
1639
+ ? scopeRetryAtMsAtGrant
1640
+ : Math.max(
1641
+ scopeRetryAtMsAtGrant,
1642
+ this.localProviderRetryAtMs.get(bucketId) ?? 0,
1643
+ );
1629
1644
  let settled = false;
1630
1645
  const refund = async (): Promise<void> => {
1631
1646
  if (!reservationId) {
@@ -1732,7 +1747,15 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
1732
1747
  });
1733
1748
  }
1734
1749
  if (waitMs > 0) {
1735
- await this.sleep(waitMs, signal);
1750
+ if (
1751
+ !(await this.waitForScheduledPermit(
1752
+ bucketId,
1753
+ localRetryAtMsAtGrant,
1754
+ waitMs,
1755
+ signal,
1756
+ ))
1757
+ )
1758
+ return false;
1736
1759
  continue;
1737
1760
  }
1738
1761
  this.validatedDispatchCadences.set(bucketId, {
@@ -72,11 +72,14 @@ export type PlayRunRecoveryDecision = Readonly<{
72
72
  export class PlayRunRecoveryError extends Error {
73
73
  readonly decision: PlayRunRecoveryDecision;
74
74
  readonly cause?: unknown;
75
+ /** Log-only provider facts; see `playRunRecoveryLogDetail`. */
76
+ readonly recoveryLogDetail?: unknown;
75
77
 
76
78
  constructor(input: {
77
79
  decision: PlayRunRecoveryDecision;
78
80
  message: string;
79
81
  cause?: unknown;
82
+ recoveryLogDetail?: unknown;
80
83
  }) {
81
84
  super(input.message);
82
85
  this.name = PLAY_RUN_RECOVERY_ERROR_NAME;
@@ -84,6 +87,9 @@ export class PlayRunRecoveryError extends Error {
84
87
  if (input.cause !== undefined) {
85
88
  this.cause = input.cause;
86
89
  }
90
+ if (input.recoveryLogDetail !== undefined) {
91
+ this.recoveryLogDetail = input.recoveryLogDetail;
92
+ }
87
93
  }
88
94
  }
89
95
 
@@ -106,7 +112,7 @@ export type PlayRunRecoveryLogDetail = Readonly<
106
112
  >;
107
113
 
108
114
  /**
109
- * Provider facts an adapter attached to the cause of a recovery error, for the
115
+ * Provider facts an adapter attached to a recovery error or its cause, for the
110
116
  * scheduler's defer log only. The decision's `reason` stays the single durable
111
117
  * recovery fact: this detail is never persisted, never returned to a caller,
112
118
  * and never read to choose a recovery action. It is bounded here, at the edge
@@ -119,9 +125,11 @@ export function playRunRecoveryLogDetail(
119
125
  try {
120
126
  if (!isPlayRunRecoveryError(error)) return null;
121
127
  const cause: unknown = error.cause;
122
- if (!cause || typeof cause !== 'object') return null;
123
- const supplied = (cause as { recoveryLogDetail?: unknown })
124
- .recoveryLogDetail;
128
+ const supplied =
129
+ error.recoveryLogDetail ??
130
+ (cause && typeof cause === 'object'
131
+ ? (cause as { recoveryLogDetail?: unknown }).recoveryLogDetail
132
+ : undefined);
125
133
  if (!supplied || typeof supplied !== 'object' || Array.isArray(supplied)) {
126
134
  return null;
127
135
  }
@@ -203,6 +211,14 @@ export function recoveryForModalSandboxCreateFailure(
203
211
  const code = errorCode(error);
204
212
  const status = errorStatus(error);
205
213
  const message = messageOf(error);
214
+ // Only this rejection carries Modal's own words. Once it opens the circuit,
215
+ // later runs defer without calling Modal, and every reason here groups
216
+ // several provider conditions.
217
+ const recoveryLogDetail = {
218
+ providerCode: code,
219
+ providerStatus: status,
220
+ providerMessage: message,
221
+ };
206
222
  const resourceExhausted =
207
223
  code === 8 ||
208
224
  (typeof code === 'string' && /RESOURCE_EXHAUSTED/i.test(code)) ||
@@ -218,6 +234,7 @@ export function recoveryForModalSandboxCreateFailure(
218
234
  message:
219
235
  'Modal sandbox capacity is unavailable before customer execution began.',
220
236
  cause: error,
237
+ recoveryLogDetail,
221
238
  });
222
239
  }
223
240
 
@@ -56,8 +56,20 @@ export const RUNTIME_CAPACITY_POLICY = {
56
56
  maxScaleDownMachinesPerReconcile: 2,
57
57
  /** Empty relay capacity must be quiet before it is reclaimed. */
58
58
  scaleDownQuietMs: 120_000,
59
+ /**
60
+ * Pooled Postgres clients per relay Machine when
61
+ * DEEPLINE_RUNTIME_GATEWAY_POOL_MAX_CLIENTS sets none, as in production.
62
+ * packages/runtime-gateway/server.ts explains the budget.
63
+ */
64
+ poolMaxClients: 6,
59
65
  },
60
66
  receiptGateway: {
67
+ /**
68
+ * Pooled Postgres clients per receipt-gateway Machine when
69
+ * DEEPLINE_RUNTIME_GATEWAY_POOL_MAX_CLIENTS sets none, as in production.
70
+ * packages/runtime-gateway/server.ts explains the budget.
71
+ */
72
+ poolMaxClients: 8,
61
73
  admissionTimeoutMs: 10_000,
62
74
  /** End identical receipt-row work before the caller can overlap a retry. */
63
75
  claimOperationTimeoutMs: 20_000,
@@ -72,35 +72,57 @@ export function acquireRateScopeSql(
72
72
  THEN CASE WHEN ${productiveRecovery} THEN ${AIMD_RECOVERY_STEP_MS}
73
73
  ELSE ${sparseRecoveryInterval} END
74
74
  ELSE LEAST(${floorRecoveryInterval}, CEIL(1000.0 / ${refillRate})) END`
75
- : `LEAST(h.rate_recovery_interval_ms, ${AIMD_RECOVERY_STEP_MS})`;
75
+ : `${AIMD_RECOVERY_STEP_MS}`;
76
76
  // A local no-progress pause remains active after the provider's shorter
77
77
  // Retry-After. Only time beyond every outstanding dispatch/hold is idle.
78
78
  const observationWasIdle = `${successesParameter}::int = 0
79
79
  AND nv.now_ms - GREATEST(h.refilled_at_ms, h.next_dispatch_at_ms, h.retry_at_ms, h.cool_until_ms)
80
80
  > GREATEST(1000, 2000.0 / ${refillRate})`;
81
- // A denied refill may fold a completion before the next floor interval is
82
- // due. Keep that fresh completion available for one later ramp rather than
83
- // requiring another provider call at the old pace. Its baseline advances on
84
- // every floor ramp, so time alone cannot reuse it for another increase.
85
- const recoverySuccess =
86
- unknownProvider && probeBaselineSupported
87
- ? `(${successesParameter}::int > 0 OR (
81
+ // A denied refill may fold a completion before the next recovery interval.
82
+ // Keep it available for one later ramp rather than requiring another call at
83
+ // the old pace. The baseline consumes it once. Unknown-provider upward probes
84
+ // above useful seed capacity retain their separate observation policy.
85
+ const recoverySuccess = probeBaselineSupported
86
+ ? `(${successesParameter}::int > 0 OR (
88
87
  h.seen_backpressure
89
- AND ${refillRate} < ${DEFAULT_UNKNOWN_PROVIDER_FAST_RECOVERY_RPS}
88
+ AND (${!unknownProvider} OR ${refillRate} < ${DEFAULT_UNKNOWN_PROVIDER_FAST_RECOVERY_RPS})
90
89
  AND h.rate_probe_started_at_ms >= h.last_ramp_at_ms
91
90
  AND h.rate_probe_started_at_ms >= h.rate_observation_started_at_ms
92
91
  AND h.rate_successes > h.rate_probe_successes_at_ramp
93
92
  AND NOT (${observationWasIdle})
94
93
  ))`
95
- : `${successesParameter}::int > 0`;
94
+ : `${successesParameter}::int > 0`;
95
+ // Known-provider feedback can arrive before its recovery interval. Wake at
96
+ // that opportunity instead of sleeping an entire old, slow token interval;
97
+ // the baseline below consumes each confirmed success for at most one ramp.
98
+ const pendingRecoveryWaitMs =
99
+ !unknownProvider && probeBaselineSupported
100
+ ? `CASE WHEN ${recoverySuccess}
101
+ AND h.seen_backpressure
102
+ AND h.rate_observation_started_at_ms IS NOT NULL
103
+ AND GREATEST(h.retry_at_ms, h.cool_until_ms) <= nv.now_ms
104
+ AND nv.now_ms < h.last_ramp_at_ms + ${backpressuredRecoveryInterval}
105
+ THEN h.last_ramp_at_ms + ${backpressuredRecoveryInterval} - nv.now_ms
106
+ ELSE NULL END`
107
+ : 'NULL::bigint';
108
+ // A provider's elapsed hold authorizes one shared recovery probe. Requiring
109
+ // a whole token at the newly learned rate would add another wait when the
110
+ // measured quota cycle is longer than Retry-After itself. The first grant
111
+ // starts an observation epoch, so competing callers cannot reuse this floor.
112
+ const postHoldProbeDue = `h.seen_backpressure
113
+ AND h.rate_observation_started_at_ms IS NULL
114
+ AND h.retry_at_ms > 0
115
+ AND GREATEST(h.retry_at_ms, h.cool_until_ms) <= nv.now_ms`;
96
116
  return `provider_hold_rows AS (
97
117
  SELECT * FROM ${table} WHERE bucket_id = $1 FOR UPDATE
98
118
  ), provider_hold AS (
99
119
  SELECT h.*,
100
- LEAST(
120
+ ${postHoldProbeDue} AS post_hold_probe_due,
121
+ ${pendingRecoveryWaitMs} AS pending_recovery_wait_ms,
122
+ GREATEST(CASE WHEN ${postHoldProbeDue} THEN 1.0 ELSE 0.0 END, LEAST(
101
123
  GREATEST(CASE WHEN h.seen_backpressure THEN 1 ELSE h.capacity END, ${refillRate}),
102
124
  h.tokens + GREATEST(0, nv.now_ms - h.refilled_at_ms) * ${refillRate} / 1000.0
103
- ) AS tokens_after_refill,
125
+ )) AS tokens_after_refill,
104
126
  ${observationWasIdle} AS observation_was_idle,
105
127
  CASE WHEN ${recoverySuccess}
106
128
  AND h.rate_observation_started_at_ms IS NOT NULL
@@ -140,7 +162,9 @@ export function updateRateScopeSql(
140
162
  const floorRampedObservation = `${unknownProvider} AND ph.seen_backpressure
141
163
  AND ph.refill_per_sec < ${DEFAULT_UNKNOWN_PROVIDER_FAST_RECOVERY_RPS}
142
164
  AND ph.next_refill_per_sec > ph.refill_per_sec`;
143
- const idleFloorObservation = `ph.observation_was_idle
165
+ const knownRampedObservation = `${!unknownProvider} AND ph.seen_backpressure
166
+ AND ph.next_refill_per_sec > ph.refill_per_sec`;
167
+ const idleFloorObservation = `${unknownProvider} AND ph.observation_was_idle
144
168
  AND ph.refill_per_sec < ${DEFAULT_UNKNOWN_PROVIDER_FAST_RECOVERY_RPS}`;
145
169
  const rebaseProbe =
146
170
  'ph.rate_probe_started_at_ms < ph.last_ramp_at_ms OR ph.rate_probe_started_at_ms < COALESCE(ph.rate_observation_started_at_ms, 0) OR ph.rate_probe_successes_at_ramp > ph.rate_successes';
@@ -163,28 +187,30 @@ export function updateRateScopeSql(
163
187
  WHEN (SELECT granted FROM grant_calc) > 0 AND ph.observation_was_idle THEN 0
164
188
  ELSE ph.rate_successes END,
165
189
  ${
166
- unknownProvider && probeBaselineSupported
190
+ probeBaselineSupported
167
191
  ? `rate_probe_successes_at_ramp = CASE
168
192
  WHEN ${recoveredObservation} THEN 0
169
193
  WHEN (SELECT granted FROM grant_calc) > 0
170
194
  AND (ph.rate_observation_started_at_ms IS NULL OR ph.observation_was_idle) THEN 0
171
195
  WHEN ${idleFloorObservation} THEN ph.rate_successes
172
- WHEN ${floorRampedObservation} OR ${probedObservation} OR (${rebaseProbe}) THEN ph.rate_successes
196
+ WHEN ${knownRampedObservation} OR ${floorRampedObservation} OR ${probedObservation} OR (${rebaseProbe}) THEN ph.rate_successes
173
197
  ELSE ph.rate_probe_successes_at_ramp END,
174
198
  rate_probe_started_at_ms = CASE
175
- WHEN ${idleFloorObservation} OR ${recoveredObservation} OR ${floorRampedObservation} OR ${probedObservation} OR (${rebaseProbe}) THEN nv.now_ms
199
+ WHEN ${knownRampedObservation} OR ${idleFloorObservation} OR ${recoveredObservation} OR ${floorRampedObservation} OR ${probedObservation} OR (${rebaseProbe}) THEN nv.now_ms
176
200
  WHEN (SELECT granted FROM grant_calc) > 0
177
201
  AND (ph.rate_observation_started_at_ms IS NULL OR ph.observation_was_idle) THEN nv.now_ms
178
202
  ELSE ph.rate_probe_started_at_ms END,`
179
203
  : ''
180
204
  }
181
- -- At the floor, a new successful epoch must not inherit failed-probe
182
- -- backoff. Above the floor, retain its useful measurement duration for
183
- -- sparse upward probes; that rate already proves successful recovery.
205
+ -- A productive known-provider epoch uses the one-second recovery window,
206
+ -- so its old measurement must not become a failed-probe streak when the
207
+ -- success counter resets. Unknown providers above the floor retain that
208
+ -- measurement for their separate sparse upward-probe policy.
184
209
  rate_recovery_interval_ms = CASE
185
- WHEN ${unknownProvider} AND (SELECT granted FROM grant_calc) > 0
210
+ WHEN (SELECT granted FROM grant_calc) > 0
186
211
  AND ph.rate_successes > 0
187
- AND ph.next_refill_per_sec <= ${DEFAULT_UNKNOWN_PROVIDER_MIN_RPS}
212
+ AND (${!unknownProvider}
213
+ OR ph.next_refill_per_sec <= ${DEFAULT_UNKNOWN_PROVIDER_MIN_RPS})
188
214
  AND (ph.observation_was_idle
189
215
  OR ph.rate_observation_started_at_ms IS NULL)
190
216
  -- Zero is an unmeasured new episode, not a failed-probe interval.
@@ -1444,7 +1444,12 @@ async function acquirePureRate(
1444
1444
  LEAST(
1445
1445
  ${effectiveCapacitySql('normalized', 'refill_per_sec', unknownProvider)},
1446
1446
  ph.tokens_after_refill,
1447
- LEAST(b.tokens, sc.capacity)
1447
+ GREATEST(CASE WHEN ph.post_hold_probe_due
1448
+ AND LEAST(b.tokens, sc.capacity)
1449
+ + GREATEST(0, nv.now_ms - b.refilled_at_ms) / 1000.0
1450
+ * CASE WHEN sc.adaptive_max_configured
1451
+ THEN sc.max_rps ELSE b.max_rps END >= 1
1452
+ THEN 1.0 ELSE 0.0 END, LEAST(b.tokens, sc.capacity)
1448
1453
  + GREATEST(0, nv.now_ms - b.refilled_at_ms) / 1000.0
1449
1454
  * LEAST(
1450
1455
  CASE
@@ -1452,7 +1457,7 @@ async function acquirePureRate(
1452
1457
  ELSE b.max_rps
1453
1458
  END,
1454
1459
  GREATEST(sc.min_rps, ph.refill_per_sec)
1455
- )
1460
+ ))
1456
1461
  ) AS tokens_after_refill
1457
1462
  FROM ${bucketTableName(input)} b
1458
1463
  JOIN seed_config sc ON sc.rule_id = b.rule_id
@@ -1683,7 +1688,7 @@ async function acquirePureRate(
1683
1688
  (SELECT cool_until_ms FROM cooling),
1684
1689
  (SELECT retry_at_ms FROM cooling)
1685
1690
  ) - (SELECT now_ms FROM refreshed LIMIT 1)${fair?.waitBranch ?? ''}
1686
- ELSE (
1691
+ ELSE LEAST((SELECT pending_recovery_wait_ms FROM provider_hold), (
1687
1692
  SELECT CEIL(
1688
1693
  MAX(
1689
1694
  (
@@ -1699,7 +1704,7 @@ async function acquirePureRate(
1699
1704
  )
1700
1705
  )::bigint
1701
1706
  FROM updated
1702
- )
1707
+ ))
1703
1708
  END AS wait_ms
1704
1709
  `,
1705
1710
  [
@@ -1964,7 +1969,12 @@ async function grantWithConcurrency(
1964
1969
  LEAST(
1965
1970
  ${effectiveCapacitySql('normalized', 'refill_per_sec', unknownProvider)},
1966
1971
  ph.tokens_after_refill,
1967
- LEAST(b.tokens, sc.capacity)
1972
+ GREATEST(CASE WHEN ph.post_hold_probe_due
1973
+ AND LEAST(b.tokens, sc.capacity)
1974
+ + GREATEST(0, nv.now_ms - b.refilled_at_ms) / 1000.0
1975
+ * CASE WHEN sc.adaptive_max_configured
1976
+ THEN sc.max_rps ELSE b.max_rps END >= 1
1977
+ THEN 1.0 ELSE 0.0 END, LEAST(b.tokens, sc.capacity)
1968
1978
  + GREATEST(0, nv.now_ms - b.refilled_at_ms) / 1000.0
1969
1979
  * LEAST(
1970
1980
  CASE
@@ -1972,7 +1982,7 @@ async function grantWithConcurrency(
1972
1982
  ELSE b.max_rps
1973
1983
  END,
1974
1984
  GREATEST(sc.min_rps, ph.refill_per_sec)
1975
- )
1985
+ ))
1976
1986
  ) AS tokens_after_refill
1977
1987
  FROM ${bucketTableName(input)} b
1978
1988
  JOIN seed_config sc ON sc.rule_id = b.rule_id
@@ -2306,7 +2316,7 @@ async function grantWithConcurrency(
2306
2316
  (SELECT retry_at_ms FROM cooling)
2307
2317
  ) - (SELECT now_ms FROM refreshed LIMIT 1)${fair?.waitBranch ?? ''}
2308
2318
  WHEN (SELECT free FROM free_slots) <= 0 THEN $7::bigint
2309
- ELSE (
2319
+ ELSE LEAST((SELECT pending_recovery_wait_ms FROM provider_hold), (
2310
2320
  SELECT CEIL(
2311
2321
  MAX(
2312
2322
  (
@@ -2323,7 +2333,7 @@ async function grantWithConcurrency(
2323
2333
  )
2324
2334
  )::bigint
2325
2335
  FROM updated
2326
- )
2336
+ ))
2327
2337
  END AS wait_ms
2328
2338
  `,
2329
2339
  [