deepline 0.3.475 → 0.3.477

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (162) hide show
  1. package/dist/bundling-sources/sdk/src/play.ts +2 -2
  2. package/dist/bundling-sources/sdk/src/release.ts +3 -3
  3. package/dist/bundling-sources/shared_libs/monitors/provider-monitor-upstream-identity.ts +19 -6
  4. package/dist/bundling-sources/shared_libs/observability/scheduled-jobs.json +4 -4
  5. package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +33 -0
  6. package/dist/bundling-sources/shared_libs/play-runtime/governor/app-runtime-rate-state-backend.ts +90 -1
  7. package/dist/bundling-sources/shared_libs/play-runtime/governor/governor.ts +1 -0
  8. package/dist/bundling-sources/shared_libs/play-runtime/governor/rate-state-backend.ts +3 -0
  9. package/dist/bundling-sources/shared_libs/play-runtime/play-run-recovery-policy.ts +21 -4
  10. package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts +6 -0
  11. package/dist/bundling-sources/shared_libs/play-runtime/runner-app/index.ts +13 -0
  12. package/dist/bundling-sources/shared_libs/play-runtime/runtime-capacity-policy.ts +12 -0
  13. package/dist/bundling-sources/shared_libs/play-runtime/sandbox-compute-usage.ts +6 -0
  14. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-rate-scope.ts +8 -6
  15. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-rate-state.ts +167 -7
  16. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts +104 -6
  17. package/dist/bundling-sources/shared_libs/play-runtime/worker-api-types.ts +42 -0
  18. package/dist/bundling-sources/shared_libs/plays/authoring-contract.ts +1 -1
  19. package/dist/cli/auth-QITP34OX.js +41 -0
  20. package/dist/cli/{auth-CFDYJ7FZ.mjs → auth-XVXRRWIQ.mjs} +5 -5
  21. package/dist/cli/{billing-XLIL3FLK.js → billing-2H7GFJDT.js} +79 -79
  22. package/dist/cli/{billing-PEXWWLRT.mjs → billing-EOPUJ4JI.mjs} +6 -6
  23. package/dist/cli/{chunk-TLPW2SHS.js → chunk-23EEVP5W.js} +6 -6
  24. package/dist/cli/{chunk-ZQYBKDPA.js → chunk-255O6APS.js} +2 -2
  25. package/dist/cli/{chunk-D6SXN34W.mjs → chunk-2GZFIWSA.mjs} +4 -4
  26. package/dist/cli/{chunk-M7K5BCWP.js → chunk-2QT6KN54.js} +10 -10
  27. package/dist/cli/{chunk-MUDOASYL.mjs → chunk-2YNLW43I.mjs} +2 -2
  28. package/dist/cli/{chunk-BSK2TGB4.js → chunk-347EE4YE.js} +8 -8
  29. package/dist/cli/{chunk-V4ILQ2DS.mjs → chunk-3A7TDUZV.mjs} +1 -1
  30. package/dist/cli/{chunk-G4SGYWKV.js → chunk-3UKKM6OS.js} +6 -6
  31. package/dist/cli/{chunk-NBVYS6VG.js → chunk-3W6E3WVU.js} +9 -9
  32. package/dist/cli/{chunk-F4FNMXI3.js → chunk-4RAENEAX.js} +53 -53
  33. package/dist/cli/{chunk-42F6LTBJ.mjs → chunk-6IJJCRE5.mjs} +3 -3
  34. package/dist/cli/{chunk-CKSRTD45.js → chunk-6KKDIISC.js} +3 -3
  35. package/dist/cli/{chunk-7SWRXZ4A.js → chunk-75NLWBYQ.js} +93 -93
  36. package/dist/cli/{chunk-KTWXY73Q.js → chunk-7CILZKLE.js} +5 -5
  37. package/dist/cli/{chunk-B7NJ5JQ2.js → chunk-ACDXGOXE.js} +4 -4
  38. package/dist/cli/{chunk-FC4C7MZM.mjs → chunk-C2AN2HTB.mjs} +4 -4
  39. package/dist/cli/{chunk-EA45HSEC.mjs → chunk-C2V2HQDW.mjs} +2 -2
  40. package/dist/cli/{chunk-YMS7VYFA.mjs → chunk-C46DWB56.mjs} +1 -1
  41. package/dist/cli/{chunk-3MQJJ6YX.js → chunk-CH4VMDB5.js} +51 -51
  42. package/dist/cli/{chunk-7U3L2S7X.mjs → chunk-DFKHXZQ7.mjs} +10 -10
  43. package/dist/cli/{chunk-VBGPRRJ6.js → chunk-DMWHEYZM.js} +2 -2
  44. package/dist/cli/{chunk-3WVA7E7O.mjs → chunk-DS3DZUES.mjs} +2 -2
  45. package/dist/cli/{chunk-Z5VSBCFV.js → chunk-E2CUSH56.js} +16 -16
  46. package/dist/cli/{chunk-AD7KL6AI.mjs → chunk-ISEUOYCB.mjs} +97 -12
  47. package/dist/cli/{chunk-VFP3MFLD.mjs → chunk-JS4UO6G4.mjs} +1 -1
  48. package/dist/cli/{chunk-HMHH7JQD.mjs → chunk-KMXBZIKO.mjs} +2 -2
  49. package/dist/cli/{chunk-M4QO3GRD.js → chunk-KWQNNPMH.js} +3 -3
  50. package/dist/cli/{chunk-3QSFSHCJ.mjs → chunk-LJYTYNQZ.mjs} +1 -1
  51. package/dist/cli/{chunk-TZRTT7EI.js → chunk-LXKGZKTL.js} +26 -26
  52. package/dist/cli/{chunk-U3F63UWD.js → chunk-M2UL4ZS5.js} +7 -7
  53. package/dist/cli/{chunk-DKR7QPEO.js → chunk-MZSLFWOT.js} +1 -1
  54. package/dist/cli/{chunk-BLW2K6YH.js → chunk-NC52JG66.js} +3 -3
  55. package/dist/cli/{chunk-HALAYT3Y.mjs → chunk-NJMYX2TP.mjs} +3 -3
  56. package/dist/cli/{chunk-NHQP4UPJ.mjs → chunk-NO7ZC2GV.mjs} +5 -5
  57. package/dist/cli/{chunk-YAAQNDP6.mjs → chunk-PQQNMMWM.mjs} +3 -3
  58. package/dist/cli/{chunk-BVAYLXMH.mjs → chunk-QODW5ILA.mjs} +2 -2
  59. package/dist/cli/{chunk-QKMWQTMI.js → chunk-R5FNCFEK.js} +3 -3
  60. package/dist/cli/{chunk-7XTBEYLS.js → chunk-RGNMMDSJ.js} +131 -46
  61. package/dist/cli/{chunk-6PRYHABU.mjs → chunk-RHIFH3GD.mjs} +1 -1
  62. package/dist/cli/{chunk-4QHAP3MR.mjs → chunk-RLCAECZH.mjs} +5 -5
  63. package/dist/cli/{chunk-DIMKSDF6.mjs → chunk-SG2SZQ4K.mjs} +2 -2
  64. package/dist/cli/{chunk-VTCXSICG.mjs → chunk-SVU2VR6Q.mjs} +2 -2
  65. package/dist/cli/{chunk-D3Q4FUFS.js → chunk-THMGJOR4.js} +8 -8
  66. package/dist/cli/{chunk-4QHQ3WYW.js → chunk-UBNS6GSV.js} +21 -21
  67. package/dist/cli/{chunk-DUGZKPRK.mjs → chunk-UYA7LJGC.mjs} +2 -2
  68. package/dist/cli/{chunk-5QS4T4KR.js → chunk-WFVEMDQL.js} +2 -2
  69. package/dist/cli/{chunk-M27SSAQ7.mjs → chunk-WRHLAAGN.mjs} +1 -1
  70. package/dist/cli/{chunk-VD3E3QMI.mjs → chunk-XQLYPHYW.mjs} +1 -1
  71. package/dist/cli/{chunk-NGOKBSBH.mjs → chunk-YZA5UJXM.mjs} +1 -1
  72. package/dist/cli/{chunk-ZANUM4XV.js → chunk-ZDPL5AIK.js} +5 -5
  73. package/dist/cli/command-docs.js +2 -2
  74. package/dist/cli/command-docs.mjs +1 -1
  75. package/dist/cli/{csv-4YHSYV2C.js → csv-ECB7TYR2.js} +13 -13
  76. package/dist/cli/{csv-WC7ILC7P.mjs → csv-IDRPTSSC.mjs} +4 -4
  77. package/dist/cli/{db-I5PV6NTC.mjs → db-FOBZD44N.mjs} +6 -6
  78. package/dist/cli/{db-APZNM2GI.js → db-S74XZZ77.js} +27 -27
  79. package/dist/cli/{describe-XH5F3X5W.mjs → describe-KMTPPI2J.mjs} +17 -17
  80. package/dist/cli/describe-PI7HCT7K.js +48 -0
  81. package/dist/cli/describe-core.d.mts +3 -3
  82. package/dist/cli/describe-core.d.ts +3 -3
  83. package/dist/cli/{executions-PIY4XM7T.mjs → executions-4JAIVE2E.mjs} +6 -6
  84. package/dist/cli/{executions-VIXFKZYU.js → executions-ZE3JMNA4.js} +8 -8
  85. package/dist/cli/{feedback-IGDHBWGT.mjs → feedback-EUX5PDBA.mjs} +4 -4
  86. package/dist/cli/{feedback-WCLIFQTR.js → feedback-W6VA47A3.js} +8 -8
  87. package/dist/cli/git-NDOGCQZY.js +25 -0
  88. package/dist/cli/{git-FRICPPTA.mjs → git-UOVG4PLH.mjs} +5 -5
  89. package/dist/cli/index.js +139 -139
  90. package/dist/cli/index.mjs +41 -41
  91. package/dist/cli/{legacy-noops-DSFOPYDY.js → legacy-noops-3H35EH5Y.js} +8 -8
  92. package/dist/cli/{legacy-noops-ROAXEG6D.mjs → legacy-noops-ZBWUNFWV.mjs} +5 -5
  93. package/dist/cli/{monitors-FZ5PRLW6.mjs → monitors-AYSGSUAK.mjs} +7 -7
  94. package/dist/cli/{monitors-KUCYQLU5.js → monitors-PI3KPLR2.js} +91 -91
  95. package/dist/cli/{org-DRNND5XU.mjs → org-2BIG7U4V.mjs} +6 -6
  96. package/dist/cli/{org-SRHMWKQR.js → org-MVS3Y6BK.js} +72 -72
  97. package/dist/cli/{pages-XNGA356T.mjs → pages-4TRRL24G.mjs} +6 -6
  98. package/dist/cli/{pages-TEJQIHTU.js → pages-VB6FOVDE.js} +17 -17
  99. package/dist/cli/{play-GCGFQSFP.js → play-BLT6IT2O.js} +340 -323
  100. package/dist/cli/{play-NE45ROIZ.mjs → play-JR3VUOZM.mjs} +37 -20
  101. package/dist/cli/{providers-57TJOSMV.mjs → providers-547DPK4S.mjs} +6 -6
  102. package/dist/cli/{providers-VHMUR63R.js → providers-BBXIAGW4.js} +9 -9
  103. package/dist/cli/{quickstart-K3SH4NXP.mjs → quickstart-NTLXD7HS.mjs} +4 -4
  104. package/dist/cli/{quickstart-BPXWO4UD.js → quickstart-RNPI5B6J.js} +6 -6
  105. package/dist/cli/{run-553AAXDU.js → run-BCCEXBOG.js} +36 -36
  106. package/dist/cli/{run-ZRWJUEN2.mjs → run-QZ4SFECD.mjs} +22 -22
  107. package/dist/cli/{search-BW6XRS4I.mjs → search-PYNELH4C.mjs} +7 -7
  108. package/dist/cli/search-YHEICYQL.js +32 -0
  109. package/dist/cli/{secrets-OYYCFGZE.mjs → secrets-O5WGREQ7.mjs} +6 -6
  110. package/dist/cli/{secrets-AMYQYUL5.js → secrets-VD35LMYJ.js} +15 -15
  111. package/dist/cli/{sessions-O4SVVOMO.mjs → sessions-2TL6Z5J5.mjs} +5 -5
  112. package/dist/cli/{sessions-RYCJUMJ2.js → sessions-NGGIBYXF.js} +6 -6
  113. package/dist/cli/{settings-QK7DHDU7.mjs → settings-6KM74AJT.mjs} +11 -11
  114. package/dist/cli/{settings-7R5FQKII.js → settings-ERFNZRHO.js} +78 -78
  115. package/dist/cli/{setup-OTQDQHDN.js → setup-6X7EJ6RG.js} +44 -44
  116. package/dist/cli/{setup-VDAAVCKK.mjs → setup-Y3BN6XRS.mjs} +9 -9
  117. package/dist/cli/skills-F7MPYC4P.js +33 -0
  118. package/dist/cli/{skills-HRK5O5JO.mjs → skills-OUBQASHI.mjs} +9 -7
  119. package/dist/cli/{tools-O7BM62QV.mjs → tools-EELZ5FF5.mjs} +16 -16
  120. package/dist/cli/tools-MCWDAK53.js +55 -0
  121. package/dist/cli/update-G5BRJOVT.js +33 -0
  122. package/dist/cli/{update-UHKHCSTY.mjs → update-TX7SW32Y.mjs} +9 -9
  123. package/dist/cli/{workflow-7Z5NCTGG.js → workflow-5AI2BJA7.js} +17 -17
  124. package/dist/cli/{workflow-T2EDR7L4.mjs → workflow-PUNQG7LG.mjs} +7 -7
  125. package/dist/{client-Bgl0_zXi.d.mts → client-AjmfA5nR.d.mts} +1 -1
  126. package/dist/{client-_ah6IuhK.d.ts → client-DMyY-QL8.d.ts} +1 -1
  127. package/dist/{client-core-C-ZF5vh9.d.ts → client-core-C705vFyV.d.ts} +1 -1
  128. package/dist/{client-core-Bfsedm0_.d.mts → client-core-CpBbvLEX.d.mts} +1 -1
  129. package/dist/{compiler-manifest-CCOqDGjU.d.mts → compiler-manifest-hqIAukR0.d.mts} +1 -1
  130. package/dist/{compiler-manifest-CCOqDGjU.d.ts → compiler-manifest-hqIAukR0.d.ts} +1 -1
  131. package/dist/{errors-Cy3kPnsi.d.ts → errors-CRO2qQ6k.d.ts} +1 -1
  132. package/dist/{errors-BLccD7tG.d.mts → errors-DIHVetNY.d.mts} +1 -1
  133. package/dist/{index-D5e4h1-H.d.ts → index-B8DVoeYt.d.ts} +1 -1
  134. package/dist/{index-4-j44HuT.d.mts → index-DVZ_X4Gb.d.mts} +1 -1
  135. package/dist/index.d.mts +9 -9
  136. package/dist/index.d.ts +9 -9
  137. package/dist/index.js +9 -9
  138. package/dist/index.mjs +6 -6
  139. package/dist/install-integrity.json +110 -110
  140. package/dist/plays/bundle-play-file.d.mts +3 -3
  141. package/dist/plays/bundle-play-file.d.ts +3 -3
  142. package/dist/plays/bundle-play-file.mjs +1 -1
  143. package/dist/plays/play-description.d.mts +2 -2
  144. package/dist/plays/play-description.d.ts +2 -2
  145. package/dist/plays/run-request.d.mts +2 -2
  146. package/dist/plays/run-request.d.ts +2 -2
  147. package/dist/release.d.mts +3 -3
  148. package/dist/release.d.ts +3 -3
  149. package/dist/release.js +2 -2
  150. package/dist/release.mjs +1 -1
  151. package/dist/server.d.mts +4 -4
  152. package/dist/server.d.ts +4 -4
  153. package/dist/server.js +5 -5
  154. package/dist/server.mjs +3 -3
  155. package/package.json +1 -1
  156. package/dist/cli/auth-7GVCW24N.js +0 -41
  157. package/dist/cli/describe-OR7FZBCD.js +0 -48
  158. package/dist/cli/git-CKN2F32Y.js +0 -25
  159. package/dist/cli/search-WF26AUDJ.js +0 -32
  160. package/dist/cli/skills-VNLMEOXW.js +0 -31
  161. package/dist/cli/tools-C3MUHL7G.js +0 -55
  162. package/dist/cli/update-TCKTU6QJ.js +0 -33
@@ -185,8 +185,8 @@ export type PlayFetchResponse = PlayAuthoringFetchResponse;
185
185
  * The changed row is delivered as the listener event's `after` in the play function's input.
186
186
  *
187
187
  * The default Play runtime is 30 minutes with the `standard` 1 GiB / 1 CPU
188
- * profile. Preview deployments also support `large` (4 GiB / 2 CPU) and
189
- * `xlarge` (8 GiB / 4 CPU) through `runtime.size`. Production uses `standard`.
188
+ * profile. Select `large` (4 GiB / 2 CPU) or `xlarge` (8 GiB / 4 CPU)
189
+ * through `runtime.size` when a step needs more memory.
190
190
  * Compute pricing follows the workspace plan; use `deepline billing quote`
191
191
  * for the session ceiling.
192
192
  * Set `runtime.timeout` for longer bounded runs; duration values are whole
@@ -205,7 +205,7 @@ export const SDK_RELEASE = {
205
205
  // getters keep their established compatibility behavior.
206
206
  // 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
207
207
  // 0.3.90 is the first deliberately versioned SDK release for API v3.
208
- version: '0.3.475',
208
+ version: '0.3.477',
209
209
  updateSummary: {
210
210
  introducedIn: '0.3.90',
211
211
  summary:
@@ -252,9 +252,9 @@ export const SDK_RELEASE = {
252
252
  },
253
253
  {
254
254
  command: 'search',
255
- introducedIn: '0.3.91',
255
+ introducedIn: '0.3.287',
256
256
  reason:
257
- 'deepline search was added in SDK CLI 0.3.91 as the one search across tools, monitors, and plays. On an older version run deepline tools search or deepline plays search, or update.',
257
+ 'deepline search was added in SDK CLI 0.3.287 as the one search across tools, monitors, and plays. On an older version run deepline tools search or deepline plays search, or update.',
258
258
  },
259
259
  {
260
260
  command: 'notifications',
@@ -46,8 +46,9 @@ type ProviderMonitorUpstreamIdentityDefinition = {
46
46
  * Payload fields that never change which findings the provider resource
47
47
  * delivers. `updates_since` only sets how far back the first delivery
48
48
  * replays, so two monitors that differ only there watch the same feed.
49
- * Claim keys written before this list existed still carry these fields;
50
- * readers resolve them through `canonicalProviderMonitorIdentityClaimKey`.
49
+ * New claim keys omit them. Keys stored before that still carry them until
50
+ * the `normalizeProviderMonitorIdentityClaimKeys` migration rewrites them;
51
+ * readers resolve both through `canonicalProviderMonitorIdentityClaimKey`.
51
52
  */
52
53
  nonConfigurationFields?: readonly string[];
53
54
  };
@@ -171,9 +172,11 @@ export function providerMonitorUpstreamIdentitySignature(
171
172
  /**
172
173
  * Exact claim identity used by the Convex conflict index. The provider's
173
174
  * declared scope remains visible in `scopeKey`; providers that allow
174
- * configuration variants add every non-scope payload field to the claim key.
175
- * Keeping the old scope JSON as the prefix lets readers recognize claims
176
- * created before this distinction was introduced.
175
+ * configuration variants add every non-scope payload field to the claim key,
176
+ * except `nonConfigurationFields`. Keeping the old scope JSON as the prefix
177
+ * lets readers recognize claims created before this distinction was
178
+ * introduced. The result is always canonical
179
+ * (`canonicalProviderMonitorIdentityClaimKey` returns it unchanged).
177
180
  */
178
181
  export function providerMonitorUpstreamIdentityClaimKey(
179
182
  tool: string,
@@ -206,9 +209,10 @@ export function providerMonitorUpstreamIdentityClaimKey(
206
209
  : []),
207
210
  ]),
208
211
  );
212
+ const ignored = new Set(definition.nonConfigurationFields ?? []);
209
213
  const configuration = Object.fromEntries(
210
214
  Object.entries(payload ?? {}).filter(
211
- ([field]) => !scopeAndAliases.has(field),
215
+ ([field]) => !scopeAndAliases.has(field) && !ignored.has(field),
212
216
  ),
213
217
  );
214
218
  if (Object.keys(configuration).length === 0) return scopeKey;
@@ -275,6 +279,15 @@ export function canonicalProviderMonitorIdentityClaimKey(
275
279
  return `${parts.scopeKey}${CONFIGURATION_SEPARATOR}${stableJson(configuration)}`;
276
280
  }
277
281
 
282
+ /**
283
+ * The scope part of a stored claim key: the whole key when it is scope-only.
284
+ * Every key for one provider scope starts with this exact JSON object, so it
285
+ * bounds an index range over that scope's claims.
286
+ */
287
+ export function providerMonitorIdentityClaimScopeKey(identityKey: string) {
288
+ return splitClaimKey(identityKey)?.scopeKey ?? identityKey;
289
+ }
290
+
278
291
  /** Whether a claim key carries configuration beyond its scope. */
279
292
  export function providerMonitorIdentityClaimKeyHasConfiguration(
280
293
  identityKey: string,
@@ -241,7 +241,7 @@
241
241
  "id": "github.eval-agent-cli-matrix",
242
242
  "name": "Eval Agent CLI Matrix",
243
243
  "scheduler": "github_actions",
244
- "cadence": "0 7 * * *",
244
+ "cadence": "0 7 * * *; 0 22 * * 4; 0 3 * * 1,3,5",
245
245
  "timezone": "America/New_York",
246
246
  "source": ".github/workflows/eval-agent-cli-matrix.yml",
247
247
  "locator": "schedule:",
@@ -265,9 +265,9 @@
265
265
  "locator": "schedule:",
266
266
  "owner": "platform",
267
267
  "criticality": "maintenance",
268
- "purpose": "Reconcile completed PR Preview Machines and release capacity only after the per-app lease fence allows it. Its named-estate lane plans every named Preview estate (eval reuse keys, runtime up preview branch lanes, one-off namespaces) with `bun scripts/fly-preview-estate-lifecycle.ts --report --named-reclaim-plan` and, only on a workflow_dispatch with reclaim_named_estates=true, scales at most 8 planned estates per pass (interrupted reclaims first, then reclaimable estates idle 24 h by lease, release or reclaim evidence) to zero through the claimed runtime-preview-namespace.yml scale-to-zero, recording the start and lane app before the seam, retiring the release lane and recording reclaimed history; scheduled passes only report it.",
269
- "successDefinition": "Fly inventory is reconciled and eligible terminal/closed PR capacity is released; every named estate is classified without a read failure, and a dispatched named reclaim leaves each target at zero Machines with its release lane retired and reclaimed history written. No eligible app or estate is a valid no-op.",
270
- "failureDefinition": "Inventory, lease resolution, scale-down, or empty-app cleanup fails or is ambiguous; a named-estate Fly or authority read fails; a named reclaim is refused inside its claim (active owner, used since discovery, ownership or lane-app refusal) or any role keeps a Machine; or any named reclaim is incomplete (started but its release lane was never retired), which fails every pass until a dispatch finishes it.",
268
+ "purpose": "Reconcile completed PR Preview Machines and release capacity only after the per-app lease fence allows it. Its named-estate lane plans every named Preview estate (eval reuse keys, runtime up preview branch lanes, one-off namespaces) with `bun scripts/fly-preview-estate-lifecycle.ts --report --named-reclaim-plan` and, only on a workflow_dispatch with reclaim_named_estates=true, scales at most 8 planned estates per pass (interrupted reclaims first, then reclaimable estates idle 24 h by lease, release or reclaim evidence) to zero through the claimed runtime-preview-namespace.yml scale-to-zero, recording the start and lane app before the seam, retiring the release lane and recording reclaimed history; scheduled passes only report it. Each pass also returns the deeplinev1/load PlanetScale branch to its parked size when no Customer Workload Suite run is active on load-tests/framework, in case a load test's own teardown never ran.",
269
+ "successDefinition": "Fly inventory is reconciled and eligible terminal/closed PR capacity is released; every named estate is classified without a read failure, and a dispatched named reclaim leaves each target at zero Machines with its release lane retired and reclaimed history written. No eligible app or estate is a valid no-op. The load database is parked, a change toward that size is in flight, or an active Customer Workload Suite run on load-tests/framework may be holding it.",
270
+ "failureDefinition": "Inventory, lease resolution, scale-down, or empty-app cleanup fails or is ambiguous; a named-estate Fly or authority read fails; a named reclaim is refused inside its claim (active owner, used since discovery, ownership or lane-app refusal) or any role keeps a Machine; or any named reclaim is incomplete (started but its release lane was never retired), which fails every pass until a dispatch finishes it; or reading or parking the load database fails.",
271
271
  "startGraceMs": 600000,
272
272
  "runTimeoutMs": 3600000,
273
273
  "failureAlert": "github_workflow_run",
@@ -2281,6 +2281,39 @@ export async function releaseRateStateViaAppRuntime(
2281
2281
  });
2282
2282
  }
2283
2283
 
2284
+ /** Persist the final success batch without acquiring another provider permit. */
2285
+ export async function flushRateStateFeedbackViaAppRuntime(
2286
+ context: WorkerRuntimeApiContext,
2287
+ input: PlayRunnerRateStateReleaseInput,
2288
+ signal: AbortSignal,
2289
+ ): Promise<void> {
2290
+ const batch = input.concurrencyObservationBatch;
2291
+ if (!batch)
2292
+ throw new Error('Rate-state feedback is missing its batch identity.');
2293
+ const result = await postAppRuntimeApi<{
2294
+ acknowledgedObservationBatch?: typeof batch;
2295
+ }>(
2296
+ {
2297
+ ...context,
2298
+ connectionRetryBudgetMs: 0,
2299
+ signal: context.signal
2300
+ ? AbortSignal.any([context.signal, signal])
2301
+ : signal,
2302
+ },
2303
+ { action: 'rate_state_release', ...input },
2304
+ );
2305
+ const acknowledged = result.acknowledgedObservationBatch;
2306
+ if (
2307
+ acknowledged?.observerId !== batch.observerId ||
2308
+ acknowledged.laneId !== batch.laneId ||
2309
+ acknowledged.sequence !== batch.sequence
2310
+ ) {
2311
+ throw new Error(
2312
+ 'Rate-state receiver did not acknowledge the success batch.',
2313
+ );
2314
+ }
2315
+ }
2316
+
2284
2317
  export async function penalizeRateStateViaAppRuntime(
2285
2318
  context: WorkerRuntimeApiContext,
2286
2319
  input: PlayRunnerRateStatePenalizeInput,
@@ -2,6 +2,7 @@ import {
2
2
  acquireRateStateViaAppRuntime,
3
3
  AppRuntimeApiTransportError,
4
4
  capacityShedRetryAfterMs,
5
+ flushRateStateFeedbackViaAppRuntime,
5
6
  penalizeRateStateViaAppRuntime,
6
7
  releaseRateStateViaAppRuntime,
7
8
  RateStateValidationUnavailableError,
@@ -54,6 +55,7 @@ export const RATE_ACQUIRE_MAX_WAIT_MS = 300_000;
54
55
  */
55
56
  export const PROVIDER_ADMISSION_WAIT_BUDGET_MS = 120_000;
56
57
  const PROVIDER_NO_PROGRESS_WARNING_MS = 180_000;
58
+ const SUCCESS_FEEDBACK_DRAIN_TIMEOUT_MS = 10_000;
57
59
 
58
60
  /**
59
61
  * Permits leased per app-runtime round trip for the direct Node substrate.
@@ -77,6 +79,10 @@ export interface AppRuntimeRatePort {
77
79
  penalize(
78
80
  input: Parameters<typeof penalizeRateStateViaAppRuntime>[1],
79
81
  ): ReturnType<typeof penalizeRateStateViaAppRuntime>;
82
+ flushFeedback?(
83
+ input: Parameters<typeof flushRateStateFeedbackViaAppRuntime>[1],
84
+ signal: AbortSignal,
85
+ ): Promise<void>;
80
86
  validate?(
81
87
  input: Parameters<typeof validateRateStateViaAppRuntime>[1],
82
88
  ): ReturnType<typeof validateRateStateViaAppRuntime>;
@@ -87,6 +93,8 @@ function contextRatePort(context: WorkerRuntimeApiContext): AppRuntimeRatePort {
87
93
  acquire: (input) => acquireRateStateViaAppRuntime(context, input),
88
94
  release: (input) => releaseRateStateViaAppRuntime(context, input),
89
95
  penalize: (input) => penalizeRateStateViaAppRuntime(context, input),
96
+ flushFeedback: (input, signal) =>
97
+ flushRateStateFeedbackViaAppRuntime(context, input, signal),
90
98
  validate: (input) => validateRateStateViaAppRuntime(context, input),
91
99
  };
92
100
  }
@@ -296,6 +304,16 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
296
304
  private readonly effectiveRps = new Map<string, number>();
297
305
  private readonly pendingSuccesses = new Map<string, number>();
298
306
  private readonly successFeedbackFlights = new Map<string, SuccessFeedback>();
307
+ private readonly successFeedbackScopes = new Map<
308
+ string,
309
+ {
310
+ bucketId: string;
311
+ rateScopeToken?: string | null;
312
+ rules: readonly PacingRule[];
313
+ }
314
+ >();
315
+ private readonly successFeedbackSupported = new Set<string>();
316
+ private successFeedbackDrain: Promise<void> | null = null;
299
317
  /**
300
318
  * Completion samples travel with the next lease refill. This keeps the
301
319
  * Gradient2 checkpoint current without adding one gateway request per row.
@@ -790,6 +808,13 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
790
808
  ): void {
791
809
  if (retryAtMs <= (this.localProviderRetryAtMs.get(bucketId) ?? 0)) return;
792
810
  this.localProviderRetryAtMs.set(bucketId, retryAtMs);
811
+ const cadence = this.validatedDispatchCadences.get(bucketId);
812
+ const serverOffsetMs = this.serverClockOffsetsMs.get(bucketId) ?? 0;
813
+ // A new hold fences old tickets and their dispatch debt. Keep debt from
814
+ // sends after an expired hold; their physical spacing is still current.
815
+ if (cadence && cadence.lastAtMs + serverOffsetMs <= retryAtMs) {
816
+ this.validatedDispatchCadences.delete(bucketId);
817
+ }
793
818
  for (const waiting of this.scheduledPermitWaits.get(bucketId) ?? []) {
794
819
  if (waiting.retryAtMsAtGrant < retryAtMs) waiting.invalidate();
795
820
  }
@@ -1066,6 +1091,11 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
1066
1091
  }
1067
1092
  : rawResponse;
1068
1093
  this.acknowledgeSuccessFeedback(successKey, feedback);
1094
+ if (response.successFeedbackSupported === true) {
1095
+ this.successFeedbackSupported.add(bucketId);
1096
+ } else {
1097
+ this.successFeedbackSupported.delete(bucketId);
1098
+ }
1069
1099
  this.recordCooldown(bucketId, response);
1070
1100
  if (response.scopeValidationSupported === true)
1071
1101
  this.validationScopes.add(bucketId);
@@ -1552,11 +1582,17 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
1552
1582
 
1553
1583
  observeSuccess(input: {
1554
1584
  bucketId: string;
1585
+ rateScopeToken?: string | null;
1555
1586
  rules: readonly PacingRule[];
1556
1587
  latencyMs?: number | null;
1557
1588
  }): void {
1558
1589
  this.providerStalls.delete(input.bucketId);
1559
1590
  const key = pendingSuccessKey(input.bucketId, input.rules);
1591
+ this.successFeedbackScopes.set(key, {
1592
+ bucketId: input.bucketId,
1593
+ rateScopeToken: input.rateScopeToken,
1594
+ rules: input.rules,
1595
+ });
1560
1596
  this.pendingSuccesses.set(
1561
1597
  key,
1562
1598
  Math.min(1_000_000, (this.pendingSuccesses.get(key) ?? 0) + 1),
@@ -1586,6 +1622,43 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
1586
1622
  this.pendingConcurrencyObservations.set(key, pending);
1587
1623
  }
1588
1624
 
1625
+ async flushSuccessFeedback(): Promise<void> {
1626
+ if (!this.port.flushFeedback) return;
1627
+ if (this.successFeedbackDrain) return await this.successFeedbackDrain;
1628
+ const signal = AbortSignal.timeout(SUCCESS_FEEDBACK_DRAIN_TIMEOUT_MS);
1629
+ const drain = async () => {
1630
+ for (const [key, scope] of this.successFeedbackScopes) {
1631
+ // Older receivers retain their existing piggyback behavior. An explicit
1632
+ // capability and matching batch acknowledgement protect rolling deploys.
1633
+ if (!this.successFeedbackSupported.has(scope.bucketId)) continue;
1634
+ for (;;) {
1635
+ if (signal.aborted) throw abortReason(signal);
1636
+ const feedback = this.snapshotSuccessFeedback(key);
1637
+ if (!feedback.concurrencyObservationBatch) break;
1638
+ await this.port.flushFeedback!(
1639
+ {
1640
+ ...scope,
1641
+ rules: [...scope.rules],
1642
+ leaseIds: [],
1643
+ ...feedback,
1644
+ schedulerSchema: this.schedulerSchema,
1645
+ },
1646
+ signal,
1647
+ );
1648
+ // A lost response leaves this exact frozen payload available to retry.
1649
+ this.acknowledgeSuccessFeedback(key, feedback);
1650
+ }
1651
+ this.successFeedbackScopes.delete(key);
1652
+ }
1653
+ };
1654
+ this.successFeedbackDrain = drain();
1655
+ try {
1656
+ await this.successFeedbackDrain;
1657
+ } finally {
1658
+ this.successFeedbackDrain = null;
1659
+ }
1660
+ }
1661
+
1589
1662
  private newPendingConcurrencyObservations(
1590
1663
  key: string,
1591
1664
  observations: Array<{ kind: 'success'; latencyMs: number }> = [],
@@ -1626,6 +1699,14 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
1626
1699
  scopeRetryAtMsAtGrant,
1627
1700
  dispatchValidationSupported,
1628
1701
  ] = scheduled;
1702
+ // Legacy grants have no shared hold snapshot. Capture their local fence
1703
+ // when assigned, while keeping the signed gateway proof unchanged.
1704
+ const localRetryAtMsAtGrant = this.validationScopes.has(bucketId)
1705
+ ? scopeRetryAtMsAtGrant
1706
+ : Math.max(
1707
+ scopeRetryAtMsAtGrant,
1708
+ this.localProviderRetryAtMs.get(bucketId) ?? 0,
1709
+ );
1629
1710
  let settled = false;
1630
1711
  const refund = async (): Promise<void> => {
1631
1712
  if (!reservationId) {
@@ -1732,7 +1813,15 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
1732
1813
  });
1733
1814
  }
1734
1815
  if (waitMs > 0) {
1735
- await this.sleep(waitMs, signal);
1816
+ if (
1817
+ !(await this.waitForScheduledPermit(
1818
+ bucketId,
1819
+ localRetryAtMsAtGrant,
1820
+ waitMs,
1821
+ signal,
1822
+ ))
1823
+ )
1824
+ return false;
1736
1825
  continue;
1737
1826
  }
1738
1827
  this.validatedDispatchCadences.set(bucketId, {
@@ -868,6 +868,7 @@ export function createPlayExecutionGovernor(
868
868
  if (!rules) return;
869
869
  input.rateState.observeSuccess?.({
870
870
  bucketId: rateScope[0],
871
+ ...(rateScope[1] ? { rateScopeToken: rateScope[1] } : {}),
871
872
  rules,
872
873
  latencyMs: success.latencyMs,
873
874
  });
@@ -197,6 +197,9 @@ export interface RateStateBackend {
197
197
  /** Provider residence measured at the HTTP boundary. */
198
198
  latencyMs?: number | null;
199
199
  }): void;
200
+
201
+ /** Bounded final feedback write after the invocation's provider work settles. */
202
+ flushSuccessFeedback?(): Promise<void>;
200
203
  }
201
204
 
202
205
  const NOOP_PERMIT: PacingPermit = {
@@ -72,11 +72,14 @@ export type PlayRunRecoveryDecision = Readonly<{
72
72
  export class PlayRunRecoveryError extends Error {
73
73
  readonly decision: PlayRunRecoveryDecision;
74
74
  readonly cause?: unknown;
75
+ /** Log-only provider facts; see `playRunRecoveryLogDetail`. */
76
+ readonly recoveryLogDetail?: unknown;
75
77
 
76
78
  constructor(input: {
77
79
  decision: PlayRunRecoveryDecision;
78
80
  message: string;
79
81
  cause?: unknown;
82
+ recoveryLogDetail?: unknown;
80
83
  }) {
81
84
  super(input.message);
82
85
  this.name = PLAY_RUN_RECOVERY_ERROR_NAME;
@@ -84,6 +87,9 @@ export class PlayRunRecoveryError extends Error {
84
87
  if (input.cause !== undefined) {
85
88
  this.cause = input.cause;
86
89
  }
90
+ if (input.recoveryLogDetail !== undefined) {
91
+ this.recoveryLogDetail = input.recoveryLogDetail;
92
+ }
87
93
  }
88
94
  }
89
95
 
@@ -106,7 +112,7 @@ export type PlayRunRecoveryLogDetail = Readonly<
106
112
  >;
107
113
 
108
114
  /**
109
- * Provider facts an adapter attached to the cause of a recovery error, for the
115
+ * Provider facts an adapter attached to a recovery error or its cause, for the
110
116
  * scheduler's defer log only. The decision's `reason` stays the single durable
111
117
  * recovery fact: this detail is never persisted, never returned to a caller,
112
118
  * and never read to choose a recovery action. It is bounded here, at the edge
@@ -119,9 +125,11 @@ export function playRunRecoveryLogDetail(
119
125
  try {
120
126
  if (!isPlayRunRecoveryError(error)) return null;
121
127
  const cause: unknown = error.cause;
122
- if (!cause || typeof cause !== 'object') return null;
123
- const supplied = (cause as { recoveryLogDetail?: unknown })
124
- .recoveryLogDetail;
128
+ const supplied =
129
+ error.recoveryLogDetail ??
130
+ (cause && typeof cause === 'object'
131
+ ? (cause as { recoveryLogDetail?: unknown }).recoveryLogDetail
132
+ : undefined);
125
133
  if (!supplied || typeof supplied !== 'object' || Array.isArray(supplied)) {
126
134
  return null;
127
135
  }
@@ -203,6 +211,14 @@ export function recoveryForModalSandboxCreateFailure(
203
211
  const code = errorCode(error);
204
212
  const status = errorStatus(error);
205
213
  const message = messageOf(error);
214
+ // Only this rejection carries Modal's own words. Once it opens the circuit,
215
+ // later runs defer without calling Modal, and every reason here groups
216
+ // several provider conditions.
217
+ const recoveryLogDetail = {
218
+ providerCode: code,
219
+ providerStatus: status,
220
+ providerMessage: message,
221
+ };
206
222
  const resourceExhausted =
207
223
  code === 8 ||
208
224
  (typeof code === 'string' && /RESOURCE_EXHAUSTED/i.test(code)) ||
@@ -218,6 +234,7 @@ export function recoveryForModalSandboxCreateFailure(
218
234
  message:
219
235
  'Modal sandbox capacity is unavailable before customer execution began.',
220
236
  cause: error,
237
+ recoveryLogDetail,
221
238
  });
222
239
  }
223
240
 
@@ -90,6 +90,8 @@ export type PlayRunnerRateStateAcquireResult = {
90
90
  scopeValidationSupported?: boolean;
91
91
  /** This gateway rechecks signed provider permits on integration execute. */
92
92
  dispatchValidationSupported?: boolean;
93
+ /** This receiver acknowledges replay-safe success batches on release. */
94
+ successFeedbackSupported?: boolean;
93
95
  /**
94
96
  * Gateway clock used to produce `scheduledAtMs`. Runners translate the
95
97
  * schedule into relative delays so cross-machine clock skew cannot compress
@@ -138,6 +140,10 @@ export type PlayRunnerRateStateReleaseInput = {
138
140
  leaseIds: string[];
139
141
  /** Idempotently refund rate reservations that never reached provider fetch. */
140
142
  refundReservationIds?: string[];
143
+ /** Flush confirmed outcomes without reserving another provider permit. */
144
+ observedSuccesses?: number;
145
+ concurrencyObservations?: PlayRunnerRateStateConcurrencyObservation[];
146
+ concurrencyObservationBatch?: PlayRunnerRateStateConcurrencyObservationBatch;
141
147
  schedulerSchema?: string | null;
142
148
  };
143
149
 
@@ -3788,6 +3788,19 @@ async function run(
3788
3788
  } finally {
3789
3789
  clearInterval(workCounterTimer);
3790
3790
  emitWorkCounters();
3791
+ try {
3792
+ await rateStateBackend.rateState?.flushSuccessFeedback?.();
3793
+ } catch (feedbackError) {
3794
+ // Feedback is advisory: preserve the Play's result or original failure.
3795
+ emitWorkerLog(
3796
+ `[warn] runtime.provider_success_feedback_flush_failed: ${
3797
+ feedbackError instanceof Error
3798
+ ? feedbackError.message
3799
+ : String(feedbackError)
3800
+ }`,
3801
+ options,
3802
+ );
3803
+ }
3791
3804
  }
3792
3805
  }
3793
3806
 
@@ -56,8 +56,20 @@ export const RUNTIME_CAPACITY_POLICY = {
56
56
  maxScaleDownMachinesPerReconcile: 2,
57
57
  /** Empty relay capacity must be quiet before it is reclaimed. */
58
58
  scaleDownQuietMs: 120_000,
59
+ /**
60
+ * Pooled Postgres clients per relay Machine when
61
+ * DEEPLINE_RUNTIME_GATEWAY_POOL_MAX_CLIENTS sets none, as in production.
62
+ * packages/runtime-gateway/server.ts explains the budget.
63
+ */
64
+ poolMaxClients: 6,
59
65
  },
60
66
  receiptGateway: {
67
+ /**
68
+ * Pooled Postgres clients per receipt-gateway Machine when
69
+ * DEEPLINE_RUNTIME_GATEWAY_POOL_MAX_CLIENTS sets none, as in production.
70
+ * packages/runtime-gateway/server.ts explains the budget.
71
+ */
72
+ poolMaxClients: 8,
61
73
  admissionTimeoutMs: 10_000,
62
74
  /** End identical receipt-row work before the caller can overlap a retry. */
63
75
  claimOperationTimeoutMs: 20_000,
@@ -88,6 +88,9 @@ export function resolveRuntimeSandboxComputeItems(input: {
88
88
  sandboxId,
89
89
  startedAt: resource.startedAt,
90
90
  endedAt,
91
+ maxBillingDurationSeconds: resource.maxBillingDurationSeconds,
92
+ observedWallTimeSeconds:
93
+ Math.max(0, observedEndedAt - resource.startedAt) / 1_000,
91
94
  }),
92
95
  );
93
96
  }
@@ -201,6 +204,9 @@ export function resolveDaytonaRuntimeComputeItems(input: {
201
204
  sandboxId,
202
205
  startedAt: resource.startedAt,
203
206
  endedAt,
207
+ maxBillingDurationSeconds: resource.maxBillingDurationSeconds,
208
+ observedWallTimeSeconds:
209
+ Math.max(0, observedEndedAt - resource.startedAt) / 1_000,
204
210
  }),
205
211
  );
206
212
  }
@@ -72,7 +72,7 @@ export function acquireRateScopeSql(
72
72
  THEN CASE WHEN ${productiveRecovery} THEN ${AIMD_RECOVERY_STEP_MS}
73
73
  ELSE ${sparseRecoveryInterval} END
74
74
  ELSE LEAST(${floorRecoveryInterval}, CEIL(1000.0 / ${refillRate})) END`
75
- : `LEAST(h.rate_recovery_interval_ms, ${AIMD_RECOVERY_STEP_MS})`;
75
+ : `${AIMD_RECOVERY_STEP_MS}`;
76
76
  // A local no-progress pause remains active after the provider's shorter
77
77
  // Retry-After. Only time beyond every outstanding dispatch/hold is idle.
78
78
  const observationWasIdle = `${successesParameter}::int = 0
@@ -202,13 +202,15 @@ export function updateRateScopeSql(
202
202
  ELSE ph.rate_probe_started_at_ms END,`
203
203
  : ''
204
204
  }
205
- -- At the floor, a new successful epoch must not inherit failed-probe
206
- -- backoff. Above the floor, retain its useful measurement duration for
207
- -- sparse upward probes; that rate already proves successful recovery.
205
+ -- A productive known-provider epoch uses the one-second recovery window,
206
+ -- so its old measurement must not become a failed-probe streak when the
207
+ -- success counter resets. Unknown providers above the floor retain that
208
+ -- measurement for their separate sparse upward-probe policy.
208
209
  rate_recovery_interval_ms = CASE
209
- WHEN ${unknownProvider} AND (SELECT granted FROM grant_calc) > 0
210
+ WHEN (SELECT granted FROM grant_calc) > 0
210
211
  AND ph.rate_successes > 0
211
- AND ph.next_refill_per_sec <= ${DEFAULT_UNKNOWN_PROVIDER_MIN_RPS}
212
+ AND (${!unknownProvider}
213
+ OR ph.next_refill_per_sec <= ${DEFAULT_UNKNOWN_PROVIDER_MIN_RPS})
212
214
  AND (ph.observation_was_idle
213
215
  OR ph.rate_observation_started_at_ms IS NULL)
214
216
  -- Zero is an unmeasured new episode, not a failed-probe interval.