deepline 0.3.331 → 0.3.333

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (102) hide show
  1. package/dist/bundling-sources/sdk/src/billing-idempotency.ts +5 -1
  2. package/dist/bundling-sources/sdk/src/release.ts +1 -1
  3. package/dist/bundling-sources/shared_libs/play-runtime/context.ts +26 -23
  4. package/dist/bundling-sources/shared_libs/play-runtime/execution-gateway-invocation-fence.ts +16 -0
  5. package/dist/bundling-sources/shared_libs/play-runtime/governor/app-runtime-rate-state-backend.ts +30 -9
  6. package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts +6 -0
  7. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-rate-scope.ts +95 -24
  8. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-rate-state.ts +132 -37
  9. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts +6 -3
  10. package/dist/bundling-sources/shared_libs/play-runtime/tool-execute-retry-policy.ts +87 -3
  11. package/dist/cli/auth-26GA3RM6.js +31 -0
  12. package/dist/cli/{auth-TVTO43JD.mjs → auth-OQ356CEE.mjs} +4 -4
  13. package/dist/cli/{billing-Y4EX5FFA.js → billing-3APFTHNY.js} +71 -71
  14. package/dist/cli/{billing-FBZ5CCU7.mjs → billing-7YVHLDMO.mjs} +4 -4
  15. package/dist/cli/{chunk-RI57LHLC.mjs → chunk-2CLENWOQ.mjs} +1 -1
  16. package/dist/cli/{chunk-KGZPY524.js → chunk-3WQJWQ5T.js} +60 -60
  17. package/dist/cli/{chunk-YIYK57LW.mjs → chunk-4DUEB2WO.mjs} +2 -2
  18. package/dist/cli/{chunk-T7E5KIPE.js → chunk-5A44CWQE.js} +48 -48
  19. package/dist/cli/{chunk-VEP3POWB.mjs → chunk-5UN6KLD2.mjs} +1 -1
  20. package/dist/cli/{chunk-L4CJXV7D.js → chunk-64CZWDWT.js} +8 -8
  21. package/dist/cli/{chunk-RRJ6AA5L.mjs → chunk-6JPZJ5W7.mjs} +2 -2
  22. package/dist/cli/{chunk-BXZN544J.mjs → chunk-ABVOFJAE.mjs} +1 -1
  23. package/dist/cli/{chunk-NHR3O4OV.js → chunk-AFVETOG3.js} +15 -15
  24. package/dist/cli/{chunk-4JYLNJMD.mjs → chunk-AGGDEQDF.mjs} +1 -1
  25. package/dist/cli/{chunk-LFV2A7GB.js → chunk-DROPDYOD.js} +1 -1
  26. package/dist/cli/{chunk-ZGWURYZK.js → chunk-F2QKAGST.js} +10 -10
  27. package/dist/cli/{chunk-O52SKQQT.mjs → chunk-F2XAQHQC.mjs} +4 -4
  28. package/dist/cli/{chunk-CKAZY2JT.mjs → chunk-H5SORD7Q.mjs} +2 -2
  29. package/dist/cli/{chunk-DU72PAVF.mjs → chunk-HNX3RC6V.mjs} +1 -1
  30. package/dist/cli/{chunk-YCYWPBRV.js → chunk-IDEK4DND.js} +2 -2
  31. package/dist/cli/{chunk-MDQOGPQY.js → chunk-IYFKFNEB.js} +225 -225
  32. package/dist/cli/{chunk-KXWOO3GI.js → chunk-KBXHB5P3.js} +26 -26
  33. package/dist/cli/{chunk-HG5NWSN2.mjs → chunk-KREEYWVL.mjs} +6 -6
  34. package/dist/cli/{chunk-EGRTGSNS.mjs → chunk-N5KGBQPG.mjs} +3 -3
  35. package/dist/cli/{chunk-PS5BFY2W.mjs → chunk-NI4FJRFJ.mjs} +5 -5
  36. package/dist/cli/{chunk-XZG75ELG.js → chunk-PRZ4S4NU.js} +6 -6
  37. package/dist/cli/{chunk-RX2WWVZ7.js → chunk-STVJ7ZPD.js} +6 -6
  38. package/dist/cli/{chunk-QEI6SNAT.mjs → chunk-SXX4ST3D.mjs} +2 -2
  39. package/dist/cli/{chunk-ELDOV5XG.js → chunk-UHHTRBJF.js} +45 -45
  40. package/dist/cli/{chunk-3AQ3L2NX.js → chunk-V77J5ISS.js} +4 -4
  41. package/dist/cli/client-IZRTEBU4.js +20 -0
  42. package/dist/cli/{client-Q7J5R5PT.mjs → client-WKS7HQRP.mjs} +3 -3
  43. package/dist/cli/{csv-WBEF4KIZ.js → csv-MMW6EKKD.js} +12 -12
  44. package/dist/cli/{csv-RVG3V4YX.mjs → csv-ZFY2LD2K.mjs} +3 -3
  45. package/dist/cli/{db-4SPPGSEV.js → db-FMOQU6QW.js} +29 -29
  46. package/dist/cli/{db-AVIQ5WHJ.mjs → db-Q2NJVILC.mjs} +4 -4
  47. package/dist/cli/{describe-IQ3FOLGV.js → describe-4KG2JAQV.js} +15 -15
  48. package/dist/cli/{describe-4UA6N4YX.mjs → describe-EWP7DMNU.mjs} +8 -8
  49. package/dist/cli/{executions-VCULCPYR.mjs → executions-UOD7FWEE.mjs} +4 -4
  50. package/dist/cli/{executions-3ZUK3YJB.js → executions-Y4XSGKN6.js} +6 -6
  51. package/dist/cli/{feedback-SZLTZXWC.mjs → feedback-3F3EURZC.mjs} +3 -3
  52. package/dist/cli/{feedback-7YKDOMWW.js → feedback-TEJJUXDI.js} +6 -6
  53. package/dist/cli/{git-H4YS7TQW.mjs → git-5YM22RGA.mjs} +4 -4
  54. package/dist/cli/git-GNEZSTVF.js +23 -0
  55. package/dist/cli/index.js +79 -79
  56. package/dist/cli/index.mjs +31 -31
  57. package/dist/cli/{legacy-noops-K6U4YL3K.mjs → legacy-noops-D5XBVUHS.mjs} +4 -4
  58. package/dist/cli/{legacy-noops-GU7X3U2Y.js → legacy-noops-T5OUSXMR.js} +7 -7
  59. package/dist/cli/{monitors-2HHSSSDD.mjs → monitors-LPZNYNQW.mjs} +4 -4
  60. package/dist/cli/{monitors-IUEWN2VX.js → monitors-VI7VBQRP.js} +85 -85
  61. package/dist/cli/{org-6NBBO7G3.mjs → org-BYIDOMJB.mjs} +4 -4
  62. package/dist/cli/{org-E5TYEFPX.js → org-OBGIV5QE.js} +60 -60
  63. package/dist/cli/play-5JYRSXAL.js +153 -0
  64. package/dist/cli/{play-C2BQ4MJG.mjs → play-Y5PQQRBO.mjs} +7 -7
  65. package/dist/cli/{providers-N2LG7Y3E.mjs → providers-GEGIGKOL.mjs} +4 -4
  66. package/dist/cli/{providers-FM6LWHA4.js → providers-VH6J7FFY.js} +7 -7
  67. package/dist/cli/{quickstart-GQQ6ODNU.mjs → quickstart-GHITFT27.mjs} +3 -3
  68. package/dist/cli/{quickstart-I4K6ZDU6.js → quickstart-ZQX5XNHG.js} +5 -5
  69. package/dist/cli/{search-NKDQSOQ5.mjs → search-AFX3DYAC.mjs} +5 -5
  70. package/dist/cli/search-YIXGQFXA.js +26 -0
  71. package/dist/cli/{secrets-FGKGASDL.mjs → secrets-4EOVXQJM.mjs} +4 -4
  72. package/dist/cli/{secrets-BBI5NYLT.js → secrets-F37M7D7R.js} +13 -13
  73. package/dist/cli/{sessions-7MWWKIUT.mjs → sessions-RX5SFHDI.mjs} +4 -4
  74. package/dist/cli/{sessions-YUTDWUVR.js → sessions-T57Q72GD.js} +5 -5
  75. package/dist/cli/{settings-U55P74SS.js → settings-3FVWW7GL.js} +74 -74
  76. package/dist/cli/{settings-JGJWBESE.mjs → settings-764O5ZEH.mjs} +7 -7
  77. package/dist/cli/{setup-XYIDQ46U.js → setup-FQVRVTR6.js} +41 -41
  78. package/dist/cli/{setup-7D577YQO.mjs → setup-X7OAHFTO.mjs} +6 -6
  79. package/dist/cli/skills-DPGB67XX.js +28 -0
  80. package/dist/cli/{skills-Y6WBNBPF.mjs → skills-O6ILHPV5.mjs} +4 -4
  81. package/dist/cli/{tools-OPTIBRZP.mjs → tools-HL4SQG35.mjs} +6 -6
  82. package/dist/cli/tools-WZCXSXER.js +40 -0
  83. package/dist/cli/update-63MANY7U.js +28 -0
  84. package/dist/cli/{update-HKJQ762M.mjs → update-A3P2RSIB.mjs} +6 -6
  85. package/dist/cli/{workflow-2ZOG35IA.mjs → workflow-44424AVE.mjs} +5 -5
  86. package/dist/cli/{workflow-EBIKBXXC.js → workflow-62AGKI6T.js} +15 -15
  87. package/dist/index.js +6 -6
  88. package/dist/index.mjs +3 -3
  89. package/dist/install-integrity.json +74 -74
  90. package/dist/release.d.mts +1 -1
  91. package/dist/release.d.ts +1 -1
  92. package/dist/release.js +2 -2
  93. package/dist/release.mjs +1 -1
  94. package/package.json +1 -1
  95. package/dist/cli/auth-YYTF3NO4.js +0 -31
  96. package/dist/cli/client-46KHPTTQ.js +0 -20
  97. package/dist/cli/git-DCZP3E3P.js +0 -23
  98. package/dist/cli/play-MPDZPEYN.js +0 -153
  99. package/dist/cli/search-6G6FUVTJ.js +0 -26
  100. package/dist/cli/skills-PXHET5MG.js +0 -28
  101. package/dist/cli/tools-BCSXPN5X.js +0 -40
  102. package/dist/cli/update-2NXG3NKT.js +0 -28
@@ -2,8 +2,12 @@ import { DeeplineError } from './errors.js';
2
2
 
3
3
  export function requireBillingIdempotencyKey(value: string): string {
4
4
  const normalized = value.trim();
5
+ // Headers accepts some control characters that HTTP delivery rejects.
5
6
  let valid =
6
- normalized.length > 0 && normalized.length <= 200 && normalized === value;
7
+ normalized.length > 0 &&
8
+ normalized.length <= 200 &&
9
+ normalized === value &&
10
+ !/[\u0000-\u0008\u000a-\u001f\u007f]/.test(value);
7
11
  if (valid) {
8
12
  try {
9
13
  valid =
@@ -200,7 +200,7 @@ export const SDK_RELEASE = {
200
200
  // getters keep their established compatibility behavior.
201
201
  // 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
202
202
  // 0.3.90 is the first deliberately versioned SDK release for API v3.
203
- version: '0.3.331',
203
+ version: '0.3.333',
204
204
  updateSummary:
205
205
  'Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.',
206
206
  packageCapabilities: {
@@ -222,7 +222,8 @@ import {
222
222
  classifyToolExecuteHttpFailure,
223
223
  createToolExecuteHttpFailureAttemptTracker,
224
224
  parseToolExecuteRetryAfterMs,
225
- parseToolExecuteRetryAtMs,
225
+ parseToolExecuteProviderRetryAtMs,
226
+ parseToolExecuteExactProviderRetryAtMs,
226
227
  parseToolExecuteAuthScopeChangedError,
227
228
  ToolExecuteAuthScopeChangedError,
228
229
  type ToolExecuteHttpFailureOutcome,
@@ -17849,8 +17850,9 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
17849
17850
  const executionRequestNonce = crypto.randomUUID();
17850
17851
  // Transport ambiguity must retain the billing identity. An ordinary
17851
17852
  // (unfenced) call advances it after a definite rejection released
17852
- // that attempt's reservation, including app pacing refusals. Fenced
17853
- // calls retain it: gateway-qualified retries own their distinct charge lifecycle.
17853
+ // that attempt's reservation. A proven app pacing refusal also
17854
+ // renews fenced billing attribution, keeping its invocation and
17855
+ // provider-operation identities unchanged. Ambiguous deliveries retain it.
17854
17856
  let deeplineRequestId = `ctx-tool-${stableDigest(executionRequestNonce).slice(0, 32)}`;
17855
17857
  // The provider operation has a semantic identity independent of
17856
17858
  // the response receipt. Its key is supplied by the caller that owns
@@ -18647,13 +18649,8 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
18647
18649
  // settle the row as failed instead of pacing.
18648
18650
  [TOOL_EXECUTE_RUNNER_ACCEPTS_HEADER]:
18649
18651
  TOOL_EXECUTE_RUNNER_ACCEPTS_VALUE,
18650
- ...(!usesDurableInvocationFence &&
18651
- !providerExecutionFenceKey
18652
- ? {
18653
- [TOOL_EXECUTE_APP_PACING_REFUSAL_HEADER]:
18654
- TOOL_EXECUTE_APP_PACING_REFUSAL_VALUE,
18655
- }
18656
- : {}),
18652
+ [TOOL_EXECUTE_APP_PACING_REFUSAL_HEADER]:
18653
+ TOOL_EXECUTE_APP_PACING_REFUSAL_VALUE,
18657
18654
  ...(providerGatewayDispatchProof
18658
18655
  ? {
18659
18656
  [TOOL_EXECUTE_PROVIDER_RATE_SCOPE_HEADER]:
@@ -18698,18 +18695,15 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
18698
18695
  recordProviderDispatch();
18699
18696
  if (dispatchedResponse.status === 429) {
18700
18697
  const observedAt = Date.now();
18701
- const retryAfter =
18702
- dispatchedResponse.headers.get('retry-after');
18703
18698
  // Preserve absolute provider deadlines even when the
18704
18699
  // response did not carry the gateway's counter
18705
18700
  // attestation. Classification below still decides
18706
18701
  // whether this 429 may update shared provider pacing.
18707
- providerRetryAtMs = retryAfter?.trim()
18708
- ? parseToolExecuteRetryAtMs(
18709
- retryAfter,
18710
- observedAt,
18711
- )
18712
- : undefined;
18702
+ providerRetryAtMs =
18703
+ parseToolExecuteProviderRetryAtMs(
18704
+ dispatchedResponse,
18705
+ observedAt,
18706
+ );
18713
18707
  }
18714
18708
  if (
18715
18709
  isAttestedProvider429Response(dispatchedResponse)
@@ -18877,6 +18871,10 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
18877
18871
  schemaVersion: toolErrorSchemaVersion,
18878
18872
  retryAfterHeader:
18879
18873
  dispatchedResponse.headers.get('retry-after'),
18874
+ retryAtMs:
18875
+ parseToolExecuteExactProviderRetryAtMs(
18876
+ dispatchedResponse,
18877
+ ),
18880
18878
  transientHttpRetrySafe:
18881
18879
  retrySafeTransientHttp || fixtureReplaySafe,
18882
18880
  executionFenced: usesDurableInvocationFence,
@@ -18898,6 +18896,10 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
18898
18896
  schemaVersion: toolErrorSchemaVersion,
18899
18897
  retryAfterHeader:
18900
18898
  dispatchedResponse.headers.get('retry-after'),
18899
+ retryAtMs:
18900
+ parseToolExecuteExactProviderRetryAtMs(
18901
+ dispatchedResponse,
18902
+ ),
18901
18903
  transientHttpRetrySafe:
18902
18904
  retrySafeTransientHttp || fixtureReplaySafe,
18903
18905
  executionFenced: usesDurableInvocationFence,
@@ -19294,6 +19296,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
19294
19296
  bodyText: text,
19295
19297
  schemaVersion: toolErrorSchemaVersion,
19296
19298
  retryAfterHeader: response.headers.get('retry-after'),
19299
+ retryAtMs: parseToolExecuteExactProviderRetryAtMs(response),
19297
19300
  transientHttpRetrySafe:
19298
19301
  retrySafeTransientHttp || fixtureReplaySafe,
19299
19302
  executionFenced: usesDurableInvocationFence,
@@ -19519,11 +19522,11 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
19519
19522
  );
19520
19523
  logProvider429RetryPhase('retry_sleep_completed');
19521
19524
  if (
19522
- !usesDurableInvocationFence &&
19523
- (failure.backpressureDelayMs !== null ||
19524
- failure.reason === 'provider_pacing_wait' ||
19525
- (failure.reason === 'rate_limit' &&
19526
- failure.knownNotDispatched))
19525
+ failure.reason === 'provider_pacing_wait' ||
19526
+ (!usesDurableInvocationFence &&
19527
+ (failure.backpressureDelayMs !== null ||
19528
+ (failure.reason === 'rate_limit' &&
19529
+ failure.knownNotDispatched)))
19527
19530
  ) {
19528
19531
  // These refusals release the prior reservation. App pacing
19529
19532
  // does not advance invocationAttempt, so use its own HTTP
@@ -62,6 +62,14 @@ export type DurableInvocationDiagnostic = {
62
62
  error?: unknown;
63
63
  };
64
64
 
65
+ /** Authenticated transport proof that this owner sent no provider request. */
66
+ export class DurableInvocationDispatchNotStartedError extends Error {
67
+ constructor(readonly reason: string) {
68
+ super(reason);
69
+ this.name = 'DurableInvocationDispatchNotStartedError';
70
+ }
71
+ }
72
+
65
73
  export class DurableInvocationConflictError extends Error {
66
74
  readonly code = 'DURABLE_INVOCATION_REQUEST_CONFLICT';
67
75
 
@@ -313,6 +321,14 @@ export async function executeWithDurableInvocationFence(input: {
313
321
  response = await input.execute();
314
322
  } catch (error) {
315
323
  diagnose('dispatch_failed', error);
324
+ if (error instanceof DurableInvocationDispatchNotStartedError) {
325
+ // An authenticated app refusal proves no provider request was sent.
326
+ // Release only this owner; ordinary transport failures stay unknown.
327
+ if (!(await input.store.abandon(begun))) {
328
+ throw new DurableInvocationInProgressError();
329
+ }
330
+ throw error;
331
+ }
316
332
  if (input.replaySafeAfterDispatch) {
317
333
  return await retryReplaySafeInvocation(begun);
318
334
  }
@@ -140,7 +140,6 @@ function sleepWithAbort(
140
140
  * hold more than a moment of a shared provider window.
141
141
  */
142
142
  const RATE_BLOCK_HORIZON_MS = 1_000;
143
- const UNKNOWN_PROVIDER_RECOVERY_BLOCK_HORIZON_MS = 500;
144
143
  const STORE_FAILURE_RETRY_DELAYS_MS = [100, 250] as const;
145
144
  /**
146
145
  * Jitter fraction applied to a server-supplied `waitMs` so a synchronized wave
@@ -667,23 +666,22 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
667
666
  // a hot map is a round trip per permit. The store's own token floor, not
668
667
  // this number, is what bounds the grant.
669
668
  const learnedRps = this.effectiveRps.get(laneKey);
669
+ const currentRps = learnedRps ?? configuredRps;
670
670
  const recoveringUnknownProvider =
671
671
  learnedRps !== undefined &&
672
672
  learnedRps < DEFAULT_UNKNOWN_PROVIDER_FAST_RECOVERY_RPS &&
673
673
  rules.some((rule) => rule.ruleId === 'default:provider-scope') &&
674
674
  rules.every((rule) => rule.ruleId.startsWith('default:'));
675
- // Below recovered capacity, the next clean observation can double the
676
- // pace after 500ms. Do not reserve a full second at the old slower pace:
677
- // that schedule debt would delay grants even after learning succeeds.
675
+ // Below recovered capacity, one dispatch interval (500/250/125ms at
676
+ // 2/4/8 RPS) can prove the next step. Reserve only that interval so cached
677
+ // old-pace tickets cannot postpone the next fresh-feedback refill.
678
678
  const horizonMs = Math.min(
679
- recoveringUnknownProvider
680
- ? UNKNOWN_PROVIDER_RECOVERY_BLOCK_HORIZON_MS
681
- : RATE_BLOCK_HORIZON_MS,
679
+ recoveringUnknownProvider ? 1_000 / currentRps : RATE_BLOCK_HORIZON_MS,
682
680
  leasedBlockTtlMs(rules),
683
681
  );
684
682
  const rateBudget = Math.max(
685
683
  1,
686
- Math.floor(((learnedRps ?? configuredRps) * horizonMs) / 1_000),
684
+ Math.floor((currentRps * horizonMs) / 1_000),
687
685
  );
688
686
  if (!Number.isFinite(rateBudget) || rateBudget <= 0) return floor;
689
687
  const waiters = this.waiters.get(laneKey) ?? 1;
@@ -1018,7 +1016,8 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
1018
1016
  const releaseGeneration = releaseSignal?.generation;
1019
1017
  const successKey = pendingSuccessKey(bucketId, ordered);
1020
1018
  const feedback = this.snapshotSuccessFeedback(successKey);
1021
- const response = await this.port.acquire({
1019
+ const requestStartedAtMs = this.now();
1020
+ const rawResponse = await this.port.acquire({
1022
1021
  bucketId,
1023
1022
  rateScopeToken,
1024
1023
  rules: withAdaptiveMaxRps(ordered),
@@ -1026,6 +1025,28 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
1026
1025
  ...feedback,
1027
1026
  schedulerSchema: this.schedulerSchema,
1028
1027
  });
1028
+ const serverElapsedMs = rawResponse.serverClockElapsedMs;
1029
+ const validServerElapsed =
1030
+ typeof rawResponse.serverNowMs === 'number' &&
1031
+ Number.isFinite(rawResponse.serverNowMs) &&
1032
+ rawResponse.serverNowMs >= 0 &&
1033
+ typeof serverElapsedMs === 'number' &&
1034
+ Number.isSafeInteger(serverElapsedMs) &&
1035
+ serverElapsedMs >= 0 &&
1036
+ serverElapsedMs <= this.now() - requestStartedAtMs;
1037
+ // The clock was already this old when the response left the server.
1038
+ // Advance only by confirmed elapsed time; unknown transit remains
1039
+ // conservative, and an invalid/legacy response keeps the old clock.
1040
+ const response = validServerElapsed
1041
+ ? {
1042
+ ...rawResponse,
1043
+ serverNowMs: rawResponse.serverNowMs! + serverElapsedMs,
1044
+ waitMs:
1045
+ rawResponse.waitMs > 0
1046
+ ? Math.max(1, rawResponse.waitMs - serverElapsedMs)
1047
+ : rawResponse.waitMs,
1048
+ }
1049
+ : rawResponse;
1029
1050
  this.acknowledgeSuccessFeedback(successKey, feedback);
1030
1051
  this.recordCooldown(bucketId, response);
1031
1052
  if (response.scopeValidationSupported === true)
@@ -96,6 +96,12 @@ export type PlayRunnerRateStateAcquireResult = {
96
96
  * provider spacing.
97
97
  */
98
98
  serverNowMs?: number;
99
+ /**
100
+ * Confirmed elapsed time after receiving the database clock and before
101
+ * returning this grant. This is a lower bound on the clock's age: uncertain
102
+ * database and HTTP transit time is excluded. Absent on older receivers.
103
+ */
104
+ serverClockElapsedMs?: number;
99
105
  /**
100
106
  * One globally reserved dispatch timestamp per granted permit. The runtime
101
107
  * may cache the block, but must not dispatch a permit before its timestamp.
@@ -4,7 +4,10 @@
4
4
  * learn from copies of the same provider feedback. These CTEs are shared by
5
5
  * pure-rate and concurrency-bearing admission, in the same atomic grant SQL.
6
6
  */
7
- import { DEFAULT_UNKNOWN_PROVIDER_FAST_RECOVERY_RPS } from '../governor/rate-state-backend';
7
+ import {
8
+ DEFAULT_UNKNOWN_PROVIDER_FAST_RECOVERY_RPS,
9
+ DEFAULT_UNKNOWN_PROVIDER_MIN_RPS,
10
+ } from '../governor/rate-state-backend';
8
11
 
9
12
  const AIMD_RECOVERY_STEP_MS = 1_000;
10
13
  // Increase relative to the learned rate, not by a fixed RPS amount: a long
@@ -14,15 +17,17 @@ const AIMD_RECOVERY_STEP_MS = 1_000;
14
17
  const AIMD_ADDITIVE_STEP_FRACTION = 0.25;
15
18
  // Keep upward probes small enough to retain headroom after a productive
16
19
  // decrease. Sparse successes use the earlier observation window; sustained
17
- // useful work can establish another small probe each second.
20
+ // useful work at nearly the current target can establish another small probe
21
+ // each second. Demand or historical successes cannot certify the next target.
18
22
  const UNKNOWN_PROVIDER_PROBE_STEP_RPS = 0.1;
19
23
  const UNKNOWN_PROVIDER_PROBE_INTERVAL_CAP_MS = 5_000;
20
- const UNKNOWN_PROVIDER_PRODUCTIVE_RATE_FRACTION = 0.8;
24
+ const UNKNOWN_PROVIDER_PRODUCTIVE_RATE_FRACTION = 0.95;
21
25
  export function acquireRateScopeSql(
22
26
  table: string,
23
27
  successesParameter: string,
24
28
  minimumRateSql = 'NULL::float8',
25
29
  unknownProvider = false,
30
+ probeBaselineSupported = true,
26
31
  ): string {
27
32
  const refillRate = `CASE WHEN ${minimumRateSql} IS NULL
28
33
  THEN h.refill_per_sec
@@ -35,11 +40,29 @@ export function acquireRateScopeSql(
35
40
  // Success counts are already applied once by the feedback writer. Measure
36
41
  // this post-hold epoch, excluding Retry-After and earlier idle history. A
37
42
  // productive second justifies a small new probe; a lone completion does not.
43
+ const probeStartedAtMs = probeBaselineSupported
44
+ ? 'GREATEST(h.rate_probe_started_at_ms, h.rate_observation_started_at_ms)'
45
+ : 'h.rate_observation_started_at_ms';
46
+ const probeSuccesses = probeBaselineSupported
47
+ ? 'GREATEST(0, h.rate_successes - h.rate_probe_successes_at_ramp)'
48
+ : 'h.rate_successes';
49
+ const productiveRateFraction = probeBaselineSupported
50
+ ? UNKNOWN_PROVIDER_PRODUCTIVE_RATE_FRACTION
51
+ : 0.8;
52
+ const storedRecoveryInterval = `LEAST(${UNKNOWN_PROVIDER_PROBE_INTERVAL_CAP_MS}, GREATEST(${AIMD_RECOVERY_STEP_MS}, h.rate_recovery_interval_ms))`;
53
+ const sparseRecoveryInterval = probeBaselineSupported
54
+ ? `CASE WHEN ${refillRate} > ${DEFAULT_UNKNOWN_PROVIDER_FAST_RECOVERY_RPS}
55
+ THEN ${UNKNOWN_PROVIDER_PROBE_INTERVAL_CAP_MS} ELSE ${storedRecoveryInterval} END`
56
+ : storedRecoveryInterval;
57
+ const floorRecoveryInterval = probeBaselineSupported
58
+ ? `GREATEST(${AIMD_RECOVERY_STEP_MS}, h.rate_recovery_interval_ms)`
59
+ : 'h.rate_recovery_interval_ms';
38
60
  const productiveRecovery = `h.rate_observation_started_at_ms IS NOT NULL
39
- AND nv.now_ms - h.rate_observation_started_at_ms >= ${AIMD_RECOVERY_STEP_MS}
40
- AND h.rate_successes * 1000.0
41
- / GREATEST(${AIMD_RECOVERY_STEP_MS}, nv.now_ms - h.rate_observation_started_at_ms)
42
- >= ${refillRate} * ${UNKNOWN_PROVIDER_PRODUCTIVE_RATE_FRACTION}`;
61
+ ${probeBaselineSupported ? 'AND h.rate_probe_started_at_ms >= h.last_ramp_at_ms AND h.rate_probe_started_at_ms >= h.rate_observation_started_at_ms' : ''}
62
+ AND nv.now_ms - ${probeStartedAtMs} >= ${AIMD_RECOVERY_STEP_MS}
63
+ AND ${probeSuccesses} * 1000.0
64
+ / GREATEST(${AIMD_RECOVERY_STEP_MS}, nv.now_ms - ${probeStartedAtMs})
65
+ >= ${refillRate} * ${productiveRateFraction}`;
43
66
  // Below useful seed capacity, a confirmed success can prove the next probe
44
67
  // after one dispatch interval at the current rate. Keeping the floor's 500ms
45
68
  // interval at 4/8 RPS waits through additional clean sends unnecessarily.
@@ -47,9 +70,29 @@ export function acquireRateScopeSql(
47
70
  const backpressuredRecoveryInterval = unknownProvider
48
71
  ? `CASE WHEN ${refillRate} >= ${DEFAULT_UNKNOWN_PROVIDER_FAST_RECOVERY_RPS}
49
72
  THEN CASE WHEN ${productiveRecovery} THEN ${AIMD_RECOVERY_STEP_MS}
50
- ELSE LEAST(${UNKNOWN_PROVIDER_PROBE_INTERVAL_CAP_MS}, GREATEST(${AIMD_RECOVERY_STEP_MS}, h.rate_recovery_interval_ms)) END
51
- ELSE LEAST(h.rate_recovery_interval_ms, CEIL(1000.0 / ${refillRate})) END`
73
+ ELSE ${sparseRecoveryInterval} END
74
+ ELSE LEAST(${floorRecoveryInterval}, CEIL(1000.0 / ${refillRate})) END`
52
75
  : `LEAST(h.rate_recovery_interval_ms, ${AIMD_RECOVERY_STEP_MS})`;
76
+ // A local no-progress pause remains active after the provider's shorter
77
+ // Retry-After. Only time beyond every outstanding dispatch/hold is idle.
78
+ const observationWasIdle = `${successesParameter}::int = 0
79
+ AND nv.now_ms - GREATEST(h.refilled_at_ms, h.next_dispatch_at_ms, h.retry_at_ms, h.cool_until_ms)
80
+ > GREATEST(1000, 2000.0 / ${refillRate})`;
81
+ // A denied refill may fold a completion before the next floor interval is
82
+ // due. Keep that fresh completion available for one later ramp rather than
83
+ // requiring another provider call at the old pace. Its baseline advances on
84
+ // every floor ramp, so time alone cannot reuse it for another increase.
85
+ const recoverySuccess =
86
+ unknownProvider && probeBaselineSupported
87
+ ? `(${successesParameter}::int > 0 OR (
88
+ h.seen_backpressure
89
+ AND ${refillRate} < ${DEFAULT_UNKNOWN_PROVIDER_FAST_RECOVERY_RPS}
90
+ AND h.rate_probe_started_at_ms >= h.last_ramp_at_ms
91
+ AND h.rate_probe_started_at_ms >= h.rate_observation_started_at_ms
92
+ AND h.rate_successes > h.rate_probe_successes_at_ramp
93
+ AND NOT (${observationWasIdle})
94
+ ))`
95
+ : `${successesParameter}::int > 0`;
53
96
  return `provider_hold_rows AS (
54
97
  SELECT * FROM ${table} WHERE bucket_id = $1 FOR UPDATE
55
98
  ), provider_hold AS (
@@ -58,12 +101,8 @@ export function acquireRateScopeSql(
58
101
  GREATEST(CASE WHEN h.seen_backpressure THEN 1 ELSE h.capacity END, ${refillRate}),
59
102
  h.tokens + GREATEST(0, nv.now_ms - h.refilled_at_ms) * ${refillRate} / 1000.0
60
103
  ) AS tokens_after_refill,
61
- ${successesParameter}::int = 0
62
- -- A local no-progress pause remains an active observation episode,
63
- -- even after the provider's shorter Retry-After has elapsed.
64
- AND nv.now_ms - GREATEST(h.refilled_at_ms, h.next_dispatch_at_ms, h.retry_at_ms, h.cool_until_ms)
65
- > GREATEST(1000, 2000.0 / ${refillRate}) AS observation_was_idle,
66
- CASE WHEN ${successesParameter}::int > 0
104
+ ${observationWasIdle} AS observation_was_idle,
105
+ CASE WHEN ${recoverySuccess}
67
106
  AND h.rate_observation_started_at_ms IS NOT NULL
68
107
  AND nv.now_ms - h.last_ramp_at_ms >= CASE WHEN h.seen_backpressure
69
108
  THEN ${backpressuredRecoveryInterval}
@@ -73,7 +112,7 @@ export function acquireRateScopeSql(
73
112
  THEN ${recoveryRate}
74
113
  ELSE ${refillRate} * 2 END)
75
114
  ELSE ${refillRate} END AS next_refill_per_sec,
76
- CASE WHEN ${successesParameter}::int > 0
115
+ CASE WHEN ${recoverySuccess}
77
116
  AND h.rate_observation_started_at_ms IS NOT NULL
78
117
  AND nv.now_ms - h.last_ramp_at_ms >= CASE WHEN h.seen_backpressure
79
118
  THEN ${backpressuredRecoveryInterval}
@@ -87,13 +126,24 @@ export function acquireRateScopeSql(
87
126
  export function updateRateScopeSql(
88
127
  table: string,
89
128
  unknownProvider = false,
129
+ probeBaselineSupported = true,
90
130
  ): string {
91
- // Floor recovery and productive probing measure different rates. Once the
92
- // geometric ramp reaches useful capacity, start a fresh measurement instead
93
- // of diluting its successful throughput with the earlier slow-start period.
131
+ // Keep the throttle epoch's productive evidence while measuring each upward
132
+ // probe from a new counter baseline. Losing the earlier useful work would
133
+ // turn an exploratory rejection into a zero-progress failure.
94
134
  const recoveredObservation = `${unknownProvider} AND ph.seen_backpressure
95
135
  AND ph.refill_per_sec < ${DEFAULT_UNKNOWN_PROVIDER_FAST_RECOVERY_RPS}
96
136
  AND ph.next_refill_per_sec >= ${DEFAULT_UNKNOWN_PROVIDER_FAST_RECOVERY_RPS}`;
137
+ const probedObservation = `${unknownProvider} AND ph.seen_backpressure
138
+ AND ph.refill_per_sec >= ${DEFAULT_UNKNOWN_PROVIDER_FAST_RECOVERY_RPS}
139
+ AND ph.next_refill_per_sec > ph.refill_per_sec`;
140
+ const floorRampedObservation = `${unknownProvider} AND ph.seen_backpressure
141
+ AND ph.refill_per_sec < ${DEFAULT_UNKNOWN_PROVIDER_FAST_RECOVERY_RPS}
142
+ AND ph.next_refill_per_sec > ph.refill_per_sec`;
143
+ const idleFloorObservation = `ph.observation_was_idle
144
+ AND ph.refill_per_sec < ${DEFAULT_UNKNOWN_PROVIDER_FAST_RECOVERY_RPS}`;
145
+ const rebaseProbe =
146
+ 'ph.rate_probe_started_at_ms < ph.last_ramp_at_ms OR ph.rate_probe_started_at_ms < COALESCE(ph.rate_observation_started_at_ms, 0) OR ph.rate_probe_successes_at_ramp > ph.rate_successes';
97
147
  return `updated_scope AS (
98
148
  UPDATE ${table} h SET
99
149
  tokens = ph.tokens_after_refill - (SELECT granted FROM grant_calc),
@@ -112,13 +162,34 @@ export function updateRateScopeSql(
112
162
  AND ph.rate_observation_started_at_ms IS NULL THEN 0
113
163
  WHEN (SELECT granted FROM grant_calc) > 0 AND ph.observation_was_idle THEN 0
114
164
  ELSE ph.rate_successes END,
115
- -- Do not turn an earlier productive measurement window into a failed
116
- -- probe streak when an unknown provider starts a new run after idle.
117
- -- Active Retry-After/local cooldowns are excluded by observation_was_idle.
165
+ ${
166
+ unknownProvider && probeBaselineSupported
167
+ ? `rate_probe_successes_at_ramp = CASE
168
+ WHEN ${recoveredObservation} THEN 0
169
+ WHEN (SELECT granted FROM grant_calc) > 0
170
+ AND (ph.rate_observation_started_at_ms IS NULL OR ph.observation_was_idle) THEN 0
171
+ WHEN ${idleFloorObservation} THEN ph.rate_successes
172
+ WHEN ${floorRampedObservation} OR ${probedObservation} OR (${rebaseProbe}) THEN ph.rate_successes
173
+ ELSE ph.rate_probe_successes_at_ramp END,
174
+ rate_probe_started_at_ms = CASE
175
+ WHEN ${idleFloorObservation} OR ${recoveredObservation} OR ${floorRampedObservation} OR ${probedObservation} OR (${rebaseProbe}) THEN nv.now_ms
176
+ WHEN (SELECT granted FROM grant_calc) > 0
177
+ AND (ph.rate_observation_started_at_ms IS NULL OR ph.observation_was_idle) THEN nv.now_ms
178
+ ELSE ph.rate_probe_started_at_ms END,`
179
+ : ''
180
+ }
181
+ -- At the floor, a new successful epoch must not inherit failed-probe
182
+ -- backoff. Above the floor, retain its useful measurement duration for
183
+ -- sparse upward probes; that rate already proves successful recovery.
118
184
  rate_recovery_interval_ms = CASE
119
185
  WHEN ${unknownProvider} AND (SELECT granted FROM grant_calc) > 0
120
- AND ph.observation_was_idle AND ph.rate_successes > 0
121
- THEN ${AIMD_RECOVERY_STEP_MS}
186
+ AND ph.rate_successes > 0
187
+ AND ph.next_refill_per_sec <= ${DEFAULT_UNKNOWN_PROVIDER_MIN_RPS}
188
+ AND (ph.observation_was_idle
189
+ OR ph.rate_observation_started_at_ms IS NULL)
190
+ -- Zero is an unmeasured new episode, not a failed-probe interval.
191
+ -- A real failure establishes the interval before it can double.
192
+ THEN ${probeBaselineSupported ? 0 : AIMD_RECOVERY_STEP_MS}
122
193
  ELSE ph.rate_recovery_interval_ms END,
123
194
  rate_throttle_ceiling_rps = CASE
124
195
  WHEN (SELECT granted FROM grant_calc) > 0