deepline 0.3.332 → 0.3.333
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/sdk/src/release.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/context.ts +26 -23
- package/dist/bundling-sources/shared_libs/play-runtime/execution-gateway-invocation-fence.ts +16 -0
- package/dist/bundling-sources/shared_libs/play-runtime/governor/app-runtime-rate-state-backend.ts +30 -9
- package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts +6 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-rate-scope.ts +95 -24
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-rate-state.ts +132 -37
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts +6 -3
- package/dist/bundling-sources/shared_libs/play-runtime/tool-execute-retry-policy.ts +87 -3
- package/dist/cli/auth-26GA3RM6.js +31 -0
- package/dist/cli/{auth-V4IWMXTG.mjs → auth-OQ356CEE.mjs} +4 -4
- package/dist/cli/{billing-BW37QW4H.js → billing-3APFTHNY.js} +71 -71
- package/dist/cli/{billing-GYW4BCPG.mjs → billing-7YVHLDMO.mjs} +4 -4
- package/dist/cli/{chunk-3E7P62NA.mjs → chunk-2CLENWOQ.mjs} +1 -1
- package/dist/cli/{chunk-GDBJYM5N.js → chunk-3WQJWQ5T.js} +60 -60
- package/dist/cli/{chunk-MXJVBJYC.mjs → chunk-4DUEB2WO.mjs} +2 -2
- package/dist/cli/{chunk-PK4EE6T3.js → chunk-5A44CWQE.js} +48 -48
- package/dist/cli/{chunk-LAJ56FLL.mjs → chunk-5UN6KLD2.mjs} +1 -1
- package/dist/cli/{chunk-ERPCUHOQ.js → chunk-64CZWDWT.js} +8 -8
- package/dist/cli/{chunk-5FHGKEAD.mjs → chunk-6JPZJ5W7.mjs} +2 -2
- package/dist/cli/{chunk-J5Z5BQGE.mjs → chunk-ABVOFJAE.mjs} +1 -1
- package/dist/cli/{chunk-U2B5KDNH.js → chunk-AFVETOG3.js} +15 -15
- package/dist/cli/{chunk-UMKN2BZK.mjs → chunk-AGGDEQDF.mjs} +1 -1
- package/dist/cli/{chunk-CYLZE6RJ.js → chunk-DROPDYOD.js} +1 -1
- package/dist/cli/{chunk-XON4H3CO.js → chunk-F2QKAGST.js} +9 -9
- package/dist/cli/{chunk-HKQMCTEE.mjs → chunk-F2XAQHQC.mjs} +4 -4
- package/dist/cli/{chunk-MSQK2FLJ.mjs → chunk-H5SORD7Q.mjs} +2 -2
- package/dist/cli/{chunk-WQI642QX.mjs → chunk-HNX3RC6V.mjs} +1 -1
- package/dist/cli/{chunk-AF6XLGZT.js → chunk-IDEK4DND.js} +2 -2
- package/dist/cli/{chunk-X6MY7BC2.js → chunk-IYFKFNEB.js} +225 -225
- package/dist/cli/{chunk-GOH5BRGN.js → chunk-KBXHB5P3.js} +26 -26
- package/dist/cli/{chunk-SBPJWA6Y.mjs → chunk-KREEYWVL.mjs} +6 -6
- package/dist/cli/{chunk-B6R2V3TH.mjs → chunk-N5KGBQPG.mjs} +3 -3
- package/dist/cli/{chunk-KYQPEIKA.mjs → chunk-NI4FJRFJ.mjs} +5 -5
- package/dist/cli/{chunk-T5TGWH4J.js → chunk-PRZ4S4NU.js} +6 -6
- package/dist/cli/{chunk-ZZWCK6F3.js → chunk-STVJ7ZPD.js} +6 -6
- package/dist/cli/{chunk-4RLN2DHZ.mjs → chunk-SXX4ST3D.mjs} +1 -1
- package/dist/cli/{chunk-TTJ7KELW.js → chunk-UHHTRBJF.js} +45 -45
- package/dist/cli/{chunk-JHF3GWKX.js → chunk-V77J5ISS.js} +4 -4
- package/dist/cli/client-IZRTEBU4.js +20 -0
- package/dist/cli/{client-TBCDCJ7T.mjs → client-WKS7HQRP.mjs} +3 -3
- package/dist/cli/{csv-EFQ7UL3M.js → csv-MMW6EKKD.js} +12 -12
- package/dist/cli/{csv-O3NEPSU3.mjs → csv-ZFY2LD2K.mjs} +3 -3
- package/dist/cli/{db-2FQ3T4LT.js → db-FMOQU6QW.js} +29 -29
- package/dist/cli/{db-RUWCFINP.mjs → db-Q2NJVILC.mjs} +4 -4
- package/dist/cli/{describe-YJFDN54P.js → describe-4KG2JAQV.js} +15 -15
- package/dist/cli/{describe-4KSVRYPM.mjs → describe-EWP7DMNU.mjs} +8 -8
- package/dist/cli/{executions-MG63LYMV.mjs → executions-UOD7FWEE.mjs} +4 -4
- package/dist/cli/{executions-O7IIWVRV.js → executions-Y4XSGKN6.js} +6 -6
- package/dist/cli/{feedback-WMYRDK2J.mjs → feedback-3F3EURZC.mjs} +3 -3
- package/dist/cli/{feedback-SR2X2H4N.js → feedback-TEJJUXDI.js} +6 -6
- package/dist/cli/{git-NIGE3SIJ.mjs → git-5YM22RGA.mjs} +4 -4
- package/dist/cli/git-GNEZSTVF.js +23 -0
- package/dist/cli/index.js +79 -79
- package/dist/cli/index.mjs +31 -31
- package/dist/cli/{legacy-noops-YXYYCDSR.mjs → legacy-noops-D5XBVUHS.mjs} +4 -4
- package/dist/cli/{legacy-noops-TXSBNOG6.js → legacy-noops-T5OUSXMR.js} +7 -7
- package/dist/cli/{monitors-QYAGRL6V.mjs → monitors-LPZNYNQW.mjs} +4 -4
- package/dist/cli/{monitors-ZZUQMDUA.js → monitors-VI7VBQRP.js} +85 -85
- package/dist/cli/{org-6NUUN2P7.mjs → org-BYIDOMJB.mjs} +4 -4
- package/dist/cli/{org-QQSCJJMM.js → org-OBGIV5QE.js} +60 -60
- package/dist/cli/play-5JYRSXAL.js +153 -0
- package/dist/cli/{play-3G2A53MD.mjs → play-Y5PQQRBO.mjs} +7 -7
- package/dist/cli/{providers-IGD3ZYTA.mjs → providers-GEGIGKOL.mjs} +4 -4
- package/dist/cli/{providers-AASOAAWT.js → providers-VH6J7FFY.js} +7 -7
- package/dist/cli/{quickstart-RRCLOSHQ.mjs → quickstart-GHITFT27.mjs} +3 -3
- package/dist/cli/{quickstart-NDUT5BVF.js → quickstart-ZQX5XNHG.js} +5 -5
- package/dist/cli/{search-37MO75IS.mjs → search-AFX3DYAC.mjs} +5 -5
- package/dist/cli/search-YIXGQFXA.js +26 -0
- package/dist/cli/{secrets-3OTOG64E.mjs → secrets-4EOVXQJM.mjs} +4 -4
- package/dist/cli/{secrets-QXY6DCJA.js → secrets-F37M7D7R.js} +13 -13
- package/dist/cli/{sessions-FML35Y64.mjs → sessions-RX5SFHDI.mjs} +4 -4
- package/dist/cli/{sessions-3P6DOM3H.js → sessions-T57Q72GD.js} +5 -5
- package/dist/cli/{settings-SD6MDI67.js → settings-3FVWW7GL.js} +74 -74
- package/dist/cli/{settings-MWFWIGHW.mjs → settings-764O5ZEH.mjs} +7 -7
- package/dist/cli/{setup-HY35HR26.js → setup-FQVRVTR6.js} +41 -41
- package/dist/cli/{setup-HDYPYPFU.mjs → setup-X7OAHFTO.mjs} +6 -6
- package/dist/cli/skills-DPGB67XX.js +28 -0
- package/dist/cli/{skills-TGEDPYTP.mjs → skills-O6ILHPV5.mjs} +4 -4
- package/dist/cli/{tools-VBARDH32.mjs → tools-HL4SQG35.mjs} +6 -6
- package/dist/cli/tools-WZCXSXER.js +40 -0
- package/dist/cli/update-63MANY7U.js +28 -0
- package/dist/cli/{update-OIDLOWGB.mjs → update-A3P2RSIB.mjs} +6 -6
- package/dist/cli/{workflow-ACHWLHPT.mjs → workflow-44424AVE.mjs} +5 -5
- package/dist/cli/{workflow-K2SJUC5Z.js → workflow-62AGKI6T.js} +15 -15
- package/dist/index.js +6 -6
- package/dist/index.mjs +3 -3
- package/dist/install-integrity.json +74 -74
- package/dist/release.d.mts +1 -1
- package/dist/release.d.ts +1 -1
- package/dist/release.js +2 -2
- package/dist/release.mjs +1 -1
- package/package.json +1 -1
- package/dist/cli/auth-6SVPO5HS.js +0 -31
- package/dist/cli/client-BPQKKACG.js +0 -20
- package/dist/cli/git-ZZU4LXWX.js +0 -23
- package/dist/cli/play-LLC44DQW.js +0 -153
- package/dist/cli/search-JWUKJSNY.js +0 -26
- package/dist/cli/skills-T3F4A4BQ.js +0 -28
- package/dist/cli/tools-PXC5VIRU.js +0 -40
- package/dist/cli/update-PIT4RK74.js +0 -28
|
@@ -200,7 +200,7 @@ export const SDK_RELEASE = {
|
|
|
200
200
|
// getters keep their established compatibility behavior.
|
|
201
201
|
// 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
|
|
202
202
|
// 0.3.90 is the first deliberately versioned SDK release for API v3.
|
|
203
|
-
version: '0.3.
|
|
203
|
+
version: '0.3.333',
|
|
204
204
|
updateSummary:
|
|
205
205
|
'Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.',
|
|
206
206
|
packageCapabilities: {
|
|
@@ -222,7 +222,8 @@ import {
|
|
|
222
222
|
classifyToolExecuteHttpFailure,
|
|
223
223
|
createToolExecuteHttpFailureAttemptTracker,
|
|
224
224
|
parseToolExecuteRetryAfterMs,
|
|
225
|
-
|
|
225
|
+
parseToolExecuteProviderRetryAtMs,
|
|
226
|
+
parseToolExecuteExactProviderRetryAtMs,
|
|
226
227
|
parseToolExecuteAuthScopeChangedError,
|
|
227
228
|
ToolExecuteAuthScopeChangedError,
|
|
228
229
|
type ToolExecuteHttpFailureOutcome,
|
|
@@ -17849,8 +17850,9 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
17849
17850
|
const executionRequestNonce = crypto.randomUUID();
|
|
17850
17851
|
// Transport ambiguity must retain the billing identity. An ordinary
|
|
17851
17852
|
// (unfenced) call advances it after a definite rejection released
|
|
17852
|
-
// that attempt's reservation
|
|
17853
|
-
//
|
|
17853
|
+
// that attempt's reservation. A proven app pacing refusal also
|
|
17854
|
+
// renews fenced billing attribution, keeping its invocation and
|
|
17855
|
+
// provider-operation identities unchanged. Ambiguous deliveries retain it.
|
|
17854
17856
|
let deeplineRequestId = `ctx-tool-${stableDigest(executionRequestNonce).slice(0, 32)}`;
|
|
17855
17857
|
// The provider operation has a semantic identity independent of
|
|
17856
17858
|
// the response receipt. Its key is supplied by the caller that owns
|
|
@@ -18647,13 +18649,8 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
18647
18649
|
// settle the row as failed instead of pacing.
|
|
18648
18650
|
[TOOL_EXECUTE_RUNNER_ACCEPTS_HEADER]:
|
|
18649
18651
|
TOOL_EXECUTE_RUNNER_ACCEPTS_VALUE,
|
|
18650
|
-
|
|
18651
|
-
|
|
18652
|
-
? {
|
|
18653
|
-
[TOOL_EXECUTE_APP_PACING_REFUSAL_HEADER]:
|
|
18654
|
-
TOOL_EXECUTE_APP_PACING_REFUSAL_VALUE,
|
|
18655
|
-
}
|
|
18656
|
-
: {}),
|
|
18652
|
+
[TOOL_EXECUTE_APP_PACING_REFUSAL_HEADER]:
|
|
18653
|
+
TOOL_EXECUTE_APP_PACING_REFUSAL_VALUE,
|
|
18657
18654
|
...(providerGatewayDispatchProof
|
|
18658
18655
|
? {
|
|
18659
18656
|
[TOOL_EXECUTE_PROVIDER_RATE_SCOPE_HEADER]:
|
|
@@ -18698,18 +18695,15 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
18698
18695
|
recordProviderDispatch();
|
|
18699
18696
|
if (dispatchedResponse.status === 429) {
|
|
18700
18697
|
const observedAt = Date.now();
|
|
18701
|
-
const retryAfter =
|
|
18702
|
-
dispatchedResponse.headers.get('retry-after');
|
|
18703
18698
|
// Preserve absolute provider deadlines even when the
|
|
18704
18699
|
// response did not carry the gateway's counter
|
|
18705
18700
|
// attestation. Classification below still decides
|
|
18706
18701
|
// whether this 429 may update shared provider pacing.
|
|
18707
|
-
providerRetryAtMs =
|
|
18708
|
-
|
|
18709
|
-
|
|
18710
|
-
|
|
18711
|
-
|
|
18712
|
-
: undefined;
|
|
18702
|
+
providerRetryAtMs =
|
|
18703
|
+
parseToolExecuteProviderRetryAtMs(
|
|
18704
|
+
dispatchedResponse,
|
|
18705
|
+
observedAt,
|
|
18706
|
+
);
|
|
18713
18707
|
}
|
|
18714
18708
|
if (
|
|
18715
18709
|
isAttestedProvider429Response(dispatchedResponse)
|
|
@@ -18877,6 +18871,10 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
18877
18871
|
schemaVersion: toolErrorSchemaVersion,
|
|
18878
18872
|
retryAfterHeader:
|
|
18879
18873
|
dispatchedResponse.headers.get('retry-after'),
|
|
18874
|
+
retryAtMs:
|
|
18875
|
+
parseToolExecuteExactProviderRetryAtMs(
|
|
18876
|
+
dispatchedResponse,
|
|
18877
|
+
),
|
|
18880
18878
|
transientHttpRetrySafe:
|
|
18881
18879
|
retrySafeTransientHttp || fixtureReplaySafe,
|
|
18882
18880
|
executionFenced: usesDurableInvocationFence,
|
|
@@ -18898,6 +18896,10 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
18898
18896
|
schemaVersion: toolErrorSchemaVersion,
|
|
18899
18897
|
retryAfterHeader:
|
|
18900
18898
|
dispatchedResponse.headers.get('retry-after'),
|
|
18899
|
+
retryAtMs:
|
|
18900
|
+
parseToolExecuteExactProviderRetryAtMs(
|
|
18901
|
+
dispatchedResponse,
|
|
18902
|
+
),
|
|
18901
18903
|
transientHttpRetrySafe:
|
|
18902
18904
|
retrySafeTransientHttp || fixtureReplaySafe,
|
|
18903
18905
|
executionFenced: usesDurableInvocationFence,
|
|
@@ -19294,6 +19296,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
19294
19296
|
bodyText: text,
|
|
19295
19297
|
schemaVersion: toolErrorSchemaVersion,
|
|
19296
19298
|
retryAfterHeader: response.headers.get('retry-after'),
|
|
19299
|
+
retryAtMs: parseToolExecuteExactProviderRetryAtMs(response),
|
|
19297
19300
|
transientHttpRetrySafe:
|
|
19298
19301
|
retrySafeTransientHttp || fixtureReplaySafe,
|
|
19299
19302
|
executionFenced: usesDurableInvocationFence,
|
|
@@ -19519,11 +19522,11 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
19519
19522
|
);
|
|
19520
19523
|
logProvider429RetryPhase('retry_sleep_completed');
|
|
19521
19524
|
if (
|
|
19522
|
-
|
|
19523
|
-
(
|
|
19524
|
-
failure.
|
|
19525
|
-
|
|
19526
|
-
|
|
19525
|
+
failure.reason === 'provider_pacing_wait' ||
|
|
19526
|
+
(!usesDurableInvocationFence &&
|
|
19527
|
+
(failure.backpressureDelayMs !== null ||
|
|
19528
|
+
(failure.reason === 'rate_limit' &&
|
|
19529
|
+
failure.knownNotDispatched)))
|
|
19527
19530
|
) {
|
|
19528
19531
|
// These refusals release the prior reservation. App pacing
|
|
19529
19532
|
// does not advance invocationAttempt, so use its own HTTP
|
package/dist/bundling-sources/shared_libs/play-runtime/execution-gateway-invocation-fence.ts
CHANGED
|
@@ -62,6 +62,14 @@ export type DurableInvocationDiagnostic = {
|
|
|
62
62
|
error?: unknown;
|
|
63
63
|
};
|
|
64
64
|
|
|
65
|
+
/** Authenticated transport proof that this owner sent no provider request. */
|
|
66
|
+
export class DurableInvocationDispatchNotStartedError extends Error {
|
|
67
|
+
constructor(readonly reason: string) {
|
|
68
|
+
super(reason);
|
|
69
|
+
this.name = 'DurableInvocationDispatchNotStartedError';
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
|
|
65
73
|
export class DurableInvocationConflictError extends Error {
|
|
66
74
|
readonly code = 'DURABLE_INVOCATION_REQUEST_CONFLICT';
|
|
67
75
|
|
|
@@ -313,6 +321,14 @@ export async function executeWithDurableInvocationFence(input: {
|
|
|
313
321
|
response = await input.execute();
|
|
314
322
|
} catch (error) {
|
|
315
323
|
diagnose('dispatch_failed', error);
|
|
324
|
+
if (error instanceof DurableInvocationDispatchNotStartedError) {
|
|
325
|
+
// An authenticated app refusal proves no provider request was sent.
|
|
326
|
+
// Release only this owner; ordinary transport failures stay unknown.
|
|
327
|
+
if (!(await input.store.abandon(begun))) {
|
|
328
|
+
throw new DurableInvocationInProgressError();
|
|
329
|
+
}
|
|
330
|
+
throw error;
|
|
331
|
+
}
|
|
316
332
|
if (input.replaySafeAfterDispatch) {
|
|
317
333
|
return await retryReplaySafeInvocation(begun);
|
|
318
334
|
}
|
package/dist/bundling-sources/shared_libs/play-runtime/governor/app-runtime-rate-state-backend.ts
CHANGED
|
@@ -140,7 +140,6 @@ function sleepWithAbort(
|
|
|
140
140
|
* hold more than a moment of a shared provider window.
|
|
141
141
|
*/
|
|
142
142
|
const RATE_BLOCK_HORIZON_MS = 1_000;
|
|
143
|
-
const UNKNOWN_PROVIDER_RECOVERY_BLOCK_HORIZON_MS = 500;
|
|
144
143
|
const STORE_FAILURE_RETRY_DELAYS_MS = [100, 250] as const;
|
|
145
144
|
/**
|
|
146
145
|
* Jitter fraction applied to a server-supplied `waitMs` so a synchronized wave
|
|
@@ -667,23 +666,22 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
|
|
|
667
666
|
// a hot map is a round trip per permit. The store's own token floor, not
|
|
668
667
|
// this number, is what bounds the grant.
|
|
669
668
|
const learnedRps = this.effectiveRps.get(laneKey);
|
|
669
|
+
const currentRps = learnedRps ?? configuredRps;
|
|
670
670
|
const recoveringUnknownProvider =
|
|
671
671
|
learnedRps !== undefined &&
|
|
672
672
|
learnedRps < DEFAULT_UNKNOWN_PROVIDER_FAST_RECOVERY_RPS &&
|
|
673
673
|
rules.some((rule) => rule.ruleId === 'default:provider-scope') &&
|
|
674
674
|
rules.every((rule) => rule.ruleId.startsWith('default:'));
|
|
675
|
-
// Below recovered capacity,
|
|
676
|
-
//
|
|
677
|
-
//
|
|
675
|
+
// Below recovered capacity, one dispatch interval (500/250/125ms at
|
|
676
|
+
// 2/4/8 RPS) can prove the next step. Reserve only that interval so cached
|
|
677
|
+
// old-pace tickets cannot postpone the next fresh-feedback refill.
|
|
678
678
|
const horizonMs = Math.min(
|
|
679
|
-
recoveringUnknownProvider
|
|
680
|
-
? UNKNOWN_PROVIDER_RECOVERY_BLOCK_HORIZON_MS
|
|
681
|
-
: RATE_BLOCK_HORIZON_MS,
|
|
679
|
+
recoveringUnknownProvider ? 1_000 / currentRps : RATE_BLOCK_HORIZON_MS,
|
|
682
680
|
leasedBlockTtlMs(rules),
|
|
683
681
|
);
|
|
684
682
|
const rateBudget = Math.max(
|
|
685
683
|
1,
|
|
686
|
-
Math.floor((
|
|
684
|
+
Math.floor((currentRps * horizonMs) / 1_000),
|
|
687
685
|
);
|
|
688
686
|
if (!Number.isFinite(rateBudget) || rateBudget <= 0) return floor;
|
|
689
687
|
const waiters = this.waiters.get(laneKey) ?? 1;
|
|
@@ -1018,7 +1016,8 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
|
|
|
1018
1016
|
const releaseGeneration = releaseSignal?.generation;
|
|
1019
1017
|
const successKey = pendingSuccessKey(bucketId, ordered);
|
|
1020
1018
|
const feedback = this.snapshotSuccessFeedback(successKey);
|
|
1021
|
-
const
|
|
1019
|
+
const requestStartedAtMs = this.now();
|
|
1020
|
+
const rawResponse = await this.port.acquire({
|
|
1022
1021
|
bucketId,
|
|
1023
1022
|
rateScopeToken,
|
|
1024
1023
|
rules: withAdaptiveMaxRps(ordered),
|
|
@@ -1026,6 +1025,28 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
|
|
|
1026
1025
|
...feedback,
|
|
1027
1026
|
schedulerSchema: this.schedulerSchema,
|
|
1028
1027
|
});
|
|
1028
|
+
const serverElapsedMs = rawResponse.serverClockElapsedMs;
|
|
1029
|
+
const validServerElapsed =
|
|
1030
|
+
typeof rawResponse.serverNowMs === 'number' &&
|
|
1031
|
+
Number.isFinite(rawResponse.serverNowMs) &&
|
|
1032
|
+
rawResponse.serverNowMs >= 0 &&
|
|
1033
|
+
typeof serverElapsedMs === 'number' &&
|
|
1034
|
+
Number.isSafeInteger(serverElapsedMs) &&
|
|
1035
|
+
serverElapsedMs >= 0 &&
|
|
1036
|
+
serverElapsedMs <= this.now() - requestStartedAtMs;
|
|
1037
|
+
// The clock was already this old when the response left the server.
|
|
1038
|
+
// Advance only by confirmed elapsed time; unknown transit remains
|
|
1039
|
+
// conservative, and an invalid/legacy response keeps the old clock.
|
|
1040
|
+
const response = validServerElapsed
|
|
1041
|
+
? {
|
|
1042
|
+
...rawResponse,
|
|
1043
|
+
serverNowMs: rawResponse.serverNowMs! + serverElapsedMs,
|
|
1044
|
+
waitMs:
|
|
1045
|
+
rawResponse.waitMs > 0
|
|
1046
|
+
? Math.max(1, rawResponse.waitMs - serverElapsedMs)
|
|
1047
|
+
: rawResponse.waitMs,
|
|
1048
|
+
}
|
|
1049
|
+
: rawResponse;
|
|
1029
1050
|
this.acknowledgeSuccessFeedback(successKey, feedback);
|
|
1030
1051
|
this.recordCooldown(bucketId, response);
|
|
1031
1052
|
if (response.scopeValidationSupported === true)
|
|
@@ -96,6 +96,12 @@ export type PlayRunnerRateStateAcquireResult = {
|
|
|
96
96
|
* provider spacing.
|
|
97
97
|
*/
|
|
98
98
|
serverNowMs?: number;
|
|
99
|
+
/**
|
|
100
|
+
* Confirmed elapsed time after receiving the database clock and before
|
|
101
|
+
* returning this grant. This is a lower bound on the clock's age: uncertain
|
|
102
|
+
* database and HTTP transit time is excluded. Absent on older receivers.
|
|
103
|
+
*/
|
|
104
|
+
serverClockElapsedMs?: number;
|
|
99
105
|
/**
|
|
100
106
|
* One globally reserved dispatch timestamp per granted permit. The runtime
|
|
101
107
|
* may cache the block, but must not dispatch a permit before its timestamp.
|
package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-rate-scope.ts
CHANGED
|
@@ -4,7 +4,10 @@
|
|
|
4
4
|
* learn from copies of the same provider feedback. These CTEs are shared by
|
|
5
5
|
* pure-rate and concurrency-bearing admission, in the same atomic grant SQL.
|
|
6
6
|
*/
|
|
7
|
-
import {
|
|
7
|
+
import {
|
|
8
|
+
DEFAULT_UNKNOWN_PROVIDER_FAST_RECOVERY_RPS,
|
|
9
|
+
DEFAULT_UNKNOWN_PROVIDER_MIN_RPS,
|
|
10
|
+
} from '../governor/rate-state-backend';
|
|
8
11
|
|
|
9
12
|
const AIMD_RECOVERY_STEP_MS = 1_000;
|
|
10
13
|
// Increase relative to the learned rate, not by a fixed RPS amount: a long
|
|
@@ -14,15 +17,17 @@ const AIMD_RECOVERY_STEP_MS = 1_000;
|
|
|
14
17
|
const AIMD_ADDITIVE_STEP_FRACTION = 0.25;
|
|
15
18
|
// Keep upward probes small enough to retain headroom after a productive
|
|
16
19
|
// decrease. Sparse successes use the earlier observation window; sustained
|
|
17
|
-
// useful work can establish another small probe
|
|
20
|
+
// useful work at nearly the current target can establish another small probe
|
|
21
|
+
// each second. Demand or historical successes cannot certify the next target.
|
|
18
22
|
const UNKNOWN_PROVIDER_PROBE_STEP_RPS = 0.1;
|
|
19
23
|
const UNKNOWN_PROVIDER_PROBE_INTERVAL_CAP_MS = 5_000;
|
|
20
|
-
const UNKNOWN_PROVIDER_PRODUCTIVE_RATE_FRACTION = 0.
|
|
24
|
+
const UNKNOWN_PROVIDER_PRODUCTIVE_RATE_FRACTION = 0.95;
|
|
21
25
|
export function acquireRateScopeSql(
|
|
22
26
|
table: string,
|
|
23
27
|
successesParameter: string,
|
|
24
28
|
minimumRateSql = 'NULL::float8',
|
|
25
29
|
unknownProvider = false,
|
|
30
|
+
probeBaselineSupported = true,
|
|
26
31
|
): string {
|
|
27
32
|
const refillRate = `CASE WHEN ${minimumRateSql} IS NULL
|
|
28
33
|
THEN h.refill_per_sec
|
|
@@ -35,11 +40,29 @@ export function acquireRateScopeSql(
|
|
|
35
40
|
// Success counts are already applied once by the feedback writer. Measure
|
|
36
41
|
// this post-hold epoch, excluding Retry-After and earlier idle history. A
|
|
37
42
|
// productive second justifies a small new probe; a lone completion does not.
|
|
43
|
+
const probeStartedAtMs = probeBaselineSupported
|
|
44
|
+
? 'GREATEST(h.rate_probe_started_at_ms, h.rate_observation_started_at_ms)'
|
|
45
|
+
: 'h.rate_observation_started_at_ms';
|
|
46
|
+
const probeSuccesses = probeBaselineSupported
|
|
47
|
+
? 'GREATEST(0, h.rate_successes - h.rate_probe_successes_at_ramp)'
|
|
48
|
+
: 'h.rate_successes';
|
|
49
|
+
const productiveRateFraction = probeBaselineSupported
|
|
50
|
+
? UNKNOWN_PROVIDER_PRODUCTIVE_RATE_FRACTION
|
|
51
|
+
: 0.8;
|
|
52
|
+
const storedRecoveryInterval = `LEAST(${UNKNOWN_PROVIDER_PROBE_INTERVAL_CAP_MS}, GREATEST(${AIMD_RECOVERY_STEP_MS}, h.rate_recovery_interval_ms))`;
|
|
53
|
+
const sparseRecoveryInterval = probeBaselineSupported
|
|
54
|
+
? `CASE WHEN ${refillRate} > ${DEFAULT_UNKNOWN_PROVIDER_FAST_RECOVERY_RPS}
|
|
55
|
+
THEN ${UNKNOWN_PROVIDER_PROBE_INTERVAL_CAP_MS} ELSE ${storedRecoveryInterval} END`
|
|
56
|
+
: storedRecoveryInterval;
|
|
57
|
+
const floorRecoveryInterval = probeBaselineSupported
|
|
58
|
+
? `GREATEST(${AIMD_RECOVERY_STEP_MS}, h.rate_recovery_interval_ms)`
|
|
59
|
+
: 'h.rate_recovery_interval_ms';
|
|
38
60
|
const productiveRecovery = `h.rate_observation_started_at_ms IS NOT NULL
|
|
39
|
-
AND
|
|
40
|
-
AND
|
|
41
|
-
|
|
42
|
-
|
|
61
|
+
${probeBaselineSupported ? 'AND h.rate_probe_started_at_ms >= h.last_ramp_at_ms AND h.rate_probe_started_at_ms >= h.rate_observation_started_at_ms' : ''}
|
|
62
|
+
AND nv.now_ms - ${probeStartedAtMs} >= ${AIMD_RECOVERY_STEP_MS}
|
|
63
|
+
AND ${probeSuccesses} * 1000.0
|
|
64
|
+
/ GREATEST(${AIMD_RECOVERY_STEP_MS}, nv.now_ms - ${probeStartedAtMs})
|
|
65
|
+
>= ${refillRate} * ${productiveRateFraction}`;
|
|
43
66
|
// Below useful seed capacity, a confirmed success can prove the next probe
|
|
44
67
|
// after one dispatch interval at the current rate. Keeping the floor's 500ms
|
|
45
68
|
// interval at 4/8 RPS waits through additional clean sends unnecessarily.
|
|
@@ -47,9 +70,29 @@ export function acquireRateScopeSql(
|
|
|
47
70
|
const backpressuredRecoveryInterval = unknownProvider
|
|
48
71
|
? `CASE WHEN ${refillRate} >= ${DEFAULT_UNKNOWN_PROVIDER_FAST_RECOVERY_RPS}
|
|
49
72
|
THEN CASE WHEN ${productiveRecovery} THEN ${AIMD_RECOVERY_STEP_MS}
|
|
50
|
-
ELSE
|
|
51
|
-
ELSE LEAST(
|
|
73
|
+
ELSE ${sparseRecoveryInterval} END
|
|
74
|
+
ELSE LEAST(${floorRecoveryInterval}, CEIL(1000.0 / ${refillRate})) END`
|
|
52
75
|
: `LEAST(h.rate_recovery_interval_ms, ${AIMD_RECOVERY_STEP_MS})`;
|
|
76
|
+
// A local no-progress pause remains active after the provider's shorter
|
|
77
|
+
// Retry-After. Only time beyond every outstanding dispatch/hold is idle.
|
|
78
|
+
const observationWasIdle = `${successesParameter}::int = 0
|
|
79
|
+
AND nv.now_ms - GREATEST(h.refilled_at_ms, h.next_dispatch_at_ms, h.retry_at_ms, h.cool_until_ms)
|
|
80
|
+
> GREATEST(1000, 2000.0 / ${refillRate})`;
|
|
81
|
+
// A denied refill may fold a completion before the next floor interval is
|
|
82
|
+
// due. Keep that fresh completion available for one later ramp rather than
|
|
83
|
+
// requiring another provider call at the old pace. Its baseline advances on
|
|
84
|
+
// every floor ramp, so time alone cannot reuse it for another increase.
|
|
85
|
+
const recoverySuccess =
|
|
86
|
+
unknownProvider && probeBaselineSupported
|
|
87
|
+
? `(${successesParameter}::int > 0 OR (
|
|
88
|
+
h.seen_backpressure
|
|
89
|
+
AND ${refillRate} < ${DEFAULT_UNKNOWN_PROVIDER_FAST_RECOVERY_RPS}
|
|
90
|
+
AND h.rate_probe_started_at_ms >= h.last_ramp_at_ms
|
|
91
|
+
AND h.rate_probe_started_at_ms >= h.rate_observation_started_at_ms
|
|
92
|
+
AND h.rate_successes > h.rate_probe_successes_at_ramp
|
|
93
|
+
AND NOT (${observationWasIdle})
|
|
94
|
+
))`
|
|
95
|
+
: `${successesParameter}::int > 0`;
|
|
53
96
|
return `provider_hold_rows AS (
|
|
54
97
|
SELECT * FROM ${table} WHERE bucket_id = $1 FOR UPDATE
|
|
55
98
|
), provider_hold AS (
|
|
@@ -58,12 +101,8 @@ export function acquireRateScopeSql(
|
|
|
58
101
|
GREATEST(CASE WHEN h.seen_backpressure THEN 1 ELSE h.capacity END, ${refillRate}),
|
|
59
102
|
h.tokens + GREATEST(0, nv.now_ms - h.refilled_at_ms) * ${refillRate} / 1000.0
|
|
60
103
|
) AS tokens_after_refill,
|
|
61
|
-
${
|
|
62
|
-
|
|
63
|
-
-- even after the provider's shorter Retry-After has elapsed.
|
|
64
|
-
AND nv.now_ms - GREATEST(h.refilled_at_ms, h.next_dispatch_at_ms, h.retry_at_ms, h.cool_until_ms)
|
|
65
|
-
> GREATEST(1000, 2000.0 / ${refillRate}) AS observation_was_idle,
|
|
66
|
-
CASE WHEN ${successesParameter}::int > 0
|
|
104
|
+
${observationWasIdle} AS observation_was_idle,
|
|
105
|
+
CASE WHEN ${recoverySuccess}
|
|
67
106
|
AND h.rate_observation_started_at_ms IS NOT NULL
|
|
68
107
|
AND nv.now_ms - h.last_ramp_at_ms >= CASE WHEN h.seen_backpressure
|
|
69
108
|
THEN ${backpressuredRecoveryInterval}
|
|
@@ -73,7 +112,7 @@ export function acquireRateScopeSql(
|
|
|
73
112
|
THEN ${recoveryRate}
|
|
74
113
|
ELSE ${refillRate} * 2 END)
|
|
75
114
|
ELSE ${refillRate} END AS next_refill_per_sec,
|
|
76
|
-
CASE WHEN ${
|
|
115
|
+
CASE WHEN ${recoverySuccess}
|
|
77
116
|
AND h.rate_observation_started_at_ms IS NOT NULL
|
|
78
117
|
AND nv.now_ms - h.last_ramp_at_ms >= CASE WHEN h.seen_backpressure
|
|
79
118
|
THEN ${backpressuredRecoveryInterval}
|
|
@@ -87,13 +126,24 @@ export function acquireRateScopeSql(
|
|
|
87
126
|
export function updateRateScopeSql(
|
|
88
127
|
table: string,
|
|
89
128
|
unknownProvider = false,
|
|
129
|
+
probeBaselineSupported = true,
|
|
90
130
|
): string {
|
|
91
|
-
//
|
|
92
|
-
//
|
|
93
|
-
//
|
|
131
|
+
// Keep the throttle epoch's productive evidence while measuring each upward
|
|
132
|
+
// probe from a new counter baseline. Losing the earlier useful work would
|
|
133
|
+
// turn an exploratory rejection into a zero-progress failure.
|
|
94
134
|
const recoveredObservation = `${unknownProvider} AND ph.seen_backpressure
|
|
95
135
|
AND ph.refill_per_sec < ${DEFAULT_UNKNOWN_PROVIDER_FAST_RECOVERY_RPS}
|
|
96
136
|
AND ph.next_refill_per_sec >= ${DEFAULT_UNKNOWN_PROVIDER_FAST_RECOVERY_RPS}`;
|
|
137
|
+
const probedObservation = `${unknownProvider} AND ph.seen_backpressure
|
|
138
|
+
AND ph.refill_per_sec >= ${DEFAULT_UNKNOWN_PROVIDER_FAST_RECOVERY_RPS}
|
|
139
|
+
AND ph.next_refill_per_sec > ph.refill_per_sec`;
|
|
140
|
+
const floorRampedObservation = `${unknownProvider} AND ph.seen_backpressure
|
|
141
|
+
AND ph.refill_per_sec < ${DEFAULT_UNKNOWN_PROVIDER_FAST_RECOVERY_RPS}
|
|
142
|
+
AND ph.next_refill_per_sec > ph.refill_per_sec`;
|
|
143
|
+
const idleFloorObservation = `ph.observation_was_idle
|
|
144
|
+
AND ph.refill_per_sec < ${DEFAULT_UNKNOWN_PROVIDER_FAST_RECOVERY_RPS}`;
|
|
145
|
+
const rebaseProbe =
|
|
146
|
+
'ph.rate_probe_started_at_ms < ph.last_ramp_at_ms OR ph.rate_probe_started_at_ms < COALESCE(ph.rate_observation_started_at_ms, 0) OR ph.rate_probe_successes_at_ramp > ph.rate_successes';
|
|
97
147
|
return `updated_scope AS (
|
|
98
148
|
UPDATE ${table} h SET
|
|
99
149
|
tokens = ph.tokens_after_refill - (SELECT granted FROM grant_calc),
|
|
@@ -112,13 +162,34 @@ export function updateRateScopeSql(
|
|
|
112
162
|
AND ph.rate_observation_started_at_ms IS NULL THEN 0
|
|
113
163
|
WHEN (SELECT granted FROM grant_calc) > 0 AND ph.observation_was_idle THEN 0
|
|
114
164
|
ELSE ph.rate_successes END,
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
165
|
+
${
|
|
166
|
+
unknownProvider && probeBaselineSupported
|
|
167
|
+
? `rate_probe_successes_at_ramp = CASE
|
|
168
|
+
WHEN ${recoveredObservation} THEN 0
|
|
169
|
+
WHEN (SELECT granted FROM grant_calc) > 0
|
|
170
|
+
AND (ph.rate_observation_started_at_ms IS NULL OR ph.observation_was_idle) THEN 0
|
|
171
|
+
WHEN ${idleFloorObservation} THEN ph.rate_successes
|
|
172
|
+
WHEN ${floorRampedObservation} OR ${probedObservation} OR (${rebaseProbe}) THEN ph.rate_successes
|
|
173
|
+
ELSE ph.rate_probe_successes_at_ramp END,
|
|
174
|
+
rate_probe_started_at_ms = CASE
|
|
175
|
+
WHEN ${idleFloorObservation} OR ${recoveredObservation} OR ${floorRampedObservation} OR ${probedObservation} OR (${rebaseProbe}) THEN nv.now_ms
|
|
176
|
+
WHEN (SELECT granted FROM grant_calc) > 0
|
|
177
|
+
AND (ph.rate_observation_started_at_ms IS NULL OR ph.observation_was_idle) THEN nv.now_ms
|
|
178
|
+
ELSE ph.rate_probe_started_at_ms END,`
|
|
179
|
+
: ''
|
|
180
|
+
}
|
|
181
|
+
-- At the floor, a new successful epoch must not inherit failed-probe
|
|
182
|
+
-- backoff. Above the floor, retain its useful measurement duration for
|
|
183
|
+
-- sparse upward probes; that rate already proves successful recovery.
|
|
118
184
|
rate_recovery_interval_ms = CASE
|
|
119
185
|
WHEN ${unknownProvider} AND (SELECT granted FROM grant_calc) > 0
|
|
120
|
-
AND ph.
|
|
121
|
-
|
|
186
|
+
AND ph.rate_successes > 0
|
|
187
|
+
AND ph.next_refill_per_sec <= ${DEFAULT_UNKNOWN_PROVIDER_MIN_RPS}
|
|
188
|
+
AND (ph.observation_was_idle
|
|
189
|
+
OR ph.rate_observation_started_at_ms IS NULL)
|
|
190
|
+
-- Zero is an unmeasured new episode, not a failed-probe interval.
|
|
191
|
+
-- A real failure establishes the interval before it can double.
|
|
192
|
+
THEN ${probeBaselineSupported ? 0 : AIMD_RECOVERY_STEP_MS}
|
|
122
193
|
ELSE ph.rate_recovery_interval_ms END,
|
|
123
194
|
rate_throttle_ceiling_rps = CASE
|
|
124
195
|
WHEN (SELECT granted FROM grant_calc) > 0
|