deepline 0.3.209 → 0.3.210
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/sdk/src/release.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/context.ts +28 -0
- package/dist/bundling-sources/shared_libs/play-runtime/governor/app-runtime-rate-state-backend.ts +95 -19
- package/dist/bundling-sources/shared_libs/play-runtime/governor/governor.ts +3 -3
- package/dist/bundling-sources/shared_libs/play-runtime/governor/policy.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/governor/rate-state-backend.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/run-failure.ts +21 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-rate-scope.ts +10 -6
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-rate-state.ts +37 -13
- package/dist/bundling-sources/shared_libs/play-runtime/tool-execute-retry-policy.ts +7 -2
- package/dist/cli/index.js +1 -1
- package/dist/cli/index.mjs +1 -1
- package/dist/index.js +1 -1
- package/dist/index.mjs +1 -1
- package/dist/release.d.mts +1 -1
- package/dist/release.d.ts +1 -1
- package/dist/release.js +1 -1
- package/dist/release.mjs +1 -1
- package/package.json +1 -1
|
@@ -200,7 +200,7 @@ export const SDK_RELEASE = {
|
|
|
200
200
|
// getters keep their established compatibility behavior.
|
|
201
201
|
// 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
|
|
202
202
|
// 0.3.90 is the first deliberately versioned SDK release for API v3.
|
|
203
|
-
version: '0.3.
|
|
203
|
+
version: '0.3.210',
|
|
204
204
|
updateSummary:
|
|
205
205
|
'Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.',
|
|
206
206
|
packageCapabilities: {
|
|
@@ -122,6 +122,7 @@ import {
|
|
|
122
122
|
import {
|
|
123
123
|
AllAdmittedRowsFailedError,
|
|
124
124
|
isProviderExhaustedFailure,
|
|
125
|
+
ProviderAdmissionWaitTimeoutError,
|
|
125
126
|
ProviderExhaustedError,
|
|
126
127
|
} from './run-failure';
|
|
127
128
|
import { PROVIDER_EXHAUSTED_MAX_WAIT_MS } from './governor/app-runtime-rate-state-backend';
|
|
@@ -18682,6 +18683,30 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
18682
18683
|
networkScope: null,
|
|
18683
18684
|
},
|
|
18684
18685
|
);
|
|
18686
|
+
const providerAdmissionTimeoutError = (
|
|
18687
|
+
error: ProviderAdmissionWaitTimeoutError,
|
|
18688
|
+
) =>
|
|
18689
|
+
createToolHttpError(
|
|
18690
|
+
toolErrorSchemaVersion,
|
|
18691
|
+
error.message,
|
|
18692
|
+
null,
|
|
18693
|
+
503,
|
|
18694
|
+
'repairable',
|
|
18695
|
+
{
|
|
18696
|
+
toolId,
|
|
18697
|
+
provider: error.provider,
|
|
18698
|
+
operation: toolId,
|
|
18699
|
+
code: error.code,
|
|
18700
|
+
origin: 'deepline',
|
|
18701
|
+
category: 'rate_limit',
|
|
18702
|
+
retryable: true,
|
|
18703
|
+
statusCode: 503,
|
|
18704
|
+
requestId: null,
|
|
18705
|
+
retryAfterMs: null,
|
|
18706
|
+
networkKind: null,
|
|
18707
|
+
networkScope: null,
|
|
18708
|
+
},
|
|
18709
|
+
);
|
|
18685
18710
|
const observeProviderWait = (retryAtMs: number) => {
|
|
18686
18711
|
const startedAtMsByScope =
|
|
18687
18712
|
(this.checkpoint.providerNoProgressStartedAtMsByScope ??= {});
|
|
@@ -19934,6 +19959,9 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
19934
19959
|
if (heartbeatFailure) {
|
|
19935
19960
|
throw heartbeatFailure;
|
|
19936
19961
|
}
|
|
19962
|
+
if (error instanceof ProviderAdmissionWaitTimeoutError) {
|
|
19963
|
+
throw providerAdmissionTimeoutError(error);
|
|
19964
|
+
}
|
|
19937
19965
|
if (error instanceof ProviderExhaustedError) {
|
|
19938
19966
|
const retryAtMs = Date.parse(error.retryAt);
|
|
19939
19967
|
observeProviderWait(
|
package/dist/bundling-sources/shared_libs/play-runtime/governor/app-runtime-rate-state-backend.ts
CHANGED
|
@@ -20,6 +20,7 @@ import {
|
|
|
20
20
|
type RateStateBackendKind,
|
|
21
21
|
} from './rate-state-backend';
|
|
22
22
|
import {
|
|
23
|
+
ProviderAdmissionWaitTimeoutError,
|
|
23
24
|
ProviderExhaustedError,
|
|
24
25
|
providerNameFromBucketId,
|
|
25
26
|
} from '../run-failure';
|
|
@@ -36,12 +37,18 @@ import type { PlayRunnerRateStateValidateResult } from '../protocol';
|
|
|
36
37
|
*/
|
|
37
38
|
export const PROVIDER_EXHAUSTED_MAX_WAIT_MS = 300_000;
|
|
38
39
|
/**
|
|
39
|
-
*
|
|
40
|
-
*
|
|
41
|
-
*
|
|
42
|
-
* starter parked here for twenty minutes with its core idle.
|
|
40
|
+
* Max total duration of a cached rate-state hold before it is yielded or its
|
|
41
|
+
* stale local no-progress warning is cleared. This is distinct from the shorter
|
|
42
|
+
* local permit-admission budget below.
|
|
43
43
|
*/
|
|
44
44
|
export const RATE_ACQUIRE_MAX_WAIT_MS = 300_000;
|
|
45
|
+
/**
|
|
46
|
+
* Total time one acquirer may spend waiting for a locally paced permit. This
|
|
47
|
+
* covers both empty-block refills and future-scheduled tickets. A spent budget
|
|
48
|
+
* is a repairable Deepline admission error, not another retry loop or a
|
|
49
|
+
* provider-owned Retry-After hold.
|
|
50
|
+
*/
|
|
51
|
+
export const PROVIDER_ADMISSION_WAIT_BUDGET_MS = 120_000;
|
|
45
52
|
const PROVIDER_NO_PROGRESS_WARNING_MS = 180_000;
|
|
46
53
|
|
|
47
54
|
/**
|
|
@@ -91,6 +98,31 @@ export const APP_RUNTIME_RATE_STATE_LEASE_BLOCK_SIZE = 8;
|
|
|
91
98
|
const LEASE_BLOCK_MIN_TTL_MS = 250;
|
|
92
99
|
const LEASE_BLOCK_MAX_TTL_MS = 5_000;
|
|
93
100
|
const MAX_ACQUIRE_SLEEP_MS = 5_000;
|
|
101
|
+
|
|
102
|
+
function abortReason(signal: AbortSignal): Error {
|
|
103
|
+
return signal.reason instanceof Error
|
|
104
|
+
? signal.reason
|
|
105
|
+
: new Error('Rate-state acquire aborted.');
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
function sleepWithAbort(ms: number, signal?: AbortSignal): Promise<void> {
|
|
109
|
+
if (signal?.aborted) return Promise.reject(abortReason(signal));
|
|
110
|
+
return new Promise((resolve, reject) => {
|
|
111
|
+
const cleanup = () => signal?.removeEventListener('abort', onAbort);
|
|
112
|
+
const timer = setTimeout(() => {
|
|
113
|
+
cleanup();
|
|
114
|
+
resolve();
|
|
115
|
+
}, ms);
|
|
116
|
+
const onAbort = () => {
|
|
117
|
+
clearTimeout(timer);
|
|
118
|
+
cleanup();
|
|
119
|
+
reject(abortReason(signal!));
|
|
120
|
+
};
|
|
121
|
+
signal?.addEventListener('abort', onAbort, { once: true });
|
|
122
|
+
if (signal?.aborted) onAbort();
|
|
123
|
+
});
|
|
124
|
+
}
|
|
125
|
+
|
|
94
126
|
/**
|
|
95
127
|
* Horizon used to size the next lease block from the rate actually in force.
|
|
96
128
|
* Kept short so a block stays inside its own TTL and a stalled runner can never
|
|
@@ -156,7 +188,7 @@ interface PendingRelease {
|
|
|
156
188
|
interface Options {
|
|
157
189
|
schedulerSchema?: string | null;
|
|
158
190
|
now?: () => number;
|
|
159
|
-
sleep?: (ms: number) => Promise<void>;
|
|
191
|
+
sleep?: (ms: number, signal?: AbortSignal) => Promise<void>;
|
|
160
192
|
onStoreFailure?: (info: { bucketId: string; error: string }) => void;
|
|
161
193
|
/** Override the transport (tests inject a counting/faulting port). */
|
|
162
194
|
port?: AppRuntimeRatePort;
|
|
@@ -177,7 +209,7 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
|
|
|
177
209
|
private readonly port: AppRuntimeRatePort;
|
|
178
210
|
private readonly schedulerSchema: string | null;
|
|
179
211
|
private readonly now: () => number;
|
|
180
|
-
private readonly sleep: (ms: number) => Promise<void>;
|
|
212
|
+
private readonly sleep: (ms: number, signal?: AbortSignal) => Promise<void>;
|
|
181
213
|
private readonly jitter: () => number;
|
|
182
214
|
private readonly onStoreFailure: (info: {
|
|
183
215
|
bucketId: string;
|
|
@@ -238,9 +270,7 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
|
|
|
238
270
|
this.port = options.port ?? contextRatePort(context);
|
|
239
271
|
this.schedulerSchema = options.schedulerSchema?.trim() || null;
|
|
240
272
|
this.now = options.now ?? (() => Date.now());
|
|
241
|
-
this.sleep =
|
|
242
|
-
options.sleep ??
|
|
243
|
-
((ms: number) => new Promise((resolve) => setTimeout(resolve, ms)));
|
|
273
|
+
this.sleep = options.sleep ?? sleepWithAbort;
|
|
244
274
|
this.jitter = options.jitter ?? Math.random;
|
|
245
275
|
this.onStoreFailure =
|
|
246
276
|
options.onStoreFailure ??
|
|
@@ -412,20 +442,47 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
|
|
|
412
442
|
const localPermit = this.drawFromBlock(bucketId, laneKey);
|
|
413
443
|
if (localPermit) {
|
|
414
444
|
const waitMs = localPermit.scheduledAtMs - this.now();
|
|
415
|
-
|
|
445
|
+
const elapsedMs = Math.max(0, this.now() - acquireStartedAt);
|
|
446
|
+
const remainingWaitMs = Math.max(
|
|
447
|
+
0,
|
|
448
|
+
PROVIDER_ADMISSION_WAIT_BUDGET_MS - elapsedMs,
|
|
449
|
+
);
|
|
450
|
+
if (
|
|
451
|
+
elapsedMs >= PROVIDER_ADMISSION_WAIT_BUDGET_MS ||
|
|
452
|
+
waitMs > remainingWaitMs
|
|
453
|
+
) {
|
|
454
|
+
await this.cancelUndispatchedPermit(localPermit);
|
|
455
|
+
throw new ProviderAdmissionWaitTimeoutError({
|
|
456
|
+
provider: providerNameFromBucketId(bucketId),
|
|
457
|
+
waitBudgetMs: PROVIDER_ADMISSION_WAIT_BUDGET_MS,
|
|
458
|
+
});
|
|
459
|
+
}
|
|
460
|
+
try {
|
|
461
|
+
if (waitMs > 0) await this.sleep(waitMs, signal);
|
|
462
|
+
} catch (error) {
|
|
463
|
+
await this.cancelUndispatchedPermit(localPermit);
|
|
464
|
+
throw error;
|
|
465
|
+
}
|
|
416
466
|
if (signal?.aborted) {
|
|
417
|
-
|
|
418
|
-
throw signal
|
|
419
|
-
|
|
420
|
-
|
|
467
|
+
await this.cancelUndispatchedPermit(localPermit);
|
|
468
|
+
throw abortReason(signal);
|
|
469
|
+
}
|
|
470
|
+
if (
|
|
471
|
+
this.now() - acquireStartedAt >= PROVIDER_ADMISSION_WAIT_BUDGET_MS
|
|
472
|
+
) {
|
|
473
|
+
await this.cancelUndispatchedPermit(localPermit);
|
|
474
|
+
throw new ProviderAdmissionWaitTimeoutError({
|
|
475
|
+
provider: providerNameFromBucketId(bucketId),
|
|
476
|
+
waitBudgetMs: PROVIDER_ADMISSION_WAIT_BUDGET_MS,
|
|
477
|
+
});
|
|
421
478
|
}
|
|
422
479
|
return localPermit.permit;
|
|
423
480
|
}
|
|
424
481
|
const waitedMs = this.now() - acquireStartedAt;
|
|
425
|
-
if (waitedMs >=
|
|
426
|
-
throw new
|
|
482
|
+
if (waitedMs >= PROVIDER_ADMISSION_WAIT_BUDGET_MS) {
|
|
483
|
+
throw new ProviderAdmissionWaitTimeoutError({
|
|
427
484
|
provider: providerNameFromBucketId(bucketId),
|
|
428
|
-
|
|
485
|
+
waitBudgetMs: PROVIDER_ADMISSION_WAIT_BUDGET_MS,
|
|
429
486
|
});
|
|
430
487
|
}
|
|
431
488
|
// Count this acquirer as waiting on the refill so the single-flight refill
|
|
@@ -1389,7 +1446,11 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
|
|
|
1389
1446
|
private drawFromBlock(
|
|
1390
1447
|
bucketId: string,
|
|
1391
1448
|
laneKey: string,
|
|
1392
|
-
): {
|
|
1449
|
+
): {
|
|
1450
|
+
permit: PacingPermit;
|
|
1451
|
+
scheduledAtMs: number;
|
|
1452
|
+
reservationId: string | null;
|
|
1453
|
+
} | null {
|
|
1393
1454
|
const block = this.liveBlock(bucketId, laneKey);
|
|
1394
1455
|
if (!block) return null;
|
|
1395
1456
|
// An emptied block stays until it expires: the next refill merges into it,
|
|
@@ -1399,9 +1460,24 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
|
|
|
1399
1460
|
return {
|
|
1400
1461
|
permit: this.permitFor(bucketId, laneKey, block, scheduled),
|
|
1401
1462
|
scheduledAtMs: scheduled[0],
|
|
1463
|
+
reservationId: scheduled[2],
|
|
1402
1464
|
};
|
|
1403
1465
|
}
|
|
1404
1466
|
|
|
1467
|
+
private async cancelUndispatchedPermit(input: {
|
|
1468
|
+
permit: PacingPermit;
|
|
1469
|
+
reservationId: string | null;
|
|
1470
|
+
}): Promise<void> {
|
|
1471
|
+
if (input.reservationId) {
|
|
1472
|
+
await input.permit.cancel();
|
|
1473
|
+
return;
|
|
1474
|
+
}
|
|
1475
|
+
// Older gateways may grant a permit without a refundable reservation id.
|
|
1476
|
+
// Still release any concurrency lease; losing one rate token is safer than
|
|
1477
|
+
// manufacturing an immediate permit during a rolling deploy.
|
|
1478
|
+
input.permit.release();
|
|
1479
|
+
}
|
|
1480
|
+
|
|
1405
1481
|
private drainBlocksForBucket(
|
|
1406
1482
|
bucketId: string,
|
|
1407
1483
|
beforeRetryAtMs?: number,
|
|
@@ -1546,7 +1622,7 @@ function withAdaptiveMaxRps(rules: readonly PacingRule[]): PacingRule[] {
|
|
|
1546
1622
|
// This one shared default rule is synthesized for unknown providers whose
|
|
1547
1623
|
// credential bucket spans operations. Keep the generic policy unchanged
|
|
1548
1624
|
// for local/coordinator backends, but let Postgres start a new provider
|
|
1549
|
-
// scope
|
|
1625
|
+
// scope at 10 RPS before learning its sustainable rate.
|
|
1550
1626
|
const seed =
|
|
1551
1627
|
rule.ruleId === 'default:provider-scope'
|
|
1552
1628
|
? {
|
|
@@ -511,9 +511,9 @@ export function createPlayExecutionGovernor(
|
|
|
511
511
|
if (!declaredPacing) {
|
|
512
512
|
// bucketId is the authenticated quota scope. Unknown tools sharing a
|
|
513
513
|
// credential must not each mint an independent copy of the default rate.
|
|
514
|
-
// Start its one shared lane at
|
|
515
|
-
// ramps on sustained success and reacts to
|
|
516
|
-
// tells us the quota is lower.
|
|
514
|
+
// Start its one shared lane at 10 RPS, matching the generic fallback.
|
|
515
|
+
// The DB-owned AIMD ramps on sustained success and reacts to
|
|
516
|
+
// Retry-After if the provider tells us the quota is lower.
|
|
517
517
|
pacing.rules = pacing.rules.map((rule) => ({
|
|
518
518
|
...rule,
|
|
519
519
|
ruleId: 'default:provider-scope',
|
|
@@ -189,7 +189,7 @@ export const SHARED_EXECUTION_POLICY: ResolvedExecutionPolicy = {
|
|
|
189
189
|
},
|
|
190
190
|
pacing: {
|
|
191
191
|
// Keep the generic fallback rate. The shared Postgres backend separately
|
|
192
|
-
// cold-starts unknown provider scopes at
|
|
192
|
+
// cold-starts unknown provider scopes at 10 RPS.
|
|
193
193
|
// Declared providers (rate-limit-definitions.ts) keep their own rates.
|
|
194
194
|
defaultProviderRequestsPerSecond: 10,
|
|
195
195
|
// This is the scheduler's resident-call envelope, not an RPS target.
|
|
@@ -15,7 +15,7 @@
|
|
|
15
15
|
export const DEFAULT_ADVISORY_PROVIDER_MAX_RPS = 100;
|
|
16
16
|
|
|
17
17
|
/** Cold-start rate for an unknown provider bucket shared across a fleet key. */
|
|
18
|
-
export const DEFAULT_UNKNOWN_PROVIDER_COLD_START_RPS =
|
|
18
|
+
export const DEFAULT_UNKNOWN_PROVIDER_COLD_START_RPS = 10;
|
|
19
19
|
|
|
20
20
|
/** AIMD state cooldown cap; provider Retry-After holds remain uncapped. */
|
|
21
21
|
export const PROVIDER_AIMD_COOLDOWN_CAP_MS = 45 * 60 * 1_000;
|
|
@@ -240,6 +240,27 @@ export function providerExhaustedMessage(input: {
|
|
|
240
240
|
);
|
|
241
241
|
}
|
|
242
242
|
|
|
243
|
+
/**
|
|
244
|
+
* The local rate governor could not offer a provider permit inside its bounded
|
|
245
|
+
* admission window. No provider request was dispatched, so this is a Deepline
|
|
246
|
+
* pacing failure rather than a provider Retry-After hold.
|
|
247
|
+
*/
|
|
248
|
+
export class ProviderAdmissionWaitTimeoutError extends Error {
|
|
249
|
+
readonly code = 'PROVIDER_ADMISSION_TIMEOUT';
|
|
250
|
+
readonly provider: string;
|
|
251
|
+
readonly waitBudgetMs: number;
|
|
252
|
+
|
|
253
|
+
constructor(input: { provider: string; waitBudgetMs: number }) {
|
|
254
|
+
const minutes = Math.max(1, Math.ceil(input.waitBudgetMs / 60_000));
|
|
255
|
+
super(
|
|
256
|
+
`PROVIDER_ADMISSION_TIMEOUT: Deepline could not grant a provider permit for ${input.provider} within its ${minutes}-minute wait budget. No provider request was sent.`,
|
|
257
|
+
);
|
|
258
|
+
this.name = 'ProviderAdmissionWaitTimeoutError';
|
|
259
|
+
this.provider = input.provider;
|
|
260
|
+
this.waitBudgetMs = input.waitBudgetMs;
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
|
|
243
264
|
/**
|
|
244
265
|
* Control signal from the DB-authoritative pacer when a provider hold exceeds
|
|
245
266
|
* the in-process wait threshold. The next provider call is not dispatched.
|
package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-rate-scope.ts
CHANGED
|
@@ -14,18 +14,22 @@ const AIMD_ADDITIVE_STEP_FRACTION = 0.25;
|
|
|
14
14
|
export function acquireRateScopeSql(
|
|
15
15
|
table: string,
|
|
16
16
|
successesParameter: string,
|
|
17
|
+
minimumRateSql = 'NULL::float8',
|
|
17
18
|
): string {
|
|
19
|
+
const refillRate = `CASE WHEN ${minimumRateSql} IS NULL
|
|
20
|
+
THEN h.refill_per_sec
|
|
21
|
+
ELSE GREATEST(h.refill_per_sec, ${minimumRateSql}) END`;
|
|
18
22
|
return `provider_hold_rows AS (
|
|
19
23
|
SELECT * FROM ${table} WHERE bucket_id = $1 FOR UPDATE
|
|
20
24
|
), provider_hold AS (
|
|
21
25
|
SELECT h.*,
|
|
22
26
|
LEAST(
|
|
23
|
-
GREATEST(CASE WHEN h.seen_backpressure THEN 1 ELSE h.capacity END,
|
|
24
|
-
h.tokens + GREATEST(0, nv.now_ms - h.refilled_at_ms) *
|
|
27
|
+
GREATEST(CASE WHEN h.seen_backpressure THEN 1 ELSE h.capacity END, ${refillRate}),
|
|
28
|
+
h.tokens + GREATEST(0, nv.now_ms - h.refilled_at_ms) * ${refillRate} / 1000.0
|
|
25
29
|
) AS tokens_after_refill,
|
|
26
30
|
${successesParameter}::int = 0
|
|
27
31
|
AND nv.now_ms - GREATEST(h.refilled_at_ms, h.next_dispatch_at_ms, h.retry_at_ms)
|
|
28
|
-
> GREATEST(1000, 2000.0 /
|
|
32
|
+
> GREATEST(1000, 2000.0 / ${refillRate}) AS observation_was_idle,
|
|
29
33
|
CASE WHEN ${successesParameter}::int > 0
|
|
30
34
|
AND h.rate_observation_started_at_ms IS NOT NULL
|
|
31
35
|
AND nv.now_ms - h.last_ramp_at_ms >= CASE WHEN h.seen_backpressure
|
|
@@ -33,9 +37,9 @@ export function acquireRateScopeSql(
|
|
|
33
37
|
ELSE h.rate_recovery_interval_ms END
|
|
34
38
|
AND h.retry_at_ms <= nv.now_ms
|
|
35
39
|
THEN LEAST(h.max_rps, CASE WHEN h.seen_backpressure
|
|
36
|
-
THEN
|
|
37
|
-
ELSE
|
|
38
|
-
ELSE
|
|
40
|
+
THEN ${refillRate} * (1 + ${AIMD_ADDITIVE_STEP_FRACTION})
|
|
41
|
+
ELSE ${refillRate} * 2 END)
|
|
42
|
+
ELSE ${refillRate} END AS next_refill_per_sec,
|
|
39
43
|
CASE WHEN ${successesParameter}::int > 0
|
|
40
44
|
AND h.rate_observation_started_at_ms IS NOT NULL
|
|
41
45
|
AND nv.now_ms - h.last_ramp_at_ms >= CASE WHEN h.seen_backpressure
|
package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-rate-state.ts
CHANGED
|
@@ -390,9 +390,9 @@ function normalizeRules(
|
|
|
390
390
|
return {
|
|
391
391
|
ruleId: rule.ruleId,
|
|
392
392
|
// Rates are fractional; only dispatched requests must be whole.
|
|
393
|
-
//
|
|
394
|
-
//
|
|
395
|
-
//
|
|
393
|
+
// Normalize every runner to the canonical 10-RPS seed for an unknown
|
|
394
|
+
// shared credential scope. The shared row's AIMD state can still be
|
|
395
|
+
// lower after backpressure and learns upward from successful calls.
|
|
396
396
|
requestsPerWindow: isUnknownProviderScope
|
|
397
397
|
? DEFAULT_UNKNOWN_PROVIDER_COLD_START_RPS
|
|
398
398
|
: rule.requestsPerWindow,
|
|
@@ -446,7 +446,13 @@ function seedForRule(rule: AcquirePacingRule): RateBucketSeed {
|
|
|
446
446
|
// Preserve its fractional refill rate, with dispatch times resolved in ms.
|
|
447
447
|
const capacity = Math.max(1, rule.requestsPerWindow);
|
|
448
448
|
const base = (rule.requestsPerWindow / rule.windowMs) * 1000;
|
|
449
|
-
|
|
449
|
+
// Undeclared providers use the shared fallback rate as their operating
|
|
450
|
+
// floor. Explicit provider Retry-After deadlines still block dispatch; a
|
|
451
|
+
// prior learned AIMD rate must not leave later runs paced below the fallback.
|
|
452
|
+
const minRps =
|
|
453
|
+
rule.ruleId === 'default:provider-scope'
|
|
454
|
+
? base
|
|
455
|
+
: Math.min(base, MIN_ADAPTIVE_RPS_FLOOR);
|
|
450
456
|
const maxRps =
|
|
451
457
|
rule.adaptiveMaxRps != null && rule.adaptiveMaxRps > 0
|
|
452
458
|
? Math.max(rule.adaptiveMaxRps, base)
|
|
@@ -461,6 +467,12 @@ function seedForRule(rule: AcquirePacingRule): RateBucketSeed {
|
|
|
461
467
|
};
|
|
462
468
|
}
|
|
463
469
|
|
|
470
|
+
function minimumRateSqlForRules(rules: readonly AcquirePacingRule[]): string {
|
|
471
|
+
return rules.some((rule) => rule.ruleId === 'default:provider-scope')
|
|
472
|
+
? `${DEFAULT_UNKNOWN_PROVIDER_COLD_START_RPS}::float8`
|
|
473
|
+
: 'NULL::float8';
|
|
474
|
+
}
|
|
475
|
+
|
|
464
476
|
function positiveInt(value: number, label: string): number {
|
|
465
477
|
if (!Number.isFinite(value) || value <= 0) {
|
|
466
478
|
throw new Error(`${label} must be a positive finite number.`);
|
|
@@ -743,11 +755,10 @@ async function seedBucketRows(
|
|
|
743
755
|
ON CONFLICT (bucket_id) DO UPDATE SET
|
|
744
756
|
retry_at_ms = GREATEST(hold.retry_at_ms, EXCLUDED.retry_at_ms),
|
|
745
757
|
refill_per_sec = CASE
|
|
746
|
-
--
|
|
747
|
-
--
|
|
748
|
-
--
|
|
749
|
-
--
|
|
750
|
-
-- evidence, and never erase a provider Retry-After hold.
|
|
758
|
+
-- If this row never moved from its seed, adopt the canonical
|
|
759
|
+
-- shared-scope seed on the first acquire after upgrade. Preserve
|
|
760
|
+
-- any rate with actual ramp or throttle evidence, and never erase
|
|
761
|
+
-- a provider Retry-After hold.
|
|
751
762
|
WHEN EXISTS (
|
|
752
763
|
SELECT 1 FROM seed_config
|
|
753
764
|
WHERE rule_id = 'default:provider-scope'
|
|
@@ -1295,7 +1306,11 @@ async function acquirePureRate(
|
|
|
1295
1306
|
adaptive_max_configured boolean
|
|
1296
1307
|
)
|
|
1297
1308
|
),
|
|
1298
|
-
${acquireRateScopeSql(
|
|
1309
|
+
${acquireRateScopeSql(
|
|
1310
|
+
scopeHoldTableName(input),
|
|
1311
|
+
'$5',
|
|
1312
|
+
minimumRateSqlForRules(rules),
|
|
1313
|
+
)},
|
|
1299
1314
|
reclaim_reservations AS (
|
|
1300
1315
|
DELETE FROM ${reservationTableName(input)}
|
|
1301
1316
|
WHERE bucket_id = $1
|
|
@@ -1735,7 +1750,11 @@ async function acquireWithConcurrency(
|
|
|
1735
1750
|
FLOOR(EXTRACT(EPOCH FROM clock_timestamp()) * 1000)::bigint
|
|
1736
1751
|
) AS now_ms
|
|
1737
1752
|
),
|
|
1738
|
-
${acquireRateScopeSql(
|
|
1753
|
+
${acquireRateScopeSql(
|
|
1754
|
+
scopeHoldTableName(input),
|
|
1755
|
+
'$10',
|
|
1756
|
+
minimumRateSqlForRules(rules),
|
|
1757
|
+
)},
|
|
1739
1758
|
reclaim_reservations AS (
|
|
1740
1759
|
DELETE FROM ${reservationTableName(input)}
|
|
1741
1760
|
WHERE bucket_id = $1
|
|
@@ -3015,6 +3034,11 @@ async function penalizeRateStateStatements(
|
|
|
3015
3034
|
return emptyRateStatePenaltyTelemetry();
|
|
3016
3035
|
}
|
|
3017
3036
|
const rules = normalizeRules((input.rules ?? []) as AcquirePacingRule[]);
|
|
3037
|
+
const minimumRateRps = rules.some(
|
|
3038
|
+
(rule) => rule.ruleId === 'default:provider-scope',
|
|
3039
|
+
)
|
|
3040
|
+
? DEFAULT_UNKNOWN_PROVIDER_COLD_START_RPS
|
|
3041
|
+
: MIN_ADAPTIVE_RPS_FLOOR;
|
|
3018
3042
|
await applyConcurrencyObservations(
|
|
3019
3043
|
client,
|
|
3020
3044
|
input,
|
|
@@ -3091,7 +3115,7 @@ async function penalizeRateStateStatements(
|
|
|
3091
3115
|
FROM held h CROSS JOIN now_value nv
|
|
3092
3116
|
), throttle_ceiling AS (
|
|
3093
3117
|
SELECT e.*,
|
|
3094
|
-
GREATEST(${
|
|
3118
|
+
GREATEST(${minimumRateRps},
|
|
3095
3119
|
CASE WHEN e.seen_backpressure AND e.rate_successes = 0 THEN
|
|
3096
3120
|
CASE WHEN $5::bigint IS NOT NULL
|
|
3097
3121
|
-- A fresh provider deadline is not a throughput measurement.
|
|
@@ -3106,7 +3130,7 @@ async function penalizeRateStateStatements(
|
|
|
3106
3130
|
FROM evidence e
|
|
3107
3131
|
), learned AS (
|
|
3108
3132
|
SELECT e.*,
|
|
3109
|
-
CASE WHEN e.new_episode THEN GREATEST(${
|
|
3133
|
+
CASE WHEN e.new_episode THEN GREATEST(${minimumRateRps}, LEAST(
|
|
3110
3134
|
e.next_ceiling_rps,
|
|
3111
3135
|
GREATEST(1, e.rate_successes) * 1000.0 / e.cycle_ms
|
|
3112
3136
|
)) ELSE e.refill_before END AS refill_after
|
|
@@ -8,7 +8,12 @@ import {
|
|
|
8
8
|
type ToolExecutionErrorSchemaVersion,
|
|
9
9
|
} from '../plays/tool-execution-error';
|
|
10
10
|
|
|
11
|
-
|
|
11
|
+
// Give an explicitly replay-safe transient 5xx one additional bounded retry.
|
|
12
|
+
// Preview gateway load runs can overlap brief Convex billing-plane 503s; this
|
|
13
|
+
// adds a second recovery chance without replaying operations that did not opt
|
|
14
|
+
// into the transient HTTP safety contract.
|
|
15
|
+
export const TOOL_EXECUTE_TRANSIENT_HTTP_MAX_ATTEMPTS = 3;
|
|
16
|
+
const TOOL_EXECUTE_CUSTOMER_DB_STORAGE_UNAVAILABLE_MAX_ATTEMPTS = 2;
|
|
12
17
|
// A provider may need to finish the original request before it can replay the
|
|
13
18
|
// result for this key. With no Retry-After signal, these two polls give the
|
|
14
19
|
// original request ten seconds to finish without repeatedly hammering it.
|
|
@@ -344,7 +349,7 @@ function decideToolExecuteHttpRetry(input: {
|
|
|
344
349
|
if (input.customerDbStorageUnavailable) {
|
|
345
350
|
return {
|
|
346
351
|
retryable: true,
|
|
347
|
-
attemptCap:
|
|
352
|
+
attemptCap: TOOL_EXECUTE_CUSTOMER_DB_STORAGE_UNAVAILABLE_MAX_ATTEMPTS,
|
|
348
353
|
reason: 'customer_db_storage_unavailable',
|
|
349
354
|
};
|
|
350
355
|
}
|
package/dist/cli/index.js
CHANGED
|
@@ -3051,7 +3051,7 @@ var SDK_RELEASE = {
|
|
|
3051
3051
|
// getters keep their established compatibility behavior.
|
|
3052
3052
|
// 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
|
|
3053
3053
|
// 0.3.90 is the first deliberately versioned SDK release for API v3.
|
|
3054
|
-
version: "0.3.
|
|
3054
|
+
version: "0.3.210",
|
|
3055
3055
|
updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.",
|
|
3056
3056
|
packageCapabilities: {
|
|
3057
3057
|
updatePreferences: 1
|
package/dist/cli/index.mjs
CHANGED
|
@@ -3046,7 +3046,7 @@ var SDK_RELEASE = {
|
|
|
3046
3046
|
// getters keep their established compatibility behavior.
|
|
3047
3047
|
// 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
|
|
3048
3048
|
// 0.3.90 is the first deliberately versioned SDK release for API v3.
|
|
3049
|
-
version: "0.3.
|
|
3049
|
+
version: "0.3.210",
|
|
3050
3050
|
updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.",
|
|
3051
3051
|
packageCapabilities: {
|
|
3052
3052
|
updatePreferences: 1
|
package/dist/index.js
CHANGED
|
@@ -864,7 +864,7 @@ var SDK_RELEASE = {
|
|
|
864
864
|
// getters keep their established compatibility behavior.
|
|
865
865
|
// 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
|
|
866
866
|
// 0.3.90 is the first deliberately versioned SDK release for API v3.
|
|
867
|
-
version: "0.3.
|
|
867
|
+
version: "0.3.210",
|
|
868
868
|
updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.",
|
|
869
869
|
packageCapabilities: {
|
|
870
870
|
updatePreferences: 1
|
package/dist/index.mjs
CHANGED
|
@@ -768,7 +768,7 @@ var SDK_RELEASE = {
|
|
|
768
768
|
// getters keep their established compatibility behavior.
|
|
769
769
|
// 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
|
|
770
770
|
// 0.3.90 is the first deliberately versioned SDK release for API v3.
|
|
771
|
-
version: "0.3.
|
|
771
|
+
version: "0.3.210",
|
|
772
772
|
updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.",
|
|
773
773
|
packageCapabilities: {
|
|
774
774
|
updatePreferences: 1
|
package/dist/release.d.mts
CHANGED
|
@@ -149,7 +149,7 @@ type SdkRelease = {
|
|
|
149
149
|
supportPolicy: SdkSupportPolicy;
|
|
150
150
|
};
|
|
151
151
|
declare const SDK_RELEASE: {
|
|
152
|
-
readonly version: "0.3.
|
|
152
|
+
readonly version: "0.3.210";
|
|
153
153
|
readonly updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.";
|
|
154
154
|
readonly packageCapabilities: {
|
|
155
155
|
readonly updatePreferences: 1;
|
package/dist/release.d.ts
CHANGED
|
@@ -149,7 +149,7 @@ type SdkRelease = {
|
|
|
149
149
|
supportPolicy: SdkSupportPolicy;
|
|
150
150
|
};
|
|
151
151
|
declare const SDK_RELEASE: {
|
|
152
|
-
readonly version: "0.3.
|
|
152
|
+
readonly version: "0.3.210";
|
|
153
153
|
readonly updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.";
|
|
154
154
|
readonly packageCapabilities: {
|
|
155
155
|
readonly updatePreferences: 1;
|
package/dist/release.js
CHANGED
|
@@ -74,7 +74,7 @@ var SDK_RELEASE = {
|
|
|
74
74
|
// getters keep their established compatibility behavior.
|
|
75
75
|
// 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
|
|
76
76
|
// 0.3.90 is the first deliberately versioned SDK release for API v3.
|
|
77
|
-
version: "0.3.
|
|
77
|
+
version: "0.3.210",
|
|
78
78
|
updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.",
|
|
79
79
|
packageCapabilities: {
|
|
80
80
|
updatePreferences: 1
|
package/dist/release.mjs
CHANGED
|
@@ -48,7 +48,7 @@ var SDK_RELEASE = {
|
|
|
48
48
|
// getters keep their established compatibility behavior.
|
|
49
49
|
// 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
|
|
50
50
|
// 0.3.90 is the first deliberately versioned SDK release for API v3.
|
|
51
|
-
version: "0.3.
|
|
51
|
+
version: "0.3.210",
|
|
52
52
|
updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.",
|
|
53
53
|
packageCapabilities: {
|
|
54
54
|
updatePreferences: 1
|