deepline 0.3.331 → 0.3.333
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/sdk/src/billing-idempotency.ts +5 -1
- package/dist/bundling-sources/sdk/src/release.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/context.ts +26 -23
- package/dist/bundling-sources/shared_libs/play-runtime/execution-gateway-invocation-fence.ts +16 -0
- package/dist/bundling-sources/shared_libs/play-runtime/governor/app-runtime-rate-state-backend.ts +30 -9
- package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts +6 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-rate-scope.ts +95 -24
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-rate-state.ts +132 -37
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts +6 -3
- package/dist/bundling-sources/shared_libs/play-runtime/tool-execute-retry-policy.ts +87 -3
- package/dist/cli/auth-26GA3RM6.js +31 -0
- package/dist/cli/{auth-TVTO43JD.mjs → auth-OQ356CEE.mjs} +4 -4
- package/dist/cli/{billing-Y4EX5FFA.js → billing-3APFTHNY.js} +71 -71
- package/dist/cli/{billing-FBZ5CCU7.mjs → billing-7YVHLDMO.mjs} +4 -4
- package/dist/cli/{chunk-RI57LHLC.mjs → chunk-2CLENWOQ.mjs} +1 -1
- package/dist/cli/{chunk-KGZPY524.js → chunk-3WQJWQ5T.js} +60 -60
- package/dist/cli/{chunk-YIYK57LW.mjs → chunk-4DUEB2WO.mjs} +2 -2
- package/dist/cli/{chunk-T7E5KIPE.js → chunk-5A44CWQE.js} +48 -48
- package/dist/cli/{chunk-VEP3POWB.mjs → chunk-5UN6KLD2.mjs} +1 -1
- package/dist/cli/{chunk-L4CJXV7D.js → chunk-64CZWDWT.js} +8 -8
- package/dist/cli/{chunk-RRJ6AA5L.mjs → chunk-6JPZJ5W7.mjs} +2 -2
- package/dist/cli/{chunk-BXZN544J.mjs → chunk-ABVOFJAE.mjs} +1 -1
- package/dist/cli/{chunk-NHR3O4OV.js → chunk-AFVETOG3.js} +15 -15
- package/dist/cli/{chunk-4JYLNJMD.mjs → chunk-AGGDEQDF.mjs} +1 -1
- package/dist/cli/{chunk-LFV2A7GB.js → chunk-DROPDYOD.js} +1 -1
- package/dist/cli/{chunk-ZGWURYZK.js → chunk-F2QKAGST.js} +10 -10
- package/dist/cli/{chunk-O52SKQQT.mjs → chunk-F2XAQHQC.mjs} +4 -4
- package/dist/cli/{chunk-CKAZY2JT.mjs → chunk-H5SORD7Q.mjs} +2 -2
- package/dist/cli/{chunk-DU72PAVF.mjs → chunk-HNX3RC6V.mjs} +1 -1
- package/dist/cli/{chunk-YCYWPBRV.js → chunk-IDEK4DND.js} +2 -2
- package/dist/cli/{chunk-MDQOGPQY.js → chunk-IYFKFNEB.js} +225 -225
- package/dist/cli/{chunk-KXWOO3GI.js → chunk-KBXHB5P3.js} +26 -26
- package/dist/cli/{chunk-HG5NWSN2.mjs → chunk-KREEYWVL.mjs} +6 -6
- package/dist/cli/{chunk-EGRTGSNS.mjs → chunk-N5KGBQPG.mjs} +3 -3
- package/dist/cli/{chunk-PS5BFY2W.mjs → chunk-NI4FJRFJ.mjs} +5 -5
- package/dist/cli/{chunk-XZG75ELG.js → chunk-PRZ4S4NU.js} +6 -6
- package/dist/cli/{chunk-RX2WWVZ7.js → chunk-STVJ7ZPD.js} +6 -6
- package/dist/cli/{chunk-QEI6SNAT.mjs → chunk-SXX4ST3D.mjs} +2 -2
- package/dist/cli/{chunk-ELDOV5XG.js → chunk-UHHTRBJF.js} +45 -45
- package/dist/cli/{chunk-3AQ3L2NX.js → chunk-V77J5ISS.js} +4 -4
- package/dist/cli/client-IZRTEBU4.js +20 -0
- package/dist/cli/{client-Q7J5R5PT.mjs → client-WKS7HQRP.mjs} +3 -3
- package/dist/cli/{csv-WBEF4KIZ.js → csv-MMW6EKKD.js} +12 -12
- package/dist/cli/{csv-RVG3V4YX.mjs → csv-ZFY2LD2K.mjs} +3 -3
- package/dist/cli/{db-4SPPGSEV.js → db-FMOQU6QW.js} +29 -29
- package/dist/cli/{db-AVIQ5WHJ.mjs → db-Q2NJVILC.mjs} +4 -4
- package/dist/cli/{describe-IQ3FOLGV.js → describe-4KG2JAQV.js} +15 -15
- package/dist/cli/{describe-4UA6N4YX.mjs → describe-EWP7DMNU.mjs} +8 -8
- package/dist/cli/{executions-VCULCPYR.mjs → executions-UOD7FWEE.mjs} +4 -4
- package/dist/cli/{executions-3ZUK3YJB.js → executions-Y4XSGKN6.js} +6 -6
- package/dist/cli/{feedback-SZLTZXWC.mjs → feedback-3F3EURZC.mjs} +3 -3
- package/dist/cli/{feedback-7YKDOMWW.js → feedback-TEJJUXDI.js} +6 -6
- package/dist/cli/{git-H4YS7TQW.mjs → git-5YM22RGA.mjs} +4 -4
- package/dist/cli/git-GNEZSTVF.js +23 -0
- package/dist/cli/index.js +79 -79
- package/dist/cli/index.mjs +31 -31
- package/dist/cli/{legacy-noops-K6U4YL3K.mjs → legacy-noops-D5XBVUHS.mjs} +4 -4
- package/dist/cli/{legacy-noops-GU7X3U2Y.js → legacy-noops-T5OUSXMR.js} +7 -7
- package/dist/cli/{monitors-2HHSSSDD.mjs → monitors-LPZNYNQW.mjs} +4 -4
- package/dist/cli/{monitors-IUEWN2VX.js → monitors-VI7VBQRP.js} +85 -85
- package/dist/cli/{org-6NBBO7G3.mjs → org-BYIDOMJB.mjs} +4 -4
- package/dist/cli/{org-E5TYEFPX.js → org-OBGIV5QE.js} +60 -60
- package/dist/cli/play-5JYRSXAL.js +153 -0
- package/dist/cli/{play-C2BQ4MJG.mjs → play-Y5PQQRBO.mjs} +7 -7
- package/dist/cli/{providers-N2LG7Y3E.mjs → providers-GEGIGKOL.mjs} +4 -4
- package/dist/cli/{providers-FM6LWHA4.js → providers-VH6J7FFY.js} +7 -7
- package/dist/cli/{quickstart-GQQ6ODNU.mjs → quickstart-GHITFT27.mjs} +3 -3
- package/dist/cli/{quickstart-I4K6ZDU6.js → quickstart-ZQX5XNHG.js} +5 -5
- package/dist/cli/{search-NKDQSOQ5.mjs → search-AFX3DYAC.mjs} +5 -5
- package/dist/cli/search-YIXGQFXA.js +26 -0
- package/dist/cli/{secrets-FGKGASDL.mjs → secrets-4EOVXQJM.mjs} +4 -4
- package/dist/cli/{secrets-BBI5NYLT.js → secrets-F37M7D7R.js} +13 -13
- package/dist/cli/{sessions-7MWWKIUT.mjs → sessions-RX5SFHDI.mjs} +4 -4
- package/dist/cli/{sessions-YUTDWUVR.js → sessions-T57Q72GD.js} +5 -5
- package/dist/cli/{settings-U55P74SS.js → settings-3FVWW7GL.js} +74 -74
- package/dist/cli/{settings-JGJWBESE.mjs → settings-764O5ZEH.mjs} +7 -7
- package/dist/cli/{setup-XYIDQ46U.js → setup-FQVRVTR6.js} +41 -41
- package/dist/cli/{setup-7D577YQO.mjs → setup-X7OAHFTO.mjs} +6 -6
- package/dist/cli/skills-DPGB67XX.js +28 -0
- package/dist/cli/{skills-Y6WBNBPF.mjs → skills-O6ILHPV5.mjs} +4 -4
- package/dist/cli/{tools-OPTIBRZP.mjs → tools-HL4SQG35.mjs} +6 -6
- package/dist/cli/tools-WZCXSXER.js +40 -0
- package/dist/cli/update-63MANY7U.js +28 -0
- package/dist/cli/{update-HKJQ762M.mjs → update-A3P2RSIB.mjs} +6 -6
- package/dist/cli/{workflow-2ZOG35IA.mjs → workflow-44424AVE.mjs} +5 -5
- package/dist/cli/{workflow-EBIKBXXC.js → workflow-62AGKI6T.js} +15 -15
- package/dist/index.js +6 -6
- package/dist/index.mjs +3 -3
- package/dist/install-integrity.json +74 -74
- package/dist/release.d.mts +1 -1
- package/dist/release.d.ts +1 -1
- package/dist/release.js +2 -2
- package/dist/release.mjs +1 -1
- package/package.json +1 -1
- package/dist/cli/auth-YYTF3NO4.js +0 -31
- package/dist/cli/client-46KHPTTQ.js +0 -20
- package/dist/cli/git-DCZP3E3P.js +0 -23
- package/dist/cli/play-MPDZPEYN.js +0 -153
- package/dist/cli/search-6G6FUVTJ.js +0 -26
- package/dist/cli/skills-PXHET5MG.js +0 -28
- package/dist/cli/tools-BCSXPN5X.js +0 -40
- package/dist/cli/update-2NXG3NKT.js +0 -28
package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-rate-state.ts
CHANGED
|
@@ -288,6 +288,36 @@ function isMissingRateThrottleCeilingColumnError(error: unknown): boolean {
|
|
|
288
288
|
);
|
|
289
289
|
}
|
|
290
290
|
|
|
291
|
+
/** Run the existing rate operation until the deploy-owned probe expansion lands. */
|
|
292
|
+
function isMissingProbeBaselineColumnError(error: unknown): boolean {
|
|
293
|
+
const candidate = error as { code?: unknown; message?: unknown } | null;
|
|
294
|
+
return (
|
|
295
|
+
candidate?.code === '42703' &&
|
|
296
|
+
typeof candidate.message === 'string' &&
|
|
297
|
+
/\b(rate_probe_successes_at_ramp|rate_probe_started_at_ms)\b/.test(
|
|
298
|
+
candidate.message,
|
|
299
|
+
)
|
|
300
|
+
);
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
async function queryRateStateWithProbeBaseline<
|
|
304
|
+
Row extends Record<string, unknown>,
|
|
305
|
+
>(
|
|
306
|
+
client: PostgresSchedulerQueryClient,
|
|
307
|
+
sql: (probeBaselineSupported: boolean) => string,
|
|
308
|
+
parameters: unknown[],
|
|
309
|
+
) {
|
|
310
|
+
try {
|
|
311
|
+
return await client.query<Row>(sql(true), parameters);
|
|
312
|
+
} catch (error) {
|
|
313
|
+
if (!isMissingProbeBaselineColumnError(error)) throw error;
|
|
314
|
+
// PostgreSQL rejects the statement before any grant or reservation writes.
|
|
315
|
+
// Retry only that statement; previously accepted success feedback is not
|
|
316
|
+
// applied twice. No DDL, migration preflight, or global schema floor.
|
|
317
|
+
return await client.query<Row>(sql(false), parameters);
|
|
318
|
+
}
|
|
319
|
+
}
|
|
320
|
+
|
|
291
321
|
/**
|
|
292
322
|
* v43 added `rate_requesters` and the lease `requester_id` column. Only an
|
|
293
323
|
* acquire that carries a requester identity touches them, so during the
|
|
@@ -565,6 +595,8 @@ export async function ensurePostgresRateStateSchema(
|
|
|
565
595
|
ADD COLUMN IF NOT EXISTS last_ramp_at_ms bigint NOT NULL DEFAULT 0,
|
|
566
596
|
ADD COLUMN IF NOT EXISTS rate_observation_started_at_ms bigint,
|
|
567
597
|
ADD COLUMN IF NOT EXISTS rate_successes bigint NOT NULL DEFAULT 0,
|
|
598
|
+
ADD COLUMN IF NOT EXISTS rate_probe_successes_at_ramp bigint NOT NULL DEFAULT 0,
|
|
599
|
+
ADD COLUMN IF NOT EXISTS rate_probe_started_at_ms bigint NOT NULL DEFAULT 0,
|
|
568
600
|
ADD COLUMN IF NOT EXISTS rate_recovery_interval_ms bigint NOT NULL DEFAULT 1000,
|
|
569
601
|
ADD COLUMN IF NOT EXISTS rate_throttle_ceiling_rps float8
|
|
570
602
|
`);
|
|
@@ -916,6 +948,8 @@ type InternalAcquireOutcome = PlayRunnerRateStateAcquireResult & {
|
|
|
916
948
|
observedRuleRows: number;
|
|
917
949
|
/** True when a seed upsert would still change a stored row (config drift). */
|
|
918
950
|
configDrift: boolean;
|
|
951
|
+
/** Local monotonic receipt time; never crosses the runtime boundary. */
|
|
952
|
+
serverClockReceivedAtMs?: number;
|
|
919
953
|
};
|
|
920
954
|
|
|
921
955
|
function acquireResultOf(
|
|
@@ -924,9 +958,17 @@ function acquireResultOf(
|
|
|
924
958
|
const result: PlayRunnerRateStateAcquireResult & {
|
|
925
959
|
observedRuleRows?: number;
|
|
926
960
|
configDrift?: boolean;
|
|
961
|
+
serverClockReceivedAtMs?: number;
|
|
927
962
|
} = { ...outcome };
|
|
928
963
|
delete result.observedRuleRows;
|
|
929
964
|
delete result.configDrift;
|
|
965
|
+
delete result.serverClockReceivedAtMs;
|
|
966
|
+
if (outcome.serverClockReceivedAtMs != null) {
|
|
967
|
+
result.serverClockElapsedMs = Math.max(
|
|
968
|
+
0,
|
|
969
|
+
Math.floor(performance.now() - outcome.serverClockReceivedAtMs),
|
|
970
|
+
);
|
|
971
|
+
}
|
|
930
972
|
return result;
|
|
931
973
|
}
|
|
932
974
|
|
|
@@ -1307,7 +1349,7 @@ async function acquirePureRate(
|
|
|
1307
1349
|
concurrency: null,
|
|
1308
1350
|
})
|
|
1309
1351
|
: null;
|
|
1310
|
-
const result = await
|
|
1352
|
+
const result = await queryRateStateWithProbeBaseline<{
|
|
1311
1353
|
granted: string | number;
|
|
1312
1354
|
wait_ms: string | number;
|
|
1313
1355
|
observed_rule_rows: string | number;
|
|
@@ -1319,7 +1361,8 @@ async function acquirePureRate(
|
|
|
1319
1361
|
reservation_ids: string[] | null;
|
|
1320
1362
|
effective_rps: string | number;
|
|
1321
1363
|
}>(
|
|
1322
|
-
|
|
1364
|
+
client,
|
|
1365
|
+
(probeBaselineSupported) => `
|
|
1323
1366
|
WITH now_value AS (
|
|
1324
1367
|
SELECT COALESCE(
|
|
1325
1368
|
$3::bigint,
|
|
@@ -1346,6 +1389,7 @@ async function acquirePureRate(
|
|
|
1346
1389
|
'$5',
|
|
1347
1390
|
minimumRateSqlForRules(rules),
|
|
1348
1391
|
unknownProvider,
|
|
1392
|
+
probeBaselineSupported,
|
|
1349
1393
|
)},
|
|
1350
1394
|
reclaim_reservations AS (
|
|
1351
1395
|
DELETE FROM ${reservationTableName(input)}
|
|
@@ -1500,7 +1544,7 @@ async function acquirePureRate(
|
|
|
1500
1544
|
GREATEST(1, FLOOR(MIN(next_refill_per_sec) / 100.0))::bigint AS burst_size
|
|
1501
1545
|
FROM refreshed
|
|
1502
1546
|
),
|
|
1503
|
-
${updateRateScopeSql(scopeHoldTableName(input), unknownProvider)},
|
|
1547
|
+
${updateRateScopeSql(scopeHoldTableName(input), unknownProvider, probeBaselineSupported)},
|
|
1504
1548
|
updated AS (
|
|
1505
1549
|
UPDATE ${bucketTableName(input)} b
|
|
1506
1550
|
SET
|
|
@@ -1669,6 +1713,8 @@ async function acquirePureRate(
|
|
|
1669
1713
|
...(requesterId ? [requesterId] : []),
|
|
1670
1714
|
],
|
|
1671
1715
|
);
|
|
1716
|
+
const serverClockReceivedAtMs =
|
|
1717
|
+
input.nowMs == null ? performance.now() : undefined;
|
|
1672
1718
|
const row = result.rows[0];
|
|
1673
1719
|
if (!row) {
|
|
1674
1720
|
// Seeding raced away or rows vanished; the caller re-seeds and retries.
|
|
@@ -1709,6 +1755,7 @@ async function acquirePureRate(
|
|
|
1709
1755
|
granted,
|
|
1710
1756
|
waitMs,
|
|
1711
1757
|
serverNowMs,
|
|
1758
|
+
serverClockReceivedAtMs,
|
|
1712
1759
|
observedRuleRows,
|
|
1713
1760
|
configDrift,
|
|
1714
1761
|
leaseIds: [],
|
|
@@ -1741,29 +1788,39 @@ async function acquireWithConcurrency(
|
|
|
1741
1788
|
seeds: RateBucketSeed[],
|
|
1742
1789
|
requested: number,
|
|
1743
1790
|
): Promise<InternalAcquireOutcome> {
|
|
1744
|
-
|
|
1745
|
-
|
|
1746
|
-
await client.query(
|
|
1747
|
-
`SELECT bucket_id FROM ${scopeHoldTableName(input)}
|
|
1748
|
-
WHERE bucket_id = $1 FOR UPDATE`,
|
|
1749
|
-
[input.bucketId],
|
|
1750
|
-
);
|
|
1751
|
-
const outcome = await grantWithConcurrency(
|
|
1752
|
-
client,
|
|
1753
|
-
input,
|
|
1754
|
-
rules,
|
|
1755
|
-
seeds,
|
|
1756
|
-
requested,
|
|
1757
|
-
);
|
|
1758
|
-
await client.query('COMMIT');
|
|
1759
|
-
return outcome;
|
|
1760
|
-
} catch (error) {
|
|
1791
|
+
let probeBaselineSupported = true;
|
|
1792
|
+
while (true) {
|
|
1793
|
+
await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
|
|
1761
1794
|
try {
|
|
1762
|
-
await client.query(
|
|
1763
|
-
|
|
1764
|
-
|
|
1795
|
+
await client.query(
|
|
1796
|
+
`SELECT bucket_id FROM ${scopeHoldTableName(input)}
|
|
1797
|
+
WHERE bucket_id = $1 FOR UPDATE`,
|
|
1798
|
+
[input.bucketId],
|
|
1799
|
+
);
|
|
1800
|
+
const outcome = await grantWithConcurrency(
|
|
1801
|
+
client,
|
|
1802
|
+
input,
|
|
1803
|
+
rules,
|
|
1804
|
+
seeds,
|
|
1805
|
+
requested,
|
|
1806
|
+
probeBaselineSupported,
|
|
1807
|
+
);
|
|
1808
|
+
await client.query('COMMIT');
|
|
1809
|
+
return outcome;
|
|
1810
|
+
} catch (error) {
|
|
1811
|
+
try {
|
|
1812
|
+
await client.query('ROLLBACK');
|
|
1813
|
+
} catch {
|
|
1814
|
+
// Preserve the original admission/storage error.
|
|
1815
|
+
}
|
|
1816
|
+
if (probeBaselineSupported && isMissingProbeBaselineColumnError(error)) {
|
|
1817
|
+
// The failed grant aborts its transaction. Reacquire the same scope lock
|
|
1818
|
+
// with a fresh snapshot; only the grant is repeated, never its feedback.
|
|
1819
|
+
probeBaselineSupported = false;
|
|
1820
|
+
continue;
|
|
1821
|
+
}
|
|
1822
|
+
throw error;
|
|
1765
1823
|
}
|
|
1766
|
-
throw error;
|
|
1767
1824
|
}
|
|
1768
1825
|
}
|
|
1769
1826
|
|
|
@@ -1773,6 +1830,7 @@ async function grantWithConcurrency(
|
|
|
1773
1830
|
rules: AcquirePacingRule[],
|
|
1774
1831
|
seeds: RateBucketSeed[],
|
|
1775
1832
|
requested: number,
|
|
1833
|
+
probeBaselineSupported: boolean,
|
|
1776
1834
|
): Promise<InternalAcquireOutcome> {
|
|
1777
1835
|
const ruleIds = rules.map((rule) => rule.ruleId);
|
|
1778
1836
|
const unknownProvider = rules.some(
|
|
@@ -1830,6 +1888,7 @@ async function grantWithConcurrency(
|
|
|
1830
1888
|
'$10',
|
|
1831
1889
|
minimumRateSqlForRules(rules),
|
|
1832
1890
|
unknownProvider,
|
|
1891
|
+
probeBaselineSupported,
|
|
1833
1892
|
)},
|
|
1834
1893
|
reclaim_reservations AS (
|
|
1835
1894
|
DELETE FROM ${reservationTableName(input)}
|
|
@@ -2068,7 +2127,7 @@ async function grantWithConcurrency(
|
|
|
2068
2127
|
GREATEST(1, FLOOR(MIN(next_refill_per_sec) / 100.0))::bigint AS burst_size
|
|
2069
2128
|
FROM refreshed
|
|
2070
2129
|
),
|
|
2071
|
-
${updateRateScopeSql(scopeHoldTableName(input), unknownProvider)},
|
|
2130
|
+
${updateRateScopeSql(scopeHoldTableName(input), unknownProvider, probeBaselineSupported)},
|
|
2072
2131
|
updated AS (
|
|
2073
2132
|
UPDATE ${bucketTableName(input)} b
|
|
2074
2133
|
SET
|
|
@@ -2283,6 +2342,8 @@ async function grantWithConcurrency(
|
|
|
2283
2342
|
...(requesterId ? [requesterId] : []),
|
|
2284
2343
|
],
|
|
2285
2344
|
);
|
|
2345
|
+
const serverClockReceivedAtMs =
|
|
2346
|
+
input.nowMs == null ? performance.now() : undefined;
|
|
2286
2347
|
const row = result.rows[0];
|
|
2287
2348
|
if (!row) {
|
|
2288
2349
|
return {
|
|
@@ -2327,6 +2388,7 @@ async function grantWithConcurrency(
|
|
|
2327
2388
|
granted,
|
|
2328
2389
|
waitMs,
|
|
2329
2390
|
serverNowMs,
|
|
2391
|
+
serverClockReceivedAtMs,
|
|
2330
2392
|
observedRuleRows,
|
|
2331
2393
|
configDrift,
|
|
2332
2394
|
leaseIds: granted > 0 ? leaseIds : [],
|
|
@@ -2410,9 +2472,15 @@ export async function validatePostgresRateStatePermit(
|
|
|
2410
2472
|
*/
|
|
2411
2473
|
export async function claimPostgresProviderDispatch(
|
|
2412
2474
|
client: PostgresSchedulerQueryClient,
|
|
2413
|
-
input: PlayRunnerRateStateValidateInput & {
|
|
2475
|
+
input: PlayRunnerRateStateValidateInput & {
|
|
2476
|
+
nowMs?: number;
|
|
2477
|
+
retryAtMsAtGrant?: number;
|
|
2478
|
+
},
|
|
2414
2479
|
): Promise<
|
|
2415
|
-
PlayRunnerRateStateValidateResult & {
|
|
2480
|
+
PlayRunnerRateStateValidateResult & {
|
|
2481
|
+
physicalCadenceWait: boolean;
|
|
2482
|
+
stalePermit?: boolean;
|
|
2483
|
+
}
|
|
2416
2484
|
> {
|
|
2417
2485
|
return withRateStateAvailabilityGuard(input, async () => {
|
|
2418
2486
|
const result = await client.query<{
|
|
@@ -2420,6 +2488,7 @@ export async function claimPostgresProviderDispatch(
|
|
|
2420
2488
|
retry_at_ms: string | number;
|
|
2421
2489
|
allowed: boolean;
|
|
2422
2490
|
cadence_wait: boolean;
|
|
2491
|
+
stale_permit: boolean;
|
|
2423
2492
|
}>(
|
|
2424
2493
|
`
|
|
2425
2494
|
WITH now_value AS (
|
|
@@ -2438,6 +2507,7 @@ export async function claimPostgresProviderDispatch(
|
|
|
2438
2507
|
GREATEST(COALESCE(s.retry_at_ms, 0), COALESCE(s.cool_until_ms, 0)) AS hold_until_ms,
|
|
2439
2508
|
s.physical_rps,
|
|
2440
2509
|
COALESCE(s.seen_backpressure, false) AS preserve_cadence_phase,
|
|
2510
|
+
COALESCE(s.retry_at_ms > $3::bigint, false) AS stale_permit,
|
|
2441
2511
|
GREATEST(1, FLOOR(s.physical_rps / 100))::int AS burst_size,
|
|
2442
2512
|
GREATEST(1, CEIL(1000.0 / s.physical_rps))::bigint AS spacing_ms
|
|
2443
2513
|
FROM now_value nv LEFT JOIN scope_rows s ON true
|
|
@@ -2453,6 +2523,7 @@ export async function claimPostgresProviderDispatch(
|
|
|
2453
2523
|
c.now_ms + c.spacing_ms * c.burst_size, 0
|
|
2454
2524
|
FROM cadence c
|
|
2455
2525
|
WHERE c.hold_until_ms <= c.now_ms AND c.physical_rps > 0
|
|
2526
|
+
AND NOT c.stale_permit
|
|
2456
2527
|
ON CONFLICT (bucket_id, rule_id) DO UPDATE SET
|
|
2457
2528
|
-- Always return the locked current cursor, including a denial.
|
|
2458
2529
|
-- A snapshot read after a blocked upsert could miss a concurrent
|
|
@@ -2466,9 +2537,16 @@ export async function claimPostgresProviderDispatch(
|
|
|
2466
2537
|
cool_until_ms = CASE
|
|
2467
2538
|
WHEN existing.next_dispatch_at_ms <= EXCLUDED.refilled_at_ms
|
|
2468
2539
|
OR LEAST(existing.tokens, EXCLUDED.capacity) >= 1 THEN 0
|
|
2469
|
-
ELSE
|
|
2470
|
-
|
|
2471
|
-
|
|
2540
|
+
ELSE CASE WHEN EXCLUDED.refill_per_sec > existing.refill_per_sec
|
|
2541
|
+
THEN LEAST(existing.next_dispatch_at_ms,
|
|
2542
|
+
GREATEST(existing.refilled_at_ms, EXCLUDED.refilled_at_ms)
|
|
2543
|
+
+ (EXCLUDED.next_dispatch_at_ms - EXCLUDED.refilled_at_ms))
|
|
2544
|
+
ELSE existing.next_dispatch_at_ms END END,
|
|
2545
|
+
-- Only a faster rate shortens existing debt. A contender can
|
|
2546
|
+
-- sample the clock before waiting behind a newer cursor grant;
|
|
2547
|
+
-- its denial must not rewind that grant at the same pace. Anchor
|
|
2548
|
+
-- recovery at the latest cursor clock, then persist its deadline
|
|
2549
|
+
-- so repeated probes converge without consuming another slot.
|
|
2472
2550
|
next_dispatch_at_ms = CASE
|
|
2473
2551
|
WHEN existing.next_dispatch_at_ms <= EXCLUDED.refilled_at_ms
|
|
2474
2552
|
-- Keep the cadence phase when a request reaches its slot
|
|
@@ -2487,23 +2565,30 @@ export async function claimPostgresProviderDispatch(
|
|
|
2487
2565
|
-- Cold discovery retains strict spacing until provider
|
|
2488
2566
|
-- backpressure establishes an adaptive cadence.
|
|
2489
2567
|
ELSE EXCLUDED.next_dispatch_at_ms END
|
|
2490
|
-
ELSE
|
|
2568
|
+
ELSE CASE WHEN EXCLUDED.refill_per_sec > existing.refill_per_sec
|
|
2569
|
+
THEN LEAST(existing.next_dispatch_at_ms,
|
|
2570
|
+
GREATEST(existing.refilled_at_ms, EXCLUDED.refilled_at_ms)
|
|
2571
|
+
+ (EXCLUDED.next_dispatch_at_ms - EXCLUDED.refilled_at_ms))
|
|
2572
|
+
ELSE existing.next_dispatch_at_ms END END,
|
|
2491
2573
|
capacity = EXCLUDED.capacity,
|
|
2492
2574
|
refill_per_sec = EXCLUDED.refill_per_sec,
|
|
2493
2575
|
base_refill_per_sec = EXCLUDED.base_refill_per_sec,
|
|
2494
2576
|
max_rps = EXCLUDED.max_rps,
|
|
2495
|
-
refilled_at_ms = EXCLUDED.refilled_at_ms
|
|
2577
|
+
refilled_at_ms = GREATEST(existing.refilled_at_ms, EXCLUDED.refilled_at_ms)
|
|
2496
2578
|
RETURNING cool_until_ms
|
|
2497
2579
|
)
|
|
2498
2580
|
SELECT c.now_ms AS server_now_ms,
|
|
2499
|
-
GREATEST(c.hold_until_ms, COALESCE(cursor.cool_until_ms, 0)
|
|
2500
|
-
|
|
2581
|
+
GREATEST(c.hold_until_ms, COALESCE(cursor.cool_until_ms, 0),
|
|
2582
|
+
CASE WHEN c.stale_permit THEN c.now_ms ELSE 0 END) AS retry_at_ms,
|
|
2583
|
+
NOT c.stale_permit AND c.hold_until_ms <= c.now_ms AND (
|
|
2501
2584
|
c.physical_rps IS NULL OR cursor.cool_until_ms = 0
|
|
2502
2585
|
) AS allowed,
|
|
2503
|
-
|
|
2586
|
+
NOT c.stale_permit AND c.hold_until_ms <= c.now_ms
|
|
2587
|
+
AND COALESCE(cursor.cool_until_ms > c.now_ms, false) AS cadence_wait,
|
|
2588
|
+
c.stale_permit
|
|
2504
2589
|
FROM cadence c LEFT JOIN cursor ON true
|
|
2505
2590
|
`,
|
|
2506
|
-
[input.bucketId, input.nowMs ?? null],
|
|
2591
|
+
[input.bucketId, input.nowMs ?? null, input.retryAtMsAtGrant ?? null],
|
|
2507
2592
|
);
|
|
2508
2593
|
const row = result.rows[0];
|
|
2509
2594
|
if (!row || typeof row.allowed !== 'boolean') {
|
|
@@ -2516,6 +2601,9 @@ export async function claimPostgresProviderDispatch(
|
|
|
2516
2601
|
physicalCadenceWait: row.cadence_wait,
|
|
2517
2602
|
serverNowMs: Number(row.server_now_ms),
|
|
2518
2603
|
retryAtMs: Number(row.retry_at_ms),
|
|
2604
|
+
...(input.retryAtMsAtGrant !== undefined
|
|
2605
|
+
? { stalePermit: row.stale_permit }
|
|
2606
|
+
: {}),
|
|
2519
2607
|
};
|
|
2520
2608
|
});
|
|
2521
2609
|
}
|
|
@@ -3362,7 +3450,13 @@ async function penalizeRateStateStatements(
|
|
|
3362
3450
|
-- no successes justifies the shared exponential pause.
|
|
3363
3451
|
WHEN h.rate_successes <= 0
|
|
3364
3452
|
AND ($5::bigint IS NOT NULL OR h.seen_backpressure)
|
|
3365
|
-
|
|
3453
|
+
-- An unknown scope can only recover above its floor after
|
|
3454
|
+
-- successful work. A cleared counter at that new epoch does
|
|
3455
|
+
-- not turn its measurement duration into consecutive failures.
|
|
3456
|
+
AND (NOT h.unknown_provider
|
|
3457
|
+
OR h.refill_per_sec <= ${DEFAULT_UNKNOWN_PROVIDER_MIN_RPS})
|
|
3458
|
+
AND (NOT h.unknown_provider OR (h.seen_backpressure
|
|
3459
|
+
AND h.rate_recovery_interval_ms > 0)) THEN
|
|
3366
3460
|
CASE WHEN h.seen_backpressure THEN
|
|
3367
3461
|
LEAST(60000, GREATEST(1000, h.rate_recovery_interval_ms * 2))
|
|
3368
3462
|
ELSE 1000 END
|
|
@@ -3392,6 +3486,7 @@ async function penalizeRateStateStatements(
|
|
|
3392
3486
|
SELECT e.*,
|
|
3393
3487
|
CASE WHEN e.new_episode THEN GREATEST(
|
|
3394
3488
|
CASE WHEN e.seen_backpressure AND e.rate_successes = 0
|
|
3489
|
+
AND e.no_progress_probe_interval_ms IS NOT NULL
|
|
3395
3490
|
THEN ${noProgressProbeRateRps}
|
|
3396
3491
|
ELSE CASE WHEN e.unknown_provider
|
|
3397
3492
|
THEN ${DEFAULT_UNKNOWN_PROVIDER_MIN_RPS} ELSE ${minimumRateRps} END END,
|
|
@@ -363,8 +363,10 @@ const POSTGRES_SCHEDULER_SCHEMA_INIT_STATEMENT_TIMEOUT_MS = 60_000;
|
|
|
363
363
|
// Existing namespace mode values and unpinned runs keep their behavior.
|
|
364
364
|
// v62 adds a deploy-owned index for per-run active outbox lease exclusion.
|
|
365
365
|
// Failed facts remain indexed: their live leases still fence the run.
|
|
366
|
+
// v63 adds optional rate-probe count and clock baselines. Older writers ignore them;
|
|
367
|
+
// unexpanded acquires retain their existing operation-local measurement.
|
|
366
368
|
// This marker records deployment work, never a runtime schema floor.
|
|
367
|
-
export const POSTGRES_SCHEDULER_SCHEMA_VERSION =
|
|
369
|
+
export const POSTGRES_SCHEDULER_SCHEMA_VERSION = 63;
|
|
368
370
|
const POSTGRES_SCHEDULER_SCHEMA_MIGRATIONS_TABLE =
|
|
369
371
|
'scheduler_schema_migrations';
|
|
370
372
|
export const POSTGRES_SCHEDULER_OPEN_RECOVERY_INDEX =
|
|
@@ -1564,9 +1566,10 @@ export async function hasCurrentPostgresSchedulerSchemaShape(
|
|
|
1564
1566
|
'tokens', 'refilled_at_ms', 'next_dispatch_at_ms', 'seen_backpressure',
|
|
1565
1567
|
'cool_until_ms', 'last_ramp_at_ms', 'rate_observation_started_at_ms',
|
|
1566
1568
|
'rate_successes', 'rate_recovery_interval_ms',
|
|
1567
|
-
'rate_throttle_ceiling_rps'
|
|
1569
|
+
'rate_throttle_ceiling_rps', 'rate_probe_successes_at_ramp',
|
|
1570
|
+
'rate_probe_started_at_ms'
|
|
1568
1571
|
)
|
|
1569
|
-
) =
|
|
1572
|
+
) = 17
|
|
1570
1573
|
AND (
|
|
1571
1574
|
SELECT count(DISTINCT column_name)
|
|
1572
1575
|
FROM information_schema.columns
|
|
@@ -107,10 +107,20 @@ export const TOOL_EXECUTE_RUNNER_ACCEPTS_HEADER = 'x-deepline-runner-accepts';
|
|
|
107
107
|
*/
|
|
108
108
|
export const TOOL_EXECUTE_RUNNER_ACCEPTS_CONCURRENCY_BACKPRESSURE =
|
|
109
109
|
'concurrency-backpressure';
|
|
110
|
+
export const TOOL_EXECUTE_RUNNER_ACCEPTS_PROVIDER_RETRY_DEADLINE =
|
|
111
|
+
'provider-retry-deadline-ms';
|
|
112
|
+
|
|
113
|
+
/** Set by a supporting gateway only when its runner also reads exact deadlines. */
|
|
114
|
+
export const TOOL_EXECUTE_PROVIDER_RETRY_DEADLINE_CAPABILITY_HEADER =
|
|
115
|
+
'x-deepline-provider-retry-deadline';
|
|
116
|
+
export const TOOL_EXECUTE_PROVIDER_RETRY_DEADLINE_CAPABILITY_VALUE =
|
|
117
|
+
'milliseconds';
|
|
118
|
+
/** Immutable provider deadline; Retry-After remains an HTTP-date for old readers. */
|
|
119
|
+
export const TOOL_EXECUTE_PROVIDER_RETRY_AT_HEADER =
|
|
120
|
+
'x-deepline-provider-retry-at-ms';
|
|
110
121
|
|
|
111
122
|
/** The capability tokens this runner build advertises on provider execute. */
|
|
112
|
-
export const TOOL_EXECUTE_RUNNER_ACCEPTS_VALUE =
|
|
113
|
-
TOOL_EXECUTE_RUNNER_ACCEPTS_CONCURRENCY_BACKPRESSURE;
|
|
123
|
+
export const TOOL_EXECUTE_RUNNER_ACCEPTS_VALUE = `${TOOL_EXECUTE_RUNNER_ACCEPTS_CONCURRENCY_BACKPRESSURE},${TOOL_EXECUTE_RUNNER_ACCEPTS_PROVIDER_RETRY_DEADLINE}`;
|
|
114
124
|
/** Runner proof headers consumed by the gateway and authenticated app boundary. */
|
|
115
125
|
export const TOOL_EXECUTE_PROVIDER_RATE_SCOPE_HEADER =
|
|
116
126
|
'x-deepline-provider-rate-scope';
|
|
@@ -161,6 +171,73 @@ export function runnerAcceptsConcurrencyBackpressure(
|
|
|
161
171
|
);
|
|
162
172
|
}
|
|
163
173
|
|
|
174
|
+
export function providerRetryDeadlineCapabilityForRunner(
|
|
175
|
+
headers: Pick<Headers, 'get'>,
|
|
176
|
+
): string | null {
|
|
177
|
+
return headers
|
|
178
|
+
.get(TOOL_EXECUTE_RUNNER_ACCEPTS_HEADER)
|
|
179
|
+
?.split(',')
|
|
180
|
+
.some(
|
|
181
|
+
(token) =>
|
|
182
|
+
token.trim().toLowerCase() ===
|
|
183
|
+
TOOL_EXECUTE_RUNNER_ACCEPTS_PROVIDER_RETRY_DEADLINE,
|
|
184
|
+
)
|
|
185
|
+
? TOOL_EXECUTE_PROVIDER_RETRY_DEADLINE_CAPABILITY_VALUE
|
|
186
|
+
: null;
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
export function providerRetryDeadlinePrecisionEnabled(
|
|
190
|
+
headers: Pick<Headers, 'get'> | undefined,
|
|
191
|
+
): boolean {
|
|
192
|
+
return (
|
|
193
|
+
headers?.get(TOOL_EXECUTE_PROVIDER_RETRY_DEADLINE_CAPABILITY_HEADER) ===
|
|
194
|
+
TOOL_EXECUTE_PROVIDER_RETRY_DEADLINE_CAPABILITY_VALUE
|
|
195
|
+
);
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
/** An exact deadline is usable only beside its matching legacy HTTP-date. */
|
|
199
|
+
export function parseToolExecuteExactProviderRetryAtMs(response: {
|
|
200
|
+
status: number;
|
|
201
|
+
headers: Pick<Headers, 'get'>;
|
|
202
|
+
}): number | undefined {
|
|
203
|
+
if (response.status !== 429) return undefined;
|
|
204
|
+
const retryAfter = response.headers.get('retry-after');
|
|
205
|
+
if (!retryAfter?.trim()) return undefined;
|
|
206
|
+
// Exact writers emit IMF-fixdate. Date.parse also accepts delta-seconds
|
|
207
|
+
// such as "1" as a calendar date, and ISO/obsolete date presentations.
|
|
208
|
+
if (
|
|
209
|
+
!/^(?:Mon|Tue|Wed|Thu|Fri|Sat|Sun), \d{2} (?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) \d{4} \d{2}:\d{2}:\d{2} GMT$/.test(
|
|
210
|
+
retryAfter,
|
|
211
|
+
)
|
|
212
|
+
)
|
|
213
|
+
return undefined;
|
|
214
|
+
const exact = response.headers.get(TOOL_EXECUTE_PROVIDER_RETRY_AT_HEADER);
|
|
215
|
+
const retryAtMs = exact && /^\d+$/.test(exact) ? Number(exact) : NaN;
|
|
216
|
+
if (
|
|
217
|
+
isAttestedProvider429Response(response) &&
|
|
218
|
+
Number.isSafeInteger(retryAtMs) &&
|
|
219
|
+
retryAtMs >= 0 &&
|
|
220
|
+
retryAtMs <= 8.64e15 &&
|
|
221
|
+
new Date(Math.ceil(retryAtMs / 1_000) * 1_000).toUTCString() === retryAfter
|
|
222
|
+
) {
|
|
223
|
+
return retryAtMs;
|
|
224
|
+
}
|
|
225
|
+
return undefined;
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
export function parseToolExecuteProviderRetryAtMs(
|
|
229
|
+
response: { status: number; headers: Pick<Headers, 'get'> },
|
|
230
|
+
nowMs = Date.now(),
|
|
231
|
+
): number | undefined {
|
|
232
|
+
if (response.status !== 429) return undefined;
|
|
233
|
+
const retryAfter = response.headers.get('retry-after');
|
|
234
|
+
if (!retryAfter?.trim()) return undefined;
|
|
235
|
+
return (
|
|
236
|
+
parseToolExecuteExactProviderRetryAtMs(response) ??
|
|
237
|
+
parseToolExecuteRetryAtMs(retryAfter, nowMs)
|
|
238
|
+
);
|
|
239
|
+
}
|
|
240
|
+
|
|
164
241
|
export class ToolExecuteAuthScopeChangedError extends Error {
|
|
165
242
|
readonly code = TOOL_EXECUTE_AUTH_SCOPE_CHANGED_CODE;
|
|
166
243
|
readonly status = 409;
|
|
@@ -612,6 +689,8 @@ export function classifyToolExecuteHttpFailure(input: {
|
|
|
612
689
|
bodyText: string;
|
|
613
690
|
schemaVersion?: ToolExecutionErrorSchemaVersion;
|
|
614
691
|
retryAfterHeader?: string | null;
|
|
692
|
+
/** Captured at response-header arrival; absent keeps the legacy header path. */
|
|
693
|
+
retryAtMs?: number;
|
|
615
694
|
transientHttpRetrySafe?: boolean;
|
|
616
695
|
/** The call runs under the gateway's durable execution fence. */
|
|
617
696
|
executionFenced?: boolean;
|
|
@@ -670,7 +749,12 @@ export function classifyToolExecuteHttpFailure(input: {
|
|
|
670
749
|
const retryAfterMs =
|
|
671
750
|
providerPacingWait && exactProviderPacingWaitMs !== null
|
|
672
751
|
? exactProviderPacingWaitMs
|
|
673
|
-
:
|
|
752
|
+
: input.status === 429 &&
|
|
753
|
+
Number.isSafeInteger(input.retryAtMs) &&
|
|
754
|
+
input.retryAtMs! >= 0 &&
|
|
755
|
+
input.retryAtMs! <= 8.64e15
|
|
756
|
+
? Math.max(0, input.retryAtMs! - (input.nowMs ?? Date.now()))
|
|
757
|
+
: parseToolExecuteRetryAfterMs(input.retryAfterHeader, input.nowMs);
|
|
674
758
|
const error = normalizeToolHttpErrorMessage({
|
|
675
759
|
toolId: input.toolId,
|
|
676
760
|
status: input.status,
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
"use strict";Object.defineProperty(exports, "__esModule", {value: true});
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
var _chunk5A44CWQEjs = require('./chunk-5A44CWQE.js');
|
|
13
|
+
require('./chunk-PRZ4S4NU.js');
|
|
14
|
+
require('./chunk-V77J5ISS.js');
|
|
15
|
+
require('./chunk-XNYIZMNQ.js');
|
|
16
|
+
require('./chunk-DROPDYOD.js');
|
|
17
|
+
require('./chunk-7P5AA3SN.js');
|
|
18
|
+
require('./chunk-OG64RBAC.js');
|
|
19
|
+
require('./chunk-COGTRVWW.js');
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
exports.EXIT_WORKSPACE_MISMATCH = _chunk5A44CWQEjs.EXIT_WORKSPACE_MISMATCH; exports.clearPendingClaim = _chunk5A44CWQEjs.clearPendingClaim; exports.clearSatisfiedPendingClaim = _chunk5A44CWQEjs.clearSatisfiedPendingClaim; exports.handleRegister = _chunk5A44CWQEjs.handleRegister; exports.handleStatus = _chunk5A44CWQEjs.handleStatus; exports.handleWait = _chunk5A44CWQEjs.handleWait; exports.readPendingAuthClaim = _chunk5A44CWQEjs.readPendingAuthClaim; exports.registerAuthCommands = _chunk5A44CWQEjs.registerAuthCommands; exports.reportSetupCompleted = _chunk5A44CWQEjs.reportSetupCompleted; exports.resolvePendingAuthClaimForResume = _chunk5A44CWQEjs.resolvePendingAuthClaimForResume;
|
|
@@ -9,11 +9,11 @@ import {
|
|
|
9
9
|
registerAuthCommands,
|
|
10
10
|
reportSetupCompleted,
|
|
11
11
|
resolvePendingAuthClaimForResume
|
|
12
|
-
} from "./chunk-
|
|
13
|
-
import "./chunk-
|
|
14
|
-
import "./chunk-
|
|
12
|
+
} from "./chunk-H5SORD7Q.mjs";
|
|
13
|
+
import "./chunk-HNX3RC6V.mjs";
|
|
14
|
+
import "./chunk-5UN6KLD2.mjs";
|
|
15
15
|
import "./chunk-DLIRSHV6.mjs";
|
|
16
|
-
import "./chunk-
|
|
16
|
+
import "./chunk-2CLENWOQ.mjs";
|
|
17
17
|
import "./chunk-NXGCO4JB.mjs";
|
|
18
18
|
import "./chunk-WZYKRD7T.mjs";
|
|
19
19
|
import "./chunk-KRD6H754.mjs";
|