deepline 0.3.141 → 0.3.142
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/sdk/src/release.ts +1 -1
- package/dist/bundling-sources/sdk/src/types.ts +65 -2
- package/dist/bundling-sources/shared_libs/observability/dlq.ts +11 -0
- package/dist/bundling-sources/shared_libs/observability/queue-health.ts +111 -0
- package/dist/bundling-sources/shared_libs/play-runtime/bettercontact-batching.ts +34 -8
- package/dist/bundling-sources/shared_libs/play-runtime/context.ts +271 -44
- package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +25 -6
- package/dist/bundling-sources/shared_libs/play-runtime/governor/app-runtime-rate-state-backend.ts +132 -8
- package/dist/bundling-sources/shared_libs/play-runtime/play-runtime-batching-registry.ts +3 -1
- package/dist/bundling-sources/shared_libs/play-runtime/runner-app/index.ts +49 -54
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +14 -7
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-rate-state.ts +118 -29
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-queue-health.ts +183 -36
- package/dist/bundling-sources/shared_libs/product-notifications/events.ts +4 -1
- package/dist/cli/index.js +242 -12
- package/dist/cli/index.mjs +242 -12
- package/dist/index.d.mts +54 -2
- package/dist/index.d.ts +54 -2
- package/dist/index.js +1 -1
- package/dist/index.mjs +1 -1
- package/dist/release.d.mts +1 -1
- package/dist/release.d.ts +1 -1
- package/dist/release.js +1 -1
- package/dist/release.mjs +1 -1
- package/package.json +1 -1
package/dist/bundling-sources/shared_libs/play-runtime/governor/app-runtime-rate-state-backend.ts
CHANGED
|
@@ -171,6 +171,13 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
|
|
|
171
171
|
private readonly blocks = new Map<string, LeasedBlock>();
|
|
172
172
|
private readonly refills = new Map<string, Promise<void>>();
|
|
173
173
|
private readonly pendingReleases = new Map<string, PendingRelease>();
|
|
174
|
+
/**
|
|
175
|
+
* Last locally synthesized dispatch slot per lane. During a rolling deploy,
|
|
176
|
+
* a gateway that omits its global schedule can return another block while
|
|
177
|
+
* the previous block's permits are still sleeping. Carrying this debt across
|
|
178
|
+
* refills prevents those fallback blocks from overlapping in one runner.
|
|
179
|
+
*/
|
|
180
|
+
private readonly nextFallbackScheduledAtMs = new Map<string, number>();
|
|
174
181
|
/**
|
|
175
182
|
* Acquirers currently parked on each bucket's refill (empty block, awaiting a
|
|
176
183
|
* grant). Demand-sizing reads this to request a block sized to the live wave
|
|
@@ -734,13 +741,41 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
|
|
|
734
741
|
}
|
|
735
742
|
// Rolling deploy compatibility: an older gateway returns no schedule. It
|
|
736
743
|
// has already debited the block under the legacy token-bucket contract, so
|
|
737
|
-
//
|
|
738
|
-
//
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
744
|
+
// preserve the grant but synthesize a conservative local schedule. Making
|
|
745
|
+
// every legacy permit immediately usable would turn a reserved block into
|
|
746
|
+
// a burst during a rolling deploy and violate the provider rate.
|
|
747
|
+
const fallbackRequestsPerSecond =
|
|
748
|
+
effectiveRequestsPerSecond > 0
|
|
749
|
+
? effectiveRequestsPerSecond
|
|
750
|
+
: Math.min(
|
|
751
|
+
...ordered
|
|
752
|
+
.map((rule) => (rule.requestsPerWindow / rule.windowMs) * 1_000)
|
|
753
|
+
.filter((rate) => Number.isFinite(rate) && rate > 0),
|
|
754
|
+
DEFAULT_ADVISORY_PROVIDER_MAX_RPS,
|
|
755
|
+
);
|
|
756
|
+
const fallbackSpacingMs = Math.max(
|
|
757
|
+
1,
|
|
758
|
+
Math.ceil(1_000 / fallbackRequestsPerSecond),
|
|
759
|
+
);
|
|
760
|
+
// The gateway deliberately emits burst groups at high rates. A flat
|
|
761
|
+
// schedule is therefore valid when it contains no more than the gateway's
|
|
762
|
+
// burst size; only a longer flat run is evidence of the rolling-deploy
|
|
763
|
+
// response shape that needs local repair.
|
|
764
|
+
const gatewayBurstSize = Math.max(
|
|
765
|
+
1,
|
|
766
|
+
Math.floor(fallbackRequestsPerSecond / 100),
|
|
767
|
+
);
|
|
743
768
|
const receivedAtMs = this.now();
|
|
769
|
+
const hasGatewaySchedule = Array.isArray(response.scheduledAtMs);
|
|
770
|
+
const gatewayScheduledAtMs: number[] = Array.isArray(
|
|
771
|
+
response.scheduledAtMs,
|
|
772
|
+
)
|
|
773
|
+
? response.scheduledAtMs.map((scheduledAtMs) => Number(scheduledAtMs))
|
|
774
|
+
: Array.from(
|
|
775
|
+
{ length: response.granted },
|
|
776
|
+
(_, index) => receivedAtMs + index * fallbackSpacingMs,
|
|
777
|
+
);
|
|
778
|
+
const serverNowMs = Number(response.serverNowMs);
|
|
744
779
|
let firstGatewayScheduledAtMs = Number.POSITIVE_INFINITY;
|
|
745
780
|
for (const scheduledAtMs of gatewayScheduledAtMs) {
|
|
746
781
|
firstGatewayScheduledAtMs = Math.min(
|
|
@@ -748,15 +783,102 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
|
|
|
748
783
|
scheduledAtMs,
|
|
749
784
|
);
|
|
750
785
|
}
|
|
751
|
-
const
|
|
786
|
+
const translatedScheduledAtMs =
|
|
752
787
|
Number.isFinite(serverNowMs) && serverNowMs >= 0
|
|
753
788
|
? gatewayScheduledAtMs.map(
|
|
754
789
|
(scheduledAtMs) =>
|
|
755
790
|
receivedAtMs +
|
|
756
791
|
Math.max(0, firstGatewayScheduledAtMs - serverNowMs) +
|
|
757
792
|
(scheduledAtMs - firstGatewayScheduledAtMs),
|
|
758
|
-
|
|
793
|
+
)
|
|
759
794
|
: gatewayScheduledAtMs;
|
|
795
|
+
const previousFallbackScheduledAtMs =
|
|
796
|
+
this.nextFallbackScheduledAtMs.get(laneKey);
|
|
797
|
+
if (
|
|
798
|
+
previousFallbackScheduledAtMs !== undefined &&
|
|
799
|
+
previousFallbackScheduledAtMs <= receivedAtMs
|
|
800
|
+
) {
|
|
801
|
+
this.nextFallbackScheduledAtMs.delete(laneKey);
|
|
802
|
+
}
|
|
803
|
+
const fallbackScheduleFloor = Math.max(
|
|
804
|
+
receivedAtMs,
|
|
805
|
+
this.nextFallbackScheduledAtMs.get(laneKey) ?? receivedAtMs,
|
|
806
|
+
);
|
|
807
|
+
let gatewayScheduleValid = hasGatewaySchedule;
|
|
808
|
+
let equalTimestampRun = 0;
|
|
809
|
+
for (let index = 0; index < translatedScheduledAtMs.length; index += 1) {
|
|
810
|
+
const candidate = translatedScheduledAtMs[index];
|
|
811
|
+
if (!Number.isFinite(candidate) || candidate < 0) {
|
|
812
|
+
gatewayScheduleValid = false;
|
|
813
|
+
break;
|
|
814
|
+
}
|
|
815
|
+
if (index === 0) {
|
|
816
|
+
equalTimestampRun = 1;
|
|
817
|
+
continue;
|
|
818
|
+
}
|
|
819
|
+
const previous = translatedScheduledAtMs[index - 1];
|
|
820
|
+
if (candidate < previous) {
|
|
821
|
+
gatewayScheduleValid = false;
|
|
822
|
+
break;
|
|
823
|
+
}
|
|
824
|
+
equalTimestampRun = candidate === previous ? equalTimestampRun + 1 : 1;
|
|
825
|
+
if (equalTimestampRun > gatewayBurstSize) {
|
|
826
|
+
gatewayScheduleValid = false;
|
|
827
|
+
break;
|
|
828
|
+
}
|
|
829
|
+
}
|
|
830
|
+
|
|
831
|
+
let scheduledAtMs: number[];
|
|
832
|
+
if (gatewayScheduleValid) {
|
|
833
|
+
// Preserve the gateway's exact cadence, including legitimate burst
|
|
834
|
+
// groups and provider-imposed gaps. Only shift the whole schedule when
|
|
835
|
+
// it is already overdue or must clear fallback debt from an older block.
|
|
836
|
+
const firstScheduledAtMs = translatedScheduledAtMs[0];
|
|
837
|
+
const shiftMs =
|
|
838
|
+
firstScheduledAtMs === undefined
|
|
839
|
+
? 0
|
|
840
|
+
: Math.max(0, fallbackScheduleFloor - firstScheduledAtMs);
|
|
841
|
+
scheduledAtMs = translatedScheduledAtMs.map(
|
|
842
|
+
(scheduledAtMs) => scheduledAtMs + shiftMs,
|
|
843
|
+
);
|
|
844
|
+
this.nextFallbackScheduledAtMs.delete(laneKey);
|
|
845
|
+
} else {
|
|
846
|
+
// A rolling gateway may expose the new schedule field before it starts
|
|
847
|
+
// populating distinct timestamps. Do not let that invalid schedule turn
|
|
848
|
+
// a reserved block into a burst. Use the conservative local cadence
|
|
849
|
+
// implied by the effective/declared rate, carrying any prior fallback
|
|
850
|
+
// debt across refills.
|
|
851
|
+
const positiveGatewayGaps = translatedScheduledAtMs.flatMap(
|
|
852
|
+
(scheduledAtMs, index) => {
|
|
853
|
+
if (index === 0) return [];
|
|
854
|
+
const gap = scheduledAtMs - translatedScheduledAtMs[index - 1];
|
|
855
|
+
return Number.isFinite(gap) && gap > 0 ? [gap] : [];
|
|
856
|
+
},
|
|
857
|
+
);
|
|
858
|
+
const repairSpacingMs = Math.min(
|
|
859
|
+
fallbackSpacingMs,
|
|
860
|
+
...(positiveGatewayGaps.length > 0
|
|
861
|
+
? [Math.min(...positiveGatewayGaps)]
|
|
862
|
+
: []),
|
|
863
|
+
);
|
|
864
|
+
let previousScheduledAtMs: number | null = null;
|
|
865
|
+
scheduledAtMs = translatedScheduledAtMs.map((candidate, index) => {
|
|
866
|
+
const earliest =
|
|
867
|
+
index === 0
|
|
868
|
+
? fallbackScheduleFloor
|
|
869
|
+
: previousScheduledAtMs! + repairSpacingMs;
|
|
870
|
+
const normalized = Math.max(candidate, earliest);
|
|
871
|
+
previousScheduledAtMs = normalized;
|
|
872
|
+
return normalized;
|
|
873
|
+
});
|
|
874
|
+
const lastScheduledAtMs = scheduledAtMs.at(-1);
|
|
875
|
+
if (lastScheduledAtMs !== undefined) {
|
|
876
|
+
this.nextFallbackScheduledAtMs.set(
|
|
877
|
+
laneKey,
|
|
878
|
+
lastScheduledAtMs + repairSpacingMs,
|
|
879
|
+
);
|
|
880
|
+
}
|
|
881
|
+
}
|
|
760
882
|
if (
|
|
761
883
|
scheduledAtMs.length !== response.granted ||
|
|
762
884
|
scheduledAtMs.some((value) => !Number.isFinite(value) || value < 0)
|
|
@@ -986,6 +1108,7 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
|
|
|
986
1108
|
if (!block) return null;
|
|
987
1109
|
if (block.expiresAt <= this.now()) {
|
|
988
1110
|
this.blocks.delete(laneKey);
|
|
1111
|
+
this.nextFallbackScheduledAtMs.delete(laneKey);
|
|
989
1112
|
// Unused window permits are forfeited on TTL expiry (the server already
|
|
990
1113
|
// decremented the provider window for them, and the rule-scoped TTL is
|
|
991
1114
|
// capped so at most one block's worth is stranded per bucket/window).
|
|
@@ -1022,6 +1145,7 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
|
|
|
1022
1145
|
for (const [laneKey, block] of this.blocks) {
|
|
1023
1146
|
if (!laneKey.startsWith(prefix)) continue;
|
|
1024
1147
|
this.blocks.delete(laneKey);
|
|
1148
|
+
this.nextFallbackScheduledAtMs.delete(laneKey);
|
|
1025
1149
|
this.releaseReserved(
|
|
1026
1150
|
bucketId,
|
|
1027
1151
|
block.rateScopeToken,
|
|
@@ -26,5 +26,7 @@ export function getPlayRuntimeBatchStrategy(
|
|
|
26
26
|
if (!operation) {
|
|
27
27
|
return null;
|
|
28
28
|
}
|
|
29
|
-
|
|
29
|
+
const canonicalOperation =
|
|
30
|
+
operation === 'fullenrich_enrich' ? 'fullenrich_bulk_enrich' : operation;
|
|
31
|
+
return PLAY_RUNTIME_BATCH_OPERATION_REGISTRY[canonicalOperation] ?? null;
|
|
30
32
|
}
|
|
@@ -109,14 +109,13 @@ import {
|
|
|
109
109
|
readRuntimeSheetDatasetRowKeys,
|
|
110
110
|
readRuntimeSheetDatasetRows,
|
|
111
111
|
releaseRuntimeWorkReceipts,
|
|
112
|
-
RuntimeSheetAttemptSupersededError,
|
|
113
112
|
startRuntimeSheetDataset,
|
|
114
113
|
supersededRuntimeMapRows,
|
|
114
|
+
type RuntimeMapRowsStaleRow,
|
|
115
115
|
type RuntimeMapRowsWriteResult,
|
|
116
116
|
} from '../runtime-api';
|
|
117
117
|
import { installPgRuntimePoolDriver } from '../runtime-pg-driver-pg';
|
|
118
118
|
import { createRuntimeToolAuthScopeDigestResolver } from '../auth-scope-resolver';
|
|
119
|
-
import { resolveMapRowOutcomeKey } from '../map-row-outcome';
|
|
120
119
|
import { createRunScopedChildPlayResolver } from './child-play-resolver';
|
|
121
120
|
import { normalizePlayContractCompatibility } from '../../plays/contracts';
|
|
122
121
|
import {
|
|
@@ -365,8 +364,7 @@ export function assertRuntimeSheetAttemptPersistedTerminalRows(input: {
|
|
|
365
364
|
}): void {
|
|
366
365
|
// Kept as a temporary source-compatible seam for callers compiled against
|
|
367
366
|
// the lease model. A partial update is classified from the terminal-write
|
|
368
|
-
// result
|
|
369
|
-
// count.
|
|
367
|
+
// result, never from this count.
|
|
370
368
|
void input;
|
|
371
369
|
}
|
|
372
370
|
|
|
@@ -453,33 +451,39 @@ export function runtimeSheetStartWriteVersion(
|
|
|
453
451
|
: {};
|
|
454
452
|
}
|
|
455
453
|
|
|
456
|
-
/**
|
|
457
|
-
|
|
458
|
-
* same-run settlement or an unversioned already-terminal row mean the sheet
|
|
459
|
-
* rows belong to a different write version than this attempt's: a newer
|
|
460
|
-
* admission (rerun, resumed attempt, concurrent run) owns them. Throw so the
|
|
461
|
-
* map fails and the persistence latch trips, instead of logging a perf line
|
|
462
|
-
* and spending provider calls on rows that can never be persisted.
|
|
463
|
-
*/
|
|
464
|
-
export function assertRuntimeSheetTerminalWriteNotSuperseded(input: {
|
|
465
|
-
tableNamespace: string;
|
|
466
|
-
runId: string;
|
|
467
|
-
writeVersion: SheetWriteVersionValue | null;
|
|
468
|
-
submittedRows: number;
|
|
454
|
+
/** Stale latest-view rows are reported, not escalated to a run failure. */
|
|
455
|
+
export function runtimeSheetTerminalWriteStaleRows(input: {
|
|
469
456
|
result: Pick<RuntimeMapRowsWriteResult, 'staleKeys' | 'staleRows'>;
|
|
470
457
|
recoveredStaleKeys: Iterable<string>;
|
|
471
|
-
}):
|
|
472
|
-
|
|
458
|
+
}): RuntimeMapRowsStaleRow[] {
|
|
459
|
+
return supersededRuntimeMapRows(input.result, {
|
|
473
460
|
exceptKeys: input.recoveredStaleKeys,
|
|
474
461
|
});
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
462
|
+
}
|
|
463
|
+
|
|
464
|
+
/** Only accepted current-sheet rows contribute to persisted lifecycle counts. */
|
|
465
|
+
export function runtimeSheetTerminalWriteCounts(input: {
|
|
466
|
+
rows: ReadonlyArray<{ key: string; status?: string }>;
|
|
467
|
+
committedKeys: Iterable<string>;
|
|
468
|
+
recoveredStaleKeys: Iterable<string>;
|
|
469
|
+
}): { persistedRows: number; succeededRows: number; failedRows: number } {
|
|
470
|
+
const acceptedKeys = new Set([
|
|
471
|
+
...input.committedKeys,
|
|
472
|
+
...input.recoveredStaleKeys,
|
|
473
|
+
]);
|
|
474
|
+
const acceptedRows = new Map(
|
|
475
|
+
input.rows
|
|
476
|
+
.filter((row) => acceptedKeys.has(row.key))
|
|
477
|
+
.map((row) => [row.key, row]),
|
|
478
|
+
);
|
|
479
|
+
const failedRows = [...acceptedRows.values()].filter(
|
|
480
|
+
(row) => row.status === 'failed',
|
|
481
|
+
).length;
|
|
482
|
+
return {
|
|
483
|
+
persistedRows: acceptedRows.size,
|
|
484
|
+
succeededRows: acceptedRows.size - failedRows,
|
|
485
|
+
failedRows,
|
|
486
|
+
};
|
|
483
487
|
}
|
|
484
488
|
|
|
485
489
|
async function vercelBypassHeaders(
|
|
@@ -2686,34 +2690,27 @@ async function run(
|
|
|
2686
2690
|
`[info] dropped ${staleKeys.size} stale runtime sheet write(s) for table=${input.tableNamespace}; ${recoveredStaleKeys.length} same-run terminal write(s) were accepted idempotently`,
|
|
2687
2691
|
options,
|
|
2688
2692
|
);
|
|
2689
|
-
|
|
2690
|
-
|
|
2691
|
-
|
|
2692
|
-
|
|
2693
|
-
|
|
2694
|
-
submittedRows: input.rows.length,
|
|
2695
|
-
result,
|
|
2696
|
-
recoveredStaleKeys,
|
|
2697
|
-
});
|
|
2698
|
-
} catch (error) {
|
|
2693
|
+
const supersededRows = runtimeSheetTerminalWriteStaleRows({
|
|
2694
|
+
result,
|
|
2695
|
+
recoveredStaleKeys,
|
|
2696
|
+
});
|
|
2697
|
+
if (supersededRows.length > 0) {
|
|
2699
2698
|
emitWorkerLog(
|
|
2700
|
-
`[
|
|
2699
|
+
`[info] ${supersededRows.length} Runtime Sheet row write(s) did not own the latest view; ` +
|
|
2700
|
+
`the existing latest-view values were preserved and this run will continue`,
|
|
2701
2701
|
options,
|
|
2702
2702
|
);
|
|
2703
|
-
throw error;
|
|
2704
2703
|
}
|
|
2705
2704
|
}
|
|
2706
|
-
const
|
|
2707
|
-
|
|
2708
|
-
|
|
2709
|
-
|
|
2710
|
-
|
|
2711
|
-
|
|
2712
|
-
|
|
2713
|
-
|
|
2714
|
-
|
|
2715
|
-
const committedTerminalRows = result.committedKeys.length;
|
|
2716
|
-
const succeededRows = Math.max(0, committedTerminalRows - failedRows);
|
|
2705
|
+
const {
|
|
2706
|
+
persistedRows: committedTerminalRows,
|
|
2707
|
+
succeededRows,
|
|
2708
|
+
failedRows,
|
|
2709
|
+
} = runtimeSheetTerminalWriteCounts({
|
|
2710
|
+
rows: input.rows,
|
|
2711
|
+
committedKeys: result.committedKeys,
|
|
2712
|
+
recoveredStaleKeys,
|
|
2713
|
+
});
|
|
2717
2714
|
const datasetPath = `datasets.${input.tableNamespace}`;
|
|
2718
2715
|
const datasetId = createRuntimeDatasetId(playName, input.tableNamespace);
|
|
2719
2716
|
const settlement = runtimeMapDatasetSettlements.get(datasetId) ?? {
|
|
@@ -2725,10 +2722,8 @@ async function run(
|
|
|
2725
2722
|
succeededRows: 0,
|
|
2726
2723
|
failedRows: 0,
|
|
2727
2724
|
};
|
|
2728
|
-
//
|
|
2729
|
-
//
|
|
2730
|
-
// same-run terminal settlements, counted when they first committed, so
|
|
2731
|
-
// they stay excluded here.
|
|
2725
|
+
// Rejected stale rows have no retained copy. Verified same-run terminal
|
|
2726
|
+
// replays count as accepted even when they performed no physical update.
|
|
2732
2727
|
settlement.persistedRows += committedTerminalRows;
|
|
2733
2728
|
settlement.succeededRows += succeededRows;
|
|
2734
2729
|
settlement.failedRows += failedRows;
|
|
@@ -7168,12 +7168,11 @@ export function supersededRuntimeMapRows(
|
|
|
7168
7168
|
}
|
|
7169
7169
|
|
|
7170
7170
|
/**
|
|
7171
|
-
* A terminal write batch was
|
|
7172
|
-
*
|
|
7173
|
-
*
|
|
7174
|
-
*
|
|
7175
|
-
*
|
|
7176
|
-
* reports `supersededKeys`; both trip the persistence latch.
|
|
7171
|
+
* A terminal write batch was explicitly marked as an attempt-wide
|
|
7172
|
+
* supersession. Retained for compatibility with older terminal-write handlers
|
|
7173
|
+
* and the Context's explicit `supersededKeys` stop signal. The current
|
|
7174
|
+
* runner-app writer reports ordinary stale latest-view rows without throwing
|
|
7175
|
+
* this error.
|
|
7177
7176
|
*/
|
|
7178
7177
|
export class RuntimeSheetAttemptSupersededError extends Error {
|
|
7179
7178
|
readonly tableNamespace: string;
|
|
@@ -8995,7 +8994,13 @@ export async function countRuntimeSheetRowOutcomes(
|
|
|
8995
8994
|
params?: unknown[],
|
|
8996
8995
|
): Promise<{ rows: Row[] }>;
|
|
8997
8996
|
},
|
|
8998
|
-
input: {
|
|
8997
|
+
input: {
|
|
8998
|
+
schema: string;
|
|
8999
|
+
table: string;
|
|
9000
|
+
runId: string;
|
|
9001
|
+
playName?: string;
|
|
9002
|
+
tableNamespace?: string;
|
|
9003
|
+
},
|
|
8999
9004
|
): Promise<RuntimeSheetRowOutcomeCounts> {
|
|
9000
9005
|
const { rows } = await client.query<{
|
|
9001
9006
|
completed: number | string;
|
|
@@ -9052,6 +9057,8 @@ export async function readRuntimeSheetRowOutcomeCounts(
|
|
|
9052
9057
|
schema: session.postgres.schema,
|
|
9053
9058
|
table: session.postgres.sheetTable,
|
|
9054
9059
|
runId: context.runId,
|
|
9060
|
+
playName: context.playName,
|
|
9061
|
+
tableNamespace: input.tableNamespace,
|
|
9055
9062
|
}),
|
|
9056
9063
|
);
|
|
9057
9064
|
}
|
package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-rate-state.ts
CHANGED
|
@@ -89,10 +89,12 @@ const BLOCK_WAIT_HORIZON_MS = 1_000;
|
|
|
89
89
|
const ADAPTIVE_BURST_HORIZON_MS = 1_000;
|
|
90
90
|
|
|
91
91
|
/**
|
|
92
|
-
* Effective (spendable) bucket capacity for rate
|
|
92
|
+
* Effective (spendable) bucket capacity for a rate-row alias. Derived per
|
|
93
93
|
* statement from the learned rate; see {@link ADAPTIVE_BURST_HORIZON_MS}.
|
|
94
94
|
*/
|
|
95
|
-
|
|
95
|
+
function effectiveCapacitySql(alias: string): string {
|
|
96
|
+
return `GREATEST(${alias}.capacity, ${alias}.refill_per_sec * ${ADAPTIVE_BURST_HORIZON_MS} / 1000.0)`;
|
|
97
|
+
}
|
|
96
98
|
|
|
97
99
|
/**
|
|
98
100
|
* Internal cooldown cap. A 429 report (penalize) can carry an arbitrarily large
|
|
@@ -1047,26 +1049,66 @@ async function acquirePureRate(
|
|
|
1047
1049
|
SELECT
|
|
1048
1050
|
b.bucket_id,
|
|
1049
1051
|
b.rule_id,
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1052
|
+
sc.capacity,
|
|
1053
|
+
sc.min_rps,
|
|
1054
|
+
CASE
|
|
1055
|
+
WHEN sc.adaptive_max_configured THEN sc.max_rps
|
|
1056
|
+
ELSE b.max_rps
|
|
1057
|
+
END AS max_rps,
|
|
1053
1058
|
b.seen_backpressure,
|
|
1054
|
-
|
|
1055
|
-
|
|
1059
|
+
sc.base_refill_per_sec,
|
|
1060
|
+
LEAST(
|
|
1061
|
+
CASE
|
|
1062
|
+
WHEN sc.adaptive_max_configured THEN sc.max_rps
|
|
1063
|
+
ELSE b.max_rps
|
|
1064
|
+
END,
|
|
1065
|
+
GREATEST(sc.min_rps, b.refill_per_sec)
|
|
1066
|
+
) AS refill_per_sec,
|
|
1056
1067
|
b.cool_until_ms,
|
|
1057
1068
|
b.last_ramp_at_ms,
|
|
1058
1069
|
b.next_dispatch_at_ms,
|
|
1059
1070
|
nv.now_ms AS now_ms,
|
|
1071
|
+
-- A warm row can have been widened by an older adaptive ceiling or
|
|
1072
|
+
-- edited out of band. Normalize it to the current seed before this
|
|
1073
|
+
-- statement computes tokens, schedules, or grants. The original
|
|
1074
|
+
-- values remain beside the normalized ones so config_drift still
|
|
1075
|
+
-- reports that the repair happened.
|
|
1076
|
+
b.capacity AS stored_capacity,
|
|
1077
|
+
b.base_refill_per_sec AS stored_base_refill_per_sec,
|
|
1078
|
+
b.min_rps AS stored_min_rps,
|
|
1079
|
+
b.max_rps AS stored_max_rps,
|
|
1080
|
+
b.adaptive_max_configured OR sc.adaptive_max_configured
|
|
1081
|
+
AS adaptive_max_configured,
|
|
1060
1082
|
-- Spendable burst: one second of the rate in force, floored at the
|
|
1061
|
-
-- declared window burst.
|
|
1062
|
-
--
|
|
1063
|
-
${
|
|
1083
|
+
-- declared window burst. The normalized row is what this acquire
|
|
1084
|
+
-- spends; the stored row remains available for drift reporting.
|
|
1085
|
+
${effectiveCapacitySql('normalized')} AS effective_capacity,
|
|
1064
1086
|
LEAST(
|
|
1065
|
-
${
|
|
1066
|
-
b.tokens
|
|
1087
|
+
${effectiveCapacitySql('normalized')},
|
|
1088
|
+
LEAST(b.tokens, sc.capacity)
|
|
1089
|
+
+ GREATEST(0, nv.now_ms - b.refilled_at_ms) / 1000.0
|
|
1090
|
+
* LEAST(
|
|
1091
|
+
CASE
|
|
1092
|
+
WHEN sc.adaptive_max_configured THEN sc.max_rps
|
|
1093
|
+
ELSE b.max_rps
|
|
1094
|
+
END,
|
|
1095
|
+
GREATEST(sc.min_rps, b.refill_per_sec)
|
|
1096
|
+
)
|
|
1067
1097
|
) AS tokens_after_refill
|
|
1068
1098
|
FROM ${bucketTableName(input)} b
|
|
1099
|
+
JOIN seed_config sc ON sc.rule_id = b.rule_id
|
|
1069
1100
|
CROSS JOIN now_value nv
|
|
1101
|
+
CROSS JOIN LATERAL (
|
|
1102
|
+
SELECT
|
|
1103
|
+
sc.capacity AS capacity,
|
|
1104
|
+
LEAST(
|
|
1105
|
+
CASE
|
|
1106
|
+
WHEN sc.adaptive_max_configured THEN sc.max_rps
|
|
1107
|
+
ELSE b.max_rps
|
|
1108
|
+
END,
|
|
1109
|
+
GREATEST(sc.min_rps, b.refill_per_sec)
|
|
1110
|
+
) AS refill_per_sec
|
|
1111
|
+
) normalized
|
|
1070
1112
|
WHERE b.bucket_id = $1
|
|
1071
1113
|
AND b.rule_id = ANY($2::text[])
|
|
1072
1114
|
ORDER BY b.rule_id
|
|
@@ -1119,6 +1161,11 @@ async function acquirePureRate(
|
|
|
1119
1161
|
updated AS (
|
|
1120
1162
|
UPDATE ${bucketTableName(input)} b
|
|
1121
1163
|
SET
|
|
1164
|
+
capacity = r.capacity,
|
|
1165
|
+
base_refill_per_sec = r.base_refill_per_sec,
|
|
1166
|
+
min_rps = r.min_rps,
|
|
1167
|
+
max_rps = r.max_rps,
|
|
1168
|
+
adaptive_max_configured = r.adaptive_max_configured,
|
|
1122
1169
|
tokens = r.tokens_after_refill - (SELECT granted FROM grant_calc),
|
|
1123
1170
|
refilled_at_ms = r.now_ms,
|
|
1124
1171
|
next_dispatch_at_ms = CASE
|
|
@@ -1200,12 +1247,12 @@ async function acquirePureRate(
|
|
|
1200
1247
|
(
|
|
1201
1248
|
SELECT COALESCE(
|
|
1202
1249
|
bool_or(
|
|
1203
|
-
r.
|
|
1204
|
-
OR r.
|
|
1205
|
-
OR r.
|
|
1250
|
+
r.stored_capacity IS DISTINCT FROM sc.capacity
|
|
1251
|
+
OR r.stored_base_refill_per_sec IS DISTINCT FROM sc.base_refill_per_sec
|
|
1252
|
+
OR r.stored_min_rps IS DISTINCT FROM sc.min_rps
|
|
1206
1253
|
OR (
|
|
1207
1254
|
sc.adaptive_max_configured
|
|
1208
|
-
AND r.
|
|
1255
|
+
AND r.stored_max_rps IS DISTINCT FROM sc.max_rps
|
|
1209
1256
|
)
|
|
1210
1257
|
),
|
|
1211
1258
|
false
|
|
@@ -1432,27 +1479,64 @@ async function acquireWithConcurrency(
|
|
|
1432
1479
|
refreshed AS (
|
|
1433
1480
|
SELECT
|
|
1434
1481
|
b.rule_id,
|
|
1435
|
-
|
|
1436
|
-
|
|
1437
|
-
|
|
1482
|
+
sc.capacity,
|
|
1483
|
+
sc.min_rps,
|
|
1484
|
+
CASE
|
|
1485
|
+
WHEN sc.adaptive_max_configured THEN sc.max_rps
|
|
1486
|
+
ELSE b.max_rps
|
|
1487
|
+
END AS max_rps,
|
|
1438
1488
|
b.seen_backpressure,
|
|
1439
|
-
|
|
1489
|
+
sc.base_refill_per_sec,
|
|
1490
|
+
LEAST(
|
|
1491
|
+
CASE
|
|
1492
|
+
WHEN sc.adaptive_max_configured THEN sc.max_rps
|
|
1493
|
+
ELSE b.max_rps
|
|
1494
|
+
END,
|
|
1495
|
+
GREATEST(sc.min_rps, b.refill_per_sec)
|
|
1496
|
+
) AS refill_per_sec,
|
|
1440
1497
|
b.learned_concurrency_limit,
|
|
1441
1498
|
b.concurrency_minimum_rtt_ms,
|
|
1442
1499
|
b.cool_until_ms,
|
|
1443
1500
|
b.last_ramp_at_ms,
|
|
1444
1501
|
b.next_dispatch_at_ms,
|
|
1445
1502
|
nv.now_ms AS now_ms,
|
|
1503
|
+
-- Normalize a warm row before deriving its grant. The stored values
|
|
1504
|
+
-- are retained for the config_drift diagnostic below.
|
|
1505
|
+
b.capacity AS stored_capacity,
|
|
1506
|
+
b.base_refill_per_sec AS stored_base_refill_per_sec,
|
|
1507
|
+
b.min_rps AS stored_min_rps,
|
|
1508
|
+
b.max_rps AS stored_max_rps,
|
|
1509
|
+
b.adaptive_max_configured OR sc.adaptive_max_configured
|
|
1510
|
+
AS adaptive_max_configured,
|
|
1446
1511
|
-- Same derived burst as the pure-rate path. The declared
|
|
1447
1512
|
-- maxConcurrency is untouched: free_slots still bounds the grant.
|
|
1448
|
-
${
|
|
1513
|
+
${effectiveCapacitySql('normalized')} AS effective_capacity,
|
|
1449
1514
|
LEAST(
|
|
1450
|
-
${
|
|
1451
|
-
b.tokens
|
|
1452
|
-
|
|
1453
|
-
|
|
1515
|
+
${effectiveCapacitySql('normalized')},
|
|
1516
|
+
LEAST(b.tokens, sc.capacity)
|
|
1517
|
+
+ GREATEST(0, nv.now_ms - b.refilled_at_ms) / 1000.0
|
|
1518
|
+
* LEAST(
|
|
1519
|
+
CASE
|
|
1520
|
+
WHEN sc.adaptive_max_configured THEN sc.max_rps
|
|
1521
|
+
ELSE b.max_rps
|
|
1522
|
+
END,
|
|
1523
|
+
GREATEST(sc.min_rps, b.refill_per_sec)
|
|
1524
|
+
)
|
|
1525
|
+
) AS tokens_after_refill
|
|
1454
1526
|
FROM ${bucketTableName(input)} b
|
|
1527
|
+
JOIN seed_config sc ON sc.rule_id = b.rule_id
|
|
1455
1528
|
CROSS JOIN now_value nv
|
|
1529
|
+
CROSS JOIN LATERAL (
|
|
1530
|
+
SELECT
|
|
1531
|
+
sc.capacity AS capacity,
|
|
1532
|
+
LEAST(
|
|
1533
|
+
CASE
|
|
1534
|
+
WHEN sc.adaptive_max_configured THEN sc.max_rps
|
|
1535
|
+
ELSE b.max_rps
|
|
1536
|
+
END,
|
|
1537
|
+
GREATEST(sc.min_rps, b.refill_per_sec)
|
|
1538
|
+
) AS refill_per_sec
|
|
1539
|
+
) normalized
|
|
1456
1540
|
WHERE b.bucket_id = $1
|
|
1457
1541
|
AND b.rule_id = ANY($2::text[])
|
|
1458
1542
|
ORDER BY b.rule_id
|
|
@@ -1560,6 +1644,11 @@ async function acquireWithConcurrency(
|
|
|
1560
1644
|
updated AS (
|
|
1561
1645
|
UPDATE ${bucketTableName(input)} b
|
|
1562
1646
|
SET
|
|
1647
|
+
capacity = r.capacity,
|
|
1648
|
+
base_refill_per_sec = r.base_refill_per_sec,
|
|
1649
|
+
min_rps = r.min_rps,
|
|
1650
|
+
max_rps = r.max_rps,
|
|
1651
|
+
adaptive_max_configured = r.adaptive_max_configured,
|
|
1563
1652
|
tokens = r.tokens_after_refill - (SELECT granted FROM grant_calc),
|
|
1564
1653
|
refilled_at_ms = r.now_ms,
|
|
1565
1654
|
next_dispatch_at_ms = CASE
|
|
@@ -1667,12 +1756,12 @@ async function acquireWithConcurrency(
|
|
|
1667
1756
|
(
|
|
1668
1757
|
SELECT COALESCE(
|
|
1669
1758
|
bool_or(
|
|
1670
|
-
r.
|
|
1671
|
-
OR r.
|
|
1672
|
-
OR r.
|
|
1759
|
+
r.stored_capacity IS DISTINCT FROM sc.capacity
|
|
1760
|
+
OR r.stored_base_refill_per_sec IS DISTINCT FROM sc.base_refill_per_sec
|
|
1761
|
+
OR r.stored_min_rps IS DISTINCT FROM sc.min_rps
|
|
1673
1762
|
OR (
|
|
1674
1763
|
sc.adaptive_max_configured
|
|
1675
|
-
AND r.
|
|
1764
|
+
AND r.stored_max_rps IS DISTINCT FROM sc.max_rps
|
|
1676
1765
|
)
|
|
1677
1766
|
),
|
|
1678
1767
|
false
|