deepline 0.3.128 → 0.3.129
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/sdk/src/client.ts +34 -9
- package/dist/bundling-sources/sdk/src/http.ts +22 -0
- package/dist/bundling-sources/sdk/src/release.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-data-plane/r2.ts +22 -0
- package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +265 -2
- package/dist/bundling-sources/shared_libs/play-runtime/bounded-batch-function.ts +682 -0
- package/dist/bundling-sources/shared_libs/play-runtime/child-execution-strategy.ts +21 -5
- package/dist/bundling-sources/shared_libs/play-runtime/context.ts +5390 -1732
- package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +107 -1
- package/dist/bundling-sources/shared_libs/play-runtime/deployment/commands/roll-execution-relay.ts +7 -0
- package/dist/bundling-sources/shared_libs/play-runtime/deployment/fly-machines.ts +229 -7
- package/dist/bundling-sources/shared_libs/play-runtime/deployment/fly-registry.ts +25 -4
- package/dist/bundling-sources/shared_libs/play-runtime/deployment/runtime-gateway-capacity.ts +71 -0
- package/dist/bundling-sources/shared_libs/play-runtime/deployment/runtime-gateway-drain.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/deployment/runtime-topology.ts +4 -1
- package/dist/bundling-sources/shared_libs/play-runtime/durable-receipt-execution.ts +694 -28
- package/dist/bundling-sources/shared_libs/play-runtime/egress-bandwidth-probe.ts +167 -0
- package/dist/bundling-sources/shared_libs/play-runtime/execute-server-timing.ts +103 -0
- package/dist/bundling-sources/shared_libs/play-runtime/execution-gateway-receipt-output-reference.ts +34 -0
- package/dist/bundling-sources/shared_libs/play-runtime/execution-gateway-response-size.ts +27 -0
- package/dist/bundling-sources/shared_libs/play-runtime/gateway-postgres-admission.ts +66 -1
- package/dist/bundling-sources/shared_libs/play-runtime/governor/app-runtime-rate-state-backend.ts +425 -71
- package/dist/bundling-sources/shared_libs/play-runtime/governor/governor.ts +210 -32
- package/dist/bundling-sources/shared_libs/play-runtime/governor/in-memory-rate-state-backend.ts +27 -13
- package/dist/bundling-sources/shared_libs/play-runtime/governor/policy.ts +19 -6
- package/dist/bundling-sources/shared_libs/play-runtime/governor/rate-state-backend.ts +61 -2
- package/dist/bundling-sources/shared_libs/play-runtime/integration-operation-reader-fence/contracts.ts +33 -1
- package/dist/bundling-sources/shared_libs/play-runtime/integration-operation-reader-fence/job.ts +42 -1
- package/dist/bundling-sources/shared_libs/play-runtime/launch-source-policy.ts +11 -11
- package/dist/bundling-sources/shared_libs/play-runtime/ledger-safe-payload.ts +49 -16
- package/dist/bundling-sources/shared_libs/play-runtime/map-memory-limits.ts +1653 -98
- package/dist/bundling-sources/shared_libs/play-runtime/memory-ledger.ts +807 -0
- package/dist/bundling-sources/shared_libs/play-runtime/memory-pressure-signal.ts +138 -0
- package/dist/bundling-sources/shared_libs/play-runtime/output-size-limits.ts +85 -3
- package/dist/bundling-sources/shared_libs/play-runtime/persistence-latch.ts +60 -2
- package/dist/bundling-sources/shared_libs/play-runtime/play-run-recovery-policy.ts +46 -1
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/terminal/contracts.ts +2 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/terminal/job.ts +1 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/terminal/ports.ts +1 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/terminal/postgres-adapter.ts +1 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runtime-semantics.ts +70 -27
- package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts +56 -0
- package/dist/bundling-sources/shared_libs/play-runtime/provider-capacity/adaptive-provider-capacity-controller.ts +192 -0
- package/dist/bundling-sources/shared_libs/play-runtime/provider-capacity/epoch-aimd-rate-controller.ts +194 -0
- package/dist/bundling-sources/shared_libs/play-runtime/provider-capacity/fixture-capacity-simulator.ts +220 -0
- package/dist/bundling-sources/shared_libs/play-runtime/provider-capacity/gradient2-concurrency-controller.ts +214 -0
- package/dist/bundling-sources/shared_libs/play-runtime/provider-capacity/provider-capacity-policy.ts +43 -0
- package/dist/bundling-sources/shared_libs/play-runtime/provider-capacity/provider-concurrency-policy.ts +30 -0
- package/dist/bundling-sources/shared_libs/play-runtime/provider-capacity/provider-rate-policy.ts +95 -0
- package/dist/bundling-sources/shared_libs/play-runtime/provider-capacity/provider-throttle-classifier.ts +40 -0
- package/dist/bundling-sources/shared_libs/play-runtime/resource-governor.ts +1535 -11
- package/dist/bundling-sources/shared_libs/play-runtime/run-failure.ts +49 -2
- package/dist/bundling-sources/shared_libs/play-runtime/run-ledger.ts +6 -1
- package/dist/bundling-sources/shared_libs/play-runtime/run-log-buffer.ts +55 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-app/egress-probe.ts +221 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-app/index.ts +339 -29
- package/dist/bundling-sources/shared_libs/play-runtime/runner-app/runtime-deadline.ts +28 -1
- package/dist/bundling-sources/shared_libs/play-runtime/runner-app/runtime-progress-push.ts +192 -165
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-lifecycle.ts +6 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-payload-transport.ts +90 -5
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-runtime-watchdog.ts +281 -6
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts +27 -1
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/local-process.ts +19 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/modal.ts +108 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/runtime-sandbox-reconciliation.ts +40 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/types.ts +11 -5
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-actions.ts +30 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-api-compression.ts +276 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +723 -115
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-capacity-policy.ts +98 -2
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-contract.ts +3 -2
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-environment.ts +41 -4
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-flow.ts +448 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-map-row-feed.ts +215 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-operation-contract.ts +11 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-receipt-store-adapter.ts +161 -14
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-receipt-value.ts +184 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-receipt-writer.ts +607 -121
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-sandbox-placement-policy.ts +4 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-sheet-row-writer.ts +21 -0
- package/dist/bundling-sources/shared_libs/play-runtime/sandbox-runs/contract.ts +15 -2
- package/dist/bundling-sources/shared_libs/play-runtime/sandbox-runs/runner-backend-adapter.ts +71 -14
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-play-deploy-task.ts +2 -2
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-runtime-worker.ts +288 -119
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-shared.ts +8 -6
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-worker-required-env.ts +28 -3
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/canonical-integration-operation-store.ts +179 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/gateway-heavy-request-admission.ts +80 -2
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/gateway-progress-registry.ts +31 -2
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/integration-operation-store-contract.ts +104 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/integration-operation-value-reference.ts +47 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/integration-operation-value-retirement.ts +193 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/integration-operation-value-storage.ts +420 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-pool.ts +41 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-progress.ts +134 -3
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-rate-state.ts +1657 -95
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts +885 -73
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/receipt-completion-coalescer.ts +452 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/receipt-gateway.ts +601 -42
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/run-terminal-transition.ts +1 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-receipt-value-storage.ts +371 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/worker-resource-metrics.ts +86 -7
- package/dist/bundling-sources/shared_libs/play-runtime/secret-redaction.ts +153 -22
- package/dist/bundling-sources/shared_libs/play-runtime/tool-call/index.ts +1 -0
- package/dist/bundling-sources/shared_libs/play-runtime/tool-call/receipt-cohort.ts +188 -38
- package/dist/bundling-sources/shared_libs/play-runtime/tool-call/runtime-step-receipts-adapter.ts +28 -13
- package/dist/bundling-sources/shared_libs/play-runtime/tool-execute-retry-policy.ts +44 -1
- package/dist/bundling-sources/shared_libs/play-runtime/tool-result.ts +56 -4
- package/dist/bundling-sources/shared_libs/play-runtime/work-receipts.ts +2 -0
- package/dist/bundling-sources/shared_libs/plays/authoring-contract.ts +16 -10
- package/dist/bundling-sources/shared_libs/plays/dataset.ts +4 -1
- package/dist/cli/index.js +250 -38
- package/dist/cli/index.mjs +250 -38
- package/dist/{compiler-manifest-DfSnbCTW.d.mts → compiler-manifest-BIqRyj5m.d.mts} +1 -1
- package/dist/{compiler-manifest-DfSnbCTW.d.ts → compiler-manifest-BIqRyj5m.d.ts} +1 -1
- package/dist/index.d.mts +4 -2
- package/dist/index.d.ts +4 -2
- package/dist/index.js +231 -35
- package/dist/index.mjs +231 -35
- package/dist/install-integrity.json +31 -2
- package/dist/plays/bundle-play-file.d.mts +2 -2
- package/dist/plays/bundle-play-file.d.ts +2 -2
- package/dist/plays/bundle-play-file.mjs +65 -1
- package/dist/release.d.mts +1 -1
- package/dist/release.d.ts +1 -1
- package/dist/release.js +1 -1
- package/dist/release.mjs +1 -1
- package/package.json +1 -1
|
@@ -35,9 +35,47 @@ export type PlayRunnerRateStateAcquireInput = {
|
|
|
35
35
|
requested: number;
|
|
36
36
|
/** Confirmed provider successes since this runner's previous reservation. */
|
|
37
37
|
observedSuccesses?: number;
|
|
38
|
+
/**
|
|
39
|
+
* Completed physical provider calls buffered by this runner since its prior
|
|
40
|
+
* reservation. The gateway folds these into the durable Gradient2 checkpoint
|
|
41
|
+
* before it issues the next provider permit.
|
|
42
|
+
*/
|
|
43
|
+
concurrencyObservations?: PlayRunnerRateStateConcurrencyObservation[];
|
|
44
|
+
/**
|
|
45
|
+
* Stable identity for the buffered observation set. A retried acquire must
|
|
46
|
+
* carry the same identity so the durable controller folds it at most once.
|
|
47
|
+
*/
|
|
48
|
+
concurrencyObservationBatch?: PlayRunnerRateStateConcurrencyObservationBatch;
|
|
38
49
|
schedulerSchema?: string | null;
|
|
39
50
|
};
|
|
40
51
|
|
|
52
|
+
/** A bounded, provider-attempt outcome used by the shared concurrency learner. */
|
|
53
|
+
export type PlayRunnerRateStateConcurrencyObservation = {
|
|
54
|
+
kind: 'success' | 'throttled';
|
|
55
|
+
latencyMs: number;
|
|
56
|
+
};
|
|
57
|
+
|
|
58
|
+
export type PlayRunnerRateStateConcurrencyObservationBatch = {
|
|
59
|
+
/** Random per-runner identity; never a customer or credential identifier. */
|
|
60
|
+
observerId: string;
|
|
61
|
+
/** Separates independently refilled rule lanes in the same runner. */
|
|
62
|
+
laneId: string;
|
|
63
|
+
/** Monotonic within that runner lane. */
|
|
64
|
+
sequence: number;
|
|
65
|
+
};
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* Admission-time facts needed to attribute a later provider 429. The gateway
|
|
69
|
+
* owns this snapshot because only it sees global concurrent leases.
|
|
70
|
+
*/
|
|
71
|
+
export type PlayRunnerRateStateConcurrencyGrant = {
|
|
72
|
+
/** Server-minted permit identity; deduplicates its later 429 report. */
|
|
73
|
+
admissionId: string;
|
|
74
|
+
inFlightAtStart: number;
|
|
75
|
+
targetRequestsPerSecond: number | null;
|
|
76
|
+
minimumRttMs: number | null;
|
|
77
|
+
};
|
|
78
|
+
|
|
41
79
|
export type PlayRunnerRateStateAcquireResult = {
|
|
42
80
|
granted: number;
|
|
43
81
|
waitMs: number;
|
|
@@ -56,6 +94,13 @@ export type PlayRunnerRateStateAcquireResult = {
|
|
|
56
94
|
effectiveRequestsPerSecond?: number;
|
|
57
95
|
/** One independently-expiring concurrency reservation per granted permit. */
|
|
58
96
|
leaseIds?: string[];
|
|
97
|
+
/** One concurrency-attribution snapshot per granted permit. */
|
|
98
|
+
concurrencyGrants?: PlayRunnerRateStateConcurrencyGrant[];
|
|
99
|
+
/**
|
|
100
|
+
* One idempotency key per granted rate permit. A runner sends one back only
|
|
101
|
+
* when that ticket was never dispatched and must be returned to the bucket.
|
|
102
|
+
*/
|
|
103
|
+
reservationIds?: string[];
|
|
59
104
|
/**
|
|
60
105
|
* Absolute epoch-ms until which the pacer's own cooldown holds (internally
|
|
61
106
|
* capped, e.g. 45min probe cadence). Advisory: exposes the current backoff to
|
|
@@ -74,6 +119,8 @@ export type PlayRunnerRateStateReleaseInput = {
|
|
|
74
119
|
rateScopeToken?: string | null;
|
|
75
120
|
rules: PacingRule[];
|
|
76
121
|
leaseIds: string[];
|
|
122
|
+
/** Idempotently refund rate reservations that never reached provider fetch. */
|
|
123
|
+
refundReservationIds?: string[];
|
|
77
124
|
schedulerSchema?: string | null;
|
|
78
125
|
};
|
|
79
126
|
|
|
@@ -81,6 +128,15 @@ export type PlayRunnerRateStatePenalizeInput = {
|
|
|
81
128
|
bucketId: string;
|
|
82
129
|
rateScopeToken?: string | null;
|
|
83
130
|
cooldownMs: number;
|
|
131
|
+
/**
|
|
132
|
+
* The same resolved rules that admitted the attempt. Optional during the
|
|
133
|
+
* rolling deploy window: old runners retain rate-only backpressure handling.
|
|
134
|
+
*/
|
|
135
|
+
rules?: PacingRule[];
|
|
136
|
+
/** Actual provider residence for this 429, when the response was observed. */
|
|
137
|
+
latencyMs?: number | null;
|
|
138
|
+
/** The server-authored permit snapshot for the rejected physical call. */
|
|
139
|
+
concurrencyGrant?: PlayRunnerRateStateConcurrencyGrant | null;
|
|
84
140
|
schedulerSchema?: string | null;
|
|
85
141
|
};
|
|
86
142
|
|
|
@@ -0,0 +1,192 @@
|
|
|
1
|
+
import {
|
|
2
|
+
EpochAimdRateController,
|
|
3
|
+
type EpochAimdRateControllerOptions,
|
|
4
|
+
type EpochAimdRateControllerSnapshot,
|
|
5
|
+
} from './epoch-aimd-rate-controller';
|
|
6
|
+
import {
|
|
7
|
+
Gradient2ConcurrencyController,
|
|
8
|
+
type Gradient2ConcurrencyControllerOptions,
|
|
9
|
+
type Gradient2ConcurrencyControllerSnapshot,
|
|
10
|
+
} from './gradient2-concurrency-controller';
|
|
11
|
+
import { classifyProviderThrottle } from './provider-throttle-classifier';
|
|
12
|
+
|
|
13
|
+
export type AdaptiveProviderCapacityControllerOptions = Readonly<{
|
|
14
|
+
rate: EpochAimdRateControllerOptions;
|
|
15
|
+
concurrency: Gradient2ConcurrencyControllerOptions;
|
|
16
|
+
}>;
|
|
17
|
+
|
|
18
|
+
export type ProviderCapacityLease = Readonly<{
|
|
19
|
+
id: number;
|
|
20
|
+
startedAtMs: number;
|
|
21
|
+
inFlightAtStart: number;
|
|
22
|
+
}>;
|
|
23
|
+
|
|
24
|
+
export type ProviderCapacityAcquisition =
|
|
25
|
+
| Readonly<{ kind: 'acquired'; lease: ProviderCapacityLease }>
|
|
26
|
+
| Readonly<{ kind: 'waiting'; retryAtMs: number }>;
|
|
27
|
+
|
|
28
|
+
export type ProviderCapacityCompletion = Readonly<{
|
|
29
|
+
kind: 'success' | 'throttled' | 'transport_error';
|
|
30
|
+
latencyMs: number;
|
|
31
|
+
completedAtMs: number;
|
|
32
|
+
retryAfterMs?: number;
|
|
33
|
+
}>;
|
|
34
|
+
|
|
35
|
+
export type AdaptiveProviderCapacityControllerSnapshot = Readonly<{
|
|
36
|
+
active: number;
|
|
37
|
+
nextAdmissionAtMs: number;
|
|
38
|
+
rate: EpochAimdRateControllerSnapshot;
|
|
39
|
+
concurrency: Gradient2ConcurrencyControllerSnapshot;
|
|
40
|
+
}>;
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* One admission point over independent rate and concurrency estimators.
|
|
44
|
+
*
|
|
45
|
+
* Callers share this instance for one provider credential scope. Controllers
|
|
46
|
+
* only supply budgets; this facade is the sole module allowed to issue a
|
|
47
|
+
* physical-call lease.
|
|
48
|
+
*/
|
|
49
|
+
export class AdaptiveProviderCapacityController {
|
|
50
|
+
readonly #rate: EpochAimdRateController;
|
|
51
|
+
readonly #concurrency: Gradient2ConcurrencyController;
|
|
52
|
+
readonly #activeLeases = new Map<number, ProviderCapacityLease>();
|
|
53
|
+
#nextLeaseId = 1;
|
|
54
|
+
#nextRateAdmissionAtMs = 0;
|
|
55
|
+
#nextConcurrencyCheckAtMs = 0;
|
|
56
|
+
#lastObservedAtMs = 0;
|
|
57
|
+
#observedConcurrencyThrottleAt: number | null = null;
|
|
58
|
+
|
|
59
|
+
constructor(options: AdaptiveProviderCapacityControllerOptions) {
|
|
60
|
+
this.#rate = new EpochAimdRateController(options.rate);
|
|
61
|
+
this.#concurrency = new Gradient2ConcurrencyController(options.concurrency);
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
tryAcquire(nowMs: number): ProviderCapacityAcquisition {
|
|
65
|
+
this.#advance(nowMs);
|
|
66
|
+
const rate = this.#rate.snapshot();
|
|
67
|
+
const concurrency = this.#concurrency.snapshot();
|
|
68
|
+
const retryAtMs = Math.max(
|
|
69
|
+
this.#nextRateAdmissionAtMs,
|
|
70
|
+
this.#nextConcurrencyCheckAtMs,
|
|
71
|
+
rate.cooldownUntilMs,
|
|
72
|
+
);
|
|
73
|
+
if (nowMs < retryAtMs) {
|
|
74
|
+
return Object.freeze({ kind: 'waiting', retryAtMs });
|
|
75
|
+
}
|
|
76
|
+
if (this.#activeLeases.size >= Math.max(1, Math.floor(concurrency.limit))) {
|
|
77
|
+
this.#nextConcurrencyCheckAtMs =
|
|
78
|
+
nowMs + Math.max(1, Math.floor((concurrency.shortRttMs ?? 100) / 4));
|
|
79
|
+
return Object.freeze({
|
|
80
|
+
kind: 'waiting',
|
|
81
|
+
retryAtMs: this.#nextConcurrencyCheckAtMs,
|
|
82
|
+
});
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
const lease = Object.freeze({
|
|
86
|
+
id: this.#nextLeaseId,
|
|
87
|
+
startedAtMs: nowMs,
|
|
88
|
+
inFlightAtStart: this.#activeLeases.size + 1,
|
|
89
|
+
});
|
|
90
|
+
this.#nextLeaseId += 1;
|
|
91
|
+
this.#activeLeases.set(lease.id, lease);
|
|
92
|
+
this.#rate.observeStart(nowMs);
|
|
93
|
+
this.#nextRateAdmissionAtMs =
|
|
94
|
+
nowMs + 1_000 / this.#rate.snapshot().targetRequestsPerSecond;
|
|
95
|
+
this.#nextConcurrencyCheckAtMs = nowMs;
|
|
96
|
+
return Object.freeze({ kind: 'acquired', lease });
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
complete(
|
|
100
|
+
lease: ProviderCapacityLease,
|
|
101
|
+
completion: ProviderCapacityCompletion,
|
|
102
|
+
): void {
|
|
103
|
+
this.#advance(completion.completedAtMs);
|
|
104
|
+
if (!this.#activeLeases.delete(lease.id)) {
|
|
105
|
+
throw new Error(
|
|
106
|
+
'Provider-capacity lease is unknown or already complete.',
|
|
107
|
+
);
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
if (completion.kind === 'success') {
|
|
111
|
+
this.#rate.observeSuccess(completion.completedAtMs);
|
|
112
|
+
this.#concurrency.observe({
|
|
113
|
+
kind: 'success',
|
|
114
|
+
latencyMs: completion.latencyMs,
|
|
115
|
+
inFlightAtStart: lease.inFlightAtStart,
|
|
116
|
+
});
|
|
117
|
+
} else if (completion.kind === 'transport_error') {
|
|
118
|
+
// Transport failure is not evidence of a provider's rate or concurrent
|
|
119
|
+
// occupancy ceiling. The runner-local HTTP window may still apply a
|
|
120
|
+
// short-lived transport-congestion response, but the shared provider
|
|
121
|
+
// controller must match Postgres grant learning, which deliberately
|
|
122
|
+
// drops transport observations.
|
|
123
|
+
} else {
|
|
124
|
+
const pressure =
|
|
125
|
+
completion.kind === 'throttled'
|
|
126
|
+
? this.#classifyThrottle(lease)
|
|
127
|
+
: 'concurrency';
|
|
128
|
+
if (pressure === 'rate' || pressure === 'ambiguous') {
|
|
129
|
+
this.#rate.observeThrottle(
|
|
130
|
+
completion.completedAtMs,
|
|
131
|
+
completion.retryAfterMs ?? 0,
|
|
132
|
+
);
|
|
133
|
+
}
|
|
134
|
+
if (pressure === 'concurrency' || pressure === 'ambiguous') {
|
|
135
|
+
this.#observedConcurrencyThrottleAt =
|
|
136
|
+
this.#observedConcurrencyThrottleAt === null
|
|
137
|
+
? lease.inFlightAtStart
|
|
138
|
+
: Math.min(
|
|
139
|
+
this.#observedConcurrencyThrottleAt,
|
|
140
|
+
lease.inFlightAtStart,
|
|
141
|
+
);
|
|
142
|
+
this.#concurrency.observe({
|
|
143
|
+
kind: completion.kind,
|
|
144
|
+
latencyMs: completion.latencyMs,
|
|
145
|
+
inFlightAtStart: lease.inFlightAtStart,
|
|
146
|
+
});
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
this.#nextConcurrencyCheckAtMs = completion.completedAtMs;
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
snapshot(): AdaptiveProviderCapacityControllerSnapshot {
|
|
153
|
+
return Object.freeze({
|
|
154
|
+
active: this.#activeLeases.size,
|
|
155
|
+
nextAdmissionAtMs: Math.max(
|
|
156
|
+
this.#nextRateAdmissionAtMs,
|
|
157
|
+
this.#nextConcurrencyCheckAtMs,
|
|
158
|
+
this.#rate.snapshot().cooldownUntilMs,
|
|
159
|
+
),
|
|
160
|
+
rate: this.#rate.snapshot(),
|
|
161
|
+
concurrency: this.#concurrency.snapshot(),
|
|
162
|
+
});
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
#advance(nowMs: number): void {
|
|
166
|
+
if (!Number.isFinite(nowMs) || nowMs < this.#lastObservedAtMs) {
|
|
167
|
+
throw new Error('Provider-capacity time must be finite and monotonic.');
|
|
168
|
+
}
|
|
169
|
+
this.#lastObservedAtMs = nowMs;
|
|
170
|
+
this.#rate.advanceTo(nowMs);
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
#classifyThrottle(
|
|
174
|
+
lease: ProviderCapacityLease,
|
|
175
|
+
): 'rate' | 'concurrency' | 'ambiguous' {
|
|
176
|
+
if (
|
|
177
|
+
this.#observedConcurrencyThrottleAt !== null &&
|
|
178
|
+
lease.inFlightAtStart >= this.#observedConcurrencyThrottleAt
|
|
179
|
+
) {
|
|
180
|
+
return 'concurrency';
|
|
181
|
+
}
|
|
182
|
+
const rate = this.#rate.snapshot().targetRequestsPerSecond;
|
|
183
|
+
const minimumRttMs = this.#concurrency.snapshot().minimumRttMs;
|
|
184
|
+
return classifyProviderThrottle({
|
|
185
|
+
grant: {
|
|
186
|
+
targetRequestsPerSecond: rate,
|
|
187
|
+
minimumRttMs,
|
|
188
|
+
},
|
|
189
|
+
inFlightAtStart: lease.inFlightAtStart,
|
|
190
|
+
});
|
|
191
|
+
}
|
|
192
|
+
}
|
|
@@ -0,0 +1,194 @@
|
|
|
1
|
+
export type EpochAimdRateControllerOptions = Readonly<{
|
|
2
|
+
initialRequestsPerSecond: number;
|
|
3
|
+
minimumRequestsPerSecond: number;
|
|
4
|
+
maximumRequestsPerSecond: number;
|
|
5
|
+
epochMs: number;
|
|
6
|
+
additiveIncrease: number;
|
|
7
|
+
decreaseFactor: number;
|
|
8
|
+
utilizationThreshold: number;
|
|
9
|
+
/** Bounded exponential cold-start probe; one or below preserves pure AIMD. */
|
|
10
|
+
startupProbeMultiplier?: number;
|
|
11
|
+
}>;
|
|
12
|
+
|
|
13
|
+
export type EpochAimdRateControllerSnapshot = Readonly<{
|
|
14
|
+
targetRequestsPerSecond: number;
|
|
15
|
+
cooldownUntilMs: number;
|
|
16
|
+
epochStartedAtMs: number;
|
|
17
|
+
startsInEpoch: number;
|
|
18
|
+
successesInEpoch: number;
|
|
19
|
+
throttleEpochs: number;
|
|
20
|
+
}>;
|
|
21
|
+
|
|
22
|
+
function positive(value: number, label: string): number {
|
|
23
|
+
if (!Number.isFinite(value) || value <= 0) {
|
|
24
|
+
throw new Error(`${label} must be finite and greater than zero.`);
|
|
25
|
+
}
|
|
26
|
+
return value;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
function fraction(value: number, label: string): number {
|
|
30
|
+
if (!Number.isFinite(value) || value <= 0 || value > 1) {
|
|
31
|
+
throw new Error(`${label} must be greater than zero and at most one.`);
|
|
32
|
+
}
|
|
33
|
+
return value;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* Epoch-based additive-increase/multiplicative-decrease rate discovery.
|
|
38
|
+
*
|
|
39
|
+
* Successful latency never changes this controller. An epoch grows only when
|
|
40
|
+
* callers used most of the offered starts and those starts completed without
|
|
41
|
+
* provider throttling. A synchronized 429 burst decreases at most once per
|
|
42
|
+
* epoch.
|
|
43
|
+
*/
|
|
44
|
+
export class EpochAimdRateController {
|
|
45
|
+
readonly #minimumRequestsPerSecond: number;
|
|
46
|
+
readonly #maximumRequestsPerSecond: number;
|
|
47
|
+
readonly #epochMs: number;
|
|
48
|
+
readonly #additiveIncrease: number;
|
|
49
|
+
readonly #decreaseFactor: number;
|
|
50
|
+
readonly #utilizationThreshold: number;
|
|
51
|
+
readonly #startupProbeMultiplier: number;
|
|
52
|
+
#targetRequestsPerSecond: number;
|
|
53
|
+
#epochStartedAtMs = 0;
|
|
54
|
+
#initialized = false;
|
|
55
|
+
#startsInEpoch = 0;
|
|
56
|
+
#successesInEpoch = 0;
|
|
57
|
+
#throttledInEpoch = false;
|
|
58
|
+
#cooldownUntilMs = 0;
|
|
59
|
+
#throttleEpochs = 0;
|
|
60
|
+
#hasObservedThrottle = false;
|
|
61
|
+
|
|
62
|
+
constructor(options: EpochAimdRateControllerOptions) {
|
|
63
|
+
this.#minimumRequestsPerSecond = positive(
|
|
64
|
+
options.minimumRequestsPerSecond,
|
|
65
|
+
'Minimum requests per second',
|
|
66
|
+
);
|
|
67
|
+
this.#maximumRequestsPerSecond = positive(
|
|
68
|
+
options.maximumRequestsPerSecond,
|
|
69
|
+
'Maximum requests per second',
|
|
70
|
+
);
|
|
71
|
+
if (this.#maximumRequestsPerSecond < this.#minimumRequestsPerSecond) {
|
|
72
|
+
throw new Error(
|
|
73
|
+
'Maximum requests per second cannot be below the minimum.',
|
|
74
|
+
);
|
|
75
|
+
}
|
|
76
|
+
this.#targetRequestsPerSecond = Math.min(
|
|
77
|
+
this.#maximumRequestsPerSecond,
|
|
78
|
+
Math.max(
|
|
79
|
+
this.#minimumRequestsPerSecond,
|
|
80
|
+
positive(
|
|
81
|
+
options.initialRequestsPerSecond,
|
|
82
|
+
'Initial requests per second',
|
|
83
|
+
),
|
|
84
|
+
),
|
|
85
|
+
);
|
|
86
|
+
this.#epochMs = positive(options.epochMs, 'Rate epoch');
|
|
87
|
+
this.#additiveIncrease = positive(
|
|
88
|
+
options.additiveIncrease,
|
|
89
|
+
'Additive increase',
|
|
90
|
+
);
|
|
91
|
+
this.#decreaseFactor = fraction(options.decreaseFactor, 'Decrease factor');
|
|
92
|
+
this.#utilizationThreshold = fraction(
|
|
93
|
+
options.utilizationThreshold,
|
|
94
|
+
'Utilization threshold',
|
|
95
|
+
);
|
|
96
|
+
this.#startupProbeMultiplier = Math.max(
|
|
97
|
+
1,
|
|
98
|
+
positive(options.startupProbeMultiplier ?? 1, 'Startup probe multiplier'),
|
|
99
|
+
);
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
observeStart(startedAtMs: number): void {
|
|
103
|
+
this.advanceTo(startedAtMs);
|
|
104
|
+
this.#startsInEpoch += 1;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
observeSuccess(completedAtMs: number): void {
|
|
108
|
+
this.advanceTo(completedAtMs);
|
|
109
|
+
this.#successesInEpoch += 1;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
observeThrottle(observedAtMs: number, retryAfterMs: number): void {
|
|
113
|
+
this.advanceTo(observedAtMs);
|
|
114
|
+
// A 429 is the only authority that ends the initial probe. From this
|
|
115
|
+
// point forward the lane retains #3943's conservative AIMD recovery.
|
|
116
|
+
this.#hasObservedThrottle = true;
|
|
117
|
+
this.#cooldownUntilMs = Math.max(
|
|
118
|
+
this.#cooldownUntilMs,
|
|
119
|
+
observedAtMs + Math.max(0, retryAfterMs),
|
|
120
|
+
);
|
|
121
|
+
if (this.#throttledInEpoch) return;
|
|
122
|
+
|
|
123
|
+
this.#throttledInEpoch = true;
|
|
124
|
+
this.#throttleEpochs += 1;
|
|
125
|
+
this.#targetRequestsPerSecond = transitionProviderRateTarget({
|
|
126
|
+
currentRequestsPerSecond: this.#targetRequestsPerSecond,
|
|
127
|
+
minimumRequestsPerSecond: this.#minimumRequestsPerSecond,
|
|
128
|
+
maximumRequestsPerSecond: this.#maximumRequestsPerSecond,
|
|
129
|
+
event: 'rate_limited',
|
|
130
|
+
additiveIncrease: this.#additiveIncrease,
|
|
131
|
+
decreaseFactor: this.#decreaseFactor,
|
|
132
|
+
});
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
advanceTo(nowMs: number): void {
|
|
136
|
+
if (!Number.isFinite(nowMs)) {
|
|
137
|
+
throw new Error('Rate-controller time must be finite and monotonic.');
|
|
138
|
+
}
|
|
139
|
+
if (!this.#initialized) {
|
|
140
|
+
this.#epochStartedAtMs = nowMs;
|
|
141
|
+
this.#initialized = true;
|
|
142
|
+
return;
|
|
143
|
+
}
|
|
144
|
+
if (nowMs < this.#epochStartedAtMs) {
|
|
145
|
+
throw new Error('Rate-controller time must be finite and monotonic.');
|
|
146
|
+
}
|
|
147
|
+
while (nowMs >= this.#epochStartedAtMs + this.#epochMs) {
|
|
148
|
+
this.#closeEpoch();
|
|
149
|
+
this.#epochStartedAtMs += this.#epochMs;
|
|
150
|
+
this.#startsInEpoch = 0;
|
|
151
|
+
this.#successesInEpoch = 0;
|
|
152
|
+
this.#throttledInEpoch = false;
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
snapshot(): EpochAimdRateControllerSnapshot {
|
|
157
|
+
return Object.freeze({
|
|
158
|
+
targetRequestsPerSecond: this.#targetRequestsPerSecond,
|
|
159
|
+
cooldownUntilMs: this.#cooldownUntilMs,
|
|
160
|
+
epochStartedAtMs: this.#epochStartedAtMs,
|
|
161
|
+
startsInEpoch: this.#startsInEpoch,
|
|
162
|
+
successesInEpoch: this.#successesInEpoch,
|
|
163
|
+
throttleEpochs: this.#throttleEpochs,
|
|
164
|
+
});
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
#closeEpoch(): void {
|
|
168
|
+
const offeredStarts =
|
|
169
|
+
(this.#targetRequestsPerSecond * this.#epochMs) / 1_000;
|
|
170
|
+
const utilized =
|
|
171
|
+
this.#startsInEpoch >= offeredStarts * this.#utilizationThreshold;
|
|
172
|
+
const healthy =
|
|
173
|
+
!this.#throttledInEpoch &&
|
|
174
|
+
this.#successesInEpoch >=
|
|
175
|
+
this.#startsInEpoch * this.#utilizationThreshold;
|
|
176
|
+
if (!utilized || !healthy) return;
|
|
177
|
+
|
|
178
|
+
this.#targetRequestsPerSecond =
|
|
179
|
+
!this.#hasObservedThrottle && this.#startupProbeMultiplier > 1
|
|
180
|
+
? Math.min(
|
|
181
|
+
this.#maximumRequestsPerSecond,
|
|
182
|
+
this.#targetRequestsPerSecond * this.#startupProbeMultiplier,
|
|
183
|
+
)
|
|
184
|
+
: transitionProviderRateTarget({
|
|
185
|
+
currentRequestsPerSecond: this.#targetRequestsPerSecond,
|
|
186
|
+
minimumRequestsPerSecond: this.#minimumRequestsPerSecond,
|
|
187
|
+
maximumRequestsPerSecond: this.#maximumRequestsPerSecond,
|
|
188
|
+
event: 'healthy_epoch',
|
|
189
|
+
additiveIncrease: this.#additiveIncrease,
|
|
190
|
+
decreaseFactor: this.#decreaseFactor,
|
|
191
|
+
});
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
import { transitionProviderRateTarget } from './provider-rate-policy';
|
|
@@ -0,0 +1,220 @@
|
|
|
1
|
+
import { AdaptiveProviderCapacityController } from './adaptive-provider-capacity-controller';
|
|
2
|
+
import { providerConcurrencyControllerOptions } from './provider-concurrency-policy';
|
|
3
|
+
import { providerRateControllerOptions } from './provider-rate-policy';
|
|
4
|
+
|
|
5
|
+
export type FixtureCapacityScenario = Readonly<{
|
|
6
|
+
operationCount: number;
|
|
7
|
+
providerRps: number;
|
|
8
|
+
providerConcurrency: number;
|
|
9
|
+
serviceTimeMs: number;
|
|
10
|
+
initialRps?: number;
|
|
11
|
+
maximumExplorationRps?: number;
|
|
12
|
+
initialConcurrency?: number;
|
|
13
|
+
maximumExplorationConcurrency?: number;
|
|
14
|
+
retryAfterMs?: number;
|
|
15
|
+
analysisWindowMs?: number;
|
|
16
|
+
}>;
|
|
17
|
+
|
|
18
|
+
export type FixtureCapacitySimulationResult = Readonly<{
|
|
19
|
+
completedOperations: number;
|
|
20
|
+
providerStarts: number;
|
|
21
|
+
provider429s: number;
|
|
22
|
+
elapsedMs: number;
|
|
23
|
+
physicalCeilingPerSecond: number;
|
|
24
|
+
steadyGoodputPerSecond: number;
|
|
25
|
+
steadyStatePhysicsUtilization: number;
|
|
26
|
+
finalTargetRps: number;
|
|
27
|
+
finalConcurrency: number;
|
|
28
|
+
maximumProviderInFlight: number;
|
|
29
|
+
}>;
|
|
30
|
+
|
|
31
|
+
type Completion = Readonly<{
|
|
32
|
+
atMs: number;
|
|
33
|
+
lease: Parameters<AdaptiveProviderCapacityController['complete']>[0];
|
|
34
|
+
kind: 'success' | 'throttled';
|
|
35
|
+
}>;
|
|
36
|
+
|
|
37
|
+
function positive(value: number, label: string): number {
|
|
38
|
+
if (!Number.isFinite(value) || value <= 0) {
|
|
39
|
+
throw new Error(`${label} must be finite and greater than zero.`);
|
|
40
|
+
}
|
|
41
|
+
return value;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
function positiveInteger(value: number, label: string): number {
|
|
45
|
+
if (!Number.isSafeInteger(value) || value <= 0) {
|
|
46
|
+
throw new Error(`${label} must be a positive safe integer.`);
|
|
47
|
+
}
|
|
48
|
+
return value;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* Deterministic fixture-only provider world.
|
|
53
|
+
*
|
|
54
|
+
* The controller receives only its conservative seed and exploration
|
|
55
|
+
* guardrails. Actual provider RPS/concurrency stay inside this simulator and
|
|
56
|
+
* reveal themselves solely as a real provider would: an unambiguous 429 with
|
|
57
|
+
* Retry-After. This makes it possible to falsify rate/concurrency discovery
|
|
58
|
+
* without a paid provider or a second scheduler implementation.
|
|
59
|
+
*/
|
|
60
|
+
export function simulateFixtureCapacityDiscovery(
|
|
61
|
+
input: FixtureCapacityScenario,
|
|
62
|
+
): FixtureCapacitySimulationResult {
|
|
63
|
+
const operationCount = positiveInteger(
|
|
64
|
+
input.operationCount,
|
|
65
|
+
'Operation count',
|
|
66
|
+
);
|
|
67
|
+
const providerRps = positive(input.providerRps, 'Provider RPS');
|
|
68
|
+
const providerConcurrency = positiveInteger(
|
|
69
|
+
input.providerConcurrency,
|
|
70
|
+
'Provider concurrency',
|
|
71
|
+
);
|
|
72
|
+
const serviceTimeMs = positive(input.serviceTimeMs, 'Provider service time');
|
|
73
|
+
const initialRps = positive(input.initialRps ?? 10, 'Initial RPS');
|
|
74
|
+
const maximumExplorationRps = positive(
|
|
75
|
+
input.maximumExplorationRps ?? 256,
|
|
76
|
+
'Maximum exploration RPS',
|
|
77
|
+
);
|
|
78
|
+
const initialConcurrency = positiveInteger(
|
|
79
|
+
input.initialConcurrency ?? 1,
|
|
80
|
+
'Initial concurrency',
|
|
81
|
+
);
|
|
82
|
+
const maximumExplorationConcurrency = positiveInteger(
|
|
83
|
+
input.maximumExplorationConcurrency ?? 256,
|
|
84
|
+
'Maximum exploration concurrency',
|
|
85
|
+
);
|
|
86
|
+
const retryAfterMs = positiveInteger(
|
|
87
|
+
input.retryAfterMs ?? 100,
|
|
88
|
+
'Retry-After',
|
|
89
|
+
);
|
|
90
|
+
const analysisWindowMs = positiveInteger(
|
|
91
|
+
input.analysisWindowMs ?? 30_000,
|
|
92
|
+
'Analysis window',
|
|
93
|
+
);
|
|
94
|
+
const controller = new AdaptiveProviderCapacityController({
|
|
95
|
+
rate: providerRateControllerOptions({
|
|
96
|
+
declaredRequestsPerSecond: initialRps,
|
|
97
|
+
maximumRequestsPerSecond: maximumExplorationRps,
|
|
98
|
+
}),
|
|
99
|
+
concurrency: providerConcurrencyControllerOptions({
|
|
100
|
+
initialLimit: initialConcurrency,
|
|
101
|
+
maximumLimit: maximumExplorationConcurrency,
|
|
102
|
+
}),
|
|
103
|
+
});
|
|
104
|
+
const pending: number[] = Array.from(
|
|
105
|
+
{ length: operationCount },
|
|
106
|
+
(_, index) => index,
|
|
107
|
+
);
|
|
108
|
+
const completions: Completion[] = [];
|
|
109
|
+
const providerStartsAtMs: number[] = [];
|
|
110
|
+
const completedAtMs: number[] = [];
|
|
111
|
+
let nowMs = 0;
|
|
112
|
+
let providerStarts = 0;
|
|
113
|
+
let provider429s = 0;
|
|
114
|
+
let maximumProviderInFlight = 0;
|
|
115
|
+
let attempts = 0;
|
|
116
|
+
|
|
117
|
+
const advanceCompletions = () => {
|
|
118
|
+
completions.sort((left, right) => left.atMs - right.atMs);
|
|
119
|
+
while (completions[0]?.atMs === nowMs) {
|
|
120
|
+
const completion = completions.shift()!;
|
|
121
|
+
controller.complete(completion.lease, {
|
|
122
|
+
kind: completion.kind,
|
|
123
|
+
latencyMs: completion.kind === 'success' ? serviceTimeMs : 1,
|
|
124
|
+
completedAtMs: nowMs,
|
|
125
|
+
...(completion.kind === 'throttled' ? { retryAfterMs } : {}),
|
|
126
|
+
});
|
|
127
|
+
if (completion.kind === 'success') completedAtMs.push(nowMs);
|
|
128
|
+
}
|
|
129
|
+
};
|
|
130
|
+
|
|
131
|
+
while (completedAtMs.length < operationCount) {
|
|
132
|
+
advanceCompletions();
|
|
133
|
+
let nextWakeAtMs = Number.POSITIVE_INFINITY;
|
|
134
|
+
while (pending.length > 0) {
|
|
135
|
+
attempts += 1;
|
|
136
|
+
if (attempts > operationCount * 10_000) {
|
|
137
|
+
throw new Error('Fixture capacity discovery did not converge.');
|
|
138
|
+
}
|
|
139
|
+
const admission = controller.tryAcquire(nowMs);
|
|
140
|
+
if (admission.kind === 'waiting') {
|
|
141
|
+
nextWakeAtMs = Math.min(nextWakeAtMs, admission.retryAtMs);
|
|
142
|
+
break;
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
const recentStarts = providerStartsAtMs.filter(
|
|
146
|
+
(startedAtMs) => startedAtMs > nowMs - 1_000,
|
|
147
|
+
);
|
|
148
|
+
providerStartsAtMs.splice(0, providerStartsAtMs.length, ...recentStarts);
|
|
149
|
+
const rateLimited = recentStarts.length >= providerRps;
|
|
150
|
+
const concurrencyLimited = completions.length >= providerConcurrency;
|
|
151
|
+
if (rateLimited || concurrencyLimited) {
|
|
152
|
+
provider429s += 1;
|
|
153
|
+
completions.push({
|
|
154
|
+
atMs: nowMs + 1,
|
|
155
|
+
lease: admission.lease,
|
|
156
|
+
kind: 'throttled',
|
|
157
|
+
});
|
|
158
|
+
nextWakeAtMs = Math.min(nextWakeAtMs, nowMs + 1);
|
|
159
|
+
break;
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
pending.shift();
|
|
163
|
+
providerStarts += 1;
|
|
164
|
+
providerStartsAtMs.push(nowMs);
|
|
165
|
+
completions.push({
|
|
166
|
+
atMs: nowMs + serviceTimeMs,
|
|
167
|
+
lease: admission.lease,
|
|
168
|
+
kind: 'success',
|
|
169
|
+
});
|
|
170
|
+
maximumProviderInFlight = Math.max(
|
|
171
|
+
maximumProviderInFlight,
|
|
172
|
+
completions.length,
|
|
173
|
+
);
|
|
174
|
+
}
|
|
175
|
+
if (completedAtMs.length === operationCount) break;
|
|
176
|
+
const completionAtMs = completions.reduce(
|
|
177
|
+
(soonest, completion) => Math.min(soonest, completion.atMs),
|
|
178
|
+
Number.POSITIVE_INFINITY,
|
|
179
|
+
);
|
|
180
|
+
const controllerWakeAtMs = controller.snapshot().nextAdmissionAtMs;
|
|
181
|
+
const nextAtMs = Math.min(
|
|
182
|
+
completionAtMs,
|
|
183
|
+
nextWakeAtMs,
|
|
184
|
+
controllerWakeAtMs > nowMs
|
|
185
|
+
? controllerWakeAtMs
|
|
186
|
+
: Number.POSITIVE_INFINITY,
|
|
187
|
+
);
|
|
188
|
+
if (!Number.isFinite(nextAtMs) || nextAtMs <= nowMs) {
|
|
189
|
+
throw new Error(
|
|
190
|
+
'Fixture capacity discovery reached an invalid wake time.',
|
|
191
|
+
);
|
|
192
|
+
}
|
|
193
|
+
nowMs = nextAtMs;
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
const steadyWindowStartedAtMs = Math.max(0, nowMs - analysisWindowMs);
|
|
197
|
+
const steadyCompletions = completedAtMs.filter(
|
|
198
|
+
(completedAt) => completedAt > steadyWindowStartedAtMs,
|
|
199
|
+
).length;
|
|
200
|
+
const measuredWindowMs = Math.max(1, nowMs - steadyWindowStartedAtMs);
|
|
201
|
+
const steadyGoodputPerSecond = (steadyCompletions * 1_000) / measuredWindowMs;
|
|
202
|
+
const physicalCeilingPerSecond = Math.min(
|
|
203
|
+
providerRps,
|
|
204
|
+
(providerConcurrency * 1_000) / serviceTimeMs,
|
|
205
|
+
);
|
|
206
|
+
const snapshot = controller.snapshot();
|
|
207
|
+
return Object.freeze({
|
|
208
|
+
completedOperations: completedAtMs.length,
|
|
209
|
+
providerStarts,
|
|
210
|
+
provider429s,
|
|
211
|
+
elapsedMs: nowMs,
|
|
212
|
+
physicalCeilingPerSecond,
|
|
213
|
+
steadyGoodputPerSecond,
|
|
214
|
+
steadyStatePhysicsUtilization:
|
|
215
|
+
steadyGoodputPerSecond / physicalCeilingPerSecond,
|
|
216
|
+
finalTargetRps: snapshot.rate.targetRequestsPerSecond,
|
|
217
|
+
finalConcurrency: snapshot.concurrency.limit,
|
|
218
|
+
maximumProviderInFlight,
|
|
219
|
+
});
|
|
220
|
+
}
|