deepline 0.3.127 → 0.3.129

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (128) hide show
  1. package/dist/bundling-sources/sdk/src/client.ts +34 -9
  2. package/dist/bundling-sources/sdk/src/http.ts +22 -0
  3. package/dist/bundling-sources/sdk/src/release.ts +1 -1
  4. package/dist/bundling-sources/shared_libs/play-data-plane/r2.ts +22 -0
  5. package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +265 -2
  6. package/dist/bundling-sources/shared_libs/play-runtime/bounded-batch-function.ts +682 -0
  7. package/dist/bundling-sources/shared_libs/play-runtime/child-execution-strategy.ts +21 -5
  8. package/dist/bundling-sources/shared_libs/play-runtime/context.ts +5390 -1732
  9. package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +107 -1
  10. package/dist/bundling-sources/shared_libs/play-runtime/deployment/commands/roll-execution-relay.ts +7 -0
  11. package/dist/bundling-sources/shared_libs/play-runtime/deployment/fly-machines.ts +229 -7
  12. package/dist/bundling-sources/shared_libs/play-runtime/deployment/fly-registry.ts +25 -4
  13. package/dist/bundling-sources/shared_libs/play-runtime/deployment/runtime-gateway-capacity.ts +71 -0
  14. package/dist/bundling-sources/shared_libs/play-runtime/deployment/runtime-gateway-drain.ts +1 -1
  15. package/dist/bundling-sources/shared_libs/play-runtime/deployment/runtime-topology.ts +4 -1
  16. package/dist/bundling-sources/shared_libs/play-runtime/durable-receipt-execution.ts +694 -28
  17. package/dist/bundling-sources/shared_libs/play-runtime/egress-bandwidth-probe.ts +167 -0
  18. package/dist/bundling-sources/shared_libs/play-runtime/execute-server-timing.ts +103 -0
  19. package/dist/bundling-sources/shared_libs/play-runtime/execution-gateway-receipt-output-reference.ts +34 -0
  20. package/dist/bundling-sources/shared_libs/play-runtime/execution-gateway-response-size.ts +27 -0
  21. package/dist/bundling-sources/shared_libs/play-runtime/gateway-postgres-admission.ts +66 -1
  22. package/dist/bundling-sources/shared_libs/play-runtime/governor/app-runtime-rate-state-backend.ts +425 -71
  23. package/dist/bundling-sources/shared_libs/play-runtime/governor/governor.ts +210 -32
  24. package/dist/bundling-sources/shared_libs/play-runtime/governor/in-memory-rate-state-backend.ts +27 -13
  25. package/dist/bundling-sources/shared_libs/play-runtime/governor/policy.ts +19 -6
  26. package/dist/bundling-sources/shared_libs/play-runtime/governor/rate-state-backend.ts +61 -2
  27. package/dist/bundling-sources/shared_libs/play-runtime/integration-operation-reader-fence/contracts.ts +33 -1
  28. package/dist/bundling-sources/shared_libs/play-runtime/integration-operation-reader-fence/job.ts +42 -1
  29. package/dist/bundling-sources/shared_libs/play-runtime/launch-source-policy.ts +11 -11
  30. package/dist/bundling-sources/shared_libs/play-runtime/ledger-safe-payload.ts +49 -16
  31. package/dist/bundling-sources/shared_libs/play-runtime/map-memory-limits.ts +1653 -98
  32. package/dist/bundling-sources/shared_libs/play-runtime/memory-ledger.ts +807 -0
  33. package/dist/bundling-sources/shared_libs/play-runtime/memory-pressure-signal.ts +138 -0
  34. package/dist/bundling-sources/shared_libs/play-runtime/output-size-limits.ts +85 -3
  35. package/dist/bundling-sources/shared_libs/play-runtime/persistence-latch.ts +60 -2
  36. package/dist/bundling-sources/shared_libs/play-runtime/play-run-recovery-policy.ts +46 -1
  37. package/dist/bundling-sources/shared_libs/play-runtime/play-runs/terminal/contracts.ts +2 -0
  38. package/dist/bundling-sources/shared_libs/play-runtime/play-runs/terminal/job.ts +1 -0
  39. package/dist/bundling-sources/shared_libs/play-runtime/play-runs/terminal/ports.ts +1 -0
  40. package/dist/bundling-sources/shared_libs/play-runtime/play-runs/terminal/postgres-adapter.ts +1 -0
  41. package/dist/bundling-sources/shared_libs/play-runtime/play-runtime-semantics.ts +70 -27
  42. package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts +56 -0
  43. package/dist/bundling-sources/shared_libs/play-runtime/provider-capacity/adaptive-provider-capacity-controller.ts +192 -0
  44. package/dist/bundling-sources/shared_libs/play-runtime/provider-capacity/epoch-aimd-rate-controller.ts +194 -0
  45. package/dist/bundling-sources/shared_libs/play-runtime/provider-capacity/fixture-capacity-simulator.ts +220 -0
  46. package/dist/bundling-sources/shared_libs/play-runtime/provider-capacity/gradient2-concurrency-controller.ts +214 -0
  47. package/dist/bundling-sources/shared_libs/play-runtime/provider-capacity/provider-capacity-policy.ts +43 -0
  48. package/dist/bundling-sources/shared_libs/play-runtime/provider-capacity/provider-concurrency-policy.ts +30 -0
  49. package/dist/bundling-sources/shared_libs/play-runtime/provider-capacity/provider-rate-policy.ts +95 -0
  50. package/dist/bundling-sources/shared_libs/play-runtime/provider-capacity/provider-throttle-classifier.ts +40 -0
  51. package/dist/bundling-sources/shared_libs/play-runtime/resource-governor.ts +1535 -11
  52. package/dist/bundling-sources/shared_libs/play-runtime/run-failure.ts +49 -2
  53. package/dist/bundling-sources/shared_libs/play-runtime/run-ledger.ts +6 -1
  54. package/dist/bundling-sources/shared_libs/play-runtime/run-log-buffer.ts +55 -0
  55. package/dist/bundling-sources/shared_libs/play-runtime/runner-app/egress-probe.ts +221 -0
  56. package/dist/bundling-sources/shared_libs/play-runtime/runner-app/index.ts +339 -29
  57. package/dist/bundling-sources/shared_libs/play-runtime/runner-app/runtime-deadline.ts +28 -1
  58. package/dist/bundling-sources/shared_libs/play-runtime/runner-app/runtime-progress-push.ts +192 -165
  59. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-lifecycle.ts +6 -0
  60. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-payload-transport.ts +90 -5
  61. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-runtime-watchdog.ts +281 -6
  62. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts +27 -1
  63. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/local-process.ts +19 -0
  64. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/modal.ts +108 -0
  65. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/runtime-sandbox-reconciliation.ts +40 -0
  66. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/types.ts +11 -5
  67. package/dist/bundling-sources/shared_libs/play-runtime/runtime-actions.ts +30 -0
  68. package/dist/bundling-sources/shared_libs/play-runtime/runtime-api-compression.ts +276 -0
  69. package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +723 -115
  70. package/dist/bundling-sources/shared_libs/play-runtime/runtime-capacity-policy.ts +98 -2
  71. package/dist/bundling-sources/shared_libs/play-runtime/runtime-contract.ts +3 -2
  72. package/dist/bundling-sources/shared_libs/play-runtime/runtime-environment.ts +41 -4
  73. package/dist/bundling-sources/shared_libs/play-runtime/runtime-flow.ts +448 -0
  74. package/dist/bundling-sources/shared_libs/play-runtime/runtime-map-row-feed.ts +215 -0
  75. package/dist/bundling-sources/shared_libs/play-runtime/runtime-operation-contract.ts +11 -0
  76. package/dist/bundling-sources/shared_libs/play-runtime/runtime-receipt-store-adapter.ts +161 -14
  77. package/dist/bundling-sources/shared_libs/play-runtime/runtime-receipt-value.ts +184 -0
  78. package/dist/bundling-sources/shared_libs/play-runtime/runtime-receipt-writer.ts +607 -121
  79. package/dist/bundling-sources/shared_libs/play-runtime/runtime-sandbox-placement-policy.ts +4 -0
  80. package/dist/bundling-sources/shared_libs/play-runtime/runtime-sheet-row-writer.ts +21 -0
  81. package/dist/bundling-sources/shared_libs/play-runtime/sandbox-runs/contract.ts +15 -2
  82. package/dist/bundling-sources/shared_libs/play-runtime/sandbox-runs/runner-backend-adapter.ts +71 -14
  83. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-play-deploy-task.ts +2 -2
  84. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-runtime-worker.ts +288 -119
  85. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-shared.ts +8 -6
  86. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-worker-required-env.ts +28 -3
  87. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/canonical-integration-operation-store.ts +179 -0
  88. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/gateway-heavy-request-admission.ts +80 -2
  89. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/gateway-progress-registry.ts +31 -2
  90. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/integration-operation-store-contract.ts +104 -0
  91. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/integration-operation-value-reference.ts +47 -0
  92. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/integration-operation-value-retirement.ts +193 -0
  93. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/integration-operation-value-storage.ts +420 -0
  94. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-pool.ts +41 -0
  95. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-progress.ts +134 -3
  96. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-rate-state.ts +1657 -95
  97. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts +885 -73
  98. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/receipt-completion-coalescer.ts +452 -0
  99. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/receipt-gateway.ts +601 -42
  100. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/run-terminal-transition.ts +1 -0
  101. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-receipt-value-storage.ts +371 -0
  102. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/worker-resource-metrics.ts +86 -7
  103. package/dist/bundling-sources/shared_libs/play-runtime/secret-redaction.ts +153 -22
  104. package/dist/bundling-sources/shared_libs/play-runtime/tool-call/index.ts +1 -0
  105. package/dist/bundling-sources/shared_libs/play-runtime/tool-call/receipt-cohort.ts +188 -38
  106. package/dist/bundling-sources/shared_libs/play-runtime/tool-call/runtime-step-receipts-adapter.ts +28 -13
  107. package/dist/bundling-sources/shared_libs/play-runtime/tool-execute-retry-policy.ts +44 -1
  108. package/dist/bundling-sources/shared_libs/play-runtime/tool-result.ts +56 -4
  109. package/dist/bundling-sources/shared_libs/play-runtime/work-receipts.ts +2 -0
  110. package/dist/bundling-sources/shared_libs/plays/authoring-contract.ts +16 -10
  111. package/dist/bundling-sources/shared_libs/plays/dataset.ts +4 -1
  112. package/dist/cli/index.js +254 -38
  113. package/dist/cli/index.mjs +254 -38
  114. package/dist/{compiler-manifest-DfSnbCTW.d.mts → compiler-manifest-BIqRyj5m.d.mts} +1 -1
  115. package/dist/{compiler-manifest-DfSnbCTW.d.ts → compiler-manifest-BIqRyj5m.d.ts} +1 -1
  116. package/dist/index.d.mts +4 -2
  117. package/dist/index.d.ts +4 -2
  118. package/dist/index.js +231 -35
  119. package/dist/index.mjs +231 -35
  120. package/dist/install-integrity.json +31 -2
  121. package/dist/plays/bundle-play-file.d.mts +2 -2
  122. package/dist/plays/bundle-play-file.d.ts +2 -2
  123. package/dist/plays/bundle-play-file.mjs +65 -1
  124. package/dist/release.d.mts +1 -1
  125. package/dist/release.d.ts +1 -1
  126. package/dist/release.js +1 -1
  127. package/dist/release.mjs +1 -1
  128. package/package.json +1 -1
@@ -35,9 +35,47 @@ export type PlayRunnerRateStateAcquireInput = {
35
35
  requested: number;
36
36
  /** Confirmed provider successes since this runner's previous reservation. */
37
37
  observedSuccesses?: number;
38
+ /**
39
+ * Completed physical provider calls buffered by this runner since its prior
40
+ * reservation. The gateway folds these into the durable Gradient2 checkpoint
41
+ * before it issues the next provider permit.
42
+ */
43
+ concurrencyObservations?: PlayRunnerRateStateConcurrencyObservation[];
44
+ /**
45
+ * Stable identity for the buffered observation set. A retried acquire must
46
+ * carry the same identity so the durable controller folds it at most once.
47
+ */
48
+ concurrencyObservationBatch?: PlayRunnerRateStateConcurrencyObservationBatch;
38
49
  schedulerSchema?: string | null;
39
50
  };
40
51
 
52
+ /** A bounded, provider-attempt outcome used by the shared concurrency learner. */
53
+ export type PlayRunnerRateStateConcurrencyObservation = {
54
+ kind: 'success' | 'throttled';
55
+ latencyMs: number;
56
+ };
57
+
58
+ export type PlayRunnerRateStateConcurrencyObservationBatch = {
59
+ /** Random per-runner identity; never a customer or credential identifier. */
60
+ observerId: string;
61
+ /** Separates independently refilled rule lanes in the same runner. */
62
+ laneId: string;
63
+ /** Monotonic within that runner lane. */
64
+ sequence: number;
65
+ };
66
+
67
+ /**
68
+ * Admission-time facts needed to attribute a later provider 429. The gateway
69
+ * owns this snapshot because only it sees global concurrent leases.
70
+ */
71
+ export type PlayRunnerRateStateConcurrencyGrant = {
72
+ /** Server-minted permit identity; deduplicates its later 429 report. */
73
+ admissionId: string;
74
+ inFlightAtStart: number;
75
+ targetRequestsPerSecond: number | null;
76
+ minimumRttMs: number | null;
77
+ };
78
+
41
79
  export type PlayRunnerRateStateAcquireResult = {
42
80
  granted: number;
43
81
  waitMs: number;
@@ -56,6 +94,13 @@ export type PlayRunnerRateStateAcquireResult = {
56
94
  effectiveRequestsPerSecond?: number;
57
95
  /** One independently-expiring concurrency reservation per granted permit. */
58
96
  leaseIds?: string[];
97
+ /** One concurrency-attribution snapshot per granted permit. */
98
+ concurrencyGrants?: PlayRunnerRateStateConcurrencyGrant[];
99
+ /**
100
+ * One idempotency key per granted rate permit. A runner sends one back only
101
+ * when that ticket was never dispatched and must be returned to the bucket.
102
+ */
103
+ reservationIds?: string[];
59
104
  /**
60
105
  * Absolute epoch-ms until which the pacer's own cooldown holds (internally
61
106
  * capped, e.g. 45min probe cadence). Advisory: exposes the current backoff to
@@ -74,6 +119,8 @@ export type PlayRunnerRateStateReleaseInput = {
74
119
  rateScopeToken?: string | null;
75
120
  rules: PacingRule[];
76
121
  leaseIds: string[];
122
+ /** Idempotently refund rate reservations that never reached provider fetch. */
123
+ refundReservationIds?: string[];
77
124
  schedulerSchema?: string | null;
78
125
  };
79
126
 
@@ -81,6 +128,15 @@ export type PlayRunnerRateStatePenalizeInput = {
81
128
  bucketId: string;
82
129
  rateScopeToken?: string | null;
83
130
  cooldownMs: number;
131
+ /**
132
+ * The same resolved rules that admitted the attempt. Optional during the
133
+ * rolling deploy window: old runners retain rate-only backpressure handling.
134
+ */
135
+ rules?: PacingRule[];
136
+ /** Actual provider residence for this 429, when the response was observed. */
137
+ latencyMs?: number | null;
138
+ /** The server-authored permit snapshot for the rejected physical call. */
139
+ concurrencyGrant?: PlayRunnerRateStateConcurrencyGrant | null;
84
140
  schedulerSchema?: string | null;
85
141
  };
86
142
 
@@ -0,0 +1,192 @@
1
+ import {
2
+ EpochAimdRateController,
3
+ type EpochAimdRateControllerOptions,
4
+ type EpochAimdRateControllerSnapshot,
5
+ } from './epoch-aimd-rate-controller';
6
+ import {
7
+ Gradient2ConcurrencyController,
8
+ type Gradient2ConcurrencyControllerOptions,
9
+ type Gradient2ConcurrencyControllerSnapshot,
10
+ } from './gradient2-concurrency-controller';
11
+ import { classifyProviderThrottle } from './provider-throttle-classifier';
12
+
13
+ export type AdaptiveProviderCapacityControllerOptions = Readonly<{
14
+ rate: EpochAimdRateControllerOptions;
15
+ concurrency: Gradient2ConcurrencyControllerOptions;
16
+ }>;
17
+
18
+ export type ProviderCapacityLease = Readonly<{
19
+ id: number;
20
+ startedAtMs: number;
21
+ inFlightAtStart: number;
22
+ }>;
23
+
24
+ export type ProviderCapacityAcquisition =
25
+ | Readonly<{ kind: 'acquired'; lease: ProviderCapacityLease }>
26
+ | Readonly<{ kind: 'waiting'; retryAtMs: number }>;
27
+
28
+ export type ProviderCapacityCompletion = Readonly<{
29
+ kind: 'success' | 'throttled' | 'transport_error';
30
+ latencyMs: number;
31
+ completedAtMs: number;
32
+ retryAfterMs?: number;
33
+ }>;
34
+
35
+ export type AdaptiveProviderCapacityControllerSnapshot = Readonly<{
36
+ active: number;
37
+ nextAdmissionAtMs: number;
38
+ rate: EpochAimdRateControllerSnapshot;
39
+ concurrency: Gradient2ConcurrencyControllerSnapshot;
40
+ }>;
41
+
42
+ /**
43
+ * One admission point over independent rate and concurrency estimators.
44
+ *
45
+ * Callers share this instance for one provider credential scope. Controllers
46
+ * only supply budgets; this facade is the sole module allowed to issue a
47
+ * physical-call lease.
48
+ */
49
+ export class AdaptiveProviderCapacityController {
50
+ readonly #rate: EpochAimdRateController;
51
+ readonly #concurrency: Gradient2ConcurrencyController;
52
+ readonly #activeLeases = new Map<number, ProviderCapacityLease>();
53
+ #nextLeaseId = 1;
54
+ #nextRateAdmissionAtMs = 0;
55
+ #nextConcurrencyCheckAtMs = 0;
56
+ #lastObservedAtMs = 0;
57
+ #observedConcurrencyThrottleAt: number | null = null;
58
+
59
+ constructor(options: AdaptiveProviderCapacityControllerOptions) {
60
+ this.#rate = new EpochAimdRateController(options.rate);
61
+ this.#concurrency = new Gradient2ConcurrencyController(options.concurrency);
62
+ }
63
+
64
+ tryAcquire(nowMs: number): ProviderCapacityAcquisition {
65
+ this.#advance(nowMs);
66
+ const rate = this.#rate.snapshot();
67
+ const concurrency = this.#concurrency.snapshot();
68
+ const retryAtMs = Math.max(
69
+ this.#nextRateAdmissionAtMs,
70
+ this.#nextConcurrencyCheckAtMs,
71
+ rate.cooldownUntilMs,
72
+ );
73
+ if (nowMs < retryAtMs) {
74
+ return Object.freeze({ kind: 'waiting', retryAtMs });
75
+ }
76
+ if (this.#activeLeases.size >= Math.max(1, Math.floor(concurrency.limit))) {
77
+ this.#nextConcurrencyCheckAtMs =
78
+ nowMs + Math.max(1, Math.floor((concurrency.shortRttMs ?? 100) / 4));
79
+ return Object.freeze({
80
+ kind: 'waiting',
81
+ retryAtMs: this.#nextConcurrencyCheckAtMs,
82
+ });
83
+ }
84
+
85
+ const lease = Object.freeze({
86
+ id: this.#nextLeaseId,
87
+ startedAtMs: nowMs,
88
+ inFlightAtStart: this.#activeLeases.size + 1,
89
+ });
90
+ this.#nextLeaseId += 1;
91
+ this.#activeLeases.set(lease.id, lease);
92
+ this.#rate.observeStart(nowMs);
93
+ this.#nextRateAdmissionAtMs =
94
+ nowMs + 1_000 / this.#rate.snapshot().targetRequestsPerSecond;
95
+ this.#nextConcurrencyCheckAtMs = nowMs;
96
+ return Object.freeze({ kind: 'acquired', lease });
97
+ }
98
+
99
+ complete(
100
+ lease: ProviderCapacityLease,
101
+ completion: ProviderCapacityCompletion,
102
+ ): void {
103
+ this.#advance(completion.completedAtMs);
104
+ if (!this.#activeLeases.delete(lease.id)) {
105
+ throw new Error(
106
+ 'Provider-capacity lease is unknown or already complete.',
107
+ );
108
+ }
109
+
110
+ if (completion.kind === 'success') {
111
+ this.#rate.observeSuccess(completion.completedAtMs);
112
+ this.#concurrency.observe({
113
+ kind: 'success',
114
+ latencyMs: completion.latencyMs,
115
+ inFlightAtStart: lease.inFlightAtStart,
116
+ });
117
+ } else if (completion.kind === 'transport_error') {
118
+ // Transport failure is not evidence of a provider's rate or concurrent
119
+ // occupancy ceiling. The runner-local HTTP window may still apply a
120
+ // short-lived transport-congestion response, but the shared provider
121
+ // controller must match Postgres grant learning, which deliberately
122
+ // drops transport observations.
123
+ } else {
124
+ const pressure =
125
+ completion.kind === 'throttled'
126
+ ? this.#classifyThrottle(lease)
127
+ : 'concurrency';
128
+ if (pressure === 'rate' || pressure === 'ambiguous') {
129
+ this.#rate.observeThrottle(
130
+ completion.completedAtMs,
131
+ completion.retryAfterMs ?? 0,
132
+ );
133
+ }
134
+ if (pressure === 'concurrency' || pressure === 'ambiguous') {
135
+ this.#observedConcurrencyThrottleAt =
136
+ this.#observedConcurrencyThrottleAt === null
137
+ ? lease.inFlightAtStart
138
+ : Math.min(
139
+ this.#observedConcurrencyThrottleAt,
140
+ lease.inFlightAtStart,
141
+ );
142
+ this.#concurrency.observe({
143
+ kind: completion.kind,
144
+ latencyMs: completion.latencyMs,
145
+ inFlightAtStart: lease.inFlightAtStart,
146
+ });
147
+ }
148
+ }
149
+ this.#nextConcurrencyCheckAtMs = completion.completedAtMs;
150
+ }
151
+
152
+ snapshot(): AdaptiveProviderCapacityControllerSnapshot {
153
+ return Object.freeze({
154
+ active: this.#activeLeases.size,
155
+ nextAdmissionAtMs: Math.max(
156
+ this.#nextRateAdmissionAtMs,
157
+ this.#nextConcurrencyCheckAtMs,
158
+ this.#rate.snapshot().cooldownUntilMs,
159
+ ),
160
+ rate: this.#rate.snapshot(),
161
+ concurrency: this.#concurrency.snapshot(),
162
+ });
163
+ }
164
+
165
+ #advance(nowMs: number): void {
166
+ if (!Number.isFinite(nowMs) || nowMs < this.#lastObservedAtMs) {
167
+ throw new Error('Provider-capacity time must be finite and monotonic.');
168
+ }
169
+ this.#lastObservedAtMs = nowMs;
170
+ this.#rate.advanceTo(nowMs);
171
+ }
172
+
173
+ #classifyThrottle(
174
+ lease: ProviderCapacityLease,
175
+ ): 'rate' | 'concurrency' | 'ambiguous' {
176
+ if (
177
+ this.#observedConcurrencyThrottleAt !== null &&
178
+ lease.inFlightAtStart >= this.#observedConcurrencyThrottleAt
179
+ ) {
180
+ return 'concurrency';
181
+ }
182
+ const rate = this.#rate.snapshot().targetRequestsPerSecond;
183
+ const minimumRttMs = this.#concurrency.snapshot().minimumRttMs;
184
+ return classifyProviderThrottle({
185
+ grant: {
186
+ targetRequestsPerSecond: rate,
187
+ minimumRttMs,
188
+ },
189
+ inFlightAtStart: lease.inFlightAtStart,
190
+ });
191
+ }
192
+ }
@@ -0,0 +1,194 @@
1
+ export type EpochAimdRateControllerOptions = Readonly<{
2
+ initialRequestsPerSecond: number;
3
+ minimumRequestsPerSecond: number;
4
+ maximumRequestsPerSecond: number;
5
+ epochMs: number;
6
+ additiveIncrease: number;
7
+ decreaseFactor: number;
8
+ utilizationThreshold: number;
9
+ /** Bounded exponential cold-start probe; one or below preserves pure AIMD. */
10
+ startupProbeMultiplier?: number;
11
+ }>;
12
+
13
+ export type EpochAimdRateControllerSnapshot = Readonly<{
14
+ targetRequestsPerSecond: number;
15
+ cooldownUntilMs: number;
16
+ epochStartedAtMs: number;
17
+ startsInEpoch: number;
18
+ successesInEpoch: number;
19
+ throttleEpochs: number;
20
+ }>;
21
+
22
+ function positive(value: number, label: string): number {
23
+ if (!Number.isFinite(value) || value <= 0) {
24
+ throw new Error(`${label} must be finite and greater than zero.`);
25
+ }
26
+ return value;
27
+ }
28
+
29
+ function fraction(value: number, label: string): number {
30
+ if (!Number.isFinite(value) || value <= 0 || value > 1) {
31
+ throw new Error(`${label} must be greater than zero and at most one.`);
32
+ }
33
+ return value;
34
+ }
35
+
36
+ /**
37
+ * Epoch-based additive-increase/multiplicative-decrease rate discovery.
38
+ *
39
+ * Successful latency never changes this controller. An epoch grows only when
40
+ * callers used most of the offered starts and those starts completed without
41
+ * provider throttling. A synchronized 429 burst decreases at most once per
42
+ * epoch.
43
+ */
44
+ export class EpochAimdRateController {
45
+ readonly #minimumRequestsPerSecond: number;
46
+ readonly #maximumRequestsPerSecond: number;
47
+ readonly #epochMs: number;
48
+ readonly #additiveIncrease: number;
49
+ readonly #decreaseFactor: number;
50
+ readonly #utilizationThreshold: number;
51
+ readonly #startupProbeMultiplier: number;
52
+ #targetRequestsPerSecond: number;
53
+ #epochStartedAtMs = 0;
54
+ #initialized = false;
55
+ #startsInEpoch = 0;
56
+ #successesInEpoch = 0;
57
+ #throttledInEpoch = false;
58
+ #cooldownUntilMs = 0;
59
+ #throttleEpochs = 0;
60
+ #hasObservedThrottle = false;
61
+
62
+ constructor(options: EpochAimdRateControllerOptions) {
63
+ this.#minimumRequestsPerSecond = positive(
64
+ options.minimumRequestsPerSecond,
65
+ 'Minimum requests per second',
66
+ );
67
+ this.#maximumRequestsPerSecond = positive(
68
+ options.maximumRequestsPerSecond,
69
+ 'Maximum requests per second',
70
+ );
71
+ if (this.#maximumRequestsPerSecond < this.#minimumRequestsPerSecond) {
72
+ throw new Error(
73
+ 'Maximum requests per second cannot be below the minimum.',
74
+ );
75
+ }
76
+ this.#targetRequestsPerSecond = Math.min(
77
+ this.#maximumRequestsPerSecond,
78
+ Math.max(
79
+ this.#minimumRequestsPerSecond,
80
+ positive(
81
+ options.initialRequestsPerSecond,
82
+ 'Initial requests per second',
83
+ ),
84
+ ),
85
+ );
86
+ this.#epochMs = positive(options.epochMs, 'Rate epoch');
87
+ this.#additiveIncrease = positive(
88
+ options.additiveIncrease,
89
+ 'Additive increase',
90
+ );
91
+ this.#decreaseFactor = fraction(options.decreaseFactor, 'Decrease factor');
92
+ this.#utilizationThreshold = fraction(
93
+ options.utilizationThreshold,
94
+ 'Utilization threshold',
95
+ );
96
+ this.#startupProbeMultiplier = Math.max(
97
+ 1,
98
+ positive(options.startupProbeMultiplier ?? 1, 'Startup probe multiplier'),
99
+ );
100
+ }
101
+
102
+ observeStart(startedAtMs: number): void {
103
+ this.advanceTo(startedAtMs);
104
+ this.#startsInEpoch += 1;
105
+ }
106
+
107
+ observeSuccess(completedAtMs: number): void {
108
+ this.advanceTo(completedAtMs);
109
+ this.#successesInEpoch += 1;
110
+ }
111
+
112
+ observeThrottle(observedAtMs: number, retryAfterMs: number): void {
113
+ this.advanceTo(observedAtMs);
114
+ // A 429 is the only authority that ends the initial probe. From this
115
+ // point forward the lane retains #3943's conservative AIMD recovery.
116
+ this.#hasObservedThrottle = true;
117
+ this.#cooldownUntilMs = Math.max(
118
+ this.#cooldownUntilMs,
119
+ observedAtMs + Math.max(0, retryAfterMs),
120
+ );
121
+ if (this.#throttledInEpoch) return;
122
+
123
+ this.#throttledInEpoch = true;
124
+ this.#throttleEpochs += 1;
125
+ this.#targetRequestsPerSecond = transitionProviderRateTarget({
126
+ currentRequestsPerSecond: this.#targetRequestsPerSecond,
127
+ minimumRequestsPerSecond: this.#minimumRequestsPerSecond,
128
+ maximumRequestsPerSecond: this.#maximumRequestsPerSecond,
129
+ event: 'rate_limited',
130
+ additiveIncrease: this.#additiveIncrease,
131
+ decreaseFactor: this.#decreaseFactor,
132
+ });
133
+ }
134
+
135
+ advanceTo(nowMs: number): void {
136
+ if (!Number.isFinite(nowMs)) {
137
+ throw new Error('Rate-controller time must be finite and monotonic.');
138
+ }
139
+ if (!this.#initialized) {
140
+ this.#epochStartedAtMs = nowMs;
141
+ this.#initialized = true;
142
+ return;
143
+ }
144
+ if (nowMs < this.#epochStartedAtMs) {
145
+ throw new Error('Rate-controller time must be finite and monotonic.');
146
+ }
147
+ while (nowMs >= this.#epochStartedAtMs + this.#epochMs) {
148
+ this.#closeEpoch();
149
+ this.#epochStartedAtMs += this.#epochMs;
150
+ this.#startsInEpoch = 0;
151
+ this.#successesInEpoch = 0;
152
+ this.#throttledInEpoch = false;
153
+ }
154
+ }
155
+
156
+ snapshot(): EpochAimdRateControllerSnapshot {
157
+ return Object.freeze({
158
+ targetRequestsPerSecond: this.#targetRequestsPerSecond,
159
+ cooldownUntilMs: this.#cooldownUntilMs,
160
+ epochStartedAtMs: this.#epochStartedAtMs,
161
+ startsInEpoch: this.#startsInEpoch,
162
+ successesInEpoch: this.#successesInEpoch,
163
+ throttleEpochs: this.#throttleEpochs,
164
+ });
165
+ }
166
+
167
+ #closeEpoch(): void {
168
+ const offeredStarts =
169
+ (this.#targetRequestsPerSecond * this.#epochMs) / 1_000;
170
+ const utilized =
171
+ this.#startsInEpoch >= offeredStarts * this.#utilizationThreshold;
172
+ const healthy =
173
+ !this.#throttledInEpoch &&
174
+ this.#successesInEpoch >=
175
+ this.#startsInEpoch * this.#utilizationThreshold;
176
+ if (!utilized || !healthy) return;
177
+
178
+ this.#targetRequestsPerSecond =
179
+ !this.#hasObservedThrottle && this.#startupProbeMultiplier > 1
180
+ ? Math.min(
181
+ this.#maximumRequestsPerSecond,
182
+ this.#targetRequestsPerSecond * this.#startupProbeMultiplier,
183
+ )
184
+ : transitionProviderRateTarget({
185
+ currentRequestsPerSecond: this.#targetRequestsPerSecond,
186
+ minimumRequestsPerSecond: this.#minimumRequestsPerSecond,
187
+ maximumRequestsPerSecond: this.#maximumRequestsPerSecond,
188
+ event: 'healthy_epoch',
189
+ additiveIncrease: this.#additiveIncrease,
190
+ decreaseFactor: this.#decreaseFactor,
191
+ });
192
+ }
193
+ }
194
+ import { transitionProviderRateTarget } from './provider-rate-policy';
@@ -0,0 +1,220 @@
1
+ import { AdaptiveProviderCapacityController } from './adaptive-provider-capacity-controller';
2
+ import { providerConcurrencyControllerOptions } from './provider-concurrency-policy';
3
+ import { providerRateControllerOptions } from './provider-rate-policy';
4
+
5
+ export type FixtureCapacityScenario = Readonly<{
6
+ operationCount: number;
7
+ providerRps: number;
8
+ providerConcurrency: number;
9
+ serviceTimeMs: number;
10
+ initialRps?: number;
11
+ maximumExplorationRps?: number;
12
+ initialConcurrency?: number;
13
+ maximumExplorationConcurrency?: number;
14
+ retryAfterMs?: number;
15
+ analysisWindowMs?: number;
16
+ }>;
17
+
18
+ export type FixtureCapacitySimulationResult = Readonly<{
19
+ completedOperations: number;
20
+ providerStarts: number;
21
+ provider429s: number;
22
+ elapsedMs: number;
23
+ physicalCeilingPerSecond: number;
24
+ steadyGoodputPerSecond: number;
25
+ steadyStatePhysicsUtilization: number;
26
+ finalTargetRps: number;
27
+ finalConcurrency: number;
28
+ maximumProviderInFlight: number;
29
+ }>;
30
+
31
+ type Completion = Readonly<{
32
+ atMs: number;
33
+ lease: Parameters<AdaptiveProviderCapacityController['complete']>[0];
34
+ kind: 'success' | 'throttled';
35
+ }>;
36
+
37
+ function positive(value: number, label: string): number {
38
+ if (!Number.isFinite(value) || value <= 0) {
39
+ throw new Error(`${label} must be finite and greater than zero.`);
40
+ }
41
+ return value;
42
+ }
43
+
44
+ function positiveInteger(value: number, label: string): number {
45
+ if (!Number.isSafeInteger(value) || value <= 0) {
46
+ throw new Error(`${label} must be a positive safe integer.`);
47
+ }
48
+ return value;
49
+ }
50
+
51
+ /**
52
+ * Deterministic fixture-only provider world.
53
+ *
54
+ * The controller receives only its conservative seed and exploration
55
+ * guardrails. Actual provider RPS/concurrency stay inside this simulator and
56
+ * reveal themselves solely as a real provider would: an unambiguous 429 with
57
+ * Retry-After. This makes it possible to falsify rate/concurrency discovery
58
+ * without a paid provider or a second scheduler implementation.
59
+ */
60
+ export function simulateFixtureCapacityDiscovery(
61
+ input: FixtureCapacityScenario,
62
+ ): FixtureCapacitySimulationResult {
63
+ const operationCount = positiveInteger(
64
+ input.operationCount,
65
+ 'Operation count',
66
+ );
67
+ const providerRps = positive(input.providerRps, 'Provider RPS');
68
+ const providerConcurrency = positiveInteger(
69
+ input.providerConcurrency,
70
+ 'Provider concurrency',
71
+ );
72
+ const serviceTimeMs = positive(input.serviceTimeMs, 'Provider service time');
73
+ const initialRps = positive(input.initialRps ?? 10, 'Initial RPS');
74
+ const maximumExplorationRps = positive(
75
+ input.maximumExplorationRps ?? 256,
76
+ 'Maximum exploration RPS',
77
+ );
78
+ const initialConcurrency = positiveInteger(
79
+ input.initialConcurrency ?? 1,
80
+ 'Initial concurrency',
81
+ );
82
+ const maximumExplorationConcurrency = positiveInteger(
83
+ input.maximumExplorationConcurrency ?? 256,
84
+ 'Maximum exploration concurrency',
85
+ );
86
+ const retryAfterMs = positiveInteger(
87
+ input.retryAfterMs ?? 100,
88
+ 'Retry-After',
89
+ );
90
+ const analysisWindowMs = positiveInteger(
91
+ input.analysisWindowMs ?? 30_000,
92
+ 'Analysis window',
93
+ );
94
+ const controller = new AdaptiveProviderCapacityController({
95
+ rate: providerRateControllerOptions({
96
+ declaredRequestsPerSecond: initialRps,
97
+ maximumRequestsPerSecond: maximumExplorationRps,
98
+ }),
99
+ concurrency: providerConcurrencyControllerOptions({
100
+ initialLimit: initialConcurrency,
101
+ maximumLimit: maximumExplorationConcurrency,
102
+ }),
103
+ });
104
+ const pending: number[] = Array.from(
105
+ { length: operationCount },
106
+ (_, index) => index,
107
+ );
108
+ const completions: Completion[] = [];
109
+ const providerStartsAtMs: number[] = [];
110
+ const completedAtMs: number[] = [];
111
+ let nowMs = 0;
112
+ let providerStarts = 0;
113
+ let provider429s = 0;
114
+ let maximumProviderInFlight = 0;
115
+ let attempts = 0;
116
+
117
+ const advanceCompletions = () => {
118
+ completions.sort((left, right) => left.atMs - right.atMs);
119
+ while (completions[0]?.atMs === nowMs) {
120
+ const completion = completions.shift()!;
121
+ controller.complete(completion.lease, {
122
+ kind: completion.kind,
123
+ latencyMs: completion.kind === 'success' ? serviceTimeMs : 1,
124
+ completedAtMs: nowMs,
125
+ ...(completion.kind === 'throttled' ? { retryAfterMs } : {}),
126
+ });
127
+ if (completion.kind === 'success') completedAtMs.push(nowMs);
128
+ }
129
+ };
130
+
131
+ while (completedAtMs.length < operationCount) {
132
+ advanceCompletions();
133
+ let nextWakeAtMs = Number.POSITIVE_INFINITY;
134
+ while (pending.length > 0) {
135
+ attempts += 1;
136
+ if (attempts > operationCount * 10_000) {
137
+ throw new Error('Fixture capacity discovery did not converge.');
138
+ }
139
+ const admission = controller.tryAcquire(nowMs);
140
+ if (admission.kind === 'waiting') {
141
+ nextWakeAtMs = Math.min(nextWakeAtMs, admission.retryAtMs);
142
+ break;
143
+ }
144
+
145
+ const recentStarts = providerStartsAtMs.filter(
146
+ (startedAtMs) => startedAtMs > nowMs - 1_000,
147
+ );
148
+ providerStartsAtMs.splice(0, providerStartsAtMs.length, ...recentStarts);
149
+ const rateLimited = recentStarts.length >= providerRps;
150
+ const concurrencyLimited = completions.length >= providerConcurrency;
151
+ if (rateLimited || concurrencyLimited) {
152
+ provider429s += 1;
153
+ completions.push({
154
+ atMs: nowMs + 1,
155
+ lease: admission.lease,
156
+ kind: 'throttled',
157
+ });
158
+ nextWakeAtMs = Math.min(nextWakeAtMs, nowMs + 1);
159
+ break;
160
+ }
161
+
162
+ pending.shift();
163
+ providerStarts += 1;
164
+ providerStartsAtMs.push(nowMs);
165
+ completions.push({
166
+ atMs: nowMs + serviceTimeMs,
167
+ lease: admission.lease,
168
+ kind: 'success',
169
+ });
170
+ maximumProviderInFlight = Math.max(
171
+ maximumProviderInFlight,
172
+ completions.length,
173
+ );
174
+ }
175
+ if (completedAtMs.length === operationCount) break;
176
+ const completionAtMs = completions.reduce(
177
+ (soonest, completion) => Math.min(soonest, completion.atMs),
178
+ Number.POSITIVE_INFINITY,
179
+ );
180
+ const controllerWakeAtMs = controller.snapshot().nextAdmissionAtMs;
181
+ const nextAtMs = Math.min(
182
+ completionAtMs,
183
+ nextWakeAtMs,
184
+ controllerWakeAtMs > nowMs
185
+ ? controllerWakeAtMs
186
+ : Number.POSITIVE_INFINITY,
187
+ );
188
+ if (!Number.isFinite(nextAtMs) || nextAtMs <= nowMs) {
189
+ throw new Error(
190
+ 'Fixture capacity discovery reached an invalid wake time.',
191
+ );
192
+ }
193
+ nowMs = nextAtMs;
194
+ }
195
+
196
+ const steadyWindowStartedAtMs = Math.max(0, nowMs - analysisWindowMs);
197
+ const steadyCompletions = completedAtMs.filter(
198
+ (completedAt) => completedAt > steadyWindowStartedAtMs,
199
+ ).length;
200
+ const measuredWindowMs = Math.max(1, nowMs - steadyWindowStartedAtMs);
201
+ const steadyGoodputPerSecond = (steadyCompletions * 1_000) / measuredWindowMs;
202
+ const physicalCeilingPerSecond = Math.min(
203
+ providerRps,
204
+ (providerConcurrency * 1_000) / serviceTimeMs,
205
+ );
206
+ const snapshot = controller.snapshot();
207
+ return Object.freeze({
208
+ completedOperations: completedAtMs.length,
209
+ providerStarts,
210
+ provider429s,
211
+ elapsedMs: nowMs,
212
+ physicalCeilingPerSecond,
213
+ steadyGoodputPerSecond,
214
+ steadyStatePhysicsUtilization:
215
+ steadyGoodputPerSecond / physicalCeilingPerSecond,
216
+ finalTargetRps: snapshot.rate.targetRequestsPerSecond,
217
+ finalConcurrency: snapshot.concurrency.limit,
218
+ maximumProviderInFlight,
219
+ });
220
+ }