deepline 0.3.127 → 0.3.129
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/sdk/src/client.ts +34 -9
- package/dist/bundling-sources/sdk/src/http.ts +22 -0
- package/dist/bundling-sources/sdk/src/release.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-data-plane/r2.ts +22 -0
- package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +265 -2
- package/dist/bundling-sources/shared_libs/play-runtime/bounded-batch-function.ts +682 -0
- package/dist/bundling-sources/shared_libs/play-runtime/child-execution-strategy.ts +21 -5
- package/dist/bundling-sources/shared_libs/play-runtime/context.ts +5390 -1732
- package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +107 -1
- package/dist/bundling-sources/shared_libs/play-runtime/deployment/commands/roll-execution-relay.ts +7 -0
- package/dist/bundling-sources/shared_libs/play-runtime/deployment/fly-machines.ts +229 -7
- package/dist/bundling-sources/shared_libs/play-runtime/deployment/fly-registry.ts +25 -4
- package/dist/bundling-sources/shared_libs/play-runtime/deployment/runtime-gateway-capacity.ts +71 -0
- package/dist/bundling-sources/shared_libs/play-runtime/deployment/runtime-gateway-drain.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/deployment/runtime-topology.ts +4 -1
- package/dist/bundling-sources/shared_libs/play-runtime/durable-receipt-execution.ts +694 -28
- package/dist/bundling-sources/shared_libs/play-runtime/egress-bandwidth-probe.ts +167 -0
- package/dist/bundling-sources/shared_libs/play-runtime/execute-server-timing.ts +103 -0
- package/dist/bundling-sources/shared_libs/play-runtime/execution-gateway-receipt-output-reference.ts +34 -0
- package/dist/bundling-sources/shared_libs/play-runtime/execution-gateway-response-size.ts +27 -0
- package/dist/bundling-sources/shared_libs/play-runtime/gateway-postgres-admission.ts +66 -1
- package/dist/bundling-sources/shared_libs/play-runtime/governor/app-runtime-rate-state-backend.ts +425 -71
- package/dist/bundling-sources/shared_libs/play-runtime/governor/governor.ts +210 -32
- package/dist/bundling-sources/shared_libs/play-runtime/governor/in-memory-rate-state-backend.ts +27 -13
- package/dist/bundling-sources/shared_libs/play-runtime/governor/policy.ts +19 -6
- package/dist/bundling-sources/shared_libs/play-runtime/governor/rate-state-backend.ts +61 -2
- package/dist/bundling-sources/shared_libs/play-runtime/integration-operation-reader-fence/contracts.ts +33 -1
- package/dist/bundling-sources/shared_libs/play-runtime/integration-operation-reader-fence/job.ts +42 -1
- package/dist/bundling-sources/shared_libs/play-runtime/launch-source-policy.ts +11 -11
- package/dist/bundling-sources/shared_libs/play-runtime/ledger-safe-payload.ts +49 -16
- package/dist/bundling-sources/shared_libs/play-runtime/map-memory-limits.ts +1653 -98
- package/dist/bundling-sources/shared_libs/play-runtime/memory-ledger.ts +807 -0
- package/dist/bundling-sources/shared_libs/play-runtime/memory-pressure-signal.ts +138 -0
- package/dist/bundling-sources/shared_libs/play-runtime/output-size-limits.ts +85 -3
- package/dist/bundling-sources/shared_libs/play-runtime/persistence-latch.ts +60 -2
- package/dist/bundling-sources/shared_libs/play-runtime/play-run-recovery-policy.ts +46 -1
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/terminal/contracts.ts +2 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/terminal/job.ts +1 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/terminal/ports.ts +1 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/terminal/postgres-adapter.ts +1 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runtime-semantics.ts +70 -27
- package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts +56 -0
- package/dist/bundling-sources/shared_libs/play-runtime/provider-capacity/adaptive-provider-capacity-controller.ts +192 -0
- package/dist/bundling-sources/shared_libs/play-runtime/provider-capacity/epoch-aimd-rate-controller.ts +194 -0
- package/dist/bundling-sources/shared_libs/play-runtime/provider-capacity/fixture-capacity-simulator.ts +220 -0
- package/dist/bundling-sources/shared_libs/play-runtime/provider-capacity/gradient2-concurrency-controller.ts +214 -0
- package/dist/bundling-sources/shared_libs/play-runtime/provider-capacity/provider-capacity-policy.ts +43 -0
- package/dist/bundling-sources/shared_libs/play-runtime/provider-capacity/provider-concurrency-policy.ts +30 -0
- package/dist/bundling-sources/shared_libs/play-runtime/provider-capacity/provider-rate-policy.ts +95 -0
- package/dist/bundling-sources/shared_libs/play-runtime/provider-capacity/provider-throttle-classifier.ts +40 -0
- package/dist/bundling-sources/shared_libs/play-runtime/resource-governor.ts +1535 -11
- package/dist/bundling-sources/shared_libs/play-runtime/run-failure.ts +49 -2
- package/dist/bundling-sources/shared_libs/play-runtime/run-ledger.ts +6 -1
- package/dist/bundling-sources/shared_libs/play-runtime/run-log-buffer.ts +55 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-app/egress-probe.ts +221 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-app/index.ts +339 -29
- package/dist/bundling-sources/shared_libs/play-runtime/runner-app/runtime-deadline.ts +28 -1
- package/dist/bundling-sources/shared_libs/play-runtime/runner-app/runtime-progress-push.ts +192 -165
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-lifecycle.ts +6 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-payload-transport.ts +90 -5
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-runtime-watchdog.ts +281 -6
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts +27 -1
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/local-process.ts +19 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/modal.ts +108 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/runtime-sandbox-reconciliation.ts +40 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/types.ts +11 -5
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-actions.ts +30 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-api-compression.ts +276 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +723 -115
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-capacity-policy.ts +98 -2
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-contract.ts +3 -2
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-environment.ts +41 -4
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-flow.ts +448 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-map-row-feed.ts +215 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-operation-contract.ts +11 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-receipt-store-adapter.ts +161 -14
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-receipt-value.ts +184 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-receipt-writer.ts +607 -121
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-sandbox-placement-policy.ts +4 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-sheet-row-writer.ts +21 -0
- package/dist/bundling-sources/shared_libs/play-runtime/sandbox-runs/contract.ts +15 -2
- package/dist/bundling-sources/shared_libs/play-runtime/sandbox-runs/runner-backend-adapter.ts +71 -14
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-play-deploy-task.ts +2 -2
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-runtime-worker.ts +288 -119
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-shared.ts +8 -6
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-worker-required-env.ts +28 -3
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/canonical-integration-operation-store.ts +179 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/gateway-heavy-request-admission.ts +80 -2
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/gateway-progress-registry.ts +31 -2
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/integration-operation-store-contract.ts +104 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/integration-operation-value-reference.ts +47 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/integration-operation-value-retirement.ts +193 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/integration-operation-value-storage.ts +420 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-pool.ts +41 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-progress.ts +134 -3
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-rate-state.ts +1657 -95
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts +885 -73
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/receipt-completion-coalescer.ts +452 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/receipt-gateway.ts +601 -42
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/run-terminal-transition.ts +1 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-receipt-value-storage.ts +371 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/worker-resource-metrics.ts +86 -7
- package/dist/bundling-sources/shared_libs/play-runtime/secret-redaction.ts +153 -22
- package/dist/bundling-sources/shared_libs/play-runtime/tool-call/index.ts +1 -0
- package/dist/bundling-sources/shared_libs/play-runtime/tool-call/receipt-cohort.ts +188 -38
- package/dist/bundling-sources/shared_libs/play-runtime/tool-call/runtime-step-receipts-adapter.ts +28 -13
- package/dist/bundling-sources/shared_libs/play-runtime/tool-execute-retry-policy.ts +44 -1
- package/dist/bundling-sources/shared_libs/play-runtime/tool-result.ts +56 -4
- package/dist/bundling-sources/shared_libs/play-runtime/work-receipts.ts +2 -0
- package/dist/bundling-sources/shared_libs/plays/authoring-contract.ts +16 -10
- package/dist/bundling-sources/shared_libs/plays/dataset.ts +4 -1
- package/dist/cli/index.js +254 -38
- package/dist/cli/index.mjs +254 -38
- package/dist/{compiler-manifest-DfSnbCTW.d.mts → compiler-manifest-BIqRyj5m.d.mts} +1 -1
- package/dist/{compiler-manifest-DfSnbCTW.d.ts → compiler-manifest-BIqRyj5m.d.ts} +1 -1
- package/dist/index.d.mts +4 -2
- package/dist/index.d.ts +4 -2
- package/dist/index.js +231 -35
- package/dist/index.mjs +231 -35
- package/dist/install-integrity.json +31 -2
- package/dist/plays/bundle-play-file.d.mts +2 -2
- package/dist/plays/bundle-play-file.d.ts +2 -2
- package/dist/plays/bundle-play-file.mjs +65 -1
- package/dist/release.d.mts +1 -1
- package/dist/release.d.ts +1 -1
- package/dist/release.js +1 -1
- package/dist/release.mjs +1 -1
- package/package.json +1 -1
|
@@ -8,20 +8,36 @@ type PendingWrite<Input, Output> = {
|
|
|
8
8
|
input: Input;
|
|
9
9
|
bytes: number;
|
|
10
10
|
signal: AbortSignal | null;
|
|
11
|
-
state: '
|
|
11
|
+
state: 'queued' | 'active' | 'settled';
|
|
12
12
|
aborted: unknown;
|
|
13
13
|
onAbort: (() => void) | null;
|
|
14
14
|
resolve: (output: Output) => void;
|
|
15
15
|
reject: (error: unknown) => void;
|
|
16
16
|
};
|
|
17
17
|
|
|
18
|
+
type CapacityWaiter = {
|
|
19
|
+
promise: Promise<void>;
|
|
20
|
+
resolve: () => void;
|
|
21
|
+
reject: (error: unknown) => void;
|
|
22
|
+
};
|
|
23
|
+
|
|
18
24
|
export type RuntimeReceiptWriterStats = {
|
|
25
|
+
/** True while at least one batch is in flight. */
|
|
19
26
|
active: boolean;
|
|
27
|
+
/** Batches currently in flight (bounded by `maxInFlightBatches`). */
|
|
28
|
+
inFlightBatches: number;
|
|
20
29
|
queued: number;
|
|
21
|
-
|
|
30
|
+
/** Inputs retained by queued or active physical batches. */
|
|
31
|
+
bufferedItems: number;
|
|
22
32
|
bufferedBytes: number;
|
|
23
33
|
};
|
|
24
34
|
|
|
35
|
+
/** Optional retained-input budget reserved for one transport lane. */
|
|
36
|
+
export type RuntimeReceiptWriterLaneBufferLimit = {
|
|
37
|
+
items?: number;
|
|
38
|
+
bytes?: number;
|
|
39
|
+
};
|
|
40
|
+
|
|
25
41
|
export type RuntimeReceiptWriterRetryEvent<Input> = {
|
|
26
42
|
phase: 'retry' | 'recovered';
|
|
27
43
|
attempt: number;
|
|
@@ -32,7 +48,7 @@ export type RuntimeReceiptWriterRetryEvent<Input> = {
|
|
|
32
48
|
elapsedMs: number;
|
|
33
49
|
retryAfterMs: number;
|
|
34
50
|
queued: number;
|
|
35
|
-
|
|
51
|
+
bufferedItems: number;
|
|
36
52
|
bufferedBytes: number;
|
|
37
53
|
error?: unknown;
|
|
38
54
|
};
|
|
@@ -50,6 +66,24 @@ export class RuntimeReceiptWriterBufferLimitError extends Error {
|
|
|
50
66
|
}
|
|
51
67
|
}
|
|
52
68
|
|
|
69
|
+
/**
|
|
70
|
+
* The writer rejected a new command before retaining its payload. The caller
|
|
71
|
+
* still owns the durable retry decision; a hidden waiter list must never turn
|
|
72
|
+
* this explicit backpressure into unbounded process memory.
|
|
73
|
+
*/
|
|
74
|
+
export class RuntimeReceiptWriterCapacityError extends Error {
|
|
75
|
+
constructor(
|
|
76
|
+
readonly limit: 'items' | 'bytes',
|
|
77
|
+
readonly current: number,
|
|
78
|
+
readonly maximum: number,
|
|
79
|
+
) {
|
|
80
|
+
super(
|
|
81
|
+
`Runtime receipt writer ${limit} capacity is full (${current}/${maximum}); retry after a physical batch settles.`,
|
|
82
|
+
);
|
|
83
|
+
this.name = 'RuntimeReceiptWriterCapacityError';
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
|
|
53
87
|
export class RuntimeReceiptWriterResultCountError extends Error {
|
|
54
88
|
constructor(
|
|
55
89
|
readonly inputCount: number,
|
|
@@ -94,6 +128,12 @@ export class RuntimeReceiptWriterRetryDeadlineError extends Error {
|
|
|
94
128
|
*/
|
|
95
129
|
export class RuntimeReceiptWriter<Input, Output> {
|
|
96
130
|
readonly #batchKey: (input: Input) => unknown;
|
|
131
|
+
readonly #batchLane: (input: Input) => string;
|
|
132
|
+
/**
|
|
133
|
+
* Receipt identities whose commands cannot overlap in transport. Callers
|
|
134
|
+
* opt into this only when they use more than one in-flight batch.
|
|
135
|
+
*/
|
|
136
|
+
readonly #orderingKeys: (input: Input) => readonly unknown[];
|
|
97
137
|
readonly #send: (
|
|
98
138
|
inputs: readonly Input[],
|
|
99
139
|
options: { signal: AbortSignal },
|
|
@@ -105,23 +145,42 @@ export class RuntimeReceiptWriter<Input, Output> {
|
|
|
105
145
|
readonly #maxBatchSize: number;
|
|
106
146
|
readonly #maxBatchBytes: number;
|
|
107
147
|
readonly #targetBatchBytes: number;
|
|
148
|
+
readonly #maxBufferedItems: number;
|
|
108
149
|
readonly #maxBufferedBytes: number;
|
|
109
150
|
readonly #maxFlushMs: number;
|
|
151
|
+
readonly #maxInFlightBatches: number;
|
|
152
|
+
readonly #maxInFlightBatchesByLane: ReadonlyMap<string, number>;
|
|
153
|
+
readonly #maxBufferedByLane: ReadonlyMap<
|
|
154
|
+
string,
|
|
155
|
+
Required<RuntimeReceiptWriterLaneBufferLimit>
|
|
156
|
+
>;
|
|
110
157
|
readonly #maxRetryElapsedMs: number;
|
|
111
158
|
readonly #onRetryEvent:
|
|
112
159
|
| ((event: RuntimeReceiptWriterRetryEvent<Input>) => void)
|
|
113
160
|
| null;
|
|
161
|
+
readonly #onBufferedBytesChanged: ((bufferedBytes: number) => void) | null;
|
|
162
|
+
/**
|
|
163
|
+
* A bounded state snapshot for diagnostics. Observers can see pressure but
|
|
164
|
+
* cannot influence admission or durable delivery.
|
|
165
|
+
*/
|
|
166
|
+
readonly #onStatsChanged: ((stats: RuntimeReceiptWriterStats) => void) | null;
|
|
114
167
|
readonly #queued: Array<PendingWrite<Input, Output>> = [];
|
|
115
|
-
readonly #blocked: Array<PendingWrite<Input, Output>> = [];
|
|
116
168
|
readonly #closeController = new AbortController();
|
|
117
169
|
|
|
170
|
+
#bufferedItems = 0;
|
|
118
171
|
#bufferedBytes = 0;
|
|
119
172
|
#nextBatchId = 0;
|
|
120
|
-
#
|
|
121
|
-
#
|
|
173
|
+
#inFlight = new Set<Promise<void>>();
|
|
174
|
+
readonly #inFlightBatchesByLane = new Map<string, number>();
|
|
175
|
+
readonly #bufferedByLane = new Map<
|
|
176
|
+
string,
|
|
177
|
+
Required<RuntimeReceiptWriterLaneBufferLimit>
|
|
178
|
+
>();
|
|
179
|
+
readonly #activeOrderingKeys: unknown[] = [];
|
|
122
180
|
#flushTimer: ReturnType<typeof setTimeout> | null = null;
|
|
123
181
|
#closed = false;
|
|
124
182
|
#closeReason: unknown = null;
|
|
183
|
+
#capacityWaiter: CapacityWaiter | null = null;
|
|
125
184
|
#drainWaiters: Array<{
|
|
126
185
|
resolve: () => void;
|
|
127
186
|
reject: (error: unknown) => void;
|
|
@@ -129,6 +188,12 @@ export class RuntimeReceiptWriter<Input, Output> {
|
|
|
129
188
|
|
|
130
189
|
constructor(options: {
|
|
131
190
|
batchKey: (input: Input) => unknown;
|
|
191
|
+
/**
|
|
192
|
+
* Optional transport lane. Per-lane limits reserve bounded control
|
|
193
|
+
* capacity without exposing queues to receipt-domain callers.
|
|
194
|
+
*/
|
|
195
|
+
batchLane?: (input: Input) => string;
|
|
196
|
+
orderingKeys?: (input: Input) => readonly unknown[];
|
|
132
197
|
send: (
|
|
133
198
|
inputs: readonly Input[],
|
|
134
199
|
options: { signal: AbortSignal },
|
|
@@ -138,12 +203,37 @@ export class RuntimeReceiptWriter<Input, Output> {
|
|
|
138
203
|
maxBatchSize?: number;
|
|
139
204
|
maxBatchBytes?: number;
|
|
140
205
|
targetBatchBytes?: number;
|
|
206
|
+
maxBufferedItems?: number;
|
|
141
207
|
maxBufferedBytes?: number;
|
|
142
208
|
maxFlushMs?: number;
|
|
209
|
+
/**
|
|
210
|
+
* Concurrent transport requests this writer may hold open. Ordering across
|
|
211
|
+
* batches is NOT preserved: every receipt command is keyed, so the store
|
|
212
|
+
* is the ordering authority. Default 1 keeps the historical strictly
|
|
213
|
+
* serialized pump.
|
|
214
|
+
*/
|
|
215
|
+
maxInFlightBatches?: number;
|
|
216
|
+
/** Maximum simultaneous physical batches per named `batchLane`. */
|
|
217
|
+
maxInFlightBatchesByLane?: Readonly<Record<string, number>>;
|
|
218
|
+
/** Retained item/byte budgets reserved for named `batchLane`s. */
|
|
219
|
+
maxBufferedByLane?: Readonly<
|
|
220
|
+
Record<string, RuntimeReceiptWriterLaneBufferLimit>
|
|
221
|
+
>;
|
|
143
222
|
maxRetryElapsedMs?: number;
|
|
144
223
|
onRetryEvent?: (event: RuntimeReceiptWriterRetryEvent<Input>) => void;
|
|
224
|
+
/**
|
|
225
|
+
* Called with the writer's buffered byte total every time it changes. The
|
|
226
|
+
* writer is the only component that knows how many serialized completion
|
|
227
|
+
* bytes it is holding, so the runtime's byte ledger reads them from here
|
|
228
|
+
* rather than measuring the same payloads a second time.
|
|
229
|
+
*/
|
|
230
|
+
onBufferedBytesChanged?: (bufferedBytes: number) => void;
|
|
231
|
+
/** Emits after this writer's retained/active state changes. */
|
|
232
|
+
onStatsChanged?: (stats: RuntimeReceiptWriterStats) => void;
|
|
145
233
|
}) {
|
|
146
234
|
this.#batchKey = options.batchKey;
|
|
235
|
+
this.#batchLane = options.batchLane ?? (() => 'default');
|
|
236
|
+
this.#orderingKeys = options.orderingKeys ?? (() => []);
|
|
147
237
|
this.#send = options.send;
|
|
148
238
|
this.#classifyRetryableError =
|
|
149
239
|
options.classifyRetryableError ?? (() => null);
|
|
@@ -162,11 +252,84 @@ export class RuntimeReceiptWriter<Input, Output> {
|
|
|
162
252
|
this.#maxBatchBytes,
|
|
163
253
|
);
|
|
164
254
|
this.#maxFlushMs = Math.max(0, Math.floor(options.maxFlushMs ?? 5));
|
|
255
|
+
this.#maxInFlightBatches = normalizePositiveInteger(
|
|
256
|
+
options.maxInFlightBatches,
|
|
257
|
+
1,
|
|
258
|
+
);
|
|
259
|
+
this.#maxInFlightBatchesByLane = new Map(
|
|
260
|
+
Object.entries(options.maxInFlightBatchesByLane ?? {}).map(
|
|
261
|
+
([lane, maximum]) => [
|
|
262
|
+
normalizeLane(lane),
|
|
263
|
+
normalizePositiveInteger(maximum, 1),
|
|
264
|
+
],
|
|
265
|
+
),
|
|
266
|
+
);
|
|
267
|
+
this.#maxBufferedByLane = new Map(
|
|
268
|
+
Object.entries(options.maxBufferedByLane ?? {}).flatMap(
|
|
269
|
+
([lane, limits]) => {
|
|
270
|
+
const items =
|
|
271
|
+
limits.items === undefined
|
|
272
|
+
? undefined
|
|
273
|
+
: normalizePositiveInteger(limits.items, 1);
|
|
274
|
+
const bytes =
|
|
275
|
+
limits.bytes === undefined
|
|
276
|
+
? undefined
|
|
277
|
+
: normalizePositiveInteger(limits.bytes, 1);
|
|
278
|
+
return items === undefined && bytes === undefined
|
|
279
|
+
? []
|
|
280
|
+
: [
|
|
281
|
+
[
|
|
282
|
+
normalizeLane(lane),
|
|
283
|
+
{
|
|
284
|
+
items: items ?? Number.MAX_SAFE_INTEGER,
|
|
285
|
+
bytes: bytes ?? Number.MAX_SAFE_INTEGER,
|
|
286
|
+
},
|
|
287
|
+
] as const,
|
|
288
|
+
];
|
|
289
|
+
},
|
|
290
|
+
),
|
|
291
|
+
);
|
|
292
|
+
this.#maxBufferedItems = normalizePositiveInteger(
|
|
293
|
+
options.maxBufferedItems,
|
|
294
|
+
// A generic writer accepts one ordinary bulk receipt request by default;
|
|
295
|
+
// the Release Worker supplies its larger, release-specific bound.
|
|
296
|
+
Math.max(this.#maxBatchSize * this.#maxInFlightBatches * 2, 1_000),
|
|
297
|
+
);
|
|
165
298
|
this.#maxRetryElapsedMs = normalizePositiveInteger(
|
|
166
299
|
options.maxRetryElapsedMs,
|
|
167
300
|
2 * 60_000,
|
|
168
301
|
);
|
|
169
302
|
this.#onRetryEvent = options.onRetryEvent ?? null;
|
|
303
|
+
this.#onBufferedBytesChanged = options.onBufferedBytesChanged ?? null;
|
|
304
|
+
this.#onStatsChanged = options.onStatsChanged ?? null;
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
#adjustBufferedBytes(deltaBytes: number): void {
|
|
308
|
+
if (deltaBytes === 0) return;
|
|
309
|
+
this.#bufferedBytes += deltaBytes;
|
|
310
|
+
this.#onBufferedBytesChanged?.(this.#bufferedBytes);
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
#adjustBufferedItems(deltaItems: number, lane: string): void {
|
|
314
|
+
if (deltaItems === 0) return;
|
|
315
|
+
this.#bufferedItems += deltaItems;
|
|
316
|
+
this.#adjustBufferedLane(lane, { items: deltaItems, bytes: 0 });
|
|
317
|
+
}
|
|
318
|
+
|
|
319
|
+
#adjustBufferedLane(
|
|
320
|
+
lane: string,
|
|
321
|
+
delta: Required<RuntimeReceiptWriterLaneBufferLimit>,
|
|
322
|
+
): void {
|
|
323
|
+
const current = this.#bufferedByLane.get(lane) ?? { items: 0, bytes: 0 };
|
|
324
|
+
const next = {
|
|
325
|
+
items: current.items + delta.items,
|
|
326
|
+
bytes: current.bytes + delta.bytes,
|
|
327
|
+
};
|
|
328
|
+
if (next.items === 0 && next.bytes === 0) {
|
|
329
|
+
this.#bufferedByLane.delete(lane);
|
|
330
|
+
return;
|
|
331
|
+
}
|
|
332
|
+
this.#bufferedByLane.set(lane, next);
|
|
170
333
|
}
|
|
171
334
|
|
|
172
335
|
write(input: Input, options: { signal?: AbortSignal } = {}): Promise<Output> {
|
|
@@ -188,13 +351,60 @@ export class RuntimeReceiptWriter<Input, Output> {
|
|
|
188
351
|
if (options.signal?.aborted) {
|
|
189
352
|
return Promise.reject(abortReason(options.signal));
|
|
190
353
|
}
|
|
354
|
+
const lane = this.#laneFor(input);
|
|
355
|
+
const laneLimit = this.#maxBufferedByLane.get(lane);
|
|
356
|
+
if (laneLimit && bytes > laneLimit.bytes) {
|
|
357
|
+
return Promise.reject(
|
|
358
|
+
new RuntimeReceiptWriterBufferLimitError(bytes, laneLimit.bytes),
|
|
359
|
+
);
|
|
360
|
+
}
|
|
361
|
+
if (this.#bufferedItems >= this.#maxBufferedItems) {
|
|
362
|
+
return Promise.reject(
|
|
363
|
+
new RuntimeReceiptWriterCapacityError(
|
|
364
|
+
'items',
|
|
365
|
+
this.#bufferedItems,
|
|
366
|
+
this.#maxBufferedItems,
|
|
367
|
+
),
|
|
368
|
+
);
|
|
369
|
+
}
|
|
370
|
+
if (!this.#canBuffer(bytes)) {
|
|
371
|
+
return Promise.reject(
|
|
372
|
+
new RuntimeReceiptWriterCapacityError(
|
|
373
|
+
'bytes',
|
|
374
|
+
this.#bufferedBytes,
|
|
375
|
+
this.#maxBufferedBytes,
|
|
376
|
+
),
|
|
377
|
+
);
|
|
378
|
+
}
|
|
379
|
+
const laneBuffered = this.#bufferedByLane.get(lane) ?? {
|
|
380
|
+
items: 0,
|
|
381
|
+
bytes: 0,
|
|
382
|
+
};
|
|
383
|
+
if (laneLimit && laneBuffered.items >= laneLimit.items) {
|
|
384
|
+
return Promise.reject(
|
|
385
|
+
new RuntimeReceiptWriterCapacityError(
|
|
386
|
+
'items',
|
|
387
|
+
laneBuffered.items,
|
|
388
|
+
laneLimit.items,
|
|
389
|
+
),
|
|
390
|
+
);
|
|
391
|
+
}
|
|
392
|
+
if (laneLimit && laneBuffered.bytes + bytes > laneLimit.bytes) {
|
|
393
|
+
return Promise.reject(
|
|
394
|
+
new RuntimeReceiptWriterCapacityError(
|
|
395
|
+
'bytes',
|
|
396
|
+
laneBuffered.bytes,
|
|
397
|
+
laneLimit.bytes,
|
|
398
|
+
),
|
|
399
|
+
);
|
|
400
|
+
}
|
|
191
401
|
|
|
192
402
|
return new Promise<Output>((resolve, reject) => {
|
|
193
403
|
const pending: PendingWrite<Input, Output> = {
|
|
194
404
|
input,
|
|
195
405
|
bytes,
|
|
196
406
|
signal: options.signal ?? null,
|
|
197
|
-
state: '
|
|
407
|
+
state: 'queued',
|
|
198
408
|
aborted: null,
|
|
199
409
|
onAbort: null,
|
|
200
410
|
resolve,
|
|
@@ -205,14 +415,52 @@ export class RuntimeReceiptWriter<Input, Output> {
|
|
|
205
415
|
once: true,
|
|
206
416
|
});
|
|
207
417
|
|
|
208
|
-
|
|
209
|
-
this.#accept(pending);
|
|
210
|
-
} else {
|
|
211
|
-
this.#blocked.push(pending);
|
|
212
|
-
}
|
|
418
|
+
this.#accept(pending);
|
|
213
419
|
});
|
|
214
420
|
}
|
|
215
421
|
|
|
422
|
+
/**
|
|
423
|
+
* Wait until a rejected caller may retry its own input without making the
|
|
424
|
+
* writer retain that input. All callers share one progress promise; the
|
|
425
|
+
* caller continues to own its command and abort signal.
|
|
426
|
+
*/
|
|
427
|
+
waitForCapacity(
|
|
428
|
+
input: Input,
|
|
429
|
+
options: { signal?: AbortSignal } = {},
|
|
430
|
+
): Promise<void> {
|
|
431
|
+
if (this.#closed) {
|
|
432
|
+
return Promise.reject(
|
|
433
|
+
this.#closeReason ?? new RuntimeReceiptWriterClosedError(),
|
|
434
|
+
);
|
|
435
|
+
}
|
|
436
|
+
if (options.signal?.aborted) {
|
|
437
|
+
return Promise.reject(abortReason(options.signal));
|
|
438
|
+
}
|
|
439
|
+
|
|
440
|
+
const bytes = Math.max(0, Math.ceil(this.#estimateBytes(input)));
|
|
441
|
+
if (bytes > this.#maxBufferedBytes || bytes > this.#maxBatchBytes) {
|
|
442
|
+
return Promise.reject(
|
|
443
|
+
new RuntimeReceiptWriterBufferLimitError(
|
|
444
|
+
bytes,
|
|
445
|
+
Math.min(this.#maxBufferedBytes, this.#maxBatchBytes),
|
|
446
|
+
),
|
|
447
|
+
);
|
|
448
|
+
}
|
|
449
|
+
const lane = this.#laneFor(input);
|
|
450
|
+
const laneLimit = this.#maxBufferedByLane.get(lane);
|
|
451
|
+
if (laneLimit && bytes > laneLimit.bytes) {
|
|
452
|
+
return Promise.reject(
|
|
453
|
+
new RuntimeReceiptWriterBufferLimitError(bytes, laneLimit.bytes),
|
|
454
|
+
);
|
|
455
|
+
}
|
|
456
|
+
this.#startPump();
|
|
457
|
+
if (this.#canAccept(bytes, lane)) return Promise.resolve();
|
|
458
|
+
|
|
459
|
+
const waiter = this.#capacityWaiter ?? this.#createCapacityWaiter();
|
|
460
|
+
if (!options.signal) return waiter.promise;
|
|
461
|
+
return this.#waitForCapacityOrAbort(waiter.promise, options.signal);
|
|
462
|
+
}
|
|
463
|
+
|
|
216
464
|
async flush(): Promise<void> {
|
|
217
465
|
this.#clearFlushTimer();
|
|
218
466
|
this.#startPump();
|
|
@@ -229,6 +477,7 @@ export class RuntimeReceiptWriter<Input, Output> {
|
|
|
229
477
|
reason === undefined ? new RuntimeReceiptWriterClosedError() : reason;
|
|
230
478
|
if (reason !== undefined) {
|
|
231
479
|
this.#closeController.abort(reason);
|
|
480
|
+
this.#rejectCapacityWaiter(reason);
|
|
232
481
|
this.#rejectWaiting(reason);
|
|
233
482
|
}
|
|
234
483
|
}
|
|
@@ -236,29 +485,61 @@ export class RuntimeReceiptWriter<Input, Output> {
|
|
|
236
485
|
await this.flush();
|
|
237
486
|
return;
|
|
238
487
|
}
|
|
239
|
-
|
|
488
|
+
// A forced close must not return while any batch still holds transport.
|
|
489
|
+
while (this.#inFlight.size > 0) {
|
|
490
|
+
await Promise.allSettled([...this.#inFlight]);
|
|
491
|
+
}
|
|
240
492
|
}
|
|
241
493
|
|
|
242
494
|
getStats(): RuntimeReceiptWriterStats {
|
|
243
495
|
return {
|
|
244
|
-
active: this.#
|
|
496
|
+
active: this.#inFlight.size > 0,
|
|
497
|
+
inFlightBatches: this.#inFlight.size,
|
|
245
498
|
queued: this.#queued.length,
|
|
246
|
-
|
|
499
|
+
bufferedItems: this.#bufferedItems,
|
|
247
500
|
bufferedBytes: this.#bufferedBytes,
|
|
248
501
|
};
|
|
249
502
|
}
|
|
250
503
|
|
|
504
|
+
#notifyStatsChanged(): void {
|
|
505
|
+
if (!this.#onStatsChanged) return;
|
|
506
|
+
try {
|
|
507
|
+
this.#onStatsChanged(this.getStats());
|
|
508
|
+
} catch {
|
|
509
|
+
// Receipt observability must never affect durable delivery.
|
|
510
|
+
}
|
|
511
|
+
}
|
|
512
|
+
|
|
251
513
|
#canBuffer(bytes: number): boolean {
|
|
252
514
|
return this.#bufferedBytes + bytes <= this.#maxBufferedBytes;
|
|
253
515
|
}
|
|
254
516
|
|
|
517
|
+
#canAccept(bytes: number, lane: string): boolean {
|
|
518
|
+
if (this.#bufferedItems >= this.#maxBufferedItems) return false;
|
|
519
|
+
if (!this.#canBuffer(bytes)) return false;
|
|
520
|
+
const laneLimit = this.#maxBufferedByLane.get(lane);
|
|
521
|
+
const laneBuffered = this.#bufferedByLane.get(lane) ?? {
|
|
522
|
+
items: 0,
|
|
523
|
+
bytes: 0,
|
|
524
|
+
};
|
|
525
|
+
return (
|
|
526
|
+
!laneLimit ||
|
|
527
|
+
(laneBuffered.items < laneLimit.items &&
|
|
528
|
+
laneBuffered.bytes + bytes <= laneLimit.bytes)
|
|
529
|
+
);
|
|
530
|
+
}
|
|
531
|
+
|
|
255
532
|
#accept(pending: PendingWrite<Input, Output>): void {
|
|
256
533
|
pending.state = 'queued';
|
|
257
534
|
this.#queued.push(pending);
|
|
258
|
-
this.#
|
|
535
|
+
const lane = this.#laneFor(pending.input);
|
|
536
|
+
this.#adjustBufferedItems(1, lane);
|
|
537
|
+
this.#adjustBufferedBytes(pending.bytes);
|
|
538
|
+
this.#adjustBufferedLane(lane, { items: 0, bytes: pending.bytes });
|
|
539
|
+
this.#notifyStatsChanged();
|
|
259
540
|
if (
|
|
260
541
|
this.#queued.length >= this.#maxBatchSize ||
|
|
261
|
-
this.#
|
|
542
|
+
this.#nextBatchBytes() >= this.#targetBatchBytes
|
|
262
543
|
) {
|
|
263
544
|
this.#clearFlushTimer();
|
|
264
545
|
this.#startPump();
|
|
@@ -267,8 +548,17 @@ export class RuntimeReceiptWriter<Input, Output> {
|
|
|
267
548
|
this.#scheduleFlush();
|
|
268
549
|
}
|
|
269
550
|
|
|
551
|
+
/**
|
|
552
|
+
* Open a batching window so results settling within it share one request.
|
|
553
|
+
* The window is a fixed delay from the first queued arrival, not a resetting
|
|
554
|
+
* debounce: a continuous producer must never starve the queue. Size and byte
|
|
555
|
+
* triggers in `#accept` bypass the window entirely, and a batch that lands
|
|
556
|
+
* while the writer is saturated is picked up by the completing batch's own
|
|
557
|
+
* `#startPump`, so under load the window adds no latency.
|
|
558
|
+
*/
|
|
270
559
|
#scheduleFlush(): void {
|
|
271
|
-
if (this.#
|
|
560
|
+
if (this.#flushTimer || this.#queued.length === 0) return;
|
|
561
|
+
if (!this.#hasFlightCapacity()) return;
|
|
272
562
|
this.#flushTimer = setTimeout(() => {
|
|
273
563
|
this.#flushTimer = null;
|
|
274
564
|
this.#startPump();
|
|
@@ -276,83 +566,225 @@ export class RuntimeReceiptWriter<Input, Output> {
|
|
|
276
566
|
}
|
|
277
567
|
|
|
278
568
|
#startPump(): void {
|
|
279
|
-
|
|
280
|
-
this.#
|
|
281
|
-
|
|
569
|
+
while (this.#queued.length > 0 && this.#hasFlightCapacity()) {
|
|
570
|
+
const batch = this.#takeBatchIfOrderAllows();
|
|
571
|
+
// An empty batch means every queued receipt conflicts with an active
|
|
572
|
+
// ordering key or belongs to a lane whose bounded capacity is full.
|
|
573
|
+
if (batch.length === 0) break;
|
|
574
|
+
this.#launchBatch(batch);
|
|
282
575
|
}
|
|
283
|
-
this.#
|
|
284
|
-
this.#pump = null;
|
|
285
|
-
if (this.#queued.length > 0) this.#startPump();
|
|
286
|
-
else this.#notifyDrained();
|
|
287
|
-
});
|
|
576
|
+
if (this.#inFlight.size === 0) this.#notifyDrained();
|
|
288
577
|
}
|
|
289
578
|
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
579
|
+
#launchBatch(batch: Array<PendingWrite<Input, Output>>): void {
|
|
580
|
+
const batchId = ++this.#nextBatchId;
|
|
581
|
+
const lane = this.#laneFor(batch[0]!.input);
|
|
582
|
+
const orderingKeys = this.#orderingKeysFor(batch);
|
|
583
|
+
this.#activeOrderingKeys.push(...orderingKeys);
|
|
584
|
+
this.#incrementInFlightLane(lane);
|
|
585
|
+
// Annotated: the callback below closes over `inFlight`, so inference
|
|
586
|
+
// would otherwise be circular.
|
|
587
|
+
const inFlight: Promise<void> = this.#runBatch(batch, batchId).finally(
|
|
588
|
+
() => {
|
|
589
|
+
this.#inFlight.delete(inFlight);
|
|
590
|
+
for (const key of orderingKeys) {
|
|
591
|
+
const index = this.#activeOrderingKeys.findIndex((active) =>
|
|
592
|
+
Object.is(active, key),
|
|
302
593
|
);
|
|
594
|
+
if (index >= 0) this.#activeOrderingKeys.splice(index, 1);
|
|
303
595
|
}
|
|
304
|
-
batch.forEach((pending, index) => {
|
|
305
|
-
if (pending.aborted !== null) {
|
|
306
|
-
this.#settleRejected(pending, pending.aborted);
|
|
307
|
-
} else {
|
|
308
|
-
this.#settleResolved(pending, outputs[index]!);
|
|
309
|
-
}
|
|
310
|
-
});
|
|
311
|
-
} catch (error) {
|
|
312
|
-
batch.forEach((pending) => {
|
|
313
|
-
this.#settleRejected(
|
|
314
|
-
pending,
|
|
315
|
-
pending.aborted !== null ? pending.aborted : error,
|
|
316
|
-
);
|
|
317
|
-
});
|
|
318
|
-
} finally {
|
|
319
|
-
this.#activeBatch = null;
|
|
320
596
|
for (const pending of batch) {
|
|
321
|
-
this.#
|
|
597
|
+
this.#adjustBufferedItems(-1, this.#laneFor(pending.input));
|
|
598
|
+
this.#adjustBufferedBytes(-pending.bytes);
|
|
599
|
+
this.#adjustBufferedLane(this.#laneFor(pending.input), {
|
|
600
|
+
items: 0,
|
|
601
|
+
bytes: -pending.bytes,
|
|
602
|
+
});
|
|
322
603
|
}
|
|
323
|
-
this.#
|
|
604
|
+
this.#decrementInFlightLane(lane);
|
|
605
|
+
this.#notifyStatsChanged();
|
|
606
|
+
this.#resolveCapacityWaiter();
|
|
607
|
+
if (this.#queued.length > 0) this.#startPump();
|
|
608
|
+
else if (this.#inFlight.size === 0) this.#notifyDrained();
|
|
609
|
+
},
|
|
610
|
+
);
|
|
611
|
+
this.#inFlight.add(inFlight);
|
|
612
|
+
this.#notifyStatsChanged();
|
|
613
|
+
}
|
|
614
|
+
|
|
615
|
+
async #runBatch(
|
|
616
|
+
batch: Array<PendingWrite<Input, Output>>,
|
|
617
|
+
batchId: number,
|
|
618
|
+
): Promise<void> {
|
|
619
|
+
try {
|
|
620
|
+
const outputs = await this.#sendWithRetry(batch, batchId);
|
|
621
|
+
if (outputs.length !== batch.length) {
|
|
622
|
+
throw new RuntimeReceiptWriterResultCountError(
|
|
623
|
+
batch.length,
|
|
624
|
+
outputs.length,
|
|
625
|
+
);
|
|
324
626
|
}
|
|
627
|
+
batch.forEach((pending, index) => {
|
|
628
|
+
if (pending.aborted !== null) {
|
|
629
|
+
this.#settleRejected(pending, pending.aborted);
|
|
630
|
+
} else {
|
|
631
|
+
this.#settleResolved(pending, outputs[index]!);
|
|
632
|
+
}
|
|
633
|
+
});
|
|
634
|
+
} catch (error) {
|
|
635
|
+
batch.forEach((pending) => {
|
|
636
|
+
this.#settleRejected(
|
|
637
|
+
pending,
|
|
638
|
+
pending.aborted !== null ? pending.aborted : error,
|
|
639
|
+
);
|
|
640
|
+
});
|
|
325
641
|
}
|
|
326
642
|
}
|
|
327
643
|
|
|
328
|
-
#
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
644
|
+
#takeBatchIfOrderAllows(): Array<PendingWrite<Input, Output>> {
|
|
645
|
+
this.#rejectAbortedQueued();
|
|
646
|
+
const indices = this.#selectRunnableBatchIndices();
|
|
647
|
+
if (indices.length === 0) return [];
|
|
648
|
+
const batch = indices.map((index) => this.#queued[index]!);
|
|
649
|
+
for (let index = indices.length - 1; index >= 0; index -= 1) {
|
|
650
|
+
this.#queued.splice(indices[index]!, 1);
|
|
334
651
|
}
|
|
335
|
-
const
|
|
336
|
-
|
|
652
|
+
for (const pending of batch) pending.state = 'active';
|
|
653
|
+
return batch;
|
|
654
|
+
}
|
|
337
655
|
|
|
338
|
-
|
|
339
|
-
|
|
656
|
+
/**
|
|
657
|
+
* Select the first FIFO-ready batch. A command blocked only because its
|
|
658
|
+
* receipt is active may be bypassed so an independent receipt can use a
|
|
659
|
+
* free flight. A normally ready command is never bypassed: that keeps the
|
|
660
|
+
* historical FIFO ordering for callers that did not provide ordering keys.
|
|
661
|
+
*/
|
|
662
|
+
#selectRunnableBatchIndices(): number[] {
|
|
663
|
+
const skippedOrderingKeys: unknown[] = [];
|
|
664
|
+
const selected: number[] = [];
|
|
665
|
+
let selectedBatchKey: unknown;
|
|
666
|
+
let selectedLane: string | null = null;
|
|
340
667
|
let batchBytes = 0;
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
668
|
+
|
|
669
|
+
for (let index = 0; index < this.#queued.length; index += 1) {
|
|
670
|
+
const pending = this.#queued[index]!;
|
|
671
|
+
if (pending.aborted !== null) continue;
|
|
672
|
+
const orderingKeys = this.#orderingKeys(pending.input);
|
|
673
|
+
const blockedByOrder = this.#orderingKeysConflict(
|
|
674
|
+
orderingKeys,
|
|
675
|
+
skippedOrderingKeys,
|
|
676
|
+
);
|
|
677
|
+
const blockedByActive = this.#orderingKeysConflict(
|
|
678
|
+
orderingKeys,
|
|
679
|
+
this.#activeOrderingKeys,
|
|
680
|
+
);
|
|
681
|
+
const lane = this.#laneFor(pending.input);
|
|
682
|
+
const laneAvailable = this.#hasFlightCapacity(lane);
|
|
683
|
+
|
|
684
|
+
if (selected.length === 0) {
|
|
685
|
+
if (blockedByOrder || blockedByActive || !laneAvailable) {
|
|
686
|
+
skippedOrderingKeys.push(...orderingKeys);
|
|
687
|
+
continue;
|
|
688
|
+
}
|
|
689
|
+
selected.push(index);
|
|
690
|
+
selectedBatchKey = this.#batchKey(pending.input);
|
|
691
|
+
selectedLane = lane;
|
|
692
|
+
batchBytes = pending.bytes;
|
|
693
|
+
continue;
|
|
694
|
+
}
|
|
695
|
+
|
|
696
|
+
// A ready, incompatible command is an ordinary FIFO boundary. It must
|
|
697
|
+
// run before any later command, even one that could share this batch.
|
|
698
|
+
if (blockedByOrder || blockedByActive || !laneAvailable) {
|
|
699
|
+
skippedOrderingKeys.push(...orderingKeys);
|
|
700
|
+
continue;
|
|
701
|
+
}
|
|
344
702
|
if (
|
|
345
|
-
|
|
346
|
-
|
|
703
|
+
selectedLane !== lane ||
|
|
704
|
+
!Object.is(this.#batchKey(pending.input), selectedBatchKey) ||
|
|
705
|
+
selected.length >= this.#maxBatchSize ||
|
|
706
|
+
batchBytes + pending.bytes > this.#targetBatchBytes
|
|
347
707
|
) {
|
|
348
708
|
break;
|
|
349
709
|
}
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
batch.push(next);
|
|
353
|
-
batchBytes += next.bytes;
|
|
710
|
+
selected.push(index);
|
|
711
|
+
batchBytes += pending.bytes;
|
|
354
712
|
}
|
|
355
|
-
return
|
|
713
|
+
return selected;
|
|
714
|
+
}
|
|
715
|
+
|
|
716
|
+
#rejectAbortedQueued(): void {
|
|
717
|
+
let changed = false;
|
|
718
|
+
for (let index = this.#queued.length - 1; index >= 0; index -= 1) {
|
|
719
|
+
const pending = this.#queued[index]!;
|
|
720
|
+
if (pending.aborted === null) continue;
|
|
721
|
+
this.#queued.splice(index, 1);
|
|
722
|
+
this.#adjustBufferedItems(-1, this.#laneFor(pending.input));
|
|
723
|
+
this.#adjustBufferedBytes(-pending.bytes);
|
|
724
|
+
this.#adjustBufferedLane(this.#laneFor(pending.input), {
|
|
725
|
+
items: 0,
|
|
726
|
+
bytes: -pending.bytes,
|
|
727
|
+
});
|
|
728
|
+
this.#settleRejected(pending, pending.aborted);
|
|
729
|
+
changed = true;
|
|
730
|
+
}
|
|
731
|
+
if (changed) this.#notifyStatsChanged();
|
|
732
|
+
}
|
|
733
|
+
|
|
734
|
+
#nextBatchBytes(): number {
|
|
735
|
+
this.#rejectAbortedQueued();
|
|
736
|
+
return this.#selectRunnableBatchIndices().reduce(
|
|
737
|
+
(total, index) => total + this.#queued[index]!.bytes,
|
|
738
|
+
0,
|
|
739
|
+
);
|
|
740
|
+
}
|
|
741
|
+
|
|
742
|
+
#orderingKeysConflict(
|
|
743
|
+
candidate: readonly unknown[],
|
|
744
|
+
existing: readonly unknown[],
|
|
745
|
+
): boolean {
|
|
746
|
+
return candidate.some((key) =>
|
|
747
|
+
existing.some((active) => Object.is(active, key)),
|
|
748
|
+
);
|
|
749
|
+
}
|
|
750
|
+
|
|
751
|
+
#laneFor(input: Input): string {
|
|
752
|
+
return normalizeLane(this.#batchLane(input));
|
|
753
|
+
}
|
|
754
|
+
|
|
755
|
+
#hasFlightCapacity(lane?: string): boolean {
|
|
756
|
+
if (this.#inFlight.size >= this.#maxInFlightBatches) return false;
|
|
757
|
+
if (lane === undefined) return true;
|
|
758
|
+
const maximum = this.#maxInFlightBatchesByLane.get(lane);
|
|
759
|
+
return (
|
|
760
|
+
maximum === undefined ||
|
|
761
|
+
(this.#inFlightBatchesByLane.get(lane) ?? 0) < maximum
|
|
762
|
+
);
|
|
763
|
+
}
|
|
764
|
+
|
|
765
|
+
#incrementInFlightLane(lane: string): void {
|
|
766
|
+
this.#inFlightBatchesByLane.set(
|
|
767
|
+
lane,
|
|
768
|
+
(this.#inFlightBatchesByLane.get(lane) ?? 0) + 1,
|
|
769
|
+
);
|
|
770
|
+
}
|
|
771
|
+
|
|
772
|
+
#decrementInFlightLane(lane: string): void {
|
|
773
|
+
const current = this.#inFlightBatchesByLane.get(lane) ?? 0;
|
|
774
|
+
if (current <= 1) this.#inFlightBatchesByLane.delete(lane);
|
|
775
|
+
else this.#inFlightBatchesByLane.set(lane, current - 1);
|
|
776
|
+
}
|
|
777
|
+
|
|
778
|
+
#orderingKeysFor(batch: readonly PendingWrite<Input, Output>[]): unknown[] {
|
|
779
|
+
const keys: unknown[] = [];
|
|
780
|
+
for (const pending of batch) {
|
|
781
|
+
for (const key of this.#orderingKeys(pending.input)) {
|
|
782
|
+
if (!keys.some((existing) => Object.is(existing, key))) {
|
|
783
|
+
keys.push(key);
|
|
784
|
+
}
|
|
785
|
+
}
|
|
786
|
+
}
|
|
787
|
+
return keys;
|
|
356
788
|
}
|
|
357
789
|
|
|
358
790
|
async #sendWithRetry(
|
|
@@ -445,7 +877,7 @@ export class RuntimeReceiptWriter<Input, Output> {
|
|
|
445
877
|
elapsedMs: Date.now() - input.startedAt,
|
|
446
878
|
retryAfterMs: input.retryAfterMs,
|
|
447
879
|
queued: this.#queued.length,
|
|
448
|
-
|
|
880
|
+
bufferedItems: this.#bufferedItems,
|
|
449
881
|
bufferedBytes: this.#bufferedBytes,
|
|
450
882
|
...(input.error === undefined ? {} : { error: input.error }),
|
|
451
883
|
});
|
|
@@ -501,32 +933,23 @@ export class RuntimeReceiptWriter<Input, Output> {
|
|
|
501
933
|
pending.aborted = abortReason(pending.signal);
|
|
502
934
|
if (pending.state === 'active') return;
|
|
503
935
|
|
|
504
|
-
const collection =
|
|
505
|
-
pending.state === 'queued' ? this.#queued : this.#blocked;
|
|
936
|
+
const collection = pending.state === 'queued' ? this.#queued : [];
|
|
506
937
|
const index = collection.indexOf(pending);
|
|
507
938
|
if (index >= 0) collection.splice(index, 1);
|
|
508
939
|
if (pending.state === 'queued') {
|
|
509
|
-
this.#
|
|
510
|
-
this.#
|
|
940
|
+
this.#adjustBufferedItems(-1, this.#laneFor(pending.input));
|
|
941
|
+
this.#adjustBufferedBytes(-pending.bytes);
|
|
942
|
+
this.#adjustBufferedLane(this.#laneFor(pending.input), {
|
|
943
|
+
items: 0,
|
|
944
|
+
bytes: -pending.bytes,
|
|
945
|
+
});
|
|
946
|
+
this.#notifyStatsChanged();
|
|
511
947
|
}
|
|
512
948
|
this.#settleRejected(pending, pending.aborted);
|
|
949
|
+
this.#resolveCapacityWaiter();
|
|
513
950
|
this.#notifyDrained();
|
|
514
951
|
}
|
|
515
952
|
|
|
516
|
-
#admitBlocked(): void {
|
|
517
|
-
while (this.#blocked.length > 0) {
|
|
518
|
-
const pending = this.#blocked[0]!;
|
|
519
|
-
if (pending.aborted !== null) {
|
|
520
|
-
this.#blocked.shift();
|
|
521
|
-
this.#settleRejected(pending, pending.aborted);
|
|
522
|
-
continue;
|
|
523
|
-
}
|
|
524
|
-
if (!this.#canBuffer(pending.bytes)) break;
|
|
525
|
-
this.#blocked.shift();
|
|
526
|
-
this.#accept(pending);
|
|
527
|
-
}
|
|
528
|
-
}
|
|
529
|
-
|
|
530
953
|
#settleResolved(pending: PendingWrite<Input, Output>, output: Output): void {
|
|
531
954
|
if (pending.state === 'settled') return;
|
|
532
955
|
this.#detachAbort(pending);
|
|
@@ -549,41 +972,70 @@ export class RuntimeReceiptWriter<Input, Output> {
|
|
|
549
972
|
|
|
550
973
|
#rejectWaiting(reason: unknown): void {
|
|
551
974
|
this.#clearFlushTimer();
|
|
975
|
+
let changed = false;
|
|
552
976
|
for (const pending of this.#queued.splice(0)) {
|
|
553
|
-
this.#
|
|
554
|
-
this.#
|
|
555
|
-
|
|
556
|
-
|
|
977
|
+
this.#adjustBufferedItems(-1, this.#laneFor(pending.input));
|
|
978
|
+
this.#adjustBufferedBytes(-pending.bytes);
|
|
979
|
+
this.#adjustBufferedLane(this.#laneFor(pending.input), {
|
|
980
|
+
items: 0,
|
|
981
|
+
bytes: -pending.bytes,
|
|
982
|
+
});
|
|
557
983
|
this.#settleRejected(pending, reason);
|
|
984
|
+
changed = true;
|
|
558
985
|
}
|
|
986
|
+
if (changed) this.#notifyStatsChanged();
|
|
987
|
+
this.#resolveCapacityWaiter();
|
|
559
988
|
this.#notifyDrained(reason);
|
|
560
989
|
}
|
|
561
990
|
|
|
562
|
-
#
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
const
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
991
|
+
#createCapacityWaiter(): CapacityWaiter {
|
|
992
|
+
let resolve!: () => void;
|
|
993
|
+
let reject!: (error: unknown) => void;
|
|
994
|
+
const promise = new Promise<void>((resolvePromise, rejectPromise) => {
|
|
995
|
+
resolve = resolvePromise;
|
|
996
|
+
reject = rejectPromise;
|
|
997
|
+
});
|
|
998
|
+
const waiter = { promise, resolve, reject };
|
|
999
|
+
this.#capacityWaiter = waiter;
|
|
1000
|
+
return waiter;
|
|
1001
|
+
}
|
|
1002
|
+
|
|
1003
|
+
#resolveCapacityWaiter(): void {
|
|
1004
|
+
const waiter = this.#capacityWaiter;
|
|
1005
|
+
if (!waiter) return;
|
|
1006
|
+
this.#capacityWaiter = null;
|
|
1007
|
+
waiter.resolve();
|
|
1008
|
+
}
|
|
1009
|
+
|
|
1010
|
+
#rejectCapacityWaiter(reason: unknown): void {
|
|
1011
|
+
const waiter = this.#capacityWaiter;
|
|
1012
|
+
if (!waiter) return;
|
|
1013
|
+
this.#capacityWaiter = null;
|
|
1014
|
+
waiter.reject(reason);
|
|
1015
|
+
}
|
|
1016
|
+
|
|
1017
|
+
async #waitForCapacityOrAbort(
|
|
1018
|
+
capacity: Promise<void>,
|
|
1019
|
+
signal: AbortSignal,
|
|
1020
|
+
): Promise<void> {
|
|
1021
|
+
await new Promise<void>((resolve, reject) => {
|
|
1022
|
+
const finish = (error?: unknown): void => {
|
|
1023
|
+
signal.removeEventListener('abort', onAbort);
|
|
1024
|
+
if (error === undefined) resolve();
|
|
1025
|
+
else reject(error);
|
|
1026
|
+
};
|
|
1027
|
+
const onAbort = () => finish(abortReason(signal));
|
|
1028
|
+
capacity.then(
|
|
1029
|
+
() => finish(),
|
|
1030
|
+
(error) => finish(error),
|
|
1031
|
+
);
|
|
1032
|
+
signal.addEventListener('abort', onAbort, { once: true });
|
|
1033
|
+
if (signal.aborted) onAbort();
|
|
1034
|
+
});
|
|
579
1035
|
}
|
|
580
1036
|
|
|
581
1037
|
#isDrained(): boolean {
|
|
582
|
-
return
|
|
583
|
-
this.#activeBatch === null &&
|
|
584
|
-
this.#queued.length === 0 &&
|
|
585
|
-
this.#blocked.length === 0
|
|
586
|
-
);
|
|
1038
|
+
return this.#inFlight.size === 0 && this.#queued.length === 0;
|
|
587
1039
|
}
|
|
588
1040
|
|
|
589
1041
|
#notifyDrained(error?: unknown): void {
|
|
@@ -602,6 +1054,32 @@ export class RuntimeReceiptWriter<Input, Output> {
|
|
|
602
1054
|
}
|
|
603
1055
|
}
|
|
604
1056
|
|
|
1057
|
+
/**
|
|
1058
|
+
* Submit one idempotent receipt command without turning local admission
|
|
1059
|
+
* pressure into either an unbounded writer queue or a persistence failure.
|
|
1060
|
+
*
|
|
1061
|
+
* The caller retains `input` while it waits on the writer's shared progress
|
|
1062
|
+
* signal. This is deliberately shared by the receipt-store Adapter and
|
|
1063
|
+
* Context's direct completion writer so all runtime receipt paths apply the
|
|
1064
|
+
* same bounded backpressure rule.
|
|
1065
|
+
*/
|
|
1066
|
+
export async function writeRuntimeReceiptWithBackpressure<Input, Output>(
|
|
1067
|
+
writer: RuntimeReceiptWriter<Input, Output>,
|
|
1068
|
+
input: Input,
|
|
1069
|
+
options: { signal?: AbortSignal } = {},
|
|
1070
|
+
): Promise<Output> {
|
|
1071
|
+
while (true) {
|
|
1072
|
+
try {
|
|
1073
|
+
return await writer.write(input, options);
|
|
1074
|
+
} catch (error) {
|
|
1075
|
+
if (!(error instanceof RuntimeReceiptWriterCapacityError)) {
|
|
1076
|
+
throw error;
|
|
1077
|
+
}
|
|
1078
|
+
await writer.waitForCapacity(input, options);
|
|
1079
|
+
}
|
|
1080
|
+
}
|
|
1081
|
+
}
|
|
1082
|
+
|
|
605
1083
|
const textEncoder = new TextEncoder();
|
|
606
1084
|
|
|
607
1085
|
function estimateJsonBytes(input: unknown): number {
|
|
@@ -616,6 +1094,14 @@ function normalizePositiveInteger(
|
|
|
616
1094
|
return Math.max(1, Math.floor(value ?? fallback));
|
|
617
1095
|
}
|
|
618
1096
|
|
|
1097
|
+
function normalizeLane(value: string): string {
|
|
1098
|
+
const lane = value.trim();
|
|
1099
|
+
if (!lane) {
|
|
1100
|
+
throw new Error('Runtime receipt writer batch lanes must be non-empty.');
|
|
1101
|
+
}
|
|
1102
|
+
return lane;
|
|
1103
|
+
}
|
|
1104
|
+
|
|
619
1105
|
function jitteredRetryDelayMs(delayMs: number): number {
|
|
620
1106
|
const normalized = Math.max(0, Math.floor(delayMs));
|
|
621
1107
|
if (normalized === 0) return 0;
|