deepline 0.3.128 → 0.3.129
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/sdk/src/client.ts +34 -9
- package/dist/bundling-sources/sdk/src/http.ts +22 -0
- package/dist/bundling-sources/sdk/src/release.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-data-plane/r2.ts +22 -0
- package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +265 -2
- package/dist/bundling-sources/shared_libs/play-runtime/bounded-batch-function.ts +682 -0
- package/dist/bundling-sources/shared_libs/play-runtime/child-execution-strategy.ts +21 -5
- package/dist/bundling-sources/shared_libs/play-runtime/context.ts +5390 -1732
- package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +107 -1
- package/dist/bundling-sources/shared_libs/play-runtime/deployment/commands/roll-execution-relay.ts +7 -0
- package/dist/bundling-sources/shared_libs/play-runtime/deployment/fly-machines.ts +229 -7
- package/dist/bundling-sources/shared_libs/play-runtime/deployment/fly-registry.ts +25 -4
- package/dist/bundling-sources/shared_libs/play-runtime/deployment/runtime-gateway-capacity.ts +71 -0
- package/dist/bundling-sources/shared_libs/play-runtime/deployment/runtime-gateway-drain.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/deployment/runtime-topology.ts +4 -1
- package/dist/bundling-sources/shared_libs/play-runtime/durable-receipt-execution.ts +694 -28
- package/dist/bundling-sources/shared_libs/play-runtime/egress-bandwidth-probe.ts +167 -0
- package/dist/bundling-sources/shared_libs/play-runtime/execute-server-timing.ts +103 -0
- package/dist/bundling-sources/shared_libs/play-runtime/execution-gateway-receipt-output-reference.ts +34 -0
- package/dist/bundling-sources/shared_libs/play-runtime/execution-gateway-response-size.ts +27 -0
- package/dist/bundling-sources/shared_libs/play-runtime/gateway-postgres-admission.ts +66 -1
- package/dist/bundling-sources/shared_libs/play-runtime/governor/app-runtime-rate-state-backend.ts +425 -71
- package/dist/bundling-sources/shared_libs/play-runtime/governor/governor.ts +210 -32
- package/dist/bundling-sources/shared_libs/play-runtime/governor/in-memory-rate-state-backend.ts +27 -13
- package/dist/bundling-sources/shared_libs/play-runtime/governor/policy.ts +19 -6
- package/dist/bundling-sources/shared_libs/play-runtime/governor/rate-state-backend.ts +61 -2
- package/dist/bundling-sources/shared_libs/play-runtime/integration-operation-reader-fence/contracts.ts +33 -1
- package/dist/bundling-sources/shared_libs/play-runtime/integration-operation-reader-fence/job.ts +42 -1
- package/dist/bundling-sources/shared_libs/play-runtime/launch-source-policy.ts +11 -11
- package/dist/bundling-sources/shared_libs/play-runtime/ledger-safe-payload.ts +49 -16
- package/dist/bundling-sources/shared_libs/play-runtime/map-memory-limits.ts +1653 -98
- package/dist/bundling-sources/shared_libs/play-runtime/memory-ledger.ts +807 -0
- package/dist/bundling-sources/shared_libs/play-runtime/memory-pressure-signal.ts +138 -0
- package/dist/bundling-sources/shared_libs/play-runtime/output-size-limits.ts +85 -3
- package/dist/bundling-sources/shared_libs/play-runtime/persistence-latch.ts +60 -2
- package/dist/bundling-sources/shared_libs/play-runtime/play-run-recovery-policy.ts +46 -1
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/terminal/contracts.ts +2 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/terminal/job.ts +1 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/terminal/ports.ts +1 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/terminal/postgres-adapter.ts +1 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runtime-semantics.ts +70 -27
- package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts +56 -0
- package/dist/bundling-sources/shared_libs/play-runtime/provider-capacity/adaptive-provider-capacity-controller.ts +192 -0
- package/dist/bundling-sources/shared_libs/play-runtime/provider-capacity/epoch-aimd-rate-controller.ts +194 -0
- package/dist/bundling-sources/shared_libs/play-runtime/provider-capacity/fixture-capacity-simulator.ts +220 -0
- package/dist/bundling-sources/shared_libs/play-runtime/provider-capacity/gradient2-concurrency-controller.ts +214 -0
- package/dist/bundling-sources/shared_libs/play-runtime/provider-capacity/provider-capacity-policy.ts +43 -0
- package/dist/bundling-sources/shared_libs/play-runtime/provider-capacity/provider-concurrency-policy.ts +30 -0
- package/dist/bundling-sources/shared_libs/play-runtime/provider-capacity/provider-rate-policy.ts +95 -0
- package/dist/bundling-sources/shared_libs/play-runtime/provider-capacity/provider-throttle-classifier.ts +40 -0
- package/dist/bundling-sources/shared_libs/play-runtime/resource-governor.ts +1535 -11
- package/dist/bundling-sources/shared_libs/play-runtime/run-failure.ts +49 -2
- package/dist/bundling-sources/shared_libs/play-runtime/run-ledger.ts +6 -1
- package/dist/bundling-sources/shared_libs/play-runtime/run-log-buffer.ts +55 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-app/egress-probe.ts +221 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-app/index.ts +339 -29
- package/dist/bundling-sources/shared_libs/play-runtime/runner-app/runtime-deadline.ts +28 -1
- package/dist/bundling-sources/shared_libs/play-runtime/runner-app/runtime-progress-push.ts +192 -165
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-lifecycle.ts +6 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-payload-transport.ts +90 -5
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-runtime-watchdog.ts +281 -6
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts +27 -1
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/local-process.ts +19 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/modal.ts +108 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/runtime-sandbox-reconciliation.ts +40 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/types.ts +11 -5
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-actions.ts +30 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-api-compression.ts +276 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +723 -115
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-capacity-policy.ts +98 -2
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-contract.ts +3 -2
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-environment.ts +41 -4
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-flow.ts +448 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-map-row-feed.ts +215 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-operation-contract.ts +11 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-receipt-store-adapter.ts +161 -14
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-receipt-value.ts +184 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-receipt-writer.ts +607 -121
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-sandbox-placement-policy.ts +4 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-sheet-row-writer.ts +21 -0
- package/dist/bundling-sources/shared_libs/play-runtime/sandbox-runs/contract.ts +15 -2
- package/dist/bundling-sources/shared_libs/play-runtime/sandbox-runs/runner-backend-adapter.ts +71 -14
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-play-deploy-task.ts +2 -2
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-runtime-worker.ts +288 -119
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-shared.ts +8 -6
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-worker-required-env.ts +28 -3
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/canonical-integration-operation-store.ts +179 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/gateway-heavy-request-admission.ts +80 -2
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/gateway-progress-registry.ts +31 -2
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/integration-operation-store-contract.ts +104 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/integration-operation-value-reference.ts +47 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/integration-operation-value-retirement.ts +193 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/integration-operation-value-storage.ts +420 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-pool.ts +41 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-progress.ts +134 -3
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-rate-state.ts +1657 -95
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts +885 -73
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/receipt-completion-coalescer.ts +452 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/receipt-gateway.ts +601 -42
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/run-terminal-transition.ts +1 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-receipt-value-storage.ts +371 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/worker-resource-metrics.ts +86 -7
- package/dist/bundling-sources/shared_libs/play-runtime/secret-redaction.ts +153 -22
- package/dist/bundling-sources/shared_libs/play-runtime/tool-call/index.ts +1 -0
- package/dist/bundling-sources/shared_libs/play-runtime/tool-call/receipt-cohort.ts +188 -38
- package/dist/bundling-sources/shared_libs/play-runtime/tool-call/runtime-step-receipts-adapter.ts +28 -13
- package/dist/bundling-sources/shared_libs/play-runtime/tool-execute-retry-policy.ts +44 -1
- package/dist/bundling-sources/shared_libs/play-runtime/tool-result.ts +56 -4
- package/dist/bundling-sources/shared_libs/play-runtime/work-receipts.ts +2 -0
- package/dist/bundling-sources/shared_libs/plays/authoring-contract.ts +16 -10
- package/dist/bundling-sources/shared_libs/plays/dataset.ts +4 -1
- package/dist/cli/index.js +250 -38
- package/dist/cli/index.mjs +250 -38
- package/dist/{compiler-manifest-DfSnbCTW.d.mts → compiler-manifest-BIqRyj5m.d.mts} +1 -1
- package/dist/{compiler-manifest-DfSnbCTW.d.ts → compiler-manifest-BIqRyj5m.d.ts} +1 -1
- package/dist/index.d.mts +4 -2
- package/dist/index.d.ts +4 -2
- package/dist/index.js +231 -35
- package/dist/index.mjs +231 -35
- package/dist/install-integrity.json +31 -2
- package/dist/plays/bundle-play-file.d.mts +2 -2
- package/dist/plays/bundle-play-file.d.ts +2 -2
- package/dist/plays/bundle-play-file.mjs +65 -1
- package/dist/release.d.mts +1 -1
- package/dist/release.d.ts +1 -1
- package/dist/release.js +1 -1
- package/dist/release.mjs +1 -1
- package/package.json +1 -1
|
@@ -20,7 +20,13 @@ import {
|
|
|
20
20
|
resolveMaxConcurrentExternalCalls,
|
|
21
21
|
resolveRowConcurrency,
|
|
22
22
|
} from './policy';
|
|
23
|
-
import {
|
|
23
|
+
import {
|
|
24
|
+
type PacingPermit,
|
|
25
|
+
type PacingRule,
|
|
26
|
+
type ProviderConcurrencyGrant,
|
|
27
|
+
type ProviderPacingGrant,
|
|
28
|
+
type RateStateBackend,
|
|
29
|
+
} from './rate-state-backend';
|
|
24
30
|
import {
|
|
25
31
|
createInMemoryAdaptiveAdmission,
|
|
26
32
|
type RuntimeAdaptiveAdmission,
|
|
@@ -36,6 +42,30 @@ export interface WorkLease {
|
|
|
36
42
|
release(): void;
|
|
37
43
|
}
|
|
38
44
|
|
|
45
|
+
/** A provider admission that may be returned only before its first fetch. */
|
|
46
|
+
export interface ProviderPermit extends WorkLease {
|
|
47
|
+
/** Canonical provider bucket that admitted this physical attempt. */
|
|
48
|
+
readonly provider: string;
|
|
49
|
+
/** Server-authored attribution context for a later provider 429. */
|
|
50
|
+
readonly concurrencyGrant?: ProviderConcurrencyGrant | null;
|
|
51
|
+
/** Effective rate-controller reading for this physical provider permit. */
|
|
52
|
+
readonly pacingGrant?: ProviderPacingGrant | null;
|
|
53
|
+
/**
|
|
54
|
+
* Return an acquired provider permit before an outbound request started.
|
|
55
|
+
* Unlike `release`, this returns the rate reservation as well as any
|
|
56
|
+
* concurrency lease. It is deliberately required at this boundary so a
|
|
57
|
+
* caller cannot silently burn a ticket while re-admitting after local work.
|
|
58
|
+
*/
|
|
59
|
+
cancel(): Promise<void>;
|
|
60
|
+
/**
|
|
61
|
+
* Return a due permit that lost a local capacity race so the next
|
|
62
|
+
* `acquireProviderPermit` for the same lane draws it locally. Frees the
|
|
63
|
+
* local concurrency slot like `cancel`, but keeps the rate ticket in the
|
|
64
|
+
* backend's lease block when that block is still live; otherwise refunds.
|
|
65
|
+
*/
|
|
66
|
+
park(): Promise<void>;
|
|
67
|
+
}
|
|
68
|
+
|
|
39
69
|
export type BudgetKind = 'toolCall' | 'retry' | 'waterfallStep';
|
|
40
70
|
|
|
41
71
|
/** Runtime identity and counters shared by inline composition. */
|
|
@@ -113,6 +143,16 @@ export interface PlayExecutionGovernor {
|
|
|
113
143
|
opts?: { signal?: AbortSignal },
|
|
114
144
|
): Promise<WorkLease>;
|
|
115
145
|
|
|
146
|
+
/** Acquire only local external-execution capacity; this never charges a budget. */
|
|
147
|
+
acquireToolExecutionSlot(opts?: { signal?: AbortSignal }): Promise<WorkLease>;
|
|
148
|
+
|
|
149
|
+
/**
|
|
150
|
+
* Take immediately available local external-execution capacity, if any.
|
|
151
|
+
* This is deliberately non-blocking: a provider ticket that is already due
|
|
152
|
+
* must be re-admitted instead of aging behind a local queue.
|
|
153
|
+
*/
|
|
154
|
+
tryAcquireToolExecutionSlot(): WorkLease | null;
|
|
155
|
+
|
|
116
156
|
/**
|
|
117
157
|
* Admit one brief runtime-to-app HTTP fetch/body exchange. Long provider or
|
|
118
158
|
* fixture residence happens before this edge and must not occupy this slot.
|
|
@@ -121,6 +161,12 @@ export interface PlayExecutionGovernor {
|
|
|
121
161
|
signal?: AbortSignal;
|
|
122
162
|
}): Promise<WorkLease>;
|
|
123
163
|
|
|
164
|
+
/**
|
|
165
|
+
* Take a runtime-to-app transport slot only if one is immediately ready.
|
|
166
|
+
* A due provider permit must re-admit rather than age behind this queue.
|
|
167
|
+
*/
|
|
168
|
+
tryAcquireIntegrationRequestSlot(): WorkLease | null;
|
|
169
|
+
|
|
124
170
|
/**
|
|
125
171
|
* Block until one physical provider HTTP attempt is admitted under the
|
|
126
172
|
* provider's shared rate/concurrency rules. Call this immediately before the
|
|
@@ -129,7 +175,16 @@ export interface PlayExecutionGovernor {
|
|
|
129
175
|
acquireProviderPermit(
|
|
130
176
|
toolId: string,
|
|
131
177
|
opts?: { signal?: AbortSignal },
|
|
132
|
-
): Promise<
|
|
178
|
+
): Promise<ProviderPermit>;
|
|
179
|
+
|
|
180
|
+
/**
|
|
181
|
+
* Admit one physical provider attempt only from capacity this process
|
|
182
|
+
* already holds: the tool's local concurrency slot and a due permit in the
|
|
183
|
+
* rate backend's local state. Synchronous and never remote, so a caller may
|
|
184
|
+
* probe it while holding an execution slot. Null until a blocking
|
|
185
|
+
* `acquireProviderPermit` has resolved the tool's pacing at least once.
|
|
186
|
+
*/
|
|
187
|
+
tryAcquireProviderPermit(toolId: string): ProviderPermit | null;
|
|
133
188
|
|
|
134
189
|
/**
|
|
135
190
|
* Suggested batch parallelism for a tool: the provider's own rate hints
|
|
@@ -171,6 +226,10 @@ export interface PlayExecutionGovernor {
|
|
|
171
226
|
*/
|
|
172
227
|
toolId?: string;
|
|
173
228
|
retryAfterMs: number;
|
|
229
|
+
/** Provider residence for the rejected physical request, when observed. */
|
|
230
|
+
latencyMs?: number | null;
|
|
231
|
+
/** Server-authored context for attributing this physical provider 429. */
|
|
232
|
+
concurrencyGrant?: ProviderConcurrencyGrant | null;
|
|
174
233
|
}): Promise<void>;
|
|
175
234
|
|
|
176
235
|
/** Feed clean provider calls back into adaptive admission. */
|
|
@@ -203,9 +262,37 @@ interface GovernorInput {
|
|
|
203
262
|
|
|
204
263
|
class Semaphore {
|
|
205
264
|
private inFlight = 0;
|
|
206
|
-
private readonly waiters: Array<() => void> = [];
|
|
265
|
+
private readonly waiters: Array<(reserved: boolean) => void> = [];
|
|
207
266
|
constructor(private readonly limit: number) {}
|
|
208
267
|
|
|
268
|
+
private lease(): WorkLease {
|
|
269
|
+
this.inFlight += 1;
|
|
270
|
+
return this.leased();
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
/** Return a lease for capacity already reserved by a waiter handoff. */
|
|
274
|
+
private leased(): WorkLease {
|
|
275
|
+
let released = false;
|
|
276
|
+
return {
|
|
277
|
+
release: () => {
|
|
278
|
+
if (released) return;
|
|
279
|
+
released = true;
|
|
280
|
+
this.inFlight = Math.max(0, this.inFlight - 1);
|
|
281
|
+
const next = this.waiters.shift();
|
|
282
|
+
if (next) {
|
|
283
|
+
// Reserve before waking. A synchronous tryAcquire between this
|
|
284
|
+
// release and the promise continuation must not overtake FIFO work.
|
|
285
|
+
this.inFlight += 1;
|
|
286
|
+
next(true);
|
|
287
|
+
}
|
|
288
|
+
},
|
|
289
|
+
};
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
tryAcquire(): WorkLease | null {
|
|
293
|
+
return this.inFlight >= this.limit ? null : this.lease();
|
|
294
|
+
}
|
|
295
|
+
|
|
209
296
|
async acquire(signal?: AbortSignal): Promise<WorkLease> {
|
|
210
297
|
// Fail fast on an already-aborted signal: the parked-promise abort listener
|
|
211
298
|
// below registers with { once: true } and never fires for a signal that was
|
|
@@ -217,10 +304,10 @@ class Semaphore {
|
|
|
217
304
|
: new Error('Slot acquire aborted.');
|
|
218
305
|
}
|
|
219
306
|
while (this.inFlight >= this.limit) {
|
|
220
|
-
await new Promise<
|
|
221
|
-
const onResolve = () => {
|
|
307
|
+
const reserved = await new Promise<boolean>((resolve, reject) => {
|
|
308
|
+
const onResolve = (wasReserved: boolean) => {
|
|
222
309
|
signal?.removeEventListener('abort', onAbort);
|
|
223
|
-
resolve();
|
|
310
|
+
resolve(wasReserved);
|
|
224
311
|
};
|
|
225
312
|
const onAbort = () => {
|
|
226
313
|
const idx = this.waiters.indexOf(onResolve);
|
|
@@ -235,21 +322,17 @@ class Semaphore {
|
|
|
235
322
|
signal?.addEventListener('abort', onAbort, { once: true });
|
|
236
323
|
});
|
|
237
324
|
if (signal?.aborted) {
|
|
325
|
+
// A release may have handed this waiter capacity immediately before
|
|
326
|
+
// cancellation became observable. Return that handoff so it cannot
|
|
327
|
+
// leak or strand the next queued call.
|
|
328
|
+
if (reserved) this.leased().release();
|
|
238
329
|
throw signal.reason instanceof Error
|
|
239
330
|
? signal.reason
|
|
240
331
|
: new Error('Slot acquire aborted.');
|
|
241
332
|
}
|
|
333
|
+
if (reserved) return this.leased();
|
|
242
334
|
}
|
|
243
|
-
this.
|
|
244
|
-
let released = false;
|
|
245
|
-
return {
|
|
246
|
-
release: () => {
|
|
247
|
-
if (released) return;
|
|
248
|
-
released = true;
|
|
249
|
-
this.inFlight = Math.max(0, this.inFlight - 1);
|
|
250
|
-
this.waiters.shift()?.();
|
|
251
|
-
},
|
|
252
|
-
};
|
|
335
|
+
return this.lease();
|
|
253
336
|
}
|
|
254
337
|
}
|
|
255
338
|
|
|
@@ -322,6 +405,63 @@ export function createPlayExecutionGovernor(
|
|
|
322
405
|
|
|
323
406
|
const orgBucket = (provider: string) => `${input.scope.orgId}:${provider}`;
|
|
324
407
|
|
|
408
|
+
/** The tool's local semaphore for a declared `maxConcurrency`, else null. */
|
|
409
|
+
const providerConcurrencySlotsFor = (
|
|
410
|
+
toolId: string,
|
|
411
|
+
rules: readonly PacingRule[],
|
|
412
|
+
): Semaphore | null => {
|
|
413
|
+
const declaredMaxConcurrency = Math.min(
|
|
414
|
+
...rules.flatMap((rule) =>
|
|
415
|
+
rule.maxConcurrency != null ? [rule.maxConcurrency] : [],
|
|
416
|
+
),
|
|
417
|
+
);
|
|
418
|
+
if (!Number.isFinite(declaredMaxConcurrency)) return null;
|
|
419
|
+
let slots = providerConcurrencySlotsByTool.get(toolId);
|
|
420
|
+
if (!slots) {
|
|
421
|
+
slots = new Semaphore(Math.max(1, declaredMaxConcurrency));
|
|
422
|
+
providerConcurrencySlotsByTool.set(toolId, slots);
|
|
423
|
+
}
|
|
424
|
+
return slots;
|
|
425
|
+
};
|
|
426
|
+
|
|
427
|
+
const providerPermitFrom = (
|
|
428
|
+
provider: string,
|
|
429
|
+
rateLease: PacingPermit,
|
|
430
|
+
localConcurrencySlot: WorkLease | null,
|
|
431
|
+
): ProviderPermit => {
|
|
432
|
+
let settled = false;
|
|
433
|
+
return {
|
|
434
|
+
provider,
|
|
435
|
+
concurrencyGrant: rateLease.concurrencyGrant ?? null,
|
|
436
|
+
pacingGrant: rateLease.pacingGrant ?? null,
|
|
437
|
+
release() {
|
|
438
|
+
if (settled) return;
|
|
439
|
+
settled = true;
|
|
440
|
+
rateLease.release();
|
|
441
|
+
localConcurrencySlot?.release();
|
|
442
|
+
},
|
|
443
|
+
async cancel() {
|
|
444
|
+
if (settled) return;
|
|
445
|
+
settled = true;
|
|
446
|
+
try {
|
|
447
|
+
await rateLease.cancel();
|
|
448
|
+
} finally {
|
|
449
|
+
localConcurrencySlot?.release();
|
|
450
|
+
}
|
|
451
|
+
},
|
|
452
|
+
async park() {
|
|
453
|
+
if (settled) return;
|
|
454
|
+
settled = true;
|
|
455
|
+
try {
|
|
456
|
+
if (rateLease.park) await rateLease.park();
|
|
457
|
+
else await rateLease.cancel();
|
|
458
|
+
} finally {
|
|
459
|
+
localConcurrencySlot?.release();
|
|
460
|
+
}
|
|
461
|
+
},
|
|
462
|
+
};
|
|
463
|
+
};
|
|
464
|
+
|
|
325
465
|
async function resolveScope(toolId: string, provider: string) {
|
|
326
466
|
const resolved = await input.resolveRateScope?.(toolId, provider);
|
|
327
467
|
const scope = [
|
|
@@ -427,8 +567,17 @@ export function createPlayExecutionGovernor(
|
|
|
427
567
|
policy,
|
|
428
568
|
|
|
429
569
|
acquireRowSlot: (opts) => rowSlots.acquire(opts?.signal),
|
|
570
|
+
async acquireToolExecutionSlot(opts) {
|
|
571
|
+
return await toolSlots.acquire(opts?.signal);
|
|
572
|
+
},
|
|
573
|
+
|
|
574
|
+
tryAcquireToolExecutionSlot: () => toolSlots.tryAcquire(),
|
|
575
|
+
|
|
576
|
+
tryAcquireIntegrationRequestSlot: () =>
|
|
577
|
+
integrationRequestSlots.tryAcquire(),
|
|
578
|
+
|
|
430
579
|
async acquireToolSlot(_toolId, opts) {
|
|
431
|
-
const slot = await
|
|
580
|
+
const slot = await governor.acquireToolExecutionSlot(opts);
|
|
432
581
|
// Charge the logical call only after its local execution slot is held, so
|
|
433
582
|
// a failed/aborted slot acquisition never consumes the budget. The
|
|
434
583
|
// physical provider admission is intentionally deferred to
|
|
@@ -461,15 +610,12 @@ export function createPlayExecutionGovernor(
|
|
|
461
610
|
rule.maxConcurrency != null ? [rule.maxConcurrency] : [],
|
|
462
611
|
),
|
|
463
612
|
);
|
|
464
|
-
const
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
}
|
|
471
|
-
return slots.acquire(opts?.signal);
|
|
472
|
-
})()
|
|
613
|
+
const localConcurrencySlots = providerConcurrencySlotsFor(
|
|
614
|
+
toolId,
|
|
615
|
+
pacing.rules,
|
|
616
|
+
);
|
|
617
|
+
const localConcurrencySlot = localConcurrencySlots
|
|
618
|
+
? await localConcurrencySlots.acquire(opts?.signal)
|
|
473
619
|
: null;
|
|
474
620
|
try {
|
|
475
621
|
const rateLease = await input.rateState.acquire({
|
|
@@ -478,18 +624,38 @@ export function createPlayExecutionGovernor(
|
|
|
478
624
|
rules: pacing.rules,
|
|
479
625
|
signal: opts?.signal,
|
|
480
626
|
});
|
|
481
|
-
return
|
|
482
|
-
release() {
|
|
483
|
-
rateLease.release();
|
|
484
|
-
localConcurrencySlot?.release();
|
|
485
|
-
},
|
|
486
|
-
};
|
|
627
|
+
return providerPermitFrom(pacing.provider, rateLease, localConcurrencySlot);
|
|
487
628
|
} catch (error) {
|
|
488
629
|
localConcurrencySlot?.release();
|
|
489
630
|
throw error;
|
|
490
631
|
}
|
|
491
632
|
},
|
|
492
633
|
|
|
634
|
+
tryAcquireProviderPermit(toolId) {
|
|
635
|
+
const tryAcquire = input.rateState.tryAcquire;
|
|
636
|
+
const provider = providerByTool.get(toolId);
|
|
637
|
+
const rules = pacingRulesByTool.get(toolId);
|
|
638
|
+
const rateScope = scopeByTool.get(toolId);
|
|
639
|
+
if (!tryAcquire || !provider || !rules || !rateScope) return null;
|
|
640
|
+
const localConcurrencySlots = providerConcurrencySlotsFor(toolId, rules);
|
|
641
|
+
const localConcurrencySlot = localConcurrencySlots
|
|
642
|
+
? localConcurrencySlots.tryAcquire()
|
|
643
|
+
: null;
|
|
644
|
+
if (localConcurrencySlots && !localConcurrencySlot) return null;
|
|
645
|
+
let rateLease: PacingPermit | null = null;
|
|
646
|
+
try {
|
|
647
|
+
rateLease = tryAcquire.call(input.rateState, {
|
|
648
|
+
bucketId: rateScope[0],
|
|
649
|
+
rateScopeToken: rateScope[1],
|
|
650
|
+
rules,
|
|
651
|
+
});
|
|
652
|
+
} finally {
|
|
653
|
+
if (!rateLease) localConcurrencySlot?.release();
|
|
654
|
+
}
|
|
655
|
+
if (!rateLease) return null;
|
|
656
|
+
return providerPermitFrom(provider, rateLease, localConcurrencySlot);
|
|
657
|
+
},
|
|
658
|
+
|
|
493
659
|
async suggestedParallelism(toolId, fallback) {
|
|
494
660
|
const pacing = await resolveAdaptivePacing(toolId);
|
|
495
661
|
const concurrencyLimits = pacing.rules.flatMap((rule) =>
|
|
@@ -556,10 +722,16 @@ export function createPlayExecutionGovernor(
|
|
|
556
722
|
});
|
|
557
723
|
}
|
|
558
724
|
const rateScope = scopeFor(bp.toolId, bp.provider);
|
|
725
|
+
const rules = bp.toolId ? pacingRulesByTool.get(bp.toolId) : undefined;
|
|
559
726
|
await input.rateState.penalize({
|
|
560
727
|
bucketId: rateScope[0],
|
|
561
728
|
...(rateScope[1] ? { rateScopeToken: rateScope[1] } : {}),
|
|
562
729
|
cooldownMs: bp.retryAfterMs,
|
|
730
|
+
...(rules ? { rules } : {}),
|
|
731
|
+
...(bp.latencyMs != null ? { latencyMs: bp.latencyMs } : {}),
|
|
732
|
+
...(bp.concurrencyGrant
|
|
733
|
+
? { concurrencyGrant: bp.concurrencyGrant }
|
|
734
|
+
: {}),
|
|
563
735
|
});
|
|
564
736
|
},
|
|
565
737
|
|
|
@@ -572,6 +744,7 @@ export function createPlayExecutionGovernor(
|
|
|
572
744
|
input.rateState.observeSuccess?.({
|
|
573
745
|
bucketId: rateScope[0],
|
|
574
746
|
rules,
|
|
747
|
+
latencyMs: success.latencyMs,
|
|
575
748
|
});
|
|
576
749
|
return;
|
|
577
750
|
}
|
|
@@ -651,10 +824,15 @@ function createInlineChildGovernor(
|
|
|
651
824
|
policy: root.policy,
|
|
652
825
|
acquireRowSlot: (opts) => root.acquireRowSlot(opts),
|
|
653
826
|
acquireToolSlot: (toolId, opts) => root.acquireToolSlot(toolId, opts),
|
|
827
|
+
acquireToolExecutionSlot: (opts) => root.acquireToolExecutionSlot(opts),
|
|
828
|
+
tryAcquireToolExecutionSlot: () => root.tryAcquireToolExecutionSlot(),
|
|
829
|
+
tryAcquireIntegrationRequestSlot: () =>
|
|
830
|
+
root.tryAcquireIntegrationRequestSlot(),
|
|
654
831
|
acquireIntegrationRequestSlot: (opts) =>
|
|
655
832
|
root.acquireIntegrationRequestSlot(opts),
|
|
656
833
|
acquireProviderPermit: (toolId, opts) =>
|
|
657
834
|
root.acquireProviderPermit(toolId, opts),
|
|
835
|
+
tryAcquireProviderPermit: (toolId) => root.tryAcquireProviderPermit(toolId),
|
|
658
836
|
suggestedParallelism: (toolId, fallback) =>
|
|
659
837
|
root.suggestedParallelism(toolId, fallback),
|
|
660
838
|
suggestedDirectToolParallelism: (toolId, fallback) =>
|
package/dist/bundling-sources/shared_libs/play-runtime/governor/in-memory-rate-state-backend.ts
CHANGED
|
@@ -3,6 +3,7 @@ import type {
|
|
|
3
3
|
PacingRule,
|
|
4
4
|
RateStateBackend,
|
|
5
5
|
} from './rate-state-backend';
|
|
6
|
+
import { noopPacingPermit } from './rate-state-backend';
|
|
6
7
|
|
|
7
8
|
/**
|
|
8
9
|
* In-memory Rate State Backend for the single-process `cjs_node20` runner.
|
|
@@ -48,7 +49,7 @@ export class InMemoryRateStateBackend implements RateStateBackend {
|
|
|
48
49
|
}): Promise<PacingPermit> {
|
|
49
50
|
const { bucketId, rules, signal } = input;
|
|
50
51
|
if (rules.length === 0) {
|
|
51
|
-
return
|
|
52
|
+
return noopPacingPermit();
|
|
52
53
|
}
|
|
53
54
|
// Stable order to avoid deadlock between concurrent acquirers on shared rules.
|
|
54
55
|
const ordered = [...rules].sort((a, b) => a.ruleId.localeCompare(b.ruleId));
|
|
@@ -95,23 +96,36 @@ export class InMemoryRateStateBackend implements RateStateBackend {
|
|
|
95
96
|
});
|
|
96
97
|
|
|
97
98
|
if (decision.acquired) {
|
|
98
|
-
let
|
|
99
|
+
let settled = false;
|
|
100
|
+
const releaseConcurrency = async (refundRateReservation: boolean) => {
|
|
101
|
+
await this.withLock(() => {
|
|
102
|
+
for (const rule of ordered) {
|
|
103
|
+
const state = this.ruleStates.get(
|
|
104
|
+
this.key(bucketId, rule.ruleId),
|
|
105
|
+
);
|
|
106
|
+
if (!state) continue;
|
|
107
|
+
if (refundRateReservation && state.startedInWindow > 0) {
|
|
108
|
+
state.startedInWindow -= 1;
|
|
109
|
+
}
|
|
110
|
+
if (rule.maxConcurrency != null && state.activeCount > 0) {
|
|
111
|
+
state.activeCount -= 1;
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
});
|
|
115
|
+
};
|
|
99
116
|
return {
|
|
100
117
|
// The concurrency decrement is intentionally not awaitable (the permit
|
|
101
118
|
// signature is sync `release(): void`). It still runs serialized under
|
|
102
119
|
// the lock; do not "fix" this by trying to await a void.
|
|
103
120
|
release: () => {
|
|
104
|
-
if (
|
|
105
|
-
|
|
106
|
-
void
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
if (state && state.activeCount > 0) state.activeCount -= 1;
|
|
113
|
-
}
|
|
114
|
-
});
|
|
121
|
+
if (settled) return;
|
|
122
|
+
settled = true;
|
|
123
|
+
void releaseConcurrency(false);
|
|
124
|
+
},
|
|
125
|
+
cancel: async () => {
|
|
126
|
+
if (settled) return;
|
|
127
|
+
settled = true;
|
|
128
|
+
await releaseConcurrency(true);
|
|
115
129
|
},
|
|
116
130
|
};
|
|
117
131
|
}
|
|
@@ -92,6 +92,14 @@ export const MAX_CONFIGURABLE_CONCURRENT_ROWS = 1_000;
|
|
|
92
92
|
// available to long-residence providers while still bounding live calls.
|
|
93
93
|
export const DEFAULT_MAX_IN_FLIGHT_TOOL_DISPATCH_GROUPS =
|
|
94
94
|
MAX_CONFIGURABLE_CONCURRENT_EXTERNAL_CALLS;
|
|
95
|
+
// One lane is one tool (or one batch bucket). Its in-flight groups are not
|
|
96
|
+
// capped below the run-wide envelope: a group counts as in flight until its
|
|
97
|
+
// slowest call returns, so with provider tails of 8-40 s a small per-lane
|
|
98
|
+
// cap pinned every slot on nearly finished groups and left ~20 calls on the
|
|
99
|
+
// wire per shape. Coalescing under saturation is handled by the round-trip
|
|
100
|
+
// adaptive window in the dispatcher instead.
|
|
101
|
+
export const DEFAULT_MAX_IN_FLIGHT_TOOL_DISPATCH_GROUPS_PER_LANE =
|
|
102
|
+
DEFAULT_MAX_IN_FLIGHT_TOOL_DISPATCH_GROUPS;
|
|
95
103
|
// Launches persisted before this field existed used the original fixed
|
|
96
104
|
// eight-slot policy. Preserve that value on their retries and resumes.
|
|
97
105
|
export const LEGACY_MAX_CONCURRENT_EXTERNAL_CALLS = 8;
|
|
@@ -153,18 +161,23 @@ export const SHARED_EXECUTION_POLICY: ResolvedExecutionPolicy = {
|
|
|
153
161
|
// Runs may request a value through the validated 1..1000 launch contract;
|
|
154
162
|
// provider pacing and the runner-owned hard ceiling still apply.
|
|
155
163
|
toolCalls: DEFAULT_MAX_CONCURRENT_EXTERNAL_CALLS,
|
|
156
|
-
// Physical runtime-to-app fetch/body admission.
|
|
157
|
-
//
|
|
158
|
-
//
|
|
159
|
-
//
|
|
160
|
-
|
|
164
|
+
// Physical runtime-to-app fetch/body admission. This is deliberately
|
|
165
|
+
// independent of provider rate limits: it protects Deepline's own
|
|
166
|
+
// transport edge. Measured at 1k rows on a warm production-shape preview
|
|
167
|
+
// estate: 256 here with a 1,024-request proxy limit took kareem from
|
|
168
|
+
// 8.7 to 11.2 rows/s and modal from 6.9 to 9.4; at 64 sockets 700-1,000
|
|
169
|
+
// resident calls waited seconds for a socket while the sandbox idled.
|
|
170
|
+
// Only a cold estate (replay samples fetched per call until cached)
|
|
171
|
+
// queues badly at this width, and that is a warm-up concern, not a cap.
|
|
172
|
+
integrationRequests: 256,
|
|
161
173
|
// Scheduling cohorts overlap provider-response tails, but each cohort may
|
|
162
174
|
// itself contain many logical calls. Keep this materially above the legacy
|
|
163
175
|
// eight-group ceiling without creating one independently flushed cohort
|
|
164
176
|
// per configured tool slot. The latter fragments batchable providers and
|
|
165
177
|
// adds receipt/gateway round trips under wide (128+) runs.
|
|
166
178
|
toolDispatchGroups: DEFAULT_MAX_IN_FLIGHT_TOOL_DISPATCH_GROUPS,
|
|
167
|
-
toolDispatchGroupsPerLane:
|
|
179
|
+
toolDispatchGroupsPerLane:
|
|
180
|
+
DEFAULT_MAX_IN_FLIGHT_TOOL_DISPATCH_GROUPS_PER_LANE,
|
|
168
181
|
},
|
|
169
182
|
budgets: {
|
|
170
183
|
// Runaway guards, not workload limits. A 5,000-row map calling several tools
|
|
@@ -30,6 +30,24 @@ export interface PacingRule {
|
|
|
30
30
|
readonly maxConcurrency: number | null;
|
|
31
31
|
}
|
|
32
32
|
|
|
33
|
+
/**
|
|
34
|
+
* Effective shared rate captured when a physical request was admitted. It is
|
|
35
|
+
* read-only diagnostic context: callers cannot widen a provider limit.
|
|
36
|
+
*/
|
|
37
|
+
export type ProviderPacingGrant = {
|
|
38
|
+
readonly targetRequestsPerSecond: number | null;
|
|
39
|
+
};
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* Global occupancy and RTT baseline captured with one concurrency permit.
|
|
43
|
+
* Pure-rate permits publish only {@link ProviderPacingGrant}.
|
|
44
|
+
*/
|
|
45
|
+
export type ProviderConcurrencyGrant = ProviderPacingGrant & {
|
|
46
|
+
readonly admissionId: string;
|
|
47
|
+
readonly inFlightAtStart: number;
|
|
48
|
+
readonly minimumRttMs: number | null;
|
|
49
|
+
};
|
|
50
|
+
|
|
33
51
|
/**
|
|
34
52
|
* Per-tool queue-hint metadata produced by the runtime tool catalog
|
|
35
53
|
* (`packages/integrations/runtime-tool-metadata.ts`) and surfaced to the runtime via
|
|
@@ -49,9 +67,26 @@ export interface PlayQueueHint {
|
|
|
49
67
|
maxConcurrency: number | null;
|
|
50
68
|
}
|
|
51
69
|
|
|
52
|
-
/** Handle returned by acquire()
|
|
70
|
+
/** Handle returned by acquire(). */
|
|
53
71
|
export interface PacingPermit {
|
|
72
|
+
/** Complete a dispatched request; frees only any concurrency held by the rules. */
|
|
54
73
|
release(): void;
|
|
74
|
+
/**
|
|
75
|
+
* Cancel a ticket that never reached the provider. Unlike `release`, this
|
|
76
|
+
* returns its rate reservation before freeing any held concurrency.
|
|
77
|
+
*/
|
|
78
|
+
cancel(): Promise<void>;
|
|
79
|
+
/**
|
|
80
|
+
* Return a due ticket that never reached the provider to the backend's local
|
|
81
|
+
* lease block, so the next acquire on the same lane draws it with no store
|
|
82
|
+
* round trip. Falls back to `cancel` when that block is gone or expired.
|
|
83
|
+
* Absent on backends that keep no local block; callers use `cancel` then.
|
|
84
|
+
*/
|
|
85
|
+
park?(): Promise<void>;
|
|
86
|
+
/** Server-authored context for attributing this request's provider 429. */
|
|
87
|
+
readonly concurrencyGrant?: ProviderConcurrencyGrant | null;
|
|
88
|
+
/** Effective shared rate for every app-runtime provider permit. */
|
|
89
|
+
readonly pacingGrant?: ProviderPacingGrant | null;
|
|
55
90
|
}
|
|
56
91
|
|
|
57
92
|
/**
|
|
@@ -85,6 +120,18 @@ export interface RateStateBackend {
|
|
|
85
120
|
signal?: AbortSignal;
|
|
86
121
|
}): Promise<PacingPermit>;
|
|
87
122
|
|
|
123
|
+
/**
|
|
124
|
+
* Take a permit for `bucketId` only if one is already due in local state.
|
|
125
|
+
* Synchronous and non-blocking: never refills, sleeps, or reaches the store.
|
|
126
|
+
* Returns null when no due permit is held locally, when the pacer is
|
|
127
|
+
* holding the provider off, or when the backend keeps no local state.
|
|
128
|
+
*/
|
|
129
|
+
tryAcquire?(input: {
|
|
130
|
+
bucketId: string;
|
|
131
|
+
rateScopeToken?: string | null;
|
|
132
|
+
rules: readonly PacingRule[];
|
|
133
|
+
}): PacingPermit | null;
|
|
134
|
+
|
|
88
135
|
/**
|
|
89
136
|
* Feed a server-observed Retry-After back so future acquires for this bucket
|
|
90
137
|
* back off. Advisory and idempotent; never un-charges an in-flight call.
|
|
@@ -93,6 +140,12 @@ export interface RateStateBackend {
|
|
|
93
140
|
bucketId: string;
|
|
94
141
|
rateScopeToken?: string | null;
|
|
95
142
|
cooldownMs: number;
|
|
143
|
+
/** Resolved provider rules for the rejected physical attempt. */
|
|
144
|
+
rules?: readonly PacingRule[];
|
|
145
|
+
/** Provider residence measured at the HTTP boundary. */
|
|
146
|
+
latencyMs?: number | null;
|
|
147
|
+
/** Server-authored context for attributing this physical provider 429. */
|
|
148
|
+
concurrencyGrant?: ProviderConcurrencyGrant | null;
|
|
96
149
|
}): void | Promise<void>;
|
|
97
150
|
|
|
98
151
|
/** Record confirmed provider success for adaptive recovery. */
|
|
@@ -100,10 +153,16 @@ export interface RateStateBackend {
|
|
|
100
153
|
bucketId: string;
|
|
101
154
|
rateScopeToken?: string | null;
|
|
102
155
|
rules: readonly PacingRule[];
|
|
156
|
+
/** Provider residence measured at the HTTP boundary. */
|
|
157
|
+
latencyMs?: number | null;
|
|
103
158
|
}): void;
|
|
104
159
|
}
|
|
105
160
|
|
|
106
|
-
const NOOP_PERMIT: PacingPermit = {
|
|
161
|
+
const NOOP_PERMIT: PacingPermit = {
|
|
162
|
+
release() {},
|
|
163
|
+
async cancel() {},
|
|
164
|
+
async park() {},
|
|
165
|
+
};
|
|
107
166
|
|
|
108
167
|
/** Permit used when a bucket has no rules — nothing to debit or release. */
|
|
109
168
|
export function noopPacingPermit(): PacingPermit {
|
|
@@ -12,6 +12,22 @@ export type IntegrationOperationStorageMode =
|
|
|
12
12
|
export const INTEGRATION_OPERATION_CANONICAL_READER_CAPABILITY =
|
|
13
13
|
'integration-operation-canonical-v1' as const;
|
|
14
14
|
|
|
15
|
+
/**
|
|
16
|
+
* A strictly stronger reader proof. A gateway that reports this capability can
|
|
17
|
+
* materialize the existing canonical Integration Operation outcome and the
|
|
18
|
+
* immutable R2 receipt-value references that may be published beside it.
|
|
19
|
+
*
|
|
20
|
+
* Keep this distinct from the original canonical capability: an already-active
|
|
21
|
+
* canonical fence may have certified an older gateway image that cannot read a
|
|
22
|
+
* receipt reference yet.
|
|
23
|
+
*/
|
|
24
|
+
export const RUNTIME_RECEIPT_VALUE_REFERENCE_READER_CAPABILITY =
|
|
25
|
+
'runtime-receipt-value-reference-v1' as const;
|
|
26
|
+
|
|
27
|
+
export type RuntimeGatewayReaderCapability =
|
|
28
|
+
| typeof INTEGRATION_OPERATION_CANONICAL_READER_CAPABILITY
|
|
29
|
+
| typeof RUNTIME_RECEIPT_VALUE_REFERENCE_READER_CAPABILITY;
|
|
30
|
+
|
|
15
31
|
export type RuntimeGatewayRole = 'receipt' | 'execution';
|
|
16
32
|
|
|
17
33
|
/**
|
|
@@ -20,7 +36,7 @@ export type RuntimeGatewayRole = 'receipt' | 'execution';
|
|
|
20
36
|
* collects this evidence; this module records and validates its shape.
|
|
21
37
|
*/
|
|
22
38
|
export type CanonicalReaderProof = {
|
|
23
|
-
capability:
|
|
39
|
+
capability: RuntimeGatewayReaderCapability;
|
|
24
40
|
sourceSha: string;
|
|
25
41
|
machineIds: Readonly<Record<RuntimeGatewayRole, readonly string[]>>;
|
|
26
42
|
};
|
|
@@ -47,6 +63,18 @@ export type ReaderFenceWriterMode = {
|
|
|
47
63
|
failure?: 'store_unavailable' | 'invalid_record';
|
|
48
64
|
};
|
|
49
65
|
|
|
66
|
+
/**
|
|
67
|
+
* The additional format is write-enabled only after the stronger, exact-fleet
|
|
68
|
+
* proof reaches the durable mirror. A missing or unreadable mirror stays
|
|
69
|
+
* inline, which is compatible with every prior gateway image.
|
|
70
|
+
*/
|
|
71
|
+
export type RuntimeReceiptValueReferenceWriterMode = {
|
|
72
|
+
enabled: boolean;
|
|
73
|
+
source: 'fence' | 'absent' | 'fail_closed';
|
|
74
|
+
epoch: number | null;
|
|
75
|
+
failure?: 'store_unavailable' | 'invalid_record';
|
|
76
|
+
};
|
|
77
|
+
|
|
50
78
|
export type ReaderFenceApplyOutcome =
|
|
51
79
|
| { kind: 'advanced'; fence: RuntimeGatewayReaderFence }
|
|
52
80
|
| { kind: 'already_applied'; fence: RuntimeGatewayReaderFence }
|
|
@@ -59,6 +87,10 @@ export type ReaderFenceApplyOutcome =
|
|
|
59
87
|
export type RuntimeGatewayReaderFenceJob = {
|
|
60
88
|
/** Resolve the only format this gateway may write. Never returns canonical on uncertainty. */
|
|
61
89
|
writerMode(input: { schema: string }): Promise<ReaderFenceWriterMode>;
|
|
90
|
+
/** Resolve whether this gateway may publish receipt value references. */
|
|
91
|
+
runtimeReceiptValueReferenceWriterMode(input: {
|
|
92
|
+
schema: string;
|
|
93
|
+
}): Promise<RuntimeReceiptValueReferenceWriterMode>;
|
|
62
94
|
/** Advance the mirror only with a strictly newer epoch, or retry the exact same fence. */
|
|
63
95
|
apply(
|
|
64
96
|
candidate: RuntimeGatewayReaderFenceCandidate,
|