deepline 0.1.291 → 0.1.293
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/sdk/src/release.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +81 -3
- package/dist/bundling-sources/shared_libs/play-runtime/context.ts +4 -46
- package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +0 -14
- package/dist/bundling-sources/shared_libs/play-runtime/gateway-postgres-admission.ts +159 -0
- package/dist/bundling-sources/shared_libs/play-runtime/governor/app-runtime-rate-state-backend.ts +50 -29
- package/dist/bundling-sources/shared_libs/play-runtime/persistence-latch.ts +3 -0
- package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts +6 -14
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +5 -2
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-capacity-policy.ts +76 -11
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-receipt-store-adapter.ts +410 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-receipt-writer.ts +494 -0
- package/dist/cli/index.js +1 -1
- package/dist/cli/index.mjs +1 -1
- package/dist/index.js +1 -1
- package/dist/index.mjs +1 -1
- package/package.json +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/receipt-completion-sink.ts +0 -343
- package/dist/bundling-sources/shared_libs/play-runtime/receipt-state-sink.ts +0 -69
- package/dist/bundling-sources/shared_libs/plays/legacy-webhook-drain.ts +0 -110
|
@@ -155,7 +155,7 @@ export const SDK_RELEASE = {
|
|
|
155
155
|
// 0.1.253 makes play-page browser opening opt-in and retires --no-open.
|
|
156
156
|
// 0.1.254 removes the internal operations tree from the published SDK CLI.
|
|
157
157
|
// Operators use the checkout-local deepline-admin binary instead.
|
|
158
|
-
version: '0.1.
|
|
158
|
+
version: '0.1.293',
|
|
159
159
|
contracts: {
|
|
160
160
|
api: {
|
|
161
161
|
name: 'sdk-http-api',
|
|
@@ -364,6 +364,9 @@ export type WorkerRuntimeApiContext = {
|
|
|
364
364
|
runtimeTestFaultHeader?: string | null;
|
|
365
365
|
fetch?: typeof fetch;
|
|
366
366
|
requestTimeoutMs?: number | null;
|
|
367
|
+
/** A higher-level writer owns retries and preserves the exact request body. */
|
|
368
|
+
retryPolicy?: 'default' | 'none';
|
|
369
|
+
signal?: AbortSignal;
|
|
367
370
|
};
|
|
368
371
|
|
|
369
372
|
const APP_RUNTIME_API_RETRY_DELAYS_MS = [100, 250, 500, 1_000] as const;
|
|
@@ -1007,6 +1010,60 @@ export class AppRuntimeApiResponseError extends Error {
|
|
|
1007
1010
|
}
|
|
1008
1011
|
}
|
|
1009
1012
|
|
|
1013
|
+
const APP_RUNTIME_CAPACITY_ERROR_CODES = new Set([
|
|
1014
|
+
'receipt_db_admission_backpressure',
|
|
1015
|
+
'scheduler_capacity',
|
|
1016
|
+
'runtime_postgres_admission_backpressure',
|
|
1017
|
+
'RUNTIME_SCHEDULER_SATURATED',
|
|
1018
|
+
]);
|
|
1019
|
+
|
|
1020
|
+
/**
|
|
1021
|
+
* The runtime persistence plane is temporarily full. This is an admission
|
|
1022
|
+
* signal, not a transport failure: callers must park the same operation rather
|
|
1023
|
+
* than spending the ordinary request retry ladder.
|
|
1024
|
+
*/
|
|
1025
|
+
export class AppRuntimeApiCapacityError extends AppRuntimeApiResponseError {
|
|
1026
|
+
readonly retryAfterMs: number;
|
|
1027
|
+
|
|
1028
|
+
constructor(input: {
|
|
1029
|
+
action: RuntimeApiRequest['action'];
|
|
1030
|
+
status: number;
|
|
1031
|
+
code: string;
|
|
1032
|
+
requestId?: string | null;
|
|
1033
|
+
detail: string;
|
|
1034
|
+
retryAfterMs: number;
|
|
1035
|
+
}) {
|
|
1036
|
+
super({
|
|
1037
|
+
action: input.action,
|
|
1038
|
+
status: input.status,
|
|
1039
|
+
code: input.code,
|
|
1040
|
+
requestId: input.requestId,
|
|
1041
|
+
retryable: true,
|
|
1042
|
+
detail: input.detail,
|
|
1043
|
+
});
|
|
1044
|
+
this.name = 'AppRuntimeApiCapacityError';
|
|
1045
|
+
this.retryAfterMs = Math.max(0, Math.floor(input.retryAfterMs));
|
|
1046
|
+
}
|
|
1047
|
+
}
|
|
1048
|
+
|
|
1049
|
+
export function isAppRuntimeApiCapacityError(
|
|
1050
|
+
error: unknown,
|
|
1051
|
+
): error is AppRuntimeApiCapacityError {
|
|
1052
|
+
if (error instanceof AppRuntimeApiCapacityError) return true;
|
|
1053
|
+
if (!error || typeof error !== 'object') return false;
|
|
1054
|
+
const candidate = error as {
|
|
1055
|
+
name?: unknown;
|
|
1056
|
+
code?: unknown;
|
|
1057
|
+
retryAfterMs?: unknown;
|
|
1058
|
+
};
|
|
1059
|
+
return (
|
|
1060
|
+
candidate.name === 'AppRuntimeApiCapacityError' &&
|
|
1061
|
+
typeof candidate.code === 'string' &&
|
|
1062
|
+
APP_RUNTIME_CAPACITY_ERROR_CODES.has(candidate.code) &&
|
|
1063
|
+
typeof candidate.retryAfterMs === 'number'
|
|
1064
|
+
);
|
|
1065
|
+
}
|
|
1066
|
+
|
|
1010
1067
|
function resolveAppRuntimeApiUrl(context: WorkerRuntimeApiContext): string {
|
|
1011
1068
|
const baseUrl = context.baseUrl.trim().replace(/\/$/, '');
|
|
1012
1069
|
return `${baseUrl}${PLAY_RUNTIME_API_COMPAT_PATH}`;
|
|
@@ -1032,7 +1089,8 @@ async function postAppRuntimeApi<TResponse>(
|
|
|
1032
1089
|
fetchImpl: runtimeFetch,
|
|
1033
1090
|
});
|
|
1034
1091
|
|
|
1035
|
-
const maxAttempts =
|
|
1092
|
+
const maxAttempts =
|
|
1093
|
+
context.retryPolicy === 'none' ? 1 : appRuntimeMaxAttempts(body.action);
|
|
1036
1094
|
const retryTelemetry: AppRuntimeRetryTelemetry = {
|
|
1037
1095
|
enabled: isRuntimeStepReceiptAction(body.action),
|
|
1038
1096
|
startedAt: Date.now(),
|
|
@@ -1066,7 +1124,12 @@ async function postAppRuntimeApi<TResponse>(
|
|
|
1066
1124
|
: {}),
|
|
1067
1125
|
},
|
|
1068
1126
|
body: JSON.stringify(body),
|
|
1069
|
-
signal:
|
|
1127
|
+
signal: context.signal
|
|
1128
|
+
? AbortSignal.any([
|
|
1129
|
+
context.signal,
|
|
1130
|
+
AbortSignal.timeout(requestTimeoutMs),
|
|
1131
|
+
])
|
|
1132
|
+
: AbortSignal.timeout(requestTimeoutMs),
|
|
1070
1133
|
});
|
|
1071
1134
|
} catch (error) {
|
|
1072
1135
|
if (
|
|
@@ -1176,6 +1239,21 @@ async function postAppRuntimeApi<TResponse>(
|
|
|
1176
1239
|
});
|
|
1177
1240
|
continue;
|
|
1178
1241
|
}
|
|
1242
|
+
const responseCode = appRuntimeErrorCode(response, responseText);
|
|
1243
|
+
if (
|
|
1244
|
+
response.status === 503 &&
|
|
1245
|
+
responseCode &&
|
|
1246
|
+
APP_RUNTIME_CAPACITY_ERROR_CODES.has(responseCode)
|
|
1247
|
+
) {
|
|
1248
|
+
throw new AppRuntimeApiCapacityError({
|
|
1249
|
+
action: body.action,
|
|
1250
|
+
status: response.status,
|
|
1251
|
+
code: responseCode,
|
|
1252
|
+
requestId: response.headers.get('x-deepline-request-id')?.trim(),
|
|
1253
|
+
detail: summarizeAppRuntimeErrorBody(responseText),
|
|
1254
|
+
retryAfterMs: appRuntimeRetryAfterMs(response, responseText) ?? 1_000,
|
|
1255
|
+
});
|
|
1256
|
+
}
|
|
1179
1257
|
if (
|
|
1180
1258
|
attempt < maxAttempts &&
|
|
1181
1259
|
isRetryableAppRuntimeResponse({
|
|
@@ -1221,7 +1299,7 @@ async function postAppRuntimeApi<TResponse>(
|
|
|
1221
1299
|
maxAttempts,
|
|
1222
1300
|
finalAttemptStartedAt: attemptStartedAt,
|
|
1223
1301
|
});
|
|
1224
|
-
const code =
|
|
1302
|
+
const code = responseCode;
|
|
1225
1303
|
const requestId = response.headers.get('x-deepline-request-id')?.trim();
|
|
1226
1304
|
throw new AppRuntimeApiResponseError({
|
|
1227
1305
|
action: body.action,
|
|
@@ -2052,30 +2052,6 @@ export class PlayContextImpl {
|
|
|
2052
2052
|
byLeaseId.set(target.leaseId, keys);
|
|
2053
2053
|
}
|
|
2054
2054
|
|
|
2055
|
-
if (
|
|
2056
|
-
this.#options.markRuntimeStepReceiptsRunning ||
|
|
2057
|
-
this.#options.markRuntimeStepReceiptRunning
|
|
2058
|
-
) {
|
|
2059
|
-
const receipts = await this.markRuntimeStepReceiptsRunning(
|
|
2060
|
-
targets.map((target) => ({
|
|
2061
|
-
key: target.receiptKey,
|
|
2062
|
-
runId: this.currentReceiptOwnerRunId,
|
|
2063
|
-
leaseId: target.leaseId,
|
|
2064
|
-
})),
|
|
2065
|
-
);
|
|
2066
|
-
for (const target of targets) {
|
|
2067
|
-
const receipt = receipts.get(target.receiptKey);
|
|
2068
|
-
if (!this.runtimeToolReceiptStillOwned(receipt, target.leaseId)) {
|
|
2069
|
-
throw new RuntimeReceiptLeaseLostError({
|
|
2070
|
-
receiptKey: target.receiptKey,
|
|
2071
|
-
runId: this.currentReceiptOwnerRunId,
|
|
2072
|
-
leaseId: target.leaseId,
|
|
2073
|
-
});
|
|
2074
|
-
}
|
|
2075
|
-
}
|
|
2076
|
-
return;
|
|
2077
|
-
}
|
|
2078
|
-
|
|
2079
2055
|
if (this.#options.heartbeatRuntimeStepReceipts) {
|
|
2080
2056
|
for (const [leaseId, keys] of byLeaseId) {
|
|
2081
2057
|
const receipts = await this.#options.heartbeatRuntimeStepReceipts({
|
|
@@ -5429,7 +5405,7 @@ export class PlayContextImpl {
|
|
|
5429
5405
|
let activeFieldName: string | null = null;
|
|
5430
5406
|
|
|
5431
5407
|
// Global row slot keeps concurrent maps in the same run under rowMax. The
|
|
5432
|
-
// worker pool
|
|
5408
|
+
// worker pool below enforces this map's requested/default concurrency.
|
|
5433
5409
|
const globalRowSlot = await this.resourceGovernor.acquireRow({
|
|
5434
5410
|
estimatedBytes: runtimeMapJsonByteLength(baseRow),
|
|
5435
5411
|
});
|
|
@@ -6602,8 +6578,9 @@ export class PlayContextImpl {
|
|
|
6602
6578
|
: physicalDirectKey,
|
|
6603
6579
|
timeoutMs: resolveToolRuntimeTimeoutMs(toolId, options?.timeoutMs),
|
|
6604
6580
|
...(directReceiptLeaseId &&
|
|
6605
|
-
(this.#options.
|
|
6606
|
-
this.#options.
|
|
6581
|
+
(this.#options.heartbeatRuntimeStepReceipts ||
|
|
6582
|
+
this.#options.getRuntimeStepReceipt ||
|
|
6583
|
+
this.#options.getRuntimeStepReceipts)
|
|
6607
6584
|
? {
|
|
6608
6585
|
beforeProviderCall: async () => {
|
|
6609
6586
|
await this.assertRuntimeToolReceiptOwnership([
|
|
@@ -6618,19 +6595,6 @@ export class PlayContextImpl {
|
|
|
6618
6595
|
},
|
|
6619
6596
|
]);
|
|
6620
6597
|
},
|
|
6621
|
-
...(this.#options.markRuntimeStepReceiptsQueued
|
|
6622
|
-
? {
|
|
6623
|
-
parkProviderCall: async () => {
|
|
6624
|
-
await this.markRuntimeStepReceiptsQueued([
|
|
6625
|
-
{
|
|
6626
|
-
key: directCacheKey,
|
|
6627
|
-
runId: this.currentReceiptOwnerRunId,
|
|
6628
|
-
leaseId: directReceiptLeaseId,
|
|
6629
|
-
},
|
|
6630
|
-
]);
|
|
6631
|
-
},
|
|
6632
|
-
}
|
|
6633
|
-
: {}),
|
|
6634
6598
|
}
|
|
6635
6599
|
: {}),
|
|
6636
6600
|
});
|
|
@@ -8016,8 +7980,6 @@ export class PlayContextImpl {
|
|
|
8016
7980
|
{
|
|
8017
7981
|
beforeProviderCall: () =>
|
|
8018
7982
|
this.assertRuntimeToolReceiptOwnership([owner]),
|
|
8019
|
-
parkProviderCall: () =>
|
|
8020
|
-
this.parkRuntimeToolReceiptsQueued([owner]),
|
|
8021
7983
|
durableCallReceiptKey: receiptKey,
|
|
8022
7984
|
executionAuthScopeDigest: owner.executionAuthScopeDigest,
|
|
8023
7985
|
receiptLeaseExpiresAt: owner.receiptLeaseExpiresAt,
|
|
@@ -8298,8 +8260,6 @@ export class PlayContextImpl {
|
|
|
8298
8260
|
this.assertRuntimeToolReceiptOwnership(
|
|
8299
8261
|
batch.memberRequests,
|
|
8300
8262
|
),
|
|
8301
|
-
parkProviderCall: () =>
|
|
8302
|
-
this.parkRuntimeToolReceiptsQueued(batch.memberRequests),
|
|
8303
8263
|
},
|
|
8304
8264
|
);
|
|
8305
8265
|
},
|
|
@@ -8447,8 +8407,6 @@ export class PlayContextImpl {
|
|
|
8447
8407
|
{
|
|
8448
8408
|
beforeProviderCall: () =>
|
|
8449
8409
|
this.assertRuntimeToolReceiptOwnership([request]),
|
|
8450
|
-
parkProviderCall: () =>
|
|
8451
|
-
this.parkRuntimeToolReceiptsQueued([request]),
|
|
8452
8410
|
...(request.receiptKey
|
|
8453
8411
|
? {
|
|
8454
8412
|
durableCallReceiptKey: request.receiptKey,
|
|
@@ -536,20 +536,6 @@ export interface ContextOptions {
|
|
|
536
536
|
artifactHash?: string | null;
|
|
537
537
|
convexUrl?: string;
|
|
538
538
|
runtimeSchedulerSchema?: string | null;
|
|
539
|
-
/**
|
|
540
|
-
* Execution profile for scheduler-backed `ctx.runPlay` child launches. Threaded
|
|
541
|
-
* from the parent launch so a child inherits the parent's scheduler profile
|
|
542
|
-
* (e.g. `absurd`) on the child `/api/v2/plays/run` submission. Absent defaults
|
|
543
|
-
* to the `absurd` profile.
|
|
544
|
-
*/
|
|
545
|
-
childRunProfile?: string | null;
|
|
546
|
-
/**
|
|
547
|
-
* The parent worker's own absurd release lane id. Threaded onto child
|
|
548
|
-
* `ctx.runPlay` launches via the `x-deepline-absurd-release` header so the
|
|
549
|
-
* child pins to the parent's lane (defense-in-depth). Null off the absurd
|
|
550
|
-
* path or on a pre-release parent.
|
|
551
|
-
*/
|
|
552
|
-
absurdReleaseId?: string | null;
|
|
553
539
|
staticPipeline?: PlayStaticPipeline | null;
|
|
554
540
|
workflowId?: string;
|
|
555
541
|
sessionId?: string;
|
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
export type GatewayPostgresLane = 'ordinary' | 'commit';
|
|
2
|
+
|
|
3
|
+
export const GATEWAY_POSTGRES_MAX_ORDINARY_ACTIVE = 8;
|
|
4
|
+
export const GATEWAY_POSTGRES_MAX_TOTAL_ACTIVE = 10;
|
|
5
|
+
|
|
6
|
+
export type GatewayPostgresAdmissionSnapshot = {
|
|
7
|
+
active: Record<GatewayPostgresLane, number> & { total: number };
|
|
8
|
+
waiting: Record<GatewayPostgresLane, number>;
|
|
9
|
+
cancelled: number;
|
|
10
|
+
commitPriorityGrants: number;
|
|
11
|
+
};
|
|
12
|
+
|
|
13
|
+
export class GatewayPostgresAdmissionCancelledError extends Error {
|
|
14
|
+
constructor() {
|
|
15
|
+
super('Gateway Postgres admission was cancelled.');
|
|
16
|
+
this.name = 'GatewayPostgresAdmissionCancelledError';
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
type Waiter = {
|
|
21
|
+
lane: GatewayPostgresLane;
|
|
22
|
+
signal: AbortSignal | null;
|
|
23
|
+
onAbort: (() => void) | null;
|
|
24
|
+
resolve: (release: () => void) => void;
|
|
25
|
+
reject: (error: Error) => void;
|
|
26
|
+
settled: boolean;
|
|
27
|
+
};
|
|
28
|
+
|
|
29
|
+
export class GatewayPostgresAdmission {
|
|
30
|
+
readonly #waiting: Record<GatewayPostgresLane, Waiter[]> = {
|
|
31
|
+
ordinary: [],
|
|
32
|
+
commit: [],
|
|
33
|
+
};
|
|
34
|
+
#ordinaryActive = 0;
|
|
35
|
+
#commitActive = 0;
|
|
36
|
+
#cancelled = 0;
|
|
37
|
+
#commitPriorityGrants = 0;
|
|
38
|
+
|
|
39
|
+
async acquire(
|
|
40
|
+
lane: GatewayPostgresLane,
|
|
41
|
+
options: { signal?: AbortSignal } = {},
|
|
42
|
+
): Promise<() => void> {
|
|
43
|
+
if (options.signal?.aborted) {
|
|
44
|
+
this.#cancelled += 1;
|
|
45
|
+
throw new GatewayPostgresAdmissionCancelledError();
|
|
46
|
+
}
|
|
47
|
+
if (this.#canGrant(lane) && this.#waiting.commit.length === 0) {
|
|
48
|
+
return this.#grant(lane);
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
return await new Promise<() => void>((resolve, reject) => {
|
|
52
|
+
const waiter: Waiter = {
|
|
53
|
+
lane,
|
|
54
|
+
signal: options.signal ?? null,
|
|
55
|
+
onAbort: null,
|
|
56
|
+
resolve,
|
|
57
|
+
reject,
|
|
58
|
+
settled: false,
|
|
59
|
+
};
|
|
60
|
+
waiter.onAbort = () => {
|
|
61
|
+
if (!this.#remove(waiter)) return;
|
|
62
|
+
this.#cancelled += 1;
|
|
63
|
+
reject(new GatewayPostgresAdmissionCancelledError());
|
|
64
|
+
};
|
|
65
|
+
waiter.signal?.addEventListener('abort', waiter.onAbort, { once: true });
|
|
66
|
+
this.#waiting[lane].push(waiter);
|
|
67
|
+
if (waiter.signal?.aborted) {
|
|
68
|
+
waiter.onAbort();
|
|
69
|
+
return;
|
|
70
|
+
}
|
|
71
|
+
this.#drain();
|
|
72
|
+
});
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
snapshot(): GatewayPostgresAdmissionSnapshot {
|
|
76
|
+
return {
|
|
77
|
+
active: {
|
|
78
|
+
ordinary: this.#ordinaryActive,
|
|
79
|
+
commit: this.#commitActive,
|
|
80
|
+
total: this.#activeTotal(),
|
|
81
|
+
},
|
|
82
|
+
waiting: {
|
|
83
|
+
ordinary: this.#waiting.ordinary.length,
|
|
84
|
+
commit: this.#waiting.commit.length,
|
|
85
|
+
},
|
|
86
|
+
cancelled: this.#cancelled,
|
|
87
|
+
commitPriorityGrants: this.#commitPriorityGrants,
|
|
88
|
+
};
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
#activeTotal(): number {
|
|
92
|
+
return this.#ordinaryActive + this.#commitActive;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
#canGrant(lane: GatewayPostgresLane): boolean {
|
|
96
|
+
if (this.#activeTotal() >= GATEWAY_POSTGRES_MAX_TOTAL_ACTIVE) return false;
|
|
97
|
+
return (
|
|
98
|
+
lane === 'commit' ||
|
|
99
|
+
this.#ordinaryActive < GATEWAY_POSTGRES_MAX_ORDINARY_ACTIVE
|
|
100
|
+
);
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
#grant(lane: GatewayPostgresLane): () => void {
|
|
104
|
+
if (lane === 'ordinary') this.#ordinaryActive += 1;
|
|
105
|
+
else this.#commitActive += 1;
|
|
106
|
+
let released = false;
|
|
107
|
+
return () => {
|
|
108
|
+
if (released) return;
|
|
109
|
+
released = true;
|
|
110
|
+
if (lane === 'ordinary') this.#ordinaryActive -= 1;
|
|
111
|
+
else this.#commitActive -= 1;
|
|
112
|
+
this.#drain();
|
|
113
|
+
};
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
#drain(): void {
|
|
117
|
+
while (this.#activeTotal() < GATEWAY_POSTGRES_MAX_TOTAL_ACTIVE) {
|
|
118
|
+
const commit = this.#waiting.commit.shift();
|
|
119
|
+
if (commit) {
|
|
120
|
+
if (this.#waiting.ordinary.length > 0) {
|
|
121
|
+
this.#commitPriorityGrants += 1;
|
|
122
|
+
}
|
|
123
|
+
this.#resolve(commit);
|
|
124
|
+
continue;
|
|
125
|
+
}
|
|
126
|
+
if (
|
|
127
|
+
this.#ordinaryActive < GATEWAY_POSTGRES_MAX_ORDINARY_ACTIVE &&
|
|
128
|
+
this.#waiting.ordinary.length > 0
|
|
129
|
+
) {
|
|
130
|
+
this.#resolve(this.#waiting.ordinary.shift()!);
|
|
131
|
+
continue;
|
|
132
|
+
}
|
|
133
|
+
return;
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
#resolve(waiter: Waiter): void {
|
|
138
|
+
waiter.settled = true;
|
|
139
|
+
this.#cleanup(waiter);
|
|
140
|
+
waiter.resolve(this.#grant(waiter.lane));
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
#remove(waiter: Waiter): boolean {
|
|
144
|
+
if (waiter.settled) return false;
|
|
145
|
+
const queue = this.#waiting[waiter.lane];
|
|
146
|
+
const index = queue.indexOf(waiter);
|
|
147
|
+
if (index < 0) return false;
|
|
148
|
+
waiter.settled = true;
|
|
149
|
+
queue.splice(index, 1);
|
|
150
|
+
this.#cleanup(waiter);
|
|
151
|
+
return true;
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
#cleanup(waiter: Waiter): void {
|
|
155
|
+
if (waiter.signal && waiter.onAbort) {
|
|
156
|
+
waiter.signal.removeEventListener('abort', waiter.onAbort);
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
}
|
package/dist/bundling-sources/shared_libs/play-runtime/governor/app-runtime-rate-state-backend.ts
CHANGED
|
@@ -193,7 +193,10 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
|
|
|
193
193
|
// Duration alone decides. Neither hold is over threshold → let it sleep.
|
|
194
194
|
if (
|
|
195
195
|
coolWait <= PROVIDER_EXHAUSTED_MAX_WAIT_MS &&
|
|
196
|
-
!(
|
|
196
|
+
!(
|
|
197
|
+
claimActive &&
|
|
198
|
+
cooldown.claimedRetryAtMs - now > PROVIDER_EXHAUSTED_MAX_WAIT_MS
|
|
199
|
+
)
|
|
197
200
|
) {
|
|
198
201
|
return;
|
|
199
202
|
}
|
|
@@ -273,20 +276,19 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
|
|
|
273
276
|
}
|
|
274
277
|
|
|
275
278
|
/**
|
|
276
|
-
*
|
|
277
|
-
*
|
|
278
|
-
*
|
|
279
|
-
* buckets
|
|
280
|
-
*
|
|
281
|
-
*
|
|
282
|
-
*
|
|
279
|
+
* Size the next lease from live demand and the provider's real capacity.
|
|
280
|
+
*
|
|
281
|
+
* Pure-rate buckets retain a small prefetch floor. Concurrency-bearing
|
|
282
|
+
* buckets do not: each unused permit pins a durable provider slot, so request
|
|
283
|
+
* only the live waiter count, capped by both the rate horizon and declared
|
|
284
|
+
* max concurrency. This lets a 30-concurrent provider fill in one round trip
|
|
285
|
+
* without reserving a single slot that cannot be used.
|
|
283
286
|
*/
|
|
284
287
|
private requestedForBucket(
|
|
285
288
|
bucketId: string,
|
|
286
289
|
rules: readonly PacingRule[],
|
|
287
290
|
): number {
|
|
288
291
|
const floor = APP_RUNTIME_RATE_STATE_LEASE_BLOCK_SIZE;
|
|
289
|
-
if (rules.some((rule) => rule.maxConcurrency != null)) return floor;
|
|
290
292
|
const configuredRps = Math.min(
|
|
291
293
|
...rules.map((rule) => (rule.requestsPerWindow / rule.windowMs) * 1_000),
|
|
292
294
|
);
|
|
@@ -294,15 +296,28 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
|
|
|
294
296
|
1,
|
|
295
297
|
Math.ceil(this.effectiveRps.get(bucketId) ?? configuredRps),
|
|
296
298
|
);
|
|
297
|
-
const
|
|
299
|
+
const rateBudget = Math.min(
|
|
298
300
|
horizonBudget,
|
|
299
301
|
...rules
|
|
300
302
|
.map((rule) => rule.requestsPerWindow)
|
|
301
303
|
.filter((value) => Number.isFinite(value) && value > 0),
|
|
302
304
|
);
|
|
303
|
-
if (!Number.isFinite(
|
|
305
|
+
if (!Number.isFinite(rateBudget) || rateBudget <= 0) return floor;
|
|
304
306
|
const waiters = this.waiters.get(bucketId) ?? 1;
|
|
305
|
-
|
|
307
|
+
const concurrencyLimits = rules.flatMap((rule) =>
|
|
308
|
+
rule.maxConcurrency != null &&
|
|
309
|
+
Number.isFinite(rule.maxConcurrency) &&
|
|
310
|
+
rule.maxConcurrency > 0
|
|
311
|
+
? [Math.trunc(rule.maxConcurrency)]
|
|
312
|
+
: [],
|
|
313
|
+
);
|
|
314
|
+
if (concurrencyLimits.length > 0) {
|
|
315
|
+
return Math.max(
|
|
316
|
+
1,
|
|
317
|
+
Math.min(waiters, Math.trunc(rateBudget), ...concurrencyLimits),
|
|
318
|
+
);
|
|
319
|
+
}
|
|
320
|
+
return Math.max(floor, Math.min(waiters, Math.trunc(rateBudget)));
|
|
306
321
|
}
|
|
307
322
|
|
|
308
323
|
/**
|
|
@@ -344,7 +359,11 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
|
|
|
344
359
|
*/
|
|
345
360
|
private providerExhaustedFromResponse(
|
|
346
361
|
bucketId: string,
|
|
347
|
-
response: {
|
|
362
|
+
response: {
|
|
363
|
+
waitMs: number;
|
|
364
|
+
coolUntilMs?: number;
|
|
365
|
+
claimedRetryAtMs?: number;
|
|
366
|
+
},
|
|
348
367
|
): ProviderExhaustedError {
|
|
349
368
|
const claimedRetryAtMs = Number(response.claimedRetryAtMs);
|
|
350
369
|
const coolUntilMs = Number(response.coolUntilMs);
|
|
@@ -507,6 +526,7 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
|
|
|
507
526
|
response: {
|
|
508
527
|
granted: number;
|
|
509
528
|
leaseIds?: unknown;
|
|
529
|
+
serverNowMs?: unknown;
|
|
510
530
|
scheduledAtMs?: unknown;
|
|
511
531
|
},
|
|
512
532
|
): void {
|
|
@@ -531,9 +551,18 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
|
|
|
531
551
|
// has already debited the block under the legacy token-bucket contract, so
|
|
532
552
|
// keep those permits immediately usable until every gateway serves the new
|
|
533
553
|
// globally reserved timestamps.
|
|
534
|
-
const
|
|
554
|
+
const gatewayScheduledAtMs = Array.isArray(response.scheduledAtMs)
|
|
535
555
|
? response.scheduledAtMs.map(Number)
|
|
536
556
|
: Array.from({ length: response.granted }, () => this.now());
|
|
557
|
+
const serverNowMs = Number(response.serverNowMs);
|
|
558
|
+
const receivedAtMs = this.now();
|
|
559
|
+
const scheduledAtMs =
|
|
560
|
+
Number.isFinite(serverNowMs) && serverNowMs >= 0
|
|
561
|
+
? gatewayScheduledAtMs.map(
|
|
562
|
+
(scheduledAtMs) =>
|
|
563
|
+
receivedAtMs + Math.max(0, scheduledAtMs - serverNowMs),
|
|
564
|
+
)
|
|
565
|
+
: gatewayScheduledAtMs;
|
|
537
566
|
if (
|
|
538
567
|
scheduledAtMs.length !== response.granted ||
|
|
539
568
|
scheduledAtMs.some((value) => !Number.isFinite(value) || value < 0)
|
|
@@ -545,12 +574,10 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
|
|
|
545
574
|
this.mergeBlock(
|
|
546
575
|
bucketId,
|
|
547
576
|
rateScopeToken,
|
|
548
|
-
scheduledAtMs.map(
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
],
|
|
553
|
-
),
|
|
577
|
+
scheduledAtMs.map((scheduledAtMs, index): [number, string | null] => [
|
|
578
|
+
scheduledAtMs,
|
|
579
|
+
hasConcurrency ? (leaseIds[index] ?? null) : null,
|
|
580
|
+
]),
|
|
554
581
|
rulesKey,
|
|
555
582
|
ordered,
|
|
556
583
|
);
|
|
@@ -639,9 +666,7 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
|
|
|
639
666
|
bucketId,
|
|
640
667
|
existing.rateScopeToken,
|
|
641
668
|
existing.rules,
|
|
642
|
-
existing.permits.flatMap(([, leaseId]) =>
|
|
643
|
-
leaseId ? [leaseId] : [],
|
|
644
|
-
),
|
|
669
|
+
existing.permits.flatMap(([, leaseId]) => (leaseId ? [leaseId] : [])),
|
|
645
670
|
);
|
|
646
671
|
this.blocks.set(bucketId, {
|
|
647
672
|
permits,
|
|
@@ -668,9 +693,7 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
|
|
|
668
693
|
bucketId,
|
|
669
694
|
block.rateScopeToken,
|
|
670
695
|
block.rules,
|
|
671
|
-
block.permits.flatMap(([, leaseId]) =>
|
|
672
|
-
leaseId ? [leaseId] : [],
|
|
673
|
-
),
|
|
696
|
+
block.permits.flatMap(([, leaseId]) => (leaseId ? [leaseId] : [])),
|
|
674
697
|
);
|
|
675
698
|
return null;
|
|
676
699
|
}
|
|
@@ -699,9 +722,7 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
|
|
|
699
722
|
bucketId,
|
|
700
723
|
block.rateScopeToken,
|
|
701
724
|
block.rules,
|
|
702
|
-
block.permits.flatMap(([, leaseId]) =>
|
|
703
|
-
leaseId ? [leaseId] : [],
|
|
704
|
-
),
|
|
725
|
+
block.permits.flatMap(([, leaseId]) => (leaseId ? [leaseId] : [])),
|
|
705
726
|
);
|
|
706
727
|
}
|
|
707
728
|
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import { isAppRuntimeApiCapacityError } from './app-runtime-api';
|
|
2
|
+
|
|
1
3
|
/**
|
|
2
4
|
* Runtime persistence-failure circuit breaker.
|
|
3
5
|
*
|
|
@@ -113,6 +115,7 @@ export function tripRuntimePersistenceLatch(
|
|
|
113
115
|
latch: RuntimePersistenceLatch,
|
|
114
116
|
error: unknown,
|
|
115
117
|
): void {
|
|
118
|
+
if (isAppRuntimeApiCapacityError(error)) return;
|
|
116
119
|
if (latch.tripped) return;
|
|
117
120
|
latch.tripped = true;
|
|
118
121
|
latch.cause = formatLatchCause(error);
|
|
@@ -36,6 +36,12 @@ export type PlayRunnerRateStateAcquireInput = {
|
|
|
36
36
|
export type PlayRunnerRateStateAcquireResult = {
|
|
37
37
|
granted: number;
|
|
38
38
|
waitMs: number;
|
|
39
|
+
/**
|
|
40
|
+
* Gateway clock used to produce `scheduledAtMs`. Runners translate the
|
|
41
|
+
* schedule into relative delays so cross-machine clock skew cannot compress
|
|
42
|
+
* provider spacing.
|
|
43
|
+
*/
|
|
44
|
+
serverNowMs?: number;
|
|
39
45
|
/**
|
|
40
46
|
* One globally reserved dispatch timestamp per granted permit. The runtime
|
|
41
47
|
* may cache the block, but must not dispatch a permit before its timestamp.
|
|
@@ -134,20 +140,6 @@ export interface PlayRunnerContextConfig {
|
|
|
134
140
|
postgresSessionUnwrapKey?: string | null;
|
|
135
141
|
rateStateBackend?: PlayRunnerRateStateBackendConfig;
|
|
136
142
|
runtimeSchedulerSchema?: string | null;
|
|
137
|
-
/**
|
|
138
|
-
* Execution profile a scheduler-backed `ctx.runPlay` child must be launched
|
|
139
|
-
* under. The runner posts it as the `profile` on the child `/api/v2/plays/run`
|
|
140
|
-
* submission so a child inherits the parent's scheduler (e.g. `absurd`) instead
|
|
141
|
-
* of defaulting away from the parent scheduler. Absent means `absurd`.
|
|
142
|
-
*/
|
|
143
|
-
childRunProfile?: string | null;
|
|
144
|
-
/**
|
|
145
|
-
* The parent worker's absurd release lane id. Posted as the
|
|
146
|
-
* `x-deepline-absurd-release` header on child `ctx.runPlay` launches so the
|
|
147
|
-
* child pins to the parent's lane (defense-in-depth). Absent off the absurd
|
|
148
|
-
* path or on a pre-release parent.
|
|
149
|
-
*/
|
|
150
|
-
absurdReleaseId?: string | null;
|
|
151
143
|
/**
|
|
152
144
|
* Push-execution wiring for the Daytona sandbox runner. When present the
|
|
153
145
|
* runner keeps the Absurd run claim alive itself (heartbeating the receipt
|
|
@@ -2236,8 +2236,8 @@ async function missingRuntimeWorkReceiptSelfHealColumns(
|
|
|
2236
2236
|
);
|
|
2237
2237
|
}
|
|
2238
2238
|
|
|
2239
|
-
// COMPAT / SELF-HEAL — owner: runtime. Removal milestone:
|
|
2240
|
-
// cutover
|
|
2239
|
+
// COMPAT / SELF-HEAL — owner: runtime. Removal milestone: runtime-ledger
|
|
2240
|
+
// cutover. The customer-Neon
|
|
2241
2241
|
// work-receipt table (lease columns + failure_kind) is the dying surface. The
|
|
2242
2242
|
// F2 rolling migration (scripts/migrate-play-runtime-storage.ts) backfills these
|
|
2243
2243
|
// columns; this lazy CREATE/ALTER self-heal is a compat-window safety net and is
|
|
@@ -4175,6 +4175,9 @@ export async function claimRuntimeWorkReceipt(
|
|
|
4175
4175
|
input.leaseTtlMs,
|
|
4176
4176
|
PLAY_RUNTIME_WORK_RECEIPT_LEASE_TTL_MS,
|
|
4177
4177
|
);
|
|
4178
|
+
// A successful lease-bearing claim is the execution transition. Keep the
|
|
4179
|
+
// stored queued status for mixed-version readers; the runner does not add
|
|
4180
|
+
// a second queued -> running write on the provider hot path.
|
|
4178
4181
|
const claimStatus =
|
|
4179
4182
|
input.leaseAware === true
|
|
4180
4183
|
? RECEIPT_STATUS_QUEUED
|