deepline 0.1.291 → 0.1.293

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -155,7 +155,7 @@ export const SDK_RELEASE = {
155
155
  // 0.1.253 makes play-page browser opening opt-in and retires --no-open.
156
156
  // 0.1.254 removes the internal operations tree from the published SDK CLI.
157
157
  // Operators use the checkout-local deepline-admin binary instead.
158
- version: '0.1.291',
158
+ version: '0.1.293',
159
159
  contracts: {
160
160
  api: {
161
161
  name: 'sdk-http-api',
@@ -364,6 +364,9 @@ export type WorkerRuntimeApiContext = {
364
364
  runtimeTestFaultHeader?: string | null;
365
365
  fetch?: typeof fetch;
366
366
  requestTimeoutMs?: number | null;
367
+ /** A higher-level writer owns retries and preserves the exact request body. */
368
+ retryPolicy?: 'default' | 'none';
369
+ signal?: AbortSignal;
367
370
  };
368
371
 
369
372
  const APP_RUNTIME_API_RETRY_DELAYS_MS = [100, 250, 500, 1_000] as const;
@@ -1007,6 +1010,60 @@ export class AppRuntimeApiResponseError extends Error {
1007
1010
  }
1008
1011
  }
1009
1012
 
1013
+ const APP_RUNTIME_CAPACITY_ERROR_CODES = new Set([
1014
+ 'receipt_db_admission_backpressure',
1015
+ 'scheduler_capacity',
1016
+ 'runtime_postgres_admission_backpressure',
1017
+ 'RUNTIME_SCHEDULER_SATURATED',
1018
+ ]);
1019
+
1020
+ /**
1021
+ * The runtime persistence plane is temporarily full. This is an admission
1022
+ * signal, not a transport failure: callers must park the same operation rather
1023
+ * than spending the ordinary request retry ladder.
1024
+ */
1025
+ export class AppRuntimeApiCapacityError extends AppRuntimeApiResponseError {
1026
+ readonly retryAfterMs: number;
1027
+
1028
+ constructor(input: {
1029
+ action: RuntimeApiRequest['action'];
1030
+ status: number;
1031
+ code: string;
1032
+ requestId?: string | null;
1033
+ detail: string;
1034
+ retryAfterMs: number;
1035
+ }) {
1036
+ super({
1037
+ action: input.action,
1038
+ status: input.status,
1039
+ code: input.code,
1040
+ requestId: input.requestId,
1041
+ retryable: true,
1042
+ detail: input.detail,
1043
+ });
1044
+ this.name = 'AppRuntimeApiCapacityError';
1045
+ this.retryAfterMs = Math.max(0, Math.floor(input.retryAfterMs));
1046
+ }
1047
+ }
1048
+
1049
+ export function isAppRuntimeApiCapacityError(
1050
+ error: unknown,
1051
+ ): error is AppRuntimeApiCapacityError {
1052
+ if (error instanceof AppRuntimeApiCapacityError) return true;
1053
+ if (!error || typeof error !== 'object') return false;
1054
+ const candidate = error as {
1055
+ name?: unknown;
1056
+ code?: unknown;
1057
+ retryAfterMs?: unknown;
1058
+ };
1059
+ return (
1060
+ candidate.name === 'AppRuntimeApiCapacityError' &&
1061
+ typeof candidate.code === 'string' &&
1062
+ APP_RUNTIME_CAPACITY_ERROR_CODES.has(candidate.code) &&
1063
+ typeof candidate.retryAfterMs === 'number'
1064
+ );
1065
+ }
1066
+
1010
1067
  function resolveAppRuntimeApiUrl(context: WorkerRuntimeApiContext): string {
1011
1068
  const baseUrl = context.baseUrl.trim().replace(/\/$/, '');
1012
1069
  return `${baseUrl}${PLAY_RUNTIME_API_COMPAT_PATH}`;
@@ -1032,7 +1089,8 @@ async function postAppRuntimeApi<TResponse>(
1032
1089
  fetchImpl: runtimeFetch,
1033
1090
  });
1034
1091
 
1035
- const maxAttempts = appRuntimeMaxAttempts(body.action);
1092
+ const maxAttempts =
1093
+ context.retryPolicy === 'none' ? 1 : appRuntimeMaxAttempts(body.action);
1036
1094
  const retryTelemetry: AppRuntimeRetryTelemetry = {
1037
1095
  enabled: isRuntimeStepReceiptAction(body.action),
1038
1096
  startedAt: Date.now(),
@@ -1066,7 +1124,12 @@ async function postAppRuntimeApi<TResponse>(
1066
1124
  : {}),
1067
1125
  },
1068
1126
  body: JSON.stringify(body),
1069
- signal: AbortSignal.timeout(requestTimeoutMs),
1127
+ signal: context.signal
1128
+ ? AbortSignal.any([
1129
+ context.signal,
1130
+ AbortSignal.timeout(requestTimeoutMs),
1131
+ ])
1132
+ : AbortSignal.timeout(requestTimeoutMs),
1070
1133
  });
1071
1134
  } catch (error) {
1072
1135
  if (
@@ -1176,6 +1239,21 @@ async function postAppRuntimeApi<TResponse>(
1176
1239
  });
1177
1240
  continue;
1178
1241
  }
1242
+ const responseCode = appRuntimeErrorCode(response, responseText);
1243
+ if (
1244
+ response.status === 503 &&
1245
+ responseCode &&
1246
+ APP_RUNTIME_CAPACITY_ERROR_CODES.has(responseCode)
1247
+ ) {
1248
+ throw new AppRuntimeApiCapacityError({
1249
+ action: body.action,
1250
+ status: response.status,
1251
+ code: responseCode,
1252
+ requestId: response.headers.get('x-deepline-request-id')?.trim(),
1253
+ detail: summarizeAppRuntimeErrorBody(responseText),
1254
+ retryAfterMs: appRuntimeRetryAfterMs(response, responseText) ?? 1_000,
1255
+ });
1256
+ }
1179
1257
  if (
1180
1258
  attempt < maxAttempts &&
1181
1259
  isRetryableAppRuntimeResponse({
@@ -1221,7 +1299,7 @@ async function postAppRuntimeApi<TResponse>(
1221
1299
  maxAttempts,
1222
1300
  finalAttemptStartedAt: attemptStartedAt,
1223
1301
  });
1224
- const code = appRuntimeErrorCode(response, responseText);
1302
+ const code = responseCode;
1225
1303
  const requestId = response.headers.get('x-deepline-request-id')?.trim();
1226
1304
  throw new AppRuntimeApiResponseError({
1227
1305
  action: body.action,
@@ -2052,30 +2052,6 @@ export class PlayContextImpl {
2052
2052
  byLeaseId.set(target.leaseId, keys);
2053
2053
  }
2054
2054
 
2055
- if (
2056
- this.#options.markRuntimeStepReceiptsRunning ||
2057
- this.#options.markRuntimeStepReceiptRunning
2058
- ) {
2059
- const receipts = await this.markRuntimeStepReceiptsRunning(
2060
- targets.map((target) => ({
2061
- key: target.receiptKey,
2062
- runId: this.currentReceiptOwnerRunId,
2063
- leaseId: target.leaseId,
2064
- })),
2065
- );
2066
- for (const target of targets) {
2067
- const receipt = receipts.get(target.receiptKey);
2068
- if (!this.runtimeToolReceiptStillOwned(receipt, target.leaseId)) {
2069
- throw new RuntimeReceiptLeaseLostError({
2070
- receiptKey: target.receiptKey,
2071
- runId: this.currentReceiptOwnerRunId,
2072
- leaseId: target.leaseId,
2073
- });
2074
- }
2075
- }
2076
- return;
2077
- }
2078
-
2079
2055
  if (this.#options.heartbeatRuntimeStepReceipts) {
2080
2056
  for (const [leaseId, keys] of byLeaseId) {
2081
2057
  const receipts = await this.#options.heartbeatRuntimeStepReceipts({
@@ -5429,7 +5405,7 @@ export class PlayContextImpl {
5429
5405
  let activeFieldName: string | null = null;
5430
5406
 
5431
5407
  // Global row slot keeps concurrent maps in the same run under rowMax. The
5432
- // worker pool above enforces this map's requested/default concurrency.
5408
+ // worker pool below enforces this map's requested/default concurrency.
5433
5409
  const globalRowSlot = await this.resourceGovernor.acquireRow({
5434
5410
  estimatedBytes: runtimeMapJsonByteLength(baseRow),
5435
5411
  });
@@ -6602,8 +6578,9 @@ export class PlayContextImpl {
6602
6578
  : physicalDirectKey,
6603
6579
  timeoutMs: resolveToolRuntimeTimeoutMs(toolId, options?.timeoutMs),
6604
6580
  ...(directReceiptLeaseId &&
6605
- (this.#options.markRuntimeStepReceiptRunning ||
6606
- this.#options.markRuntimeStepReceiptsRunning)
6581
+ (this.#options.heartbeatRuntimeStepReceipts ||
6582
+ this.#options.getRuntimeStepReceipt ||
6583
+ this.#options.getRuntimeStepReceipts)
6607
6584
  ? {
6608
6585
  beforeProviderCall: async () => {
6609
6586
  await this.assertRuntimeToolReceiptOwnership([
@@ -6618,19 +6595,6 @@ export class PlayContextImpl {
6618
6595
  },
6619
6596
  ]);
6620
6597
  },
6621
- ...(this.#options.markRuntimeStepReceiptsQueued
6622
- ? {
6623
- parkProviderCall: async () => {
6624
- await this.markRuntimeStepReceiptsQueued([
6625
- {
6626
- key: directCacheKey,
6627
- runId: this.currentReceiptOwnerRunId,
6628
- leaseId: directReceiptLeaseId,
6629
- },
6630
- ]);
6631
- },
6632
- }
6633
- : {}),
6634
6598
  }
6635
6599
  : {}),
6636
6600
  });
@@ -8016,8 +7980,6 @@ export class PlayContextImpl {
8016
7980
  {
8017
7981
  beforeProviderCall: () =>
8018
7982
  this.assertRuntimeToolReceiptOwnership([owner]),
8019
- parkProviderCall: () =>
8020
- this.parkRuntimeToolReceiptsQueued([owner]),
8021
7983
  durableCallReceiptKey: receiptKey,
8022
7984
  executionAuthScopeDigest: owner.executionAuthScopeDigest,
8023
7985
  receiptLeaseExpiresAt: owner.receiptLeaseExpiresAt,
@@ -8298,8 +8260,6 @@ export class PlayContextImpl {
8298
8260
  this.assertRuntimeToolReceiptOwnership(
8299
8261
  batch.memberRequests,
8300
8262
  ),
8301
- parkProviderCall: () =>
8302
- this.parkRuntimeToolReceiptsQueued(batch.memberRequests),
8303
8263
  },
8304
8264
  );
8305
8265
  },
@@ -8447,8 +8407,6 @@ export class PlayContextImpl {
8447
8407
  {
8448
8408
  beforeProviderCall: () =>
8449
8409
  this.assertRuntimeToolReceiptOwnership([request]),
8450
- parkProviderCall: () =>
8451
- this.parkRuntimeToolReceiptsQueued([request]),
8452
8410
  ...(request.receiptKey
8453
8411
  ? {
8454
8412
  durableCallReceiptKey: request.receiptKey,
@@ -536,20 +536,6 @@ export interface ContextOptions {
536
536
  artifactHash?: string | null;
537
537
  convexUrl?: string;
538
538
  runtimeSchedulerSchema?: string | null;
539
- /**
540
- * Execution profile for scheduler-backed `ctx.runPlay` child launches. Threaded
541
- * from the parent launch so a child inherits the parent's scheduler profile
542
- * (e.g. `absurd`) on the child `/api/v2/plays/run` submission. Absent defaults
543
- * to the `absurd` profile.
544
- */
545
- childRunProfile?: string | null;
546
- /**
547
- * The parent worker's own absurd release lane id. Threaded onto child
548
- * `ctx.runPlay` launches via the `x-deepline-absurd-release` header so the
549
- * child pins to the parent's lane (defense-in-depth). Null off the absurd
550
- * path or on a pre-release parent.
551
- */
552
- absurdReleaseId?: string | null;
553
539
  staticPipeline?: PlayStaticPipeline | null;
554
540
  workflowId?: string;
555
541
  sessionId?: string;
@@ -0,0 +1,159 @@
1
+ export type GatewayPostgresLane = 'ordinary' | 'commit';
2
+
3
+ export const GATEWAY_POSTGRES_MAX_ORDINARY_ACTIVE = 8;
4
+ export const GATEWAY_POSTGRES_MAX_TOTAL_ACTIVE = 10;
5
+
6
+ export type GatewayPostgresAdmissionSnapshot = {
7
+ active: Record<GatewayPostgresLane, number> & { total: number };
8
+ waiting: Record<GatewayPostgresLane, number>;
9
+ cancelled: number;
10
+ commitPriorityGrants: number;
11
+ };
12
+
13
+ export class GatewayPostgresAdmissionCancelledError extends Error {
14
+ constructor() {
15
+ super('Gateway Postgres admission was cancelled.');
16
+ this.name = 'GatewayPostgresAdmissionCancelledError';
17
+ }
18
+ }
19
+
20
+ type Waiter = {
21
+ lane: GatewayPostgresLane;
22
+ signal: AbortSignal | null;
23
+ onAbort: (() => void) | null;
24
+ resolve: (release: () => void) => void;
25
+ reject: (error: Error) => void;
26
+ settled: boolean;
27
+ };
28
+
29
+ export class GatewayPostgresAdmission {
30
+ readonly #waiting: Record<GatewayPostgresLane, Waiter[]> = {
31
+ ordinary: [],
32
+ commit: [],
33
+ };
34
+ #ordinaryActive = 0;
35
+ #commitActive = 0;
36
+ #cancelled = 0;
37
+ #commitPriorityGrants = 0;
38
+
39
+ async acquire(
40
+ lane: GatewayPostgresLane,
41
+ options: { signal?: AbortSignal } = {},
42
+ ): Promise<() => void> {
43
+ if (options.signal?.aborted) {
44
+ this.#cancelled += 1;
45
+ throw new GatewayPostgresAdmissionCancelledError();
46
+ }
47
+ if (this.#canGrant(lane) && this.#waiting.commit.length === 0) {
48
+ return this.#grant(lane);
49
+ }
50
+
51
+ return await new Promise<() => void>((resolve, reject) => {
52
+ const waiter: Waiter = {
53
+ lane,
54
+ signal: options.signal ?? null,
55
+ onAbort: null,
56
+ resolve,
57
+ reject,
58
+ settled: false,
59
+ };
60
+ waiter.onAbort = () => {
61
+ if (!this.#remove(waiter)) return;
62
+ this.#cancelled += 1;
63
+ reject(new GatewayPostgresAdmissionCancelledError());
64
+ };
65
+ waiter.signal?.addEventListener('abort', waiter.onAbort, { once: true });
66
+ this.#waiting[lane].push(waiter);
67
+ if (waiter.signal?.aborted) {
68
+ waiter.onAbort();
69
+ return;
70
+ }
71
+ this.#drain();
72
+ });
73
+ }
74
+
75
+ snapshot(): GatewayPostgresAdmissionSnapshot {
76
+ return {
77
+ active: {
78
+ ordinary: this.#ordinaryActive,
79
+ commit: this.#commitActive,
80
+ total: this.#activeTotal(),
81
+ },
82
+ waiting: {
83
+ ordinary: this.#waiting.ordinary.length,
84
+ commit: this.#waiting.commit.length,
85
+ },
86
+ cancelled: this.#cancelled,
87
+ commitPriorityGrants: this.#commitPriorityGrants,
88
+ };
89
+ }
90
+
91
+ #activeTotal(): number {
92
+ return this.#ordinaryActive + this.#commitActive;
93
+ }
94
+
95
+ #canGrant(lane: GatewayPostgresLane): boolean {
96
+ if (this.#activeTotal() >= GATEWAY_POSTGRES_MAX_TOTAL_ACTIVE) return false;
97
+ return (
98
+ lane === 'commit' ||
99
+ this.#ordinaryActive < GATEWAY_POSTGRES_MAX_ORDINARY_ACTIVE
100
+ );
101
+ }
102
+
103
+ #grant(lane: GatewayPostgresLane): () => void {
104
+ if (lane === 'ordinary') this.#ordinaryActive += 1;
105
+ else this.#commitActive += 1;
106
+ let released = false;
107
+ return () => {
108
+ if (released) return;
109
+ released = true;
110
+ if (lane === 'ordinary') this.#ordinaryActive -= 1;
111
+ else this.#commitActive -= 1;
112
+ this.#drain();
113
+ };
114
+ }
115
+
116
+ #drain(): void {
117
+ while (this.#activeTotal() < GATEWAY_POSTGRES_MAX_TOTAL_ACTIVE) {
118
+ const commit = this.#waiting.commit.shift();
119
+ if (commit) {
120
+ if (this.#waiting.ordinary.length > 0) {
121
+ this.#commitPriorityGrants += 1;
122
+ }
123
+ this.#resolve(commit);
124
+ continue;
125
+ }
126
+ if (
127
+ this.#ordinaryActive < GATEWAY_POSTGRES_MAX_ORDINARY_ACTIVE &&
128
+ this.#waiting.ordinary.length > 0
129
+ ) {
130
+ this.#resolve(this.#waiting.ordinary.shift()!);
131
+ continue;
132
+ }
133
+ return;
134
+ }
135
+ }
136
+
137
+ #resolve(waiter: Waiter): void {
138
+ waiter.settled = true;
139
+ this.#cleanup(waiter);
140
+ waiter.resolve(this.#grant(waiter.lane));
141
+ }
142
+
143
+ #remove(waiter: Waiter): boolean {
144
+ if (waiter.settled) return false;
145
+ const queue = this.#waiting[waiter.lane];
146
+ const index = queue.indexOf(waiter);
147
+ if (index < 0) return false;
148
+ waiter.settled = true;
149
+ queue.splice(index, 1);
150
+ this.#cleanup(waiter);
151
+ return true;
152
+ }
153
+
154
+ #cleanup(waiter: Waiter): void {
155
+ if (waiter.signal && waiter.onAbort) {
156
+ waiter.signal.removeEventListener('abort', waiter.onAbort);
157
+ }
158
+ }
159
+ }
@@ -193,7 +193,10 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
193
193
  // Duration alone decides. Neither hold is over threshold → let it sleep.
194
194
  if (
195
195
  coolWait <= PROVIDER_EXHAUSTED_MAX_WAIT_MS &&
196
- !(claimActive && cooldown.claimedRetryAtMs - now > PROVIDER_EXHAUSTED_MAX_WAIT_MS)
196
+ !(
197
+ claimActive &&
198
+ cooldown.claimedRetryAtMs - now > PROVIDER_EXHAUSTED_MAX_WAIT_MS
199
+ )
197
200
  ) {
198
201
  return;
199
202
  }
@@ -273,20 +276,19 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
273
276
  }
274
277
 
275
278
  /**
276
- * How many permits to request on the next refill for a bucket. Buckets with any
277
- * `maxConcurrency` rule stay at the fixed floor block: each permit pins a
278
- * durable concurrency slot, so over-reserving would strand real slots. Pure-rate
279
- * buckets demand-size: clamp the live waiter count to
280
- * [block floor, window budget] where the window budget is the smallest
281
- * `requestsPerWindow` across the pure-rate rules (never reserve more than one
282
- * window can hold). A 200-row wave then lands in ~one round trip.
279
+ * Size the next lease from live demand and the provider's real capacity.
280
+ *
281
+ * Pure-rate buckets retain a small prefetch floor. Concurrency-bearing
282
+ * buckets do not: each unused permit pins a durable provider slot, so request
283
+ * only the live waiter count, capped by both the rate horizon and declared
284
+ * max concurrency. This lets a 30-concurrent provider fill in one round trip
285
+ * without reserving a single slot that cannot be used.
283
286
  */
284
287
  private requestedForBucket(
285
288
  bucketId: string,
286
289
  rules: readonly PacingRule[],
287
290
  ): number {
288
291
  const floor = APP_RUNTIME_RATE_STATE_LEASE_BLOCK_SIZE;
289
- if (rules.some((rule) => rule.maxConcurrency != null)) return floor;
290
292
  const configuredRps = Math.min(
291
293
  ...rules.map((rule) => (rule.requestsPerWindow / rule.windowMs) * 1_000),
292
294
  );
@@ -294,15 +296,28 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
294
296
  1,
295
297
  Math.ceil(this.effectiveRps.get(bucketId) ?? configuredRps),
296
298
  );
297
- const windowBudget = Math.min(
299
+ const rateBudget = Math.min(
298
300
  horizonBudget,
299
301
  ...rules
300
302
  .map((rule) => rule.requestsPerWindow)
301
303
  .filter((value) => Number.isFinite(value) && value > 0),
302
304
  );
303
- if (!Number.isFinite(windowBudget) || windowBudget <= 0) return floor;
305
+ if (!Number.isFinite(rateBudget) || rateBudget <= 0) return floor;
304
306
  const waiters = this.waiters.get(bucketId) ?? 1;
305
- return Math.max(floor, Math.min(waiters, Math.trunc(windowBudget)));
307
+ const concurrencyLimits = rules.flatMap((rule) =>
308
+ rule.maxConcurrency != null &&
309
+ Number.isFinite(rule.maxConcurrency) &&
310
+ rule.maxConcurrency > 0
311
+ ? [Math.trunc(rule.maxConcurrency)]
312
+ : [],
313
+ );
314
+ if (concurrencyLimits.length > 0) {
315
+ return Math.max(
316
+ 1,
317
+ Math.min(waiters, Math.trunc(rateBudget), ...concurrencyLimits),
318
+ );
319
+ }
320
+ return Math.max(floor, Math.min(waiters, Math.trunc(rateBudget)));
306
321
  }
307
322
 
308
323
  /**
@@ -344,7 +359,11 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
344
359
  */
345
360
  private providerExhaustedFromResponse(
346
361
  bucketId: string,
347
- response: { waitMs: number; coolUntilMs?: number; claimedRetryAtMs?: number },
362
+ response: {
363
+ waitMs: number;
364
+ coolUntilMs?: number;
365
+ claimedRetryAtMs?: number;
366
+ },
348
367
  ): ProviderExhaustedError {
349
368
  const claimedRetryAtMs = Number(response.claimedRetryAtMs);
350
369
  const coolUntilMs = Number(response.coolUntilMs);
@@ -507,6 +526,7 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
507
526
  response: {
508
527
  granted: number;
509
528
  leaseIds?: unknown;
529
+ serverNowMs?: unknown;
510
530
  scheduledAtMs?: unknown;
511
531
  },
512
532
  ): void {
@@ -531,9 +551,18 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
531
551
  // has already debited the block under the legacy token-bucket contract, so
532
552
  // keep those permits immediately usable until every gateway serves the new
533
553
  // globally reserved timestamps.
534
- const scheduledAtMs = Array.isArray(response.scheduledAtMs)
554
+ const gatewayScheduledAtMs = Array.isArray(response.scheduledAtMs)
535
555
  ? response.scheduledAtMs.map(Number)
536
556
  : Array.from({ length: response.granted }, () => this.now());
557
+ const serverNowMs = Number(response.serverNowMs);
558
+ const receivedAtMs = this.now();
559
+ const scheduledAtMs =
560
+ Number.isFinite(serverNowMs) && serverNowMs >= 0
561
+ ? gatewayScheduledAtMs.map(
562
+ (scheduledAtMs) =>
563
+ receivedAtMs + Math.max(0, scheduledAtMs - serverNowMs),
564
+ )
565
+ : gatewayScheduledAtMs;
537
566
  if (
538
567
  scheduledAtMs.length !== response.granted ||
539
568
  scheduledAtMs.some((value) => !Number.isFinite(value) || value < 0)
@@ -545,12 +574,10 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
545
574
  this.mergeBlock(
546
575
  bucketId,
547
576
  rateScopeToken,
548
- scheduledAtMs.map(
549
- (scheduledAtMs, index): [number, string | null] => [
550
- scheduledAtMs,
551
- hasConcurrency ? (leaseIds[index] ?? null) : null,
552
- ],
553
- ),
577
+ scheduledAtMs.map((scheduledAtMs, index): [number, string | null] => [
578
+ scheduledAtMs,
579
+ hasConcurrency ? (leaseIds[index] ?? null) : null,
580
+ ]),
554
581
  rulesKey,
555
582
  ordered,
556
583
  );
@@ -639,9 +666,7 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
639
666
  bucketId,
640
667
  existing.rateScopeToken,
641
668
  existing.rules,
642
- existing.permits.flatMap(([, leaseId]) =>
643
- leaseId ? [leaseId] : [],
644
- ),
669
+ existing.permits.flatMap(([, leaseId]) => (leaseId ? [leaseId] : [])),
645
670
  );
646
671
  this.blocks.set(bucketId, {
647
672
  permits,
@@ -668,9 +693,7 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
668
693
  bucketId,
669
694
  block.rateScopeToken,
670
695
  block.rules,
671
- block.permits.flatMap(([, leaseId]) =>
672
- leaseId ? [leaseId] : [],
673
- ),
696
+ block.permits.flatMap(([, leaseId]) => (leaseId ? [leaseId] : [])),
674
697
  );
675
698
  return null;
676
699
  }
@@ -699,9 +722,7 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
699
722
  bucketId,
700
723
  block.rateScopeToken,
701
724
  block.rules,
702
- block.permits.flatMap(([, leaseId]) =>
703
- leaseId ? [leaseId] : [],
704
- ),
725
+ block.permits.flatMap(([, leaseId]) => (leaseId ? [leaseId] : [])),
705
726
  );
706
727
  }
707
728
 
@@ -1,3 +1,5 @@
1
+ import { isAppRuntimeApiCapacityError } from './app-runtime-api';
2
+
1
3
  /**
2
4
  * Runtime persistence-failure circuit breaker.
3
5
  *
@@ -113,6 +115,7 @@ export function tripRuntimePersistenceLatch(
113
115
  latch: RuntimePersistenceLatch,
114
116
  error: unknown,
115
117
  ): void {
118
+ if (isAppRuntimeApiCapacityError(error)) return;
116
119
  if (latch.tripped) return;
117
120
  latch.tripped = true;
118
121
  latch.cause = formatLatchCause(error);
@@ -36,6 +36,12 @@ export type PlayRunnerRateStateAcquireInput = {
36
36
  export type PlayRunnerRateStateAcquireResult = {
37
37
  granted: number;
38
38
  waitMs: number;
39
+ /**
40
+ * Gateway clock used to produce `scheduledAtMs`. Runners translate the
41
+ * schedule into relative delays so cross-machine clock skew cannot compress
42
+ * provider spacing.
43
+ */
44
+ serverNowMs?: number;
39
45
  /**
40
46
  * One globally reserved dispatch timestamp per granted permit. The runtime
41
47
  * may cache the block, but must not dispatch a permit before its timestamp.
@@ -134,20 +140,6 @@ export interface PlayRunnerContextConfig {
134
140
  postgresSessionUnwrapKey?: string | null;
135
141
  rateStateBackend?: PlayRunnerRateStateBackendConfig;
136
142
  runtimeSchedulerSchema?: string | null;
137
- /**
138
- * Execution profile a scheduler-backed `ctx.runPlay` child must be launched
139
- * under. The runner posts it as the `profile` on the child `/api/v2/plays/run`
140
- * submission so a child inherits the parent's scheduler (e.g. `absurd`) instead
141
- * of defaulting away from the parent scheduler. Absent means `absurd`.
142
- */
143
- childRunProfile?: string | null;
144
- /**
145
- * The parent worker's absurd release lane id. Posted as the
146
- * `x-deepline-absurd-release` header on child `ctx.runPlay` launches so the
147
- * child pins to the parent's lane (defense-in-depth). Absent off the absurd
148
- * path or on a pre-release parent.
149
- */
150
- absurdReleaseId?: string | null;
151
143
  /**
152
144
  * Push-execution wiring for the Daytona sandbox runner. When present the
153
145
  * runner keeps the Absurd run claim alive itself (heartbeating the receipt
@@ -2236,8 +2236,8 @@ async function missingRuntimeWorkReceiptSelfHealColumns(
2236
2236
  );
2237
2237
  }
2238
2238
 
2239
- // COMPAT / SELF-HEAL — owner: runtime. Removal milestone: execution-ledger
2240
- // cutover M1 (docs/play-runtime-execution-ledger-plan.md). The customer-Neon
2239
+ // COMPAT / SELF-HEAL — owner: runtime. Removal milestone: runtime-ledger
2240
+ // cutover. The customer-Neon
2241
2241
  // work-receipt table (lease columns + failure_kind) is the dying surface. The
2242
2242
  // F2 rolling migration (scripts/migrate-play-runtime-storage.ts) backfills these
2243
2243
  // columns; this lazy CREATE/ALTER self-heal is a compat-window safety net and is
@@ -4175,6 +4175,9 @@ export async function claimRuntimeWorkReceipt(
4175
4175
  input.leaseTtlMs,
4176
4176
  PLAY_RUNTIME_WORK_RECEIPT_LEASE_TTL_MS,
4177
4177
  );
4178
+ // A successful lease-bearing claim is the execution transition. Keep the
4179
+ // stored queued status for mixed-version readers; the runner does not add
4180
+ // a second queued -> running write on the provider hot path.
4178
4181
  const claimStatus =
4179
4182
  input.leaseAware === true
4180
4183
  ? RECEIPT_STATUS_QUEUED