deepline 0.3.184 → 0.3.186

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. package/dist/bundling-sources/sdk/src/release.ts +1 -1
  2. package/dist/bundling-sources/sdk/src/types.ts +4 -0
  3. package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +53 -0
  4. package/dist/bundling-sources/shared_libs/play-runtime/context.ts +751 -193
  5. package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +4 -0
  6. package/dist/bundling-sources/shared_libs/play-runtime/durable-receipt-execution.ts +3 -1
  7. package/dist/bundling-sources/shared_libs/play-runtime/fixture-behavior.ts +1 -0
  8. package/dist/bundling-sources/shared_libs/play-runtime/governor/app-runtime-rate-state-backend.ts +437 -150
  9. package/dist/bundling-sources/shared_libs/play-runtime/governor/governor.ts +47 -4
  10. package/dist/bundling-sources/shared_libs/play-runtime/governor/policy.ts +3 -1
  11. package/dist/bundling-sources/shared_libs/play-runtime/governor/rate-state-backend.ts +18 -0
  12. package/dist/bundling-sources/shared_libs/play-runtime/launch-identity.ts +229 -0
  13. package/dist/bundling-sources/shared_libs/play-runtime/map-execution-frame.ts +3 -0
  14. package/dist/bundling-sources/shared_libs/play-runtime/play-runs/admission/contracts.ts +16 -1
  15. package/dist/bundling-sources/shared_libs/play-runtime/play-runs/admission/postgres-adapter.ts +3 -0
  16. package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts +22 -2
  17. package/dist/bundling-sources/shared_libs/play-runtime/resource-governor.ts +6 -1
  18. package/dist/bundling-sources/shared_libs/play-runtime/run-failure.ts +11 -5
  19. package/dist/bundling-sources/shared_libs/play-runtime/runtime-flow.ts +102 -12
  20. package/dist/bundling-sources/shared_libs/play-runtime/runtime-operation-contract.ts +8 -0
  21. package/dist/bundling-sources/shared_libs/play-runtime/runtime-telemetry.ts +9 -0
  22. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backend.ts +7 -0
  23. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-runtime-worker.ts +136 -28
  24. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-rate-scope.ts +80 -0
  25. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-rate-state.ts +897 -207
  26. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts +271 -25
  27. package/dist/bundling-sources/shared_libs/play-runtime/suspension.ts +21 -0
  28. package/dist/bundling-sources/shared_libs/play-runtime/tool-execute-retry-policy.ts +120 -34
  29. package/dist/bundling-sources/shared_libs/play-runtime/tool-result.ts +82 -2
  30. package/dist/bundling-sources/shared_libs/plays/tool-result-types.ts +7 -2
  31. package/dist/cli/index.js +29 -3
  32. package/dist/cli/index.mjs +29 -3
  33. package/dist/index.d.mts +4 -0
  34. package/dist/index.d.ts +4 -0
  35. package/dist/index.js +1 -1
  36. package/dist/index.mjs +1 -1
  37. package/dist/install-integrity.json +2 -0
  38. package/dist/release.d.mts +1 -1
  39. package/dist/release.d.ts +1 -1
  40. package/dist/release.js +1 -1
  41. package/dist/release.mjs +1 -1
  42. package/package.json +1 -1
@@ -21,6 +21,7 @@ import {
21
21
  resolveRowConcurrency,
22
22
  } from './policy';
23
23
  import {
24
+ DEFAULT_UNKNOWN_PROVIDER_COLD_START_RPS,
24
25
  type PacingPermit,
25
26
  type PacingRule,
26
27
  type ProviderConcurrencyGrant,
@@ -50,6 +51,8 @@ export interface ProviderPermit extends WorkLease {
50
51
  readonly concurrencyGrant?: ProviderConcurrencyGrant | null;
51
52
  /** Effective rate-controller reading for this physical provider permit. */
52
53
  readonly pacingGrant?: ProviderPacingGrant | null;
54
+ /** Signed shared-scope proof consumed by the execution gateway after queueing. */
55
+ readonly gatewayDispatchProof?: PacingPermit['gatewayDispatchProof'];
53
56
  /**
54
57
  * Return an acquired provider permit before an outbound request started.
55
58
  * Unlike `release`, this returns the rate reservation as well as any
@@ -64,6 +67,8 @@ export interface ProviderPermit extends WorkLease {
64
67
  * backend's lease block when that block is still live; otherwise refunds.
65
68
  */
66
69
  park(): Promise<void>;
70
+ /** Recheck shared provider state immediately before network dispatch. */
71
+ validate?(): Promise<boolean>;
67
72
  }
68
73
 
69
74
  export type BudgetKind = 'toolCall' | 'retry' | 'waterfallStep';
@@ -174,7 +179,7 @@ export interface PlayExecutionGovernor {
174
179
  */
175
180
  acquireProviderPermit(
176
181
  toolId: string,
177
- opts?: { signal?: AbortSignal },
182
+ opts?: { signal?: AbortSignal; yieldProviderHolds?: boolean },
178
183
  ): Promise<ProviderPermit>;
179
184
 
180
185
  /**
@@ -226,6 +231,8 @@ export interface PlayExecutionGovernor {
226
231
  */
227
232
  toolId?: string;
228
233
  retryAfterMs: number;
234
+ /** Absolute provider retry deadline from the observed response. */
235
+ retryAtMs?: number;
229
236
  /** Provider residence for the rejected physical request, when observed. */
230
237
  latencyMs?: number | null;
231
238
  /** Server-authored context for attributing this physical provider 429. */
@@ -434,6 +441,15 @@ export function createPlayExecutionGovernor(
434
441
  provider,
435
442
  concurrencyGrant: rateLease.concurrencyGrant ?? null,
436
443
  pacingGrant: rateLease.pacingGrant ?? null,
444
+ gatewayDispatchProof: rateLease.gatewayDispatchProof ?? null,
445
+ ...(rateLease.validate
446
+ ? {
447
+ validate: async () => {
448
+ if (settled) return false;
449
+ return await rateLease.validate!();
450
+ },
451
+ }
452
+ : {}),
437
453
  release() {
438
454
  if (settled) return;
439
455
  settled = true;
@@ -487,16 +503,37 @@ export function createPlayExecutionGovernor(
487
503
  // Passthrough: hand the DB-backed pacer the declared rules verbatim (or
488
504
  // the default-pacing shape when undeclared). No in-memory admission — the
489
505
  // row is authoritative.
490
- const pacing = declaredPacing ?? defaultPacingForTool(toolId, policy);
506
+ const fallback = defaultPacingForTool(toolId, policy);
507
+ const pacing = declaredPacing ?? {
508
+ ...fallback,
509
+ provider: resolvedPacing?.provider ?? fallback.provider,
510
+ };
511
+ if (!declaredPacing) {
512
+ // bucketId is the authenticated quota scope. Unknown tools sharing a
513
+ // credential must not each mint an independent copy of the default rate.
514
+ // Start its one shared lane at one probe/sec; the DB-owned AIMD quickly
515
+ // ramps on sustained success and reacts to Retry-After if the provider
516
+ // tells us the quota is lower.
517
+ pacing.rules = pacing.rules.map((rule) => ({
518
+ ...rule,
519
+ ruleId: 'default:provider-scope',
520
+ requestsPerWindow: DEFAULT_UNKNOWN_PROVIDER_COLD_START_RPS,
521
+ windowMs: 1_000,
522
+ }));
523
+ }
491
524
  providerByTool.set(toolId, pacing.provider);
492
525
  pacingRulesByTool.set(toolId, pacing.rules);
493
526
  return pacing;
494
527
  }
528
+ const fallback = defaultPacingForTool(toolId, policy);
495
529
  const pacing = adaptiveAdmission.resolvePacing({
496
530
  orgId: input.scope.orgId,
497
531
  toolId,
498
532
  declaredPacing,
499
- fallbackPacing: defaultPacingForTool(toolId, policy),
533
+ fallbackPacing: {
534
+ ...fallback,
535
+ provider: resolvedPacing?.provider ?? fallback.provider,
536
+ },
500
537
  });
501
538
  providerByTool.set(toolId, pacing.provider);
502
539
  pacingRulesByTool.set(toolId, pacing.rules);
@@ -623,8 +660,13 @@ export function createPlayExecutionGovernor(
623
660
  rateScopeToken: rateScope[1],
624
661
  rules: pacing.rules,
625
662
  signal: opts?.signal,
663
+ yieldProviderHolds: opts?.yieldProviderHolds,
626
664
  });
627
- return providerPermitFrom(pacing.provider, rateLease, localConcurrencySlot);
665
+ return providerPermitFrom(
666
+ pacing.provider,
667
+ rateLease,
668
+ localConcurrencySlot,
669
+ );
628
670
  } catch (error) {
629
671
  localConcurrencySlot?.release();
630
672
  throw error;
@@ -727,6 +769,7 @@ export function createPlayExecutionGovernor(
727
769
  bucketId: rateScope[0],
728
770
  ...(rateScope[1] ? { rateScopeToken: rateScope[1] } : {}),
729
771
  cooldownMs: bp.retryAfterMs,
772
+ ...(bp.retryAtMs != null ? { retryAtMs: bp.retryAtMs } : {}),
730
773
  ...(rules ? { rules } : {}),
731
774
  ...(bp.latencyMs != null ? { latencyMs: bp.latencyMs } : {}),
732
775
  ...(bp.concurrencyGrant
@@ -188,7 +188,9 @@ export const SHARED_EXECUTION_POLICY: ResolvedExecutionPolicy = {
188
188
  maxWaterfallStepExecutions: 5_000_000,
189
189
  },
190
190
  pacing: {
191
- // Undeclared providers; declared providers (rate-limit-definitions.ts) win.
191
+ // Keep the generic fallback rate. The shared Postgres backend separately
192
+ // cold-starts unknown provider scopes at one probe/sec.
193
+ // Declared providers (rate-limit-definitions.ts) keep their own rates.
192
194
  defaultProviderRequestsPerSecond: 10,
193
195
  // This is the scheduler's resident-call envelope, not an RPS target.
194
196
  // Keep it aligned with the explicit run ceiling so an RPS-only provider
@@ -14,6 +14,12 @@
14
14
  */
15
15
  export const DEFAULT_ADVISORY_PROVIDER_MAX_RPS = 100;
16
16
 
17
+ /** Cold-start rate for an unknown provider bucket shared across a fleet key. */
18
+ export const DEFAULT_UNKNOWN_PROVIDER_COLD_START_RPS = 1;
19
+
20
+ /** AIMD state cooldown cap; provider Retry-After holds remain uncapped. */
21
+ export const PROVIDER_AIMD_COOLDOWN_CAP_MS = 45 * 60 * 1_000;
22
+
17
23
  /** A single resolved rate rule for a bucket (from rate-limit-definitions). */
18
24
  export interface PacingRule {
19
25
  readonly ruleId: string;
@@ -83,10 +89,18 @@ export interface PacingPermit {
83
89
  * Absent on backends that keep no local block; callers use `cancel` then.
84
90
  */
85
91
  park?(): Promise<void>;
92
+ /** Recheck the durable provider-scope hold immediately before dispatch. */
93
+ validate?(): Promise<boolean>;
86
94
  /** Server-authored context for attributing this request's provider 429. */
87
95
  readonly concurrencyGrant?: ProviderConcurrencyGrant | null;
88
96
  /** Effective shared rate for every app-runtime provider permit. */
89
97
  readonly pacingGrant?: ProviderPacingGrant | null;
98
+ /** Signed shared-scope proof for revalidation after the execution-gateway queue. */
99
+ readonly gatewayDispatchProof?: {
100
+ readonly bucketId: string;
101
+ readonly rateScopeToken: string;
102
+ readonly retryAtMsAtGrant: number;
103
+ } | null;
90
104
  }
91
105
 
92
106
  /**
@@ -118,6 +132,8 @@ export interface RateStateBackend {
118
132
  rateScopeToken?: string | null;
119
133
  rules: readonly PacingRule[];
120
134
  signal?: AbortSignal;
135
+ /** Managed Plays hand a shared provider hold to durable row suspension. */
136
+ yieldProviderHolds?: boolean;
121
137
  }): Promise<PacingPermit>;
122
138
 
123
139
  /**
@@ -140,6 +156,8 @@ export interface RateStateBackend {
140
156
  bucketId: string;
141
157
  rateScopeToken?: string | null;
142
158
  cooldownMs: number;
159
+ /** Absolute upstream retry deadline, stable across transport retries. */
160
+ retryAtMs?: number;
143
161
  /** Resolved provider rules for the rejected physical attempt. */
144
162
  rules?: readonly PacingRule[];
145
163
  /** Provider residence measured at the HTTP boundary. */
@@ -0,0 +1,229 @@
1
+ import { sha256Hex, stableStringify } from '../plays/row-identity';
2
+ import { LEGACY_MAX_CONCURRENT_EXTERNAL_CALLS } from './governor/policy';
3
+ import type { PlaySchedulerSubmitInput } from './scheduler-backend';
4
+
5
+ /** Optional launch-envelope field used by the mixed-release admission fence. */
6
+ export const PLAY_LAUNCH_IDENTITY_DIGEST_FIELD = 'launchIdentityDigest';
7
+ const PLAY_LAUNCH_IDENTITY_DIGEST_NAMESPACE = 'deepline:play-launch-identity';
8
+
9
+ export type PlayLaunchFieldRole = 'identity' | 'detail';
10
+
11
+ /**
12
+ * Keep this list exhaustive over the launch envelope. A newly added field must
13
+ * be classified before it can participate in launch admission.
14
+ */
15
+ export const PLAY_LAUNCH_ENVELOPE_FIELD_ROLES: {
16
+ readonly [K in keyof PlaySchedulerSubmitInput]-?: PlayLaunchFieldRole;
17
+ } = {
18
+ orgId: 'identity',
19
+ playName: 'identity',
20
+ playReference: 'identity',
21
+ definitionScope: 'identity',
22
+ revisionId: 'identity',
23
+ artifactStorageKey: 'identity',
24
+ artifactHash: 'identity',
25
+ graphHash: 'identity',
26
+ workflowFamilyKey: 'identity',
27
+ inputSha256: 'identity',
28
+ source: 'identity',
29
+ triggerSource: 'identity',
30
+ parentRunId: 'identity',
31
+ rootRunId: 'identity',
32
+ asyncLaunchKey: 'identity',
33
+ asyncInvocationKey: 'identity',
34
+ asyncRequestFingerprint: 'identity',
35
+ force: 'identity',
36
+ forceToolRefresh: 'identity',
37
+ maxConcurrentExternalCalls: 'identity',
38
+ maxConcurrentRows: 'identity',
39
+ integrationMode: 'identity',
40
+ runtimeSandboxPlacementPolicyId: 'identity',
41
+ input: 'identity',
42
+ inputFile: 'identity',
43
+ inlineInputFile: 'identity',
44
+ inlineCsv: 'identity',
45
+ packagedFiles: 'identity',
46
+ runId: 'detail',
47
+ existingOnly: 'detail',
48
+ asyncParentRunAttempt: 'detail',
49
+ asyncParentRunLeaseToken: 'detail',
50
+ asyncParentAttemptRequired: 'detail',
51
+ asyncAncestryPlayIds: 'detail',
52
+ playId: 'detail',
53
+ schedulerSchema: 'detail',
54
+ requiresRunCreatedProjection: 'detail',
55
+ runtimeArtifact: 'detail',
56
+ runtimeProtocolVersion: 'detail',
57
+ staticPipeline: 'detail',
58
+ inputSummary: 'detail',
59
+ inputBytes: 'detail',
60
+ inputFileId: 'detail',
61
+ replayedFromRunId: 'detail',
62
+ totalRows: 'detail',
63
+ contractSnapshot: 'detail',
64
+ executionPlan: 'detail',
65
+ preloadedDbSessions: 'detail',
66
+ preloadedDbSessionUnwrapKey: 'detail',
67
+ executorToken: 'detail',
68
+ runtimeAuthority: 'detail',
69
+ runtimeApiBaseUrl: 'detail',
70
+ runtimeTokenOrigin: 'detail',
71
+ baseUrl: 'detail',
72
+ coordinatorUrl: 'detail',
73
+ coordinatorInternalToken: 'detail',
74
+ runtimeDeployVersion: 'detail',
75
+ absurdReleaseId: 'detail',
76
+ absurdReleaseEnvironment: 'detail',
77
+ vercelProtectionBypassToken: 'detail',
78
+ preferredReceiptGatewayMachineId: 'detail',
79
+ runtimeTestFaultHeader: 'detail',
80
+ testPolicyOverrides: 'detail',
81
+ submittedAtMs: 'detail',
82
+ schedulerBackend: 'detail',
83
+ executionProfile: 'detail',
84
+ runtimeBackend: 'detail',
85
+ queueMaxDelaySeconds: 'detail',
86
+ queuePriority: 'detail',
87
+ userEmail: 'detail',
88
+ userId: 'detail',
89
+ launchIdentityDigest: 'detail',
90
+ };
91
+
92
+ export const PLAY_LAUNCH_IDENTITY_FIELDS_COMPARED_OUTSIDE_DIGEST: readonly string[] =
93
+ Object.freeze(['runtimeSandboxPlacementPolicyId']);
94
+
95
+ export const PLAY_LAUNCH_IDENTITY_FIELDS: readonly string[] = Object.freeze(
96
+ Object.entries(PLAY_LAUNCH_ENVELOPE_FIELD_ROLES)
97
+ .filter(([, role]) => role === 'identity')
98
+ .map(([field]) => field)
99
+ .sort(),
100
+ );
101
+
102
+ export const PLAY_LAUNCH_DETAIL_FIELDS: readonly string[] = Object.freeze(
103
+ Object.entries(PLAY_LAUNCH_ENVELOPE_FIELD_ROLES)
104
+ .filter(([, role]) => role === 'detail')
105
+ .map(([field]) => field)
106
+ .sort(),
107
+ );
108
+
109
+ export type PlayLaunchIdentitySource = Readonly<
110
+ Partial<Record<keyof PlaySchedulerSubmitInput, unknown>>
111
+ >;
112
+
113
+ function text(value: unknown): string | null {
114
+ return typeof value === 'string' && value.trim() ? value.trim() : null;
115
+ }
116
+
117
+ function flag(value: unknown): boolean {
118
+ return value === true;
119
+ }
120
+
121
+ function count(value: unknown): number | null {
122
+ return typeof value === 'number' && Number.isFinite(value) ? value : null;
123
+ }
124
+
125
+ function record(value: unknown): Record<string, unknown> | null {
126
+ return value && typeof value === 'object' && !Array.isArray(value)
127
+ ? (value as Record<string, unknown>)
128
+ : null;
129
+ }
130
+
131
+ function stagedFileIdentity(value: unknown): Record<string, unknown> | null {
132
+ const file = record(value);
133
+ if (!file) return null;
134
+ return {
135
+ logicalPath:
136
+ text(file.logicalPath) ?? text(file.path) ?? text(file.playPath),
137
+ fileName: text(file.fileName) ?? text(file.name),
138
+ contentType: text(file.contentType),
139
+ bytes: count(file.bytes),
140
+ contentDigest:
141
+ typeof file.contentBase64 === 'string'
142
+ ? sha256Hex(file.contentBase64)
143
+ : typeof file.inlineText === 'string'
144
+ ? sha256Hex(file.inlineText)
145
+ : null,
146
+ };
147
+ }
148
+
149
+ function stagedFileListIdentity(
150
+ value: unknown,
151
+ ): Array<Record<string, unknown>> {
152
+ if (!Array.isArray(value)) return [];
153
+ return value
154
+ .map((entry) => stagedFileIdentity(entry))
155
+ .filter((entry): entry is Record<string, unknown> => entry !== null)
156
+ .sort((left, right) =>
157
+ stableStringify(left).localeCompare(stableStringify(right)),
158
+ );
159
+ }
160
+
161
+ /** Return only customer intent; transport, authority, and release details are excluded. */
162
+ export function buildPlayLaunchIdentity(
163
+ launch: PlayLaunchIdentitySource,
164
+ ): Record<string, unknown> {
165
+ const inlineCsv = record(launch.inlineCsv);
166
+ return {
167
+ orgId: text(launch.orgId),
168
+ playName: text(launch.playName),
169
+ playReference: text(launch.playReference) ?? text(launch.playName),
170
+ definitionScope: text(launch.definitionScope),
171
+ revisionId: text(launch.revisionId),
172
+ artifactStorageKey: text(launch.artifactStorageKey),
173
+ artifactHash: text(launch.artifactHash),
174
+ graphHash: text(launch.graphHash),
175
+ workflowFamilyKey: text(launch.workflowFamilyKey),
176
+ inputSha256: text(launch.inputSha256),
177
+ source: text(launch.source),
178
+ triggerSource: text(launch.triggerSource),
179
+ parentRunId: text(launch.parentRunId),
180
+ rootRunId: text(launch.rootRunId),
181
+ asyncLaunchKey: text(launch.asyncLaunchKey),
182
+ asyncInvocationKey: text(launch.asyncInvocationKey),
183
+ asyncRequestFingerprint: text(launch.asyncRequestFingerprint),
184
+ force: flag(launch.force),
185
+ forceToolRefresh: flag(launch.forceToolRefresh),
186
+ integrationMode: text(launch.integrationMode),
187
+ maxConcurrentExternalCalls:
188
+ count(launch.maxConcurrentExternalCalls) ??
189
+ LEGACY_MAX_CONCURRENT_EXTERNAL_CALLS,
190
+ maxConcurrentRows: count(launch.maxConcurrentRows),
191
+ inputDigest: sha256Hex(stableStringify(launch.input ?? {})),
192
+ inputFile: stagedFileIdentity(launch.inputFile),
193
+ inlineInputFile: stagedFileIdentity(launch.inlineInputFile),
194
+ inlineCsv: inlineCsv
195
+ ? {
196
+ name: text(inlineCsv.name),
197
+ rowsDigest: sha256Hex(stableStringify(inlineCsv.rows ?? [])),
198
+ }
199
+ : null,
200
+ packagedFiles: stagedFileListIdentity(launch.packagedFiles),
201
+ };
202
+ }
203
+
204
+ export const PLAY_LAUNCH_DIGESTED_IDENTITY_FIELDS: readonly string[] =
205
+ Object.freeze(
206
+ PLAY_LAUNCH_IDENTITY_FIELDS.filter(
207
+ (field) =>
208
+ !PLAY_LAUNCH_IDENTITY_FIELDS_COMPARED_OUTSIDE_DIGEST.includes(field),
209
+ ),
210
+ );
211
+
212
+ export function buildPlayLaunchIdentityDigest(
213
+ launch: PlayLaunchIdentitySource,
214
+ ): string {
215
+ return sha256Hex(
216
+ `${PLAY_LAUNCH_IDENTITY_DIGEST_NAMESPACE}\0${stableStringify(
217
+ buildPlayLaunchIdentity(launch),
218
+ )}`,
219
+ );
220
+ }
221
+
222
+ export function withPlayLaunchIdentityDigest<
223
+ T extends PlaySchedulerSubmitInput,
224
+ >(launch: T): T {
225
+ return {
226
+ ...launch,
227
+ launchIdentityDigest: buildPlayLaunchIdentityDigest(launch),
228
+ };
229
+ }
@@ -15,6 +15,9 @@ export function cloneMapFrame(frame: MapExecutionFrame): MapExecutionFrame {
15
15
  suspendedInputRows: frame.suspendedInputRows.map((row) => ({
16
16
  inputIndex: row.inputIndex,
17
17
  data: { ...row.data },
18
+ ...(row.authoredInputDigest
19
+ ? { authoredInputDigest: row.authoredInputDigest }
20
+ : {}),
18
21
  })),
19
22
  }
20
23
  : {}),
@@ -42,16 +42,31 @@ export type PlayRunAdmissionJob = {
42
42
  admit(input: PlayRunAdmissionInput): Promise<PlayRunAdmissionOutcome>;
43
43
  };
44
44
 
45
+ export type PlayRunLaunchConflictReason = 'launch_identity_mismatch';
46
+
45
47
  /** A stable, machine-readable rejection for a reused run id with new intent. */
46
48
  export class PlayRunLaunchConflictError extends Error {
47
49
  readonly code = 'PLAY_RUN_LAUNCH_CONFLICT';
50
+ readonly conflictReason: PlayRunLaunchConflictReason | undefined;
51
+ readonly conflictingFields: string[] | undefined;
48
52
 
49
- constructor(runId: string, options?: { cause?: unknown }) {
53
+ constructor(
54
+ runId: string,
55
+ options?: {
56
+ cause?: unknown;
57
+ conflictReason?: PlayRunLaunchConflictReason;
58
+ conflictingFields?: readonly string[];
59
+ },
60
+ ) {
50
61
  super(
51
62
  `Play run "${runId}" already exists with a different launch.`,
52
63
  options,
53
64
  );
54
65
  this.name = 'PlayRunLaunchConflictError';
66
+ this.conflictReason = options?.conflictReason;
67
+ this.conflictingFields = options?.conflictingFields
68
+ ? [...options.conflictingFields]
69
+ : undefined;
55
70
  }
56
71
  }
57
72
 
@@ -54,6 +54,9 @@ export function createPostgresPlayRunAdmissionPorts(input: {
54
54
  artifactHash: admission.launch.artifactHash,
55
55
  graphHash: admission.launch.graphHash,
56
56
  inputSha256: admission.launch.inputSha256,
57
+ launchIdentityDigest: admission.launch.launchIdentityDigest,
58
+ conflictReason: conflict.conflictReason ?? null,
59
+ conflictingFields: conflict.conflictingFields ?? null,
57
60
  error: conflict,
58
61
  });
59
62
  throw conflict;
@@ -79,6 +79,8 @@ export type PlayRunnerRateStateConcurrencyGrant = {
79
79
  export type PlayRunnerRateStateAcquireResult = {
80
80
  granted: number;
81
81
  waitMs: number;
82
+ /** This receiver serves rate_state_validate. Absent on pre-expansion readers. */
83
+ scopeValidationSupported?: boolean;
82
84
  /**
83
85
  * Gateway clock used to produce `scheduledAtMs`. Runners translate the
84
86
  * schedule into relative delays so cross-machine clock skew cannot compress
@@ -108,8 +110,8 @@ export type PlayRunnerRateStateAcquireResult = {
108
110
  */
109
111
  coolUntilMs?: number;
110
112
  /**
111
- * The most recent server Retry-After stored verbatim (uncapped) as an absolute
112
- * epoch-ms deadline. Purely observational — never gates a grant on its own.
113
+ * Most recent provider Retry-After deadline, surfaced for diagnostics. The
114
+ * authoritative admission gate is the durable provider-scope hold row.
113
115
  */
114
116
  claimedRetryAtMs?: number;
115
117
  };
@@ -128,6 +130,12 @@ export type PlayRunnerRateStatePenalizeInput = {
128
130
  bucketId: string;
129
131
  rateScopeToken?: string | null;
130
132
  cooldownMs: number;
133
+ /** Absolute provider retry deadline captured with the observed response. */
134
+ retryAtMs?: number;
135
+ /** Unflushed confirmed successes, sharing acquire's replay-safe identity. */
136
+ observedSuccesses?: number;
137
+ concurrencyObservations?: PlayRunnerRateStateConcurrencyObservation[];
138
+ concurrencyObservationBatch?: PlayRunnerRateStateConcurrencyObservationBatch;
131
139
  /**
132
140
  * The same resolved rules that admitted the attempt. Optional during the
133
141
  * rolling deploy window: old runners retain rate-only backpressure handling.
@@ -140,6 +148,18 @@ export type PlayRunnerRateStatePenalizeInput = {
140
148
  schedulerSchema?: string | null;
141
149
  };
142
150
 
151
+ export type PlayRunnerRateStateValidateInput = {
152
+ bucketId: string;
153
+ rateScopeToken?: string | null;
154
+ schedulerSchema?: string | null;
155
+ };
156
+
157
+ export type PlayRunnerRateStateValidateResult = {
158
+ allowed: boolean;
159
+ serverNowMs: number;
160
+ retryAtMs: number;
161
+ };
162
+
143
163
  export type PlayRunnerBudgetCharge = {
144
164
  key: string;
145
165
  amount: number;
@@ -711,6 +711,7 @@ export interface RuntimeResourceGovernor {
711
711
  acquireProviderPermit(input: {
712
712
  toolId: string;
713
713
  signal?: AbortSignal;
714
+ yieldProviderHolds?: boolean;
714
715
  }): Promise<RuntimeProviderPermit>;
715
716
  /**
716
717
  * Non-blocking provider admission from capacity this process already holds.
@@ -748,6 +749,7 @@ export interface RuntimeResourceGovernor {
748
749
  provider: string;
749
750
  toolId?: string;
750
751
  retryAfterMs: number;
752
+ retryAtMs?: number;
751
753
  /** Provider residence for the rejected physical request, when observed. */
752
754
  latencyMs?: number | null;
753
755
  /** Server-authored context for attributing this physical provider 429. */
@@ -1585,7 +1587,10 @@ export function createRuntimeResourceGovernor(input: {
1585
1587
  const startedAt = Date.now();
1586
1588
  const lease = await input.executionGovernor.acquireProviderPermit(
1587
1589
  providerInput.toolId,
1588
- { signal: providerInput.signal },
1590
+ {
1591
+ signal: providerInput.signal,
1592
+ yieldProviderHolds: providerInput.yieldProviderHolds,
1593
+ },
1589
1594
  );
1590
1595
  observe({
1591
1596
  providerResourceKey: lease.provider,
@@ -241,15 +241,17 @@ export function providerExhaustedMessage(input: {
241
241
  }
242
242
 
243
243
  /**
244
- * Thrown by the DB-authoritative pacer (`app_runtime_postgres` rate-state
245
- * backend) when a provider's hold exceeds the tolerable wait
246
- * (PROVIDER_EXHAUSTED_MAX_WAIT_MS). The call is skipped BEFORE any tool
247
- * execution request, so nothing is dispatched and nothing is billed. Row-failure
248
- * isolation still applies: this is a row/step outcome, not a run-fatal error.
244
+ * Control signal from the DB-authoritative pacer when a provider hold exceeds
245
+ * the in-process wait threshold. The next provider call is not dispatched.
246
+ * Managed durable Plays translate this into a persisted run/row wait and resume
247
+ * after the deadline; callers without durable boundaries receive a repairable
248
+ * provider-exhausted outcome. This signal is not a terminal row failure in the
249
+ * managed durable path.
249
250
  */
250
251
  export class ProviderExhaustedError extends Error {
251
252
  readonly code = PROVIDER_EXHAUSTED_CODE;
252
253
  readonly provider: string;
254
+ readonly publicDetails: ToolExecutionError['publicDetails'];
253
255
  /** ISO 8601 retry deadline (from the server Retry-After / cooldown). */
254
256
  readonly retryAt: string;
255
257
  constructor(input: { provider: string; retryAtMs: number; cause?: unknown }) {
@@ -264,6 +266,10 @@ export class ProviderExhaustedError extends Error {
264
266
  );
265
267
  this.name = 'ProviderExhaustedError';
266
268
  this.provider = input.provider;
269
+ this.publicDetails =
270
+ input.cause instanceof ToolExecutionError
271
+ ? input.cause.publicDetails
272
+ : null;
267
273
  this.retryAt = Number.isFinite(Number(input.retryAtMs))
268
274
  ? new Date(Number(input.retryAtMs)).toISOString()
269
275
  : new Date(0).toISOString();