deepline 0.3.184 → 0.3.186
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/sdk/src/release.ts +1 -1
- package/dist/bundling-sources/sdk/src/types.ts +4 -0
- package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +53 -0
- package/dist/bundling-sources/shared_libs/play-runtime/context.ts +751 -193
- package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +4 -0
- package/dist/bundling-sources/shared_libs/play-runtime/durable-receipt-execution.ts +3 -1
- package/dist/bundling-sources/shared_libs/play-runtime/fixture-behavior.ts +1 -0
- package/dist/bundling-sources/shared_libs/play-runtime/governor/app-runtime-rate-state-backend.ts +437 -150
- package/dist/bundling-sources/shared_libs/play-runtime/governor/governor.ts +47 -4
- package/dist/bundling-sources/shared_libs/play-runtime/governor/policy.ts +3 -1
- package/dist/bundling-sources/shared_libs/play-runtime/governor/rate-state-backend.ts +18 -0
- package/dist/bundling-sources/shared_libs/play-runtime/launch-identity.ts +229 -0
- package/dist/bundling-sources/shared_libs/play-runtime/map-execution-frame.ts +3 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/admission/contracts.ts +16 -1
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/admission/postgres-adapter.ts +3 -0
- package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts +22 -2
- package/dist/bundling-sources/shared_libs/play-runtime/resource-governor.ts +6 -1
- package/dist/bundling-sources/shared_libs/play-runtime/run-failure.ts +11 -5
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-flow.ts +102 -12
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-operation-contract.ts +8 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-telemetry.ts +9 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backend.ts +7 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-runtime-worker.ts +136 -28
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-rate-scope.ts +80 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-rate-state.ts +897 -207
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts +271 -25
- package/dist/bundling-sources/shared_libs/play-runtime/suspension.ts +21 -0
- package/dist/bundling-sources/shared_libs/play-runtime/tool-execute-retry-policy.ts +120 -34
- package/dist/bundling-sources/shared_libs/play-runtime/tool-result.ts +82 -2
- package/dist/bundling-sources/shared_libs/plays/tool-result-types.ts +7 -2
- package/dist/cli/index.js +29 -3
- package/dist/cli/index.mjs +29 -3
- package/dist/index.d.mts +4 -0
- package/dist/index.d.ts +4 -0
- package/dist/index.js +1 -1
- package/dist/index.mjs +1 -1
- package/dist/install-integrity.json +2 -0
- package/dist/release.d.mts +1 -1
- package/dist/release.d.ts +1 -1
- package/dist/release.js +1 -1
- package/dist/release.mjs +1 -1
- package/package.json +1 -1
|
@@ -21,6 +21,7 @@ import {
|
|
|
21
21
|
resolveRowConcurrency,
|
|
22
22
|
} from './policy';
|
|
23
23
|
import {
|
|
24
|
+
DEFAULT_UNKNOWN_PROVIDER_COLD_START_RPS,
|
|
24
25
|
type PacingPermit,
|
|
25
26
|
type PacingRule,
|
|
26
27
|
type ProviderConcurrencyGrant,
|
|
@@ -50,6 +51,8 @@ export interface ProviderPermit extends WorkLease {
|
|
|
50
51
|
readonly concurrencyGrant?: ProviderConcurrencyGrant | null;
|
|
51
52
|
/** Effective rate-controller reading for this physical provider permit. */
|
|
52
53
|
readonly pacingGrant?: ProviderPacingGrant | null;
|
|
54
|
+
/** Signed shared-scope proof consumed by the execution gateway after queueing. */
|
|
55
|
+
readonly gatewayDispatchProof?: PacingPermit['gatewayDispatchProof'];
|
|
53
56
|
/**
|
|
54
57
|
* Return an acquired provider permit before an outbound request started.
|
|
55
58
|
* Unlike `release`, this returns the rate reservation as well as any
|
|
@@ -64,6 +67,8 @@ export interface ProviderPermit extends WorkLease {
|
|
|
64
67
|
* backend's lease block when that block is still live; otherwise refunds.
|
|
65
68
|
*/
|
|
66
69
|
park(): Promise<void>;
|
|
70
|
+
/** Recheck shared provider state immediately before network dispatch. */
|
|
71
|
+
validate?(): Promise<boolean>;
|
|
67
72
|
}
|
|
68
73
|
|
|
69
74
|
export type BudgetKind = 'toolCall' | 'retry' | 'waterfallStep';
|
|
@@ -174,7 +179,7 @@ export interface PlayExecutionGovernor {
|
|
|
174
179
|
*/
|
|
175
180
|
acquireProviderPermit(
|
|
176
181
|
toolId: string,
|
|
177
|
-
opts?: { signal?: AbortSignal },
|
|
182
|
+
opts?: { signal?: AbortSignal; yieldProviderHolds?: boolean },
|
|
178
183
|
): Promise<ProviderPermit>;
|
|
179
184
|
|
|
180
185
|
/**
|
|
@@ -226,6 +231,8 @@ export interface PlayExecutionGovernor {
|
|
|
226
231
|
*/
|
|
227
232
|
toolId?: string;
|
|
228
233
|
retryAfterMs: number;
|
|
234
|
+
/** Absolute provider retry deadline from the observed response. */
|
|
235
|
+
retryAtMs?: number;
|
|
229
236
|
/** Provider residence for the rejected physical request, when observed. */
|
|
230
237
|
latencyMs?: number | null;
|
|
231
238
|
/** Server-authored context for attributing this physical provider 429. */
|
|
@@ -434,6 +441,15 @@ export function createPlayExecutionGovernor(
|
|
|
434
441
|
provider,
|
|
435
442
|
concurrencyGrant: rateLease.concurrencyGrant ?? null,
|
|
436
443
|
pacingGrant: rateLease.pacingGrant ?? null,
|
|
444
|
+
gatewayDispatchProof: rateLease.gatewayDispatchProof ?? null,
|
|
445
|
+
...(rateLease.validate
|
|
446
|
+
? {
|
|
447
|
+
validate: async () => {
|
|
448
|
+
if (settled) return false;
|
|
449
|
+
return await rateLease.validate!();
|
|
450
|
+
},
|
|
451
|
+
}
|
|
452
|
+
: {}),
|
|
437
453
|
release() {
|
|
438
454
|
if (settled) return;
|
|
439
455
|
settled = true;
|
|
@@ -487,16 +503,37 @@ export function createPlayExecutionGovernor(
|
|
|
487
503
|
// Passthrough: hand the DB-backed pacer the declared rules verbatim (or
|
|
488
504
|
// the default-pacing shape when undeclared). No in-memory admission — the
|
|
489
505
|
// row is authoritative.
|
|
490
|
-
const
|
|
506
|
+
const fallback = defaultPacingForTool(toolId, policy);
|
|
507
|
+
const pacing = declaredPacing ?? {
|
|
508
|
+
...fallback,
|
|
509
|
+
provider: resolvedPacing?.provider ?? fallback.provider,
|
|
510
|
+
};
|
|
511
|
+
if (!declaredPacing) {
|
|
512
|
+
// bucketId is the authenticated quota scope. Unknown tools sharing a
|
|
513
|
+
// credential must not each mint an independent copy of the default rate.
|
|
514
|
+
// Start its one shared lane at one probe/sec; the DB-owned AIMD quickly
|
|
515
|
+
// ramps on sustained success and reacts to Retry-After if the provider
|
|
516
|
+
// tells us the quota is lower.
|
|
517
|
+
pacing.rules = pacing.rules.map((rule) => ({
|
|
518
|
+
...rule,
|
|
519
|
+
ruleId: 'default:provider-scope',
|
|
520
|
+
requestsPerWindow: DEFAULT_UNKNOWN_PROVIDER_COLD_START_RPS,
|
|
521
|
+
windowMs: 1_000,
|
|
522
|
+
}));
|
|
523
|
+
}
|
|
491
524
|
providerByTool.set(toolId, pacing.provider);
|
|
492
525
|
pacingRulesByTool.set(toolId, pacing.rules);
|
|
493
526
|
return pacing;
|
|
494
527
|
}
|
|
528
|
+
const fallback = defaultPacingForTool(toolId, policy);
|
|
495
529
|
const pacing = adaptiveAdmission.resolvePacing({
|
|
496
530
|
orgId: input.scope.orgId,
|
|
497
531
|
toolId,
|
|
498
532
|
declaredPacing,
|
|
499
|
-
fallbackPacing:
|
|
533
|
+
fallbackPacing: {
|
|
534
|
+
...fallback,
|
|
535
|
+
provider: resolvedPacing?.provider ?? fallback.provider,
|
|
536
|
+
},
|
|
500
537
|
});
|
|
501
538
|
providerByTool.set(toolId, pacing.provider);
|
|
502
539
|
pacingRulesByTool.set(toolId, pacing.rules);
|
|
@@ -623,8 +660,13 @@ export function createPlayExecutionGovernor(
|
|
|
623
660
|
rateScopeToken: rateScope[1],
|
|
624
661
|
rules: pacing.rules,
|
|
625
662
|
signal: opts?.signal,
|
|
663
|
+
yieldProviderHolds: opts?.yieldProviderHolds,
|
|
626
664
|
});
|
|
627
|
-
return providerPermitFrom(
|
|
665
|
+
return providerPermitFrom(
|
|
666
|
+
pacing.provider,
|
|
667
|
+
rateLease,
|
|
668
|
+
localConcurrencySlot,
|
|
669
|
+
);
|
|
628
670
|
} catch (error) {
|
|
629
671
|
localConcurrencySlot?.release();
|
|
630
672
|
throw error;
|
|
@@ -727,6 +769,7 @@ export function createPlayExecutionGovernor(
|
|
|
727
769
|
bucketId: rateScope[0],
|
|
728
770
|
...(rateScope[1] ? { rateScopeToken: rateScope[1] } : {}),
|
|
729
771
|
cooldownMs: bp.retryAfterMs,
|
|
772
|
+
...(bp.retryAtMs != null ? { retryAtMs: bp.retryAtMs } : {}),
|
|
730
773
|
...(rules ? { rules } : {}),
|
|
731
774
|
...(bp.latencyMs != null ? { latencyMs: bp.latencyMs } : {}),
|
|
732
775
|
...(bp.concurrencyGrant
|
|
@@ -188,7 +188,9 @@ export const SHARED_EXECUTION_POLICY: ResolvedExecutionPolicy = {
|
|
|
188
188
|
maxWaterfallStepExecutions: 5_000_000,
|
|
189
189
|
},
|
|
190
190
|
pacing: {
|
|
191
|
-
//
|
|
191
|
+
// Keep the generic fallback rate. The shared Postgres backend separately
|
|
192
|
+
// cold-starts unknown provider scopes at one probe/sec.
|
|
193
|
+
// Declared providers (rate-limit-definitions.ts) keep their own rates.
|
|
192
194
|
defaultProviderRequestsPerSecond: 10,
|
|
193
195
|
// This is the scheduler's resident-call envelope, not an RPS target.
|
|
194
196
|
// Keep it aligned with the explicit run ceiling so an RPS-only provider
|
|
@@ -14,6 +14,12 @@
|
|
|
14
14
|
*/
|
|
15
15
|
export const DEFAULT_ADVISORY_PROVIDER_MAX_RPS = 100;
|
|
16
16
|
|
|
17
|
+
/** Cold-start rate for an unknown provider bucket shared across a fleet key. */
|
|
18
|
+
export const DEFAULT_UNKNOWN_PROVIDER_COLD_START_RPS = 1;
|
|
19
|
+
|
|
20
|
+
/** AIMD state cooldown cap; provider Retry-After holds remain uncapped. */
|
|
21
|
+
export const PROVIDER_AIMD_COOLDOWN_CAP_MS = 45 * 60 * 1_000;
|
|
22
|
+
|
|
17
23
|
/** A single resolved rate rule for a bucket (from rate-limit-definitions). */
|
|
18
24
|
export interface PacingRule {
|
|
19
25
|
readonly ruleId: string;
|
|
@@ -83,10 +89,18 @@ export interface PacingPermit {
|
|
|
83
89
|
* Absent on backends that keep no local block; callers use `cancel` then.
|
|
84
90
|
*/
|
|
85
91
|
park?(): Promise<void>;
|
|
92
|
+
/** Recheck the durable provider-scope hold immediately before dispatch. */
|
|
93
|
+
validate?(): Promise<boolean>;
|
|
86
94
|
/** Server-authored context for attributing this request's provider 429. */
|
|
87
95
|
readonly concurrencyGrant?: ProviderConcurrencyGrant | null;
|
|
88
96
|
/** Effective shared rate for every app-runtime provider permit. */
|
|
89
97
|
readonly pacingGrant?: ProviderPacingGrant | null;
|
|
98
|
+
/** Signed shared-scope proof for revalidation after the execution-gateway queue. */
|
|
99
|
+
readonly gatewayDispatchProof?: {
|
|
100
|
+
readonly bucketId: string;
|
|
101
|
+
readonly rateScopeToken: string;
|
|
102
|
+
readonly retryAtMsAtGrant: number;
|
|
103
|
+
} | null;
|
|
90
104
|
}
|
|
91
105
|
|
|
92
106
|
/**
|
|
@@ -118,6 +132,8 @@ export interface RateStateBackend {
|
|
|
118
132
|
rateScopeToken?: string | null;
|
|
119
133
|
rules: readonly PacingRule[];
|
|
120
134
|
signal?: AbortSignal;
|
|
135
|
+
/** Managed Plays hand a shared provider hold to durable row suspension. */
|
|
136
|
+
yieldProviderHolds?: boolean;
|
|
121
137
|
}): Promise<PacingPermit>;
|
|
122
138
|
|
|
123
139
|
/**
|
|
@@ -140,6 +156,8 @@ export interface RateStateBackend {
|
|
|
140
156
|
bucketId: string;
|
|
141
157
|
rateScopeToken?: string | null;
|
|
142
158
|
cooldownMs: number;
|
|
159
|
+
/** Absolute upstream retry deadline, stable across transport retries. */
|
|
160
|
+
retryAtMs?: number;
|
|
143
161
|
/** Resolved provider rules for the rejected physical attempt. */
|
|
144
162
|
rules?: readonly PacingRule[];
|
|
145
163
|
/** Provider residence measured at the HTTP boundary. */
|
|
@@ -0,0 +1,229 @@
|
|
|
1
|
+
import { sha256Hex, stableStringify } from '../plays/row-identity';
|
|
2
|
+
import { LEGACY_MAX_CONCURRENT_EXTERNAL_CALLS } from './governor/policy';
|
|
3
|
+
import type { PlaySchedulerSubmitInput } from './scheduler-backend';
|
|
4
|
+
|
|
5
|
+
/** Optional launch-envelope field used by the mixed-release admission fence. */
|
|
6
|
+
export const PLAY_LAUNCH_IDENTITY_DIGEST_FIELD = 'launchIdentityDigest';
|
|
7
|
+
const PLAY_LAUNCH_IDENTITY_DIGEST_NAMESPACE = 'deepline:play-launch-identity';
|
|
8
|
+
|
|
9
|
+
export type PlayLaunchFieldRole = 'identity' | 'detail';
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* Keep this list exhaustive over the launch envelope. A newly added field must
|
|
13
|
+
* be classified before it can participate in launch admission.
|
|
14
|
+
*/
|
|
15
|
+
export const PLAY_LAUNCH_ENVELOPE_FIELD_ROLES: {
|
|
16
|
+
readonly [K in keyof PlaySchedulerSubmitInput]-?: PlayLaunchFieldRole;
|
|
17
|
+
} = {
|
|
18
|
+
orgId: 'identity',
|
|
19
|
+
playName: 'identity',
|
|
20
|
+
playReference: 'identity',
|
|
21
|
+
definitionScope: 'identity',
|
|
22
|
+
revisionId: 'identity',
|
|
23
|
+
artifactStorageKey: 'identity',
|
|
24
|
+
artifactHash: 'identity',
|
|
25
|
+
graphHash: 'identity',
|
|
26
|
+
workflowFamilyKey: 'identity',
|
|
27
|
+
inputSha256: 'identity',
|
|
28
|
+
source: 'identity',
|
|
29
|
+
triggerSource: 'identity',
|
|
30
|
+
parentRunId: 'identity',
|
|
31
|
+
rootRunId: 'identity',
|
|
32
|
+
asyncLaunchKey: 'identity',
|
|
33
|
+
asyncInvocationKey: 'identity',
|
|
34
|
+
asyncRequestFingerprint: 'identity',
|
|
35
|
+
force: 'identity',
|
|
36
|
+
forceToolRefresh: 'identity',
|
|
37
|
+
maxConcurrentExternalCalls: 'identity',
|
|
38
|
+
maxConcurrentRows: 'identity',
|
|
39
|
+
integrationMode: 'identity',
|
|
40
|
+
runtimeSandboxPlacementPolicyId: 'identity',
|
|
41
|
+
input: 'identity',
|
|
42
|
+
inputFile: 'identity',
|
|
43
|
+
inlineInputFile: 'identity',
|
|
44
|
+
inlineCsv: 'identity',
|
|
45
|
+
packagedFiles: 'identity',
|
|
46
|
+
runId: 'detail',
|
|
47
|
+
existingOnly: 'detail',
|
|
48
|
+
asyncParentRunAttempt: 'detail',
|
|
49
|
+
asyncParentRunLeaseToken: 'detail',
|
|
50
|
+
asyncParentAttemptRequired: 'detail',
|
|
51
|
+
asyncAncestryPlayIds: 'detail',
|
|
52
|
+
playId: 'detail',
|
|
53
|
+
schedulerSchema: 'detail',
|
|
54
|
+
requiresRunCreatedProjection: 'detail',
|
|
55
|
+
runtimeArtifact: 'detail',
|
|
56
|
+
runtimeProtocolVersion: 'detail',
|
|
57
|
+
staticPipeline: 'detail',
|
|
58
|
+
inputSummary: 'detail',
|
|
59
|
+
inputBytes: 'detail',
|
|
60
|
+
inputFileId: 'detail',
|
|
61
|
+
replayedFromRunId: 'detail',
|
|
62
|
+
totalRows: 'detail',
|
|
63
|
+
contractSnapshot: 'detail',
|
|
64
|
+
executionPlan: 'detail',
|
|
65
|
+
preloadedDbSessions: 'detail',
|
|
66
|
+
preloadedDbSessionUnwrapKey: 'detail',
|
|
67
|
+
executorToken: 'detail',
|
|
68
|
+
runtimeAuthority: 'detail',
|
|
69
|
+
runtimeApiBaseUrl: 'detail',
|
|
70
|
+
runtimeTokenOrigin: 'detail',
|
|
71
|
+
baseUrl: 'detail',
|
|
72
|
+
coordinatorUrl: 'detail',
|
|
73
|
+
coordinatorInternalToken: 'detail',
|
|
74
|
+
runtimeDeployVersion: 'detail',
|
|
75
|
+
absurdReleaseId: 'detail',
|
|
76
|
+
absurdReleaseEnvironment: 'detail',
|
|
77
|
+
vercelProtectionBypassToken: 'detail',
|
|
78
|
+
preferredReceiptGatewayMachineId: 'detail',
|
|
79
|
+
runtimeTestFaultHeader: 'detail',
|
|
80
|
+
testPolicyOverrides: 'detail',
|
|
81
|
+
submittedAtMs: 'detail',
|
|
82
|
+
schedulerBackend: 'detail',
|
|
83
|
+
executionProfile: 'detail',
|
|
84
|
+
runtimeBackend: 'detail',
|
|
85
|
+
queueMaxDelaySeconds: 'detail',
|
|
86
|
+
queuePriority: 'detail',
|
|
87
|
+
userEmail: 'detail',
|
|
88
|
+
userId: 'detail',
|
|
89
|
+
launchIdentityDigest: 'detail',
|
|
90
|
+
};
|
|
91
|
+
|
|
92
|
+
export const PLAY_LAUNCH_IDENTITY_FIELDS_COMPARED_OUTSIDE_DIGEST: readonly string[] =
|
|
93
|
+
Object.freeze(['runtimeSandboxPlacementPolicyId']);
|
|
94
|
+
|
|
95
|
+
export const PLAY_LAUNCH_IDENTITY_FIELDS: readonly string[] = Object.freeze(
|
|
96
|
+
Object.entries(PLAY_LAUNCH_ENVELOPE_FIELD_ROLES)
|
|
97
|
+
.filter(([, role]) => role === 'identity')
|
|
98
|
+
.map(([field]) => field)
|
|
99
|
+
.sort(),
|
|
100
|
+
);
|
|
101
|
+
|
|
102
|
+
export const PLAY_LAUNCH_DETAIL_FIELDS: readonly string[] = Object.freeze(
|
|
103
|
+
Object.entries(PLAY_LAUNCH_ENVELOPE_FIELD_ROLES)
|
|
104
|
+
.filter(([, role]) => role === 'detail')
|
|
105
|
+
.map(([field]) => field)
|
|
106
|
+
.sort(),
|
|
107
|
+
);
|
|
108
|
+
|
|
109
|
+
export type PlayLaunchIdentitySource = Readonly<
|
|
110
|
+
Partial<Record<keyof PlaySchedulerSubmitInput, unknown>>
|
|
111
|
+
>;
|
|
112
|
+
|
|
113
|
+
function text(value: unknown): string | null {
|
|
114
|
+
return typeof value === 'string' && value.trim() ? value.trim() : null;
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
function flag(value: unknown): boolean {
|
|
118
|
+
return value === true;
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
function count(value: unknown): number | null {
|
|
122
|
+
return typeof value === 'number' && Number.isFinite(value) ? value : null;
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
function record(value: unknown): Record<string, unknown> | null {
|
|
126
|
+
return value && typeof value === 'object' && !Array.isArray(value)
|
|
127
|
+
? (value as Record<string, unknown>)
|
|
128
|
+
: null;
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
function stagedFileIdentity(value: unknown): Record<string, unknown> | null {
|
|
132
|
+
const file = record(value);
|
|
133
|
+
if (!file) return null;
|
|
134
|
+
return {
|
|
135
|
+
logicalPath:
|
|
136
|
+
text(file.logicalPath) ?? text(file.path) ?? text(file.playPath),
|
|
137
|
+
fileName: text(file.fileName) ?? text(file.name),
|
|
138
|
+
contentType: text(file.contentType),
|
|
139
|
+
bytes: count(file.bytes),
|
|
140
|
+
contentDigest:
|
|
141
|
+
typeof file.contentBase64 === 'string'
|
|
142
|
+
? sha256Hex(file.contentBase64)
|
|
143
|
+
: typeof file.inlineText === 'string'
|
|
144
|
+
? sha256Hex(file.inlineText)
|
|
145
|
+
: null,
|
|
146
|
+
};
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
function stagedFileListIdentity(
|
|
150
|
+
value: unknown,
|
|
151
|
+
): Array<Record<string, unknown>> {
|
|
152
|
+
if (!Array.isArray(value)) return [];
|
|
153
|
+
return value
|
|
154
|
+
.map((entry) => stagedFileIdentity(entry))
|
|
155
|
+
.filter((entry): entry is Record<string, unknown> => entry !== null)
|
|
156
|
+
.sort((left, right) =>
|
|
157
|
+
stableStringify(left).localeCompare(stableStringify(right)),
|
|
158
|
+
);
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
/** Return only customer intent; transport, authority, and release details are excluded. */
|
|
162
|
+
export function buildPlayLaunchIdentity(
|
|
163
|
+
launch: PlayLaunchIdentitySource,
|
|
164
|
+
): Record<string, unknown> {
|
|
165
|
+
const inlineCsv = record(launch.inlineCsv);
|
|
166
|
+
return {
|
|
167
|
+
orgId: text(launch.orgId),
|
|
168
|
+
playName: text(launch.playName),
|
|
169
|
+
playReference: text(launch.playReference) ?? text(launch.playName),
|
|
170
|
+
definitionScope: text(launch.definitionScope),
|
|
171
|
+
revisionId: text(launch.revisionId),
|
|
172
|
+
artifactStorageKey: text(launch.artifactStorageKey),
|
|
173
|
+
artifactHash: text(launch.artifactHash),
|
|
174
|
+
graphHash: text(launch.graphHash),
|
|
175
|
+
workflowFamilyKey: text(launch.workflowFamilyKey),
|
|
176
|
+
inputSha256: text(launch.inputSha256),
|
|
177
|
+
source: text(launch.source),
|
|
178
|
+
triggerSource: text(launch.triggerSource),
|
|
179
|
+
parentRunId: text(launch.parentRunId),
|
|
180
|
+
rootRunId: text(launch.rootRunId),
|
|
181
|
+
asyncLaunchKey: text(launch.asyncLaunchKey),
|
|
182
|
+
asyncInvocationKey: text(launch.asyncInvocationKey),
|
|
183
|
+
asyncRequestFingerprint: text(launch.asyncRequestFingerprint),
|
|
184
|
+
force: flag(launch.force),
|
|
185
|
+
forceToolRefresh: flag(launch.forceToolRefresh),
|
|
186
|
+
integrationMode: text(launch.integrationMode),
|
|
187
|
+
maxConcurrentExternalCalls:
|
|
188
|
+
count(launch.maxConcurrentExternalCalls) ??
|
|
189
|
+
LEGACY_MAX_CONCURRENT_EXTERNAL_CALLS,
|
|
190
|
+
maxConcurrentRows: count(launch.maxConcurrentRows),
|
|
191
|
+
inputDigest: sha256Hex(stableStringify(launch.input ?? {})),
|
|
192
|
+
inputFile: stagedFileIdentity(launch.inputFile),
|
|
193
|
+
inlineInputFile: stagedFileIdentity(launch.inlineInputFile),
|
|
194
|
+
inlineCsv: inlineCsv
|
|
195
|
+
? {
|
|
196
|
+
name: text(inlineCsv.name),
|
|
197
|
+
rowsDigest: sha256Hex(stableStringify(inlineCsv.rows ?? [])),
|
|
198
|
+
}
|
|
199
|
+
: null,
|
|
200
|
+
packagedFiles: stagedFileListIdentity(launch.packagedFiles),
|
|
201
|
+
};
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
export const PLAY_LAUNCH_DIGESTED_IDENTITY_FIELDS: readonly string[] =
|
|
205
|
+
Object.freeze(
|
|
206
|
+
PLAY_LAUNCH_IDENTITY_FIELDS.filter(
|
|
207
|
+
(field) =>
|
|
208
|
+
!PLAY_LAUNCH_IDENTITY_FIELDS_COMPARED_OUTSIDE_DIGEST.includes(field),
|
|
209
|
+
),
|
|
210
|
+
);
|
|
211
|
+
|
|
212
|
+
export function buildPlayLaunchIdentityDigest(
|
|
213
|
+
launch: PlayLaunchIdentitySource,
|
|
214
|
+
): string {
|
|
215
|
+
return sha256Hex(
|
|
216
|
+
`${PLAY_LAUNCH_IDENTITY_DIGEST_NAMESPACE}\0${stableStringify(
|
|
217
|
+
buildPlayLaunchIdentity(launch),
|
|
218
|
+
)}`,
|
|
219
|
+
);
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
export function withPlayLaunchIdentityDigest<
|
|
223
|
+
T extends PlaySchedulerSubmitInput,
|
|
224
|
+
>(launch: T): T {
|
|
225
|
+
return {
|
|
226
|
+
...launch,
|
|
227
|
+
launchIdentityDigest: buildPlayLaunchIdentityDigest(launch),
|
|
228
|
+
};
|
|
229
|
+
}
|
|
@@ -15,6 +15,9 @@ export function cloneMapFrame(frame: MapExecutionFrame): MapExecutionFrame {
|
|
|
15
15
|
suspendedInputRows: frame.suspendedInputRows.map((row) => ({
|
|
16
16
|
inputIndex: row.inputIndex,
|
|
17
17
|
data: { ...row.data },
|
|
18
|
+
...(row.authoredInputDigest
|
|
19
|
+
? { authoredInputDigest: row.authoredInputDigest }
|
|
20
|
+
: {}),
|
|
18
21
|
})),
|
|
19
22
|
}
|
|
20
23
|
: {}),
|
|
@@ -42,16 +42,31 @@ export type PlayRunAdmissionJob = {
|
|
|
42
42
|
admit(input: PlayRunAdmissionInput): Promise<PlayRunAdmissionOutcome>;
|
|
43
43
|
};
|
|
44
44
|
|
|
45
|
+
export type PlayRunLaunchConflictReason = 'launch_identity_mismatch';
|
|
46
|
+
|
|
45
47
|
/** A stable, machine-readable rejection for a reused run id with new intent. */
|
|
46
48
|
export class PlayRunLaunchConflictError extends Error {
|
|
47
49
|
readonly code = 'PLAY_RUN_LAUNCH_CONFLICT';
|
|
50
|
+
readonly conflictReason: PlayRunLaunchConflictReason | undefined;
|
|
51
|
+
readonly conflictingFields: string[] | undefined;
|
|
48
52
|
|
|
49
|
-
constructor(
|
|
53
|
+
constructor(
|
|
54
|
+
runId: string,
|
|
55
|
+
options?: {
|
|
56
|
+
cause?: unknown;
|
|
57
|
+
conflictReason?: PlayRunLaunchConflictReason;
|
|
58
|
+
conflictingFields?: readonly string[];
|
|
59
|
+
},
|
|
60
|
+
) {
|
|
50
61
|
super(
|
|
51
62
|
`Play run "${runId}" already exists with a different launch.`,
|
|
52
63
|
options,
|
|
53
64
|
);
|
|
54
65
|
this.name = 'PlayRunLaunchConflictError';
|
|
66
|
+
this.conflictReason = options?.conflictReason;
|
|
67
|
+
this.conflictingFields = options?.conflictingFields
|
|
68
|
+
? [...options.conflictingFields]
|
|
69
|
+
: undefined;
|
|
55
70
|
}
|
|
56
71
|
}
|
|
57
72
|
|
package/dist/bundling-sources/shared_libs/play-runtime/play-runs/admission/postgres-adapter.ts
CHANGED
|
@@ -54,6 +54,9 @@ export function createPostgresPlayRunAdmissionPorts(input: {
|
|
|
54
54
|
artifactHash: admission.launch.artifactHash,
|
|
55
55
|
graphHash: admission.launch.graphHash,
|
|
56
56
|
inputSha256: admission.launch.inputSha256,
|
|
57
|
+
launchIdentityDigest: admission.launch.launchIdentityDigest,
|
|
58
|
+
conflictReason: conflict.conflictReason ?? null,
|
|
59
|
+
conflictingFields: conflict.conflictingFields ?? null,
|
|
57
60
|
error: conflict,
|
|
58
61
|
});
|
|
59
62
|
throw conflict;
|
|
@@ -79,6 +79,8 @@ export type PlayRunnerRateStateConcurrencyGrant = {
|
|
|
79
79
|
export type PlayRunnerRateStateAcquireResult = {
|
|
80
80
|
granted: number;
|
|
81
81
|
waitMs: number;
|
|
82
|
+
/** This receiver serves rate_state_validate. Absent on pre-expansion readers. */
|
|
83
|
+
scopeValidationSupported?: boolean;
|
|
82
84
|
/**
|
|
83
85
|
* Gateway clock used to produce `scheduledAtMs`. Runners translate the
|
|
84
86
|
* schedule into relative delays so cross-machine clock skew cannot compress
|
|
@@ -108,8 +110,8 @@ export type PlayRunnerRateStateAcquireResult = {
|
|
|
108
110
|
*/
|
|
109
111
|
coolUntilMs?: number;
|
|
110
112
|
/**
|
|
111
|
-
*
|
|
112
|
-
*
|
|
113
|
+
* Most recent provider Retry-After deadline, surfaced for diagnostics. The
|
|
114
|
+
* authoritative admission gate is the durable provider-scope hold row.
|
|
113
115
|
*/
|
|
114
116
|
claimedRetryAtMs?: number;
|
|
115
117
|
};
|
|
@@ -128,6 +130,12 @@ export type PlayRunnerRateStatePenalizeInput = {
|
|
|
128
130
|
bucketId: string;
|
|
129
131
|
rateScopeToken?: string | null;
|
|
130
132
|
cooldownMs: number;
|
|
133
|
+
/** Absolute provider retry deadline captured with the observed response. */
|
|
134
|
+
retryAtMs?: number;
|
|
135
|
+
/** Unflushed confirmed successes, sharing acquire's replay-safe identity. */
|
|
136
|
+
observedSuccesses?: number;
|
|
137
|
+
concurrencyObservations?: PlayRunnerRateStateConcurrencyObservation[];
|
|
138
|
+
concurrencyObservationBatch?: PlayRunnerRateStateConcurrencyObservationBatch;
|
|
131
139
|
/**
|
|
132
140
|
* The same resolved rules that admitted the attempt. Optional during the
|
|
133
141
|
* rolling deploy window: old runners retain rate-only backpressure handling.
|
|
@@ -140,6 +148,18 @@ export type PlayRunnerRateStatePenalizeInput = {
|
|
|
140
148
|
schedulerSchema?: string | null;
|
|
141
149
|
};
|
|
142
150
|
|
|
151
|
+
export type PlayRunnerRateStateValidateInput = {
|
|
152
|
+
bucketId: string;
|
|
153
|
+
rateScopeToken?: string | null;
|
|
154
|
+
schedulerSchema?: string | null;
|
|
155
|
+
};
|
|
156
|
+
|
|
157
|
+
export type PlayRunnerRateStateValidateResult = {
|
|
158
|
+
allowed: boolean;
|
|
159
|
+
serverNowMs: number;
|
|
160
|
+
retryAtMs: number;
|
|
161
|
+
};
|
|
162
|
+
|
|
143
163
|
export type PlayRunnerBudgetCharge = {
|
|
144
164
|
key: string;
|
|
145
165
|
amount: number;
|
|
@@ -711,6 +711,7 @@ export interface RuntimeResourceGovernor {
|
|
|
711
711
|
acquireProviderPermit(input: {
|
|
712
712
|
toolId: string;
|
|
713
713
|
signal?: AbortSignal;
|
|
714
|
+
yieldProviderHolds?: boolean;
|
|
714
715
|
}): Promise<RuntimeProviderPermit>;
|
|
715
716
|
/**
|
|
716
717
|
* Non-blocking provider admission from capacity this process already holds.
|
|
@@ -748,6 +749,7 @@ export interface RuntimeResourceGovernor {
|
|
|
748
749
|
provider: string;
|
|
749
750
|
toolId?: string;
|
|
750
751
|
retryAfterMs: number;
|
|
752
|
+
retryAtMs?: number;
|
|
751
753
|
/** Provider residence for the rejected physical request, when observed. */
|
|
752
754
|
latencyMs?: number | null;
|
|
753
755
|
/** Server-authored context for attributing this physical provider 429. */
|
|
@@ -1585,7 +1587,10 @@ export function createRuntimeResourceGovernor(input: {
|
|
|
1585
1587
|
const startedAt = Date.now();
|
|
1586
1588
|
const lease = await input.executionGovernor.acquireProviderPermit(
|
|
1587
1589
|
providerInput.toolId,
|
|
1588
|
-
{
|
|
1590
|
+
{
|
|
1591
|
+
signal: providerInput.signal,
|
|
1592
|
+
yieldProviderHolds: providerInput.yieldProviderHolds,
|
|
1593
|
+
},
|
|
1589
1594
|
);
|
|
1590
1595
|
observe({
|
|
1591
1596
|
providerResourceKey: lease.provider,
|
|
@@ -241,15 +241,17 @@ export function providerExhaustedMessage(input: {
|
|
|
241
241
|
}
|
|
242
242
|
|
|
243
243
|
/**
|
|
244
|
-
*
|
|
245
|
-
*
|
|
246
|
-
*
|
|
247
|
-
*
|
|
248
|
-
*
|
|
244
|
+
* Control signal from the DB-authoritative pacer when a provider hold exceeds
|
|
245
|
+
* the in-process wait threshold. The next provider call is not dispatched.
|
|
246
|
+
* Managed durable Plays translate this into a persisted run/row wait and resume
|
|
247
|
+
* after the deadline; callers without durable boundaries receive a repairable
|
|
248
|
+
* provider-exhausted outcome. This signal is not a terminal row failure in the
|
|
249
|
+
* managed durable path.
|
|
249
250
|
*/
|
|
250
251
|
export class ProviderExhaustedError extends Error {
|
|
251
252
|
readonly code = PROVIDER_EXHAUSTED_CODE;
|
|
252
253
|
readonly provider: string;
|
|
254
|
+
readonly publicDetails: ToolExecutionError['publicDetails'];
|
|
253
255
|
/** ISO 8601 retry deadline (from the server Retry-After / cooldown). */
|
|
254
256
|
readonly retryAt: string;
|
|
255
257
|
constructor(input: { provider: string; retryAtMs: number; cause?: unknown }) {
|
|
@@ -264,6 +266,10 @@ export class ProviderExhaustedError extends Error {
|
|
|
264
266
|
);
|
|
265
267
|
this.name = 'ProviderExhaustedError';
|
|
266
268
|
this.provider = input.provider;
|
|
269
|
+
this.publicDetails =
|
|
270
|
+
input.cause instanceof ToolExecutionError
|
|
271
|
+
? input.cause.publicDetails
|
|
272
|
+
: null;
|
|
267
273
|
this.retryAt = Number.isFinite(Number(input.retryAtMs))
|
|
268
274
|
? new Date(Number(input.retryAtMs)).toISOString()
|
|
269
275
|
: new Date(0).toISOString();
|