deepline 0.1.252 → 0.1.254
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/sdk/src/release.ts +8 -7
- package/dist/bundling-sources/sdk/src/types.ts +2 -0
- package/dist/bundling-sources/shared_libs/play-runtime/context.ts +261 -162
- package/dist/bundling-sources/shared_libs/play-runtime/governor/governor.ts +37 -35
- package/dist/bundling-sources/shared_libs/play-runtime/ledger-safe-payload.ts +20 -5
- package/dist/bundling-sources/shared_libs/play-runtime/output-size-limits.ts +462 -29
- package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts +16 -0
- package/dist/bundling-sources/shared_libs/play-runtime/resource-governor.ts +11 -0
- package/dist/bundling-sources/shared_libs/play-runtime/run-failure.ts +44 -1
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-lifecycle.ts +7 -4
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-payload-transport.ts +98 -13
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-runtime-watchdog.ts +117 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts +19 -3
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/types.ts +1 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-api-paths.ts +4 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-constants.ts +13 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-contract.ts +22 -0
- package/dist/bundling-sources/shared_libs/play-runtime/secret-resolution-retry-policy.ts +80 -0
- package/dist/cli/index.js +80 -51
- package/dist/cli/index.mjs +80 -51
- package/dist/index.d.mts +2 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.js +13 -10
- package/dist/index.mjs +13 -10
- package/package.json +1 -1
|
@@ -116,7 +116,8 @@ export const SDK_RELEASE = {
|
|
|
116
116
|
// 0.1.252 hard-cuts the customer Monitor catalog to the two launched
|
|
117
117
|
// Deepline-native radars. Older clients must update before discovering,
|
|
118
118
|
// checking, or deploying an unlaunched monitor integration.
|
|
119
|
-
|
|
119
|
+
// 0.1.253 makes play-page browser opening opt-in and retires --no-open.
|
|
120
|
+
version: '0.1.254',
|
|
120
121
|
apiContract: '2026-07-native-monitor-launch-hard-cutover',
|
|
121
122
|
supportPolicy: {
|
|
122
123
|
minimumSupported: '0.1.53',
|
|
@@ -124,9 +125,9 @@ export const SDK_RELEASE = {
|
|
|
124
125
|
commandMinimumSupported: [
|
|
125
126
|
{
|
|
126
127
|
command: 'enrich',
|
|
127
|
-
minimumSupported: '0.1.
|
|
128
|
+
minimumSupported: '0.1.253',
|
|
128
129
|
reason:
|
|
129
|
-
'Older SDK CLI enrich generated stale play source; SDK CLI enrich 0.1.153 generated removed StepOptions fields in play source; SDK CLI enrich before 0.1.175 could silently succeed on the empty-waterfall bug when a requested waterfall returned no values for selected rows; SDK CLI enrich before 0.1.230 could convert extractor failures into successful unmatched results; SDK CLI enrich before 0.1.238 could drop a nested provider email match during generic pick("email") extraction and materialize it as NO_MATCH.',
|
|
130
|
+
'Older SDK CLI enrich generated stale play source; SDK CLI enrich 0.1.153 generated removed StepOptions fields in play source; SDK CLI enrich before 0.1.175 could silently succeed on the empty-waterfall bug when a requested waterfall returned no values for selected rows; SDK CLI enrich before 0.1.230 could convert extractor failures into successful unmatched results; SDK CLI enrich before 0.1.238 could drop a nested provider email match during generic pick("email") extraction and materialize it as NO_MATCH; SDK CLI enrich before 0.1.253 opens the generated play page by default and accepts the removed --no-open flag.',
|
|
130
131
|
},
|
|
131
132
|
{
|
|
132
133
|
command: 'plays',
|
|
@@ -136,16 +137,16 @@ export const SDK_RELEASE = {
|
|
|
136
137
|
},
|
|
137
138
|
{
|
|
138
139
|
command: 'plays run',
|
|
139
|
-
minimumSupported: '0.1.
|
|
140
|
+
minimumSupported: '0.1.253',
|
|
140
141
|
reason:
|
|
141
|
-
'Older SDK CLI versions predate dataset-native Play results and can emit the retired esm_workers artifact contract.',
|
|
142
|
+
'Older SDK CLI versions predate dataset-native Play results and can emit the retired esm_workers artifact contract; versions before 0.1.253 open the play page by default and accept the removed --no-open flag.',
|
|
142
143
|
},
|
|
143
144
|
{
|
|
144
145
|
command: 'run',
|
|
145
146
|
displayCommand: 'plays run',
|
|
146
|
-
minimumSupported: '0.1.
|
|
147
|
+
minimumSupported: '0.1.253',
|
|
147
148
|
reason:
|
|
148
|
-
'Older SDK CLI versions predate dataset-native Play results and can emit the retired esm_workers artifact contract.',
|
|
149
|
+
'Older SDK CLI versions predate dataset-native Play results and can emit the retired esm_workers artifact contract; versions before 0.1.253 open the play page by default and accept the removed --no-open flag.',
|
|
149
150
|
},
|
|
150
151
|
{
|
|
151
152
|
command: 'plays check',
|
|
@@ -551,6 +551,8 @@ export interface PlayRunPackage {
|
|
|
551
551
|
durationMs?: number | null;
|
|
552
552
|
error?: string;
|
|
553
553
|
};
|
|
554
|
+
/** Bounded customer-safe warnings about output projection or availability. */
|
|
555
|
+
warnings?: string[];
|
|
554
556
|
/** Step-level summaries emitted by the runtime. */
|
|
555
557
|
steps: Array<Record<string, unknown>>;
|
|
556
558
|
/** Named output summaries, including dataset handles and scalar outputs. */
|
|
@@ -42,7 +42,17 @@ import {
|
|
|
42
42
|
mapRowOutcomeRuntimeFields,
|
|
43
43
|
resolveMapRowOutcomeKey,
|
|
44
44
|
} from './map-row-outcome';
|
|
45
|
-
import {
|
|
45
|
+
import {
|
|
46
|
+
PLAY_RUNTIME_API_COMPAT_PATH,
|
|
47
|
+
PLAY_RUNTIME_OPERATION_ATTEMPT_HEADER,
|
|
48
|
+
PLAY_RUNTIME_OPERATION_ID_HEADER,
|
|
49
|
+
} from './runtime-api-paths';
|
|
50
|
+
import {
|
|
51
|
+
isRetryableSecretResolutionStatus,
|
|
52
|
+
secretResolutionRetryDecision,
|
|
53
|
+
SECRET_RESOLUTION_MAX_ATTEMPTS,
|
|
54
|
+
type SecretResolutionRetryDecision,
|
|
55
|
+
} from './secret-resolution-retry-policy';
|
|
46
56
|
import {
|
|
47
57
|
createRootRunExecutionScope,
|
|
48
58
|
deriveChildRunExecutionScope,
|
|
@@ -519,6 +529,26 @@ function loadSafeFetch(): Promise<SafeFetchModule> {
|
|
|
519
529
|
return safeFetchModule;
|
|
520
530
|
}
|
|
521
531
|
|
|
532
|
+
function waitForSecretResolutionRetry(ms: number): Promise<void> {
|
|
533
|
+
if (ms <= 0) return Promise.resolve();
|
|
534
|
+
return new Promise<void>((resolve) => setTimeout(resolve, ms));
|
|
535
|
+
}
|
|
536
|
+
|
|
537
|
+
function cancelRuntimeResponseBody(response: Response): void {
|
|
538
|
+
try {
|
|
539
|
+
void response.body?.cancel().catch(() => {});
|
|
540
|
+
} catch {
|
|
541
|
+
// Connection cleanup must never replace the resolver failure or retry.
|
|
542
|
+
}
|
|
543
|
+
}
|
|
544
|
+
|
|
545
|
+
const NO_SECRET_RESOLUTION_RETRY: SecretResolutionRetryDecision = {
|
|
546
|
+
retry: false,
|
|
547
|
+
retryDelayMs: 0,
|
|
548
|
+
retryAfterMs: null,
|
|
549
|
+
retryAfterExceedsCap: false,
|
|
550
|
+
};
|
|
551
|
+
|
|
522
552
|
function isUnsafeOutboundUrlError(error: unknown): boolean {
|
|
523
553
|
return error instanceof Error && error.name === 'UnsafeOutboundUrlError';
|
|
524
554
|
}
|
|
@@ -1543,106 +1573,160 @@ export class PlayContextImpl {
|
|
|
1543
1573
|
this.#options.runId
|
|
1544
1574
|
) {
|
|
1545
1575
|
const url = `${this.#options.baseUrl.replace(/\/$/, '')}${PLAY_RUNTIME_API_COMPAT_PATH}`;
|
|
1546
|
-
const
|
|
1547
|
-
const
|
|
1548
|
-
|
|
1549
|
-
|
|
1550
|
-
|
|
1551
|
-
|
|
1552
|
-
|
|
1553
|
-
|
|
1554
|
-
|
|
1555
|
-
|
|
1556
|
-
|
|
1557
|
-
|
|
1558
|
-
|
|
1559
|
-
|
|
1560
|
-
|
|
1561
|
-
|
|
1562
|
-
|
|
1563
|
-
|
|
1564
|
-
|
|
1565
|
-
|
|
1566
|
-
|
|
1567
|
-
|
|
1568
|
-
|
|
1569
|
-
|
|
1570
|
-
|
|
1571
|
-
|
|
1572
|
-
|
|
1573
|
-
|
|
1574
|
-
|
|
1575
|
-
|
|
1576
|
-
|
|
1577
|
-
|
|
1578
|
-
|
|
1579
|
-
|
|
1576
|
+
const operationId = `ctx-secret-operation-${crypto.randomUUID()}`;
|
|
1577
|
+
const operationStartedAt = Date.now();
|
|
1578
|
+
for (
|
|
1579
|
+
let attempt = 1;
|
|
1580
|
+
attempt <= SECRET_RESOLUTION_MAX_ATTEMPTS;
|
|
1581
|
+
attempt += 1
|
|
1582
|
+
) {
|
|
1583
|
+
const requestId = `ctx-secret-${crypto.randomUUID()}`;
|
|
1584
|
+
const attemptStartedAt = Date.now();
|
|
1585
|
+
let response: Response;
|
|
1586
|
+
try {
|
|
1587
|
+
response = await fetch(url, {
|
|
1588
|
+
method: 'POST',
|
|
1589
|
+
headers: {
|
|
1590
|
+
Authorization: `Bearer ${this.#options.executorToken}`,
|
|
1591
|
+
'Content-Type': 'application/json',
|
|
1592
|
+
'x-deepline-request-id': requestId,
|
|
1593
|
+
[PLAY_RUNTIME_OPERATION_ID_HEADER]: operationId,
|
|
1594
|
+
[PLAY_RUNTIME_OPERATION_ATTEMPT_HEADER]: String(attempt),
|
|
1595
|
+
...(await this.vercelProtectionHeaders()),
|
|
1596
|
+
},
|
|
1597
|
+
body: JSON.stringify({
|
|
1598
|
+
action: 'resolve_secret',
|
|
1599
|
+
name: auth.secret.name,
|
|
1600
|
+
playName: this.#options.playName,
|
|
1601
|
+
}),
|
|
1602
|
+
});
|
|
1603
|
+
} catch (error) {
|
|
1604
|
+
const diagnostic = describeTransportError(error);
|
|
1605
|
+
const retry = secretResolutionRetryDecision({
|
|
1606
|
+
attempt,
|
|
1607
|
+
retryAfter: null,
|
|
1608
|
+
});
|
|
1609
|
+
this.log(
|
|
1610
|
+
`[runtime.secret_resolution_failure] ${JSON.stringify({
|
|
1611
|
+
stage: 'transport',
|
|
1612
|
+
secret_name: auth.secret.name,
|
|
1613
|
+
gateway_origin: transportGatewayOriginForDiagnostic(url),
|
|
1614
|
+
operation_id: operationId,
|
|
1615
|
+
request_id: requestId,
|
|
1616
|
+
attempt,
|
|
1617
|
+
max_attempts: SECRET_RESOLUTION_MAX_ATTEMPTS,
|
|
1618
|
+
retrying: retry.retry,
|
|
1619
|
+
retry_delay_ms: retry.retryDelayMs,
|
|
1620
|
+
elapsed_ms: Date.now() - attemptStartedAt,
|
|
1621
|
+
operation_elapsed_ms: Date.now() - operationStartedAt,
|
|
1622
|
+
error: diagnostic,
|
|
1623
|
+
})}`,
|
|
1624
|
+
);
|
|
1625
|
+
if (!retry.retry) {
|
|
1626
|
+
throw new Error(
|
|
1627
|
+
`Secret ${auth.secret.name} resolution transport failed (request_id=${requestId}): ${diagnostic.message ?? 'unknown transport error'}`,
|
|
1628
|
+
);
|
|
1629
|
+
}
|
|
1630
|
+
await waitForSecretResolutionRetry(retry.retryDelayMs);
|
|
1631
|
+
continue;
|
|
1632
|
+
}
|
|
1580
1633
|
|
|
1581
|
-
|
|
1582
|
-
|
|
1583
|
-
|
|
1584
|
-
|
|
1585
|
-
|
|
1586
|
-
|
|
1587
|
-
|
|
1588
|
-
|
|
1589
|
-
|
|
1590
|
-
|
|
1591
|
-
|
|
1592
|
-
|
|
1593
|
-
|
|
1594
|
-
|
|
1595
|
-
|
|
1596
|
-
|
|
1597
|
-
|
|
1598
|
-
|
|
1599
|
-
|
|
1600
|
-
|
|
1601
|
-
|
|
1602
|
-
|
|
1603
|
-
|
|
1604
|
-
|
|
1605
|
-
|
|
1634
|
+
const retryAfter = response.headers.get('retry-after');
|
|
1635
|
+
const retry = isRetryableSecretResolutionStatus(response.status)
|
|
1636
|
+
? secretResolutionRetryDecision({ attempt, retryAfter })
|
|
1637
|
+
: NO_SECRET_RESOLUTION_RETRY;
|
|
1638
|
+
const responseDiagnostic = {
|
|
1639
|
+
stage: response.ok ? 'invalid_response' : 'http_response',
|
|
1640
|
+
secret_name: auth.secret.name,
|
|
1641
|
+
gateway_origin: transportGatewayOriginForDiagnostic(url),
|
|
1642
|
+
operation_id: operationId,
|
|
1643
|
+
request_id: requestId,
|
|
1644
|
+
attempt,
|
|
1645
|
+
max_attempts: SECRET_RESOLUTION_MAX_ATTEMPTS,
|
|
1646
|
+
retrying: retry.retry,
|
|
1647
|
+
retry_delay_ms: retry.retryDelayMs,
|
|
1648
|
+
elapsed_ms: Date.now() - attemptStartedAt,
|
|
1649
|
+
operation_elapsed_ms: Date.now() - operationStartedAt,
|
|
1650
|
+
http_status: response.status,
|
|
1651
|
+
status_text: response.statusText || null,
|
|
1652
|
+
content_type: response.headers.get('content-type'),
|
|
1653
|
+
server: response.headers.get('server'),
|
|
1654
|
+
vercel_request_id:
|
|
1655
|
+
response.headers.get('x-vercel-id') ??
|
|
1656
|
+
response.headers.get('x-vercel-request-id'),
|
|
1657
|
+
response_request_id: response.headers.get('x-deepline-request-id'),
|
|
1658
|
+
error_code: response.headers.get('x-deepline-error-code'),
|
|
1659
|
+
retry_after: retryAfter,
|
|
1660
|
+
retry_after_ms: retry.retryAfterMs,
|
|
1661
|
+
retry_after_exceeds_cap: retry.retryAfterExceedsCap,
|
|
1662
|
+
};
|
|
1663
|
+
if (!response.ok) {
|
|
1664
|
+
this.log(
|
|
1665
|
+
`[runtime.secret_resolution_failure] ${JSON.stringify(responseDiagnostic)}`,
|
|
1666
|
+
);
|
|
1667
|
+
cancelRuntimeResponseBody(response);
|
|
1668
|
+
if (retry.retry) {
|
|
1669
|
+
await waitForSecretResolutionRetry(retry.retryDelayMs);
|
|
1670
|
+
continue;
|
|
1671
|
+
}
|
|
1672
|
+
throw new Error(
|
|
1673
|
+
`Secret ${auth.secret.name} is not available to this run (resolution status=${response.status}, request_id=${requestId}).`,
|
|
1674
|
+
);
|
|
1675
|
+
}
|
|
1606
1676
|
|
|
1607
|
-
|
|
1608
|
-
|
|
1609
|
-
|
|
1610
|
-
|
|
1611
|
-
|
|
1612
|
-
|
|
1613
|
-
|
|
1614
|
-
|
|
1615
|
-
|
|
1616
|
-
|
|
1617
|
-
|
|
1618
|
-
|
|
1619
|
-
|
|
1620
|
-
|
|
1621
|
-
|
|
1622
|
-
|
|
1677
|
+
let payload: unknown;
|
|
1678
|
+
try {
|
|
1679
|
+
payload = await response.json();
|
|
1680
|
+
} catch (error) {
|
|
1681
|
+
this.log(
|
|
1682
|
+
`[runtime.secret_resolution_failure] ${JSON.stringify({
|
|
1683
|
+
...responseDiagnostic,
|
|
1684
|
+
response_kind: 'invalid_json',
|
|
1685
|
+
parse_error_name:
|
|
1686
|
+
error instanceof Error ? error.name : typeof error,
|
|
1687
|
+
})}`,
|
|
1688
|
+
);
|
|
1689
|
+
throw new Error(
|
|
1690
|
+
`Secret ${auth.secret.name} is not available to this run (invalid resolution response, request_id=${requestId}).`,
|
|
1691
|
+
);
|
|
1692
|
+
}
|
|
1623
1693
|
|
|
1624
|
-
|
|
1625
|
-
|
|
1626
|
-
|
|
1627
|
-
|
|
1628
|
-
|
|
1629
|
-
|
|
1630
|
-
|
|
1631
|
-
|
|
1632
|
-
|
|
1633
|
-
|
|
1634
|
-
|
|
1635
|
-
|
|
1636
|
-
|
|
1637
|
-
|
|
1638
|
-
|
|
1639
|
-
|
|
1640
|
-
|
|
1641
|
-
|
|
1642
|
-
|
|
1643
|
-
|
|
1644
|
-
|
|
1645
|
-
|
|
1694
|
+
value =
|
|
1695
|
+
payload &&
|
|
1696
|
+
typeof payload === 'object' &&
|
|
1697
|
+
!Array.isArray(payload) &&
|
|
1698
|
+
typeof (payload as { value?: unknown }).value === 'string'
|
|
1699
|
+
? (payload as { value: string }).value
|
|
1700
|
+
: null;
|
|
1701
|
+
if (!value) {
|
|
1702
|
+
this.log(
|
|
1703
|
+
`[runtime.secret_resolution_failure] ${JSON.stringify({
|
|
1704
|
+
...responseDiagnostic,
|
|
1705
|
+
response_kind:
|
|
1706
|
+
payload === null
|
|
1707
|
+
? 'null'
|
|
1708
|
+
: Array.isArray(payload)
|
|
1709
|
+
? 'array'
|
|
1710
|
+
: typeof payload,
|
|
1711
|
+
})}`,
|
|
1712
|
+
);
|
|
1713
|
+
throw new Error(
|
|
1714
|
+
`Secret ${auth.secret.name} is not available to this run (missing resolution value, request_id=${requestId}).`,
|
|
1715
|
+
);
|
|
1716
|
+
}
|
|
1717
|
+
if (attempt > 1) {
|
|
1718
|
+
this.log(
|
|
1719
|
+
`[runtime.secret_resolution_recovered] ${JSON.stringify({
|
|
1720
|
+
secret_name: auth.secret.name,
|
|
1721
|
+
gateway_origin: transportGatewayOriginForDiagnostic(url),
|
|
1722
|
+
operation_id: operationId,
|
|
1723
|
+
request_id: requestId,
|
|
1724
|
+
attempts: attempt,
|
|
1725
|
+
operation_elapsed_ms: Date.now() - operationStartedAt,
|
|
1726
|
+
})}`,
|
|
1727
|
+
);
|
|
1728
|
+
}
|
|
1729
|
+
break;
|
|
1646
1730
|
}
|
|
1647
1731
|
} else {
|
|
1648
1732
|
throw new Error(
|
|
@@ -8282,9 +8366,9 @@ export class PlayContextImpl {
|
|
|
8282
8366
|
const url = `${this.#options.baseUrl}/api/v2/integrations/${encodeURIComponent(toolId)}/execute`;
|
|
8283
8367
|
const timeoutMs = resolveToolRuntimeTimeoutMs(toolId, options?.timeoutMs);
|
|
8284
8368
|
|
|
8285
|
-
//
|
|
8286
|
-
//
|
|
8287
|
-
//
|
|
8369
|
+
// Hold one logical tool slot and charge one logical tool call for this
|
|
8370
|
+
// execution. Per-provider rate admission is intentionally deferred until
|
|
8371
|
+
// each physical fetch is ready to leave this process.
|
|
8288
8372
|
const admissionStartedAt = Date.now();
|
|
8289
8373
|
const toolSlot = await this.resourceGovernor.acquireTool({
|
|
8290
8374
|
orgId: this.#options.orgId ?? null,
|
|
@@ -8408,7 +8492,6 @@ export class PlayContextImpl {
|
|
|
8408
8492
|
`[perf] tool call id=${toolId} phase=before_provider_ownership elapsed_ms=${Date.now() - ownershipStartedAt}`,
|
|
8409
8493
|
);
|
|
8410
8494
|
}
|
|
8411
|
-
providerCallStartedAt = Date.now();
|
|
8412
8495
|
const heartbeatIntervalMs = hasReceiptHeartbeat
|
|
8413
8496
|
? runtimeLeaseHeartbeatIntervalFromExpiry({
|
|
8414
8497
|
leaseExpiresAt: options?.receiptLeaseExpiresAt,
|
|
@@ -8438,71 +8521,87 @@ export class PlayContextImpl {
|
|
|
8438
8521
|
: null;
|
|
8439
8522
|
receiptHeartbeatSupervisor?.start();
|
|
8440
8523
|
try {
|
|
8524
|
+
// Complete all local/receipt preparation before taking the
|
|
8525
|
+
// shared provider permit. Once admitted, the next awaited work
|
|
8526
|
+
// is the fetch itself, so independently delayed runners cannot
|
|
8527
|
+
// compress real provider arrivals after their tickets.
|
|
8528
|
+
const protectionHeaders = await this.vercelProtectionHeaders();
|
|
8529
|
+
const providerPermit =
|
|
8530
|
+
await this.resourceGovernor.acquireProviderPermit({
|
|
8531
|
+
toolId,
|
|
8532
|
+
signal: abortController?.signal,
|
|
8533
|
+
});
|
|
8441
8534
|
const integrationFetchStartedAt = Date.now();
|
|
8442
|
-
|
|
8443
|
-
|
|
8444
|
-
|
|
8445
|
-
|
|
8446
|
-
|
|
8447
|
-
|
|
8448
|
-
|
|
8449
|
-
|
|
8450
|
-
|
|
8451
|
-
|
|
8452
|
-
|
|
8453
|
-
|
|
8454
|
-
|
|
8455
|
-
|
|
8456
|
-
}
|
|
8457
|
-
: {}),
|
|
8458
|
-
...(await this.vercelProtectionHeaders()),
|
|
8459
|
-
},
|
|
8460
|
-
body: JSON.stringify({
|
|
8461
|
-
payload: input,
|
|
8462
|
-
metadata: {
|
|
8463
|
-
parent_run_id: this.#options.runId,
|
|
8464
|
-
...(durableCallReceiptKey
|
|
8465
|
-
? {
|
|
8466
|
-
durable_call_receipt_key: durableCallReceiptKey,
|
|
8467
|
-
...(executionAuthScopeDigest
|
|
8468
|
-
? {
|
|
8469
|
-
execution_auth_scope_digest:
|
|
8470
|
-
executionAuthScopeDigest,
|
|
8471
|
-
}
|
|
8472
|
-
: {}),
|
|
8473
|
-
}
|
|
8474
|
-
: {}),
|
|
8475
|
-
...(options?.customerDbDataset
|
|
8535
|
+
providerCallStartedAt = integrationFetchStartedAt;
|
|
8536
|
+
try {
|
|
8537
|
+
response = await fetch(url, {
|
|
8538
|
+
method: 'POST',
|
|
8539
|
+
signal: abortController?.signal,
|
|
8540
|
+
headers: {
|
|
8541
|
+
'Content-Type': 'application/json',
|
|
8542
|
+
Authorization: `Bearer ${this.#options.executorToken}`,
|
|
8543
|
+
[EXECUTE_RESPONSE_CONTRACT_HEADER]:
|
|
8544
|
+
V2_EXECUTE_RESPONSE_CONTRACT,
|
|
8545
|
+
[EXECUTE_RESPONSE_INTENT_HEADER]: 'dataset',
|
|
8546
|
+
[EXECUTE_TOOL_METADATA_HEADER]: 'true',
|
|
8547
|
+
'x-deepline-request-id': deeplineRequestId,
|
|
8548
|
+
...(providerIdempotencyKey
|
|
8476
8549
|
? {
|
|
8477
|
-
|
|
8478
|
-
|
|
8479
|
-
offset: options.customerDbDataset.offset,
|
|
8480
|
-
page_size: options.customerDbDataset.pageSize,
|
|
8481
|
-
total_rows: options.customerDbDataset.totalRows,
|
|
8482
|
-
},
|
|
8483
|
-
...(isCustomerDbDatasetTool(toolId)
|
|
8484
|
-
? {
|
|
8485
|
-
customer_db_dataset: {
|
|
8486
|
-
limit: options.customerDbDataset.limit,
|
|
8487
|
-
offset: options.customerDbDataset.offset,
|
|
8488
|
-
page_size:
|
|
8489
|
-
options.customerDbDataset.pageSize,
|
|
8490
|
-
total_rows:
|
|
8491
|
-
options.customerDbDataset.totalRows,
|
|
8492
|
-
},
|
|
8493
|
-
}
|
|
8494
|
-
: {}),
|
|
8550
|
+
'x-deepline-idempotency-key':
|
|
8551
|
+
providerIdempotencyKey,
|
|
8495
8552
|
}
|
|
8496
8553
|
: {}),
|
|
8497
|
-
...
|
|
8498
|
-
? { provider_idempotency_key: providerIdempotencyKey }
|
|
8499
|
-
: {}),
|
|
8554
|
+
...protectionHeaders,
|
|
8500
8555
|
},
|
|
8501
|
-
|
|
8502
|
-
|
|
8503
|
-
: {
|
|
8504
|
-
|
|
8505
|
-
|
|
8556
|
+
body: JSON.stringify({
|
|
8557
|
+
payload: input,
|
|
8558
|
+
metadata: {
|
|
8559
|
+
parent_run_id: this.#options.runId,
|
|
8560
|
+
...(durableCallReceiptKey
|
|
8561
|
+
? {
|
|
8562
|
+
durable_call_receipt_key: durableCallReceiptKey,
|
|
8563
|
+
...(executionAuthScopeDigest
|
|
8564
|
+
? {
|
|
8565
|
+
execution_auth_scope_digest:
|
|
8566
|
+
executionAuthScopeDigest,
|
|
8567
|
+
}
|
|
8568
|
+
: {}),
|
|
8569
|
+
}
|
|
8570
|
+
: {}),
|
|
8571
|
+
...(options?.customerDbDataset
|
|
8572
|
+
? {
|
|
8573
|
+
query_result_dataset: {
|
|
8574
|
+
limit: options.customerDbDataset.limit,
|
|
8575
|
+
offset: options.customerDbDataset.offset,
|
|
8576
|
+
page_size: options.customerDbDataset.pageSize,
|
|
8577
|
+
total_rows: options.customerDbDataset.totalRows,
|
|
8578
|
+
},
|
|
8579
|
+
...(isCustomerDbDatasetTool(toolId)
|
|
8580
|
+
? {
|
|
8581
|
+
customer_db_dataset: {
|
|
8582
|
+
limit: options.customerDbDataset.limit,
|
|
8583
|
+
offset: options.customerDbDataset.offset,
|
|
8584
|
+
page_size:
|
|
8585
|
+
options.customerDbDataset.pageSize,
|
|
8586
|
+
total_rows:
|
|
8587
|
+
options.customerDbDataset.totalRows,
|
|
8588
|
+
},
|
|
8589
|
+
}
|
|
8590
|
+
: {}),
|
|
8591
|
+
}
|
|
8592
|
+
: {}),
|
|
8593
|
+
...(providerIdempotencyKey
|
|
8594
|
+
? { provider_idempotency_key: providerIdempotencyKey }
|
|
8595
|
+
: {}),
|
|
8596
|
+
},
|
|
8597
|
+
...(this.#options.integrationMode
|
|
8598
|
+
? { integration_mode: this.#options.integrationMode }
|
|
8599
|
+
: {}),
|
|
8600
|
+
}),
|
|
8601
|
+
});
|
|
8602
|
+
} finally {
|
|
8603
|
+
providerPermit.release();
|
|
8604
|
+
}
|
|
8506
8605
|
if (runtimeReceiptReadTraceEnabled) {
|
|
8507
8606
|
this.log(
|
|
8508
8607
|
`[perf] tool call id=${toolId} phase=integration_fetch_headers elapsed_ms=${Date.now() - integrationFetchStartedAt} status=${response.status}`,
|
|
@@ -6,7 +6,8 @@
|
|
|
6
6
|
* at once" policy. See ADR 0007 + CONTEXT.md.
|
|
7
7
|
*
|
|
8
8
|
* Surface (small, by design):
|
|
9
|
-
* - acquireRowSlot / acquireToolSlot →
|
|
9
|
+
* - acquireRowSlot / acquireToolSlot → local execution leases
|
|
10
|
+
* - acquireProviderPermit → per-attempt provider admission
|
|
10
11
|
* - chargeBudget → throws on breach
|
|
11
12
|
* - forkInlineChild → inline recursion scope
|
|
12
13
|
* - resolveRowConcurrency / reportProviderBackpressure / snapshot
|
|
@@ -102,15 +103,24 @@ export interface PlayExecutionGovernor {
|
|
|
102
103
|
/** Block until a map-row slot is free. */
|
|
103
104
|
acquireRowSlot(opts?: { signal?: AbortSignal }): Promise<WorkLease>;
|
|
104
105
|
/**
|
|
105
|
-
*
|
|
106
|
-
*
|
|
107
|
-
* lease. Parked future permits never consume scarce execution slots.
|
|
106
|
+
* Acquire the local execution slot and charge one logical tool-call budget.
|
|
107
|
+
* Provider pacing deliberately happens separately at the HTTP-attempt edge.
|
|
108
108
|
*/
|
|
109
109
|
acquireToolSlot(
|
|
110
110
|
toolId: string,
|
|
111
111
|
opts?: { signal?: AbortSignal },
|
|
112
112
|
): Promise<WorkLease>;
|
|
113
113
|
|
|
114
|
+
/**
|
|
115
|
+
* Block until one physical provider HTTP attempt is admitted under the
|
|
116
|
+
* provider's shared rate/concurrency rules. Call this immediately before the
|
|
117
|
+
* outbound request and again for every retry.
|
|
118
|
+
*/
|
|
119
|
+
acquireProviderPermit(
|
|
120
|
+
toolId: string,
|
|
121
|
+
opts?: { signal?: AbortSignal },
|
|
122
|
+
): Promise<WorkLease>;
|
|
123
|
+
|
|
114
124
|
/**
|
|
115
125
|
* Suggested batch parallelism for a tool: the provider's own rate hints
|
|
116
126
|
* tightened to the policy's suggested ceiling. No hints → the fallback.
|
|
@@ -373,44 +383,34 @@ export function createPlayExecutionGovernor(
|
|
|
373
383
|
policy,
|
|
374
384
|
|
|
375
385
|
acquireRowSlot: (opts) => rowSlots.acquire(opts?.signal),
|
|
376
|
-
async acquireToolSlot(
|
|
377
|
-
|
|
378
|
-
//
|
|
386
|
+
async acquireToolSlot(_toolId, opts) {
|
|
387
|
+
const slot = await toolSlots.acquire(opts?.signal);
|
|
388
|
+
// Charge the logical call only after its local execution slot is held, so
|
|
389
|
+
// a failed/aborted slot acquisition never consumes the budget. The
|
|
390
|
+
// physical provider admission is intentionally deferred to
|
|
391
|
+
// acquireProviderPermit at the outbound HTTP edge.
|
|
392
|
+
try {
|
|
393
|
+
await chargeBudget('toolCall');
|
|
394
|
+
} catch (error) {
|
|
395
|
+
slot.release();
|
|
396
|
+
throw error;
|
|
397
|
+
}
|
|
398
|
+
return slot;
|
|
399
|
+
},
|
|
400
|
+
|
|
401
|
+
async acquireProviderPermit(toolId, opts) {
|
|
402
|
+
// The rate ticket is an admission for a physical outbound attempt, not a
|
|
403
|
+
// reservation made while receipt ownership, headers, or local tool slots
|
|
404
|
+
// may still be waiting. This is what keeps real provider arrivals inside
|
|
405
|
+
// the declared rolling window under concurrent runners.
|
|
379
406
|
const pacing = await resolveAdaptivePacing(toolId);
|
|
380
407
|
const rateScope = await resolveScope(toolId, pacing.provider);
|
|
381
|
-
|
|
408
|
+
return await input.rateState.acquire({
|
|
382
409
|
bucketId: rateScope[0],
|
|
383
410
|
rateScopeToken: rateScope[1],
|
|
384
411
|
rules: pacing.rules,
|
|
385
412
|
signal: opts?.signal,
|
|
386
413
|
});
|
|
387
|
-
// 2. Acquire a scarce execution slot only after the provider permit is
|
|
388
|
-
// due. If the slot wait aborts, refund any concurrency-bearing permit.
|
|
389
|
-
let slot: WorkLease;
|
|
390
|
-
try {
|
|
391
|
-
slot = await toolSlots.acquire(opts?.signal);
|
|
392
|
-
} catch (error) {
|
|
393
|
-
permit.release();
|
|
394
|
-
throw error;
|
|
395
|
-
}
|
|
396
|
-
// 3. Charge the budget only once the call is actually cleared to run, so a
|
|
397
|
-
// failed/aborted acquisition never permanently consumes tool budget.
|
|
398
|
-
try {
|
|
399
|
-
await chargeBudget('toolCall');
|
|
400
|
-
} catch (error) {
|
|
401
|
-
permit.release();
|
|
402
|
-
slot.release();
|
|
403
|
-
throw error;
|
|
404
|
-
}
|
|
405
|
-
let released = false;
|
|
406
|
-
return {
|
|
407
|
-
release: () => {
|
|
408
|
-
if (released) return;
|
|
409
|
-
released = true;
|
|
410
|
-
permit.release();
|
|
411
|
-
slot.release();
|
|
412
|
-
},
|
|
413
|
-
};
|
|
414
414
|
},
|
|
415
415
|
|
|
416
416
|
async suggestedParallelism(toolId, fallback) {
|
|
@@ -541,6 +541,8 @@ function createInlineChildGovernor(
|
|
|
541
541
|
policy: root.policy,
|
|
542
542
|
acquireRowSlot: (opts) => root.acquireRowSlot(opts),
|
|
543
543
|
acquireToolSlot: (toolId, opts) => root.acquireToolSlot(toolId, opts),
|
|
544
|
+
acquireProviderPermit: (toolId, opts) =>
|
|
545
|
+
root.acquireProviderPermit(toolId, opts),
|
|
544
546
|
suggestedParallelism: (toolId, fallback) =>
|
|
545
547
|
root.suggestedParallelism(toolId, fallback),
|
|
546
548
|
chargeBudget: (kind, amount) => root.chargeBudget(kind, amount),
|