deepline 0.1.230 → 0.1.232
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/apps/play-runner-workers/src/entry.ts +31 -15
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/tool-dispatch.ts +18 -12
- package/dist/bundling-sources/sdk/src/release.ts +2 -2
- package/dist/bundling-sources/shared_libs/play-runtime/context.ts +18 -3
- package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +8 -0
- package/dist/bundling-sources/shared_libs/play-runtime/governor/app-runtime-rate-state-backend.ts +174 -55
- package/dist/bundling-sources/shared_libs/play-runtime/governor/coordinator-rate-state-backend.ts +13 -9
- package/dist/bundling-sources/shared_libs/play-runtime/governor/governor.ts +66 -27
- package/dist/bundling-sources/shared_libs/play-runtime/governor/rate-state-backend.ts +12 -1
- package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts +12 -0
- package/dist/bundling-sources/shared_libs/play-runtime/resource-governor.ts +4 -3
- package/dist/cli/index.js +82 -4
- package/dist/cli/index.mjs +82 -4
- package/dist/index.js +2 -2
- package/dist/index.mjs +2 -2
- package/package.json +1 -1
|
@@ -1454,7 +1454,7 @@ async function executeTool(
|
|
|
1454
1454
|
providerIdempotencyKey?: string | null;
|
|
1455
1455
|
},
|
|
1456
1456
|
workflowStep?: WorkflowStep,
|
|
1457
|
-
onProviderBackpressure?: (retryAfterMs: number) => void
|
|
1457
|
+
onProviderBackpressure?: (retryAfterMs: number) => void | Promise<void>,
|
|
1458
1458
|
onRetryAttempt?: () => void | Promise<void>,
|
|
1459
1459
|
transientHttpRetrySafe = false,
|
|
1460
1460
|
abortSignal?: AbortSignal,
|
|
@@ -1500,7 +1500,7 @@ async function executeToolWithLifecycle(
|
|
|
1500
1500
|
},
|
|
1501
1501
|
workflowStep: WorkflowStep | undefined,
|
|
1502
1502
|
callbacks: WorkerCtxCallbacks | undefined,
|
|
1503
|
-
onProviderBackpressure?: (retryAfterMs: number) => void
|
|
1503
|
+
onProviderBackpressure?: (retryAfterMs: number) => void | Promise<void>,
|
|
1504
1504
|
onRetryAttempt?: () => void | Promise<void>,
|
|
1505
1505
|
transientHttpRetrySafe = false,
|
|
1506
1506
|
abortSignal?: AbortSignal,
|
|
@@ -1690,7 +1690,7 @@ async function callToolDirect(
|
|
|
1690
1690
|
executionAuthScopeDigest?: string | null;
|
|
1691
1691
|
providerIdempotencyKey?: string | null;
|
|
1692
1692
|
},
|
|
1693
|
-
onProviderBackpressure?: (retryAfterMs: number) => void
|
|
1693
|
+
onProviderBackpressure?: (retryAfterMs: number) => void | Promise<void>,
|
|
1694
1694
|
// Invoked once per in-process retry attempt (429 / retryable 5xx / synthetic
|
|
1695
1695
|
// transient) so the Governor charges chargeBudget('retry') per attempt — the
|
|
1696
1696
|
// same runaway guard the cjs runner applies (context.ts charges retry on each
|
|
@@ -1877,7 +1877,7 @@ async function callToolDirect(
|
|
|
1877
1877
|
if (failure.backpressureDelayMs !== null) {
|
|
1878
1878
|
// Feed the provider's backpressure into the shared pacer even on the
|
|
1879
1879
|
// final attempt so the (org, provider) bucket backs off across isolates.
|
|
1880
|
-
onProviderBackpressure?.(failure.backpressureDelayMs);
|
|
1880
|
+
await onProviderBackpressure?.(failure.backpressureDelayMs);
|
|
1881
1881
|
}
|
|
1882
1882
|
if (!failure.shouldRetry) {
|
|
1883
1883
|
throw lastError;
|
|
@@ -2616,7 +2616,7 @@ function isWorkerFetchResponse(value: unknown): value is WorkerFetchResponse {
|
|
|
2616
2616
|
async function postRuntimeEgressFetch(
|
|
2617
2617
|
req: RunRequest,
|
|
2618
2618
|
payload: RuntimeEgressFetchPayload,
|
|
2619
|
-
onProviderBackpressure?: (retryAfterMs: number) => void
|
|
2619
|
+
onProviderBackpressure?: (retryAfterMs: number) => void | Promise<void>,
|
|
2620
2620
|
onRetryAttempt?: () => void | Promise<void>,
|
|
2621
2621
|
): Promise<WorkerFetchResponse> {
|
|
2622
2622
|
let lastError: Error | null = null;
|
|
@@ -2667,7 +2667,7 @@ async function postRuntimeEgressFetch(
|
|
|
2667
2667
|
Number.isFinite(retryAfterSeconds) && retryAfterSeconds > 0
|
|
2668
2668
|
? Math.ceil(retryAfterSeconds * 1000)
|
|
2669
2669
|
: 1_000;
|
|
2670
|
-
onProviderBackpressure?.(retryAfterMs);
|
|
2670
|
+
await onProviderBackpressure?.(retryAfterMs);
|
|
2671
2671
|
if (attempt >= TOOL_EXECUTE_RATE_LIMIT_MAX_ATTEMPTS) {
|
|
2672
2672
|
throw lastError;
|
|
2673
2673
|
}
|
|
@@ -3776,7 +3776,7 @@ function createWorkerToolAuthScopeDigestResolver(req: RunRequest) {
|
|
|
3776
3776
|
// (RuntimeToolAuthScopeDigestResolver), whose `invalidate` member is
|
|
3777
3777
|
// REQUIRED — the AUTH_SCOPE_CHANGED re-claim wiring below calls it directly,
|
|
3778
3778
|
// and a resolver without eviction support must be a type error.
|
|
3779
|
-
|
|
3779
|
+
const digestResolver = createRuntimeToolAuthScopeDigestResolver({
|
|
3780
3780
|
missingToolIdMessage: 'Tool auth scope needs tool id.',
|
|
3781
3781
|
fetchDigest: async (normalized) => {
|
|
3782
3782
|
const runScopedPath =
|
|
@@ -3807,12 +3807,19 @@ function createWorkerToolAuthScopeDigestResolver(req: RunRequest) {
|
|
|
3807
3807
|
return digest;
|
|
3808
3808
|
},
|
|
3809
3809
|
});
|
|
3810
|
+
return digestResolver;
|
|
3810
3811
|
}
|
|
3811
3812
|
|
|
3812
3813
|
function createWorkerPacingResolver(req: RunRequest): WorkerPacingResolver {
|
|
3813
3814
|
const cache = new Map<
|
|
3814
3815
|
string,
|
|
3815
|
-
Promise<
|
|
3816
|
+
Promise<
|
|
3817
|
+
| (ResolvedPacingPolicy & {
|
|
3818
|
+
retrySafeTransientHttp: boolean;
|
|
3819
|
+
r: { bucketId: string; token: string } | null;
|
|
3820
|
+
})
|
|
3821
|
+
| null
|
|
3822
|
+
>
|
|
3816
3823
|
>();
|
|
3817
3824
|
return (toolId: string) => {
|
|
3818
3825
|
const normalized = String(toolId || '').trim();
|
|
@@ -3840,6 +3847,7 @@ function createWorkerPacingResolver(req: RunRequest): WorkerPacingResolver {
|
|
|
3840
3847
|
provider?: unknown;
|
|
3841
3848
|
queueHints?: unknown;
|
|
3842
3849
|
retry?: unknown;
|
|
3850
|
+
r?: { bucketId: string; token: string };
|
|
3843
3851
|
} | null;
|
|
3844
3852
|
if (!body) return null;
|
|
3845
3853
|
const pacing = pacingPolicyFromUnknownQueueHints(body.queueHints);
|
|
@@ -3857,6 +3865,7 @@ function createWorkerPacingResolver(req: RunRequest): WorkerPacingResolver {
|
|
|
3857
3865
|
: '',
|
|
3858
3866
|
rules: [],
|
|
3859
3867
|
}),
|
|
3868
|
+
r: body.r ?? null,
|
|
3860
3869
|
retrySafeTransientHttp: retry.retrySafeTransientHttp === true,
|
|
3861
3870
|
};
|
|
3862
3871
|
})();
|
|
@@ -3883,11 +3892,13 @@ function resumeGovernanceFromRequest(req: RunRequest): GovernanceSnapshot {
|
|
|
3883
3892
|
};
|
|
3884
3893
|
}
|
|
3885
3894
|
|
|
3886
|
-
function createGovernorForRun(
|
|
3895
|
+
function createGovernorForRun(
|
|
3896
|
+
req: RunRequest,
|
|
3897
|
+
resolvePacing: WorkerPacingResolver,
|
|
3898
|
+
): {
|
|
3887
3899
|
governor: PlayExecutionGovernor;
|
|
3888
3900
|
resolvePacing: WorkerPacingResolver;
|
|
3889
3901
|
} {
|
|
3890
|
-
const resolvePacing = createWorkerPacingResolver(req);
|
|
3891
3902
|
const governor = createPlayExecutionGovernor({
|
|
3892
3903
|
adapter: 'esm_workers',
|
|
3893
3904
|
scope: {
|
|
@@ -3906,6 +3917,8 @@ function createGovernorForRun(req: RunRequest): {
|
|
|
3906
3917
|
}),
|
|
3907
3918
|
),
|
|
3908
3919
|
resolvePacing,
|
|
3920
|
+
resolveRateScope: async (toolId) =>
|
|
3921
|
+
(await resolvePacing(toolId))?.r ?? null,
|
|
3909
3922
|
resume: resumeGovernanceFromRequest(req),
|
|
3910
3923
|
});
|
|
3911
3924
|
return { governor, resolvePacing };
|
|
@@ -3944,12 +3957,15 @@ function createMinimalWorkerCtx(
|
|
|
3944
3957
|
leasedSheetNamespaces?: Set<string>,
|
|
3945
3958
|
receiptSalvage?: ReceiptSalvageBuffer,
|
|
3946
3959
|
): unknown {
|
|
3947
|
-
const
|
|
3948
|
-
|
|
3949
|
-
const resolveToolActionCacheVersion =
|
|
3950
|
-
createWorkerToolActionCacheVersionResolver(req);
|
|
3960
|
+
const resolveToolPacing = createWorkerPacingResolver(req);
|
|
3961
|
+
const { governor } = createGovernorForRun(req, resolveToolPacing);
|
|
3951
3962
|
const resolveToolAuthScopeDigest =
|
|
3952
3963
|
createWorkerToolAuthScopeDigestResolver(req);
|
|
3964
|
+
const resolveToolActionCacheVersion =
|
|
3965
|
+
createWorkerToolActionCacheVersionResolver(req);
|
|
3966
|
+
// Play-call depth/count/per-parent budgets, child-play concurrency, and the
|
|
3967
|
+
// lineage snapshot are owned by the Governor (createGovernorForRun above).
|
|
3968
|
+
// The worker keeps only substrate mechanism here.
|
|
3953
3969
|
const stepCallCounts: Record<string, number> = {};
|
|
3954
3970
|
const secretRedactor = createSecretRedactionContext();
|
|
3955
3971
|
|
|
@@ -4004,7 +4020,7 @@ function createMinimalWorkerCtx(
|
|
|
4004
4020
|
input: Record<string, unknown>;
|
|
4005
4021
|
workflowStep?: unknown;
|
|
4006
4022
|
callbacks?: WorkerCtxCallbacks;
|
|
4007
|
-
onProviderBackpressure?: (retryAfterMs: number) => void
|
|
4023
|
+
onProviderBackpressure?: (retryAfterMs: number) => void | Promise<void>;
|
|
4008
4024
|
onRetryAttempt?: () => void | Promise<void>;
|
|
4009
4025
|
transientHttpRetrySafe?: boolean;
|
|
4010
4026
|
durableCallReceiptKey?: string | null;
|
|
@@ -73,7 +73,11 @@ import type { WorkerWorkBudgetMeter } from './work-budget';
|
|
|
73
73
|
export type WorkerPacingResolver = (
|
|
74
74
|
toolId: string,
|
|
75
75
|
) => Promise<
|
|
76
|
-
(ResolvedPacingPolicy & {
|
|
76
|
+
| (ResolvedPacingPolicy & {
|
|
77
|
+
retrySafeTransientHttp: boolean;
|
|
78
|
+
r: { bucketId: string; token: string } | null;
|
|
79
|
+
})
|
|
80
|
+
| null
|
|
77
81
|
>;
|
|
78
82
|
|
|
79
83
|
export type WorkerToolActionCacheVersionResolver = (
|
|
@@ -111,7 +115,7 @@ export type WorkerToolDispatchExecuteTool = (input: {
|
|
|
111
115
|
input: Record<string, unknown>;
|
|
112
116
|
workflowStep?: WorkflowStepLike;
|
|
113
117
|
callbacks?: WorkerToolDispatchCallbacks;
|
|
114
|
-
onProviderBackpressure?: (retryAfterMs: number) => void
|
|
118
|
+
onProviderBackpressure?: (retryAfterMs: number) => void | Promise<void>;
|
|
115
119
|
onRetryAttempt?: () => void | Promise<void>;
|
|
116
120
|
transientHttpRetrySafe?: boolean;
|
|
117
121
|
durableCallReceiptKey?: string | null;
|
|
@@ -561,17 +565,19 @@ export class WorkerToolBatchScheduler {
|
|
|
561
565
|
* isolates. Provider comes from the same pacing resolver the Governor uses
|
|
562
566
|
* (the worker has no local catalog), so callers pass only the toolId.
|
|
563
567
|
*/
|
|
564
|
-
private reportBackpressure(
|
|
568
|
+
private async reportBackpressure(
|
|
569
|
+
toolId: string,
|
|
570
|
+
retryAfterMs: number,
|
|
571
|
+
): Promise<void> {
|
|
565
572
|
if (retryAfterMs <= 0) return;
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
})();
|
|
573
|
+
const pacing = await this.options.resolvePacing(toolId).catch(() => null);
|
|
574
|
+
if (pacing?.provider) {
|
|
575
|
+
await this.options.governor.reportProviderBackpressure({
|
|
576
|
+
provider: pacing.provider,
|
|
577
|
+
toolId,
|
|
578
|
+
retryAfterMs,
|
|
579
|
+
});
|
|
580
|
+
}
|
|
575
581
|
}
|
|
576
582
|
|
|
577
583
|
async execute(
|
|
@@ -108,10 +108,10 @@ export const SDK_RELEASE = {
|
|
|
108
108
|
// 0.1.175 ships loud `deepline enrich` empty-waterfall reporting.
|
|
109
109
|
// 0.1.220 deprecates SDK CLI versions below 0.1.219 and updates them
|
|
110
110
|
// automatically without blocking their current command.
|
|
111
|
-
version: '0.1.
|
|
111
|
+
version: '0.1.232',
|
|
112
112
|
apiContract: '2026-06-dataset-handle-results-hard-cutover',
|
|
113
113
|
supportPolicy: {
|
|
114
|
-
latest: '0.1.
|
|
114
|
+
latest: '0.1.232',
|
|
115
115
|
minimumSupported: '0.1.53',
|
|
116
116
|
deprecatedBelow: '0.1.219',
|
|
117
117
|
commandMinimumSupported: [
|
|
@@ -57,10 +57,12 @@ export {
|
|
|
57
57
|
import {
|
|
58
58
|
createDefaultGovernanceSnapshot,
|
|
59
59
|
createPlayExecutionGovernor,
|
|
60
|
+
defaultPacingForTool,
|
|
60
61
|
type GovernanceSnapshot,
|
|
61
62
|
type PacingResolver,
|
|
62
63
|
type PlayExecutionGovernor,
|
|
63
64
|
} from './governor/governor';
|
|
65
|
+
import { resolveExecutionPolicy } from './governor/policy';
|
|
64
66
|
import {
|
|
65
67
|
createRuntimeResourceGovernor,
|
|
66
68
|
type RuntimeResourceGovernor,
|
|
@@ -1142,12 +1144,20 @@ function cloneMapFrame(frame: MapExecutionFrame): MapExecutionFrame {
|
|
|
1142
1144
|
*/
|
|
1143
1145
|
function createPacingResolver(
|
|
1144
1146
|
getToolQueueHints: ContextOptions['getToolQueueHints'],
|
|
1147
|
+
getToolProvider: ContextOptions['getToolProvider'],
|
|
1145
1148
|
): PacingResolver {
|
|
1146
1149
|
return async (toolId: string) => {
|
|
1147
1150
|
const builtin = pacingPolicyForTool(toolId, []);
|
|
1148
1151
|
if (builtin) return builtin;
|
|
1149
1152
|
const hints = getToolQueueHints ? await getToolQueueHints(toolId) : [];
|
|
1150
|
-
|
|
1153
|
+
const declared = pacingPolicyForTool(toolId, hints);
|
|
1154
|
+
if (declared) return declared;
|
|
1155
|
+
const provider = (await getToolProvider?.(toolId))?.trim();
|
|
1156
|
+
if (!provider) return null;
|
|
1157
|
+
return {
|
|
1158
|
+
...defaultPacingForTool(toolId, resolveExecutionPolicy('cjs_node20')),
|
|
1159
|
+
provider,
|
|
1160
|
+
};
|
|
1151
1161
|
};
|
|
1152
1162
|
}
|
|
1153
1163
|
|
|
@@ -1375,7 +1385,11 @@ export class PlayContextImpl {
|
|
|
1375
1385
|
},
|
|
1376
1386
|
rateState: options.rateState ?? new InMemoryRateStateBackend(),
|
|
1377
1387
|
budgetState: options.budgetState,
|
|
1378
|
-
resolvePacing: createPacingResolver(
|
|
1388
|
+
resolvePacing: createPacingResolver(
|
|
1389
|
+
options.getToolQueueHints,
|
|
1390
|
+
options.getToolProvider,
|
|
1391
|
+
),
|
|
1392
|
+
resolveRateScope: options.getToolRateScope,
|
|
1379
1393
|
// A root run keeps depth 0 (its first child play is depth 1) but still
|
|
1380
1394
|
// seeds its own play id into the ancestry/currentPlayId so the cycle guard
|
|
1381
1395
|
// catches a child re-invoking the root. createDefaultGovernanceSnapshot
|
|
@@ -8533,8 +8547,9 @@ export class PlayContextImpl {
|
|
|
8533
8547
|
? await this.#options.getToolQueueHints(toolId)
|
|
8534
8548
|
: [];
|
|
8535
8549
|
const provider = hints[0]?.provider?.trim() || `tool:${toolId}`;
|
|
8536
|
-
this.resourceGovernor.reportProviderBackpressure({
|
|
8550
|
+
await this.resourceGovernor.reportProviderBackpressure({
|
|
8537
8551
|
provider,
|
|
8552
|
+
toolId,
|
|
8538
8553
|
retryAfterMs,
|
|
8539
8554
|
});
|
|
8540
8555
|
}
|
|
@@ -641,12 +641,20 @@ export interface ContextOptions {
|
|
|
641
641
|
runtimeSheetBackedMapDatasets?: boolean;
|
|
642
642
|
resolvePlay?: (playRef: string) => Promise<ResolvedPlayExecution | null>;
|
|
643
643
|
getToolQueueHints?: (toolId: string) => Promise<readonly PlayQueueHint[]>;
|
|
644
|
+
getToolProvider?: (toolId: string) => Promise<string | null>;
|
|
644
645
|
getToolRetryPolicy?: (toolId: string) => Promise<{
|
|
645
646
|
retrySafeTransientHttp?: boolean;
|
|
646
647
|
requiresExecutionFence?: boolean;
|
|
647
648
|
} | null>;
|
|
648
649
|
getToolActionCacheVersion?: (toolId: string) => Promise<string> | string;
|
|
649
650
|
getToolAuthScopeDigest?: (toolId: string) => Promise<string> | string;
|
|
651
|
+
getToolRateScope?: (
|
|
652
|
+
toolId: string,
|
|
653
|
+
provider: string,
|
|
654
|
+
) =>
|
|
655
|
+
| Promise<{ bucketId: string; token: string } | null>
|
|
656
|
+
| { bucketId: string; token: string }
|
|
657
|
+
| null;
|
|
650
658
|
invalidateToolAuthScopeDigest?: (toolId: string) => void;
|
|
651
659
|
getToolTargetGetters?: (
|
|
652
660
|
toolId: string,
|