deepline 0.1.243 → 0.1.244
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/bundling-sources/sdk/src/plays/bundle-play-file.ts +5 -25
- package/dist/bundling-sources/sdk/src/release.ts +23 -21
- package/dist/bundling-sources/sdk/src/types.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/backend.ts +2 -21
- package/dist/bundling-sources/shared_libs/play-runtime/governor/governor.ts +5 -6
- package/dist/bundling-sources/shared_libs/play-runtime/governor/policy.ts +6 -13
- package/dist/bundling-sources/shared_libs/play-runtime/governor/rate-state-backend.ts +9 -26
- package/dist/bundling-sources/shared_libs/play-runtime/internal-step-ids.ts +1 -2
- package/dist/bundling-sources/shared_libs/play-runtime/map-row-identity.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/output-size-limits.ts +191 -29
- package/dist/bundling-sources/shared_libs/play-runtime/play-latency-trace.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/providers.ts +1 -36
- package/dist/bundling-sources/shared_libs/play-runtime/run-ledger.ts +2 -3
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/index.ts +0 -7
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +1 -12
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-pg-driver-pg.ts +2 -3
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-pg-driver.ts +5 -15
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backend.ts +6 -21
- package/dist/bundling-sources/shared_libs/play-runtime/tool-result.ts +3 -4
- package/dist/bundling-sources/shared_libs/plays/artifact-kind-guard.ts +16 -29
- package/dist/bundling-sources/shared_libs/plays/artifact-types.ts +4 -8
- package/dist/bundling-sources/shared_libs/plays/bundling/index.ts +21 -712
- package/dist/bundling-sources/shared_libs/plays/bundling/limits.ts +1 -18
- package/dist/cli/index.js +29 -71
- package/dist/cli/index.mjs +29 -71
- package/dist/{compiler-manifest-CYcwzSOJ.d.ts → compiler-manifest-DlE7dnRm.d.mts} +2 -8
- package/dist/{compiler-manifest-CYcwzSOJ.d.mts → compiler-manifest-DlE7dnRm.d.ts} +2 -8
- package/dist/index.d.mts +2 -2
- package/dist/index.d.ts +2 -2
- package/dist/index.js +24 -22
- package/dist/index.mjs +24 -22
- package/dist/plays/bundle-play-file.d.mts +7 -22
- package/dist/plays/bundle-play-file.d.ts +7 -22
- package/dist/plays/bundle-play-file.mjs +26 -614
- package/package.json +2 -2
- package/dist/bundling-sources/apps/play-runner-workers/src/coordinator-entry.ts +0 -4504
- package/dist/bundling-sources/apps/play-runner-workers/src/dedup-do.ts +0 -1386
- package/dist/bundling-sources/apps/play-runner-workers/src/durable-object-deploy-handoff.ts +0 -30
- package/dist/bundling-sources/apps/play-runner-workers/src/entry.ts +0 -8385
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/README.md +0 -21
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/batching.ts +0 -251
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/coordinator-progress.ts +0 -289
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/csv-rows.ts +0 -85
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/customer-console.ts +0 -23
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/dataset-handles.ts +0 -528
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/harness-receipt-store.ts +0 -229
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/ledger-event-batches.ts +0 -81
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/live-progress.ts +0 -22
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/map-chunk-plan.ts +0 -920
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/map-latency-profile.ts +0 -90
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/output-datasets.ts +0 -475
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/receipts.ts +0 -570
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/result-dataset-persistence.ts +0 -50
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/row-isolation.ts +0 -5
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/run-work-dispatcher.ts +0 -519
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/tool-batch.ts +0 -107
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/tool-dispatch.ts +0 -2541
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/tool-http-errors.ts +0 -7
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/tool-receipts.ts +0 -227
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/work-budget.ts +0 -130
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/worker-platform-budget.ts +0 -39
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/workflow-preview.ts +0 -26
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime-api-retry.ts +0 -23
- package/dist/bundling-sources/apps/play-runner-workers/src/workflow-instance-create.ts +0 -47
- package/dist/bundling-sources/apps/play-runner-workers/src/workflow-retry-state.ts +0 -167
- package/dist/bundling-sources/apps/play-runner-workers/src/workflow-retry.ts +0 -91
- package/dist/bundling-sources/sdk/src/plays/harness-stub.ts +0 -329
- package/dist/bundling-sources/sdk/src/worker-play-entry.ts +0 -240
- package/dist/bundling-sources/shared_libs/play-runtime/governor/coordinator-rate-state-backend.ts +0 -252
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-pg-driver-neon-serverless.ts +0 -217
|
@@ -1,227 +0,0 @@
|
|
|
1
|
-
import {
|
|
2
|
-
markToolExecuteResultExecutionOutcome,
|
|
3
|
-
type ToolExecutionOutcome,
|
|
4
|
-
} from '../../../../shared_libs/play-runtime/tool-execution-outcome';
|
|
5
|
-
|
|
6
|
-
export type WorkerToolReceiptSelection = {
|
|
7
|
-
claimableReceiptKey: string | null;
|
|
8
|
-
forceDurableRefresh: boolean;
|
|
9
|
-
forceFailedDurableRefresh: boolean;
|
|
10
|
-
};
|
|
11
|
-
|
|
12
|
-
export type WorkerToolReceiptGroup<TRequest> = {
|
|
13
|
-
claimableReceiptKey: string;
|
|
14
|
-
forceDurableRefresh: boolean;
|
|
15
|
-
forceFailedDurableRefresh: boolean;
|
|
16
|
-
requests: TRequest[];
|
|
17
|
-
};
|
|
18
|
-
|
|
19
|
-
export type WorkerToolReceiptGroupPlan<TRequest> = {
|
|
20
|
-
localRequests: TRequest[];
|
|
21
|
-
durableGroups: WorkerToolReceiptGroup<TRequest>[];
|
|
22
|
-
};
|
|
23
|
-
|
|
24
|
-
export function canReclaimTimedOutWorkerToolReceipt(input: {
|
|
25
|
-
ownerRunId?: string | null;
|
|
26
|
-
currentRunId: string;
|
|
27
|
-
updatedAt?: string | null;
|
|
28
|
-
nowMs?: number;
|
|
29
|
-
}): boolean {
|
|
30
|
-
const currentRunId = input.currentRunId.trim();
|
|
31
|
-
return Boolean(currentRunId);
|
|
32
|
-
}
|
|
33
|
-
|
|
34
|
-
/**
|
|
35
|
-
* Staleness floor for the dead-owner reclaim fast path. A `running` receipt
|
|
36
|
-
* left by a different run whose last write is older than this is treated as an
|
|
37
|
-
* orphan from a dead run — no live worker will ever complete it. Running
|
|
38
|
-
* receipts carry no heartbeat (updatedAt is frozen at claim time), so this is a
|
|
39
|
-
* conservative constant, not a heartbeat multiple.
|
|
40
|
-
*/
|
|
41
|
-
export const RUNNING_RECEIPT_DEAD_OWNER_STALENESS_MS = 60_000;
|
|
42
|
-
export const RUNNING_RECEIPT_POLL_INTERVAL_MS = 250;
|
|
43
|
-
|
|
44
|
-
/** Bound a follower poll by when a different-run owner becomes reclaimable. */
|
|
45
|
-
export function resolveRunningReceiptWaitMaxAttempts(input: {
|
|
46
|
-
ownerRunId?: string | null;
|
|
47
|
-
currentRunId: string;
|
|
48
|
-
updatedAt?: string | null;
|
|
49
|
-
requestedMaxAttempts: number;
|
|
50
|
-
nowMs?: number;
|
|
51
|
-
stalenessFloorMs?: number;
|
|
52
|
-
pollIntervalMs?: number;
|
|
53
|
-
}): number {
|
|
54
|
-
const requested = Math.max(1, Math.floor(input.requestedMaxAttempts));
|
|
55
|
-
const ownerRunId = input.ownerRunId?.trim() ?? '';
|
|
56
|
-
const currentRunId = input.currentRunId.trim();
|
|
57
|
-
if (!ownerRunId || !currentRunId || ownerRunId === currentRunId) {
|
|
58
|
-
return requested;
|
|
59
|
-
}
|
|
60
|
-
const updatedAtMs = input.updatedAt ? Date.parse(input.updatedAt) : NaN;
|
|
61
|
-
if (!Number.isFinite(updatedAtMs)) return 1;
|
|
62
|
-
const remainingMs = Math.max(
|
|
63
|
-
0,
|
|
64
|
-
updatedAtMs +
|
|
65
|
-
(input.stalenessFloorMs ?? RUNNING_RECEIPT_DEAD_OWNER_STALENESS_MS) -
|
|
66
|
-
(input.nowMs ?? Date.now()),
|
|
67
|
-
);
|
|
68
|
-
const attemptsUntilReclaim = Math.max(
|
|
69
|
-
1,
|
|
70
|
-
Math.floor(
|
|
71
|
-
remainingMs / (input.pollIntervalMs ?? RUNNING_RECEIPT_POLL_INTERVAL_MS),
|
|
72
|
-
) + 1,
|
|
73
|
-
);
|
|
74
|
-
return Math.min(requested, attemptsUntilReclaim);
|
|
75
|
-
}
|
|
76
|
-
|
|
77
|
-
export function canReclaimFailedWorkerToolReceipt(input: {
|
|
78
|
-
error?: string | null;
|
|
79
|
-
}): boolean {
|
|
80
|
-
const error = input.error?.trim();
|
|
81
|
-
return Boolean(
|
|
82
|
-
error &&
|
|
83
|
-
(/\b(cancell?ed|aborted|terminate[d]?)\b/i.test(error) ||
|
|
84
|
-
/\bmax runtime\b/i.test(error) ||
|
|
85
|
-
/persistence failure|durable receipts? could not be marked/i.test(error)),
|
|
86
|
-
);
|
|
87
|
-
}
|
|
88
|
-
|
|
89
|
-
export function selectWorkerToolReceipt(input: {
|
|
90
|
-
durableReceiptKey?: string | null;
|
|
91
|
-
localRetryCacheKey: string;
|
|
92
|
-
force: boolean;
|
|
93
|
-
forceFailed?: boolean;
|
|
94
|
-
allowLocalRetryReceipts: boolean;
|
|
95
|
-
}): WorkerToolReceiptSelection {
|
|
96
|
-
const durableReceiptKey = input.durableReceiptKey?.trim() || null;
|
|
97
|
-
const localRetryReceiptKey =
|
|
98
|
-
durableReceiptKey || !input.allowLocalRetryReceipts
|
|
99
|
-
? null
|
|
100
|
-
: input.localRetryCacheKey.trim() || null;
|
|
101
|
-
return {
|
|
102
|
-
claimableReceiptKey: durableReceiptKey ?? localRetryReceiptKey,
|
|
103
|
-
forceDurableRefresh: input.force === true && durableReceiptKey !== null,
|
|
104
|
-
forceFailedDurableRefresh:
|
|
105
|
-
input.force !== true &&
|
|
106
|
-
input.forceFailed === true &&
|
|
107
|
-
durableReceiptKey !== null,
|
|
108
|
-
};
|
|
109
|
-
}
|
|
110
|
-
|
|
111
|
-
export function planWorkerToolReceiptGroups<TRequest>(
|
|
112
|
-
requests: TRequest[],
|
|
113
|
-
input: {
|
|
114
|
-
allowLocalRetryReceipts: boolean;
|
|
115
|
-
getReceiptInput: (request: TRequest) => {
|
|
116
|
-
durableReceiptKey?: string | null;
|
|
117
|
-
localRetryCacheKey: string;
|
|
118
|
-
force: boolean;
|
|
119
|
-
forceFailed?: boolean;
|
|
120
|
-
};
|
|
121
|
-
},
|
|
122
|
-
): WorkerToolReceiptGroupPlan<TRequest> {
|
|
123
|
-
const localRequests: TRequest[] = [];
|
|
124
|
-
const groupsByKey = new Map<string, WorkerToolReceiptGroup<TRequest>>();
|
|
125
|
-
for (const request of requests) {
|
|
126
|
-
const {
|
|
127
|
-
claimableReceiptKey,
|
|
128
|
-
forceDurableRefresh,
|
|
129
|
-
forceFailedDurableRefresh,
|
|
130
|
-
} = selectWorkerToolReceipt({
|
|
131
|
-
...input.getReceiptInput(request),
|
|
132
|
-
allowLocalRetryReceipts: input.allowLocalRetryReceipts,
|
|
133
|
-
});
|
|
134
|
-
if (!claimableReceiptKey) {
|
|
135
|
-
localRequests.push(request);
|
|
136
|
-
continue;
|
|
137
|
-
}
|
|
138
|
-
const group = groupsByKey.get(claimableReceiptKey) ?? {
|
|
139
|
-
claimableReceiptKey,
|
|
140
|
-
forceDurableRefresh: false,
|
|
141
|
-
forceFailedDurableRefresh: false,
|
|
142
|
-
requests: [],
|
|
143
|
-
};
|
|
144
|
-
group.requests.push(request);
|
|
145
|
-
group.forceDurableRefresh ||= forceDurableRefresh;
|
|
146
|
-
group.forceFailedDurableRefresh ||= forceFailedDurableRefresh;
|
|
147
|
-
groupsByKey.set(claimableReceiptKey, group);
|
|
148
|
-
}
|
|
149
|
-
return {
|
|
150
|
-
localRequests,
|
|
151
|
-
durableGroups: [...groupsByKey.values()],
|
|
152
|
-
};
|
|
153
|
-
}
|
|
154
|
-
|
|
155
|
-
export function resolveWorkerToolReceiptGroupWaitMaxAttempts<TRequest>(
|
|
156
|
-
requests: TRequest[],
|
|
157
|
-
getWaitMaxAttempts: (request: TRequest) => number,
|
|
158
|
-
): number {
|
|
159
|
-
return requests.reduce((maxAttempts, request) => {
|
|
160
|
-
const attempts = getWaitMaxAttempts(request);
|
|
161
|
-
return Number.isFinite(attempts) && attempts > maxAttempts
|
|
162
|
-
? attempts
|
|
163
|
-
: maxAttempts;
|
|
164
|
-
}, 1);
|
|
165
|
-
}
|
|
166
|
-
|
|
167
|
-
export type WorkerToolRuntimeTimeoutRequest = {
|
|
168
|
-
input: Record<string, unknown>;
|
|
169
|
-
runtimeTimeoutMs?: number;
|
|
170
|
-
};
|
|
171
|
-
|
|
172
|
-
export type WorkerToolRuntimeTimeoutClaim = {
|
|
173
|
-
request: WorkerToolRuntimeTimeoutRequest;
|
|
174
|
-
followers: WorkerToolRuntimeTimeoutRequest[];
|
|
175
|
-
};
|
|
176
|
-
|
|
177
|
-
export function resolveWorkerToolRuntimeTimeoutMs(
|
|
178
|
-
claimedRequests: WorkerToolRuntimeTimeoutClaim[],
|
|
179
|
-
input: {
|
|
180
|
-
resolveOwnerTimeoutMs: (
|
|
181
|
-
request: WorkerToolRuntimeTimeoutRequest,
|
|
182
|
-
) => number | undefined;
|
|
183
|
-
},
|
|
184
|
-
): number | undefined {
|
|
185
|
-
let timeoutMs = 0;
|
|
186
|
-
for (const claimed of claimedRequests) {
|
|
187
|
-
const ownerTimeoutMs = input.resolveOwnerTimeoutMs(claimed.request);
|
|
188
|
-
if (
|
|
189
|
-
typeof ownerTimeoutMs === 'number' &&
|
|
190
|
-
Number.isFinite(ownerTimeoutMs) &&
|
|
191
|
-
ownerTimeoutMs > timeoutMs
|
|
192
|
-
) {
|
|
193
|
-
timeoutMs = ownerTimeoutMs;
|
|
194
|
-
}
|
|
195
|
-
for (const follower of claimed.followers) {
|
|
196
|
-
const followerTimeoutMs = follower.runtimeTimeoutMs;
|
|
197
|
-
if (
|
|
198
|
-
typeof followerTimeoutMs === 'number' &&
|
|
199
|
-
Number.isFinite(followerTimeoutMs) &&
|
|
200
|
-
followerTimeoutMs > timeoutMs
|
|
201
|
-
) {
|
|
202
|
-
timeoutMs = followerTimeoutMs;
|
|
203
|
-
}
|
|
204
|
-
}
|
|
205
|
-
}
|
|
206
|
-
return timeoutMs > 0 ? timeoutMs : undefined;
|
|
207
|
-
}
|
|
208
|
-
|
|
209
|
-
export function markWorkerToolReceiptResultCached(
|
|
210
|
-
value: unknown,
|
|
211
|
-
cacheKey: string,
|
|
212
|
-
receiptKey = cacheKey,
|
|
213
|
-
): unknown {
|
|
214
|
-
return markWorkerToolReceiptResultExecution(value, {
|
|
215
|
-
kind: 'cache',
|
|
216
|
-
cacheKey,
|
|
217
|
-
receiptKey,
|
|
218
|
-
attachedToReceiptKey: receiptKey,
|
|
219
|
-
});
|
|
220
|
-
}
|
|
221
|
-
|
|
222
|
-
export function markWorkerToolReceiptResultExecution(
|
|
223
|
-
value: unknown,
|
|
224
|
-
outcome: ToolExecutionOutcome,
|
|
225
|
-
): unknown {
|
|
226
|
-
return markToolExecuteResultExecutionOutcome(value, outcome);
|
|
227
|
-
}
|
|
@@ -1,130 +0,0 @@
|
|
|
1
|
-
import { WORKER_PLATFORM_YIELD_LIMITS } from './worker-platform-budget';
|
|
2
|
-
|
|
3
|
-
export type WorkerWorkBudgetEventKind =
|
|
4
|
-
| 'tool'
|
|
5
|
-
| 'provider'
|
|
6
|
-
| 'receipt'
|
|
7
|
-
| 'heartbeat'
|
|
8
|
-
| 'sheet'
|
|
9
|
-
| 'log'
|
|
10
|
-
| 'egress';
|
|
11
|
-
|
|
12
|
-
export type WorkerWorkBudgetSnapshot = {
|
|
13
|
-
startedAt: number;
|
|
14
|
-
updatedAt: number;
|
|
15
|
-
elapsedMs: number;
|
|
16
|
-
counts: Record<WorkerWorkBudgetEventKind, number>;
|
|
17
|
-
elapsedMsByKind: Record<WorkerWorkBudgetEventKind, number>;
|
|
18
|
-
};
|
|
19
|
-
|
|
20
|
-
export type WorkerWorkBudgetYieldReason =
|
|
21
|
-
| 'elapsed'
|
|
22
|
-
| 'subrequest'
|
|
23
|
-
| 'provider'
|
|
24
|
-
| 'tool'
|
|
25
|
-
| 'receipt'
|
|
26
|
-
| 'heartbeat'
|
|
27
|
-
| 'sheet'
|
|
28
|
-
| 'log'
|
|
29
|
-
| 'egress';
|
|
30
|
-
|
|
31
|
-
export type WorkerWorkBudgetYieldDecision = {
|
|
32
|
-
shouldYield: boolean;
|
|
33
|
-
reason: WorkerWorkBudgetYieldReason | null;
|
|
34
|
-
snapshot: WorkerWorkBudgetSnapshot;
|
|
35
|
-
};
|
|
36
|
-
|
|
37
|
-
export type WorkerWorkBudgetYieldLimits = Partial<
|
|
38
|
-
Record<WorkerWorkBudgetYieldReason, number>
|
|
39
|
-
>;
|
|
40
|
-
|
|
41
|
-
export type WorkerWorkBudgetMeter = {
|
|
42
|
-
count(kind: WorkerWorkBudgetEventKind, amount?: number): void;
|
|
43
|
-
snapshot(): WorkerWorkBudgetSnapshot;
|
|
44
|
-
resetWindow(): void;
|
|
45
|
-
};
|
|
46
|
-
|
|
47
|
-
const WORK_BUDGET_EVENT_KINDS: WorkerWorkBudgetEventKind[] = [
|
|
48
|
-
'tool',
|
|
49
|
-
'provider',
|
|
50
|
-
'receipt',
|
|
51
|
-
'heartbeat',
|
|
52
|
-
'sheet',
|
|
53
|
-
'log',
|
|
54
|
-
'egress',
|
|
55
|
-
];
|
|
56
|
-
|
|
57
|
-
function zeroCounters(): Record<WorkerWorkBudgetEventKind, number> {
|
|
58
|
-
return Object.fromEntries(
|
|
59
|
-
WORK_BUDGET_EVENT_KINDS.map((kind) => [kind, 0]),
|
|
60
|
-
) as Record<WorkerWorkBudgetEventKind, number>;
|
|
61
|
-
}
|
|
62
|
-
|
|
63
|
-
export function createWorkerWorkBudgetMeter(input?: {
|
|
64
|
-
nowMs?: () => number;
|
|
65
|
-
}): WorkerWorkBudgetMeter {
|
|
66
|
-
const nowMs = input?.nowMs ?? Date.now;
|
|
67
|
-
let startedAt = nowMs();
|
|
68
|
-
let updatedAt = startedAt;
|
|
69
|
-
let counts = zeroCounters();
|
|
70
|
-
let elapsedMsByKind = zeroCounters();
|
|
71
|
-
|
|
72
|
-
return {
|
|
73
|
-
count(kind, amount = 1) {
|
|
74
|
-
counts[kind] += amount;
|
|
75
|
-
updatedAt = nowMs();
|
|
76
|
-
},
|
|
77
|
-
snapshot() {
|
|
78
|
-
const now = nowMs();
|
|
79
|
-
return {
|
|
80
|
-
startedAt,
|
|
81
|
-
updatedAt,
|
|
82
|
-
elapsedMs: Math.max(0, now - startedAt),
|
|
83
|
-
counts: { ...counts },
|
|
84
|
-
elapsedMsByKind: { ...elapsedMsByKind },
|
|
85
|
-
};
|
|
86
|
-
},
|
|
87
|
-
resetWindow() {
|
|
88
|
-
startedAt = nowMs();
|
|
89
|
-
updatedAt = startedAt;
|
|
90
|
-
counts = zeroCounters();
|
|
91
|
-
elapsedMsByKind = zeroCounters();
|
|
92
|
-
},
|
|
93
|
-
};
|
|
94
|
-
}
|
|
95
|
-
|
|
96
|
-
export const WORKER_WORK_BUDGET_YIELD_LIMITS = WORKER_PLATFORM_YIELD_LIMITS;
|
|
97
|
-
|
|
98
|
-
export function shouldYieldWorkerWorkBudget(input: {
|
|
99
|
-
meter: WorkerWorkBudgetMeter;
|
|
100
|
-
limits?: WorkerWorkBudgetYieldLimits;
|
|
101
|
-
projectedSubrequests?: number;
|
|
102
|
-
}): WorkerWorkBudgetYieldDecision {
|
|
103
|
-
const snapshot = input.meter.snapshot();
|
|
104
|
-
const limits = {
|
|
105
|
-
...WORKER_WORK_BUDGET_YIELD_LIMITS,
|
|
106
|
-
...(input.limits ?? {}),
|
|
107
|
-
};
|
|
108
|
-
if (snapshot.elapsedMs >= limits.elapsed) {
|
|
109
|
-
return { shouldYield: true, reason: 'elapsed', snapshot };
|
|
110
|
-
}
|
|
111
|
-
const subrequests =
|
|
112
|
-
snapshot.counts.provider +
|
|
113
|
-
snapshot.counts.receipt +
|
|
114
|
-
snapshot.counts.heartbeat +
|
|
115
|
-
snapshot.counts.sheet +
|
|
116
|
-
snapshot.counts.egress;
|
|
117
|
-
const projectedSubrequests = Math.max(
|
|
118
|
-
0,
|
|
119
|
-
Math.floor(input.projectedSubrequests ?? 0),
|
|
120
|
-
);
|
|
121
|
-
if (subrequests + projectedSubrequests >= limits.subrequest) {
|
|
122
|
-
return { shouldYield: true, reason: 'subrequest', snapshot };
|
|
123
|
-
}
|
|
124
|
-
for (const kind of WORK_BUDGET_EVENT_KINDS) {
|
|
125
|
-
if (snapshot.counts[kind] >= limits[kind]) {
|
|
126
|
-
return { shouldYield: true, reason: kind, snapshot };
|
|
127
|
-
}
|
|
128
|
-
}
|
|
129
|
-
return { shouldYield: false, reason: null, snapshot };
|
|
130
|
-
}
|
package/dist/bundling-sources/apps/play-runner-workers/src/runtime/worker-platform-budget.ts
DELETED
|
@@ -1,39 +0,0 @@
|
|
|
1
|
-
import type { WorkerWorkBudgetYieldLimits } from './work-budget';
|
|
2
|
-
|
|
3
|
-
// Preview accounts can still enforce the free/internal-service subrequest
|
|
4
|
-
// ceiling even when wrangler config asks for more. This is a planner budget,
|
|
5
|
-
// not the advertised platform ceiling: unbatched tool chunks also spend
|
|
6
|
-
// subrequests on sheet writes, progress/ledger flushes, service-binding hops,
|
|
7
|
-
// and workflow replay around the visible provider calls. Individual log lines
|
|
8
|
-
// are buffered and batched into those flushes; they are not one subrequest each.
|
|
9
|
-
// Keep real headroom here;
|
|
10
|
-
// paid deployments may raise it only once the active account limit is proven by
|
|
11
|
-
// an E2E run, not by config alone.
|
|
12
|
-
export const WORKER_PLATFORM_SUBREQUEST_SOFT_LIMIT = 700;
|
|
13
|
-
|
|
14
|
-
// Fresh unbatched tool calls spend more than the visible provider RPC:
|
|
15
|
-
// durable receipt claim/complete, provider execution, route/service-binding
|
|
16
|
-
// hops, and row progress/sheet writes all share the Worker subrequest pool.
|
|
17
|
-
export const WORKER_PLATFORM_SUBREQUESTS_PER_UNBATCHED_TOOL_CALL = 8;
|
|
18
|
-
|
|
19
|
-
export const WORKER_PLATFORM_YIELD_LIMITS: Required<WorkerWorkBudgetYieldLimits> =
|
|
20
|
-
{
|
|
21
|
-
elapsed: 45_000,
|
|
22
|
-
subrequest: WORKER_PLATFORM_SUBREQUEST_SOFT_LIMIT,
|
|
23
|
-
provider: WORKER_PLATFORM_SUBREQUEST_SOFT_LIMIT,
|
|
24
|
-
tool: 1_500,
|
|
25
|
-
receipt: 2_000,
|
|
26
|
-
heartbeat: 1_000,
|
|
27
|
-
sheet: 800,
|
|
28
|
-
log: 1_500,
|
|
29
|
-
egress: 1_000,
|
|
30
|
-
};
|
|
31
|
-
|
|
32
|
-
export function estimateUnbatchedToolCallSubrequests(input: {
|
|
33
|
-
toolCalls: number;
|
|
34
|
-
}): number {
|
|
35
|
-
return (
|
|
36
|
-
Math.max(0, Math.floor(input.toolCalls)) *
|
|
37
|
-
WORKER_PLATFORM_SUBREQUESTS_PER_UNBATCHED_TOOL_CALL
|
|
38
|
-
);
|
|
39
|
-
}
|
|
@@ -1,26 +0,0 @@
|
|
|
1
|
-
export const WORKFLOW_MAP_CHUNK_PREVIEW_MAX_BYTES = 64 * 1024;
|
|
2
|
-
|
|
3
|
-
const jsonByteLength = (value: unknown): number =>
|
|
4
|
-
new TextEncoder().encode(JSON.stringify(value)).byteLength;
|
|
5
|
-
|
|
6
|
-
export function boundedWorkflowPreviewRows<T extends Record<string, unknown>>(
|
|
7
|
-
rows: readonly T[],
|
|
8
|
-
input: {
|
|
9
|
-
maxRows: number;
|
|
10
|
-
maxBytes: number;
|
|
11
|
-
serialize?: (row: T) => T;
|
|
12
|
-
},
|
|
13
|
-
): T[] {
|
|
14
|
-
const preview: T[] = [];
|
|
15
|
-
let bytes = 2; // Opening and closing array brackets.
|
|
16
|
-
for (const row of rows) {
|
|
17
|
-
if (preview.length >= input.maxRows) break;
|
|
18
|
-
const serialized = input.serialize ? input.serialize(row) : row;
|
|
19
|
-
const rowBytes = jsonByteLength(serialized) + (preview.length > 0 ? 1 : 0);
|
|
20
|
-
if (rowBytes > input.maxBytes) continue;
|
|
21
|
-
if (bytes + rowBytes > input.maxBytes) break;
|
|
22
|
-
preview.push(serialized);
|
|
23
|
-
bytes += rowBytes;
|
|
24
|
-
}
|
|
25
|
-
return preview;
|
|
26
|
-
}
|
|
@@ -1,23 +0,0 @@
|
|
|
1
|
-
const PERMANENT_RUNTIME_API_STATUS_RE = /runtime API 40[0134]:/i;
|
|
2
|
-
const RETRYABLE_RUNTIME_API_ERROR_RE =
|
|
3
|
-
/runtime API (408|429|500|50[2-4]|530):|time(?:d out|out)|fetch failed|ECONNRESET|ECONNREFUSED|UND_ERR_CONNECT_TIMEOUT|DeploymentNotFound|DEPLOYMENT_NOT_FOUND|requested deployment .*exist|The deployment could not be found on Vercel|Authentication infrastructure/i;
|
|
4
|
-
|
|
5
|
-
function isVercelDeploymentMissingRuntimeApi404(message: string): boolean {
|
|
6
|
-
return (
|
|
7
|
-
/runtime API 404:/i.test(message) &&
|
|
8
|
-
/DeploymentNotFound|DEPLOYMENT_NOT_FOUND|requested deployment .*exist|The deployment could not be found on Vercel/i.test(
|
|
9
|
-
message,
|
|
10
|
-
)
|
|
11
|
-
);
|
|
12
|
-
}
|
|
13
|
-
|
|
14
|
-
export function isRetryableWorkerRuntimeApiError(error: unknown): boolean {
|
|
15
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
16
|
-
if (
|
|
17
|
-
PERMANENT_RUNTIME_API_STATUS_RE.test(message) &&
|
|
18
|
-
!isVercelDeploymentMissingRuntimeApi404(message)
|
|
19
|
-
) {
|
|
20
|
-
return false;
|
|
21
|
-
}
|
|
22
|
-
return RETRYABLE_RUNTIME_API_ERROR_RE.test(message);
|
|
23
|
-
}
|
|
@@ -1,47 +0,0 @@
|
|
|
1
|
-
export type WorkflowInstanceLike = {
|
|
2
|
-
id: string;
|
|
3
|
-
};
|
|
4
|
-
|
|
5
|
-
export type WorkflowInstanceCreateResult<T extends WorkflowInstanceLike> = {
|
|
6
|
-
instance: T;
|
|
7
|
-
startMode: 'created' | 'reattached_existing';
|
|
8
|
-
};
|
|
9
|
-
|
|
10
|
-
export function isWorkflowInstanceAlreadyExistsError(error: unknown): boolean {
|
|
11
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
12
|
-
return /instance[._ -]?already[._ -]?exists|Instance already exists/i.test(
|
|
13
|
-
message,
|
|
14
|
-
);
|
|
15
|
-
}
|
|
16
|
-
|
|
17
|
-
export async function createOrAttachWorkflowInstance<
|
|
18
|
-
T extends WorkflowInstanceLike,
|
|
19
|
-
>(input: {
|
|
20
|
-
create: () => Promise<T>;
|
|
21
|
-
getExisting: () => Promise<T>;
|
|
22
|
-
onCreateFailure?: (error: unknown) => Promise<void>;
|
|
23
|
-
}): Promise<WorkflowInstanceCreateResult<T>> {
|
|
24
|
-
try {
|
|
25
|
-
return { instance: await input.create(), startMode: 'created' };
|
|
26
|
-
} catch (error) {
|
|
27
|
-
if (!isWorkflowInstanceAlreadyExistsError(error)) {
|
|
28
|
-
await input.onCreateFailure?.(error);
|
|
29
|
-
throw error;
|
|
30
|
-
}
|
|
31
|
-
try {
|
|
32
|
-
return {
|
|
33
|
-
instance: await input.getExisting(),
|
|
34
|
-
startMode: 'reattached_existing',
|
|
35
|
-
};
|
|
36
|
-
} catch (attachError) {
|
|
37
|
-
await input.onCreateFailure?.(attachError);
|
|
38
|
-
const message =
|
|
39
|
-
attachError instanceof Error
|
|
40
|
-
? attachError.message
|
|
41
|
-
: String(attachError);
|
|
42
|
-
throw new Error(
|
|
43
|
-
`Workflow instance already exists but could not be reattached: ${message}`,
|
|
44
|
-
);
|
|
45
|
-
}
|
|
46
|
-
}
|
|
47
|
-
}
|
|
@@ -1,167 +0,0 @@
|
|
|
1
|
-
import type { ExecutionPlan } from '../../../shared_libs/play-runtime/execution-plan';
|
|
2
|
-
import type { PlayRunInputPayload } from '../../../shared_libs/play-runtime/scheduler-backend';
|
|
3
|
-
import type { PreloadedRuntimeDbSession } from '../../../shared_libs/play-runtime/db-session';
|
|
4
|
-
|
|
5
|
-
import {
|
|
6
|
-
PLAY_SUBMIT_INPUT_INLINE_MAX_BYTES,
|
|
7
|
-
PLAY_SUBMIT_INPUT_MAX_BYTES,
|
|
8
|
-
} from '../../../shared_libs/play-runtime/submit-limits';
|
|
9
|
-
|
|
10
|
-
export const WORKFLOW_RETRY_STATE_TARGET_BYTES =
|
|
11
|
-
PLAY_SUBMIT_INPUT_INLINE_MAX_BYTES;
|
|
12
|
-
export const WORKFLOW_RETRY_PARAMS_EXTERNALIZE_AFTER_BYTES =
|
|
13
|
-
WORKFLOW_RETRY_STATE_TARGET_BYTES;
|
|
14
|
-
export const WORKFLOW_RETRY_PARAMS_MAX_BYTES = PLAY_SUBMIT_INPUT_MAX_BYTES;
|
|
15
|
-
|
|
16
|
-
export type WorkflowRetryParamsRef = {
|
|
17
|
-
storageKind: 'r2';
|
|
18
|
-
storageKey: string;
|
|
19
|
-
bytes: number;
|
|
20
|
-
hash: string;
|
|
21
|
-
expiresAt: number;
|
|
22
|
-
};
|
|
23
|
-
|
|
24
|
-
export type WorkflowRetryPlayParams = {
|
|
25
|
-
runId: string;
|
|
26
|
-
runAttempt?: number;
|
|
27
|
-
playId: string;
|
|
28
|
-
playName: string;
|
|
29
|
-
artifactStorageKey: string;
|
|
30
|
-
artifactHash: string;
|
|
31
|
-
graphHash: string;
|
|
32
|
-
input: PlayRunInputPayload;
|
|
33
|
-
inputFile?: {
|
|
34
|
-
name?: string;
|
|
35
|
-
r2Key?: string;
|
|
36
|
-
storageKey?: string;
|
|
37
|
-
path?: string;
|
|
38
|
-
fileName?: string;
|
|
39
|
-
logicalPath?: string;
|
|
40
|
-
contentType?: string;
|
|
41
|
-
bytes?: number;
|
|
42
|
-
} | null;
|
|
43
|
-
inlineCsv?: { name: string; rows: Record<string, unknown>[] } | null;
|
|
44
|
-
packagedFiles?: Array<{
|
|
45
|
-
playPath: string;
|
|
46
|
-
storageKey: string;
|
|
47
|
-
contentType?: string;
|
|
48
|
-
bytes?: number;
|
|
49
|
-
inlineText?: string;
|
|
50
|
-
}> | null;
|
|
51
|
-
contractSnapshot?: unknown;
|
|
52
|
-
executionPlan?: ExecutionPlan | null;
|
|
53
|
-
preloadedDbSessions?: PreloadedRuntimeDbSession[] | null;
|
|
54
|
-
preloadedDbSessionRef?: {
|
|
55
|
-
runId: string;
|
|
56
|
-
sessionCount: number;
|
|
57
|
-
expiresAt: number;
|
|
58
|
-
} | null;
|
|
59
|
-
dynamicWorkerCode?: string | null;
|
|
60
|
-
executorToken: string;
|
|
61
|
-
baseUrl: string;
|
|
62
|
-
runtimeApiBaseUrl?: string | null;
|
|
63
|
-
callbackUrl?: string | null;
|
|
64
|
-
orgId: string;
|
|
65
|
-
userEmail: string;
|
|
66
|
-
userId?: string | null;
|
|
67
|
-
runtimeBackend: string;
|
|
68
|
-
dedupBackend: string;
|
|
69
|
-
totalRows?: number;
|
|
70
|
-
coordinatorUrl?: string | null;
|
|
71
|
-
coordinatorInternalToken?: string | null;
|
|
72
|
-
runtimeTestFaultHeader?: string | null;
|
|
73
|
-
};
|
|
74
|
-
|
|
75
|
-
export type WorkflowRetryStatePayload<TParams = WorkflowRetryPlayParams> =
|
|
76
|
-
| {
|
|
77
|
-
params: TParams;
|
|
78
|
-
paramsRef?: null;
|
|
79
|
-
paramsBytes: number;
|
|
80
|
-
}
|
|
81
|
-
| {
|
|
82
|
-
params: null;
|
|
83
|
-
paramsRef: WorkflowRetryParamsRef;
|
|
84
|
-
paramsBytes: number;
|
|
85
|
-
};
|
|
86
|
-
|
|
87
|
-
export function buildWorkflowRetryParams<
|
|
88
|
-
TParams extends WorkflowRetryPlayParams,
|
|
89
|
-
>(params: TParams): TParams {
|
|
90
|
-
const retryParams = {
|
|
91
|
-
...params,
|
|
92
|
-
dynamicWorkerCode: null,
|
|
93
|
-
contractSnapshot: stripRetrySourceSnapshot(params.contractSnapshot),
|
|
94
|
-
packagedFiles: stripRetryPackagedFiles(params.packagedFiles),
|
|
95
|
-
} satisfies WorkflowRetryPlayParams as TParams;
|
|
96
|
-
if (jsonByteLength(retryParams) <= WORKFLOW_RETRY_STATE_TARGET_BYTES) {
|
|
97
|
-
return retryParams;
|
|
98
|
-
}
|
|
99
|
-
return {
|
|
100
|
-
...retryParams,
|
|
101
|
-
contractSnapshot: stripRetryContractSnapshotToArtifact(
|
|
102
|
-
retryParams.contractSnapshot,
|
|
103
|
-
params,
|
|
104
|
-
),
|
|
105
|
-
} satisfies WorkflowRetryPlayParams as TParams;
|
|
106
|
-
}
|
|
107
|
-
|
|
108
|
-
export function jsonByteLength(value: unknown): number {
|
|
109
|
-
return new TextEncoder().encode(JSON.stringify(value)).length;
|
|
110
|
-
}
|
|
111
|
-
|
|
112
|
-
export function workflowRetryParamsStorageKey(input: {
|
|
113
|
-
runId: string;
|
|
114
|
-
hash: string;
|
|
115
|
-
}): string {
|
|
116
|
-
const safeRunId = input.runId
|
|
117
|
-
.toLowerCase()
|
|
118
|
-
.replace(/[^a-z0-9_-]+/g, '-')
|
|
119
|
-
.slice(0, 140);
|
|
120
|
-
return `plays/workflow-retry-params/${safeRunId || 'run'}/${input.hash}.json`;
|
|
121
|
-
}
|
|
122
|
-
|
|
123
|
-
function stripRetryPackagedFiles(
|
|
124
|
-
files: WorkflowRetryPlayParams['packagedFiles'],
|
|
125
|
-
): WorkflowRetryPlayParams['packagedFiles'] {
|
|
126
|
-
return (
|
|
127
|
-
files?.map((file) => ({
|
|
128
|
-
playPath: file.playPath,
|
|
129
|
-
storageKey: file.storageKey,
|
|
130
|
-
contentType: file.contentType,
|
|
131
|
-
bytes: file.bytes,
|
|
132
|
-
})) ?? null
|
|
133
|
-
);
|
|
134
|
-
}
|
|
135
|
-
|
|
136
|
-
function stripRetrySourceSnapshot(snapshot: unknown): unknown {
|
|
137
|
-
if (!isRecord(snapshot)) return snapshot;
|
|
138
|
-
const rest = { ...snapshot };
|
|
139
|
-
delete rest.sourceCode;
|
|
140
|
-
delete rest.sourceFiles;
|
|
141
|
-
delete rest.bundledCode;
|
|
142
|
-
return rest;
|
|
143
|
-
}
|
|
144
|
-
|
|
145
|
-
function stripRetryContractSnapshotToArtifact(
|
|
146
|
-
snapshot: unknown,
|
|
147
|
-
params: WorkflowRetryPlayParams,
|
|
148
|
-
): unknown {
|
|
149
|
-
if (!isRecord(snapshot)) return snapshot;
|
|
150
|
-
return {
|
|
151
|
-
source: snapshot.source ?? 'artifact',
|
|
152
|
-
revisionVersion: snapshot.revisionVersion ?? null,
|
|
153
|
-
staticPipeline: snapshot.staticPipeline ?? null,
|
|
154
|
-
billingLimit: snapshot.billingLimit ?? null,
|
|
155
|
-
artifactMetadata: {
|
|
156
|
-
storageKey: params.artifactStorageKey,
|
|
157
|
-
artifactHash: params.artifactHash,
|
|
158
|
-
graphHash: params.graphHash,
|
|
159
|
-
},
|
|
160
|
-
codeFormat: snapshot.codeFormat ?? null,
|
|
161
|
-
compatibility: snapshot.compatibility ?? null,
|
|
162
|
-
};
|
|
163
|
-
}
|
|
164
|
-
|
|
165
|
-
function isRecord(value: unknown): value is Record<string, unknown> {
|
|
166
|
-
return Boolean(value) && typeof value === 'object' && !Array.isArray(value);
|
|
167
|
-
}
|