deepline 0.1.199 → 0.1.201
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/apps/play-runner-workers/src/coordinator-entry.ts +5 -1
- package/dist/bundling-sources/apps/play-runner-workers/src/entry.ts +67 -18
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/README.md +1 -1
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/harness-receipt-store.ts +16 -0
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/receipts.ts +40 -9
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/tool-dispatch.ts +108 -39
- package/dist/bundling-sources/sdk/src/http.ts +5 -2
- package/dist/bundling-sources/sdk/src/play.ts +1 -1
- package/dist/bundling-sources/sdk/src/plays/harness-stub.ts +7 -0
- package/dist/bundling-sources/sdk/src/release.ts +2 -2
- package/dist/bundling-sources/sdk/src/worker-play-entry.ts +11 -55
- package/dist/bundling-sources/shared_libs/play-runtime/adaptive-tool-dispatcher.ts +355 -0
- package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +172 -73
- package/dist/bundling-sources/shared_libs/play-runtime/context.ts +1794 -377
- package/dist/bundling-sources/shared_libs/play-runtime/ctx-contract.ts +3 -0
- package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +67 -4
- package/dist/bundling-sources/shared_libs/play-runtime/durable-receipt-execution.ts +161 -39
- package/dist/bundling-sources/shared_libs/play-runtime/governor/adaptive-admission.ts +218 -0
- package/dist/bundling-sources/shared_libs/play-runtime/governor/app-runtime-rate-state-backend.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/governor/governor.ts +115 -25
- package/dist/bundling-sources/shared_libs/play-runtime/governor/policy.ts +17 -17
- package/dist/bundling-sources/shared_libs/play-runtime/lease-policy.ts +22 -0
- package/dist/bundling-sources/shared_libs/play-runtime/ledger-safe-payload.ts +2 -1
- package/dist/bundling-sources/shared_libs/play-runtime/live-events.ts +0 -1
- package/dist/bundling-sources/shared_libs/play-runtime/map-memory-limits.ts +265 -0
- package/dist/bundling-sources/shared_libs/play-runtime/output-size-limits.ts +135 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-input.ts +17 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-latency-trace.ts +0 -3
- package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts +10 -1
- package/dist/bundling-sources/shared_libs/play-runtime/providers.ts +9 -15
- package/dist/bundling-sources/shared_libs/play-runtime/receipt-completion-sink.ts +262 -0
- package/dist/bundling-sources/shared_libs/play-runtime/receipt-heartbeat-supervisor.ts +64 -0
- package/dist/bundling-sources/shared_libs/play-runtime/receipt-sql.ts +76 -8
- package/dist/bundling-sources/shared_libs/play-runtime/receipt-state-sink.ts +59 -0
- package/dist/bundling-sources/shared_libs/play-runtime/receipt-status.ts +3 -2
- package/dist/bundling-sources/shared_libs/play-runtime/resource-governor.ts +269 -0
- package/dist/bundling-sources/shared_libs/play-runtime/run-ledger.ts +45 -3
- package/dist/bundling-sources/shared_libs/play-runtime/run-terminal-source.ts +0 -1
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-lifecycle.ts +433 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-payload-transport.ts +227 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts +882 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/local-process.ts +573 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/bundle.ts +264 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/index.ts +33 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/runner-events.ts +35 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/types.ts +53 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-actions.ts +0 -1
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +561 -105
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-constants.ts +12 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-infra-mode.ts +23 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-sheet-attempt-state-machine.ts +23 -2
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backend.ts +16 -24
- package/dist/bundling-sources/shared_libs/play-runtime/serial-task-queue.ts +26 -0
- package/dist/bundling-sources/shared_libs/play-runtime/submit-limits.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/vercel-protection.ts +81 -0
- package/dist/bundling-sources/shared_libs/play-runtime/work-receipt-state-machine.ts +54 -9
- package/dist/bundling-sources/shared_libs/play-runtime/work-receipts.ts +17 -3
- package/dist/bundling-sources/shared_libs/play-runtime/worker-api-types.ts +0 -20
- package/dist/bundling-sources/shared_libs/plays/dataset.ts +105 -4
- package/dist/bundling-sources/shared_libs/plays/row-identity.ts +4 -11
- package/dist/cli/index.js +167 -54
- package/dist/cli/index.mjs +167 -54
- package/dist/index.d.mts +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.js +8 -8
- package/dist/index.mjs +8 -8
- package/dist/plays/bundle-play-file.mjs +15 -20
- package/package.json +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/hatchet-cold-execution-diagnosis.ts +0 -321
- package/dist/bundling-sources/shared_libs/play-runtime/hatchet-cold-execution-target.ts +0 -158
- package/dist/bundling-sources/shared_libs/temporal/constants.ts +0 -39
- package/dist/bundling-sources/shared_libs/temporal/preview-config.ts +0 -150
package/dist/bundling-sources/shared_libs/play-runtime/governor/app-runtime-rate-state-backend.ts
CHANGED
|
@@ -12,7 +12,7 @@ import {
|
|
|
12
12
|
} from './rate-state-backend';
|
|
13
13
|
|
|
14
14
|
/**
|
|
15
|
-
* Permits leased per app-runtime round trip for the
|
|
15
|
+
* Permits leased per app-runtime round trip for the direct Node substrate.
|
|
16
16
|
*
|
|
17
17
|
* Eight keeps DB/API round trips amortized for batch-heavy maps while limiting
|
|
18
18
|
* stale-block over-issue to 8 x live runner processes per bucket/window.
|
|
@@ -19,11 +19,11 @@ import {
|
|
|
19
19
|
resolveExecutionPolicy,
|
|
20
20
|
resolveRowConcurrency,
|
|
21
21
|
} from './policy';
|
|
22
|
+
import { type PacingRule, type RateStateBackend } from './rate-state-backend';
|
|
22
23
|
import {
|
|
23
|
-
|
|
24
|
-
type
|
|
25
|
-
|
|
26
|
-
} from './rate-state-backend';
|
|
24
|
+
createInMemoryAdaptiveAdmission,
|
|
25
|
+
type RuntimeAdaptiveAdmission,
|
|
26
|
+
} from './adaptive-admission';
|
|
27
27
|
|
|
28
28
|
export interface WorkLease {
|
|
29
29
|
/** Free the slot / pacing permit. Idempotent. MUST be called in a finally. */
|
|
@@ -74,6 +74,25 @@ export type PacingResolver = (
|
|
|
74
74
|
toolId: string,
|
|
75
75
|
) => Promise<{ provider: string; rules: PacingRule[] } | null>;
|
|
76
76
|
|
|
77
|
+
export function defaultPacingForTool(
|
|
78
|
+
toolId: string,
|
|
79
|
+
policy: ResolvedExecutionPolicy,
|
|
80
|
+
): { provider: string; rules: PacingRule[] } {
|
|
81
|
+
return {
|
|
82
|
+
// Unknown providers still need a stable bucket. Tool id is the only
|
|
83
|
+
// substrate-neutral key available when the runtime has no queue hints.
|
|
84
|
+
provider: `tool:${toolId}`,
|
|
85
|
+
rules: [
|
|
86
|
+
{
|
|
87
|
+
ruleId: `default:${toolId}`,
|
|
88
|
+
requestsPerWindow: policy.pacing.defaultProviderRequestsPerSecond,
|
|
89
|
+
windowMs: 1_000,
|
|
90
|
+
maxConcurrency: null,
|
|
91
|
+
},
|
|
92
|
+
],
|
|
93
|
+
};
|
|
94
|
+
}
|
|
95
|
+
|
|
77
96
|
export interface PlayExecutionGovernor {
|
|
78
97
|
readonly adapter: AdapterId;
|
|
79
98
|
readonly policy: ResolvedExecutionPolicy;
|
|
@@ -132,6 +151,12 @@ export interface PlayExecutionGovernor {
|
|
|
132
151
|
retryAfterMs: number;
|
|
133
152
|
}): void;
|
|
134
153
|
|
|
154
|
+
/** Feed clean provider calls back into adaptive admission. */
|
|
155
|
+
observeProviderSuccess(input: {
|
|
156
|
+
provider: string;
|
|
157
|
+
latencyMs?: number | null;
|
|
158
|
+
}): void;
|
|
159
|
+
|
|
135
160
|
snapshot(): GovernanceSnapshot;
|
|
136
161
|
}
|
|
137
162
|
|
|
@@ -140,6 +165,7 @@ interface GovernorInput {
|
|
|
140
165
|
scope: { orgId: string; rootRunId: string };
|
|
141
166
|
rateState: RateStateBackend;
|
|
142
167
|
resolvePacing: PacingResolver;
|
|
168
|
+
adaptiveAdmission?: RuntimeAdaptiveAdmission;
|
|
143
169
|
resume?: GovernanceSnapshot;
|
|
144
170
|
}
|
|
145
171
|
|
|
@@ -226,35 +252,79 @@ export function createPlayExecutionGovernor(
|
|
|
226
252
|
const rowSlots = new Semaphore(policy.concurrency.rowMax);
|
|
227
253
|
const childPlaySlots = new Semaphore(policy.concurrency.childPlay);
|
|
228
254
|
const toolSlots = new Semaphore(policy.concurrency.toolCalls);
|
|
255
|
+
const adaptiveAdmission =
|
|
256
|
+
input.adaptiveAdmission ??
|
|
257
|
+
createInMemoryAdaptiveAdmission({
|
|
258
|
+
initialRequestsPerSecond: policy.pacing.defaultProviderRequestsPerSecond,
|
|
259
|
+
maxRequestsPerSecond: policy.pacing.suggestedMaxParallelism,
|
|
260
|
+
});
|
|
229
261
|
|
|
230
262
|
const bucketId = (provider: string) => `${input.scope.orgId}:${provider}`;
|
|
231
263
|
|
|
264
|
+
async function resolveAdaptivePacing(toolId: string): Promise<{
|
|
265
|
+
provider: string;
|
|
266
|
+
rules: PacingRule[];
|
|
267
|
+
}> {
|
|
268
|
+
const resolvedPacing = await input.resolvePacing(toolId);
|
|
269
|
+
const declaredPacing =
|
|
270
|
+
resolvedPacing && resolvedPacing.rules.length > 0 ? resolvedPacing : null;
|
|
271
|
+
return adaptiveAdmission.resolvePacing({
|
|
272
|
+
orgId: input.scope.orgId,
|
|
273
|
+
toolId,
|
|
274
|
+
declaredPacing,
|
|
275
|
+
fallbackPacing: defaultPacingForTool(toolId, policy),
|
|
276
|
+
});
|
|
277
|
+
}
|
|
278
|
+
|
|
232
279
|
function chargeBudget(kind: BudgetKind, amount = 1): void {
|
|
233
280
|
switch (kind) {
|
|
234
281
|
case 'playCall':
|
|
235
282
|
state.playCallCount += amount;
|
|
236
283
|
if (state.playCallCount > policy.budgets.maxPlayCallCount)
|
|
237
|
-
throw new GovernorBudgetError(
|
|
284
|
+
throw new GovernorBudgetError(
|
|
285
|
+
'playCall',
|
|
286
|
+
state.playCallCount,
|
|
287
|
+
policy.budgets.maxPlayCallCount,
|
|
288
|
+
);
|
|
238
289
|
return;
|
|
239
290
|
case 'toolCall':
|
|
240
291
|
state.toolCallCount += amount;
|
|
241
292
|
if (state.toolCallCount > policy.budgets.maxToolCallCount)
|
|
242
|
-
throw new GovernorBudgetError(
|
|
293
|
+
throw new GovernorBudgetError(
|
|
294
|
+
'toolCall',
|
|
295
|
+
state.toolCallCount,
|
|
296
|
+
policy.budgets.maxToolCallCount,
|
|
297
|
+
);
|
|
243
298
|
return;
|
|
244
299
|
case 'retry':
|
|
245
300
|
state.retryCount += amount;
|
|
246
301
|
if (state.retryCount > policy.budgets.maxRetryCount)
|
|
247
|
-
throw new GovernorBudgetError(
|
|
302
|
+
throw new GovernorBudgetError(
|
|
303
|
+
'retry',
|
|
304
|
+
state.retryCount,
|
|
305
|
+
policy.budgets.maxRetryCount,
|
|
306
|
+
);
|
|
248
307
|
return;
|
|
249
308
|
case 'descendant':
|
|
250
309
|
state.descendantCount += amount;
|
|
251
310
|
if (state.descendantCount > policy.budgets.maxDescendants)
|
|
252
|
-
throw new GovernorBudgetError(
|
|
311
|
+
throw new GovernorBudgetError(
|
|
312
|
+
'descendant',
|
|
313
|
+
state.descendantCount,
|
|
314
|
+
policy.budgets.maxDescendants,
|
|
315
|
+
);
|
|
253
316
|
return;
|
|
254
317
|
case 'waterfallStep':
|
|
255
318
|
state.waterfallStepExecutions += amount;
|
|
256
|
-
if (
|
|
257
|
-
|
|
319
|
+
if (
|
|
320
|
+
state.waterfallStepExecutions >
|
|
321
|
+
policy.budgets.maxWaterfallStepExecutions
|
|
322
|
+
)
|
|
323
|
+
throw new GovernorBudgetError(
|
|
324
|
+
'waterfallStep',
|
|
325
|
+
state.waterfallStepExecutions,
|
|
326
|
+
policy.budgets.maxWaterfallStepExecutions,
|
|
327
|
+
);
|
|
258
328
|
return;
|
|
259
329
|
}
|
|
260
330
|
}
|
|
@@ -273,15 +343,12 @@ export function createPlayExecutionGovernor(
|
|
|
273
343
|
// resolver, so callers only need the toolId. No rules → no pacing.
|
|
274
344
|
let permit: { release(): void };
|
|
275
345
|
try {
|
|
276
|
-
const pacing = await
|
|
277
|
-
permit =
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
signal: opts?.signal,
|
|
283
|
-
})
|
|
284
|
-
: noopPacingPermit();
|
|
346
|
+
const pacing = await resolveAdaptivePacing(toolId);
|
|
347
|
+
permit = await input.rateState.acquire({
|
|
348
|
+
bucketId: bucketId(pacing.provider),
|
|
349
|
+
rules: pacing.rules,
|
|
350
|
+
signal: opts?.signal,
|
|
351
|
+
});
|
|
285
352
|
} catch (error) {
|
|
286
353
|
slot.release();
|
|
287
354
|
throw error;
|
|
@@ -307,8 +374,7 @@ export function createPlayExecutionGovernor(
|
|
|
307
374
|
},
|
|
308
375
|
|
|
309
376
|
async suggestedParallelism(toolId, fallback) {
|
|
310
|
-
const pacing = await
|
|
311
|
-
if (!pacing || pacing.rules.length === 0) return fallback;
|
|
377
|
+
const pacing = await resolveAdaptivePacing(toolId);
|
|
312
378
|
const limits = pacing.rules.flatMap((rule) =>
|
|
313
379
|
rule.maxConcurrency != null
|
|
314
380
|
? [rule.requestsPerWindow, rule.maxConcurrency]
|
|
@@ -324,16 +390,26 @@ export function createPlayExecutionGovernor(
|
|
|
324
390
|
|
|
325
391
|
forkChild(childInput) {
|
|
326
392
|
if (state.ancestryPlayIds.includes(childInput.childPlayName)) {
|
|
327
|
-
const chain = [...state.ancestryPlayIds, childInput.childPlayName].join(
|
|
393
|
+
const chain = [...state.ancestryPlayIds, childInput.childPlayName].join(
|
|
394
|
+
' -> ',
|
|
395
|
+
);
|
|
328
396
|
throw new Error(`Recursive play graph detected: ${chain}`);
|
|
329
397
|
}
|
|
330
398
|
const nextDepth = state.callDepth + 1;
|
|
331
399
|
if (nextDepth > policy.budgets.maxPlayCallDepth)
|
|
332
|
-
throw new GovernorBudgetError(
|
|
400
|
+
throw new GovernorBudgetError(
|
|
401
|
+
'playDepth',
|
|
402
|
+
nextDepth,
|
|
403
|
+
policy.budgets.maxPlayCallDepth,
|
|
404
|
+
);
|
|
333
405
|
const parentKey = state.currentPlayId;
|
|
334
406
|
const nextParent = (state.parentChildCalls[parentKey] ?? 0) + 1;
|
|
335
407
|
if (nextParent > policy.budgets.maxChildPlayCallsPerParent)
|
|
336
|
-
throw new GovernorBudgetError(
|
|
408
|
+
throw new GovernorBudgetError(
|
|
409
|
+
'childPerParent',
|
|
410
|
+
nextParent,
|
|
411
|
+
policy.budgets.maxChildPlayCallsPerParent,
|
|
412
|
+
);
|
|
337
413
|
// Charge the run-wide play/descendant budgets on the parent. Charged at
|
|
338
414
|
// fork time (not after the caller's child-play slot acquire) and never
|
|
339
415
|
// refunded if that acquire fails — see the forkChild interface doc.
|
|
@@ -357,15 +433,29 @@ export function createPlayExecutionGovernor(
|
|
|
357
433
|
};
|
|
358
434
|
},
|
|
359
435
|
|
|
360
|
-
resolveRowConcurrency: (requested) =>
|
|
436
|
+
resolveRowConcurrency: (requested) =>
|
|
437
|
+
resolveRowConcurrency(policy, requested),
|
|
361
438
|
|
|
362
439
|
reportProviderBackpressure(bp) {
|
|
440
|
+
adaptiveAdmission.observeProviderBackpressure({
|
|
441
|
+
orgId: input.scope.orgId,
|
|
442
|
+
provider: bp.provider,
|
|
443
|
+
retryAfterMs: bp.retryAfterMs,
|
|
444
|
+
});
|
|
363
445
|
input.rateState.penalize({
|
|
364
446
|
bucketId: bucketId(bp.provider),
|
|
365
447
|
cooldownMs: bp.retryAfterMs,
|
|
366
448
|
});
|
|
367
449
|
},
|
|
368
450
|
|
|
451
|
+
observeProviderSuccess(success) {
|
|
452
|
+
adaptiveAdmission.observeProviderSuccess({
|
|
453
|
+
orgId: input.scope.orgId,
|
|
454
|
+
provider: success.provider,
|
|
455
|
+
latencyMs: success.latencyMs,
|
|
456
|
+
});
|
|
457
|
+
},
|
|
458
|
+
|
|
369
459
|
snapshot: () => ({
|
|
370
460
|
...state,
|
|
371
461
|
ancestryPlayIds: [...state.ancestryPlayIds],
|
|
@@ -8,12 +8,12 @@
|
|
|
8
8
|
* CONTEXT.md.
|
|
9
9
|
*
|
|
10
10
|
* Tuning philosophy (product decision):
|
|
11
|
-
* - Caps are
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
* - The real-time bounds on
|
|
15
|
-
*
|
|
16
|
-
*
|
|
11
|
+
* - Caps are explicit resource admission boundaries. A run should queue behind
|
|
12
|
+
* them inside the executor instead of manufacturing thousands of promises,
|
|
13
|
+
* receipts, sockets, or row payload copies up front.
|
|
14
|
+
* - The real-time bounds on outbound work are still per-provider pacing and
|
|
15
|
+
* provider latency. Row concurrency exists to keep the sandbox healthy while
|
|
16
|
+
* enough work is admitted to saturate normal provider RPS.
|
|
17
17
|
* - Every value is an EXPLICIT, finite cap. Nothing here is unbounded — there
|
|
18
18
|
* is no `null`, no "off", no implicit infinity. Unaccounted resource use
|
|
19
19
|
* must be impossible.
|
|
@@ -90,15 +90,15 @@ export interface ResolvedExecutionPolicy {
|
|
|
90
90
|
*/
|
|
91
91
|
export const SHARED_EXECUTION_POLICY: ResolvedExecutionPolicy = {
|
|
92
92
|
concurrency: {
|
|
93
|
-
//
|
|
94
|
-
//
|
|
95
|
-
//
|
|
96
|
-
//
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
//
|
|
100
|
-
//
|
|
101
|
-
rowMax:
|
|
93
|
+
// Logical row target. A 1,000-row window can fill five 200-item provider
|
|
94
|
+
// batches or keep a 100-RPS unbatched provider fed; the runtime's shared
|
|
95
|
+
// resident-byte admission calculator clamps fat rows below this target.
|
|
96
|
+
// Pending rows remain indexes, not promises or pre-claimed receipts.
|
|
97
|
+
rowDefault: 1_000,
|
|
98
|
+
// Finite global ceiling across concurrent maps in one run. This intentionally
|
|
99
|
+
// matches the default: throughput grows through provider pacing/batching,
|
|
100
|
+
// while row payload size can only reduce admission through the byte budget.
|
|
101
|
+
rowMax: 1_000,
|
|
102
102
|
// Concurrent child-play launches. Generous; each child is a real launch, so
|
|
103
103
|
// this is the one value most likely to need a documented esm_workers
|
|
104
104
|
// override if isolate pressure shows up in E2E.
|
|
@@ -121,13 +121,13 @@ export const SHARED_EXECUTION_POLICY: ResolvedExecutionPolicy = {
|
|
|
121
121
|
pacing: {
|
|
122
122
|
// Undeclared providers; declared providers (rate-limit-definitions.ts) win.
|
|
123
123
|
defaultProviderRequestsPerSecond: 10,
|
|
124
|
-
suggestedMaxParallelism:
|
|
124
|
+
suggestedMaxParallelism: 256,
|
|
125
125
|
// Worker isolate pacing knobs. The budget module owns platform accounting;
|
|
126
126
|
// the Governor owns the policy values that decide when work is chunked.
|
|
127
127
|
workerYieldElapsedMs: 45_000,
|
|
128
128
|
// Fallback wave width for batched tool chunks that carry no provider pacing
|
|
129
129
|
// hint. This is only a floor for the undeclared case: it stays bounded by
|
|
130
|
-
// suggestedMaxParallelism
|
|
130
|
+
// suggestedMaxParallelism above, and declared providers still get
|
|
131
131
|
// tightened by their per-provider pacing (rate-limit-definitions.ts). Raised
|
|
132
132
|
// 4 -> 16 so full batches fan out wide enough to close the map-throughput gap
|
|
133
133
|
// without over-running providers that actually declare limits.
|
|
@@ -50,3 +50,25 @@ export function runtimeLeaseHeartbeatIntervalMs(ttlMs: number): number {
|
|
|
50
50
|
}
|
|
51
51
|
return Math.max(1, Math.floor(ttlMs / PLAY_RUNTIME_LEASE_HEARTBEAT_DIVISOR));
|
|
52
52
|
}
|
|
53
|
+
|
|
54
|
+
/**
|
|
55
|
+
* The receipt store is authoritative for lease duration. Executors may not
|
|
56
|
+
* inherit the coordinator's environment, so derive renewal cadence from the
|
|
57
|
+
* expiry returned with the actual claim instead of assuming a local TTL.
|
|
58
|
+
*/
|
|
59
|
+
export function runtimeLeaseHeartbeatIntervalFromExpiry(input: {
|
|
60
|
+
leaseExpiresAt?: string | null;
|
|
61
|
+
fallbackTtlMs: number;
|
|
62
|
+
nowMs?: number;
|
|
63
|
+
}): number {
|
|
64
|
+
const expiresAtMs = input.leaseExpiresAt
|
|
65
|
+
? Date.parse(input.leaseExpiresAt)
|
|
66
|
+
: Number.NaN;
|
|
67
|
+
const nowMs = input.nowMs ?? Date.now();
|
|
68
|
+
const remainingMs = expiresAtMs - nowMs;
|
|
69
|
+
return runtimeLeaseHeartbeatIntervalMs(
|
|
70
|
+
Number.isFinite(remainingMs) && remainingMs > 0
|
|
71
|
+
? remainingMs
|
|
72
|
+
: input.fallbackTtlMs,
|
|
73
|
+
);
|
|
74
|
+
}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { createSecretRedactionContext } from './secret-redaction';
|
|
2
|
+
import { RUNTIME_RECEIPT_OUTPUT_MAX_BYTES } from './output-size-limits';
|
|
2
3
|
|
|
3
|
-
export const MAX_LEDGER_RESULT_PAYLOAD_BYTES =
|
|
4
|
+
export const MAX_LEDGER_RESULT_PAYLOAD_BYTES = RUNTIME_RECEIPT_OUTPUT_MAX_BYTES;
|
|
4
5
|
export const MAX_LEDGER_LOG_LINES_PER_EVENT = 500;
|
|
5
6
|
export const MAX_LEDGER_LOG_LINE_LENGTH = 2_000;
|
|
6
7
|
|
|
@@ -0,0 +1,265 @@
|
|
|
1
|
+
export const NODE_RUNTIME_MAP_MEMORY_BUDGET_BYTES = 512 * 1024 * 1024;
|
|
2
|
+
export const NODE_RUNTIME_MAP_ACTIVE_ROWS_MEMORY_BUDGET_BYTES =
|
|
3
|
+
384 * 1024 * 1024;
|
|
4
|
+
export const NODE_RUNTIME_MAP_MATERIALIZED_MEMORY_MULTIPLIER = 4;
|
|
5
|
+
export const NODE_RUNTIME_MAP_ACTIVE_ROW_MEMORY_MULTIPLIER = 3;
|
|
6
|
+
export const NODE_RUNTIME_MAP_ROW_OVERHEAD_BYTES = 512;
|
|
7
|
+
|
|
8
|
+
export type RuntimeMapMemoryLimits = {
|
|
9
|
+
materializedBudgetBytes: number;
|
|
10
|
+
activeRowsBudgetBytes: number;
|
|
11
|
+
};
|
|
12
|
+
|
|
13
|
+
export type RuntimeMapMemoryEstimate = {
|
|
14
|
+
rowCount: number;
|
|
15
|
+
serializedBytes: number;
|
|
16
|
+
estimatedResidentBytes: number;
|
|
17
|
+
largestRowBytes: number;
|
|
18
|
+
};
|
|
19
|
+
|
|
20
|
+
export type RuntimeMapRowAdmission = {
|
|
21
|
+
concurrency: number;
|
|
22
|
+
estimatedBytesPerActiveRow: number;
|
|
23
|
+
estimatedResidentBytes: number;
|
|
24
|
+
};
|
|
25
|
+
|
|
26
|
+
export class RuntimeMapMemoryLimitError extends Error {
|
|
27
|
+
readonly code = 'RUNTIME_MAP_MEMORY_LIMIT_EXCEEDED';
|
|
28
|
+
constructor(input: {
|
|
29
|
+
mapName: string;
|
|
30
|
+
phase: 'materialize' | 'active_rows' | 'retained_rows';
|
|
31
|
+
estimate: RuntimeMapMemoryEstimate;
|
|
32
|
+
budgetBytes: number;
|
|
33
|
+
activeRows?: number;
|
|
34
|
+
rowConcurrency?: number;
|
|
35
|
+
}) {
|
|
36
|
+
const active =
|
|
37
|
+
input.activeRows === undefined
|
|
38
|
+
? ''
|
|
39
|
+
: ` activeRows=${input.activeRows}` +
|
|
40
|
+
(input.rowConcurrency === undefined
|
|
41
|
+
? ''
|
|
42
|
+
: ` rowConcurrency=${input.rowConcurrency}`);
|
|
43
|
+
const stopPoint =
|
|
44
|
+
input.phase === 'materialize'
|
|
45
|
+
? ' before provider calls'
|
|
46
|
+
: input.phase === 'active_rows'
|
|
47
|
+
? ' before row resolver bodies start'
|
|
48
|
+
: '';
|
|
49
|
+
super(
|
|
50
|
+
`Runtime map memory limit exceeded for ctx.dataset("${input.mapName}") during ${input.phase}: ` +
|
|
51
|
+
`estimatedResidentBytes=${input.estimate.estimatedResidentBytes} ` +
|
|
52
|
+
`budgetBytes=${input.budgetBytes} rowCount=${input.estimate.rowCount} ` +
|
|
53
|
+
`serializedBytes=${input.estimate.serializedBytes} largestRowBytes=${input.estimate.largestRowBytes}${active}. ` +
|
|
54
|
+
`This run would risk sandbox OOM. Deepline stops here${stopPoint} instead of silently lowering concurrency. ` +
|
|
55
|
+
'Reduce row payload size, stage bulky blobs, or split the input into smaller runs.',
|
|
56
|
+
);
|
|
57
|
+
this.name = 'RuntimeMapMemoryLimitError';
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
const textEncoder =
|
|
62
|
+
typeof TextEncoder === 'undefined' ? null : new TextEncoder();
|
|
63
|
+
|
|
64
|
+
export function runtimeMapJsonByteLength(value: unknown): number {
|
|
65
|
+
const serialized = JSON.stringify(value);
|
|
66
|
+
if (serialized === undefined) return 0;
|
|
67
|
+
return textEncoder?.encode(serialized).byteLength ?? serialized.length;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
export function defaultRuntimeMapMemoryLimits(): RuntimeMapMemoryLimits {
|
|
71
|
+
return {
|
|
72
|
+
materializedBudgetBytes: NODE_RUNTIME_MAP_MEMORY_BUDGET_BYTES,
|
|
73
|
+
activeRowsBudgetBytes: NODE_RUNTIME_MAP_ACTIVE_ROWS_MEMORY_BUDGET_BYTES,
|
|
74
|
+
};
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
export function resolveRuntimeMapMemoryLimits(
|
|
78
|
+
override?: Partial<RuntimeMapMemoryLimits> | null,
|
|
79
|
+
): RuntimeMapMemoryLimits {
|
|
80
|
+
const defaults = defaultRuntimeMapMemoryLimits();
|
|
81
|
+
return {
|
|
82
|
+
materializedBudgetBytes: positiveBudget(
|
|
83
|
+
override?.materializedBudgetBytes,
|
|
84
|
+
defaults.materializedBudgetBytes,
|
|
85
|
+
),
|
|
86
|
+
activeRowsBudgetBytes: positiveBudget(
|
|
87
|
+
override?.activeRowsBudgetBytes,
|
|
88
|
+
defaults.activeRowsBudgetBytes,
|
|
89
|
+
),
|
|
90
|
+
};
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
function positiveBudget(value: number | undefined, fallback: number): number {
|
|
94
|
+
if (!Number.isFinite(value) || value === undefined) return fallback;
|
|
95
|
+
return Math.max(1, Math.floor(value));
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
export function estimateRuntimeMapMaterializedMemory(input: {
|
|
99
|
+
rowCount: number;
|
|
100
|
+
serializedBytes: number;
|
|
101
|
+
largestRowBytes: number;
|
|
102
|
+
}): RuntimeMapMemoryEstimate {
|
|
103
|
+
return {
|
|
104
|
+
rowCount: input.rowCount,
|
|
105
|
+
serializedBytes: input.serializedBytes,
|
|
106
|
+
largestRowBytes: input.largestRowBytes,
|
|
107
|
+
estimatedResidentBytes:
|
|
108
|
+
input.serializedBytes * NODE_RUNTIME_MAP_MATERIALIZED_MEMORY_MULTIPLIER +
|
|
109
|
+
input.rowCount * NODE_RUNTIME_MAP_ROW_OVERHEAD_BYTES,
|
|
110
|
+
};
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
export function estimateRuntimeMapRowsMemory(
|
|
114
|
+
rows: readonly unknown[],
|
|
115
|
+
): RuntimeMapMemoryEstimate {
|
|
116
|
+
let serializedBytes = 0;
|
|
117
|
+
let largestRowBytes = 0;
|
|
118
|
+
for (const row of rows) {
|
|
119
|
+
const rowBytes = runtimeMapJsonByteLength(row);
|
|
120
|
+
serializedBytes += rowBytes;
|
|
121
|
+
largestRowBytes = Math.max(largestRowBytes, rowBytes);
|
|
122
|
+
}
|
|
123
|
+
return estimateRuntimeMapMaterializedMemory({
|
|
124
|
+
rowCount: rows.length,
|
|
125
|
+
serializedBytes,
|
|
126
|
+
largestRowBytes,
|
|
127
|
+
});
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
type RuntimeMapRowsTrackerInput = {
|
|
131
|
+
mapName: string;
|
|
132
|
+
budgetBytes?: number;
|
|
133
|
+
};
|
|
134
|
+
|
|
135
|
+
type RuntimeMapRowsTracker = {
|
|
136
|
+
track(row: unknown): void;
|
|
137
|
+
estimate(): RuntimeMapMemoryEstimate;
|
|
138
|
+
};
|
|
139
|
+
|
|
140
|
+
function createRuntimeMapRowsTracker(
|
|
141
|
+
input: RuntimeMapRowsTrackerInput,
|
|
142
|
+
phase: 'materialize' | 'retained_rows',
|
|
143
|
+
): RuntimeMapRowsTracker {
|
|
144
|
+
const budgetBytes = positiveBudget(
|
|
145
|
+
input.budgetBytes,
|
|
146
|
+
NODE_RUNTIME_MAP_MEMORY_BUDGET_BYTES,
|
|
147
|
+
);
|
|
148
|
+
let rowCount = 0;
|
|
149
|
+
let serializedBytes = 0;
|
|
150
|
+
let largestRowBytes = 0;
|
|
151
|
+
const estimate = () =>
|
|
152
|
+
estimateRuntimeMapMaterializedMemory({
|
|
153
|
+
rowCount,
|
|
154
|
+
serializedBytes,
|
|
155
|
+
largestRowBytes,
|
|
156
|
+
});
|
|
157
|
+
return {
|
|
158
|
+
track(row) {
|
|
159
|
+
const rowBytes = runtimeMapJsonByteLength(row);
|
|
160
|
+
rowCount += 1;
|
|
161
|
+
serializedBytes += rowBytes;
|
|
162
|
+
largestRowBytes = Math.max(largestRowBytes, rowBytes);
|
|
163
|
+
const current = estimate();
|
|
164
|
+
if (current.estimatedResidentBytes > budgetBytes) {
|
|
165
|
+
throw new RuntimeMapMemoryLimitError({
|
|
166
|
+
mapName: input.mapName,
|
|
167
|
+
phase,
|
|
168
|
+
estimate: current,
|
|
169
|
+
budgetBytes,
|
|
170
|
+
});
|
|
171
|
+
}
|
|
172
|
+
},
|
|
173
|
+
estimate,
|
|
174
|
+
};
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
export function createRuntimeMapMaterializationTracker(
|
|
178
|
+
input: RuntimeMapRowsTrackerInput,
|
|
179
|
+
): RuntimeMapRowsTracker {
|
|
180
|
+
return createRuntimeMapRowsTracker(input, 'materialize');
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
export function createRuntimeMapRetainedRowsTracker(
|
|
184
|
+
input: RuntimeMapRowsTrackerInput,
|
|
185
|
+
): RuntimeMapRowsTracker {
|
|
186
|
+
return createRuntimeMapRowsTracker(input, 'retained_rows');
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
export function assertRuntimeMapActiveRowsWithinMemoryLimit(input: {
|
|
190
|
+
mapName: string;
|
|
191
|
+
rowCount: number;
|
|
192
|
+
rowConcurrency: number;
|
|
193
|
+
largestRowBytes: number;
|
|
194
|
+
activeRowsBudgetBytes?: number;
|
|
195
|
+
}): void {
|
|
196
|
+
const activeRows = Math.min(
|
|
197
|
+
Math.max(0, input.rowCount),
|
|
198
|
+
Math.max(1, Math.floor(input.rowConcurrency)),
|
|
199
|
+
);
|
|
200
|
+
const estimate: RuntimeMapMemoryEstimate = {
|
|
201
|
+
rowCount: input.rowCount,
|
|
202
|
+
serializedBytes: input.largestRowBytes * activeRows,
|
|
203
|
+
largestRowBytes: input.largestRowBytes,
|
|
204
|
+
estimatedResidentBytes:
|
|
205
|
+
input.largestRowBytes *
|
|
206
|
+
activeRows *
|
|
207
|
+
NODE_RUNTIME_MAP_ACTIVE_ROW_MEMORY_MULTIPLIER +
|
|
208
|
+
activeRows * NODE_RUNTIME_MAP_ROW_OVERHEAD_BYTES,
|
|
209
|
+
};
|
|
210
|
+
const budgetBytes = positiveBudget(
|
|
211
|
+
input.activeRowsBudgetBytes,
|
|
212
|
+
NODE_RUNTIME_MAP_ACTIVE_ROWS_MEMORY_BUDGET_BYTES,
|
|
213
|
+
);
|
|
214
|
+
if (estimate.estimatedResidentBytes > budgetBytes) {
|
|
215
|
+
throw new RuntimeMapMemoryLimitError({
|
|
216
|
+
mapName: input.mapName,
|
|
217
|
+
phase: 'active_rows',
|
|
218
|
+
estimate,
|
|
219
|
+
budgetBytes,
|
|
220
|
+
activeRows,
|
|
221
|
+
rowConcurrency: input.rowConcurrency,
|
|
222
|
+
});
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
/**
|
|
227
|
+
* Resolve a row-worker window from the shared concurrency target and a concrete
|
|
228
|
+
* resident-byte budget. Pending rows remain indexes, so only this admitted
|
|
229
|
+
* window contributes active row copies/promises. If one row cannot fit, fail
|
|
230
|
+
* before any resolver body starts; otherwise memory pressure is normal
|
|
231
|
+
* backpressure and reduces the window explicitly.
|
|
232
|
+
*/
|
|
233
|
+
export function resolveRuntimeMapRowAdmission(input: {
|
|
234
|
+
rowCount: number;
|
|
235
|
+
requestedConcurrency: number;
|
|
236
|
+
largestRowBytes: number;
|
|
237
|
+
activeRowsBudgetBytes?: number;
|
|
238
|
+
}): RuntimeMapRowAdmission {
|
|
239
|
+
const rowCount = Math.max(0, Math.floor(input.rowCount));
|
|
240
|
+
const requestedConcurrency = Math.max(
|
|
241
|
+
1,
|
|
242
|
+
Math.floor(input.requestedConcurrency),
|
|
243
|
+
);
|
|
244
|
+
const largestRowBytes = Math.max(0, Math.floor(input.largestRowBytes));
|
|
245
|
+
const budgetBytes = positiveBudget(
|
|
246
|
+
input.activeRowsBudgetBytes,
|
|
247
|
+
NODE_RUNTIME_MAP_ACTIVE_ROWS_MEMORY_BUDGET_BYTES,
|
|
248
|
+
);
|
|
249
|
+
const estimatedBytesPerActiveRow =
|
|
250
|
+
largestRowBytes * NODE_RUNTIME_MAP_ACTIVE_ROW_MEMORY_MULTIPLIER +
|
|
251
|
+
NODE_RUNTIME_MAP_ROW_OVERHEAD_BYTES;
|
|
252
|
+
|
|
253
|
+
const maxConcurrencyByMemory = Math.floor(
|
|
254
|
+
budgetBytes / Math.max(1, estimatedBytesPerActiveRow),
|
|
255
|
+
);
|
|
256
|
+
const concurrency =
|
|
257
|
+
rowCount === 0
|
|
258
|
+
? 0
|
|
259
|
+
: Math.min(rowCount, requestedConcurrency, maxConcurrencyByMemory);
|
|
260
|
+
return {
|
|
261
|
+
concurrency,
|
|
262
|
+
estimatedBytesPerActiveRow,
|
|
263
|
+
estimatedResidentBytes: estimatedBytesPerActiveRow * concurrency,
|
|
264
|
+
};
|
|
265
|
+
}
|