deepline 0.1.199 → 0.1.201

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. package/dist/bundling-sources/apps/play-runner-workers/src/coordinator-entry.ts +5 -1
  2. package/dist/bundling-sources/apps/play-runner-workers/src/entry.ts +67 -18
  3. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/README.md +1 -1
  4. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/harness-receipt-store.ts +16 -0
  5. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/receipts.ts +40 -9
  6. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/tool-dispatch.ts +108 -39
  7. package/dist/bundling-sources/sdk/src/http.ts +5 -2
  8. package/dist/bundling-sources/sdk/src/play.ts +1 -1
  9. package/dist/bundling-sources/sdk/src/plays/harness-stub.ts +7 -0
  10. package/dist/bundling-sources/sdk/src/release.ts +2 -2
  11. package/dist/bundling-sources/sdk/src/worker-play-entry.ts +11 -55
  12. package/dist/bundling-sources/shared_libs/play-runtime/adaptive-tool-dispatcher.ts +355 -0
  13. package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +172 -73
  14. package/dist/bundling-sources/shared_libs/play-runtime/context.ts +1794 -377
  15. package/dist/bundling-sources/shared_libs/play-runtime/ctx-contract.ts +3 -0
  16. package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +67 -4
  17. package/dist/bundling-sources/shared_libs/play-runtime/durable-receipt-execution.ts +161 -39
  18. package/dist/bundling-sources/shared_libs/play-runtime/governor/adaptive-admission.ts +218 -0
  19. package/dist/bundling-sources/shared_libs/play-runtime/governor/app-runtime-rate-state-backend.ts +1 -1
  20. package/dist/bundling-sources/shared_libs/play-runtime/governor/governor.ts +115 -25
  21. package/dist/bundling-sources/shared_libs/play-runtime/governor/policy.ts +17 -17
  22. package/dist/bundling-sources/shared_libs/play-runtime/lease-policy.ts +22 -0
  23. package/dist/bundling-sources/shared_libs/play-runtime/ledger-safe-payload.ts +2 -1
  24. package/dist/bundling-sources/shared_libs/play-runtime/live-events.ts +0 -1
  25. package/dist/bundling-sources/shared_libs/play-runtime/map-memory-limits.ts +265 -0
  26. package/dist/bundling-sources/shared_libs/play-runtime/output-size-limits.ts +135 -0
  27. package/dist/bundling-sources/shared_libs/play-runtime/play-input.ts +17 -0
  28. package/dist/bundling-sources/shared_libs/play-runtime/play-latency-trace.ts +0 -3
  29. package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts +10 -1
  30. package/dist/bundling-sources/shared_libs/play-runtime/providers.ts +9 -15
  31. package/dist/bundling-sources/shared_libs/play-runtime/receipt-completion-sink.ts +262 -0
  32. package/dist/bundling-sources/shared_libs/play-runtime/receipt-heartbeat-supervisor.ts +64 -0
  33. package/dist/bundling-sources/shared_libs/play-runtime/receipt-sql.ts +76 -8
  34. package/dist/bundling-sources/shared_libs/play-runtime/receipt-state-sink.ts +59 -0
  35. package/dist/bundling-sources/shared_libs/play-runtime/receipt-status.ts +3 -2
  36. package/dist/bundling-sources/shared_libs/play-runtime/resource-governor.ts +269 -0
  37. package/dist/bundling-sources/shared_libs/play-runtime/run-ledger.ts +45 -3
  38. package/dist/bundling-sources/shared_libs/play-runtime/run-terminal-source.ts +0 -1
  39. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-lifecycle.ts +433 -0
  40. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-payload-transport.ts +227 -0
  41. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts +882 -0
  42. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/local-process.ts +573 -0
  43. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/bundle.ts +264 -0
  44. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/index.ts +33 -0
  45. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/runner-events.ts +35 -0
  46. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/types.ts +53 -0
  47. package/dist/bundling-sources/shared_libs/play-runtime/runtime-actions.ts +0 -1
  48. package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +561 -105
  49. package/dist/bundling-sources/shared_libs/play-runtime/runtime-constants.ts +12 -0
  50. package/dist/bundling-sources/shared_libs/play-runtime/runtime-infra-mode.ts +23 -0
  51. package/dist/bundling-sources/shared_libs/play-runtime/runtime-sheet-attempt-state-machine.ts +23 -2
  52. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backend.ts +16 -24
  53. package/dist/bundling-sources/shared_libs/play-runtime/serial-task-queue.ts +26 -0
  54. package/dist/bundling-sources/shared_libs/play-runtime/submit-limits.ts +1 -1
  55. package/dist/bundling-sources/shared_libs/play-runtime/vercel-protection.ts +81 -0
  56. package/dist/bundling-sources/shared_libs/play-runtime/work-receipt-state-machine.ts +54 -9
  57. package/dist/bundling-sources/shared_libs/play-runtime/work-receipts.ts +17 -3
  58. package/dist/bundling-sources/shared_libs/play-runtime/worker-api-types.ts +0 -20
  59. package/dist/bundling-sources/shared_libs/plays/dataset.ts +105 -4
  60. package/dist/bundling-sources/shared_libs/plays/row-identity.ts +4 -11
  61. package/dist/cli/index.js +167 -54
  62. package/dist/cli/index.mjs +167 -54
  63. package/dist/index.d.mts +1 -1
  64. package/dist/index.d.ts +1 -1
  65. package/dist/index.js +8 -8
  66. package/dist/index.mjs +8 -8
  67. package/dist/plays/bundle-play-file.mjs +15 -20
  68. package/package.json +1 -1
  69. package/dist/bundling-sources/shared_libs/play-runtime/hatchet-cold-execution-diagnosis.ts +0 -321
  70. package/dist/bundling-sources/shared_libs/play-runtime/hatchet-cold-execution-target.ts +0 -158
  71. package/dist/bundling-sources/shared_libs/temporal/constants.ts +0 -39
  72. package/dist/bundling-sources/shared_libs/temporal/preview-config.ts +0 -150
@@ -12,7 +12,7 @@ import {
12
12
  } from './rate-state-backend';
13
13
 
14
14
  /**
15
- * Permits leased per app-runtime round trip for the Hatchet/Node substrate.
15
+ * Permits leased per app-runtime round trip for the direct Node substrate.
16
16
  *
17
17
  * Eight keeps DB/API round trips amortized for batch-heavy maps while limiting
18
18
  * stale-block over-issue to 8 x live runner processes per bucket/window.
@@ -19,11 +19,11 @@ import {
19
19
  resolveExecutionPolicy,
20
20
  resolveRowConcurrency,
21
21
  } from './policy';
22
+ import { type PacingRule, type RateStateBackend } from './rate-state-backend';
22
23
  import {
23
- noopPacingPermit,
24
- type PacingRule,
25
- type RateStateBackend,
26
- } from './rate-state-backend';
24
+ createInMemoryAdaptiveAdmission,
25
+ type RuntimeAdaptiveAdmission,
26
+ } from './adaptive-admission';
27
27
 
28
28
  export interface WorkLease {
29
29
  /** Free the slot / pacing permit. Idempotent. MUST be called in a finally. */
@@ -74,6 +74,25 @@ export type PacingResolver = (
74
74
  toolId: string,
75
75
  ) => Promise<{ provider: string; rules: PacingRule[] } | null>;
76
76
 
77
+ export function defaultPacingForTool(
78
+ toolId: string,
79
+ policy: ResolvedExecutionPolicy,
80
+ ): { provider: string; rules: PacingRule[] } {
81
+ return {
82
+ // Unknown providers still need a stable bucket. Tool id is the only
83
+ // substrate-neutral key available when the runtime has no queue hints.
84
+ provider: `tool:${toolId}`,
85
+ rules: [
86
+ {
87
+ ruleId: `default:${toolId}`,
88
+ requestsPerWindow: policy.pacing.defaultProviderRequestsPerSecond,
89
+ windowMs: 1_000,
90
+ maxConcurrency: null,
91
+ },
92
+ ],
93
+ };
94
+ }
95
+
77
96
  export interface PlayExecutionGovernor {
78
97
  readonly adapter: AdapterId;
79
98
  readonly policy: ResolvedExecutionPolicy;
@@ -132,6 +151,12 @@ export interface PlayExecutionGovernor {
132
151
  retryAfterMs: number;
133
152
  }): void;
134
153
 
154
+ /** Feed clean provider calls back into adaptive admission. */
155
+ observeProviderSuccess(input: {
156
+ provider: string;
157
+ latencyMs?: number | null;
158
+ }): void;
159
+
135
160
  snapshot(): GovernanceSnapshot;
136
161
  }
137
162
 
@@ -140,6 +165,7 @@ interface GovernorInput {
140
165
  scope: { orgId: string; rootRunId: string };
141
166
  rateState: RateStateBackend;
142
167
  resolvePacing: PacingResolver;
168
+ adaptiveAdmission?: RuntimeAdaptiveAdmission;
143
169
  resume?: GovernanceSnapshot;
144
170
  }
145
171
 
@@ -226,35 +252,79 @@ export function createPlayExecutionGovernor(
226
252
  const rowSlots = new Semaphore(policy.concurrency.rowMax);
227
253
  const childPlaySlots = new Semaphore(policy.concurrency.childPlay);
228
254
  const toolSlots = new Semaphore(policy.concurrency.toolCalls);
255
+ const adaptiveAdmission =
256
+ input.adaptiveAdmission ??
257
+ createInMemoryAdaptiveAdmission({
258
+ initialRequestsPerSecond: policy.pacing.defaultProviderRequestsPerSecond,
259
+ maxRequestsPerSecond: policy.pacing.suggestedMaxParallelism,
260
+ });
229
261
 
230
262
  const bucketId = (provider: string) => `${input.scope.orgId}:${provider}`;
231
263
 
264
+ async function resolveAdaptivePacing(toolId: string): Promise<{
265
+ provider: string;
266
+ rules: PacingRule[];
267
+ }> {
268
+ const resolvedPacing = await input.resolvePacing(toolId);
269
+ const declaredPacing =
270
+ resolvedPacing && resolvedPacing.rules.length > 0 ? resolvedPacing : null;
271
+ return adaptiveAdmission.resolvePacing({
272
+ orgId: input.scope.orgId,
273
+ toolId,
274
+ declaredPacing,
275
+ fallbackPacing: defaultPacingForTool(toolId, policy),
276
+ });
277
+ }
278
+
232
279
  function chargeBudget(kind: BudgetKind, amount = 1): void {
233
280
  switch (kind) {
234
281
  case 'playCall':
235
282
  state.playCallCount += amount;
236
283
  if (state.playCallCount > policy.budgets.maxPlayCallCount)
237
- throw new GovernorBudgetError('playCall', state.playCallCount, policy.budgets.maxPlayCallCount);
284
+ throw new GovernorBudgetError(
285
+ 'playCall',
286
+ state.playCallCount,
287
+ policy.budgets.maxPlayCallCount,
288
+ );
238
289
  return;
239
290
  case 'toolCall':
240
291
  state.toolCallCount += amount;
241
292
  if (state.toolCallCount > policy.budgets.maxToolCallCount)
242
- throw new GovernorBudgetError('toolCall', state.toolCallCount, policy.budgets.maxToolCallCount);
293
+ throw new GovernorBudgetError(
294
+ 'toolCall',
295
+ state.toolCallCount,
296
+ policy.budgets.maxToolCallCount,
297
+ );
243
298
  return;
244
299
  case 'retry':
245
300
  state.retryCount += amount;
246
301
  if (state.retryCount > policy.budgets.maxRetryCount)
247
- throw new GovernorBudgetError('retry', state.retryCount, policy.budgets.maxRetryCount);
302
+ throw new GovernorBudgetError(
303
+ 'retry',
304
+ state.retryCount,
305
+ policy.budgets.maxRetryCount,
306
+ );
248
307
  return;
249
308
  case 'descendant':
250
309
  state.descendantCount += amount;
251
310
  if (state.descendantCount > policy.budgets.maxDescendants)
252
- throw new GovernorBudgetError('descendant', state.descendantCount, policy.budgets.maxDescendants);
311
+ throw new GovernorBudgetError(
312
+ 'descendant',
313
+ state.descendantCount,
314
+ policy.budgets.maxDescendants,
315
+ );
253
316
  return;
254
317
  case 'waterfallStep':
255
318
  state.waterfallStepExecutions += amount;
256
- if (state.waterfallStepExecutions > policy.budgets.maxWaterfallStepExecutions)
257
- throw new GovernorBudgetError('waterfallStep', state.waterfallStepExecutions, policy.budgets.maxWaterfallStepExecutions);
319
+ if (
320
+ state.waterfallStepExecutions >
321
+ policy.budgets.maxWaterfallStepExecutions
322
+ )
323
+ throw new GovernorBudgetError(
324
+ 'waterfallStep',
325
+ state.waterfallStepExecutions,
326
+ policy.budgets.maxWaterfallStepExecutions,
327
+ );
258
328
  return;
259
329
  }
260
330
  }
@@ -273,15 +343,12 @@ export function createPlayExecutionGovernor(
273
343
  // resolver, so callers only need the toolId. No rules → no pacing.
274
344
  let permit: { release(): void };
275
345
  try {
276
- const pacing = await input.resolvePacing(toolId);
277
- permit =
278
- pacing && pacing.rules.length > 0
279
- ? await input.rateState.acquire({
280
- bucketId: bucketId(pacing.provider),
281
- rules: pacing.rules,
282
- signal: opts?.signal,
283
- })
284
- : noopPacingPermit();
346
+ const pacing = await resolveAdaptivePacing(toolId);
347
+ permit = await input.rateState.acquire({
348
+ bucketId: bucketId(pacing.provider),
349
+ rules: pacing.rules,
350
+ signal: opts?.signal,
351
+ });
285
352
  } catch (error) {
286
353
  slot.release();
287
354
  throw error;
@@ -307,8 +374,7 @@ export function createPlayExecutionGovernor(
307
374
  },
308
375
 
309
376
  async suggestedParallelism(toolId, fallback) {
310
- const pacing = await input.resolvePacing(toolId);
311
- if (!pacing || pacing.rules.length === 0) return fallback;
377
+ const pacing = await resolveAdaptivePacing(toolId);
312
378
  const limits = pacing.rules.flatMap((rule) =>
313
379
  rule.maxConcurrency != null
314
380
  ? [rule.requestsPerWindow, rule.maxConcurrency]
@@ -324,16 +390,26 @@ export function createPlayExecutionGovernor(
324
390
 
325
391
  forkChild(childInput) {
326
392
  if (state.ancestryPlayIds.includes(childInput.childPlayName)) {
327
- const chain = [...state.ancestryPlayIds, childInput.childPlayName].join(' -> ');
393
+ const chain = [...state.ancestryPlayIds, childInput.childPlayName].join(
394
+ ' -> ',
395
+ );
328
396
  throw new Error(`Recursive play graph detected: ${chain}`);
329
397
  }
330
398
  const nextDepth = state.callDepth + 1;
331
399
  if (nextDepth > policy.budgets.maxPlayCallDepth)
332
- throw new GovernorBudgetError('playDepth', nextDepth, policy.budgets.maxPlayCallDepth);
400
+ throw new GovernorBudgetError(
401
+ 'playDepth',
402
+ nextDepth,
403
+ policy.budgets.maxPlayCallDepth,
404
+ );
333
405
  const parentKey = state.currentPlayId;
334
406
  const nextParent = (state.parentChildCalls[parentKey] ?? 0) + 1;
335
407
  if (nextParent > policy.budgets.maxChildPlayCallsPerParent)
336
- throw new GovernorBudgetError('childPerParent', nextParent, policy.budgets.maxChildPlayCallsPerParent);
408
+ throw new GovernorBudgetError(
409
+ 'childPerParent',
410
+ nextParent,
411
+ policy.budgets.maxChildPlayCallsPerParent,
412
+ );
337
413
  // Charge the run-wide play/descendant budgets on the parent. Charged at
338
414
  // fork time (not after the caller's child-play slot acquire) and never
339
415
  // refunded if that acquire fails — see the forkChild interface doc.
@@ -357,15 +433,29 @@ export function createPlayExecutionGovernor(
357
433
  };
358
434
  },
359
435
 
360
- resolveRowConcurrency: (requested) => resolveRowConcurrency(policy, requested),
436
+ resolveRowConcurrency: (requested) =>
437
+ resolveRowConcurrency(policy, requested),
361
438
 
362
439
  reportProviderBackpressure(bp) {
440
+ adaptiveAdmission.observeProviderBackpressure({
441
+ orgId: input.scope.orgId,
442
+ provider: bp.provider,
443
+ retryAfterMs: bp.retryAfterMs,
444
+ });
363
445
  input.rateState.penalize({
364
446
  bucketId: bucketId(bp.provider),
365
447
  cooldownMs: bp.retryAfterMs,
366
448
  });
367
449
  },
368
450
 
451
+ observeProviderSuccess(success) {
452
+ adaptiveAdmission.observeProviderSuccess({
453
+ orgId: input.scope.orgId,
454
+ provider: success.provider,
455
+ latencyMs: success.latencyMs,
456
+ });
457
+ },
458
+
369
459
  snapshot: () => ({
370
460
  ...state,
371
461
  ancestryPlayIds: [...state.ancestryPlayIds],
@@ -8,12 +8,12 @@
8
8
  * CONTEXT.md.
9
9
  *
10
10
  * Tuning philosophy (product decision):
11
- * - Caps are GENEROUS and tuned for fast execution. They are runaway /
12
- * anti-starvation guards, NOT per-workload throttles. A legitimate large run
13
- * should never hit them.
14
- * - The real-time bounds on a run are (a) the wall-clock runtime cap and
15
- * (b) per-provider rate pacing. The counters below only stop pathological
16
- * recursion / fan-out.
11
+ * - Caps are explicit resource admission boundaries. A run should queue behind
12
+ * them inside the executor instead of manufacturing thousands of promises,
13
+ * receipts, sockets, or row payload copies up front.
14
+ * - The real-time bounds on outbound work are still per-provider pacing and
15
+ * provider latency. Row concurrency exists to keep the sandbox healthy while
16
+ * enough work is admitted to saturate normal provider RPS.
17
17
  * - Every value is an EXPLICIT, finite cap. Nothing here is unbounded — there
18
18
  * is no `null`, no "off", no implicit infinity. Unaccounted resource use
19
19
  * must be impossible.
@@ -90,15 +90,15 @@ export interface ResolvedExecutionPolicy {
90
90
  */
91
91
  export const SHARED_EXECUTION_POLICY: ResolvedExecutionPolicy = {
92
92
  concurrency: {
93
- // Map row concurrency is platform-controlled customers cannot set it — so
94
- // every map runs at the ceiling: high enough that per-provider pacing (not
95
- // the row pool) is the bottleneck even for multi-step waterfall rows. Pure-JS
96
- // maps use a separate fast path and are not bound by this. Kept == rowMax so
97
- // the default and the cap are the same single platform value.
98
- rowDefault: 2_000,
99
- // Hard cap. Above this, more in-flight rows only park memory — outbound is
100
- // already bounded by tool concurrency + per-provider pacing.
101
- rowMax: 2_000,
93
+ // Logical row target. A 1,000-row window can fill five 200-item provider
94
+ // batches or keep a 100-RPS unbatched provider fed; the runtime's shared
95
+ // resident-byte admission calculator clamps fat rows below this target.
96
+ // Pending rows remain indexes, not promises or pre-claimed receipts.
97
+ rowDefault: 1_000,
98
+ // Finite global ceiling across concurrent maps in one run. This intentionally
99
+ // matches the default: throughput grows through provider pacing/batching,
100
+ // while row payload size can only reduce admission through the byte budget.
101
+ rowMax: 1_000,
102
102
  // Concurrent child-play launches. Generous; each child is a real launch, so
103
103
  // this is the one value most likely to need a documented esm_workers
104
104
  // override if isolate pressure shows up in E2E.
@@ -121,13 +121,13 @@ export const SHARED_EXECUTION_POLICY: ResolvedExecutionPolicy = {
121
121
  pacing: {
122
122
  // Undeclared providers; declared providers (rate-limit-definitions.ts) win.
123
123
  defaultProviderRequestsPerSecond: 10,
124
- suggestedMaxParallelism: 50,
124
+ suggestedMaxParallelism: 256,
125
125
  // Worker isolate pacing knobs. The budget module owns platform accounting;
126
126
  // the Governor owns the policy values that decide when work is chunked.
127
127
  workerYieldElapsedMs: 45_000,
128
128
  // Fallback wave width for batched tool chunks that carry no provider pacing
129
129
  // hint. This is only a floor for the undeclared case: it stays bounded by
130
- // suggestedMaxParallelism (50) above, and declared providers still get
130
+ // suggestedMaxParallelism above, and declared providers still get
131
131
  // tightened by their per-provider pacing (rate-limit-definitions.ts). Raised
132
132
  // 4 -> 16 so full batches fan out wide enough to close the map-throughput gap
133
133
  // without over-running providers that actually declare limits.
@@ -50,3 +50,25 @@ export function runtimeLeaseHeartbeatIntervalMs(ttlMs: number): number {
50
50
  }
51
51
  return Math.max(1, Math.floor(ttlMs / PLAY_RUNTIME_LEASE_HEARTBEAT_DIVISOR));
52
52
  }
53
+
54
+ /**
55
+ * The receipt store is authoritative for lease duration. Executors may not
56
+ * inherit the coordinator's environment, so derive renewal cadence from the
57
+ * expiry returned with the actual claim instead of assuming a local TTL.
58
+ */
59
+ export function runtimeLeaseHeartbeatIntervalFromExpiry(input: {
60
+ leaseExpiresAt?: string | null;
61
+ fallbackTtlMs: number;
62
+ nowMs?: number;
63
+ }): number {
64
+ const expiresAtMs = input.leaseExpiresAt
65
+ ? Date.parse(input.leaseExpiresAt)
66
+ : Number.NaN;
67
+ const nowMs = input.nowMs ?? Date.now();
68
+ const remainingMs = expiresAtMs - nowMs;
69
+ return runtimeLeaseHeartbeatIntervalMs(
70
+ Number.isFinite(remainingMs) && remainingMs > 0
71
+ ? remainingMs
72
+ : input.fallbackTtlMs,
73
+ );
74
+ }
@@ -1,6 +1,7 @@
1
1
  import { createSecretRedactionContext } from './secret-redaction';
2
+ import { RUNTIME_RECEIPT_OUTPUT_MAX_BYTES } from './output-size-limits';
2
3
 
3
- export const MAX_LEDGER_RESULT_PAYLOAD_BYTES = 2_500_000;
4
+ export const MAX_LEDGER_RESULT_PAYLOAD_BYTES = RUNTIME_RECEIPT_OUTPUT_MAX_BYTES;
4
5
  export const MAX_LEDGER_LOG_LINES_PER_EVENT = 500;
5
6
  export const MAX_LEDGER_LOG_LINE_LENGTH = 2_000;
6
7
 
@@ -1,6 +1,5 @@
1
1
  export type PlayLiveEventSource =
2
2
  | 'worker'
3
- | 'temporal'
4
3
  | 'convex'
5
4
  | 'play'
6
5
  | 'system';
@@ -0,0 +1,265 @@
1
+ export const NODE_RUNTIME_MAP_MEMORY_BUDGET_BYTES = 512 * 1024 * 1024;
2
+ export const NODE_RUNTIME_MAP_ACTIVE_ROWS_MEMORY_BUDGET_BYTES =
3
+ 384 * 1024 * 1024;
4
+ export const NODE_RUNTIME_MAP_MATERIALIZED_MEMORY_MULTIPLIER = 4;
5
+ export const NODE_RUNTIME_MAP_ACTIVE_ROW_MEMORY_MULTIPLIER = 3;
6
+ export const NODE_RUNTIME_MAP_ROW_OVERHEAD_BYTES = 512;
7
+
8
+ export type RuntimeMapMemoryLimits = {
9
+ materializedBudgetBytes: number;
10
+ activeRowsBudgetBytes: number;
11
+ };
12
+
13
+ export type RuntimeMapMemoryEstimate = {
14
+ rowCount: number;
15
+ serializedBytes: number;
16
+ estimatedResidentBytes: number;
17
+ largestRowBytes: number;
18
+ };
19
+
20
+ export type RuntimeMapRowAdmission = {
21
+ concurrency: number;
22
+ estimatedBytesPerActiveRow: number;
23
+ estimatedResidentBytes: number;
24
+ };
25
+
26
+ export class RuntimeMapMemoryLimitError extends Error {
27
+ readonly code = 'RUNTIME_MAP_MEMORY_LIMIT_EXCEEDED';
28
+ constructor(input: {
29
+ mapName: string;
30
+ phase: 'materialize' | 'active_rows' | 'retained_rows';
31
+ estimate: RuntimeMapMemoryEstimate;
32
+ budgetBytes: number;
33
+ activeRows?: number;
34
+ rowConcurrency?: number;
35
+ }) {
36
+ const active =
37
+ input.activeRows === undefined
38
+ ? ''
39
+ : ` activeRows=${input.activeRows}` +
40
+ (input.rowConcurrency === undefined
41
+ ? ''
42
+ : ` rowConcurrency=${input.rowConcurrency}`);
43
+ const stopPoint =
44
+ input.phase === 'materialize'
45
+ ? ' before provider calls'
46
+ : input.phase === 'active_rows'
47
+ ? ' before row resolver bodies start'
48
+ : '';
49
+ super(
50
+ `Runtime map memory limit exceeded for ctx.dataset("${input.mapName}") during ${input.phase}: ` +
51
+ `estimatedResidentBytes=${input.estimate.estimatedResidentBytes} ` +
52
+ `budgetBytes=${input.budgetBytes} rowCount=${input.estimate.rowCount} ` +
53
+ `serializedBytes=${input.estimate.serializedBytes} largestRowBytes=${input.estimate.largestRowBytes}${active}. ` +
54
+ `This run would risk sandbox OOM. Deepline stops here${stopPoint} instead of silently lowering concurrency. ` +
55
+ 'Reduce row payload size, stage bulky blobs, or split the input into smaller runs.',
56
+ );
57
+ this.name = 'RuntimeMapMemoryLimitError';
58
+ }
59
+ }
60
+
61
+ const textEncoder =
62
+ typeof TextEncoder === 'undefined' ? null : new TextEncoder();
63
+
64
+ export function runtimeMapJsonByteLength(value: unknown): number {
65
+ const serialized = JSON.stringify(value);
66
+ if (serialized === undefined) return 0;
67
+ return textEncoder?.encode(serialized).byteLength ?? serialized.length;
68
+ }
69
+
70
+ export function defaultRuntimeMapMemoryLimits(): RuntimeMapMemoryLimits {
71
+ return {
72
+ materializedBudgetBytes: NODE_RUNTIME_MAP_MEMORY_BUDGET_BYTES,
73
+ activeRowsBudgetBytes: NODE_RUNTIME_MAP_ACTIVE_ROWS_MEMORY_BUDGET_BYTES,
74
+ };
75
+ }
76
+
77
+ export function resolveRuntimeMapMemoryLimits(
78
+ override?: Partial<RuntimeMapMemoryLimits> | null,
79
+ ): RuntimeMapMemoryLimits {
80
+ const defaults = defaultRuntimeMapMemoryLimits();
81
+ return {
82
+ materializedBudgetBytes: positiveBudget(
83
+ override?.materializedBudgetBytes,
84
+ defaults.materializedBudgetBytes,
85
+ ),
86
+ activeRowsBudgetBytes: positiveBudget(
87
+ override?.activeRowsBudgetBytes,
88
+ defaults.activeRowsBudgetBytes,
89
+ ),
90
+ };
91
+ }
92
+
93
+ function positiveBudget(value: number | undefined, fallback: number): number {
94
+ if (!Number.isFinite(value) || value === undefined) return fallback;
95
+ return Math.max(1, Math.floor(value));
96
+ }
97
+
98
+ export function estimateRuntimeMapMaterializedMemory(input: {
99
+ rowCount: number;
100
+ serializedBytes: number;
101
+ largestRowBytes: number;
102
+ }): RuntimeMapMemoryEstimate {
103
+ return {
104
+ rowCount: input.rowCount,
105
+ serializedBytes: input.serializedBytes,
106
+ largestRowBytes: input.largestRowBytes,
107
+ estimatedResidentBytes:
108
+ input.serializedBytes * NODE_RUNTIME_MAP_MATERIALIZED_MEMORY_MULTIPLIER +
109
+ input.rowCount * NODE_RUNTIME_MAP_ROW_OVERHEAD_BYTES,
110
+ };
111
+ }
112
+
113
+ export function estimateRuntimeMapRowsMemory(
114
+ rows: readonly unknown[],
115
+ ): RuntimeMapMemoryEstimate {
116
+ let serializedBytes = 0;
117
+ let largestRowBytes = 0;
118
+ for (const row of rows) {
119
+ const rowBytes = runtimeMapJsonByteLength(row);
120
+ serializedBytes += rowBytes;
121
+ largestRowBytes = Math.max(largestRowBytes, rowBytes);
122
+ }
123
+ return estimateRuntimeMapMaterializedMemory({
124
+ rowCount: rows.length,
125
+ serializedBytes,
126
+ largestRowBytes,
127
+ });
128
+ }
129
+
130
+ type RuntimeMapRowsTrackerInput = {
131
+ mapName: string;
132
+ budgetBytes?: number;
133
+ };
134
+
135
+ type RuntimeMapRowsTracker = {
136
+ track(row: unknown): void;
137
+ estimate(): RuntimeMapMemoryEstimate;
138
+ };
139
+
140
+ function createRuntimeMapRowsTracker(
141
+ input: RuntimeMapRowsTrackerInput,
142
+ phase: 'materialize' | 'retained_rows',
143
+ ): RuntimeMapRowsTracker {
144
+ const budgetBytes = positiveBudget(
145
+ input.budgetBytes,
146
+ NODE_RUNTIME_MAP_MEMORY_BUDGET_BYTES,
147
+ );
148
+ let rowCount = 0;
149
+ let serializedBytes = 0;
150
+ let largestRowBytes = 0;
151
+ const estimate = () =>
152
+ estimateRuntimeMapMaterializedMemory({
153
+ rowCount,
154
+ serializedBytes,
155
+ largestRowBytes,
156
+ });
157
+ return {
158
+ track(row) {
159
+ const rowBytes = runtimeMapJsonByteLength(row);
160
+ rowCount += 1;
161
+ serializedBytes += rowBytes;
162
+ largestRowBytes = Math.max(largestRowBytes, rowBytes);
163
+ const current = estimate();
164
+ if (current.estimatedResidentBytes > budgetBytes) {
165
+ throw new RuntimeMapMemoryLimitError({
166
+ mapName: input.mapName,
167
+ phase,
168
+ estimate: current,
169
+ budgetBytes,
170
+ });
171
+ }
172
+ },
173
+ estimate,
174
+ };
175
+ }
176
+
177
+ export function createRuntimeMapMaterializationTracker(
178
+ input: RuntimeMapRowsTrackerInput,
179
+ ): RuntimeMapRowsTracker {
180
+ return createRuntimeMapRowsTracker(input, 'materialize');
181
+ }
182
+
183
+ export function createRuntimeMapRetainedRowsTracker(
184
+ input: RuntimeMapRowsTrackerInput,
185
+ ): RuntimeMapRowsTracker {
186
+ return createRuntimeMapRowsTracker(input, 'retained_rows');
187
+ }
188
+
189
+ export function assertRuntimeMapActiveRowsWithinMemoryLimit(input: {
190
+ mapName: string;
191
+ rowCount: number;
192
+ rowConcurrency: number;
193
+ largestRowBytes: number;
194
+ activeRowsBudgetBytes?: number;
195
+ }): void {
196
+ const activeRows = Math.min(
197
+ Math.max(0, input.rowCount),
198
+ Math.max(1, Math.floor(input.rowConcurrency)),
199
+ );
200
+ const estimate: RuntimeMapMemoryEstimate = {
201
+ rowCount: input.rowCount,
202
+ serializedBytes: input.largestRowBytes * activeRows,
203
+ largestRowBytes: input.largestRowBytes,
204
+ estimatedResidentBytes:
205
+ input.largestRowBytes *
206
+ activeRows *
207
+ NODE_RUNTIME_MAP_ACTIVE_ROW_MEMORY_MULTIPLIER +
208
+ activeRows * NODE_RUNTIME_MAP_ROW_OVERHEAD_BYTES,
209
+ };
210
+ const budgetBytes = positiveBudget(
211
+ input.activeRowsBudgetBytes,
212
+ NODE_RUNTIME_MAP_ACTIVE_ROWS_MEMORY_BUDGET_BYTES,
213
+ );
214
+ if (estimate.estimatedResidentBytes > budgetBytes) {
215
+ throw new RuntimeMapMemoryLimitError({
216
+ mapName: input.mapName,
217
+ phase: 'active_rows',
218
+ estimate,
219
+ budgetBytes,
220
+ activeRows,
221
+ rowConcurrency: input.rowConcurrency,
222
+ });
223
+ }
224
+ }
225
+
226
+ /**
227
+ * Resolve a row-worker window from the shared concurrency target and a concrete
228
+ * resident-byte budget. Pending rows remain indexes, so only this admitted
229
+ * window contributes active row copies/promises. If one row cannot fit, fail
230
+ * before any resolver body starts; otherwise memory pressure is normal
231
+ * backpressure and reduces the window explicitly.
232
+ */
233
+ export function resolveRuntimeMapRowAdmission(input: {
234
+ rowCount: number;
235
+ requestedConcurrency: number;
236
+ largestRowBytes: number;
237
+ activeRowsBudgetBytes?: number;
238
+ }): RuntimeMapRowAdmission {
239
+ const rowCount = Math.max(0, Math.floor(input.rowCount));
240
+ const requestedConcurrency = Math.max(
241
+ 1,
242
+ Math.floor(input.requestedConcurrency),
243
+ );
244
+ const largestRowBytes = Math.max(0, Math.floor(input.largestRowBytes));
245
+ const budgetBytes = positiveBudget(
246
+ input.activeRowsBudgetBytes,
247
+ NODE_RUNTIME_MAP_ACTIVE_ROWS_MEMORY_BUDGET_BYTES,
248
+ );
249
+ const estimatedBytesPerActiveRow =
250
+ largestRowBytes * NODE_RUNTIME_MAP_ACTIVE_ROW_MEMORY_MULTIPLIER +
251
+ NODE_RUNTIME_MAP_ROW_OVERHEAD_BYTES;
252
+
253
+ const maxConcurrencyByMemory = Math.floor(
254
+ budgetBytes / Math.max(1, estimatedBytesPerActiveRow),
255
+ );
256
+ const concurrency =
257
+ rowCount === 0
258
+ ? 0
259
+ : Math.min(rowCount, requestedConcurrency, maxConcurrencyByMemory);
260
+ return {
261
+ concurrency,
262
+ estimatedBytesPerActiveRow,
263
+ estimatedResidentBytes: estimatedBytesPerActiveRow * concurrency,
264
+ };
265
+ }