deepline 0.3.128 → 0.3.129
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/sdk/src/client.ts +34 -9
- package/dist/bundling-sources/sdk/src/http.ts +22 -0
- package/dist/bundling-sources/sdk/src/release.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-data-plane/r2.ts +22 -0
- package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +265 -2
- package/dist/bundling-sources/shared_libs/play-runtime/bounded-batch-function.ts +682 -0
- package/dist/bundling-sources/shared_libs/play-runtime/child-execution-strategy.ts +21 -5
- package/dist/bundling-sources/shared_libs/play-runtime/context.ts +5390 -1732
- package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +107 -1
- package/dist/bundling-sources/shared_libs/play-runtime/deployment/commands/roll-execution-relay.ts +7 -0
- package/dist/bundling-sources/shared_libs/play-runtime/deployment/fly-machines.ts +229 -7
- package/dist/bundling-sources/shared_libs/play-runtime/deployment/fly-registry.ts +25 -4
- package/dist/bundling-sources/shared_libs/play-runtime/deployment/runtime-gateway-capacity.ts +71 -0
- package/dist/bundling-sources/shared_libs/play-runtime/deployment/runtime-gateway-drain.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/deployment/runtime-topology.ts +4 -1
- package/dist/bundling-sources/shared_libs/play-runtime/durable-receipt-execution.ts +694 -28
- package/dist/bundling-sources/shared_libs/play-runtime/egress-bandwidth-probe.ts +167 -0
- package/dist/bundling-sources/shared_libs/play-runtime/execute-server-timing.ts +103 -0
- package/dist/bundling-sources/shared_libs/play-runtime/execution-gateway-receipt-output-reference.ts +34 -0
- package/dist/bundling-sources/shared_libs/play-runtime/execution-gateway-response-size.ts +27 -0
- package/dist/bundling-sources/shared_libs/play-runtime/gateway-postgres-admission.ts +66 -1
- package/dist/bundling-sources/shared_libs/play-runtime/governor/app-runtime-rate-state-backend.ts +425 -71
- package/dist/bundling-sources/shared_libs/play-runtime/governor/governor.ts +210 -32
- package/dist/bundling-sources/shared_libs/play-runtime/governor/in-memory-rate-state-backend.ts +27 -13
- package/dist/bundling-sources/shared_libs/play-runtime/governor/policy.ts +19 -6
- package/dist/bundling-sources/shared_libs/play-runtime/governor/rate-state-backend.ts +61 -2
- package/dist/bundling-sources/shared_libs/play-runtime/integration-operation-reader-fence/contracts.ts +33 -1
- package/dist/bundling-sources/shared_libs/play-runtime/integration-operation-reader-fence/job.ts +42 -1
- package/dist/bundling-sources/shared_libs/play-runtime/launch-source-policy.ts +11 -11
- package/dist/bundling-sources/shared_libs/play-runtime/ledger-safe-payload.ts +49 -16
- package/dist/bundling-sources/shared_libs/play-runtime/map-memory-limits.ts +1653 -98
- package/dist/bundling-sources/shared_libs/play-runtime/memory-ledger.ts +807 -0
- package/dist/bundling-sources/shared_libs/play-runtime/memory-pressure-signal.ts +138 -0
- package/dist/bundling-sources/shared_libs/play-runtime/output-size-limits.ts +85 -3
- package/dist/bundling-sources/shared_libs/play-runtime/persistence-latch.ts +60 -2
- package/dist/bundling-sources/shared_libs/play-runtime/play-run-recovery-policy.ts +46 -1
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/terminal/contracts.ts +2 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/terminal/job.ts +1 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/terminal/ports.ts +1 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/terminal/postgres-adapter.ts +1 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runtime-semantics.ts +70 -27
- package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts +56 -0
- package/dist/bundling-sources/shared_libs/play-runtime/provider-capacity/adaptive-provider-capacity-controller.ts +192 -0
- package/dist/bundling-sources/shared_libs/play-runtime/provider-capacity/epoch-aimd-rate-controller.ts +194 -0
- package/dist/bundling-sources/shared_libs/play-runtime/provider-capacity/fixture-capacity-simulator.ts +220 -0
- package/dist/bundling-sources/shared_libs/play-runtime/provider-capacity/gradient2-concurrency-controller.ts +214 -0
- package/dist/bundling-sources/shared_libs/play-runtime/provider-capacity/provider-capacity-policy.ts +43 -0
- package/dist/bundling-sources/shared_libs/play-runtime/provider-capacity/provider-concurrency-policy.ts +30 -0
- package/dist/bundling-sources/shared_libs/play-runtime/provider-capacity/provider-rate-policy.ts +95 -0
- package/dist/bundling-sources/shared_libs/play-runtime/provider-capacity/provider-throttle-classifier.ts +40 -0
- package/dist/bundling-sources/shared_libs/play-runtime/resource-governor.ts +1535 -11
- package/dist/bundling-sources/shared_libs/play-runtime/run-failure.ts +49 -2
- package/dist/bundling-sources/shared_libs/play-runtime/run-ledger.ts +6 -1
- package/dist/bundling-sources/shared_libs/play-runtime/run-log-buffer.ts +55 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-app/egress-probe.ts +221 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-app/index.ts +339 -29
- package/dist/bundling-sources/shared_libs/play-runtime/runner-app/runtime-deadline.ts +28 -1
- package/dist/bundling-sources/shared_libs/play-runtime/runner-app/runtime-progress-push.ts +192 -165
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-lifecycle.ts +6 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-payload-transport.ts +90 -5
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-runtime-watchdog.ts +281 -6
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts +27 -1
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/local-process.ts +19 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/modal.ts +108 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/runtime-sandbox-reconciliation.ts +40 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/types.ts +11 -5
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-actions.ts +30 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-api-compression.ts +276 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +723 -115
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-capacity-policy.ts +98 -2
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-contract.ts +3 -2
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-environment.ts +41 -4
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-flow.ts +448 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-map-row-feed.ts +215 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-operation-contract.ts +11 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-receipt-store-adapter.ts +161 -14
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-receipt-value.ts +184 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-receipt-writer.ts +607 -121
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-sandbox-placement-policy.ts +4 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-sheet-row-writer.ts +21 -0
- package/dist/bundling-sources/shared_libs/play-runtime/sandbox-runs/contract.ts +15 -2
- package/dist/bundling-sources/shared_libs/play-runtime/sandbox-runs/runner-backend-adapter.ts +71 -14
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-play-deploy-task.ts +2 -2
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-runtime-worker.ts +288 -119
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-shared.ts +8 -6
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-worker-required-env.ts +28 -3
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/canonical-integration-operation-store.ts +179 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/gateway-heavy-request-admission.ts +80 -2
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/gateway-progress-registry.ts +31 -2
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/integration-operation-store-contract.ts +104 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/integration-operation-value-reference.ts +47 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/integration-operation-value-retirement.ts +193 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/integration-operation-value-storage.ts +420 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-pool.ts +41 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-progress.ts +134 -3
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-rate-state.ts +1657 -95
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts +885 -73
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/receipt-completion-coalescer.ts +452 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/receipt-gateway.ts +601 -42
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/run-terminal-transition.ts +1 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-receipt-value-storage.ts +371 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/worker-resource-metrics.ts +86 -7
- package/dist/bundling-sources/shared_libs/play-runtime/secret-redaction.ts +153 -22
- package/dist/bundling-sources/shared_libs/play-runtime/tool-call/index.ts +1 -0
- package/dist/bundling-sources/shared_libs/play-runtime/tool-call/receipt-cohort.ts +188 -38
- package/dist/bundling-sources/shared_libs/play-runtime/tool-call/runtime-step-receipts-adapter.ts +28 -13
- package/dist/bundling-sources/shared_libs/play-runtime/tool-execute-retry-policy.ts +44 -1
- package/dist/bundling-sources/shared_libs/play-runtime/tool-result.ts +56 -4
- package/dist/bundling-sources/shared_libs/play-runtime/work-receipts.ts +2 -0
- package/dist/bundling-sources/shared_libs/plays/authoring-contract.ts +16 -10
- package/dist/bundling-sources/shared_libs/plays/dataset.ts +4 -1
- package/dist/cli/index.js +250 -38
- package/dist/cli/index.mjs +250 -38
- package/dist/{compiler-manifest-DfSnbCTW.d.mts → compiler-manifest-BIqRyj5m.d.mts} +1 -1
- package/dist/{compiler-manifest-DfSnbCTW.d.ts → compiler-manifest-BIqRyj5m.d.ts} +1 -1
- package/dist/index.d.mts +4 -2
- package/dist/index.d.ts +4 -2
- package/dist/index.js +231 -35
- package/dist/index.mjs +231 -35
- package/dist/install-integrity.json +31 -2
- package/dist/plays/bundle-play-file.d.mts +2 -2
- package/dist/plays/bundle-play-file.d.ts +2 -2
- package/dist/plays/bundle-play-file.mjs +65 -1
- package/dist/release.d.mts +1 -1
- package/dist/release.d.ts +1 -1
- package/dist/release.js +1 -1
- package/dist/release.mjs +1 -1
- package/package.json +1 -1
|
@@ -1,13 +1,32 @@
|
|
|
1
|
+
import { readFileSync } from 'node:fs';
|
|
2
|
+
import { availableParallelism } from 'node:os';
|
|
3
|
+
import {
|
|
4
|
+
PerformanceObserver,
|
|
5
|
+
constants as perfConstants,
|
|
6
|
+
} from 'node:perf_hooks';
|
|
7
|
+
import { getHeapStatistics } from 'node:v8';
|
|
8
|
+
|
|
9
|
+
import type {
|
|
10
|
+
RuntimeMemoryLedger,
|
|
11
|
+
RuntimeMemoryLedgerCategory,
|
|
12
|
+
RuntimeMemoryLedgerSnapshot,
|
|
13
|
+
} from './memory-ledger';
|
|
14
|
+
import {
|
|
15
|
+
RUNTIME_MEMORY_LEDGER_CATEGORIES,
|
|
16
|
+
RUNTIME_MEMORY_LEDGER_INITIAL_FACTOR,
|
|
17
|
+
runtimeJsonByteLength,
|
|
18
|
+
} from './memory-ledger';
|
|
19
|
+
import {
|
|
20
|
+
onMemoryPressure as subscribeRuntimeMemoryPressure,
|
|
21
|
+
reportMemoryLedger as publishRuntimeMemoryLedger,
|
|
22
|
+
} from './memory-pressure-signal';
|
|
23
|
+
|
|
1
24
|
export const NODE_RUNTIME_MAP_MEMORY_BUDGET_BYTES = 512 * 1024 * 1024;
|
|
2
|
-
export const NODE_RUNTIME_MAP_ACTIVE_ROWS_MEMORY_BUDGET_BYTES =
|
|
3
|
-
384 * 1024 * 1024;
|
|
4
25
|
export const NODE_RUNTIME_MAP_MATERIALIZED_MEMORY_MULTIPLIER = 4;
|
|
5
|
-
export const NODE_RUNTIME_MAP_ACTIVE_ROW_MEMORY_MULTIPLIER = 3;
|
|
6
26
|
export const NODE_RUNTIME_MAP_ROW_OVERHEAD_BYTES = 512;
|
|
7
27
|
|
|
8
28
|
export type RuntimeMapMemoryLimits = {
|
|
9
29
|
materializedBudgetBytes: number;
|
|
10
|
-
activeRowsBudgetBytes: number;
|
|
11
30
|
};
|
|
12
31
|
|
|
13
32
|
export type RuntimeMapMemoryEstimate = {
|
|
@@ -17,12 +36,1633 @@ export type RuntimeMapMemoryEstimate = {
|
|
|
17
36
|
largestRowBytes: number;
|
|
18
37
|
};
|
|
19
38
|
|
|
20
|
-
export
|
|
39
|
+
export const RUNTIME_GC_KINDS = [
|
|
40
|
+
'minor',
|
|
41
|
+
'major',
|
|
42
|
+
'incremental',
|
|
43
|
+
'weakcb',
|
|
44
|
+
'other',
|
|
45
|
+
] as const;
|
|
46
|
+
export type RuntimeGcKind = (typeof RUNTIME_GC_KINDS)[number];
|
|
47
|
+
|
|
48
|
+
export type RuntimeGcKindStats = {
|
|
49
|
+
count: number;
|
|
50
|
+
totalMs: number;
|
|
51
|
+
maxMs: number;
|
|
52
|
+
/** Epoch ms at which the last collection of this kind finished. */
|
|
53
|
+
lastCompletedAtMs: number | null;
|
|
54
|
+
};
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* What the collector actually cost, from V8 rather than from a guess.
|
|
58
|
+
*
|
|
59
|
+
* A 1-CPU sandbox holding a ~500 MB heap of megabyte response bodies pays for
|
|
60
|
+
* every major collection in event-loop lag: the jai runner showed p90 0.6 s /
|
|
61
|
+
* p99 0.8 s while a light map showed none. Without this the lag is visible but
|
|
62
|
+
* unattributable, and admission cannot tell a slow provider from a stalled
|
|
63
|
+
* process.
|
|
64
|
+
*/
|
|
65
|
+
export type RuntimeGcObservation = {
|
|
66
|
+
/** False when this runtime delivers no `gc` performance entries. */
|
|
67
|
+
observing: boolean;
|
|
68
|
+
count: number;
|
|
69
|
+
totalMs: number;
|
|
70
|
+
maxMs: number;
|
|
71
|
+
byKind: Record<RuntimeGcKind, RuntimeGcKindStats>;
|
|
72
|
+
};
|
|
73
|
+
|
|
74
|
+
export type RuntimeGcObserver = {
|
|
75
|
+
snapshot(): RuntimeGcObservation;
|
|
76
|
+
close(): void;
|
|
77
|
+
};
|
|
78
|
+
|
|
79
|
+
function emptyGcKindStats(): RuntimeGcKindStats {
|
|
80
|
+
return { count: 0, totalMs: 0, maxMs: 0, lastCompletedAtMs: null };
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
function gcKindFromEntry(entry: {
|
|
84
|
+
kind?: number;
|
|
85
|
+
detail?: { kind?: number } | null;
|
|
86
|
+
}): RuntimeGcKind {
|
|
87
|
+
const kind = entry.detail?.kind ?? entry.kind;
|
|
88
|
+
switch (kind) {
|
|
89
|
+
case perfConstants.NODE_PERFORMANCE_GC_MINOR:
|
|
90
|
+
return 'minor';
|
|
91
|
+
case perfConstants.NODE_PERFORMANCE_GC_MAJOR:
|
|
92
|
+
return 'major';
|
|
93
|
+
case perfConstants.NODE_PERFORMANCE_GC_INCREMENTAL:
|
|
94
|
+
return 'incremental';
|
|
95
|
+
case perfConstants.NODE_PERFORMANCE_GC_WEAKCB:
|
|
96
|
+
return 'weakcb';
|
|
97
|
+
default:
|
|
98
|
+
return 'other';
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/**
|
|
103
|
+
* Tally every collection V8 reports. Installed once per process: the entries
|
|
104
|
+
* are process-wide, so a per-map observer would only multiply the callbacks
|
|
105
|
+
* without changing the numbers. Unref'd, so it never holds the runner open.
|
|
106
|
+
*/
|
|
107
|
+
export function createRuntimeGcObserver(): RuntimeGcObserver {
|
|
108
|
+
const byKind = Object.fromEntries(
|
|
109
|
+
RUNTIME_GC_KINDS.map((kind) => [kind, emptyGcKindStats()]),
|
|
110
|
+
) as Record<RuntimeGcKind, RuntimeGcKindStats>;
|
|
111
|
+
let count = 0;
|
|
112
|
+
let totalMs = 0;
|
|
113
|
+
let maxMs = 0;
|
|
114
|
+
let observing = false;
|
|
115
|
+
let observer: PerformanceObserver | null = null;
|
|
116
|
+
try {
|
|
117
|
+
observer = new PerformanceObserver((list) => {
|
|
118
|
+
for (const entry of list.getEntries()) {
|
|
119
|
+
const durationMs = Number.isFinite(entry.duration) ? entry.duration : 0;
|
|
120
|
+
const stats =
|
|
121
|
+
byKind[
|
|
122
|
+
gcKindFromEntry(
|
|
123
|
+
entry as unknown as {
|
|
124
|
+
kind?: number;
|
|
125
|
+
detail?: { kind?: number } | null;
|
|
126
|
+
},
|
|
127
|
+
)
|
|
128
|
+
];
|
|
129
|
+
stats.count += 1;
|
|
130
|
+
stats.totalMs += durationMs;
|
|
131
|
+
stats.maxMs = Math.max(stats.maxMs, durationMs);
|
|
132
|
+
// `startTime` is milliseconds since `timeOrigin`, so this is the same
|
|
133
|
+
// clock the admission cooldown compares against.
|
|
134
|
+
stats.lastCompletedAtMs =
|
|
135
|
+
performance.timeOrigin + entry.startTime + durationMs;
|
|
136
|
+
count += 1;
|
|
137
|
+
totalMs += durationMs;
|
|
138
|
+
maxMs = Math.max(maxMs, durationMs);
|
|
139
|
+
}
|
|
140
|
+
});
|
|
141
|
+
observer.observe({ entryTypes: ['gc'] });
|
|
142
|
+
(observer as { unref?: () => void }).unref?.();
|
|
143
|
+
observing = true;
|
|
144
|
+
} catch {
|
|
145
|
+
// A runtime without `gc` performance entries is not a failure: admission
|
|
146
|
+
// reports that it is not observing rather than inventing zeroes that look
|
|
147
|
+
// like a collector that never ran.
|
|
148
|
+
observer = null;
|
|
149
|
+
observing = false;
|
|
150
|
+
}
|
|
151
|
+
return {
|
|
152
|
+
snapshot: () => ({
|
|
153
|
+
observing,
|
|
154
|
+
count,
|
|
155
|
+
totalMs,
|
|
156
|
+
maxMs,
|
|
157
|
+
byKind: Object.fromEntries(
|
|
158
|
+
RUNTIME_GC_KINDS.map((kind) => [kind, { ...byKind[kind] }]),
|
|
159
|
+
) as Record<RuntimeGcKind, RuntimeGcKindStats>,
|
|
160
|
+
}),
|
|
161
|
+
close: () => {
|
|
162
|
+
observer?.disconnect();
|
|
163
|
+
observer = null;
|
|
164
|
+
observing = false;
|
|
165
|
+
},
|
|
166
|
+
};
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
let sharedGcObserver: RuntimeGcObserver | null = null;
|
|
170
|
+
|
|
171
|
+
/** The process-wide collector tally, installed on first use. */
|
|
172
|
+
export function runtimeGcObserver(): RuntimeGcObserver {
|
|
173
|
+
sharedGcObserver ??= createRuntimeGcObserver();
|
|
174
|
+
return sharedGcObserver;
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
/** A sampled V8 heap view; no customer data is retained in this diagnostic. */
|
|
178
|
+
export type RuntimeMapHeapSnapshot = {
|
|
179
|
+
usedHeapBytes: number;
|
|
180
|
+
heapLimitBytes: number;
|
|
181
|
+
/**
|
|
182
|
+
* Native memory owned by JS objects (decoded response Buffers, ArrayBuffers).
|
|
183
|
+
* V8's heapUsed excludes it, yet it is freed with its owners, so it is part
|
|
184
|
+
* of the live footprint the map can actually shrink.
|
|
185
|
+
*/
|
|
186
|
+
externalBytes?: number;
|
|
187
|
+
/**
|
|
188
|
+
* The process working set. It is what the sandbox cgroup kills on, but Node
|
|
189
|
+
* rarely returns freed pages to the OS, so RSS only proves proximity to the
|
|
190
|
+
* cliff; it never proves that memory is still in use.
|
|
191
|
+
*/
|
|
192
|
+
residentBytes?: number;
|
|
193
|
+
};
|
|
194
|
+
|
|
195
|
+
export type RuntimeMapHeapAdmissionSnapshot = {
|
|
196
|
+
activeRows: number;
|
|
197
|
+
activeRowPeak: number;
|
|
198
|
+
admittedRows: number;
|
|
199
|
+
/** Rows that have released their lease. */
|
|
200
|
+
settledRows: number;
|
|
201
|
+
/** Rows queued for admission right now: the latch this window can hold. */
|
|
202
|
+
waitingRows: number;
|
|
21
203
|
concurrency: number;
|
|
22
|
-
|
|
23
|
-
|
|
204
|
+
maxConcurrency: number;
|
|
205
|
+
heapLimitBytes: number | null;
|
|
206
|
+
heapTargetBytes: number | null;
|
|
207
|
+
heapUsedBytes: number | null;
|
|
208
|
+
heapUsedPeakBytes: number | null;
|
|
209
|
+
/** heapUsed + external: the footprint that shrinks when rows settle. */
|
|
210
|
+
liveBytes: number | null;
|
|
211
|
+
liveBaselineBytes: number | null;
|
|
212
|
+
livePeakBytes: number | null;
|
|
213
|
+
/**
|
|
214
|
+
* Bytes reserved per admitted row while it has not started materializing a
|
|
215
|
+
* provider response. In deferred-body mode this is the learned row promise,
|
|
216
|
+
* capped at the unknown full-row prior; decoded responses are separately
|
|
217
|
+
* admitted at materialization.
|
|
218
|
+
*/
|
|
219
|
+
rowReserveBytes: number | null;
|
|
220
|
+
/** Whether decoded provider bodies are reserved at materialization. */
|
|
221
|
+
providerBodyReservationDeferred: boolean;
|
|
222
|
+
/** Compact per-row reserve used while body reservation is deferred. */
|
|
223
|
+
waitingRowReserveBytes: number | null;
|
|
224
|
+
/**
|
|
225
|
+
* The ledger's settled p90 per-row peak scaled by the calibration factor:
|
|
226
|
+
* the live bytes one more row is expected to cost. Null until enough rows
|
|
227
|
+
* have settled, where the unmeasured prior stands instead.
|
|
228
|
+
*/
|
|
229
|
+
measuredRowBytes: number | null;
|
|
230
|
+
/**
|
|
231
|
+
* Settled per-row peaks behind `measuredRowBytes`. A reserve with none
|
|
232
|
+
* behind it is still the unmeasured prior, which is why the readout carries
|
|
233
|
+
* the count.
|
|
234
|
+
*/
|
|
235
|
+
footprintSamples: number;
|
|
236
|
+
/**
|
|
237
|
+
* Retained for readout compatibility. Under byte admission the per-row cost
|
|
238
|
+
* comes from the ledger's settled owners, never from a probe quotient, so no
|
|
239
|
+
* minimum in-flight width gates it; this echoes the configured input.
|
|
240
|
+
*/
|
|
241
|
+
footprintSampleMinActiveRows: number;
|
|
242
|
+
/** Ledger bytes x calibration factor: the live footprint the ledger predicts. */
|
|
243
|
+
predictedLiveBytes: number | null;
|
|
244
|
+
/**
|
|
245
|
+
* `liveBaselineBytes + predictedLiveBytes`: what admission divides headroom
|
|
246
|
+
* against. The ledger, not the probe, is the estimator — with a large old
|
|
247
|
+
* space V8 lets garbage pile toward the limit before a major GC, so
|
|
248
|
+
* `heapUsed + external` is mostly garbage between collections and pacing on
|
|
249
|
+
* it collapses the window to one row (jai run 34074320144: 139/1000 rows in
|
|
250
|
+
* 25 minutes with the calibration factor pinned at its clamp).
|
|
251
|
+
*/
|
|
252
|
+
occupiedBytes: number | null;
|
|
253
|
+
/**
|
|
254
|
+
* The byte ledger at the last probe, with what the collector cost alongside
|
|
255
|
+
* it; null when no ledger is attached.
|
|
256
|
+
*/
|
|
257
|
+
memory: RuntimeMapMemorySnapshot | null;
|
|
258
|
+
/** Watchdog pressure signals delivered from outside this process. */
|
|
259
|
+
externalPressureSignals: number;
|
|
260
|
+
/** Probes that paused new row admission while pressure was active. */
|
|
261
|
+
admissionHolds: number;
|
|
262
|
+
/**
|
|
263
|
+
* Whether memory pressure is currently pausing new row admission. The
|
|
264
|
+
* computed concurrency remains the next safe width so a short pressure
|
|
265
|
+
* episode cannot turn the rest of a map into a serial tail.
|
|
266
|
+
*/
|
|
267
|
+
admissionPaused: boolean;
|
|
268
|
+
/**
|
|
269
|
+
* What the collector cost this process. Also carried inside `memory`, and
|
|
270
|
+
* repeated here because the terminal runtime-resources summary keeps the
|
|
271
|
+
* per-map admission snapshot but drops its copy of the process-wide ledger.
|
|
272
|
+
*/
|
|
273
|
+
gc: RuntimeGcObservation | null;
|
|
274
|
+
/** Whether this runner can collect on demand (`node --expose-gc`). */
|
|
275
|
+
gcAvailable: boolean;
|
|
276
|
+
/** Collections this admission asked for itself. */
|
|
277
|
+
gcExplicitCount: number;
|
|
278
|
+
/** Wall time spent inside those explicit collections. */
|
|
279
|
+
gcExplicitTotalMs: number;
|
|
280
|
+
/**
|
|
281
|
+
* Age of the last collection admission counted on — its own, or a major one
|
|
282
|
+
* V8 ran on its own account. Null before the first.
|
|
283
|
+
*/
|
|
284
|
+
lastGcAgeMs: number | null;
|
|
285
|
+
limitedByHeap: boolean;
|
|
286
|
+
/** The sandbox memory limit the RSS guard is derived from, when readable. */
|
|
287
|
+
cgroupLimitBytes: number | null;
|
|
288
|
+
/**
|
|
289
|
+
* CPU cores the sandbox may actually use (cgroup quota / period), so a
|
|
290
|
+
* readout labels a 2 vCPU session as one instead of asserting it in prose.
|
|
291
|
+
* Null only when neither cgroup interface nor the host count is readable.
|
|
292
|
+
*/
|
|
293
|
+
cpuCount: number | null;
|
|
294
|
+
residentGuardBytes: number | null;
|
|
295
|
+
residentBytes: number | null;
|
|
296
|
+
residentPeakBytes: number | null;
|
|
297
|
+
limitedByResidentMemory: boolean;
|
|
298
|
+
/** Distinguishes V8 graph pressure from Buffer/native response pressure. */
|
|
299
|
+
heapPressureEvents: number;
|
|
300
|
+
residentPressureEvents: number;
|
|
301
|
+
/** Admission was held because the event loop could not keep up. */
|
|
302
|
+
limitedByEventLoop: boolean;
|
|
303
|
+
eventLoopPressureEvents: number;
|
|
304
|
+
/** Row ceiling set by event-loop lag; maxConcurrency when it never lagged. */
|
|
305
|
+
eventLoopCeiling: number;
|
|
306
|
+
/** The last event-loop lag reading admission acted on, if it had one. */
|
|
307
|
+
eventLoopLagMs: number | null;
|
|
308
|
+
memoryPressureEvents: number;
|
|
309
|
+
};
|
|
310
|
+
|
|
311
|
+
/**
|
|
312
|
+
* The subset of a heap probe the admission decision needs as non-null numbers.
|
|
313
|
+
* Returned rather than read back off the mutable snapshot fields so a
|
|
314
|
+
* post-collection re-probe replaces the reading it supersedes.
|
|
315
|
+
*/
|
|
316
|
+
type HeapReading = {
|
|
317
|
+
heapLimitBytes: number;
|
|
318
|
+
heapTargetBytes: number;
|
|
319
|
+
liveBytes: number;
|
|
320
|
+
liveBaselineBytes: number;
|
|
321
|
+
residentBytes: number | null;
|
|
322
|
+
residentGuardBytes: number;
|
|
323
|
+
};
|
|
324
|
+
|
|
325
|
+
/** The byte ledger plus the collector cost measured over the same process. */
|
|
326
|
+
export type RuntimeMapMemorySnapshot = RuntimeMemoryLedgerSnapshot & {
|
|
327
|
+
/** Process-wide collector tally, or null when nothing is observing. */
|
|
328
|
+
gc: RuntimeGcObservation | null;
|
|
24
329
|
};
|
|
25
330
|
|
|
331
|
+
export type RuntimeMapHeapAdmissionLease = {
|
|
332
|
+
release(): void;
|
|
333
|
+
};
|
|
334
|
+
|
|
335
|
+
export type RuntimeMapHeapAdmission = {
|
|
336
|
+
acquire(): Promise<RuntimeMapHeapAdmissionLease>;
|
|
337
|
+
/** Sample V8 and admit any newly safe waiting rows. Useful for deterministic tests. */
|
|
338
|
+
reconcile(): void;
|
|
339
|
+
/** Reject waiters after a row failure, or release the timer after normal completion. */
|
|
340
|
+
close(error?: unknown): RuntimeMapHeapAdmissionSnapshot;
|
|
341
|
+
snapshot(): RuntimeMapHeapAdmissionSnapshot;
|
|
342
|
+
};
|
|
343
|
+
|
|
344
|
+
// Upper bound of the opening wave, before any probe has run. It is not a row
|
|
345
|
+
// limit in its own right: the construction-time clamp below divides probed
|
|
346
|
+
// headroom by the unmeasured per-row reserve and almost always lands far under
|
|
347
|
+
// it (146 rows on a 1 GiB sandbox), and every later width comes from bytes.
|
|
348
|
+
// It exists because acquire() is called by every row worker before the first
|
|
349
|
+
// probe, so the wave has to be bounded by *something* in the same tick.
|
|
350
|
+
const RUNTIME_MAP_HEAP_INITIAL_CONCURRENCY = 256;
|
|
351
|
+
const RUNTIME_MAP_HEAP_PROBE_INTERVAL_MS = 25;
|
|
352
|
+
// Live memory (heapUsed + external) may fill this share of the heap limit.
|
|
353
|
+
// Live is not what the cgroup kills on: RSS runs about 1.3-1.5x live plus the
|
|
354
|
+
// runtime's own baseline, so a 65% live target on the 960 MiB heap limit of a
|
|
355
|
+
// 1 GiB sandbox authorizes 624 MiB of live bytes and lands RSS on the cliff.
|
|
356
|
+
// The measured jai shape (~2.5-4 MB live per row) proves both sides of it: one
|
|
357
|
+
// run survived at 364 rows in flight only by peaking at 964 MB RSS inside a
|
|
358
|
+
// 1 GiB cgroup, and the next died of RUNTIME_SANDBOX_OOM at 205 settled rows
|
|
359
|
+
// with 256 in flight. Half the heap limit keeps the RSS that follows from a
|
|
360
|
+
// full window inside the sandbox instead of one settling wave away from it.
|
|
361
|
+
const RUNTIME_MAP_HEAP_TARGET_UTILIZATION = 0.5;
|
|
362
|
+
/**
|
|
363
|
+
* Event-loop lag (EWMA of timer delay) above which admission holds. Memory
|
|
364
|
+
* headroom alone let a one-vCPU sandbox keep 1,000 rows active pinned to the
|
|
365
|
+
* heap target, where V8 collected continuously and the runner's own
|
|
366
|
+
* heartbeats and result writes could no longer get scheduled.
|
|
367
|
+
*/
|
|
368
|
+
export const RUNTIME_MAP_EVENT_LOOP_LAG_HOLD_MS = 250;
|
|
369
|
+
/**
|
|
370
|
+
* While the loop lags, the row ceiling is cut to this fraction of the active
|
|
371
|
+
* set once per adjust interval; once it keeps up, the ceiling grows back by
|
|
372
|
+
* a tenth per interval. Holding alone could only stop new rows: the window
|
|
373
|
+
* had already ramped to its cap before the lag sampler could register it.
|
|
374
|
+
*/
|
|
375
|
+
const RUNTIME_MAP_EVENT_LOOP_SHRINK_FACTOR = 0.75;
|
|
376
|
+
const RUNTIME_MAP_EVENT_LOOP_GROW_FRACTION = 0.1;
|
|
377
|
+
export const RUNTIME_MAP_EVENT_LOOP_ADJUST_INTERVAL_MS = 2_000;
|
|
378
|
+
// RSS is the number the sandbox actually kills on, so the guard belongs on the
|
|
379
|
+
// cgroup limit, not on the heap limit. The runner now sets its old space to
|
|
380
|
+
// roughly half the cgroup, so a guard expressed as a share of heap_size_limit
|
|
381
|
+
// would sit at ~364 MB of a 1 GiB sandbox — below the RSS a healthy map holds
|
|
382
|
+
// once Node stops returning freed pages, which would latch the window shut.
|
|
383
|
+
// The watchdog signals pressure at 0.8 of the cgroup and aborts at 0.92, so a
|
|
384
|
+
// guard above 0.8 could never fire before the watchdog had already spoken.
|
|
385
|
+
const RUNTIME_MAP_RESIDENT_GUARD_CGROUP_FRACTION = 0.8;
|
|
386
|
+
// Fallback when the cgroup limit is unreadable (a local run, a host without
|
|
387
|
+
// cgroups). Derivation: old space is set to ~50% of the cgroup, so the cgroup
|
|
388
|
+
// is ~2x heap_size_limit; a full old space plus the young generation
|
|
389
|
+
// (~16 MiB semi-space, three of them live during a scavenge) plus the runner's
|
|
390
|
+
// own native baseline lands RSS near 1.45x heap_size_limit before the cgroup
|
|
391
|
+
// is in danger. Measured 1k runs agree: a kareem map peaked at 379 MB RSS on
|
|
392
|
+
// a 256 MB heap limit.
|
|
393
|
+
const RUNTIME_MAP_RESIDENT_GUARD_HEAP_MULTIPLE = 1.45;
|
|
394
|
+
const CGROUP_V2_MEMORY_MAX_PATH = '/sys/fs/cgroup/memory.max';
|
|
395
|
+
const CGROUP_V1_MEMORY_LIMIT_PATH =
|
|
396
|
+
'/sys/fs/cgroup/memory/memory.limit_in_bytes';
|
|
397
|
+
// Minimum wall time between admission-requested collections. A full collection
|
|
398
|
+
// of a ~500 MB heap of megabyte response bodies on a 1-CPU sandbox costs
|
|
399
|
+
// 0.3-1 s of event-loop lag — the jai runner measured p90 0.6 s / p99 0.8 s —
|
|
400
|
+
// so an explicit collection is a real cost, not a free measurement. Five
|
|
401
|
+
// seconds keeps it under a few percent of wall while still answering the
|
|
402
|
+
// garbage question faster than a map can fill the sandbox. The observer below
|
|
403
|
+
// makes most of these unnecessary anyway: if V8 already ran a major collection
|
|
404
|
+
// inside the interval, admission re-probes instead of paying for its own.
|
|
405
|
+
const RUNTIME_MAP_GC_MIN_INTERVAL_MS = 5_000;
|
|
406
|
+
// RSS near the cliff only means danger while live memory is itself
|
|
407
|
+
// substantial. Node keeps freed pages, so a high RSS over a small live
|
|
408
|
+
// footprint is a stale plateau: new rows reuse those pages and RSS does not
|
|
409
|
+
// grow. Below this share of the live target the guard is ignored.
|
|
410
|
+
const RUNTIME_MAP_RESIDENT_GUARD_LIVE_FRACTION = 0.5;
|
|
411
|
+
// New rows admitted per probe: a 1,000-row window ramps in ~0.4 s while every
|
|
412
|
+
// step is checked against live memory, instead of one blind launch wave.
|
|
413
|
+
const RUNTIME_MAP_HEAP_GROWTH_STEP_MAX = 64;
|
|
414
|
+
// Reserve per row before the ledger has settled enough rows to state one. It
|
|
415
|
+
// must cover a large provider response outright: a thousand optimistic guesses
|
|
416
|
+
// arriving as one wave of decoded bodies is exactly how a 1 GiB sandbox dies
|
|
417
|
+
// before the ledger can say anything. 3 MiB opens ~150 rows of an unknown
|
|
418
|
+
// shape on the 960 MiB heap limit of a 1 GiB sandbox, where 2 MiB opened
|
|
419
|
+
// ~300 and a heavy shape reached the cliff before a single row had settled.
|
|
420
|
+
// The cost is paid only until 16 rows settle: a light shape's measured
|
|
421
|
+
// footprint replaces this prior within the first probes and the window
|
|
422
|
+
// widens, while a heavy shape never gets to the cliff to begin with.
|
|
423
|
+
export const RUNTIME_MAP_ROW_RESERVE_DEFAULT_BYTES = 3 * 1024 * 1024;
|
|
424
|
+
// Publish cadence for the byte ledger the watchdog attaches to a memory abort.
|
|
425
|
+
const RUNTIME_MAP_LEDGER_PUBLISH_INTERVAL_MS = 1_000;
|
|
426
|
+
const RUNTIME_MAP_ROW_RESERVE_MIN_BYTES = 64 * 1024;
|
|
427
|
+
// Settled rows the ledger needs before its p90 per-row peak replaces the
|
|
428
|
+
// prior. Sixteen settled peaks make a p90 mean something without making the
|
|
429
|
+
// map wait for a tenth of itself to finish.
|
|
430
|
+
const RUNTIME_MAP_FOOTPRINT_SAMPLE_ROWS = 16;
|
|
431
|
+
// Retained input default for the snapshot field of the same name. Byte
|
|
432
|
+
// admission derives the per-row cost from the ledger's settled owners, so no
|
|
433
|
+
// in-flight width gates a measurement any more; see the snapshot type.
|
|
434
|
+
const RUNTIME_MAP_ROW_FOOTPRINT_SAMPLE_MIN_ACTIVE_ROWS = 16;
|
|
435
|
+
const RUNTIME_MAP_ROW_FOOTPRINT_SAMPLE_MIN_ACTIVE_ROWS_DIVISOR = 4;
|
|
436
|
+
// Rows the window may exceed the opening width by, per row that has fully
|
|
437
|
+
// settled. This paces growth; it does not cap the map. Admission cannot be
|
|
438
|
+
// undone: a row admitted while a downstream lane is slow holds every body it
|
|
439
|
+
// has already fetched until that lane drains, so a window that grows faster
|
|
440
|
+
// than rows finish fills the sandbox with bodies the ledger has not yet had a
|
|
441
|
+
// settled row to price (jai: 1,000 rows each holding an enformion result while
|
|
442
|
+
// ~490 agent calls queued on a 30-permit lane). Rows keep arriving for as long
|
|
443
|
+
// as rows keep finishing, so a map of any size still runs to completion.
|
|
444
|
+
const RUNTIME_MAP_ROWS_PER_SETTLED_ROW = 4;
|
|
445
|
+
|
|
446
|
+
/**
|
|
447
|
+
* The runner's real heap, rather than input JSON size, is the authority for
|
|
448
|
+
* how many live resolver graphs it may hold. A failed probe deliberately
|
|
449
|
+
* disables this adaptive layer; the existing structural input limits remain.
|
|
450
|
+
*/
|
|
451
|
+
/**
|
|
452
|
+
* The sandbox memory limit as the kernel states it, read once at construction.
|
|
453
|
+
*
|
|
454
|
+
* V8's `heap_size_limit` is a JS-heap ceiling the runner chooses; the cgroup
|
|
455
|
+
* limit is what the sandbox is actually killed on and what the watchdog
|
|
456
|
+
* measures against. An unreadable or absent limit is not a failure — a local
|
|
457
|
+
* run has no cgroup — so the caller falls back to a documented multiple of the
|
|
458
|
+
* heap limit rather than guessing a sandbox size.
|
|
459
|
+
*/
|
|
460
|
+
export function readCgroupMemoryLimitBytes(options?: {
|
|
461
|
+
/** Seam for tests; production reads the two cgroup interface files. */
|
|
462
|
+
readFile?: (path: string) => string;
|
|
463
|
+
}): number | null {
|
|
464
|
+
const readFile =
|
|
465
|
+
options?.readFile ?? ((path: string) => readFileSync(path, 'utf8'));
|
|
466
|
+
for (const path of [CGROUP_V2_MEMORY_MAX_PATH, CGROUP_V1_MEMORY_LIMIT_PATH]) {
|
|
467
|
+
let raw: string;
|
|
468
|
+
try {
|
|
469
|
+
raw = readFile(path).trim();
|
|
470
|
+
} catch {
|
|
471
|
+
continue;
|
|
472
|
+
}
|
|
473
|
+
// cgroup v2 writes the literal "max" for no limit; v1 writes a sentinel
|
|
474
|
+
// near 2^63. Neither is a memory limit.
|
|
475
|
+
if (!raw || raw === 'max') continue;
|
|
476
|
+
const value = Number(raw);
|
|
477
|
+
if (!Number.isFinite(value) || value <= 0) continue;
|
|
478
|
+
if (value >= Number.MAX_SAFE_INTEGER) continue;
|
|
479
|
+
return value;
|
|
480
|
+
}
|
|
481
|
+
return null;
|
|
482
|
+
}
|
|
483
|
+
|
|
484
|
+
const CGROUP_V2_CPU_MAX_PATH = '/sys/fs/cgroup/cpu.max';
|
|
485
|
+
const CGROUP_V1_CPU_QUOTA_PATH = '/sys/fs/cgroup/cpu/cpu.cfs_quota_us';
|
|
486
|
+
const CGROUP_V1_CPU_PERIOD_PATH = '/sys/fs/cgroup/cpu/cpu.cfs_period_us';
|
|
487
|
+
|
|
488
|
+
/**
|
|
489
|
+
* The CPU the sandbox may actually use: cgroup v2 `cpu.max` ("quota period"
|
|
490
|
+
* or "max period"), else cgroup v1 quota/period, else the host's parallelism.
|
|
491
|
+
* The host count is the wrong answer inside a sandbox (Daytona reports the
|
|
492
|
+
* host's 48-64 cores against a quota of one), so it is only the fallback for
|
|
493
|
+
* an unlimited or unreadable cgroup.
|
|
494
|
+
*/
|
|
495
|
+
export function readCgroupCpuLimitCores(options?: {
|
|
496
|
+
readFile?: (path: string) => string;
|
|
497
|
+
hostCpuCount?: () => number;
|
|
498
|
+
}): number | null {
|
|
499
|
+
const readFile =
|
|
500
|
+
options?.readFile ?? ((path: string) => readFileSync(path, 'utf8'));
|
|
501
|
+
const hostCpuCount = options?.hostCpuCount ?? availableParallelism;
|
|
502
|
+
const quotaCores = (quotaRaw: string, periodRaw: string): number | null => {
|
|
503
|
+
const quota = Number(quotaRaw);
|
|
504
|
+
const period = Number(periodRaw);
|
|
505
|
+
if (!Number.isFinite(quota) || !Number.isFinite(period)) return null;
|
|
506
|
+
if (quota <= 0 || period <= 0) return null;
|
|
507
|
+
return Math.max(quota / period, 0.01);
|
|
508
|
+
};
|
|
509
|
+
try {
|
|
510
|
+
const [quota, period] = readFile(CGROUP_V2_CPU_MAX_PATH)
|
|
511
|
+
.trim()
|
|
512
|
+
.split(/\s+/, 2);
|
|
513
|
+
if (quota && quota !== 'max' && period) {
|
|
514
|
+
const cores = quotaCores(quota, period);
|
|
515
|
+
if (cores !== null) return cores;
|
|
516
|
+
}
|
|
517
|
+
} catch {
|
|
518
|
+
try {
|
|
519
|
+
const cores = quotaCores(
|
|
520
|
+
readFile(CGROUP_V1_CPU_QUOTA_PATH).trim(),
|
|
521
|
+
readFile(CGROUP_V1_CPU_PERIOD_PATH).trim(),
|
|
522
|
+
);
|
|
523
|
+
if (cores !== null) return cores;
|
|
524
|
+
} catch {
|
|
525
|
+
// No cgroup CPU interface: fall through to the host count.
|
|
526
|
+
}
|
|
527
|
+
}
|
|
528
|
+
const host = hostCpuCount();
|
|
529
|
+
return Number.isFinite(host) && host > 0 ? host : null;
|
|
530
|
+
}
|
|
531
|
+
|
|
532
|
+
export function readRuntimeMapHeapSnapshot(): RuntimeMapHeapSnapshot | null {
|
|
533
|
+
try {
|
|
534
|
+
const memoryUsage = process.memoryUsage();
|
|
535
|
+
const usedHeapBytes = memoryUsage.heapUsed;
|
|
536
|
+
const heapLimitBytes = getHeapStatistics().heap_size_limit;
|
|
537
|
+
if (
|
|
538
|
+
!Number.isFinite(usedHeapBytes) ||
|
|
539
|
+
!Number.isFinite(heapLimitBytes) ||
|
|
540
|
+
usedHeapBytes < 0 ||
|
|
541
|
+
heapLimitBytes <= 0
|
|
542
|
+
) {
|
|
543
|
+
return null;
|
|
544
|
+
}
|
|
545
|
+
return {
|
|
546
|
+
usedHeapBytes,
|
|
547
|
+
heapLimitBytes,
|
|
548
|
+
externalBytes:
|
|
549
|
+
Number.isFinite(memoryUsage.external) && memoryUsage.external >= 0
|
|
550
|
+
? memoryUsage.external
|
|
551
|
+
: undefined,
|
|
552
|
+
residentBytes: Number.isFinite(memoryUsage.rss)
|
|
553
|
+
? memoryUsage.rss
|
|
554
|
+
: undefined,
|
|
555
|
+
};
|
|
556
|
+
} catch {
|
|
557
|
+
return null;
|
|
558
|
+
}
|
|
559
|
+
}
|
|
560
|
+
|
|
561
|
+
/**
|
|
562
|
+
* What the runtime is occupying, and what one more unit of work may cost.
|
|
563
|
+
*
|
|
564
|
+
* This is the single statement of the memory invariant's arithmetic. Map row
|
|
565
|
+
* admission and completed-receipt hydration both admit against it, so neither
|
|
566
|
+
* can drift into a private notion of "full" — and neither states a limit as a
|
|
567
|
+
* count of anything.
|
|
568
|
+
*/
|
|
569
|
+
export type RuntimeMemoryOccupancy = {
|
|
570
|
+
/** Live bytes the process may hold: a share of the V8 heap limit. */
|
|
571
|
+
heapTargetBytes: number;
|
|
572
|
+
/** `heapUsed + external` at the probe this was computed from. */
|
|
573
|
+
liveBytes: number;
|
|
574
|
+
liveBaselineBytes: number;
|
|
575
|
+
/**
|
|
576
|
+
* Sum over ledger categories of bytes x that category's fitted factor, or
|
|
577
|
+
* null with no ledger attached.
|
|
578
|
+
*/
|
|
579
|
+
predictedLiveBytes: number | null;
|
|
580
|
+
/** `baseline + predictedLive`: what headroom is measured against. */
|
|
581
|
+
occupiedBytes: number;
|
|
582
|
+
/** Live bytes still available under the target. Never negative. */
|
|
583
|
+
headroomBytes: number;
|
|
584
|
+
/** The ledger's settled p90 per-owner peak in live bytes; null if unmeasured. */
|
|
585
|
+
measuredOwnerBytes: number | null;
|
|
586
|
+
/** Live bytes one more owner may cost: the measurement, or the prior. */
|
|
587
|
+
ownerReserveBytes: number;
|
|
588
|
+
/**
|
|
589
|
+
* The same reserve expressed in ledger bytes, which is the unit a
|
|
590
|
+
* reservation is registered in. Registering live bytes would double-count
|
|
591
|
+
* the calibration factor the moment the ledger scaled them again.
|
|
592
|
+
*/
|
|
593
|
+
ownerReserveLedgerBytes: number;
|
|
594
|
+
/**
|
|
595
|
+
* The mix-weighted factor the ledger currently implies. Kept for readouts
|
|
596
|
+
* and for the no-ledger fallback; a lane converting its own reservation to
|
|
597
|
+
* live bytes must use `categoryFactors` for the category it registers on,
|
|
598
|
+
* because the mix at the moment it asks is not its own mix.
|
|
599
|
+
*/
|
|
600
|
+
calibrationFactor: number;
|
|
601
|
+
/** Live bytes per ledger byte, per category: the conversion each lane owns. */
|
|
602
|
+
categoryFactors: Record<RuntimeMemoryLedgerCategory, number>;
|
|
603
|
+
/** Settled owner peaks behind `measuredOwnerBytes`. */
|
|
604
|
+
footprintSamples: number;
|
|
605
|
+
};
|
|
606
|
+
|
|
607
|
+
/**
|
|
608
|
+
* The occupancy arithmetic, with no probing and no state. Both admission
|
|
609
|
+
* paths call this so the invariant is stated once.
|
|
610
|
+
*/
|
|
611
|
+
export function runtimeMemoryOccupancy(input: {
|
|
612
|
+
heapLimitBytes: number;
|
|
613
|
+
liveBytes: number;
|
|
614
|
+
liveBaselineBytes: number;
|
|
615
|
+
ledger: RuntimeMemoryLedgerSnapshot | null;
|
|
616
|
+
footprintSampleRows?: number;
|
|
617
|
+
ownerReserveDefaultBytes?: number;
|
|
618
|
+
ownerReserveMinBytes?: number;
|
|
619
|
+
}): RuntimeMemoryOccupancy {
|
|
620
|
+
const footprintSampleRows = positiveBudget(
|
|
621
|
+
input.footprintSampleRows,
|
|
622
|
+
RUNTIME_MAP_FOOTPRINT_SAMPLE_ROWS,
|
|
623
|
+
);
|
|
624
|
+
const ownerReserveDefaultBytes = positiveBudget(
|
|
625
|
+
input.ownerReserveDefaultBytes,
|
|
626
|
+
RUNTIME_MAP_ROW_RESERVE_DEFAULT_BYTES,
|
|
627
|
+
);
|
|
628
|
+
const ownerReserveMinBytes = positiveBudget(
|
|
629
|
+
input.ownerReserveMinBytes,
|
|
630
|
+
RUNTIME_MAP_ROW_RESERVE_MIN_BYTES,
|
|
631
|
+
);
|
|
632
|
+
const heapTargetBytes = Math.floor(
|
|
633
|
+
input.heapLimitBytes * RUNTIME_MAP_HEAP_TARGET_UTILIZATION,
|
|
634
|
+
);
|
|
635
|
+
const ledger = input.ledger;
|
|
636
|
+
const predictedLiveBytes = ledger?.predictedLiveBytes ?? null;
|
|
637
|
+
// The ledger is the estimator and the probe is only the guard; without a
|
|
638
|
+
// ledger the probe is all there is.
|
|
639
|
+
const occupiedBytes =
|
|
640
|
+
predictedLiveBytes === null
|
|
641
|
+
? input.liveBytes
|
|
642
|
+
: input.liveBaselineBytes + predictedLiveBytes;
|
|
643
|
+
const categoryFactors = runtimeMemoryCategoryFactors(ledger);
|
|
644
|
+
// A settled owner peak is provider-body bytes: the reserve is what one more
|
|
645
|
+
// row's decoded body costs. Scaling it by the mix-weighted factor priced a
|
|
646
|
+
// row from whatever else the process happened to be holding at the probe.
|
|
647
|
+
const rowFactor = categoryFactors.providerBody;
|
|
648
|
+
const measuredOwnerBytes =
|
|
649
|
+
ledger &&
|
|
650
|
+
ledger.ownerPeakSamples >= footprintSampleRows &&
|
|
651
|
+
ledger.ownerPeakBytesP90 !== null
|
|
652
|
+
? ledger.ownerPeakBytesP90 * rowFactor
|
|
653
|
+
: null;
|
|
654
|
+
const ownerReserveBytes =
|
|
655
|
+
measuredOwnerBytes === null
|
|
656
|
+
? ownerReserveDefaultBytes
|
|
657
|
+
: Math.max(ownerReserveMinBytes, measuredOwnerBytes);
|
|
658
|
+
const calibrationFactor =
|
|
659
|
+
ledger && ledger.calibrationFactor > 0
|
|
660
|
+
? ledger.calibrationFactor
|
|
661
|
+
: RUNTIME_MEMORY_LEDGER_INITIAL_FACTOR;
|
|
662
|
+
return {
|
|
663
|
+
heapTargetBytes,
|
|
664
|
+
liveBytes: input.liveBytes,
|
|
665
|
+
liveBaselineBytes: input.liveBaselineBytes,
|
|
666
|
+
predictedLiveBytes,
|
|
667
|
+
occupiedBytes,
|
|
668
|
+
headroomBytes: Math.max(0, heapTargetBytes - occupiedBytes),
|
|
669
|
+
measuredOwnerBytes,
|
|
670
|
+
ownerReserveBytes,
|
|
671
|
+
ownerReserveLedgerBytes: Math.max(
|
|
672
|
+
1,
|
|
673
|
+
Math.ceil(ownerReserveBytes / rowFactor),
|
|
674
|
+
),
|
|
675
|
+
calibrationFactor,
|
|
676
|
+
categoryFactors,
|
|
677
|
+
footprintSamples: ledger?.ownerPeakSamples ?? 0,
|
|
678
|
+
};
|
|
679
|
+
}
|
|
680
|
+
|
|
681
|
+
/**
|
|
682
|
+
* The per-category live-bytes-per-ledger-byte conversion a lane should use.
|
|
683
|
+
*
|
|
684
|
+
* With a ledger attached these are its fitted factors. Without one — a runtime
|
|
685
|
+
* whose heap cannot be read, or a snapshot published before per-category
|
|
686
|
+
* calibration existed — every category falls back to whatever single factor
|
|
687
|
+
* that snapshot carries, so an old readout still converts the way it used to
|
|
688
|
+
* rather than silently switching to a prior it never measured.
|
|
689
|
+
*/
|
|
690
|
+
export function runtimeMemoryCategoryFactors(
|
|
691
|
+
ledger: RuntimeMemoryLedgerSnapshot | null,
|
|
692
|
+
): Record<RuntimeMemoryLedgerCategory, number> {
|
|
693
|
+
const factors = {} as Record<RuntimeMemoryLedgerCategory, number>;
|
|
694
|
+
const legacyFactor =
|
|
695
|
+
ledger && ledger.calibrationFactor > 0
|
|
696
|
+
? ledger.calibrationFactor
|
|
697
|
+
: RUNTIME_MEMORY_LEDGER_INITIAL_FACTOR;
|
|
698
|
+
for (const category of RUNTIME_MEMORY_LEDGER_CATEGORIES) {
|
|
699
|
+
const fitted = ledger?.categoryFactors?.[category];
|
|
700
|
+
factors[category] =
|
|
701
|
+
typeof fitted === 'number' && Number.isFinite(fitted) && fitted > 0
|
|
702
|
+
? fitted
|
|
703
|
+
: legacyFactor;
|
|
704
|
+
}
|
|
705
|
+
return factors;
|
|
706
|
+
}
|
|
707
|
+
|
|
708
|
+
export type RuntimeMemoryOccupancyReader = {
|
|
709
|
+
/**
|
|
710
|
+
* Current occupancy, or null when this runtime has no readable heap. A null
|
|
711
|
+
* reading disables byte admission for its caller rather than inventing a
|
|
712
|
+
* limit out of a probe that did not answer.
|
|
713
|
+
*/
|
|
714
|
+
read(): RuntimeMemoryOccupancy | null;
|
|
715
|
+
/**
|
|
716
|
+
* Answer the garbage question the same way map admission does: let V8's own
|
|
717
|
+
* recent major collection stand, otherwise collect (rate limited), then
|
|
718
|
+
* re-probe. Returns the post-collection reading, or null when no collection
|
|
719
|
+
* happened and the caller's reading still stands.
|
|
720
|
+
*/
|
|
721
|
+
collectAndRead(): RuntimeMemoryOccupancy | null;
|
|
722
|
+
snapshot(): {
|
|
723
|
+
/** Heap probes taken, and probes served from the interval cache. */
|
|
724
|
+
probes: number;
|
|
725
|
+
cachedProbes: number;
|
|
726
|
+
/** Collections this reader asked for, and what they cost. */
|
|
727
|
+
gcCount: number;
|
|
728
|
+
gcTotalMs: number;
|
|
729
|
+
gcAvailable: boolean;
|
|
730
|
+
};
|
|
731
|
+
};
|
|
732
|
+
|
|
733
|
+
/**
|
|
734
|
+
* A process-wide occupancy reader.
|
|
735
|
+
*
|
|
736
|
+
* Map row admission owns its own probe loop because it also carries the RSS
|
|
737
|
+
* guard, the watchdog hold, and calibration. A lane that only needs to ask
|
|
738
|
+
* "do these bytes fit" gets this instead: the same arithmetic, the same
|
|
739
|
+
* collect-before-you-conclude rule, and a probe cached for the same interval
|
|
740
|
+
* so asking the question often costs no more than asking it once.
|
|
741
|
+
*/
|
|
742
|
+
export function createRuntimeMemoryOccupancyReader(input: {
|
|
743
|
+
memoryLedger: RuntimeMemoryLedger;
|
|
744
|
+
heapProbe?: () => RuntimeMapHeapSnapshot | null;
|
|
745
|
+
/** Omit for the runner's `--expose-gc` hook; null states it cannot collect. */
|
|
746
|
+
collectGarbage?: (() => void) | null;
|
|
747
|
+
observeGc?: (() => RuntimeGcObservation) | null;
|
|
748
|
+
probeIntervalMs?: number;
|
|
749
|
+
gcMinIntervalMs?: number;
|
|
750
|
+
footprintSampleRows?: number;
|
|
751
|
+
ownerReserveDefaultBytes?: number;
|
|
752
|
+
ownerReserveMinBytes?: number;
|
|
753
|
+
now?: () => number;
|
|
754
|
+
}): RuntimeMemoryOccupancyReader {
|
|
755
|
+
const heapProbe = input.heapProbe ?? readRuntimeMapHeapSnapshot;
|
|
756
|
+
const probeIntervalMs = positiveBudget(
|
|
757
|
+
input.probeIntervalMs,
|
|
758
|
+
RUNTIME_MAP_HEAP_PROBE_INTERVAL_MS,
|
|
759
|
+
);
|
|
760
|
+
const gcMinIntervalMs = positiveBudget(
|
|
761
|
+
input.gcMinIntervalMs,
|
|
762
|
+
RUNTIME_MAP_GC_MIN_INTERVAL_MS,
|
|
763
|
+
);
|
|
764
|
+
const runtimeGc =
|
|
765
|
+
input.collectGarbage === undefined
|
|
766
|
+
? ((globalThis as { gc?: () => void }).gc?.bind(globalThis) ?? null)
|
|
767
|
+
: input.collectGarbage;
|
|
768
|
+
const gcObserver =
|
|
769
|
+
input.observeGc === undefined
|
|
770
|
+
? () => runtimeGcObserver().snapshot()
|
|
771
|
+
: input.observeGc;
|
|
772
|
+
const now = input.now ?? Date.now;
|
|
773
|
+
let heapLimitBytes: number | null = null;
|
|
774
|
+
let liveBytes: number | null = null;
|
|
775
|
+
let liveBaselineBytes: number | null = null;
|
|
776
|
+
let lastProbeAtMs: number | null = null;
|
|
777
|
+
let lastGcAtMs: number | null = null;
|
|
778
|
+
let probes = 0;
|
|
779
|
+
let cachedProbes = 0;
|
|
780
|
+
let gcCount = 0;
|
|
781
|
+
let gcTotalMs = 0;
|
|
782
|
+
|
|
783
|
+
const sampleHeap = (): boolean => {
|
|
784
|
+
const heap = heapProbe();
|
|
785
|
+
if (!heap) return false;
|
|
786
|
+
probes += 1;
|
|
787
|
+
heapLimitBytes = heap.heapLimitBytes;
|
|
788
|
+
liveBytes = heap.usedHeapBytes + Math.max(0, heap.externalBytes ?? 0);
|
|
789
|
+
// The baseline is the first reading this reader ever took, which for a run
|
|
790
|
+
// is taken before any row work: the same "before the work" meaning map
|
|
791
|
+
// admission gives it.
|
|
792
|
+
liveBaselineBytes ??= liveBytes;
|
|
793
|
+
lastProbeAtMs = now();
|
|
794
|
+
return true;
|
|
795
|
+
};
|
|
796
|
+
|
|
797
|
+
const occupancy = (): RuntimeMemoryOccupancy | null => {
|
|
798
|
+
if (heapLimitBytes === null || liveBytes === null) return null;
|
|
799
|
+
return runtimeMemoryOccupancy({
|
|
800
|
+
heapLimitBytes,
|
|
801
|
+
liveBytes,
|
|
802
|
+
liveBaselineBytes: liveBaselineBytes ?? liveBytes,
|
|
803
|
+
// Read every time, never cached: a reservation registered a microsecond
|
|
804
|
+
// ago must be visible to the next admission decision, or the lane admits
|
|
805
|
+
// twice against the same headroom.
|
|
806
|
+
ledger: input.memoryLedger.snapshot(),
|
|
807
|
+
...(input.footprintSampleRows === undefined
|
|
808
|
+
? {}
|
|
809
|
+
: { footprintSampleRows: input.footprintSampleRows }),
|
|
810
|
+
...(input.ownerReserveDefaultBytes === undefined
|
|
811
|
+
? {}
|
|
812
|
+
: { ownerReserveDefaultBytes: input.ownerReserveDefaultBytes }),
|
|
813
|
+
...(input.ownerReserveMinBytes === undefined
|
|
814
|
+
? {}
|
|
815
|
+
: { ownerReserveMinBytes: input.ownerReserveMinBytes }),
|
|
816
|
+
});
|
|
817
|
+
};
|
|
818
|
+
|
|
819
|
+
return {
|
|
820
|
+
read() {
|
|
821
|
+
const staleAtMs = lastProbeAtMs;
|
|
822
|
+
if (staleAtMs === null || now() - staleAtMs >= probeIntervalMs) {
|
|
823
|
+
if (!sampleHeap() && heapLimitBytes === null) return null;
|
|
824
|
+
} else {
|
|
825
|
+
cachedProbes += 1;
|
|
826
|
+
}
|
|
827
|
+
return occupancy();
|
|
828
|
+
},
|
|
829
|
+
collectAndRead() {
|
|
830
|
+
const startedAt = now();
|
|
831
|
+
const lastMajorAtMs =
|
|
832
|
+
gcObserver?.().byKind.major.lastCompletedAtMs ?? null;
|
|
833
|
+
if (
|
|
834
|
+
lastMajorAtMs !== null &&
|
|
835
|
+
startedAt - lastMajorAtMs < gcMinIntervalMs &&
|
|
836
|
+
lastMajorAtMs > (lastGcAtMs ?? Number.NEGATIVE_INFINITY)
|
|
837
|
+
) {
|
|
838
|
+
// V8 already swept inside the interval; re-probing reads live objects
|
|
839
|
+
// without paying for a second sweep of them.
|
|
840
|
+
lastGcAtMs = lastMajorAtMs;
|
|
841
|
+
return sampleHeap() ? occupancy() : null;
|
|
842
|
+
}
|
|
843
|
+
if (runtimeGc === null) return null;
|
|
844
|
+
if (lastGcAtMs !== null && startedAt - lastGcAtMs < gcMinIntervalMs) {
|
|
845
|
+
return null;
|
|
846
|
+
}
|
|
847
|
+
runtimeGc();
|
|
848
|
+
const finishedAt = now();
|
|
849
|
+
gcCount += 1;
|
|
850
|
+
gcTotalMs += Math.max(0, finishedAt - startedAt);
|
|
851
|
+
lastGcAtMs = finishedAt;
|
|
852
|
+
return sampleHeap() ? occupancy() : null;
|
|
853
|
+
},
|
|
854
|
+
snapshot: () => ({
|
|
855
|
+
probes,
|
|
856
|
+
cachedProbes,
|
|
857
|
+
gcCount,
|
|
858
|
+
gcTotalMs,
|
|
859
|
+
gcAvailable: runtimeGc !== null,
|
|
860
|
+
}),
|
|
861
|
+
};
|
|
862
|
+
}
|
|
863
|
+
|
|
864
|
+
/**
|
|
865
|
+
* Admit row resolver bodies against the bytes the runtime says it is holding.
|
|
866
|
+
* This is intentionally separate from provider admission: it owns only
|
|
867
|
+
* resident JS graphs, while AIMD continues to decide provider dispatch.
|
|
868
|
+
* Pending rows are waiters, not created resolver bodies, so a wide requested
|
|
869
|
+
* concurrency cannot create a one-shot heap spike before the first provider
|
|
870
|
+
* response arrives.
|
|
871
|
+
*
|
|
872
|
+
* Nothing here caps the map by a row count. The opening wave is bounded by
|
|
873
|
+
* bytes at construction, every later width is bytes divided by a per-row
|
|
874
|
+
* reserve, and settlement only paces how fast that width may be reached. A
|
|
875
|
+
* map of any size runs to completion; it just never holds more bytes than the
|
|
876
|
+
* sandbox has.
|
|
877
|
+
*
|
|
878
|
+
* Policy, in order:
|
|
879
|
+
* 1. Occupancy is `baseline + ledger.liveBytes x factor`. The ledger is the
|
|
880
|
+
* estimator and the heap probe is only a guard. `heapUsed + external` is
|
|
881
|
+
* not a live measurement: with a 512 MiB old space V8 lets garbage pile
|
|
882
|
+
* toward the limit before it runs a major GC, so between collections the
|
|
883
|
+
* probe reads mostly garbage. Pacing on it collapsed a 1,000-row map to
|
|
884
|
+
* one row at a time for 25 minutes with the calibration factor pinned at
|
|
885
|
+
* its clamp (jai, run 34074320144), because every reading said the
|
|
886
|
+
* sandbox was full of rows it had already finished.
|
|
887
|
+
* 2. Headroom above occupancy is divided by a per-row reserve. The reserve
|
|
888
|
+
* is the unmeasured prior until the ledger has settled enough rows to
|
|
889
|
+
* state a p90 per-row peak, then that peak scaled by the ledger's
|
|
890
|
+
* calibration factor. It is never a probe quotient: dividing heap growth
|
|
891
|
+
* by rows in flight read 0.9 MB/row with bodies on the wire and 48 MB/row
|
|
892
|
+
* over a draining tail, and admission acted on both.
|
|
893
|
+
* 3. Observed live crossing the target is a question, not a verdict. The
|
|
894
|
+
* answer is a collection: admission collects (at most once every
|
|
895
|
+
* `RUNTIME_MAP_GC_MIN_INTERVAL_MS`), re-probes, and holds only if the
|
|
896
|
+
* post-collection reading is still over. The same post-collection probes
|
|
897
|
+
* are the only ones the ledger calibrates from, so the factor describes
|
|
898
|
+
* live objects instead of uncollected garbage.
|
|
899
|
+
* 4. RSS near the cgroup limit is a hard guard, and a watchdog pressure
|
|
900
|
+
* signal from outside the process is another. Both pause new admission,
|
|
901
|
+
* release the map's durable results, collect, and reprobe; both release
|
|
902
|
+
* only once live memory has visibly dropped. The last safe width is
|
|
903
|
+
* retained while paused, so an episode cannot turn the tail serial.
|
|
904
|
+
*/
|
|
905
|
+
export function createRuntimeMapHeapAdmission(input: {
|
|
906
|
+
maxConcurrency: number;
|
|
907
|
+
heapProbe?: () => RuntimeMapHeapSnapshot | null;
|
|
908
|
+
initialConcurrency?: number;
|
|
909
|
+
probeIntervalMs?: number;
|
|
910
|
+
/** Settled rows the ledger needs before its p90 per-row peak is trusted. */
|
|
911
|
+
footprintSampleRows?: number;
|
|
912
|
+
/** Retained for the snapshot field of the same name; see its documentation. */
|
|
913
|
+
footprintSampleMinActiveRows?: number;
|
|
914
|
+
/** Per-row reserve used until the ledger has priced a row. */
|
|
915
|
+
rowReserveBytes?: number;
|
|
916
|
+
/** Floor under the ledger reserve so one light row cannot launch a wave. */
|
|
917
|
+
rowReserveMinBytes?: number;
|
|
918
|
+
/**
|
|
919
|
+
* Keep a decoded provider body out of row admission until the response is
|
|
920
|
+
* about to be read. A shared payload gate then reserves it by bytes, so rows
|
|
921
|
+
* waiting on provider residence retain only compact local state here.
|
|
922
|
+
*/
|
|
923
|
+
deferProviderBodyReservationToMaterialization?: boolean;
|
|
924
|
+
/** Compact waiting-row floor retained in deferred-body diagnostics. */
|
|
925
|
+
waitingRowReserveBytes?: number;
|
|
926
|
+
/** Rows admitted per probe while growing toward the safe width. */
|
|
927
|
+
growthStepRows?: number;
|
|
928
|
+
/** Rows the window may grow past the opening width per settled row. */
|
|
929
|
+
rowsPerSettledRow?: number;
|
|
930
|
+
/**
|
|
931
|
+
* The runtime's byte ledger. Admission calibrates it on every probe and
|
|
932
|
+
* prices a row from its settled owners. Without one the unmeasured prior
|
|
933
|
+
* stands for the whole map, which is honest: nothing measured a row.
|
|
934
|
+
*/
|
|
935
|
+
memoryLedger?: RuntimeMemoryLedger | null;
|
|
936
|
+
/**
|
|
937
|
+
* Gives the map a chance to discard results which have already crossed its
|
|
938
|
+
* durable egress boundary before memory constrains more row work.
|
|
939
|
+
*/
|
|
940
|
+
onMemoryPressure?: () => void;
|
|
941
|
+
/** Watchdog pressure subscription. Overridable for deterministic tests. */
|
|
942
|
+
subscribeMemoryPressure?: (listener: () => void) => () => void;
|
|
943
|
+
/**
|
|
944
|
+
* Recent event-loop lag in milliseconds. While it is at or above
|
|
945
|
+
* `eventLoopLagHoldMs` admission holds exactly as it does for memory
|
|
946
|
+
* pressure: active rows drain as they settle and nothing new starts.
|
|
947
|
+
*/
|
|
948
|
+
getEventLoopLagMs?: () => number | null;
|
|
949
|
+
eventLoopLagHoldMs?: number;
|
|
950
|
+
eventLoopAdjustIntervalMs?: number;
|
|
951
|
+
/** Publishes the snapshot the watchdog attaches to a memory abort. */
|
|
952
|
+
reportLedger?: (snapshot: RuntimeMapHeapAdmissionSnapshot) => void;
|
|
953
|
+
/**
|
|
954
|
+
* Explicit collection. Omit to use the runner's own `--expose-gc` hook; pass
|
|
955
|
+
* null to state that this runner cannot collect, which changes what
|
|
956
|
+
* admission is allowed to conclude from a probe.
|
|
957
|
+
*/
|
|
958
|
+
collectGarbage?: (() => void) | null;
|
|
959
|
+
/** Minimum wall time between admission-requested collections. */
|
|
960
|
+
gcMinIntervalMs?: number;
|
|
961
|
+
/**
|
|
962
|
+
* Reads the collector tally. Omit to use the process-wide observer; pass
|
|
963
|
+
* null to run without one (its cost then reads as unobserved, not as zero).
|
|
964
|
+
* `lastCompletedAtMs` is on the same clock as `now`, so a test that injects
|
|
965
|
+
* `now` must inject this too rather than let the two disagree.
|
|
966
|
+
*/
|
|
967
|
+
observeGc?: (() => RuntimeGcObservation) | null;
|
|
968
|
+
/**
|
|
969
|
+
* The sandbox memory limit the RSS guard is derived from. Omit to read the
|
|
970
|
+
* cgroup; pass null to state explicitly that no limit is known.
|
|
971
|
+
*/
|
|
972
|
+
cgroupLimitBytes?: number | null;
|
|
973
|
+
/** CPU cores available to the runner. Omit to read the cgroup quota. */
|
|
974
|
+
cpuCount?: number | null;
|
|
975
|
+
/** Clock for the collection cooldown. Injectable for deterministic tests. */
|
|
976
|
+
now?: () => number;
|
|
977
|
+
}): RuntimeMapHeapAdmission {
|
|
978
|
+
const maxConcurrency = positiveBudget(input.maxConcurrency, 1);
|
|
979
|
+
const initialConcurrency = Math.min(
|
|
980
|
+
maxConcurrency,
|
|
981
|
+
positiveBudget(
|
|
982
|
+
input.initialConcurrency,
|
|
983
|
+
RUNTIME_MAP_HEAP_INITIAL_CONCURRENCY,
|
|
984
|
+
),
|
|
985
|
+
);
|
|
986
|
+
const probeIntervalMs = positiveBudget(
|
|
987
|
+
input.probeIntervalMs,
|
|
988
|
+
RUNTIME_MAP_HEAP_PROBE_INTERVAL_MS,
|
|
989
|
+
);
|
|
990
|
+
const footprintSampleRows = positiveBudget(
|
|
991
|
+
input.footprintSampleRows,
|
|
992
|
+
RUNTIME_MAP_FOOTPRINT_SAMPLE_ROWS,
|
|
993
|
+
);
|
|
994
|
+
const footprintSampleMinActiveRows = positiveBudget(
|
|
995
|
+
input.footprintSampleMinActiveRows,
|
|
996
|
+
Math.max(
|
|
997
|
+
RUNTIME_MAP_ROW_FOOTPRINT_SAMPLE_MIN_ACTIVE_ROWS,
|
|
998
|
+
Math.floor(
|
|
999
|
+
initialConcurrency /
|
|
1000
|
+
RUNTIME_MAP_ROW_FOOTPRINT_SAMPLE_MIN_ACTIVE_ROWS_DIVISOR,
|
|
1001
|
+
),
|
|
1002
|
+
),
|
|
1003
|
+
);
|
|
1004
|
+
const defaultRowReserveBytes = positiveBudget(
|
|
1005
|
+
input.rowReserveBytes,
|
|
1006
|
+
RUNTIME_MAP_ROW_RESERVE_DEFAULT_BYTES,
|
|
1007
|
+
);
|
|
1008
|
+
const growthStepRows = positiveBudget(
|
|
1009
|
+
input.growthStepRows,
|
|
1010
|
+
RUNTIME_MAP_HEAP_GROWTH_STEP_MAX,
|
|
1011
|
+
);
|
|
1012
|
+
const rowReserveMinBytes = positiveBudget(
|
|
1013
|
+
input.rowReserveMinBytes,
|
|
1014
|
+
RUNTIME_MAP_ROW_RESERVE_MIN_BYTES,
|
|
1015
|
+
);
|
|
1016
|
+
const providerBodyReservationDeferred =
|
|
1017
|
+
input.deferProviderBodyReservationToMaterialization === true;
|
|
1018
|
+
const waitingRowReserveBytes = providerBodyReservationDeferred
|
|
1019
|
+
? positiveBudget(
|
|
1020
|
+
input.waitingRowReserveBytes,
|
|
1021
|
+
RUNTIME_MAP_ROW_RESERVE_MIN_BYTES,
|
|
1022
|
+
)
|
|
1023
|
+
: null;
|
|
1024
|
+
const rowsPerSettledRow = positiveBudget(
|
|
1025
|
+
input.rowsPerSettledRow,
|
|
1026
|
+
RUNTIME_MAP_ROWS_PER_SETTLED_ROW,
|
|
1027
|
+
);
|
|
1028
|
+
const heapProbe = input.heapProbe ?? readRuntimeMapHeapSnapshot;
|
|
1029
|
+
const memoryLedger = input.memoryLedger ?? null;
|
|
1030
|
+
const subscribeMemoryPressure =
|
|
1031
|
+
input.subscribeMemoryPressure ??
|
|
1032
|
+
((listener: () => void) =>
|
|
1033
|
+
subscribeRuntimeMemoryPressure(() => listener()));
|
|
1034
|
+
const reportLedger = input.reportLedger ?? publishRuntimeMemoryLedger;
|
|
1035
|
+
let lastLedgerPublishMs = 0;
|
|
1036
|
+
// A runner without `--expose-gc` cannot separate live bytes from garbage.
|
|
1037
|
+
// That is not a failure mode to paper over: it changes what a probe proves,
|
|
1038
|
+
// so admission records it and calibrates differently rather than pretending
|
|
1039
|
+
// it collected.
|
|
1040
|
+
const runtimeGc =
|
|
1041
|
+
input.collectGarbage === undefined
|
|
1042
|
+
? ((globalThis as { gc?: () => void }).gc?.bind(globalThis) ?? null)
|
|
1043
|
+
: input.collectGarbage;
|
|
1044
|
+
const gcAvailable = runtimeGc !== null;
|
|
1045
|
+
const gcMinIntervalMs = positiveBudget(
|
|
1046
|
+
input.gcMinIntervalMs,
|
|
1047
|
+
RUNTIME_MAP_GC_MIN_INTERVAL_MS,
|
|
1048
|
+
);
|
|
1049
|
+
const now = input.now ?? Date.now;
|
|
1050
|
+
const cgroupLimitBytes =
|
|
1051
|
+
input.cgroupLimitBytes === undefined
|
|
1052
|
+
? readCgroupMemoryLimitBytes()
|
|
1053
|
+
: input.cgroupLimitBytes;
|
|
1054
|
+
const cpuCount =
|
|
1055
|
+
input.cpuCount === undefined ? readCgroupCpuLimitCores() : input.cpuCount;
|
|
1056
|
+
const gcObserver =
|
|
1057
|
+
input.observeGc === undefined
|
|
1058
|
+
? () => runtimeGcObserver().snapshot()
|
|
1059
|
+
: input.observeGc;
|
|
1060
|
+
let gcExplicitCount = 0;
|
|
1061
|
+
let lastNaturalMajorCalibratedAtMs: number | null = null;
|
|
1062
|
+
let gcExplicitTotalMs = 0;
|
|
1063
|
+
let lastGcAtMs: number | null = null;
|
|
1064
|
+
/**
|
|
1065
|
+
* The last collected verdict on observed live memory. Sticky between
|
|
1066
|
+
* collections: inside the cooldown a probe carries no new evidence, so it
|
|
1067
|
+
* may neither arm the hold on fresh garbage nor release one that a
|
|
1068
|
+
* collection confirmed.
|
|
1069
|
+
*/
|
|
1070
|
+
let observedOverTargetConfirmed = false;
|
|
1071
|
+
/** Set by the watchdog handler so the next probe counts as collected. */
|
|
1072
|
+
let nextProbeIsPostCollection = false;
|
|
1073
|
+
let concurrency = initialConcurrency;
|
|
1074
|
+
let activeRows = 0;
|
|
1075
|
+
let activeRowPeak = 0;
|
|
1076
|
+
let admittedRows = 0;
|
|
1077
|
+
let settledRows = 0;
|
|
1078
|
+
let heapLimitBytes: number | null = null;
|
|
1079
|
+
let heapTargetBytes: number | null = null;
|
|
1080
|
+
let heapUsedBytes: number | null = null;
|
|
1081
|
+
let heapUsedPeakBytes: number | null = null;
|
|
1082
|
+
let liveBytes: number | null = null;
|
|
1083
|
+
let liveBaselineBytes: number | null = null;
|
|
1084
|
+
let livePeakBytes: number | null = null;
|
|
1085
|
+
let rowReserveBytes: number | null = null;
|
|
1086
|
+
let measuredRowBytes: number | null = null;
|
|
1087
|
+
let footprintSamples = 0;
|
|
1088
|
+
let predictedLiveBytes: number | null = null;
|
|
1089
|
+
let occupiedBytes: number | null = null;
|
|
1090
|
+
let memorySnapshot: RuntimeMemoryLedgerSnapshot | null = null;
|
|
1091
|
+
let externalPressureSignals = 0;
|
|
1092
|
+
let admissionHolds = 0;
|
|
1093
|
+
let admissionPaused = false;
|
|
1094
|
+
// Live memory observed when the watchdog last signalled. The hold releases
|
|
1095
|
+
// only once live memory has fallen below it, so a signal cannot be answered
|
|
1096
|
+
// by a probe taken before the collection it asked for had any effect.
|
|
1097
|
+
let externalHoldLiveBytes: number | null = null;
|
|
1098
|
+
let limitedByHeap = false;
|
|
1099
|
+
let residentGuardBytes: number | null = null;
|
|
1100
|
+
let residentBytes: number | null = null;
|
|
1101
|
+
let residentPeakBytes: number | null = null;
|
|
1102
|
+
let limitedByResidentMemory = false;
|
|
1103
|
+
let memoryPressureActive = false;
|
|
1104
|
+
let heapPressureEvents = 0;
|
|
1105
|
+
let residentPressureEvents = 0;
|
|
1106
|
+
let limitedByEventLoop = false;
|
|
1107
|
+
let eventLoopPressureEvents = 0;
|
|
1108
|
+
let eventLoopPressureActive = false;
|
|
1109
|
+
let eventLoopLagMs: number | null = null;
|
|
1110
|
+
const eventLoopLagHoldMs = positiveBudget(
|
|
1111
|
+
input.eventLoopLagHoldMs,
|
|
1112
|
+
RUNTIME_MAP_EVENT_LOOP_LAG_HOLD_MS,
|
|
1113
|
+
);
|
|
1114
|
+
const eventLoopAdjustIntervalMs =
|
|
1115
|
+
input.eventLoopAdjustIntervalMs !== undefined &&
|
|
1116
|
+
Number.isFinite(input.eventLoopAdjustIntervalMs) &&
|
|
1117
|
+
input.eventLoopAdjustIntervalMs >= 0
|
|
1118
|
+
? input.eventLoopAdjustIntervalMs
|
|
1119
|
+
: RUNTIME_MAP_EVENT_LOOP_ADJUST_INTERVAL_MS;
|
|
1120
|
+
let eventLoopCeiling = maxConcurrency;
|
|
1121
|
+
let lastEventLoopAdjustAtMs: number | null = null;
|
|
1122
|
+
let memoryPressureEvents = 0;
|
|
1123
|
+
let closed = false;
|
|
1124
|
+
let closeError: unknown = null;
|
|
1125
|
+
const waiters: Array<{
|
|
1126
|
+
resolve: (lease: RuntimeMapHeapAdmissionLease) => void;
|
|
1127
|
+
reject: (error: unknown) => void;
|
|
1128
|
+
}> = [];
|
|
1129
|
+
{
|
|
1130
|
+
// The baseline must predate the first admitted row, or the ledger's
|
|
1131
|
+
// calibration would credit early bodies to the map's own overhead. It must
|
|
1132
|
+
// also be a collected reading: every calibration sample below is taken
|
|
1133
|
+
// after a collection, and subtracting an uncollected baseline from one
|
|
1134
|
+
// would charge the map's own garbage against it and read the factor low —
|
|
1135
|
+
// the direction that over-admits. One collection per map buys a baseline
|
|
1136
|
+
// the samples are actually comparable to.
|
|
1137
|
+
if (runtimeGc !== null) {
|
|
1138
|
+
const startedAt = now();
|
|
1139
|
+
runtimeGc();
|
|
1140
|
+
const finishedAt = now();
|
|
1141
|
+
gcExplicitCount += 1;
|
|
1142
|
+
gcExplicitTotalMs += Math.max(0, finishedAt - startedAt);
|
|
1143
|
+
lastGcAtMs = finishedAt;
|
|
1144
|
+
}
|
|
1145
|
+
const initial = heapProbe();
|
|
1146
|
+
if (initial) {
|
|
1147
|
+
liveBaselineBytes =
|
|
1148
|
+
initial.usedHeapBytes + Math.max(0, initial.externalBytes ?? 0);
|
|
1149
|
+
// The opening width is spent before any probe: every row worker calls
|
|
1150
|
+
// acquire() at once, and a granted lease cannot be taken back. So the
|
|
1151
|
+
// unmeasured reserve has to bound the first wave too, not just the
|
|
1152
|
+
// width the first probe computes — otherwise a heavy shape puts the
|
|
1153
|
+
// full opening width of bodies in flight on a sandbox with room for
|
|
1154
|
+
// half of them (jai: OOM at 205 settled rows, 256 in flight from the
|
|
1155
|
+
// first probe on).
|
|
1156
|
+
const openingHeadroomBytes =
|
|
1157
|
+
Math.floor(
|
|
1158
|
+
initial.heapLimitBytes * RUNTIME_MAP_HEAP_TARGET_UTILIZATION,
|
|
1159
|
+
) - liveBaselineBytes;
|
|
1160
|
+
// A body is deferred only after a response has reached its byte gate;
|
|
1161
|
+
// before the first representative row settles, this map has not yet
|
|
1162
|
+
// measured what that gate will admit or how much surrounding resolver
|
|
1163
|
+
// state that body retains. The compact waiting-row allowance describes
|
|
1164
|
+
// one row already known to be waiting on I/O, not an unknown first wave.
|
|
1165
|
+
// Starting the latter on 64 KiB turns a 1 GiB sandbox into its arbitrary
|
|
1166
|
+
// 256-row cap before a single body can price the shape (JAI did exactly
|
|
1167
|
+
// that). The unknown full-row prior bounds that one irreversible wave.
|
|
1168
|
+
const openingRowReserveBytes = defaultRowReserveBytes;
|
|
1169
|
+
concurrency = Math.max(
|
|
1170
|
+
1,
|
|
1171
|
+
Math.min(
|
|
1172
|
+
concurrency,
|
|
1173
|
+
Math.floor(openingHeadroomBytes / openingRowReserveBytes),
|
|
1174
|
+
),
|
|
1175
|
+
);
|
|
1176
|
+
}
|
|
1177
|
+
}
|
|
1178
|
+
|
|
1179
|
+
const snapshot = (): RuntimeMapHeapAdmissionSnapshot => {
|
|
1180
|
+
const gcObservation = gcObserver?.() ?? null;
|
|
1181
|
+
return {
|
|
1182
|
+
activeRows,
|
|
1183
|
+
activeRowPeak,
|
|
1184
|
+
admittedRows,
|
|
1185
|
+
settledRows,
|
|
1186
|
+
waitingRows: waiters.length,
|
|
1187
|
+
concurrency,
|
|
1188
|
+
maxConcurrency,
|
|
1189
|
+
heapLimitBytes,
|
|
1190
|
+
heapTargetBytes,
|
|
1191
|
+
heapUsedBytes,
|
|
1192
|
+
heapUsedPeakBytes,
|
|
1193
|
+
liveBytes,
|
|
1194
|
+
liveBaselineBytes,
|
|
1195
|
+
livePeakBytes,
|
|
1196
|
+
rowReserveBytes,
|
|
1197
|
+
providerBodyReservationDeferred,
|
|
1198
|
+
waitingRowReserveBytes,
|
|
1199
|
+
measuredRowBytes,
|
|
1200
|
+
footprintSamples,
|
|
1201
|
+
footprintSampleMinActiveRows,
|
|
1202
|
+
predictedLiveBytes,
|
|
1203
|
+
occupiedBytes,
|
|
1204
|
+
memory:
|
|
1205
|
+
memorySnapshot === null
|
|
1206
|
+
? null
|
|
1207
|
+
: { ...memorySnapshot, gc: gcObservation },
|
|
1208
|
+
externalPressureSignals,
|
|
1209
|
+
admissionHolds,
|
|
1210
|
+
admissionPaused,
|
|
1211
|
+
gc: gcObservation,
|
|
1212
|
+
gcAvailable,
|
|
1213
|
+
gcExplicitCount,
|
|
1214
|
+
gcExplicitTotalMs,
|
|
1215
|
+
lastGcAgeMs: lastGcAtMs === null ? null : Math.max(0, now() - lastGcAtMs),
|
|
1216
|
+
limitedByHeap,
|
|
1217
|
+
cgroupLimitBytes,
|
|
1218
|
+
cpuCount,
|
|
1219
|
+
residentGuardBytes,
|
|
1220
|
+
residentBytes,
|
|
1221
|
+
residentPeakBytes,
|
|
1222
|
+
limitedByResidentMemory,
|
|
1223
|
+
heapPressureEvents,
|
|
1224
|
+
residentPressureEvents,
|
|
1225
|
+
memoryPressureEvents,
|
|
1226
|
+
limitedByEventLoop,
|
|
1227
|
+
eventLoopPressureEvents,
|
|
1228
|
+
eventLoopCeiling,
|
|
1229
|
+
eventLoopLagMs,
|
|
1230
|
+
};
|
|
1231
|
+
};
|
|
1232
|
+
|
|
1233
|
+
const drain = (
|
|
1234
|
+
maximumAdmissions = Number.POSITIVE_INFINITY,
|
|
1235
|
+
allowWhilePaused = false,
|
|
1236
|
+
) => {
|
|
1237
|
+
let admissions = 0;
|
|
1238
|
+
while (
|
|
1239
|
+
!closed &&
|
|
1240
|
+
// A pressure pause retains exactly one progress row after the active
|
|
1241
|
+
// set drains. The waiter may arrive after the reconciliation that
|
|
1242
|
+
// noticed the zero width, so make the allowance in drain rather than
|
|
1243
|
+
// relying on a waiter already existing at that instant.
|
|
1244
|
+
(allowWhilePaused || !admissionPaused || activeRows === 0) &&
|
|
1245
|
+
admissions < maximumAdmissions &&
|
|
1246
|
+
activeRows < concurrency &&
|
|
1247
|
+
waiters.length > 0
|
|
1248
|
+
) {
|
|
1249
|
+
const waiter = waiters.shift();
|
|
1250
|
+
if (!waiter) return;
|
|
1251
|
+
activeRows += 1;
|
|
1252
|
+
activeRowPeak = Math.max(activeRowPeak, activeRows);
|
|
1253
|
+
admittedRows += 1;
|
|
1254
|
+
admissions += 1;
|
|
1255
|
+
let released = false;
|
|
1256
|
+
waiter.resolve({
|
|
1257
|
+
release() {
|
|
1258
|
+
if (released) return;
|
|
1259
|
+
released = true;
|
|
1260
|
+
activeRows = Math.max(0, activeRows - 1);
|
|
1261
|
+
settledRows += 1;
|
|
1262
|
+
// Re-pricing here is deliberate and load-bearing: a settling row is
|
|
1263
|
+
// the only evidence the ledger gets about what a row of this shape
|
|
1264
|
+
// actually cost, and `probeIntervalMs` may be far longer than a row
|
|
1265
|
+
// (the admission tests set it to 60s precisely to prove settlement
|
|
1266
|
+
// drives pricing, not the timer). The per-row cost is kept sane by
|
|
1267
|
+
// memoizing the ledger's percentiles rather than by skipping this.
|
|
1268
|
+
reconcile();
|
|
1269
|
+
},
|
|
1270
|
+
});
|
|
1271
|
+
}
|
|
1272
|
+
};
|
|
1273
|
+
|
|
1274
|
+
const pauseAdmission = () => {
|
|
1275
|
+
admissionHolds += 1;
|
|
1276
|
+
admissionPaused = true;
|
|
1277
|
+
};
|
|
1278
|
+
|
|
1279
|
+
const applyHeapSample = (heap: RuntimeMapHeapSnapshot): HeapReading => {
|
|
1280
|
+
heapLimitBytes = heap.heapLimitBytes;
|
|
1281
|
+
heapUsedBytes = heap.usedHeapBytes;
|
|
1282
|
+
heapUsedPeakBytes = Math.max(heapUsedPeakBytes ?? 0, heap.usedHeapBytes);
|
|
1283
|
+
heapTargetBytes = Math.floor(
|
|
1284
|
+
heap.heapLimitBytes * RUNTIME_MAP_HEAP_TARGET_UTILIZATION,
|
|
1285
|
+
);
|
|
1286
|
+
liveBytes = heap.usedHeapBytes + Math.max(0, heap.externalBytes ?? 0);
|
|
1287
|
+
liveBaselineBytes ??= liveBytes;
|
|
1288
|
+
livePeakBytes = Math.max(livePeakBytes ?? 0, liveBytes);
|
|
1289
|
+
residentBytes = heap.residentBytes ?? null;
|
|
1290
|
+
residentPeakBytes =
|
|
1291
|
+
residentBytes == null
|
|
1292
|
+
? residentPeakBytes
|
|
1293
|
+
: Math.max(residentPeakBytes ?? 0, residentBytes);
|
|
1294
|
+
// The sandbox kills on RSS against the cgroup, so the guard is a share of
|
|
1295
|
+
// the cgroup limit whenever the kernel will state one. heap_size_limit is
|
|
1296
|
+
// only the fallback, and then at a multiple rather than a fraction: the
|
|
1297
|
+
// runner's old space is about half the cgroup, so RSS legitimately runs
|
|
1298
|
+
// well above the heap limit.
|
|
1299
|
+
residentGuardBytes =
|
|
1300
|
+
cgroupLimitBytes === null
|
|
1301
|
+
? Math.floor(
|
|
1302
|
+
heap.heapLimitBytes * RUNTIME_MAP_RESIDENT_GUARD_HEAP_MULTIPLE,
|
|
1303
|
+
)
|
|
1304
|
+
: Math.floor(
|
|
1305
|
+
cgroupLimitBytes * RUNTIME_MAP_RESIDENT_GUARD_CGROUP_FRACTION,
|
|
1306
|
+
);
|
|
1307
|
+
return {
|
|
1308
|
+
heapLimitBytes: heap.heapLimitBytes,
|
|
1309
|
+
heapTargetBytes,
|
|
1310
|
+
liveBytes,
|
|
1311
|
+
liveBaselineBytes: liveBaselineBytes ?? liveBytes,
|
|
1312
|
+
residentBytes,
|
|
1313
|
+
residentGuardBytes,
|
|
1314
|
+
};
|
|
1315
|
+
};
|
|
1316
|
+
|
|
1317
|
+
/**
|
|
1318
|
+
* Answer the garbage question with a collected reading.
|
|
1319
|
+
*
|
|
1320
|
+
* V8's own major collections count: if one finished inside the interval the
|
|
1321
|
+
* heap has already been swept, so admission re-probes instead of paying
|
|
1322
|
+
* 0.3-1 s of event-loop lag for a second sweep of the same objects. An
|
|
1323
|
+
* explicit collection is the fallback, and it is rate limited because on a
|
|
1324
|
+
* 1-CPU sandbox it is one of the most expensive things this process can do.
|
|
1325
|
+
*
|
|
1326
|
+
* Returns the post-collection reading, or null when no collection happened
|
|
1327
|
+
* and the caller's reading still stands.
|
|
1328
|
+
*/
|
|
1329
|
+
const collectAndReprobe = (): HeapReading | null => {
|
|
1330
|
+
const startedAt = now();
|
|
1331
|
+
const lastMajorAtMs = gcObserver?.().byKind.major.lastCompletedAtMs ?? null;
|
|
1332
|
+
if (
|
|
1333
|
+
lastMajorAtMs !== null &&
|
|
1334
|
+
startedAt - lastMajorAtMs < gcMinIntervalMs &&
|
|
1335
|
+
lastMajorAtMs > (lastGcAtMs ?? Number.NEGATIVE_INFINITY)
|
|
1336
|
+
) {
|
|
1337
|
+
lastGcAtMs = lastMajorAtMs;
|
|
1338
|
+
const swept = heapProbe();
|
|
1339
|
+
return swept ? applyHeapSample(swept) : null;
|
|
1340
|
+
}
|
|
1341
|
+
if (runtimeGc === null) return null;
|
|
1342
|
+
if (lastGcAtMs !== null && startedAt - lastGcAtMs < gcMinIntervalMs) {
|
|
1343
|
+
return null;
|
|
1344
|
+
}
|
|
1345
|
+
runtimeGc();
|
|
1346
|
+
const finishedAt = now();
|
|
1347
|
+
gcExplicitCount += 1;
|
|
1348
|
+
gcExplicitTotalMs += Math.max(0, finishedAt - startedAt);
|
|
1349
|
+
lastGcAtMs = finishedAt;
|
|
1350
|
+
const collected = heapProbe();
|
|
1351
|
+
return collected ? applyHeapSample(collected) : null;
|
|
1352
|
+
};
|
|
1353
|
+
|
|
1354
|
+
/**
|
|
1355
|
+
* Memory headroom bounds the width; settlement paces how fast it may be
|
|
1356
|
+
* reached, because only a finished row proves the shape drains.
|
|
1357
|
+
*/
|
|
1358
|
+
const applyWidth = (admissibleRows: number): void => {
|
|
1359
|
+
const settledAllowance =
|
|
1360
|
+
initialConcurrency + settledRows * rowsPerSettledRow;
|
|
1361
|
+
const safeWidth = Math.max(
|
|
1362
|
+
1,
|
|
1363
|
+
Math.min(
|
|
1364
|
+
maxConcurrency,
|
|
1365
|
+
activeRows + admissibleRows,
|
|
1366
|
+
settledAllowance,
|
|
1367
|
+
eventLoopCeiling,
|
|
1368
|
+
),
|
|
1369
|
+
);
|
|
1370
|
+
concurrency =
|
|
1371
|
+
safeWidth >= concurrency
|
|
1372
|
+
? Math.min(safeWidth, concurrency + growthStepRows)
|
|
1373
|
+
: safeWidth;
|
|
1374
|
+
};
|
|
1375
|
+
|
|
1376
|
+
const reconcile = () => {
|
|
1377
|
+
if (closed) return;
|
|
1378
|
+
const heap = heapProbe();
|
|
1379
|
+
if (!heap) {
|
|
1380
|
+
admissionPaused = false;
|
|
1381
|
+
concurrency = maxConcurrency;
|
|
1382
|
+
drain();
|
|
1383
|
+
return;
|
|
1384
|
+
}
|
|
1385
|
+
let reading = applyHeapSample(heap);
|
|
1386
|
+
// A collection the watchdog just asked for makes this probe a collected
|
|
1387
|
+
// one, which is the only kind the ledger may calibrate from.
|
|
1388
|
+
let postCollectionSample = nextProbeIsPostCollection;
|
|
1389
|
+
nextProbeIsPostCollection = false;
|
|
1390
|
+
// V8's own major collections count too: a probe taken right after one
|
|
1391
|
+
// reads live objects, not garbage, so a light map that never crosses the
|
|
1392
|
+
// target still calibrates instead of keeping the prior for its whole life.
|
|
1393
|
+
const naturalMajorAtMs =
|
|
1394
|
+
gcObserver?.().byKind.major.lastCompletedAtMs ?? null;
|
|
1395
|
+
if (
|
|
1396
|
+
naturalMajorAtMs !== null &&
|
|
1397
|
+
naturalMajorAtMs >
|
|
1398
|
+
(lastNaturalMajorCalibratedAtMs ?? Number.NEGATIVE_INFINITY)
|
|
1399
|
+
) {
|
|
1400
|
+
lastNaturalMajorCalibratedAtMs = naturalMajorAtMs;
|
|
1401
|
+
postCollectionSample = true;
|
|
1402
|
+
}
|
|
1403
|
+
|
|
1404
|
+
// Observed live over the target is a question about garbage, not a verdict
|
|
1405
|
+
// about rows. Collect, re-probe, and let the post-collection reading
|
|
1406
|
+
// answer it. Without a collection the reading cannot be trusted either
|
|
1407
|
+
// way, so a runner that cannot collect keeps the old, conservative rule:
|
|
1408
|
+
// over target is over target.
|
|
1409
|
+
if (reading.liveBytes < reading.heapTargetBytes) {
|
|
1410
|
+
// Under target even counting garbage: there is nothing to confirm.
|
|
1411
|
+
observedOverTargetConfirmed = false;
|
|
1412
|
+
} else {
|
|
1413
|
+
const collected = collectAndReprobe();
|
|
1414
|
+
if (collected) {
|
|
1415
|
+
reading = collected;
|
|
1416
|
+
postCollectionSample = true;
|
|
1417
|
+
}
|
|
1418
|
+
if (postCollectionSample || !gcAvailable) {
|
|
1419
|
+
observedOverTargetConfirmed =
|
|
1420
|
+
reading.liveBytes >= reading.heapTargetBytes;
|
|
1421
|
+
}
|
|
1422
|
+
// Inside the collection cooldown there is no new evidence, so the last
|
|
1423
|
+
// collected verdict stands. That is what keeps a confirmed hold held
|
|
1424
|
+
// and keeps a disproved one from re-arming on fresh garbage.
|
|
1425
|
+
}
|
|
1426
|
+
|
|
1427
|
+
// Calibrate only from collected memory. A pre-collection probe divides
|
|
1428
|
+
// garbage by the ledger and reads the clamp — that is how a 1,000-row map
|
|
1429
|
+
// ended up pricing every row at six times its real cost.
|
|
1430
|
+
// The ledger only sees objects that have reached an owning lane. A
|
|
1431
|
+
// deferred-body map still needs a bounded admission promise between a
|
|
1432
|
+
// response arriving and the body gate taking it; otherwise a learned
|
|
1433
|
+
// multi-megabyte row can be widened as if it cost the 64 KiB waiting
|
|
1434
|
+
// prior, and no later pressure signal can retract that launch wave. The
|
|
1435
|
+
// body gate remains the authority for responses larger than the unknown
|
|
1436
|
+
// full-row prior, so a slow provider does not become a synthetic
|
|
1437
|
+
// row-concurrency ceiling. The calculation below still subtracts
|
|
1438
|
+
// registered growth to avoid charging observed bytes twice.
|
|
1439
|
+
const preCalibrationSnapshot = memoryLedger?.snapshot() ?? null;
|
|
1440
|
+
const preCalibrationOccupancy = runtimeMemoryOccupancy({
|
|
1441
|
+
heapLimitBytes: reading.heapLimitBytes,
|
|
1442
|
+
liveBytes: reading.liveBytes,
|
|
1443
|
+
liveBaselineBytes: reading.liveBaselineBytes,
|
|
1444
|
+
ledger: preCalibrationSnapshot,
|
|
1445
|
+
footprintSampleRows,
|
|
1446
|
+
ownerReserveDefaultBytes: defaultRowReserveBytes,
|
|
1447
|
+
ownerReserveMinBytes: rowReserveMinBytes,
|
|
1448
|
+
});
|
|
1449
|
+
const reserveForAdmittedRow = (occupancy: RuntimeMemoryOccupancy) =>
|
|
1450
|
+
providerBodyReservationDeferred
|
|
1451
|
+
? Math.max(
|
|
1452
|
+
waitingRowReserveBytes ?? rowReserveMinBytes,
|
|
1453
|
+
Math.min(defaultRowReserveBytes, occupancy.ownerReserveBytes),
|
|
1454
|
+
)
|
|
1455
|
+
: occupancy.ownerReserveBytes;
|
|
1456
|
+
const admittedRowReserveBytes =
|
|
1457
|
+
activeRows * reserveForAdmittedRow(preCalibrationOccupancy);
|
|
1458
|
+
if (postCollectionSample || !gcAvailable) {
|
|
1459
|
+
memoryLedger?.calibrate({
|
|
1460
|
+
observedLiveBytes: reading.liveBytes,
|
|
1461
|
+
baselineBytes: reading.liveBaselineBytes,
|
|
1462
|
+
admittedRowReserveBytes,
|
|
1463
|
+
});
|
|
1464
|
+
}
|
|
1465
|
+
memorySnapshot = memoryLedger?.snapshot() ?? null;
|
|
1466
|
+
// The ledger is the occupancy estimator; the probe above is the guard.
|
|
1467
|
+
// Stated once, in `runtimeMemoryOccupancy`, so the receipt hydration lane
|
|
1468
|
+
// admits against the same arithmetic instead of a second opinion.
|
|
1469
|
+
const occupancy = runtimeMemoryOccupancy({
|
|
1470
|
+
heapLimitBytes: reading.heapLimitBytes,
|
|
1471
|
+
liveBytes: reading.liveBytes,
|
|
1472
|
+
liveBaselineBytes: reading.liveBaselineBytes,
|
|
1473
|
+
ledger: memorySnapshot,
|
|
1474
|
+
footprintSampleRows,
|
|
1475
|
+
ownerReserveDefaultBytes: defaultRowReserveBytes,
|
|
1476
|
+
ownerReserveMinBytes: rowReserveMinBytes,
|
|
1477
|
+
});
|
|
1478
|
+
predictedLiveBytes = occupancy.predictedLiveBytes;
|
|
1479
|
+
occupiedBytes = occupancy.occupiedBytes;
|
|
1480
|
+
footprintSamples = occupancy.footprintSamples;
|
|
1481
|
+
const ledgerRowBytes = occupancy.measuredOwnerBytes;
|
|
1482
|
+
measuredRowBytes = ledgerRowBytes;
|
|
1483
|
+
|
|
1484
|
+
const overHeapTarget =
|
|
1485
|
+
occupiedBytes >= reading.heapTargetBytes || observedOverTargetConfirmed;
|
|
1486
|
+
// RSS at the cliff is pressure only while live memory is substantial;
|
|
1487
|
+
// a high RSS over a small live footprint is freed pages that new rows
|
|
1488
|
+
// will reuse, and holding on it would latch the window as the old clamp
|
|
1489
|
+
// did.
|
|
1490
|
+
const overResidentGuard =
|
|
1491
|
+
reading.residentBytes != null &&
|
|
1492
|
+
reading.residentBytes >= reading.residentGuardBytes &&
|
|
1493
|
+
reading.liveBytes >=
|
|
1494
|
+
reading.heapTargetBytes * RUNTIME_MAP_RESIDENT_GUARD_LIVE_FRACTION;
|
|
1495
|
+
// A watchdog signal holds until the collection it asked for has actually
|
|
1496
|
+
// shown up as a fall in live memory.
|
|
1497
|
+
const underExternalHold =
|
|
1498
|
+
externalHoldLiveBytes !== null &&
|
|
1499
|
+
reading.liveBytes >= externalHoldLiveBytes;
|
|
1500
|
+
if (externalHoldLiveBytes !== null && !underExternalHold) {
|
|
1501
|
+
externalHoldLiveBytes = null;
|
|
1502
|
+
}
|
|
1503
|
+
|
|
1504
|
+
// The watchdog reads the published ledger only when it aborts, so the
|
|
1505
|
+
// file need not track every 25 ms probe; publishing is synchronous I/O
|
|
1506
|
+
// on the runner's event loop and one write per second is plenty.
|
|
1507
|
+
const nowMs = Date.now();
|
|
1508
|
+
if (nowMs - lastLedgerPublishMs >= RUNTIME_MAP_LEDGER_PUBLISH_INTERVAL_MS) {
|
|
1509
|
+
lastLedgerPublishMs = nowMs;
|
|
1510
|
+
reportLedger(snapshot());
|
|
1511
|
+
}
|
|
1512
|
+
|
|
1513
|
+
if (overHeapTarget || overResidentGuard || underExternalHold) {
|
|
1514
|
+
// Existing rows are never interrupted. Pause new admission until
|
|
1515
|
+
// settled rows free enough live memory to grow again. Keep the computed
|
|
1516
|
+
// width intact: replacing it with the momentary active count makes a
|
|
1517
|
+
// transient pause compound into a serial tail as rows finish. Once the
|
|
1518
|
+
// active count reaches zero, retain one progress row; otherwise a map
|
|
1519
|
+
// whose baseline is already over target could strand every queued row.
|
|
1520
|
+
limitedByHeap ||= overHeapTarget;
|
|
1521
|
+
limitedByResidentMemory ||= overResidentGuard;
|
|
1522
|
+
if (!memoryPressureActive) {
|
|
1523
|
+
memoryPressureActive = true;
|
|
1524
|
+
memoryPressureEvents += 1;
|
|
1525
|
+
if (overHeapTarget) heapPressureEvents += 1;
|
|
1526
|
+
if (overResidentGuard) residentPressureEvents += 1;
|
|
1527
|
+
input.onMemoryPressure?.();
|
|
1528
|
+
}
|
|
1529
|
+
pauseAdmission();
|
|
1530
|
+
if (activeRows === 0) drain(1, true);
|
|
1531
|
+
return;
|
|
1532
|
+
}
|
|
1533
|
+
memoryPressureActive = false;
|
|
1534
|
+
eventLoopLagMs = input.getEventLoopLagMs?.() ?? null;
|
|
1535
|
+
const adjustDue =
|
|
1536
|
+
lastEventLoopAdjustAtMs === null ||
|
|
1537
|
+
now() - lastEventLoopAdjustAtMs >= eventLoopAdjustIntervalMs;
|
|
1538
|
+
if (eventLoopLagMs !== null && eventLoopLagMs >= eventLoopLagHoldMs) {
|
|
1539
|
+
limitedByEventLoop = true;
|
|
1540
|
+
if (!eventLoopPressureActive) {
|
|
1541
|
+
eventLoopPressureActive = true;
|
|
1542
|
+
eventLoopPressureEvents += 1;
|
|
1543
|
+
lastEventLoopAdjustAtMs = null;
|
|
1544
|
+
}
|
|
1545
|
+
if (
|
|
1546
|
+
lastEventLoopAdjustAtMs === null ||
|
|
1547
|
+
now() - lastEventLoopAdjustAtMs >= eventLoopAdjustIntervalMs
|
|
1548
|
+
) {
|
|
1549
|
+
lastEventLoopAdjustAtMs = now();
|
|
1550
|
+
// Never below the opening width: the hold above still keeps a
|
|
1551
|
+
// saturated loop from admitting, and a ceiling of a handful of rows
|
|
1552
|
+
// would take minutes of additive growth to recover from.
|
|
1553
|
+
eventLoopCeiling = Math.max(
|
|
1554
|
+
initialConcurrency,
|
|
1555
|
+
Math.floor(
|
|
1556
|
+
Math.min(eventLoopCeiling, Math.max(activeRows, 1)) *
|
|
1557
|
+
RUNTIME_MAP_EVENT_LOOP_SHRINK_FACTOR,
|
|
1558
|
+
),
|
|
1559
|
+
);
|
|
1560
|
+
concurrency = Math.min(concurrency, eventLoopCeiling);
|
|
1561
|
+
}
|
|
1562
|
+
pauseAdmission();
|
|
1563
|
+
if (activeRows === 0) drain(1, true);
|
|
1564
|
+
return;
|
|
1565
|
+
}
|
|
1566
|
+
eventLoopPressureActive = false;
|
|
1567
|
+
if (eventLoopLagMs !== null && eventLoopCeiling < maxConcurrency && adjustDue) {
|
|
1568
|
+
lastEventLoopAdjustAtMs = now();
|
|
1569
|
+
eventLoopCeiling = Math.min(
|
|
1570
|
+
maxConcurrency,
|
|
1571
|
+
eventLoopCeiling +
|
|
1572
|
+
Math.max(
|
|
1573
|
+
1,
|
|
1574
|
+
Math.ceil(eventLoopCeiling * RUNTIME_MAP_EVENT_LOOP_GROW_FRACTION),
|
|
1575
|
+
),
|
|
1576
|
+
);
|
|
1577
|
+
}
|
|
1578
|
+
admissionPaused = false;
|
|
1579
|
+
|
|
1580
|
+
// The reserve is what one more row may cost: the ledger's settled p90
|
|
1581
|
+
// per-row peak scaled by its calibration factor, or the unmeasured prior
|
|
1582
|
+
// until enough rows have settled to state one. It is floored so a run of
|
|
1583
|
+
// light rows cannot authorize a launch wave for a heavy one.
|
|
1584
|
+
rowReserveBytes = reserveForAdmittedRow(occupancy);
|
|
1585
|
+
// Rows already admitted may not have shown their bodies yet. Charge them
|
|
1586
|
+
// the reserve minus whatever growth occupancy already reflects, so a
|
|
1587
|
+
// static reading between probes cannot be mistaken for free headroom.
|
|
1588
|
+
const growthBytes = Math.max(0, occupiedBytes - reading.liveBaselineBytes);
|
|
1589
|
+
const unmaterializedBytes = Math.max(
|
|
1590
|
+
0,
|
|
1591
|
+
activeRows * rowReserveBytes - growthBytes,
|
|
1592
|
+
);
|
|
1593
|
+
const admissibleRows = Math.floor(
|
|
1594
|
+
Math.max(
|
|
1595
|
+
0,
|
|
1596
|
+
reading.heapTargetBytes - occupiedBytes - unmaterializedBytes,
|
|
1597
|
+
) / rowReserveBytes,
|
|
1598
|
+
);
|
|
1599
|
+
applyWidth(admissibleRows);
|
|
1600
|
+
drain();
|
|
1601
|
+
};
|
|
1602
|
+
|
|
1603
|
+
const unsubscribeMemoryPressure = subscribeMemoryPressure(() => {
|
|
1604
|
+
if (closed) return;
|
|
1605
|
+
externalPressureSignals += 1;
|
|
1606
|
+
// Pause first, then release durable results and collect, then reprobe. The
|
|
1607
|
+
// order matters: a signal that reprobed first would widen the window on
|
|
1608
|
+
// the reading that provoked the signal.
|
|
1609
|
+
pauseAdmission();
|
|
1610
|
+
// Before the first interval probe the only live reading is the baseline
|
|
1611
|
+
// taken at construction; a signal that early still holds against it.
|
|
1612
|
+
externalHoldLiveBytes = liveBytes ?? liveBaselineBytes;
|
|
1613
|
+
// One episode, one egress release: the reprobe below re-enters the hold
|
|
1614
|
+
// branch, and it must not re-announce pressure the map is already under.
|
|
1615
|
+
if (!memoryPressureActive) {
|
|
1616
|
+
memoryPressureActive = true;
|
|
1617
|
+
memoryPressureEvents += 1;
|
|
1618
|
+
input.onMemoryPressure?.();
|
|
1619
|
+
}
|
|
1620
|
+
// The watchdog asked for memory back, so this collection is not on the
|
|
1621
|
+
// probe cooldown: it is the point of the signal.
|
|
1622
|
+
if (runtimeGc !== null) {
|
|
1623
|
+
const startedAt = now();
|
|
1624
|
+
runtimeGc();
|
|
1625
|
+
const finishedAt = now();
|
|
1626
|
+
gcExplicitCount += 1;
|
|
1627
|
+
gcExplicitTotalMs += Math.max(0, finishedAt - startedAt);
|
|
1628
|
+
lastGcAtMs = finishedAt;
|
|
1629
|
+
nextProbeIsPostCollection = true;
|
|
1630
|
+
}
|
|
1631
|
+
reconcile();
|
|
1632
|
+
});
|
|
1633
|
+
|
|
1634
|
+
const timer = setInterval(reconcile, probeIntervalMs);
|
|
1635
|
+
if (typeof timer === 'object' && 'unref' in timer) timer.unref();
|
|
1636
|
+
|
|
1637
|
+
return {
|
|
1638
|
+
async acquire() {
|
|
1639
|
+
if (closed) {
|
|
1640
|
+
throw closeError ?? new Error('Runtime map row admission is closed.');
|
|
1641
|
+
}
|
|
1642
|
+
return await new Promise<RuntimeMapHeapAdmissionLease>(
|
|
1643
|
+
(resolve, reject) => {
|
|
1644
|
+
waiters.push({ resolve, reject });
|
|
1645
|
+
drain();
|
|
1646
|
+
},
|
|
1647
|
+
);
|
|
1648
|
+
},
|
|
1649
|
+
reconcile,
|
|
1650
|
+
close(error) {
|
|
1651
|
+
if (!closed) {
|
|
1652
|
+
closed = true;
|
|
1653
|
+
closeError = error ?? new Error('Runtime map row admission is closed.');
|
|
1654
|
+
clearInterval(timer);
|
|
1655
|
+
unsubscribeMemoryPressure();
|
|
1656
|
+
while (waiters.length > 0) {
|
|
1657
|
+
waiters.shift()?.reject(closeError);
|
|
1658
|
+
}
|
|
1659
|
+
}
|
|
1660
|
+
return snapshot();
|
|
1661
|
+
},
|
|
1662
|
+
snapshot,
|
|
1663
|
+
};
|
|
1664
|
+
}
|
|
1665
|
+
|
|
26
1666
|
export class RuntimeMapMemoryLimitError extends Error {
|
|
27
1667
|
readonly code = 'RUNTIME_MAP_MEMORY_LIMIT_EXCEEDED';
|
|
28
1668
|
constructor(input: {
|
|
@@ -58,19 +1698,16 @@ export class RuntimeMapMemoryLimitError extends Error {
|
|
|
58
1698
|
}
|
|
59
1699
|
}
|
|
60
1700
|
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
return textEncoder?.encode(serialized).byteLength ?? serialized.length;
|
|
68
|
-
}
|
|
1701
|
+
/**
|
|
1702
|
+
* Serialized size in the ledger's unit. The implementation moved to
|
|
1703
|
+
* `memory-ledger.ts` so callers that must not pull `node:v8` into their module
|
|
1704
|
+
* graph can measure bytes too; this name stays for the map's own call sites.
|
|
1705
|
+
*/
|
|
1706
|
+
export const runtimeMapJsonByteLength = runtimeJsonByteLength;
|
|
69
1707
|
|
|
70
1708
|
export function defaultRuntimeMapMemoryLimits(): RuntimeMapMemoryLimits {
|
|
71
1709
|
return {
|
|
72
1710
|
materializedBudgetBytes: NODE_RUNTIME_MAP_MEMORY_BUDGET_BYTES,
|
|
73
|
-
activeRowsBudgetBytes: NODE_RUNTIME_MAP_ACTIVE_ROWS_MEMORY_BUDGET_BYTES,
|
|
74
1711
|
};
|
|
75
1712
|
}
|
|
76
1713
|
|
|
@@ -83,10 +1720,6 @@ export function resolveRuntimeMapMemoryLimits(
|
|
|
83
1720
|
override?.materializedBudgetBytes,
|
|
84
1721
|
defaults.materializedBudgetBytes,
|
|
85
1722
|
),
|
|
86
|
-
activeRowsBudgetBytes: positiveBudget(
|
|
87
|
-
override?.activeRowsBudgetBytes,
|
|
88
|
-
defaults.activeRowsBudgetBytes,
|
|
89
|
-
),
|
|
90
1723
|
};
|
|
91
1724
|
}
|
|
92
1725
|
|
|
@@ -185,81 +1818,3 @@ export function createRuntimeMapRetainedRowsTracker(
|
|
|
185
1818
|
): RuntimeMapRowsTracker {
|
|
186
1819
|
return createRuntimeMapRowsTracker(input, 'retained_rows');
|
|
187
1820
|
}
|
|
188
|
-
|
|
189
|
-
export function assertRuntimeMapActiveRowsWithinMemoryLimit(input: {
|
|
190
|
-
mapName: string;
|
|
191
|
-
rowCount: number;
|
|
192
|
-
rowConcurrency: number;
|
|
193
|
-
largestRowBytes: number;
|
|
194
|
-
activeRowsBudgetBytes?: number;
|
|
195
|
-
}): void {
|
|
196
|
-
const activeRows = Math.min(
|
|
197
|
-
Math.max(0, input.rowCount),
|
|
198
|
-
Math.max(1, Math.floor(input.rowConcurrency)),
|
|
199
|
-
);
|
|
200
|
-
const estimate: RuntimeMapMemoryEstimate = {
|
|
201
|
-
rowCount: input.rowCount,
|
|
202
|
-
serializedBytes: input.largestRowBytes * activeRows,
|
|
203
|
-
largestRowBytes: input.largestRowBytes,
|
|
204
|
-
estimatedResidentBytes:
|
|
205
|
-
input.largestRowBytes *
|
|
206
|
-
activeRows *
|
|
207
|
-
NODE_RUNTIME_MAP_ACTIVE_ROW_MEMORY_MULTIPLIER +
|
|
208
|
-
activeRows * NODE_RUNTIME_MAP_ROW_OVERHEAD_BYTES,
|
|
209
|
-
};
|
|
210
|
-
const budgetBytes = positiveBudget(
|
|
211
|
-
input.activeRowsBudgetBytes,
|
|
212
|
-
NODE_RUNTIME_MAP_ACTIVE_ROWS_MEMORY_BUDGET_BYTES,
|
|
213
|
-
);
|
|
214
|
-
if (estimate.estimatedResidentBytes > budgetBytes) {
|
|
215
|
-
throw new RuntimeMapMemoryLimitError({
|
|
216
|
-
mapName: input.mapName,
|
|
217
|
-
phase: 'active_rows',
|
|
218
|
-
estimate,
|
|
219
|
-
budgetBytes,
|
|
220
|
-
activeRows,
|
|
221
|
-
rowConcurrency: input.rowConcurrency,
|
|
222
|
-
});
|
|
223
|
-
}
|
|
224
|
-
}
|
|
225
|
-
|
|
226
|
-
/**
|
|
227
|
-
* Resolve a row-worker window from the shared concurrency target and a concrete
|
|
228
|
-
* resident-byte budget. Pending rows remain indexes, so only this admitted
|
|
229
|
-
* window contributes active row copies/promises. If one row cannot fit, fail
|
|
230
|
-
* before any resolver body starts; otherwise memory pressure is normal
|
|
231
|
-
* backpressure and reduces the window explicitly.
|
|
232
|
-
*/
|
|
233
|
-
export function resolveRuntimeMapRowAdmission(input: {
|
|
234
|
-
rowCount: number;
|
|
235
|
-
requestedConcurrency: number;
|
|
236
|
-
largestRowBytes: number;
|
|
237
|
-
activeRowsBudgetBytes?: number;
|
|
238
|
-
}): RuntimeMapRowAdmission {
|
|
239
|
-
const rowCount = Math.max(0, Math.floor(input.rowCount));
|
|
240
|
-
const requestedConcurrency = Math.max(
|
|
241
|
-
1,
|
|
242
|
-
Math.floor(input.requestedConcurrency),
|
|
243
|
-
);
|
|
244
|
-
const largestRowBytes = Math.max(0, Math.floor(input.largestRowBytes));
|
|
245
|
-
const budgetBytes = positiveBudget(
|
|
246
|
-
input.activeRowsBudgetBytes,
|
|
247
|
-
NODE_RUNTIME_MAP_ACTIVE_ROWS_MEMORY_BUDGET_BYTES,
|
|
248
|
-
);
|
|
249
|
-
const estimatedBytesPerActiveRow =
|
|
250
|
-
largestRowBytes * NODE_RUNTIME_MAP_ACTIVE_ROW_MEMORY_MULTIPLIER +
|
|
251
|
-
NODE_RUNTIME_MAP_ROW_OVERHEAD_BYTES;
|
|
252
|
-
|
|
253
|
-
const maxConcurrencyByMemory = Math.floor(
|
|
254
|
-
budgetBytes / Math.max(1, estimatedBytesPerActiveRow),
|
|
255
|
-
);
|
|
256
|
-
const concurrency =
|
|
257
|
-
rowCount === 0
|
|
258
|
-
? 0
|
|
259
|
-
: Math.min(rowCount, requestedConcurrency, maxConcurrencyByMemory);
|
|
260
|
-
return {
|
|
261
|
-
concurrency,
|
|
262
|
-
estimatedBytesPerActiveRow,
|
|
263
|
-
estimatedResidentBytes: estimatedBytesPerActiveRow * concurrency,
|
|
264
|
-
};
|
|
265
|
-
}
|