@code-yeongyu/senpi-codemode 2026.10.5 → 2026.10.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +22 -0
- package/README.md +16 -2
- package/package.json +4 -4
- package/src/bridge/kernel-tools-protocol.ts +1 -1
- package/src/config/feature-settings.ts +110 -0
- package/src/config/memory-settings.ts +8 -0
- package/src/config/settings-constants.ts +7 -0
- package/src/config/settings-overrides.ts +72 -0
- package/src/config/settings.ts +35 -78
- package/src/config/unknown-settings.ts +19 -0
- package/src/extension/bridge-tool-call.ts +37 -0
- package/src/extension/idle-parking-kernel.ts +272 -0
- package/src/extension/kernel-registration.ts +25 -0
- package/src/extension/kernel-tools-probe.ts +24 -0
- package/src/extension/session-manager.ts +42 -55
- package/src/kernels/jl/kernel.ts +3 -1
- package/src/kernels/jl/runner.jl +154 -0
- package/src/kernels/js/context-manager.ts +1 -1
- package/src/kernels/js/kernel-tools-types.ts +2 -1
- package/src/kernels/js/worker-global-sizes.js +84 -14
- package/src/kernels/py/prelude.py +6 -1
- package/src/kernels/rb/kernel.ts +3 -1
- package/src/kernels/rb/runner.rb +141 -0
- package/src/kernels/shared/kernel-memory-host.ts +14 -6
- package/src/kernels/{js/kernel-tools-host.ts → shared/kernel-tools-pump.ts} +2 -2
- package/src/kernels/shared/process-group-cpu.ts +139 -0
- package/src/kernels/shared/subprocess-contract.ts +3 -0
- package/src/kernels/shared/subprocess-kernel.ts +33 -1
- package/src/kernels/shared/subprocess-startup.ts +90 -0
- package/src/tool/run-eval-cell.ts +5 -6
package/src/kernels/jl/runner.jl
CHANGED
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
# allow: SIZE_OK — parser, stream capture, bridge calls, and the persistent execution loop share Main globals.
|
|
2
|
+
write(stdout, "{\"type\":\"status\",\"event\":{\"op\":\"kernel-startup\",\"stage\":\"stdlib-imports\"}}\n")
|
|
3
|
+
flush(stdout)
|
|
2
4
|
using Sockets
|
|
3
5
|
|
|
4
6
|
const SENPI_ORIGINAL_STDOUT = stdout
|
|
@@ -7,6 +9,8 @@ out_read, out_write = redirect_stdout()
|
|
|
7
9
|
err_read, err_write = redirect_stderr()
|
|
8
10
|
redirect_stdin(devnull)
|
|
9
11
|
|
|
12
|
+
write(SENPI_ORIGINAL_STDOUT, "{\"type\":\"status\",\"event\":{\"op\":\"kernel-startup\",\"stage\":\"runtime-init\"}}\n")
|
|
13
|
+
flush(SENPI_ORIGINAL_STDOUT)
|
|
10
14
|
include("prelude.jl")
|
|
11
15
|
|
|
12
16
|
const senpi_connection = Dict{String, Any}()
|
|
@@ -281,6 +285,152 @@ function senpi_set_connection(value)
|
|
|
281
285
|
end
|
|
282
286
|
end
|
|
283
287
|
|
|
288
|
+
const senpi_memory = Ref{Any}(nothing)
|
|
289
|
+
|
|
290
|
+
const SENPI_MEMORY_INTERNALS = Set([:senpi_current_cell, :senpi_memory, :senpi_connection, :senpi_frame_io, :senpi_stdout_capture, :senpi_stderr_capture, :senpi_protocol_stdin])
|
|
291
|
+
|
|
292
|
+
const SENPI_SIZER_SAMPLE = 1_000
|
|
293
|
+
const SENPI_SIZER_NODE_BUDGET = 5_000
|
|
294
|
+
const SENPI_SIZER_MAX_DEPTH = 64
|
|
295
|
+
const SENPI_SIZER_POINTER = 8
|
|
296
|
+
const SENPI_SIZER_OBJECT = 16
|
|
297
|
+
const SENPI_SIZER_MIN_REPORTED = 1024 * 1024
|
|
298
|
+
|
|
299
|
+
# Sizes a global with sampling and a node budget per global (leaves count too), so one deep global
|
|
300
|
+
# never hides the ones measured after it and a huge container costs a bounded walk. Only concrete Base
|
|
301
|
+
# containers are iterated; any other AbstractDict or AbstractSet is sized as an opaque struct, so no user
|
|
302
|
+
# length or iterate method runs during a memory report.
|
|
303
|
+
mutable struct SenpiSizer
|
|
304
|
+
seen::IdDict{Any, Nothing}
|
|
305
|
+
nodes::Int
|
|
306
|
+
approximate::Bool
|
|
307
|
+
end
|
|
308
|
+
SenpiSizer() = SenpiSizer(IdDict{Any, Nothing}(), 0, false)
|
|
309
|
+
|
|
310
|
+
senpi_over_budget(sizer::SenpiSizer) = sizer.nodes >= SENPI_SIZER_NODE_BUDGET
|
|
311
|
+
|
|
312
|
+
function senpi_size(sizer::SenpiSizer, value, depth::Int)::Int
|
|
313
|
+
sizer.nodes += 1
|
|
314
|
+
value isa Type && return 0
|
|
315
|
+
value isa Union{Number, Char, Bool, Symbol, Nothing} && return isbits(value) ? sizeof(value) : 0
|
|
316
|
+
value isa String && return SENPI_SIZER_OBJECT + sizeof(value)
|
|
317
|
+
if ismutable(value)
|
|
318
|
+
haskey(sizer.seen, value) && return 0
|
|
319
|
+
sizer.seen[value] = nothing
|
|
320
|
+
end
|
|
321
|
+
if depth >= SENPI_SIZER_MAX_DEPTH || senpi_over_budget(sizer)
|
|
322
|
+
sizer.approximate = true
|
|
323
|
+
return SENPI_SIZER_OBJECT
|
|
324
|
+
end
|
|
325
|
+
if value isa Array
|
|
326
|
+
isbitstype(eltype(value)) && return SENPI_SIZER_OBJECT + sizeof(value)
|
|
327
|
+
return SENPI_SIZER_OBJECT + length(value) * SENPI_SIZER_POINTER +
|
|
328
|
+
senpi_sampled(sizer, length(value), depth) do index
|
|
329
|
+
isassigned(value, index) ? value[index] : nothing
|
|
330
|
+
end
|
|
331
|
+
elseif value isa Union{Dict, IdDict}
|
|
332
|
+
count = length(value)
|
|
333
|
+
total = 0
|
|
334
|
+
taken = 0
|
|
335
|
+
for (key, item) in value
|
|
336
|
+
senpi_over_budget(sizer) && break
|
|
337
|
+
total += senpi_size(sizer, key, depth + 1) + senpi_size(sizer, item, depth + 1)
|
|
338
|
+
taken += 1
|
|
339
|
+
taken >= SENPI_SIZER_SAMPLE && break
|
|
340
|
+
end
|
|
341
|
+
taken < count && (sizer.approximate = true)
|
|
342
|
+
return SENPI_SIZER_OBJECT + count * 2 * SENPI_SIZER_POINTER + (taken == 0 ? 0 : div(total * count, taken))
|
|
343
|
+
elseif value isa Union{Set, Tuple}
|
|
344
|
+
count = length(value)
|
|
345
|
+
total = 0
|
|
346
|
+
taken = 0
|
|
347
|
+
for item in value
|
|
348
|
+
senpi_over_budget(sizer) && break
|
|
349
|
+
total += senpi_size(sizer, item, depth + 1)
|
|
350
|
+
taken += 1
|
|
351
|
+
taken >= SENPI_SIZER_SAMPLE && break
|
|
352
|
+
end
|
|
353
|
+
taken < count && (sizer.approximate = true)
|
|
354
|
+
return SENPI_SIZER_OBJECT + count * SENPI_SIZER_POINTER + (taken == 0 ? 0 : div(total * count, taken))
|
|
355
|
+
end
|
|
356
|
+
isbits(value) && return sizeof(value)
|
|
357
|
+
fields = fieldcount(typeof(value))
|
|
358
|
+
return SENPI_SIZER_OBJECT + fields * SENPI_SIZER_POINTER +
|
|
359
|
+
senpi_sampled(sizer, fields, depth) do index
|
|
360
|
+
isdefined(value, index) ? getfield(value, index) : nothing
|
|
361
|
+
end
|
|
362
|
+
end
|
|
363
|
+
|
|
364
|
+
# Up to SENPI_SIZER_SAMPLE evenly spaced elements; once the budget runs out part-way, the elements measured
|
|
365
|
+
# so far stand in for the rest.
|
|
366
|
+
function senpi_sampled(at, sizer::SenpiSizer, count::Int, depth::Int)::Int
|
|
367
|
+
count == 0 && return 0
|
|
368
|
+
picks = min(count, SENPI_SIZER_SAMPLE)
|
|
369
|
+
picks < count && (sizer.approximate = true)
|
|
370
|
+
step = count / picks
|
|
371
|
+
total = 0
|
|
372
|
+
measured = 0
|
|
373
|
+
for sample in 1:picks
|
|
374
|
+
if senpi_over_budget(sizer)
|
|
375
|
+
sizer.approximate = true
|
|
376
|
+
break
|
|
377
|
+
end
|
|
378
|
+
total += senpi_size(sizer, at(1 + floor(Int, (sample - 1) * step)), depth + 1)
|
|
379
|
+
measured += 1
|
|
380
|
+
end
|
|
381
|
+
return measured == 0 ? 0 : round(Int, total / measured * count)
|
|
382
|
+
end
|
|
383
|
+
|
|
384
|
+
function senpi_largest_globals(limit::Int)
|
|
385
|
+
try
|
|
386
|
+
sizer = SenpiSizer()
|
|
387
|
+
candidates = Tuple{String, Int, Bool}[]
|
|
388
|
+
Base.invokelatest() do
|
|
389
|
+
for name in names(Main, all = true)
|
|
390
|
+
name in SENPI_MEMORY_INTERNALS && continue
|
|
391
|
+
name in (:Main, :Base, :Core, :Ans, :ans) && continue
|
|
392
|
+
lowered = lowercase(string(name))
|
|
393
|
+
startswith(lowered, "senpi_") && continue
|
|
394
|
+
startswith(string(name), "Senpi") && continue
|
|
395
|
+
startswith(string(name), "#") && continue
|
|
396
|
+
isdefined(Main, name) || continue
|
|
397
|
+
value = getfield(Main, name)
|
|
398
|
+
value isa Module && continue
|
|
399
|
+
value isa IO && continue
|
|
400
|
+
value isa Type && continue
|
|
401
|
+
value isa Function && continue
|
|
402
|
+
sizer.approximate = false
|
|
403
|
+
sizer.nodes = 0
|
|
404
|
+
bytes = try
|
|
405
|
+
senpi_size(sizer, value, 0) + SENPI_SIZER_POINTER
|
|
406
|
+
catch
|
|
407
|
+
0
|
|
408
|
+
end
|
|
409
|
+
bytes >= SENPI_SIZER_MIN_REPORTED && push!(candidates, (string(name), bytes, sizer.approximate))
|
|
410
|
+
end
|
|
411
|
+
end
|
|
412
|
+
sort!(candidates, by = entry -> entry[2], rev = true)
|
|
413
|
+
[approximate ? Dict{String, Any}("name" => name, "bytes" => bytes, "approximate" => true) : Dict{String, Any}("name" => name, "bytes" => bytes) for (name, bytes, approximate) in candidates[1:min(limit, end)]]
|
|
414
|
+
catch
|
|
415
|
+
Dict{String, Any}[]
|
|
416
|
+
end
|
|
417
|
+
end
|
|
418
|
+
|
|
419
|
+
function senpi_memory_report()
|
|
420
|
+
try
|
|
421
|
+
senpi_memory[] === nothing && return nothing
|
|
422
|
+
# The host reads the interpreter's current footprint and decides on the notice; the runner only
|
|
423
|
+
# names its largest globals, with a bounded walk, so a cell never pays for an unbounded scan.
|
|
424
|
+
named = senpi_largest_globals(5)
|
|
425
|
+
# The host replaces liveBytes with the interpreter footprint it reads itself; only globals is used.
|
|
426
|
+
report = Dict{String, Any}("liveBytes" => 0, "measure" => "footprint")
|
|
427
|
+
isempty(named) || (report["globals"] = named)
|
|
428
|
+
report
|
|
429
|
+
catch
|
|
430
|
+
nothing
|
|
431
|
+
end
|
|
432
|
+
end
|
|
433
|
+
|
|
284
434
|
function senpi_run_cell(message)
|
|
285
435
|
cell_id = string(get(message, "cellId", ""))
|
|
286
436
|
code = string(get(message, "code", ""))
|
|
@@ -297,6 +447,8 @@ function senpi_run_cell(message)
|
|
|
297
447
|
yield()
|
|
298
448
|
frame = Dict{String, Any}("type" => "result", "cellId" => cell_id, "ok" => true, "durationMs" => round(Int, (time() - started) * 1000))
|
|
299
449
|
value !== nothing && senpi_should_display_result(parsed) && (frame["valueRepr"] = senpi_json(value))
|
|
450
|
+
memory = senpi_memory_report()
|
|
451
|
+
memory !== nothing && (frame["memory"] = memory)
|
|
300
452
|
senpi_emit(frame)
|
|
301
453
|
catch error
|
|
302
454
|
senpi_emit(Dict("type" => "result", "cellId" => cell_id, "ok" => false, "error" => senpi_error(error), "durationMs" => round(Int, (time() - started) * 1000)))
|
|
@@ -313,7 +465,9 @@ while !eof(SENPI_ORIGINAL_STDIN)
|
|
|
313
465
|
message isa AbstractDict || error("Bridge frame must be an object")
|
|
314
466
|
kind = get(message, "type", nothing)
|
|
315
467
|
if kind == "init"
|
|
468
|
+
senpi_emit(Dict("type" => "status", "event" => Dict("op" => "kernel-startup", "stage" => "host-init")))
|
|
316
469
|
senpi_set_connection(get(message, "connection", nothing))
|
|
470
|
+
senpi_memory[] = get(message, "memory", nothing)
|
|
317
471
|
senpi_emit(Dict("type" => "ready"))
|
|
318
472
|
elseif kind == "run"
|
|
319
473
|
senpi_run_cell(message)
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type { HostToKernelMessage, KernelToHostMessage } from "../../bridge/protocol.ts";
|
|
2
2
|
import { CHILD_LIFECYCLE_OP, INTERRUPT_ACK_OP } from "../../bridge/reserved.ts";
|
|
3
3
|
import type { KernelInterruptHandle } from "../../tool/types.ts";
|
|
4
|
+
import { KernelToolHostPump } from "../shared/kernel-tools-pump.ts";
|
|
4
5
|
import { ActiveCellControl } from "./active-cell-control.ts";
|
|
5
6
|
import { DEFAULT_INTERRUPT_BOUNDS, type WorkerRetirement } from "./interrupt-bounds.ts";
|
|
6
7
|
import {
|
|
@@ -14,7 +15,6 @@ import {
|
|
|
14
15
|
} from "./kernel-contract.ts";
|
|
15
16
|
import { type JavaScriptMemoryReading, KernelMemoryBridge } from "./kernel-memory-bridge.ts";
|
|
16
17
|
import { kernelToolError } from "./kernel-tools-errors.ts";
|
|
17
|
-
import { KernelToolHostPump } from "./kernel-tools-host.ts";
|
|
18
18
|
import type {
|
|
19
19
|
KernelToolsDescribeResult,
|
|
20
20
|
KernelToolsInvokeOptions,
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { ExtensionKernelTools, KernelToolInvokeOptions, KernelToolInvokeScope } from "@code-yeongyu/senpi";
|
|
2
|
+
import type { EvalLanguage } from "../../tool/types.ts";
|
|
2
3
|
import type { KernelToolErrorCode, KernelToolHostDenial, KernelToolHostDenialReason } from "./kernel-tools-errors.ts";
|
|
3
4
|
|
|
4
5
|
export type { KernelToolErrorCode, KernelToolHostDenial, KernelToolHostDenialReason };
|
|
@@ -7,7 +8,7 @@ export type KernelToolDescriptor = {
|
|
|
7
8
|
readonly name: string;
|
|
8
9
|
readonly description: string;
|
|
9
10
|
readonly input_schema: unknown;
|
|
10
|
-
readonly language:
|
|
11
|
+
readonly language: EvalLanguage;
|
|
11
12
|
readonly kernel_generation: number;
|
|
12
13
|
readonly definition_revision: number;
|
|
13
14
|
};
|
|
@@ -9,6 +9,54 @@ const FUNCTION_BYTES = 64;
|
|
|
9
9
|
const INTERNAL_GLOBAL_PREFIX = "__senpi";
|
|
10
10
|
const MIN_REPORTED_BYTES = 1024 * 1024;
|
|
11
11
|
|
|
12
|
+
function intrinsicGetter(prototype, name) {
|
|
13
|
+
return Object.getOwnPropertyDescriptor(prototype, name)?.get;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
// Built-in getters captured at load: a user subclass that overrides `byteLength` or `size` is never called.
|
|
17
|
+
const TYPED_ARRAY_BYTE_LENGTH = intrinsicGetter(Object.getPrototypeOf(Uint8Array.prototype), "byteLength");
|
|
18
|
+
const DATA_VIEW_BYTE_LENGTH = intrinsicGetter(DataView.prototype, "byteLength");
|
|
19
|
+
const ARRAY_BUFFER_BYTE_LENGTH = intrinsicGetter(ArrayBuffer.prototype, "byteLength");
|
|
20
|
+
const SHARED_ARRAY_BUFFER_BYTE_LENGTH =
|
|
21
|
+
typeof SharedArrayBuffer === "function" ? intrinsicGetter(SharedArrayBuffer.prototype, "byteLength") : undefined;
|
|
22
|
+
const MAP_SIZE = intrinsicGetter(Map.prototype, "size");
|
|
23
|
+
const SET_SIZE = intrinsicGetter(Set.prototype, "size");
|
|
24
|
+
const BLOB_SIZE = typeof Blob === "function" ? intrinsicGetter(Blob.prototype, "size") : undefined;
|
|
25
|
+
// Iteration captured at load too: a user who replaces Map.prototype.entries or an iterator's next() is never called.
|
|
26
|
+
const MAP_ENTRIES = Map.prototype.entries;
|
|
27
|
+
const SET_VALUES = Set.prototype.values;
|
|
28
|
+
const MAP_ITERATOR_NEXT = Object.getPrototypeOf(new Map().entries()).next;
|
|
29
|
+
const SET_ITERATOR_NEXT = Object.getPrototypeOf(new Set().values()).next;
|
|
30
|
+
|
|
31
|
+
function read(getter, value) {
|
|
32
|
+
return Reflect.apply(getter, value, []);
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
const GET_PROTOTYPE_OF = Object.getPrototypeOf;
|
|
36
|
+
const BLOB_PROTOTYPE = typeof Blob === "function" ? Blob.prototype : undefined;
|
|
37
|
+
|
|
38
|
+
function inheritsFromBlob(value) {
|
|
39
|
+
for (let prototype = GET_PROTOTYPE_OF(value); prototype !== null; prototype = GET_PROTOTYPE_OF(prototype)) {
|
|
40
|
+
// A Proxy in the chain would run its getPrototypeOf trap on the next step.
|
|
41
|
+
if (types.isProxy(prototype)) return false;
|
|
42
|
+
if (prototype === BLOB_PROTOTYPE) return true;
|
|
43
|
+
}
|
|
44
|
+
return false;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
// The Blob brand: a cheap prototype-chain pre-check (no user hook; proxies never reach here) so ordinary
|
|
48
|
+
// objects skip the try, then the size getter as the proof, which throws on a forged look-alike.
|
|
49
|
+
// instanceof would call a user-redefinable Symbol.hasInstance.
|
|
50
|
+
function isBlob(value) {
|
|
51
|
+
if (BLOB_SIZE === undefined || !inheritsFromBlob(value)) return false;
|
|
52
|
+
try {
|
|
53
|
+
read(BLOB_SIZE, value);
|
|
54
|
+
return true;
|
|
55
|
+
} catch {
|
|
56
|
+
return false;
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
|
|
12
60
|
export function captureGlobalBaseline() {
|
|
13
61
|
return new Set(Object.getOwnPropertyNames(globalThis));
|
|
14
62
|
}
|
|
@@ -55,11 +103,12 @@ function createSizer() {
|
|
|
55
103
|
return (total / SAMPLED_ELEMENTS) * length;
|
|
56
104
|
}
|
|
57
105
|
|
|
58
|
-
function entriesOf(
|
|
106
|
+
function entriesOf(iterator, next, count, depth, sizeOf) {
|
|
59
107
|
const taken = [];
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
if (
|
|
108
|
+
while (taken.length < SAMPLED_ELEMENTS) {
|
|
109
|
+
const step = Reflect.apply(next, iterator, []);
|
|
110
|
+
if (step.done) break;
|
|
111
|
+
taken.push(step.value);
|
|
63
112
|
}
|
|
64
113
|
if (taken.length < count) approximate = true;
|
|
65
114
|
let total = 0;
|
|
@@ -67,21 +116,42 @@ function createSizer() {
|
|
|
67
116
|
return total * (count / Math.max(1, taken.length));
|
|
68
117
|
}
|
|
69
118
|
|
|
119
|
+
// An own data element, never a getter: an accessor is left unsized and makes the estimate approximate.
|
|
120
|
+
function elementOf(array, index) {
|
|
121
|
+
const descriptor = Object.getOwnPropertyDescriptor(array, index);
|
|
122
|
+
if (descriptor === undefined) return undefined;
|
|
123
|
+
if ("value" in descriptor) return descriptor.value;
|
|
124
|
+
approximate = true;
|
|
125
|
+
return undefined;
|
|
126
|
+
}
|
|
127
|
+
|
|
70
128
|
function objectSize(value, depth) {
|
|
71
129
|
if (types.isProxy(value)) return OBJECT_BYTES;
|
|
72
|
-
if (
|
|
73
|
-
if (types.
|
|
74
|
-
if (
|
|
75
|
-
if (
|
|
76
|
-
if (value
|
|
77
|
-
|
|
78
|
-
|
|
130
|
+
if (types.isTypedArray(value)) return read(TYPED_ARRAY_BYTE_LENGTH, value);
|
|
131
|
+
if (types.isDataView(value)) return read(DATA_VIEW_BYTE_LENGTH, value);
|
|
132
|
+
if (types.isArrayBuffer(value)) return read(ARRAY_BUFFER_BYTE_LENGTH, value);
|
|
133
|
+
if (types.isSharedArrayBuffer(value) && SHARED_ARRAY_BUFFER_BYTE_LENGTH) return read(SHARED_ARRAY_BUFFER_BYTE_LENGTH, value);
|
|
134
|
+
if (isBlob(value)) return read(BLOB_SIZE, value);
|
|
135
|
+
if (Array.isArray(value)) {
|
|
136
|
+
const length = value.length;
|
|
137
|
+
return OBJECT_BYTES + length * POINTER_BYTES + sampled(length, (index) => elementOf(value, index), depth);
|
|
138
|
+
}
|
|
139
|
+
if (types.isMap(value)) {
|
|
140
|
+
const count = read(MAP_SIZE, value);
|
|
141
|
+
const entries = Reflect.apply(MAP_ENTRIES, value, []);
|
|
142
|
+
return OBJECT_BYTES + count * 2 * POINTER_BYTES + entriesOf(entries, MAP_ITERATOR_NEXT, count, depth, ([key, item], at) => size(key, at) + size(item, at));
|
|
143
|
+
}
|
|
144
|
+
if (types.isSet(value)) {
|
|
145
|
+
const count = read(SET_SIZE, value);
|
|
146
|
+
return OBJECT_BYTES + count * POINTER_BYTES + entriesOf(Reflect.apply(SET_VALUES, value, []), SET_ITERATOR_NEXT, count, depth, size);
|
|
79
147
|
}
|
|
80
|
-
if (value instanceof Set) return OBJECT_BYTES + value.size * POINTER_BYTES + entriesOf(Set.prototype.values.call(value), value.size, depth, size);
|
|
81
148
|
const keys = Object.keys(value);
|
|
82
149
|
const propertyValue = (index) => {
|
|
83
150
|
const descriptor = Object.getOwnPropertyDescriptor(value, keys[index]);
|
|
84
|
-
|
|
151
|
+
if (descriptor === undefined) return undefined;
|
|
152
|
+
if ("value" in descriptor) return descriptor.value;
|
|
153
|
+
approximate = true;
|
|
154
|
+
return undefined;
|
|
85
155
|
};
|
|
86
156
|
return OBJECT_BYTES + keys.length * POINTER_BYTES + sampled(keys.length, propertyValue, depth);
|
|
87
157
|
}
|
|
@@ -114,7 +184,7 @@ function createSizer() {
|
|
|
114
184
|
try {
|
|
115
185
|
return { bytes: POINTER_BYTES + size(value, 0), approximate };
|
|
116
186
|
} catch (error) {
|
|
117
|
-
// A
|
|
187
|
+
// A forged built-in look-alike (a Blob-prototype object without its slot) throws from the intrinsic getter; it is left unsized.
|
|
118
188
|
if (error instanceof Error) return undefined;
|
|
119
189
|
throw error;
|
|
120
190
|
}
|
|
@@ -354,6 +354,11 @@ def write(path: str | Path, content: str) -> Path:
|
|
|
354
354
|
return target
|
|
355
355
|
|
|
356
356
|
|
|
357
|
+
# The bridge is a loopback call: urllib's default opener would route it through a configured proxy
|
|
358
|
+
# (the environment everywhere, the registry on Windows), which a 127.0.0.1 request must never use.
|
|
359
|
+
_BRIDGE_OPENER = urllib.request.build_opener(urllib.request.ProxyHandler({}))
|
|
360
|
+
|
|
361
|
+
|
|
357
362
|
def bridge_post(path: str, payload: dict[str, Any]) -> Any:
|
|
358
363
|
port = CONNECTION.get("port")
|
|
359
364
|
token = CONNECTION.get("token")
|
|
@@ -369,7 +374,7 @@ def bridge_post(path: str, payload: dict[str, Any]) -> Any:
|
|
|
369
374
|
emit_status(TIMEOUT_PAUSE_OP, force=True)
|
|
370
375
|
try:
|
|
371
376
|
try:
|
|
372
|
-
with
|
|
377
|
+
with _BRIDGE_OPENER.open(request, timeout=60) as response:
|
|
373
378
|
response_data = response.read()
|
|
374
379
|
except urllib.error.HTTPError as exc:
|
|
375
380
|
response_data = exc.read()
|
package/src/kernels/rb/kernel.ts
CHANGED
|
@@ -2,6 +2,7 @@ import { join } from "node:path";
|
|
|
2
2
|
import type { BridgeConnectionConfig, KernelToHostMessage } from "../../bridge/protocol.ts";
|
|
3
3
|
import type { SessionEnvironment } from "../session-env.ts";
|
|
4
4
|
import type { KernelLifecycle } from "../shared/kernel-death.ts";
|
|
5
|
+
import { readProcessGroupCpuTime } from "../shared/process-group-cpu.ts";
|
|
5
6
|
import { type CodemodeRuntimeAssetEnvironment, requireCodemodeRuntimeAsset } from "../shared/runtime-asset.ts";
|
|
6
7
|
import { SubprocessKernel, type SubprocessKernelMemory, type SubprocessSpawn } from "../shared/subprocess-kernel.ts";
|
|
7
8
|
|
|
@@ -14,7 +15,7 @@ export interface RubyKernelStartOptions extends KernelLifecycle {
|
|
|
14
15
|
readonly command?: string;
|
|
15
16
|
readonly spawn?: SubprocessSpawn;
|
|
16
17
|
readonly onMessage?: (message: KernelToHostMessage) => void;
|
|
17
|
-
/**
|
|
18
|
+
/** Memory management: the runner reports its largest globals and the host reads the interpreter footprint for the ceiling. */
|
|
18
19
|
readonly memory?: SubprocessKernelMemory;
|
|
19
20
|
}
|
|
20
21
|
|
|
@@ -43,6 +44,7 @@ export class RubyKernel extends SubprocessKernel {
|
|
|
43
44
|
onMessage: options.onMessage,
|
|
44
45
|
memory: options.memory && { language: "rb", ...options.memory },
|
|
45
46
|
onDeath: options.onDeath,
|
|
47
|
+
startup: { label: "Ruby", readGroupCpuTime: readProcessGroupCpuTime },
|
|
46
48
|
});
|
|
47
49
|
}
|
|
48
50
|
}
|
package/src/kernels/rb/runner.rb
CHANGED
|
@@ -1,6 +1,10 @@
|
|
|
1
|
+
$stdout.write("{\"type\":\"status\",\"event\":{\"op\":\"kernel-startup\",\"stage\":\"stdlib-imports\"}}\n")
|
|
2
|
+
$stdout.flush
|
|
1
3
|
require "json"
|
|
2
4
|
require "net/http"
|
|
3
5
|
require "uri"
|
|
6
|
+
$stdout.write("{\"type\":\"status\",\"event\":{\"op\":\"kernel-startup\",\"stage\":\"runtime-init\"}}\n")
|
|
7
|
+
$stdout.flush
|
|
4
8
|
|
|
5
9
|
$__senpi_binding = TOPLEVEL_BINDING
|
|
6
10
|
$__senpi_frame_mutex = Mutex.new
|
|
@@ -119,6 +123,135 @@ rescue JSON::GeneratorError
|
|
|
119
123
|
value.inspect
|
|
120
124
|
end
|
|
121
125
|
|
|
126
|
+
$__senpi_memory = nil
|
|
127
|
+
|
|
128
|
+
SENPI_SIZER_SAMPLE = 1_000
|
|
129
|
+
SENPI_SIZER_NODE_BUDGET = 5_000
|
|
130
|
+
SENPI_SIZER_MAX_DEPTH = 64
|
|
131
|
+
SENPI_SIZER_POINTER = 8
|
|
132
|
+
SENPI_SIZER_OBJECT = 40
|
|
133
|
+
SENPI_SIZER_MIN_REPORTED = 1024 * 1024
|
|
134
|
+
SENPI_ARRAY_AT = Array.instance_method(:[])
|
|
135
|
+
SENPI_ARRAY_SIZE = Array.instance_method(:size)
|
|
136
|
+
SENPI_HASH_SIZE = Hash.instance_method(:size)
|
|
137
|
+
SENPI_HASH_EACH = Hash.instance_method(:each_pair)
|
|
138
|
+
SENPI_STRING_BYTESIZE = String.instance_method(:bytesize)
|
|
139
|
+
SENPI_IVARS = Kernel.instance_method(:instance_variables)
|
|
140
|
+
SENPI_IVAR_GET = Kernel.instance_method(:instance_variable_get)
|
|
141
|
+
|
|
142
|
+
# Sizes a global without running user code: built-ins are read through their own unbound methods, so an
|
|
143
|
+
# override in a subclass is never called. Collections are sampled, and each global gets its own node
|
|
144
|
+
# budget (leaves count against it too), so one deep global never hides the ones measured after it.
|
|
145
|
+
class SenpiGlobalSizer
|
|
146
|
+
attr_reader :approximate
|
|
147
|
+
|
|
148
|
+
def initialize
|
|
149
|
+
@seen = {}.compare_by_identity
|
|
150
|
+
@nodes = 0
|
|
151
|
+
@approximate = false
|
|
152
|
+
end
|
|
153
|
+
|
|
154
|
+
def measure(value)
|
|
155
|
+
@approximate = false
|
|
156
|
+
@nodes = 0
|
|
157
|
+
SENPI_SIZER_POINTER + size(value, 0)
|
|
158
|
+
end
|
|
159
|
+
|
|
160
|
+
private
|
|
161
|
+
|
|
162
|
+
def size(value, depth)
|
|
163
|
+
@nodes += 1
|
|
164
|
+
return 0 if NilClass === value || TrueClass === value || FalseClass === value || Symbol === value
|
|
165
|
+
return ObjectSpace.memsize_of(value) if Integer === value || Float === value
|
|
166
|
+
return 0 if @seen.key?(value)
|
|
167
|
+
@seen[value] = true
|
|
168
|
+
return SENPI_SIZER_OBJECT + SENPI_STRING_BYTESIZE.bind_call(value) if String === value
|
|
169
|
+
if depth >= SENPI_SIZER_MAX_DEPTH || @nodes >= SENPI_SIZER_NODE_BUDGET
|
|
170
|
+
@approximate = true
|
|
171
|
+
return ObjectSpace.memsize_of(value)
|
|
172
|
+
end
|
|
173
|
+
if Array === value
|
|
174
|
+
length = SENPI_ARRAY_SIZE.bind_call(value)
|
|
175
|
+
SENPI_SIZER_OBJECT + length * SENPI_SIZER_POINTER + sampled(length, depth) { |index| SENPI_ARRAY_AT.bind_call(value, index) }
|
|
176
|
+
elsif Hash === value
|
|
177
|
+
hash_size(value, depth)
|
|
178
|
+
else
|
|
179
|
+
ivars = SENPI_IVARS.bind_call(value)
|
|
180
|
+
ObjectSpace.memsize_of(value) + sampled(ivars.length, depth) { |index| SENPI_IVAR_GET.bind_call(value, ivars[index]) }
|
|
181
|
+
end
|
|
182
|
+
end
|
|
183
|
+
|
|
184
|
+
def hash_size(hash, depth)
|
|
185
|
+
count = SENPI_HASH_SIZE.bind_call(hash)
|
|
186
|
+
taken = 0
|
|
187
|
+
total = 0
|
|
188
|
+
SENPI_HASH_EACH.bind_call(hash) do |key, item|
|
|
189
|
+
total += size(key, depth + 1) + size(item, depth + 1)
|
|
190
|
+
taken += 1
|
|
191
|
+
break if taken >= SENPI_SIZER_SAMPLE
|
|
192
|
+
end
|
|
193
|
+
@approximate = true if taken < count
|
|
194
|
+
SENPI_SIZER_OBJECT + count * 2 * SENPI_SIZER_POINTER + (taken.zero? ? 0 : total * count / taken)
|
|
195
|
+
end
|
|
196
|
+
|
|
197
|
+
# Up to SENPI_SIZER_SAMPLE evenly spaced elements; when the walk budget runs out part-way, the elements
|
|
198
|
+
# measured so far stand in for the rest, so a cut-short container is scaled up, not under-counted.
|
|
199
|
+
def sampled(length, depth)
|
|
200
|
+
return 0 if length.zero?
|
|
201
|
+
picks = [length, SENPI_SIZER_SAMPLE].min
|
|
202
|
+
@approximate = true if picks < length
|
|
203
|
+
step = length.to_f / picks
|
|
204
|
+
total = 0
|
|
205
|
+
measured = 0
|
|
206
|
+
picks.times do |sample|
|
|
207
|
+
if @nodes >= SENPI_SIZER_NODE_BUDGET
|
|
208
|
+
@approximate = true
|
|
209
|
+
break
|
|
210
|
+
end
|
|
211
|
+
total += size(yield((sample * step).floor), depth + 1)
|
|
212
|
+
measured += 1
|
|
213
|
+
end
|
|
214
|
+
measured.zero? ? 0 : (total.to_f / measured * length).round
|
|
215
|
+
end
|
|
216
|
+
end
|
|
217
|
+
|
|
218
|
+
def __senpi_largest_globals(limit)
|
|
219
|
+
require "objspace"
|
|
220
|
+
sizer = SenpiGlobalSizer.new
|
|
221
|
+
sized = []
|
|
222
|
+
measure = lambda do |name, value|
|
|
223
|
+
bytes = sizer.measure(value)
|
|
224
|
+
next if bytes < SENPI_SIZER_MIN_REPORTED
|
|
225
|
+
entry = { "name" => name, "bytes" => bytes }
|
|
226
|
+
entry["approximate"] = true if sizer.approximate
|
|
227
|
+
sized << entry
|
|
228
|
+
end
|
|
229
|
+
$__senpi_binding.local_variables.each do |name|
|
|
230
|
+
next if $__senpi_memory_baseline_locals.include?(name)
|
|
231
|
+
measure.call(name.to_s, $__senpi_binding.local_variable_get(name))
|
|
232
|
+
end
|
|
233
|
+
global_variables.each do |name|
|
|
234
|
+
next if $__senpi_memory_baseline_globals.include?(name) || name.to_s.start_with?("$__senpi_")
|
|
235
|
+
measure.call(name.to_s, eval(name.to_s))
|
|
236
|
+
end
|
|
237
|
+
sized.sort_by { |entry| -entry["bytes"] }.first(limit)
|
|
238
|
+
rescue StandardError
|
|
239
|
+
[]
|
|
240
|
+
end
|
|
241
|
+
|
|
242
|
+
# The host reads the interpreter's footprint and decides on the notice; the runner only names its
|
|
243
|
+
# largest globals. Nothing here spawns a process, so the user's `$?` is left alone.
|
|
244
|
+
def __senpi_memory_report
|
|
245
|
+
return nil if $__senpi_memory.nil?
|
|
246
|
+
named = __senpi_largest_globals(5)
|
|
247
|
+
# The host replaces liveBytes with the interpreter footprint it reads itself; only globals is used.
|
|
248
|
+
report = { "liveBytes" => 0, "measure" => "footprint" }
|
|
249
|
+
report["globals"] = named unless named.empty?
|
|
250
|
+
report
|
|
251
|
+
rescue StandardError
|
|
252
|
+
nil
|
|
253
|
+
end
|
|
254
|
+
|
|
122
255
|
SENPI_NON_DISPLAY_NODES = %i[
|
|
123
256
|
LASGN IASGN GASGN CVASGN DASGN OP_ASGN OP_CDECL CDECL MASGN CASGN
|
|
124
257
|
DEFN DEFS CLASS MODULE SCLASS ALIAS UNDEF
|
|
@@ -172,6 +305,8 @@ def __senpi_run_cell(message)
|
|
|
172
305
|
"durationMs" => ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - started) * 1000).round,
|
|
173
306
|
}
|
|
174
307
|
frame["valueRepr"] = __senpi_value_repr(value) if !value.nil? && __senpi_should_display_result?(source)
|
|
308
|
+
memory = __senpi_memory_report
|
|
309
|
+
frame["memory"] = memory unless memory.nil?
|
|
175
310
|
__senpi_emit(frame)
|
|
176
311
|
rescue Exception => error
|
|
177
312
|
__senpi_emit({
|
|
@@ -187,11 +322,17 @@ def __senpi_run_cell(message)
|
|
|
187
322
|
end
|
|
188
323
|
end
|
|
189
324
|
|
|
325
|
+
# Everything defined so far belongs to the runner or the interpreter, never to the user's cells.
|
|
326
|
+
$__senpi_memory_baseline_locals = $__senpi_binding.local_variables
|
|
327
|
+
$__senpi_memory_baseline_globals = global_variables
|
|
328
|
+
|
|
190
329
|
$__senpi_protocol_stdin.each_line do |line|
|
|
191
330
|
message = JSON.parse(line)
|
|
192
331
|
case message["type"]
|
|
193
332
|
when "init"
|
|
333
|
+
__senpi_emit({ "type" => "status", "event" => { "op" => "kernel-startup", "stage" => "host-init" } })
|
|
194
334
|
$__senpi_connection = message["connection"]
|
|
335
|
+
$__senpi_memory = message["memory"]
|
|
195
336
|
__senpi_emit({ "type" => "ready" })
|
|
196
337
|
when "run"
|
|
197
338
|
__senpi_run_cell(message)
|
|
@@ -10,8 +10,9 @@ export type FootprintReader = (pid: number) => { readonly bytes: number } | unde
|
|
|
10
10
|
|
|
11
11
|
export interface KernelMemoryHostOptions {
|
|
12
12
|
/**
|
|
13
|
-
* Host-measured kernel (rb, jl): the
|
|
14
|
-
*
|
|
13
|
+
* Host-measured kernel (rb, jl): the host reads the interpreter's footprint after each result and
|
|
14
|
+
* replaces the runner's live reading with it; the runner still names its largest globals, so the
|
|
15
|
+
* notice and the ceiling both apply, with every reading counting (no collection precedes it).
|
|
15
16
|
*/
|
|
16
17
|
readonly readFootprint?: FootprintReader;
|
|
17
18
|
}
|
|
@@ -31,12 +32,15 @@ export class KernelMemoryHost {
|
|
|
31
32
|
this.#policy =
|
|
32
33
|
this.#readFootprint === undefined
|
|
33
34
|
? new KernelMemoryPolicy(language, thresholds)
|
|
34
|
-
: new KernelMemoryPolicy(language,
|
|
35
|
+
: new KernelMemoryPolicy(language, thresholds, { collects: false });
|
|
35
36
|
}
|
|
36
37
|
|
|
37
38
|
annotate(result: ResultMessage, pid?: number): ResultMessage {
|
|
39
|
+
const runnerReport = result.memory;
|
|
38
40
|
const report =
|
|
39
|
-
this.#readFootprint === undefined
|
|
41
|
+
this.#readFootprint === undefined
|
|
42
|
+
? runnerReport
|
|
43
|
+
: this.#footprintReport(this.#readFootprint, pid, runnerReport);
|
|
40
44
|
return report === undefined ? result : { ...result, memory: this.#policy.annotate(report) };
|
|
41
45
|
}
|
|
42
46
|
|
|
@@ -54,10 +58,14 @@ export class KernelMemoryHost {
|
|
|
54
58
|
this.#policy.kernelRetired();
|
|
55
59
|
}
|
|
56
60
|
|
|
57
|
-
#footprintReport(read: FootprintReader, pid: number | undefined) {
|
|
61
|
+
#footprintReport(read: FootprintReader, pid: number | undefined, runnerReport: ResultMessage["memory"]) {
|
|
58
62
|
if (pid === undefined) return undefined;
|
|
59
63
|
const footprint = read(pid);
|
|
60
64
|
if (footprint === undefined) return undefined;
|
|
61
|
-
return {
|
|
65
|
+
return {
|
|
66
|
+
liveBytes: Math.round(footprint.bytes),
|
|
67
|
+
measure: "footprint" as const,
|
|
68
|
+
...(runnerReport?.globals === undefined ? {} : { globals: runnerReport.globals }),
|
|
69
|
+
};
|
|
62
70
|
}
|
|
63
71
|
}
|
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
import type { HostToKernelMessage, KernelToHostMessage } from "../../bridge/protocol.ts";
|
|
2
2
|
import { generateCorrelationId } from "../../bridge/protocol.ts";
|
|
3
3
|
import { RESERVED_AGENT_TOOL } from "../../bridge/reserved.ts";
|
|
4
|
-
import { kernelToolError } from "
|
|
4
|
+
import { kernelToolError } from "../js/kernel-tools-errors.ts";
|
|
5
5
|
import type {
|
|
6
6
|
KernelToolsDescribeResult,
|
|
7
7
|
KernelToolsInvokeOptions,
|
|
8
8
|
KernelToolsInvokeRequest,
|
|
9
9
|
KernelToolsInvokeScope,
|
|
10
|
-
} from "
|
|
10
|
+
} from "../js/kernel-tools-types.ts";
|
|
11
11
|
|
|
12
12
|
type KernelToolReply = Extract<
|
|
13
13
|
KernelToHostMessage,
|