@code-yeongyu/senpi-codemode 2026.10.7 → 2026.10.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. package/CHANGELOG.md +38 -0
  2. package/README.md +67 -5
  3. package/package.json +5 -4
  4. package/src/bridge/http-server.ts +9 -1
  5. package/src/bridge/kernel-tools-protocol.ts +8 -0
  6. package/src/bridge/protocol.ts +4 -0
  7. package/src/bridges/eval-virtual-schemas.ts +22 -0
  8. package/src/environments/editable-check.ts +116 -0
  9. package/src/environments/install-lock.ts +223 -0
  10. package/src/environments/py-environment.ts +98 -0
  11. package/src/environments/py-installer.ts +247 -0
  12. package/src/environments/python-environments.ts +85 -0
  13. package/src/environments/revision-store.ts +110 -0
  14. package/src/extension/bridge-tool-call.ts +2 -0
  15. package/src/extension/cell-kernel-tools.ts +28 -0
  16. package/src/extension/kernel-registration.ts +5 -0
  17. package/src/extension/kernel-replacement.ts +20 -0
  18. package/src/extension/runtime-factory.ts +12 -0
  19. package/src/extension/session-manager-proxy.ts +6 -0
  20. package/src/extension/session-manager.ts +24 -2
  21. package/src/index.ts +1 -0
  22. package/src/kernels/jl/prelude.jl +6 -1
  23. package/src/kernels/js/context-manager.ts +15 -1
  24. package/src/kernels/js/kernel-tools-errors.js +2 -0
  25. package/src/kernels/js/kernel-tools-errors.ts +2 -0
  26. package/src/kernels/js/local-module-loader.ts +9 -4
  27. package/src/kernels/js/worker-runtime.js +3 -1
  28. package/src/kernels/js/workpool.js +5 -2
  29. package/src/kernels/py/kernel-contract.ts +11 -2
  30. package/src/kernels/py/kernel-tools-host.ts +97 -0
  31. package/src/kernels/py/kernel.ts +48 -10
  32. package/src/kernels/py/prelude.py +771 -18
  33. package/src/kernels/py/process.ts +15 -4
  34. package/src/kernels/py/startup.ts +58 -20
  35. package/src/kernels/py/transport.ts +49 -8
  36. package/src/kernels/rb/workpool.rb +7 -1
  37. package/src/kernels/sandbox/vendor/pi-codemode/LICENSE +21 -0
  38. package/src/kernels/sandbox/vendor/pi-codemode/VENDORED.md +42 -0
  39. package/src/kernels/sandbox/vendor/pi-codemode/identifier.ts +14 -0
  40. package/src/kernels/sandbox/vendor/pi-codemode/runtime/host.ts +514 -0
  41. package/src/kernels/sandbox/vendor/pi-codemode/runtime/prelude-source.ts +431 -0
  42. package/src/kernels/sandbox/vendor/pi-codemode/runtime/protocol.ts +61 -0
  43. package/src/kernels/sandbox/vendor/pi-codemode/runtime/worker.ts +231 -0
  44. package/src/kernels/sandbox/vendor/pi-codemode/types.ts +185 -0
  45. package/src/kernels/sandbox/vendor/pi-codemode/wasm.ts +37 -0
  46. package/src/kernels/shared/cell-source-at-start.ts +18 -0
  47. package/src/kernels/shared/kernel-tools-pump.ts +6 -0
  48. package/src/kernels/shared/process-cpu.ts +69 -0
  49. package/src/kernels/shared/subprocess-queue.ts +1 -0
  50. package/src/tool/cell-handler.ts +1 -1
  51. package/src/tool/eval-tool-options.ts +3 -0
  52. package/src/tool/load-cell.ts +73 -0
  53. package/src/tool/magic-cell-host.ts +65 -0
  54. package/src/tool/magic-cells.ts +65 -0
  55. package/src/tool/run-eval-cell.ts +30 -0
  56. package/src/tool/types.ts +27 -0
package/CHANGELOG.md CHANGED
@@ -12,6 +12,44 @@
12
12
 
13
13
  ### Removed
14
14
 
15
+ ## [2026.10.9] - 2026-10-04
16
+
17
+ ### Breaking Changes
18
+
19
+ ### Added
20
+
21
+ - Python `agent(prompt, tools=[...])` grants the child the cell's `@tool` functions by name, like JavaScript's `agent(prompt, { tools })`; anything other than a list of names is refused with `invalid_tools` ([#2731](https://github.com/code-yeongyu/senpi/issues/2731)).
22
+ - `workpool(agent, name, {mode, tools})` forwards `tools`, a list of kernel-tool names the cell defined, to the host workpool unchanged in all four languages, so pool workers can call them; anything other than a list of names is refused with `invalid_tools` before reaching the host. Which kernels' tools a host accepts is the host's call: JavaScript `tool(fn)` tools work on omo today, while Python `@tool` tools need omo#9529 ([#2452](https://github.com/code-yeongyu/senpi/issues/2452)).
23
+ - A Python or JavaScript cell that is only `%load <path>` runs that local file as the cell: its definitions persist, Python tracebacks name the file and its sibling modules import, and JavaScript resolves the file's relative imports from its directory; remote URLs are refused ([#2452](https://github.com/code-yeongyu/senpi/issues/2452)).
24
+ - A Python cell that is only `%pip install <requirements>` installs packages without restarting the kernel; the next cell imports them. Packages go into the session's own environment (or `<cwd>/.senpi/python-packages` after `%environment project`), never the interpreter's site-packages or the user site, and a failed or cancelled install leaves the previous packages active ([#2452](https://github.com/code-yeongyu/senpi/issues/2452)).
25
+
26
+ ### Changed
27
+
28
+ ### Fixed
29
+
30
+ - A Python cell can now grant its `@tool` functions to a child ([#2731](https://github.com/code-yeongyu/senpi/issues/2731)). Host calls from Python cells (`tool.task(..., tools=[...])`, `agent(..., tools=[...])`, `workpool(..., tools=[...])`) reached the host with no kernel-tools capability, so every such grant was refused as unavailable. Each call now carries its cell, and the host gives it that cell's capability while the cell runs; a call from another, unknown or finished cell gets none.
31
+ - `%pip` parsing follow-ups ([#2689](https://github.com/code-yeongyu/senpi/pull/2689)): a `%pip` or `%environment` line after code now says to put it on its own cell instead of "Unsupported line magic"; a comment line ending in a backslash no longer swallows the `%pip` line after it; inside double quotes a backslash is kept unless it escapes a quote, backslash, `$` or a backtick, as a POSIX shell does.
32
+
33
+ ### Removed
34
+
35
+ ## [2026.10.8] - 2026-10-04
36
+
37
+ ### Breaking Changes
38
+
39
+ ### Added
40
+
41
+ - Internal groundwork for installing Python packages from a cell: per-session environment revisions that are published only after a successful install (a failed or interrupted install leaves the previous revision active), a per-root install lock, and a pip installer that always targets the session's own directory. Not exposed to cells yet ([#2452](https://github.com/code-yeongyu/senpi/issues/2452)).
42
+ - Python kernel tools: `@tool` registers a function that in-process children can call, with its schema inferred from type hints; callbacks are served while the kernel is idle or its cell waits on a host call, never during a running computation, and a reset or redefinition makes old descriptors stale. `tool.defined()` / `tool.undefine()` in Python, and `tool_schema("eval:kernel-tools")` ([#2452](https://github.com/code-yeongyu/senpi/issues/2452)).
43
+ - Internal groundwork for isolated sandbox cells: a vendored copy of the pi codemode runtime (QuickJS in a worker) with two opt-in host options, output streaming bounded by a credit window and a store policy that keeps no state. Nothing uses it yet ([#2452](https://github.com/code-yeongyu/senpi/issues/2452)).
44
+
45
+ ### Changed
46
+
47
+ ### Fixed
48
+
49
+ - A cold Python kernel start on a busy machine no longer fails as a hang: startup keeps waiting while the interpreter is still using CPU or writing output, and fails only when it has gone completely still (naming the stage), or after 120 s without becoming ready ([#2718](https://github.com/code-yeongyu/senpi/issues/2718)).
50
+
51
+ ### Removed
52
+
15
53
  ## [2026.10.7] - 2026-10-04
16
54
 
17
55
  ### Breaking Changes
package/README.md CHANGED
@@ -100,6 +100,51 @@ getter and may name a file that does not exist yet. It honors session-directory
100
100
  and in-memory sessions; it cannot be derived reliably from `PI_SESSION_FILE`. If the host
101
101
  omits the getter, the variable is unset rather than inherited from the launching process.
102
102
 
103
+ ### Loading a file
104
+
105
+ A Python or JavaScript cell whose only line is `%load <path>` reads that file and
106
+ runs it as the cell, so its definitions persist like any cell. The path is relative
107
+ to the session directory, or a `local://` or `file://` URL; other URL schemes are
108
+ refused and nothing is fetched. The cell runs as its file: Python compiles it under
109
+ the file's name, so tracebacks name the file and line, and sets `__file__` and puts
110
+ the file's directory first on the import path, so `from sibling import g` resolves
111
+ next to it. JavaScript evaluates it under the file's name and resolves its relative
112
+ imports from the file's directory. The file is read when the cell's turn comes in
113
+ the kernel's queue, so a `%load` queued behind the cell that writes the file runs
114
+ what that cell wrote. When the cell ends, Python takes the file's directory back off
115
+ the import path and restores `__file__`, so later cells import as before. A file that
116
+ cannot be read fails the cell with a message naming the path as written
117
+ (`file not found: <path>`, `permission denied: <path>`); files over 8 MiB, and
118
+ anything that is not a regular file, are refused.
119
+
120
+ ### Python packages
121
+
122
+ A Python cell whose only line is `%pip install <requirements>` installs packages
123
+ without restarting the kernel: variables, the process and its working directory
124
+ stay as they were. The cell runs in the kernel's queue like any other cell (it waits
125
+ for the cell ahead of it, can be cancelled while queued, and detaches past the
126
+ foreground window), but the install runs on the host, so its time does not count
127
+ against the run budget. The next cell imports the new packages; modules a cell
128
+ already imported stay cached until `reset`.
129
+
130
+ Packages go into the session's own environment, never into the interpreter's
131
+ site-packages or the user site: pip always runs as
132
+ `<kernel interpreter> -m pip install --target <environment>`, and flags that pick
133
+ another destination (`--target`, `--user`, `--prefix`, `--root`, `--home`,
134
+ `-e`) are refused. Each install builds a new revision of the environment and
135
+ publishes it only when pip succeeds, so a failed or cancelled install leaves the
136
+ previous packages active and importable. A cell that mixes `%pip` with code is
137
+ refused; put `%pip` on its own cell.
138
+
139
+ `%environment managed` (the default) keeps the environment under the session's
140
+ artifacts directory, or `environments.managedRoot` when set; `%environment project`
141
+ installs into `<cwd>/.senpi/python-packages` instead, shared by every session in
142
+ that project, with concurrent installs serialised by a lock. Setting
143
+ `environments.autoProvision` to `false` turns installs off
144
+ (`environment_installer_unavailable`). Failures carry pip's error output and one
145
+ of `environment_install_failed`, `environment_install_cancelled`,
146
+ `environment_installer_unavailable` or `environment_resolution_conflict`.
147
+
103
148
  ## Settings
104
149
 
105
150
  Configuration is loaded in this order:
@@ -202,11 +247,12 @@ options object and asynchronous helpers are `await`-able.
202
247
  | `tool_schema(name?)` | Returns a tool's parameter schema without calling it; omit `name` to list tool names. |
203
248
  | `tool(fn, metadata?)` (js) | Registers a named function as a kernel tool for in-process children. `metadata.name` registers it under that name instead of the function's; arguments are still passed in the function's parameter order. |
204
249
  | `tool.defined()` / `tool.undefine(name)` (js) | List the kernel tools this kernel defines (sorted), and remove one (`true` if it existed). A descriptor taken before `undefine` can no longer be invoked. Both names are reserved: a kernel tool can't be registered as `defined` or `undefine` (`reserved_tool_name`), and a host tool with either name is shadowed in the `tool` namespace, so it can't be called from a JavaScript cell. |
250
+ | `@tool` / `@tool(name=, description=, schema=)` (py) | Registers a Python function as a kernel tool for in-process children; the schema is inferred from its type hints (`tool_schema("eval:kernel-tools")` lists the rules). Callbacks run while the kernel is idle or its cell is parked in a host call, never in the middle of a running computation. `tool.defined()` and `tool.undefine(name)` work as in JavaScript. Ruby and Julia kernels answer `tools_unavailable`. |
205
251
  | `completion(prompt, model?, system?, schema?)` | Requests a one-shot host completion; `schema` asks the host to parse structured output. |
206
252
  | `agent(prompt, ...)` | Delegates to the configured active `taskTools.task` tool. Supports background handles and structured JSON results. |
207
253
  | `wait(handles, timeout?, mode?)` | Blocks the cell until the given handles settle (agent handle records, `handle()` views, completion handles, closed workpools, or saved `{kind, id, run_epoch}` references). `mode` is `all` (values in input order; the first failed, cancelled, or lost handle raises), `any` (`{index, ref, value}` of the first success), or `settled` (one outcome per input slot). `timeout` is wall-clock seconds from entry; on expiry `eval_wait_timeout` is raised and nothing is cancelled. Rides the bridge-call path, so the run budget pauses while parked. Agent and workpool handles need the host's `EvalHandleHost` capability (`eval_wait_unavailable` without it); completion handles always work. Julia extends `Base.wait` for handle views (`wait(handle(node))`). Details: `tool_schema("eval:wait")`. |
208
254
  | `handle(node \| ref \| {pool_id})` | Returns a rich view: the legacy record's fields plus a non-enumerable `control` (Python: attribute on a `dict` subclass; Ruby: singleton method) with `status()`, `output(format?, offset?, limit?)`, `send(message)` (agent handles only), `cancel()` (idempotent for that run epoch; never touches a successor run), and `wait(timeout?)`. Every control call is fenced by owner, id, and `run_epoch` inside the task owner (`eval_handle_stale`, `eval_handle_forbidden`). `completion(prompt, handle: true)` returns such a view for a host completion. The `agent(..., handle: true)` record itself is unchanged. Details: `tool_schema("eval:helpers")`. |
209
- | `workpool(agent, name, mode?)` | Creates a thin adapter over the normal host `workpool` tool; exposes `pool_id`, `push(items)`, `close()`, `inspect()`, and `cancel()`. JS awaits creation and operations. |
255
+ | `workpool(agent, name, mode?, tools?)` | Creates a thin adapter over the normal host `workpool` tool; exposes `pool_id`, `push(items)`, `close()`, `inspect()`, and `cancel()`. JS awaits creation and operations. `tools` is a list of kernel-tool names this cell defined; the pool's workers may call exactly those, the host refuses any name the caller doesn't hold, and a value that isn't a list of names raises `invalid_tools`. Ruby and Julia define no kernel tools, so they have nothing to grant. |
210
256
  | `output(ids, format?, offset?, limit?)` | Delegates transcript retrieval to the configured active `taskTools.output` tool. |
211
257
  | `parallel(thunks)` | Runs thunks through the configured bounded pool while preserving input order. |
212
258
  | `pipeline(items, ...stages)` | Applies stages left to right with a barrier between stages. |
@@ -255,8 +301,11 @@ Any isolation metadata supplied by the host on a handle is preserved as
255
301
  `{subagent_type, prompt, model?}` as its plain-data agent spec. Mode is `fresh`
256
302
  or `keep_alive`: pass `{mode: "fresh"}` in JS, `mode="fresh"` in Python/Julia,
257
303
  or `mode: "fresh"` in Ruby. Omission is forwarded unchanged to the engine;
258
- hosts without an approved default still require an explicit mode. Custom tool
259
- names are not enabled by this adapter.
304
+ hosts without an approved default still require an explicit mode. `tools`
305
+ (kernel-tool names, for example from Python `@tool` or JavaScript `tool(fn)`) is
306
+ forwarded unchanged to the host's `create`, which validates it: `{tools: ["add"]}`
307
+ in JS, `tools=["add"]` in Python/Julia, `tools: ["add"]` in Ruby. There is no
308
+ `pool.wait()`; the host delivers the aggregate.
260
309
 
261
310
  `push` forwards `[{key, input}]` and returns the host receipt without waiting
262
311
  for admission. Operations return the same `{text, details, images?, hasError?}`
@@ -461,8 +510,21 @@ Homebrew (put `/opt/homebrew/opt/ruby/bin` first on `PATH`; macOS's system Ruby
461
510
  before `--write-baseline`.
462
511
 
463
512
  The report is gitignored `gate-report.json` by default (`--report <path>` overrides it).
464
- `test/gate/allowlist.json` contains reviewed additive changes keyed by plan node;
465
- it cannot authorize removal or modification of a legacy entry. The test-only
513
+ `test/gate/allowlist.json` contains reviewed additive changes keyed by plan node.
514
+ A pull request that changes the committed `baseline.json` itself (an intended
515
+ schema or prompt change, a renamed contract, a deleted module) must list every
516
+ cell it edits or removes under that node's `changes`, each as
517
+ `{ "key": "<section>/<cell>", "reason": "..." }`. In a pull request the gate reads
518
+ the baseline at the merge base and fails any baseline edit or removal that is
519
+ not listed, any edited or added cell that differs from what the head measures,
520
+ and any listed removal the head still measures. A cell written into the baseline that
521
+ was absent at the merge base must be listed under some node's `additions` (an entry an earlier
522
+ pull request added counts, so a re-record can write in cells approved before); only `changes`
523
+ entries must be new in this pull request. Reviewers read each `changes`
524
+ entry: it is the only record of why a protected surface moved. If the merge
525
+ base cannot be resolved in a pull request, the gate fails closed; the gate job
526
+ checks out full history for this. A local run outside a pull request reviews
527
+ baseline changes only with `--base-ref <ref>`. The test-only
466
528
  `SENPI_CODEMODE_GATE_MUTATE=drop-phase` report mutation proves that helper removal
467
529
  is rejected. `SENPI_CODEMODE_GATE_MUTATE=leak-kernel` leaves the real kernel
468
530
  alive at the teardown witness, then closes it in `finally`; nonzero process,
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@code-yeongyu/senpi-codemode",
3
- "version": "2026.10.7",
3
+ "version": "2026.10.9",
4
4
  "description": "Source-only senpi extension package for codemode evaluation tools",
5
5
  "type": "module",
6
6
  "main": "./src/index.ts",
@@ -32,14 +32,15 @@
32
32
  },
33
33
  "dependencies": {
34
34
  "@babel/parser": "8.0.4",
35
- "@earendil-works/pi-ai": "npm:@code-yeongyu/senpi-ai@2026.10.7",
35
+ "@earendil-works/pi-ai": "npm:@code-yeongyu/senpi-ai@2026.10.9",
36
+ "quickjs-wasi": "3.6.2",
36
37
  "typebox": "1.3.34"
37
38
  },
38
39
  "peerDependencies": {
39
- "@code-yeongyu/senpi": "2026.10.7"
40
+ "@code-yeongyu/senpi": "2026.10.9"
40
41
  },
41
42
  "devDependencies": {
42
- "@code-yeongyu/senpi": "2026.10.7"
43
+ "@code-yeongyu/senpi": "2026.10.9"
43
44
  },
44
45
  "keywords": [
45
46
  "senpi",
@@ -8,6 +8,8 @@ const LOOPBACK_HOST = "127.0.0.1";
8
8
 
9
9
  export interface BridgeHttpCallRequest {
10
10
  callId: string;
11
+ /** The calling run's secret; the host resolves that run's kernel-tools capability from it. */
12
+ cellToken?: string;
11
13
  toolName: string;
12
14
  args: unknown;
13
15
  signal: AbortSignal;
@@ -132,7 +134,13 @@ async function dispatchCall(body: unknown, options: BridgeServerOptions, signal:
132
134
  try {
133
135
  return {
134
136
  ok: true,
135
- value: await options.onCall({ callId: body.callId, toolName: body.toolName, args: body.args, signal }),
137
+ value: await options.onCall({
138
+ callId: body.callId,
139
+ ...(typeof body.cellToken === "string" ? { cellToken: body.cellToken } : {}),
140
+ toolName: body.toolName,
141
+ args: body.args,
142
+ signal,
143
+ }),
136
144
  };
137
145
  } catch (error) {
138
146
  return { ok: false, error: bridgeError(error) };
@@ -92,12 +92,20 @@ export const kernelToolKernelToHostSchemas = [
92
92
  requestId: Type.String({ minLength: 1 }),
93
93
  ok: Type.Literal(true),
94
94
  value: Type.Unknown(),
95
+ /** Python: text the tool printed during this call (never the parent cell's output). */
96
+ output: Type.Optional(Type.String()),
95
97
  }),
96
98
  Type.Object({
97
99
  type: Type.Literal("kernel-tool-invoke-reply"),
98
100
  requestId: Type.String({ minLength: 1 }),
99
101
  ok: Type.Literal(false),
100
102
  error: kernelToolErrorSchema,
103
+ output: Type.Optional(Type.String()),
104
+ }),
105
+ /** Python: the kernel's tool names after a define or undefine, for cross-language collision checks. */
106
+ Type.Object({
107
+ type: Type.Literal("kernel-tools-defined"),
108
+ names: Type.Array(Type.String({ minLength: 1 })),
101
109
  }),
102
110
  ] as const;
103
111
 
@@ -44,6 +44,10 @@ const hostToKernelMessageSchema = Type.Union([
44
44
  cellId: Type.String({ minLength: 1 }),
45
45
  code: Type.String(),
46
46
  timeoutMs: Type.Optional(Type.Integer({ minimum: 1 })),
47
+ /** Python only: the published environment revision to import from, applied before the cell runs; empty removes the previous one. */
48
+ envRoot: Type.Optional(Type.String()),
49
+ /** The file a `%load` cell runs; the runner compiles under its name and imports beside it. */
50
+ sourceFile: Type.Optional(Type.String({ minLength: 1 })),
47
51
  /** Python only: active tool globals to install before the cell and deactivated ones to remove. */
48
52
  preludes: Type.Optional(
49
53
  Type.Object({
@@ -52,6 +52,7 @@ const helpersEntry: EvalSchemaResult = {
52
52
  "control.status() goes through the host subscription; control.send/cancel/output are fenced by owner, id and run_epoch inside the task owner (stale -> eval_handle_stale, foreign -> eval_handle_forbidden); control.output() returns only that epoch's transcript. The plain output() helper and task tools are unchanged.",
53
53
  "The legacy agent(..., {handle: true}) record is unchanged: .output stays a string and no enumerable key is added. A saved {kind, id, run_epoch} (or the record itself) can be rebound with handle(ref) in a later cell or after a kernel reset while the owner and run epoch are valid.",
54
54
  "completion(prompt, {handle: true}) (py: handle=True) returns a control handle after preflight validation; provider failures surface in its outcome; its deadline derives from the creating cell's hard deadline. Completion handles are codemode-owned and need no host capability; once the session generation is dropped they fail with eval_handle_stale.",
55
+ "workpool(agent, name, {mode?, tools?}) (py/jl: mode=, tools=; rb: mode:, tools:) forwards tools (kernel-tool names, e.g. from @tool or tool(fn)) unchanged to the host workpool tool's create; the host checks them. There is no pool.wait(): the host delivers the aggregate and the kernel stays free to serve kernel tools.",
55
56
  "See tool_schema('eval:wait') for the barrier semantics.",
56
57
  ].join("\n"),
57
58
  parameters: {
@@ -69,9 +70,30 @@ const helpersEntry: EvalSchemaResult = {
69
70
  },
70
71
  };
71
72
 
73
+ const kernelToolsEntry: EvalSchemaResult = {
74
+ name: "eval:kernel-tools",
75
+ description: [
76
+ "Kernel tools: functions a cell defines that in-process children can call (js and py; rb/jl answer tools_unavailable).",
77
+ "js: tool(fn, {name?, description?, schema?}) registers a named function; arguments are passed in its parameter order; without schema only the parameter names are inferred and every one is required.",
78
+ 'py: @tool or @tool(name=..., description=..., schema=...) registers the function and returns it unchanged. The schema is inferred from type hints (str, int, float, bool, None, list[T], dict[str, T], Literal, unions and Optional as nullable, Annotated[T, "description"], keyword-only parameters); a JSON default makes a parameter optional; string annotations (from __future__ import annotations) resolve through the function\'s globals. Positional-only, *args, **kwargs and unresolvable or unsupported annotations are refused with invalid_tool_definition unless schema= is given. The function is never called to infer anything.',
79
+ "tool.defined() lists this kernel's kernel tools (sorted); tool.undefine(name) removes one (true if it existed). The names defined and undefine are reserved.",
80
+ "Grants: a child is given descriptors (name, kernel_generation, definition_revision) before it starts. A reset, restart or redefinition makes an old descriptor fail with kernel_tool_stale; the new definition never satisfies it. A name defined in both js and py fails describe with tool_name_collision.",
81
+ "py callbacks run while the kernel is idle or its cell is parked in a host call (a tool call, wait(), an install); a cell busy in pure computation is never interrupted, the callback waits. A callback's printed text goes to its own reply, never the parent cell's output. Cancelling a call ends it at its next host call (kernel_tool_cancelled); a computation already running finishes and its result is dropped. An awaited object from the parent cell's event loop fails with kernel_tool_loop_mismatch.",
82
+ ].join("\n"),
83
+ parameters: {
84
+ type: "object",
85
+ properties: {
86
+ tool: { description: "js: tool(fn, metadata?); py: @tool / @tool(name=, description=, schema=)" },
87
+ "tool.defined": { description: "() -> names[]" },
88
+ "tool.undefine": { description: "(name) -> boolean" },
89
+ },
90
+ },
91
+ };
92
+
72
93
  const VIRTUAL_ENTRIES: ReadonlyMap<string, EvalSchemaResult> = new Map([
73
94
  [waitEntry.name, waitEntry],
74
95
  [helpersEntry.name, helpersEntry],
96
+ [kernelToolsEntry.name, kernelToolsEntry],
75
97
  ]);
76
98
 
77
99
  export function virtualEvalSchema(name: string): EvalSchemaResult | undefined {
@@ -0,0 +1,116 @@
1
+ import { lstat, readdir, readFile, realpath } from "node:fs/promises";
2
+ import { join, relative, resolve, sep } from "node:path";
3
+ import { EnvironmentError } from "./py-installer.ts";
4
+
5
+ /**
6
+ * Refuses a staged revision that links to code outside itself: a PEP 660 editable install
7
+ * (`direct_url.json` with `dir_info.editable`), a legacy `setup.py develop` (`.egg-link`), a `.pth` path
8
+ * line that resolves outside the revision, or a symlink anywhere in it that does. `.pth` lines starting
9
+ * with `import` are code that runs at startup, like any installed package's code, and are not inspected.
10
+ * A refused revision is never published.
11
+ */
12
+ export async function assertNoEditableInstalls(staging: string): Promise<void> {
13
+ const root = await realpath(staging).catch(() => resolve(staging));
14
+ let entries: string[];
15
+ try {
16
+ entries = await readdir(staging);
17
+ } catch {
18
+ return;
19
+ }
20
+ for (const entry of entries) {
21
+ const path = join(staging, entry);
22
+ if (entry.endsWith(".egg-link"))
23
+ refuse(entry.replace(/\.egg-link$/u, ""), "a legacy editable install (.egg-link)");
24
+ if (
25
+ entry.endsWith(".dist-info") &&
26
+ isEditable(await readFile(join(path, "direct_url.json"), "utf8").catch(() => ""))
27
+ ) {
28
+ refuse(entry.replace(/\.dist-info$/u, ""), "an editable install");
29
+ }
30
+ if (entry.endsWith(".pth")) {
31
+ for (const line of (await readFile(path, "utf8").catch(() => "")).split(/\r?\n/u)) {
32
+ const text = line.trim();
33
+ if (text === "" || text.startsWith("#") || text.startsWith("import ")) continue;
34
+ if (!(await inside(root, resolve(staging, text))))
35
+ refuse(entry, `a path outside the environment (${text})`);
36
+ }
37
+ }
38
+ }
39
+ await refuseOutsideLinks(root, staging);
40
+ }
41
+
42
+ /** Every symlink in the revision, at any depth, must resolve inside it; links are checked, not followed. */
43
+ async function refuseOutsideLinks(root: string, dir: string): Promise<void> {
44
+ for (const entry of await readdir(dir)) {
45
+ const path = join(dir, entry);
46
+ const info = await lstat(path);
47
+ if (info.isSymbolicLink()) {
48
+ if (!(await inside(root, path))) refuse(relative(root, path), "a link that points outside the environment");
49
+ } else if (info.isDirectory()) {
50
+ await refuseOutsideLinks(root, path);
51
+ }
52
+ }
53
+ }
54
+
55
+ /**
56
+ * pip reports what it installed; each of those distributions must be in the revision. A configuration
57
+ * that redirects pip elsewhere (a target, root or prefix from any config file or variable, on any pip
58
+ * version) leaves them missing here, so the install fails instead of publishing an empty revision.
59
+ */
60
+ export async function assertInstalledInRevision(stdout: string, staging: string): Promise<void> {
61
+ const line = stdout.split("\n").find((entry) => entry.startsWith("Successfully installed "));
62
+ if (line === undefined) return;
63
+ const present = new Set((await readdir(staging).catch(() => [] as string[])).flatMap(installedName));
64
+ for (const token of line.slice("Successfully installed ".length).trim().split(/\s+/u)) {
65
+ const name = token.slice(0, token.lastIndexOf("-"));
66
+ if (!present.has(canonicalName(name))) {
67
+ throw new EnvironmentError(
68
+ "environment_install_failed",
69
+ `${name} was installed outside the session's environment (pip's configuration redirected it); nothing was published`,
70
+ );
71
+ }
72
+ }
73
+ }
74
+
75
+ /**
76
+ * The project a metadata entry records: `name-version.dist-info` (wheels), or `name-version[-pyX.Y].egg-info`
77
+ * (a setup.py project pip <= 23.0 installs without the wheel package). Both escape `-` in the name to `_`.
78
+ */
79
+ function installedName(entry: string): string[] {
80
+ if (entry.endsWith(".dist-info")) {
81
+ const stem = entry.slice(0, -".dist-info".length);
82
+ return [canonicalName(stem.slice(0, stem.lastIndexOf("-")))];
83
+ }
84
+ if (entry.endsWith(".egg-info")) return [canonicalName(entry.split("-")[0] ?? "")];
85
+ return [];
86
+ }
87
+
88
+ function canonicalName(name: string): string {
89
+ return name.toLowerCase().replace(/[-_.]+/gu, "_");
90
+ }
91
+
92
+ async function inside(root: string, path: string): Promise<boolean> {
93
+ const real = await realpath(path).catch(() => resolve(path));
94
+ const rel = relative(root, real);
95
+ return (
96
+ rel === "" || (!rel.startsWith(`..${sep}`) && rel !== ".." && !rel.startsWith(sep) && !/^[A-Za-z]:/u.test(rel))
97
+ );
98
+ }
99
+
100
+ function refuse(name: string, what: string): never {
101
+ throw new EnvironmentError(
102
+ "environment_install_failed",
103
+ `${name} was installed as ${what}; packages must live in the session's environment root, so editable and linked installs are not allowed`,
104
+ );
105
+ }
106
+
107
+ function isEditable(text: string): boolean {
108
+ try {
109
+ const value: unknown = JSON.parse(text);
110
+ if (typeof value !== "object" || value === null || !("dir_info" in value)) return false;
111
+ const dirInfo = value.dir_info;
112
+ return typeof dirInfo === "object" && dirInfo !== null && "editable" in dirInfo && dirInfo.editable === true;
113
+ } catch {
114
+ return false;
115
+ }
116
+ }
@@ -0,0 +1,223 @@
1
+ import { randomUUID } from "node:crypto";
2
+ import { watch } from "node:fs";
3
+ import { link, readdir, readFile, rm, stat, writeFile } from "node:fs/promises";
4
+ import { hostname } from "node:os";
5
+ import { join } from "node:path";
6
+
7
+ const LOCK_FILE = ".install.lock";
8
+ const REAP_PREFIX = ".install.lock.reap.";
9
+ const RECHECK_MS = 500;
10
+ // An unparseable lock can only come from a crash in an older version's create-then-write; it is stale once old.
11
+ const UNREADABLE_STALE_MS = 5_000;
12
+ const WAIT_NOTICE_MS = 30_000;
13
+
14
+ type Holder = { readonly pid: number; readonly host: string; readonly nonce: string };
15
+
16
+ export type LockWaitNotice = {
17
+ readonly holder: { readonly pid: number; readonly host: string } | undefined;
18
+ readonly waitedMs: number;
19
+ };
20
+
21
+ /**
22
+ * Serialises installs into one environment root across sessions and processes.
23
+ *
24
+ * The lock file is published complete (written to a unique temp file, then `link`ed into place, which fails
25
+ * if a lock exists), so it is never seen empty. A lock whose holder is a dead process on this host, or
26
+ * that is unreadable and old, is stale. Each lock incarnation has its own key (its holder's nonce), and
27
+ * only the waiter that exclusively creates the reap claim for that key may remove it, after re-reading the
28
+ * lock under the claim and finding the same incarnation. A stale incarnation's holder is dead and nobody
29
+ * else may remove it, so it can't change between that re-read and the removal: a live lock is never
30
+ * deleted. A claimer that dies passes the claim on to the next claim in the chain.
31
+ */
32
+ export async function withRootLock<T>(
33
+ base: string,
34
+ fn: () => Promise<T>,
35
+ signal?: AbortSignal,
36
+ onWait?: (notice: LockWaitNotice) => void,
37
+ ): Promise<T> {
38
+ const path = join(base, LOCK_FILE);
39
+ const mine: Holder = { pid: process.pid, host: hostname(), nonce: randomUUID() };
40
+ const started = Date.now();
41
+ let noticed = false;
42
+ for (;;) {
43
+ signal?.throwIfAborted();
44
+ if (await tryCreate(base, path, mine)) break;
45
+ if (await takeOverIfStale(base, path)) continue;
46
+ if (!noticed && Date.now() - started >= WAIT_NOTICE_MS) {
47
+ noticed = true;
48
+ const holder = await readHolder(path);
49
+ onWait?.({ holder: holder && { pid: holder.pid, host: holder.host }, waitedMs: Date.now() - started });
50
+ }
51
+ await waitForRelease(base, path, signal);
52
+ }
53
+ try {
54
+ return await fn();
55
+ } finally {
56
+ const holder = await readHolder(path);
57
+ if (holder?.nonce === mine.nonce) await rm(path, { force: true });
58
+ }
59
+ }
60
+
61
+ async function tryCreate(base: string, path: string, holder: Holder): Promise<boolean> {
62
+ const temp = join(base, `.install.lock.${holder.nonce}`);
63
+ await writeFile(temp, JSON.stringify(holder), { mode: 0o600 });
64
+ try {
65
+ await link(temp, path);
66
+ return true;
67
+ } catch (error) {
68
+ if (errorCode(error) === "EEXIST") return false;
69
+ throw error;
70
+ } finally {
71
+ await rm(temp, { force: true });
72
+ }
73
+ }
74
+
75
+ async function takeOverIfStale(base: string, path: string): Promise<boolean> {
76
+ const key = await staleKey(path);
77
+ if (key === undefined) return false;
78
+ const claim = await claimReap(base, key);
79
+ if (claim === undefined) return false;
80
+ try {
81
+ // Under the claim nobody else may remove this incarnation, and its holder is dead, so it is still the
82
+ // lock judged stale exactly when it still has the same key.
83
+ if ((await staleKey(path)) !== key) return false;
84
+ await rm(path, { force: true });
85
+ await removeDeadClaims(base, key);
86
+ return true;
87
+ } finally {
88
+ await rm(claim, { force: true });
89
+ }
90
+ }
91
+
92
+ /** Once the incarnation is gone, its claims only matter to waiters that will find a different lock. */
93
+ async function removeDeadClaims(base: string, key: string): Promise<void> {
94
+ const prefix = `${REAP_PREFIX}${key}.`;
95
+ for (const name of (await readdir(base)).filter((entry) => entry.startsWith(prefix))) {
96
+ const holder = await readHolder(join(base, name));
97
+ if (holder !== undefined && holder.host === hostname() && !isAlive(holder.pid))
98
+ await rm(join(base, name), { force: true });
99
+ }
100
+ }
101
+
102
+ /**
103
+ * Creates the next free reap claim for one lock incarnation: `<key>.0`, or the claim after the last one
104
+ * whose holder died. Returns undefined when a live waiter already holds it.
105
+ */
106
+ async function claimReap(base: string, key: string): Promise<string | undefined> {
107
+ for (let depth = 0; ; depth++) {
108
+ const claim = join(base, `${REAP_PREFIX}${key}.${depth}`);
109
+ if (await tryCreate(base, claim, { pid: process.pid, host: hostname(), nonce: randomUUID() })) return claim;
110
+ const holder = await readHolder(claim);
111
+ // Gone again (its holder finished): the caller re-reads the lock and retries.
112
+ if (holder === undefined) return undefined;
113
+ if (holder.host !== hostname() || isAlive(holder.pid)) return undefined;
114
+ }
115
+ }
116
+
117
+ async function reapInProgress(base: string, key: string): Promise<boolean> {
118
+ const prefix = `${REAP_PREFIX}${key}.`;
119
+ let names: string[];
120
+ try {
121
+ names = (await readdir(base)).filter((name) => name.startsWith(prefix));
122
+ } catch {
123
+ return false;
124
+ }
125
+ for (const name of names) {
126
+ const holder = await readHolder(join(base, name));
127
+ if (holder !== undefined && holder.host === hostname() && isAlive(holder.pid)) return true;
128
+ }
129
+ return false;
130
+ }
131
+
132
+ /** The incarnation key of a stale lock (its holder's nonce, or its inode when unreadable), else undefined. */
133
+ async function staleKey(path: string): Promise<string | undefined> {
134
+ let content: string;
135
+ try {
136
+ content = await readFile(path, "utf8");
137
+ } catch {
138
+ return undefined;
139
+ }
140
+ const holder = parseHolder(content);
141
+ if (holder !== undefined && (holder.host !== hostname() || isAlive(holder.pid))) return undefined;
142
+ if (holder !== undefined && /^[\w-]+$/u.test(holder.nonce)) return holder.nonce;
143
+ try {
144
+ const info = await stat(path);
145
+ // A dead holder's lock without a usable nonce, or an unreadable old one, is keyed by its inode.
146
+ return holder !== undefined || Date.now() - info.mtimeMs > UNREADABLE_STALE_MS ? `ino${info.ino}` : undefined;
147
+ } catch {
148
+ return undefined;
149
+ }
150
+ }
151
+
152
+ async function readHolder(path: string): Promise<Holder | undefined> {
153
+ try {
154
+ return parseHolder(await readFile(path, "utf8"));
155
+ } catch {
156
+ return undefined;
157
+ }
158
+ }
159
+
160
+ function parseHolder(content: string): Holder | undefined {
161
+ let value: unknown;
162
+ try {
163
+ value = JSON.parse(content);
164
+ } catch {
165
+ return undefined;
166
+ }
167
+ if (typeof value !== "object" || value === null) return undefined;
168
+ const pid = "pid" in value ? value.pid : undefined;
169
+ const host = "host" in value ? value.host : undefined;
170
+ const nonce = "nonce" in value ? value.nonce : undefined;
171
+ return typeof pid === "number" && typeof host === "string"
172
+ ? { pid, host, nonce: typeof nonce === "string" ? nonce : "" }
173
+ : undefined;
174
+ }
175
+
176
+ function isAlive(pid: number): boolean {
177
+ try {
178
+ process.kill(pid, 0);
179
+ return true;
180
+ } catch (error) {
181
+ return errorCode(error) !== "ESRCH";
182
+ }
183
+ }
184
+
185
+ function errorCode(error: unknown): string | undefined {
186
+ return error instanceof Error && "code" in error && typeof error.code === "string" ? error.code : undefined;
187
+ }
188
+
189
+ /** Wakes when the lock file goes away; the periodic recheck covers dropped file-watch events and staleness. */
190
+ function waitForRelease(base: string, path: string, signal?: AbortSignal): Promise<void> {
191
+ return new Promise((resolve, reject) => {
192
+ const watcher = watch(base, () => check());
193
+ const timer = setInterval(() => check(), RECHECK_MS);
194
+ const onAbort = () => done(() => reject(signal?.reason));
195
+ signal?.addEventListener("abort", onAbort, { once: true });
196
+ let settled = false;
197
+ function done(settle: () => void): void {
198
+ if (settled) return;
199
+ settled = true;
200
+ watcher.close();
201
+ clearInterval(timer);
202
+ signal?.removeEventListener("abort", onAbort);
203
+ settle();
204
+ }
205
+ function check(): void {
206
+ void stat(path).then(
207
+ // A stale lock is worth retrying only when no live waiter is already reaping it; otherwise wait.
208
+ () =>
209
+ staleKey(path).then(
210
+ async (key) => key !== undefined && !(await reapInProgress(base, key)) && done(resolve),
211
+ ),
212
+ () => done(resolve),
213
+ );
214
+ }
215
+ // An abort during the claim inspection that led here fired before this listener existed.
216
+ if (signal?.aborted) {
217
+ onAbort();
218
+ return;
219
+ }
220
+ // The holder may have released between the failed create and the watcher starting; no event would follow.
221
+ check();
222
+ });
223
+ }