opencode-ipynb 0.1.2 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +187 -0
- package/README.md +197 -41
- package/dist/domain/cell.d.ts +4 -3
- package/dist/domain/errors.d.ts +3 -31
- package/dist/domain/execution.d.ts +4 -1
- package/dist/domain/notebook.d.ts +6 -3
- package/dist/domain/output.d.ts +0 -1
- package/dist/format/diagnostics.d.ts +0 -1
- package/dist/format/markdown.d.ts +1 -2
- package/dist/format/{diff.d.ts → notebook.d.ts} +1 -1
- package/dist/format/outputs.d.ts +2 -2
- package/dist/index.d.ts +3 -1
- package/dist/index.js +11294 -28152
- package/dist/load-document.d.ts +10 -0
- package/dist/lsp.d.ts +58 -0
- package/dist/lsp.js +172 -0
- package/dist/notebook-diagnostics.d.ts +52 -0
- package/dist/notebook-hooks.d.ts +18 -0
- package/dist/plugin-options.d.ts +4 -1
- package/dist/plugin-v2.d.ts +31 -0
- package/dist/plugin.d.ts +2 -3
- package/dist/registrations.d.ts +5 -0
- package/dist/rpc.d.ts +134 -0
- package/dist/rpc.js +84 -0
- package/dist/services/KernelManager.d.ts +45 -0
- package/dist/services/NotebookCleanService.d.ts +7 -7
- package/dist/services/NotebookEditService.d.ts +15 -12
- package/dist/services/NotebookExecutionService.d.ts +41 -12
- package/dist/services/NotebookExportService.d.ts +3 -7
- package/dist/services/NotebookFileService.d.ts +31 -16
- package/dist/services/NotebookInspectService.d.ts +15 -19
- package/dist/services/NotebookOutputService.d.ts +7 -10
- package/dist/services/NotebookReadService.d.ts +4 -10
- package/dist/services/PathService.d.ts +10 -8
- package/dist/services/PermissionService.d.ts +13 -8
- package/dist/services/PythonService.d.ts +14 -43
- package/dist/services/index.d.ts +23 -28
- package/dist/services/revision.d.ts +9 -0
- package/dist/tool.d.ts +45 -0
- package/dist/tools/_readPermission.d.ts +2 -0
- package/dist/tools/_resolveOptions.d.ts +13 -2
- package/dist/tools/cell_delete.d.ts +4 -13
- package/dist/tools/cell_insert.d.ts +10 -21
- package/dist/tools/cell_move.d.ts +5 -15
- package/dist/tools/clean.d.ts +8 -21
- package/dist/tools/doctor.d.ts +3 -11
- package/dist/tools/edit.d.ts +10 -21
- package/dist/tools/export.d.ts +12 -23
- package/dist/tools/inspect.d.ts +7 -19
- package/dist/tools/kernel.d.ts +13 -21
- package/dist/tools/lsp.d.ts +9 -0
- package/dist/tools/outputs.d.ts +15 -29
- package/dist/tools/read.d.ts +12 -29
- package/dist/tools/repro.d.ts +6 -14
- package/dist/tools/run.d.ts +17 -34
- package/dist/utils/ansi.d.ts +1 -1
- package/dist/utils/atomic-replace.d.ts +2 -0
- package/dist/utils/attachments.d.ts +2 -3
- package/dist/utils/execution-order.d.ts +7 -0
- package/dist/utils/fence.d.ts +1 -0
- package/dist/utils/i18n.d.ts +1 -2
- package/dist/utils/imports.d.ts +0 -1
- package/dist/utils/json-patch.d.ts +58 -0
- package/dist/utils/json.d.ts +16 -2
- package/dist/utils/limits.d.ts +11 -2
- package/dist/utils/markdown-safe.d.ts +2 -0
- package/dist/utils/metadata.d.ts +2 -0
- package/dist/utils/mime.d.ts +1 -1
- package/dist/utils/paths.d.ts +2 -2
- package/dist/utils/realpath.d.ts +1 -0
- package/dist/utils/rename-hook.d.ts +6 -0
- package/dist/utils/truncate.d.ts +0 -1
- package/dist/utils/warnings.d.ts +15 -0
- package/dist/utils/windows.d.ts +3 -0
- package/dist/virtual-materialize.d.ts +25 -0
- package/package.json +30 -18
- package/python/ipynb_runner.py +418 -31
- package/python/requirements.txt +5 -4
- package/server.js +3 -0
- package/dist/domain/cell.d.ts.map +0 -1
- package/dist/domain/errors.d.ts.map +0 -1
- package/dist/domain/execution.d.ts.map +0 -1
- package/dist/domain/notebook.d.ts.map +0 -1
- package/dist/domain/output.d.ts.map +0 -1
- package/dist/format/diagnostics.d.ts.map +0 -1
- package/dist/format/diff.d.ts.map +0 -1
- package/dist/format/markdown.d.ts.map +0 -1
- package/dist/format/outputs.d.ts.map +0 -1
- package/dist/index.d.ts.map +0 -1
- package/dist/plugin-options.d.ts.map +0 -1
- package/dist/plugin.d.ts.map +0 -1
- package/dist/services/DiffService.d.ts +0 -17
- package/dist/services/DiffService.d.ts.map +0 -1
- package/dist/services/NotebookCleanService.d.ts.map +0 -1
- package/dist/services/NotebookEditService.d.ts.map +0 -1
- package/dist/services/NotebookExecutionService.d.ts.map +0 -1
- package/dist/services/NotebookExportService.d.ts.map +0 -1
- package/dist/services/NotebookFileService.d.ts.map +0 -1
- package/dist/services/NotebookInspectService.d.ts.map +0 -1
- package/dist/services/NotebookOutputService.d.ts.map +0 -1
- package/dist/services/NotebookReadService.d.ts.map +0 -1
- package/dist/services/PathService.d.ts.map +0 -1
- package/dist/services/PermissionService.d.ts.map +0 -1
- package/dist/services/PythonService.d.ts.map +0 -1
- package/dist/services/index.d.ts.map +0 -1
- package/dist/tools/_resolveOptions.d.ts.map +0 -1
- package/dist/tools/cell_delete.d.ts.map +0 -1
- package/dist/tools/cell_insert.d.ts.map +0 -1
- package/dist/tools/cell_move.d.ts.map +0 -1
- package/dist/tools/clean.d.ts.map +0 -1
- package/dist/tools/doctor.d.ts.map +0 -1
- package/dist/tools/edit.d.ts.map +0 -1
- package/dist/tools/export.d.ts.map +0 -1
- package/dist/tools/inspect.d.ts.map +0 -1
- package/dist/tools/kernel.d.ts.map +0 -1
- package/dist/tools/outputs.d.ts.map +0 -1
- package/dist/tools/read.d.ts.map +0 -1
- package/dist/tools/repro.d.ts.map +0 -1
- package/dist/tools/run.d.ts.map +0 -1
- package/dist/utils/ansi.d.ts.map +0 -1
- package/dist/utils/attachments.d.ts.map +0 -1
- package/dist/utils/fiber.d.ts +0 -2
- package/dist/utils/fiber.d.ts.map +0 -1
- package/dist/utils/i18n.d.ts.map +0 -1
- package/dist/utils/imports.d.ts.map +0 -1
- package/dist/utils/json.d.ts.map +0 -1
- package/dist/utils/limits.d.ts.map +0 -1
- package/dist/utils/mime.d.ts.map +0 -1
- package/dist/utils/paths.d.ts.map +0 -1
- package/dist/utils/truncate.d.ts.map +0 -1
package/python/ipynb_runner.py
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
"""Real notebook runner for the opencode-ipynb plugin
|
|
1
|
+
"""Real notebook runner for the opencode-ipynb plugin.
|
|
2
2
|
|
|
3
3
|
Reads a JSON request from stdin and writes a JSON response to stdout. Modes:
|
|
4
4
|
|
|
@@ -7,7 +7,7 @@ Reads a JSON request from stdin and writes a JSON response to stdout. Modes:
|
|
|
7
7
|
- ``env``: read the notebook's kernelspec / language_info, dump the
|
|
8
8
|
current Python interpreter version + executable + platform, and run
|
|
9
9
|
``pip freeze --local``. Does NOT require nbformat / nbclient.
|
|
10
|
-
- ``serve`` (
|
|
10
|
+
- ``serve`` (warm-kernel mode): enter a long-lived loop. The first
|
|
11
11
|
request carries ``filePath`` (and optional ``kernel`` / ``timeoutMs``);
|
|
12
12
|
subsequent requests are normal ``cell`` / ``all`` / ``range`` / ``from``
|
|
13
13
|
/ ``env`` requests. Each request has an integer ``id``; the response
|
|
@@ -23,10 +23,18 @@ venv still produces a structured error instead of a traceback.
|
|
|
23
23
|
|
|
24
24
|
from __future__ import annotations
|
|
25
25
|
|
|
26
|
+
MAX_ERROR_TEXT = 8_000
|
|
27
|
+
MAX_RAW_OUTPUT_BYTES = 32 * 1024 * 1024
|
|
28
|
+
MAX_RAW_OUTPUT_TOTAL_BYTES = 48 * 1024 * 1024
|
|
29
|
+
|
|
26
30
|
import json
|
|
27
31
|
import os
|
|
32
|
+
import signal
|
|
28
33
|
import subprocess
|
|
29
34
|
import sys
|
|
35
|
+
import tempfile
|
|
36
|
+
import queue
|
|
37
|
+
import threading
|
|
30
38
|
import time
|
|
31
39
|
import traceback
|
|
32
40
|
from typing import Any, Dict, Iterable, List, Optional
|
|
@@ -34,6 +42,14 @@ from typing import Any, Dict, Iterable, List, Optional
|
|
|
34
42
|
|
|
35
43
|
PROBE_DEPS = ("nbformat", "nbclient", "jupyter_client", "ipykernel")
|
|
36
44
|
|
|
45
|
+
# Windows redirected stdio often uses the locale code page; requests and
|
|
46
|
+
# responses are UTF-8 JSON, so force it explicitly.
|
|
47
|
+
for _stream in (sys.stdin, sys.stdout, sys.stderr):
|
|
48
|
+
try:
|
|
49
|
+
_stream.reconfigure(encoding="utf-8", errors="strict")
|
|
50
|
+
except Exception:
|
|
51
|
+
pass
|
|
52
|
+
|
|
37
53
|
|
|
38
54
|
def _read_request() -> Dict[str, Any]:
|
|
39
55
|
raw = sys.stdin.read()
|
|
@@ -66,7 +82,11 @@ def _read_request_line() -> Dict[str, Any]:
|
|
|
66
82
|
return json.loads(line)
|
|
67
83
|
|
|
68
84
|
|
|
69
|
-
def _write_response(
|
|
85
|
+
def _write_response(
|
|
86
|
+
response: Dict[str, Any], nonce: Optional[str] = None
|
|
87
|
+
) -> None:
|
|
88
|
+
if nonce is not None:
|
|
89
|
+
response["nonce"] = nonce
|
|
70
90
|
sys.stdout.write(json.dumps(response))
|
|
71
91
|
sys.stdout.write("\n")
|
|
72
92
|
sys.stdout.flush()
|
|
@@ -83,17 +103,119 @@ def _read_notebook_raw(file_path: str) -> Dict[str, Any]:
|
|
|
83
103
|
return data
|
|
84
104
|
|
|
85
105
|
|
|
106
|
+
PIP_FREEZE_CODE = (
|
|
107
|
+
"import sys; "
|
|
108
|
+
"sys.path[:] = [p for p in sys.path if p not in ('', '.')]; "
|
|
109
|
+
"from pip._internal.cli.main import main; "
|
|
110
|
+
"raise SystemExit(main(['freeze', '--local']))"
|
|
111
|
+
)
|
|
112
|
+
|
|
113
|
+
|
|
86
114
|
def _run_pip_freeze(python_executable: str, timeout_s: int = 30) -> List[str]:
|
|
87
|
-
"""Run ``pip freeze --local``
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
)
|
|
115
|
+
"""Run ``pip freeze --local`` without importing project files.
|
|
116
|
+
|
|
117
|
+
``python -m pip`` puts the current directory first on sys.path, so a
|
|
118
|
+
repository-supplied ``pip.py`` would execute during a read-only report.
|
|
119
|
+
Use ``-c`` plus explicit sys.path sanitation instead of ``-m``: this
|
|
120
|
+
keeps user-site pip available (unlike ``-I``) while removing the cwd
|
|
121
|
+
entries before pip is imported. ``-E`` drops PYTHONPATH/PYTHONHOME.
|
|
122
|
+
"""
|
|
123
|
+
env = os.environ.copy()
|
|
124
|
+
env.pop("PYTHONPATH", None)
|
|
125
|
+
env.pop("PYTHONHOME", None)
|
|
126
|
+
try:
|
|
127
|
+
raw = subprocess.check_output(
|
|
128
|
+
[python_executable, "-E", "-c", PIP_FREEZE_CODE],
|
|
129
|
+
cwd=tempfile.gettempdir(),
|
|
130
|
+
env=env,
|
|
131
|
+
stderr=subprocess.PIPE,
|
|
132
|
+
timeout=timeout_s,
|
|
133
|
+
)
|
|
134
|
+
except subprocess.CalledProcessError:
|
|
135
|
+
# A venv created with --without-pip (or a bare interpreter) can still
|
|
136
|
+
# run notebooks; the package list is diagnostic, so report it as empty
|
|
137
|
+
# instead of failing the whole env report.
|
|
138
|
+
return []
|
|
93
139
|
text = raw.decode("utf-8", errors="replace")
|
|
94
140
|
return [line.strip() for line in text.splitlines() if line.strip()]
|
|
95
141
|
|
|
96
142
|
|
|
143
|
+
def _teardown_client(client: Any) -> None:
|
|
144
|
+
"""Release whatever the installed nbclient version exposes."""
|
|
145
|
+
for name in ("shutdown_kernel", "cleanup_kernel", "cleanup_resources"):
|
|
146
|
+
method = getattr(client, name, None)
|
|
147
|
+
if callable(method):
|
|
148
|
+
try:
|
|
149
|
+
method()
|
|
150
|
+
except Exception:
|
|
151
|
+
pass
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def _kernel_pid(client: Any) -> Optional[int]:
|
|
155
|
+
km = getattr(client, "km", None)
|
|
156
|
+
provisioner = getattr(km, "provisioner", None)
|
|
157
|
+
pid = getattr(provisioner, "pid", None)
|
|
158
|
+
if not isinstance(pid, int):
|
|
159
|
+
pid = getattr(km, "kernel_pid", None)
|
|
160
|
+
return pid if isinstance(pid, int) and pid > 0 else None
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def _kill_kernel_process_group(client: Any) -> None:
|
|
164
|
+
"""Kill the kernel and every process it started.
|
|
165
|
+
|
|
166
|
+
Jupyter starts the kernel in its own session, so killing the helper's
|
|
167
|
+
process group is not enough. The kernel is the leader of its own group.
|
|
168
|
+
"""
|
|
169
|
+
pid = _kernel_pid(client)
|
|
170
|
+
if pid is None:
|
|
171
|
+
return
|
|
172
|
+
kill_signal = getattr(signal, "SIGKILL", signal.SIGTERM)
|
|
173
|
+
try:
|
|
174
|
+
if hasattr(os, "killpg") and hasattr(os, "getpgid"):
|
|
175
|
+
os.killpg(os.getpgid(pid), kill_signal)
|
|
176
|
+
else:
|
|
177
|
+
os.kill(pid, kill_signal)
|
|
178
|
+
except ProcessLookupError:
|
|
179
|
+
pass
|
|
180
|
+
except Exception:
|
|
181
|
+
try:
|
|
182
|
+
os.kill(pid, kill_signal)
|
|
183
|
+
except Exception:
|
|
184
|
+
pass
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
_ACTIVE_CLIENT: Any = None
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
def _handle_sigterm(signum: int, frame: Any) -> None:
|
|
191
|
+
"""Best-effort cleanup when the wrapper aborts a one-shot/serve run."""
|
|
192
|
+
try:
|
|
193
|
+
if _ACTIVE_CLIENT is not None:
|
|
194
|
+
_kill_kernel_process_group(_ACTIVE_CLIENT)
|
|
195
|
+
finally:
|
|
196
|
+
os._exit(143)
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
signal.signal(signal.SIGTERM, _handle_sigterm)
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def _kernel_env(req: Dict[str, Any]) -> Dict[str, str]:
|
|
203
|
+
"""Environment for the notebook kernel only.
|
|
204
|
+
|
|
205
|
+
The helper process itself runs with PYTHONPATH/PYTHONHOME removed so a
|
|
206
|
+
project cannot shadow stdlib imports. The original values are carried in
|
|
207
|
+
the request and restored for the kernel, where user code is expected to
|
|
208
|
+
run with the environment the user configured.
|
|
209
|
+
"""
|
|
210
|
+
env = {key: value for key, value in os.environ.items()}
|
|
211
|
+
extra = req.get("kernelEnv")
|
|
212
|
+
if isinstance(extra, dict):
|
|
213
|
+
for key, value in extra.items():
|
|
214
|
+
if isinstance(key, str) and isinstance(value, str):
|
|
215
|
+
env[key] = value
|
|
216
|
+
return env
|
|
217
|
+
|
|
218
|
+
|
|
97
219
|
def _build_env_report(
|
|
98
220
|
file_path: str,
|
|
99
221
|
started: float,
|
|
@@ -103,7 +225,7 @@ def _build_env_report(
|
|
|
103
225
|
"""
|
|
104
226
|
try:
|
|
105
227
|
nb = _read_notebook_raw(file_path)
|
|
106
|
-
except FileNotFoundError
|
|
228
|
+
except FileNotFoundError:
|
|
107
229
|
return _error_response(
|
|
108
230
|
None,
|
|
109
231
|
started,
|
|
@@ -191,6 +313,14 @@ def _build_env_report(
|
|
|
191
313
|
}
|
|
192
314
|
|
|
193
315
|
|
|
316
|
+
def _require_positive_int(req: Dict[str, Any], field: str) -> None:
|
|
317
|
+
if field not in req:
|
|
318
|
+
return
|
|
319
|
+
value = req[field]
|
|
320
|
+
if isinstance(value, bool) or not isinstance(value, int) or value <= 0:
|
|
321
|
+
raise ValueError(f"{field} must be a positive integer, got {value!r}")
|
|
322
|
+
|
|
323
|
+
|
|
194
324
|
def _validate_request(req: Dict[str, Any]) -> None:
|
|
195
325
|
required = {"filePath", "mode"}
|
|
196
326
|
missing = required - req.keys()
|
|
@@ -198,13 +328,17 @@ def _validate_request(req: Dict[str, Any]) -> None:
|
|
|
198
328
|
raise ValueError(f"missing required fields: {sorted(missing)}")
|
|
199
329
|
if req["mode"] not in {"cell", "range", "all", "from", "env", "serve"}:
|
|
200
330
|
raise ValueError(f"invalid mode: {req['mode']!r}")
|
|
331
|
+
_require_positive_int(req, "timeoutMs")
|
|
332
|
+
_require_positive_int(req, "maxOutputChars")
|
|
201
333
|
|
|
202
334
|
|
|
203
335
|
def _select_cell_indexes(req: Dict[str, Any], total: int) -> List[int]:
|
|
204
336
|
"""Return the cell indexes (across the WHOLE notebook) the caller wants to run."""
|
|
205
337
|
mode = req["mode"]
|
|
206
338
|
if total <= 0:
|
|
207
|
-
|
|
339
|
+
if mode == "all":
|
|
340
|
+
return []
|
|
341
|
+
raise ValueError("notebook has no cells")
|
|
208
342
|
if mode == "all":
|
|
209
343
|
return list(range(total))
|
|
210
344
|
if mode == "cell":
|
|
@@ -223,6 +357,10 @@ def _select_cell_indexes(req: Dict[str, Any], total: int) -> List[int]:
|
|
|
223
357
|
raise ValueError("mode='range' requires integer start and end")
|
|
224
358
|
if start < 0 or end < 0 or start > end:
|
|
225
359
|
raise ValueError(f"invalid range start={start} end={end}")
|
|
360
|
+
if start >= total:
|
|
361
|
+
raise ValueError(
|
|
362
|
+
f"range start {start} out of range (notebook has {total} cells)"
|
|
363
|
+
)
|
|
226
364
|
return list(range(start, min(end, total - 1) + 1))
|
|
227
365
|
if mode == "from":
|
|
228
366
|
idx = req.get("cellIndex")
|
|
@@ -297,6 +435,7 @@ def _build_cell_summary(
|
|
|
297
435
|
status: str,
|
|
298
436
|
max_output_chars: int,
|
|
299
437
|
duration_ms: int,
|
|
438
|
+
include_raw: bool = False,
|
|
300
439
|
) -> Dict[str, Any]:
|
|
301
440
|
summary: Dict[str, Any] = {
|
|
302
441
|
"cellIndex": cell_index,
|
|
@@ -308,8 +447,15 @@ def _build_cell_summary(
|
|
|
308
447
|
if ec is not None:
|
|
309
448
|
summary["executionCount"] = int(ec)
|
|
310
449
|
raw_outputs = list(cell.get("outputs", []) or [])
|
|
311
|
-
if raw_outputs:
|
|
312
|
-
|
|
450
|
+
if raw_outputs and include_raw:
|
|
451
|
+
try:
|
|
452
|
+
encoded = json.dumps(raw_outputs)
|
|
453
|
+
except Exception:
|
|
454
|
+
encoded = ""
|
|
455
|
+
if len(encoded) <= MAX_RAW_OUTPUT_BYTES:
|
|
456
|
+
summary["rawOutputs"] = raw_outputs
|
|
457
|
+
else:
|
|
458
|
+
summary["rawOutputsOmitted"] = True
|
|
313
459
|
for out in raw_outputs:
|
|
314
460
|
addition = _summarize_output(out)
|
|
315
461
|
_merge_summaries(summary, addition)
|
|
@@ -318,8 +464,21 @@ def _build_cell_summary(
|
|
|
318
464
|
for key in ("stdout", "stderr", "resultPreview"):
|
|
319
465
|
if key in summary and len(summary[key]) > max_output_chars:
|
|
320
466
|
summary[key] = summary[key][:max_output_chars] + (
|
|
321
|
-
|
|
467
|
+
"\n... (truncated, use maxOutputChars to increase)"
|
|
322
468
|
)
|
|
469
|
+
for err in summary.get("errors") or []:
|
|
470
|
+
for key in ("ename", "evalue"):
|
|
471
|
+
value = str(err.get(key) or "")
|
|
472
|
+
if len(value) > MAX_ERROR_TEXT:
|
|
473
|
+
err[key] = value[:MAX_ERROR_TEXT] + "\n... (truncated)"
|
|
474
|
+
else:
|
|
475
|
+
err[key] = value
|
|
476
|
+
tb = [str(line) for line in (err.get("traceback") or [])]
|
|
477
|
+
joined = "\n".join(tb)
|
|
478
|
+
if len(joined) > MAX_ERROR_TEXT:
|
|
479
|
+
err["traceback"] = [joined[:MAX_ERROR_TEXT] + "\n... (truncated)"]
|
|
480
|
+
else:
|
|
481
|
+
err["traceback"] = tb
|
|
323
482
|
return summary
|
|
324
483
|
|
|
325
484
|
|
|
@@ -342,9 +501,11 @@ def _error_response(
|
|
|
342
501
|
"error": {
|
|
343
502
|
"kind": kind,
|
|
344
503
|
"cellIndex": cell_index,
|
|
345
|
-
"ename": ename,
|
|
346
|
-
"evalue": evalue,
|
|
347
|
-
"traceback":
|
|
504
|
+
"ename": str(ename)[:MAX_ERROR_TEXT],
|
|
505
|
+
"evalue": str(evalue)[:MAX_ERROR_TEXT],
|
|
506
|
+
"traceback": [
|
|
507
|
+
line[:MAX_ERROR_TEXT] for line in list(traceback_lines or [])[:200]
|
|
508
|
+
],
|
|
348
509
|
},
|
|
349
510
|
}
|
|
350
511
|
if request_id is not None:
|
|
@@ -365,6 +526,23 @@ def _resolve_kernel_name(req: Dict[str, Any], nb: Any) -> Optional[str]:
|
|
|
365
526
|
return None
|
|
366
527
|
|
|
367
528
|
|
|
529
|
+
def _cap_raw_outputs(outputs: List[Dict[str, Any]]) -> None:
|
|
530
|
+
remaining = MAX_RAW_OUTPUT_TOTAL_BYTES
|
|
531
|
+
for summary in outputs:
|
|
532
|
+
raw = summary.get("rawOutputs")
|
|
533
|
+
if raw is None:
|
|
534
|
+
continue
|
|
535
|
+
try:
|
|
536
|
+
size = len(json.dumps(raw))
|
|
537
|
+
except Exception:
|
|
538
|
+
size = MAX_RAW_OUTPUT_TOTAL_BYTES + 1
|
|
539
|
+
if size > remaining:
|
|
540
|
+
summary.pop("rawOutputs", None)
|
|
541
|
+
summary["rawOutputsOmitted"] = True
|
|
542
|
+
else:
|
|
543
|
+
remaining -= size
|
|
544
|
+
|
|
545
|
+
|
|
368
546
|
def _run_nbclient(
|
|
369
547
|
req: Dict[str, Any],
|
|
370
548
|
nb: Any,
|
|
@@ -422,10 +600,14 @@ def _run_nbclient(
|
|
|
422
600
|
# into the kernel subprocess Popen call. Execute the selected cells
|
|
423
601
|
# explicitly so mode=cell/range/from never runs cells outside the
|
|
424
602
|
# requested slice.
|
|
425
|
-
|
|
603
|
+
global _ACTIVE_CLIENT
|
|
604
|
+
_ACTIVE_CLIENT = client
|
|
605
|
+
with client.setup_kernel(env=_kernel_env(req)):
|
|
426
606
|
for idx in target_code_cells:
|
|
427
607
|
client.execute_cell(nb.cells[idx], idx)
|
|
428
608
|
except Exception as exc:
|
|
609
|
+
if type(exc).__name__ == "CellTimeoutError":
|
|
610
|
+
_kill_kernel_process_group(client)
|
|
429
611
|
failing_cell_index = getattr(exc, "cell_index", None)
|
|
430
612
|
if not isinstance(failing_cell_index, int) or failing_cell_index < 0:
|
|
431
613
|
msg = str(exc)
|
|
@@ -461,18 +643,26 @@ def _run_nbclient(
|
|
|
461
643
|
status="ok",
|
|
462
644
|
max_output_chars=int(req.get("maxOutputChars", 12_000)),
|
|
463
645
|
duration_ms=0,
|
|
646
|
+
include_raw=bool(
|
|
647
|
+
req.get("save", False) or req.get("includeRawOutputs", False)
|
|
648
|
+
),
|
|
464
649
|
)
|
|
465
650
|
)
|
|
466
651
|
|
|
652
|
+
_cap_raw_outputs(outputs)
|
|
467
653
|
return {
|
|
468
654
|
"success": True,
|
|
469
|
-
"executedCells":
|
|
655
|
+
"executedCells": target_code_cells,
|
|
470
656
|
"durationMs": int((time.time() - started) * 1000),
|
|
471
657
|
"saved": False,
|
|
472
658
|
"outputs": outputs,
|
|
473
659
|
}
|
|
474
660
|
|
|
475
661
|
|
|
662
|
+
class KernelBroken(Exception):
|
|
663
|
+
"""Raised when a timed-out cell requires a fresh kernel."""
|
|
664
|
+
|
|
665
|
+
|
|
476
666
|
def _run_serve_request(
|
|
477
667
|
req: Dict[str, Any],
|
|
478
668
|
client: Any,
|
|
@@ -498,6 +688,23 @@ def _run_serve_request(
|
|
|
498
688
|
# omits it — keep the wire format terse for the hot path.
|
|
499
689
|
if "filePath" not in req and isinstance(file_path, str):
|
|
500
690
|
req["filePath"] = file_path
|
|
691
|
+
requested_path = req.get("filePath")
|
|
692
|
+
if (
|
|
693
|
+
isinstance(requested_path, str)
|
|
694
|
+
and isinstance(file_path, str)
|
|
695
|
+
and os.path.realpath(requested_path) != os.path.realpath(file_path)
|
|
696
|
+
):
|
|
697
|
+
# One serve process owns one kernel for one notebook. Executing a
|
|
698
|
+
# different notebook would silently run against the init document.
|
|
699
|
+
return _error_response(
|
|
700
|
+
req,
|
|
701
|
+
started,
|
|
702
|
+
"PythonRunnerError",
|
|
703
|
+
"NotebookMismatch",
|
|
704
|
+
f"request names {requested_path}, but this kernel serves {file_path}",
|
|
705
|
+
-1,
|
|
706
|
+
request_id=req_id,
|
|
707
|
+
)
|
|
501
708
|
try:
|
|
502
709
|
_validate_request(req)
|
|
503
710
|
except Exception as exc:
|
|
@@ -511,6 +718,27 @@ def _run_serve_request(
|
|
|
511
718
|
request_id=req_id,
|
|
512
719
|
)
|
|
513
720
|
|
|
721
|
+
# A warm kernel keeps Python state, but the notebook source must follow
|
|
722
|
+
# edits on disk. Reload the cells before every request; the kernel client
|
|
723
|
+
# keeps its process and namespace, while the NotebookNode is refreshed.
|
|
724
|
+
try:
|
|
725
|
+
import nbformat # type: ignore
|
|
726
|
+
|
|
727
|
+
fresh = nbformat.read(file_path, as_version=4)
|
|
728
|
+
nb.cells = fresh.cells
|
|
729
|
+
nb.metadata = fresh.metadata
|
|
730
|
+
except Exception as exc:
|
|
731
|
+
return _error_response(
|
|
732
|
+
req,
|
|
733
|
+
started,
|
|
734
|
+
"PythonRunnerError",
|
|
735
|
+
type(exc).__name__,
|
|
736
|
+
f"Failed to reload notebook {file_path}: {exc}",
|
|
737
|
+
-1,
|
|
738
|
+
traceback.format_exception(exc),
|
|
739
|
+
request_id=req_id,
|
|
740
|
+
)
|
|
741
|
+
|
|
514
742
|
if req["mode"] == "serve":
|
|
515
743
|
return _error_response(
|
|
516
744
|
req,
|
|
@@ -554,7 +782,7 @@ def _run_serve_request(
|
|
|
554
782
|
if not target_code_cells:
|
|
555
783
|
response = {
|
|
556
784
|
"success": True,
|
|
557
|
-
"executedCells":
|
|
785
|
+
"executedCells": [],
|
|
558
786
|
"durationMs": int((time.time() - started) * 1000),
|
|
559
787
|
"outputs": [],
|
|
560
788
|
}
|
|
@@ -562,7 +790,15 @@ def _run_serve_request(
|
|
|
562
790
|
response["id"] = req_id
|
|
563
791
|
return response
|
|
564
792
|
|
|
793
|
+
# nbclient's timeout is per cell; the caller's budget covers the whole
|
|
794
|
+
# request, so the request-level wall clock lives in the TypeScript layer
|
|
795
|
+
# (KernelManager arms it when the payload is written). Giving each cell the
|
|
796
|
+
# full budget here only bounds a runaway single cell.
|
|
565
797
|
timeout_s = max(1, int(req.get("timeoutMs", 120_000) / 1000))
|
|
798
|
+
try:
|
|
799
|
+
client.timeout = timeout_s
|
|
800
|
+
except Exception:
|
|
801
|
+
pass
|
|
566
802
|
# Per-cell durations: capture start/end around each `execute_cell` call so
|
|
567
803
|
# the `cell_duration_ms` field reflects what actually happened, not the
|
|
568
804
|
# batch total attributed to the first cell.
|
|
@@ -589,7 +825,7 @@ def _run_serve_request(
|
|
|
589
825
|
break
|
|
590
826
|
else:
|
|
591
827
|
failing_cell_index = target_code_cells[0] if target_code_cells else -1
|
|
592
|
-
|
|
828
|
+
response = _error_response(
|
|
593
829
|
req,
|
|
594
830
|
started,
|
|
595
831
|
"CellExecutionError",
|
|
@@ -599,6 +835,21 @@ def _run_serve_request(
|
|
|
599
835
|
traceback.format_exception(exc),
|
|
600
836
|
request_id=req_id,
|
|
601
837
|
)
|
|
838
|
+
# A kernel that died under nbclient leaves the client bound to a dead
|
|
839
|
+
# process; tear it down so the next request starts a fresh kernel
|
|
840
|
+
# instead of failing until an explicit restart.
|
|
841
|
+
if type(exc).__name__ in ("DeadKernelError", "KernelRestartRequired"):
|
|
842
|
+
_kill_kernel_process_group(client)
|
|
843
|
+
_teardown_client(client)
|
|
844
|
+
raise KernelBroken("kernel died during execution") from exc
|
|
845
|
+
if type(exc).__name__ == "CellTimeoutError":
|
|
846
|
+
# nbclient leaves the cell running. Kill the kernel and its
|
|
847
|
+
# descendants so the next request starts clean instead of
|
|
848
|
+
# queueing behind a runaway cell.
|
|
849
|
+
_kill_kernel_process_group(client)
|
|
850
|
+
_teardown_client(client)
|
|
851
|
+
raise KernelBroken("kernel stopped after a cell timeout") from exc
|
|
852
|
+
return response
|
|
602
853
|
|
|
603
854
|
outputs: List[Dict[str, Any]] = []
|
|
604
855
|
for i in cell_indexes:
|
|
@@ -612,12 +863,16 @@ def _run_serve_request(
|
|
|
612
863
|
status="ok",
|
|
613
864
|
max_output_chars=int(req.get("maxOutputChars", 12_000)),
|
|
614
865
|
duration_ms=cell_durations_ms.get(i, 0),
|
|
866
|
+
include_raw=bool(
|
|
867
|
+
req.get("save", False) or req.get("includeRawOutputs", False)
|
|
868
|
+
),
|
|
615
869
|
)
|
|
616
870
|
)
|
|
617
871
|
|
|
872
|
+
_cap_raw_outputs(outputs)
|
|
618
873
|
response = {
|
|
619
874
|
"success": True,
|
|
620
|
-
"executedCells":
|
|
875
|
+
"executedCells": target_code_cells,
|
|
621
876
|
"durationMs": int((time.time() - started) * 1000),
|
|
622
877
|
"outputs": outputs,
|
|
623
878
|
}
|
|
@@ -626,8 +881,101 @@ def _run_serve_request(
|
|
|
626
881
|
return response
|
|
627
882
|
|
|
628
883
|
|
|
884
|
+
def _start_request_reader(
|
|
885
|
+
client: Any,
|
|
886
|
+
preempt: threading.Event,
|
|
887
|
+
) -> "queue.Queue[Optional[Dict[str, Any]]]":
|
|
888
|
+
"""Read requests on one thread so a queued shutdown can preempt a cell.
|
|
889
|
+
|
|
890
|
+
The serve loop executes cells synchronously; if it also owned the read,
|
|
891
|
+
a ``shutdown`` sent during a long cell would only be seen afterwards. The
|
|
892
|
+
reader enqueues requests in order and, on ``shutdown`` (or EOF), kills the
|
|
893
|
+
kernel group before delivering the sentinel so the in-flight cell raises.
|
|
894
|
+
"""
|
|
895
|
+
requests: "queue.Queue[Optional[Dict[str, Any]]]" = queue.Queue()
|
|
896
|
+
|
|
897
|
+
def read() -> None:
|
|
898
|
+
while True:
|
|
899
|
+
try:
|
|
900
|
+
line = sys.stdin.readline()
|
|
901
|
+
except Exception:
|
|
902
|
+
requests.put(None)
|
|
903
|
+
return
|
|
904
|
+
if line == "":
|
|
905
|
+
requests.put(None)
|
|
906
|
+
return
|
|
907
|
+
try:
|
|
908
|
+
req = json.loads(line)
|
|
909
|
+
except Exception:
|
|
910
|
+
requests.put({"__invalid_json__": line})
|
|
911
|
+
continue
|
|
912
|
+
if isinstance(req, dict) and req.get("mode") == "shutdown":
|
|
913
|
+
# Kill only if the loop is busy with a cell; the loop clears the
|
|
914
|
+
# event the moment it starts waiting for the next request, so a
|
|
915
|
+
# shutdown that is already queued ahead of finished work does
|
|
916
|
+
# not tear down the kernel mid-run.
|
|
917
|
+
if preempt.is_set():
|
|
918
|
+
try:
|
|
919
|
+
_kill_kernel_process_group(client)
|
|
920
|
+
except Exception:
|
|
921
|
+
pass
|
|
922
|
+
requests.put(None)
|
|
923
|
+
return
|
|
924
|
+
if isinstance(req, dict):
|
|
925
|
+
requests.put(req)
|
|
926
|
+
else:
|
|
927
|
+
requests.put({"__invalid_request__": True})
|
|
928
|
+
|
|
929
|
+
threading.Thread(target=read, name="request-reader", daemon=True).start()
|
|
930
|
+
return requests
|
|
931
|
+
|
|
932
|
+
|
|
933
|
+
def _start_parent_watchdog() -> None:
|
|
934
|
+
"""Exit and kill the kernel when the parent process disappears.
|
|
935
|
+
|
|
936
|
+
The helper is spawned detached with a blocking stdin read, so a SIGKILL
|
|
937
|
+
of the plugin would otherwise leave the helper and its kernel alive until
|
|
938
|
+
the running cell finishes. The parent passes a pipe whose write end dies
|
|
939
|
+
with it; EOF on that descriptor means we are orphaned. When no descriptor
|
|
940
|
+
is passed, fall back to watching our parent pid.
|
|
941
|
+
"""
|
|
942
|
+
raw = os.environ.pop("OPENCODE_IPYNB_PARENT_FD", None)
|
|
943
|
+
fd: Optional[int] = None
|
|
944
|
+
if raw is not None:
|
|
945
|
+
try:
|
|
946
|
+
candidate = int(raw)
|
|
947
|
+
os.fstat(candidate)
|
|
948
|
+
fd = candidate
|
|
949
|
+
except (TypeError, ValueError, OSError):
|
|
950
|
+
fd = None
|
|
951
|
+
|
|
952
|
+
def watch() -> None:
|
|
953
|
+
try:
|
|
954
|
+
if fd is not None:
|
|
955
|
+
while True:
|
|
956
|
+
chunk = os.read(fd, 1)
|
|
957
|
+
if not chunk:
|
|
958
|
+
break
|
|
959
|
+
else:
|
|
960
|
+
parent = os.getppid()
|
|
961
|
+
while os.getppid() == parent:
|
|
962
|
+
time.sleep(0.25)
|
|
963
|
+
except Exception:
|
|
964
|
+
pass
|
|
965
|
+
client = _ACTIVE_CLIENT
|
|
966
|
+
if client is not None:
|
|
967
|
+
try:
|
|
968
|
+
_kill_kernel_process_group(client)
|
|
969
|
+
except Exception:
|
|
970
|
+
pass
|
|
971
|
+
os._exit(0)
|
|
972
|
+
|
|
973
|
+
thread = threading.Thread(target=watch, name="parent-watchdog", daemon=True)
|
|
974
|
+
thread.start()
|
|
975
|
+
|
|
976
|
+
|
|
629
977
|
def _serve_mode(init_req: Dict[str, Any]) -> int:
|
|
630
|
-
"""
|
|
978
|
+
"""Warm-kernel loop. Initializes one kernel and serves NDJSON
|
|
631
979
|
requests until ``{"mode": "shutdown"}`` or EOF on stdin.
|
|
632
980
|
"""
|
|
633
981
|
file_path = init_req.get("filePath")
|
|
@@ -698,10 +1046,24 @@ def _serve_mode(init_req: Dict[str, Any]) -> int:
|
|
|
698
1046
|
# NOT use ``setup_kernel`` (a context manager) because we want the
|
|
699
1047
|
# kernel to outlive the first request — subsequent requests reuse
|
|
700
1048
|
# the same kernel client via ``client.execute_cell``.
|
|
1049
|
+
global _ACTIVE_CLIENT
|
|
1050
|
+
_ACTIVE_CLIENT = client
|
|
701
1051
|
client.km = client.create_kernel_manager()
|
|
702
|
-
client.start_new_kernel()
|
|
1052
|
+
client.start_new_kernel(env=_kernel_env(init_req))
|
|
703
1053
|
client.start_new_kernel_client()
|
|
704
1054
|
except Exception as exc:
|
|
1055
|
+
# A kernel may already be running even though the client failed.
|
|
1056
|
+
# Jupyter starts it in its own session, so the parent cannot reap it
|
|
1057
|
+
# from the helper's process group.
|
|
1058
|
+
try:
|
|
1059
|
+
client.shutdown_kernel(now=True)
|
|
1060
|
+
except Exception:
|
|
1061
|
+
pass
|
|
1062
|
+
try:
|
|
1063
|
+
_kill_kernel_process_group(client)
|
|
1064
|
+
except Exception:
|
|
1065
|
+
pass
|
|
1066
|
+
_ACTIVE_CLIENT = None
|
|
705
1067
|
_write_response(
|
|
706
1068
|
_error_response(
|
|
707
1069
|
init_req,
|
|
@@ -717,7 +1079,9 @@ def _serve_mode(init_req: Dict[str, Any]) -> int:
|
|
|
717
1079
|
|
|
718
1080
|
ready_response: Dict[str, Any] = {
|
|
719
1081
|
"ready": True,
|
|
1082
|
+
"nonceVersion": 1,
|
|
720
1083
|
"pid": os.getpid(),
|
|
1084
|
+
"kernelPid": _kernel_pid(client),
|
|
721
1085
|
"kernelName": kernel_name,
|
|
722
1086
|
"durationMs": int((time.time() - started) * 1000),
|
|
723
1087
|
}
|
|
@@ -725,21 +1089,23 @@ def _serve_mode(init_req: Dict[str, Any]) -> int:
|
|
|
725
1089
|
ready_response["id"] = init_req["id"]
|
|
726
1090
|
_write_response(ready_response)
|
|
727
1091
|
|
|
1092
|
+
_start_parent_watchdog()
|
|
1093
|
+
busy = threading.Event()
|
|
1094
|
+
incoming = _start_request_reader(client, busy)
|
|
728
1095
|
requests_handled = 0
|
|
729
1096
|
try:
|
|
730
1097
|
while True:
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
except EOFError:
|
|
1098
|
+
req = incoming.get()
|
|
1099
|
+
if req is None:
|
|
734
1100
|
break
|
|
735
|
-
|
|
1101
|
+
if "__invalid_json__" in req:
|
|
736
1102
|
_write_response(
|
|
737
1103
|
_error_response(
|
|
738
1104
|
None,
|
|
739
1105
|
time.time(),
|
|
740
1106
|
"PythonRunnerError",
|
|
741
1107
|
"InvalidJSON",
|
|
742
|
-
|
|
1108
|
+
"could not parse request",
|
|
743
1109
|
-1,
|
|
744
1110
|
)
|
|
745
1111
|
)
|
|
@@ -761,9 +1127,28 @@ def _serve_mode(init_req: Dict[str, Any]) -> int:
|
|
|
761
1127
|
mode = req.get("mode")
|
|
762
1128
|
if mode == "shutdown":
|
|
763
1129
|
break
|
|
1130
|
+
request_nonce = req.get("nonce")
|
|
1131
|
+
if not isinstance(request_nonce, str):
|
|
1132
|
+
request_nonce = None
|
|
764
1133
|
|
|
1134
|
+
busy.set()
|
|
765
1135
|
try:
|
|
766
1136
|
response = _run_serve_request(req, client, nb, file_path)
|
|
1137
|
+
except KernelBroken as exc:
|
|
1138
|
+
req_id = req.get("id") if isinstance(req.get("id"), int) else None
|
|
1139
|
+
_write_response(
|
|
1140
|
+
_error_response(
|
|
1141
|
+
req,
|
|
1142
|
+
time.time(),
|
|
1143
|
+
"PythonRunnerError",
|
|
1144
|
+
"KernelRestartRequired",
|
|
1145
|
+
str(exc),
|
|
1146
|
+
-1,
|
|
1147
|
+
request_id=req_id,
|
|
1148
|
+
),
|
|
1149
|
+
request_nonce,
|
|
1150
|
+
)
|
|
1151
|
+
break
|
|
767
1152
|
except Exception as exc:
|
|
768
1153
|
req_id = req.get("id") if isinstance(req.get("id"), int) else None
|
|
769
1154
|
response = _error_response(
|
|
@@ -777,11 +1162,13 @@ def _serve_mode(init_req: Dict[str, Any]) -> int:
|
|
|
777
1162
|
request_id=req_id,
|
|
778
1163
|
)
|
|
779
1164
|
|
|
780
|
-
|
|
1165
|
+
busy.clear()
|
|
1166
|
+
_write_response(response, request_nonce)
|
|
781
1167
|
requests_handled += 1
|
|
782
1168
|
except KeyboardInterrupt:
|
|
783
1169
|
return 0
|
|
784
1170
|
finally:
|
|
1171
|
+
_kill_kernel_process_group(client)
|
|
785
1172
|
try:
|
|
786
1173
|
client.shutdown_kernel()
|
|
787
1174
|
except Exception:
|
|
@@ -875,7 +1262,7 @@ def _run() -> int:
|
|
|
875
1262
|
|
|
876
1263
|
try:
|
|
877
1264
|
import nbformat # type: ignore
|
|
878
|
-
from nbclient import NotebookClient # type: ignore
|
|
1265
|
+
from nbclient import NotebookClient # type: ignore # noqa: F401
|
|
879
1266
|
except ImportError as exc:
|
|
880
1267
|
sys.stdout.write(
|
|
881
1268
|
json.dumps(
|