opencode-ipynb 2.0.0 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +150 -0
- package/README.md +12 -6
- package/dist/domain/cell.d.ts +2 -2
- package/dist/domain/notebook.d.ts +5 -2
- package/dist/format/notebook.d.ts +0 -1
- package/dist/format/outputs.d.ts +0 -1
- package/dist/index.js +2969 -717
- package/dist/load-document.d.ts +0 -6
- package/dist/lsp.d.ts +4 -43
- package/dist/lsp.js +45 -2
- package/dist/notebook-diagnostics.d.ts +15 -18
- package/dist/notebook-hooks.d.ts +0 -5
- package/dist/plugin-v2.d.ts +3 -7
- package/dist/rpc.d.ts +0 -7
- package/dist/services/KernelManager.d.ts +9 -0
- package/dist/services/NotebookCleanService.d.ts +4 -0
- package/dist/services/NotebookEditService.d.ts +10 -1
- package/dist/services/NotebookExecutionService.d.ts +28 -3
- package/dist/services/NotebookFileService.d.ts +18 -9
- package/dist/services/NotebookInspectService.d.ts +0 -5
- package/dist/services/NotebookOutputService.d.ts +3 -0
- package/dist/services/PathService.d.ts +5 -1
- package/dist/services/PermissionService.d.ts +10 -3
- package/dist/services/PythonService.d.ts +5 -15
- package/dist/services/index.d.ts +7 -5
- package/dist/services/revision.d.ts +2 -8
- package/dist/tool.d.ts +1 -22
- package/dist/tools/_readPermission.d.ts +0 -6
- package/dist/utils/ansi.d.ts +0 -1
- package/dist/utils/atomic-replace.d.ts +2 -6
- package/dist/utils/execution-order.d.ts +0 -1
- package/dist/utils/fence.d.ts +0 -1
- package/dist/utils/imports.d.ts +0 -6
- package/dist/utils/json-patch.d.ts +58 -0
- package/dist/utils/json.d.ts +16 -1
- package/dist/utils/limits.d.ts +0 -1
- package/dist/utils/markdown-safe.d.ts +0 -5
- package/dist/utils/metadata.d.ts +0 -1
- package/dist/utils/paths.d.ts +2 -0
- package/dist/utils/realpath.d.ts +1 -0
- package/dist/utils/rename-hook.d.ts +6 -0
- package/dist/utils/warnings.d.ts +15 -0
- package/dist/utils/windows.d.ts +2 -3
- package/dist/virtual-materialize.d.ts +12 -23
- package/package.json +2 -2
- package/python/ipynb_runner.py +189 -32
package/python/ipynb_runner.py
CHANGED
|
@@ -33,6 +33,8 @@ import signal
|
|
|
33
33
|
import subprocess
|
|
34
34
|
import sys
|
|
35
35
|
import tempfile
|
|
36
|
+
import queue
|
|
37
|
+
import threading
|
|
36
38
|
import time
|
|
37
39
|
import traceback
|
|
38
40
|
from typing import Any, Dict, Iterable, List, Optional
|
|
@@ -80,7 +82,11 @@ def _read_request_line() -> Dict[str, Any]:
|
|
|
80
82
|
return json.loads(line)
|
|
81
83
|
|
|
82
84
|
|
|
83
|
-
def _write_response(
|
|
85
|
+
def _write_response(
|
|
86
|
+
response: Dict[str, Any], nonce: Optional[str] = None
|
|
87
|
+
) -> None:
|
|
88
|
+
if nonce is not None:
|
|
89
|
+
response["nonce"] = nonce
|
|
84
90
|
sys.stdout.write(json.dumps(response))
|
|
85
91
|
sys.stdout.write("\n")
|
|
86
92
|
sys.stdout.flush()
|
|
@@ -117,17 +123,34 @@ def _run_pip_freeze(python_executable: str, timeout_s: int = 30) -> List[str]:
|
|
|
117
123
|
env = os.environ.copy()
|
|
118
124
|
env.pop("PYTHONPATH", None)
|
|
119
125
|
env.pop("PYTHONHOME", None)
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
126
|
+
try:
|
|
127
|
+
raw = subprocess.check_output(
|
|
128
|
+
[python_executable, "-E", "-c", PIP_FREEZE_CODE],
|
|
129
|
+
cwd=tempfile.gettempdir(),
|
|
130
|
+
env=env,
|
|
131
|
+
stderr=subprocess.PIPE,
|
|
132
|
+
timeout=timeout_s,
|
|
133
|
+
)
|
|
134
|
+
except subprocess.CalledProcessError:
|
|
135
|
+
# A venv created with --without-pip (or a bare interpreter) can still
|
|
136
|
+
# run notebooks; the package list is diagnostic, so report it as empty
|
|
137
|
+
# instead of failing the whole env report.
|
|
138
|
+
return []
|
|
127
139
|
text = raw.decode("utf-8", errors="replace")
|
|
128
140
|
return [line.strip() for line in text.splitlines() if line.strip()]
|
|
129
141
|
|
|
130
142
|
|
|
143
|
+
def _teardown_client(client: Any) -> None:
|
|
144
|
+
"""Release whatever the installed nbclient version exposes."""
|
|
145
|
+
for name in ("shutdown_kernel", "cleanup_kernel", "cleanup_resources"):
|
|
146
|
+
method = getattr(client, name, None)
|
|
147
|
+
if callable(method):
|
|
148
|
+
try:
|
|
149
|
+
method()
|
|
150
|
+
except Exception:
|
|
151
|
+
pass
|
|
152
|
+
|
|
153
|
+
|
|
131
154
|
def _kernel_pid(client: Any) -> Optional[int]:
|
|
132
155
|
km = getattr(client, "km", None)
|
|
133
156
|
provisioner = getattr(km, "provisioner", None)
|
|
@@ -202,7 +225,7 @@ def _build_env_report(
|
|
|
202
225
|
"""
|
|
203
226
|
try:
|
|
204
227
|
nb = _read_notebook_raw(file_path)
|
|
205
|
-
except FileNotFoundError
|
|
228
|
+
except FileNotFoundError:
|
|
206
229
|
return _error_response(
|
|
207
230
|
None,
|
|
208
231
|
started,
|
|
@@ -290,6 +313,14 @@ def _build_env_report(
|
|
|
290
313
|
}
|
|
291
314
|
|
|
292
315
|
|
|
316
|
+
def _require_positive_int(req: Dict[str, Any], field: str) -> None:
|
|
317
|
+
if field not in req:
|
|
318
|
+
return
|
|
319
|
+
value = req[field]
|
|
320
|
+
if isinstance(value, bool) or not isinstance(value, int) or value <= 0:
|
|
321
|
+
raise ValueError(f"{field} must be a positive integer, got {value!r}")
|
|
322
|
+
|
|
323
|
+
|
|
293
324
|
def _validate_request(req: Dict[str, Any]) -> None:
|
|
294
325
|
required = {"filePath", "mode"}
|
|
295
326
|
missing = required - req.keys()
|
|
@@ -297,6 +328,8 @@ def _validate_request(req: Dict[str, Any]) -> None:
|
|
|
297
328
|
raise ValueError(f"missing required fields: {sorted(missing)}")
|
|
298
329
|
if req["mode"] not in {"cell", "range", "all", "from", "env", "serve"}:
|
|
299
330
|
raise ValueError(f"invalid mode: {req['mode']!r}")
|
|
331
|
+
_require_positive_int(req, "timeoutMs")
|
|
332
|
+
_require_positive_int(req, "maxOutputChars")
|
|
300
333
|
|
|
301
334
|
|
|
302
335
|
def _select_cell_indexes(req: Dict[str, Any], total: int) -> List[int]:
|
|
@@ -431,7 +464,7 @@ def _build_cell_summary(
|
|
|
431
464
|
for key in ("stdout", "stderr", "resultPreview"):
|
|
432
465
|
if key in summary and len(summary[key]) > max_output_chars:
|
|
433
466
|
summary[key] = summary[key][:max_output_chars] + (
|
|
434
|
-
|
|
467
|
+
"\n... (truncated, use maxOutputChars to increase)"
|
|
435
468
|
)
|
|
436
469
|
for err in summary.get("errors") or []:
|
|
437
470
|
for key in ("ename", "evalue"):
|
|
@@ -471,8 +504,7 @@ def _error_response(
|
|
|
471
504
|
"ename": str(ename)[:MAX_ERROR_TEXT],
|
|
472
505
|
"evalue": str(evalue)[:MAX_ERROR_TEXT],
|
|
473
506
|
"traceback": [
|
|
474
|
-
line[:MAX_ERROR_TEXT]
|
|
475
|
-
for line in list(traceback_lines or [])[:200]
|
|
507
|
+
line[:MAX_ERROR_TEXT] for line in list(traceback_lines or [])[:200]
|
|
476
508
|
],
|
|
477
509
|
},
|
|
478
510
|
}
|
|
@@ -611,7 +643,9 @@ def _run_nbclient(
|
|
|
611
643
|
status="ok",
|
|
612
644
|
max_output_chars=int(req.get("maxOutputChars", 12_000)),
|
|
613
645
|
duration_ms=0,
|
|
614
|
-
include_raw=bool(
|
|
646
|
+
include_raw=bool(
|
|
647
|
+
req.get("save", False) or req.get("includeRawOutputs", False)
|
|
648
|
+
),
|
|
615
649
|
)
|
|
616
650
|
)
|
|
617
651
|
|
|
@@ -654,6 +688,23 @@ def _run_serve_request(
|
|
|
654
688
|
# omits it — keep the wire format terse for the hot path.
|
|
655
689
|
if "filePath" not in req and isinstance(file_path, str):
|
|
656
690
|
req["filePath"] = file_path
|
|
691
|
+
requested_path = req.get("filePath")
|
|
692
|
+
if (
|
|
693
|
+
isinstance(requested_path, str)
|
|
694
|
+
and isinstance(file_path, str)
|
|
695
|
+
and os.path.realpath(requested_path) != os.path.realpath(file_path)
|
|
696
|
+
):
|
|
697
|
+
# One serve process owns one kernel for one notebook. Executing a
|
|
698
|
+
# different notebook would silently run against the init document.
|
|
699
|
+
return _error_response(
|
|
700
|
+
req,
|
|
701
|
+
started,
|
|
702
|
+
"PythonRunnerError",
|
|
703
|
+
"NotebookMismatch",
|
|
704
|
+
f"request names {requested_path}, but this kernel serves {file_path}",
|
|
705
|
+
-1,
|
|
706
|
+
request_id=req_id,
|
|
707
|
+
)
|
|
657
708
|
try:
|
|
658
709
|
_validate_request(req)
|
|
659
710
|
except Exception as exc:
|
|
@@ -739,9 +790,11 @@ def _run_serve_request(
|
|
|
739
790
|
response["id"] = req_id
|
|
740
791
|
return response
|
|
741
792
|
|
|
793
|
+
# nbclient's timeout is per cell; the caller's budget covers the whole
|
|
794
|
+
# request, so the request-level wall clock lives in the TypeScript layer
|
|
795
|
+
# (KernelManager arms it when the payload is written). Giving each cell the
|
|
796
|
+
# full budget here only bounds a runaway single cell.
|
|
742
797
|
timeout_s = max(1, int(req.get("timeoutMs", 120_000) / 1000))
|
|
743
|
-
# The initial client timeout belongs to the first serve request. Update it
|
|
744
|
-
# per request so a short first call does not freeze every later timeout.
|
|
745
798
|
try:
|
|
746
799
|
client.timeout = timeout_s
|
|
747
800
|
except Exception:
|
|
@@ -782,19 +835,19 @@ def _run_serve_request(
|
|
|
782
835
|
traceback.format_exception(exc),
|
|
783
836
|
request_id=req_id,
|
|
784
837
|
)
|
|
838
|
+
# A kernel that died under nbclient leaves the client bound to a dead
|
|
839
|
+
# process; tear it down so the next request starts a fresh kernel
|
|
840
|
+
# instead of failing until an explicit restart.
|
|
841
|
+
if type(exc).__name__ in ("DeadKernelError", "KernelRestartRequired"):
|
|
842
|
+
_kill_kernel_process_group(client)
|
|
843
|
+
_teardown_client(client)
|
|
844
|
+
raise KernelBroken("kernel died during execution") from exc
|
|
785
845
|
if type(exc).__name__ == "CellTimeoutError":
|
|
786
846
|
# nbclient leaves the cell running. Kill the kernel and its
|
|
787
847
|
# descendants so the next request starts clean instead of
|
|
788
848
|
# queueing behind a runaway cell.
|
|
789
849
|
_kill_kernel_process_group(client)
|
|
790
|
-
|
|
791
|
-
client.shutdown_kernel()
|
|
792
|
-
except Exception:
|
|
793
|
-
pass
|
|
794
|
-
try:
|
|
795
|
-
client.cleanup_kernel()
|
|
796
|
-
except Exception:
|
|
797
|
-
pass
|
|
850
|
+
_teardown_client(client)
|
|
798
851
|
raise KernelBroken("kernel stopped after a cell timeout") from exc
|
|
799
852
|
return response
|
|
800
853
|
|
|
@@ -810,7 +863,9 @@ def _run_serve_request(
|
|
|
810
863
|
status="ok",
|
|
811
864
|
max_output_chars=int(req.get("maxOutputChars", 12_000)),
|
|
812
865
|
duration_ms=cell_durations_ms.get(i, 0),
|
|
813
|
-
include_raw=bool(
|
|
866
|
+
include_raw=bool(
|
|
867
|
+
req.get("save", False) or req.get("includeRawOutputs", False)
|
|
868
|
+
),
|
|
814
869
|
)
|
|
815
870
|
)
|
|
816
871
|
|
|
@@ -826,6 +881,99 @@ def _run_serve_request(
|
|
|
826
881
|
return response
|
|
827
882
|
|
|
828
883
|
|
|
884
|
+
def _start_request_reader(
|
|
885
|
+
client: Any,
|
|
886
|
+
preempt: threading.Event,
|
|
887
|
+
) -> "queue.Queue[Optional[Dict[str, Any]]]":
|
|
888
|
+
"""Read requests on one thread so a queued shutdown can preempt a cell.
|
|
889
|
+
|
|
890
|
+
The serve loop executes cells synchronously; if it also owned the read,
|
|
891
|
+
a ``shutdown`` sent during a long cell would only be seen afterwards. The
|
|
892
|
+
reader enqueues requests in order and, on ``shutdown`` (or EOF), kills the
|
|
893
|
+
kernel group before delivering the sentinel so the in-flight cell raises.
|
|
894
|
+
"""
|
|
895
|
+
requests: "queue.Queue[Optional[Dict[str, Any]]]" = queue.Queue()
|
|
896
|
+
|
|
897
|
+
def read() -> None:
|
|
898
|
+
while True:
|
|
899
|
+
try:
|
|
900
|
+
line = sys.stdin.readline()
|
|
901
|
+
except Exception:
|
|
902
|
+
requests.put(None)
|
|
903
|
+
return
|
|
904
|
+
if line == "":
|
|
905
|
+
requests.put(None)
|
|
906
|
+
return
|
|
907
|
+
try:
|
|
908
|
+
req = json.loads(line)
|
|
909
|
+
except Exception:
|
|
910
|
+
requests.put({"__invalid_json__": line})
|
|
911
|
+
continue
|
|
912
|
+
if isinstance(req, dict) and req.get("mode") == "shutdown":
|
|
913
|
+
# Kill only if the loop is busy with a cell; the loop clears the
|
|
914
|
+
# event the moment it starts waiting for the next request, so a
|
|
915
|
+
# shutdown that is already queued ahead of finished work does
|
|
916
|
+
# not tear down the kernel mid-run.
|
|
917
|
+
if preempt.is_set():
|
|
918
|
+
try:
|
|
919
|
+
_kill_kernel_process_group(client)
|
|
920
|
+
except Exception:
|
|
921
|
+
pass
|
|
922
|
+
requests.put(None)
|
|
923
|
+
return
|
|
924
|
+
if isinstance(req, dict):
|
|
925
|
+
requests.put(req)
|
|
926
|
+
else:
|
|
927
|
+
requests.put({"__invalid_request__": True})
|
|
928
|
+
|
|
929
|
+
threading.Thread(target=read, name="request-reader", daemon=True).start()
|
|
930
|
+
return requests
|
|
931
|
+
|
|
932
|
+
|
|
933
|
+
def _start_parent_watchdog() -> None:
|
|
934
|
+
"""Exit and kill the kernel when the parent process disappears.
|
|
935
|
+
|
|
936
|
+
The helper is spawned detached with a blocking stdin read, so a SIGKILL
|
|
937
|
+
of the plugin would otherwise leave the helper and its kernel alive until
|
|
938
|
+
the running cell finishes. The parent passes a pipe whose write end dies
|
|
939
|
+
with it; EOF on that descriptor means we are orphaned. When no descriptor
|
|
940
|
+
is passed, fall back to watching our parent pid.
|
|
941
|
+
"""
|
|
942
|
+
raw = os.environ.pop("OPENCODE_IPYNB_PARENT_FD", None)
|
|
943
|
+
fd: Optional[int] = None
|
|
944
|
+
if raw is not None:
|
|
945
|
+
try:
|
|
946
|
+
candidate = int(raw)
|
|
947
|
+
os.fstat(candidate)
|
|
948
|
+
fd = candidate
|
|
949
|
+
except (TypeError, ValueError, OSError):
|
|
950
|
+
fd = None
|
|
951
|
+
|
|
952
|
+
def watch() -> None:
|
|
953
|
+
try:
|
|
954
|
+
if fd is not None:
|
|
955
|
+
while True:
|
|
956
|
+
chunk = os.read(fd, 1)
|
|
957
|
+
if not chunk:
|
|
958
|
+
break
|
|
959
|
+
else:
|
|
960
|
+
parent = os.getppid()
|
|
961
|
+
while os.getppid() == parent:
|
|
962
|
+
time.sleep(0.25)
|
|
963
|
+
except Exception:
|
|
964
|
+
pass
|
|
965
|
+
client = _ACTIVE_CLIENT
|
|
966
|
+
if client is not None:
|
|
967
|
+
try:
|
|
968
|
+
_kill_kernel_process_group(client)
|
|
969
|
+
except Exception:
|
|
970
|
+
pass
|
|
971
|
+
os._exit(0)
|
|
972
|
+
|
|
973
|
+
thread = threading.Thread(target=watch, name="parent-watchdog", daemon=True)
|
|
974
|
+
thread.start()
|
|
975
|
+
|
|
976
|
+
|
|
829
977
|
def _serve_mode(init_req: Dict[str, Any]) -> int:
|
|
830
978
|
"""Warm-kernel loop. Initializes one kernel and serves NDJSON
|
|
831
979
|
requests until ``{"mode": "shutdown"}`` or EOF on stdin.
|
|
@@ -931,6 +1079,7 @@ def _serve_mode(init_req: Dict[str, Any]) -> int:
|
|
|
931
1079
|
|
|
932
1080
|
ready_response: Dict[str, Any] = {
|
|
933
1081
|
"ready": True,
|
|
1082
|
+
"nonceVersion": 1,
|
|
934
1083
|
"pid": os.getpid(),
|
|
935
1084
|
"kernelPid": _kernel_pid(client),
|
|
936
1085
|
"kernelName": kernel_name,
|
|
@@ -940,21 +1089,23 @@ def _serve_mode(init_req: Dict[str, Any]) -> int:
|
|
|
940
1089
|
ready_response["id"] = init_req["id"]
|
|
941
1090
|
_write_response(ready_response)
|
|
942
1091
|
|
|
1092
|
+
_start_parent_watchdog()
|
|
1093
|
+
busy = threading.Event()
|
|
1094
|
+
incoming = _start_request_reader(client, busy)
|
|
943
1095
|
requests_handled = 0
|
|
944
1096
|
try:
|
|
945
1097
|
while True:
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
except EOFError:
|
|
1098
|
+
req = incoming.get()
|
|
1099
|
+
if req is None:
|
|
949
1100
|
break
|
|
950
|
-
|
|
1101
|
+
if "__invalid_json__" in req:
|
|
951
1102
|
_write_response(
|
|
952
1103
|
_error_response(
|
|
953
1104
|
None,
|
|
954
1105
|
time.time(),
|
|
955
1106
|
"PythonRunnerError",
|
|
956
1107
|
"InvalidJSON",
|
|
957
|
-
|
|
1108
|
+
"could not parse request",
|
|
958
1109
|
-1,
|
|
959
1110
|
)
|
|
960
1111
|
)
|
|
@@ -976,7 +1127,11 @@ def _serve_mode(init_req: Dict[str, Any]) -> int:
|
|
|
976
1127
|
mode = req.get("mode")
|
|
977
1128
|
if mode == "shutdown":
|
|
978
1129
|
break
|
|
1130
|
+
request_nonce = req.get("nonce")
|
|
1131
|
+
if not isinstance(request_nonce, str):
|
|
1132
|
+
request_nonce = None
|
|
979
1133
|
|
|
1134
|
+
busy.set()
|
|
980
1135
|
try:
|
|
981
1136
|
response = _run_serve_request(req, client, nb, file_path)
|
|
982
1137
|
except KernelBroken as exc:
|
|
@@ -990,7 +1145,8 @@ def _serve_mode(init_req: Dict[str, Any]) -> int:
|
|
|
990
1145
|
str(exc),
|
|
991
1146
|
-1,
|
|
992
1147
|
request_id=req_id,
|
|
993
|
-
)
|
|
1148
|
+
),
|
|
1149
|
+
request_nonce,
|
|
994
1150
|
)
|
|
995
1151
|
break
|
|
996
1152
|
except Exception as exc:
|
|
@@ -1006,7 +1162,8 @@ def _serve_mode(init_req: Dict[str, Any]) -> int:
|
|
|
1006
1162
|
request_id=req_id,
|
|
1007
1163
|
)
|
|
1008
1164
|
|
|
1009
|
-
|
|
1165
|
+
busy.clear()
|
|
1166
|
+
_write_response(response, request_nonce)
|
|
1010
1167
|
requests_handled += 1
|
|
1011
1168
|
except KeyboardInterrupt:
|
|
1012
1169
|
return 0
|
|
@@ -1105,7 +1262,7 @@ def _run() -> int:
|
|
|
1105
1262
|
|
|
1106
1263
|
try:
|
|
1107
1264
|
import nbformat # type: ignore
|
|
1108
|
-
from nbclient import NotebookClient # type: ignore
|
|
1265
|
+
from nbclient import NotebookClient # type: ignore # noqa: F401
|
|
1109
1266
|
except ImportError as exc:
|
|
1110
1267
|
sys.stdout.write(
|
|
1111
1268
|
json.dumps(
|