beatrina 0.8.6

package/engines/python/adapter.py

@@ -0,0 +1,577 @@
+"""adapter.py — Jupyter messages in, Beatrix frames out.
+
+Ported from `../carmar-python/py/carmapy/kernel_adapter.py` (read 2026-09-14).
+It does not evaluate anything itself; `ipykernel` does that, in its own OS
+process, and this translates.
+
+THE RULE THAT COST THREE WRONG MEASUREMENTS OVER THERE
+(`../carmar-python/notes/probes/RESULTS.md` P1):
+
+    A cell is finished when iopub reports `status: idle` for its parent
+    msg_id — NEVER when execute_reply arrives.
+
+The channels are independent and the reply routinely precedes the output it
+describes. Closing on the reply drops output the user was supposed to see.
+
+What is NEW here, against carmapy's version:
+
+  * `input_request`. The chunk runs with `allow_stdin=True` and the stdin
+    channel is drained in the same loop as iopub and shell, so `input()` in a
+    Python chunk raises the page's question instead of hanging the kernel.
+    carmapy ran every cell with stdin disallowed.
+  * The error frame is emitted in worker.R's SHAPE — `stderr` for the
+    traceback text plus a `traceback` frame — because `{"type":"error"}` is a
+    frame `src/r-kernel.js` does not handle, so carmapy's version would have
+    been dropped by the page it was written for. Measured against the R worker
+    on 2026-09-14 (`tmp/probe-r-frames.mjs`).
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import queue
+import sys
+import time
+from typing import Awaitable, Callable
+
+from jupyter_client.manager import AsyncKernelManager
+
+from bootstrap import BOOTSTRAP, CARMAR_MIME, CONTROL_MIME, REPLY_MIME
+
+#: debugger.py's PROMPT. Restated, not imported: debugger.py is kernel-side
+#: source and is never imported into this process. test/host-python-debug
+#: pins the two spellings against each other.
+DEBUG_PROMPT = "\x1bcarmar-debug\x1b"
+#: debugger.py's CarmarDebugAbort, as ipykernel names it on the error message.
+DEBUG_ABORT = "CarmarDebugAbort"
+
+Emit = Callable[[dict], Awaitable[None]]
+
+COMPLETE_TIMEOUT = 1.5      # matches lib/r-complete.js's 1500 ms
+BUSY_BACKOFF = 10.0         # matches BACKOFF_MS; must always heal, never latch
+
+
+class PythonSession:
+    """One Python session: the notebook's `worker.R` equivalent.
+
+    Owns exactly one ipykernel process. Not a pool, not a registry — one
+    document, one kernel, which is both Jupyter's rule and CarmaR's.
+    """
+
+    def __init__(self, kernel_name: str = "python3", on_input=None, on_debug=None):
+        self.kernel_name = kernel_name
+        self.km: AsyncKernelManager | None = None
+        self.kc = None
+        self.busy: bool = False
+        self.busy_since: float | None = None
+        self._retry_after: float = 0.0
+        self._current: str | None = None      # wire id of the running cell
+        # The worker's cwd, volunteered by the kernel after every cell (see
+        # bootstrap.py). Relative paths resolve against it, so an `os.chdir()`
+        # in a chunk is known when that cell ends.
+        self.cwd: str = ""
+        self._reply: dict | None = None       # the viewer answer for the drain
+        # `on_input(prompt)` is awaited when the kernel asks a question on the
+        # stdin channel. The worker turns it into an `input_request` frame; the
+        # answer comes back through `answer_input()` from the command reader,
+        # which is a DIFFERENT task — that is the whole reason the drain loop
+        # must keep yielding rather than block on the answer.
+        self._on_input = on_input
+        self._input_open = False
+        # The debugger's pause is an `input()` whose prompt starts with
+        # DEBUG_PROMPT (engines/python/debugger.py). It is NOT a question for
+        # the user: `on_debug(payload_json)` turns it into a `debug` frame, and
+        # `answer_debug()` — never `answer_input()` — writes the command back.
+        self._on_debug = on_debug
+        self._debug_open = False
+        # Commands that arrived while the paused chunk was BETWEEN prompts — an
+        # `eval` still printing when `where` is pressed. R's Browse prompt reads
+        # raw console lines, so they queue on its stdin for free; here they
+        # queue in this list and the next prompt takes the first one. Cleared
+        # with the run: a step can never leak into the next chunk's pause.
+        self._debug_pending: list[dict] = []
+        #: Set by `_translate` while a chunk runs, read by the worker when it
+        #: builds the `done` frame: was this a Stop, and if it failed, why.
+        #: Two fields rather than one status, because "interrupted" and "the
+        #: shell reply said error" are different facts that arrive on
+        #: different channels.
+        self.interrupted = False
+        self.failure: str | None = None
+        #: `{width, height, res, format}` the chunk asked for, so a figure is
+        #: sized by what the page requested rather than by a constant.
+        self.dims: dict | None = None
+        # ONE READER AT A TIME. `_drain_until_idle` consumes the shell, iopub
+        # and stdin channels, so two of them running concurrently would steal
+        # each other's messages — a `view` sent during a cell would eat that
+        # cell's execute_reply and the cell would never finish. The lock makes
+        # `view` QUEUE behind a running cell, which is R's behaviour for the
+        # same op (`view` is a worker command there too).
+        self._chan = asyncio.Lock()
+
+    # ---------------------------------------------------------------- lifecycle
+
+    async def start(self) -> dict:
+        # IPC, not TCP, and it is a boundary decision rather than a preference.
+        # host-plan §2: "an engine never opens a port". A TCP-transport
+        # ipykernel opens FIVE listening loopback ports that nothing in Beatrix
+        # gates, and ipykernel says so itself — it writes a warning to stderr on
+        # every start ("running over TCP without encryption… susceptible to
+        # eavesdropping"), which the framer then attributes to whatever chunk
+        # happens to be running, so the user's first Python cell prints a
+        # security warning about a port they did not ask for. Unix domain
+        # sockets in the kernel's own runtime directory have no port, no
+        # listener, and file permissions for a gate. Windows has no AF_UNIX for
+        # ZMQ, so it keeps TCP on loopback.
+        transport = "tcp" if sys.platform == "win32" else "ipc"
+        self.km = AsyncKernelManager(kernel_name=self.kernel_name, transport=transport)
+        await self.km.start_kernel()
+        self.kc = self.km.client()
+        self.kc.start_channels()
+        await self.kc.wait_for_ready(timeout=60)
+        await self._bootstrap()
+        return {
+            "pid": self.km.provisioner.process.pid if self.km.provisioner else None,
+            "python": sys.version.split()[0],
+            "cwd": await self._cwd(),
+            "executable": sys.executable,
+        }
+
+    async def _bootstrap(self) -> None:
+        """Install the DataFrame formatter. Silent to the USER, never to us.
+
+        carmapy's first version passed emit=None and discarded the result, so
+        when the formatter failed to register (pandas 3.0 moved
+        DataFrame.__module__) the kernel came up reporting success and quietly
+        rendered every DataFrame as raw HTML. A bootstrap that cannot fail
+        loudly is a silent failure by construction.
+        """
+        errors: list[str] = []
+
+        async def capture(frame):
+            if frame.get("type") == "traceback":
+                errors.append(frame.get("message", ""))
+
+        msg_id = self.kc.execute(BOOTSTRAP, silent=True, store_history=False)
+        status = await self._drain_until_idle(msg_id, capture, None, timeout=60)
+        if status != "ok" or errors:
+            raise RuntimeError(
+                "the Python engine's bootstrap failed; DataFrames would render "
+                "as raw HTML.\n" + "\n".join(errors))
+
+    async def _cwd(self) -> str:
+        out: list[str] = []
+
+        async def cap(frame):
+            if frame.get("type") == "stdout":
+                out.append(frame["text"])
+
+        await self.execute("import os as _os; print(_os.getcwd())", "cwd-probe", cap,
+                           store_history=False, allow_stdin=False)
+        self.cwd = "".join(out).strip()
+        return self.cwd
+
+    async def shutdown(self) -> None:
+        if self.kc:
+            try:
+                self.kc.stop_channels()
+            except Exception:
+                pass
+        if self.km:
+            try:
+                await self.km.shutdown_kernel(now=True)
+            except Exception:
+                pass
+            # The IPC transport leaves five socket files behind; the connection
+            # file too. Nothing else removes them, and a session per document
+            # would litter the runtime directory for as long as the machine is
+            # up.
+            # AsyncKernelManager's is a COROUTINE (`_async_cleanup_resources`),
+            # so calling it and discarding the result deletes nothing and warns
+            # "coroutine was never awaited" — measured 2026-09-14. Awaited when
+            # it is one, called when it is not.
+            try:
+                maybe = self.km.cleanup_resources()
+                if asyncio.iscoroutine(maybe):
+                    await maybe
+            except Exception:
+                pass
+
+    # ---------------------------------------------------------------- execution
+
+    async def execute(self, code: str, wire_id: str, emit: Emit | None,
+                      store_history: bool = True, allow_stdin: bool = True,
+                      srcname: str | None = None, breaks: dict | None = None) -> str:
+        """Run one chunk. Emits frames as they arrive; returns the final status.
+
+        `srcname` ("chunk:<stableId>") names the compiled cell, which is the
+        address a breakpoint uses; `breaks` is every breakpoint the page has
+        set. Both are handed to the kernel in ONE silent call immediately
+        before the chunk, under the same channel lock, so no other cell can be
+        compiled in between and take the name.
+        """
+        async with self._chan:
+            self.busy = True
+            self.busy_since = time.monotonic()
+            self._current = wire_id
+            self.interrupted = False
+            self.failure = None
+            try:
+                if srcname:
+                    spec = json.dumps(json.dumps({"srcname": srcname, "breaks": breaks or {}}))
+                    prep = self.kc.execute(f"__carmar_debug_prepare({spec})",
+                                           store_history=False, allow_stdin=False)
+                    await self._drain_until_idle(prep, None, None, timeout=30)
+                msg_id = self.kc.execute(code, store_history=store_history,
+                                         allow_stdin=allow_stdin)
+                status = await self._drain_until_idle(msg_id, emit, wire_id)
+                # The shell reply says "error" for a Stop too, so the two facts
+                # are reconciled here and the caller gets R's vocabulary.
+                # BOTH conditions: a Stop pressed a millisecond before a chunk
+                # would have finished anyway leaves `interrupted` set against a
+                # run that succeeded, and R reports that run `ok`.
+                if self.interrupted and status != "ok":
+                    return "interrupted"
+                return status
+            finally:
+                self.busy = False
+                self.busy_since = None
+                self._current = None
+                self._input_open = False
+                self._debug_open = False
+                self._debug_pending = []
+
+    async def request(self, op: str, spec: dict) -> dict:
+        """A request answered by the kernel — `view`, `colstats`, `env`, `rm`.
+
+        It runs in the evaluating session because that is where the objects
+        are, which is exactly why R routes `view`/`colstats` to `worker.R`. It
+        therefore QUEUES behind a running cell, which is R's behaviour too.
+
+        The wire never carries code. `spec` is JSON, passed as ONE argument to
+        a function the bootstrap installed; the kernel-side half parses it. A
+        composed `__carmapy_view(df, sort='...')` string would be the browser
+        writing Python, which is the door this design does not have.
+        """
+        async with self._chan:
+            self._reply = None
+            payload = json.dumps(json.dumps(spec, allow_nan=False))
+            code = f"__carmapy_reply({op!r}, {payload})"
+            msg_id = self.kc.execute(code, store_history=False, allow_stdin=False)
+            await self._drain_until_idle(msg_id, None, None, timeout=300)
+            reply, self._reply = self._reply, None
+        if reply is None:
+            return {"error": "the session returned no answer"}
+        return reply
+
+    async def _drain_until_idle(self, msg_id: str, emit: Emit | None,
+                                wire_id: str | None, timeout: float = 86400) -> str:
+        """Consume iopub until idle for THIS parent, and the shell reply.
+
+        Both conditions. See the module docstring — this is the P1 rule and
+        the single most important line of behaviour in the file.
+        """
+        status = "ok"
+        idle = False
+        reply = False
+        deadline = time.monotonic() + timeout
+
+        while time.monotonic() < deadline and not (idle and reply):
+            got = False
+            try:
+                m = await self.kc.get_iopub_msg(timeout=0.02)
+                got = True
+                if m.get("parent_header", {}).get("msg_id") == msg_id:
+                    frames = self._translate(m, wire_id)
+                    if emit:
+                        for f in frames:
+                            await emit(f)
+                    if (m["header"]["msg_type"] == "status"
+                            and m["content"]["execution_state"] == "idle"):
+                        idle = True
+            except (queue.Empty, asyncio.TimeoutError):
+                pass
+            except Exception:
+                pass
+            try:
+                r = await self.kc.get_shell_msg(timeout=0.01)
+                got = True
+                if r.get("parent_header", {}).get("msg_id") == msg_id:
+                    status = r["content"].get("status", "ok")
+                    reply = True
+            except (queue.Empty, asyncio.TimeoutError):
+                pass
+            except Exception:
+                pass
+            # The stdin channel, drained in the SAME loop. A question asked
+            # while a cell runs is answered by another task (`answer_input`),
+            # so this must never block waiting for it.
+            if self._on_input is not None and not self._input_open and not self._debug_open:
+                try:
+                    s = await self.kc.get_stdin_msg(timeout=0.01)
+                    got = True
+                    asked = s["header"]["msg_type"] == "input_request" \
+                        and s.get("parent_header", {}).get("msg_id") == msg_id
+                    prompt = s["content"].get("prompt", "") if asked else ""
+                    if asked and prompt.startswith(DEBUG_PROMPT) and self._on_debug is not None:
+                        self._debug_open = True
+                        await self._on_debug(prompt[len(DEBUG_PROMPT):], wire_id)
+                        if self._debug_pending:
+                            self._deliver_debug(self._debug_pending.pop(0))
+                    elif asked:
+                        self._input_open = True
+                        await self._on_input(prompt, wire_id)
+                except (queue.Empty, asyncio.TimeoutError):
+                    pass
+                except Exception:
+                    pass
+            if not got:
+                await asyncio.sleep(0)          # yield to the event loop
+        return status
+
+    @property
+    def input_open(self) -> bool:
+        return self._input_open
+
+    @property
+    def current(self) -> str | None:
+        """The wire id of the cell running right now, or None."""
+        return self._current
+
+    @property
+    def debug_open(self) -> bool:
+        """Is the running chunk paused in the debugger right now?"""
+        return self._debug_open
+
+    def answer_debug(self, command: dict) -> bool:
+        """Deliver one debugger command to the paused chunk — now if it is at
+        its prompt, at its next prompt if it is between two. False when no
+        chunk is running: a step for a run that already finished does nothing.
+        """
+        if self.kc is None or self._current is None:
+            return False
+        if not self._debug_open:
+            self._debug_pending.append(command)
+            return True
+        self._deliver_debug(command)
+        return True
+
+    def _deliver_debug(self, command: dict) -> None:
+        self.kc.input(json.dumps(command, separators=(",", ":")))
+        self._debug_open = False
+
+    def answer_input(self, value: str) -> bool:
+        """Deliver one line to a waiting `input()`. False when nothing asked."""
+        if not self._input_open or self.kc is None:
+            return False
+        self.kc.input(value)
+        self._input_open = False
+        return True
+
+    def _translate(self, m: dict, wire_id: str | None) -> list[dict]:
+        """One Jupyter message -> zero or more Beatrix frames."""
+        t = m["header"]["msg_type"]
+        c = m["content"]
+
+        if t == "stream":
+            kind = "stderr" if c.get("name") == "stderr" else "stdout"
+            text = c.get("text", "")
+            # ipykernel 7 reports a Stop TWO ways and the choice is not ours:
+            # an `error` message with ename KeyboardInterrupt when the signal
+            # lands inside the user's frame, and a bare "\nKeyboardInterrupt"
+            # written to stderr when it lands while the previous one is still
+            # unwinding (measured 2026-09-14 — the interrupt ladder re-sends at
+            # +60 ms, so the second shape is the common one). R prints NOTHING
+            # when a chunk is interrupted, so neither does this: the word alone,
+            # while a Stop this worker sent is outstanding, is the acknowledgement
+            # and not output the user wrote.
+            if kind == "stderr" and self.interrupted \
+                    and set(text.split()) <= {"KeyboardInterrupt"}:
+                return []
+            # The page accumulates and joins on "\n"; keep R's line semantics.
+            return [{"type": kind, "id": wire_id, "text": text.rstrip("\n")}]
+
+        if t == "error":
+            ename, evalue = c.get("ename") or "Error", c.get("evalue")
+            headline = f"{ename}: {evalue}" if evalue else str(ename)
+            # A KeyboardInterrupt is STOP, not a failure. R's worker ends an
+            # interrupted chunk with `done {status:"interrupted"}` and prints
+            # no traceback (measured against the R worker, 2026-09-14), and a
+            # page that showed a stack trace every time someone pressed Stop
+            # would be reporting a bug that does not exist.
+            if ename == "KeyboardInterrupt":
+                self.interrupted = True
+                return []
+            # The debugger's Stop: R reports it as `interrupted` with the
+            # message "Stopped from the debugger" (worker.R carmar_abort_cell),
+            # and so does this — no traceback for a button the user pressed.
+            if ename == DEBUG_ABORT:
+                self.interrupted = True
+                self.failure = "Stopped from the debugger"
+                return []
+            self.failure = headline
+            tb = "\n".join(c.get("traceback", []))
+            # worker.R's shape, measured: the text of a failure reaches the
+            # page as `stderr`, and ONE `traceback` frame carries the headline
+            # and the line. `{"type":"error"}` is not in src/r-kernel.js's
+            # switch and would be silently dropped.
+            return [
+                {"type": "stderr", "id": wire_id, "text": tb or headline},
+                {"type": "traceback", "id": wire_id, "message": headline,
+                 "line": _error_line(c), "call": ename, "frames": []},
+            ]
+
+        if t in ("execute_result", "display_data"):
+            return self._from_bundle(c.get("data", {}) or {}, wire_id,
+                                     c.get("metadata") or {})
+
+        return []
+
+    def _from_bundle(self, data: dict, wire_id: str | None, meta: dict | None = None) -> list[dict]:
+        """The MIME bundle, richest representation first.
+
+        Order matters: a DataFrame also carries text/html and text/plain, and
+        emitting all three would paint the same value three times.
+        """
+        # The kernel's two private MIMEs are read FIRST and produce no frame.
+        # A viewer answer belongs to the caller waiting on it, not to the
+        # notebook's output pane, and the cwd is state the worker records.
+        if CONTROL_MIME in data:
+            payload = data[CONTROL_MIME]
+            if isinstance(payload, str):
+                payload = json.loads(payload)
+            if isinstance(payload.get("cwd"), str):
+                self.cwd = payload["cwd"]
+            return []
+
+        if REPLY_MIME in data:
+            payload = data[REPLY_MIME]
+            self._reply = json.loads(payload) if isinstance(payload, str) else payload
+            return []
+
+        if CARMAR_MIME in data:
+            payload = data[CARMAR_MIME]
+            if isinstance(payload, str):
+                payload = json.loads(payload)
+            return [{"type": "dataframe", "id": wire_id, "source": None, **payload}]
+
+        # The page sizes a figure by pixels ÷ (res ÷ 96), so `res` is not
+        # optional — a 1500px plot at 300 dpi is a five-inch figure. Three
+        # sources, best first: the message's own metadata, the dims the chunk
+        # asked for, then the page's defaults.
+        png_meta = (meta or {}).get("image/png") or {}
+        dims = self.dims or {}
+
+        def _size(key, fallback):
+            for src in (png_meta, dims):
+                v = src.get(key)
+                if isinstance(v, (int, float)) and v > 0:
+                    return v
+            return fallback
+
+        if "image/png" in data:
+            return [{"type": "plot", "id": wire_id, "mime": "image/png",
+                     "width": _size("width", 672), "height": _size("height", 480),
+                     "res": _size("res", 96),
+                     "data": data["image/png"].replace("\n", "")}]
+
+        if "image/svg+xml" in data:
+            return [{"type": "plot", "id": wire_id, "mime": "image/svg+xml",
+                     "width": _size("width", 672), "height": _size("height", 480),
+                     "res": _size("res", 96),
+                     "data": data["image/svg+xml"]}]
+
+        if "text/html" in data:
+            return [{"type": "widget", "id": wire_id, "kind": "html",
+                     "class": "html", "html": data["text/html"]}]
+
+        if "text/plain" in data:
+            return [{"type": "stdout", "id": wire_id, "text": data["text/plain"]}]
+
+        return []
+
+    # ---------------------------------------------------------------- control
+
+    async def interrupt(self) -> None:
+        """Stop.
+
+        `interrupt_mode` is `signal` for the stock python3 kernelspec, so this
+        is a SIGINT to the kernel process — which jupyter_client started in its
+        own session (measured 2026-09-14: kernel sid == kernel pid), so the
+        host's group signal to this worker can never reach it. That is exactly
+        why the worker catches SIGINT and calls this.
+        """
+        if self.km is not None:
+            await self.km.interrupt_kernel()
+
+    async def force_stop(self) -> None:
+        """The second button. A C extension that neither releases the GIL nor
+        checks signals cannot be interrupted by any message — only killed."""
+        if self.km is not None:
+            await self.km.shutdown_kernel(now=True)
+
+    # ---------------------------------------------------------------- intelligence
+
+    async def complete(self, line: str, cursor: int) -> dict | None:
+        """LIVE completion — the tier a static analyser cannot serve.
+
+        Returns None rather than waiting when the kernel is busy. That guard is
+        MANDATORY, not an optimisation: carmapy's P3 measured
+        `complete_request` getting no answer in 3.0 s while a cell ran, because
+        it rides the shell channel and is served in order. Without this, every
+        keystroke pause during a long run buys a hang.
+
+        The backoff always heals on its own and never latches until a reload —
+        R learned that the hard way when a four-second timeout marked
+        completion unsupported for the rest of the page.
+        """
+        # `_chan.locked()` as well as `busy`: a `view` on a million-row frame
+        # holds the channel without being a running CELL, and a completion that
+        # read the shell channel underneath it would steal the viewer's reply.
+        if self.busy or self._chan.locked() or time.monotonic() < self._retry_after:
+            return None
+        msg_id = self.kc.complete(line, cursor)
+        deadline = time.monotonic() + COMPLETE_TIMEOUT
+        while time.monotonic() < deadline:
+            try:
+                r = await self.kc.get_shell_msg(timeout=0.05)
+            except (queue.Empty, asyncio.TimeoutError):
+                await asyncio.sleep(0)
+                continue
+            except Exception:
+                break
+            if r.get("parent_header", {}).get("msg_id") == msg_id:
+                c = r["content"]
+                self._retry_after = 0.0
+                return {"start": c.get("cursor_start", cursor),
+                        "end": c.get("cursor_end", cursor),
+                        "matches": c.get("matches", [])}
+        self._retry_after = time.monotonic() + BUSY_BACKOFF
+        return None
+
+
+def _error_line(content: dict):
+    """The 1-based line inside THE CHUNK THAT RAN, or None.
+
+    ipykernel's traceback frames for a cell carry the line as
+    `Cell In[3], line 4`, or as a bare `----> 4` arrow. Two things the first
+    version missed, both measured 2026-09-14 against IPython 9.11:
+
+      * The text is ANSI-COLOURED — `Cell`, `In[3]` and `, line 4` each sit in
+        their own escape sequence — so neither pattern matched anything and
+        every Python traceback reached the page with `line: null`. The escapes
+        are stripped before reading.
+      * The FIRST cell frame is the chunk that ran; later ones are deeper
+        calls, possibly into a function another chunk defined. R's `line` is
+        the line within the running chunk (worker.R `at_line`), so it is the
+        first match, not the last.
+    """
+    import re
+    text = re.sub(r"\x1b\[[0-9;]*[A-Za-z]", "", "\n".join(content.get("traceback", [])))
+    hits = re.findall(r"[Cc]ell In\[\d+\],\s*line\s*(\d+)", text)
+    if not hits:
+        hits = re.findall(r"-+>\s*(\d+)", text)
+    try:
+        return int(hits[0]) if hits else None
+    except Exception:
+        return None