beatrina 0.8.6

package/engines/python/bootstrap.py

@@ -0,0 +1,309 @@
+"""bootstrap.py — code executed silently inside the ipykernel at startup.
+
+Ported from `../carmar-python/py/carmapy/bootstrap.py` (read 2026-09-14), with
+one addition: `__carmapy_reply` now also answers `env`, so the notebook's
+Environment pane has something to show for a Python session.
+
+This is how a pandas DataFrame becomes Beatrix's `dataframe` frame rather than
+a blob of HTML: we register an IPython display formatter for a private MIME
+type, so the ordinary displayhook — the thing Jupyter already does well — hands
+the adapter a payload already in the frame's shape.
+
+Borrowing the MIME-bundle displayhook instead of re-implementing a displayhook
+is the entire reason the Python worker is 600 lines rather than 3,000.
+
+Two rules copied from `emit_dataframe` in `spike/worker.R`, because they are
+correctness, not polish:
+
+  * A returned value is a NOTEBOOK PREVIEW, not the data viewer. Bounding rows
+    alone is insufficient — one cell can hold a fitted model or megabytes of
+    text — so complex cells are flattened to a description and text is clipped,
+    then a final wire-size ceiling is enforced.
+  * The preview is 200 rows, because the page's own ROWS_PER_PAGE is 25 and a
+    smaller preview makes the row navigator unreachable.
+"""
+
+import base64 as _base64
+from pathlib import Path as _Path
+
+CARMAR_MIME = "application/vnd.carmar.dataframe+json"
+# The reply channel, and the kernel's own housekeeping, each on their own
+# private MIME. Two, not one: a `view` reply is a REPLY the worker is waiting
+# on by wire id, while the cwd is UNSOLICITED state the worker records and
+# never forwards. Folding them together would make "is anyone waiting for
+# this?" a field rather than a type.
+REPLY_MIME = "application/vnd.carmapy.reply+json"
+CONTROL_MIME = "application/vnd.carmapy.control+json"
+
+# `dataview.py` is shipped into the kernel as SOURCE, base64-encoded.
+#
+# Source rather than an import, because the kernel may be any Python
+# environment the user chose and Beatrix's engine package need not be installed
+# in it. Base64 rather than an embedded string literal, because the bootstrap
+# is Python source, so it eventually contains a quote, a backslash or a `%`,
+# and one of those breaks the template the first time it appears.
+_DATAVIEW_B64 = _base64.b64encode(
+    _Path(__file__).with_name("dataview.py").read_bytes()).decode()
+# `debugger.py`, the same way and for the same reasons. Installed EAGERLY, not
+# on first use: it names the next cell after its chunk (the breakpoint address),
+# and a function defined before the first breakpoint was set must already carry
+# that name, exactly as R parses every chunk under its srcname whether or not
+# anyone is debugging.
+_DEBUGGER_B64 = _base64.b64encode(
+    _Path(__file__).with_name("debugger.py").read_bytes()).decode()
+# `handoff.py` (share/use), the same way: standard library only.
+_HANDOFF_B64 = _base64.b64encode(
+    _Path(__file__).with_name("handoff.py").read_bytes()).decode()
+
+BOOTSTRAP = r'''
+def __carmar_install_formatters():
+    import json
+
+    PREVIEW_ROWS = 200
+    MAX_CELL_CHARS = 512
+    MAX_WIRE_BYTES = 512 * 1024
+
+    def _cell(v):
+        """Flatten anything that is not a plain scalar to a description."""
+        if v is None:
+            return None
+        if isinstance(v, bool) or isinstance(v, int):
+            return v
+        if isinstance(v, float):
+            # NaN/Inf are not JSON; the page reads null the way R's na="null" does.
+            return v if v == v and v not in (float("inf"), float("-inf")) else None
+        if isinstance(v, str):
+            return v if len(v) <= MAX_CELL_CHARS else v[:MAX_CELL_CHARS] + "…"
+        try:
+            import numpy as np
+            if isinstance(v, np.generic):
+                return _cell(v.item())
+        except Exception:
+            pass
+        s = repr(v)
+        if len(s) > MAX_CELL_CHARS:
+            s = s[:MAX_CELL_CHARS] + "…"
+        return s
+
+    def _frame(df):
+        total_rows, total_cols = int(df.shape[0]), int(df.shape[1])
+        head = df.head(PREVIEW_ROWS)
+        cols = [str(c) for c in head.columns]
+        types = [str(t) for t in head.dtypes]
+        rows = [{cols[i]: _cell(v) for i, v in enumerate(rec)}
+                for rec in head.itertuples(index=False, name=None)]
+
+        payload = {
+            "columns": cols,
+            "types": types,
+            "nrow": total_rows,
+            "ncol": total_cols,
+            "shown": len(rows),
+            "shownCols": len(cols),
+            "truncated": total_rows > len(rows) or total_cols > len(cols),
+        }
+
+        # Final wire ceiling. Halving is what R did and it is wrong from a big
+        # head (it costs a third of the rows that would have fit), so estimate
+        # the per-row cost and keep what fits, as view_payload does.
+        body = json.dumps({**payload, "rows": rows})
+        if len(body.encode()) > MAX_WIRE_BYTES and rows:
+            per = len(body.encode()) / max(1, len(rows))
+            keep = max(1, int(MAX_WIRE_BYTES // per))
+            rows = rows[:keep]
+            payload["shown"] = len(rows)
+            payload["truncated"] = True
+        payload["rows"] = rows
+        return json.dumps(payload)
+
+    try:
+        ip = get_ipython()
+    except NameError:
+        return
+    fmts = ip.display_formatter.formatters
+    if "__CARMAR_MIME__" not in fmts:
+        from IPython.core.formatters import BaseFormatter
+        from traitlets import Unicode, ObjectName
+        class _CarmarDF(BaseFormatter):
+            format_type = Unicode("__CARMAR_MIME__")
+            print_method = ObjectName("_repr_carmar_df_")
+            _return_type = str
+        fmts["__CARMAR_MIME__"] = _CarmarDF(parent=ip.display_formatter)
+    # pandas 3.0 sets DataFrame.__module__ = "pandas"; every older example in
+    # the ecosystem says "pandas.core.frame", which on pandas 3.x matches
+    # NOTHING and fails silently — the formatter registers, reports active, and
+    # is never called. Register both spellings; a deferred name that never
+    # resolves costs nothing.
+    for mod in ("pandas", "pandas.core.frame"):
+        fmts["__CARMAR_MIME__"].for_type_by_name(mod, "DataFrame", _frame)
+
+    # ---- figures -------------------------------------------------------
+    # The R worker captures a plot without being asked; Python must too, or
+    # `plt.plot(...)` produces nothing and the user is told to type a magic.
+    # Both halves are LAZY: neither imports matplotlib, so a session that never
+    # plots pays nothing.
+    #
+    #  (1) a PNG formatter for Figure, so an explicitly displayed figure renders
+    #  (2) a post-cell hook that shows figures the user created but never
+    #      displayed — what %matplotlib inline does, without the magic
+
+    def _fig_png(fig):
+        import io, base64
+        buf = io.BytesIO()
+        fig.savefig(buf, format="png", dpi=96, bbox_inches="tight")
+        return base64.b64encode(buf.getvalue()).decode()
+
+    png = fmts.get("image/png")
+    if png is not None:
+        png.for_type_by_name("matplotlib.figure", "Figure", _fig_png)
+
+    def _flush_figures(*a, **k):
+        import sys
+        plt = sys.modules.get("matplotlib.pyplot")
+        if plt is None:
+            return
+        try:
+            from IPython.display import display
+            for num in plt.get_fignums():
+                display(plt.figure(num))
+            plt.close("all")
+        except Exception:
+            pass
+
+    ip.events.register("post_run_cell", _flush_figures)
+
+    # ---- the working directory, reported without being asked --------------
+    # Every relative path the host resolves is resolved against the WORKER's
+    # cwd, and an `os.chdir()` inside a cell moves it. R gets this free — its
+    # worker prints getwd() into the done frame it is already building. Here
+    # the frame is built outside the kernel, so the kernel has to volunteer the
+    # value, and post_run_cell is the one place it is certainly current: the
+    # cell that moved it has finished.
+    def _carmapy_cwd(*a, **k):
+        import os, json
+        from IPython.display import publish_display_data
+        publish_display_data({"__CARMAPY_CONTROL__": json.dumps({"cwd": os.getcwd()})})
+
+    ip.events.register("post_run_cell", _carmapy_cwd)
+
+    # ---- the data viewer, and the environment listing ----------------------
+    # `dataview.py`'s source, exec'd on FIRST USE and never at boot. Lazy so a
+    # session that never opens the viewer does not pay pandas' import to start.
+    state = {"ns": None}
+
+    def _dataview():
+        if state["ns"] is None:
+            import base64
+            ns = {"__name__": "carmapy_dataview"}
+            exec(compile(base64.b64decode("__CARMAPY_DATAVIEW_B64__").decode("utf-8"),
+                         "<beatrina dataview>", "exec"), ns)
+            state["ns"] = ns
+        return state["ns"]
+
+    def _env_rows(user_ns):
+        """The Environment pane's rows, in `env`'s shape (spike/worker.R).
+
+        One row per binding the USER made: IPython's own names (`In`, `Out`,
+        `_i3`, `get_ipython`), dunders and this module's own helpers are not
+        the user's variables and would bury them. `bytes` is sys.getsizeof,
+        which is the object's own footprint and not a deep size — R's
+        object.size is not a deep size either, and claiming one would be worse
+        than reporting the shallow one both languages actually have.
+        """
+        import sys, types
+        hidden = {"In", "Out", "get_ipython", "exit", "quit", "open", "share", "use"}
+        rows = []
+        for name, value in list(user_ns.items()):
+            if name.startswith("_") or name in hidden or name.startswith("__carmapy"):
+                continue
+            # An `import pandas as pd` IS a binding the user made, but a pane
+            # that lists every module buries the three variables the notebook
+            # is about. R's Environment pane has the same rule by accident —
+            # a package is attached, not bound — so this makes it explicit.
+            if isinstance(value, types.ModuleType):
+                continue
+            cls = type(value).__name__
+            try:
+                shape = getattr(value, "shape", None)
+                if shape is not None:
+                    dims = "×".join(str(int(d)) for d in shape)
+                elif isinstance(value, (list, tuple, dict, set, str, bytes)):
+                    dims = str(len(value))
+                else:
+                    dims = ""
+            except Exception:
+                dims = ""
+            try:
+                summary = repr(value)
+                if len(summary) > 120:
+                    summary = summary[:120] + "…"
+            except Exception:
+                summary = "<unrepresentable>"
+            kind = "function" if callable(value) else "value"
+            try:
+                nbytes = int(sys.getsizeof(value))
+            except Exception:
+                nbytes = 0
+            rows.append({"name": name, "class": cls, "dims": dims,
+                         "bytes": nbytes, "summary": summary, "kind": kind})
+        rows.sort(key=lambda r: r["name"])
+        return {"objects": rows}
+
+    def __carmapy_reply(op, spec_json):
+        """Answer one request into a private MIME on iopub.
+
+        NOT a return value and not stdout. A display_data message carries the
+        parent msg_id, so the worker can match the answer to the request it
+        sent — and user code printing the same text cannot be mistaken for a
+        reply, which is the property R buys with its 24-character sentinel.
+        """
+        import json
+        from IPython.display import publish_display_data
+        try:
+            spec = json.loads(spec_json)
+            if op == "env":
+                payload = _env_rows(get_ipython().user_ns)
+            elif op == "rm":
+                ns = get_ipython().user_ns
+                names = spec.get("names") or []
+                if isinstance(names, str):
+                    names = [names]
+                for nm in names:
+                    ns.pop(nm, None)
+                payload = {"removed": True}
+            else:
+                dv = _dataview()
+                fn = dv["view_reply"] if op == "view" else dv["colstats_reply"]
+                payload = fn(spec, get_ipython().user_ns)
+        except Exception as exc:
+            # An error FIELD, never a raised exception: pointing the viewer at
+            # a 40-column frame of fitted models is a normal thing to do by
+            # accident, and it must cost a message, not the session.
+            payload = {"error": "%s: %s" % (type(exc).__name__, exc)}
+        publish_display_data({"__CARMAPY_REPLY__": json.dumps(payload, allow_nan=False)})
+
+    ip.user_ns["__carmapy_reply"] = __carmapy_reply
+
+    # ---- the debugger (docs/wp/wp8b-python-debugger.md) ------------------
+    import base64 as _b64
+    dbg = {"__name__": "carmar_debugger"}
+    exec(compile(_b64.b64decode("__CARMAR_DEBUGGER_B64__").decode("utf-8"),
+                 "<beatrina debugger>", "exec"), dbg)
+    dbg["install"](ip)
+    ip.user_ns["__carmar_debug_prepare"] = dbg["prepare"]
+
+    # ---- share / use (engines/python/handoff.py) -------------------------
+    hand = {"__name__": "beatrix_handoff"}
+    exec(compile(_b64.b64decode("__BEATRIX_HANDOFF_B64__").decode("utf-8"),
+                 "<beatrina handoff>", "exec"), hand)
+    ip.user_ns["share"] = hand["share"]
+    ip.user_ns["use"] = hand["use"]
+
+__carmar_install_formatters()
+del __carmar_install_formatters
+'''.replace("__CARMAR_MIME__", CARMAR_MIME) \
+   .replace("__CARMAPY_REPLY__", REPLY_MIME) \
+   .replace("__CARMAPY_CONTROL__", CONTROL_MIME) \
+   .replace("__CARMAPY_DATAVIEW_B64__", _DATAVIEW_B64) \
+   .replace("__CARMAR_DEBUGGER_B64__", _DEBUGGER_B64) \
+   .replace("__BEATRIX_HANDOFF_B64__", _HANDOFF_B64)