beatrina 0.8.6

package/engines/python/dataview.py

@@ -0,0 +1,735 @@
+"""dataview.py — the data viewer's two ops, `view` and `colstats`, in pandas.
+
+Port of `spike/worker.R`'s `view_payload`, `view_filter`, `match_filter`,
+`describe_column` and `colstats_payload`. Every rule below was read out of that
+file, not recalled.
+
+WHERE THIS RUNS, AND WHY IT IS NOT THE SUPERVISOR. `view` needs the objects in
+the running session, so it is answered by the evaluating child — exactly as R
+routes `view`/`colstats` to `worker.R` and not to `serve.R`. It therefore queues
+behind a running cell, which is R's behaviour too. The file ops do NOT (see
+`fileio.py`): a Save must not wait for a forty-second fit, but a data viewer
+asking about `df` while `df` is being rebuilt has nothing to answer with.
+
+This module is a SINGLE BODY WITH TWO CALLERS, the shape `fileio.R` records:
+the test process imports it directly (fast, no kernel), and `bootstrap.py`
+ships its source into the kernel, where it is exec'd on FIRST USE. Lazy for
+B4's reason — a session that never opens the viewer must not pay pandas'
+import at boot.
+
+THE TWO RULES THAT ARE CORRECTNESS, NOT POLISH:
+
+  1. The preview is bounded by BYTES as well as rows. 500 rows of 20 KB strings
+     is a 10 MB frame that stalls the socket, and one cell can hold a fitted
+     model. Rows are capped, columns are capped, CELLS are clipped, and then the
+     page is measured as it will actually ship.
+
+  2. When shedding rows to fit that ceiling, ESTIMATE the per-row cost and keep
+     what fits — never halve repeatedly. `emit_dataframe` halved (200 → 100 →
+     50) and it cost a third of the rows that would have fit; the number it
+     lands on is also one the reader cannot predict from anything on screen.
+     `view_payload` one file over had the right rule, and this is that rule.
+
+ONE TRAP R HAS THAT PYTHON DOES NOT, and one Python has that R does not.
+R's `I()` wrappers are absent here because a Python list of length 1 is still a
+list — jsonlite's auto_unbox, which shipped `"bins": 10` for a constant column
+and broke every client that mapped over it, has no Python equivalent. In
+exchange, `json.dumps` will happily write bare `NaN`/`Infinity`, which are not
+JSON and which `JSON.parse` rejects: every number that reaches a payload goes
+through `_num()`, which maps them to null the way jsonlite's `na = "null"` does.
+"""
+
+from __future__ import annotations
+
+import json
+import math
+import re
+
+MAX_VIEW_ROWS = 500
+MAX_VIEW_COLS = 100
+MAX_VIEW_BYTES = 512 * 1024
+MAX_VIEW_CELL_CHARS = 512
+MAX_VIEW_LABEL_CHARS = 128
+MAX_STATS_LEVELS = 15
+STATS_BINS = 24
+VIEW_BINS = 12
+
+
+# ── formatting ──────────────────────────────────────────────────────────────
+# Numbers come back BOTH ways — as strings formatted HERE (so the card never
+# re-invents significant digits in JavaScript) and, where a drawing needs them,
+# as raw numerics. The two never disagree because they come from one value.
+
+def _num(x):
+    """A JSON-safe number, or None. NaN and ±Inf are not JSON."""
+    if x is None:
+        return None
+    try:
+        v = float(x)
+    except (TypeError, ValueError):
+        return None
+    if v != v or math.isinf(v):
+        return None
+    return v
+
+
+def fmt_num(x) -> str:
+    try:
+        v = float(x)
+    except (TypeError, ValueError):
+        return "NA"
+    if v != v:
+        return "NaN"
+    if math.isinf(v):
+        return "Inf" if v > 0 else "-Inf"
+    if abs(v) >= 1e5 or (abs(v) < 1e-3 and v != 0):
+        return f"{v:.2e}"                      # R's format(digits = 3, scientific)
+    s = f"{v:.3f}".rstrip("0").rstrip(".")
+    return "0" if s in ("", "-", "-0") else s
+
+
+def fmt_count(x) -> str:
+    try:
+        return f"{int(x):,}"
+    except (TypeError, ValueError):
+        return "NA"
+
+
+def fmt_pct(part, whole) -> str:
+    """A share as a bare percentage: "85.3%".
+
+    Every percentage on the card comes from here. R and JavaScript do not round
+    halves the same way — R's round() goes to even, so 50/4000 is 1.2% while
+    JavaScript's toFixed(1) gives 1.3% — and a card that computed some of its
+    own percentages printed both, for the same count, two inches apart. Python's
+    format() rounds half to even like R, so one formatter is one answer.
+    Always one decimal: a column reading "1.2% / 1.1% / 1% / 1%" loses the eye.
+    """
+    whole = _num(whole)
+    if whole is None or whole <= 0:
+        return ""
+    return f"{100.0 * float(part) / whole:.1f}%"
+
+
+def fmt_share(part, whole) -> str:
+    """A share as count and percentage: "3,412 (85.3%)"."""
+    w = _num(whole)
+    if w is None or w <= 0:
+        return fmt_count(part)
+    return f"{fmt_count(part)} ({fmt_pct(part, whole)})"
+
+
+def _pair(label, value):
+    return {"label": label, "value": value}
+
+
+def as_count(x, default: int) -> int:
+    """Coerce a wire-supplied count, FALLING BACK rather than erroring: a
+    malformed offset from a client must degrade to the default, not kill the
+    pane."""
+    if isinstance(x, bool) or x is None:
+        return default
+    try:
+        return int(x)
+    except (TypeError, ValueError):
+        return default
+
+
+# ── searching and filtering ─────────────────────────────────────────────────
+
+def _escape_regex(s: str) -> str:
+    return re.escape(str(s))
+
+
+def as_search_text(col):
+    """A column as searchable text, whatever it holds."""
+    import pandas as pd
+    if isinstance(col.dtype, pd.CategoricalDtype):
+        col = col.astype("object")
+    return col.astype("string").fillna("")
+
+
+def contains_ci(text, needle):
+    """Case-insensitive substring match.
+
+    R's note here is worth keeping because the Python trap is the mirror image:
+    `grepl(fixed = TRUE, ignore.case = TRUE)` silently ignores ignore.case, so
+    searching "Cohesion" found nothing in a column of "cohesion". pandas'
+    `str.contains(case=False)` honours the flag but treats the needle as a
+    REGEX, so `f[1]` would be a character class and `a.c` would match "abc".
+    Escaping the needle and keeping regex on gives both properties at once —
+    every metacharacter inert, folding that works for non-ASCII too.
+    """
+    return text.str.contains(_escape_regex(needle), case=False, na=False, regex=True)
+
+
+def match_filter(col, spec):
+    """One column filter spec → a boolean keep-mask.
+
+    NEVER evaluates the spec: comparisons and ranges are parsed, everything
+    else is literal text.
+    """
+    import numpy as np
+    import pandas as pd
+
+    spec = str(spec).strip()
+    n = len(col)
+    if not spec:
+        return np.ones(n, dtype=bool)
+    na = col.isna().to_numpy()
+    if spec.upper() == "NA":
+        return na
+
+    if pd.api.types.is_bool_dtype(col.dtype):
+        wanted = spec.upper()
+        if wanted in ("TRUE", "T"):
+            return (~na) & col.fillna(False).to_numpy(dtype=bool)
+        if wanted in ("FALSE", "F"):
+            return (~na) & (~col.fillna(True).to_numpy(dtype=bool))
+
+    if pd.api.types.is_numeric_dtype(col.dtype) and not pd.api.types.is_bool_dtype(col.dtype):
+        values = pd.to_numeric(col, errors="coerce").to_numpy(dtype="float64", na_value=np.nan)
+        bounds = spec.split("..")
+        if len(bounds) == 2:
+            lo, hi = _float_or_none(bounds[0]), _float_or_none(bounds[1])
+            if lo is not None and hi is not None:
+                return (~na) & (values >= lo) & (values <= hi)
+        hit = re.match(r"^\s*(>=|<=|!=|==|=|>|<)\s*(-?[0-9]+(?:\.[0-9]+)?(?:[eE][+-]?[0-9]+)?)\s*$", spec)
+        if hit:
+            target = float(hit.group(2))
+            op = hit.group(1)
+            with np.errstate(invalid="ignore"):
+                mask = {">=": values >= target, "<=": values <= target,
+                        "!=": values != target, "==": values == target,
+                        "=": values == target, ">": values > target,
+                        "<": values < target}[op]
+            return (~na) & mask
+        target = _float_or_none(spec)
+        if target is not None:
+            return (~na) & (values == target)
+
+    text = as_search_text(col)
+    if spec.startswith("="):
+        want = spec[1:].lower()
+        return (~na) & (text.str.lower() == want).to_numpy(dtype=bool)
+    if spec.startswith("!"):
+        # NA rows are KEPT by a negated filter — R's `is.na(col) | !contains`.
+        return na | (~contains_ci(text, spec[1:]).to_numpy(dtype=bool))
+    return (~na) & contains_ci(text, spec).to_numpy(dtype=bool)
+
+
+def _float_or_none(s):
+    try:
+        return float(str(s).strip())
+    except (TypeError, ValueError):
+        return None
+
+
+def view_filter(obj, query=None, filters=None):
+    """Narrow a frame by the viewer's search box and its per-column filters.
+
+    Shared with the statistics card, so a profile of "the rows you are looking
+    at" cannot disagree with the rows the grid is actually showing. A panel that
+    quietly profiled the whole frame while the grid showed a filtered subset
+    would be worse than no panel: both numbers look authoritative and only one
+    answers the question asked.
+    """
+    import numpy as np
+
+    count = 0
+    if isinstance(filters, dict) and filters:
+        for name in list(filters.keys()):
+            if name not in obj.columns:
+                continue
+            spec = filters[name]
+            if spec is None or not str(spec).strip():
+                continue
+            obj = obj[match_filter(obj[name], spec)]
+            count += 1
+    query = str(query).strip() if isinstance(query, str) else ""
+    if query and len(obj):
+        keep = np.zeros(len(obj), dtype=bool)
+        for name in obj.columns:
+            keep |= contains_ci(as_search_text(obj[name]), query).to_numpy(dtype=bool)
+        obj = obj[keep]
+    return obj, count, query
+
+
+# ── column description (per page, must stay cheap) ──────────────────────────
+
+def _kind_of(col):
+    import pandas as pd
+    dtype = col.dtype
+    if pd.api.types.is_bool_dtype(dtype):
+        return "logical"
+    if pd.api.types.is_datetime64_any_dtype(dtype) or isinstance(dtype, pd.PeriodDtype):
+        return "date"
+    if pd.api.types.is_numeric_dtype(dtype):
+        return "numeric"
+    return "categorical"
+
+
+def _epoch_seconds(col):
+    """Datetimes as epoch SECONDS, whatever resolution pandas chose.
+
+    NOT `.astype("int64") / 1e9`. pandas 3 stores a datetime column as
+    `datetime64[us]` by default, so that expression divides MICROseconds by a
+    billion and every date in 2020 reads as the 19th of January 1970 — a value
+    that is finite, plausibly shaped, orders correctly and is completely wrong.
+    Found by printing the first real reply, not by a test. Subtracting the epoch
+    and asking for total seconds is resolution-independent, and it is also what
+    R's `as.numeric` on a POSIXct means.
+    """
+    import pandas as pd
+    stamps = pd.to_datetime(col, errors="coerce")
+    tz = getattr(stamps.dtype, "tz", None)
+    epoch = pd.Timestamp("1970-01-01", tz=tz) if tz is not None else pd.Timestamp("1970-01-01")
+    return (stamps - epoch).dt.total_seconds().to_numpy()
+
+
+def _show_time(v) -> str:
+    """An epoch-seconds value as a timestamp, seconds resolution — what R's
+    `format.POSIXct` prints, so a quartile does not arrive with six decimals."""
+    import pandas as pd
+    try:
+        return pd.Timestamp(float(v), unit="s").strftime("%Y-%m-%d %H:%M:%S")
+    except (ValueError, OverflowError, OSError):
+        return fmt_num(v)
+
+
+def _numeric_values(col, kind):
+    """The column's present values as float64 — epoch seconds for a date."""
+    import numpy as np
+    import pandas as pd
+    if kind == "date":
+        return _epoch_seconds(col.dropna())
+    return pd.to_numeric(col.dropna(), errors="coerce").to_numpy(dtype="float64",
+                                                                 na_value=np.nan)
+
+
+def describe_column(col, name: str) -> dict:
+    """Describe one column the way a data viewer shows it: type, a sparkline's
+    worth of shape, a stat label, and how much is missing.
+
+    The bins are computed HERE rather than shipping the column: a 500k-row
+    numeric vector is 4 MB of JSON and 12 counts is 60 bytes, and the viewer
+    only ever draws the 12.
+    """
+    import numpy as np
+
+    n = int(len(col))
+    # `isna().sum()` and not `isna().to_numpy()`: this runs for every column of
+    # every page, and materialising a million-element boolean to count it — and
+    # then slicing with it — is measurably the difference between a 230 ms page
+    # and a 150 ms one. Only the logical branch needs the mask itself.
+    missing = int(col.isna().sum())
+    kind = _kind_of(col)
+    base = {"name": name, "class": str(col.dtype), "n": n, "missing": missing}
+    miss_pair = [_pair("Missing", fmt_share(missing, n))] if missing > 0 else []
+
+    if kind in ("numeric", "date"):
+        values = _numeric_values(col, kind)
+        values = values[np.isfinite(values)]
+        if values.size == 0:
+            return {**base, "kind": kind, "bins": [], "stat": "all missing",
+                    "summary": [_pair("Missing", fmt_share(missing, n))]}
+        lo, hi = float(values.min()), float(values.max())
+        # np.histogram, not value_counts: an empty bin must still be a bin, or a
+        # gap in the data silently shortens the sparkline and every bar after it
+        # shifts left. (R's note against table(); the same trap, the same fix.)
+        if hi == lo:
+            bins = [int(values.size)]
+        else:
+            bins = [int(v) for v in np.histogram(values, bins=VIEW_BINS,
+                                                 range=(lo, hi))[0]]
+        med = float(np.median(values))
+        mu = float(values.mean())
+        sigma = float(values.std(ddof=1)) if values.size > 1 else float("nan")
+        # A date's VALUES stay epoch seconds, because the sparkline draws them,
+        # but every string a reader sees is a date. R has no date branch here at
+        # all — a Date falls through to the categorical arm, which builds a
+        # value_counts over a million distinct timestamps to print a level count
+        # nobody wants. A range of "1.58e+06 – 1.58e+06" was the first draft of
+        # this line and is the reason the branch exists.
+        show = _show_time if kind == "date" else fmt_num
+        return {**base, "kind": kind, "bins": bins,
+                "min": _num(lo), "max": _num(hi), "median": _num(med),
+                "mean": _num(mu), "sd": _num(sigma),
+                "summary": [_pair("Mean", show(mu)),
+                            _pair("Std. dev.", fmt_num(sigma)),
+                            _pair("Minimum", show(lo)),
+                            _pair("Median", show(med)),
+                            _pair("Maximum", show(hi))] + miss_pair,
+                "stat": f"{show(lo)} – {show(hi)}"}
+
+    if kind == "logical":
+        na = col.isna().to_numpy()
+        truthy = col.fillna(False).to_numpy(dtype=bool)
+        yes = int((truthy & ~na).sum())
+        no = int((~truthy & ~na).sum())
+        return {**base, "kind": "logical", "bins": [yes, no],
+                "levels": ["TRUE", "FALSE"],
+                "summary": [_pair("TRUE", fmt_share(yes, yes + no)),
+                            _pair("FALSE", fmt_share(no, yes + no))] + miss_pair,
+                "stat": f"{yes} true / {no} false"}
+
+    # `value_counts()` straight off the column, NOT off `as_search_text()`: the
+    # search text is a full second copy of a million strings (27 ms measured) and
+    # value_counts already drops missing and already groups whatever the dtype
+    # holds. Only the handful of level LABELS that ship need stringifying.
+    tab = col.value_counts()
+    present = n - missing
+    top = tab.head(VIEW_BINS)
+    # The commonest levels answer "what is in here" far better than the level
+    # COUNT alone, and they are already computed for the sparkline. Four are
+    # listed when four is all there is, so a column with exactly four levels does
+    # not show three and silently swallow the last one.
+    lead = tab.head(len(tab) if len(tab) <= 4 else 3)
+    tops = [_pair(str(k)[:MAX_VIEW_LABEL_CHARS], fmt_share(int(v), present))
+            for k, v in lead.items()]
+    return {**base, "kind": "categorical",
+            "bins": [int(v) for v in top.to_numpy()],
+            "levels": [str(k)[:MAX_VIEW_LABEL_CHARS] for k in top.index],
+            "nlevels": int(len(tab)),
+            "summary": [_pair("Distinct levels", fmt_count(len(tab)))] + tops + miss_pair,
+            "stat": f"{len(tab)} level{'' if len(tab) == 1 else 's'}"}
+
+
+# ── the statistics card (one column, on demand) ─────────────────────────────
+
+def colstats_payload(col, column_name: str) -> dict:
+    """Everything a statistics card shows about ONE column.
+
+    Deliberately separate from describe_column(): that one runs for every column
+    of every page and must stay cheap, so it ships 12 bins and a label. This one
+    runs when a reader asks about a single column and can afford quantiles,
+    shape moments and a level table.
+    """
+    import numpy as np
+    import pandas as pd
+
+    n = int(len(col))
+    ok = col.dropna()
+    missing = n - int(len(ok))
+    kind = _kind_of(col)
+    base = {"column": column_name, "class": str(col.dtype), "n": n,
+            "missing": missing, "present": int(len(ok)),
+            "distinct": int(ok.nunique(dropna=True))}
+
+    if kind in ("numeric", "date"):
+        dated = kind == "date"
+        x = _numeric_values(col, kind)
+        x = x[np.isfinite(x)]
+        if x.size == 0:
+            return {**base, "kind": kind,
+                    "summary": [_pair("Present", "0 — every value is missing")]}
+
+        show = _show_time if dated else fmt_num
+        q1, med, q3 = (float(v) for v in np.quantile(x, [0.25, 0.5, 0.75]))
+        iqr = q3 - q1
+        m = float(x.mean())
+        s = float(x.std(ddof=1)) if x.size > 1 else float("nan")
+        # Fences are Tukey's, so "outlier" here means what a boxplot means by it
+        # — not a normal-theory z cut, which would be a different claim about
+        # data nobody has shown is normal.
+        lo_fence, hi_fence = q1 - 1.5 * iqr, q3 + 1.5 * iqr
+        outliers = int(((x < lo_fence) | (x > hi_fence)).sum())
+        # Moments by hand rather than by dependency: both are one line.
+        if math.isfinite(s) and s > 0:
+            z = (x - m) / s
+            skew, kurt = float((z ** 3).mean()), float((z ** 4).mean() - 3)
+        else:
+            skew = kurt = float("nan")
+        lo, hi = float(x.min()), float(x.max())
+        if hi == lo:
+            bins = [int(x.size)]
+        else:
+            bins = [int(v) for v in np.histogram(x, bins=STATS_BINS, range=(lo, hi))[0]]
+        mad = float(np.median(np.abs(x - med))) * 1.4826   # R's mad(), constant included
+        summary = [
+            _pair("Mean", show(m)), _pair("Std. dev.", fmt_num(s)),
+            _pair("Minimum", show(lo)), _pair("1st quartile", show(q1)),
+            _pair("Median", show(med)), _pair("3rd quartile", show(q3)),
+            _pair("Maximum", show(hi)), _pair("IQR", fmt_num(iqr)),
+            _pair("Median abs. dev.", fmt_num(mad)),
+            _pair("Skewness", fmt_num(skew)), _pair("Excess kurtosis", fmt_num(kurt)),
+            _pair("Outliers (1.5 IQR)", fmt_share(outliers, int(x.size)))]
+        if not dated:
+            summary += [_pair("Zeros", fmt_share(int((x == 0).sum()), int(x.size))),
+                        _pair("Negative", fmt_share(int((x < 0).sum()), int(x.size)))]
+        else:
+            summary += [_pair("Span", f"{fmt_num((hi - lo) / 86400.0)} days")]
+        return {**base, "kind": kind, "summary": summary, "bins": bins,
+                "binMin": _num(lo), "binMax": _num(hi),
+                "box": {"min": _num(lo), "q1": _num(q1), "median": _num(med),
+                        "q3": _num(q3), "max": _num(hi),
+                        "lower": _num(max(lo_fence, lo)),
+                        "upper": _num(min(hi_fence, hi)),
+                        "outliers": outliers}}
+
+    if kind == "logical":
+        na = col.isna().to_numpy()
+        truthy = col.fillna(False).to_numpy(dtype=bool)
+        yes = int((truthy & ~na).sum())
+        no = int((~truthy & ~na).sum())
+        return {**base, "kind": "logical",
+                "summary": [_pair("TRUE", fmt_share(yes, yes + no)),
+                            _pair("FALSE", fmt_share(no, yes + no)),
+                            _pair("Missing", fmt_share(missing, n))],
+                "levels": ["TRUE", "FALSE"], "counts": [yes, no],
+                "shares": [fmt_pct(yes, yes + no), fmt_pct(no, yes + no)],
+                "other": 0}
+
+    text = as_search_text(ok)
+    tab = text.value_counts()
+    top = tab.head(MAX_STATS_LEVELS)
+    widths = text.str.len().to_numpy()
+    total = int(len(text))
+    summary = [
+        _pair("Distinct levels", fmt_count(len(tab))),
+        _pair("Most common", f"{tab.index[0]} — {fmt_share(int(tab.iloc[0]), total)}"
+              if len(tab) else "—"),
+        _pair("Least common", f"{tab.index[-1]} — {fmt_share(int(tab.iloc[-1]), total)}"
+              if len(tab) else "—"),
+        _pair("Empty strings", fmt_share(int((text == "").sum()), total)),
+        _pair("Shortest", fmt_count(int(widths.min())) if widths.size else "—"),
+        _pair("Longest", fmt_count(int(widths.max())) if widths.size else "—"),
+        _pair("Mean length", fmt_num(float(widths.mean())) if widths.size else "—")]
+    counts = [int(v) for v in top.to_numpy()]
+    other = total - sum(counts)
+    return {**base, "kind": "categorical", "summary": summary,
+            "levels": [str(k)[:MAX_VIEW_LABEL_CHARS] for k in top.index],
+            "counts": counts,
+            "shares": [fmt_pct(c, total) for c in counts],
+            "other": other, "otherShare": fmt_pct(other, total),
+            "nlevels": int(len(tab))}
+
+
+# ── the page ────────────────────────────────────────────────────────────────
+
+def to_frame(obj):
+    """Anything → a DataFrame, or None for "not a table"."""
+    import pandas as pd
+    if isinstance(obj, pd.DataFrame):
+        return obj
+    try:
+        if isinstance(obj, pd.Series):
+            return obj.to_frame(name=obj.name if obj.name is not None else "value")
+        return pd.DataFrame(obj)
+    except Exception:
+        return None
+
+
+def _cell(v, limit: int):
+    """One display value, JSON-safe and bounded.
+
+    A single cell can dwarf the row and column caps (logs, embedded documents,
+    accidental blobs, a fitted model in an object column). The viewer is a
+    preview, so a complex value becomes a description and text is clipped BEFORE
+    JSON encoding — which is also the hard upper bound when the page has one row
+    and row shedding cannot help.
+    """
+    if v is None:
+        return None, False
+    if isinstance(v, bool):
+        return v, False
+    if isinstance(v, int):
+        return v, False
+    if isinstance(v, float):
+        return _num(v), False
+    if isinstance(v, str):
+        if len(v) > limit:
+            return v[:limit] + "…", True
+        return v, False
+    try:
+        import numpy as np
+        if isinstance(v, np.generic):
+            return _cell(v.item(), limit)
+    except Exception:
+        pass
+    try:
+        import pandas as pd
+        if v is pd.NaT or (not isinstance(v, (list, dict, tuple, set)) and pd.isna(v)):
+            return None, False
+    except Exception:
+        pass
+    s = str(v)
+    if len(s) > limit:
+        return s[:limit] + "…", True
+    return s, False
+
+
+def _uniquify(names):
+    """R's make.unique, so two columns clipped to the same label stay two."""
+    seen, out = {}, []
+    for raw in names:
+        name = str(raw)[:MAX_VIEW_LABEL_CHARS]
+        if name in seen:
+            seen[name] += 1
+            name = f"{name}.{seen[name]}"
+        else:
+            seen[name] = 0
+        out.append(name)
+    return out
+
+
+def view_payload(obj, shown_name, offset=None, limit=None, sort=None, desc=False,
+                 col_offset=None, col_limit=None, query=None, filters=None) -> dict:
+    """The true shape, whole-column descriptions, and one row × column window.
+
+    Sorting happens HERE, not in the client: the client only ever holds one
+    page, so a client-side sort would order 200 rows of a million-row frame and
+    call it sorted. It runs on the full frame BEFORE any windowing, so a column
+    window still sees globally ordered rows.
+
+    The reply always states both what EXISTS (`nrow`/`ncol`) and what it
+    actually CONTAINS (`shown`/`shownCols`, the effective `limit`/`colLimit`,
+    the clamp flags) — nothing downstream should have to guess.
+    """
+    frame = to_frame(obj)
+    if frame is None:
+        return {"name": shown_name, "error": "not a table"}
+
+    total_rows = int(len(frame))
+    offset = max(0, as_count(offset, 0))
+    limit_req = max(1, as_count(limit, 200))
+    limit = min(limit_req, MAX_VIEW_ROWS)
+    col_offset = max(0, as_count(col_offset, 0))
+    col_limit_req = max(1, as_count(col_limit, 30))
+    col_limit = min(col_limit_req, MAX_VIEW_COLS)
+
+    frame, filter_count, query = view_filter(frame, query, filters)
+    if isinstance(sort, str) and sort in set(str(c) for c in frame.columns):
+        # kind="stable" so ties keep their original order: an unstable sort makes
+        # the same page of the same frame come back shuffled between requests.
+        frame = frame.sort_values(by=sort, ascending=not bool(desc), kind="stable",
+                                  na_position="last")
+
+    ncol = int(frame.shape[1])
+    cidx = list(range(col_offset, min(col_offset + col_limit, ncol)))
+    ridx_hi = min(offset + limit, len(frame))
+    page = frame.iloc[offset:ridx_hi, cidx] if cidx else frame.iloc[offset:ridx_hi, []]
+
+    source_names = [str(c) for c in page.columns]
+    display_names = _uniquify(source_names)
+
+    clipped = 0
+    rows = []
+    # itertuples over iterrows: iterrows boxes every row into a Series, which on
+    # a 500-row × 100-column page is 50,000 object allocations for a preview.
+    for record in page.itertuples(index=False, name=None):
+        row = {}
+        for i, value in enumerate(record):
+            cell, was_clipped = _cell(value, MAX_VIEW_CELL_CHARS)
+            clipped += int(was_clipped)
+            row[display_names[i]] = cell
+        rows.append(row)
+
+    # PAYLOAD GUARD. The caps above bound CELLS, not bytes. Measure the page as
+    # it will actually ship and shed rows until it fits, reporting the shrunken
+    # window rather than silently serving it.
+    body = len(json.dumps(rows, allow_nan=False).encode("utf-8"))
+    if body > MAX_VIEW_BYTES and len(rows) > 1:
+        # ESTIMATE, never halve. Halving (200 → 100 → 50) is invisible from a
+        # small head and wasteful from a large one: a text-heavy frame that would
+        # fit 74 rows lands on 50, and the number it lands on is one the reader
+        # cannot predict from anything on screen.
+        per_row = body / len(rows)
+        keep = max(1, int(MAX_VIEW_BYTES // per_row))
+        if keep < len(rows):
+            rows = rows[:keep]
+            limit = keep
+            # The estimate assumes rows cost about the same; one enormous cell
+            # breaks that, so the halving loop stays underneath it as the
+            # fallback and almost never runs.
+            while (len(json.dumps(rows, allow_nan=False).encode("utf-8")) > MAX_VIEW_BYTES
+                   and len(rows) > 1):
+                rows = rows[:max(1, len(rows) // 2)]
+                limit = len(rows)
+
+    # `columns` describes the WINDOWED SET only, but each description covers the
+    # WHOLE column — the sparkline must show the full distribution, not a page's.
+    columns = [describe_column(frame.iloc[:, index], display)
+               for index, display in zip(cidx, display_names)]
+
+    return {"name": shown_name, "nrow": int(len(frame)), "ncol": ncol,
+            "totalRows": total_rows,
+            "filtered": bool(query) or filter_count > 0, "filterCount": filter_count,
+            "offset": offset, "limit": limit,
+            "colOffset": col_offset, "colLimit": col_limit,
+            "limitClamped": limit < limit_req,
+            "colLimitClamped": col_limit < col_limit_req,
+            "clippedCells": clipped,
+            "columns": columns, "rows": rows,
+            "shown": len(rows), "shownCols": len(display_names)}
+
+
+# ── the two entry points the kernel calls ───────────────────────────────────
+
+def _resolve(name, namespace):
+    """The object behind a viewer request.
+
+    A bare identifier is a LOOKUP; anything else is evaluated, which is what
+    `emit_view` does (`eval(parse(text = name), globalenv())`) and is safe here
+    for the same reason it is safe there — this is the evaluating session, and
+    the string came from a name the user chose in their own notebook. The lookup
+    comes first so the common case runs no code at all.
+    """
+    if not isinstance(name, str) or not name.strip():
+        return None, "bad name"
+    if name in namespace:
+        return namespace[name], None
+    if name.isidentifier():
+        return None, "not found"
+    try:
+        return eval(name, namespace), None       # noqa: S307 — see the docstring
+    except Exception:
+        return None, "not found"
+
+
+def view_reply(spec: dict, namespace: dict) -> dict:
+    name = spec.get("name")
+    shown = spec.get("label") or name
+    obj, err = _resolve(name, namespace)
+    if err:
+        return {"name": shown, "error": err}
+    return view_payload(obj, shown,
+                        offset=spec.get("offset"), limit=spec.get("limit"),
+                        sort=spec.get("sort"), desc=spec.get("desc"),
+                        col_offset=spec.get("colOffset"),
+                        col_limit=spec.get("colLimit"),
+                        query=spec.get("query"), filters=spec.get("filters"))
+
+
+def colstats_reply(spec: dict, namespace: dict) -> dict:
+    """One column, profiled over the rows the GRID IS SHOWING.
+
+    Takes the same `query` and `filters` as `view`, so the panel describes the
+    visible data rather than a different population that happens to share a name.
+    """
+    name = spec.get("name")
+    column = spec.get("column")
+    column = column[0] if isinstance(column, list) and column else column
+    if not isinstance(column, str) or not column:
+        return {"name": name, "column": column, "error": "not found"}
+    obj, err = _resolve(name, namespace)
+    if err:
+        return {"name": name, "column": column, "error": err}
+    frame = to_frame(obj)
+    if frame is None:
+        return {"name": name, "column": column, "error": "not found"}
+    total = int(len(frame))
+    frame, count, query = view_filter(frame, spec.get("query"), spec.get("filters"))
+    names = [str(c) for c in frame.columns]
+    # The viewer truncates long column names for display; match on the truncated
+    # name too, or a card opened from a clipped header can never find its column.
+    if column in names:
+        hit = names.index(column)
+    else:
+        clipped = [n[:MAX_VIEW_LABEL_CHARS] for n in names]
+        hit = clipped.index(column) if column in clipped else -1
+    if hit < 0:
+        return {"name": name, "column": column, "error": "no such column"}
+    return {"name": name, **colstats_payload(frame.iloc[:, hit], column),
+            "rows": int(len(frame)), "totalRows": total,
+            "filtered": bool(query) or count > 0}