@voxgig/sdkgen 1.5.0 → 2.0.0

package/project/.sdk/tm/py-data/pkg/frames.py

@@ -0,0 +1,236 @@
+# ProjectName Data — records to DataFrame.
+#
+# The one genuinely new runtime component in this package. Everything else
+# either delegates to the sibling ProjectName SDK or is generated.
+#
+# Pipeline, per accessor call:
+#   1. normalise  — SDK dataclasses / dicts / scalars -> plain dicts
+#   2. flatten    — one level of nesting -> dotted columns (json_normalize)
+#   3. dtype      — model-declared nullable pandas dtypes, coerced safely
+#   4. order      — model field order first, extras appended
+#
+# Step 3 uses NULLABLE dtypes ('Int64', not 'int64') throughout. See the
+# PANDAS_DTYPE note in sdkgen's canonType.ts: API payloads omit optional
+# fields freely, so a column that looks integral in one page can arrive with
+# nulls in the next. NumPy's int64/bool cannot hold NA and would silently
+# upcast to float64/object mid-fetch, making a column's dtype depend on which
+# rows happened to come back.
+
+from __future__ import annotations
+
+import dataclasses
+from typing import Any, Iterable
+
+import pandas as pd
+
+
+# Flatten depth accepted by the accessors: 1 (default, one level), 0/"none"
+# (no flattening), or "full" (recursive).
+FLATTEN_DEFAULT = 1
+
+
+def to_record(item: Any) -> Any:
+    """One SDK result item -> a plain dict (or the value itself if scalar).
+
+    The ops are NOT symmetric, and this is the load-bearing detail:
+
+      * `list()` yields ENTITY OBJECTS (ProjectNameXEntity), whose record is
+        held privately and read through the public `data_get()`.
+      * `load()` / `create()` yield a plain dict.
+
+    So an entity is unwrapped via `data_get()` before anything else. An
+    earlier version fell through to the `__dict__` branch, which filters
+    underscore-prefixed attributes and therefore returned `{}` for every
+    row — a frame with the right number of rows and NO columns.
+    """
+    if item is None:
+        return None
+
+    # SDK entity object: read the record through its public accessor rather
+    # than reaching into `_data`, so feature hooks (GetData) still fire.
+    getter = getattr(item, "data_get", None)
+    if callable(getter):
+        try:
+            rec = getter()
+            if isinstance(rec, dict):
+                return rec
+        except Exception:
+            pass
+
+    if dataclasses.is_dataclass(item) and not isinstance(item, type):
+        # asdict() recurses into nested dataclasses, which is what we want:
+        # flattening below then decides how deep to expose them as columns.
+        return dataclasses.asdict(item)
+    if isinstance(item, dict):
+        return item
+    # Objects with a __dict__ (defensive: a hand-written entity, a namespace).
+    d = getattr(item, "__dict__", None)
+    if isinstance(d, dict):
+        return {k: v for k, v in d.items() if not k.startswith("_")}
+    return item
+
+
+def to_records(items: Any) -> list:
+    """An op result -> a list of plain records.
+
+    A `list` op yields a list; a `load`/`create` op yields one record. Both
+    are accepted so the same engine serves frame and series accessors.
+    """
+    if items is None:
+        return []
+    if isinstance(items, dict):
+        return [to_record(items)]
+    if isinstance(items, (list, tuple)):
+        return [to_record(i) for i in items]
+    if isinstance(items, Iterable) and not isinstance(items, (str, bytes)):
+        return [to_record(i) for i in items]
+    return [to_record(items)]
+
+
+def _max_level(flatten: Any) -> Any:
+    """Accessor `flatten=` kwarg -> json_normalize max_level.
+
+    1 (default) is deliberate. Full recursive flattening turns a deeply
+    nested payload into an unusable 400-column frame; one level gives the
+    dotted columns people actually group by ('model_card.author') and leaves
+    genuinely deep structures boxed in object columns, still reachable with
+    normal pandas.
+    """
+    if flatten is None or flatten is False or flatten == 0 or flatten == "none":
+        return 0
+    if flatten is True or flatten == "full":
+        return None  # json_normalize: no limit
+    if isinstance(flatten, int):
+        return flatten
+    return FLATTEN_DEFAULT
+
+
+def build_frame(
+    items: Any,
+    *,
+    dtypes: dict | None = None,
+    order: list | None = None,
+    flatten: Any = FLATTEN_DEFAULT,
+    dtype: bool = True,
+    parse_dates: Any = None,
+) -> pd.DataFrame:
+    """Records -> DataFrame, typed and ordered from the entity model.
+
+    dtypes / order are supplied by the generated accessor from the model's
+    field list; callers passing raw records (records_df) get neither, which
+    is fine — pandas' own inference then applies.
+    """
+    records = to_records(items)
+
+    if len(records) == 0:
+        # An empty result still returns the RIGHT columns with the RIGHT
+        # dtypes, so downstream code (df.empty checks, concat, groupby on a
+        # known column) behaves identically to a populated frame. A bare
+        # pd.DataFrame() would raise KeyError on the first column access.
+        cols = list(order or [])
+        empty = pd.DataFrame({c: pd.Series(dtype=(dtypes or {}).get(c, "object"))
+                              for c in cols})
+        return empty
+
+    # Scalars (a list op returning bare strings) -> a single 'value' column.
+    if not isinstance(records[0], dict):
+        return pd.DataFrame({"value": records})
+
+    max_level = _max_level(flatten)
+    if max_level == 0:
+        df = pd.DataFrame(records)
+    else:
+        df = pd.json_normalize(records, sep=".", max_level=max_level)
+
+    if dtype and dtypes:
+        df = apply_dtypes(df, dtypes)
+
+    if parse_dates:
+        df = apply_dates(df, parse_dates)
+
+    if order:
+        df = order_columns(df, order)
+
+    return df
+
+
+def apply_dtypes(df: pd.DataFrame, dtypes: dict) -> pd.DataFrame:
+    """Coerce declared columns to their model dtype, one column at a time.
+
+    Per-column rather than a single df.astype(): one unconvertible column
+    must not fail the whole frame. A column that will not convert is LEFT AS
+    IS — an analyst with a usable frame and one object column is far better
+    served than one holding an exception. The API is the authority on its own
+    data; our model is a hint.
+    """
+    for col, want in dtypes.items():
+        if col not in df.columns:
+            continue
+        if str(df[col].dtype) == want:
+            continue
+        try:
+            df[col] = df[col].astype(want)
+        except (TypeError, ValueError):
+            # Numeric rescue: an API that sends "42" for an integer field is
+            # common enough to be worth handling.
+            #
+            # But to_numeric(errors="coerce") turns anything it cannot parse
+            # into NA, which would DESTROY a genuinely non-numeric column
+            # while appearing to succeed. So the rescue is only accepted when
+            # it is lossless: every value that was non-null before must still
+            # be non-null after. Otherwise the column is left exactly as it
+            # came from the API.
+            if want in ("Int64", "Float64"):
+                coerced = pd.to_numeric(df[col], errors="coerce")
+                lost = coerced.isna() & df[col].notna()
+                if not lost.any():
+                    try:
+                        df[col] = coerced.astype(want)
+                    except (TypeError, ValueError):
+                        pass
+    return df
+
+
+def apply_dates(df: pd.DataFrame, parse_dates: Any) -> pd.DataFrame:
+    """Parse the named columns as UTC datetimes.
+
+    EXPLICIT opt-in, never inferred. The model carries no string formats —
+    apidef has no $DATE sentinel — so guessing which string columns are dates
+    would be a heuristic on column names, and a wrong guess mangles data
+    silently. `errors="coerce"` turns unparseable values into NaT rather than
+    raising, keeping the frame usable.
+    """
+    cols = [parse_dates] if isinstance(parse_dates, str) else list(parse_dates)
+    for col in cols:
+        if col in df.columns:
+            df[col] = pd.to_datetime(df[col], errors="coerce", utc=True)
+    return df
+
+
+def order_columns(df: pd.DataFrame, order: list) -> pd.DataFrame:
+    """Model-declared column order first, everything else appended as-is.
+
+    Stable column order across calls beats alphabetical: an analyst's
+    df.iloc[:, :5] should mean the same thing on Monday and Tuesday, and the
+    model lists required fields first.
+    """
+    known = [c for c in order if c in df.columns]
+    extra = [c for c in df.columns if c not in known]
+    return df[known + extra]
+
+
+def records_df(items: Any, **kwargs) -> pd.DataFrame:
+    """Shape ANY op result into a DataFrame.
+
+    The public escape hatch for endpoints without a generated accessor —
+    action ops, non-entity endpoints, or a raw call made through `.sdk`.
+    """
+    return build_frame(items, **kwargs)
+
+
+def to_series(item: Any, **kwargs) -> pd.Series:
+    """One record -> a Series (the natural shape for a single `load`)."""
+    df = build_frame(item, **kwargs)
+    if len(df) == 0:
+        return pd.Series(dtype="object")
+    return df.iloc[0]

package/project/.sdk/tm/py-data/src/feature/README.md

@@ -0,0 +1,8 @@
+# py-data features
+
+The py-data target does not generate per-feature source. Features are
+applied to the sibling Python SDK by the `py` target; this package
+consumes that SDK as a single dependency and reads the state its
+features publish (notably `paging`, whose recorded cursor drives the
+eager fetch in `fetch.py`). This file exists only to satisfy the
+standard `tm/<target>/src/feature/` folder convention.

package/project/.sdk/tm/py-data/test/test_auth.py

@@ -0,0 +1,95 @@
+# ProjectName Data — credential discovery tests.
+
+from __future__ import annotations
+
+import os
+
+import pytest
+
+from projectname_data import auth
+from projectname_data.auth import resolve, from_env, missing_token_message, missing_base_message, NOT_FOUND
+
+
+class TestResolve:
+
+    def test_explicit_argument_wins(self, monkeypatch):
+        monkeypatch.setenv("ACME_APIKEY", "from-env")
+        tok, base, src = resolve("ACME", token="explicit")
+        assert tok == "explicit"
+        assert src == "argument"
+
+    def test_env_fallback(self, monkeypatch):
+        monkeypatch.setenv("ACME_APIKEY", "from-env")
+        tok, base, src = resolve("ACME")
+        assert tok == "from-env"
+        assert src == "env"
+
+    def test_missing_returns_none(self, monkeypatch):
+        monkeypatch.delenv("ACME_APIKEY", raising=False)
+        tok, base, src = resolve("ACME")
+        assert tok is None
+        assert src is None
+
+    def test_base_url_from_env(self, monkeypatch):
+        monkeypatch.setenv("ACME_APIKEY", "k")
+        monkeypatch.setenv("ACME_BASE", "https://tenant.example.com")
+        tok, base, src = resolve("ACME")
+        assert base == "https://tenant.example.com"
+
+    def test_explicit_base_wins(self, monkeypatch):
+        monkeypatch.setenv("ACME_BASE", "https://env.example.com")
+        tok, base, src = resolve("ACME", token="k", base_url="https://arg.example.com")
+        assert base == "https://arg.example.com"
+
+    def test_custom_keys(self, monkeypatch):
+        monkeypatch.setenv("WEIRD_TOKEN", "k")
+        tok, base, src = resolve("ACME", token_key="WEIRD_TOKEN")
+        assert tok == "k"
+
+    def test_empty_string_is_not_a_credential(self, monkeypatch):
+        monkeypatch.setenv("ACME_APIKEY", "")
+        tok, base, src = resolve("ACME")
+        assert tok is None
+
+    def test_colab_is_tried_before_env(self, monkeypatch):
+        monkeypatch.setenv("ACME_APIKEY", "from-env")
+        monkeypatch.setattr(auth, "from_colab",
+                            lambda key: "from-colab" if key == "ACME_APIKEY" else NOT_FOUND)
+        tok, base, src = resolve("ACME")
+        assert tok == "from-colab"
+        assert src == "colab"
+
+    def test_colab_failure_falls_through_to_env(self, monkeypatch):
+        # Outside Colab the import fails; inside Colab an ungranted secret
+        # raises. Neither is an error here — both mean "try the next source".
+        monkeypatch.setenv("ACME_APIKEY", "from-env")
+        monkeypatch.setattr(auth, "from_colab", lambda key: NOT_FOUND)
+        tok, base, src = resolve("ACME")
+        assert tok == "from-env"
+
+
+class TestFromEnv:
+
+    def test_present(self, monkeypatch):
+        monkeypatch.setenv("X_Y", "v")
+        assert from_env("X_Y") == "v"
+
+    def test_absent(self, monkeypatch):
+        monkeypatch.delenv("X_Y", raising=False)
+        assert from_env("X_Y") is NOT_FOUND
+
+
+class TestMessages:
+
+    def test_token_message_names_the_real_key(self):
+        msg = missing_token_message("ACME")
+        assert "ACME_APIKEY" in msg
+        # It must teach, not just report: both environments, copy-pasteable.
+        assert "userdata.set" in msg
+        assert "os.environ" in msg
+        assert "data(token=" in msg
+
+    def test_base_message_names_the_real_key(self):
+        msg = missing_base_message("ACME")
+        assert "ACME_BASE" in msg
+        assert "base_url=" in msg

package/project/.sdk/tm/py-data/test/test_fetch.py

@@ -0,0 +1,152 @@
+# ProjectName Data — paging drain tests.
+#
+# Driven with a fake client rather than the SDK: the drain loop's contract is
+# with the paging state the SDK's paging feature publishes on the client
+# (`client._paging["last"]`), so a fake that publishes the same shape tests
+# the real contract without a network or a generated SDK in the loop.
+
+from __future__ import annotations
+
+import pytest
+
+from projectname_data.fetch import drain, _next_ctrl, _last_paging, MAX_PAGES_DEFAULT
+
+
+class FakeClient:
+    """Publishes paging state exactly as the SDK's paging feature does."""
+
+    def __init__(self, pages):
+        self.pages = pages
+        self.calls = []
+        self._paging = None
+
+    def call(self, ctrl):
+        self.calls.append(ctrl)
+        idx = len(self.calls) - 1
+        if idx >= len(self.pages):
+            return []
+        rows, paging = self.pages[idx]
+        self._paging = {"last": paging}
+        return rows
+
+
+def paging(has_more=False, cursor=None, next_page=None):
+    return {
+        "page": None, "totalCount": None, "nextPage": next_page,
+        "next": None, "cursor": cursor, "hasMore": has_more,
+    }
+
+
+class TestDrain:
+
+    def test_single_page_makes_one_call(self):
+        c = FakeClient([([{"a": 1}], paging(has_more=False))])
+        rows = drain(c.call, c, quiet=True)
+        assert rows == [{"a": 1}]
+        assert len(c.calls) == 1
+
+    def test_cursor_paging_drains_every_page(self):
+        c = FakeClient([
+            ([{"a": 1}], paging(has_more=True, cursor="c1")),
+            ([{"a": 2}], paging(has_more=True, cursor="c2")),
+            ([{"a": 3}], paging(has_more=False)),
+        ])
+        rows = drain(c.call, c, quiet=True)
+        assert [r["a"] for r in rows] == [1, 2, 3]
+        assert c.calls[1] == {"paging": {"cursor": "c1"}}
+        assert c.calls[2] == {"paging": {"cursor": "c2"}}
+
+    def test_page_number_paging(self):
+        c = FakeClient([
+            ([{"a": 1}], paging(has_more=True, next_page=2)),
+            ([{"a": 2}], paging(has_more=False)),
+        ])
+        rows = drain(c.call, c, quiet=True)
+        assert len(rows) == 2
+        assert c.calls[1] == {"paging": {"page": 2}}
+
+    def test_cursor_wins_over_page_number(self):
+        # When a server offers both, the cursor is the one with consistency
+        # guarantees.
+        nxt = _next_ctrl(paging(has_more=True, cursor="c1", next_page=5))
+        assert nxt == {"paging": {"cursor": "c1"}}
+
+    def test_limit_truncates_and_stops_early(self):
+        c = FakeClient([
+            ([{"a": 1}, {"a": 2}], paging(has_more=True, cursor="c1")),
+            ([{"a": 3}], paging(has_more=False)),
+        ])
+        rows = drain(c.call, c, limit=2, quiet=True)
+        assert len(rows) == 2
+        assert len(c.calls) == 1
+
+    def test_no_paging_state_degrades_to_one_call(self):
+        # An SDK with the paging feature inactive genuinely has one page.
+        # That is correct degradation, not an error.
+        class NoPaging:
+            def __init__(self):
+                self.n = 0
+
+            def call(self, ctrl):
+                self.n += 1
+                return [{"a": 1}]
+
+        c = NoPaging()
+        rows = drain(c.call, c, quiet=True)
+        assert len(rows) == 1
+        assert c.n == 1
+
+    def test_empty_page_claiming_more_stops(self):
+        c = FakeClient([
+            ([], paging(has_more=True, cursor="c1")),
+            ([{"a": 1}], paging(has_more=False)),
+        ])
+        rows = drain(c.call, c, quiet=True)
+        assert rows == []
+        assert len(c.calls) == 1
+
+    def test_repeated_cursor_stops_and_warns(self):
+        # A server that never advances its cursor would otherwise hang the
+        # notebook cell forever.
+        c = FakeClient([
+            ([{"a": 1}], paging(has_more=True, cursor="same")),
+            ([{"a": 2}], paging(has_more=True, cursor="same")),
+            ([{"a": 3}], paging(has_more=True, cursor="same")),
+        ])
+        with pytest.warns(UserWarning, match="stopped advancing"):
+            rows = drain(c.call, c, quiet=True)
+        assert len(rows) == 2
+
+    def test_max_pages_backstop_warns(self):
+        pages = [([{"a": i}], paging(has_more=True, cursor=f"c{i}"))
+                 for i in range(10)]
+        c = FakeClient(pages)
+        with pytest.warns(UserWarning, match="max_pages"):
+            rows = drain(c.call, c, max_pages=3, quiet=True)
+        assert len(rows) == 3
+
+    def test_has_more_without_a_usable_token_stops(self):
+        # A Link: rel="next" URL with no cursor/page is not something we can
+        # turn into a ctrl; stopping beats guessing at query parsing.
+        c = FakeClient([([{"a": 1}], paging(has_more=True))])
+        rows = drain(c.call, c, quiet=True)
+        assert len(rows) == 1
+        assert len(c.calls) == 1
+
+
+class TestPagingHelpers:
+
+    def test_last_paging_absent(self):
+        class Bare:
+            pass
+
+        assert _last_paging(Bare()) is None
+
+    def test_last_paging_present(self):
+        class C:
+            _paging = {"last": {"hasMore": True}}
+
+        assert _last_paging(C()) == {"hasMore": True}
+
+    def test_max_pages_default_is_a_backstop_not_a_limit(self):
+        assert MAX_PAGES_DEFAULT >= 1000