@adhd/apigen-python-env 0.1.5 → 0.2.1

package/python/apigen_python/errors.py

@@ -0,0 +1,99 @@
+"""
+apigen_python.errors — §9.1 canonical error taxonomy for the Python host.
+
+Mirrors the TS @adhd/apigen-errors package exactly so the Python runtime
+raises the same codes and the gateway maps them identically.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Literal
+
+# ---------------------------------------------------------------------------
+# §9 — Canonical error code set (gRPC-style)
+# ---------------------------------------------------------------------------
+
+ApiErrorCode = Literal[
+    "invalid_argument",
+    "unauthenticated",
+    "permission_denied",
+    "not_found",
+    "internal",
+]
+
+ERROR_CODES: tuple[ApiErrorCode, ...] = (
+    "invalid_argument",
+    "unauthenticated",
+    "permission_denied",
+    "not_found",
+    "internal",
+)
+
+# ---------------------------------------------------------------------------
+# §9.1 — Per-transport status maps  (normative — must mirror SPEC §9 table)
+# ---------------------------------------------------------------------------
+
+HTTP_STATUS: dict[ApiErrorCode, int] = {
+    "invalid_argument": 400,
+    "unauthenticated": 401,
+    "permission_denied": 403,
+    "not_found": 404,
+    "internal": 500,
+}
+
+GRPC_CODE: dict[ApiErrorCode, str] = {
+    "invalid_argument": "INVALID_ARGUMENT",
+    "unauthenticated": "UNAUTHENTICATED",
+    "permission_denied": "PERMISSION_DENIED",
+    "not_found": "NOT_FOUND",
+    "internal": "INTERNAL",
+}
+
+CLI_EXIT_CODE: dict[ApiErrorCode, int] = {
+    "invalid_argument": 2,
+    "unauthenticated": 3,
+    "permission_denied": 3,
+    "not_found": 4,
+    "internal": 1,
+}
+
+MCP_ERROR_KIND: dict[ApiErrorCode, Literal["error"]] = {
+    "invalid_argument": "error",
+    "unauthenticated": "error",
+    "permission_denied": "error",
+    "not_found": "error",
+    "internal": "error",
+}
+
+# ---------------------------------------------------------------------------
+# ApiError — the raised error class
+# ---------------------------------------------------------------------------
+
+
+class ApiError(Exception):
+    """The canonical apigen error (§9).
+
+    Every transport adapter catches this and maps ``code`` to the native
+    status using the tables above.
+    """
+
+    def __init__(
+        self,
+        code: ApiErrorCode,
+        message: str,
+        details: Any = None,
+    ) -> None:
+        super().__init__(message)
+        self.code: ApiErrorCode = code
+        self.message: str = message
+        self.details: Any = details
+
+    def to_json(self) -> dict[str, Any]:
+        """Serialise to a plain dict suitable for JSON transport."""
+        out: dict[str, Any] = {"code": self.code, "message": self.message}
+        if self.details is not None:
+            out["details"] = self.details
+        return out
+
+    def __repr__(self) -> str:
+        return f"ApiError(code={self.code!r}, message={self.message!r})"

package/python/apigen_python/extractor.py

@@ -0,0 +1,578 @@
+"""
+apigen_python.extractor — Python source → canonical Operation[] descriptors (SPEC §4 / §14).
+
+Extracts every public export from a Python module and emits a list of Operation
+dicts that match the canonical §4 JSON shape:
+
+  id, host('python'), namespace, path, kind, async, streaming, safe,
+  input (JSON-Schema-2020-12 object), output (JSON-Schema-2020-12),
+  envelope ({} — middleware is TS-side only for now), typeText (None).
+
+Usage (subprocess / CLI):
+    python -m apigen_python.extractor <module_file_or_path> --emit-json [--namespace <ns>]
+    → JSON array on stdout
+
+`--emit-json` is REQUIRED (apigen-serve-core py-flask-serve-split /
+py-extract-serve-split-findings.md §1.4): this CLI is now a stable, two-phase
+subprocess-protocol contract the TS `py-flask`/`py-grpc` plugins spawn as
+phase 1 (extract-only) before computing route/verb via the REAL
+`@adhd/apigen-engine-naming` `project()` and re-spawning `flask_server`/
+`grpc_server` with the result (`--plan-file`). Requiring the flag explicitly
+— rather than making JSON-on-stdout the silent, implicit default — means a
+future non-JSON output mode (e.g. a human-readable tree) can be added to
+this CLI later without silently changing what the two-phase callers depend
+on.
+
+Rules:
+- Every exported name (not starting with '_') that is callable → kind='action'.
+- Every exported name that holds a serialisable primitive/dict/list → kind='query'.
+- Non-serialisable, non-callable exports → skipped + warned to stderr.
+- 'ctx' first-parameter is excluded from input schema (§4 inv:ctx-name-only).
+- Async functions → async:true; generators / async-generators → streaming:true.
+- kind='query' → safe:true by default; kind='action' → safe:false.
+"""
+
+from __future__ import annotations
+
+import ast
+import importlib.util
+import inspect
+import json
+import os
+import re
+import sys
+import types
+import warnings
+from pathlib import Path
+from typing import Any
+
+
+# ---------------------------------------------------------------------------
+# §4 Segment helpers
+# ---------------------------------------------------------------------------
+
+def _tokenise(raw: str) -> list[str]:
+    """Tokenise a camelCase / snake_case / kebab-case name into lower-cased words.
+
+    Examples:
+        'humanizeBytes' → ['humanize', 'bytes']
+        'get_user'      → ['get', 'user']
+        'file-name'     → ['file', 'name']
+    """
+    # Insert split boundary between lower→upper transitions (camelCase).
+    s = re.sub(r"([a-z0-9])([A-Z])", r"\1_\2", raw)
+    # Split on non-alphanumeric separators.
+    parts = re.split(r"[^a-zA-Z0-9]+", s)
+    return [p.lower() for p in parts if p]
+
+
+def _seg(raw: str) -> dict[str, Any]:
+    """Build a casing-neutral Segment dict from a raw token."""
+    return {"raw": raw, "words": _tokenise(raw)}
+
+
+def _normalise_filename(raw: str) -> str:
+    """Strip extension; dots/underscores → hyphens (SPEC §5)."""
+    no_ext = re.sub(r"\.[^.]+$", "", raw)
+    return re.sub(r"[._]+", "-", no_ext)
+
+
+# ---------------------------------------------------------------------------
+# JSON-Schema inference (stdlib only — no jsonschema dependency for extraction)
+# ---------------------------------------------------------------------------
+
+def _py_type_hint_to_schema(annotation: Any) -> dict[str, Any]:
+    """Best-effort conversion of a Python type annotation to JSON Schema 2020-12.
+
+    Only covers the common cases; falls back to {} (any) for complex generics.
+    Returns a plain dict.
+
+    Logical-type mappings (mirrors the TS apigen-logical codecs):
+      decimal.Decimal  → {"type": "string", "format": "decimal"}
+      datetime.datetime → {"type": "string", "format": "date-time"}
+      uuid.UUID        → {"type": "string", "format": "uuid"}
+    """
+    import typing
+
+    origin = getattr(annotation, "__origin__", None)
+    args = getattr(annotation, "__args__", ())
+
+    # None / NoneType
+    if annotation is type(None):
+        return {"type": "null"}
+
+    # Logical types — must be checked before the primitive table so that
+    # `Decimal` / `datetime` / `UUID` are not silently swallowed as {}
+    # (they are not in the _primitives dict, so they would fall through to
+    # the dataclass check and ultimately to the {} fallback without this guard).
+    try:
+        from decimal import Decimal as _Decimal
+        if annotation is _Decimal:
+            return {"type": "string", "format": "decimal"}
+    except ImportError:
+        pass
+
+    try:
+        from datetime import datetime as _Datetime
+        if annotation is _Datetime:
+            return {"type": "string", "format": "date-time"}
+    except ImportError:
+        pass
+
+    try:
+        from uuid import UUID as _UUID
+        if annotation is _UUID:
+            return {"type": "string", "format": "uuid"}
+    except ImportError:
+        pass
+
+    # Primitives
+    _primitives: dict[Any, str] = {
+        str: "string",
+        int: "integer",
+        float: "number",
+        bool: "boolean",
+        bytes: "string",
+    }
+    if annotation in _primitives:
+        return {"type": _primitives[annotation]}
+
+    # typing.Optional[X] == Union[X, None]
+    if origin is typing.Union:
+        non_none = [a for a in args if a is not type(None)]
+        if len(non_none) == 1 and type(None) in args:
+            inner = _py_type_hint_to_schema(non_none[0])
+            return {"oneOf": [inner, {"type": "null"}]}
+        schemas = [_py_type_hint_to_schema(a) for a in args]
+        return {"oneOf": schemas}
+
+    # list / List[T]
+    if origin is list:
+        items = _py_type_hint_to_schema(args[0]) if args else {}
+        return {"type": "array", "items": items}
+
+    # dict / Dict[K, V]
+    if origin is dict:
+        value_schema = _py_type_hint_to_schema(args[1]) if len(args) > 1 else {}
+        return {"type": "object", "additionalProperties": value_schema}
+
+    # dataclasses.dataclass
+    try:
+        import dataclasses
+        if dataclasses.is_dataclass(annotation) and isinstance(annotation, type):
+            props: dict[str, Any] = {}
+            required: list[str] = []
+            for f in dataclasses.fields(annotation):
+                props[f.name] = _py_type_hint_to_schema(f.type)
+                if f.default is dataclasses.MISSING and f.default_factory is dataclasses.MISSING:  # type: ignore[misc]
+                    required.append(f.name)
+            schema: dict[str, Any] = {"type": "object", "properties": props}
+            if required:
+                schema["required"] = required
+            return schema
+    except Exception:
+        pass
+
+    # Fall back: open object schema (any)
+    return {}
+
+
+def _resolve_hints(fn: Any) -> dict[str, Any]:
+    """Resolve all annotations for a callable, handling PEP-563 stringized forms.
+
+    When a module uses ``from __future__ import annotations`` all annotations are
+    stored as strings rather than type objects.  ``typing.get_type_hints`` evaluates
+    those strings against the function's module globals (and the optional
+    ``localns`` dict), returning a ``{param_name: type}`` mapping with real type
+    objects.
+
+    Falls back gracefully if resolution fails (e.g. a forward reference that
+    cannot be resolved in the execution context) — unresolvable params get ``{}``
+    (any) schema, preserving backward compatibility.
+
+    Args:
+        fn: Any callable (function, method, lambda).
+
+    Returns:
+        Dict mapping parameter / return annotation names to resolved type objects.
+        The return type is keyed under ``"return"`` (matching ``typing.get_type_hints``
+        convention).
+    """
+    import typing
+
+    try:
+        # include_extras=False strips Annotated[] wrappers to bare types.
+        hints = typing.get_type_hints(fn, include_extras=False)
+        return hints
+    except Exception:
+        # Fall back to raw __annotations__ (may be strings for PEP-563 modules).
+        raw = getattr(fn, "__annotations__", {})
+        return {k: v for k, v in raw.items() if not isinstance(v, str)}
+
+
+def _params_to_input_schema(sig: inspect.Signature, fn: Any = None) -> dict[str, Any]:
+    """Convert a function's signature to a §4 input JSON-Schema object.
+
+    - Excludes the first parameter named 'ctx' (§4 inv:ctx-name-only).
+    - Each remaining parameter becomes a property.
+    - Parameters without defaults are required.
+    - Resolves PEP-563 stringized annotations via ``typing.get_type_hints`` when
+      ``fn`` is supplied, so that ``Decimal``/``datetime``/``UUID`` type hints work
+      correctly even in modules that use ``from __future__ import annotations``.
+
+    Args:
+        sig: The ``inspect.Signature`` for the function.
+        fn:  Optional callable — used to resolve string annotations (PEP 563).
+    """
+    props: dict[str, Any] = {}
+    required: list[str] = []
+
+    # Resolve annotations via get_type_hints when a callable is supplied.
+    # This handles both eager annotations (fn.__annotations__ = real types) and
+    # PEP-563 stringized annotations (fn.__annotations__ = strings).
+    resolved_hints: dict[str, Any] = _resolve_hints(fn) if fn is not None else {}
+
+    params = list(sig.parameters.values())
+    # Drop 'self' for bound methods.
+    if params and params[0].name == "self":
+        params = params[1:]
+    # Drop 'ctx' first-param (name-only match, SPEC §4).
+    if params and params[0].name == "ctx":
+        params = params[1:]
+    # Drop *args / **kwargs from schema (not serialisable as named params).
+    named = [
+        p for p in params
+        if p.kind not in (
+            inspect.Parameter.VAR_POSITIONAL,
+            inspect.Parameter.VAR_KEYWORD,
+        )
+    ]
+
+    for p in named:
+        # Prefer the get_type_hints-resolved annotation; fall back to the
+        # raw signature annotation (which may be a string for PEP-563 modules).
+        ann = resolved_hints.get(p.name, p.annotation)
+        if ann is inspect.Parameter.empty or ann is None:
+            schema_frag: dict[str, Any] = {}
+        else:
+            schema_frag = _py_type_hint_to_schema(ann)
+        props[p.name] = schema_frag
+        if p.default is inspect.Parameter.empty:
+            required.append(p.name)
+
+    result: dict[str, Any] = {"type": "object", "properties": props}
+    if required:
+        result["required"] = required
+    return result
+
+
+def _return_to_output_schema(sig: inspect.Signature, fn: Any = None) -> dict[str, Any]:
+    """Convert a function's return annotation to a §4 output JSON-Schema.
+
+    Promise<T> / Awaitable[T] and generator wrappers are unwrapped to T.
+    Resolves PEP-563 stringized return annotations via ``typing.get_type_hints``
+    when ``fn`` is supplied.
+
+    Args:
+        sig: The ``inspect.Signature`` for the function.
+        fn:  Optional callable — used to resolve string annotations (PEP 563).
+    """
+    # Prefer get_type_hints-resolved return annotation.
+    resolved_hints: dict[str, Any] = _resolve_hints(fn) if fn is not None else {}
+    ann = resolved_hints.get("return", sig.return_annotation)
+    if ann is inspect.Parameter.empty:
+        return {}
+    # Unwrap Coroutine / Awaitable / Generator wrappers.
+    import typing
+
+    origin = getattr(ann, "__origin__", None)
+    # AsyncGenerator[T, ...] → T
+    if origin is not None:
+        name = getattr(origin, "__name__", "") or getattr(origin, "_name", "")
+        if name in ("AsyncGenerator", "Generator", "AsyncIterator", "Iterator"):
+            args = getattr(ann, "__args__", ())
+            if args:
+                return _py_type_hint_to_schema(args[0])
+    # Coroutine[_, _, T] → T
+    try:
+        if origin is not None and hasattr(origin, "__name__") and origin.__name__ == "Coroutine":
+            args = getattr(ann, "__args__", ())
+            if len(args) >= 3:
+                return _py_type_hint_to_schema(args[2])
+    except Exception:
+        pass
+
+    return _py_type_hint_to_schema(ann)
+
+
+# ---------------------------------------------------------------------------
+# Operation building
+# ---------------------------------------------------------------------------
+
+def _is_streaming(fn: Any) -> bool:
+    """True if the function is an async generator or regular generator."""
+    return inspect.isasyncgenfunction(fn) or inspect.isgeneratorfunction(fn)
+
+
+def _is_async(fn: Any) -> bool:
+    """True if the function is a coroutine function or async generator."""
+    return inspect.iscoroutinefunction(fn) or inspect.isasyncgenfunction(fn)
+
+
+def _is_serialisable(value: Any) -> bool:
+    """Cheaply check if a non-callable export can be JSON-serialised."""
+    try:
+        json.dumps(value)
+        return True
+    except (TypeError, ValueError):
+        return False
+
+
+def _derive_id(namespace_raw: str, path: list[dict[str, Any]]) -> str:
+    """Derive the canonical §4 id: namespace/path (kebab-joined).
+
+    Mirrors the TS implementation: namespace slug + "/" + path segments joined
+    with "/" where each segment is tokenised words joined with "-".
+    """
+    def seg_to_slug(s: dict[str, Any]) -> str:
+        return "-".join(s["words"])
+
+    parts = [namespace_raw] + [seg_to_slug(s) for s in path]
+    return "/".join(parts)
+
+
+def extract_module(
+    module_path: str,
+    namespace: str | None = None,
+) -> list[dict[str, Any]]:
+    """Extract Operation descriptors from a Python source file.
+
+    Args:
+        module_path: Absolute or relative path to the .py file.
+        namespace: The §4 namespace slug. Defaults to the file stem.
+
+    Returns:
+        List of Operation dicts (§4 JSON shape).
+
+    Note:
+        Supports modules that use ``from __future__ import annotations`` (PEP 563).
+        The module is registered in ``sys.modules`` before ``exec_module`` so that
+        ``dataclasses._is_type`` (which calls ``sys.modules.get(cls.__module__)``)
+        can resolve the module and ``typing.get_type_hints`` can resolve stringized
+        annotations to real type objects.
+    """
+    path = Path(module_path).resolve()
+    file_stem = path.stem
+
+    # Normalise the file segment: strip extension, dots/underscores → hyphens (SPEC §5).
+    norm_file = _normalise_filename(path.name)
+    ns_raw = namespace or norm_file
+    namespace_seg = _seg(ns_raw)
+
+    # Dynamically import the module.
+    spec = importlib.util.spec_from_file_location("_apigen_target_", str(path))
+    if spec is None or spec.loader is None:
+        raise ImportError(f"Cannot load module from {path}")
+    mod = importlib.util.module_from_spec(spec)
+
+    # CANONICAL PATTERN (importlib docs): register in sys.modules BEFORE exec_module
+    # so that:
+    #   1. dataclasses._is_type() can resolve cls.__module__ via sys.modules — without
+    #      this, @dataclass definitions crash with AttributeError when the module uses
+    #      `from __future__ import annotations` (PEP 563).
+    #   2. typing.get_type_hints() can resolve string annotations back to real types
+    #      (it needs the module's globals to be reachable via sys.modules).
+    sys.modules[spec.name] = mod
+    try:
+        spec.loader.exec_module(mod)  # type: ignore[union-attr]
+    except Exception:
+        # On failure, remove the partial module so subsequent imports are not poisoned.
+        sys.modules.pop(spec.name, None)
+        raise
+
+    operations: list[dict[str, Any]] = []
+
+    # Collect public names (not starting with '_').
+    # Use __all__ if defined, otherwise all non-private names.
+    if hasattr(mod, "__all__"):
+        export_names: list[str] = list(mod.__all__)
+    else:
+        export_names = [
+            name for name in dir(mod)
+            if not name.startswith("_")
+        ]
+
+    # For index files (stem == 'index' or 'main'), drop the file segment from path (SPEC §5).
+    include_file_seg = file_stem.lower() not in ("index", "main")
+
+    # Determine the module's __name__ as assigned by importlib — we use this to
+    # filter out names that were *imported into* the module (e.g. `from decimal
+    # import Decimal`) from names that were *defined in* the module.  An imported
+    # callable's `__module__` attribute points to its defining package, not to
+    # `_apigen_target_`; a locally-defined function has `__module__ == mod.__name__`.
+    mod_name: str = mod.__name__  # '_apigen_target_'
+
+    for name in export_names:
+        try:
+            value = getattr(mod, name)
+        except AttributeError:
+            continue
+
+        # -----------------------------------------------------------------
+        # FIX: skip names that are imported from another module, not defined
+        # here.  This prevents `from decimal import Decimal` or
+        # `from datetime import datetime` from being extracted as operations.
+        #
+        # Rule: a callable or type whose `__module__` differs from the loaded
+        # module's `__name__` is an *import* — skip it.
+        #
+        # Non-callables (module-level constants) have no `__module__`, so
+        # they bypass this check and fall through to the serialisability gate
+        # below (CONSTANT = 42 → kind='query').
+        #
+        # Exception: when the module explicitly declares `__all__`, trust the
+        # author's intent and do NOT filter by __module__ (they may re-export
+        # third-party callables deliberately).
+        # -----------------------------------------------------------------
+        if not hasattr(mod, "__all__"):
+            obj_module = getattr(value, "__module__", None)
+            if obj_module is not None and obj_module != mod_name:
+                # Imported callable or type — not defined in this module.
+                continue
+
+        # Build the §4 path.
+        if include_file_seg:
+            path_segs = [_seg(norm_file), _seg(name)]
+        else:
+            path_segs = [_seg(name)]
+
+        op_id = _derive_id(ns_raw, path_segs)
+
+        if callable(value) and not isinstance(value, type):
+            # --- action (callable export) ---
+            try:
+                sig = inspect.signature(value)
+            except (ValueError, TypeError):
+                sig = inspect.Signature()
+
+            streaming = _is_streaming(value)
+            is_async = _is_async(value)
+            # Pass `value` so PEP-563 stringized annotations are resolved via
+            # typing.get_type_hints before JSON-schema inference.
+            input_schema = _params_to_input_schema(sig, fn=value)
+            output_schema = _return_to_output_schema(sig, fn=value)
+
+            op: dict[str, Any] = {
+                "id": op_id,
+                "host": "python",
+                "namespace": namespace_seg,
+                "path": path_segs,
+                "kind": "action",
+                "async": is_async,
+                "streaming": streaming,
+                "safe": False,
+                "input": input_schema,
+                "output": output_schema,
+                "envelope": {},
+                "typeText": None,
+            }
+            operations.append(op)
+
+        elif isinstance(value, type):
+            # Class export → constructor + public instance methods (SPEC §10).
+            # Constructor.
+            try:
+                sig = inspect.signature(value.__init__)
+            except (ValueError, TypeError):
+                sig = inspect.Signature()
+
+            input_schema = _params_to_input_schema(sig, fn=value.__init__)
+            ctor_path = path_segs + [] if include_file_seg else path_segs
+            ctor_op: dict[str, Any] = {
+                "id": op_id,
+                "host": "python",
+                "namespace": namespace_seg,
+                "path": path_segs,
+                "kind": "constructor",
+                "async": False,
+                "streaming": False,
+                "safe": False,
+                "input": input_schema,
+                "output": {"type": "object", "properties": {"instanceId": {"type": "string"}}},
+                "envelope": {},
+                "typeText": None,
+            }
+            operations.append(ctor_op)
+
+        elif not callable(value):
+            # Serialisable const → query.
+            if _is_serialisable(value):
+                # Infer schema from value type.
+                val_type = type(value)
+                type_map = {str: "string", int: "integer", float: "number", bool: "boolean",
+                            list: "array", dict: "object"}
+                json_type = type_map.get(val_type, "object")
+                output_schema = {"type": json_type}
+
+                query_op: dict[str, Any] = {
+                    "id": op_id,
+                    "host": "python",
+                    "namespace": namespace_seg,
+                    "path": path_segs,
+                    "kind": "query",
+                    "async": False,
+                    "streaming": False,
+                    "safe": True,
+                    "input": {"type": "object", "properties": {}},
+                    "output": output_schema,
+                    "envelope": {},
+                    "typeText": None,
+                }
+                operations.append(query_op)
+            else:
+                warnings.warn(
+                    f"apigen-python extractor: skipping non-serialisable, non-callable export '{name}'",
+                    stacklevel=2,
+                )
+
+    return operations
+
+
+# ---------------------------------------------------------------------------
+# CLI entry point (subprocess protocol: JSON on stdout)
+# ---------------------------------------------------------------------------
+
+def _main() -> None:
+    import argparse
+
+    parser = argparse.ArgumentParser(
+        description="Extract apigen Operation descriptors from a Python source file."
+    )
+    parser.add_argument("source", help="Path to the Python source file")
+    parser.add_argument(
+        "--namespace", "-n", default=None,
+        help="Namespace slug (default: normalised file stem)"
+    )
+    parser.add_argument(
+        "--emit-json", action="store_true", required=True,
+        help=(
+            "Required. Emit the extracted Operation[] as JSON on stdout. "
+            "This is the stable extract-only half of the apigen-serve-core "
+            "two-phase extract/serve protocol (see module docstring) — "
+            "required rather than implicit so this CLI's default behavior "
+            "is never silently relied upon by a caller that didn't ask for "
+            "it."
+        ),
+    )
+    args = parser.parse_args()
+
+    try:
+        ops = extract_module(args.source, namespace=args.namespace)
+        print(json.dumps(ops, indent=2))
+    except Exception as exc:
+        print(f"error: {exc}", file=sys.stderr)
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    _main()