@ai-matrx/desktop-protocol 0.7.0 → 0.11.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ai-matrx/desktop-protocol",
-  "version": "0.7.0",
+  "version": "0.11.0",
   "description": "The one wire contract between Matrx 2 (the desktop app) and every client that drives it — phone, web app, extension, server: zod schemas for the envelope and every cap.op, the 16-byte binary frame codec, a reconnecting request/stream/credit/reattach client, React hooks, and a generated JSON Schema + Pydantic twin that CI keeps byte-identical.",
   "type": "module",
   "main": "./dist/index.cjs",
@@ -28,6 +28,8 @@
   },
   "files": [
     "dist",
+    "generated/desktop-protocol.schema.json",
+    "scripts/gen_pydantic.py",
     "README.md",
     "LICENSE",
     "CHANGELOG.md"

package/scripts/gen_pydantic.py

@@ -0,0 +1,335 @@
+"""Generate the Pydantic v2 twin of @ai-matrx/desktop-protocol from its JSON Schema.
+
+Called by ``pnpm generate`` (scripts/generate.ts) and by aidream's pytest contract check. Stdlib
+only; the one external tool is datamodel-codegen, pinned and run through ``uvx`` so every machine
+and CI job produces byte-identical output (``--formatters builtin``: no black/isort version drift).
+
+Two post-processing steps make the generated module a faithful twin of the zod source:
+
+1. ``StrictModel`` — the generated base of every object model. datamodel-codegen renders an
+   optional, NON-nullable field (zod ``.optional()``) as ``X | None = None``, so Pydantic accepts
+   an explicit ``null`` that zod refuses (parity corpus case 19). The base refuses an explicit
+   ``None`` for every field whose default is ``None`` and whose type is not ``Any``. That rule is
+   exact because generate.ts refuses any schema property that is BOTH optional and nullable.
+2. ``OPS`` / ``EVENTS`` — the op and event registries (from the schema's ``x-ops``/``x-events``),
+   so the server routes and validates by the same table the TypeScript Router uses.
+3. ``SERVICES`` and, per supervised Python service (``x-services``), its handler ``Protocol``:
+   ``<Name>ServiceHandlers`` has one ``async`` method per op, typed with that op's generated
+   models, so ``pyright --strict`` fails a service whose handlers drift from the contract.
+   ``CONTRACT_SHA256`` is the schema file's hash, exchanged in the service hello.
+
+Usage: gen_pydantic.py --schema <desktop-protocol.schema.json> --out <desktop_protocol.py>
+"""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import re
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+
+CODEGEN_VERSION = "0.83.0"
+CODEGEN_ARGS = [
+    "--input-file-type",
+    "jsonschema",
+    "--output-model-type",
+    "pydantic_v2.BaseModel",
+    # The PLAN.md flags (accepted 2026-10-01):
+    "--strict-nullable",
+    "--extra-fields",
+    "forbid",
+    "--enum-field-as-literal",
+    "all",
+    "--use-annotated",
+    # Determinism:
+    "--target-python-version",
+    "3.13",
+    "--disable-timestamp",
+    "--formatters",
+    "builtin",
+]
+
+HEADER = '''# ruff: noqa
+# AUTO-GENERATED — DO NOT EDIT.
+# Source: aidream/apps/shared/desktop-protocol (@ai-matrx/desktop-protocol, zod), via
+#   `pnpm generate` → generated/desktop-protocol.schema.json → datamodel-codegen {version}
+#   → scripts/gen_pydantic.py. `pnpm generate --check` and
+#   aidream/_generated/tests/test_desktop_protocol_contract.py fail on any hand edit.
+'''
+
+STRICT_BASE = '''
+
+class StrictModel(BaseModel):
+    """Generated base of every object model: refuses an explicit ``null`` for an optional field.
+
+    zod ``.optional()`` means "may be absent", never "may be null"; datamodel-codegen still types
+    such a field ``X | None = None``. A field whose default is ``None`` and whose type is not
+    ``Any`` is optional-non-nullable by construction (generate.ts refuses optional+nullable), so
+    an explicit ``None`` for it is rejected exactly as zod rejects it.
+    """
+
+    @model_validator(mode="before")
+    @classmethod
+    def _refuse_explicit_null(cls, data: Any) -> Any:
+        if isinstance(data, dict):
+            for name, field in cls.model_fields.items():
+                key = field.alias or name
+                if (
+                    key in data
+                    and data[key] is None
+                    and not field.is_required()
+                    and field.default is None
+                    and field.annotation is not Any
+                ):
+                    raise ValueError(f"{key}: explicit null is not allowed (omit the field instead)")
+        return data
+
+
+def _js_number(text: str) -> float | int:
+    value = float(text)
+    return int(value) if value.is_integer() and abs(value) <= 9007199254740991 else value
+
+
+def validate_wire[M: BaseModel](model: type[M], raw: str | bytes) -> M:
+    """THE way to validate one wire message in Python — the verdict matches zod's exactly.
+
+    Strict mode (no "2" -> 2 coercion, as zod), plus JavaScript's one number type: JSON.parse
+    makes ``2.0`` the integer 2, so an integral float is an integer here too.
+    """
+    data = json.loads(raw, parse_float=_js_number)
+    return model.model_validate_json(json.dumps(data), strict=True)
+'''
+
+
+def run_codegen(schema: Path, out: Path) -> str:
+    cmd = [
+        "uvx",
+        "--from",
+        f"datamodel-code-generator=={CODEGEN_VERSION}",
+        "datamodel-codegen",
+        "--input",
+        str(schema),
+        "--output",
+        str(out),
+        *CODEGEN_ARGS,
+    ]
+    proc = subprocess.run(cmd, capture_output=True, text=True, check=False)
+    if proc.returncode != 0:
+        sys.stderr.write(proc.stdout + proc.stderr)
+        raise SystemExit(f"datamodel-codegen {CODEGEN_VERSION} failed (exit {proc.returncode})")
+    return out.read_text(encoding="utf-8")
+
+
+def post_process(code: str, schema_doc: dict, contract_sha256: str) -> str:
+    lines = code.splitlines()
+    # Drop codegen's own banner (it names a temp path on some machines); ours replaces it.
+    while lines and lines[0].startswith("#"):
+        lines.pop(0)
+    code = "\n".join(lines).lstrip("\n") + "\n"
+
+    # Imports the base needs.
+    m = re.search(r"^from pydantic import (.+)$", code, flags=re.M)
+    if m is None:
+        raise SystemExit("gen_pydantic: no `from pydantic import` line — codegen output changed shape")
+    names = {n.strip() for n in m.group(1).split(",")} | {"BaseModel", "RootModel", "model_validator"}
+    code = code[: m.start()] + "from pydantic import " + ", ".join(sorted(names)) + code[m.end() :]
+    m = re.search(r"^from typing import (.+)$", code, flags=re.M)
+    if m is None:
+        raise SystemExit("gen_pydantic: no `from typing import` line — codegen output changed shape")
+    names = {n.strip() for n in m.group(1).split(",")} | {"Any", "Literal", "NamedTuple", "Protocol"}
+    code = code[: m.start()] + "from typing import " + ", ".join(sorted(names)) + code[m.end() :]
+
+    # `constr(...)` in a type position (record keys) is a call in a type expression — pyright
+    # rejects it. Rewrite to the equivalent Annotated[str, StringConstraints(...)].
+    code = rewrite_constr(code)
+
+    # Every object model inherits the strict base.
+    count = len(re.findall(r"^class \w+\(BaseModel\):", code, flags=re.M))
+    if count == 0:
+        raise SystemExit("gen_pydantic: no BaseModel classes found — codegen output changed shape")
+    code = re.sub(r"^(class \w+)\(BaseModel\):", r"\1(StrictModel):", code, flags=re.M)
+
+    # Insert the base right after the import block (before the first class).
+    first_class = re.search(r"^class ", code, flags=re.M)
+    assert first_class is not None
+    head, tail = code[: first_class.start()].rstrip("\n"), code[first_class.start() :]
+    if "from __future__ import annotations\n" not in head:
+        raise SystemExit("gen_pydantic: no __future__ import — codegen output changed shape")
+    head = head.replace("from __future__ import annotations\n", "from __future__ import annotations\n\nimport json\n", 1)
+    code = head + "\n" + STRICT_BASE + "\n\n" + tail
+
+    code = code.rstrip("\n") + "\n" + registries(schema_doc) + services(schema_doc, contract_sha256)
+    return HEADER.format(version=CODEGEN_VERSION) + "\n" + code
+
+
+def rewrite_constr(code: str) -> str:
+    out: list[str] = []
+    i = 0
+    found = False
+    while (j := code.find("constr(", i)) != -1:
+        if j > 0 and (code[j - 1].isalnum() or code[j - 1] == "_"):
+            out.append(code[i : j + 7])
+            i = j + 7
+            continue
+        found = True
+        depth, k, quote = 0, j + 6, None
+        while True:
+            ch = code[k]
+            if quote:
+                if ch == "\\":
+                    k += 2
+                    continue
+                if ch == quote:
+                    quote = None
+            elif ch in "'\"":
+                quote = ch
+            elif ch == "(":
+                depth += 1
+            elif ch == ")":
+                depth -= 1
+                if depth == 0:
+                    break
+            k += 1
+        out.append(code[i:j] + "Annotated[str, StringConstraints(" + code[j + 7 : k] + ")]")
+        i = k + 1
+    out.append(code[i:])
+    code = "".join(out)
+    if found:
+        code = re.sub(r"^(from pydantic import .*?)\bconstr, ", r"\1", code, count=1, flags=re.M)
+        code = re.sub(r"^(from pydantic import .*?), constr$", r"\1", code, count=1, flags=re.M)
+        m = re.search(r"^from pydantic import (.+)$", code, flags=re.M)
+        assert m is not None
+        names = {n.strip() for n in m.group(1).split(",")} | {"StringConstraints"}
+        code = code[: m.start()] + "from pydantic import " + ", ".join(sorted(names)) + code[m.end() :]
+    return code
+
+
+def registries(doc: dict) -> str:
+    ops = doc.get("x-ops", {})
+    events = doc.get("x-events", {})
+    out = [
+        "",
+        "",
+        "# ── Registries (from x-ops / x-events) ──────────────────────────────────────────────",
+        "",
+        "",
+        "class OpSpec(NamedTuple):",
+        "    params: type[BaseModel]",
+        "    result: type[BaseModel] | type[RootModel[Any]]",
+        "    kind: Literal['unary', 'stream', 'resource']",
+        "    since: str",
+        "    errors: tuple[str, ...]",
+        "",
+        "",
+        "OPS: dict[str, OpSpec] = {",
+    ]
+    for name, spec in ops.items():
+        errs = ", ".join(repr(e) for e in spec["errors"])
+        errs = f"({errs},)" if len(spec["errors"]) == 1 else f"({errs})"
+        out.append(
+            f"    {name!r}: OpSpec({spec['params']}, {spec['result']}, {spec['kind']!r}, {spec['since']!r}, {errs}),"
+        )
+    out += ["}", "", "EVENTS: dict[str, type[BaseModel]] = {"]
+    for name, model in events.items():
+        out.append(f"    {name!r}: {model},")
+    out.append("}")
+    out += [
+        "",
+        f"#: The newest protocol this contract speaks (\"major.minor\"); announce it, negotiate down.",
+        f"PROTOCOL_VERSION = {doc.get('x-protocol-version', '1.0')!r}",
+        "",
+        "#: Fields added after x.0, by schema location (src/versioning.ts): optional on read; a reader",
+        "#: that needs one checks the peer's negotiated version and says so when the peer is older.",
+        "FIELDS_SINCE: dict[str, dict[str, str]] = {",
+    ]
+    for path, fields in doc.get("x-fields-since", {}).items():
+        out.append(f"    {path!r}: {{{', '.join(f'{k!r}: {v!r}' for k, v in fields.items())}}},")
+    out.append("}")
+    out += [
+        "",
+        "#: 1.0 fields a later minor lets be null, and what an older peer is sent instead.",
+        "NULL_SINCE: dict[str, dict[str, Any]] = {",
+    ]
+    for path, rule in doc.get("x-null-since", {}).items():
+        out.append(f"    {path!r}: {{'since': {rule['since']!r}, 'older_peers_get': {rule['olderPeersGet']!r}}},")
+    out.append("}")
+    return "\n".join(out) + "\n"
+
+
+def services(doc: dict, contract_sha256: str) -> str:
+    out = [
+        "",
+        "",
+        "# ── Supervised Python services (from x-services) ────────────────────────────────────",
+        "",
+        "#: SHA-256 of the JSON Schema this module was generated from. The core sends its own in the",
+        "#: service hello; a service built from a different contract refuses the connection.",
+        f"CONTRACT_SHA256 = {contract_sha256!r}",
+        "",
+        "",
+        "class ServiceOpSpec(NamedTuple):",
+        "    params: type[BaseModel]",
+        "    result: type[BaseModel]",
+        "    #: The handler method that serves the op (the op name after the service prefix).",
+        "    method: str",
+        "",
+        "",
+    ]
+    table: list[str] = ["SERVICES: dict[str, dict[str, ServiceOpSpec]] = {"]
+    progress: list[str] = ["SERVICE_PROGRESS: dict[str, type[BaseModel]] = {"]
+    for service, spec in doc.get("x-services", {}).items():
+        cls = "".join(part.capitalize() for part in service.split("_"))
+        out += [
+            f"class {cls}ServiceCall(Protocol):",
+            f'    """What one `{service}` handler may do besides return its result."""',
+            "",
+            "    @property",
+            "    def cancelled(self) -> bool: ...",
+            "",
+            f"    def progress(self, payload: {spec['progress']}, /) -> None: ...",
+            "",
+            "",
+            f"class {cls}ServiceHandlers(Protocol):",
+            f'    """The `{service}` service implements exactly these ops (pyright --strict holds it to the contract)."""',
+            "",
+        ]
+        table.append(f"    {service!r}: {{")
+        for op, o in spec["ops"].items():
+            prefix = f"{service}."
+            if not op.startswith(prefix):
+                raise SystemExit(f"gen_pydantic: service op {op} does not start with {prefix}")
+            method = op[len(prefix) :]
+            if not method.isidentifier():
+                raise SystemExit(f"gen_pydantic: service op {op} is not a valid method name")
+            out.append(
+                f"    async def {method}(self, params: {o['params']}, call: {cls}ServiceCall, /) -> {o['result']}: ..."
+            )
+            table.append(f"        {op!r}: ServiceOpSpec({o['params']}, {o['result']}, {method!r}),")
+        out += ["", ""]
+        table.append("    },")
+        progress.append(f"    {service!r}: {spec['progress']},")
+    table.append("}")
+    progress.append("}")
+    out += table + ["", ""] + progress
+    return "\n".join(out) + "\n"
+
+
+def main() -> None:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--schema", required=True, type=Path)
+    ap.add_argument("--out", required=True, type=Path)
+    args = ap.parse_args()
+    raw_schema = args.schema.read_bytes()
+    doc = json.loads(raw_schema.decode("utf-8"))
+    with tempfile.TemporaryDirectory() as tmp:
+        raw = run_codegen(args.schema, Path(tmp) / "raw.py")
+    code = post_process(raw, doc, hashlib.sha256(raw_schema).hexdigest())
+    args.out.write_text(code, encoding="utf-8")
+
+
+if __name__ == "__main__":
+    main()