machinaos 0.0.78 → 0.0.80

package/{machina → cli}/commands/start.py

@@ -15,19 +15,25 @@ import asyncio
 import json
 import os
 import subprocess
-import sys
 import time
 from pathlib import Path
 
 import typer
 
-from machina.colors import console
-from machina.config import load_config
-from machina.platform_ import IS_WINDOWS, IS_WSL, platform_name, project_root
-from machina.buildenv import validate_build, venv_python
-from machina.ports import kill_port
-from machina.run import which_argv
-from machina.supervisor import Manager, RestartPolicy, ServiceSpec
+from cli._common import build_backend_spec, error_block, free_all_ports, preflight
+from cli.colors import console
+from cli.platform_ import (
+    IS_WINDOWS,
+    IS_WSL,
+    platform_name,
+    server_dir,
+    server_venv,
+    static_client_script,
+)
+from cli.buildenv import validate_build
+from cli.run import uv_run
+from cli.supervisor import Manager, ServiceSpec
+from cli.commands._temporal_specs import temporal_specs
 
 
 def _sqlalchemy_preflight(root: Path) -> None:
@@ -38,28 +44,55 @@ def _sqlalchemy_preflight(root: Path) -> None:
     calls even after exclusions are added. Catching it here gives a
     clear remediation message instead of letting uvicorn hang silently.
     See ``docs-internal/errors.md`` #1 / #1a.
+
+    Runs the probe inside the server ``.venv`` through ``uv run`` with
+    ``cwd=server/`` so uv discovers ``server/pyproject.toml`` (the only
+    place uv-managed Python deps live -- root ``pyproject.toml`` is the
+    standalone CLI package). Same environment the supervised services
+    will use, no path-to-interpreter logic on this side.
     """
-    py = venv_python(root)
-    if py is None:
-        return
     started = time.monotonic()
     try:
         subprocess.run(
-            [str(py), "-c", "import sqlalchemy"],
+            uv_run("python", "-c", "import sqlalchemy"),
+            cwd=str(server_dir(root)),
             timeout=15,
             check=True,
             capture_output=True,
         )
-    except (subprocess.TimeoutExpired, subprocess.CalledProcessError):
+    except (subprocess.TimeoutExpired, subprocess.CalledProcessError) as exc:
         elapsed = time.monotonic() - started
-        console.print(f"[red]Error: Python venv health check failed ({elapsed:.1f}s).[/]")
-        console.print("  sqlalchemy import hung or crashed.")
-        console.print("  Likely cause: Windows Defender scan cache or stale kernel state.")
-        console.print("  Fix options:")
-        console.print("    1. Restart-Service WinDefend  (admin PowerShell)")
-        console.print("    2. Reboot the machine")
-        console.print(f"    3. Add {root / 'server' / '.venv'} to Defender exclusions")
-        console.print("  See docs-internal/errors.md #1 / #1a for details.")
+        stderr_tail = ""
+        if isinstance(exc, subprocess.CalledProcessError) and exc.stderr:
+            stderr_bytes = (
+                exc.stderr if isinstance(exc.stderr, bytes) else exc.stderr.encode()
+            )
+            stderr_tail = stderr_bytes.decode(errors="replace").strip()
+        details: list[str] = ["sqlalchemy import hung or crashed."]
+        if stderr_tail:
+            details.append(f"subprocess stderr: {stderr_tail}")
+        if IS_WINDOWS:
+            details.extend(
+                [
+                    "Likely cause on Windows: Defender scan cache or stale kernel state.",
+                    "Fix options:",
+                    "  1. Restart-Service WinDefend  (admin PowerShell)",
+                    "  2. Reboot the machine",
+                    f"  3. Add {server_venv(root)} to Defender exclusions",
+                ]
+            )
+        else:
+            details.extend(
+                [
+                    f"Check that {server_venv(root)} exists and is populated.",
+                    "Run `machina build` to recreate the server venv.",
+                ]
+            )
+        details.append("See docs-internal/errors.md #1 / #1a for details.")
+        error_block(
+            f"Python venv health check failed ({elapsed:.1f}s).",
+            details,
+        )
         raise typer.Exit(code=1)
     elapsed = time.monotonic() - started
     if elapsed > 5.0:
@@ -69,18 +102,15 @@ def _sqlalchemy_preflight(root: Path) -> None:
         )
 
 
-def _temporal_running() -> bool:
-    """Check whether the temporal server is already up via the ``temporal`` CLI."""
-    try:
-        out = subprocess.run(
-            which_argv(["temporal", "status"]),
-            capture_output=True,
-            text=True,
-            timeout=5,
-        )
-        return any(token in out.stdout for token in ("running", "UP", "up"))
-    except (FileNotFoundError, subprocess.TimeoutExpired):
-        return False
+def _temporal_running(cfg) -> bool:
+    """Check whether a Temporal frontend is already listening on the
+    configured gRPC port. TCP probe instead of spawning a CLI -- works
+    without the legacy ``temporal-server`` npm wrapper (we install our
+    own binary via pooch at first boot of the supervised
+    ``TemporalServerRuntime``)."""
+    from cli.tcp import probe_tcp_port_sync
+
+    return probe_tcp_port_sync(cfg.temporal_port)
 
 
 def _read_version(root: Path) -> str:
@@ -92,61 +122,38 @@ def _read_version(root: Path) -> str:
 
 
 def _build_specs(root: Path, cfg, *, temporal_running: bool) -> list[ServiceSpec]:
-    static_client = root / "scripts" / "serve-client.js"
-    server_dir = root / "server"
-
     # Match the existing behaviour: production uses python:start
     # (bind 127.0.0.1) on Windows/WSL, python:daemon (bind 0.0.0.0)
     # on POSIX.
     backend_host = "127.0.0.1" if (IS_WINDOWS or IS_WSL) else "0.0.0.0"
-    backend_argv = [
-        "uv", "run", "uvicorn", "main:app",
-        "--host", backend_host,
-        "--port", str(cfg.backend_port),
-        "--log-level", "warning",
-    ]
 
     specs: list[ServiceSpec] = [
         ServiceSpec(
             name="client",
-            argv=["node", str(static_client)],
+            argv=["node", str(static_client_script(root))],
             cwd=root,
             ready_port=cfg.client_port,
         ),
-        ServiceSpec(
-            name="server",
-            argv=backend_argv,
-            cwd=server_dir,
-            ready_port=cfg.backend_port,
-        ),
+        build_backend_spec(cfg, host=backend_host, root=root),
     ]
     if not temporal_running:
-        specs.append(
-            ServiceSpec(
-                name="temporal",
-                argv=["temporal", "api"],
-                cwd=root,
-                ready_port=cfg.temporal_port,
-            )
-        )
+        specs.extend(temporal_specs(root, cfg))
     return specs
 
 
 def start_command() -> None:
-    root = project_root()
-    cfg = load_config()
+    cfg, root = preflight()
     os.environ.setdefault("PYTHONUTF8", "1")
 
     validate_build(root, require_client_dist=True)
     _sqlalchemy_preflight(root)
 
-    temporal_running = _temporal_running()
+    temporal_running = _temporal_running(cfg)
     if temporal_running:
         console.print("[dim]Temporal already running, skipping[/]")
 
     console.log("Freeing ports...")
-    for port in cfg.all_ports:
-        kill_port(port)
+    free_all_ports(cfg)
     console.log("Ports ready")
 
     version = _read_version(root)

package/{machina → cli}/commands/stop.py

@@ -11,30 +11,27 @@ import time
 
 import typer
 
-from machina.colors import console
-from machina.config import load_config
-from machina.platform_ import platform_name, project_root
-from machina.ports import (
+from cli._common import free_all_ports, preflight
+from cli.colors import console
+from cli.platform_ import platform_name
+from cli.ports import (
     kill_by_pattern,
     kill_orphaned_machina_processes,
-    kill_port,
 )
 
 
 def stop_command() -> None:
-    cfg = load_config()
-    root = project_root()
+    cfg, root = preflight()
 
     console.print()
     console.print("[bold]Stopping MachinaOS services...[/]")
     console.print(f"Platform: {platform_name()}")
     console.print(f"Ports:    {', '.join(str(p) for p in cfg.all_ports)}")
-    console.print(f"Temporal: enabled" if cfg.temporal_enabled else "Temporal: disabled")
+    console.print("Temporal: enabled" if cfg.temporal_enabled else "Temporal: disabled")
     console.print()
 
     all_stopped = True
-    for port in cfg.all_ports:
-        result = kill_port(port)
+    for port, result in zip(cfg.all_ports, free_all_ports(cfg)):
         status = "[green]\\[OK][/]" if result.port_free else "[red]\\[!!][/]"
         if result.port_free:
             if result.killed_pids:

package/{machina → cli}/commands/version.py

@@ -13,9 +13,9 @@ from pathlib import Path
 
 import typer
 
-from machina.colors import console
-from machina.platform_ import project_root
-from machina.run import capture
+from cli.colors import console
+from cli.platform_ import project_root
+from cli.run import capture
 
 
 app = typer.Typer(
@@ -36,7 +36,9 @@ def _git_describe(root: Path) -> str | None:
         return described
     listed = capture(["git", "tag", "-l", "--sort=-version:refname"], cwd=root)
     if listed:
-        return next((line.strip() for line in listed.splitlines() if line.strip()), None)
+        return next(
+            (line.strip() for line in listed.splitlines() if line.strip()), None
+        )
     return None
 
 
@@ -58,7 +60,9 @@ def _update_package_json(path: Path, new_version: str) -> bool:
 
 
 @app.command("sync", help="Sync package.json versions from latest git tag.")
-def sync(tag: str | None = typer.Argument(None, help="Git tag to use (defaults to latest).")) -> None:
+def sync(
+    tag: str | None = typer.Argument(None, help="Git tag to use (defaults to latest)."),
+) -> None:
     root = project_root()
     resolved_tag = tag or _git_describe(root)
     if not resolved_tag:
@@ -67,7 +71,9 @@ def sync(tag: str | None = typer.Argument(None, help="Git tag to use (defaults t
 
     version = _tag_to_version(resolved_tag)
     if not _VALID_VERSION.match(version):
-        console.print(f"[red]Error: Invalid version from tag {resolved_tag!r}: {version!r}[/]")
+        console.print(
+            f"[red]Error: Invalid version from tag {resolved_tag!r}: {version!r}[/]"
+        )
         console.print("Expected semver format (e.g. v0.0.11 or 0.0.11).")
         raise typer.Exit(code=1)
 

package/cli/config.py

@@ -0,0 +1,170 @@
+"""Project configuration -- typed view over ``.env.dev`` / ``.env`` / ``.env.template``.
+
+Stdlib-only env-var loader. Three env files, lowest precedence first:
+
+  1. ``.env.template`` — canonical production defaults. Ships with every
+     install. ``DATA_DIR=~/.machina`` (user home) is the daemon
+     behaviour: ``machina start`` / ``machina daemon`` use these.
+  2. ``.env`` — user overrides (created by ``scripts/postinstall.js`` on
+     ``npm install -g machinaos``). Gitignored.
+  3. ``.env.dev`` — dev-mode overrides. Loaded ONLY by ``machina dev``
+     via :func:`load_dev_overrides`. Pins ``DATA_DIR=.machina`` so
+     per-checkout dev state lives at ``<repo>/.machina/`` instead of
+     ``~/.machina/``. Committed to git.
+
+Process env (``os.environ``) wins over all three. Merged file values
+are pushed into ``os.environ`` via ``setdefault`` so downstream
+consumers that read ``os.environ`` directly see the same view as this
+module.
+
+No Python-side hardcoded defaults: env files are the single source of
+truth. If ``.env.template`` is missing (broken install), we error out
+loudly with a remediation hint rather than silently substituting
+duplicate hardcoded values that drift from the shipped template.
+"""
+
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass
+from functools import lru_cache
+from pathlib import Path
+
+from cli.platform_ import project_root
+
+
+def _load_env_file(path: Path) -> dict[str, str]:
+    out: dict[str, str] = {}
+    if not path.exists():
+        return out
+    for raw in path.read_text(encoding="utf-8").splitlines():
+        line = raw.strip()
+        if not line or line.startswith("#") or "=" not in line:
+            continue
+        key, _, value = line.partition("=")
+        value = value.strip()
+        if len(value) >= 2 and value[0] == value[-1] and value[0] in ("'", '"'):
+            value = value[1:-1]
+        out[key.strip()] = value
+    return out
+
+
+def _require(env: dict[str, str], key: str) -> str:
+    raw = env.get(key)
+    if raw is None:
+        raise KeyError(
+            f"{key} missing from .env / .env.template -- "
+            "reinstall (``npm install -g machinaos``) or restore the template."
+        )
+    return raw
+
+
+def _int(env: dict[str, str], key: str) -> int:
+    return int(_require(env, key))
+
+
+def _bool(env: dict[str, str], key: str) -> bool:
+    return _require(env, key).strip().lower() in ("1", "true", "yes", "on")
+
+
+@dataclass(frozen=True)
+class Config:
+    """Typed view over the merged ``.env.template`` + ``.env`` + process
+    env. No field defaults -- every value originates in the env files so
+    the published shape matches what users see (and can edit) in their
+    ``.env``."""
+
+    client_port: int
+    backend_port: int
+    whatsapp_port: int
+    nodejs_port: int
+    temporal_address: str
+    temporal_enabled: bool
+    temporal_ui_port: int
+
+    @property
+    def temporal_port(self) -> int:
+        return int(self.temporal_address.rsplit(":", 1)[-1])
+
+    @property
+    def all_ports(self) -> list[int]:
+        # Ports the supervisor frees pre-spawn and verifies are released
+        # on ``machina stop``. ``temporal_port`` (gRPC) and
+        # ``temporal_ui_port`` are bound by the same ``temporal server
+        # start-dev`` process, so killing one kills both — listing both
+        # only matters when cleaning up stale orphans on either port.
+        return [
+            self.client_port,
+            self.backend_port,
+            self.whatsapp_port,
+            self.nodejs_port,
+            self.temporal_port,
+            self.temporal_ui_port,
+        ]
+
+
+def load_dev_overrides(root: Path | None = None) -> None:
+    """Layer ``.env.dev`` on top of ``.env.template`` + ``.env``.
+
+    Called by ``machina dev`` BEFORE :func:`load_config` so dev-only env
+    vars (notably ``DATA_DIR=.machina``, the per-checkout dev state
+    root) land in ``os.environ`` first. ``load_config`` then sees them
+    via its own ``setdefault`` pass and skips its template fallbacks.
+
+    Uses ``setdefault`` so an explicit shell override
+    (``DATA_DIR=/foo machina dev``) still wins. Missing ``.env.dev``
+    file is a no-op — production-style invocations never trigger this.
+
+    The daemon path (``machina start`` / ``machina daemon``) does not
+    call this, so daemon installs keep the ``.env.template`` default
+    of ``DATA_DIR=~/.machina``.
+    """
+    root = root or project_root()
+    for key, value in _load_env_file(root / ".env.dev").items():
+        os.environ.setdefault(key, value)
+
+
+@lru_cache(maxsize=1)
+def load_config(root: Path | None = None) -> Config:
+    """Load + merge ``.env.template`` (baseline) and ``.env`` (overrides).
+
+    Precedence (low -> high): ``.env.template`` < ``.env`` < ``os.environ``.
+
+    Merged file values are pushed into ``os.environ`` so any downstream
+    ``os.environ.get(...)`` lookup sees the same view -- without
+    overwriting existing process env vars (matches python-dotenv's
+    default ``override=False`` semantics).
+    """
+    root = root or project_root()
+    template = root / ".env.template"
+    user = root / ".env"
+
+    if not template.exists():
+        raise FileNotFoundError(
+            f".env.template not found at {template}. "
+            "Reinstall (``npm install -g machinaos``) or restore the template "
+            "from the source tree."
+        )
+
+    # Layer: .env.template (canonical defaults) <- .env (user override)
+    merged: dict[str, str] = _load_env_file(template)
+    merged.update(_load_env_file(user))
+
+    # Push file values into the process environment so downstream
+    # ``os.environ.get(...)`` reads see them. Process env still wins --
+    # only keys missing from the process environment are inserted.
+    for key, value in merged.items():
+        os.environ.setdefault(key, value)
+
+    # Final view layers process env on top of file env.
+    env = {**merged, **os.environ}
+
+    return Config(
+        client_port=_int(env, "VITE_CLIENT_PORT"),
+        backend_port=_int(env, "PYTHON_BACKEND_PORT"),
+        whatsapp_port=_int(env, "WHATSAPP_RPC_PORT"),
+        nodejs_port=_int(env, "NODEJS_EXECUTOR_PORT"),
+        temporal_address=_require(env, "TEMPORAL_SERVER_ADDRESS"),
+        temporal_enabled=_bool(env, "TEMPORAL_ENABLED"),
+        temporal_ui_port=_int(env, "TEMPORAL_UI_PORT"),
+    )

package/cli/platform_.py

@@ -0,0 +1,169 @@
+"""Platform / shell / venv detection + canonical path-prefix helpers.
+
+All on-disk locations the CLI needs (server tree, the server's uv venv,
+client static script, build-output index, OS-native user dirs) are
+derived from one prefix -- :func:`project_root` -- via the helpers
+below. No callsite composes ``root / "server"`` / ``root / ".venv"``
+inline; every consumer imports the named helper. This keeps layout
+knowledge centralised so a future repo rename / directory relocation
+is a single-file edit.
+
+The CLI is **independent of the server's uv environment**: ``machina``
+is installable via pipx / system Python without involving uv, and
+shells out to ``uv run --no-sync ...`` (via :func:`cli.run.uv_run`)
+for any server-side command. :func:`server_venv` exists so build /
+clean / preflight code can check or report that venv's location
+without composing it inline.
+
+``server_venv`` respects uv's documented ``UV_PROJECT_ENVIRONMENT``
+override (https://docs.astral.sh/uv/concepts/projects/config/) so
+power users with non-default venv locations still work.
+
+OS-native user dirs (data, cache, config, log) are derived through
+``platformdirs`` (https://pypi.org/project/platformdirs/) -- the same
+library the server uses for its pooch binary cache -- so the two halves
+of the project agree on per-OS conventions.
+"""
+
+from __future__ import annotations
+
+import os
+import sys
+from pathlib import Path
+
+
+IS_WINDOWS = sys.platform == "win32"
+IS_MACOS = sys.platform == "darwin"
+IS_LINUX = sys.platform.startswith("linux")
+IS_WSL = IS_LINUX and ("WSL_DISTRO_NAME" in os.environ or "WSLENV" in os.environ)
+IS_GIT_BASH = IS_WINDOWS and bool(
+    os.environ.get("MSYSTEM") or "bash" in (os.environ.get("SHELL") or "")
+)
+
+
+def platform_name() -> str:
+    """Human-readable platform label."""
+    if IS_GIT_BASH:
+        return "Git Bash"
+    if IS_WSL:
+        return "WSL"
+    if IS_WINDOWS:
+        return "Windows"
+    if IS_MACOS:
+        return "macOS"
+    return "Linux"
+
+
+def project_root() -> Path:
+    """Resolve the project root from a module under ``cli/``.
+
+    Layout: ``<project_root>/cli/<this_file>`` -> parents[1].
+    """
+    return Path(__file__).resolve().parents[1]
+
+
+# ---------------------------------------------------------------------------
+# Path-prefix helpers -- canonical layout derived from ``project_root``.
+# ---------------------------------------------------------------------------
+
+
+def _root(root: Path | None) -> Path:
+    return root if root is not None else project_root()
+
+
+def server_dir(root: Path | None = None) -> Path:
+    """The ``server/`` directory -- the Python backend + plugin tree.
+    ``server/`` is its own standalone uv project (its ``pyproject.toml``
+    declares the dependency closure ``uv sync`` resolves)."""
+    return _root(root) / "server"
+
+
+def server_venv(root: Path | None = None) -> Path:
+    """The server's uv-managed project environment directory.
+
+    ``server/`` is a standalone uv project, so uv places its venv at
+    ``server/.venv`` by default per https://docs.astral.sh/uv/concepts/
+    projects/layout/. The ``UV_PROJECT_ENVIRONMENT`` env var overrides
+    this -- relative overrides resolve against ``server/`` (matching
+    uv's own behaviour), absolute paths are used verbatim.
+
+    The CLI itself does NOT live in this venv; ``machina`` is
+    independently installable (pipx / system python). This helper
+    exists so build / clean / preflight code can check or report the
+    server venv's location without anyone composing the path inline.
+    """
+    base = server_dir(root)
+    override = os.environ.get("UV_PROJECT_ENVIRONMENT")
+    if override:
+        path = Path(override)
+        return path if path.is_absolute() else base / path
+    return base / ".venv"
+
+
+def node_modules_dir(root: Path | None = None) -> Path:
+    """The pnpm/npm-installed dependency tree at the workspace root."""
+    return _root(root) / "node_modules"
+
+
+def client_dist_entry(root: Path | None = None) -> Path:
+    """The Vite-built client entrypoint -- proof that the client side
+    of ``machina build`` completed."""
+    return _root(root) / "client" / "dist" / "index.html"
+
+
+def static_client_script(root: Path | None = None) -> Path:
+    """The Node.js static-server script that serves the built client
+    (used by ``machina start``)."""
+    return _root(root) / "scripts" / "serve-client.js"
+
+
+# ---------------------------------------------------------------------------
+# OS-native user directories via ``platformdirs``
+# (https://pypi.org/project/platformdirs/). ``platformdirs`` is imported
+# inside each helper, NOT at module level, so the rest of
+# ``cli.platform_`` (path-prefix helpers, project_root, the platform
+# booleans) keeps loading even when the wheel hasn't been installed
+# yet -- the recovery-verb scenario (``machina clean`` against a
+# half-broken env). Callers that actually need a user dir trigger the
+# import on first call; if missing, the natural ``ModuleNotFoundError``
+# surfaces at that callsite, not at every CLI import.
+# ---------------------------------------------------------------------------
+
+_APP_NAME = "MachinaOs"
+
+
+def user_data_dir() -> Path:
+    """User data directory.
+
+    Honours the project's ``DATA_DIR`` env override (see
+    ``.env.template``) -- the convention shared with the server's
+    ``core.paths.machina_root``. Otherwise delegates to platformdirs.
+    """
+    override = os.environ.get("DATA_DIR")
+    if override:
+        return Path(override).expanduser()
+    import platformdirs
+
+    return platformdirs.user_data_path(_APP_NAME, appauthor=_APP_NAME)
+
+
+def user_cache_dir() -> Path:
+    """User cache directory (downloaded binaries / pooch caches).
+    Matches ``server/core/paths.py::packages_dir``."""
+    import platformdirs
+
+    return platformdirs.user_cache_path(_APP_NAME, appauthor=_APP_NAME)
+
+
+def user_config_dir() -> Path:
+    """User config directory."""
+    import platformdirs
+
+    return platformdirs.user_config_path(_APP_NAME, appauthor=_APP_NAME)
+
+
+def user_log_dir() -> Path:
+    """User log directory."""
+    import platformdirs
+
+    return platformdirs.user_log_path(_APP_NAME, appauthor=_APP_NAME)