@zerwiz/ymir 0.1.33 → 0.1.35

package/apps/smidja-factory/templates/smidja/smidja_modules/quality.py

@@ -0,0 +1,240 @@
+"""Deterministic lint, typecheck, build, and test blocks.
+
+A known command is not a judgement call. Anything whose invocation you can write
+down belongs here as code — it runs in milliseconds, costs nothing, and returns
+the same answer every time. Agents are for the parts that need reading and
+deciding.
+
+╔══════════════════════════════════════════════════════════════════════════════╗
+║  REPLACE THE PLACEHOLDER COMMANDS BELOW.                                     ║
+║                                                                              ║
+║  Every block ships as an `echo` that exits 0 and announces it is fake. They   ║
+║  are placeholders on purpose: a stamped repo has no way to guess your test    ║
+║  runner, and a wrong-but-plausible command that silently passes is worse      ║
+║  than one that says so out loud.                                             ║
+║                                                                              ║
+║  For each block you want: swap `_placeholder(...)` for the real argv, e.g.    ║
+║      argv=["bun", "test", "apps/web/server.test.ts"]                         ║
+║      argv=["uv", "run", "pytest", "-q"]                                      ║
+║      argv=["npm", "run", "lint"]                                             ║
+║  Delete the blocks you don't need, and drop them from run_quality()'s list.   ║
+║                                                                              ║
+║  Two rules when you write the real command:                                  ║
+║    1. argv LIST, never a shell string — no quoting bugs, no shell injection.  ║
+║    2. Call binaries by BARE NAME. These blocks inherit the operator's         ║
+║       environment (see utils.operator_env), so `bun`, `uv`, `pytest` resolve  ║
+║       exactly as they do in their terminal. Never hard-code an absolute path  ║
+║       like /Users/you/.bun/bin/bun — that bakes your machine into the trace.  ║
+╚══════════════════════════════════════════════════════════════════════════════╝
+"""
+
+from __future__ import annotations
+
+import shlex
+import subprocess
+import time
+from pathlib import Path
+from typing import Callable
+
+from .data_types import (EventRecord, QualityCheckResult, QualityCheckSpec, QualityResult,
+                         VerifyOutput)
+from .utils import now_iso, operator_env
+
+# How much of a failing command's output rides back inside the envelope. Enough
+# for a builder to act on without opening the artifact; bounded so a runaway
+# stack trace can't swamp the next agent's context.
+TAIL_CHARS = 4_000
+
+
+def _placeholder(name: str) -> list[str]:
+    """A command that does nothing and admits it. Replace every call to this."""
+    return ["echo", f"PLACEHOLDER {name}: edit smidja/smidja_modules/quality.py and "
+                    f"replace this echo with the real {name} command"]
+
+
+def _check_dir(run, name: str) -> Path:
+    seq = run.phases[-1].seq if run.phases else 0
+    path = run.context_handoff_dir / "quality" / f"{seq:02d}_{name}"
+    path.mkdir(parents=True, exist_ok=True)
+    return path
+
+
+def _run(spec: QualityCheckSpec, run) -> QualityCheckResult:
+    phase = run.phases[-1]
+    output_dir = _check_dir(run, spec.name)
+    output_artifact = output_dir / "command.log"
+    command = shlex.join(spec.argv)
+    env = operator_env()             # the engineer's own shell environment
+
+    run.console.note(f"quality {spec.name}: {command}")
+    started_at = now_iso()
+    clock = time.monotonic()
+    stdout = ""
+    stderr = ""
+    try:
+        completed = subprocess.run(
+            spec.argv,
+            cwd=run.repo_root,
+            env=env,
+            capture_output=True,
+            text=True,
+            timeout=spec.timeout_seconds,
+        )
+        returncode = completed.returncode
+        stdout = completed.stdout
+        stderr = completed.stderr
+    except subprocess.TimeoutExpired as error:
+        returncode = 124
+        stdout = error.stdout or ""
+        stderr = (error.stderr or "") + f"\nTimed out after {spec.timeout_seconds}s."
+    except OSError as error:
+        # A missing binary lands here as exit 127 with the real message — no
+        # pre-flight probe needed, and none wanted.
+        returncode = 127
+        stderr = str(error)
+
+    duration = time.monotonic() - clock
+    output_artifact.write_text(
+        f"$ {command}\nexit: {returncode}\nduration_seconds: {duration:.3f}\n"
+        f"\n--- stdout ---\n{stdout}\n--- stderr ---\n{stderr}\n"
+    )
+    passed = returncode == 0
+    run.tracer.event(EventRecord(
+        smidja_id=run.smidja_id,
+        phase_id=phase.phase_id,
+        type="tool_call",
+        name=f"quality:{spec.name}",
+        payload={
+            "area": spec.area,
+            "operation": spec.operation,
+            "command": command,
+            "returncode": returncode,
+            "passed": passed,
+            "output_artifact": str(output_artifact),
+        },
+        started_at=started_at,
+        ended_at=now_iso(),
+    ))
+    run.console.note(
+        f"quality {spec.name}: {'passed' if passed else 'failed'} "
+        f"(exit {returncode}, {duration:.1f}s)"
+    )
+    return QualityCheckResult(
+        name=spec.name,
+        area=spec.area,
+        operation=spec.operation,
+        command=command,
+        returncode=returncode,
+        passed=passed,
+        duration_seconds=duration,
+        output_artifact=str(output_artifact),
+        output_tail=(stdout + stderr)[-TAIL_CHARS:],
+    )
+
+
+# ── Blocks ────────────────────────────────────────────────────────────────────
+# Replace every argv below. See the banner at the top of this file.
+
+def test(run) -> QualityCheckResult:
+    """Run the project's test suite. The highest-value block to wire up first."""
+    return _run(QualityCheckSpec(
+        name="test",
+        area="backend",
+        operation="build",
+        argv=_placeholder("test"),        # e.g. ["bun", "test"] or ["uv", "run", "pytest", "-q"]
+        timeout_seconds=600,
+    ), run)
+
+
+def lint(run) -> QualityCheckResult:
+    return _run(QualityCheckSpec(
+        name="lint",
+        area="backend",
+        operation="lint",
+        argv=_placeholder("lint"),        # e.g. ["bun", "x", "oxlint@1.36.0", "src"]
+    ), run)
+
+
+def typecheck(run) -> QualityCheckResult:
+    return _run(QualityCheckSpec(
+        name="typecheck",
+        area="backend",
+        operation="typecheck",
+        argv=_placeholder("typecheck"),   # e.g. ["bun", "x", "tsc", "--noEmit"]
+    ), run)
+
+
+def build(run) -> QualityCheckResult:
+    output_dir = _check_dir(run, "build") / "bundle"
+    return _run(QualityCheckSpec(
+        name="build",
+        area="backend",
+        operation="build",
+        argv=_placeholder("build"),       # e.g. ["bun", "build", "src/index.ts", "--outdir", str(output_dir)]
+    ), run)
+
+
+def run_tests(run) -> QualityResult:
+    """The test suite alone, as a QualityResult — the deterministic test phase.
+
+    This is what replaces a `tester` agent once the command is written down. An
+    agent rediscovering the runner on every run costs a fortune to learn what a
+    subprocess already knows; the repair loop is unchanged, because a failure
+    still reaches the builder through `as_envelope` below.
+    """
+    check = test(run)
+    failures = ([] if check.passed else
+                [f"{check.name}: `{check.command}` exited {check.returncode}\n"
+                 f"{check.output_tail}".rstrip()])
+    return QualityResult(passed=check.passed, checks=[check], failures=failures,
+                         artifacts=[check.output_artifact])
+
+
+def as_envelope(result: QualityResult, what: str) -> VerifyOutput:
+    """Wrap a deterministic result so an agent can be handed it directly.
+
+    Agents hand each other typed envelopes; code blocks return QualityResult.
+    This is the adapter, so a failing lint or test run flows back into the
+    builder through exactly the same door an agent's report would — the smidja
+    script is the only thing that knows the difference.
+    """
+    return VerifyOutput(
+        status="success" if result.passed else "fail",
+        summary=(f"{what}: all {len(result.checks)} check(s) passed" if result.passed
+                 else f"{what}: {len(result.failures)} of {len(result.checks)} check(s) failed"),
+        artifacts=result.artifacts,
+        notes_for_next_agent=("" if result.passed else
+                              "Fix every failure below. The output is verbatim from the "
+                              "command — trust it over any summary."),
+        passed=result.passed,
+        failures=result.failures,
+    )
+
+
+def run_quality(run) -> QualityResult:
+    """Run every block and collect ALL failures — one pass tells you everything.
+
+    Ordering contract for the caller: a failing block does NOT fail the phase.
+    The runner did its job; the CODE is what failed. Hand this result to the
+    builder and let the bounded repair loop decide the run's fate.
+    """
+    blocks: list[Callable] = [
+        test,
+        lint,
+        typecheck,
+        build,
+    ]
+    checks = [block(run) for block in blocks]
+    # A failure is the command, its exit code, and what it actually printed —
+    # everything a builder needs to repair without opening a log or being told
+    # what the error "means" by a parser that guessed.
+    failures = [
+        f"{check.name}: `{check.command}` exited {check.returncode}\n{check.output_tail}".rstrip()
+        for check in checks if not check.passed
+    ]
+    return QualityResult(
+        passed=not failures,
+        checks=checks,
+        failures=failures,
+        artifacts=[check.output_artifact for check in checks],
+    )

package/apps/smidja-factory/templates/smidja/smidja_modules/runner.py

@@ -0,0 +1,155 @@
+"""The Run object: config + smidja_id + agent_map + tracer + console, bound once.
+
+`run.phase(PhaseParams(...))` is the ONE phase primitive — a context manager
+for all three kinds (engineer, agent, code). Success must be earned: every
+phase defaults to fail; only a clean exit flips it (agent phases additionally
+require a parsed envelope + green gates, enforced inside ph.call).
+"""
+
+from __future__ import annotations
+
+import json
+import time
+from contextlib import contextmanager
+from pathlib import Path
+
+from . import agents, git_helper
+from .console import Console
+from .data_types import AgentCall, EnvelopeBase, EventRecord, Phase, PhaseParams
+from .utils import ensure_dir, now_iso
+
+
+class PhaseHandle:
+    def __init__(self, run: "Run", phase: Phase):
+        self.run = run
+        self.phase = phase
+
+    def log(self, **payload) -> None:
+        self.run.tracer.event(EventRecord(smidja_id=self.run.smidja_id,
+                                          phase_id=self.phase.phase_id,
+                                          type="log", name=self.phase.params.name,
+                                          payload=payload))
+        self.run.console.note(", ".join(f"{k}: {v}" for k, v in payload.items()))
+        if self.phase.params.kind == "engineer" and "input" in payload:
+            self.run.tracer.session_request(self.run.smidja_id, str(payload["input"]))
+
+    def call(self, call: AgentCall) -> EnvelopeBase:
+        if self.phase.params.kind != "agent":
+            raise RuntimeError("ph.call() is only valid inside an agent phase")
+        return agents.execute(self.run, self.phase, call)
+
+
+class Run:
+    def __init__(self, cfg, smidja_id: str, tracer, engineer: str,
+                 repo_root: str | None = None,
+                 data_base: str | Path | None = None):
+        self.cfg = cfg
+        self.smidja_id = smidja_id
+        self.tracer = tracer
+        self.console = Console(tracer, smidja_id)
+        self.engineer = engineer
+        self.phases: list[Phase] = []
+        self._subagent_counter = 0   # unique ids for orchestrator task-dispatch lanes
+        # Lanes opened at tool_execution_start (G2) keyed by (phase_id, call_id)
+        # so the terminal materialization closes the SAME lane instead of
+        # opening a duplicate. Additive — absent lanes fall back to old behavior.
+        self._open_subagent_lanes: dict[str, dict[str, str]] = {}
+        self.tokens = 0
+        self.cost = 0.0
+        self._seq = tracer.max_phase_seq(smidja_id)   # a joined run continues the sequence
+        # Project-aware anchor: session.ensure passes the project repo root +
+        # its smidja/smidja_data base (agents work in the project, sessions + db
+        # live there too). Falls back to the config-relative defaults.
+        self.repo_root = Path(repo_root) if repo_root \
+            else git_helper.repo_root()             # where every agent is spawned to work
+        base = Path(data_base) if data_base \
+            else self.repo_root / cfg.defaults.data_dir
+        self.session_dir = ensure_dir(base / "sessions" / smidja_id)
+        self.context_handoff_dir = ensure_dir(self.session_dir / "context_handoff")
+        self._agent_map_path = self.session_dir / "agent_map.json"
+        self.agent_map: dict = (json.loads(self._agent_map_path.read_text())
+                                if self._agent_map_path.exists() else {})
+
+    # ── agent map (smidja_id -> per-agent coding-agent session ids) ────────────
+    def save_agent_map(self, agent: str, entry: dict) -> None:
+        self.agent_map[agent] = entry
+        self._agent_map_path.write_text(json.dumps(self.agent_map, indent=2))
+
+    # ── usage (run totals mirror what the tracer accumulates in sqlite) ─────
+    def add_usage(self, tokens: int, cost: float) -> None:
+        self.tokens += tokens
+        self.cost += cost
+        self.tracer.session_add_usage(self.smidja_id, tokens, cost)
+
+    # ── the phase primitive ─────────────────────────────────────────────────
+    @contextmanager
+    def phase(self, params: PhaseParams):
+        self._seq += 1
+        phase = Phase(phase_id=f"{self.smidja_id}_{self._seq:02d}_{params.name}",
+                      smidja_id=self.smidja_id, seq=self._seq, params=params,
+                      status="running", started_at=now_iso())
+        self.phases.append(phase)
+        self.tracer.phase_upsert(phase)
+        self.tracer.event(EventRecord(smidja_id=self.smidja_id, phase_id=phase.phase_id,
+                                      type="phase_start", name=params.name,
+                                      payload={"kind": params.kind, "owner": params.owner,
+                                               "description": params.description}))
+        self.console.phase_started(phase)
+        clock = time.monotonic()
+        try:
+            yield PhaseHandle(self, phase)
+        except BaseException as error:
+            phase.status = "fail"                      # success must be earned
+            phase.error = str(error)[:1000]
+            phase.ended_at = now_iso()
+            self.tracer.event(EventRecord(smidja_id=self.smidja_id, phase_id=phase.phase_id,
+                                          type="error", name=params.name,
+                                          payload={"error": phase.error}))
+            self.tracer.event(EventRecord(smidja_id=self.smidja_id, phase_id=phase.phase_id,
+                                          type="phase_end", name=params.name,
+                                          payload={"status": "fail"}))
+            self.tracer.phase_upsert(phase)
+            self.tracer.session_finish(self.smidja_id, ok=False)
+            self.console.phase_ended(phase, time.monotonic() - clock)
+            self.console.session_finished(False, self.tokens, self.cost,
+                                          self.cfg.observability.db)
+            raise
+        else:
+            phase.status = "success"
+            phase.ended_at = now_iso()
+            self.tracer.event(EventRecord(smidja_id=self.smidja_id, phase_id=phase.phase_id,
+                                          type="phase_end", name=params.name,
+                                          payload={"status": "success"}))
+            self.tracer.phase_upsert(phase)
+            self.console.phase_ended(phase, time.monotonic() - clock)
+
+    # ── run outcome ─────────────────────────────────────────────────────────
+    def finish(self, accepted: bool = True, reason: str = "") -> int:
+        """Finalize the run and return its exit code. Call this exactly once.
+
+        Two criteria, not one. Every phase must have passed, AND the smidja's own
+        acceptance test must hold. They are different questions on purpose: a
+        test phase that ran the suite did its job even when the suite came back
+        red, so the PHASE succeeds while the RUN must not.
+
+        This replaces a `succeeded` property that answered only the first
+        question — and, being a property with side effects, wrote the session
+        status and printed the banner before the caller's `and test.passed` was
+        ever evaluated. A run whose suite never passed was recorded green in the
+        db, on the terminal, and in the UI while exiting 1. Anyone reading the
+        trace saw success; only a CI job checking `$?` saw the truth. One call
+        now settles the db, the banner, and the exit code together, so the three
+        cannot disagree.
+        """
+        phases_ok = bool(self.phases) and all(p.status == "success" for p in self.phases)
+        ok = phases_ok and accepted
+        if phases_ok and not accepted:
+            note = reason or "the run's acceptance criterion was not met"
+            self.tracer.event(EventRecord(
+                smidja_id=self.smidja_id,
+                phase_id=self.phases[-1].phase_id if self.phases else "",
+                type="error", name="not_accepted", payload={"reason": note}))
+            self.console.note(f"not accepted: {note}")
+        self.tracer.session_finish(self.smidja_id, ok=ok)
+        self.console.session_finished(ok, self.tokens, self.cost, self.cfg.observability.db)
+        return 0 if ok else 1

package/apps/smidja-factory/templates/smidja/smidja_modules/session.py

@@ -0,0 +1,50 @@
+"""Session lifecycle: pin-or-create an smidja_id, build the Run object.
+
+`ensure(cfg, smidja_id)` joins the session if it exists or creates it under
+exactly that id (pinned ids for repeatable runs); omitted, a fresh id is
+minted and printed so the next smidja can pick it up.
+"""
+
+from __future__ import annotations
+
+import os
+import signal
+import sys
+from pathlib import Path
+
+from .data_types import smidjaConfig
+from .runner import Run
+from .tracer import Tracer
+from .utils import engineer_name, new_id
+
+
+def _finalize_when_killed(run: Run) -> None:
+    """A killed run still closes its own trace.
+
+    Python's default SIGTERM handling exits without unwinding, so `just kill`
+    (or any `kill <pid>`) would leave the session reading `running` forever and
+    its process rows open — the trace would claim work is in flight that is
+    already dead. Turning the signal into SystemExit both finalizes here and
+    lets the phase context manager record the phase as failed on the way out.
+    """
+    def handler(signum, _frame):
+        run.tracer.session_finish(run.smidja_id, ok=False)   # also closes process rows
+        raise SystemExit(128 + signum)
+
+    for sig in (signal.SIGTERM, signal.SIGINT):
+        signal.signal(sig, handler)
+
+
+def ensure(cfg: smidjaConfig, smidja_id: str | None = None) -> Run:
+    smidja_id = smidja_id or new_id(8)
+    tracer = Tracer(cfg.observability.db,
+                    f"{cfg.defaults.data_dir}/sessions/{smidja_id}/events.jsonl")
+    run = Run(cfg=cfg, smidja_id=smidja_id, tracer=tracer, engineer=engineer_name())
+    tracer.session_start(smidja_id, run.engineer, smidja_name=Path(sys.argv[0]).stem)
+    # This process is the run. Record it before any phase opens, so a run that
+    # hangs in its first agent call is still killable by smidja_id.
+    tracer.process_start(smidja_id, "smidja", "", os.getpid(),
+                         " ".join([Path(sys.argv[0]).name, *sys.argv[1:]]))
+    _finalize_when_killed(run)
+    run.console.session_started(smidja_id, run.engineer)
+    return run

package/apps/smidja-factory/templates/smidja/smidja_modules/stall_watch.py

@@ -0,0 +1,149 @@
+"""LM Studio GPU-kernel stall watchdog (G3) — ADDITIVE recovery layer.
+
+Recurring failure (3× this session, audit 2026-09-01): after ~3 tool
+round-trips the LM Studio GPU kernel pegs 98–100% and the model stops emitting
+tokens; unload/reload clears it, but nothing was watching, so the run sat
+"running" until the runner's own watchdog gave up.
+
+This module watches one active phase: while its pi child is alive, it samples
+(a) the agent's `raw_output.jsonl` mtime and (b) the GPU utilization. When the
+stream has produced NO new bytes for `stall_s` seconds AND the GPU is pegged
+(`gpu_pct` or more), it:
+
+  1. traces a `model_stall` event (so the UI + audit show what happened),
+  2. unloads the stalled model and reloads it (clears the kernel),
+  3. SIGKILLs the stuck child so the phase fails fast with a clear reason
+     instead of hanging forever — the existing model-fallback retry in
+     agents.py resumes the same session cleanly on the fallback model.
+
+Nothing existing is altered: the watchdog is a separate thread the caller
+starts per dispatch and stops when the child exits. Env knobs (all additive,
+all best-effort):
+
+    SMIDJA_STALL_WATCH=1          on by default; 0 disables
+    SMIDJA_STALL_SECS=180         no-new-bytes threshold before recovery
+    SMIDJA_STALL_GPU_PCT=90       GPU util at/above which a stall counts
+    SMIDJA_STALL_POLL=5           watchdog sampling interval
+"""
+
+from __future__ import annotations
+
+import os
+import signal
+import threading
+import time
+
+from . import lm_local
+
+_ENABLED = os.environ.get("SMIDJA_STALL_WATCH", "1") == "1"
+_STALL_S = int(os.environ.get("SMIDJA_STALL_SECS", "180"))
+_GPU_PCT = int(os.environ.get("SMIDJA_STALL_GPU_PCT", "90"))
+_POLL_S = float(os.environ.get("SMIDJA_STALL_POLL", "5"))
+
+
+class StallWatch:
+    """Watch one phase's raw stream; unload→reload + kill on a GPU-pegged stall."""
+
+    def __init__(self, smidja_id: str, phase_id: str, agent: str,
+                 model_key: str, raw_path: str, tracer,
+                 poll_s: float = _POLL_S, stall_s: int = _STALL_S,
+                 gpu_pct: int = _GPU_PCT) -> None:
+        self.smidja_id = smidja_id
+        self.phase_id = phase_id
+        self.agent = agent
+        self.model_key = model_key
+        self.raw_path = raw_path
+        self.tracer = tracer
+        self.poll_s = poll_s
+        self.stall_s = stall_s
+        self.gpu_pct = gpu_pct
+        self._pid: int | None = None
+        self._stop = threading.Event()
+        self._thread: threading.Thread | None = None
+
+    # ── lifecycle ────────────────────────────────────────────────────────────
+
+    def arm(self, pid: int) -> None:
+        """Start watching a live child (call from on_spawn)."""
+        if not _ENABLED:
+            return
+        self._pid = pid
+        self._stop.clear()
+        self._thread = threading.Thread(target=self._run, name="stall-watch", daemon=True)
+        self._thread.start()
+
+    def disarm(self) -> None:
+        """Stop watching (call from on_exit)."""
+        self._stop.set()
+        if self._thread:
+            self._thread.join(timeout=2)
+
+    # ── watchdog loop ────────────────────────────────────────────────────────
+
+    def _last_write(self) -> float:
+        try:
+            return os.path.getmtime(self.raw_path)
+        except OSError:
+            return time.time()
+
+    def _trace(self, type_: str, payload: dict) -> None:
+        try:
+            from .data_types import EventRecord  # late import — avoids cycles
+            self.tracer.event(EventRecord(
+                smidja_id=self.smidja_id, phase_id=self.phase_id,
+                type=type_, name=self.agent, payload=payload))
+        except Exception:
+            pass
+
+    def _recover(self, stalled_since: float) -> None:
+        self._trace("model_stall", {
+            "agent": self.agent, "model": self.model_key,
+            "stalled_seconds": int(time.time() - stalled_since),
+            "recovery": "unload→reload",
+            "gpu_pct": lm_local.gpu_util_pct(),
+        })
+        lm_local.unload_one(self.model_key)
+        lm_local.load(self.model_key)
+        # The child is stuck on a dead kernel — kill it so the phase fails fast
+        # and the existing fallback retry in agents.py resumes the session.
+        if self._pid:
+            try:
+                os.kill(self._pid, signal.SIGKILL)
+            except ProcessLookupError:
+                pass
+
+    def _run(self) -> None:
+        last = self._last_write()
+        stalled_since: float | None = None
+        while not self._stop.is_set():
+            time.sleep(self.poll_s)
+            now = self._last_write()
+            if now > last:                       # stream advanced → healthy
+                last, stalled_since = now, None
+                continue
+            if stalled_since is None:
+                stalled_since = time.time()
+                continue
+            if time.time() - stalled_since < self.stall_s:
+                continue
+            gpu = lm_local.gpu_util_pct()
+            if gpu is None or gpu >= self.gpu_pct:
+                self._recover(stalled_since)
+                return
+            stalled_since = time.time()          # GPU not pegged → re-arm timer
+
+
+def watch_for(request, run, phase, agent) -> StallWatch | None:
+    """Build a watchdog for a pi/LM-Studio dispatch, or None when not applicable
+    (opencode surface, cloud model, watchdog disabled). Additive helper."""
+    if not _ENABLED:
+        return None
+    if agent.coding_agent != "pi":
+        return None
+    if not getattr(request, "model", "").startswith("lmstudio/"):
+        return None
+    return StallWatch(
+        smidja_id=run.smidja_id, phase_id=phase.phase_id, agent=agent.name,
+        model_key=request.model.split("/", 1)[1],
+        raw_path=request.raw_output_path, tracer=run.tracer,
+    )