@zerwiz/ymir 0.1.33 → 0.1.35

package/apps/smidja-factory/templates/smidja/smidja_modules/gates.py

@@ -0,0 +1,108 @@
+"""Validation gates: verify the envelope's CLAIMS, never guesses.
+
+A gate is `gate(envelope, run) -> GateReport` — one check per item it looked at.
+Violations are derived from the failed checks and sent back to the SAME agent
+session as a correction. Every check is recorded either way, so a green gate
+says WHAT it verified instead of only that it passed.
+
+Gates check what is mechanically checkable; plan quality is a reviewer's job.
+"""
+
+from __future__ import annotations
+
+import json
+import subprocess
+from pathlib import Path
+
+from .data_types import EnvelopeBase, GateReport
+
+TAIL_CHARS = 1000        # command output kept as evidence on a failure
+
+
+def _size(path: Path) -> str:
+    n = path.stat().st_size
+    return f"{n}B" if n < 1024 else f"{n / 1024:.1f}KB"
+
+
+def artifacts_exist(envelope: EnvelopeBase, run) -> GateReport:
+    report = GateReport()
+    for a in envelope.artifacts:
+        p = Path(a)
+        report.check(a, p.exists(),
+                     f"exists, {_size(p)}" if p.exists() else "declared artifact does not exist")
+    return report
+
+
+def files_non_empty(envelope: EnvelopeBase, run) -> GateReport:
+    report = GateReport()
+    for a in envelope.artifacts:
+        p = Path(a)
+        if not (p.exists() and p.is_file()):
+            continue                       # existence is artifacts_exist's job
+        empty = p.stat().st_size == 0
+        report.check(a, not empty, "declared artifact is empty" if empty else _size(p))
+    return report
+
+
+def json_parses(envelope: EnvelopeBase, run) -> GateReport:
+    report = GateReport()
+    for a in envelope.artifacts:
+        p = Path(a)
+        if p.suffix != ".json" or not p.exists():
+            continue
+        try:
+            parsed = json.loads(p.read_text())
+            report.check(a, True, f"parses, {type(parsed).__name__}")
+        except json.JSONDecodeError as e:
+            report.check(a, False, f"declared JSON artifact does not parse: {e}")
+    return report
+
+
+def diff_matches_claims(envelope: EnvelopeBase, run) -> GateReport:
+    """Every file claimed changed must exist on disk."""
+    report = GateReport()
+    for f in getattr(envelope, "changed_files", []):
+        p = Path(f)
+        report.check(f, p.exists(),
+                     f"exists, {_size(p)}" if p.exists() else "claimed changed file does not exist")
+    return report
+
+
+def verdict_consistent(envelope: EnvelopeBase, run) -> GateReport:
+    """A review's verdict must agree with the findings it just wrote down.
+
+    Nothing here judges the code — that is the reviewer's job. This checks the
+    envelope against itself: an approval that ships blocking items, or a
+    rejection that names no problem, is a claim the harness can refute without
+    reading a line of the diff.
+    """
+    report = GateReport()
+    approved = bool(getattr(envelope, "approved", False))
+    blocking = list(getattr(envelope, "blocking", []))
+    unmet = [f.requirement for f in getattr(envelope, "findings", []) if not f.met]
+
+    report.check("approved vs blocking", not (approved and blocking),
+                 "no blocking items" if not blocking
+                 else f"{len(blocking)} blocking item(s) while approved=true"
+                 if approved else f"{len(blocking)} blocking item(s), not approved")
+    report.check("approved vs findings", not (approved and unmet),
+                 "every requirement met" if not unmet
+                 else f"{len(unmet)} unmet requirement(s) while approved=true"
+                 if approved else f"{len(unmet)} unmet requirement(s), not approved")
+    report.check("rejection names a problem", approved or bool(blocking or unmet),
+                 "verdict is supported" if approved or blocking or unmet
+                 else "approved=false but no blocking item or unmet requirement was given")
+    return report
+
+
+def tests_pass(command: str):
+    """Gate smidja: the given shell command must exit 0."""
+    def gate(envelope: EnvelopeBase, run) -> GateReport:
+        result = subprocess.run(command, shell=True, capture_output=True, text=True)
+        ok = result.returncode == 0
+        note = f"exit {result.returncode}"
+        if not ok:
+            note += "\n" + (result.stdout + result.stderr)[-TAIL_CHARS:]
+        return GateReport().check(command, ok, note)
+    gate.__name__ = f"tests_pass({command})"
+    return gate

package/apps/smidja-factory/templates/smidja/smidja_modules/git_helper.py

@@ -0,0 +1,120 @@
+"""Low-level git operations for code phases. All low-level logic lives in smidja_modules."""
+
+from __future__ import annotations
+
+import subprocess
+from pathlib import Path
+
+
+def _git(*args: str) -> str:
+    result = subprocess.run(["git", *args], capture_output=True, text=True)
+    if result.returncode != 0:
+        raise RuntimeError(f"git {' '.join(args)} failed: {result.stderr.strip()}")
+    return result.stdout.strip()
+
+
+def current_branch() -> str:
+    return _git("rev-parse", "--abbrev-ref", "HEAD")
+
+
+def create_branch(name: str) -> str:
+    _git("checkout", "-b", name)
+    return name
+
+
+def is_repo() -> bool:
+    result = subprocess.run(["git", "rev-parse", "--git-dir"],
+                            capture_output=True, text=True)
+    return result.returncode == 0
+
+
+def repo_root() -> Path:
+    """Absolute root of the codebase — where agents are spawned to work.
+
+    The git toplevel when there is one, else the process cwd (smidja run fine in a
+    non-git dir; only a commit phase requires a repo). Always absolute, so it is
+    safe to hand to a subprocess regardless of where the smidja was launched from.
+    """
+    if is_repo():
+        return Path(_git("rev-parse", "--show-toplevel")).resolve()
+    return Path.cwd().resolve()
+
+
+def commit_all(message: str) -> str:
+    """Stage the working tree and commit it. Returns the new short sha."""
+    if not is_repo():
+        raise RuntimeError(
+            "not a git repository — a commit phase needs one. Run `git init` in the "
+            "repo root (and make a first commit) before running an smidja that commits.")
+    _git("add", "-A")
+    if not _git("status", "--porcelain"):
+        raise RuntimeError("nothing to commit — the preceding phases changed no files")
+    _git("commit", "-m", message)
+    return _git("rev-parse", "--short", "HEAD")
+
+
+def changed_files() -> list[str]:
+    out = _git("status", "--porcelain")
+    return [line[3:] for line in out.splitlines() if line]
+
+
+# ── diff plumbing (composed into a ChangeSet by documentation.py) ────────────
+
+def ref_exists(ref: str) -> bool:
+    """True when `ref` resolves to a commit. Never raises — this is a question."""
+    result = subprocess.run(["git", "rev-parse", "--verify", "--quiet", f"{ref}^{{commit}}"],
+                            capture_output=True, text=True)
+    return result.returncode == 0
+
+
+def rev(ref: str = "HEAD") -> str:
+    return _git("rev-parse", ref)
+
+
+def short_sha(ref: str = "HEAD") -> str:
+    return _git("rev-parse", "--short", ref)
+
+
+def merge_base(ref: str, other: str = "HEAD") -> str:
+    """The commit where `ref` and `other` diverged — the honest base of a branch.
+
+    On the base branch itself this returns HEAD, which makes the diff exactly
+    "what is not committed yet". Off it, the diff is the whole branch plus the
+    working tree. One command covers both cases, so no smidja has to branch on it.
+    """
+    return _git("merge-base", ref, other)
+
+
+def is_dirty() -> bool:
+    return bool(_git("status", "--porcelain"))
+
+
+def untracked_files() -> list[str]:
+    out = _git("ls-files", "--others", "--exclude-standard")
+    return [line for line in out.splitlines() if line]
+
+
+def diff_files(base: str) -> list[str]:
+    """Tracked files that differ between `base` and the working tree."""
+    out = _git("diff", "--name-only", base)
+    return [line for line in out.splitlines() if line]
+
+
+def diff_stat(base: str) -> str:
+    return _git("diff", "--stat", base)
+
+
+def diff_counts(base: str) -> tuple[int, int]:
+    """(insertions, deletions) across the diff. Binary files count as neither."""
+    insertions = deletions = 0
+    for line in _git("diff", "--numstat", base).splitlines():
+        added, removed, *_ = line.split("\t")
+        if added.isdigit():
+            insertions += int(added)
+        if removed.isdigit():
+            deletions += int(removed)
+    return insertions, deletions
+
+
+def diff_text(base: str) -> str:
+    return _git("diff", base)

package/apps/smidja-factory/templates/smidja/smidja_modules/lm_local.py

@@ -0,0 +1,144 @@
+"""LM Studio local model management — load/unload via the native /api/v1 API.
+
+Why: the smidja runs one coding agent per phase; each pi invocation asks LM
+Studio to (re)load a model on demand. If models from earlier phases are left
+resident, the next load attempt has no room in VRAM and the llama.cpp engine
+refuses (HTTP 400) or crashes the engine (SIGABRT) — the failure mode that made
+whole rosters "broken". Between uses we therefore give the target model room:
+if it is already loaded we leave it (several models may be resident at once —
+repeated phases on the same model stay fast), otherwise we unload whatever else
+is loaded first.
+
+Uses LM Studio's native v1 API (0.4+), not the OpenAI-compatible DELETE which
+server-side returns 200 but does NOT free the VRAM allocation:
+    GET  /api/v1/models                -> per-model loaded_instances[]
+    POST /api/v1/models/load   {model}
+    POST /api/v1/models/unload {instance_id}
+
+Controlled by:
+    SMIDJA_LM_SERVER       default http://localhost:1234
+    SMIDJA_UNLOAD_MODELS   1 (default) unload non-targets when needed; 0 = disable
+Everything is best-effort and never raises — a missing/unreachable server or a
+non-LM-Studio backend is a no-op.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import subprocess
+import urllib.error
+import urllib.request
+
+SERVER = os.environ.get("SMIDJA_LM_SERVER", "http://localhost:1234")
+
+
+def _get(path: str, timeout: float = 5.0):
+    with urllib.request.urlopen(SERVER + path, timeout=timeout) as resp:
+        return json.load(resp)
+
+
+def _post(path: str, payload: dict, timeout: float = 15.0):
+    req = urllib.request.Request(
+        SERVER + path,
+        data=json.dumps(payload).encode(),
+        headers={"Content-Type": "application/json"},
+        method="POST",
+    )
+    with urllib.request.urlopen(req, timeout=timeout) as resp:
+        return json.load(resp)
+
+
+def loaded_ids() -> set[str]:
+    """Set of LM Studio model keys currently resident in memory."""
+    try:
+        data = _get("/api/v1/models")
+        return {m["key"] for m in data.get("models", []) if m.get("loaded_instances")}
+    except Exception:
+        return set()
+
+
+def load(model_key: str) -> bool:
+    """Load a single model by key (additive — used by the stall watchdog's
+    unload→reload recovery). Best-effort; True if the server accepted the load."""
+    try:
+        _post("/api/v1/models/load", {"model": model_key}, timeout=60)
+        return True
+    except Exception:
+        return False
+
+
+def gpu_util_pct() -> float | None:
+    """Current NVIDIA GPU utilization percent via nvidia-smi, or None if the
+    GPU isn't reachable / this isn't an NVIDIA box. Additive — used by the
+    stall watchdog to distinguish a GPU-kernel pegged stall from an ordinary
+    slow model."""
+    try:
+        out = subprocess.run(
+            ["nvidia-smi", "--query-gpu=utilization.gpu",
+             "--format=csv,noheader,nounits"],
+            capture_output=True, text=True, timeout=5,
+        )
+        if out.returncode != 0 or not out.stdout.strip():
+            return None
+        vals = [int(v) for v in out.stdout.strip().splitlines() if v.strip().isdigit()]
+        return float(max(vals)) if vals else None
+    except Exception:
+        return None
+
+
+def unload_all() -> int:
+    """Unload every loaded LM Studio model; return the number unloaded."""
+    unloaded = 0
+    try:
+        data = _get("/api/v1/models")
+    except Exception:
+        return unloaded
+    for m in data.get("models", []):
+        for inst in m.get("loaded_instances") or []:
+            iid = inst.get("id")
+            if not iid:
+                continue
+            try:
+                _post("/api/v1/models/unload", {"instance_id": iid}, timeout=20)
+                unloaded += 1
+            except Exception:
+                pass
+    return unloaded
+
+
+def unload_one(model_key: str) -> bool:
+    """Unload a single model by key; best-effort. True if it was resident."""
+    try:
+        data = _get("/api/v1/models")
+        for m in data.get("models", []):
+            if m.get("key") != model_key:
+                continue
+            for inst in m.get("loaded_instances") or []:
+                iid = inst.get("id")
+                if iid:
+                    _post("/api/v1/models/unload", {"instance_id": iid}, timeout=20)
+                    return True
+    except Exception:
+        pass
+    return False
+
+
+def make_room_for(model_key: str) -> None:
+    """Ensure `model_key` can load on LM Studio without VRAM overflow.
+
+    - Model already loaded  -> keep it (and any other residents that fit).
+    - Model not loaded      -> unload all residents first, then it loads fresh.
+    - Disabled / unreachable -> no-op.
+    """
+    if os.environ.get("SMIDJA_UNLOAD_MODELS", "1") == "0":
+        return
+    try:
+        data = _get("/api/v1/models")
+        loaded = [m["key"] for m in data.get("models", []) if m.get("loaded_instances")]
+    except Exception:
+        return  # not LM Studio or server down — do nothing
+    if model_key in loaded:
+        return
+    if loaded:
+        unload_all()

package/apps/smidja-factory/templates/smidja/smidja_modules/permissions.py

@@ -0,0 +1,194 @@
+"""What an agent may CHANGE, enforced in code after the fact.
+
+`tools:` is a capability list, not a sandbox, and two holes make it
+unenforceable on its own:
+
+  * `bash` runs anything. A builder handed bash to run a test suite can also
+    run `git checkout smidja/` — which is not hypothetical: one did, discarding
+    uncommitted changes to the very quality check it was about to be judged by.
+  * `write` reaches any path, not just the one report file an agent was given
+    it for. A reviewer configured with "no edit, so it cannot quietly fix"
+    could still rewrite the code it was reviewing.
+
+So permission is verified the way every other claim in this system is —
+after the fact, against the repo itself. `snapshot()` fingerprints the working
+tree's change-set before an agent runs; `enforce()` compares it afterwards and
+fails the phase if the agent touched anything outside its allowlist.
+
+Comparing change-sets, rather than watching for writes, is what catches the
+`git checkout` case: a path that was modified before the agent ran and is clean
+afterwards has been reverted, and a reversion is a modification. Appearing,
+disappearing, and changing all count.
+
+A breach is NOT a gate violation. Gates are for work an agent can be asked to
+redo; a breach cannot be corrected by re-prompting, because the write already
+happened. It aborts the phase and names every offending path.
+
+Two keys drive it, both in smidja.config.yaml:
+    defaults.protected_files   paths no agent may touch unless it names them itself
+    agents[].writes      None = unrestricted · [] = read-only · [...] = only these
+"""
+
+from __future__ import annotations
+
+import re
+import subprocess
+from pathlib import Path
+
+from .data_types import AgentConfig, smidjaConfig
+
+
+class PermissionBreach(RuntimeError):
+    """An agent modified a path it was not permitted to modify."""
+
+
+def _git(args: list[str], cwd) -> str:
+    result = subprocess.run(["git", *args], cwd=cwd, capture_output=True, text=True)
+    return result.stdout if result.returncode == 0 else ""
+
+
+def snapshot(run) -> dict[str, str]:
+    """Fingerprint every path the working tree currently differs on.
+
+    Tracked files carry their numstat counts, so an edit to an already-dirty
+    file still registers as a change. Untracked files are listed by name.
+    Gitignored paths never appear, which is why the session runtime under
+    `data_dir` — where handoff files legitimately land — needs no special case.
+    """
+    fingerprints: dict[str, str] = {}
+    for line in _git(["diff", "HEAD", "--numstat"], run.repo_root).splitlines():
+        fields = line.split("\t")
+        if len(fields) >= 3:
+            path = fields[-1].strip()
+            fingerprints[path] = f"{fields[0]},{fields[1]}"
+    for path in _git(["ls-files", "--others", "--exclude-standard"],
+                     run.repo_root).splitlines():
+        if path.strip():
+            fingerprints[path.strip()] = "untracked"
+    return fingerprints
+
+
+def changed_paths(before: dict[str, str], after: dict[str, str]) -> list[str]:
+    """Every path whose state differs — appeared, vanished, or was rewritten."""
+    return sorted({p for p in set(before) | set(after)
+                   if before.get(p) != after.get(p)})
+
+
+def _glob(pattern: str) -> re.Pattern:
+    """Translate a pattern, with `*` stopping at a path separator.
+
+    fnmatch would let `*` cross `/`, which quietly widens every pattern:
+    `smidja/smidja_*.py` would match `smidja/smidja_data/sessions/x/y.py` as well as the
+    smidja scripts it means. `**` is the way to say "cross directories".
+    """
+    out, i = [], 0
+    while i < len(pattern):
+        char = pattern[i]
+        if pattern.startswith("**", i):
+            out.append(".*")
+            i += 2
+        elif char == "*":
+            out.append("[^/]*")
+            i += 1
+        elif char == "?":
+            out.append("[^/]")
+            i += 1
+        else:
+            out.append(re.escape(char))
+            i += 1
+    return re.compile("".join(out))
+
+
+def _matches(path: str, pattern: str) -> bool:
+    if pattern.endswith("/"):                      # directory prefix
+        return path.startswith(pattern)
+    if "*" in pattern or "?" in pattern:
+        return _glob(pattern).fullmatch(path) is not None
+    return path == pattern
+
+
+def always_writable(cfg: smidjaConfig) -> list[str]:
+    """The session runtime, which EVERY agent must be able to write.
+
+    `context_handoff/` is the one place agents hand work to each other, and an
+    agent's own prompts, raw_output.jsonl, and envelope.json land beside it.
+    Scout writes its findings there, the reviewer its review, the planner its
+    plan — a read-only agent is read-only with respect to the REPO, never with
+    respect to its own report.
+
+    This is granted from `data_dir` rather than left to .gitignore. The runtime
+    is normally ignored, so it never even appears in a snapshot — but an agent's
+    ability to record its work must not hang on a gitignore entry that someone
+    can delete or that a changed `data_dir` can outgrow.
+    """
+    return [cfg.defaults.data_dir.rstrip("/") + "/"]
+
+
+def permitted(path: str, agent: AgentConfig, cfg: smidjaConfig) -> bool:
+    """Session runtime first, then the agent's own list, then what is protected."""
+    if any(_matches(path, p) for p in always_writable(cfg)):
+        return True
+    if any(_matches(path, p) for p in (agent.writes or [])):
+        return True                      # naming a path is what unlocks a protected one
+    if any(_matches(path, p) for p in cfg.defaults.protected_files):
+        return False
+    return agent.writes is None          # None = unrestricted, [] = no repo writes
+
+
+def _roll_back(run, path: str, before: dict[str, str], after: dict[str, str]) -> str:
+    """Undo one unauthorized change. Returns a word describing what happened.
+
+    Only changes the agent INTRODUCED are undone. A path that was already dirty
+    when the agent started is left exactly as it is: the operator had
+    uncommitted work there, and discarding it to tidy up would be the same harm
+    this module exists to prevent, committed by the cleanup instead of the agent.
+    """
+    if path in before:
+        # Already dirty beforehand. If it is gone from the diff now, the agent
+        # reverted an engineer's uncommitted work and the content is not ours
+        # to reconstruct — say so loudly rather than pretend it was handled.
+        return "REVERTED-BY-AGENT (uncommitted work lost, cannot restore)" \
+            if path not in after else "left as-is (was already modified)"
+    if after.get(path) == "untracked":
+        try:
+            (Path(run.repo_root) / path).unlink()
+            return "deleted"
+        except OSError as error:
+            return f"could not delete ({error})"
+    result = subprocess.run(["git", "checkout", "--", path],
+                            cwd=run.repo_root, capture_output=True, text=True)
+    return "rolled back" if result.returncode == 0 else "could not roll back"
+
+
+def enforce(run, phase, agent: AgentConfig, before: dict[str, str]) -> list[str]:
+    """Compare the tree against `before`; undo and raise if the agent overstepped.
+
+    Returns the paths it legitimately changed, so the trace records what an
+    agent actually touched rather than only what it claimed in its envelope.
+
+    Detection alone would leave the repo holding the unauthorized change while
+    reporting a failure, so anything the agent introduced outside its allowlist
+    is rolled back before the phase dies. What it cannot undo, it names.
+    """
+    after = snapshot(run)
+    touched = changed_paths(before, after)
+    # A path that was ALREADY dirty when the agent started and is still dirty
+    # now ("left as-is") is the operator's own uncommitted work, not an
+    # agent change. Failing a run over it (a) blames the agent for work it
+    # never did and (b) punishes every run that starts on a dirty tree. Only
+    # agent-attributable changes are breaches: files the agent CREATED (not
+    # in `before`) or files it DELETED/reverted (in `before`, gone in `after`).
+    # Additive — a clean baseline behaves exactly as before.
+    agent_changes = [p for p in touched
+                     if p not in before or p not in after]
+    breaches = [p for p in agent_changes if not permitted(p, agent, run.cfg)]
+    if not breaches:
+        return touched
+
+    outcomes = {p: _roll_back(run, p, before, after) for p in breaches}
+    scope = ("read-only" if agent.writes == []
+             else f"limited to {agent.writes}" if agent.writes
+             else f"barred from {run.cfg.defaults.protected_files}")
+    detail = "\n".join(f"  - {p} — {outcome}" for p, outcome in outcomes.items())
+    raise PermissionBreach(
+        f"{agent.name} is {scope} but modified {len(breaches)} path(s):\n{detail}")

package/apps/smidja-factory/templates/smidja/smidja_modules/prompts.py

@@ -0,0 +1,21 @@
+"""Prompt rendering: load system/user refs from config, replace {{placeholders}}."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+
+def render(template_path: str | Path, variables: dict[str, str]) -> str:
+    text = Path(template_path).read_text()
+    for key, value in variables.items():
+        text = text.replace("{{" + key + "}}", value)
+    return text
+
+
+def save(directory: str | Path, name: str, content: str) -> Path:
+    """Save the exact prompt sent, before execution — the audit copy."""
+    directory = Path(directory)
+    directory.mkdir(parents=True, exist_ok=True)
+    path = directory / name
+    path.write_text(content)
+    return path