@agentikos/omega-os 0.2.0 → 0.19.6

package/omega/Agentik_Engine/omega_engine/integrations/graphify.py

@@ -0,0 +1,234 @@
+"""Graphify integration — `safishamsi/graphify` as a Claude Code skill.
+
+What graphify actually is (verified against the upstream CLI at runtime):
+
+  * `graphifyy` (PyPI) installs a CLI named `graphify`.
+  * `graphify install` registers a Claude Code **skill** at
+    `~/.claude/skills/graphify/`. From inside Claude Code, typing
+    `/graphify .` triggers the skill — Claude reads the files and writes
+    the graph to `<project>/graphify-out/`.
+  * The CLI itself does NOT extract graphs. It manages the skill
+    (`install`, `uninstall`) and queries an existing graph (`path`,
+    `explain`, `diagnose`, `global list`).
+
+So this wrapper is intentionally lean. It:
+
+  1. detects whether the CLI is installed (`is_graphify_installed`)
+  2. registers the skill via `graphify install` (`graphify_install_skill`)
+  3. reads an existing `graphify-out/graph.json` and parses it
+     (`graphify_load_graph` → `GraphifyReport`)
+  4. merges the parsed graph into the engine's `GraphRetriever` so
+     multi-RAG can answer relational queries (`graphify_into_rag`)
+
+The graph is BUILT by Claude inside Claude Code (`/graphify .`). The engine
+CONSUMES the resulting `graph.json` — never the other way around.
+"""
+from __future__ import annotations
+
+import json
+import shutil
+import subprocess
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+
+_CLI = "graphify"
+_DEFAULT_OUT = "graphify-out"
+_DEFAULT_JSON = "graph.json"
+
+
+class GraphifyError(RuntimeError):
+    """Base class for graphify integration errors."""
+
+
+class GraphifyNotInstalled(GraphifyError):
+    """The `graphify` CLI is not on PATH."""
+
+    def __init__(self) -> None:
+        super().__init__(
+            "graphify CLI not found. Install with: `pip install graphifyy && "
+            "graphify install`"
+        )
+
+
+class GraphifyGraphMissing(GraphifyError):
+    """A `graphify-out/graph.json` was expected but not found."""
+
+
+@dataclass
+class GraphifyReport:
+    """Parsed result of a graphify run.
+
+    `nodes`/`edges` are the parsed structure. `summary` is whatever
+    top-level metadata graphify wrote alongside (mode, repo path, version,
+    statistics). `json_path` is the actual file we read — usually
+    `output_dir/graph.json` but callers can point at any path.
+    """
+
+    output_dir: Path
+    nodes: list[dict[str, Any]] = field(default_factory=list)
+    edges: list[dict[str, Any]] = field(default_factory=list)
+    summary: dict[str, Any] = field(default_factory=dict)
+    json_path: Path | None = None
+
+    @property
+    def html_path(self) -> Path:
+        return self.output_dir / "graph.html"
+
+    @property
+    def report_path(self) -> Path:
+        return self.output_dir / "GRAPH_REPORT.md"
+
+
+def is_graphify_installed() -> bool:
+    """True iff the `graphify` CLI is on PATH."""
+    return shutil.which(_CLI) is not None
+
+
+def graphify_version() -> str | None:
+    """Return the CLI's reported version, or None if missing/unreadable."""
+    if not is_graphify_installed():
+        return None
+    try:
+        out = subprocess.run(
+            [_CLI, "--version"],
+            check=False, capture_output=True, text=True, timeout=30,
+        )
+    except (OSError, subprocess.SubprocessError):
+        return None
+    text = (out.stdout or out.stderr or "").strip()
+    return text or None
+
+
+def graphify_install_skill(*, platform: str = "claude") -> dict[str, Any]:
+    """Register the graphify skill into a Claude-Code-compatible platform.
+
+    By default writes `~/.claude/skills/graphify/SKILL.md` (the Claude Code
+    skill that powers the `/graphify` slash command). Other supported
+    platforms include `cursor`, `gemini`, `codex` — see `graphify --help`.
+
+    Idempotent: re-running over an installed skill is safe (graphify itself
+    handles the overwrite).
+    """
+    if not is_graphify_installed():
+        raise GraphifyNotInstalled()
+    try:
+        proc = subprocess.run(
+            [_CLI, "install", "--platform", platform],
+            check=False, capture_output=True, text=True, timeout=120,
+        )
+    except subprocess.TimeoutExpired as exc:
+        raise GraphifyError("graphify install timed out") from exc
+    if proc.returncode != 0:
+        raise GraphifyError(
+            f"graphify install (platform={platform}) exited "
+            f"{proc.returncode}: "
+            f"{(proc.stderr or proc.stdout or '').strip()[:500]}"
+        )
+    return {
+        "platform": platform,
+        "stdout": (proc.stdout or "").strip(),
+    }
+
+
+def graphify_load_graph(
+    target: str | Path,
+    *,
+    json_path: str | Path | None = None,
+) -> GraphifyReport:
+    """Load an existing `graphify-out/graph.json` and parse it.
+
+    The graph is produced by typing `/graphify .` inside Claude Code at
+    `target/`. This function never invokes Claude — it only consumes the
+    artefact Claude produced.
+
+    Args:
+        target: the project root. We look for `target/graphify-out/graph.json`.
+        json_path: override the JSON path explicitly (skips the default lookup).
+
+    Raises:
+        GraphifyGraphMissing: no graph.json found at the resolved location.
+        GraphifyError: the file exists but isn't parseable.
+    """
+    if json_path is not None:
+        jp = Path(json_path).resolve()
+    else:
+        jp = (Path(target) / _DEFAULT_OUT / _DEFAULT_JSON).resolve()
+
+    if not jp.exists():
+        raise GraphifyGraphMissing(
+            f"no graph.json at {jp}. "
+            f"Build one with `/graphify .` inside Claude Code at {target!s}."
+        )
+
+    try:
+        data = json.loads(jp.read_text())
+    except (json.JSONDecodeError, OSError) as exc:
+        raise GraphifyError(f"unparsable JSON at {jp}: {exc}") from exc
+
+    nodes = list(data.get("nodes") or [])
+    edges = list(data.get("edges") or data.get("links") or [])
+    summary = {
+        k: v for k, v in data.items() if k not in {"nodes", "edges", "links"}
+    }
+    return GraphifyReport(
+        output_dir=jp.parent, nodes=nodes, edges=edges, summary=summary,
+        json_path=jp,
+    )
+
+
+def graphify_into_rag(report: GraphifyReport, graph_retriever: Any) -> dict[str, int]:
+    """Feed a `GraphifyReport` into the engine's `GraphRetriever`.
+
+    Each graphify node becomes a graph node (its `id` or `name` becomes the
+    key, the path/file/kind/symbol becomes metadata). Each edge becomes a
+    typed undirected edge — `edge_type` from `kind`/`type`/`relation`.
+
+    Returns counts so callers can log "added 1,234 nodes and 4,567 edges".
+    The retriever isn't imported eagerly so the engine works on machines
+    without graphify installed.
+    """
+    n_nodes = n_edges = 0
+    for node in report.nodes:
+        node_id = str(node.get("id") or node.get("name") or "").strip()
+        if not node_id:
+            continue
+        text = str(
+            node.get("text")
+            or node.get("description")
+            or node.get("name")
+            or node_id
+        )
+        meta = {
+            k: v for k, v in node.items()
+            if k not in {"id", "name", "text", "description"}
+        }
+        graph_retriever.add_node(node_id, text=text, metadata=meta)
+        n_nodes += 1
+
+    for edge in report.edges:
+        a = str(
+            edge.get("source")
+            or edge.get("from")
+            or edge.get("a")
+            or ""
+        ).strip()
+        b = str(
+            edge.get("target")
+            or edge.get("to")
+            or edge.get("b")
+            or ""
+        ).strip()
+        if not a or not b:
+            continue
+        etype = str(
+            edge.get("kind")
+            or edge.get("type")
+            or edge.get("relation")
+            or "rel"
+        )
+        graph_retriever.add_edge(a, b, edge_type=etype)
+        n_edges += 1
+
+    return {"nodes": n_nodes, "edges": n_edges}

package/omega/Agentik_Engine/omega_engine/learning.py

@@ -0,0 +1,268 @@
+"""Smith — the reflection loop that turns past failures into systemic upgrades.
+
+The audit history table records every score across time, but no one's
+reading it. This module does the reading + finds patterns + writes
+structured insights to ``Agentik_Extra/staging/promotion/`` where the
+educator pipeline can promote the best ones into the SSOT.
+
+What we look at:
+
+  * **Audit history** — which audits drop below threshold the most? On
+    which projects? On which kinds of files?
+  * **Worker outcomes** — which task descriptions correlate with
+    audit rejection? Where do workers go pending most often?
+  * **Mission durations** — which roles eat the most tokens?
+  * **Cost per mission** — when did Sonnet vs. Opus pay off?
+
+What we propose:
+
+  * **Audit threshold tweaks** — if codeaudit always fails at 78 on
+    project X but the operator never fixes the same finding kind,
+    suggest a per-project threshold override.
+  * **Skill recommendations** — patterns of repeated work suggest a
+    new skill the operator should create.
+  * **Model upgrades** — when a worker fails twice on Haiku, propose
+    bumping that intent class to Sonnet via the classifier.
+
+This is *proposal only*. The educator staging pipeline (which already
+exists) gates any actual change. Smith never edits the live tree
+directly.
+"""
+from __future__ import annotations
+
+import json
+import os
+import sqlite3
+import time
+from collections import Counter, defaultdict
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+
+@dataclass
+class Insight:
+    """One actionable observation Smith found."""
+    kind: str                  # "weak_audit" | "pending_pattern" | "cost_outlier" | "skill_suggestion"
+    title: str
+    detail: str
+    evidence: list[str] = field(default_factory=list)
+    severity: str = "info"     # "info" | "warn" | "high"
+    proposed_action: str = ""
+
+
+@dataclass
+class ReflectionReport:
+    generated_at: int
+    omega_home: str
+    insights: list[Insight] = field(default_factory=list)
+    counts: dict[str, int] = field(default_factory=dict)
+
+    def to_dict(self) -> dict:
+        return {
+            "generated_at": self.generated_at,
+            "omega_home": self.omega_home,
+            "insights": [
+                {"kind": i.kind, "title": i.title, "detail": i.detail,
+                 "evidence": i.evidence, "severity": i.severity,
+                 "proposed_action": i.proposed_action}
+                for i in self.insights
+            ],
+            "counts": self.counts,
+        }
+
+
+def _audit_db(home: Path) -> Path:
+    return home / "Agentik_Runtime" / "audits.db"
+
+
+def _event_db(home: Path) -> Path:
+    return home / "Agentik_Runtime" / "eventlog" / "omega.db"
+
+
+def _staging_dir(home: Path) -> Path:
+    d = home / "Agentik_Extra" / "staging" / "promotion"
+    d.mkdir(parents=True, exist_ok=True)
+    return d
+
+
+def _analyse_audit_history(home: Path) -> list[Insight]:
+    """Find audits that consistently fail or trend down."""
+    db = _audit_db(home)
+    if not db.exists():
+        return []
+    insights: list[Insight] = []
+    conn = sqlite3.connect(str(db))
+    try:
+        # Average score per audit_id over the last 30 days
+        cutoff = int(time.time()) - 30 * 86400
+        rows = conn.execute(
+            "SELECT audit_id, AVG(score) AS avg_score, COUNT(*) AS n, "
+            "       SUM(CASE WHEN verified = 1 THEN 1 ELSE 0 END) AS verified "
+            "FROM audit_runs WHERE timestamp > ? "
+            "GROUP BY audit_id HAVING n >= 3",
+            (cutoff,),
+        ).fetchall()
+        for audit_id, avg_score, n, verified in rows:
+            avg = float(avg_score or 0)
+            if avg < 75.0:
+                insights.append(Insight(
+                    kind="weak_audit",
+                    title=f"{audit_id} averages {avg:.1f}/100 over {n} runs",
+                    detail=(
+                        f"{audit_id} is failing consistently. "
+                        f"{verified}/{n} runs reached the threshold."
+                    ),
+                    evidence=[f"30d_avg={avg:.1f}", f"runs={n}",
+                              f"verified={verified}"],
+                    severity="warn" if avg >= 60 else "high",
+                    proposed_action=(
+                        f"Run `omega audit run {audit_id} --fix` to see if "
+                        "the smart batcher can clear it. If the same "
+                        "findings recur, lower the threshold OR write a "
+                        "checker skill that prevents that class of failure."
+                    ),
+                ))
+        # Trend: compare last 7d vs prior 23d for each audit
+        for audit_id in {r[0] for r in rows}:
+            r_recent = conn.execute(
+                "SELECT AVG(score) FROM audit_runs "
+                "WHERE audit_id = ? AND timestamp > ?",
+                (audit_id, int(time.time()) - 7 * 86400),
+            ).fetchone()
+            r_old = conn.execute(
+                "SELECT AVG(score) FROM audit_runs "
+                "WHERE audit_id = ? AND timestamp BETWEEN ? AND ?",
+                (audit_id, cutoff, int(time.time()) - 7 * 86400),
+            ).fetchone()
+            recent = float(r_recent[0] or 0)
+            old = float(r_old[0] or 0)
+            if recent > 0 and old > 0 and (old - recent) > 10:
+                insights.append(Insight(
+                    kind="weak_audit",
+                    title=f"{audit_id} dropped from {old:.0f} → {recent:.0f}",
+                    detail=(
+                        f"Regression of {old - recent:.0f} points in the "
+                        "last 7 days. Something landed that's hurting "
+                        "this dimension."
+                    ),
+                    evidence=[f"recent_avg={recent:.1f}",
+                              f"prior_avg={old:.1f}"],
+                    severity="high",
+                    proposed_action=(
+                        f"Bisect with `omega audit diff <last_good_run> "
+                        f"<recent_run>` to see which findings are new."
+                    ),
+                ))
+    finally:
+        conn.close()
+    return insights
+
+
+def _analyse_done_jsons(home: Path) -> list[Insight]:
+    """Find patterns in worker .done.json statuses."""
+    sess = home / "Agentik_Runtime" / "sessions"
+    if not sess.is_dir():
+        return []
+    status_counter: Counter = Counter()
+    pending_actions: Counter = Counter()
+    for done in sess.glob("*/.done.json"):
+        try:
+            data = json.loads(done.read_text())
+        except (json.JSONDecodeError, OSError):
+            continue
+        status_counter[data.get("status", "unknown")] += 1
+        for action in (data.get("pending_actions") or []):
+            pending_actions[action[:60]] += 1
+
+    insights: list[Insight] = []
+    total = sum(status_counter.values())
+    if total >= 10:
+        failed_pct = status_counter.get("failed", 0) / total
+        pending_pct = status_counter.get("pending", 0) / total
+        if failed_pct > 0.2:
+            insights.append(Insight(
+                kind="pending_pattern",
+                title=f"{failed_pct:.0%} of workers ended status=failed",
+                detail=(
+                    f"Out of {total} recent worker outcomes, "
+                    f"{status_counter['failed']} failed. The audit "
+                    "gate is catching real issues — good. But the "
+                    "rate is high, look at common errors."
+                ),
+                evidence=[f"total={total}",
+                          f"failed={status_counter.get('failed', 0)}",
+                          f"pending={status_counter.get('pending', 0)}"],
+                severity="warn",
+                proposed_action=(
+                    "Run `omega memory search 'errors'` to spot "
+                    "common failure modes; consider escalating those "
+                    "intent classes to a stronger model."
+                ),
+            ))
+        if pending_pct > 0.3:
+            insights.append(Insight(
+                kind="pending_pattern",
+                title=f"{pending_pct:.0%} workers end pending with un-actioned items",
+                detail="A lot of work is being deferred. Either tasks are too big or context is missing.",
+                evidence=[f"pending={status_counter.get('pending', 0)}",
+                          f"total={total}"],
+                severity="warn",
+                proposed_action=(
+                    "Break missions into smaller intents OR seed the "
+                    "envelope with more parent context (envelope.py)."
+                ),
+            ))
+
+    # Repeated pending actions = a skill begging to be created
+    repeat = [(text, n) for text, n in pending_actions.most_common(5)
+              if n >= 3]
+    for text, n in repeat:
+        insights.append(Insight(
+            kind="skill_suggestion",
+            title=f"Repeated pending action ({n}x): {text}",
+            detail=(
+                f"This pending_action appeared {n} times in recent "
+                "worker reports. It's a candidate for its own skill or "
+                "a reusable helper."
+            ),
+            evidence=[f"frequency={n}"],
+            severity="info",
+            proposed_action=(
+                f"Create a SKILL.md in Agentik_SSOT/skills/ that "
+                f"automates: {text[:80]}"
+            ),
+        ))
+    return insights
+
+
+def reflect(
+    omega_home: str | Path, *, write_proposals: bool = True,
+) -> ReflectionReport:
+    """Run a full Smith reflection. Returns the structured report.
+
+    With ``write_proposals=True`` (default), insights are also written
+    as JSON files in ``Agentik_Extra/staging/promotion/smith-<ts>-N.json``
+    where the existing educator pipeline picks them up.
+    """
+    home = Path(omega_home or os.environ.get("OMEGA_HOME")
+                or Path.home() / "Omega")
+    insights = _analyse_audit_history(home) + _analyse_done_jsons(home)
+    report = ReflectionReport(
+        generated_at=int(time.time()),
+        omega_home=str(home),
+        insights=insights,
+        counts={"insights": len(insights)},
+    )
+    if write_proposals and insights:
+        out = _staging_dir(home)
+        for i, ins in enumerate(insights, start=1):
+            (out / f"smith-{report.generated_at}-{i}.json").write_text(
+                json.dumps({
+                    "kind": ins.kind, "title": ins.title,
+                    "detail": ins.detail, "evidence": ins.evidence,
+                    "severity": ins.severity,
+                    "proposed_action": ins.proposed_action,
+                }, indent=2),
+            )
+    return report