@0dai-dev/cli 4.3.5 → 4.3.7

package/lib/python/structural_memory.py

@@ -0,0 +1,325 @@
+#!/usr/bin/env python3
+"""Read-only structural-memory hints for 0dai context surfaces.
+
+This module does not rewrite memory, experience, or graph records. It derives a
+small role/status/source envelope from existing metadata so agents can tell raw
+artifacts, hypotheses, accepted knowledge, specs, and decisions apart.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import pathlib
+from collections import Counter
+from typing import Any
+
+
+ROLE_RAW_ARTIFACT = "raw_artifact"
+ROLE_HYPOTHESIS = "hypothesis"
+ROLE_ACCEPTED_KNOWLEDGE = "accepted_knowledge"
+ROLE_SPEC = "spec"
+ROLE_DECISION = "decision"
+ROLE_RISK = "risk"
+ROLE_EVENT = "event"
+ROLE_PROCESS = "process"
+ROLE_REFERENCE = "reference"
+ROLE_SYSTEM = "system"
+ROLE_UNKNOWN = "unknown"
+
+STATUS_ACCEPTED = "accepted"
+STATUS_CANDIDATE = "candidate"
+STATUS_UNVERIFIED = "unverified"
+STATUS_LOW_CONFIDENCE = "low_confidence"
+STATUS_ACTIVE = "active"
+STATUS_ARCHIVED = "archived"
+
+RAW_TAGS = {"raw", "log", "logs", "runtime", "artifact", "audit", "telemetry", "transcript"}
+HYPOTHESIS_TAGS = {"hypothesis", "idea", "draft", "candidate", "proposal", "experiment"}
+SPEC_TAGS = {"spec", "specification", "requirement", "contract", "acceptance"}
+DECISION_TAGS = {"decision", "adr", "rule", "directive", "policy"}
+REFERENCE_TAGS = {"source", "reference", "external", "article", "link"}
+
+MEMORY_CATEGORY_ROLES = {
+    "constraint": ROLE_SPEC,
+    "preference": ROLE_ACCEPTED_KNOWLEDGE,
+    "pattern": ROLE_ACCEPTED_KNOWLEDGE,
+    "lesson": ROLE_ACCEPTED_KNOWLEDGE,
+    "gotcha": ROLE_RISK,
+    "fact": ROLE_ACCEPTED_KNOWLEDGE,
+}
+
+GRAPH_TYPE_ROLES = {
+    "Component": ROLE_SYSTEM,
+    "Technology": ROLE_SYSTEM,
+    "Endpoint": ROLE_SYSTEM,
+    "Decision": ROLE_DECISION,
+    "Constraint": ROLE_SPEC,
+    "Requirement": ROLE_SPEC,
+    "TestPlan": ROLE_SPEC,
+    "Risk": ROLE_RISK,
+    "Artifact": ROLE_RAW_ARTIFACT,
+    "Event": ROLE_EVENT,
+    "Session": ROLE_EVENT,
+    "Deliberation": ROLE_PROCESS,
+    "Outcome": ROLE_ACCEPTED_KNOWLEDGE,
+    "DesignArtifact": ROLE_REFERENCE,
+    "MarketEntity": ROLE_REFERENCE,
+}
+
+
+def _tags(record: dict[str, Any]) -> set[str]:
+    raw = record.get("tags") or []
+    if isinstance(raw, str):
+        raw = raw.split(",")
+    return {str(tag).strip().lower() for tag in raw if str(tag).strip()}
+
+
+def _source(record: dict[str, Any], key: str = "source") -> str:
+    value = record.get(key) or ""
+    return str(value).strip() or "unknown"
+
+
+def _confidence(record: dict[str, Any]) -> float:
+    try:
+        return float(record.get("confidence", 0.0))
+    except (TypeError, ValueError):
+        return 0.0
+
+
+def classify_memory_entry(entry: dict[str, Any]) -> dict[str, Any]:
+    """Return structural hints for one ai/memory entry."""
+    tags = _tags(entry)
+    category = str(entry.get("category") or "fact").lower()
+    confidence = _confidence(entry)
+    verified = bool(entry.get("verified"))
+    source = _source(entry)
+
+    role = MEMORY_CATEGORY_ROLES.get(category, ROLE_UNKNOWN)
+    signals: list[str] = []
+
+    if tags & RAW_TAGS:
+        role = ROLE_RAW_ARTIFACT
+        signals.append("raw_tag")
+    elif tags & HYPOTHESIS_TAGS:
+        role = ROLE_HYPOTHESIS
+        signals.append("hypothesis_tag")
+    elif tags & SPEC_TAGS:
+        role = ROLE_SPEC
+        signals.append("spec_tag")
+    elif tags & DECISION_TAGS:
+        role = ROLE_DECISION
+        signals.append("decision_tag")
+    elif tags & REFERENCE_TAGS:
+        role = ROLE_REFERENCE
+        signals.append("reference_tag")
+
+    if verified:
+        status = STATUS_ACCEPTED
+    elif confidence < 0.5:
+        status = STATUS_LOW_CONFIDENCE
+    elif role in {ROLE_HYPOTHESIS, ROLE_RAW_ARTIFACT, ROLE_REFERENCE}:
+        status = STATUS_CANDIDATE
+    else:
+        status = STATUS_UNVERIFIED
+
+    drift: list[str] = []
+    if verified and confidence < 0.8:
+        drift.append("verified_low_confidence")
+    if category in {"constraint", "preference", "lesson", "pattern"} and role == ROLE_RAW_ARTIFACT:
+        drift.append("accepted_category_marked_raw")
+    if category == "fact" and role == ROLE_HYPOTHESIS:
+        drift.append("fact_marked_hypothesis")
+
+    return {
+        "role": role,
+        "status": status,
+        "source": source,
+        "confidence": confidence,
+        "signals": signals,
+        "drift": drift,
+    }
+
+
+def enrich_memory_entry(entry: dict[str, Any]) -> dict[str, Any]:
+    enriched = dict(entry)
+    enriched["structural"] = classify_memory_entry(entry)
+    return enriched
+
+
+def classify_graph_node(node: dict[str, Any]) -> dict[str, Any]:
+    """Return structural hints for one project graph node."""
+    node_type = str(node.get("type") or "")
+    role = GRAPH_TYPE_ROLES.get(node_type, ROLE_UNKNOWN)
+    raw_status = str(node.get("status") or STATUS_ACTIVE).lower()
+    source = _source(node, key="source_type")
+
+    if raw_status in {"deprecated", "archived", "superseded", "deleted"}:
+        status = STATUS_ARCHIVED
+    elif raw_status in {"confirmed", "accepted", "active"}:
+        status = STATUS_ACCEPTED if role in {ROLE_DECISION, ROLE_SPEC, ROLE_ACCEPTED_KNOWLEDGE} else STATUS_ACTIVE
+    elif raw_status in {"draft", "candidate", "proposed"}:
+        status = STATUS_CANDIDATE
+    else:
+        status = raw_status or STATUS_ACTIVE
+
+    drift: list[str] = []
+    if role == ROLE_UNKNOWN:
+        drift.append("unknown_graph_node_role")
+    if node_type == "Decision" and raw_status not in {"active", "confirmed", "accepted", "superseded", "deprecated", "archived"}:
+        drift.append("decision_status_not_normalized")
+
+    return {
+        "role": role,
+        "status": status,
+        "source": source,
+        "signals": [f"graph_type:{node_type}"] if node_type else [],
+        "drift": drift,
+    }
+
+
+def _load_memory_entries(target: pathlib.Path) -> list[dict[str, Any]]:
+    path = target / "ai" / "memory" / "memory.jsonl"
+    if not path.is_file():
+        return []
+    entries: list[dict[str, Any]] = []
+    for line in path.read_text(encoding="utf-8").splitlines():
+        line = line.strip()
+        if not line:
+            continue
+        try:
+            payload = json.loads(line)
+        except json.JSONDecodeError:
+            continue
+        if isinstance(payload, dict) and not payload.get("deleted"):
+            entries.append(payload)
+    return entries
+
+
+def _load_graph_nodes(target: pathlib.Path) -> list[dict[str, Any]]:
+    path = target / "ai" / "manifest" / "project_graph.json"
+    if not path.is_file():
+        return []
+    try:
+        graph = json.loads(path.read_text(encoding="utf-8"))
+    except (json.JSONDecodeError, OSError):
+        return []
+    nodes = graph.get("nodes") if isinstance(graph, dict) else {}
+    if isinstance(nodes, dict):
+        return [n for n in nodes.values() if isinstance(n, dict)]
+    if isinstance(nodes, list):
+        return [n for n in nodes if isinstance(n, dict)]
+    return []
+
+
+def _experience_role(subdir: str) -> str:
+    return {
+        "events": ROLE_RAW_ARTIFACT,
+        "outbox": ROLE_RAW_ARTIFACT,
+        "candidates": ROLE_HYPOTHESIS,
+        "accepted": ROLE_ACCEPTED_KNOWLEDGE,
+    }.get(subdir, ROLE_UNKNOWN)
+
+
+def _experience_status(subdir: str) -> str:
+    return {
+        "events": STATUS_UNVERIFIED,
+        "outbox": STATUS_CANDIDATE,
+        "candidates": STATUS_CANDIDATE,
+        "accepted": STATUS_ACCEPTED,
+    }.get(subdir, STATUS_UNVERIFIED)
+
+
+def _experience_counts(target: pathlib.Path) -> tuple[Counter, Counter, int]:
+    roles: Counter = Counter()
+    statuses: Counter = Counter()
+    total = 0
+    root = target / "ai" / "experience"
+    for subdir in ("events", "outbox", "candidates", "accepted"):
+        base = root / subdir
+        if not base.is_dir():
+            continue
+        for path in base.rglob("*"):
+            if path.is_file() and path.suffix in {".json", ".jsonl", ".md"}:
+                total += 1
+                roles[_experience_role(subdir)] += 1
+                statuses[_experience_status(subdir)] += 1
+    return roles, statuses, total
+
+
+def summarize_project(target: pathlib.Path) -> dict[str, Any]:
+    """Build a read-only structural-memory summary for a project."""
+    target = target.resolve()
+    memory_entries = _load_memory_entries(target)
+    graph_nodes = _load_graph_nodes(target)
+
+    role_counts: Counter = Counter()
+    status_counts: Counter = Counter()
+    source_counts: Counter = Counter()
+    warnings: list[dict[str, str]] = []
+
+    for entry in memory_entries:
+        structural = classify_memory_entry(entry)
+        role_counts[structural["role"]] += 1
+        status_counts[structural["status"]] += 1
+        source_counts[structural["source"]] += 1
+        for drift in structural["drift"]:
+            warnings.append({"surface": "memory", "id": str(entry.get("id") or ""), "drift": drift})
+
+    for node in graph_nodes:
+        structural = classify_graph_node(node)
+        role_counts[structural["role"]] += 1
+        status_counts[structural["status"]] += 1
+        source_counts[structural["source"]] += 1
+        for drift in structural["drift"]:
+            warnings.append({"surface": "graph", "id": str(node.get("id") or ""), "drift": drift})
+
+    exp_roles, exp_statuses, exp_total = _experience_counts(target)
+    role_counts.update(exp_roles)
+    status_counts.update(exp_statuses)
+
+    return {
+        "schema": 1,
+        "target": str(target),
+        "total_records": len(memory_entries) + len(graph_nodes) + exp_total,
+        "surfaces": {
+            "memory": len(memory_entries),
+            "graph": len(graph_nodes),
+            "experience": exp_total,
+        },
+        "roles": dict(sorted(role_counts.items())),
+        "statuses": dict(sorted(status_counts.items())),
+        "sources": dict(sorted(source_counts.items())),
+        "drift_warnings": warnings[:20],
+        "drift_warning_count": len(warnings),
+    }
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="Show read-only structural-memory hints.")
+    parser.add_argument("--target", default=".", help="Project root")
+    parser.add_argument("--json", action="store_true", help="Print JSON")
+    args = parser.parse_args()
+
+    summary = summarize_project(pathlib.Path(args.target))
+    if args.json:
+        print(json.dumps(summary, indent=2, ensure_ascii=False))
+        return 0
+
+    print("Structural memory:")
+    print(f"  records: {summary['total_records']}")
+    print(
+        "  surfaces: "
+        + ", ".join(f"{key}={value}" for key, value in summary["surfaces"].items())
+    )
+    if summary["roles"]:
+        print(
+            "  roles: "
+            + ", ".join(f"{key}={value}" for key, value in summary["roles"].items())
+        )
+    if summary["drift_warning_count"]:
+        print(f"  drift warnings: {summary['drift_warning_count']}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/lib/python/swarm_cost.py

@@ -0,0 +1,177 @@
+"""Swarm Cost — normalize telemetry data from different agents.
+
+Implements Layer 3.5 cost monitoring.
+
+Note on accuracy (2026-04-26 update per operator directive):
+- Rates table updated for current pool models (gpt-5.5, gpt-5.3-codex,
+  claude-opus-4.7, claude-sonnet-4.6, gemini-3, qoder-native).
+- For subscription-mode sessions (auth_mode=subscription per
+  ai/manifest/agent-pool.yaml), the USD estimate is "subscription-covered"
+  (SC) — operator's flat monthly fee covers it within quota. The estimate
+  is still computed (for cross-mode comparison + future API-mode parity)
+  but rendered with `(SC)` suffix when displayed.
+- For api-key sessions (RuAPI proxy / opencode anthropic / ghost ensemble),
+  the USD estimate maps to actual incremental spend.
+- CLI footers expose real subscription quota usage (5h block %, weekly %)
+  — see hop session pane, e.g. "5h 87% · weekly 86%". That's the
+  authoritative subscription cost signal; estimate_cost() is for paper
+  comparison only.
+"""
+from __future__ import annotations
+
+import json
+import pathlib
+
+# Per-agent rates (USD per 1M tokens, public list-price reference).
+# Updated 2026-04-26: added current pool models.
+DEFAULT_RATES = {
+    # Anthropic (current — Claude 4 family)
+    "claude-opus-4.7": (15.0, 75.0),       # (input, output)
+    "claude-opus-4-7": (15.0, 75.0),       # alias
+    "claude-opus-4.6": (15.0, 75.0),
+    "claude-opus-4-6": (15.0, 75.0),
+    "claude-sonnet-4.6": (3.0, 15.0),
+    "claude-sonnet-4-6": (3.0, 15.0),
+    "claude-haiku-4.5": (0.80, 4.0),
+    "claude-haiku-4-5": (0.80, 4.0),
+    "opus": (15.0, 75.0),                  # CLI alias
+    "sonnet": (3.0, 15.0),                 # CLI alias
+    "haiku": (0.80, 4.0),                  # CLI alias
+    # Anthropic (legacy aliases)
+    "claude-opus": (15.0, 75.0),
+    "claude-sonnet": (3.0, 15.0),
+    "claude-3-5-sonnet": (3.0, 15.0),
+    # OpenAI / codex (current pool)
+    "gpt-5.5": (10.0, 40.0),               # 2026 pricing per OpenAI public list
+    "gpt-5.4": (10.0, 30.0),
+    "gpt-5.4-mini": (0.15, 0.60),
+    "gpt-5.3-codex": (12.0, 36.0),         # codex-tuned, deep tier
+    "gpt-5.3": (8.0, 24.0),
+    "gpt-5.2-codex": (10.0, 30.0),
+    # Google / gemini (current)
+    "gemini-3": (3.50, 10.50),
+    "gemini-3.1-pro": (3.50, 10.50),
+    "gemini-2.5-pro": (3.50, 10.50),
+    "gemini-1.5-pro": (3.50, 10.50),
+    "gemini-1.5-flash": (0.35, 1.05),
+    # Qoder native (subscription-only, public list approximate)
+    "qoder-default": (5.0, 20.0),
+    "qoder-performance": (5.0, 20.0),
+    "qoder-ultimate": (10.0, 40.0),
+}
+
+# Sessions with subscription-flat-cost (per ai/manifest/agent-pool.yaml
+# auth_mode: subscription). USD estimate for these is paper-only — the
+# operator's monthly subscription fee covers it within quota. Add (SC)
+# suffix when rendering.
+SUBSCRIPTION_AGENTS = {
+    "claude",       # Claude Max via /root/.claude/ (claude-2/3 pool)
+    "codex",        # Codex via OpenAI subscription (codex-1/2/3 pool)
+    "gemini",       # Gemini via Google Code Assist Pro (gemini-1/2/3 pool)
+    "qoder",        # Qoder native subscription
+}
+
+# Sessions with api-key (real incremental spend via RuAPI / direct API):
+API_KEY_AGENTS = {
+    "ruapi-opus47", "ruapi-sonnet46", "ruapi-gemini31",
+    "opencode",                              # opencode via api-key per manifest
+    "ghost", "ghost-opus47", "ghost-gemini31",
+    "zeus", "athena", "hephaestus", "hermes", "ares", "apollo",   # Greek pantheon
+}
+
+
+def is_subscription_covered(agent: str) -> bool:
+    """True if this agent's USD estimate is subscription-covered (paper only)."""
+    base = (agent or "").lower().split("-")[0]   # claude-2 → claude
+    return base in SUBSCRIPTION_AGENTS or agent in SUBSCRIPTION_AGENTS
+
+
+def format_cost(usd: float, agent: str = "") -> str:
+    """Render USD with (SC) suffix when subscription-covered.
+
+    Examples:
+      format_cost(3.16, "codex")     → "$3.1600 (SC)"
+      format_cost(3.16, "ruapi-opus47") → "$3.1600"
+      format_cost(0.0, "claude")     → "$0.0000 (SC)"
+    """
+    base = f"${usd:.4f}"
+    if is_subscription_covered(agent):
+        return f"{base} (SC)"
+    return base
+
+
+class LaneCostReport:
+    def __init__(self, lane_id: str, agent: str, model: str):
+        self.lane_id = lane_id
+        self.agent = agent
+        self.model = model
+        self.input_tokens = 0
+        self.output_tokens = 0
+        self.usd_estimate = 0.0
+        self.wall_clock_seconds = 0
+
+    @property
+    def subscription_covered(self) -> bool:
+        return is_subscription_covered(self.agent)
+
+    def to_dict(self) -> dict:
+        return {
+            "lane_id": self.lane_id,
+            "agent": self.agent,
+            "model": self.model,
+            "input_tokens": self.input_tokens,
+            "output_tokens": self.output_tokens,
+            "usd_estimate": round(self.usd_estimate, 6),
+            "subscription_covered": self.subscription_covered,
+            "wall_clock_seconds": self.wall_clock_seconds,
+        }
+
+
+def estimate_cost(model: str, input_tokens: int, output_tokens: int) -> float:
+    """Calculate USD cost based on token counts (paper estimate, list-price).
+
+    For subscription-covered sessions this is the equivalent api-mode cost,
+    NOT actual incremental spend. Use is_subscription_covered() / format_cost()
+    when displaying.
+    """
+    rates = DEFAULT_RATES.get(model, (10.0, 30.0))   # fallback mid-range
+    in_cost = (input_tokens / 1_000_000) * rates[0]
+    out_cost = (output_tokens / 1_000_000) * rates[1]
+    return in_cost + out_cost
+
+
+def get_lane_telemetry(lane_id: str, target: pathlib.Path) -> LaneCostReport | None:
+    """Sample lane telemetry from ai/work/swarm/active/<lane_id>.json."""
+    active_path = target / "ai" / "work" / "swarm" / "active" / f"{lane_id}.json"
+    if not active_path.is_file():
+        # Check done dir as fallback
+        active_path = target / "ai" / "work" / "swarm" / "done" / f"{lane_id}.json"
+        if not active_path.is_file():
+            return None
+
+    try:
+        data = json.loads(active_path.read_text(encoding="utf-8"))
+        report = LaneCostReport(
+            lane_id=lane_id,
+            agent=data.get("agent", "unknown"),
+            model=data.get("model", "unknown"),
+        )
+        report.input_tokens = int(data.get("tokens_input", 0))
+        report.output_tokens = int(data.get("tokens_output", 0))
+        report.usd_estimate = estimate_cost(report.model, report.input_tokens, report.output_tokens)
+
+        # Calculate wall clock if timestamps are present
+        start = data.get("started_at")
+        if start:
+            # Simple epoch diff for now
+            pass
+
+        return report
+    except (json.JSONDecodeError, OSError, ValueError):
+        return None
+
+
+if __name__ == "__main__":
+    print(f"Cost of 1M Opus 4.7 tokens (500K in / 500K out): {format_cost(estimate_cost('claude-opus-4.7', 500000, 500000), 'claude')}")
+    print(f"Cost of 1M gpt-5.5 tokens (500K in / 500K out): {format_cost(estimate_cost('gpt-5.5', 500000, 500000), 'codex')}")
+    print(f"Cost of 1M ruapi-opus47 tokens: {format_cost(estimate_cost('claude-opus-4.7', 500000, 500000), 'ruapi-opus47')}")