arkaos 3.78.0 → 4.0.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "arkaos",
-  "version": "3.78.0",
+  "version": "4.0.1",
   "description": "The Operating System for AI Agent Teams",
   "type": "module",
   "bin": {

package/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "arkaos-core"
-version = "3.78.0"
+version = "4.0.1"
 description = "Core engine for ArkaOS — The Operating System for AI Agent Teams"
 readme = "README.md"
 license = {text = "MIT"}

package/scripts/bench/__init__.py

@@ -0,0 +1,5 @@
+"""ArkaOS benchmark harness.
+
+Reproducible, honest measurements of the core engine. No fabricated numbers:
+every value in the docs comes from running `python scripts/bench/run.py`.
+"""

package/scripts/bench/harness.py

@@ -0,0 +1,138 @@
+"""ArkaOS benchmark harness -- core engine measurements.
+
+Three honest measurements:
+
+1. Synapse injection latency (engine-only, no vector store) -- cold vs warm,
+   plus per-layer compute time so the "cached layers are sub-millisecond"
+   claim can be verified against the "full engine costs N ms" reality.
+2. Subagent handoff artifact size -- measured token estimate vs the documented
+   ~379-token claim.
+3. Routing accuracy -- DepartmentLayer keyword detection over a fixed labelled
+   prompt set.
+
+All numbers are reproducible. Timings vary by machine; routing accuracy and
+handoff sizes are deterministic.
+"""
+from __future__ import annotations
+
+import statistics
+import sys
+import time
+from pathlib import Path
+from typing import Callable
+
+_REPO_ROOT = Path(__file__).resolve().parents[2]
+if str(_REPO_ROOT) not in sys.path:
+    sys.path.insert(0, str(_REPO_ROOT))
+
+
+def _percentiles(samples_ms: list[float]) -> dict:
+    """Summarise a list of millisecond samples."""
+    ordered = sorted(samples_ms)
+    return {
+        "runs": len(ordered),
+        "min": round(ordered[0], 3),
+        "p50": round(statistics.median(ordered), 3),
+        "mean": round(statistics.mean(ordered), 3),
+        "max": round(ordered[-1], 3),
+    }
+
+
+def _time_call(fn: Callable[[], object]) -> float:
+    """Time a single call, return elapsed milliseconds."""
+    start = time.perf_counter()
+    fn()
+    return (time.perf_counter() - start) * 1000.0
+
+
+def bench_synapse_latency(runs: int = 50) -> dict:
+    """Measure Synapse engine injection latency (cold vs warm) + per-layer ms."""
+    from core.synapse.engine import create_default_engine
+    from core.synapse.layers import PromptContext
+
+    engine = create_default_engine()
+    ctx = PromptContext(
+        user_input="fix the authentication bug in the login controller",
+        cwd="/tmp/project", git_branch="feat/auth", project_name="demo",
+        project_stack="laravel 11", active_agent="backend-dev",
+    )
+    cold = [_time_call(lambda: (engine.clear_cache(), engine.inject(ctx))) for _ in range(runs)]
+    engine.inject(ctx)  # warm the cache
+    warm = [_time_call(lambda: engine.inject(ctx)) for _ in range(runs)]
+    last = engine.metrics[-1] if engine.metrics else {}
+    profile = {
+        "layers_computed": last.get("layers_computed"),
+        "layers_skipped": last.get("layers_skipped"),
+        "tokens_injected": last.get("tokens_injected"),
+    }
+    return {"layer_count": engine.layer_count,
+            "cold_ms": _percentiles(cold), "warm_ms": _percentiles(warm),
+            "injection_profile": profile}
+
+
+def bench_subagent_handoff() -> dict:
+    """Measure a representative handoff artifact's token estimate."""
+    from core.runtime.subagent import HandoffArtifact
+
+    artifact = HandoffArtifact(
+        task_id="task-0042",
+        task_description="Implement Stripe subscription billing with idempotent webhooks",
+        agent_id="backend-dev", agent_role="Senior Backend Developer",
+        agent_disc="D:80 I:50 S:45 C:78", department="dev",
+        relevant_files=["app/Services/BillingService.php",
+                        "app/Http/Controllers/WebhookController.php",
+                        "tests/Feature/BillingTest.php"],
+        context_summary=("Laravel 11 app, Cashier installed. Customer model has "
+                         "stripe_id. Need tiered pricing with volume discounts."),
+        constraints=["SOLID + Services/Repositories", "Feature tests required",
+                     "Idempotent webhook handling"],
+        expected_output="Tested, secure billing implementation with passing suite",
+        quality_criteria=["80%+ coverage", "OWASP reviewed", "Conventional commits"],
+    )
+    return {"documented_claim": 379,
+            "measured_tokens": artifact.estimated_tokens,
+            "prompt_chars": len(artifact.to_prompt())}
+
+
+# Fixed labelled prompt set for routing accuracy. (prompt, expected_department)
+_ROUTING_SET: list[tuple[str, str]] = [
+    ("fix the authentication bug in the login controller", "dev"),
+    ("refactor the payment service and add unit tests", "dev"),
+    ("create a go-to-market plan for our new SaaS", "saas"),
+    ("design a brand identity with logo and color palette", "brand"),
+    ("write viral content hooks for our TikTok channel", "content"),
+    ("build a high-converting landing page funnel", "landing"),
+    ("audit our online store conversion rate", "ecom"),
+    ("model our Q3 budget and cash flow forecast", "finance"),
+    ("run a competitive analysis with Porter's Five Forces", "strategy"),
+    ("plan the next sprint and groom the backlog", "pm"),
+    ("set up an SEO and paid ads growth campaign", "marketing"),
+    ("automate our client onboarding with an SOP", "ops"),
+]
+
+
+def bench_routing_accuracy() -> dict:
+    """Measure DepartmentLayer keyword routing over the labelled prompt set."""
+    from core.synapse.layers import DepartmentLayer, PromptContext
+
+    layer = DepartmentLayer()
+    hits, details = 0, []
+    for prompt, expected in _ROUTING_SET:
+        result = layer.compute(PromptContext(user_input=prompt))
+        detected = (result.content or "").strip()
+        ok = detected == expected
+        hits += int(ok)
+        details.append({"prompt": prompt, "expected": expected,
+                        "detected": detected or "(none)", "ok": ok})
+    total = len(_ROUTING_SET)
+    return {"total": total, "correct": hits,
+            "accuracy_pct": round(100.0 * hits / total, 1), "details": details}
+
+
+def run_all(runs: int = 50) -> dict:
+    """Run every benchmark and return a combined result dict."""
+    return {
+        "synapse_latency": bench_synapse_latency(runs=runs),
+        "subagent_handoff": bench_subagent_handoff(),
+        "routing_accuracy": bench_routing_accuracy(),
+    }

package/scripts/bench/run.py

@@ -0,0 +1,136 @@
+#!/usr/bin/env python3
+"""Run the ArkaOS benchmark harness and persist results.
+
+Writes:
+- benchmarks/results.json  -- machine-readable, consumed by the wiki
+- benchmarks/results.md     -- human-readable summary table
+
+Usage:
+    python scripts/bench/run.py                 # default 50 runs
+    python scripts/bench/run.py --runs 100
+    python scripts/bench/run.py --runs 30 --no-write   # print only
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import platform
+import sys
+from pathlib import Path
+
+_REPO_ROOT = Path(__file__).resolve().parents[2]
+if str(_REPO_ROOT) not in sys.path:
+    sys.path.insert(0, str(_REPO_ROOT))
+
+from scripts.bench import harness  # noqa: E402
+
+_OUT_DIR = _REPO_ROOT / "benchmarks"
+
+
+def _environment() -> dict:
+    """Capture the machine environment (numbers are machine-relative)."""
+    return {
+        "python": platform.python_version(),
+        "platform": platform.platform(),
+        "machine": platform.machine(),
+    }
+
+
+def _synapse_section(sl: dict) -> list[str]:
+    """Render the Synapse latency section."""
+    prof = sl["injection_profile"]
+    return [
+        "## Synapse context injection (engine-only, no vector store)",
+        "",
+        f"- Registered layers: **{sl['layer_count']}**",
+        f"- Cold injection (cache cleared each run): "
+        f"p50 **{sl['cold_ms']['p50']} ms**, mean {sl['cold_ms']['mean']} ms, "
+        f"min {sl['cold_ms']['min']} ms, max {sl['cold_ms']['max']} ms "
+        f"({sl['cold_ms']['runs']} runs)",
+        f"- Warm injection (cached): "
+        f"p50 **{sl['warm_ms']['p50']} ms**, mean {sl['warm_ms']['mean']} ms "
+        f"({sl['warm_ms']['runs']} runs)",
+        "- The small cold/warm delta is expected: cacheable layers are a "
+        "minority of total compute, so warming the cache saves only a few ms.",
+        f"- Representative injection: {prof['layers_computed']} layers computed, "
+        f"{prof['layers_skipped']} skipped, {prof['tokens_injected']} tokens injected",
+        "",
+    ]
+
+
+def _handoff_section(ho: dict) -> list[str]:
+    """Render the subagent handoff section."""
+    return [
+        "## Subagent handoff artifact",
+        "",
+        f"- Measured (representative artifact): **{ho['measured_tokens']} word-tokens** "
+        f"({ho['prompt_chars']} chars). 'word-tokens' is a whitespace-split estimate, "
+        "not a BPE tokenizer count.",
+        f"- Previously documented claim: {ho['documented_claim']}",
+        "",
+    ]
+
+
+def _routing_section(ra: dict) -> list[str]:
+    """Render the routing accuracy section + table."""
+    out = [
+        "## Routing accuracy (DepartmentLayer keyword detection)",
+        "",
+        f"- **{ra['correct']}/{ra['total']} = {ra['accuracy_pct']}%** on a fixed "
+        "labelled prompt set",
+        "",
+        "| Prompt | Expected | Detected | OK |",
+        "|---|---|---|:--:|",
+    ]
+    for d in ra["details"]:
+        mark = "yes" if d["ok"] else "no"
+        prompt = d["prompt"] if len(d["prompt"]) <= 48 else d["prompt"][:45] + "..."
+        out.append(f"| {prompt} | {d['expected']} | {d['detected']} | {mark} |")
+    out.append("")
+    return out
+
+
+def render_markdown(results: dict, env: dict) -> str:
+    """Render a human-readable benchmark summary."""
+    header = [
+        "# ArkaOS Benchmarks",
+        "",
+        "> Generated by `python scripts/bench/run.py`. Timings are "
+        "machine-relative; routing accuracy and handoff size are deterministic.",
+        "",
+        f"**Environment:** Python {env['python']} - {env['platform']}",
+        "",
+    ]
+    return "\n".join(header
+                     + _synapse_section(results["synapse_latency"])
+                     + _handoff_section(results["subagent_handoff"])
+                     + _routing_section(results["routing_accuracy"]))
+
+
+def main() -> int:
+    """Entry point."""
+    parser = argparse.ArgumentParser(description="Run ArkaOS benchmarks")
+    parser.add_argument("--runs", type=int, default=50, help="Latency samples (default 50)")
+    parser.add_argument("--no-write", action="store_true", help="Print only, do not write files")
+    args = parser.parse_args()
+
+    env = _environment()
+    results = harness.run_all(runs=args.runs)
+    payload = {"environment": env, "results": results}
+    md = render_markdown(results, env)
+
+    if args.no_write:
+        print(json.dumps(payload, indent=2))
+        print("\n" + md)
+        return 0
+
+    _OUT_DIR.mkdir(exist_ok=True)
+    (_OUT_DIR / "results.json").write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8")
+    (_OUT_DIR / "results.md").write_text(md + "\n", encoding="utf-8")
+    print(f"Wrote {_OUT_DIR / 'results.json'} and {_OUT_DIR / 'results.md'}")
+    print("\n" + md)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())