arkaos 4.13.2 → 4.14.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "arkaos",
-  "version": "4.13.2",
+  "version": "4.14.0",
   "description": "The Operating System for AI Agent Teams",
   "type": "module",
   "bin": {

package/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "arkaos-core"
-version = "4.13.2"
+version = "4.14.0"
 description = "Core engine for ArkaOS — The Operating System for AI Agent Teams"
 readme = "README.md"
 license = {text = "MIT"}
@@ -32,6 +32,14 @@ knowledge = [
     "fastembed>=0.8.0",
     "sqlite-vec>=0.1.1",
 ]
+mcp = [
+    # The arka-tools MCP server runs under its own `uv` env in production
+    # (mcps/arka-tools/pyproject.toml); this extra lets the CORE test suite
+    # import the FastMCP decorators so the arka-tools security tests
+    # (OWASP LLM01 neutralization, write-gating) run on every Python
+    # version, not just where mcp happens to be installed.
+    "mcp[cli]>=1.2.0",
+]
 dashboard = [
     "fastapi>=0.115.0",
     "uvicorn>=0.32.0",

package/scripts/marketplace_gen.py

@@ -0,0 +1,310 @@
+"""Marketplace generator — per-department plugins from the curated cut.
+
+F2-7b (ruflo-teardown campaign, distribution endgame): the ArkaOS
+marketplace answers ruflo's 38-plugin distribution surface WITHOUT its
+344-SKILL.md context blowout. The curated core stays on the installer
+channel (multi-runtime); every sub-skill OUTSIDE the curated set ships
+à la carte in a per-department plugin, where the plugin namespace
+(``arkaos-<dept>:<skill>``) also disambiguates the later-wins collision
+slugs natively.
+
+Single source of truth: ``config/skills-curated.yaml`` + the
+``departments/`` tree. This script GENERATES (never hand-typed —
+docs_stats/gate_manifest precedent):
+
+- ``plugins/arkaos-<dept>/`` — one plugin per department with a
+  non-empty complement (dept sub-skills minus curated), skills exported
+  with the marketplace transformations;
+- ``.claude-plugin/marketplace.json`` — EXTENDED, never replaced: the
+  ``name``/``owner`` fields and the legacy ``arkaos-dev-skills`` entry
+  are preserved verbatim (that plugin may already be installed on user
+  machines and its 10-skill list is locked by
+  ``test_marketplace_export.py``); top-level ``version`` tracks the
+  VERSION file (kills the stale ``metadata.version`` for good);
+- ``knowledge/skills-manifest.json`` — slug -> {depts, curated,
+  plugins, collision} map consumed by /do install hints, the doctor and
+  the F2-7c skill-budget linter.
+
+Regenerate + commit after touching the YAML, any SKILL.md set, or
+VERSION:
+
+    python scripts/marketplace_gen.py
+
+``tests/python/test_marketplace_gen.py`` drift-gates the committed tree
+against a fresh in-memory run, byte for byte.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+import shutil
+import sys
+from pathlib import Path
+
+import yaml
+
+REPO_ROOT = Path(__file__).resolve().parent.parent
+if str(REPO_ROOT / "scripts") not in sys.path:
+    sys.path.insert(0, str(REPO_ROOT / "scripts"))
+
+from marketplace_export import _convert  # noqa: E402
+
+CURATED_YAML = REPO_ROOT / "config" / "skills-curated.yaml"
+DEPARTMENTS_DIR = REPO_ROOT / "departments"
+PLUGINS_DIR = REPO_ROOT / "plugins"
+MARKETPLACE_JSON = REPO_ROOT / ".claude-plugin" / "marketplace.json"
+SKILLS_MANIFEST_JSON = REPO_ROOT / "knowledge" / "skills-manifest.json"
+VERSION = (REPO_ROOT / "VERSION").read_text(encoding="utf-8").strip()
+
+MARKETPLACE_NAME = "arkaos"
+LEGACY_PLUGIN = "arkaos-dev-skills"
+_RESOURCE_DIRS = ("scripts", "references", "assets")
+
+# The export _convert strips slash-command suffixes but lets markdown
+# links to ArkaOS commands through (observed leak:
+# "[Browser Integration Pattern](/arka)" in marketplace/skills/
+# code-review). Fifth transformation: unwrap them to plain text.
+_ARKA_LINK_PATTERN = re.compile(r"\[([^\]]+)\]\(/arka[^)]*\)")
+
+_DEPT_LABELS = {
+    "brand": "Brand & Design",
+    "community": "Communities & Groups",
+    "content": "Content & Viralization",
+    "dev": "Development",
+    "ecom": "E-Commerce",
+    "finance": "Finance & Investment",
+    "kb": "Knowledge Management",
+    "landing": "Landing Pages & Funnels",
+    "leadership": "Leadership & People",
+    "marketing": "Marketing & Growth",
+    "ops": "Operations & Automation",
+    "org": "Organization & Teams",
+    "pm": "Project Management",
+    "quality": "Quality",
+    "saas": "SaaS & Micro-SaaS",
+    "sales": "Sales & Negotiation",
+    "strategy": "Strategy & Innovation",
+}
+
+
+def load_curated() -> dict[str, list[str]]:
+    config = yaml.safe_load(CURATED_YAML.read_text(encoding="utf-8"))
+    return {
+        dept: sorted(slugs)
+        for dept, slugs in config["curated_subskills"].items()
+    }
+
+
+def dept_subskills() -> dict[str, list[str]]:
+    """All sub-skill slugs per department, from disk."""
+    result: dict[str, list[str]] = {}
+    for dept_dir in sorted(DEPARTMENTS_DIR.iterdir()):
+        skills_dir = dept_dir / "skills"
+        if not skills_dir.is_dir():
+            continue
+        slugs = sorted(
+            p.name for p in skills_dir.iterdir()
+            if (p / "SKILL.md").is_file()
+        )
+        if slugs:
+            result[dept_dir.name] = slugs
+    return result
+
+
+def collision_slugs(subskills: dict[str, list[str]]) -> dict[str, list[str]]:
+    """Slugs owned by more than one department (later-wins on deploy)."""
+    owners: dict[str, list[str]] = {}
+    for dept, slugs in subskills.items():
+        for slug in slugs:
+            owners.setdefault(slug, []).append(dept)
+    return {s: sorted(d) for s, d in owners.items() if len(d) > 1}
+
+
+_NAME_LINE_PATTERN = re.compile(r"^name:\s*.*$", flags=re.MULTILINE)
+
+
+def convert_skill(body: str, slug: str) -> str:
+    """The 4 marketplace_export transformations + two plugin-specific
+    ones: unwrap markdown links to ArkaOS commands, and FORCE the
+    frontmatter name to the directory slug — the in-tree files carry
+    `name: <dept>/<name>` (235 of 265) and sometimes a name that is not
+    the dir slug at all (`logo` in `logo-brief/`); the plugin runtime
+    rejects name/dir mismatches."""
+    out = _ARKA_LINK_PATTERN.sub(r"\1", _convert(body))
+    return _NAME_LINE_PATTERN.sub(f"name: {slug}", out, count=1)
+
+
+def _plugin_description(dept: str, slugs: list[str]) -> str:
+    label = _DEPT_LABELS.get(dept, dept.title())
+    shown = ", ".join(slugs[:8])
+    more = f", +{len(slugs) - 8} more" if len(slugs) > 8 else ""
+    return (
+        f"ArkaOS {label} skills, à la carte — {len(slugs)} framework-backed "
+        f"skills beyond the curated core: {shown}{more}. Generated by "
+        f"scripts/marketplace_gen.py from departments/{dept}/skills/."
+    )
+
+
+def _emit_skill(src_dir: Path, dest_dir: Path) -> None:
+    dest_dir.mkdir(parents=True, exist_ok=True)
+    body = (src_dir / "SKILL.md").read_text(encoding="utf-8")
+    (dest_dir / "SKILL.md").write_text(
+        convert_skill(body, dest_dir.name), encoding="utf-8")
+    for resource in _RESOURCE_DIRS:
+        src = src_dir / resource
+        if src.is_dir():
+            shutil.copytree(src, dest_dir / resource, dirs_exist_ok=True)
+
+
+def build_plugins() -> dict[str, list[str]]:
+    """Write plugins/arkaos-<dept>/ trees. Returns dept -> plugin slugs."""
+    curated = load_curated()
+    subskills = dept_subskills()
+    if PLUGINS_DIR.exists():
+        shutil.rmtree(PLUGINS_DIR)
+    emitted: dict[str, list[str]] = {}
+    for dept, slugs in subskills.items():
+        complement = [s for s in slugs if s not in set(curated.get(dept, []))]
+        if not complement:
+            continue
+        plugin_dir = PLUGINS_DIR / f"arkaos-{dept}"
+        for slug in complement:
+            _emit_skill(
+                DEPARTMENTS_DIR / dept / "skills" / slug,
+                plugin_dir / "skills" / slug,
+            )
+        _write_plugin_manifest(plugin_dir, dept, complement)
+        emitted[dept] = complement
+    return emitted
+
+
+def _write_plugin_manifest(
+    plugin_dir: Path, dept: str, slugs: list[str]
+) -> None:
+    manifest_dir = plugin_dir / ".claude-plugin"
+    manifest_dir.mkdir(parents=True, exist_ok=True)
+    plugin_json = {
+        "name": f"arkaos-{dept}",
+        "description": _plugin_description(dept, slugs),
+        # Version lives ONLY here (plugin.json wins silently over a
+        # marketplace-entry version per the plugin spec — never both).
+        "version": VERSION,
+        "author": {"name": "WizardingCode", "email": "hello@wizardingcode.io"},
+        "license": "MIT",
+    }
+    (manifest_dir / "plugin.json").write_text(
+        json.dumps(plugin_json, indent=2, ensure_ascii=False) + "\n",
+        encoding="utf-8",
+    )
+
+
+def build_marketplace(emitted: dict[str, list[str]]) -> dict:
+    """Extend the existing manifest — name/owner/legacy entry verbatim."""
+    manifest = json.loads(MARKETPLACE_JSON.read_text(encoding="utf-8"))
+    if manifest.get("name") != MARKETPLACE_NAME:
+        raise ValueError(
+            f"marketplace name changed ({manifest.get('name')!r}) — the "
+            f"registered name is load-bearing on user machines; refusing"
+        )
+    legacy = [p for p in manifest["plugins"] if p.get("name") == LEGACY_PLUGIN]
+    if len(legacy) != 1:
+        raise ValueError("legacy arkaos-dev-skills entry missing — refusing")
+    # Top-level version tracks VERSION (metadata.version is deprecated
+    # upstream; the metadata KEY stays one release for backwards
+    # compatibility, but its value tracks VERSION too — the stale 2.71.0
+    # it carried was exactly the hand-typed drift this generator kills).
+    manifest["version"] = VERSION
+    if isinstance(manifest.get("metadata"), dict):
+        manifest["metadata"]["version"] = VERSION
+    dept_entries = [
+        {
+            "name": f"arkaos-{dept}",
+            "source": f"./plugins/arkaos-{dept}",
+            "category": dept,
+            "description": _plugin_description(dept, slugs),
+        }
+        for dept, slugs in sorted(emitted.items())
+    ]
+    manifest["plugins"] = legacy + dept_entries
+    return manifest
+
+
+def build_skills_manifest(emitted: dict[str, list[str]]) -> dict:
+    curated = load_curated()
+    subskills = dept_subskills()
+    collisions = collision_slugs(subskills)
+    skills: dict[str, dict] = {}
+    for dept, slugs in subskills.items():
+        for slug in slugs:
+            entry = skills.setdefault(slug, {
+                "depts": [], "curated": False, "plugins": [],
+                "collision": slug in collisions,
+            })
+            entry["depts"].append(dept)
+            if slug in set(curated.get(dept, [])):
+                entry["curated"] = True
+            elif dept in emitted and slug in emitted[dept]:
+                entry["plugins"].append(f"arkaos-{dept}@{MARKETPLACE_NAME}")
+    for entry in skills.values():
+        entry["depts"].sort()
+        entry["plugins"].sort()
+    return {
+        "_meta": {
+            "generator": "scripts/marketplace_gen.py",
+            "version": VERSION,
+            "marketplace": MARKETPLACE_NAME,
+        },
+        "structural": _structural_surface(),
+        "skills": dict(sorted(skills.items())),
+    }
+
+
+def _structural_surface() -> dict:
+    return {
+        "main": "arka",
+        "hubs": sorted(
+            f"arka-{d.name}" for d in DEPARTMENTS_DIR.iterdir()
+            if (d / "SKILL.md").is_file()
+        ),
+        "meta": sorted(
+            f"arka-{p.name}" for p in (REPO_ROOT / "arka" / "skills").iterdir()
+            if (p / "SKILL.md").is_file()
+        ),
+    }
+
+
+def generate() -> dict:
+    """Full generation. Returns summary counts for the CLI output."""
+    emitted = build_plugins()
+    marketplace = build_marketplace(emitted)
+    MARKETPLACE_JSON.write_text(
+        json.dumps(marketplace, indent=2, ensure_ascii=False) + "\n",
+        encoding="utf-8",
+    )
+    skills_manifest = build_skills_manifest(emitted)
+    SKILLS_MANIFEST_JSON.write_text(
+        json.dumps(skills_manifest, indent=2, ensure_ascii=False) + "\n",
+        encoding="utf-8",
+    )
+    return {
+        "plugins": len(emitted),
+        "plugin_skills": sum(len(s) for s in emitted.values()),
+        "curated": sum(len(s) for s in load_curated().values()),
+        "collisions": len(collision_slugs(dept_subskills())),
+    }
+
+
+def main() -> int:
+    summary = generate()
+    print(
+        f"marketplace generated: {summary['plugins']} dept plugins, "
+        f"{summary['plugin_skills']} plugin skills, "
+        f"{summary['curated']} curated in core, "
+        f"{summary['collisions']} collision slugs (plugin-namespaced)"
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/scripts/tools/docs_stats.py

@@ -20,13 +20,12 @@ import re
 import subprocess
 import sys
 from pathlib import Path
-from typing import Optional
 
 _TEST_DEF_RE = re.compile(r"^\s*(?:async\s+)?def\s+test_\w+", re.MULTILINE)
 _COLLECTED_RE = re.compile(r"(\d+)\s+tests?\s+collected")
 
 
-def repo_root(start: Optional[Path] = None) -> Path:
+def repo_root(start: Path | None = None) -> Path:
     """Find the repo root by walking up to a dir with VERSION + departments/."""
     cur = (start or Path(__file__).resolve()).resolve()
     candidates = [cur, *cur.parents] if cur.is_dir() else [cur.parent, *cur.parents]
@@ -64,8 +63,11 @@ def count_skills(root: Path) -> dict:
         return len(list(base.rglob("SKILL.md"))) if base.is_dir() else 0
 
     dept, arka, market = _n("departments"), _n("arka"), _n("marketplace")
+    # plugins/ is a GENERATED export-copy (scripts/marketplace_gen.py)
+    # of non-curated department skills — informative only, never part
+    # of the core headline (the sources remain under departments/).
     return {"departments": dept, "arka": arka, "marketplace": market,
-            "core": dept + arka}
+            "plugins": _n("plugins"), "core": dept + arka}
 
 
 def count_adrs(root: Path) -> int:
@@ -83,7 +85,7 @@ def count_test_functions(root: Path) -> int:
                for f in tdir.rglob("test_*.py"))
 
 
-def collect_pytest_cases(root: Path) -> Optional[int]:
+def collect_pytest_cases(root: Path) -> int | None:
     """Authoritative pytest case count via --collect-only. None on failure."""
     try:
         out = subprocess.run(

package/scripts/tools/skill_budget.py

@@ -0,0 +1,209 @@
+"""Skill-budget linter — the curated default must stay small (F2-7c).
+
+Measures the ALWAYS-ON context surface of the default skill install:
+the frontmatter descriptions of every skill the curated cut deploys
+(main /arka + 17 hubs + 14 meta + curated sub-skills, resolved from
+``knowledge/skills-manifest.json`` — the generated classification the
+marketplace generator emits). Skill bodies are invoke-only and do not
+count; descriptions are what the runtime can inject per session.
+
+Honesty note, printed with every report: live sessions currently
+observe a smaller footprint (~2.3k tokens) because the runtime
+truncates most entries to name-only — but that truncation is
+NON-CONTRACTUAL runtime behavior. This budget sizes the GUARANTEED
+surface (chars of description; tokens estimated at chars/4).
+
+Thresholds (calibrated against the measured 2026-07-12 baseline:
+69 skills, 34,929 chars):
+
+    skills   WARN > 72 (90% of cap)   FAIL > 80
+    total    WARN > 36,000 chars      FAIL > 40,000 (~10k tokens)
+    per-skill WARN > 800 chars        FAIL > 1,024
+    structural completeness           FAIL (flow & co. must be default)
+    curated slug in a collision set   FAIL
+
+CI gate: ``tests/python/test_skill_budget.py`` (repo-only, no install
+needed). CLI: ``python scripts/tools/skill_budget.py [--json]`` —
+exit 1 when any FAIL finding exists. stdlib only (docs_stats pattern).
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import re
+import sys
+from dataclasses import asdict, dataclass
+from pathlib import Path
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+MANIFEST_PATH = REPO_ROOT / "knowledge" / "skills-manifest.json"
+
+MAX_SKILLS_FAIL = 80
+MAX_SKILLS_WARN = 72
+TOTAL_CHARS_FAIL = 40_000
+TOTAL_CHARS_WARN = 36_000
+PER_SKILL_FAIL = 1_024
+PER_SKILL_WARN = 800
+
+_FRONTMATTER_RE = re.compile(r"^---\n(.*?)\n---", re.DOTALL)
+_DESCRIPTION_RE = re.compile(r"^description:\s*(.*)$", re.MULTILINE)
+_ROOT_KEY_RE = re.compile(r"^[A-Za-z_-]+:")
+
+
+@dataclass(frozen=True)
+class Finding:
+    level: str  # "FAIL" | "WARN"
+    check: str
+    detail: str
+
+
+def frontmatter_description(skill_md: Path) -> str:
+    """Extract the description value, including folded (>) blocks."""
+    match = _FRONTMATTER_RE.match(skill_md.read_text(encoding="utf-8"))
+    if not match:
+        return ""
+    frontmatter = match.group(1)
+    desc = _DESCRIPTION_RE.search(frontmatter)
+    if not desc:
+        return ""
+    value = desc.group(1).strip()
+    if value not in (">", "|", ">-", "|-"):
+        return value
+    folded: list[str] = []
+    for line in frontmatter[desc.end():].split("\n"):
+        if line.startswith((" ", "\t")):
+            folded.append(line.strip())
+        elif _ROOT_KEY_RE.match(line):
+            break
+    return " ".join(folded).strip()
+
+
+def default_surface(manifest: dict) -> dict[str, Path]:
+    """Deployed-name -> source SKILL.md for the curated default install."""
+    surface: dict[str, Path] = {"arka": REPO_ROOT / "arka" / "SKILL.md"}
+    for hub in manifest["structural"]["hubs"]:
+        dept = hub.removeprefix("arka-")
+        surface[hub] = REPO_ROOT / "departments" / dept / "SKILL.md"
+    for meta in manifest["structural"]["meta"]:
+        slug = meta.removeprefix("arka-")
+        surface[meta] = REPO_ROOT / "arka" / "skills" / slug / "SKILL.md"
+    for slug, entry in manifest["skills"].items():
+        if entry["curated"]:
+            dept = entry["depts"][0]
+            surface[f"arka-{slug}"] = (
+                REPO_ROOT / "departments" / dept / "skills" / slug / "SKILL.md"
+            )
+    return surface
+
+
+def _threshold_findings(count: int, total: int) -> list[Finding]:
+    findings: list[Finding] = []
+    if count > MAX_SKILLS_FAIL:
+        findings.append(Finding(
+            "FAIL", "skill-count",
+            f"default install carries {count} skills (cap {MAX_SKILLS_FAIL})"))
+    elif count > MAX_SKILLS_WARN:
+        findings.append(Finding(
+            "WARN", "skill-count",
+            f"{count} skills — approaching the {MAX_SKILLS_FAIL} cap"))
+    if total > TOTAL_CHARS_FAIL:
+        findings.append(Finding(
+            "FAIL", "total-description-chars",
+            f"{total} chars of always-on descriptions "
+            f"(cap {TOTAL_CHARS_FAIL} ~= {TOTAL_CHARS_FAIL // 4} tokens)"))
+    elif total > TOTAL_CHARS_WARN:
+        findings.append(Finding(
+            "WARN", "total-description-chars",
+            f"{total} chars — approaching the {TOTAL_CHARS_FAIL} cap"))
+    return findings
+
+
+def _structural_findings(manifest: dict) -> list[Finding]:
+    findings: list[Finding] = []
+    meta = set(manifest["structural"]["meta"])
+    for required in ("arka-flow", "arka-forge", "arka-fusion"):
+        if required not in meta:
+            findings.append(Finding(
+                "FAIL", "structural-completeness",
+                f"{required} missing from the structural default — "
+                f"evidence-flow and planning are NON-NEGOTIABLE"))
+    for slug, entry in manifest["skills"].items():
+        if entry["curated"] and entry["collision"]:
+            findings.append(Finding(
+                "FAIL", "curated-collision",
+                f"curated slug {slug!r} is a later-wins collision — it "
+                f"must live only in plugins"))
+    return findings
+
+
+def audit(manifest_path: Path | None = None) -> dict:
+    """Full audit. Returns {summary, findings, note}.
+
+    ``manifest_path`` resolves the module attribute at CALL time so
+    tests can monkeypatch ``MANIFEST_PATH`` (a def-time default would
+    freeze the original binding).
+    """
+    manifest_path = manifest_path or MANIFEST_PATH
+    manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
+    surface = default_surface(manifest)
+    sizes = {
+        name: len(frontmatter_description(path))
+        for name, path in surface.items() if path.is_file()
+    }
+    missing = sorted(set(surface) - set(sizes))
+    total = sum(sizes.values())
+    findings = _threshold_findings(len(surface), total)
+    findings.extend(_structural_findings(manifest))
+    for name in missing:
+        findings.append(Finding(
+            "FAIL", "missing-source", f"{name}: source SKILL.md not found"))
+    for name, size in sorted(sizes.items(), key=lambda kv: -kv[1]):
+        if size > PER_SKILL_FAIL:
+            findings.append(Finding(
+                "FAIL", "per-skill-description",
+                f"{name}: {size} chars (cap {PER_SKILL_FAIL})"))
+        elif size > PER_SKILL_WARN:
+            findings.append(Finding(
+                "WARN", "per-skill-description",
+                f"{name}: {size} chars (warn over {PER_SKILL_WARN})"))
+    return {
+        "summary": {
+            "skills": len(surface),
+            "total_description_chars": total,
+            "estimated_tokens": total // 4,
+            "fails": sum(1 for f in findings if f.level == "FAIL"),
+            "warns": sum(1 for f in findings if f.level == "WARN"),
+        },
+        "findings": [asdict(f) for f in findings],
+        "note": (
+            "Budget sizes the GUARANTEED always-on surface (description "
+            "chars). Live sessions may observe less (~2.3k tokens) because "
+            "the runtime truncates entries to name-only — that truncation "
+            "is non-contractual and not relied upon."
+        ),
+    }
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(
+        prog="python scripts/tools/skill_budget.py",
+        description="Audit the curated default skill surface. Exit 1 on FAIL.",
+    )
+    parser.add_argument("--json", action="store_true", dest="as_json")
+    args = parser.parse_args(argv)
+    report = audit()
+    if args.as_json:
+        print(json.dumps(report, indent=2))
+    else:
+        s = report["summary"]
+        print(f"skills: {s['skills']} | always-on description chars: "
+              f"{s['total_description_chars']} (~{s['estimated_tokens']} tokens)")
+        for f in report["findings"]:
+            print(f"  [{f['level']}] {f['check']}: {f['detail']}")
+        print(report["note"])
+    return 1 if report["summary"]["fails"] else 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())