arkaos 4.26.0 → 4.28.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "arkaos",
-  "version": "4.26.0",
+  "version": "4.28.0",
   "description": "The Operating System for AI Agent Teams",
   "type": "module",
   "bin": {

package/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "arkaos-core"
-version = "4.26.0"
+version = "4.28.0"
 description = "Core engine for ArkaOS — The Operating System for AI Agent Teams"
 readme = "README.md"
 license = {text = "MIT"}

package/scripts/tools/evals_import.py

@@ -0,0 +1,183 @@
+#!/usr/bin/env python3
+"""Import the marketing-skills eval corpus into config/evals/<dept>.yaml.
+
+Converts the 45 upstream ``evals/evals.json`` files from
+coreyhaines31/marketingskills (MIT) into the ArkaOS EvalTask schema
+(``core/evals/schema.py``): one YAML list per department, appended after
+each department's hand-written seed tasks.
+
+Mapping per case:
+  id                  = "<dept>-<arkaos-slug>-NN"   (kebab, unique)
+  department          = the owning ArkaOS department
+  prompt              = upstream prompt verbatim
+  expected_properties = upstream assertions[] (the verifiable checklist)
+  rubric              = upstream expected_output (free-text judging guidance)
+  tags                = [<arkaos-slug>, "imported"]
+
+Deterministic and idempotent: re-running replaces the ``imported`` block
+(everything tagged ``imported``) and preserves the seed tasks above it.
+Third-party attribution: docs/THIRD-PARTY-NOTICES.md + skills-provenance.
+
+Usage:
+  arka-py scripts/tools/evals_import.py --source <clone>/skills [--check]
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+import yaml
+
+# Corey skill name -> (ArkaOS department, ArkaOS skill slug). The department
+# decides which config/evals/<dept>.yaml the cases land in; the slug builds
+# the id and the tag. Mirrors the campaign disposition table.
+SKILL_MAP: dict[str, tuple[str, str]] = {
+    # marketing
+    "ab-testing": ("marketing", "ab-test"),
+    "ad-creative": ("marketing", "ad-creative"),
+    "ads": ("marketing", "paid-campaign"),
+    "ai-seo": ("marketing", "ai-seo"),
+    "analytics": ("marketing", "analytics-report"),
+    "aso": ("marketing", "aso"),
+    "co-marketing": ("marketing", "co-marketing"),
+    "cold-email": ("marketing", "cold-email"),
+    "competitor-profiling": ("marketing", "competitor-profile"),
+    "competitors": ("marketing", "competitor-analysis"),
+    "content-strategy": ("marketing", "content-strategy"),
+    "customer-research": ("marketing", "customer-research"),
+    "directory-submissions": ("marketing", "directory-submit"),
+    "emails": ("marketing", "email-sequence"),
+    "free-tools": ("marketing", "free-tools"),
+    "marketing-council": ("marketing", "council"),
+    "marketing-ideas": ("marketing", "marketing-ideas"),
+    "marketing-loops": ("marketing", "marketing-loops"),
+    "marketing-plan": ("marketing", "marketing-plan"),
+    "marketing-psychology": ("marketing", "marketing-psychology"),
+    "product-marketing": ("marketing", "product-marketing"),
+    "programmatic-seo": ("marketing", "programmatic-seo"),
+    "schema": ("marketing", "schema-markup"),
+    "seo-audit": ("marketing", "seo-audit"),
+    "social": ("marketing", "social-strategy"),
+    "sms": ("marketing", "sms-campaign"),
+    "copy-editing": ("marketing", "human-writing"),
+    # landing
+    "copywriting": ("landing", "copy-framework"),
+    "cro": ("landing", "optimize-page"),
+    "lead-magnets": ("landing", "lead-magnet"),
+    "popups": ("landing", "popup-design"),
+    "signup": ("landing", "signup-optimize"),
+    "site-architecture": ("landing", "page-architect"),
+    "referrals": ("landing", "affiliate-bridge"),
+    # saas
+    "churn-prevention": ("saas", "churn-analysis"),
+    "onboarding": ("saas", "onboarding-optimize"),
+    "paywalls": ("saas", "paywall-design"),
+    "pricing": ("saas", "pricing-strategy"),
+    "launch": ("saas", "launch-execute"),
+    # sales
+    "prospecting": ("sales", "prospecting"),
+    "revops": ("sales", "revops"),
+    "sales-enablement": ("sales", "sales-enablement"),
+    # community
+    "community-marketing": ("community", "community-marketing"),
+    # content
+    "image": ("content", "image-create"),
+    "video": ("content", "video-produce"),
+}
+
+IMPORTED_TAG = "imported"
+_HEADER = "# --- BEGIN imported (marketing-skills, MIT) — regenerated by scripts/tools/evals_import.py ---"
+_FOOTER = "# --- END imported ---"
+
+
+def _case_to_task(case: dict, dept: str, slug: str, n: int) -> dict:
+    props = [a.strip() for a in case.get("assertions", []) if a.strip()]
+    if not props:
+        # Schema requires >= 1 property; fall back to the expected_output.
+        props = [case.get("expected_output", "meets the skill's stated behavior").strip()]
+    task = {
+        "id": f"{dept}-{slug}-{n:02d}",
+        "department": dept,
+        "prompt": case["prompt"].strip(),
+        "expected_properties": props,
+    }
+    rubric = (case.get("expected_output") or "").strip()
+    if rubric:
+        task["rubric"] = rubric
+    task["tags"] = [slug, IMPORTED_TAG]
+    return task
+
+
+def build_tasks_by_dept(source: Path) -> dict[str, list[dict]]:
+    by_dept: dict[str, list[dict]] = {}
+    for corey_skill, (dept, slug) in sorted(SKILL_MAP.items()):
+        ev = source / corey_skill / "evals" / "evals.json"
+        if not ev.exists():
+            continue
+        cases = json.loads(ev.read_text(encoding="utf-8")).get("evals", [])
+        for i, case in enumerate(cases, start=1):
+            by_dept.setdefault(dept, []).append(_case_to_task(case, dept, slug, i))
+    return by_dept
+
+
+def _split_seed(text: str) -> str:
+    """Return the seed content (imported block removed), or "" if none.
+
+    An empty seed must render as "" — not "\\n" — so a freshly created
+    dept file (no hand-written seed) round-trips identically on re-run.
+    """
+    head = text.split(_HEADER)[0] if _HEADER in text else text
+    seed = head.rstrip()
+    return seed + "\n" if seed else ""
+
+
+def render(evals_dir: Path, by_dept: dict[str, list[dict]]) -> dict[str, str]:
+    """Compute the desired content of each touched dept yaml (seed + imported)."""
+    out: dict[str, str] = {}
+    for dept, tasks in sorted(by_dept.items()):
+        path = evals_dir / f"{dept}.yaml"
+        seed = _split_seed(path.read_text(encoding="utf-8")) if path.exists() else ""
+        block = yaml.safe_dump(
+            tasks, sort_keys=False, allow_unicode=True, width=88, default_flow_style=False
+        )
+        body = seed
+        if seed and not seed.endswith("\n\n"):
+            body = seed.rstrip("\n") + "\n\n"
+        out[dept] = f"{body}{_HEADER}\n{block}{_FOOTER}\n"
+    return out
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--source", required=True, help="path to the marketing-skills skills/ dir")
+    ap.add_argument("--evals-dir", default="config/evals")
+    ap.add_argument("--check", action="store_true", help="exit 1 if files would change")
+    args = ap.parse_args()
+
+    source = Path(args.source)
+    evals_dir = Path(args.evals_dir)
+    by_dept = build_tasks_by_dept(source)
+    desired = render(evals_dir, by_dept)
+
+    total = sum(len(v) for v in by_dept.values())
+    changed = False
+    for dept, content in desired.items():
+        path = evals_dir / f"{dept}.yaml"
+        current = path.read_text(encoding="utf-8") if path.exists() else ""
+        if current != content:
+            changed = True
+            if not args.check:
+                path.write_text(content, encoding="utf-8")
+    dept_counts = ", ".join(f"{d}:{len(v)}" for d, v in sorted(by_dept.items()))
+    print(f"imported {total} eval cases across {len(by_dept)} departments ({dept_counts})")
+    if args.check and changed:
+        print("config/evals/*.yaml is stale — run evals_import.py", file=sys.stderr)
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())