arkaos 2.16.1 → 2.17.0

package/core/sync/content_syncer.py

@@ -0,0 +1,167 @@
+"""Content syncer for the ArkaOS Sync Engine.
+
+Syncs CLAUDE.md (with intelligent merge), rules, hooks, and a generated
+constitution excerpt into each project's .claude/ directory.
+"""
+
+from __future__ import annotations
+
+import os
+import shutil
+from pathlib import Path
+
+import yaml
+
+from core.sync.content_merger import merge_managed_content
+from core.sync.schema import ContentSyncResult, Project
+
+
+def _core_root() -> Path:
+    # Honors ARKAOS_CORE_ROOT env var for tests; falls back to repo root.
+    env = os.environ.get("ARKAOS_CORE_ROOT")
+    if env:
+        return Path(env)
+    return Path(__file__).resolve().parents[2]
+
+
+def sync_project_content(project: Project) -> ContentSyncResult:
+    """Sync CLAUDE.md, rules, hooks, and constitution excerpt for a project."""
+    try:
+        return _do_sync(project)
+    except Exception as exc:  # noqa: BLE001
+        return ContentSyncResult(
+            path=project.path, status="error", error=str(exc)
+        )
+
+
+def _do_sync(project: Project) -> ContentSyncResult:
+    core = _core_root()
+    version = (core / "VERSION").read_text().strip()
+    project_claude = Path(project.path) / ".claude"
+    project_claude.mkdir(parents=True, exist_ok=True)
+
+    updated: list[str] = []
+    unchanged: list[str] = []
+    errored: list[str] = []
+
+    _sync_claude_md(core, project, project_claude, version, updated, unchanged, errored)
+    _sync_rules(core, project_claude, updated, unchanged, errored)
+    _sync_hooks(core, project_claude, updated, unchanged, errored)
+    _sync_constitution(core, project_claude, updated, unchanged, errored)
+
+    if errored:
+        status = "error"
+    elif updated:
+        status = "updated"
+    else:
+        status = "unchanged"
+    return ContentSyncResult(
+        path=project.path,
+        status=status,
+        artefacts_updated=updated,
+        artefacts_unchanged=unchanged,
+        artefacts_errored=errored,
+    )
+
+
+def _sync_claude_md(
+    core: Path,
+    project: Project,
+    project_claude: Path,
+    version: str,
+    updated: list[str],
+    unchanged: list[str],
+    errored: list[str],
+) -> None:
+    base = (core / "config" / "user-claude.md").read_text()
+    overlays_dir = core / "config" / "standards" / "claude-md-overlays"
+    overlays: list[str] = []
+    for stack in project.stack:
+        overlay = overlays_dir / f"{stack}.md"
+        if overlay.exists():
+            overlays.append(overlay.read_text())
+
+    managed_content = "\n\n".join([base, *overlays]).strip()
+    target_file = project_claude / "CLAUDE.md"
+    target_text = target_file.read_text() if target_file.exists() else ""
+
+    result = merge_managed_content(target_text, managed_content, version)
+    if result.status == "error":
+        errored.append(f"CLAUDE.md: {result.error}")
+        sidecar = target_file.with_suffix(".md.arkaos-new")
+        sidecar.write_text(managed_content)
+        return
+    if result.status == "unchanged":
+        unchanged.append("CLAUDE.md")
+        return
+    target_file.write_text(result.new_text)
+    updated.append("CLAUDE.md")
+
+
+def _sync_rules(
+    core: Path,
+    project_claude: Path,
+    updated: list[str],
+    unchanged: list[str],
+    errored: list[str],
+) -> None:
+    # Copies/updates rules from core standards; does not delete orphan files.
+    src = core / "config" / "standards"
+    dst = project_claude / "rules"
+    dst.mkdir(parents=True, exist_ok=True)
+    for rule in src.glob("*.md"):
+        target = dst / rule.name
+        src_text = rule.read_text()
+        if target.exists() and target.read_text() == src_text:
+            unchanged.append(f"rules/{rule.name}")
+            continue
+        target.write_text(src_text)
+        updated.append(f"rules/{rule.name}")
+
+
+def _sync_hooks(
+    core: Path,
+    project_claude: Path,
+    updated: list[str],
+    unchanged: list[str],
+    errored: list[str],
+) -> None:
+    src = core / "config" / "hooks"
+    dst = project_claude / "hooks"
+    dst.mkdir(parents=True, exist_ok=True)
+    for hook in src.glob("*.sh"):
+        target = dst / hook.name
+        src_text = hook.read_text()
+        if target.exists() and target.read_text() == src_text:
+            unchanged.append(f"hooks/{hook.name}")
+            continue
+        shutil.copy2(hook, target)
+        target.chmod(0o755)
+        updated.append(f"hooks/{hook.name}")
+
+
+def _sync_constitution(
+    core: Path,
+    project_claude: Path,
+    updated: list[str],
+    unchanged: list[str],
+    errored: list[str],
+) -> None:
+    src = core / "config" / "constitution.yaml"
+    target = project_claude / "constitution-applicable.md"
+    data = yaml.safe_load(src.read_text()) or {}
+    rules = data.get("rules", [])
+    lines = ["# ArkaOS Constitution — Applicable Rules", ""]
+    for rule in rules:
+        lines.append(f"- **{rule.get('name', '?')}** — {rule.get('level', '?')}")
+    body = "\n".join(lines) + "\n"
+    if target.exists() and target.read_text() == body:
+        unchanged.append("constitution-applicable.md")
+        return
+    target.write_text(body)
+    updated.append("constitution-applicable.md")
+
+
+def sync_all_content(projects: list[Project]) -> list[ContentSyncResult]:
+    """Sync content artefacts for all projects."""
+    return [sync_project_content(p) for p in projects]

package/core/sync/engine.py

@@ -13,9 +13,12 @@ from pathlib import Path
 
 from core.sync.manifest import build_manifest
 from core.sync.discovery import discover_all_projects
+from core.sync.mcp_optimizer import optimize_all_mcps
 from core.sync.mcp_syncer import sync_all_mcps
 from core.sync.settings_syncer import sync_all_settings
 from core.sync.descriptor_syncer import sync_all_descriptors
+from core.sync.agent_provisioner import sync_all_agents
+from core.sync.content_syncer import sync_all_content
 from core.sync.reporter import build_report, format_report, write_sync_state
 from core.sync.schema import SyncReport
 
@@ -37,8 +40,23 @@ def run_sync(arkaos_home: Path, skills_dir: Path, home_path: str) -> SyncReport:
 
     registry_path = skills_dir / "arka" / "mcps" / "registry.json"
     mcp_results = sync_all_mcps(projects, registry_path, home_path)
+
+    policy_path = Path(__file__).resolve().parents[2] / "config" / "mcp-policy.yaml"
+    vault_path = Path.home() / ".arkaos" / "secrets.json"
+    cache_path = Path.home() / ".arkaos" / "mcp-decisions.cache.json"
+    if policy_path.exists():
+        mcp_results = optimize_all_mcps(
+            projects,
+            mcp_results,
+            policy_path,
+            vault_path if vault_path.exists() else None,
+            cache_path,
+        )
+
     settings_results = sync_all_settings(mcp_results)
     descriptor_results = sync_all_descriptors(projects)
+    content_results = sync_all_content(projects)
+    agent_results = sync_all_agents(projects)
 
     report = build_report(
         previous_version,
@@ -47,6 +65,8 @@ def run_sync(arkaos_home: Path, skills_dir: Path, home_path: str) -> SyncReport:
         settings_results,
         descriptor_results,
         [],
+        content_results=content_results,
+        agent_results=agent_results,
         new_features=manifest.new_features,
         deprecated_features=manifest.deprecated_features,
     )

package/core/sync/mcp_optimizer.py

@@ -0,0 +1,187 @@
+"""MCP Optimizer — narrows the active MCP list per project via policy,
+AI fallback, and per-project override; injects env secrets from a vault;
+generates .env.arkaos.example for missing values.
+
+Runs between mcp_syncer (produces full .mcp.json) and settings_syncer
+(writes enabledMcpjsonServers). Deferred MCPs are simply absent from the
+returned ``final_mcp_list`` — their definitions remain in ``.mcp.json``
+so user can opt-in later.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import stat
+from pathlib import Path
+
+import yaml
+
+from core.sync.ai_mcp_decider import decide_ambiguous
+from core.sync.policy_loader import decide, load_policy
+from core.sync.schema import McpSyncResult, Project
+
+
+def optimize_project_mcps(
+    project: Project,
+    mcp_result: McpSyncResult,
+    policy_path: Path,
+    vault_path: Path | None,
+    cache_path: Path,
+    call_ai=None,
+) -> McpSyncResult:
+    """Return a new McpSyncResult with deferred MCPs removed from final_mcp_list."""
+    try:
+        return _do_optimize(project, mcp_result, policy_path, vault_path, cache_path, call_ai)
+    except Exception as exc:  # noqa: BLE001
+        return mcp_result.model_copy(update={
+            "optimizer_warnings": list(mcp_result.optimizer_warnings) + [f"optimization failed: {exc}"],
+        })
+
+
+def _do_optimize(
+    project: Project,
+    mcp_result: McpSyncResult,
+    policy_path: Path,
+    vault_path: Path | None,
+    cache_path: Path,
+    call_ai=None,
+) -> McpSyncResult:
+    if mcp_result.status == "error" or not mcp_result.final_mcp_list:
+        return mcp_result
+
+    warnings: list[str] = list(mcp_result.optimizer_warnings)
+
+    policy = load_policy(policy_path)
+    pd = decide(policy, mcp_result.final_mcp_list, project.stack, project.ecosystem)
+    ai_decisions = decide_ambiguous(
+        pd.ambiguous, project.stack, project.ecosystem, cache_path, call_ai
+    )
+
+    active = set(pd.active)
+    deferred = set(pd.deferred)
+    for name, decision in ai_decisions.items():
+        (active if decision == "active" else deferred).add(name)
+
+    override, override_warnings = _load_override(Path(project.path))
+    warnings.extend(override_warnings)
+
+    # Fix 3: collision handling — force_active takes precedence
+    force_active = set(override.get("force_active", []))
+    force_deferred = set(override.get("force_deferred", []))
+    collisions = force_active & force_deferred
+    for c in sorted(collisions):
+        warnings.append(f"override collision for MCP '{c}': force_active takes precedence")
+
+    active = (active | force_active) - (force_deferred - force_active)
+    deferred = (deferred - force_active) | (force_deferred - force_active)
+
+    inject_warnings = _inject_env_vars(Path(project.path), vault_path, project.name)
+    warnings.extend(inject_warnings)
+
+    return mcp_result.model_copy(update={
+        "final_mcp_list": sorted(active),
+        "mcps_deferred": sorted(deferred),
+        "optimizer_warnings": warnings,
+    })
+
+
+def _load_override(project_path: Path) -> tuple[dict, list[str]]:
+    """Load per-project override YAML. Returns (data, warnings)."""
+    override = project_path / ".arkaos" / "mcp-override.yaml"
+    if not override.exists():
+        return {}, []
+    try:
+        return yaml.safe_load(override.read_text()) or {}, []
+    except (yaml.YAMLError, OSError) as exc:
+        return {}, [f"override YAML parse error: {exc}"]
+
+
+def _merge_env(servers: dict, merged_env: dict) -> tuple[bool, dict[str, str]]:
+    """Inject known env vars into server configs. Returns (changed, missing)."""
+    missing: dict[str, str] = {}
+    changed = False
+    for server_name, config in servers.items():
+        env = config.get("env") or {}
+        for var_name, current in env.items():
+            if current:
+                continue
+            if var_name in merged_env:
+                env[var_name] = merged_env[var_name]
+                changed = True
+            else:
+                missing[var_name] = server_name
+        if env:
+            config["env"] = env
+    return changed, missing
+
+
+def _inject_env_vars(project_path: Path, vault_path: Path | None, project_name: str) -> list[str]:
+    """Inject vault secrets into .mcp.json env vars. Returns warnings."""
+    mcp_file = project_path / ".mcp.json"
+    if not mcp_file.exists():
+        return []
+
+    try:
+        data = json.loads(mcp_file.read_text())
+    except (json.JSONDecodeError, OSError):
+        return [".mcp.json malformed; skipped env injection"]
+
+    servers = data.get("mcpServers", {})
+    vault, vault_warnings = _load_vault(vault_path) if vault_path else ({}, [])
+
+    global_env = vault.get("global", {})
+    project_env = vault.get("per_project", {}).get(project_name, {})
+    merged_env = {**global_env, **project_env}
+
+    changed, missing = _merge_env(servers, merged_env)
+
+    if changed:
+        mcp_file.write_text(json.dumps(data, indent=2) + "\n")
+
+    if missing:
+        _write_env_example(project_path, missing)
+
+    return vault_warnings
+
+
+def _load_vault(path: Path) -> tuple[dict, list[str]]:
+    """Load the secrets vault JSON. Returns (data, warnings)."""
+    if not path.exists():
+        return {}, []
+    try:
+        st = path.stat()
+    except OSError:
+        return {}, []
+    # Refuse world- or group-readable files
+    if st.st_mode & (stat.S_IRWXG | stat.S_IRWXO):
+        return {}, ["vault permissions too permissive (group/world readable); secrets not injected"]
+    try:
+        return json.loads(path.read_text()), []
+    except (json.JSONDecodeError, OSError):
+        return {}, ["vault JSON parse error; secrets not injected"]
+
+
+def _write_env_example(project_path: Path, missing: dict[str, str]) -> None:
+    lines = ["# Auto-generated by ArkaOS MCP Optimizer", ""]
+    for var, server in sorted(missing.items()):
+        lines.append(f"# required by {server}")
+        lines.append(f"{var}=")
+    (project_path / ".env.arkaos.example").write_text("\n".join(lines) + "\n")
+
+
+def optimize_all_mcps(
+    projects: list[Project],
+    mcp_results: list[McpSyncResult],
+    policy_path: Path,
+    vault_path: Path | None,
+    cache_path: Path,
+) -> list[McpSyncResult]:
+    by_path = {r.path: r for r in mcp_results}
+    out: list[McpSyncResult] = []
+    for p in projects:
+        mr = by_path.get(p.path)
+        if mr is None:
+            continue
+        out.append(optimize_project_mcps(p, mr, policy_path, vault_path, cache_path))
+    return out

package/core/sync/policy_loader.py

@@ -0,0 +1,94 @@
+"""MCP policy loader and matcher for the ArkaOS Sync Engine."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from pathlib import Path
+
+import yaml
+
+
+@dataclass
+class PolicyRule:
+    match: dict
+    active: list[str] = field(default_factory=list)
+    deferred: list[str] = field(default_factory=list)
+    ambiguous: list[str] = field(default_factory=list)
+
+
+@dataclass
+class Policy:
+    rules: list[PolicyRule]
+
+
+@dataclass
+class PolicyDecision:
+    active: list[str]
+    deferred: list[str]
+    ambiguous: list[str]
+
+
+def load_policy(path: Path) -> Policy:
+    """Load and parse an mcp-policy.yaml file."""
+    data = yaml.safe_load(path.read_text()) or {}
+    rules = [
+        PolicyRule(
+            match=r.get("match", {}),
+            active=list(r.get("active", [])),
+            deferred=list(r.get("deferred", [])),
+            ambiguous=list(r.get("ambiguous", [])),
+        )
+        for r in data.get("policies", [])
+    ]
+    return Policy(rules=rules)
+
+
+def decide(
+    policy: Policy,
+    mcps: list[str],
+    stack: list[str],
+    ecosystem: str | None,
+) -> PolicyDecision:
+    """Apply the first matching rule and classify each MCP.
+
+    MCPs explicitly listed in the matched rule go to active/deferred/ambiguous
+    as declared. Any MCP not explicitly classified falls through to ambiguous
+    (the AI fallback decides).
+    """
+    if not mcps:
+        return PolicyDecision(active=[], deferred=[], ambiguous=[])
+
+    rule = _first_match(policy, stack, ecosystem)
+    if rule is None:
+        return PolicyDecision(active=[], deferred=[], ambiguous=list(mcps))
+
+    active: list[str] = []
+    deferred: list[str] = []
+    ambiguous: list[str] = []
+
+    for mcp in mcps:
+        if mcp in rule.active:
+            active.append(mcp)
+        elif mcp in rule.deferred:
+            deferred.append(mcp)
+        else:
+            ambiguous.append(mcp)
+
+    return PolicyDecision(active=active, deferred=deferred, ambiguous=ambiguous)
+
+
+def _first_match(
+    policy: Policy, stack: list[str], ecosystem: str | None
+) -> PolicyRule | None:
+    stack_set = {s.lower() for s in stack}
+    for rule in policy.rules:
+        match = rule.match
+        if match.get("default"):
+            return rule
+        stack_inc = match.get("stack_includes")
+        if stack_inc and any(s.lower() in stack_set for s in stack_inc):
+            return rule
+        eco = match.get("ecosystem")
+        if eco and ecosystem and eco == ecosystem:
+            return rule
+    return None

package/core/sync/reporter.py

@@ -10,6 +10,8 @@ from datetime import datetime, timezone
 from pathlib import Path
 
 from core.sync.schema import (
+    AgentProvisionResult,
+    ContentSyncResult,
     DescriptorSyncResult,
     McpSyncResult,
     SettingsSyncResult,
@@ -34,9 +36,18 @@ def build_report(
     skill_results: list[SkillSyncResult],
     new_features: list[str] | None = None,
     deprecated_features: list[str] | None = None,
+    content_results: list[ContentSyncResult] | None = None,
+    agent_results: list[AgentProvisionResult] | None = None,
 ) -> SyncReport:
     """Aggregate all sync results into a SyncReport."""
-    errors = _collect_errors(mcp_results, settings_results, descriptor_results, skill_results)
+    errors = _collect_errors(
+        mcp_results,
+        settings_results,
+        descriptor_results,
+        skill_results,
+        content_results=content_results,
+        agent_results=agent_results,
+    )
     return SyncReport(
         previous_version=previous_version,
         current_version=current_version,
@@ -46,6 +57,8 @@ def build_report(
         settings_results=settings_results,
         descriptor_results=descriptor_results,
         skill_results=skill_results,
+        content_results=content_results or [],
+        agent_results=agent_results or [],
         errors=errors,
     )
 
@@ -75,6 +88,8 @@ def format_report(report: SyncReport) -> str:
         _format_phase_line("Settings", report.settings_results),
         _format_phase_line("Descriptors", report.descriptor_results),
         _format_skill_line(report.skill_results),
+        _format_content_line(report.content_results),
+        _format_agents_line(report.agent_results),
     ]
 
     key_changes = _format_key_changes(report)
@@ -82,6 +97,11 @@ def format_report(report: SyncReport) -> str:
         lines += ["", "  Key changes:"]
         lines += [f"  - {c}" for c in key_changes]
 
+    total_deferred = sum(len(r.mcps_deferred) for r in report.mcp_results)
+    projects_with_deferred = sum(1 for r in report.mcp_results if r.mcps_deferred)
+    if total_deferred > 0:
+        lines += ["", f"  Deferred MCPs: {total_deferred} across {projects_with_deferred} projects."]
+
     lines += [
         "",
         f"  Errors: {len(report.errors)}",
@@ -100,11 +120,15 @@ def _collect_errors(
     settings: list[SettingsSyncResult],
     desc: list[DescriptorSyncResult],
     skills: list[SkillSyncResult],
+    content_results: list[ContentSyncResult] | None = None,
+    agent_results: list[AgentProvisionResult] | None = None,
 ) -> list[str]:
     errors: list[str] = []
     for r in mcp:
         if r.error:
             errors.append(f"MCP({r.path}): {r.error}")
+        for w in r.optimizer_warnings:
+            errors.append(f"MCP Optimizer({r.path}): {w}")
     for r in settings:
         if r.error:
             errors.append(f"Settings({r.path}): {r.error}")
@@ -114,6 +138,16 @@ def _collect_errors(
     for r in skills:
         if r.error:
             errors.append(f"Skill({r.skill_name}): {r.error}")
+    for r in content_results or []:
+        if r.error:
+            errors.append(f"Content({r.path}): {r.error}")
+        for artefact_error in r.artefacts_errored:
+            errors.append(f"Content({r.path}): {artefact_error}")
+    for r in agent_results or []:
+        if r.error:
+            errors.append(f"Agents({r.path}): {r.error}")
+        for a in r.agents_errored:
+            errors.append(f"Agents({r.path}): missing core file for {a}")
     return errors
 
 
@@ -176,3 +210,17 @@ def _add_skill_changes(results: list[SkillSyncResult], changes: list[str]) -> No
     for r in results:
         for feature in r.features_added:
             changes.append(f"'{feature}' added to: {r.skill_name}")
+
+
+def _format_content_line(results: list[ContentSyncResult]) -> str:
+    total = len(results)
+    updated = _count_updated(results)
+    unchanged = _count_unchanged(results)
+    return f"  {'Content:':<14}{total} synced ({updated} updated, {unchanged} unchanged)"
+
+
+def _format_agents_line(results: list[AgentProvisionResult]) -> str:
+    total = len(results)
+    updated = _count_updated(results)
+    unchanged = _count_unchanged(results)
+    return f"  {'Agents:':<14}{total} synced ({updated} updated, {unchanged} unchanged)"

package/core/sync/schema.py

@@ -47,8 +47,10 @@ class McpSyncResult(BaseModel):
     mcps_removed: list[str] = Field(default_factory=list)
     mcps_updated: list[str] = Field(default_factory=list)
     mcps_preserved: list[str] = Field(default_factory=list)
+    mcps_deferred: list[str] = Field(default_factory=list)
     final_mcp_list: list[str] = Field(default_factory=list)
     error: str | None = None
+    optimizer_warnings: list[str] = Field(default_factory=list)
 
 
 class SettingsSyncResult(BaseModel):
@@ -80,6 +82,28 @@ class SkillSyncResult(BaseModel):
     error: str | None = None
 
 
+class ContentSyncResult(BaseModel):
+    """Result of syncing content artefacts (CLAUDE.md, rules, hooks, constitution) for a project."""
+
+    path: str
+    status: str
+    artefacts_updated: list[str] = Field(default_factory=list)
+    artefacts_unchanged: list[str] = Field(default_factory=list)
+    artefacts_errored: list[str] = Field(default_factory=list)
+    error: str | None = None
+
+
+class AgentProvisionResult(BaseModel):
+    """Result of syncing baseline agents for a project."""
+
+    path: str
+    status: str
+    agents_added: list[str] = Field(default_factory=list)
+    agents_unchanged: list[str] = Field(default_factory=list)
+    agents_errored: list[str] = Field(default_factory=list)
+    error: str | None = None
+
+
 class SyncReport(BaseModel):
     """Aggregated report produced at the end of a full /arka update run."""
 
@@ -91,4 +115,17 @@ class SyncReport(BaseModel):
     settings_results: list[SettingsSyncResult] = Field(default_factory=list)
     descriptor_results: list[DescriptorSyncResult] = Field(default_factory=list)
     skill_results: list[SkillSyncResult] = Field(default_factory=list)
+    content_results: list[ContentSyncResult] = Field(default_factory=list)
+    agent_results: list[AgentProvisionResult] = Field(default_factory=list)
     errors: list[str] = Field(default_factory=list)
+
+
+class SyncError(BaseModel):
+    """Structured sync error with grep-able code and context."""
+
+    phase: str
+    project_path: str
+    code: str
+    message: str
+    context: dict = Field(default_factory=dict)
+    retry_count: int = 0