@softspark/ai-toolkit 2.1.3 → 2.3.0

package/scripts/install_steps/install_state.py

@@ -60,6 +60,31 @@ def get_installed_profile() -> str:
     return state.get("profile", "")
 
 
+def get_mcp_templates() -> list[str]:
+    """Return list of globally tracked MCP template names."""
+    state = load_state()
+    templates = state.get("mcp_templates", [])
+    return templates if isinstance(templates, list) else []
+
+
+def record_mcp_template(name: str) -> None:
+    """Add a template name to the tracked set in state.json."""
+    state = load_state()
+    templates = set(state.get("mcp_templates", []))
+    templates.add(name)
+    state["mcp_templates"] = sorted(templates)
+    save_state(state)
+
+
+def remove_mcp_template(name: str) -> None:
+    """Remove a template name from the tracked set in state.json."""
+    state = load_state()
+    templates = set(state.get("mcp_templates", []))
+    templates.discard(name)
+    state["mcp_templates"] = sorted(templates)
+    save_state(state)
+
+
 def _now_iso() -> str:
     """Return current UTC time in ISO 8601 format."""
     return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
@@ -138,6 +163,10 @@ def print_status() -> None:
         langs = [m.replace("rules-", "") for m in detected]
         print(f"  Detected:   {', '.join(langs)}")
 
+    mcp = state.get("mcp_templates", [])
+    if mcp:
+        print(f"  MCP:        {', '.join(mcp)}")
+
     extends = state.get("extends")
     if extends:
         print(f"  Extends:    {extends.get('source', 'unknown')}")

package/scripts/install_steps/markers.py

@@ -32,14 +32,23 @@ def install_marker_files(claude_dir: Path, only: str, skip: str,
 
 
 def inject_rules(claude_dir: Path, target_dir: Path, rules_dir: Path,
-                 only: str, skip: str, dry_run: bool) -> None:
-    """Inject rules into CLAUDE.md."""
+                 only: str, skip: str, dry_run: bool,
+                 refresh_urls: bool = False) -> None:
+    """Inject rules into CLAUDE.md.
+
+    When refresh_urls is True, re-fetches URL-sourced rules before injection.
+    Only the global install path should set this to True (once per update).
+    """
     claude_md = claude_dir / "CLAUDE.md"
 
     if dry_run:
         _inject_rules_dry_run(rules_dir)
         return
 
+    # Refresh URL-sourced rules before injection (global update only)
+    if refresh_urls:
+        _refresh_url_rules(rules_dir)
+
     if not claude_md.is_file():
         claude_md.touch()
         print("  Created: ~/.claude/CLAUDE.md")
@@ -66,6 +75,30 @@ def inject_rules(claude_dir: Path, target_dir: Path, rules_dir: Path,
     print(f"  Rules injected: {' '.join(rules_injected)}")
 
 
+def _refresh_url_rules(rules_dir: Path) -> None:
+    """Re-fetch all URL-sourced rules. Warn on failure, use cached copy."""
+    from rule_sources import get_url_rules, fetch_url, register_url_source
+
+    url_rules = get_url_rules(rules_dir)
+    if not url_rules:
+        return
+
+    for rule_name, url in url_rules.items():
+        rule_file = rules_dir / f"{rule_name}.md"
+        try:
+            data = fetch_url(url)
+            rule_file.write_bytes(data)
+            register_url_source(rules_dir, rule_name, url)
+            print(f"  Refreshed: {rule_name} (from {url})")
+        except Exception as exc:
+            if rule_file.is_file():
+                print(f"  Warning: could not refresh '{rule_name}' from {url}: {exc}")
+                print(f"           Using cached version.")
+            else:
+                print(f"  Warning: could not fetch '{rule_name}' from {url}: {exc}")
+                print(f"           No cached version — rule will be skipped.")
+
+
 def _inject_rules_dry_run(rules_dir: Path) -> None:
     rules_src = app_dir / "rules"
     rule_names = " ".join(

package/scripts/install_steps/project_registry.py

@@ -8,15 +8,43 @@ Stdlib-only — no external dependencies.
 """
 from __future__ import annotations
 
+import contextlib
+import fcntl
 import json
+import os
 import sys
+import tempfile
+import time
 from datetime import datetime, timezone
 from pathlib import Path
-from typing import Any
+from typing import Any, Generator
 
 sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
 from paths import PROJECTS_FILE
 
+# Max retries when reading a partially-written file
+_LOAD_RETRIES = 3
+_LOAD_RETRY_DELAY = 0.05  # 50ms
+
+
+@contextlib.contextmanager
+def _registry_lock() -> Generator[None, None, None]:
+    """Exclusive file lock for read-modify-write on projects.json.
+
+    Prevents concurrent processes from interleaving loads and saves,
+    which can silently drop entries.
+    """
+    path = _registry_path()
+    path.parent.mkdir(parents=True, exist_ok=True)
+    lock_path = path.with_suffix(".lock")
+    fd = os.open(str(lock_path), os.O_CREAT | os.O_RDWR)
+    try:
+        fcntl.flock(fd, fcntl.LOCK_EX)
+        yield
+    finally:
+        fcntl.flock(fd, fcntl.LOCK_UN)
+        os.close(fd)
+
 
 def _registry_path() -> Path:
     """Return the canonical path to projects.json."""
@@ -32,28 +60,66 @@ def _now_iso() -> str:
 # ---------------------------------------------------------------------------
 
 def load_registry() -> list[dict[str, Any]]:
-    """Load project registry. Returns empty list if missing/corrupt."""
+    """Load project registry with retry for partially-written files.
+
+    Retries on JSONDecodeError (another process mid-write).
+    Returns empty list only if the file genuinely doesn't exist.
+    """
     path = _registry_path()
     if not path.is_file():
         return []
-    try:
-        with open(path, encoding="utf-8") as f:
-            data = json.load(f)
-        if isinstance(data, dict):
-            projects = data.get("projects", [])
-            return projects if isinstance(projects, list) else []
-        return []
-    except (json.JSONDecodeError, OSError):
-        return []
+
+    last_err: Exception | None = None
+    for attempt in range(_LOAD_RETRIES):
+        try:
+            with open(path, encoding="utf-8") as f:
+                data = json.load(f)
+            if isinstance(data, dict):
+                projects = data.get("projects", [])
+                return projects if isinstance(projects, list) else []
+            return []
+        except json.JSONDecodeError as exc:
+            last_err = exc
+            if attempt < _LOAD_RETRIES - 1:
+                time.sleep(_LOAD_RETRY_DELAY)
+        except OSError:
+            return []
+
+    # All retries exhausted — file is genuinely corrupt, not mid-write
+    import sys as _sys
+    print(
+        f"Warning: {path} is corrupt after {_LOAD_RETRIES} retries: {last_err}",
+        file=_sys.stderr,
+    )
+    return []
 
 
 def save_registry(projects: list[dict[str, Any]]) -> None:
-    """Save project registry."""
+    """Save project registry atomically (write-to-temp + rename).
+
+    Uses os.rename which is atomic on POSIX, preventing other processes
+    from reading a partially-written file.
+    """
     path = _registry_path()
     path.parent.mkdir(parents=True, exist_ok=True)
-    with open(path, "w", encoding="utf-8") as f:
-        json.dump({"projects": projects}, f, indent=2)
-        f.write("\n")
+
+    fd, tmp_path = tempfile.mkstemp(
+        dir=str(path.parent), prefix=".projects_", suffix=".tmp"
+    )
+    try:
+        with os.fdopen(fd, "w", encoding="utf-8") as f:
+            json.dump({"projects": projects}, f, indent=2)
+            f.write("\n")
+            f.flush()
+            os.fsync(f.fileno())
+        os.rename(tmp_path, str(path))
+    except BaseException:
+        # Clean up temp file on failure
+        try:
+            os.unlink(tmp_path)
+        except OSError:
+            pass
+        raise
 
 
 # ---------------------------------------------------------------------------
@@ -68,47 +134,52 @@ def register_project(
     """Register a project directory. Returns True if newly added, False if updated.
 
     Idempotent — updates existing entry if path already registered.
+    Uses file lock to prevent concurrent read-modify-write races.
     """
     project_path = str(Path(project_path).resolve())
-    projects = load_registry()
-    now = _now_iso()
-
-    for p in projects:
-        if p.get("path") == project_path:
-            # Update existing
-            p["last_updated"] = now
-            if profile:
-                p["profile"] = profile
-            if extends:
-                p["extends"] = extends
-            elif "extends" in p and not extends:
-                # Clear extends if project no longer uses it
-                pass
-            save_registry(projects)
-            return False
 
-    # New registration
-    projects.append({
-        "path": project_path,
-        "registered_at": now,
-        "last_updated": now,
-        "profile": profile or "standard",
-        "extends": extends or "",
-    })
-    save_registry(projects)
-    return True
+    with _registry_lock():
+        projects = load_registry()
+        now = _now_iso()
+
+        for p in projects:
+            if p.get("path") == project_path:
+                # Update existing
+                p["last_updated"] = now
+                if profile:
+                    p["profile"] = profile
+                if extends:
+                    p["extends"] = extends
+                elif "extends" in p and not extends:
+                    # Clear extends if project no longer uses it
+                    pass
+                save_registry(projects)
+                return False
+
+        # New registration
+        projects.append({
+            "path": project_path,
+            "registered_at": now,
+            "last_updated": now,
+            "profile": profile or "standard",
+            "extends": extends or "",
+        })
+        save_registry(projects)
+        return True
 
 
 def unregister_project(project_path: str | Path) -> bool:
     """Unregister a project. Returns True if found and removed."""
     project_path = str(Path(project_path).resolve())
-    projects = load_registry()
-    original_len = len(projects)
-    projects = [p for p in projects if p.get("path") != project_path]
-    if len(projects) < original_len:
-        save_registry(projects)
-        return True
-    return False
+
+    with _registry_lock():
+        projects = load_registry()
+        original_len = len(projects)
+        projects = [p for p in projects if p.get("path") != project_path]
+        if len(projects) < original_len:
+            save_registry(projects)
+            return True
+        return False
 
 
 def list_projects() -> list[dict[str, Any]]:
@@ -121,18 +192,19 @@ def list_projects() -> list[dict[str, Any]]:
 
 def prune_stale() -> list[str]:
     """Remove projects whose directories no longer exist. Returns pruned paths."""
-    projects = load_registry()
-    pruned: list[str] = []
-    kept: list[dict[str, Any]] = []
-
-    for p in projects:
-        if Path(p["path"]).is_dir():
-            kept.append(p)
-        else:
-            pruned.append(p["path"])
-
-    if pruned:
-        save_registry(kept)
+    with _registry_lock():
+        projects = load_registry()
+        pruned: list[str] = []
+        kept: list[dict[str, Any]] = []
+
+        for p in projects:
+            if Path(p["path"]).is_dir():
+                kept.append(p)
+            else:
+                pruned.append(p["path"])
+
+        if pruned:
+            save_registry(kept)
 
     return pruned
 

package/scripts/mcp_editors.py

@@ -15,8 +15,8 @@ except ModuleNotFoundError:  # pragma: no cover - Python 3.11+ should have tomll
 EDITOR_SPECS: dict[str, dict[str, str | None]] = {
     "claude": {
         "label": "Claude Code",
-        "project_path": ".claude/settings.local.json",
-        "global_path": ".claude/settings.json",
+        "project_path": ".mcp.json",
+        "global_path": ".claude.json",
         "format": "json",
         "doc_scope": "project + global",
     },

package/scripts/mcp_manager.py

@@ -130,6 +130,11 @@ def cmd_show(name: str) -> None:
         for key, val in env_vars:
             print(f"  {key} = {val}")
 
+    post_install = data.get("postInstall")
+    if post_install:
+        print()
+        print(f"Setup: {post_install}")
+
 
 def cmd_add(names: list[str], target_dir: Path) -> None:
     """Add one or more MCP templates to .mcp.json."""
@@ -155,6 +160,13 @@ def cmd_add(names: list[str], target_dir: Path) -> None:
     write_mcp_config(target_dir, config)
     print(f"Added: {', '.join(added)}")
 
+    # Show postInstall hints from added templates
+    for name in names:
+        data = load_template(name)
+        post_install = data.get("postInstall")
+        if post_install:
+            print(f"\n  Note ({name}): {post_install}")
+
 
 def cmd_install(
     names: list[str],
@@ -203,6 +215,20 @@ def cmd_install(
     for path in updated:
         print(f"Updated: {path}")
 
+    # Track globally installed templates in state.json
+    if eff_scope == "global" and names:
+        from install_steps.install_state import record_mcp_template
+        for name in names:
+            record_mcp_template(name)
+
+    # Show postInstall hints from installed templates
+    if names:
+        for name in names:
+            data = load_template(name)
+            post_install = data.get("postInstall")
+            if post_install:
+                print(f"\n  Note ({name}): {post_install}")
+
 
 def cmd_remove(name: str, target_dir: Path | None, *, editors: list[str], scope: str | None) -> None:
     """Remove an MCP server from .mcp.json."""
@@ -223,6 +249,9 @@ def cmd_remove(name: str, target_dir: Path | None, *, editors: list[str], scope:
             )
         else:
             updated = remove_servers(editors, [name], scope="global")
+            # Untrack globally removed template from state.json
+            from install_steps.install_state import remove_mcp_template
+            remove_mcp_template(name)
         for path in updated:
             print(f"Updated: {path}")
         print(f"Removed: {name}")

package/scripts/remove_rule.py

@@ -43,6 +43,11 @@ def main() -> None:
     else:
         print(f"Not registered: '{rule_name}' not found in {rules_dir}")
 
+    # 1b. Clean up URL source metadata (if any)
+    from rule_sources import unregister_source
+    if unregister_source(rules_dir, rule_name):
+        print(f"Removed URL source for '{rule_name}'")
+
     # 2. Strip injected block from .claude/CLAUDE.md
     found = remove_rule_section(rule_name, target_dir)
     if found:

package/scripts/rule_sources.py

@@ -0,0 +1,138 @@
+#!/usr/bin/env python3
+"""URL source registry for remotely-sourced rules.
+
+Tracks which rules were registered from a URL so that `ai-toolkit update`
+can re-fetch the latest version before injection.
+
+Metadata stored in ~/.softspark/ai-toolkit/rules/sources.json.
+
+Stdlib-only — no external dependencies.
+"""
+from __future__ import annotations
+
+import json
+import os
+import ssl
+import sys
+import tempfile
+import urllib.request
+import urllib.error
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+from paths import RULES_DIR
+
+_SOURCES_FILENAME = "sources.json"
+_FETCH_TIMEOUT = 30  # seconds
+_FETCH_MAX_BYTES = 10 * 1024 * 1024  # 10MB
+
+
+# ---------------------------------------------------------------------------
+# Load / Save
+# ---------------------------------------------------------------------------
+
+def _sources_path(rules_dir: Path | None = None) -> Path:
+    return (rules_dir or RULES_DIR) / _SOURCES_FILENAME
+
+
+def load_sources(rules_dir: Path | None = None) -> dict[str, dict[str, Any]]:
+    """Load sources.json. Returns {} if missing or corrupt."""
+    path = _sources_path(rules_dir)
+    if not path.is_file():
+        return {}
+    try:
+        with open(path, encoding="utf-8") as f:
+            data = json.load(f)
+        if isinstance(data, dict):
+            return data.get("rules", {})
+        return {}
+    except (json.JSONDecodeError, OSError):
+        return {}
+
+
+def save_sources(rules_dir: Path | None = None,
+                 sources: dict[str, dict[str, Any]] | None = None) -> None:
+    """Write sources.json atomically."""
+    rules_dir = rules_dir or RULES_DIR
+    path = _sources_path(rules_dir)
+    path.parent.mkdir(parents=True, exist_ok=True)
+
+    payload = json.dumps({"schema_version": 1, "rules": sources or {}}, indent=2)
+
+    fd, tmp_path = tempfile.mkstemp(
+        dir=str(path.parent), prefix=".sources_", suffix=".tmp"
+    )
+    try:
+        with os.fdopen(fd, "w", encoding="utf-8") as f:
+            f.write(payload)
+            f.write("\n")
+            f.flush()
+            os.fsync(f.fileno())
+        os.rename(tmp_path, str(path))
+    except BaseException:
+        try:
+            os.unlink(tmp_path)
+        except OSError:
+            pass
+        raise
+
+
+# ---------------------------------------------------------------------------
+# CRUD
+# ---------------------------------------------------------------------------
+
+def register_url_source(rules_dir: Path | None, rule_name: str, url: str) -> None:
+    """Add or update a URL source entry."""
+    rules_dir = rules_dir or RULES_DIR
+    sources = load_sources(rules_dir)
+    sources[rule_name] = {
+        "url": url,
+        "fetched_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
+    }
+    save_sources(rules_dir, sources)
+
+
+def unregister_source(rules_dir: Path | None, rule_name: str) -> bool:
+    """Remove a source entry. Returns True if found and removed."""
+    rules_dir = rules_dir or RULES_DIR
+    sources = load_sources(rules_dir)
+    if rule_name in sources:
+        del sources[rule_name]
+        save_sources(rules_dir, sources)
+        return True
+    return False
+
+
+def get_url_rules(rules_dir: Path | None = None) -> dict[str, str]:
+    """Return {rule_name: url} for all URL-sourced rules."""
+    sources = load_sources(rules_dir)
+    return {name: entry["url"] for name, entry in sources.items() if "url" in entry}
+
+
+# ---------------------------------------------------------------------------
+# Fetch
+# ---------------------------------------------------------------------------
+
+def fetch_url(url: str) -> bytes:
+    """Fetch URL content. HTTPS only, 30s timeout, 10MB cap.
+
+    Raises:
+        ValueError: if URL is not HTTPS
+        urllib.error.URLError: on network failure
+    """
+    if not url.startswith("https://"):
+        raise ValueError(
+            f"Only HTTPS URLs are supported (got: {url.split('://')[0]}://)"
+        )
+
+    ctx = ssl.create_default_context()
+    with urllib.request.urlopen(url, timeout=_FETCH_TIMEOUT, context=ctx) as resp:
+        data = resp.read(_FETCH_MAX_BYTES)
+
+    # Basic binary detection — reject if null bytes present
+    if b"\x00" in data:
+        raise ValueError(f"URL returned binary content, expected markdown: {url}")
+
+    return data

package/scripts/update_projects.py

@@ -17,7 +17,7 @@ from pathlib import Path
 from typing import Any
 
 sys.path.insert(0, str(Path(__file__).resolve().parent))
-from install_steps.project_registry import get_active_projects, prune_stale
+from install_steps.project_registry import get_active_projects, prune_stale, register_project
 
 
 def _update_project(project: dict[str, Any], install_script: str, extra_args: list[str]) -> dict:
@@ -91,13 +91,17 @@ def main() -> None:
         print(f"  Updating {len(projects)} registered project(s)...")
         print()
 
+    # --skip-register: parallel installs must NOT write to projects.json
+    # concurrently. We re-register sequentially after all installs complete.
+    parallel_args = extra_args + ["--skip-register"]
+
     # Run in parallel (max 8 workers — don't overwhelm the system)
     max_workers = min(len(projects), 8)
     results: list[dict] = []
 
     with ThreadPoolExecutor(max_workers=max_workers) as pool:
         futures = {
-            pool.submit(_update_project, p, install_script, extra_args): p
+            pool.submit(_update_project, p, install_script, parallel_args): p
             for p in projects
         }
         for future in as_completed(futures):
@@ -118,6 +122,15 @@ def main() -> None:
                     for line in result["error"].strip().split("\n"):
                         print(f"      ERROR: {line}")
 
+    # Re-register projects sequentially (safe — no concurrent writes)
+    for result in results:
+        if result["success"]:
+            register_project(
+                result["path"],
+                profile=result.get("profile", "standard"),
+                extends=result.get("extends", ""),
+            )
+
     # Summary
     passed = sum(1 for r in results if r["success"])
     failed = len(results) - passed

package/scripts/validate.py

@@ -610,10 +610,43 @@ def validate_metadata_contracts(
     else:
         print(f"  OK: tests ({actual_tests})")
 
+    # Cross-validate versions: package.json vs manifest.json vs plugin.json
+    _validate_version_sync(tk_dir, vr)
+
     print()
     return actual_tests
 
 
+def _validate_version_sync(tk_dir: Path, vr: ValidationResult) -> None:
+    """Ensure package.json, manifest.json, and plugin.json versions match."""
+    import json as _json
+
+    version_files = {
+        "package.json": tk_dir / "package.json",
+        "manifest.json": tk_dir / "manifest.json",
+        "plugin.json": tk_dir / "app" / ".claude-plugin" / "plugin.json",
+    }
+
+    versions: dict[str, str] = {}
+    for name, path in version_files.items():
+        if path.is_file():
+            try:
+                data = _json.loads(path.read_text(encoding="utf-8"))
+                versions[name] = data.get("version", "")
+            except Exception:
+                pass
+
+    if len(versions) < 2:
+        return  # Not enough files to compare (e.g., installed copy without source)
+
+    unique = set(versions.values())
+    if len(unique) == 1:
+        print(f"  OK: version sync ({unique.pop()})")
+    else:
+        detail = ", ".join(f"{k}={v}" for k, v in versions.items())
+        vr.error(f"Version mismatch across files: {detail}")
+
+
 def validate_content_quality(tk_dir: Path, vr: ValidationResult) -> None:
     """Check content quality: name matches directory, non-empty body."""
     print()