@softspark/ai-toolkit 2.1.2 → 2.2.0

package/scripts/add_rule.py

@@ -1,52 +1,94 @@
 #!/usr/bin/env python3
-"""add-rule -- Register a rule file in ~/.softspark/ai-toolkit/rules/.
+"""add-rule -- Register a rule file or URL in ~/.softspark/ai-toolkit/rules/.
 
 Registered rules are automatically injected into all AI tool configs
-on next 'ai-toolkit install' or 'ai-toolkit update':
+on next 'ai-toolkit install' or 'ai-toolkit update'.
+URL-sourced rules are auto-refreshed on every update.
+
   Global: Claude, Cursor, Windsurf, Gemini, Augment
   Local (--local): all of the above + Copilot, Cline, Roo, Aider, Antigravity
 
 Usage:
-  add_rule.py <rule-file> [rule-name]
+  add_rule.py <rule-file-or-url> [rule-name]
 
 Arguments:
-  rule-file   Path to .md file with the rule content
-  rule-name   Override the rule name (default: filename without .md)
+  rule-file-or-url  Path to .md file or HTTPS URL to register globally
+  rule-name         Override the rule name (default: filename without .md)
 """
 from __future__ import annotations
 
 import re
 import shutil
 import sys
+import urllib.parse
 from pathlib import Path
 
 sys.path.insert(0, str(Path(__file__).resolve().parent))
 
 
+def _name_from_url(url: str) -> str:
+    """Derive a rule name from a URL's last path segment."""
+    parsed = urllib.parse.urlparse(url)
+    filename = parsed.path.rstrip("/").split("/")[-1]
+    stem = filename.rsplit(".", 1)[0] if "." in filename else filename
+    return re.sub(r"[^a-zA-Z0-9_-]", "", stem)
+
+
 def main() -> None:
-    """Register a rule file in the global rules directory."""
+    """Register a rule file or URL in the global rules directory."""
     if len(sys.argv) < 2:
-        print("Usage: add_rule.py <rule-file> [rule-name]", file=sys.stderr)
+        print("Usage: add_rule.py <rule-file-or-url> [rule-name]", file=sys.stderr)
         sys.exit(1)
 
-    rule_file = Path(sys.argv[1])
-    if not rule_file.is_file():
-        print(f"Rule file not found: {rule_file}", file=sys.stderr)
-        sys.exit(1)
+    source = sys.argv[1]
+    is_url = source.startswith("https://") or source.startswith("http://")
 
-    rule_name = sys.argv[2] if len(sys.argv) > 2 else rule_file.stem
-    rule_name = re.sub(r"[^a-zA-Z0-9_-]", "", rule_name)
-    if not rule_name:
-        print("Error: rule name is empty after sanitization", file=sys.stderr)
+    if is_url and source.startswith("http://"):
+        print("Error: only HTTPS URLs are supported. Use https:// for security.", file=sys.stderr)
         sys.exit(1)
+
     from paths import RULES_DIR
     rules_dir = RULES_DIR
     rules_dir.mkdir(parents=True, exist_ok=True)
 
-    dest = rules_dir / f"{rule_name}.md"
-    shutil.copy2(rule_file, dest)
+    if is_url:
+        from rule_sources import fetch_url, register_url_source
+
+        rule_name = sys.argv[2] if len(sys.argv) > 2 else _name_from_url(source)
+        rule_name = re.sub(r"[^a-zA-Z0-9_-]", "", rule_name)
+        if not rule_name:
+            print("Error: could not derive rule name from URL. Provide one explicitly.", file=sys.stderr)
+            sys.exit(1)
+
+        try:
+            data = fetch_url(source)
+        except Exception as exc:
+            print(f"Error fetching URL: {exc}", file=sys.stderr)
+            sys.exit(1)
+
+        dest = rules_dir / f"{rule_name}.md"
+        dest.write_bytes(data)
+        register_url_source(rules_dir, rule_name, source)
+
+        print(f"Registered: '{rule_name}' -> {dest}")
+        print(f"Source URL: {source} (auto-refreshed on update)")
+    else:
+        rule_file = Path(source)
+        if not rule_file.is_file():
+            print(f"Rule file not found: {rule_file}", file=sys.stderr)
+            sys.exit(1)
+
+        rule_name = sys.argv[2] if len(sys.argv) > 2 else rule_file.stem
+        rule_name = re.sub(r"[^a-zA-Z0-9_-]", "", rule_name)
+        if not rule_name:
+            print("Error: rule name is empty after sanitization", file=sys.stderr)
+            sys.exit(1)
+
+        dest = rules_dir / f"{rule_name}.md"
+        shutil.copy2(rule_file, dest)
+
+        print(f"Registered: '{rule_name}' -> {dest}")
 
-    print(f"Registered: '{rule_name}' -> {dest}")
     print()
     print("Apply now:")
     print("  ai-toolkit update            # global (Claude, Cursor, Windsurf, Gemini, Augment)")

package/scripts/generate_agents_md.py

@@ -6,13 +6,13 @@ Usage: ./scripts/generate_agents_md.py > AGENTS.md
 """
 from __future__ import annotations
 
-import os
 import sys
 from pathlib import Path
 
 sys.path.insert(0, str(Path(__file__).resolve().parent))
+import subprocess
+
 from _common import agents_dir, frontmatter_field
-from paths import RULES_DIR
 
 
 def main() -> None:
@@ -88,18 +88,16 @@ def main() -> None:
         print("---")
         print()
 
-    # Registered custom rules — skip when called by inject_with_rules()
-    if not os.environ.get("_TOOLKIT_INJECT_MODE"):
-        if RULES_DIR.is_dir():
-            for rule_file in sorted(RULES_DIR.glob("*.md")):
-                rule_name = rule_file.stem
-                print(f"<!-- TOOLKIT:{rule_name} START -->")
-                print("<!-- Auto-injected by ai-toolkit. Re-run to update. -->")
-                print()
-                print(rule_file.read_text(encoding="utf-8").rstrip())
-                print()
-                print(f"<!-- TOOLKIT:{rule_name} END -->")
-            print()
+    # Codex CLI configuration block (agents, skills, guidelines)
+    codex_script = Path(__file__).resolve().parent / "generate_codex.py"
+    result = subprocess.run(
+        ["python3", str(codex_script)],
+        capture_output=True, text=True,
+    )
+    if result.returncode == 0 and result.stdout.strip():
+        print(result.stdout.rstrip())
+
+    # Note: custom rules are included via generate_codex.py output above
 
 
 if __name__ == "__main__":

package/scripts/generate_codex.py

@@ -8,7 +8,6 @@ Usage: ./scripts/generate_codex.py > AGENTS.md
 """
 from __future__ import annotations
 
-import os
 import sys
 from pathlib import Path
 
@@ -89,19 +88,17 @@ def main() -> None:
 
     print_toolkit_end()
 
-    # Registered custom rules — skip when called by inject_with_rules()
-    # (which handles injection separately via inject_section)
-    if not os.environ.get("_TOOLKIT_INJECT_MODE"):
-        if RULES_DIR.is_dir():
-            for rule_file in sorted(RULES_DIR.glob("*.md")):
-                rule_name = rule_file.stem
-                print()
-                print(f"<!-- TOOLKIT:{rule_name} START -->")
-                print("<!-- Auto-injected by ai-toolkit. Re-run to update. -->")
-                print()
-                print(rule_file.read_text(encoding="utf-8").rstrip())
-                print()
-                print(f"<!-- TOOLKIT:{rule_name} END -->")
+    # Registered custom rules from ~/.softspark/ai-toolkit/rules/
+    if RULES_DIR.is_dir():
+        for rule_file in sorted(RULES_DIR.glob("*.md")):
+            rule_name = rule_file.stem
+            print()
+            print(f"<!-- TOOLKIT:{rule_name} START -->")
+            print("<!-- Auto-injected by ai-toolkit. Re-run to update. -->")
+            print()
+            print(rule_file.read_text(encoding="utf-8").rstrip())
+            print()
+            print(f"<!-- TOOLKIT:{rule_name} END -->")
 
 
 if __name__ == "__main__":

package/scripts/generator_base.py

@@ -26,7 +26,6 @@ Usage::
 """
 from __future__ import annotations
 
-import os
 import sys
 from pathlib import Path
 
@@ -143,15 +142,14 @@ def render_generator(config: dict) -> None:
         print()
         print_toolkit_end()
 
-    # Registered custom rules — skip when called by inject_with_rules()
-    if not os.environ.get("_TOOLKIT_INJECT_MODE"):
-        if RULES_DIR.is_dir():
-            for rule_file in sorted(RULES_DIR.glob("*.md")):
-                rule_name = rule_file.stem
-                print()
-                print(f"<!-- TOOLKIT:{rule_name} START -->")
-                print("<!-- Auto-injected by ai-toolkit. Re-run to update. -->")
-                print()
-                print(rule_file.read_text(encoding="utf-8").rstrip())
-                print()
-                print(f"<!-- TOOLKIT:{rule_name} END -->")
+    # Registered custom rules from ~/.softspark/ai-toolkit/rules/
+    if RULES_DIR.is_dir():
+        for rule_file in sorted(RULES_DIR.glob("*.md")):
+            rule_name = rule_file.stem
+            print()
+            print(f"<!-- TOOLKIT:{rule_name} START -->")
+            print("<!-- Auto-injected by ai-toolkit. Re-run to update. -->")
+            print()
+            print(rule_file.read_text(encoding="utf-8").rstrip())
+            print()
+            print(f"<!-- TOOLKIT:{rule_name} END -->")

package/scripts/install.py

@@ -212,6 +212,7 @@ def parse_args(argv: list[str]) -> dict:
         "editors": "",
         "config": "",
         "refresh_base": False,
+        "skip_register": False,
     }
     i = 0
     while i < len(argv):
@@ -268,6 +269,8 @@ def parse_args(argv: list[str]) -> dict:
             cfg["config"] = argv[i] if i < len(argv) else ""
         elif arg == "--refresh-base":
             cfg["refresh_base"] = True
+        elif arg == "--skip-register":
+            cfg["skip_register"] = True
         elif arg.startswith("-"):
             print(f"Unknown option: {arg}")
             sys.exit(1)
@@ -426,7 +429,8 @@ def install_claude_code(target_dir: Path, hooks_scripts_dir: Path,
     print()
     print(f"  Available: {count_agents()} agents, {count_skills()} skills")
 
-    inject_rules(claude_dir, target_dir, rules_dir, only, skip, dry_run)
+    inject_rules(claude_dir, target_dir, rules_dir, only, skip, dry_run,
+                 refresh_urls=True)
 
 
 VALID_PERSONAS = ("backend-lead", "frontend-lead", "devops-eng", "junior-dev")
@@ -711,7 +715,9 @@ def main() -> None:
         )
 
         # Register project in global registry (for `ai-toolkit update` propagation)
-        if local:
+        # Skipped when called from update_projects.py (--skip-register) to avoid
+        # concurrent writes to projects.json during parallel updates.
+        if local and not cfg.get("skip_register"):
             extends_source = ""
             if extends_info:
                 extends_source = extends_info.get("source", "")

package/scripts/install_steps/ai_tools.py

@@ -1,7 +1,6 @@
 """Install AI tool configs (Cursor, Windsurf, Gemini, Augment, Codex) and local project setup."""
 from __future__ import annotations
 
-import os
 import shutil
 import subprocess
 from pathlib import Path
@@ -79,14 +78,12 @@ def inject_with_rules(
     else:
         cmd = ["bash", str(scripts_dir / generator_script)]
 
-    env = {**os.environ, "_TOOLKIT_INJECT_MODE": "1"}
-    result = subprocess.run(cmd, capture_output=True, text=True, env=env)
+    result = subprocess.run(cmd, capture_output=True, text=True)
     if result.returncode != 0:
         print(f"  ERROR: {generator_script} failed: {result.stderr.strip()}")
         return
 
     generated = result.stdout
-    start_marker = "<!-- TOOLKIT:ai-toolkit START -->"
 
     target_file = Path(target_file)
     target_file.parent.mkdir(parents=True, exist_ok=True)
@@ -94,26 +91,28 @@ def inject_with_rules(
         target_file.touch()
 
     existing = target_file.read_text(encoding="utf-8")
-    if start_marker in existing:
-        existing = _strip_section(existing, "ai-toolkit")
+    # Strip ALL toolkit sections from existing — generated output is the
+    # complete source of truth (includes ai-toolkit block + custom rules)
+    import re
+    existing = re.sub(
+        r"<!-- TOOLKIT:[^ ]+ START -->.*?<!-- TOOLKIT:[^ ]+ END -->\n?",
+        "",
+        existing,
+        flags=re.DOTALL,
+    )
     existing = _trim_trailing_blanks(existing)
-    existing = existing.lstrip("\n")  # no leading blank lines
+    existing = existing.lstrip("\n")
 
     parts: list[str] = []
     if existing.strip():
         parts.append(existing)
-        parts.append("")
+        parts.append("")  # blank line separator
     parts.append(generated.rstrip("\n"))
 
     output = "\n".join(parts) + "\n"
     output = _collapse_blank_runs(output)
-    output = output.lstrip("\n")  # no leading blank lines
+    output = output.lstrip("\n")
     target_file.write_text(output, encoding="utf-8")
-
-    if rules_dir.is_dir():
-        for rule_file in sorted(rules_dir.glob("*.md")):
-            inject_section(rule_file, target_file, rule_file.stem)
-
     print(f"  Updated: {target_file}")
 
 

package/scripts/install_steps/markers.py

@@ -32,14 +32,23 @@ def install_marker_files(claude_dir: Path, only: str, skip: str,
 
 
 def inject_rules(claude_dir: Path, target_dir: Path, rules_dir: Path,
-                 only: str, skip: str, dry_run: bool) -> None:
-    """Inject rules into CLAUDE.md."""
+                 only: str, skip: str, dry_run: bool,
+                 refresh_urls: bool = False) -> None:
+    """Inject rules into CLAUDE.md.
+
+    When refresh_urls is True, re-fetches URL-sourced rules before injection.
+    Only the global install path should set this to True (once per update).
+    """
     claude_md = claude_dir / "CLAUDE.md"
 
     if dry_run:
         _inject_rules_dry_run(rules_dir)
         return
 
+    # Refresh URL-sourced rules before injection (global update only)
+    if refresh_urls:
+        _refresh_url_rules(rules_dir)
+
     if not claude_md.is_file():
         claude_md.touch()
         print("  Created: ~/.claude/CLAUDE.md")
@@ -66,6 +75,30 @@ def inject_rules(claude_dir: Path, target_dir: Path, rules_dir: Path,
     print(f"  Rules injected: {' '.join(rules_injected)}")
 
 
+def _refresh_url_rules(rules_dir: Path) -> None:
+    """Re-fetch all URL-sourced rules. Warn on failure, use cached copy."""
+    from rule_sources import get_url_rules, fetch_url, register_url_source
+
+    url_rules = get_url_rules(rules_dir)
+    if not url_rules:
+        return
+
+    for rule_name, url in url_rules.items():
+        rule_file = rules_dir / f"{rule_name}.md"
+        try:
+            data = fetch_url(url)
+            rule_file.write_bytes(data)
+            register_url_source(rules_dir, rule_name, url)
+            print(f"  Refreshed: {rule_name} (from {url})")
+        except Exception as exc:
+            if rule_file.is_file():
+                print(f"  Warning: could not refresh '{rule_name}' from {url}: {exc}")
+                print(f"           Using cached version.")
+            else:
+                print(f"  Warning: could not fetch '{rule_name}' from {url}: {exc}")
+                print(f"           No cached version — rule will be skipped.")
+
+
 def _inject_rules_dry_run(rules_dir: Path) -> None:
     rules_src = app_dir / "rules"
     rule_names = " ".join(

package/scripts/install_steps/project_registry.py

@@ -8,15 +8,43 @@ Stdlib-only — no external dependencies.
 """
 from __future__ import annotations
 
+import contextlib
+import fcntl
 import json
+import os
 import sys
+import tempfile
+import time
 from datetime import datetime, timezone
 from pathlib import Path
-from typing import Any
+from typing import Any, Generator
 
 sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
 from paths import PROJECTS_FILE
 
+# Max retries when reading a partially-written file
+_LOAD_RETRIES = 3
+_LOAD_RETRY_DELAY = 0.05  # 50ms
+
+
+@contextlib.contextmanager
+def _registry_lock() -> Generator[None, None, None]:
+    """Exclusive file lock for read-modify-write on projects.json.
+
+    Prevents concurrent processes from interleaving loads and saves,
+    which can silently drop entries.
+    """
+    path = _registry_path()
+    path.parent.mkdir(parents=True, exist_ok=True)
+    lock_path = path.with_suffix(".lock")
+    fd = os.open(str(lock_path), os.O_CREAT | os.O_RDWR)
+    try:
+        fcntl.flock(fd, fcntl.LOCK_EX)
+        yield
+    finally:
+        fcntl.flock(fd, fcntl.LOCK_UN)
+        os.close(fd)
+
 
 def _registry_path() -> Path:
     """Return the canonical path to projects.json."""
@@ -32,28 +60,66 @@ def _now_iso() -> str:
 # ---------------------------------------------------------------------------
 
 def load_registry() -> list[dict[str, Any]]:
-    """Load project registry. Returns empty list if missing/corrupt."""
+    """Load project registry with retry for partially-written files.
+
+    Retries on JSONDecodeError (another process mid-write).
+    Returns empty list only if the file genuinely doesn't exist.
+    """
     path = _registry_path()
     if not path.is_file():
         return []
-    try:
-        with open(path, encoding="utf-8") as f:
-            data = json.load(f)
-        if isinstance(data, dict):
-            projects = data.get("projects", [])
-            return projects if isinstance(projects, list) else []
-        return []
-    except (json.JSONDecodeError, OSError):
-        return []
+
+    last_err: Exception | None = None
+    for attempt in range(_LOAD_RETRIES):
+        try:
+            with open(path, encoding="utf-8") as f:
+                data = json.load(f)
+            if isinstance(data, dict):
+                projects = data.get("projects", [])
+                return projects if isinstance(projects, list) else []
+            return []
+        except json.JSONDecodeError as exc:
+            last_err = exc
+            if attempt < _LOAD_RETRIES - 1:
+                time.sleep(_LOAD_RETRY_DELAY)
+        except OSError:
+            return []
+
+    # All retries exhausted — file is genuinely corrupt, not mid-write
+    import sys as _sys
+    print(
+        f"Warning: {path} is corrupt after {_LOAD_RETRIES} retries: {last_err}",
+        file=_sys.stderr,
+    )
+    return []
 
 
 def save_registry(projects: list[dict[str, Any]]) -> None:
-    """Save project registry."""
+    """Save project registry atomically (write-to-temp + rename).
+
+    Uses os.rename which is atomic on POSIX, preventing other processes
+    from reading a partially-written file.
+    """
     path = _registry_path()
     path.parent.mkdir(parents=True, exist_ok=True)
-    with open(path, "w", encoding="utf-8") as f:
-        json.dump({"projects": projects}, f, indent=2)
-        f.write("\n")
+
+    fd, tmp_path = tempfile.mkstemp(
+        dir=str(path.parent), prefix=".projects_", suffix=".tmp"
+    )
+    try:
+        with os.fdopen(fd, "w", encoding="utf-8") as f:
+            json.dump({"projects": projects}, f, indent=2)
+            f.write("\n")
+            f.flush()
+            os.fsync(f.fileno())
+        os.rename(tmp_path, str(path))
+    except BaseException:
+        # Clean up temp file on failure
+        try:
+            os.unlink(tmp_path)
+        except OSError:
+            pass
+        raise
 
 
 # ---------------------------------------------------------------------------
@@ -68,47 +134,52 @@ def register_project(
     """Register a project directory. Returns True if newly added, False if updated.
 
     Idempotent — updates existing entry if path already registered.
+    Uses file lock to prevent concurrent read-modify-write races.
     """
     project_path = str(Path(project_path).resolve())
-    projects = load_registry()
-    now = _now_iso()
-
-    for p in projects:
-        if p.get("path") == project_path:
-            # Update existing
-            p["last_updated"] = now
-            if profile:
-                p["profile"] = profile
-            if extends:
-                p["extends"] = extends
-            elif "extends" in p and not extends:
-                # Clear extends if project no longer uses it
-                pass
-            save_registry(projects)
-            return False
 
-    # New registration
-    projects.append({
-        "path": project_path,
-        "registered_at": now,
-        "last_updated": now,
-        "profile": profile or "standard",
-        "extends": extends or "",
-    })
-    save_registry(projects)
-    return True
+    with _registry_lock():
+        projects = load_registry()
+        now = _now_iso()
+
+        for p in projects:
+            if p.get("path") == project_path:
+                # Update existing
+                p["last_updated"] = now
+                if profile:
+                    p["profile"] = profile
+                if extends:
+                    p["extends"] = extends
+                elif "extends" in p and not extends:
+                    # Clear extends if project no longer uses it
+                    pass
+                save_registry(projects)
+                return False
+
+        # New registration
+        projects.append({
+            "path": project_path,
+            "registered_at": now,
+            "last_updated": now,
+            "profile": profile or "standard",
+            "extends": extends or "",
+        })
+        save_registry(projects)
+        return True
 
 
 def unregister_project(project_path: str | Path) -> bool:
     """Unregister a project. Returns True if found and removed."""
     project_path = str(Path(project_path).resolve())
-    projects = load_registry()
-    original_len = len(projects)
-    projects = [p for p in projects if p.get("path") != project_path]
-    if len(projects) < original_len:
-        save_registry(projects)
-        return True
-    return False
+
+    with _registry_lock():
+        projects = load_registry()
+        original_len = len(projects)
+        projects = [p for p in projects if p.get("path") != project_path]
+        if len(projects) < original_len:
+            save_registry(projects)
+            return True
+        return False
 
 
 def list_projects() -> list[dict[str, Any]]:
@@ -121,18 +192,19 @@ def list_projects() -> list[dict[str, Any]]:
 
 def prune_stale() -> list[str]:
     """Remove projects whose directories no longer exist. Returns pruned paths."""
-    projects = load_registry()
-    pruned: list[str] = []
-    kept: list[dict[str, Any]] = []
-
-    for p in projects:
-        if Path(p["path"]).is_dir():
-            kept.append(p)
-        else:
-            pruned.append(p["path"])
-
-    if pruned:
-        save_registry(kept)
+    with _registry_lock():
+        projects = load_registry()
+        pruned: list[str] = []
+        kept: list[dict[str, Any]] = []
+
+        for p in projects:
+            if Path(p["path"]).is_dir():
+                kept.append(p)
+            else:
+                pruned.append(p["path"])
+
+        if pruned:
+            save_registry(kept)
 
     return pruned
 

package/scripts/remove_rule.py

@@ -43,6 +43,11 @@ def main() -> None:
     else:
         print(f"Not registered: '{rule_name}' not found in {rules_dir}")
 
+    # 1b. Clean up URL source metadata (if any)
+    from rule_sources import unregister_source
+    if unregister_source(rules_dir, rule_name):
+        print(f"Removed URL source for '{rule_name}'")
+
     # 2. Strip injected block from .claude/CLAUDE.md
     found = remove_rule_section(rule_name, target_dir)
     if found: