octo-agent 0.11.2

data/lib/octo/default_skills/skill-creator/scripts/quick_validate.py

@@ -0,0 +1,103 @@
+#!/usr/bin/env python3
+"""
+Quick validation script for skills - minimal version
+"""
+
+import sys
+import os
+import re
+import yaml
+from pathlib import Path
+
+def validate_skill(skill_path):
+    """Basic validation of a skill"""
+    skill_path = Path(skill_path)
+
+    # Check SKILL.md exists
+    skill_md = skill_path / 'SKILL.md'
+    if not skill_md.exists():
+        return False, "SKILL.md not found"
+
+    # Read and validate frontmatter
+    content = skill_md.read_text()
+    if not content.startswith('---'):
+        return False, "No YAML frontmatter found"
+
+    # Extract frontmatter
+    match = re.match(r'^---\n(.*?)\n---', content, re.DOTALL)
+    if not match:
+        return False, "Invalid frontmatter format"
+
+    frontmatter_text = match.group(1)
+
+    # Parse YAML frontmatter
+    try:
+        frontmatter = yaml.safe_load(frontmatter_text)
+        if not isinstance(frontmatter, dict):
+            return False, "Frontmatter must be a YAML dictionary"
+    except yaml.YAMLError as e:
+        return False, f"Invalid YAML in frontmatter: {e}"
+
+    # Define allowed properties
+    ALLOWED_PROPERTIES = {'name', 'description', 'license', 'allowed-tools', 'metadata', 'compatibility'}
+
+    # Check for unexpected properties (excluding nested keys under metadata)
+    unexpected_keys = set(frontmatter.keys()) - ALLOWED_PROPERTIES
+    if unexpected_keys:
+        return False, (
+            f"Unexpected key(s) in SKILL.md frontmatter: {', '.join(sorted(unexpected_keys))}. "
+            f"Allowed properties are: {', '.join(sorted(ALLOWED_PROPERTIES))}"
+        )
+
+    # Check required fields
+    if 'name' not in frontmatter:
+        return False, "Missing 'name' in frontmatter"
+    if 'description' not in frontmatter:
+        return False, "Missing 'description' in frontmatter"
+
+    # Extract name for validation
+    name = frontmatter.get('name', '')
+    if not isinstance(name, str):
+        return False, f"Name must be a string, got {type(name).__name__}"
+    name = name.strip()
+    if name:
+        # Check naming convention (kebab-case: lowercase with hyphens)
+        if not re.match(r'^[a-z0-9-]+$', name):
+            return False, f"Name '{name}' should be kebab-case (lowercase letters, digits, and hyphens only)"
+        if name.startswith('-') or name.endswith('-') or '--' in name:
+            return False, f"Name '{name}' cannot start/end with hyphen or contain consecutive hyphens"
+        # Check name length (max 64 characters per spec)
+        if len(name) > 64:
+            return False, f"Name is too long ({len(name)} characters). Maximum is 64 characters."
+
+    # Extract and validate description
+    description = frontmatter.get('description', '')
+    if not isinstance(description, str):
+        return False, f"Description must be a string, got {type(description).__name__}"
+    description = description.strip()
+    if description:
+        # Check for angle brackets
+        if '<' in description or '>' in description:
+            return False, "Description cannot contain angle brackets (< or >)"
+        # Check description length (max 1024 characters per spec)
+        if len(description) > 1024:
+            return False, f"Description is too long ({len(description)} characters). Maximum is 1024 characters."
+
+    # Validate compatibility field if present (optional)
+    compatibility = frontmatter.get('compatibility', '')
+    if compatibility:
+        if not isinstance(compatibility, str):
+            return False, f"Compatibility must be a string, got {type(compatibility).__name__}"
+        if len(compatibility) > 500:
+            return False, f"Compatibility is too long ({len(compatibility)} characters). Maximum is 500 characters."
+
+    return True, "Skill is valid!"
+
+if __name__ == "__main__":
+    if len(sys.argv) != 2:
+        print("Usage: python quick_validate.py <skill_directory>")
+        sys.exit(1)
+    
+    valid, message = validate_skill(sys.argv[1])
+    print(message)
+    sys.exit(0 if valid else 1)

data/lib/octo/default_skills/skill-creator/scripts/run_eval.py

@@ -0,0 +1,317 @@
+#!/usr/bin/env python3
+"""Run trigger evaluation for a skill description.
+
+Tests whether Octo's agent triggers (invokes) a skill for a set of queries.
+Runs octo agent --json in persistent mode, sends queries via stdin NDJSON,
+detects {"type":"tool_call","name":"invoke_skill","args":{"skill_name":"<name>"}}
+events, and returns pass/fail results as JSON.
+
+Executes queries serially (Octo is single-agent, no parallel workers).
+"""
+
+import argparse
+import json
+import os
+import select
+import shutil
+import subprocess
+import sys
+import time
+import uuid
+from pathlib import Path
+
+from scripts.utils import parse_skill_md
+
+
+OCTO_BIN = shutil.which("octo") or "/Users/sizzy/.local/share/mise/shims/octo"
+SKILLS_DIR = Path.home() / ".octo" / "skills"
+
+
+def find_project_root() -> Path:
+    """Find the project root by walking up from cwd, used for --path arg."""
+    current = Path.cwd()
+    for parent in [current, *current.parents]:
+        if (parent / ".octo").is_dir():
+            return parent
+    return current
+
+
+def _read_ndjson_lines(proc, timeout: float) -> list[dict]:
+    """Read NDJSON lines from proc.stdout until timeout or process exits."""
+    lines = []
+    buffer = b""
+    start = time.time()
+    while time.time() - start < timeout:
+        ready = select.select([proc.stdout], [], [], 0.5)[0]
+        if ready:
+            chunk = os.read(proc.stdout.fileno(), 8192)
+            if chunk:
+                buffer += chunk
+                while b"\n" in buffer:
+                    line_b, buffer = buffer.split(b"\n", 1)
+                    line = line_b.decode("utf-8", errors="replace").strip()
+                    if not line:
+                        continue
+                    try:
+                        lines.append(json.loads(line))
+                    except json.JSONDecodeError:
+                        pass
+        if proc.poll() is not None:
+            # drain remaining
+            remaining = proc.stdout.read()
+            if remaining:
+                for line in remaining.decode("utf-8", errors="replace").splitlines():
+                    line = line.strip()
+                    if line:
+                        try:
+                            lines.append(json.loads(line))
+                        except json.JSONDecodeError:
+                            pass
+            break
+    return lines
+
+
+def run_single_query(
+    query: str,
+    skill_name: str,
+    skill_description: str,
+    timeout: int,
+    project_root: str,
+) -> bool:
+    """Run a single query via octo agent --json and detect skill trigger.
+
+    Creates a temp skill in ~/.octo/skills/, starts octo agent in JSON mode,
+    sends the query, watches for invoke_skill tool_call event targeting our temp skill.
+    """
+    unique_id = uuid.uuid4().hex[:8]
+    temp_skill_name = f"{skill_name}-eval-{unique_id}"
+    temp_skill_dir = SKILLS_DIR / temp_skill_name
+
+    try:
+        # Write temporary skill
+        temp_skill_dir.mkdir(parents=True, exist_ok=True)
+        skill_md = (
+            f"---\n"
+            f"name: {temp_skill_name}\n"
+            f"description: {skill_description}\n"
+            f"---\n\n"
+            f"# {skill_name}\n\n"
+            f"This skill handles: {skill_description}\n"
+        )
+        (temp_skill_dir / "SKILL.md").write_text(skill_md)
+
+        # Launch octo agent in persistent JSON mode
+        proc = subprocess.Popen(
+            [OCTO_BIN, "agent", "--json", "--mode", "auto_approve",
+             "--path", project_root],
+            stdin=subprocess.PIPE,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.DEVNULL,
+            bufsize=0,
+        )
+
+        try:
+            # Wait for "system" ready event before sending query
+            start = time.time()
+            buffer = b""
+            ready_received = False
+            while time.time() - start < 10:
+                r = select.select([proc.stdout], [], [], 0.5)[0]
+                if r:
+                    chunk = os.read(proc.stdout.fileno(), 4096)
+                    if chunk:
+                        buffer += chunk
+                        while b"\n" in buffer:
+                            line_b, buffer = buffer.split(b"\n", 1)
+                            line = line_b.strip()
+                            if line:
+                                try:
+                                    evt = json.loads(line)
+                                    if evt.get("type") == "system":
+                                        ready_received = True
+                                except json.JSONDecodeError:
+                                    pass
+                if ready_received:
+                    break
+
+            # Send query
+            msg = (json.dumps({"type": "message", "content": query}) + "\n").encode()
+            proc.stdin.write(msg)
+            proc.stdin.flush()
+
+            # Read events until "complete" or timeout
+            triggered = False
+            start = time.time()
+            buffer = b""
+            while time.time() - start < timeout:
+                r = select.select([proc.stdout], [], [], 0.5)[0]
+                if r:
+                    chunk = os.read(proc.stdout.fileno(), 8192)
+                    if chunk:
+                        buffer += chunk
+                        while b"\n" in buffer:
+                            line_b, buffer = buffer.split(b"\n", 1)
+                            line = line_b.decode("utf-8", errors="replace").strip()
+                            if not line:
+                                continue
+                            try:
+                                event = json.loads(line)
+                            except json.JSONDecodeError:
+                                continue
+
+                            # Detect skill trigger
+                            if event.get("type") == "tool_call" and event.get("name") == "invoke_skill":
+                                args = event.get("args", {})
+                                invoked = args.get("skill_name", "")
+                                if invoked == temp_skill_name:
+                                    return True  # triggered — exit early
+
+                            # Task complete
+                            if event.get("type") == "complete":
+                                return triggered
+
+                if proc.poll() is not None:
+                    break
+
+            return triggered
+
+        finally:
+            # Gracefully exit the agent
+            try:
+                proc.stdin.write((json.dumps({"type": "exit"}) + "\n").encode())
+                proc.stdin.flush()
+            except Exception:
+                pass
+            if proc.poll() is None:
+                proc.kill()
+                proc.wait()
+
+    finally:
+        # Always remove temp skill directory
+        if temp_skill_dir.exists():
+            shutil.rmtree(temp_skill_dir, ignore_errors=True)
+
+
+def run_eval(
+    eval_set: list[dict],
+    skill_name: str,
+    description: str,
+    timeout: int,
+    project_root: Path,
+    runs_per_query: int = 1,
+    trigger_threshold: float = 0.5,
+) -> dict:
+    """Run the full eval set serially and return results.
+
+    Note: Octo is single-agent — queries are executed serially, not in parallel.
+    Each query spawns a fresh octo agent process to avoid session contamination.
+    """
+    results = []
+    query_triggers: dict[str, list[bool]] = {}
+    query_items: dict[str, dict] = {}
+
+    for item in eval_set:
+        query = item["query"]
+        query_items[query] = item
+        if query not in query_triggers:
+            query_triggers[query] = []
+
+        for run_idx in range(runs_per_query):
+            try:
+                triggered = run_single_query(
+                    query=query,
+                    skill_name=skill_name,
+                    skill_description=description,
+                    timeout=timeout,
+                    project_root=str(project_root),
+                )
+                query_triggers[query].append(triggered)
+            except Exception as e:
+                print(f"Warning: query failed (run {run_idx}): {e}", file=sys.stderr)
+                query_triggers[query].append(False)
+
+    for query, triggers in query_triggers.items():
+        item = query_items[query]
+        trigger_rate = sum(triggers) / len(triggers)
+        should_trigger = item["should_trigger"]
+        if should_trigger:
+            did_pass = trigger_rate >= trigger_threshold
+        else:
+            did_pass = trigger_rate < trigger_threshold
+        results.append({
+            "query": query,
+            "should_trigger": should_trigger,
+            "trigger_rate": trigger_rate,
+            "triggers": sum(triggers),
+            "runs": len(triggers),
+            "pass": did_pass,
+        })
+
+    passed = sum(1 for r in results if r["pass"])
+    total = len(results)
+
+    return {
+        "skill_name": skill_name,
+        "description": description,
+        "results": results,
+        "summary": {
+            "total": total,
+            "passed": passed,
+            "failed": total - passed,
+        },
+    }
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Run trigger evaluation for a skill description (Octo)")
+    parser.add_argument("--eval-set", required=True, help="Path to eval set JSON file")
+    parser.add_argument("--skill-path", required=True, help="Path to skill directory")
+    parser.add_argument("--description", default=None, help="Override description to test")
+    parser.add_argument("--timeout", type=int, default=45, help="Timeout per query in seconds")
+    parser.add_argument("--runs-per-query", type=int, default=1, help="Number of runs per query (serially)")
+    parser.add_argument("--trigger-threshold", type=float, default=0.5, help="Trigger rate threshold")
+    parser.add_argument("--verbose", action="store_true", help="Print progress to stderr")
+    # --num-workers kept for CLI compat but ignored (Octo is serial)
+    parser.add_argument("--num-workers", type=int, default=1, help="Ignored — Octo runs serially")
+    parser.add_argument("--model", default=None, help="Ignored — model comes from ~/.octo/config.yml")
+    args = parser.parse_args()
+
+    eval_set = json.loads(Path(args.eval_set).read_text())
+    skill_path = Path(args.skill_path)
+
+    if not (skill_path / "SKILL.md").exists():
+        print(f"Error: No SKILL.md found at {skill_path}", file=sys.stderr)
+        sys.exit(1)
+
+    name, original_description, content = parse_skill_md(skill_path)
+    description = args.description or original_description
+    project_root = find_project_root()
+
+    if args.verbose:
+        print(f"Evaluating skill: {name}", file=sys.stderr)
+        print(f"Description: {description}", file=sys.stderr)
+        print(f"Queries: {len(eval_set)}, runs-per-query: {args.runs_per_query}", file=sys.stderr)
+
+    output = run_eval(
+        eval_set=eval_set,
+        skill_name=name,
+        description=description,
+        timeout=args.timeout,
+        project_root=project_root,
+        runs_per_query=args.runs_per_query,
+        trigger_threshold=args.trigger_threshold,
+    )
+
+    if args.verbose:
+        summary = output["summary"]
+        print(f"Results: {summary['passed']}/{summary['total']} passed", file=sys.stderr)
+        for r in output["results"]:
+            status = "PASS" if r["pass"] else "FAIL"
+            rate_str = f"{r['triggers']}/{r['runs']}"
+            print(f"  [{status}] rate={rate_str} expected={r['should_trigger']}: {r['query'][:70]}", file=sys.stderr)
+
+    print(json.dumps(output, indent=2))
+
+
+if __name__ == "__main__":
+    main()