@dev-sergeev/datalab-kit 0.1.5 → 0.1.6

package/skills/skill-creator/scripts/pi_runtime.py

@@ -0,0 +1,177 @@
+"""Pi JSON CLI backend for the Anthropic skill-creator scripts.
+
+Added by Datalab Kit for Pi; upstream source pinned at
+anthropics/skills dbd4588f9e1033efb41dad4bef2f7947c8993d44 (Apache-2.0).
+"""
+
+import json
+import os
+import selectors
+import signal
+import subprocess
+import tempfile
+import threading
+import time
+from contextlib import contextmanager
+from pathlib import Path
+
+
+def pi_command(model: str | None = None) -> list[str]:
+    """Use the host Pi installation without importing package-local Node modules."""
+    # Keep configured extensions: they may register the selected model provider.
+    # Discovery isolation concerns skills/context, not the user's credentials.
+    command = [
+        os.environ.get("PI_EXECUTABLE_PATH") or "pi",
+        "--print", "--mode", "json", "--no-session",
+        "--no-skills", "--no-prompt-templates", "--no-themes",
+        "--no-context-files", "--no-approve",
+        # Empty explicit sources suppress global SYSTEM.md/APPEND_SYSTEM.md
+        # discovery while retaining Pi's built-in system prompt.
+        "--system-prompt", "", "--append-system-prompt", "",
+    ]
+    if model:
+        command.extend(["--model", model])
+    return command
+
+
+@contextmanager
+def interrupt_on_termination():
+    """Give CLI SIGTERM the same resource cleanup guarantees as Ctrl-C."""
+    if threading.current_thread() is not threading.main_thread():
+        yield
+        return
+    previous = signal.getsignal(signal.SIGTERM)
+
+    def terminate(signum, frame):
+        raise SystemExit(128 + signum)
+
+    signal.signal(signal.SIGTERM, terminate)
+    try:
+        yield
+    finally:
+        signal.signal(signal.SIGTERM, previous)
+
+
+def _stop_process(process: subprocess.Popen) -> None:
+    """Reap Pi and terminate its subprocess group, including tool children."""
+    if os.name == "posix":
+        # Also kill descendants if the group leader has already exited.
+        try:
+            os.killpg(process.pid, signal.SIGTERM)
+        except ProcessLookupError:
+            pass
+        try:
+            process.wait(timeout=2)
+        except subprocess.TimeoutExpired:
+            pass
+        try:
+            os.killpg(process.pid, signal.SIGKILL)
+        except ProcessLookupError:
+            pass
+    elif process.poll() is None:
+        process.kill()
+    process.wait()
+
+
+def run_pi(
+    prompt: str,
+    model: str | None,
+    timeout: int,
+    cwd: Path,
+    options: list[str],
+    on_event=None,
+    cancel: threading.Event | None = None,
+) -> str:
+    """Run one fresh Pi session, optionally stopping at a decisive tool event.
+
+    An on_event callback returning True marks an intentional early completion.
+    Otherwise a valid agent_end, successful process exit and non-error assistant
+    messages are required. Provider failures can exit zero in Pi JSON mode.
+    """
+    command = pi_command(model) + options
+    # A file-backed stdin avoids argument-size limits and pipe write deadlocks.
+    with tempfile.TemporaryFile() as stdin, tempfile.TemporaryFile() as stderr:
+        stdin.write(prompt.encode("utf-8"))
+        stdin.seek(0)
+        process = subprocess.Popen(
+            command, stdin=stdin, stdout=subprocess.PIPE, stderr=stderr,
+            cwd=cwd, start_new_session=(os.name == "posix"),
+        )
+        deadline = time.monotonic() + timeout
+        text = []
+        ended = False
+        assistant_seen = False
+        buffer = b""
+
+        def consume(line):
+            nonlocal ended, assistant_seen
+            if not line.strip():
+                return False
+            try:
+                event = json.loads(line)
+            except (ValueError, UnicodeDecodeError) as error:
+                raise RuntimeError("Pi emitted invalid JSON on stdout") from error
+            if not isinstance(event, dict):
+                raise RuntimeError("Pi emitted a non-object JSON event")
+            if event.get("type") == "message_end":
+                message = event.get("message", {})
+                if message.get("role") == "assistant":
+                    assistant_seen = True
+                    if message.get("stopReason") in ("error", "aborted"):
+                        raise RuntimeError(f"Pi model failed: {message.get('errorMessage') or message['stopReason']}")
+                    text.extend(block["text"] for block in message.get("content", []) if block.get("type") == "text")
+            if event.get("type") == "agent_end":
+                ended = True
+            return bool(on_event and on_event(event))
+
+        try:
+            with selectors.DefaultSelector() as selector:
+                selector.register(process.stdout, selectors.EVENT_READ)
+                while True:
+                    if cancel is not None and cancel.is_set():
+                        raise RuntimeError("Pi evaluation cancelled")
+                    remaining = deadline - time.monotonic()
+                    if remaining <= 0:
+                        raise TimeoutError(f"Pi did not complete within {timeout} seconds")
+                    if not selector.select(min(remaining, 0.1)):
+                        continue
+                    chunk = os.read(process.stdout.fileno(), 65536)
+                    if not chunk:
+                        if buffer and consume(buffer):
+                            return "".join(text)
+                        break
+                    buffer += chunk
+                    while b"\n" in buffer:
+                        line, buffer = buffer.split(b"\n", 1)
+                        if consume(line):
+                            return "".join(text)
+            remaining = deadline - time.monotonic()
+            if remaining <= 0:
+                raise TimeoutError(f"Pi did not complete within {timeout} seconds")
+            try:
+                returncode = process.wait(timeout=remaining)
+            except subprocess.TimeoutExpired as error:
+                raise TimeoutError(f"Pi did not complete within {timeout} seconds") from error
+            if returncode != 0:
+                raise RuntimeError(f"Pi exited {returncode}")
+            if not ended or not assistant_seen:
+                raise RuntimeError("Pi exited without a complete assistant session")
+            return "".join(text)
+        except Exception as error:
+            stderr.seek(0)
+            diagnostic = stderr.read().decode("utf-8", errors="replace").strip()
+            if diagnostic:
+                raise RuntimeError(f"{error}\nPi stderr: {diagnostic}") from error
+            raise
+        finally:
+            _stop_process(process)
+            process.stdout.close()
+
+
+def call_pi(prompt: str, model: str | None, timeout: int = 300) -> str:
+    """Generate text without tools, skill discovery or writes in the package."""
+    with tempfile.TemporaryDirectory(prefix="pi-skill-description-") as workdir:
+        response = run_pi(prompt, model, timeout, Path(workdir), ["--no-tools"])
+    if not response.strip():
+        raise RuntimeError("Pi returned no description text")
+    return response

package/skills/skill-creator/scripts/quick_validate.py

@@ -0,0 +1,107 @@
+#!/usr/bin/env python3
+"""
+Quick validation script for skills - minimal version
+
+Changed by Datalab Kit: accept Pi's disable-model-invocation metadata.
+Upstream: anthropics/skills dbd4588f9e1033efb41dad4bef2f7947c8993d44
+(Apache-2.0).
+"""
+
+import sys
+import os
+import re
+import yaml
+from pathlib import Path
+
+def validate_skill(skill_path):
+    """Basic validation of a skill"""
+    skill_path = Path(skill_path)
+
+    # Check SKILL.md exists
+    skill_md = skill_path / 'SKILL.md'
+    if not skill_md.exists():
+        return False, "SKILL.md not found"
+
+    # Read and validate frontmatter
+    content = skill_md.read_text()
+    if not content.startswith('---'):
+        return False, "No YAML frontmatter found"
+
+    # Extract frontmatter
+    match = re.match(r'^---\n(.*?)\n---', content, re.DOTALL)
+    if not match:
+        return False, "Invalid frontmatter format"
+
+    frontmatter_text = match.group(1)
+
+    # Parse YAML frontmatter
+    try:
+        frontmatter = yaml.safe_load(frontmatter_text)
+        if not isinstance(frontmatter, dict):
+            return False, "Frontmatter must be a YAML dictionary"
+    except yaml.YAMLError as e:
+        return False, f"Invalid YAML in frontmatter: {e}"
+
+    # Define allowed properties
+    ALLOWED_PROPERTIES = {'name', 'description', 'license', 'allowed-tools', 'metadata', 'compatibility', 'disable-model-invocation'}
+
+    # Check for unexpected properties (excluding nested keys under metadata)
+    unexpected_keys = set(frontmatter.keys()) - ALLOWED_PROPERTIES
+    if unexpected_keys:
+        return False, (
+            f"Unexpected key(s) in SKILL.md frontmatter: {', '.join(sorted(unexpected_keys))}. "
+            f"Allowed properties are: {', '.join(sorted(ALLOWED_PROPERTIES))}"
+        )
+
+    # Check required fields
+    if 'name' not in frontmatter:
+        return False, "Missing 'name' in frontmatter"
+    if 'description' not in frontmatter:
+        return False, "Missing 'description' in frontmatter"
+
+    # Extract name for validation
+    name = frontmatter.get('name', '')
+    if not isinstance(name, str):
+        return False, f"Name must be a string, got {type(name).__name__}"
+    name = name.strip()
+    if name:
+        # Check naming convention (kebab-case: lowercase with hyphens)
+        if not re.match(r'^[a-z0-9-]+$', name):
+            return False, f"Name '{name}' should be kebab-case (lowercase letters, digits, and hyphens only)"
+        if name.startswith('-') or name.endswith('-') or '--' in name:
+            return False, f"Name '{name}' cannot start/end with hyphen or contain consecutive hyphens"
+        # Check name length (max 64 characters per spec)
+        if len(name) > 64:
+            return False, f"Name is too long ({len(name)} characters). Maximum is 64 characters."
+
+    # Extract and validate description
+    description = frontmatter.get('description', '')
+    if not isinstance(description, str):
+        return False, f"Description must be a string, got {type(description).__name__}"
+    description = description.strip()
+    if description:
+        # Check for angle brackets
+        if '<' in description or '>' in description:
+            return False, "Description cannot contain angle brackets (< or >)"
+        # Check description length (max 1024 characters per spec)
+        if len(description) > 1024:
+            return False, f"Description is too long ({len(description)} characters). Maximum is 1024 characters."
+
+    # Validate compatibility field if present (optional)
+    compatibility = frontmatter.get('compatibility', '')
+    if compatibility:
+        if not isinstance(compatibility, str):
+            return False, f"Compatibility must be a string, got {type(compatibility).__name__}"
+        if len(compatibility) > 500:
+            return False, f"Compatibility is too long ({len(compatibility)} characters). Maximum is 500 characters."
+
+    return True, "Skill is valid!"
+
+if __name__ == "__main__":
+    if len(sys.argv) != 2:
+        print("Usage: python quick_validate.py <skill_directory>")
+        sys.exit(1)
+    
+    valid, message = validate_skill(sys.argv[1])
+    print(message)
+    sys.exit(0 if valid else 1)

package/skills/skill-creator/scripts/run_eval.py

@@ -0,0 +1,215 @@
+#!/usr/bin/env python3
+"""Run trigger evaluation for a skill description.
+
+Tests whether a skill's description causes Pi to trigger (read the skill)
+for a set of queries. Outputs results as JSON.
+
+Changed by Datalab Kit: isolated Pi JSON CLI backend, error propagation and
+cancellation cleanup. Upstream: anthropics/skills
+dbd4588f9e1033efb41dad4bef2f7947c8993d44 (Apache-2.0).
+"""
+
+import argparse
+import json
+import sys
+import threading
+import tempfile
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from pathlib import Path
+
+import yaml
+
+from scripts.pi_runtime import interrupt_on_termination, run_pi
+
+from scripts.utils import parse_skill_md
+
+
+def find_project_root() -> Path:
+    """Use the caller's workspace; no host-specific discovery files are written."""
+    return Path.cwd()
+
+
+def run_single_query(
+    query: str,
+    skill_name: str,
+    skill_description: str,
+    timeout: int,
+    project_root: str,
+    model: str | None = None,
+    cancel: threading.Event | None = None,
+) -> bool:
+    """Evaluate natural skill selection in a fresh, isolated Pi session.
+
+    Only the candidate description is advertised, not the creator or any other
+    installed skills. The read-only tool allowlist prevents evaluation queries
+    from modifying the caller's workspace. A read of this exact SKILL.md is
+    the trigger signal; narration and reads of unrelated files do not count.
+    """
+    with tempfile.TemporaryDirectory(prefix="pi-skill-trigger-") as workdir:
+        root = Path(workdir)
+        candidate = root / "SKILL.md"
+        frontmatter = yaml.safe_dump(
+            {"name": skill_name, "description": skill_description},
+            sort_keys=False, allow_unicode=True,
+        )
+        candidate.write_text(
+            f"---\n{frontmatter}---\n\n# {skill_name}\n\n"
+            "This is a description-trigger evaluation. The skill has been loaded.\n",
+            encoding="utf-8",
+        )
+        triggered = False
+
+        def detect_trigger(event):
+            nonlocal triggered
+            if event.get("type") != "tool_execution_start" or event.get("toolName") != "read":
+                return False
+            path = event.get("args", {}).get("path")
+            if not isinstance(path, str):
+                return False
+            requested = Path(path).expanduser()
+            if not requested.is_absolute():
+                requested = root / requested
+            triggered = requested.resolve() == candidate.resolve()
+            return triggered
+
+        run_pi(
+            query, model, timeout, root,
+            ["--skill", str(candidate), "--tools", "read"],
+            on_event=detect_trigger, cancel=cancel,
+        )
+        return triggered
+
+
+def run_eval(
+    eval_set: list[dict],
+    skill_name: str,
+    description: str,
+    num_workers: int,
+    timeout: int,
+    project_root: Path,
+    runs_per_query: int = 1,
+    trigger_threshold: float = 0.5,
+    model: str | None = None,
+) -> dict:
+    """Run the full eval set and return results."""
+    results = []
+
+    cancel = threading.Event()
+    executor = ThreadPoolExecutor(max_workers=num_workers)
+    try:
+        future_to_info = {}
+        for item in eval_set:
+            for run_idx in range(runs_per_query):
+                future = executor.submit(
+                    run_single_query,
+                    item["query"],
+                    skill_name,
+                    description,
+                    timeout,
+                    str(project_root),
+                    model,
+                    cancel,
+                )
+                future_to_info[future] = (item, run_idx)
+
+        query_triggers: dict[str, list[bool]] = {}
+        query_items: dict[str, dict] = {}
+        for future in as_completed(future_to_info):
+            item, _ = future_to_info[future]
+            query = item["query"]
+            query_items[query] = item
+            if query not in query_triggers:
+                query_triggers[query] = []
+            try:
+                query_triggers[query].append(future.result())
+            except Exception as error:
+                raise RuntimeError(f"Trigger evaluation failed for {query!r}: {error}") from error
+    finally:
+        cancel.set()
+        executor.shutdown(wait=True, cancel_futures=True)
+
+    for query, triggers in query_triggers.items():
+        item = query_items[query]
+        trigger_rate = sum(triggers) / len(triggers)
+        should_trigger = item["should_trigger"]
+        if should_trigger:
+            did_pass = trigger_rate >= trigger_threshold
+        else:
+            did_pass = trigger_rate < trigger_threshold
+        results.append({
+            "query": query,
+            "should_trigger": should_trigger,
+            "trigger_rate": trigger_rate,
+            "triggers": sum(triggers),
+            "runs": len(triggers),
+            "pass": did_pass,
+        })
+
+    passed = sum(1 for r in results if r["pass"])
+    total = len(results)
+
+    return {
+        "skill_name": skill_name,
+        "description": description,
+        "results": results,
+        "summary": {
+            "total": total,
+            "passed": passed,
+            "failed": total - passed,
+        },
+    }
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Run trigger evaluation for a skill description")
+    parser.add_argument("--eval-set", required=True, help="Path to eval set JSON file")
+    parser.add_argument("--skill-path", required=True, help="Path to skill directory")
+    parser.add_argument("--description", default=None, help="Override description to test")
+    parser.add_argument("--num-workers", type=int, default=10, help="Number of parallel workers")
+    parser.add_argument("--timeout", type=int, default=30, help="Timeout per query in seconds")
+    parser.add_argument("--runs-per-query", type=int, default=3, help="Number of runs per query")
+    parser.add_argument("--trigger-threshold", type=float, default=0.5, help="Trigger rate threshold")
+    parser.add_argument("--model", default=None, help="Pi model provider/id[:thinking] (default: configured model)")
+    parser.add_argument("--verbose", action="store_true", help="Print progress to stderr")
+    args = parser.parse_args()
+
+    eval_set = json.loads(Path(args.eval_set).read_text())
+    skill_path = Path(args.skill_path)
+
+    if not (skill_path / "SKILL.md").exists():
+        print(f"Error: No SKILL.md found at {skill_path}", file=sys.stderr)
+        sys.exit(1)
+
+    name, original_description, content = parse_skill_md(skill_path)
+    description = args.description or original_description
+    project_root = find_project_root()
+
+    if args.verbose:
+        print(f"Evaluating: {description}", file=sys.stderr)
+
+    output = run_eval(
+        eval_set=eval_set,
+        skill_name=name,
+        description=description,
+        num_workers=args.num_workers,
+        timeout=args.timeout,
+        project_root=project_root,
+        runs_per_query=args.runs_per_query,
+        trigger_threshold=args.trigger_threshold,
+        model=args.model,
+    )
+
+    if args.verbose:
+        summary = output["summary"]
+        print(f"Results: {summary['passed']}/{summary['total']} passed", file=sys.stderr)
+        for r in output["results"]:
+            status = "PASS" if r["pass"] else "FAIL"
+            rate_str = f"{r['triggers']}/{r['runs']}"
+            print(f"  [{status}] rate={rate_str} expected={r['should_trigger']}: {r['query'][:70]}", file=sys.stderr)
+
+    print(json.dumps(output, indent=2))
+
+
+if __name__ == "__main__":
+    with interrupt_on_termination():
+        main()