@devrev-computer/skills 1.0.0

package/skills/skill-creator/scripts/run_loop.py

@@ -0,0 +1,328 @@
+#!/usr/bin/env python3
+"""Run the eval + improve loop until all pass or max iterations reached.
+
+Combines run_eval.py and improve_description.py in a loop, tracking history
+and returning the best description found. Supports train/test split to prevent
+overfitting.
+"""
+
+import argparse
+import json
+import random
+import sys
+import tempfile
+import time
+import webbrowser
+from pathlib import Path
+
+from scripts.generate_report import generate_html
+from scripts.improve_description import improve_description
+from scripts.run_eval import find_project_root, run_eval
+from scripts.utils import parse_skill_md
+
+
+def split_eval_set(eval_set: list[dict], holdout: float, seed: int = 42) -> tuple[list[dict], list[dict]]:
+    """Split eval set into train and test sets, stratified by should_trigger."""
+    random.seed(seed)
+
+    # Separate by should_trigger
+    trigger = [e for e in eval_set if e["should_trigger"]]
+    no_trigger = [e for e in eval_set if not e["should_trigger"]]
+
+    # Shuffle each group
+    random.shuffle(trigger)
+    random.shuffle(no_trigger)
+
+    # Calculate split points
+    n_trigger_test = max(1, int(len(trigger) * holdout))
+    n_no_trigger_test = max(1, int(len(no_trigger) * holdout))
+
+    # Split
+    test_set = trigger[:n_trigger_test] + no_trigger[:n_no_trigger_test]
+    train_set = trigger[n_trigger_test:] + no_trigger[n_no_trigger_test:]
+
+    return train_set, test_set
+
+
+def run_loop(
+    eval_set: list[dict],
+    skill_path: Path,
+    description_override: str | None,
+    num_workers: int,
+    timeout: int,
+    max_iterations: int,
+    runs_per_query: int,
+    trigger_threshold: float,
+    holdout: float,
+    model: str,
+    verbose: bool,
+    live_report_path: Path | None = None,
+    log_dir: Path | None = None,
+) -> dict:
+    """Run the eval + improvement loop."""
+    project_root = find_project_root()
+    name, original_description, content = parse_skill_md(skill_path)
+    current_description = description_override or original_description
+
+    # Split into train/test if holdout > 0
+    if holdout > 0:
+        train_set, test_set = split_eval_set(eval_set, holdout)
+        if verbose:
+            print(f"Split: {len(train_set)} train, {len(test_set)} test (holdout={holdout})", file=sys.stderr)
+    else:
+        train_set = eval_set
+        test_set = []
+
+    history = []
+    exit_reason = "unknown"
+
+    for iteration in range(1, max_iterations + 1):
+        if verbose:
+            print(f"\n{'='*60}", file=sys.stderr)
+            print(f"Iteration {iteration}/{max_iterations}", file=sys.stderr)
+            print(f"Description: {current_description}", file=sys.stderr)
+            print(f"{'='*60}", file=sys.stderr)
+
+        # Evaluate train + test together in one batch for parallelism
+        all_queries = train_set + test_set
+        t0 = time.time()
+        all_results = run_eval(
+            eval_set=all_queries,
+            skill_name=name,
+            description=current_description,
+            num_workers=num_workers,
+            timeout=timeout,
+            project_root=project_root,
+            runs_per_query=runs_per_query,
+            trigger_threshold=trigger_threshold,
+            model=model,
+        )
+        eval_elapsed = time.time() - t0
+
+        # Split results back into train/test by matching queries
+        train_queries_set = {q["query"] for q in train_set}
+        train_result_list = [r for r in all_results["results"] if r["query"] in train_queries_set]
+        test_result_list = [r for r in all_results["results"] if r["query"] not in train_queries_set]
+
+        train_passed = sum(1 for r in train_result_list if r["pass"])
+        train_total = len(train_result_list)
+        train_summary = {"passed": train_passed, "failed": train_total - train_passed, "total": train_total}
+        train_results = {"results": train_result_list, "summary": train_summary}
+
+        if test_set:
+            test_passed = sum(1 for r in test_result_list if r["pass"])
+            test_total = len(test_result_list)
+            test_summary = {"passed": test_passed, "failed": test_total - test_passed, "total": test_total}
+            test_results = {"results": test_result_list, "summary": test_summary}
+        else:
+            test_results = None
+            test_summary = None
+
+        history.append({
+            "iteration": iteration,
+            "description": current_description,
+            "train_passed": train_summary["passed"],
+            "train_failed": train_summary["failed"],
+            "train_total": train_summary["total"],
+            "train_results": train_results["results"],
+            "test_passed": test_summary["passed"] if test_summary else None,
+            "test_failed": test_summary["failed"] if test_summary else None,
+            "test_total": test_summary["total"] if test_summary else None,
+            "test_results": test_results["results"] if test_results else None,
+            # For backward compat with report generator
+            "passed": train_summary["passed"],
+            "failed": train_summary["failed"],
+            "total": train_summary["total"],
+            "results": train_results["results"],
+        })
+
+        # Write live report if path provided
+        if live_report_path:
+            partial_output = {
+                "original_description": original_description,
+                "best_description": current_description,
+                "best_score": "in progress",
+                "iterations_run": len(history),
+                "holdout": holdout,
+                "train_size": len(train_set),
+                "test_size": len(test_set),
+                "history": history,
+            }
+            live_report_path.write_text(generate_html(partial_output, auto_refresh=True, skill_name=name))
+
+        if verbose:
+            def print_eval_stats(label, results, elapsed):
+                pos = [r for r in results if r["should_trigger"]]
+                neg = [r for r in results if not r["should_trigger"]]
+                tp = sum(r["triggers"] for r in pos)
+                pos_runs = sum(r["runs"] for r in pos)
+                fn = pos_runs - tp
+                fp = sum(r["triggers"] for r in neg)
+                neg_runs = sum(r["runs"] for r in neg)
+                tn = neg_runs - fp
+                total = tp + tn + fp + fn
+                precision = tp / (tp + fp) if (tp + fp) > 0 else 1.0
+                recall = tp / (tp + fn) if (tp + fn) > 0 else 1.0
+                accuracy = (tp + tn) / total if total > 0 else 0.0
+                print(f"{label}: {tp+tn}/{total} correct, precision={precision:.0%} recall={recall:.0%} accuracy={accuracy:.0%} ({elapsed:.1f}s)", file=sys.stderr)
+                for r in results:
+                    status = "PASS" if r["pass"] else "FAIL"
+                    rate_str = f"{r['triggers']}/{r['runs']}"
+                    print(f"  [{status}] rate={rate_str} expected={r['should_trigger']}: {r['query'][:60]}", file=sys.stderr)
+
+            print_eval_stats("Train", train_results["results"], eval_elapsed)
+            if test_summary:
+                print_eval_stats("Test ", test_results["results"], 0)
+
+        if train_summary["failed"] == 0:
+            exit_reason = f"all_passed (iteration {iteration})"
+            if verbose:
+                print(f"\nAll train queries passed on iteration {iteration}!", file=sys.stderr)
+            break
+
+        if iteration == max_iterations:
+            exit_reason = f"max_iterations ({max_iterations})"
+            if verbose:
+                print(f"\nMax iterations reached ({max_iterations}).", file=sys.stderr)
+            break
+
+        # Improve the description based on train results
+        if verbose:
+            print(f"\nImproving description...", file=sys.stderr)
+
+        t0 = time.time()
+        # Strip test scores from history so improvement model can't see them
+        blinded_history = [
+            {k: v for k, v in h.items() if not k.startswith("test_")}
+            for h in history
+        ]
+        new_description = improve_description(
+            skill_name=name,
+            skill_content=content,
+            current_description=current_description,
+            eval_results=train_results,
+            history=blinded_history,
+            model=model,
+            log_dir=log_dir,
+            iteration=iteration,
+        )
+        improve_elapsed = time.time() - t0
+
+        if verbose:
+            print(f"Proposed ({improve_elapsed:.1f}s): {new_description}", file=sys.stderr)
+
+        current_description = new_description
+
+    # Find the best iteration by TEST score (or train if no test set)
+    if test_set:
+        best = max(history, key=lambda h: h["test_passed"] or 0)
+        best_score = f"{best['test_passed']}/{best['test_total']}"
+    else:
+        best = max(history, key=lambda h: h["train_passed"])
+        best_score = f"{best['train_passed']}/{best['train_total']}"
+
+    if verbose:
+        print(f"\nExit reason: {exit_reason}", file=sys.stderr)
+        print(f"Best score: {best_score} (iteration {best['iteration']})", file=sys.stderr)
+
+    return {
+        "exit_reason": exit_reason,
+        "original_description": original_description,
+        "best_description": best["description"],
+        "best_score": best_score,
+        "best_train_score": f"{best['train_passed']}/{best['train_total']}",
+        "best_test_score": f"{best['test_passed']}/{best['test_total']}" if test_set else None,
+        "final_description": current_description,
+        "iterations_run": len(history),
+        "holdout": holdout,
+        "train_size": len(train_set),
+        "test_size": len(test_set),
+        "history": history,
+    }
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Run eval + improve loop")
+    parser.add_argument("--eval-set", required=True, help="Path to eval set JSON file")
+    parser.add_argument("--skill-path", required=True, help="Path to skill directory")
+    parser.add_argument("--description", default=None, help="Override starting description")
+    parser.add_argument("--num-workers", type=int, default=10, help="Number of parallel workers")
+    parser.add_argument("--timeout", type=int, default=30, help="Timeout per query in seconds")
+    parser.add_argument("--max-iterations", type=int, default=5, help="Max improvement iterations")
+    parser.add_argument("--runs-per-query", type=int, default=3, help="Number of runs per query")
+    parser.add_argument("--trigger-threshold", type=float, default=0.5, help="Trigger rate threshold")
+    parser.add_argument("--holdout", type=float, default=0.4, help="Fraction of eval set to hold out for testing (0 to disable)")
+    parser.add_argument("--model", required=True, help="Model for improvement")
+    parser.add_argument("--verbose", action="store_true", help="Print progress to stderr")
+    parser.add_argument("--report", default="auto", help="Generate HTML report at this path (default: 'auto' for temp file, 'none' to disable)")
+    parser.add_argument("--results-dir", default=None, help="Save all outputs (results.json, report.html, log.txt) to a timestamped subdirectory here")
+    args = parser.parse_args()
+
+    eval_set = json.loads(Path(args.eval_set).read_text())
+    skill_path = Path(args.skill_path)
+
+    if not (skill_path / "SKILL.md").exists():
+        print(f"Error: No SKILL.md found at {skill_path}", file=sys.stderr)
+        sys.exit(1)
+
+    name, _, _ = parse_skill_md(skill_path)
+
+    # Set up live report path
+    if args.report != "none":
+        if args.report == "auto":
+            timestamp = time.strftime("%Y%m%d_%H%M%S")
+            live_report_path = Path(tempfile.gettempdir()) / f"skill_description_report_{skill_path.name}_{timestamp}.html"
+        else:
+            live_report_path = Path(args.report)
+        # Open the report immediately so the user can watch
+        live_report_path.write_text("<html><body><h1>Starting optimization loop...</h1><meta http-equiv='refresh' content='5'></body></html>")
+        webbrowser.open(str(live_report_path))
+    else:
+        live_report_path = None
+
+    # Determine output directory (create before run_loop so logs can be written)
+    if args.results_dir:
+        timestamp = time.strftime("%Y-%m-%d_%H%M%S")
+        results_dir = Path(args.results_dir) / timestamp
+        results_dir.mkdir(parents=True, exist_ok=True)
+    else:
+        results_dir = None
+
+    log_dir = results_dir / "logs" if results_dir else None
+
+    output = run_loop(
+        eval_set=eval_set,
+        skill_path=skill_path,
+        description_override=args.description,
+        num_workers=args.num_workers,
+        timeout=args.timeout,
+        max_iterations=args.max_iterations,
+        runs_per_query=args.runs_per_query,
+        trigger_threshold=args.trigger_threshold,
+        holdout=args.holdout,
+        model=args.model,
+        verbose=args.verbose,
+        live_report_path=live_report_path,
+        log_dir=log_dir,
+    )
+
+    # Save JSON output
+    json_output = json.dumps(output, indent=2)
+    print(json_output)
+    if results_dir:
+        (results_dir / "results.json").write_text(json_output)
+
+    # Write final HTML report (without auto-refresh)
+    if live_report_path:
+        live_report_path.write_text(generate_html(output, auto_refresh=False, skill_name=name))
+        print(f"\nReport: {live_report_path}", file=sys.stderr)
+
+    if results_dir and live_report_path:
+        (results_dir / "report.html").write_text(generate_html(output, auto_refresh=False, skill_name=name))
+
+    if results_dir:
+        print(f"Results saved to: {results_dir}", file=sys.stderr)
+
+
+if __name__ == "__main__":
+    main()

package/skills/skill-creator/scripts/utils.py

@@ -0,0 +1,47 @@
+"""Shared utilities for skill-creator scripts."""
+
+from pathlib import Path
+
+
+
+def parse_skill_md(skill_path: Path) -> tuple[str, str, str]:
+    """Parse a SKILL.md file, returning (name, description, full_content)."""
+    content = (skill_path / "SKILL.md").read_text()
+    lines = content.split("\n")
+
+    if lines[0].strip() != "---":
+        raise ValueError("SKILL.md missing frontmatter (no opening ---)")
+
+    end_idx = None
+    for i, line in enumerate(lines[1:], start=1):
+        if line.strip() == "---":
+            end_idx = i
+            break
+
+    if end_idx is None:
+        raise ValueError("SKILL.md missing frontmatter (no closing ---)")
+
+    name = ""
+    description = ""
+    frontmatter_lines = lines[1:end_idx]
+    i = 0
+    while i < len(frontmatter_lines):
+        line = frontmatter_lines[i]
+        if line.startswith("name:"):
+            name = line[len("name:"):].strip().strip('"').strip("'")
+        elif line.startswith("description:"):
+            value = line[len("description:"):].strip()
+            # Handle YAML multiline indicators (>, |, >-, |-)
+            if value in (">", "|", ">-", "|-"):
+                continuation_lines: list[str] = []
+                i += 1
+                while i < len(frontmatter_lines) and (frontmatter_lines[i].startswith("  ") or frontmatter_lines[i].startswith("\t")):
+                    continuation_lines.append(frontmatter_lines[i].strip())
+                    i += 1
+                description = " ".join(continuation_lines)
+                continue
+            else:
+                description = value.strip('"').strip("'")
+        i += 1
+
+    return name, description, content

package/skills/trace-diagnosis/trace-diagnosis.md

@@ -0,0 +1,186 @@
+---
+skill-name: trace-diagnosis
+user-invocable: true
+description: Diagnose issues in Computer agent execution traces. Fetches MELTS traces, analyzes tool calls, errors, latency and response quality, creates an engineering ticket. Use when debugging agent failures or investigating why the Computer agent gave wrong answers.
+arguments:
+  - name: context
+    description: Description of the issue or the conversation context where the agent failed
+    required: false
+---
+
+# TraceDiagnosis
+
+Diagnoses issues in Computer agent execution traces. Fetches MELTS traces for recent user messages, analyzes tool calls, errors, latency and response quality, and provides a triage summary.
+
+## Tools
+
+1. **FetchTraces** — Fetches MELTS execution traces for the current conversation. Returns the skill call chain, inputs, outputs, errors, timing, and token usage.
+2. **FetchKnowledgeBase** — Returns content from the **Computer Agent Common Issues** reference article only. Use it for known root causes and fixes. Do not use the customer org's knowledge base or any other org articles for this analysis.
+3. **CreateTicket** — Creates a ticket with the full technical analysis as the description.
+
+## Your Output to the User
+
+Your text output should be two parts: (1) one line stating the root cause only, in plain language; (2) one line stating that we have used their data to report this bug to the DevRev team, and the ticket ID. Keep it brief — 2 to 3 sentences total.
+
+Example output:
+"I analyzed the traces and found the issue: the agent used a search tool instead of a database query for your request. I've created ticket TKT-181 to track this for the engineering team."
+
+Do not include in your output:
+- Headings, sections, or markdown formatting
+- "What should have happened" or correct execution paths
+- Skill or tool names (KnowledgeSearch, NLToSQL, GetKGSchema, HybridSearch, ExecuteSQL, FetchObjectContext, etc.)
+- Technical terms (SQL, namespace, DON ID, aggregation, semantic search, DQL, etc.)
+- Step-by-step tables or bullet point breakdowns
+- Code blocks or SQL examples
+- Recommendations or workarounds
+- Apologies or follow-up questions
+
+All technical details go exclusively into the CreateTicket description where only engineers see them.
+
+## Task Sequence
+
+1. Call **FetchTraces** to get execution traces for this conversation. Check the result carefully — if the output contains "timeout", "error", or is empty, treat it as a failure even if the tool call itself reports success. If traces are unavailable, use the SHORT ticket template (Reference B2) instead of the full template.
+2. Call **FetchKnowledgeBase** with the error or failure pattern to find known root causes.
+3. Call **CreateTicket**:
+   - If you have trace data: use the FULL ticket template (Reference B1) with actual data from the traces.
+   - If traces are unavailable: use the SHORT ticket template (Reference B2). Do NOT use the full template and fill it with "not available".
+4. Output (1) one line with the root cause only, and (2) one line that we've reported this with the ticket ID.
+
+## Reference A: Failure Patterns
+
+Use these to classify the failure when writing the CreateTicket description:
+
+- **A: SQL Silent Fallback** — GetNodeSchema fails/wrong DON ID → HybridSearch used for analytical query → wrong counts
+- **B: SQL Generation Error** — NLToSQL/ExecuteSQL → table/column not found, forbidden expression
+- **C: Code Counting Error** — ExecuteSQL returns N rows → ExecuteCode → different count
+- **D: Invalid Search Params** — HybridSearch → namespace not permitted, bad params
+- **E: Object Fetch Failure** — FetchObjectContext → mongo no documents, invalid ID, 400/404 error
+- **F: Permission Error** — GetKGSchema → "not authorised" → downstream blocked
+- **G: Timeout/503** — Any skill → 503, timeout, deadline exceeded
+- **H: Context Overflow** — Multiple calls succeed → sudden "unable to process"
+- **I: No Skills Called** — Agent responded without calling any tools
+- **J: Wrong Skill Selected** — Wrong tool for the job (e.g. HybridSearch for counts)
+
+## Reference B1: Full Ticket Template (when trace data IS available)
+
+**Title**: [TraceDiagnosis] — [what failed in ≤10 words]
+
+**Description**:
+
+```
+## Summary
+[2-3 sentences: what the user asked, what went wrong, what the impact was]
+
+## Failure Pattern
+[Pattern letter]: [1 sentence explaining why this pattern was identified]
+
+## Turn 1 Analysis: "[user's exact query for turn 1]"
+
+**Agent Response**: "[first 200 chars of agent's response]..."
+
+### Skill Calls
+
+#### Skill Call 1: [SkillName]
+- **Query**: [the search query, SQL query, or natural language input — verbatim from trace]
+- **Namespace** (if applicable): [the namespace value passed]
+- **Full Input**: [complete input JSON or all key fields]
+- **Output**: [full output or key fields, truncate if >500 chars with "[truncated]"]
+- **Duration**: [X]ms
+- **Status**: Success / Failed ([HTTP status code])
+- **Error**: [exact error message if failed, "none" if success]
+
+[Continue for ALL skill calls in this turn.]
+
+### Turn 1 Errors
+1. **[SkillName] — [error type]**: [exact error message]
+   - Triggering input: [what caused it]
+   - Downstream impact: [what broke because of this]
+
+### Turn 1 Assessment
+[What went wrong — 2-3 sentences]
+
+**Token Usage**: [prompt_tokens] prompt / [completion_tokens] completion
+**Total Duration**: [X]ms
+
+---
+
+[Add more turns as needed]
+
+---
+
+## Root Cause Analysis
+
+### What Failed
+[Name the specific skill, error, input that caused it. Include DON IDs, error codes.]
+
+### Why It Failed
+[Technical root cause. Reference KB match if found.]
+
+### Silent Fallbacks
+[List cases where skill failed and agent silently switched approach.]
+
+### Knowledge Base Match
+[Quote relevant root cause and fix from KB. If no match: "No match found."]
+
+## Impact
+[How this affected the user]
+
+## Severity
+[High / Medium / Low]
+
+## Recommendation
+*Only if FetchKnowledgeBase found a match.*
+- **Workaround**: [1 sentence]
+- **Engineering Fix**: [1 sentence]
+- **Known Issue**: [Yes/No]
+
+## Reproduction
+- **DM ID**: [conversation ID]
+- **Timestamp**: [when failing query was sent]
+- **User**: [user name and DON ID]
+- **Org**: [org slug / DON]
+- **Agent**: [agent ID]
+- **Repro query**: "[exact user query]"
+```
+
+## Reference B2: Short Ticket Template (when trace data is NOT available)
+
+**Title**: [TraceDiagnosis] Trace Unavailable — [brief description]
+
+**Description**:
+
+```
+## Summary
+[2-3 sentences: what the user asked and what appeared to go wrong]
+
+## Trace Status
+FetchTraces returned: [exact error message or "timeout"].
+
+## What We Know
+- **User query**: "[exact user query]"
+- **Agent response summary**: [brief summary]
+- **Suspected pattern**: [Pattern letter or "Unknown"]
+- **Knowledge base match**: [if found]
+
+## What Engineers Need To Do
+1. Pull MELTS traces manually
+2. Identify full skill call chain and error cascade
+3. Confirm suspected failure pattern
+
+## Reproduction
+- **DM ID**: [if available]
+- **User**: [user name and DON ID]
+- **Org**: [org slug / DON]
+- **Repro query**: "[exact user query]"
+```
+
+## Rules
+
+1. Your text output to the user is two parts only: root cause + ticket ID. No technical details.
+2. Always call FetchTraces first. Inspect result for "timeout", "error", or empty — treat as failure.
+3. Always create a ticket. Every diagnosis gets a ticket.
+4. Choose the right template: B1 when you have traces, B2 when you don't. Never fill B1 with "not available".
+5. When using B1: be exhaustive — engineers should never need to re-pull traces.
+6. When using B2: keep it short and actionable.
+7. Never fabricate trace data. Only include fields where you have actual data.
+8. For known root causes, use only FetchKnowledgeBase output. Do not search customer org's data.