npm - @rudderhq/agent-runtime-gemini-local - Versions diffs - 0.2.1 → 0.2.2-canary.0 - Mend

@rudderhq/agent-runtime-gemini-local 0.2.1 → 0.2.2-canary.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (42) hide show

package/skills/conversation-to-skill/scripts/generate_report.py ADDED Viewed

@@ -0,0 +1,335 @@
+#!/usr/bin/env python3
+"""Generate an HTML report from run_loop.py output.
+Takes the JSON output from run_loop.py and generates a visual HTML report
+showing each description attempt with check/x for each test case.
+Distinguishes between train and test queries.
+"""
+import argparse
+import html
+import json
+import sys
+from pathlib import Path
+def generate_html(data: dict, auto_refresh: bool = False, skill_name: str = "") -> str:
+    """Generate HTML report from loop output data. If auto_refresh is True, adds a meta refresh tag."""
+    history = data.get("history", [])
+    title_prefix = html.escape(skill_name + " \u2014 ") if skill_name else ""
+    if not history:
+        return f"""<!DOCTYPE html>
+<html>
+<head>
+    <meta charset="utf-8">
+    <title>{title_prefix}Skill Description Optimization</title>
+</head>
+<body>
+    <h1>{title_prefix}Skill Description Optimization</h1>
+    <p>No optimization history was found in the provided JSON.</p>
+</body>
+</html>
+"""
+    # Get all unique queries from train and test sets, with should_trigger info
+    train_queries: list[dict] = []
+    test_queries: list[dict] = []
+    for r in history[0].get("train_results", history[0].get("results", [])):
+        train_queries.append({"query": r["query"], "should_trigger": r.get("should_trigger", True)})
+    for r in history[0].get("test_results") or []:
+        test_queries.append({"query": r["query"], "should_trigger": r.get("should_trigger", True)})
+    refresh_tag = '    <meta http-equiv="refresh" content="5">\n' if auto_refresh else ""
+    html_parts = ["""<!DOCTYPE html>
+<html>
+<head>
+    <meta charset="utf-8">
+""" + refresh_tag + """    <title>""" + title_prefix + """Skill Description Optimization</title>
+    <link rel="preconnect" href="https://fonts.googleapis.com">
+    <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
+    <link href="https://fonts.googleapis.com/css2?family=Poppins:wght@500;600&family=Lora:wght@400;500&display=swap" rel="stylesheet">
+    <style>
+        body {
+            font-family: 'Lora', Georgia, serif;
+            max-width: 100%;
+            margin: 0 auto;
+            padding: 20px;
+            background: #faf9f5;
+            color: #141413;
+        }
+        h1 { font-family: 'Poppins', sans-serif; color: #141413; }
+        .explainer {
+            background: white;
+            padding: 15px;
+            border-radius: 6px;
+            margin-bottom: 20px;
+            border: 1px solid #e8e6dc;
+            color: #b0aea5;
+            font-size: 0.875rem;
+            line-height: 1.6;
+        }
+        .summary {
+            background: white;
+            padding: 15px;
+            border-radius: 6px;
+            margin-bottom: 20px;
+            border: 1px solid #e8e6dc;
+        }
+        .summary p { margin: 5px 0; }
+        .best { color: #788c5d; font-weight: bold; }
+        .table-container {
+            overflow-x: auto;
+            width: 100%;
+        }
+        table {
+            border-collapse: collapse;
+            background: white;
+            border: 1px solid #e8e6dc;
+            border-radius: 6px;
+            font-size: 12px;
+            min-width: 100%;
+        }
+        th, td {
+            padding: 8px;
+            text-align: left;
+            border: 1px solid #e8e6dc;
+            white-space: normal;
+            word-wrap: break-word;
+        }
+        th {
+            font-family: 'Poppins', sans-serif;
+            background: #141413;
+            color: #faf9f5;
+            font-weight: 500;
+        }
+        th.test-col {
+            background: #6a9bcc;
+        }
+        th.query-col { min-width: 200px; }
+        td.description {
+            font-family: monospace;
+            font-size: 11px;
+            word-wrap: break-word;
+            max-width: 400px;
+        }
+        td.result {
+            text-align: center;
+            font-size: 16px;
+            min-width: 40px;
+        }
+        td.test-result {
+            background: #f0f6fc;
+        }
+        .pass { color: #788c5d; }
+        .fail { color: #c44; }
+        .rate {
+            font-size: 9px;
+            color: #b0aea5;
+            display: block;
+        }
+        tr:hover { background: #faf9f5; }
+        .score {
+            display: inline-block;
+            padding: 2px 6px;
+            border-radius: 4px;
+            font-weight: bold;
+            font-size: 11px;
+        }
+        .score-good { background: #eef2e8; color: #788c5d; }
+        .score-ok { background: #fef3c7; color: #d97706; }
+        .score-bad { background: #fceaea; color: #c44; }
+        .train-label { color: #b0aea5; font-size: 10px; }
+        .test-label { color: #6a9bcc; font-size: 10px; font-weight: bold; }
+        .best-row { background: #f5f8f2; }
+        th.positive-col { border-bottom: 3px solid #788c5d; }
+        th.negative-col { border-bottom: 3px solid #c44; }
+        th.test-col.positive-col { border-bottom: 3px solid #788c5d; }
+        th.test-col.negative-col { border-bottom: 3px solid #c44; }
+        .legend { font-family: 'Poppins', sans-serif; display: flex; gap: 20px; margin-bottom: 10px; font-size: 13px; align-items: center; }
+        .legend-item { display: flex; align-items: center; gap: 6px; }
+        .legend-swatch { width: 16px; height: 16px; border-radius: 3px; display: inline-block; }
+        .swatch-positive { background: #141413; border-bottom: 3px solid #788c5d; }
+        .swatch-negative { background: #141413; border-bottom: 3px solid #c44; }
+        .swatch-test { background: #6a9bcc; }
+        .swatch-train { background: #141413; }
+    </style>
+</head>
+<body>
+    <h1>""" + title_prefix + """Skill Description Optimization</h1>
+    <div class="explainer">
+        <strong>Optimizing your skill's description.</strong> This page updates automatically as Claude tests different versions of your skill's description. Each row is an iteration — a new description attempt. The columns show test queries: green checkmarks mean the skill triggered correctly (or correctly didn't trigger), red crosses mean it got it wrong. The "Train" score shows performance on queries used to improve the description; the "Test" score shows performance on held-out queries the optimizer hasn't seen. When it's done, Claude will apply the best-performing description to your skill.
+    </div>
+"""]
+    # Summary section
+    best_test_score = data.get('best_test_score')
+    best_train_score = data.get('best_train_score')
+    html_parts.append(f"""
+    <div class="summary">
+        <p><strong>Original:</strong> {html.escape(data.get('original_description', 'N/A'))}</p>
+        <p class="best"><strong>Best:</strong> {html.escape(data.get('best_description', 'N/A'))}</p>
+        <p><strong>Best Score:</strong> {data.get('best_score', 'N/A')} {'(test)' if best_test_score else '(train)'}</p>
+        <p><strong>Iterations:</strong> {data.get('iterations_run', 0)} | <strong>Train:</strong> {data.get('train_size', '?')} | <strong>Test:</strong> {data.get('test_size', '?')}</p>
+    </div>
+""")
+    # Legend
+    html_parts.append("""
+    <div class="legend">
+        <span style="font-weight:600">Query columns:</span>
+        <span class="legend-item"><span class="legend-swatch swatch-positive"></span> Should trigger</span>
+        <span class="legend-item"><span class="legend-swatch swatch-negative"></span> Should NOT trigger</span>
+        <span class="legend-item"><span class="legend-swatch swatch-train"></span> Train</span>
+        <span class="legend-item"><span class="legend-swatch swatch-test"></span> Test</span>
+    </div>
+""")
+    # Table header
+    html_parts.append("""
+    <div class="table-container">
+    <table>
+        <thead>
+            <tr>
+                <th>Iter</th>
+                <th>Train</th>
+                <th>Test</th>
+                <th class="query-col">Description</th>
+""")
+    # Add column headers for train queries
+    for qinfo in train_queries:
+        polarity = "positive-col" if qinfo["should_trigger"] else "negative-col"
+        html_parts.append(f'                <th class="{polarity}">{html.escape(qinfo["query"])}</th>\n')
+    # Add column headers for test queries (different color)
+    for qinfo in test_queries:
+        polarity = "positive-col" if qinfo["should_trigger"] else "negative-col"
+        html_parts.append(f'                <th class="test-col {polarity}">{html.escape(qinfo["query"])}</th>\n')
+    html_parts.append("""            </tr>
+        </thead>
+        <tbody>
+""")
+    # Find best iteration for highlighting
+    if test_queries:
+        best_iter = max(history, key=lambda h: h.get("test_passed") or 0).get("iteration")
+    else:
+        best_iter = max(history, key=lambda h: h.get("train_passed", h.get("passed", 0))).get("iteration")
+    # Add rows for each iteration
+    for h in history:
+        iteration = h.get("iteration", "?")
+        description = h.get("description", "")
+        train_results = h.get("train_results", h.get("results", [])) or []
+        test_results = h.get("test_results") or []
+        # Create lookups for results by query
+        train_by_query = {r["query"]: r for r in train_results}
+        test_by_query = {r["query"]: r for r in test_results} if test_results else {}
+        # Compute aggregate correct/total runs across all retries
+        def aggregate_runs(results: list[dict] | None) -> tuple[int, int]:
+            if not results:
+                return 0, 0
+            correct = 0
+            total = 0
+            for r in results:
+                runs = r.get("runs", 0)
+                triggers = r.get("triggers", 0)
+                total += runs
+                if r.get("should_trigger", True):
+                    correct += triggers
+                else:
+                    correct += runs - triggers
+            return correct, total
+        train_correct, train_runs = aggregate_runs(train_results)
+        test_correct, test_runs = aggregate_runs(test_results)
+        # Determine score classes
+        def score_class(correct: int, total: int) -> str:
+            if total > 0:
+                ratio = correct / total
+                if ratio >= 0.8:
+                    return "score-good"
+                elif ratio >= 0.5:
+                    return "score-ok"
+            return "score-bad"
+        train_class = score_class(train_correct, train_runs)
+        test_class = score_class(test_correct, test_runs)
+        row_class = "best-row" if iteration == best_iter else ""
+        html_parts.append(f"""            <tr class="{row_class}">
+                <td>{iteration}</td>
+                <td><span class="score {train_class}">{train_correct}/{train_runs}</span></td>
+                <td><span class="score {test_class}">{test_correct}/{test_runs}</span></td>
+                <td class="description">{html.escape(description)}</td>
+""")
+        # Add result for each train query
+        for qinfo in train_queries:
+            r = train_by_query.get(qinfo["query"], {})
+            did_pass = r.get("pass", False)
+            triggers = r.get("triggers", 0)
+            runs = r.get("runs", 0)
+            icon = "✓" if did_pass else "✗"
+            css_class = "pass" if did_pass else "fail"
+            html_parts.append(f'                <td class="result {css_class}">{icon}<span class="rate">{triggers}/{runs}</span></td>\n')
+        # Add result for each test query (with different background)
+        for qinfo in test_queries:
+            r = test_by_query.get(qinfo["query"], {})
+            did_pass = r.get("pass", False)
+            triggers = r.get("triggers", 0)
+            runs = r.get("runs", 0)
+            icon = "✓" if did_pass else "✗"
+            css_class = "pass" if did_pass else "fail"
+            html_parts.append(f'                <td class="result test-result {css_class}">{icon}<span class="rate">{triggers}/{runs}</span></td>\n')
+        html_parts.append("            </tr>\n")
+    html_parts.append("""        </tbody>
+    </table>
+    </div>
+""")
+    html_parts.append("""
+</body>
+</html>
+""")
+    return "".join(html_parts)
+def main():
+    parser = argparse.ArgumentParser(description="Generate HTML report from run_loop output")
+    parser.add_argument("input", help="Path to JSON output from run_loop.py (or - for stdin)")
+    parser.add_argument("-o", "--output", default=None, help="Output HTML file (default: stdout)")
+    parser.add_argument("--skill-name", default="", help="Skill name to include in the report title")
+    args = parser.parse_args()
+    if args.input == "-":
+        data = json.load(sys.stdin)
+    else:
+        data = json.loads(Path(args.input).read_text())
+    html_output = generate_html(data, skill_name=args.skill_name)
+    if args.output:
+        Path(args.output).write_text(html_output)
+        print(f"Report written to {args.output}", file=sys.stderr)
+    else:
+        print(html_output)
+if __name__ == "__main__":
+    main()

package/skills/conversation-to-skill/scripts/improve_description.py ADDED Viewed

@@ -0,0 +1,197 @@
+#!/usr/bin/env python3
+"""Improve a skill description based on eval results."""
+from __future__ import annotations
+import argparse
+import json
+import sys
+from pathlib import Path
+if __package__ in (None, ""):
+    sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+from scripts.model_backends import detect_backend, extract_tagged_text, generate_text
+from scripts.utils import parse_skill_md
+def improve_description(
+    *,
+    backend: str,
+    skill_name: str,
+    skill_content: str,
+    current_description: str,
+    eval_results: dict,
+    history: list[dict],
+    model: str | None = None,
+    test_results: dict | None = None,
+    log_dir: Path | None = None,
+    iteration: int | None = None,
+) -> str:
+    """Call the selected backend to improve the description."""
+    backend = detect_backend(backend)
+    failed_triggers = [
+        item for item in eval_results["results"]
+        if item["should_trigger"] and not item["pass"]
+    ]
+    false_triggers = [
+        item for item in eval_results["results"]
+        if not item["should_trigger"] and not item["pass"]
+    ]
+    train_score = f"{eval_results['summary']['passed']}/{eval_results['summary']['total']}"
+    if test_results:
+        test_score = f"{test_results['summary']['passed']}/{test_results['summary']['total']}"
+        scores_summary = f"Train: {train_score}, Test: {test_score}"
+    else:
+        scores_summary = f"Train: {train_score}"
+    prompt = f"""You are optimizing a reusable AI skill description.
+The skill system uses progressive disclosure:
+- The model first sees only the skill name and description.
+- If it chooses the skill, it reads the full SKILL.md and bundled resources.
+Your job is to improve only the description so that the skill triggers for the right requests and stays quiet for adjacent or simpler ones.
+Skill name: {skill_name}
+Current description:
+<current_description>
+{current_description}
+</current_description>
+Current scores: {scores_summary}
+"""
+    if failed_triggers:
+        prompt += "FAILED TO TRIGGER:\n"
+        for item in failed_triggers:
+            prompt += f'  - "{item["query"]}" (triggered {item["triggers"]}/{item["runs"]} times)\n'
+        prompt += "\n"
+    if false_triggers:
+        prompt += "FALSE TRIGGERS:\n"
+        for item in false_triggers:
+            prompt += f'  - "{item["query"]}" (triggered {item["triggers"]}/{item["runs"]} times)\n'
+        prompt += "\n"
+    if history:
+        prompt += "PREVIOUS ATTEMPTS. Avoid repeating the same structure:\n\n"
+        for attempt in history:
+            train_s = f"{attempt.get('train_passed', attempt.get('passed', 0))}/{attempt.get('train_total', attempt.get('total', 0))}"
+            test_s = f"{attempt.get('test_passed', '?')}/{attempt.get('test_total', '?')}" if attempt.get('test_passed') is not None else None
+            score_str = f"train={train_s}" + (f", test={test_s}" if test_s else "")
+            prompt += f"<attempt {score_str}>\n"
+            prompt += f'Description: "{attempt["description"]}"\n'
+            if "results" in attempt:
+                for result in attempt["results"]:
+                    status = "PASS" if result["pass"] else "FAIL"
+                    prompt += f'  [{status}] "{result["query"][:80]}" (triggered {result["triggers"]}/{result["runs"]})\n'
+            prompt += "</attempt>\n\n"
+    prompt += f"""Skill content for context:
+<skill_content>
+{skill_content}
+</skill_content>
+Write a new description that generalizes from the failures instead of listing lots of exact prompts.
+Constraints:
+- 100 to 200 words preferred
+- Phrase it in terms of user intent, not internal implementation
+- Make it distinctive enough to win against nearby skills
+- Keep it portable across agents like Codex, Claude Code, and other tool-using assistants
+Return only the description wrapped in <new_description> tags.
+"""
+    response_text = generate_text(prompt, backend=backend, model=model, timeout=300)
+    description = extract_tagged_text(response_text, "new_description")
+    transcript = {
+        "iteration": iteration,
+        "backend": backend,
+        "prompt": prompt,
+        "response": response_text,
+        "final_description": description,
+        "char_count": len(description),
+    }
+    if len(description) > 1024:
+        shorten_prompt = (
+            f"The description is {len(description)} characters. Rewrite it below 1024 characters without "
+            "losing the key trigger intent. Return only <new_description>...</new_description>."
+        )
+        shortened_response = generate_text(
+            "\n\n".join([prompt, response_text, shorten_prompt]),
+            backend=backend,
+            model=model,
+            timeout=300,
+        )
+        description = extract_tagged_text(shortened_response, "new_description")
+        transcript["rewrite_prompt"] = shorten_prompt
+        transcript["rewrite_response"] = shortened_response
+        transcript["rewrite_char_count"] = len(description)
+        transcript["final_description"] = description
+    if log_dir:
+        log_dir.mkdir(parents=True, exist_ok=True)
+        log_file = log_dir / f"improve_iter_{iteration or 'unknown'}.json"
+        log_file.write_text(json.dumps(transcript, indent=2))
+    return description
+def main():
+    parser = argparse.ArgumentParser(description="Improve a skill description based on eval results")
+    parser.add_argument("--eval-results", required=True, help="Path to eval results JSON")
+    parser.add_argument("--skill-path", required=True, help="Path to skill directory")
+    parser.add_argument("--history", default=None, help="Path to history JSON")
+    parser.add_argument("--model", default=None, help="Optional backend model identifier")
+    parser.add_argument("--backend", default="auto", choices=["auto", "claude", "codex"], help="Generation backend")
+    parser.add_argument("--verbose", action="store_true", help="Print progress to stderr")
+    args = parser.parse_args()
+    skill_path = Path(args.skill_path)
+    if not (skill_path / "SKILL.md").exists():
+        print(f"Error: No SKILL.md found at {skill_path}", file=sys.stderr)
+        sys.exit(1)
+    eval_results = json.loads(Path(args.eval_results).read_text())
+    history = json.loads(Path(args.history).read_text()) if args.history else []
+    name, _, content = parse_skill_md(skill_path)
+    current_description = eval_results["description"]
+    if args.verbose:
+        print(f"Current: {current_description}", file=sys.stderr)
+        print(f"Score: {eval_results['summary']['passed']}/{eval_results['summary']['total']}", file=sys.stderr)
+    new_description = improve_description(
+        backend=args.backend,
+        skill_name=name,
+        skill_content=content,
+        current_description=current_description,
+        eval_results=eval_results,
+        history=history,
+        model=args.model,
+    )
+    if args.verbose:
+        print(f"Improved: {new_description}", file=sys.stderr)
+    output = {
+        "description": new_description,
+        "history": history + [{
+            "description": current_description,
+            "passed": eval_results["summary"]["passed"],
+            "failed": eval_results["summary"]["failed"],
+            "total": eval_results["summary"]["total"],
+            "results": eval_results["results"],
+        }],
+    }
+    print(json.dumps(output, indent=2))
+if __name__ == "__main__":
+    main()

package/skills/conversation-to-skill/scripts/model_backends.py ADDED Viewed

@@ -0,0 +1,115 @@
+#!/usr/bin/env python3
+"""Helpers for running host-specific model backends.
+This skill can be used from several agent environments. Some environments expose
+real skill triggering, others only expose a model CLI. This module centralizes
+those differences so the higher-level workflow can stay mostly host-neutral.
+"""
+from __future__ import annotations
+import json
+import os
+import re
+import shutil
+import subprocess
+from pathlib import Path
+def command_exists(name: str) -> bool:
+    return shutil.which(name) is not None
+def detect_backend(preferred: str | None = None) -> str:
+    if preferred and preferred != "auto":
+        return preferred
+    if command_exists("claude"):
+        return "claude"
+    if command_exists("codex"):
+        return "codex"
+    raise RuntimeError("No supported backend found. Install `claude` or `codex`, or pass --backend explicitly.")
+def _run(
+    cmd: list[str],
+    *,
+    cwd: Path | None = None,
+    timeout: int = 300,
+    strip_claudecode: bool = False,
+    retries: int = 0,
+    retry_timeout_scale: float = 1.5,
+) -> str:
+    env = dict(os.environ)
+    if strip_claudecode:
+        env.pop("CLAUDECODE", None)
+    attempt = 0
+    current_timeout = timeout
+    while True:
+        try:
+            completed = subprocess.run(
+                cmd,
+                cwd=str(cwd) if cwd else None,
+                env=env,
+                capture_output=True,
+                text=True,
+                timeout=current_timeout,
+                check=True,
+            )
+            return completed.stdout.strip()
+        except subprocess.TimeoutExpired:
+            if attempt >= retries:
+                raise
+            attempt += 1
+            current_timeout = max(current_timeout + 1, int(current_timeout * retry_timeout_scale))
+def generate_text(
+    prompt: str,
+    *,
+    backend: str,
+    model: str | None = None,
+    cwd: Path | None = None,
+    timeout: int = 300,
+) -> str:
+    backend = detect_backend(backend)
+    if backend == "claude":
+        cmd = ["claude", "-p", prompt]
+        if model:
+            cmd.extend(["--model", model])
+        return _run(cmd, cwd=cwd, timeout=timeout, strip_claudecode=True)
+    if backend == "codex":
+        cmd = ["codex", "exec", "-s", "read-only"]
+        if model:
+            cmd.extend(["-m", model])
+        if cwd:
+            cmd.extend(["-C", str(cwd)])
+            if not any((parent / ".git").exists() for parent in [cwd, *cwd.parents]):
+                cmd.append("--skip-git-repo-check")
+        cmd.append(prompt)
+        # Codex routing judgments can occasionally run close to the timeout.
+        return _run(cmd, cwd=cwd, timeout=timeout, retries=1)
+    raise RuntimeError(f"Unsupported backend: {backend}")
+def extract_tagged_text(text: str, tag: str) -> str:
+    match = re.search(rf"<{tag}>(.*?)</{tag}>", text, re.DOTALL)
+    if match:
+        return match.group(1).strip().strip('"')
+    return text.strip().strip('"')
+def extract_first_json_object(text: str) -> dict:
+    decoder = json.JSONDecoder()
+    for index, char in enumerate(text):
+        if char != "{":
+            continue
+        try:
+            obj, _ = decoder.raw_decode(text[index:])
+            if isinstance(obj, dict):
+                return obj
+        except json.JSONDecodeError:
+            continue
+    raise ValueError(f"No JSON object found in backend output: {text[:200]}")