myaidev-method 0.3.3 → 0.3.5

package/skills/myai-skill-builder/scripts/run_eval.py

@@ -0,0 +1,105 @@
+#!/usr/bin/env python3
+"""Run a single description trigger evaluation.
+
+Tests whether a given prompt would trigger a skill based on its description.
+Used by run_loop.py for description optimization.
+
+Usage:
+    python3 run_eval.py --description "Skill description text" --query "User prompt to test"
+
+Output:
+    JSON with trigger prediction and reasoning.
+"""
+
+import argparse
+import json
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+
+
+def evaluate_trigger(description, query):
+    """Evaluate whether a skill description would trigger for a given query.
+
+    Uses a simple heuristic: check if the key concepts in the description
+    match the query's intent. For more accurate results, use claude -p.
+    """
+    # Normalize for comparison
+    desc_lower = description.lower()
+    query_lower = query.lower()
+
+    # Extract "when" clause keywords from description
+    when_idx = desc_lower.find("when")
+    if when_idx >= 0:
+        when_clause = desc_lower[when_idx:]
+    else:
+        when_clause = desc_lower
+
+    # Simple keyword overlap scoring
+    desc_words = set(when_clause.split())
+    query_words = set(query_lower.split())
+
+    # Remove common stop words
+    stop_words = {"a", "an", "the", "is", "are", "was", "were", "be", "been",
+                  "being", "have", "has", "had", "do", "does", "did", "will",
+                  "would", "could", "should", "may", "might", "can", "to",
+                  "of", "in", "for", "on", "with", "at", "by", "from", "or",
+                  "and", "not", "no", "but", "if", "this", "that", "it", "i",
+                  "you", "use", "when"}
+    desc_words -= stop_words
+    query_words -= stop_words
+
+    overlap = desc_words & query_words
+    score = len(overlap) / max(len(desc_words), 1)
+
+    return {
+        "would_trigger": score > 0.15,
+        "confidence": min(score * 2, 1.0),
+        "matching_keywords": list(overlap),
+        "score": score
+    }
+
+
+def evaluate_trigger_with_claude(description, query):
+    """Use claude CLI to evaluate trigger accuracy (more accurate but slower)."""
+    prompt = f"""Given this skill description:
+"{description}"
+
+Would this user message trigger the skill? Answer with ONLY "yes" or "no":
+"{query}"
+"""
+    try:
+        result = subprocess.run(
+            ["claude", "-p", prompt, "--output-format", "text"],
+            capture_output=True, text=True, timeout=30
+        )
+        answer = result.stdout.strip().lower()
+        return {
+            "would_trigger": answer.startswith("yes"),
+            "confidence": 0.9 if answer in ("yes", "no") else 0.5,
+            "raw_answer": answer
+        }
+    except (subprocess.TimeoutExpired, FileNotFoundError):
+        # Fall back to heuristic
+        return evaluate_trigger(description, query)
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Evaluate a single description trigger")
+    parser.add_argument("--description", required=True, help="Skill description text")
+    parser.add_argument("--query", required=True, help="User prompt to test")
+    parser.add_argument("--use-claude", action="store_true",
+                        help="Use claude CLI for evaluation (slower but more accurate)")
+    args = parser.parse_args()
+
+    if args.use_claude:
+        result = evaluate_trigger_with_claude(args.description, args.query)
+    else:
+        result = evaluate_trigger(args.description, args.query)
+
+    print(json.dumps(result, indent=2))
+
+
+if __name__ == "__main__":
+    main()

package/skills/myai-skill-builder/scripts/run_loop.py

@@ -0,0 +1,211 @@
+#!/usr/bin/env python3
+"""Description optimization loop with train/test split.
+
+Generates trigger eval queries, tests the current description,
+and suggests improvements based on accuracy metrics.
+
+Usage:
+    python3 run_loop.py --skill-dir .claude/skills/my-skill
+    python3 run_loop.py --skill-dir .claude/skills/my-skill --use-claude
+    python3 run_loop.py --queries queries.json --description "Skill desc"
+
+Input:
+    Either a --skill-dir (reads SKILL.md frontmatter) or explicit --description.
+    Optionally --queries pointing to a JSON file with pre-defined trigger queries.
+
+Output:
+    Prints accuracy metrics and saves results to description_eval.json in skill dir.
+"""
+
+import argparse
+import json
+import random
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parent))
+from utils import load_json, save_json, now_iso
+from run_eval import evaluate_trigger, evaluate_trigger_with_claude
+
+
+def parse_frontmatter(skill_path):
+    """Extract name and description from SKILL.md frontmatter."""
+    path = Path(skill_path) / "SKILL.md"
+    if not path.exists():
+        print(f"Error: {path} not found", file=sys.stderr)
+        sys.exit(1)
+
+    content = path.read_text()
+    if not content.startswith("---"):
+        print("Error: No frontmatter found in SKILL.md", file=sys.stderr)
+        sys.exit(1)
+
+    end = content.index("---", 3)
+    frontmatter = content[3:end]
+
+    name = ""
+    description = ""
+    for line in frontmatter.strip().split("\n"):
+        if line.startswith("name:"):
+            name = line.split(":", 1)[1].strip().strip('"').strip("'")
+        elif line.startswith("description:"):
+            description = line.split(":", 1)[1].strip().strip('"').strip("'")
+
+    return name, description
+
+
+def load_queries(queries_path):
+    """Load trigger queries from a JSON file."""
+    data = load_json(queries_path)
+    return data.get("should_trigger", []), data.get("should_not_trigger", [])
+
+
+def train_test_split(items, train_ratio=0.7):
+    """Split items into train and test sets."""
+    shuffled = items[:]
+    random.shuffle(shuffled)
+    split_idx = max(1, int(len(shuffled) * train_ratio))
+    return shuffled[:split_idx], shuffled[split_idx:]
+
+
+def evaluate_description(description, should_trigger, should_not_trigger, use_claude=False):
+    """Evaluate a description against trigger queries."""
+    eval_fn = evaluate_trigger_with_claude if use_claude else evaluate_trigger
+
+    results = {"true_positive": 0, "false_positive": 0,
+               "true_negative": 0, "false_negative": 0,
+               "details": []}
+
+    for query in should_trigger:
+        result = eval_fn(description, query)
+        triggered = result["would_trigger"]
+        if triggered:
+            results["true_positive"] += 1
+        else:
+            results["false_negative"] += 1
+        results["details"].append({
+            "query": query, "expected": True, "actual": triggered,
+            "correct": triggered, "confidence": result.get("confidence", 0)
+        })
+
+    for query in should_not_trigger:
+        result = eval_fn(description, query)
+        triggered = result["would_trigger"]
+        if not triggered:
+            results["true_negative"] += 1
+        else:
+            results["false_positive"] += 1
+        results["details"].append({
+            "query": query, "expected": False, "actual": triggered,
+            "correct": not triggered, "confidence": result.get("confidence", 0)
+        })
+
+    total = len(should_trigger) + len(should_not_trigger)
+    correct = results["true_positive"] + results["true_negative"]
+    results["accuracy"] = correct / total if total > 0 else 0
+    results["precision"] = (
+        results["true_positive"] / (results["true_positive"] + results["false_positive"])
+        if (results["true_positive"] + results["false_positive"]) > 0 else 0
+    )
+    results["recall"] = (
+        results["true_positive"] / (results["true_positive"] + results["false_negative"])
+        if (results["true_positive"] + results["false_negative"]) > 0 else 0
+    )
+
+    return results
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Description optimization loop")
+    parser.add_argument("--skill-dir", help="Path to skill directory")
+    parser.add_argument("--description", help="Explicit description to test")
+    parser.add_argument("--queries", help="Path to queries JSON file")
+    parser.add_argument("--use-claude", action="store_true",
+                        help="Use claude CLI for evaluation")
+    parser.add_argument("--train-only", action="store_true",
+                        help="Only evaluate against training set")
+    args = parser.parse_args()
+
+    # Get description
+    if args.description:
+        description = args.description
+        skill_name = "unknown"
+    elif args.skill_dir:
+        skill_name, description = parse_frontmatter(args.skill_dir)
+    else:
+        print("Error: Provide --skill-dir or --description", file=sys.stderr)
+        sys.exit(1)
+
+    if not description:
+        print("Error: No description found", file=sys.stderr)
+        sys.exit(1)
+
+    # Get queries
+    if args.queries:
+        should_trigger, should_not_trigger = load_queries(args.queries)
+    else:
+        print("Error: Provide --queries with trigger eval queries", file=sys.stderr)
+        sys.exit(1)
+
+    # Split into train/test
+    train_yes, test_yes = train_test_split(should_trigger)
+    train_no, test_no = train_test_split(should_not_trigger)
+
+    print(f"Skill: {skill_name}")
+    print(f"Description: {description[:80]}...")
+    print(f"Queries: {len(should_trigger)} should-trigger, {len(should_not_trigger)} should-not-trigger")
+    print(f"Train/Test split: {len(train_yes)+len(train_no)} / {len(test_yes)+len(test_no)}")
+    print()
+
+    # Evaluate on training set
+    print("=== Training Set ===")
+    train_results = evaluate_description(description, train_yes, train_no, args.use_claude)
+    print(f"Accuracy:  {train_results['accuracy']:.1%}")
+    print(f"Precision: {train_results['precision']:.1%}")
+    print(f"Recall:    {train_results['recall']:.1%}")
+
+    # Show misclassifications
+    misclassified = [d for d in train_results["details"] if not d["correct"]]
+    if misclassified:
+        print(f"\nMisclassified ({len(misclassified)}):")
+        for m in misclassified:
+            label = "FN" if m["expected"] else "FP"
+            print(f"  [{label}] {m['query']}")
+
+    # Evaluate on test set (unless train-only)
+    if not args.train_only and (test_yes or test_no):
+        print("\n=== Test Set ===")
+        test_results = evaluate_description(description, test_yes, test_no, args.use_claude)
+        print(f"Accuracy:  {test_results['accuracy']:.1%}")
+        print(f"Precision: {test_results['precision']:.1%}")
+        print(f"Recall:    {test_results['recall']:.1%}")
+    else:
+        test_results = None
+
+    # Save results
+    output = {
+        "timestamp": now_iso(),
+        "skill_name": skill_name,
+        "description": description,
+        "train_results": {
+            "accuracy": train_results["accuracy"],
+            "precision": train_results["precision"],
+            "recall": train_results["recall"],
+            "misclassified": misclassified
+        }
+    }
+    if test_results:
+        output["test_results"] = {
+            "accuracy": test_results["accuracy"],
+            "precision": test_results["precision"],
+            "recall": test_results["recall"],
+        }
+
+    if args.skill_dir:
+        save_json(Path(args.skill_dir) / "description_eval.json", output)
+    else:
+        print(json.dumps(output, indent=2))
+
+
+if __name__ == "__main__":
+    main()

package/skills/myai-skill-builder/scripts/utils.py

@@ -0,0 +1,123 @@
+#!/usr/bin/env python3
+"""Shared utilities for skill-builder scripts."""
+
+import json
+import os
+import sys
+from pathlib import Path
+from datetime import datetime, timezone
+
+
+def load_json(path):
+    """Load and parse a JSON file. Exit with error if not found or invalid."""
+    path = Path(path)
+    if not path.exists():
+        print(f"Error: {path} not found", file=sys.stderr)
+        sys.exit(1)
+    try:
+        with open(path, "r") as f:
+            return json.load(f)
+    except json.JSONDecodeError as e:
+        print(f"Error: Invalid JSON in {path}: {e}", file=sys.stderr)
+        sys.exit(1)
+
+
+def save_json(path, data, indent=2):
+    """Save data as formatted JSON."""
+    path = Path(path)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with open(path, "w") as f:
+        json.dump(data, f, indent=indent)
+    print(f"Saved: {path}")
+
+
+def find_workspace(skill_dir):
+    """Find or create the workspace directory for a skill."""
+    skill_dir = Path(skill_dir)
+    slug = skill_dir.name
+    workspace = skill_dir.parent.parent.parent / f"{slug}-workspace"
+    return workspace
+
+
+def find_latest_iteration(workspace):
+    """Find the highest iteration number in a workspace."""
+    workspace = Path(workspace)
+    if not workspace.exists():
+        return 0
+    iterations = [
+        int(d.name.split("-")[1])
+        for d in workspace.iterdir()
+        if d.is_dir() and d.name.startswith("iteration-")
+    ]
+    return max(iterations) if iterations else 0
+
+
+def next_iteration_dir(workspace):
+    """Get the path for the next iteration directory."""
+    workspace = Path(workspace)
+    n = find_latest_iteration(workspace) + 1
+    return workspace / f"iteration-{n}"
+
+
+def collect_grading_files(iteration_dir):
+    """Collect all grading.json files from an iteration directory."""
+    iteration_dir = Path(iteration_dir)
+    results = []
+    for eval_dir in sorted(iteration_dir.iterdir()):
+        if not eval_dir.is_dir() or not eval_dir.name.startswith("eval-"):
+            continue
+        for config in ["with_skill", "without_skill"]:
+            grading_path = eval_dir / config / "grading.json"
+            if grading_path.exists():
+                results.append(load_json(grading_path))
+    return results
+
+
+def collect_timing_files(iteration_dir):
+    """Collect all timing.json files from an iteration directory."""
+    iteration_dir = Path(iteration_dir)
+    results = []
+    for eval_dir in sorted(iteration_dir.iterdir()):
+        if not eval_dir.is_dir() or not eval_dir.name.startswith("eval-"):
+            continue
+        for config in ["with_skill", "without_skill"]:
+            timing_path = eval_dir / config / "timing.json"
+            if timing_path.exists():
+                results.append(load_json(timing_path))
+    return results
+
+
+def now_iso():
+    """Return current time as ISO 8601 string."""
+    return datetime.now(timezone.utc).isoformat()
+
+
+def mean(values):
+    """Calculate mean of a list of numbers."""
+    if not values:
+        return 0
+    return sum(values) / len(values)
+
+
+def stddev(values):
+    """Calculate population standard deviation."""
+    if len(values) < 2:
+        return 0
+    m = mean(values)
+    variance = sum((x - m) ** 2 for x in values) / len(values)
+    return variance ** 0.5
+
+
+def format_percent(value):
+    """Format a float as a percentage string."""
+    return f"{value * 100:.1f}%"
+
+
+def format_duration(ms):
+    """Format milliseconds as a human-readable duration."""
+    if ms < 1000:
+        return f"{ms}ms"
+    elif ms < 60000:
+        return f"{ms / 1000:.1f}s"
+    else:
+        return f"{ms / 60000:.1f}m"

package/skills/myai-visual-generator/SKILL.md

@@ -0,0 +1,125 @@
+---
+name: myai-visual-generator
+description: "Generates high-quality images, videos, infographics, and research visuals using AI APIs (Gemini, Imagen, DALL-E, GPT Image, FLUX, Veo) with fal.ai MCP model discovery. Use when creating hero images, blog post visuals, diagrams, infographics, architecture diagrams, flowcharts, sequence diagrams, data visualizations, or video content. Also use when the user mentions generating images, creating visuals, making diagrams, or needs any kind of visual content for articles, presentations, or projects."
+argument-hint: "[prompt] [--visual-type=image] [--type=hero] [--service=gemini]"
+allowed-tools: [Read, Write, Bash, Task, "mcp__fal-ai__*"]
+context: fork
+---
+
+# Visual Generator
+
+Generate images, videos, infographics, and research visuals using AI generation APIs, Mermaid rendering, HTML conversion, and PaperBanana.
+
+## Bundled Scripts
+
+The generation SDK is bundled at `${CLAUDE_SKILL_DIR}/scripts/visual-generation-utils.js`. It provides:
+- `validateAPIKeys()` — check which services are available
+- `generateImage(prompt, options)` — auto-routed image generation
+- `generateVideo(prompt, options)` — Veo 3 video generation
+- `estimateCost(service, options)` — cost estimation
+- `selectBestService(preferred)` — smart service selection with fallback
+- `buildInfographicPrompt(config)` — structured infographic prompt builder
+- `getServiceInfo(service)` — service metadata lookup
+
+Run it via Node.js:
+```bash
+node -e "import('${CLAUDE_SKILL_DIR}/scripts/visual-generation-utils.js').then(m => m.generateImage('prompt', {type: 'hero'}).then(r => console.log(r)))"
+```
+
+Or reference it from a script you write during generation.
+
+## Quick Start
+
+```
+/myai-visual-generator "Modern cloud-native architecture" --type=hero
+/myai-visual-generator "API request lifecycle" --visual-type=infographic --type=flowchart
+/myai-visual-generator "SaaS KPI dashboard" --visual-type=infographic --type=infographic-data
+/myai-visual-generator "Product demo" --visual-type=video
+```
+
+## Arguments
+
+Parse from: `$ARGUMENTS`
+
+| Parameter | Description | Default |
+|-----------|-------------|---------|
+| `prompt` | What the visual should depict | Required |
+| `--visual-type` | `image`, `video`, `infographic`, `research-visuals` | image |
+| `--type` | Sub-type (see routing below) | hero |
+| `--service` | Preferred generation service | auto |
+| `--quality` | low, medium, high | medium |
+| `--size` | 1024x1024, 1792x1024, 1024x1792 | 1024x1024 |
+
+## Visual Type Routing
+
+1. Ask **visual type** first: `image`, `video`, `infographic`, or `research-visuals`
+2. Then ask the relevant **sub-type** only:
+   - **Image**: `hero` or `illustration`
+   - **Video**: user intent (demo, tutorial, walkthrough)
+   - **Infographic**: `flowchart` or `sequence-diagram` (quick mode)
+   - **Research**: `research-diagram`, `research-plot`, or `research-evaluation`
+3. Collect only mandatory inputs; use defaults for everything else
+
+### Routing Table
+
+| Visual Type + Sub-type | Path |
+|----------------------|------|
+| Image (hero, illustration) | AI generation via bundled SDK |
+| Video | Veo 3 via bundled SDK |
+| Infographic (from article `*.md`) | Mermaid analyze → design → render → insert |
+| Infographic (standalone) | Mermaid/Gemini diagram path or HTML conversion |
+| Research visuals | PaperBanana CLI |
+| Text-heavy data visuals | HTML-to-screenshot (Playwright) |
+
+**Critical routing rule**: When the user needs pixel-perfect text/numbers (metrics, data, labels), use HTML conversion instead of AI generation. AI models can hallucinate text. See [services reference](references/services.md) for the full decision matrix.
+
+## Workflow
+
+1. **Check config**: Run `validateAPIKeys()` from the bundled SDK to see what's available
+2. **Gather requirements**: Visual type, sub-type, prompt, preferences (minimum questions)
+3. **Route**: Pick the generation path based on the routing table
+4. **Estimate cost**: Show estimated cost and check against budget before generating
+5. **Generate**: Execute via the appropriate path
+6. **Save**: Organize into `content-assets/images/YYYY-MM-DD/` or `content-assets/videos/YYYY-MM-DD/`
+7. **Return**: Provide markdown embed code and generation report
+
+## File Organization
+
+```
+content-assets/
+  ├── images/
+  │   └── YYYY-MM-DD/
+  │       └── {type}-{description}-{id}.png
+  └── videos/
+      └── YYYY-MM-DD/
+          └── video-{description}-{id}.mp4
+```
+
+## Additional Resources
+
+- **Service catalog, pricing, MCP tools**: See [references/services.md](references/services.md)
+- **Infographic/Mermaid pipeline details**: See [references/infographic-pipeline.md](references/infographic-pipeline.md)
+- **Research visuals (PaperBanana)**: See [references/research-visuals.md](references/research-visuals.md)
+- **Generation SDK source**: See [scripts/visual-generation-utils.js](scripts/visual-generation-utils.js) for the full API
+
+## Quality Guardrails
+
+- Verify all visible text in generated images is free from typos
+- For text-heavy visuals, prefer GPT Image 1.5 or HTML conversion
+- If generated text has mistakes, regenerate with clearer constraints or switch to HTML conversion
+- Validate rendered Mermaid outputs exist and are >1KB before inserting references
+
+## Error Handling
+
+- **No API keys**: Show which keys are missing and how to configure them (`/myai-configure visual`)
+- **Rate limiting**: Wait 60s and retry, or switch service
+- **Budget exceeded**: Show usage and suggest increasing limits in `.env`
+- **MCP unavailable**: Fall back to SDK-only approach automatically
+- **Mermaid render fails**: Save source `.mmd` files and provide manual render instructions
+
+## Integration
+
+- Works with `content-writer` skill via `--with-images` flag
+- Automatic asset organization and cost tracking
+- Markdown reference generation for easy embedding
+- fal.ai MCP model discovery when `FAL_KEY` is configured