@openhands/extensions 0.1.0 → 0.3.0

package/tests/test_skills_have_readme.py

@@ -0,0 +1,66 @@
+"""Test that all skills have a README.md file in their directory."""
+
+import os
+from pathlib import Path
+
+
+def get_skills_directory():
+    """Get the path to the skills directory."""
+    # Get the directory containing this test file
+    test_dir = Path(__file__).parent
+    # Go up one level to the repo root, then into skills/
+    return test_dir.parent / "skills"
+
+
+def test_all_skills_have_readme():
+    """Verify that every skill directory contains a README.md file."""
+    skills_dir = get_skills_directory()
+    
+    # Get all subdirectories in the skills directory (excluding files)
+    skill_dirs = [
+        d for d in skills_dir.iterdir()
+        if d.is_dir() and not d.name.startswith('.')
+    ]
+    
+    assert len(skill_dirs) > 0, "No skill directories found"
+    
+    missing_readmes = []
+    for skill_dir in skill_dirs:
+        readme_path = skill_dir / "README.md"
+        if not readme_path.exists():
+            missing_readmes.append(skill_dir.name)
+    
+    assert len(missing_readmes) == 0, (
+        f"The following skills are missing README.md: {', '.join(missing_readmes)}"
+    )
+
+
+def test_readme_is_readable():
+    """Verify that README.md files are readable (symlinks resolve correctly)."""
+    skills_dir = get_skills_directory()
+    
+    skill_dirs = [
+        d for d in skills_dir.iterdir()
+        if d.is_dir() and not d.name.startswith('.')
+    ]
+    
+    unreadable_readmes = []
+    for skill_dir in skill_dirs:
+        readme_path = skill_dir / "README.md"
+        if readme_path.exists():
+            try:
+                content = readme_path.read_text()
+                if len(content) == 0:
+                    unreadable_readmes.append(f"{skill_dir.name} (empty)")
+            except Exception as e:
+                unreadable_readmes.append(f"{skill_dir.name} ({e})")
+    
+    assert len(unreadable_readmes) == 0, (
+        f"The following README.md files are unreadable: {', '.join(unreadable_readmes)}"
+    )
+
+
+if __name__ == "__main__":
+    test_all_skills_have_readme()
+    test_readme_is_readable()
+    print("All tests passed!")

package/tests/test_sync_extensions.py

@@ -0,0 +1,292 @@
+"""Tests for scripts/sync_extensions.py core functions."""
+
+import sys
+from pathlib import Path
+
+import pytest
+
+sys.path.insert(0, str(Path(__file__).parent.parent / "scripts"))
+from sync_extensions import (
+    CMD_MARKER,
+    CommandSpec,
+    REPO_ROOT,
+    _entry_type,
+    build_command_content,
+    collect_needed_commands,
+    generate_catalog,
+    load_marketplaces,
+    parse_frontmatter,
+    slash_triggers,
+    sync_commands,
+)
+
+
+# ── parse_frontmatter ────────────────────────────────────────────────
+
+class TestParseFrontmatter:
+    def test_basic(self):
+        text = (
+            "---\n"
+            "name: test-skill\n"
+            "description: A test skill\n"
+            "triggers:\n"
+            "  - /test\n"
+            "---\n"
+            "Body"
+        )
+        meta = parse_frontmatter(text)
+        assert meta["name"] == "test-skill"
+        assert meta["description"] == "A test skill"
+        assert meta["triggers"] == ["/test"]
+
+    def test_no_frontmatter(self):
+        assert parse_frontmatter("No frontmatter here") == {}
+
+    def test_empty_frontmatter(self):
+        meta = parse_frontmatter("---\n---\nBody")
+        assert meta.get("triggers", []) == []
+        assert "name" not in meta
+
+    def test_multiple_triggers(self):
+        text = (
+            "---\n"
+            "name: multi\n"
+            "triggers:\n"
+            "  - /foo\n"
+            "  - /bar\n"
+            "  - baz\n"
+            "---\n"
+        )
+        meta = parse_frontmatter(text)
+        assert meta["triggers"] == ["/foo", "/bar", "baz"]
+
+    def test_no_triggers_key(self):
+        text = "---\nname: no-triggers\ndescription: desc\n---\n"
+        meta = parse_frontmatter(text)
+        assert meta["triggers"] == []
+
+    def test_triggers_with_comments(self):
+        text = (
+            "---\n"
+            "triggers:\n"
+            "  - /cmd\n"
+            "  # this is a comment\n"
+            "  - /other\n"
+            "---\n"
+        )
+        meta = parse_frontmatter(text)
+        assert "/cmd" in meta["triggers"]
+        assert "/other" in meta["triggers"]
+
+    def test_folded_block_scalar_description(self):
+        text = (
+            "---\n"
+            "name: test\n"
+            "description: >\n"
+            "  This is a long\n"
+            "  description text\n"
+            "other: value\n"
+            "---\n"
+        )
+        meta = parse_frontmatter(text)
+        assert meta["description"] == "This is a long description text"
+
+    def test_unicode(self):
+        text = (
+            "---\n"
+            "name: unic\u00f6de-skill\n"
+            "description: \u00dcn\u00efc\u00f6d\u00e9 description \U0001f680\n"
+            "---\n"
+        )
+        meta = parse_frontmatter(text)
+        assert meta["name"] == "unic\u00f6de-skill"
+        assert "\U0001f680" in meta["description"]
+
+
+class TestParseFrontmatterEdgeCases:
+    """Edge-case tests ensuring graceful degradation."""
+
+    def test_malformed_yaml_missing_close(self):
+        assert parse_frontmatter("---\nname: incomplete\n") == {}
+
+    def test_invalid_yaml_returns_empty(self):
+        assert parse_frontmatter("---\n: [invalid yaml{{\n---\n") == {}
+
+    def test_colon_in_description(self):
+        text = '---\nname: test\ndescription: "key: value pair"\n---\n'
+        meta = parse_frontmatter(text)
+        assert meta["description"] == "key: value pair"
+
+    def test_quoted_values_stripped(self):
+        text = "---\nname: \"quoted-name\"\ndescription: 'single-quoted'\n---\n"
+        meta = parse_frontmatter(text)
+        assert meta["name"] == "quoted-name"
+        assert meta["description"] == "single-quoted"
+
+    def test_triggers_end_at_next_key(self):
+        text = (
+            "---\n"
+            "triggers:\n"
+            "  - /a\n"
+            "  - /b\n"
+            "name: after-triggers\n"
+            "---\n"
+        )
+        meta = parse_frontmatter(text)
+        assert meta["triggers"] == ["/a", "/b"]
+        assert meta["name"] == "after-triggers"
+
+    def test_null_trigger_skipped(self):
+        text = (
+            "---\n"
+            "triggers:\n"
+            "  - /cmd\n"
+            "  -\n"
+            "  - /other\n"
+            "---\n"
+        )
+        meta = parse_frontmatter(text)
+        assert "/cmd" in meta["triggers"]
+        assert "/other" in meta["triggers"]
+
+    def test_frontmatter_only_block(self):
+        meta = parse_frontmatter("---\nname: only\n---")
+        assert meta["name"] == "only"
+
+
+# ── slash_triggers ────────────────────────────────────────────────────
+
+class TestSlashTriggers:
+    def test_filters_slash_only(self):
+        meta = {"triggers": ["/init", "keyword", "/build"]}
+        assert slash_triggers(meta) == ["/init", "/build"]
+
+    def test_empty(self):
+        assert slash_triggers({}) == []
+        assert slash_triggers({"triggers": []}) == []
+
+
+# ── _entry_type ───────────────────────────────────────────────────────
+
+class TestEntryType:
+    def test_skills_path(self):
+        assert _entry_type("./skills/foo") == "skill"
+
+    def test_plugins_path(self):
+        assert _entry_type("./plugins/bar") == "plugin"
+
+    def test_relative_plugins(self):
+        assert _entry_type("../plugins/baz") == "plugin"
+
+    def test_relative_skills(self):
+        assert _entry_type("../skills/qux") == "skill"
+
+    def test_fallback_is_skill(self):
+        assert _entry_type("./nonexistent-xyz") == "skill"
+
+
+# ── collect_needed_commands ───────────────────────────────────────────
+
+class TestCollectNeededCommands:
+    def test_returns_command_specs(self):
+        specs = collect_needed_commands()
+        assert all(isinstance(s, CommandSpec) for s in specs)
+        assert len(specs) > 0
+
+    def test_paths_are_under_commands_dir(self):
+        for spec in collect_needed_commands():
+            assert spec.path.parent.name == "commands"
+            assert spec.path.suffix == ".md"
+
+
+# ── load_marketplaces ────────────────────────────────────────────────
+
+class TestLoadMarketplaces:
+    def test_loads_at_least_one(self):
+        mps = load_marketplaces()
+        assert len(mps) > 0
+
+    def test_marketplaces_have_required_fields(self):
+        for mp in load_marketplaces():
+            assert "plugins" in mp
+            assert "_file" in mp
+
+
+# ── generate_catalog ─────────────────────────────────────────────────
+
+class TestGenerateCatalog:
+    def test_catalog_contains_marketplace_names(self):
+        catalog = generate_catalog()
+        assert "openhands-extensions" in catalog
+
+    def test_catalog_has_table_header(self):
+        catalog = generate_catalog()
+        assert "| Name | Type | Description | Commands |" in catalog
+
+    def test_catalog_counts_are_consistent(self):
+        catalog = generate_catalog()
+        assert "marketplace(s)" in catalog
+        assert "extensions" in catalog
+
+    def test_catalog_only_has_skill_or_plugin_types(self):
+        """Every table row's Type column must be 'skill' or 'plugin'."""
+        catalog = generate_catalog()
+        types_found: list[str] = []
+        for line in catalog.splitlines():
+            # Skip non-table lines, header, and separator
+            if not line.startswith("|") or line.startswith("|---") or "Type" in line:
+                continue
+            cols = [c.strip() for c in line.split("|")]
+            # cols[0] is empty (before first |), cols[1]=Name, cols[2]=Type
+            if len(cols) >= 3:
+                types_found.append(cols[2])
+        assert len(types_found) > 0, "Expected at least one table entry"
+        invalid = [t for t in types_found if t not in ("skill", "plugin")]
+        assert not invalid, f"Found invalid types in catalog: {invalid}"
+
+
+# ── sync_commands ────────────────────────────────────────────────────
+
+class TestSyncCommands:
+    def test_check_mode_reports_no_problems_when_in_sync(self):
+        problems = sync_commands(check=True)
+        assert problems == [], f"Unexpected problems: {problems}"
+
+    def test_manually_edited_file_detected_in_check_mode(self, tmp_path, monkeypatch):
+        """A command file without the header should be flagged in --check mode."""
+        # Create a skill with a slash trigger
+        skill_dir = tmp_path / "skills" / "test-skill"
+        skill_dir.mkdir(parents=True)
+        skill_md = skill_dir / "SKILL.md"
+        skill_md.write_text(
+            "---\nname: test-skill\ndescription: Test\ntriggers:\n  - /test-cmd\n---\nBody\n"
+        )
+        # Create a manually-edited command file (no header)
+        cmd_dir = skill_dir / "commands"
+        cmd_dir.mkdir()
+        (cmd_dir / "test-cmd.md").write_text("Custom content, no header\n")
+
+        monkeypatch.setattr("sync_extensions.REPO_ROOT", tmp_path)
+        monkeypatch.setattr("sync_extensions.SKILL_DIRS", [tmp_path / "skills"])
+        monkeypatch.setattr("sync_extensions._slash_cmd_cache", {})
+
+        problems = sync_commands(check=True)
+        assert any("manually-edited" in p for p in problems)
+
+
+# ── marketplace source paths ─────────────────────────────────────────
+
+class TestMarketplaceSourcePaths:
+    def test_all_source_paths_exist(self):
+        """Every source path in every marketplace should resolve on disk."""
+        import json
+
+        for mp in load_marketplaces():
+            mp_file = mp["_file"]
+            for plugin in mp["plugins"]:
+                src = plugin.get("source", "")
+                resolved = REPO_ROOT / src.lstrip("./")
+                assert resolved.exists(), (
+                    f"{mp_file.name}: {plugin.get('name', '?')} source "
+                    f"'{src}' does not exist at {resolved}"
+                )

package/tests/test_workflow_sync.py

@@ -0,0 +1,46 @@
+#!/usr/bin/env python3
+"""Test that workflow files in .github/workflows/ match copies in plugins/*/workflows/."""
+
+from pathlib import Path
+
+
+def test_workflow_files_are_in_sync():
+    """Ensure workflow files in .github/workflows/ are identical to copies in plugin dirs."""
+    repo_root = Path(__file__).parent.parent
+    workflows_dir = repo_root / ".github" / "workflows"
+
+    if not workflows_dir.exists():
+        return  # No workflows directory
+
+    mismatches = []
+    # Check all plugins/*/workflows/ directories
+    for copy_path in repo_root.glob("plugins/*/workflows"):
+        if not copy_path.is_dir():
+            continue
+
+        for copy_file in copy_path.glob("*.yml"):
+            canonical_file = workflows_dir / copy_file.name
+            if not canonical_file.exists():
+                continue
+
+            canonical_content = canonical_file.read_text()
+            copy_content = copy_file.read_text()
+
+            if canonical_content != copy_content:
+                rel_path = copy_path.relative_to(repo_root)
+                mismatches.append(
+                    f"{rel_path}/{copy_file.name} differs from "
+                    f".github/workflows/{copy_file.name}"
+                )
+
+    if mismatches:
+        raise AssertionError(
+            "Workflow files are out of sync:\n"
+            + "\n".join(f"  - {m}" for m in mismatches)
+            + "\n\nRun: cp .github/workflows/<file> <destination> to sync"
+        )
+
+
+if __name__ == "__main__":
+    test_workflow_files_are_in_sync()
+    print("All workflow files are in sync!")

package/utils/analysis/README.md

@@ -0,0 +1,7 @@
+# Signal Analysis Utilities
+
+This directory contains utilities for analyzing signals collected from Laminar to understand AI agent behavior patterns.
+
+## Available Utilities
+
+- **[laminar_signals/](laminar_signals/)** - Analyze Laminar signal events using LLMs with customizable Jinja2 prompt templates

package/utils/analysis/laminar_signals/README.md

@@ -0,0 +1,211 @@
+# Laminar Signal Analysis
+
+Analyze Laminar signal events using LLMs to identify issues and areas for improvement. Uses function calling to get structured output with clickable trace URLs.
+
+**Primary Focus**: Identifying problems and generating actionable recommendations to improve AI agent behavior.
+
+## Directory Structure
+
+```
+laminar_signals/
+├── analyze.py          # Main analysis script
+├── README.md           # This file
+└── templates/
+    ├── default.j2      # Generic template for any signal type
+    └── pr_review.j2    # Specialized template for PR review signals
+```
+
+## Prerequisites
+
+- Python 3.9+
+- `jinja2` package
+
+```bash
+pip install jinja2
+```
+
+## Environment Variables
+
+| Variable | Required | Default | Description |
+|----------|----------|---------|-------------|
+| `LMNR_PROJECT_API_KEY` | Yes | - | Laminar project API key |
+| `LMNR_PROJECT_ID` | No | - | Laminar project ID (for generating clickable trace URLs) |
+| `LLM_API_KEY` | Yes | - | API key for the LLM |
+| `LLM_MODEL` | No | `gemini-3-pro-preview` | Model to use |
+| `LLM_BASE_URL` | No | `https://llm-proxy.app.all-hands.dev` | Base URL for LLM API |
+
+**Note:** To enable clickable trace URLs in the output, set `LMNR_PROJECT_ID` to your Laminar project ID. 
+You can find this in your Laminar dashboard URL: `https://laminar.sh/project/{PROJECT_ID}/...`
+
+## Usage
+
+```bash
+# List available signals
+python analyze.py --list-signals
+
+# Analyze PR review signals (outputs markdown by default)
+python analyze.py --signal "pr review suggestion and analysis"
+
+# Analyze with skill context for grounded recommendations
+python analyze.py --signal "pr review suggestion and analysis" \
+    --skill-dir ../../../plugins/pr-review
+
+# Output as JSON (structured data with trace URLs)
+python analyze.py --signal "pr review suggestion and analysis" --format json
+
+# Analyze with custom prompt template
+python analyze.py --signal "my-signal" --prompt-file my_prompt.j2
+
+# Analyze last 30 days and save to file
+python analyze.py --signal "my-signal" --days 30 --output results.md
+```
+
+## How It Works
+
+1. **Fetches Signals**: Queries the Laminar SQL API to retrieve signal events with trace IDs.
+
+2. **Generates Prompt**: Uses a Jinja2 template focused on identifying issues and improvements.
+
+3. **Loads Skill Context** (optional): If `--skill-dir` is provided, loads the skill/plugin content so recommendations can reference specific prompt verbiage.
+
+4. **LLM Analysis**: Uses **function calling** to get structured output with:
+   - `issues`: Problems and failures with severity, frequency, and clickable trace URLs
+   - `recommendations`: Specific fixes with priority levels (grounded in skill content when provided)
+   - `metrics`: Quantitative statistics (issue rate, etc.)
+   - `strengths`: Brief note on what's working (secondary focus)
+
+5. **Output**: Markdown report (default) or JSON for programmatic use.
+
+## Grounding Recommendations in Skills
+
+Use `--skill-dir` to provide the current skill/plugin configuration. This allows the LLM to:
+- Reference specific sections of the prompt that need changes
+- Suggest exact wording modifications
+- Ground recommendations in the actual instructions the agent follows
+
+```bash
+# Use a skill directory
+python analyze.py --signal "..." --skill-dir path/to/skills/code-review
+
+# Use a plugin directory (loads nested skills and prompt.py)
+python analyze.py --signal "..." --skill-dir path/to/plugins/pr-review
+```
+
+The script automatically loads:
+- `SKILL.md` from the directory
+- `scripts/prompt.py` if present (for plugins)
+- Any nested skills in `skills/` subdirectory
+
+## Output Structure
+
+### Issues (Primary Focus)
+
+Each issue includes:
+- **Title**: Short description
+- **Description**: Detailed explanation of the problem
+- **Severity**: critical / high / medium / low
+- **Frequency**: How often this occurs
+- **Trace URLs**: Clickable links to example traces in Laminar
+
+### Recommendations
+
+Each recommendation includes:
+- **Title**: Short description
+- **Description**: How to implement the fix
+- **Addresses**: Which issues this fixes
+- **Priority**: high / medium / low
+
+### Example JSON Output
+
+```json
+{
+  "issues": [
+    {
+      "title": "Premature Approval of Functional Defects",
+      "description": "The AI approves code with known issues...",
+      "severity": "critical",
+      "frequency": "10-15% of traces",
+      "trace_urls": [
+        "https://www.lmnr.ai/traces/a0dbd0c7-116d-776b-9c4b-c7f5f4d0a97e",
+        "https://www.lmnr.ai/traces/5de277ff-38f2-e173-4b03-c7332301ea58"
+      ]
+    }
+  ],
+  "recommendations": [
+    {
+      "title": "Enforce Logic Verification Over Style",
+      "description": "Modify the prompt to separate style from logic analysis...",
+      "addresses": ["Premature Approval of Functional Defects"],
+      "priority": "high"
+    }
+  ],
+  "metrics": {
+    "total_signals": 50,
+    "issue_rate": "30%",
+    "key_statistics": [...]
+  },
+  "strengths": [
+    {"title": "Security Analysis", "description": "..."}
+  ],
+  "summary": "Executive summary of critical improvements needed..."
+}
+```
+
+### Example Markdown Output
+
+```markdown
+# Agent Improvement Report
+
+**Signal:** `pr review suggestion and analysis`
+
+## Executive Summary
+
+The AI agent demonstrates strong security awareness but suffers from...
+
+## Issues Requiring Attention
+
+### 1. [CRITICAL] Premature Approval of Functional Defects
+
+The AI approves code with known issues...
+
+**Frequency:** 10-15% of traces
+
+**Example traces:**
+- https://www.lmnr.ai/traces/a0dbd0c7-116d-776b-9c4b-c7f5f4d0a97e
+- https://www.lmnr.ai/traces/5de277ff-38f2-e173-4b03-c7332301ea58
+
+## Recommended Fixes
+
+### 1. [HIGH PRIORITY] Enforce Logic Verification Over Style
+
+Modify the prompt to separate style from logic analysis...
+
+*Fixes: Premature Approval of Functional Defects*
+```
+
+## Custom Prompt Templates
+
+Create a Jinja2 template (`.j2`) with access to:
+
+| Variable | Type | Description |
+|----------|------|-------------|
+| `signals` | `list[dict]` | List of parsed signal objects |
+| `num_signals` | `int` | Number of signals |
+| `signal_name` | `str` | Name of the signal |
+
+Each signal object contains:
+- `trace_url`: Full URL to view the trace in Laminar
+- `trace_id`: Trace UUID
+- `timestamp`: When the signal was created
+- `payload`: Parsed payload as dict
+- `payload_json`: Formatted JSON string
+- All payload fields flattened to top level
+
+## Laminar SQL API Reference
+
+The script uses Laminar's SQL API at `/v1/sql/query`. Key tables:
+
+- `signal_events`: Signal events with trace_id and payload
+- `traces`: Trace-level aggregates
+
+See [Laminar SQL Editor documentation](https://docs.laminar.sh/platform/sql-editor) for details.