@galaxyproject/orbit 0.0.0 → 0.9.0

package/extensions/orbit/vendor/skills/skills/galaxy-integration/jupyterlite/examples/vcp_variant_map.ipynb

@@ -0,0 +1,443 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "cell-0",
+   "metadata": {},
+   "source": [
+    "# V/C/P Gene Variant Geographic Map\n",
+    "\n",
+    "This notebook visualizes the mean allele frequencies of V/C/P gene variants across geographic locations.\n",
+    "\n",
+    "**V/C/P Gene**: The measles P gene encodes three proteins via overlapping reading frames:\n",
+    "- **Phosphoprotein (P)**: Essential for viral RNA synthesis\n",
+    "- **V protein**: Interferon antagonist\n",
+    "- **C protein**: Interferes with host innate immunity\n",
+    "\n",
+    "We create three maps:\n",
+    "1. US states\n",
+    "2. Canadian provinces\n",
+    "3. Romania (Bucharest)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "cell-1",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import gxy\n",
+    "import pandas as pd\n",
+    "import altair as alt\n",
+    "from vega_datasets import data as vega_data\n",
+    "\n",
+    "# Suppress warnings\n",
+    "import warnings\n",
+    "warnings.filterwarnings('ignore', message='.*narwhals.*')\n",
+    "warnings.filterwarnings('ignore')\n",
+    "\n",
+    "print(\"Libraries loaded\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "cell-2",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Dataset IDs - change these to match your history\n",
+    "VARIANTS_PER_SAMPLE = 13433   # Per-sample variants with AF\n",
+    "METADATA_CLEAN = 13681        # Sample metadata (sample_id, location, date)\n",
+    "ANNOTATED_VARIANTS = 13682    # Aggregated variants with gene annotation"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "cell-3",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Download all datasets\n",
+    "paths = await gxy.get([VARIANTS_PER_SAMPLE, METADATA_CLEAN, ANNOTATED_VARIANTS])\n",
+    "print(f\"Downloaded {len(paths)} files\")\n",
+    "for p in paths:\n",
+    "    print(f\"  {p}\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "cell-4",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Load per-sample variants (no header)\n",
+    "per_sample_cols = ['Sample', 'CHROM', 'POS', 'REF', 'ALT', 'QUAL', 'DP', 'AF', 'DP4', 'POSREFALT']\n",
+    "per_sample = pd.read_csv(paths[0], sep='\\t', names=per_sample_cols, skiprows=1)\n",
+    "print(f\"Per-sample variants: {len(per_sample)} rows\")\n",
+    "\n",
+    "# Load metadata (has header)\n",
+    "metadata = pd.read_csv(paths[1], sep='\\t')\n",
+    "print(f\"Sample metadata: {len(metadata)} samples\")\n",
+    "\n",
+    "# Load annotated variants (has header)\n",
+    "annotated = pd.read_csv(paths[2], sep='\\t')\n",
+    "print(f\"Annotated variants: {len(annotated)} variants\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "cell-5",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Filter to V/C/P gene variants only\n",
+    "vcp_variants = annotated[annotated['gene'] == 'P/V/C'].copy()\n",
+    "print(f\"V/C/P gene variants: {len(vcp_variants)}\")\n",
+    "print(f\"\\nBy product:\")\n",
+    "print(vcp_variants['product'].value_counts().to_string())"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "cell-6",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Get list of V/C/P variant IDs\n",
+    "vcp_var_ids = set(vcp_variants['var_id'].tolist())\n",
+    "print(f\"V/C/P variant IDs: {len(vcp_var_ids)}\")\n",
+    "\n",
+    "# Filter per-sample variants to only V/C/P\n",
+    "per_sample_vcp = per_sample[per_sample['POSREFALT'].isin(vcp_var_ids)].copy()\n",
+    "print(f\"Per-sample V/C/P variants: {len(per_sample_vcp)} rows\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "cell-7",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Join with metadata to get location\n",
+    "merged = per_sample_vcp.merge(\n",
+    "    metadata,\n",
+    "    left_on='Sample',\n",
+    "    right_on='sample_id',\n",
+    "    how='inner'\n",
+    ")\n",
+    "print(f\"Merged rows: {len(merged)}\")\n",
+    "print(f\"Unique samples: {merged['Sample'].nunique()}\")\n",
+    "print(f\"Unique locations: {merged['location'].nunique()}\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "cell-8",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Aggregate by location - calculate mean AF\n",
+    "location_stats = merged.groupby('location').agg(\n",
+    "    mean_af=('AF', 'mean'),\n",
+    "    sample_count=('Sample', 'nunique'),\n",
+    "    variant_count=('POSREFALT', 'count')\n",
+    ").reset_index()\n",
+    "\n",
+    "print(\"=== Mean AF by Location ===\")\n",
+    "print(location_stats.sort_values('mean_af', ascending=False).to_string(index=False))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "cell-9",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Parse location into country and state/province\n",
+    "def parse_location(loc):\n",
+    "    if ':' in loc:\n",
+    "        parts = loc.split(':', 1)\n",
+    "        return parts[0], parts[1]\n",
+    "    return loc, loc\n",
+    "\n",
+    "location_stats[['country', 'region']] = location_stats['location'].apply(\n",
+    "    lambda x: pd.Series(parse_location(x))\n",
+    ")\n",
+    "\n",
+    "# Separate by country\n",
+    "us_data = location_stats[location_stats['country'] == 'USA'].copy()\n",
+    "canada_data = location_stats[location_stats['country'] == 'Canada'].copy()\n",
+    "romania_data = location_stats[location_stats['country'].str.contains('Romania', na=False)].copy()\n",
+    "\n",
+    "print(f\"US locations: {len(us_data)}\")\n",
+    "print(f\"Canada locations: {len(canada_data)}\")\n",
+    "print(f\"Romania locations: {len(romania_data)}\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "cell-10",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# === US State Map ===\n",
+    "\n",
+    "# Load US states topojson\n",
+    "states = alt.topo_feature(vega_data.us_10m.url, 'states')\n",
+    "\n",
+    "# State name to ID mapping (FIPS codes)\n",
+    "state_ids = {\n",
+    "    'Alabama': 1, 'Alaska': 2, 'Arizona': 4, 'Arkansas': 5, 'California': 6,\n",
+    "    'Colorado': 8, 'Connecticut': 9, 'Delaware': 10, 'Florida': 12, 'Georgia': 13,\n",
+    "    'Hawaii': 15, 'Idaho': 16, 'Illinois': 17, 'Indiana': 18, 'Iowa': 19,\n",
+    "    'Kansas': 20, 'Kentucky': 21, 'Louisiana': 22, 'Maine': 23, 'Maryland': 24,\n",
+    "    'Massachusetts': 25, 'Michigan': 26, 'Minnesota': 27, 'Mississippi': 28,\n",
+    "    'Missouri': 29, 'Montana': 30, 'Nebraska': 31, 'Nevada': 32, 'New Hampshire': 33,\n",
+    "    'New Jersey': 34, 'New Mexico': 35, 'New York': 36, 'North Carolina': 37,\n",
+    "    'North Dakota': 38, 'Ohio': 39, 'Oklahoma': 40, 'Oregon': 41, 'Pennsylvania': 42,\n",
+    "    'Rhode Island': 44, 'South Carolina': 45, 'South Dakota': 46, 'Tennessee': 47,\n",
+    "    'Texas': 48, 'Utah': 49, 'Vermont': 50, 'Virginia': 51, 'Washington': 53,\n",
+    "    'West Virginia': 54, 'Wisconsin': 55, 'Wyoming': 56\n",
+    "}\n",
+    "\n",
+    "# Add FIPS ID to data\n",
+    "us_data['id'] = us_data['region'].map(state_ids)\n",
+    "us_data = us_data.dropna(subset=['id'])\n",
+    "us_data['id'] = us_data['id'].astype(int)\n",
+    "\n",
+    "print(\"US data with state IDs:\")\n",
+    "print(us_data[['region', 'id', 'mean_af', 'sample_count']].to_string(index=False))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "cell-11",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Create US map\n",
+    "us_background = alt.Chart(states).mark_geoshape(\n",
+    "    fill='lightgray',\n",
+    "    stroke='white'\n",
+    ").project(\n",
+    "    type='albersUsa'\n",
+    ").properties(\n",
+    "    width=600,\n",
+    "    height=400\n",
+    ")\n",
+    "\n",
+    "us_choropleth = alt.Chart(states).mark_geoshape(\n",
+    "    stroke='white'\n",
+    ").encode(\n",
+    "    color=alt.Color(\n",
+    "        'mean_af:Q',\n",
+    "        scale=alt.Scale(scheme='blues'),\n",
+    "        legend=alt.Legend(title='Mean AF')\n",
+    "    ),\n",
+    "    tooltip=[\n",
+    "        alt.Tooltip('region:N', title='State'),\n",
+    "        alt.Tooltip('mean_af:Q', title='Mean AF', format='.4f'),\n",
+    "        alt.Tooltip('sample_count:Q', title='Samples'),\n",
+    "        alt.Tooltip('variant_count:Q', title='Variant Calls')\n",
+    "    ]\n",
+    ").transform_lookup(\n",
+    "    lookup='id',\n",
+    "    from_=alt.LookupData(us_data, 'id', ['mean_af', 'region', 'sample_count', 'variant_count'])\n",
+    ").project(\n",
+    "    type='albersUsa'\n",
+    ").properties(\n",
+    "    width=600,\n",
+    "    height=400,\n",
+    "    title='V/C/P Gene Variant Mean AF by US State'\n",
+    ")\n",
+    "\n",
+    "us_map = us_background + us_choropleth\n",
+    "us_map"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "cell-12",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# === Canada Province Map ===\n",
+    "\n",
+    "# Canada provinces topojson URL\n",
+    "canada_url = 'https://raw.githubusercontent.com/codeforamerica/click_that_hood/master/public/data/canada.geojson'\n",
+    "\n",
+    "print(\"Canada data:\")\n",
+    "print(canada_data[['region', 'mean_af', 'sample_count']].to_string(index=False))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "cell-13",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Create Canada bar chart (simpler than choropleth for few provinces)\n",
+    "canada_chart = alt.Chart(canada_data).mark_bar().encode(\n",
+    "    x=alt.X('region:N', title='Province', sort='-y'),\n",
+    "    y=alt.Y('mean_af:Q', title='Mean Allele Frequency'),\n",
+    "    color=alt.Color(\n",
+    "        'mean_af:Q',\n",
+    "        scale=alt.Scale(scheme='greens'),\n",
+    "        legend=None\n",
+    "    ),\n",
+    "    tooltip=[\n",
+    "        alt.Tooltip('region:N', title='Province'),\n",
+    "        alt.Tooltip('mean_af:Q', title='Mean AF', format='.4f'),\n",
+    "        alt.Tooltip('sample_count:Q', title='Samples'),\n",
+    "        alt.Tooltip('variant_count:Q', title='Variant Calls')\n",
+    "    ]\n",
+    ").properties(\n",
+    "    width=400,\n",
+    "    height=300,\n",
+    "    title='V/C/P Gene Variant Mean AF by Canadian Province'\n",
+    ")\n",
+    "\n",
+    "canada_chart"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "cell-14",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# === Romania Visualization ===\n",
+    "\n",
+    "print(\"Romania data:\")\n",
+    "print(romania_data[['location', 'mean_af', 'sample_count', 'variant_count']].to_string(index=False))\n",
+    "\n",
+    "# Romania has only Bucharest, show as single indicator\n",
+    "romania_chart = alt.Chart(romania_data).mark_bar().encode(\n",
+    "    x=alt.X('location:N', title='Location'),\n",
+    "    y=alt.Y('mean_af:Q', title='Mean Allele Frequency'),\n",
+    "    color=alt.value('coral'),\n",
+    "    tooltip=[\n",
+    "        alt.Tooltip('location:N', title='Location'),\n",
+    "        alt.Tooltip('mean_af:Q', title='Mean AF', format='.4f'),\n",
+    "        alt.Tooltip('sample_count:Q', title='Samples'),\n",
+    "        alt.Tooltip('variant_count:Q', title='Variant Calls')\n",
+    "    ]\n",
+    ").properties(\n",
+    "    width=200,\n",
+    "    height=300,\n",
+    "    title='V/C/P Gene Variant Mean AF - Romania'\n",
+    ")\n",
+    "\n",
+    "romania_chart"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "cell-15",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# === Combined Summary ===\n",
+    "\n",
+    "# Combined comparison bar chart\n",
+    "all_regions = pd.concat([us_data, canada_data, romania_data])\n",
+    "all_regions['label'] = all_regions['country'] + ': ' + all_regions['region']\n",
+    "\n",
+    "combined_chart = alt.Chart(all_regions).mark_bar().encode(\n",
+    "    x=alt.X('label:N', title='Location', sort='-y'),\n",
+    "    y=alt.Y('mean_af:Q', title='Mean Allele Frequency'),\n",
+    "    color=alt.Color(\n",
+    "        'country:N',\n",
+    "        scale=alt.Scale(domain=['USA', 'Canada', 'Romania'], range=['steelblue', 'seagreen', 'coral']),\n",
+    "        legend=alt.Legend(title='Country')\n",
+    "    ),\n",
+    "    tooltip=[\n",
+    "        alt.Tooltip('label:N', title='Location'),\n",
+    "        alt.Tooltip('mean_af:Q', title='Mean AF', format='.4f'),\n",
+    "        alt.Tooltip('sample_count:Q', title='Samples'),\n",
+    "        alt.Tooltip('variant_count:Q', title='Variant Calls')\n",
+    "    ]\n",
+    ").properties(\n",
+    "    width=700,\n",
+    "    height=400,\n",
+    "    title='V/C/P Gene Variant Mean AF by Region (All Locations)'\n",
+    ")\n",
+    "\n",
+    "combined_chart"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "cell-16",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# === Summary Statistics ===\n",
+    "\n",
+    "print(\"=\"*60)\n",
+    "print(\"V/C/P GENE VARIANT ANALYSIS SUMMARY\")\n",
+    "print(\"=\"*60)\n",
+    "\n",
+    "print(f\"\\nTotal V/C/P variants analyzed: {len(vcp_variants)}\")\n",
+    "print(f\"Total per-sample variant calls: {len(per_sample_vcp)}\")\n",
+    "print(f\"Unique samples: {merged['Sample'].nunique()}\")\n",
+    "print(f\"Unique locations: {len(location_stats)}\")\n",
+    "\n",
+    "print(f\"\\n--- By Country ---\")\n",
+    "for country in ['USA', 'Canada', 'Romania']:\n",
+    "    subset = all_regions[all_regions['country'].str.contains(country, na=False)]\n",
+    "    if len(subset) > 0:\n",
+    "        print(f\"\\n{country}:\")\n",
+    "        print(f\"  Regions: {len(subset)}\")\n",
+    "        print(f\"  Total samples: {subset['sample_count'].sum()}\")\n",
+    "        print(f\"  Mean AF range: {subset['mean_af'].min():.4f} - {subset['mean_af'].max():.4f}\")\n",
+    "\n",
+    "print(f\"\\n--- Overall ---\")\n",
+    "print(f\"Overall mean AF: {merged['AF'].mean():.4f}\")\n",
+    "print(f\"AF std dev: {merged['AF'].std():.4f}\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "cell-17",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Save summary to Galaxy\n",
+    "location_stats.to_csv('vcp_variant_by_location.tsv', sep='\\t', index=False)\n",
+    "await gxy.put('vcp_variant_by_location.tsv', output='V/C/P Variants by Location', ext='tabular')\n",
+    "print(\"Summary saved to Galaxy history!\")"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "name": "python",
+   "version": "3.11"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

package/extensions/orbit/vendor/skills/skills/galaxy-integration/scripts/README.md

@@ -0,0 +1,165 @@
+# Scripts for Galaxy Integration
+
+Automation scripts to support tool discovery, workflow validation, and workflow testing on Galaxy instances.
+
+---
+
+## Overview: Which Script to Use?
+
+| Task | Script | Requires Galaxy API? |
+|------|--------|---------------------|
+| **Check tool on Galaxy instance** | `galaxy_tool_checker.py` | ✅ Yes |
+| **Validate .ga workflow** | `galaxy_tool_checker.py` | ✅ Yes |
+| **Test workflow execution** | `galaxy_tool_checker.py` | ✅ Yes |
+
+---
+
+## galaxy_tool_checker.py
+
+**Purpose**: Check tool availability, validate workflows, and test workflow execution on live Galaxy instances using BioBlend.
+
+**Key Features**:
+- Batch tool availability checking
+- Workflow validation (.ga files)
+- Workflow testing (import and run)
+- JSON output for automation
+- Low token usage compared to MCP
+
+**Installation**:
+```bash
+pip install bioblend
+
+# Optional: For .env file support
+pip install python-dotenv
+```
+
+**Usage Examples**:
+
+### Check Single Tool
+```bash
+python galaxy-integration/scripts/galaxy_tool_checker.py \
+    --url https://usegalaxy.org \
+    --api-key YOUR_API_KEY \
+    --tool hyphy
+```
+
+### Check Multiple Tools
+```bash
+python galaxy-integration/scripts/galaxy_tool_checker.py \
+    --url https://usegalaxy.org \
+    --api-key YOUR_API_KEY \
+    --tool hyphy iqtree seqkit mafft \
+    --output report.json
+```
+
+### Check Tools from File
+```bash
+# Create tools.txt with one tool name per line
+python galaxy-integration/scripts/galaxy_tool_checker.py \
+    --url https://usegalaxy.org \
+    --api-key YOUR_API_KEY \
+    --tool-list tools.txt
+```
+
+### Validate Workflow
+```bash
+python galaxy-integration/scripts/galaxy_tool_checker.py \
+    --url https://usegalaxy.org \
+    --api-key YOUR_API_KEY \
+    --workflow my_workflow.ga \
+    --verbose
+```
+
+### Test Workflow (Import and Run)
+```bash
+python galaxy-integration/scripts/galaxy_tool_checker.py \
+    --url https://usegalaxy.org \
+    --api-key YOUR_API_KEY \
+    --workflow my_workflow.ga \
+    --test \
+    --history "Test Run" \
+    --wait
+```
+
+**Exit Codes**:
+- `0`: Success (all tools found / workflow valid)
+- `1`: Failure (tools missing / workflow invalid)
+
+**See Also**:
+- `../galaxy-integration.md` - Complete integration guide
+
+---
+
+## When to Use Which Script
+
+| Scenario | Use | Why |
+|----------|-----|-----|
+| **Check if tool exists on YOUR Galaxy instance** | `galaxy_tool_checker.py` | Connects to actual Galaxy |
+| **Check multiple tools at once** | `galaxy_tool_checker.py --tool-list` | Batch checking |
+| **Validate .ga workflow structure** | `galaxy_tool_checker.py --workflow` | Checks all tools exist |
+| **Test workflow execution on Galaxy** | `galaxy_tool_checker.py --workflow --test` | Actually runs it |
+| **CI/CD automation** | `galaxy_tool_checker.py` | Scriptable with exit codes |
+
+**Summary**: 
+- `galaxy_tool_checker.py` = Check/validate/test on actual Galaxy instance (needs API key)
+
+---
+
+## Configuration: Environment Variables and .env Files
+
+### Option 1: .env File (Recommended)
+
+Create a `.env` file in your working directory (recommended: the **skills repository root**):
+
+```bash
+# Copy the example
+# From the skills repository root:
+cp .env.example .env
+
+# Edit with your credentials
+nano .env
+```
+
+`.env` file contents:
+```bash
+GALAXY_URL=https://usegalaxy.org/
+GALAXY_API_KEY=your_api_key_here
+```
+
+**Security**: The `.env` file is gitignored by default. Never commit API keys to git!
+
+**Note**: `galaxy_tool_checker.py` looks for a `.env` in the current directory **and parent directories**, so placing `.env` at the skills repo root lets you run the script from `nf-to-galaxy/scripts/` without copying credentials into subdirectories.
+
+### Option 2: Environment Variables
+
+```bash
+export GALAXY_URL="https://usegalaxy.org"
+export GALAXY_API_KEY="your_api_key_here"
+```
+
+### Option 3: Command Line Arguments
+
+```bash
+python galaxy_tool_checker.py \
+    --url https://usegalaxy.org \
+    --api-key YOUR_KEY \
+    --tool hyphy
+```
+
+### Getting Your API Key
+
+1. Go to your Galaxy instance (e.g., https://usegalaxy.org)
+2. Log in
+3. Navigate to: **User → Preferences → Manage API Key**
+4. Click "Create a new key" if you don't have one
+5. Copy the key to your `.env` file or environment variable
+
+**Priority Order**: Command line args → Environment variables → .env file
+
+---
+
+## Related Documentation
+
+- **`../README.md`** - Galaxy integration overview and setup
+- **`../SKILL.md`** - Agent entrypoint for Galaxy integration
+- **`../galaxy-integration.md`** - Galaxy MCP and BioBlend integration guide