@pikaa-ai/pikaa 0.2.4 → 0.3.0

package/skills/neuropixels-analysis/scripts/run_sorting.py

@@ -0,0 +1,98 @@
+#!/usr/bin/env python
+"""
+Run spike sorting on preprocessed recording.
+
+Usage:
+    python run_sorting.py preprocessed/ --sorter kilosort4 --output sorting/
+"""
+
+import argparse
+from pathlib import Path
+
+import spikeinterface.full as si
+
+
+# Default parameters for each sorter
+SORTER_DEFAULTS = {
+    'kilosort4': {
+        'batch_size': 30000,
+        'nblocks': 1,
+        'Th_learned': 8,
+        'Th_universal': 9,
+    },
+    'kilosort3': {
+        'do_CAR': False,  # Already done in preprocessing
+    },
+    'spykingcircus2': {
+        'apply_preprocessing': False,
+    },
+    'mountainsort5': {
+        'filter': False,
+        'whiten': False,
+    },
+}
+
+
+def run_sorting(
+    input_path: str,
+    output_dir: str,
+    sorter: str = 'kilosort4',
+    sorter_params: dict = None,
+    n_jobs: int = -1,
+):
+    """Run spike sorting."""
+
+    print(f"Loading preprocessed recording from: {input_path}")
+    recording = si.load_extractor(Path(input_path) / 'preprocessed')
+
+    print(f"Recording: {recording.get_num_channels()} channels, {recording.get_total_duration():.1f}s")
+
+    # Get sorter parameters
+    params = SORTER_DEFAULTS.get(sorter, {}).copy()
+    if sorter_params:
+        params.update(sorter_params)
+
+    print(f"Running {sorter} with params: {params}")
+
+    output_path = Path(output_dir)
+
+    # Run sorter (note: parameter is 'folder' not 'output_folder' in newer SpikeInterface)
+    sorting = si.run_sorter(
+        sorter,
+        recording,
+        folder=output_path / f'{sorter}_output',
+        verbose=True,
+        **params,
+    )
+
+    print(f"\nSorting complete!")
+    print(f"  Units found: {len(sorting.unit_ids)}")
+    print(f"  Total spikes: {sum(len(sorting.get_unit_spike_train(uid)) for uid in sorting.unit_ids)}")
+
+    # Save sorting
+    sorting.save(folder=output_path / 'sorting')
+    print(f"  Saved to: {output_path / 'sorting'}")
+
+    return sorting
+
+
+def main():
+    parser = argparse.ArgumentParser(description='Run spike sorting')
+    parser.add_argument('input', help='Path to preprocessed recording')
+    parser.add_argument('--output', '-o', default='sorting/', help='Output directory')
+    parser.add_argument('--sorter', '-s', default='kilosort4',
+                       choices=['kilosort4', 'kilosort3', 'spykingcircus2', 'mountainsort5'])
+    parser.add_argument('--n-jobs', type=int, default=-1, help='Number of parallel jobs')
+
+    args = parser.parse_args()
+
+    run_sorting(
+        args.input,
+        args.output,
+        sorter=args.sorter,
+        n_jobs=args.n_jobs,
+    )
+
+
+if __name__ == '__main__':
+    main()

package/skills/nextflow/SKILL.md

@@ -0,0 +1,195 @@
+---
+name: nextflow
+description: Build, run, and debug Nextflow data pipelines and nf-core workflows end to end. Use whenever the user mentions Nextflow, nf-core, .nf files, nextflow.config, DSL2, processes/channels/operators, samplesheets, or wants to run a community pipeline (e.g. nf-core/rnaseq, nf-core/sarek), write or test a module/subworkflow with nf-test, configure executors/containers (Docker, Singularity/Apptainer, Conda, Wave), scale a workflow to HPC/SLURM or cloud (AWS Batch, Google Batch, Azure, Kubernetes), or debug a failed/-resume run. Make sure to use this skill for any reproducible scientific/bioinformatics workflow work even if the user does not say the word "Nextflow", and for authoring nf-core-compliant pipelines, modules, configs, and linting.
+license: Apache-2.0
+metadata:
+  version: "1.1"
+  skill-author: K-Dense Inc.
+---
+
+# Nextflow
+
+## Overview
+
+Nextflow is a workflow language and runtime for building **reproducible, portable, scalable** data pipelines. It is dominant in bioinformatics but works for any data-heavy computation. nf-core is a community curating production-grade Nextflow pipelines, reusable modules, and the `nf-core` tooling on top of Nextflow.
+
+Key ideas:
+- **Dataflow programming**: pipelines are `process` tasks connected by **channels**. Nextflow infers execution order and parallelism from data dependencies — there is no explicit scheduler to write.
+- **Write once, run anywhere**: the same pipeline runs locally, on HPC (SLURM, SGE, LSF, PBS), and on cloud (AWS Batch, Google Batch, Azure Batch, Kubernetes) by changing config/profiles, not code.
+- **Reproducibility**: per-task containers (Docker/Singularity/Apptainer/Conda/Wave) + `-resume` caching + pinned pipeline revisions.
+- **DSL2** is the modern, required syntax: modular `process`/`workflow`/`include` definitions.
+
+This skill covers both **running** existing pipelines and **developing** your own (Nextflow language + nf-core conventions, testing with nf-test, configuration, and deployment).
+
+## When to Use This Skill
+
+Use this skill when the user wants to:
+- Run an nf-core or custom Nextflow pipeline, or debug a failing/resuming run.
+- Write or modify `.nf` scripts, `nextflow.config`, profiles, or `nextflow_schema.json`.
+- Author or test nf-core-style modules/subworkflows (`main.nf`, `meta.yml`, `tests/`, nf-test).
+- Configure executors, containers, or resources; scale to HPC or cloud.
+- Build a reproducible scientific/bioinformatics workflow (even if "Nextflow" is not named).
+- Understand processes, channels, operators, `take`/`emit`, `publishDir`, `ext.args`, meta maps.
+
+## Setup
+
+Nextflow needs **Bash** and **Java 17 or newer** (17–25 supported). Verify with `java -version`.
+
+```bash
+# Install Nextflow (self-contained launcher)
+curl -s https://get.nextflow.io | bash      # creates ./nextflow
+sudo mv nextflow /usr/local/bin/             # put on PATH
+nextflow info                                # verify
+
+# Or via conda/bioconda (also gets a managed Java)
+conda create -n nf -c bioconda -c conda-forge nextflow nf-core
+```
+
+```bash
+# nf-core tools (Python) for creating/linting/running nf-core assets
+uv pip install nf-core            # or: conda install -c bioconda nf-core
+nf-core --version
+```
+
+Pin the engine for reproducibility: `export NXF_VER=24.10.0` (use an [edge] release only if needed). For air-gapped/HPC, see `references/running-pipelines.md` (offline mode) and `references/configuration.md`.
+
+## Two Modes of Work
+
+Decide which path the user is on — it changes everything:
+
+| Goal | Start here |
+|------|-----------|
+| **Run** an existing pipeline (nf-core or a `.nf` you were given) | `references/running-pipelines.md` |
+| **Develop** a new pipeline / module / subworkflow | `references/language.md` + `references/developing.md` |
+| **Configure / scale** (HPC, cloud, containers, resources) | `references/configuration.md` + `references/containers.md` |
+| **Test** modules/pipelines | `references/testing.md` |
+
+## Quick Start
+
+### Run an nf-core pipeline
+
+Always smoke-test with the bundled `test` profile first; it uses tiny data and proves your environment works.
+
+```bash
+# 1. Confirm setup works (downloads pipeline + tiny test data)
+nextflow run nf-core/rnaseq -profile test,docker --outdir results
+
+# 2. Real run: pin a revision (-r), pick a container engine, pass inputs
+nextflow run nf-core/rnaseq -r 3.14.0 \
+  -profile docker \
+  --input samplesheet.csv \
+  --genome GRCh38 \
+  --outdir results \
+  -resume
+```
+
+- `-profile` (single dash) selects bundled config profiles; **combine** them comma-separated, e.g. `test,docker`. Container/infra profiles (`docker`, `singularity`, `conda`) are mutually exclusive — pick one.
+- `--input`, `--genome`, `--outdir` (double dash) are **pipeline** parameters. nf-core pipelines take a **samplesheet CSV**, not loose files.
+- `-resume` reuses cached results from the last run. `-r <version>` pins a release for reproducibility.
+
+Use `nf-core pipelines launch <name>` for an interactive, schema-validated way to build the command and a `-params-file`. See `references/running-pipelines.md`.
+
+### Write a minimal pipeline
+
+```nextflow
+#!/usr/bin/env nextflow
+
+process SAYHELLO {
+    tag "$greeting"
+    publishDir "results", mode: 'copy'
+
+    input:
+    val greeting
+
+    output:
+    path "${greeting}.txt"
+
+    script:
+    """
+    echo '$greeting world' > ${greeting}.txt
+    """
+}
+
+workflow {
+    channel.of('hello', 'bonjour', 'hola') | SAYHELLO
+}
+```
+
+```bash
+nextflow run main.nf            # add -resume on reruns
+```
+
+The full language (processes, channels, operators, DSL2 workflows with `take`/`main`/`emit`, modules) is in `references/language.md`.
+
+## Core Concepts at a Glance
+
+- **Process**: a unit of work that runs a script (Bash by default). Declares `input:`, `output:`, optional `directives` (resources, container, `publishDir`, `tag`, `errorStrategy`), and a `script:`/`shell:`/`exec:` block. Each task runs in its own isolated work directory (`work/xx/yy…`).
+- **Channel**: the async queues that connect processes. **Queue channels** are consumable streams; **value channels** hold a single reusable value. Created with factories like `channel.of`, `channel.fromPath`, `channel.fromFilePairs`, `channel.value`.
+- **Operator**: transforms/combines channels — `map`, `filter`, `collect`, `groupTuple`, `join`, `combine`, `mix`, `flatten`, `branch`, `multiMap`, `splitCsv`, `view`, `set`.
+- **Workflow**: composes processes. DSL2 workflows can declare `take:` (inputs), `main:` (logic), `emit:` (named outputs) and be `include`d as subworkflows. The unnamed `workflow {}` is the entry point.
+- **Module**: a `.nf` file exposing processes/workflows via `include { NAME } from './path'` (supports `as` aliasing).
+- **Configuration**: `nextflow.config` sets `params`, `process` directives, `executor`, container engines, and named `profiles`. Selectors `withName:`/`withLabel:` target specific processes. See `references/configuration.md`.
+- **meta map** (nf-core): the convention of carrying a metadata map (`[ id:'sample1', single_end:false ]`) alongside files in input/output tuples so samples stay labeled through the pipeline. See `references/developing.md`.
+
+## nf-core tools CLI
+
+nf-core tools (v3+) group subcommands under `pipelines`, `modules`, and `subworkflows`. (Bare forms like `nf-core lint` still work but warn — prefer the grouped form.)
+
+| Command | Purpose |
+|---------|---------|
+| `nf-core pipelines list` | List/search nf-core pipelines (`--json`, keywords) |
+| `nf-core pipelines create` | Scaffold a new pipeline from the nf-core template |
+| `nf-core pipelines launch <name>` | Interactive, schema-driven run command + params file |
+| `nf-core pipelines download <name>` | Download pipeline + containers for offline/HPC use |
+| `nf-core pipelines lint` | Lint a pipeline against nf-core standards (run in repo root) |
+| `nf-core pipelines schema build` | Build/edit `nextflow_schema.json` via web GUI |
+| `nf-core pipelines create-params-file <name>` | Generate a documented YAML params file |
+| `nf-core pipelines bump-version` / `sync` | Bump version / sync with template updates |
+| `nf-core modules list/info/install/update/remove` | Manage modules from nf-core/modules |
+| `nf-core modules create` / `lint` / `test` | Author, lint, and nf-test a module |
+| `nf-core modules patch` / `bump-versions` | Patch an installed module / bump tool versions |
+| `nf-core subworkflows install/create/lint/test` | Same lifecycle for subworkflows |
+
+Full command reference, flags, and examples: `references/nf-core-tools.md`.
+
+## Essential `nextflow` CLI
+
+| Command | Purpose |
+|---------|---------|
+| `nextflow run <pipeline> -profile <p> --outdir <dir>` | Run a pipeline (path, `.nf`, or `user/repo`) |
+| `-resume` | Reuse cached results from prior run |
+| `-r <rev>` | Run a specific git revision/tag/branch |
+| `-params-file params.yml` | Supply parameters from YAML/JSON |
+| `-c custom.config` | Layer in an extra config file |
+| `-with-report -with-trace -with-timeline -with-dag flow.html` | Execution report, trace, timeline, DAG |
+| `-stub-run` | Run `stub:` blocks only (dry-run plumbing) |
+| `nextflow log` | Inspect past runs |
+| `nextflow clean -f -before <run>` | Delete old `work/` data |
+| `nextflow pull / drop / list / info <repo>` | Manage cached remote pipelines |
+
+Config, executors, caching internals, and tracing details: `references/configuration.md`.
+
+## Best Practices (high-value habits)
+
+- **Always `test` first**: `-profile test,docker` (or `singularity`/`conda`) before real data — fast and catches environment problems.
+- **Pin everything**: pipeline revision (`-r`), `NXF_VER`, and tool versions (containers). Don't run `latest` for science you'll publish.
+- **Use `-resume`** and understand caching: a task re-runs if its inputs, script, or container change. See cache-debugging in `references/configuration.md`.
+- **Parameterize via config/params-file**, not hardcoded paths. Keep `params` and profiles in `nextflow.config`.
+- **One container/conda env per process**; never rely on tools installed on the host.
+- **For nf-core dev**: reuse existing modules (`nf-core modules install`) before writing new ones; pass tool flags through `ext.args` (not hardcoded in the script); always include a `stub:` block and nf-test tests; run `nf-core pipelines lint` and `prettier` before committing.
+- **Right-size resources** with `process_low/medium/high` labels and `errorStrategy 'retry'` with dynamic `task.attempt` scaling instead of one giant request.
+- **Write forward-compatible syntax**: the strict-syntax parser becomes the default in Nextflow 26.04. Prefer lowercase `channel.of(...)`, explicit closure params (`{ v -> ... }`), `def` for all variables, and `emit:`-named outputs. Check with `nextflow lint`.
+
+## Reference Files
+
+Read the relevant file when you need depth — each is self-contained:
+
+- `references/language.md` — DSL2 language: processes, directives, channels, operators, workflows (`take`/`emit`), modules, dynamic resources, error handling.
+- `references/configuration.md` — `nextflow.config`, scopes, `profiles`, `withName`/`withLabel` selectors, executors (local/SLURM/cloud), caching/`-resume` internals, tracing/reports, the `nextflow` CLI.
+- `references/containers.md` — Docker, Singularity/Apptainer, Podman, Conda, Wave containers; choosing and enabling engines; common gotchas.
+- `references/running-pipelines.md` — finding/running nf-core pipelines, samplesheets, params files, reference genomes (iGenomes), offline runs, institutional configs, Seqera Platform.
+- `references/nf-core-tools.md` — complete `nf-core` CLI reference (pipelines/modules/subworkflows), flags, and workflows.
+- `references/developing.md` — authoring nf-core pipelines & modules: template layout, module `main.nf`/`meta.yml`, meta maps, `ext.args`/`modules.config`, subworkflows, resource labels, linting & Harshil alignment style.
+- `references/testing.md` — nf-test for modules/subworkflows/pipelines: test structure, assertions, snapshots, tags, running tests, CI.
+
+Official docs: Nextflow https://www.nextflow.io/docs/latest/ · nf-core https://nf-co.re/docs/ · Training https://training.nextflow.io/

package/skills/nextflow/references/configuration.md

@@ -0,0 +1,276 @@
+# Nextflow Configuration, Executors & CLI
+
+How to configure, scale, cache, observe, and drive Nextflow runs. Source: https://www.nextflow.io/docs/latest/config.html and related pages.
+
+## Table of Contents
+
+- [nextflow.config and scopes](#nextflowconfig-and-scopes)
+- [Profiles](#profiles)
+- [Process selectors](#process-selectors-withname-withlabel)
+- [Executors](#executors)
+- [Cloud executors](#cloud-executors)
+- [Caching and -resume](#caching-and--resume)
+- [Tracing and reports](#tracing-and-reports)
+- [The nextflow CLI](#the-nextflow-cli)
+- [Environment variables](#environment-variables)
+
+## nextflow.config and scopes
+
+`nextflow.config` configures a run **without touching pipeline code**. Files are auto-loaded and merged in **increasing precedence**: (1) `$NXF_HOME/config` (`~/.nextflow/config`), (2) `nextflow.config` in the **project** dir (where the script lives), (3) `nextflow.config` in the **launch** dir (current working dir), (4) each `-c custom.config` (repeatable). Using `-C file` instead loads **only** that file and ignores every other source. CLI `--param`/`-params-file` override config params. Values are grouped into **scopes**.
+
+```groovy
+// nextflow.config
+params {
+    input  = null
+    outdir = 'results'
+    genome = 'GRCh38'
+}
+
+process {
+    cpus   = 2
+    memory = '4 GB'
+    errorStrategy = 'retry'
+    maxRetries    = 2
+    container = 'ubuntu:22.04'
+}
+
+executor {
+    queueSize     = 100        // max parallel tasks submitted
+    submitRateLimit = '10/1min'
+}
+
+docker.enabled = true          // pick ONE container engine
+// singularity.enabled = true
+// conda.enabled = true
+
+timeline.enabled = true
+report.enabled   = true
+trace.enabled    = true
+
+manifest {
+    name = 'my/pipeline'
+    nextflowVersion = '!>=24.04.0'   // enforce a minimum engine version
+}
+```
+
+Key scopes: `params`, `process`, `executor`, `docker`/`singularity`/`apptainer`/`podman`/`conda`/`wave`, `aws`/`google`/`azure`/`k8s`, `tower` (Seqera Platform), `report`/`timeline`/`trace`/`dag`, `manifest`, `env`, `cleanup`.
+
+Assignment uses `=`. Use the dotted form (`docker.enabled = true`) or block form (`docker { enabled = true }`) interchangeably.
+
+## Profiles
+
+A **profile** is a named bundle of config, activated with `-profile`. nf-core pipelines ship profiles for each container engine plus a `test` profile.
+
+```groovy
+profiles {
+    standard { process.executor = 'local' }
+
+    docker      { docker.enabled = true; docker.runOptions = '-u $(id -u):$(id -g)' }
+    singularity { singularity.enabled = true; singularity.autoMounts = true }
+    conda       { conda.enabled = true }
+
+    slurm {
+        process.executor = 'slurm'
+        process.queue    = 'compute'
+    }
+
+    test {
+        params.input  = "${baseDir}/assets/test_samplesheet.csv"
+        params.genome = 'R64-1-1'
+    }
+}
+```
+
+Activate (combine with commas; **order matters** — later profiles override earlier):
+
+```bash
+nextflow run main.nf -profile test,singularity
+```
+
+Container/infra profiles (`docker`, `singularity`, `conda`) are mutually exclusive — choose one. If no `-profile` is given, the `standard` profile (if defined) is used.
+
+> Ordering caveat: with the legacy parser, profiles are applied in the **order they are defined in the config**, regardless of CLI order; with the strict parser (default in 26.04) they apply in **CLI order**. To stay safe, avoid combining profiles that set the *same* option to conflicting values.
+
+## Process selectors (withName, withLabel)
+
+Target configuration at specific processes inside the `process` scope. This is how nf-core sets per-tool resources and arguments without editing modules.
+
+```groovy
+process {
+    // by resource label
+    withLabel: 'process_low'    { cpus = 2;  memory = 6.GB;  time = 4.h }
+    withLabel: 'process_medium' { cpus = 6;  memory = 36.GB; time = 8.h }
+    withLabel: 'process_high'   { cpus = 12; memory = 72.GB; time = 16.h }
+
+    // by process name (regex / fully-qualified WORKFLOW:SUB:PROCESS)
+    withName: 'FASTQC' { cpus = 4 }
+    withName: '.*:ALIGN' {
+        container = 'quay.io/biocontainers/bwa:0.7.17--hed695b0_7'
+        ext.args  = '-M'                 // injected into the script as task.ext.args
+        publishDir = [ path: { "${params.outdir}/bam" }, mode: 'copy' ]
+    }
+}
+```
+
+Precedence: `withName` overrides `withLabel` overrides generic `process` settings. nf-core keeps all `withName` blocks in `conf/modules.config`.
+
+## Executors
+
+The executor maps tasks onto compute. Default is `local` (subprocesses on the current machine). Set globally or per-process.
+
+```groovy
+process.executor = 'slurm'        // global
+// or
+process { withLabel: 'process_high' { executor = 'slurm'; queue = 'bigmem' } }
+```
+
+| Executor | Platform |
+|----------|----------|
+| `local` | The current machine (default) |
+| `slurm` | SLURM clusters |
+| `sge` / `uge` | (Univa) Grid Engine |
+| `lsf` | IBM Spectrum LSF |
+| `pbs` / `pbspro` | PBS / Torque / PBS Pro |
+| `awsbatch` | AWS Batch |
+| `google-batch` | Google Cloud Batch |
+| `azurebatch` | Azure Batch |
+| `k8s` | Kubernetes |
+| `flux`, `hyperqueue`, `oar`, `moab` | Other schedulers |
+
+SLURM example with cluster options:
+
+```groovy
+process {
+    executor   = 'slurm'
+    queue      = 'compute'             // SLURM partition
+    clusterOptions = '--account=lab123'
+}
+executor {
+    queueSize       = 200              // max jobs queued at once
+    submitRateLimit = '10/1min'        // throttle submissions
+    perCpuMemAllocation = true         // emit --mem-per-cpu instead of --mem (some clusters require this)
+}
+```
+
+## Cloud executors
+
+Each cloud needs its scope configured (region, work bucket, etc.). Always set `workDir` to cloud storage.
+
+```groovy
+// AWS Batch
+process.executor = 'awsbatch'
+process.queue    = 'my-batch-queue'
+aws {
+    region = 'us-east-1'
+    batch.cliPath = '/home/ec2-user/miniconda/bin/aws'
+}
+workDir = 's3://my-bucket/work'
+```
+
+```groovy
+// Google Cloud Batch
+process.executor = 'google-batch'
+google.project   = 'my-gcp-project'
+google.location  = 'us-central1'
+workDir = 'gs://my-bucket/work'
+```
+
+For Azure use `azurebatch` + the `azure` scope with `workDir = 'az://...'`. For Kubernetes use `k8s` + `nextflow kuberun`. **Wave + Fusion** can accelerate cloud I/O (see `references/containers.md`). Seqera Platform (Tower) monitoring: set `tower.enabled = true` and `TOWER_ACCESS_TOKEN`, or run with `-with-tower`.
+
+## Caching and -resume
+
+`-resume` skips tasks whose inputs are unchanged, reusing outputs from the `work/` directory.
+
+```bash
+nextflow run main.nf -resume                 # resume the last run
+nextflow run main.nf -resume <session-id>    # resume a specific session (see `nextflow log`)
+```
+
+How caching works: each task gets a hash of its inputs (file content/metadata), the script text, the container, and key directives. If the hash matches a previous run, the cached output is reused.
+
+**Debugging cache misses** (a task re-runs when you expected a hit):
+- Run `nextflow log <run> -f hash,name,status,workdir` to inspect tasks.
+- Diff the task hashes of two runs: `nextflow -log a.log run … -dump-hashes json` then `nextflow -log b.log run … -resume -dump-hashes json`, and compare the `cache hash` entries to see exactly what changed.
+- Common causes: a changed input file timestamp/content, an edited script, a different container tag, a non-deterministic input order, an undeclared closure variable (use `def`), or using `cache false`.
+- `cache 'lenient'` hashes file size+timestamp (path) instead of content — useful on shared filesystems where content hashing is slow or timestamps shift. `cache 'deep'` hashes file content.
+- Caching is per **work directory**; deleting `work/` or changing `-w`/`workDir` loses the cache.
+
+**Work directory management**: `work/` grows fast. Clean with:
+
+```bash
+nextflow clean -f -before <run_name>    # remove work data before a run
+nextflow clean -f -but <run_name>       # keep one run, remove others
+```
+
+Set `cleanup = true` in config to auto-remove `work/` on successful completion (note: this disables `-resume`).
+
+## Tracing and reports
+
+Observability flags (or enable the matching config scope):
+
+| Flag | Produces |
+|------|----------|
+| `-with-report report.html` | Resource-usage HTML report per task |
+| `-with-timeline timeline.html` | Timeline of task execution |
+| `-with-trace trace.txt` | Tab-separated trace of every task (cpu, mem, time, status) |
+| `-with-dag flow.html` (or `.png`/`.mmd`/`.dot`) | Workflow DAG diagram |
+| `-with-tower` | Stream metrics to Seqera Platform |
+| `-with-weblog <url>` | POST run events to an HTTP endpoint |
+
+```bash
+nextflow run main.nf -profile docker \
+  -with-report -with-trace -with-timeline -with-dag flow.html
+```
+
+## The nextflow CLI
+
+```bash
+nextflow run <pipeline> [options]   # <pipeline> = path, .nf file, or github user/repo
+```
+
+| Option | Meaning |
+|--------|---------|
+| `-profile a,b` | Activate config profiles (comma-separated, ordered) |
+| `-resume [id]` | Reuse cached results |
+| `-r <rev>` | Git revision: tag, branch, or commit (for remote pipelines) |
+| `-params-file f.yml` | Load params from YAML/JSON |
+| `-c file.config` | Add an extra config file (highest-precedence config) |
+| `-w <dir>` / `workDir` | Set the work directory (local path or cloud URI) |
+| `-stub-run` (`-stub`) | Execute `stub:` blocks instead of real scripts |
+| `-entry <name>` | Run a specific named workflow as the entry point |
+| `-process.<dir> <val>` | Override a process directive at launch |
+| `-bg` | Run in background; `-ansi-log false` for plain logs |
+| `-dump-channels` | Print channel contents (with `.dump()`) |
+| `-preview` | Build the DAG without executing |
+
+Other top-level commands:
+
+```bash
+nextflow log [run] [-f fields]      # list past runs / fields of a run
+nextflow pull   user/repo           # download/update a remote pipeline
+nextflow list                       # list downloaded pipelines
+nextflow info   user/repo           # show pipeline metadata
+nextflow drop   user/repo           # delete a downloaded pipeline
+nextflow clean  -f -before <run>    # delete work data
+nextflow config [-profile p]        # print the resolved configuration
+nextflow inspect <pipeline>         # resolve per-process containers without running
+nextflow lint   <file|dir>          # check/format scripts & config (strict syntax)
+nextflow self-update                # update the Nextflow engine
+```
+
+## Environment variables
+
+| Variable | Effect |
+|----------|--------|
+| `NXF_VER` | Pin the Nextflow engine version for the run |
+| `NXF_WORK` | Default work directory |
+| `NXF_HOME` | Nextflow home (`~/.nextflow`) |
+| `NXF_SINGULARITY_CACHEDIR` / `NXF_APPTAINER_CACHEDIR` | Where SIF images are cached (set on HPC!) |
+| `NXF_CONDA_CACHEDIR` | Cached conda envs |
+| `NXF_CLOUDCACHE_PATH` | Store the task cache in object storage (e.g. `s3://bucket/cache`) for cloud runs |
+| `NXF_SYNTAX_PARSER=v2` | Opt into the strict-syntax parser (default in 26.04) |
+| `NXF_OPTS` | JVM options, e.g. `-Xms2g -Xmx4g` for big runs |
+| `TOWER_ACCESS_TOKEN` | Seqera Platform token (with `-with-tower`) |
+| `NXF_OFFLINE=true` | Disable network calls (offline/air-gapped runs) |
+
+On HPC, always set a shared `NXF_SINGULARITY_CACHEDIR` so image pulls are reused across jobs. See `references/running-pipelines.md` for offline execution.

package/skills/nextflow/references/containers.md

@@ -0,0 +1,92 @@
+# Software Dependencies: Containers & Conda
+
+Nextflow runs each process in an isolated software environment so pipelines are reproducible and portable. Never depend on tools installed on the host. Source: https://www.nextflow.io/docs/latest/container.html , https://www.nextflow.io/docs/latest/conda.html , https://www.nextflow.io/docs/latest/wave.html
+
+## Choosing an engine
+
+| Engine | Use when | Enable |
+|--------|----------|--------|
+| **Docker** | Local dev / laptops / CI with root or docker group | `docker.enabled = true` |
+| **Singularity / Apptainer** | HPC clusters (no root, shared FS) — most common in academia | `singularity.enabled = true` (or `apptainer.enabled = true`) |
+| **Podman** | Rootless alternative to Docker | `podman.enabled = true` |
+| **Charliecloud / Sarus / Shifter** | Site-specific HPC runtimes | `charliecloud.enabled = true`, etc. |
+| **Conda / Mamba** | No container runtime available; quick envs | `conda.enabled = true` |
+| **Wave** | On-the-fly container builds from conda/Dockerfiles, private registries, cloud speedups | `wave.enabled = true` |
+
+Enable exactly **one** container engine. nf-core ships these as profiles, so users typically just pass `-profile docker` / `-profile singularity` / `-profile conda`.
+
+## The container directive
+
+Each process declares its image; the engine config decides how it runs.
+
+```nextflow
+process SAMTOOLS_SORT {
+    container 'quay.io/biocontainers/samtools:1.19.2--h50ea8bc_0'
+    conda     'bioconda::samtools=1.19.2'    // fallback when -profile conda is used
+    script:
+    """
+    samtools sort -@ $task.cpus -o sorted.bam $input
+    """
+}
+```
+
+nf-core modules declare **both** a `container` (often a Biocontainers/Galaxy depot image) and a `conda` line, so the same module works under any engine. In nf-core modules the `conda` directive references a separate file — `conda "${moduleDir}/environment.yml"` — rather than an inline string. Biocontainers images live on `quay.io/biocontainers/...` (and `https://depot.galaxyproject.org/singularity/...` for Singularity), auto-built from Bioconda recipes.
+
+## Docker
+
+```groovy
+docker {
+    enabled    = true
+    runOptions = '-u $(id -u):$(id -g)'   // avoid root-owned output files
+}
+```
+
+## Singularity / Apptainer
+
+```groovy
+singularity {
+    enabled    = true
+    autoMounts = true                      // auto-bind host paths
+    cacheDir   = '/shared/singularity'     // or set NXF_SINGULARITY_CACHEDIR
+}
+```
+
+- Nextflow auto-converts Docker images to SIF on first use and caches them. On clusters, set a **shared** `cacheDir`/`NXF_SINGULARITY_CACHEDIR` so all jobs reuse pulls.
+- Bind extra paths with `runOptions = '-B /scratch'` if `autoMounts` misses them.
+- Apptainer (the renamed Singularity) uses the same options under the `apptainer` scope.
+
+## Conda / Mamba
+
+```groovy
+conda {
+    enabled    = true
+    useMamba   = true                       // faster solver
+    channels   = 'conda-forge,bioconda'     // priority order (this is the default since 26.04)
+    cacheDir   = '/shared/conda_envs'
+}
+process.conda = 'bioconda::bwa=0.7.17 bioconda::samtools=1.19'
+```
+
+Conda is the least reproducible option (solver drift, no OS isolation); prefer containers for published results. Use `NXF_CONDA_CACHEDIR` to reuse built envs.
+
+## Wave + Fusion
+
+**Wave** builds/augments containers on demand from a `conda` directive or a Dockerfile, pushes to a registry, and can mount private registries. **Fusion** is a virtual distributed file system that lets tasks read/write cloud object storage (S3/GCS) as if local — big speedups on cloud.
+
+```groovy
+wave {
+    enabled  = true
+    strategy = 'conda'           // build images from process conda directives
+}
+fusion.enabled = true            // pair with Wave on cloud executors
+tower.accessToken = secrets.TOWER_ACCESS_TOKEN   // some Wave features use Seqera creds
+```
+
+## Common gotchas
+
+- **Two engines enabled at once** → errors or surprising behavior. Enable one (use profiles).
+- **Root-owned outputs** with Docker → set `runOptions = '-u $(id -u):$(id -g)'`.
+- **Singularity can't see input files** → enable `autoMounts` or add `-B` binds; ensure the work dir and inputs are on bound paths.
+- **HPC pull storms / quota blowups** → set a shared `NXF_SINGULARITY_CACHEDIR` and pre-pull with `nf-core pipelines download` (see `references/running-pipelines.md`).
+- **Pinning**: always use a fully versioned image tag (and digest where possible). `latest` breaks reproducibility.
+- **Offline**: pre-stage all images (Singularity SIFs or a local Docker registry) and set `NXF_OFFLINE=true`.