@pikaa-ai/pikaa 0.3.1 → 0.3.4

package/skills/pathogen-variant-surveillance/scripts/reporting_lag.py

@@ -1,217 +0,0 @@
-#!/usr/bin/env python3
-# /// script
-# requires-python = ">=3.11"
-# dependencies = []
-# ///
-"""Measure how long sequences take to appear, and how far back to trust the data.
-
-The single most common way to get variant surveillance wrong is to compute
-prevalence over the last few weeks. Those weeks are not a sample of what was
-circulating -- they are a sample of whichever laboratories report fastest, and
-they keep growing for months. This script measures that filling-in curve from
-the instance itself and turns it into a cutoff date.
-
-    python3 reporting_lag.py --instance sars-cov-2 --where country=USA
-    python3 reporting_lag.py --instance h5n1 --target 0.8
-
-Method: take monthly collection cohorts old enough to have settled, group each
-by submission date, and compute what fraction of the cohort's present-day total
-had arrived by each lag. Averaging across cohorts gives the curve.
-
-The curve is a **lower bound** on the true lag: a cohort's denominator is what
-has arrived so far, and even old cohorts still gain sequences. Treat the
-recommended cutoff as the least conservative one defensible.
-"""
-from __future__ import annotations
-
-import argparse
-import sys
-from datetime import date, timedelta
-
-from lapis_client import (
-    LapisError,
-    aggregated,
-    data_version,
-    describe_instance,
-    emit,
-    pick_date_field,
-    range_keys,
-    resolve_base_url,
-)
-
-COLUMNS = ("lag_days", "mean_complete", "min_complete", "max_complete", "cohorts")
-OFFSETS = (7, 14, 21, 30, 45, 60, 90, 120, 180)
-
-
-def month_window(anchor: date, months_back: int) -> tuple[date, date]:
-    """First and last day of the month ``months_back`` before ``anchor``."""
-    total = anchor.year * 12 + (anchor.month - 1) - months_back
-    year, month = divmod(total, 12)
-    first = date(year, month + 1, 1)
-    nxt = date(year + 1, 1, 1) if month == 11 else date(year, month + 2, 1)
-    return first, nxt - timedelta(days=1)
-
-
-def cohort_curve(
-    base_url: str,
-    filters: dict,
-    submission_field: str,
-    cohort_end: date,
-) -> tuple[dict[int, float], int, int] | None:
-    """Cumulative completeness by lag for one collection cohort.
-
-    Returns ``(curve, dated, undated)``. Sequences with no parseable submission
-    date are excluded from the denominator rather than counted as never having
-    arrived — the question is "of the ones we can date, how fast did they come"
-    — but they are returned so the caller can say how many were set aside.
-    """
-    rows = aggregated(base_url, filters, [submission_field])
-    lags: list[tuple[int, int]] = []
-    dated = 0
-    undated = 0
-    for row in rows:
-        value = str(row.get(submission_field) or "")
-        n = int(row.get("count") or 0)
-        try:
-            submitted = date.fromisoformat(value[:10])
-        except ValueError:
-            undated += n
-            continue
-        lags.append((max(0, (submitted - cohort_end).days), n))
-        dated += n
-    if dated == 0:
-        return None
-    lags.sort()
-    curve: dict[int, float] = {}
-    for offset in OFFSETS:
-        arrived = sum(n for lag, n in lags if lag <= offset)
-        curve[offset] = arrived / dated
-    return curve, dated, undated
-
-
-def build_parser() -> argparse.ArgumentParser:
-    parser = argparse.ArgumentParser(
-        description="Measure sequence reporting lag and recommend a trust cutoff.",
-    )
-    parser.add_argument("--instance", default="sars-cov-2", help="registry name (default: sars-cov-2)")
-    parser.add_argument("--base-url", help="any other LAPIS deployment")
-    parser.add_argument("--date-field", help="override the auto-detected collection-date column")
-    parser.add_argument("--submission-field", help="override the auto-detected submission-date column")
-    parser.add_argument("--where", action="append", default=[], metavar="KEY=VALUE",
-                        help="extra filter, repeatable (lag differs sharply by country)")
-    parser.add_argument("--cohorts", type=int, default=6, help="monthly cohorts to average (default: 6)")
-    parser.add_argument("--skip-months", type=int, default=3,
-                        help="most recent months to exclude as unsettled (default: 3)")
-    parser.add_argument("--target", type=float, default=0.9,
-                        help="completeness the cutoff should reach (default: 0.9)")
-    parser.add_argument("--until", help="anchor date, YYYY-MM-DD (default: today)")
-    parser.add_argument("--format", choices=("table", "tsv", "json"), default="table")
-    return parser
-
-
-def main(argv: list[str] | None = None) -> int:
-    args = build_parser().parse_args(argv)
-
-    try:
-        base_url = resolve_base_url(args.instance, args.base_url)
-        schema = describe_instance(base_url)
-        collection_field = pick_date_field(schema, "collection", args.date_field)
-        submission_field = pick_date_field(schema, "submission", args.submission_field)
-        if collection_field == submission_field:
-            raise LapisError(
-                f"collection and submission both resolved to {collection_field!r}; "
-                "the lag would be identically zero. Set --submission-field explicitly."
-            )
-        where: dict[str, str] = {}
-        for pair in args.where:
-            key, sep, value = pair.partition("=")
-            if not sep or key.split(".")[0] not in schema["types"]:
-                raise LapisError(f"bad --where {pair!r}")
-            where[key] = value
-    except LapisError as exc:
-        print(f"error: {exc}", file=sys.stderr)
-        return 2
-
-    anchor = date.fromisoformat(args.until) if args.until else date.today()
-    from_key, to_key = range_keys(collection_field)
-
-    curves: list[dict[int, float]] = []
-    sizes: list[int] = []
-    undated_total = 0
-    try:
-        for back in range(args.skip_months, args.skip_months + max(1, args.cohorts)):
-            start, end = month_window(anchor, back)
-            result = cohort_curve(
-                base_url,
-                {from_key: start.isoformat(), to_key: end.isoformat(), **where},
-                submission_field,
-                end,
-            )
-            if result:
-                curve, dated, undated = result
-                curves.append(curve)
-                sizes.append(dated)
-                undated_total += undated
-    except LapisError as exc:
-        print(f"error: {exc}", file=sys.stderr)
-        return 2
-
-    if not curves:
-        print(
-            "error: no cohort in the requested range holds any sequence. "
-            "Widen --where, raise --cohorts, or lower --skip-months.",
-            file=sys.stderr,
-        )
-        return 1
-
-    rows = []
-    for offset in OFFSETS:
-        values = [c[offset] for c in curves]
-        rows.append(
-            {
-                "lag_days": offset,
-                "mean_complete": f"{sum(values) / len(values):.3f}",
-                "min_complete": f"{min(values):.3f}",
-                "max_complete": f"{max(values):.3f}",
-                "cohorts": len(values),
-            }
-        )
-
-    reached = [o for o in OFFSETS if sum(c[o] for c in curves) / len(curves) >= args.target]
-    print(emit(rows, COLUMNS, args.format))
-
-    sys.stdout.flush()
-    if args.format != "json":
-        print(
-            f"\n# {schema['name']} via {base_url} | data version {data_version(base_url)}"
-            f"\n# collection dates from {collection_field}, submission from {submission_field}"
-            f"\n# {len(curves)} monthly cohorts, {sum(sizes)} datable sequences"
-            + (f" ({undated_total} excluded for having no submission date)"
-               if undated_total else "")
-            + (f" | filters {where}" if where else ""),
-            file=sys.stderr,
-        )
-        if reached:
-            cutoff = anchor - timedelta(days=reached[0])
-            print(
-                f"# {args.target:.0%} of a cohort has arrived by {reached[0]} days.\n"
-                f"# Trust collection dates up to {cutoff.isoformat()}; treat anything "
-                f"later as provisional.",
-                file=sys.stderr,
-            )
-        else:
-            print(
-                f"# no lag up to {OFFSETS[-1]} days reaches {args.target:.0%} completeness "
-                f"(best {max(sum(c[o] for c in curves) / len(curves) for o in OFFSETS):.0%}). "
-                f"Recent weeks cannot support a prevalence estimate here.",
-                file=sys.stderr,
-            )
-        print(
-            "# This curve is a lower bound: cohort denominators are still growing.",
-            file=sys.stderr,
-        )
-    return 0
-
-
-if __name__ == "__main__":
-    sys.exit(main())

package/skills/pathogen-variant-surveillance/scripts/resolve_lineage.py

@@ -1,198 +0,0 @@
-#!/usr/bin/env python3
-# /// script
-# requires-python = ">=3.11"
-# dependencies = []
-# ///
-"""Resolve a lineage name against the live nomenclature before trusting it.
-
-Pango names are not stable identifiers. They are minted continuously, aliased
-through a key that must be fetched to be read, and **withdrawn or redesignated
-after the fact** -- so a remembered lineage fact is not merely stale, it can be
-actively wrong. This script answers, for each name: does it still exist, what
-does it expand to, what is it descended from, and how many sequences carry it.
-
-    python3 resolve_lineage.py XFG.23.1.3 PQ.17 PC.2
-    python3 resolve_lineage.py XFG --descendants
-    python3 resolve_lineage.py 2.3.4.4b --instance h5n1
-
-Exit code is 1 when any name is withdrawn or unknown, so it works as a gate on
-a manuscript's lineage list.
-"""
-from __future__ import annotations
-
-import argparse
-import re
-import sys
-
-from lapis_client import (
-    LapisError,
-    children_map,
-    count,
-    data_version,
-    descendants,
-    describe_instance,
-    emit,
-    fetch_lineage_notes,
-    fetch_pango_aliases,
-    lineage_definition,
-    lineage_field_candidates,
-    parent_chain,
-    pango_provenance,
-    pick_lineage_field,
-    recombinant_parents,
-    resolve_base_url,
-    unalias_full,
-)
-
-COLUMNS = (
-    "query",
-    "status",
-    "unaliased",
-    "parent",
-    "recombinant_of",
-    "descendants",
-    "sequences",
-    "detail",
-)
-PANGO_FIELDS = {"pangoLineage", "nextcladePangoLineage"}
-REDESIGNATED = re.compile(r"[Rr]edesignated as ([A-Za-z][A-Za-z0-9.]*)")
-
-
-def normalise(name: str) -> str:
-    """Uppercase the alphabetic head so ``xfg.1`` matches ``XFG.1``."""
-    head, sep, tail = name.strip().partition(".")
-    return f"{head.upper()}{sep}{tail}"
-
-
-def build_parser() -> argparse.ArgumentParser:
-    parser = argparse.ArgumentParser(
-        description="Check lineage names against the live nomenclature.",
-    )
-    parser.add_argument("names", nargs="+", help="lineage or clade names to resolve")
-    parser.add_argument("--instance", default="sars-cov-2", help="registry name (default: sars-cov-2)")
-    parser.add_argument("--base-url", help="any other LAPIS deployment")
-    parser.add_argument("--lineage-field", help="override the auto-detected lineage column")
-    parser.add_argument("--descendants", action="store_true",
-                        help="list every descendant lineage instead of counting them")
-    parser.add_argument("--no-counts", action="store_true",
-                        help="skip the sequence counts (one fewer request per name)")
-    parser.add_argument("--format", choices=("table", "tsv", "json"), default="table")
-    return parser
-
-
-def main(argv: list[str] | None = None) -> int:
-    args = build_parser().parse_args(argv)
-
-    try:
-        base_url = resolve_base_url(args.instance, args.base_url)
-        schema = describe_instance(base_url)
-        lineage_field, has_index = pick_lineage_field(schema, args.lineage_field)
-        # Fetched here, not inside the provenance f-string: a network failure at
-        # print time would raise after the table had already been written.
-        version = data_version(base_url)
-    except LapisError as exc:
-        print(f"error: {exc}", file=sys.stderr)
-        return 2
-
-    definition: dict = {}
-    if has_index:
-        try:
-            definition = lineage_definition(base_url, lineage_field)
-        except LapisError as exc:
-            print(f"warning: no lineage definition for {lineage_field}: {exc}", file=sys.stderr)
-
-    aliases: dict = {}
-    notes: dict = {}
-    if lineage_field in PANGO_FIELDS:
-        try:
-            aliases = fetch_pango_aliases()
-            notes = fetch_lineage_notes()
-        except LapisError as exc:
-            print(f"warning: pango-designation unreachable: {exc}", file=sys.stderr)
-
-    # Inverted once: the SARS-CoV-2 definition holds ~5,500 entries, so
-    # rebuilding it per name makes a list of lineages quadratic.
-    children = children_map(definition)
-
-    rows: list[dict] = []
-    failures = 0
-    for raw in args.names:
-        name = normalise(raw)
-        note = notes.get(name, {})
-        in_definition = name in definition
-
-        if note.get("status") == "withdrawn":
-            status = "withdrawn"
-        elif note.get("status") == "designated" or in_definition:
-            status = "current"
-        elif notes or definition:
-            status = "unknown"
-        else:
-            status = "unverified"
-        if status in {"withdrawn", "unknown"}:
-            failures += 1
-
-        detail = note.get("note", "")
-        successor = REDESIGNATED.search(detail)
-        if successor:
-            detail = f"now {successor.group(1)}; {detail}"
-        elif status == "unknown":
-            detail = "no such name in the live nomenclature for this instance"
-
-        chain = parent_chain(name, definition) if definition else []
-        kids = descendants(name, definition, children) if definition else []
-
-        sequences: object = ""
-        if not args.no_counts:
-            try:
-                query = f"{name}*" if has_index else name
-                sequences = count(base_url, {lineage_field: query})
-            except LapisError as exc:
-                # An indexed column rejects an unknown lineage outright rather
-                # than answering 0 -- which is the one place a typo is caught
-                # for you. An unindexed column would have returned 0 instead.
-                sequences = "n/a" if "not a valid lineage" in str(exc) else "error"
-
-        rows.append(
-            {
-                "query": raw,
-                "status": status,
-                "unaliased": unalias_full(name, aliases) if aliases else "",
-                "parent": chain[0] if chain else "",
-                "recombinant_of": "+".join(recombinant_parents(name, aliases)) if aliases else "",
-                "descendants": ", ".join(kids) if args.descendants else len(kids),
-                "sequences": sequences,
-                "detail": detail,
-            }
-        )
-
-    print(emit(rows, COLUMNS, args.format))
-    sys.stdout.flush()
-
-    if args.format != "json":
-        print(
-            f"\n# {schema['name']} via {base_url} | data version {version}"
-            f"\n# lineage column {lineage_field}"
-            f"{' with a lineage index' if has_index else ' with no lineage index'}"
-            + (
-                f"\n# nomenclature from pango-designation ({len(notes)} names, "
-                f"{sum(1 for v in notes.values() if v['status'] == 'withdrawn')} withdrawn)"
-                f"\n# source blobs {pango_provenance()}"
-                if notes
-                else ""
-            )
-            + ("\n# 'sequences' counts the lineage and its descendants" if has_index else ""),
-            file=sys.stderr,
-        )
-        others = [n for n, _ in lineage_field_candidates(schema) if n != lineage_field]
-        if others:
-            print(
-                f"# other lineage-like columns here: {', '.join(others)} "
-                f"(select one with --lineage-field)",
-                file=sys.stderr,
-            )
-    return 1 if failures else 0
-
-
-if __name__ == "__main__":
-    sys.exit(main())

package/skills/pathway-enrichment/references/databases-and-gene-sets.md

@@ -1,140 +0,0 @@
-# Databases, Gene Sets, and Gene-ID Mapping
-
-## Contents
-- [Picking libraries by question](#picking-libraries-by-question)
-- [The main gene-set databases](#the-main-gene-set-databases)
-- [MSigDB collections](#msigdb-collections)
-- [g:Profiler (alternative ORA, custom background, 500+ organisms)](#gprofiler)
-- [Gene-ID types and conversion](#gene-id-types-and-conversion)
-- [Organism handling](#organism-handling)
-- [Pathway/interaction APIs (Reactome, KEGG, STRING)](#pathwayinteraction-apis)
-- [Activity inference (decoupler: PROGENy, DoRothEA/CollecTRI)](#activity-inference)
-
-## Picking libraries by question
-
-Match the database to the biological question instead of running everything:
-
-| Question | Best gene sets |
-|----------|----------------|
-| "What are the broad themes?" | MSigDB **Hallmark** (50 curated, low redundancy) |
-| "What mechanism/process?" | **GO Biological Process** |
-| "Which curated pathways?" | **Reactome**, **KEGG**, **WikiPathways** |
-| "Molecular function / localization?" | GO MF / GO CC |
-| "Immune signatures?" | MSigDB **C7** (ImmuneSigDB) |
-| "Oncogenic / perturbation?" | MSigDB **C6** (oncogenic), **C2:CGP** |
-| "TF targets / regulons?" | MSigDB **C3**, ChEA, or decoupler (below) |
-| "Disease/phenotype association?" | g:Profiler HP, DisGeNET, GWAS Catalog |
-
-Start narrow (Hallmark + one of GO:BP / Reactome). Add libraries only if the
-question needs them — each extra library multiplies the testing burden.
-
-## The main gene-set databases
-
-- **GO (Gene Ontology)** — three namespaces: Biological Process (BP), Molecular
-  Function (MF), Cellular Component (CC). Hierarchical → highly redundant; collapse
-  terms after testing (see `interpretation.md`).
-- **KEGG** — manually curated metabolic & signaling pathways. Compact, well known.
-- **Reactome** — large, expert-curated, hierarchical human pathway set; good
-  granularity. APIs in `database-lookup`.
-- **WikiPathways** — community-curated pathways; complements KEGG/Reactome.
-- **MSigDB** — collections of collections (Hallmark, curated, GO, immune, etc.);
-  the standard source of GMT files for GSEA.
-
-## MSigDB collections
-
-| Collection | Contents |
-|-----------|----------|
-| **H** (`h.all`) | Hallmark — 50 refined, non-redundant signatures (best default for GSEA) |
-| **C2:CP** | Canonical Pathways: `c2.cp.kegg_medicus`, `c2.cp.reactome`, `c2.cp.wikipathways`, `c2.cp.biocarta` |
-| **C2:CGP** | Chemical & genetic perturbations |
-| **C3** | Regulatory targets (TFT, miRNA) |
-| **C5** | Ontology: `c5.go.bp`, `c5.go.mf`, `c5.go.cc`, `c5.hpo` |
-| **C6** | Oncogenic signatures |
-| **C7** | ImmuneSigDB |
-| **C8** | Cell-type signatures |
-
-Fetch via gseapy: `gp.Msigdb().get_gmt(category="h.all", dbver="2024.1.Hs")`
-(use `dbver="…Mm"` for mouse symbols). See `gseapy.md`.
-
-## g:Profiler
-
-The official client (`gprofiler-official`) is the best path when you need a
-**custom background**, **many organisms** (~500), or g:Profiler's `g:SCS`
-multiple-testing correction. It performs ORA over GO, KEGG, Reactome,
-WikiPathways, miRTarBase, CORUM, HP, and more in one call.
-
-```python
-from gprofiler import GProfiler
-
-gp = GProfiler(return_dataframe=True)
-res = gp.profile(
-    organism="hsapiens",                      # mmusculus, dmelanogaster, ...
-    query=gene_list,                          # symbols, Ensembl, Entrez — auto-detected
-    sources=["GO:BP", "KEGG", "REAC", "WP"],  # restrict sources
-    user_threshold=0.05,
-    significance_threshold_method="g_SCS",    # default; or "fdr" / "bonferroni"
-    domain_scope="custom",                    # use a custom statistical background
-    background=expressed_genes,               # the tested/expressed universe
-    no_iea=False,                             # True = drop electronic GO annotations
-)
-# columns: source, native, name, p_value, term_size, query_size,
-#          intersection_size, effective_domain_size, intersections
-```
-
-`gp.convert(organism="hsapiens", query=ids, target_namespace="ENTREZGENE")` maps
-IDs; `gp.orth(...)` maps orthologs across organisms.
-
-## Gene-ID types and conversion
-
-Enrichr and MSigDB libraries are keyed by **gene symbols**. Convert other ID
-types before ORA/GSEA, or matches silently drop.
-
-| You have | Convert with |
-|----------|--------------|
-| Ensembl gene IDs (`ENSG…`) | `gp.Biomart`, g:Profiler `g:Convert`, or `mygene` |
-| Entrez IDs | `mygene`, g:Profiler |
-| Mouse symbols → human | g:Profiler `g:Orth`, `mygene` (then run human libraries) |
-
-`mygene` example:
-```python
-import mygene
-mg = mygene.MyGeneInfo()
-hits = mg.querymany(ensembl_ids, scopes="ensembl.gene",
-                    fields="symbol", species="human", as_dataframe=True)
-symbols = hits["symbol"].dropna().tolist()
-```
-Strip Ensembl version suffixes first (`ENSG00000141510.16` → `ENSG00000141510`).
-The `gget` skill (`gget info`) is another quick ID-mapping path.
-
-## Organism handling
-
-- Human symbols are UPPERCASE (`TP53`); mouse symbols are Title-case (`Trp53`).
-- Set `organism=` for `gp.enrichr` (Enrichr) and use the matching MSigDB `dbver`
-  (`…Hs` vs `…Mm`) or g:Profiler `organism=` code.
-- Don't run human libraries on mouse symbols — convert or map orthologs first.
-
-## Pathway/interaction APIs
-
-For raw pathway content or network context (not enrichment statistics), use the
-`database-lookup` skill, which wraps:
-- **Reactome** content + Analysis Service (submit a gene list, get pathway
-  over-representation).
-- **KEGG** pathways/compounds.
-- **STRING** — protein–protein interactions plus its own functional-enrichment
-  endpoint for a submitted gene set; pairs well with `networkx` for network views.
-- **Gene Ontology / QuickGO** term metadata.
-
-## Activity inference
-
-When the goal is **pathway or TF activity** (a continuous score per sample/cell)
-rather than over-representation of a list, use `decoupler`. It runs multiple
-enrichment/activity methods (ORA, GSEA, univariate linear models, etc.) against
-curated priors:
-- **PROGENy** — 14 signaling pathway responsive signatures.
-- **DoRothEA / CollecTRI** — TF→target regulons for TF-activity inference.
-- **MSigDB** priors via its OmniPath integration.
-
-decoupler integrates natively with AnnData/Scanpy (per-cell activities) and with
-per-sample pseudobulk matrices. APIs evolve between major versions — check the
-current decoupler docs (https://decoupler-py.readthedocs.io/) for exact function
-names before writing code.