@inflexa-ai/harness 0.6.0 → 0.7.0

package/dist/reference-data/catalog.js

@@ -21,6 +21,14 @@ export const ReferenceSha256Schema = z.string().regex(SHA256, "Expected a lowerc
  * redistributes nothing — so the URL must be a real third-party https endpoint.
  */
 const ReferenceArtifactUrlSchema = z.url().refine((value) => value.startsWith("https://"), "Reference artifacts must be fetched over https");
+/**
+ * Logical content format, independent of compression: a `.txt.gz` mapping table
+ * is `tsv`, not `gz`. Agents pick a reader from this rather than guessing from
+ * the extension, so it stays a free token instead of an enum — a new source must
+ * remain cheap to add, and an unrecognized format degrades to "unknown reader"
+ * rather than failing catalog validation.
+ */
+export const ReferenceArtifactFormatSchema = z.string().regex(/^[a-z0-9]+(?:[+-][a-z0-9]+)*$/, "Expected a lowercase format token");
 /**
  * One final file distributed for a reference dataset: a stable install-relative
  * path and the third-party https URL it is fetched from. The catalog carries no
@@ -30,17 +38,32 @@ const ReferenceArtifactUrlSchema = z.url().refine((value) => value.startsWith("h
  * one uniform trust-on-first-use model; there is deliberately no per-artifact
  * integrity class, so adding a source is only ever a URL, and no checked-in
  * digest is ours to maintain or lets go stale when a `current` upstream rebuilds.
+ *
+ * `format` and `contents` exist because skills name no paths: an agent asked for
+ * "a TF-target regulon network" can only match that against an inventory entry if
+ * the entry says what it holds and how to read it. `contents` should name the
+ * shape a caller must know — key columns, identifier space — not restate the title.
  */
 export const ReferenceArtifactSchema = z.strictObject({
     path: ReferenceArtifactPathSchema,
     url: ReferenceArtifactUrlSchema,
+    format: ReferenceArtifactFormatSchema,
+    contents: z.string().min(1),
 });
+/**
+ * Common-name organism the dataset describes, lowercase (`human`, `mouse`, `rat`).
+ * Omitted for species-agnostic or multi-species sources. This is the axis a wrong
+ * choice is most silently wrong on — human regulons over mouse counts still run —
+ * so it is a first-class field rather than something inferred from the id suffix.
+ */
+export const ReferenceOrganismSchema = z.string().regex(/^[a-z][a-z0-9]*(?:[ -][a-z0-9]+)*$/, "Expected a lowercase organism common name");
 /** Provenance and licensing information for one supported reference dataset. */
 export const ReferenceDatasetSchema = z.object({
     id: z.string().regex(SAFE_ID, "Expected a stable lowercase dataset id"),
     version: z.string().regex(SAFE_VERSION, "Expected a safe dataset version path segment"),
     title: z.string().min(1),
     description: z.string().min(1),
+    organism: ReferenceOrganismSchema.optional(),
     sourceUrl: z.url(),
     license: z.object({
         identifier: z.string().min(1),
@@ -97,9 +120,11 @@ function deepFreeze(value) {
 /**
  * Canonical release catalog. Every artifact is fetched directly from the third
  * party that publishes it; this project hosts, mirrors, and redistributes
- * nothing. Entries are added with a real upstream https URL, provenance, and
- * licensing data — and nothing else per file: no size, no digest to compute or
- * keep in sync.
+ * nothing. Entries are added with a real upstream https URL, provenance and
+ * licensing data, and a description of what the file holds — but never a size or
+ * digest to compute and keep in sync. The descriptive half is what makes a
+ * dataset findable: this catalog is the only place that says what a reference
+ * file is, so nothing downstream has to encode a path or a filename to use it.
  *
  * `version` is the dataset's upstream release identifier. Datasets built on an
  * upstream that has no immutable release — NCBI regenerates `gene_info` daily,
@@ -114,6 +139,7 @@ export const REFERENCE_DATA_CATALOG = deepFreeze(ReferenceDataCatalogSchema.pars
             version: "current",
             title: "NCBI human gene identifiers",
             description: "Entrez Gene records for Homo sapiens (NCBI taxonomy 9606): identifiers, approved symbols, synonyms, and cross-references (Ensembl, HGNC) in the dbXrefs column. Tab-separated, gzipped. NCBI rebuilds this file in place, so it is verified against what you downloaded rather than a checked-in digest.",
+            organism: "human",
             sourceUrl: "https://ftp.ncbi.nlm.nih.gov/gene/DATA/GENE_INFO/Mammalia/",
             license: {
                 identifier: "NCBI-Molecular-Data-Usage-Policy",
@@ -124,6 +150,8 @@ export const REFERENCE_DATA_CATALOG = deepFreeze(ReferenceDataCatalogSchema.pars
                 {
                     path: "Homo_sapiens.gene_info.gz",
                     url: "https://ftp.ncbi.nlm.nih.gov/gene/DATA/GENE_INFO/Mammalia/Homo_sapiens.gene_info.gz",
+                    format: "tsv",
+                    contents: "One row per gene, gzipped, header line prefixed '#'. Key columns: GeneID (Entrez), Symbol (HGNC), Synonyms, dbXrefs (holds Ensembl: and HGNC: cross-references, '|'-separated), type_of_gene.",
                 },
             ],
         },
@@ -132,6 +160,7 @@ export const REFERENCE_DATA_CATALOG = deepFreeze(ReferenceDataCatalogSchema.pars
             version: "current",
             title: "NCBI mouse gene identifiers",
             description: "Entrez Gene records for Mus musculus (NCBI taxonomy 10090): identifiers, approved symbols, synonyms, and cross-references (Ensembl, MGI) in the dbXrefs column. Tab-separated, gzipped. NCBI rebuilds this file in place, so it is verified against what you downloaded rather than a checked-in digest.",
+            organism: "mouse",
             sourceUrl: "https://ftp.ncbi.nlm.nih.gov/gene/DATA/GENE_INFO/Mammalia/",
             license: {
                 identifier: "NCBI-Molecular-Data-Usage-Policy",
@@ -142,6 +171,8 @@ export const REFERENCE_DATA_CATALOG = deepFreeze(ReferenceDataCatalogSchema.pars
                 {
                     path: "Mus_musculus.gene_info.gz",
                     url: "https://ftp.ncbi.nlm.nih.gov/gene/DATA/GENE_INFO/Mammalia/Mus_musculus.gene_info.gz",
+                    format: "tsv",
+                    contents: "One row per gene, gzipped, header line prefixed '#'. Key columns: GeneID (Entrez), Symbol (MGI), Synonyms, dbXrefs (holds Ensembl: and MGI: cross-references, '|'-separated), type_of_gene.",
                 },
             ],
         },
@@ -150,6 +181,7 @@ export const REFERENCE_DATA_CATALOG = deepFreeze(ReferenceDataCatalogSchema.pars
             version: "current",
             title: "NCBI rat gene identifiers",
             description: "Entrez Gene records for Rattus norvegicus (NCBI taxonomy 10116): identifiers, approved symbols, synonyms, and cross-references (Ensembl, RGD) in the dbXrefs column. Tab-separated, gzipped. NCBI rebuilds this file in place, so it is verified against what you downloaded rather than a checked-in digest.",
+            organism: "rat",
             sourceUrl: "https://ftp.ncbi.nlm.nih.gov/gene/DATA/GENE_INFO/Mammalia/",
             license: {
                 identifier: "NCBI-Molecular-Data-Usage-Policy",
@@ -160,6 +192,8 @@ export const REFERENCE_DATA_CATALOG = deepFreeze(ReferenceDataCatalogSchema.pars
                 {
                     path: "Rattus_norvegicus.gene_info.gz",
                     url: "https://ftp.ncbi.nlm.nih.gov/gene/DATA/GENE_INFO/Mammalia/Rattus_norvegicus.gene_info.gz",
+                    format: "tsv",
+                    contents: "One row per gene, gzipped, header line prefixed '#'. Key columns: GeneID (Entrez), Symbol (RGD), Synonyms, dbXrefs (holds Ensembl: and RGD: cross-references, '|'-separated), type_of_gene.",
                 },
             ],
         },
@@ -176,7 +210,14 @@ export const REFERENCE_DATA_CATALOG = deepFreeze(ReferenceDataCatalogSchema.pars
                 url: "https://www.ncbi.nlm.nih.gov/home/about/policies/",
             },
             recommendation: { group: "gene-identifiers", recommended: false },
-            artifacts: [{ path: "gene2ensembl.gz", url: "https://ftp.ncbi.nlm.nih.gov/gene/DATA/gene2ensembl.gz" }],
+            artifacts: [
+                {
+                    path: "gene2ensembl.gz",
+                    url: "https://ftp.ncbi.nlm.nih.gov/gene/DATA/gene2ensembl.gz",
+                    format: "tsv",
+                    contents: "Every species in one gzipped table — filter on #tax_id first (9606 human, 10090 mouse, 10116 rat). Columns: #tax_id, GeneID, Ensembl_gene_identifier, RNA_nucleotide_accession.version, Ensembl_rna_identifier, protein_accession.version, Ensembl_protein_identifier.",
+                },
+            ],
         },
         {
             // ~2.2 GB all-species monolithic table — the largest catalog entry. Opt-in like reactome-mappings.
@@ -190,13 +231,21 @@ export const REFERENCE_DATA_CATALOG = deepFreeze(ReferenceDataCatalogSchema.pars
                 url: "https://www.ncbi.nlm.nih.gov/home/about/policies/",
             },
             recommendation: { group: "gene-identifiers", recommended: false },
-            artifacts: [{ path: "gene2refseq.gz", url: "https://ftp.ncbi.nlm.nih.gov/gene/DATA/gene2refseq.gz" }],
+            artifacts: [
+                {
+                    path: "gene2refseq.gz",
+                    url: "https://ftp.ncbi.nlm.nih.gov/gene/DATA/gene2refseq.gz",
+                    format: "tsv",
+                    contents: "Every species in one gzipped table (~2.2 GB) — filter on #tax_id first, and prefer a streaming read. Columns: #tax_id, GeneID, status, RNA_nucleotide_accession.version, protein_accession.version, genomic_nucleotide_accession.version, Symbol.",
+                },
+            ],
         },
         {
             id: "uniprot-idmapping-human",
             version: "current",
             title: "UniProt human ID mapping",
             description: "UniProtKB accession ↔ Entrez, Ensembl, RefSeq and other identifiers for Homo sapiens (idmapping_selected, ~61 MB gzipped). UniProt overwrites `current_release` each cycle, so it is verified against what you downloaded rather than a checked-in digest.",
+            organism: "human",
             sourceUrl: "https://ftp.uniprot.org/pub/databases/uniprot/current_release/knowledgebase/idmapping/by_organism/",
             license: { identifier: "CC-BY-4.0", url: "https://www.uniprot.org/help/license" },
             recommendation: { group: "gene-identifiers", recommended: true },
@@ -204,6 +253,8 @@ export const REFERENCE_DATA_CATALOG = deepFreeze(ReferenceDataCatalogSchema.pars
                 {
                     path: "HUMAN_9606_idmapping_selected.tab.gz",
                     url: "https://ftp.uniprot.org/pub/databases/uniprot/current_release/knowledgebase/idmapping/by_organism/HUMAN_9606_idmapping_selected.tab.gz",
+                    format: "tsv",
+                    contents: "Gzipped, 22 columns, NO header. Positional: 1 UniProtKB-AC, 2 UniProtKB-ID, 3 GeneID (Entrez), 4 RefSeq, 19 Ensembl, 20 Ensembl_TRS, 21 Ensembl_PRO. Multi-valued cells are '; '-separated.",
                 },
             ],
         },
@@ -212,6 +263,7 @@ export const REFERENCE_DATA_CATALOG = deepFreeze(ReferenceDataCatalogSchema.pars
             version: "current",
             title: "UniProt mouse ID mapping",
             description: "UniProtKB accession ↔ Entrez, Ensembl, RefSeq and other identifiers for Mus musculus (idmapping_selected, ~18 MB gzipped). UniProt overwrites `current_release` each cycle, so it is verified against what you downloaded rather than a checked-in digest.",
+            organism: "mouse",
             sourceUrl: "https://ftp.uniprot.org/pub/databases/uniprot/current_release/knowledgebase/idmapping/by_organism/",
             license: { identifier: "CC-BY-4.0", url: "https://www.uniprot.org/help/license" },
             recommendation: { group: "gene-identifiers", recommended: true },
@@ -219,6 +271,8 @@ export const REFERENCE_DATA_CATALOG = deepFreeze(ReferenceDataCatalogSchema.pars
                 {
                     path: "MOUSE_10090_idmapping_selected.tab.gz",
                     url: "https://ftp.uniprot.org/pub/databases/uniprot/current_release/knowledgebase/idmapping/by_organism/MOUSE_10090_idmapping_selected.tab.gz",
+                    format: "tsv",
+                    contents: "Gzipped, 22 columns, NO header. Positional: 1 UniProtKB-AC, 2 UniProtKB-ID, 3 GeneID (Entrez), 4 RefSeq, 19 Ensembl, 20 Ensembl_TRS, 21 Ensembl_PRO. Multi-valued cells are '; '-separated.",
                 },
             ],
         },
@@ -227,6 +281,7 @@ export const REFERENCE_DATA_CATALOG = deepFreeze(ReferenceDataCatalogSchema.pars
             version: "current",
             title: "UniProt rat ID mapping",
             description: "UniProtKB accession ↔ Entrez, Ensembl, RefSeq and other identifiers for Rattus norvegicus (idmapping_selected, ~6 MB gzipped). UniProt overwrites `current_release` each cycle, so it is verified against what you downloaded rather than a checked-in digest.",
+            organism: "rat",
             sourceUrl: "https://ftp.uniprot.org/pub/databases/uniprot/current_release/knowledgebase/idmapping/by_organism/",
             license: { identifier: "CC-BY-4.0", url: "https://www.uniprot.org/help/license" },
             recommendation: { group: "gene-identifiers", recommended: true },
@@ -234,6 +289,8 @@ export const REFERENCE_DATA_CATALOG = deepFreeze(ReferenceDataCatalogSchema.pars
                 {
                     path: "RAT_10116_idmapping_selected.tab.gz",
                     url: "https://ftp.uniprot.org/pub/databases/uniprot/current_release/knowledgebase/idmapping/by_organism/RAT_10116_idmapping_selected.tab.gz",
+                    format: "tsv",
+                    contents: "Gzipped, 22 columns, NO header. Positional: 1 UniProtKB-AC, 2 UniProtKB-ID, 3 GeneID (Entrez), 4 RefSeq, 19 Ensembl, 20 Ensembl_TRS, 21 Ensembl_PRO. Multi-valued cells are '; '-separated.",
                 },
             ],
         },
@@ -246,8 +303,18 @@ export const REFERENCE_DATA_CATALOG = deepFreeze(ReferenceDataCatalogSchema.pars
             license: { identifier: "CC0-1.0", url: "https://reactome.org/license" },
             recommendation: { group: "pathways", recommended: true },
             artifacts: [
-                { path: "ReactomePathways.gmt", url: "https://reactome.org/download/current/ReactomePathways.gmt" },
-                { path: "ReactomePathways.txt", url: "https://reactome.org/download/current/ReactomePathways.txt" },
+                {
+                    path: "ReactomePathways.gmt",
+                    url: "https://reactome.org/download/current/ReactomePathways.gmt",
+                    format: "gmt",
+                    contents: "One pathway per line, tab-separated: pathway name, Reactome stable ID, then member gene symbols. Covers every Reactome species in one file — filter by pathway name suffix or join ReactomePathways.txt to restrict to one.",
+                },
+                {
+                    path: "ReactomePathways.txt",
+                    url: "https://reactome.org/download/current/ReactomePathways.txt",
+                    format: "tsv",
+                    contents: "Pathway index, no header: stable ID, display name, species name. Join on stable ID to restrict the GMT to one species.",
+                },
             ],
         },
         {
@@ -266,14 +333,20 @@ export const REFERENCE_DATA_CATALOG = deepFreeze(ReferenceDataCatalogSchema.pars
                 {
                     path: "NCBI2Reactome_All_Levels.txt",
                     url: "https://reactome.org/download/current/NCBI2Reactome_All_Levels.txt",
+                    format: "tsv",
+                    contents: "No header, every species: Entrez gene ID, Reactome pathway stable ID, pathway URL, pathway name, evidence code, species name. Filter on the species column.",
                 },
                 {
                     path: "Ensembl2Reactome_All_Levels.txt",
                     url: "https://reactome.org/download/current/Ensembl2Reactome_All_Levels.txt",
+                    format: "tsv",
+                    contents: "No header, every species: Ensembl gene ID, Reactome pathway stable ID, pathway URL, pathway name, evidence code, species name. Filter on the species column.",
                 },
                 {
                     path: "UniProt2Reactome_All_Levels.txt",
                     url: "https://reactome.org/download/current/UniProt2Reactome_All_Levels.txt",
+                    format: "tsv",
+                    contents: "No header, every species: UniProtKB accession, Reactome pathway stable ID, pathway URL, pathway name, evidence code, species name. Filter on the species column.",
                 },
             ],
         },
@@ -282,6 +355,7 @@ export const REFERENCE_DATA_CATALOG = deepFreeze(ReferenceDataCatalogSchema.pars
             version: "2026.07.10",
             title: "WikiPathways human pathways",
             description: "Community-curated Homo sapiens pathway gene sets (GMT) from the immutable 2026-07-10 WikiPathways snapshot.",
+            organism: "human",
             sourceUrl: "https://data.wikipathways.org/20260710/gmt/",
             license: { identifier: "CC0-1.0", url: "https://www.wikipathways.org/about/terms.html" },
             recommendation: { group: "pathways", recommended: true },
@@ -289,6 +363,8 @@ export const REFERENCE_DATA_CATALOG = deepFreeze(ReferenceDataCatalogSchema.pars
                 {
                     path: "wikipathways_Homo_sapiens.gmt",
                     url: "https://data.wikipathways.org/20260710/gmt/wikipathways-20260710-gmt-Homo_sapiens.gmt",
+                    format: "gmt",
+                    contents: "One pathway per line, tab-separated: '%'-delimited descriptor (name, WikiPathways ID, species), pathway URL, then member gene symbols (HGNC). Split the first field on '%' to recover a readable set name.",
                 },
             ],
         },
@@ -297,6 +373,7 @@ export const REFERENCE_DATA_CATALOG = deepFreeze(ReferenceDataCatalogSchema.pars
             version: "2026.07.10",
             title: "WikiPathways mouse pathways",
             description: "Community-curated Mus musculus pathway gene sets (GMT) from the immutable 2026-07-10 WikiPathways snapshot.",
+            organism: "mouse",
             sourceUrl: "https://data.wikipathways.org/20260710/gmt/",
             license: { identifier: "CC0-1.0", url: "https://www.wikipathways.org/about/terms.html" },
             recommendation: { group: "pathways", recommended: true },
@@ -304,6 +381,8 @@ export const REFERENCE_DATA_CATALOG = deepFreeze(ReferenceDataCatalogSchema.pars
                 {
                     path: "wikipathways_Mus_musculus.gmt",
                     url: "https://data.wikipathways.org/20260710/gmt/wikipathways-20260710-gmt-Mus_musculus.gmt",
+                    format: "gmt",
+                    contents: "One pathway per line, tab-separated: '%'-delimited descriptor (name, WikiPathways ID, species), pathway URL, then member gene symbols (MGI). Split the first field on '%' to recover a readable set name.",
                 },
             ],
         },
@@ -312,6 +391,7 @@ export const REFERENCE_DATA_CATALOG = deepFreeze(ReferenceDataCatalogSchema.pars
             version: "2026.07.10",
             title: "WikiPathways rat pathways",
             description: "Community-curated Rattus norvegicus pathway gene sets (GMT) from the immutable 2026-07-10 WikiPathways snapshot.",
+            organism: "rat",
             sourceUrl: "https://data.wikipathways.org/20260710/gmt/",
             license: { identifier: "CC0-1.0", url: "https://www.wikipathways.org/about/terms.html" },
             recommendation: { group: "pathways", recommended: true },
@@ -319,6 +399,8 @@ export const REFERENCE_DATA_CATALOG = deepFreeze(ReferenceDataCatalogSchema.pars
                 {
                     path: "wikipathways_Rattus_norvegicus.gmt",
                     url: "https://data.wikipathways.org/20260710/gmt/wikipathways-20260710-gmt-Rattus_norvegicus.gmt",
+                    format: "gmt",
+                    contents: "One pathway per line, tab-separated: '%'-delimited descriptor (name, WikiPathways ID, species), pathway URL, then member gene symbols (RGD). Split the first field on '%' to recover a readable set name.",
                 },
             ],
         },
@@ -327,6 +409,7 @@ export const REFERENCE_DATA_CATALOG = deepFreeze(ReferenceDataCatalogSchema.pars
             version: "2026.1",
             title: "MSigDB human hallmark gene sets",
             description: "The 50 MSigDB hallmark gene sets for Homo sapiens (symbols GMT) — coherent, well-defined biological states and processes for enrichment. From the immutable MSigDB 2026.1 human release.",
+            organism: "human",
             sourceUrl: "https://data.broadinstitute.org/gsea-msigdb/msigdb/",
             license: { identifier: "MSigDB-License", url: "https://www.gsea-msigdb.org/gsea/msigdb/license.jsp" },
             recommendation: { group: "pathways", recommended: true },
@@ -334,6 +417,8 @@ export const REFERENCE_DATA_CATALOG = deepFreeze(ReferenceDataCatalogSchema.pars
                 {
                     path: "h.all.v2026.1.Hs.symbols.gmt",
                     url: "https://data.broadinstitute.org/gsea-msigdb/msigdb/release/2026.1.Hs/h.all.v2026.1.Hs.symbols.gmt",
+                    format: "gmt",
+                    contents: "50 hallmark sets, one per line, tab-separated: set name (HALLMARK_*), source URL, then member gene symbols (HGNC). Hallmark only — the C2/C5/C6/C7/C8 collections are not part of this dataset.",
                 },
             ],
         },
@@ -342,6 +427,7 @@ export const REFERENCE_DATA_CATALOG = deepFreeze(ReferenceDataCatalogSchema.pars
             version: "2026.1",
             title: "MSigDB mouse hallmark gene sets",
             description: "The MSigDB mouse hallmark gene sets for Mus musculus (symbols GMT) — the murine ortholog-mapped hallmark collection for enrichment. From the immutable MSigDB 2026.1 mouse release.",
+            organism: "mouse",
             sourceUrl: "https://data.broadinstitute.org/gsea-msigdb/msigdb/",
             license: { identifier: "MSigDB-License", url: "https://www.gsea-msigdb.org/gsea/msigdb/license.jsp" },
             recommendation: { group: "pathways", recommended: true },
@@ -349,6 +435,8 @@ export const REFERENCE_DATA_CATALOG = deepFreeze(ReferenceDataCatalogSchema.pars
                 {
                     path: "mh.all.v2026.1.Mm.symbols.gmt",
                     url: "https://data.broadinstitute.org/gsea-msigdb/msigdb/release/2026.1.Mm/mh.all.v2026.1.Mm.symbols.gmt",
+                    format: "gmt",
+                    contents: "Mouse hallmark sets, one per line, tab-separated: set name, source URL, then member gene symbols (MGI). Hallmark only — the C2/C5/C6/C7/C8 collections are not part of this dataset.",
                 },
             ],
         },
@@ -357,6 +445,7 @@ export const REFERENCE_DATA_CATALOG = deepFreeze(ReferenceDataCatalogSchema.pars
             version: "2.0",
             title: "CollecTRI human regulatory network",
             description: "A literature-curated human transcription-factor target interaction network (CSV) from the immutable Zenodo record 8192729.",
+            organism: "human",
             sourceUrl: "https://zenodo.org/records/8192729",
             license: { identifier: "CC-BY-4.0", url: "https://zenodo.org/records/8192729" },
             recommendation: { group: "regulatory-networks", recommended: true },
@@ -364,6 +453,83 @@ export const REFERENCE_DATA_CATALOG = deepFreeze(ReferenceDataCatalogSchema.pars
                 {
                     path: "CollecTRI_regulons.csv",
                     url: "https://zenodo.org/records/8192729/files/CollecTRI_regulons.csv",
+                    format: "csv",
+                    contents: "TF-target regulons, one interaction per row: source (TF symbol), target (gene symbol), weight (+1 activating, -1 repressing), then curation provenance columns. This is the prior a TF-activity method consumes — read it with a CSV reader and pass the frame directly as decoupler's network argument. It is CSV, not Parquet.",
+                },
+            ],
+        },
+        {
+            // Sourced from the package repository's data/ directory rather than the OmniPath web
+            // service that `dc.op.progeny()` calls: a live query endpoint is not an artifact, and
+            // an install that depends on one breaks whenever the service does.
+            id: "progeny-human",
+            version: "current",
+            title: "PROGENy human pathway weights",
+            description: "PROGENy footprint-based pathway responsive genes for Homo sapiens — per-gene weights for 14 signalling pathways (EGFR, MAPK, PI3K, TNFa, p53, Hypoxia and others), the standard prior for pathway-activity inference from expression. Tracks the package's default branch, so it is verified against what you downloaded rather than a checked-in digest.",
+            organism: "human",
+            sourceUrl: "https://github.com/saezlab/progeny",
+            license: { identifier: "Apache-2.0", url: "https://github.com/saezlab/progeny/blob/master/LICENSE" },
+            recommendation: { group: "pathway-activity", recommended: true },
+            artifacts: [
+                {
+                    path: "model_human_full.rda",
+                    url: "https://raw.githubusercontent.com/saezlab/progeny/master/data/model_human_full.rda",
+                    format: "rda",
+                    contents: "R serialized data frame `model_human_full` of pathway weights: columns gene, weight, p.value, pathway. Load in R with load(); from Python read it through rpy2 rather than a pandas reader. Rename to source/target/weight if a decoupler-style network is expected.",
+                },
+            ],
+        },
+        {
+            id: "progeny-mouse",
+            version: "current",
+            title: "PROGENy mouse pathway weights",
+            description: "PROGENy footprint-based pathway responsive genes for Mus musculus — per-gene weights for 14 signalling pathways, the standard prior for pathway-activity inference from expression. Tracks the package's default branch, so it is verified against what you downloaded rather than a checked-in digest.",
+            organism: "mouse",
+            sourceUrl: "https://github.com/saezlab/progeny",
+            license: { identifier: "Apache-2.0", url: "https://github.com/saezlab/progeny/blob/master/LICENSE" },
+            recommendation: { group: "pathway-activity", recommended: true },
+            artifacts: [
+                {
+                    path: "model_mouse_full.rda",
+                    url: "https://raw.githubusercontent.com/saezlab/progeny/master/data/model_mouse_full.rda",
+                    format: "rda",
+                    contents: "R serialized data frame `model_mouse_full` of pathway weights: columns gene, weight, p.value, pathway. Load in R with load(); from Python read it through rpy2 rather than a pandas reader. Rename to source/target/weight if a decoupler-style network is expected.",
+                },
+            ],
+        },
+        {
+            id: "dorothea-human",
+            version: "current",
+            title: "DoRothEA human TF regulons",
+            description: "DoRothEA transcription-factor regulons for Homo sapiens with per-interaction confidence levels A-E, where A is the best-supported. The confidence-filtered alternative to CollecTRI for TF-activity inference; most analyses keep levels A-C. Tracks the package's default branch, so it is verified against what you downloaded rather than a checked-in digest.",
+            organism: "human",
+            sourceUrl: "https://github.com/saezlab/dorothea",
+            license: { identifier: "GPL-3.0", url: "https://github.com/saezlab/dorothea/blob/master/LICENSE" },
+            recommendation: { group: "regulatory-networks", recommended: true },
+            artifacts: [
+                {
+                    path: "dorothea_hs.rda",
+                    url: "https://raw.githubusercontent.com/saezlab/dorothea/master/data/dorothea_hs.rda",
+                    format: "rda",
+                    contents: "R serialized data frame `dorothea_hs` of TF-target regulons: columns tf, confidence (A-E), target, mor (+1 activating, -1 repressing). Load in R with load(); from Python read it through rpy2. Filter on confidence before use, then rename tf/target/mor to source/target/weight for a decoupler-style network.",
+                },
+            ],
+        },
+        {
+            id: "dorothea-mouse",
+            version: "current",
+            title: "DoRothEA mouse TF regulons",
+            description: "DoRothEA transcription-factor regulons for Mus musculus with per-interaction confidence levels A-E, where A is the best-supported. The confidence-filtered alternative to CollecTRI for TF-activity inference; most analyses keep levels A-C. Tracks the package's default branch, so it is verified against what you downloaded rather than a checked-in digest.",
+            organism: "mouse",
+            sourceUrl: "https://github.com/saezlab/dorothea",
+            license: { identifier: "GPL-3.0", url: "https://github.com/saezlab/dorothea/blob/master/LICENSE" },
+            recommendation: { group: "regulatory-networks", recommended: true },
+            artifacts: [
+                {
+                    path: "dorothea_mm.rda",
+                    url: "https://raw.githubusercontent.com/saezlab/dorothea/master/data/dorothea_mm.rda",
+                    format: "rda",
+                    contents: "R serialized data frame `dorothea_mm` of TF-target regulons: columns tf, confidence (A-E), target, mor (+1 activating, -1 repressing). Load in R with load(); from Python read it through rpy2. Filter on confidence before use, then rename tf/target/mor to source/target/weight for a decoupler-style network.",
                 },
             ],
         },
@@ -372,6 +538,7 @@ export const REFERENCE_DATA_CATALOG = deepFreeze(ReferenceDataCatalogSchema.pars
             version: "8",
             title: "GTEx v8 normal tissue expression",
             description: "GTEx v8 median gene TPM by tissue (GCT) with the sample attributes table, for normal-reference workflows. Immutable v8 release files.",
+            organism: "human",
             sourceUrl: "https://gtexportal.org/home/downloads/adult-gtex/bulk_tissue_expression",
             license: { identifier: "GTEx-Portal-Data-License", url: "https://gtexportal.org/home/license" },
             recommendation: { group: "normal-expression", recommended: true },
@@ -379,10 +546,14 @@ export const REFERENCE_DATA_CATALOG = deepFreeze(ReferenceDataCatalogSchema.pars
                 {
                     path: "GTEx_Analysis_2017-06-05_v8_RNASeQCv1.1.9_gene_median_tpm.gct.gz",
                     url: "https://storage.googleapis.com/adult-gtex/bulk-gex/v8/rna-seq/GTEx_Analysis_2017-06-05_v8_RNASeQCv1.1.9_gene_median_tpm.gct.gz",
+                    format: "gct",
+                    contents: "Gzipped GCT: skip 2 header lines, then columns Name (versioned Ensembl gene ID — strip the suffix to join) and Description (symbol), followed by one median-TPM column per tissue.",
                 },
                 {
                     path: "GTEx_Analysis_v8_Annotations_SampleAttributesDS.txt",
                     url: "https://storage.googleapis.com/adult-gtex/annotations/v8/metadata-files/GTEx_Analysis_v8_Annotations_SampleAttributesDS.txt",
+                    format: "tsv",
+                    contents: "Per-sample attributes; key columns SAMPID, SMTS (tissue) and SMTSD (detailed tissue). Only needed to interpret or regroup the tissue columns.",
                 },
             ],
         },
@@ -391,16 +562,25 @@ export const REFERENCE_DATA_CATALOG = deepFreeze(ReferenceDataCatalogSchema.pars
             version: "current",
             title: "Human Protein Atlas",
             description: "The full Human Protein Atlas table (proteinatlas.tsv.zip, ~7 MB): per-gene RNA/protein tissue expression, tissue specificity, subcellular location, secretome and blood-concentration annotations for Homo sapiens. HPA republishes this file per release, so it is verified against what you downloaded rather than a checked-in digest.",
+            organism: "human",
             sourceUrl: "https://www.proteinatlas.org/about/download",
             license: { identifier: "CC-BY-SA-3.0", url: "https://www.proteinatlas.org/about/licence" },
             recommendation: { group: "normal-expression", recommended: true },
-            artifacts: [{ path: "proteinatlas.tsv.zip", url: "https://www.proteinatlas.org/download/proteinatlas.tsv.zip" }],
+            artifacts: [
+                {
+                    path: "proteinatlas.tsv.zip",
+                    url: "https://www.proteinatlas.org/download/proteinatlas.tsv.zip",
+                    format: "zip",
+                    contents: "Zip holding a single proteinatlas.tsv — one row per gene with Ensembl and Gene columns plus RNA/protein tissue expression, tissue specificity, subcellular location, secretome and blood-concentration annotations. Read the member from the archive; do not expect a bare .tsv on disk.",
+                },
+            ],
         },
         {
             id: "celltypist-immune",
             version: "2",
             title: "CellTypist immune cell models",
             description: "CellTypist Immune All low- and high-resolution models for immune cell-type annotation. Load by absolute path — CellTypist's by-name lookup expects its own cache layout, which the reference store deliberately does not impersonate.",
+            organism: "human",
             sourceUrl: "https://www.celltypist.org/models",
             license: { identifier: "NOASSERTION" },
             recommendation: { group: "cell-typing", recommended: true },
@@ -408,10 +588,14 @@ export const REFERENCE_DATA_CATALOG = deepFreeze(ReferenceDataCatalogSchema.pars
                 {
                     path: "Immune_All_Low.pkl",
                     url: "https://celltypist.cog.sanger.ac.uk/models/Pan_Immune_CellTypist/v2/Immune_All_Low.pkl",
+                    format: "pickle",
+                    contents: "Trained CellTypist model, fine-grained immune labels. Load by absolute path through CellTypist's model loader; passing the bare name makes it search its own cache and fail.",
                 },
                 {
                     path: "Immune_All_High.pkl",
                     url: "https://celltypist.cog.sanger.ac.uk/models/Pan_Immune_CellTypist/v2/Immune_All_High.pkl",
+                    format: "pickle",
+                    contents: "Trained CellTypist model, coarse-grained immune labels. Load by absolute path through CellTypist's model loader; passing the bare name makes it search its own cache and fail.",
                 },
             ],
         },
@@ -420,6 +604,7 @@ export const REFERENCE_DATA_CATALOG = deepFreeze(ReferenceDataCatalogSchema.pars
             version: "2",
             title: "CellTypist pan-fetal human model",
             description: "CellTypist Pan_Fetal_Human model — cell-type annotation across human fetal tissues. Load by absolute path; CellTypist's by-name lookup expects its own cache layout, which the reference store deliberately does not impersonate.",
+            organism: "human",
             sourceUrl: "https://www.celltypist.org/models",
             license: { identifier: "NOASSERTION" },
             recommendation: { group: "cell-typing", recommended: true },
@@ -427,6 +612,8 @@ export const REFERENCE_DATA_CATALOG = deepFreeze(ReferenceDataCatalogSchema.pars
                 {
                     path: "Pan_Fetal_Human.pkl",
                     url: "https://celltypist.cog.sanger.ac.uk/models/Pan_Fetal_Suo/v2/Pan_Fetal_Human.pkl",
+                    format: "pickle",
+                    contents: "Trained CellTypist model covering human fetal tissues. Load by absolute path through CellTypist's model loader; passing the bare name makes it search its own cache and fail.",
                 },
             ],
         },
@@ -435,6 +622,7 @@ export const REFERENCE_DATA_CATALOG = deepFreeze(ReferenceDataCatalogSchema.pars
             version: "1",
             title: "CellTypist COVID-19 immune model",
             description: "CellTypist COVID19_Immune_Landscape model — immune cell-type annotation trained on the cross-study COVID-19 immune atlas. Load by absolute path; CellTypist's by-name lookup expects its own cache layout, which the reference store deliberately does not impersonate.",
+            organism: "human",
             sourceUrl: "https://www.celltypist.org/models",
             license: { identifier: "NOASSERTION" },
             recommendation: { group: "cell-typing", recommended: true },
@@ -442,6 +630,8 @@ export const REFERENCE_DATA_CATALOG = deepFreeze(ReferenceDataCatalogSchema.pars
                 {
                     path: "COVID19_Immune_Landscape.pkl",
                     url: "https://celltypist.cog.sanger.ac.uk/models/COVID19_Immune_Ren/v1/COVID19_Immune_Landscape.pkl",
+                    format: "pickle",
+                    contents: "Trained CellTypist model from the cross-study COVID-19 immune atlas. Load by absolute path through CellTypist's model loader; passing the bare name makes it search its own cache and fail.",
                 },
             ],
         },
@@ -457,6 +647,8 @@ export const REFERENCE_DATA_CATALOG = deepFreeze(ReferenceDataCatalogSchema.pars
                 {
                     path: "PanglaoDB_markers_27_Mar_2020.tsv.gz",
                     url: "https://panglaodb.se/markers/PanglaoDB_markers_27_Mar_2020.tsv.gz",
+                    format: "tsv",
+                    contents: "Gzipped marker table covering human and mouse together — filter the species column ('Hs', 'Mm', or both) before use. Key columns: species, official gene symbol, cell type, organ, ubiquitousness index.",
                 },
             ],
         },
@@ -465,6 +657,7 @@ export const REFERENCE_DATA_CATALOG = deepFreeze(ReferenceDataCatalogSchema.pars
             version: "1.0.0",
             title: "Azimuth human PBMC reference",
             description: "Azimuth annotated reference for human peripheral blood mononuclear cells: the Seurat reference object plus its Annoy nearest-neighbour index, for supervised mapping and cell-type label transfer. Immutable Zenodo release 4546839.",
+            organism: "human",
             sourceUrl: "https://zenodo.org/records/4546839",
             license: { identifier: "CC-BY-4.0", url: "https://zenodo.org/records/4546839" },
             recommendation: { group: "cell-typing", recommended: true },
@@ -472,10 +665,14 @@ export const REFERENCE_DATA_CATALOG = deepFreeze(ReferenceDataCatalogSchema.pars
                 {
                     path: "ref.Rds",
                     url: "https://zenodo.org/records/4546839/files/ref.Rds",
+                    format: "rds",
+                    contents: "Seurat reference object (PBMC) for Azimuth label transfer, read with readRDS. Requires idx.annoy in the same directory.",
                 },
                 {
                     path: "idx.annoy",
                     url: "https://zenodo.org/records/4546839/files/idx.annoy",
+                    format: "annoy",
+                    contents: "Annoy neighbour index paired with ref.Rds. Never opened directly — Azimuth loads it via the reference object, which needs both files side by side.",
                 },
             ],
         },
@@ -485,6 +682,7 @@ export const REFERENCE_DATA_CATALOG = deepFreeze(ReferenceDataCatalogSchema.pars
             version: "1.0.0",
             title: "Azimuth human tonsil reference",
             description: "Azimuth annotated reference for human tonsil: the Seurat reference object plus its Annoy nearest-neighbour index, for supervised mapping and cell-type label transfer. Immutable Zenodo release 7032928, roughly 443 MB.",
+            organism: "human",
             sourceUrl: "https://zenodo.org/records/7032928",
             license: { identifier: "CC-BY-4.0", url: "https://zenodo.org/records/7032928" },
             recommendation: { group: "cell-typing", recommended: false },
@@ -492,10 +690,14 @@ export const REFERENCE_DATA_CATALOG = deepFreeze(ReferenceDataCatalogSchema.pars
                 {
                     path: "ref.Rds",
                     url: "https://zenodo.org/records/7032928/files/ref.Rds",
+                    format: "rds",
+                    contents: "Seurat reference object (tonsil) for Azimuth label transfer, read with readRDS. Requires idx.annoy in the same directory.",
                 },
                 {
                     path: "idx.annoy",
                     url: "https://zenodo.org/records/7032928/files/idx.annoy",
+                    format: "annoy",
+                    contents: "Annoy neighbour index paired with ref.Rds. Never opened directly — Azimuth loads it via the reference object, which needs both files side by side.",
                 },
             ],
         },
@@ -505,6 +707,7 @@ export const REFERENCE_DATA_CATALOG = deepFreeze(ReferenceDataCatalogSchema.pars
             version: "GRCh38.p13",
             title: "PharmCAT GRCh38 reference FASTA",
             description: "The GRCh38.p13 reference genome FASTA that PharmCAT's VCF preprocessor expects (bgzip'd with a .fai/.gzi faidx), bundled as a tar — roughly 842 MB. Immutable Zenodo release 7288118. Needed only for pharmacogenomics (PharmCAT) workflows.",
+            organism: "human",
             sourceUrl: "https://zenodo.org/records/7288118",
             license: { identifier: "CC-BY-4.0", url: "https://zenodo.org/records/7288118" },
             recommendation: { group: "pharmacogenomics", recommended: false },
@@ -512,6 +715,8 @@ export const REFERENCE_DATA_CATALOG = deepFreeze(ReferenceDataCatalogSchema.pars
                 {
                     path: "PharmCAT_GRCh38_reference_fasta.tar",
                     url: "https://zenodo.org/records/7288118/files/GRCh38_reference_fasta.tar?download=1",
+                    format: "tar",
+                    contents: "Tar bundle of the GRCh38.p13 bgzip-compressed FASTA with its .fai and .gzi indexes. Extract to a writable directory first, then point the tool at the extracted .fna.bgz — the store is read-only, and nothing reads this tar in place.",
                 },
             ],
         },

package/dist/tasks/data-profile.d.ts

@@ -52,6 +52,18 @@ export interface DataProfileDeps {
     readonly embedding: EmbeddingProvider;
     /** Absolute path to the skills tree (one subdirectory per skill). */
     readonly skillsDir: string;
+    /**
+     * Host path of the library store's `packages.txt`. Omit when the host mounts the
+     * store at the sandbox's own path; a host whose store is baked into the image must
+     * inject its extracted copy, or the profiler reads the inventory as unknown.
+     */
+    readonly packagesFile?: string;
+    /**
+     * Host path of the reference store. Omit only when none is provisioned: the
+     * profiler carries reference discovery, and without this it reports an installed
+     * store as absent.
+     */
+    readonly refStorePath?: string;
 }
 /**
  * Workflow input. JSON-serialisable — DBOS persists it as the workflow's

package/dist/tasks/data-profile.js

@@ -216,6 +216,8 @@ export async function runDataProfileBody(input, deps) {
                 embedding: deps.embedding,
                 model: deps.model,
                 skillsDir: deps.skillsDir,
+                ...(deps.packagesFile ? { packagesFile: deps.packagesFile } : {}),
+                ...(deps.refStorePath ? { refStorePath: deps.refStorePath } : {}),
                 bioKeys: deps.bioKeys,
                 step: {
                     sandbox,

package/dist/tools/research/generate-plan.d.ts

@@ -41,6 +41,13 @@ export interface GeneratePlanDeps {
      * default guidance and validation skips the ceiling check.
      */
     readonly resourcePolicy?: ResourcePolicy;
+    /**
+     * Host path of the reference store, giving the planner reference discovery.
+     * Absent, the tool reports the store as unavailable and the planner keeps
+     * planning as though nothing is guaranteed present — which is what the prompt
+     * already tells it to do.
+     */
+    readonly refStorePath?: string;
 }
 /** Build the `generate_plan` tool bound to its provider and pool. */
 export declare function createGeneratePlanTool(deps: GeneratePlanDeps): Tool;