@inflexa-ai/harness 0.24.0 → 0.25.0

package/dist/input-scan/scan.js

@@ -0,0 +1,182 @@
+/**
+ * The input scan — one deterministic pass over a staged input tree.
+ *
+ * Runs in the harness process over the workspace read seam, which is
+ * sandbox-independent: the walk touches names and metadata, and format detection
+ * compares a bounded byte prefix against a table. Neither is a decode, so neither
+ * needs a container. Only the per-shape header readout (`enrich.ts`) runs a decoder,
+ * and it runs in the sandbox.
+ *
+ * The scan is the SOLE enumeration pass. Before it, the profiler agent discovered
+ * structure by issuing one command per input file, so its cost scaled with file
+ * count and its step budget did not.
+ */
+import { unwrapOrThrow } from "../lib/result.js";
+import { MAGIC_PREFIX_BYTES, detectFormat, extensionChain } from "./formats.js";
+import { observeShapes } from "./shapes.js";
+/**
+ * Files walked before the scan reports truncation instead of completing.
+ *
+ * Header decode is O(shapes), but the walk and its per-file prefix read are O(files),
+ * and that arm needs a stated ceiling rather than an implicit one. 100k files is ~30s
+ * of stat-and-prefix work against a 20-minute profile deadline, and two orders of
+ * magnitude above the largest tree the platform has seen (3513). Past it the scan
+ * stops and SAYS it stopped: a silently sampled manifest would describe a fraction of
+ * the tree while reading as a description of all of it, which is the failure mode this
+ * capability exists to remove.
+ */
+export const MAX_SCANNED_FILES = 100_000;
+/** Walk the tree breadth-first, reading a bounded prefix of each file for format detection. */
+async function walk(args) {
+    const { session, fs, root, signal } = args;
+    const limit = args.limit ?? MAX_SCANNED_FILES;
+    const files = [];
+    const queue = [root];
+    let truncated = false;
+    while (queue.length > 0) {
+        signal?.throwIfAborted();
+        const dir = queue.shift();
+        const listing = unwrapOrThrow(await fs.list({ session, path: dir }));
+        // A directory that vanished mid-walk, or one the seam refuses, is not a scan
+        // failure — the tree is being described, not validated.
+        if (listing.kind !== "ok")
+            continue;
+        for (const entry of listing.entries) {
+            const path = dir === "" ? entry.name : `${dir}/${entry.name}`;
+            if (entry.type === "directory") {
+                queue.push(path);
+                continue;
+            }
+            if (files.length >= limit) {
+                truncated = true;
+                return { files, truncated };
+            }
+            const extensions = extensionChain(entry.name);
+            const read = unwrapOrThrow(await fs.readBytes({ session, path, length: MAGIC_PREFIX_BYTES }));
+            const prefix = read.kind === "ok" ? read.bytes : null;
+            const { format, wrapper } = detectFormat({ path, extensions, prefix });
+            files.push({
+                path,
+                size: entry.size ?? 0,
+                extensions,
+                format,
+                ...(wrapper ? { wrapper } : {}),
+            });
+        }
+    }
+    return { files, truncated };
+}
+function formatCensus(files) {
+    const counts = new Map();
+    for (const file of files)
+        counts.set(file.format, (counts.get(file.format) ?? 0) + 1);
+    return [...counts.entries()].map(([format, count]) => ({ format, count })).sort((a, b) => b.count - a.count || a.format.localeCompare(b.format, "en"));
+}
+/** Assemble a manifest from an already-walked file set. Pure — the seam of the unit tests. */
+export function buildManifest(root, files, truncated, limit = MAX_SCANNED_FILES) {
+    const observed = observeShapes(files);
+    const manifest = {
+        root,
+        fileCount: files.length,
+        totalBytes: files.reduce((sum, f) => sum + f.size, 0),
+        truncated,
+        ...(truncated ? { scanLimit: limit } : {}),
+        shapesTruncated: observed.shapesTruncated,
+        formats: formatCensus(files),
+        shapes: observed.shapes,
+        valueOverlaps: observed.valueOverlaps,
+        unstructured: observed.unstructured,
+    };
+    return { files, manifest, positionValues: observed.positionValues };
+}
+/** Walk a staged input tree and observe its shapes. No model, no container. */
+export async function scanInputTree(args) {
+    const limit = args.limit ?? MAX_SCANNED_FILES;
+    const { files, truncated } = await walk(args);
+    return buildManifest(args.root, files, truncated, limit);
+}
+function formatBytes(bytes) {
+    if (bytes < 1024)
+        return `${bytes} B`;
+    const units = ["KB", "MB", "GB", "TB"];
+    let value = bytes / 1024;
+    let unit = 0;
+    while (value >= 1024 && unit < units.length - 1) {
+        value /= 1024;
+        unit++;
+    }
+    return `${value < 10 ? value.toFixed(1) : Math.round(value)} ${units[unit]}`;
+}
+/**
+ * Render a manifest for a briefing or a tool result.
+ *
+ * Bounded by construction — every list it draws from is already capped — so a tree of
+ * three thousand files costs the same context as a tree of thirty. The wording states
+ * observations only: it never calls a shape a kind, and it says outright that the
+ * grouping decision is the reader's.
+ */
+export function renderInputScanManifest(manifest) {
+    const lines = [];
+    lines.push(`Input scan of ${manifest.root} — ${manifest.fileCount} files, ${formatBytes(manifest.totalBytes)}.`);
+    if (manifest.truncated) {
+        lines.push(`INCOMPLETE: the walk stopped at its ${manifest.scanLimit ?? MAX_SCANNED_FILES}-file ceiling, so this describes part of the tree.`);
+    }
+    lines.push(`Formats: ${manifest.formats.map((f) => `${f.format} (${f.count})`).join(", ") || "none"}`);
+    lines.push("");
+    lines.push("These are OBSERVATIONS of filename structure, not a grouping of the dataset.");
+    lines.push("A shape is a set of files whose names differ only at the marked positions; deciding");
+    lines.push("what is one kind of data, and what the varying positions mean, is your judgement.");
+    if (manifest.shapes.length > 0) {
+        lines.push("");
+        lines.push(`Shapes (${manifest.shapes.length}${manifest.shapesTruncated ? ", capped — the remainder is in the aggregate below" : ""}):`);
+        for (const shape of manifest.shapes) {
+            const location = shape.directory ? `${shape.directory}/` : "";
+            const wrapper = shape.wrapper ? `, ${shape.wrapper}-compressed` : "";
+            lines.push(`- ${shape.id}: ${shape.fileCount} files at ${location}${shape.pattern} (${shape.format}${wrapper}, ${formatBytes(shape.totalBytes)})`);
+            for (const position of shape.variablePositions) {
+                const prefix = position.prefix ? ` after "${position.prefix}"` : "";
+                const shown = position.sampleValues.join(", ");
+                const more = position.distinctValues > position.sampleValues.length ? ", …" : "";
+                lines.push(`  - position <${position.index}>${prefix}: ${position.distinctValues} distinct values — ${shown}${more}`);
+            }
+            for (const pair of shape.cooccurrence) {
+                const crossing = pair.observedPairs === pair.possiblePairs ? "fully crossed" : `partially crossed (${pair.possiblePairs} possible)`;
+                lines.push(`  - positions <${pair.positions[0]}>×<${pair.positions[1]}>: ${pair.observedPairs} observed combinations, ${crossing}`);
+            }
+            if (shape.header) {
+                const fields = Object.entries(shape.header.fields)
+                    .map(([key, value]) => `${key}=${value}`)
+                    .join(", ");
+                lines.push(`  - header (${shape.header.path}): ${fields || shape.header.unavailable || "no fields read"}`);
+            }
+            else {
+                lines.push(`  - example: ${shape.examplePaths[0] ?? "—"}`);
+            }
+        }
+    }
+    if (manifest.valueOverlaps.length > 0) {
+        lines.push("");
+        lines.push("Value overlap between shapes (evidence, NOT an assertion that they share an axis):");
+        for (const overlap of manifest.valueOverlaps) {
+            const gaps = [];
+            if (overlap.onlyInFirst > 0)
+                gaps.push(`${overlap.onlyInFirst} only in ${overlap.shapes[0]} (${overlap.onlyInFirstSample.join(", ")})`);
+            if (overlap.onlyInSecond > 0)
+                gaps.push(`${overlap.onlyInSecond} only in ${overlap.shapes[1]} (${overlap.onlyInSecondSample.join(", ")})`);
+            const gapText = gaps.length > 0 ? `; ${gaps.join("; ")}` : "; no gaps";
+            lines.push(`- ${overlap.shapes[0]}<${overlap.positions[0]}> vs ${overlap.shapes[1]}<${overlap.positions[1]}>: ` +
+                `${overlap.sharedValues} shared values${gapText}`);
+        }
+    }
+    if (manifest.unstructured.count > 0) {
+        lines.push("");
+        lines.push(`Files sharing no name structure with any other (${manifest.unstructured.count}, ${formatBytes(manifest.unstructured.totalBytes)}):`);
+        for (const entry of manifest.unstructured.sample) {
+            lines.push(`- ${entry.path} (${entry.format}, ${formatBytes(entry.size)})`);
+        }
+        if (manifest.unstructured.count > manifest.unstructured.sample.length) {
+            lines.push(`- … ${manifest.unstructured.count - manifest.unstructured.sample.length} more not listed`);
+        }
+    }
+    return lines.join("\n");
+}

package/dist/input-scan/shapes.d.ts

@@ -0,0 +1,46 @@
+/**
+ * Shape observation — pure functions over filenames and sizes, no I/O.
+ *
+ * A shape is a mechanical fact: these filenames differ only here. Establishing it
+ * needs nothing but names, sizes, and the format already detected, which is why
+ * shapes are observed BEFORE any decode and why header enrichment costs O(shapes)
+ * rather than O(files).
+ *
+ * Grouping proceeds in two passes. The first masks digit runs, so `PATIENT_001` and
+ * `PATIENT_002` land together while `tumor_001` and `normal_001` do not. The second
+ * merges groups that differ in exactly one alphabetic position, which is what makes
+ * `tumor`/`normal` one shape with a categorical position rather than two shapes an
+ * agent has to notice are related. A file whose structure matches no other survives
+ * both passes alone and is reported in the aggregate, never as a shape of one.
+ */
+import type { FileShape, ScannedFile, ShapeValueOverlap, UnstructuredFiles } from "./types.js";
+/** Shapes reported before the tail is folded into the aggregate. */
+export declare const MAX_SHAPES = 30;
+/** Values sampled per variable position. The sample is the agent's decision material. */
+export declare const MAX_SAMPLE_VALUES = 20;
+type AtomKind = "alpha" | "digit" | "delim";
+interface Atom {
+    readonly kind: AtomKind;
+    readonly value: string;
+}
+/**
+ * Split a filename stem into runs of letters, runs of digits, and the delimiters
+ * between them. Splitting INSIDE a token is what makes `PT001_D0_rep1` three
+ * positions rather than three opaque tokens — the nested design is only visible
+ * once `PT` and `001` are separate.
+ */
+export declare function tokenizeStem(stem: string): readonly Atom[];
+/** The name stem — the basename with its extension chain removed. */
+export declare function nameStem(fileName: string): string;
+export interface ObservedShapes {
+    readonly shapes: readonly FileShape[];
+    readonly unstructured: UnstructuredFiles;
+    readonly valueOverlaps: readonly ShapeValueOverlap[];
+    /** Complete per-position value sets, keyed by shape id — the manifest carries only a sample. */
+    readonly positionValues: ReadonlyMap<string, readonly (readonly string[])[]>;
+    /** True when the shape list was capped and the remainder folded into the aggregate. */
+    readonly shapesTruncated: boolean;
+}
+/** Observe the shapes of a scanned file set. Pure — no I/O, no container, no model. */
+export declare function observeShapes(files: readonly ScannedFile[]): ObservedShapes;
+export {};

package/dist/input-scan/types.d.ts

@@ -0,0 +1,172 @@
+/**
+ * Input-scan vocabulary.
+ *
+ * The scan reports what is OBSERVABLE about a staged input tree; it never decides
+ * how the tree is grouped (see the input-scan-manifest spec). That separation is
+ * carried by the vocabulary itself: a shape is "these filenames differ only here",
+ * a kind is "these files are the same sort of thing", and only the profiler agent
+ * can establish the second. Nothing here is named `kinds` or `axes`, and no type
+ * here is assignable to the profiler's output — a manifest an agent could copy
+ * into its submission would make its central judgement invisible.
+ */
+/** One file the walk found. `path` is relative to the analysis root. */
+export interface ScannedFile {
+    readonly path: string;
+    readonly size: number;
+    /** Trailing extension components, outermost last: `sample.vcf.gz` → `["vcf", "gz"]`. */
+    readonly extensions: readonly string[];
+    /** Format identified from magic bytes, falling back to the extension chain, else `"unknown"`. */
+    readonly format: string;
+    /** Compression wrapper reported ALONGSIDE the inner format, never in place of it. */
+    readonly wrapper?: string;
+}
+/**
+ * A filename position whose token differs across a shape's members.
+ *
+ * `sampleValues` is normative, not decoration: a count alone cannot distinguish an
+ * identifier (`0001`…`1171`) from a categorical label (`tumor`/`normal`) from a shard
+ * index (`chr1`…`chr22`), and those readings imply entirely different kinds.
+ */
+export interface VariablePosition {
+    /** Token index within the shape's name pattern — the `<n>` placeholder it fills. */
+    readonly index: number;
+    /** Literal text immediately preceding the token, when any (`PT` in `PT001`). */
+    readonly prefix?: string;
+    readonly distinctValues: number;
+    /** Bounded sample of the values observed at this position. */
+    readonly sampleValues: readonly string[];
+}
+/**
+ * How two variable positions vary together. A fully-crossed pair is a nested design
+ * (every subject × every timepoint); a partially-crossed one is not, and the
+ * difference is design evidence a flat file count destroys.
+ */
+export interface PositionCooccurrence {
+    readonly positions: readonly [number, number];
+    /** Distinct value pairs observed. */
+    readonly observedPairs: number;
+    /** Pairs the two value sets could form. `observedPairs === possiblePairs` is a full crossing. */
+    readonly possiblePairs: number;
+}
+/** Header fields read from a bounded prefix of one member of a shape. */
+export interface HeaderReadout {
+    /** The member the readout came from. */
+    readonly path: string;
+    /** Decoder-reported fields, flat and bounded (`delimiter`, `columns`, `sampleCount`, …). */
+    readonly fields: Readonly<Record<string, string | number | boolean>>;
+    /** Why no readout is present, when the decode did not produce one. */
+    readonly unavailable?: string;
+}
+/**
+ * A set of files that are mechanically indistinguishable by name structure, format,
+ * and location. A shape is an observation, never a grouping of the dataset: the
+ * agent may split one shape into several kinds, cover several shapes with one kind,
+ * or ignore the shape entirely.
+ */
+export interface FileShape {
+    /** Stable within one manifest (`shape-1`, `shape-2`, …) so overlaps can name shapes. */
+    readonly id: string;
+    /** Directory the members share, relative to the analysis root. */
+    readonly directory: string;
+    /** Name pattern with each variable position rendered as `<n>`. */
+    readonly pattern: string;
+    readonly format: string;
+    readonly wrapper?: string;
+    readonly extensions: readonly string[];
+    readonly fileCount: number;
+    readonly totalBytes: number;
+    readonly variablePositions: readonly VariablePosition[];
+    readonly cooccurrence: readonly PositionCooccurrence[];
+    /** Bounded sample of member paths. */
+    readonly examplePaths: readonly string[];
+    /** Present once sandbox enrichment has run for this shape. */
+    readonly header?: HeaderReadout;
+}
+/**
+ * Value-set overlap between two shapes' variable positions, reported WITH its gaps.
+ *
+ * Correspondence across shapes is heuristic — it depends on stripping shape-specific
+ * text before comparing, and near-miss namings defeat it. Reported as evidence it is
+ * useful; asserted as a shared axis it is a guess a consumer cannot tell was guessed.
+ */
+export interface ShapeValueOverlap {
+    readonly shapes: readonly [string, string];
+    readonly positions: readonly [number, number];
+    readonly sharedValues: number;
+    /** Values present in the first shape and absent from the second. */
+    readonly onlyInFirst: number;
+    readonly onlyInSecond: number;
+    /** Bounded samples of the two gaps, so a near-complete correspondence names its misses. */
+    readonly onlyInFirstSample: readonly string[];
+    readonly onlyInSecondSample: readonly string[];
+}
+/** One member of the no-shared-structure aggregate. */
+export interface UnstructuredEntry {
+    readonly path: string;
+    readonly size: number;
+    readonly format: string;
+}
+/**
+ * Files whose name structure matches no other file, in aggregate.
+ *
+ * Never one shape per file: a tree of arbitrarily named files would otherwise produce
+ * a shape count proportional to the file count, which is the unbounded output this
+ * capability exists to prevent. Whether these are notable singletons worth prose or
+ * an unclassified remainder is the agent's determination.
+ */
+export interface UnstructuredFiles {
+    readonly count: number;
+    readonly totalBytes: number;
+    /**
+     * Bounded sample covering distinct formats first, then the largest files. A
+     * size-ordered sample would bury the metadata sheet, the README, and the paper —
+     * one file each — under whatever large files failed to group.
+     */
+    readonly sample: readonly UnstructuredEntry[];
+}
+/** How many files carry each detected format, across the whole scanned tree. */
+export interface FormatCount {
+    readonly format: string;
+    readonly count: number;
+}
+/**
+ * The agent-facing manifest: bounded by construction, so injecting it into a briefing
+ * or returning it from `scan_inputs` costs a fixed amount of context whatever the tree
+ * size. The complete per-file list lives on {@link InputScan}, host-side.
+ */
+export interface InputScanManifest {
+    /** Path scanned, relative to the analysis root. */
+    readonly root: string;
+    readonly fileCount: number;
+    readonly totalBytes: number;
+    /**
+     * True when the walk hit its file ceiling and stopped. The shapes below then
+     * describe a prefix of the tree, not all of it — stated rather than silently
+     * sampled.
+     */
+    readonly truncated: boolean;
+    /** The ceiling that bound, when `truncated`. */
+    readonly scanLimit?: number;
+    /** True when the shape list itself was capped; the remainder is folded into `unstructured`. */
+    readonly shapesTruncated: boolean;
+    readonly formats: readonly FormatCount[];
+    readonly shapes: readonly FileShape[];
+    readonly valueOverlaps: readonly ShapeValueOverlap[];
+    readonly unstructured: UnstructuredFiles;
+}
+/**
+ * The scan's full result. `files` is the complete enumeration — it stays host-side and
+ * backs coverage and the index projection; `manifest` is the bounded projection the
+ * agent reads.
+ */
+export interface InputScan {
+    readonly files: readonly ScannedFile[];
+    readonly manifest: InputScanManifest;
+    /**
+     * Per shape, the COMPLETE value set of each variable position, keyed by shape id.
+     * Bounded output belongs to the manifest; the entity tier of the index needs every
+     * value, and re-deriving them from filenames downstream would duplicate the
+     * tokenisation.
+     */
+    readonly positionValues: ReadonlyMap<string, readonly (readonly string[])[]>;
+}

package/dist/input-scan/types.js

@@ -0,0 +1,12 @@
+/**
+ * Input-scan vocabulary.
+ *
+ * The scan reports what is OBSERVABLE about a staged input tree; it never decides
+ * how the tree is grouped (see the input-scan-manifest spec). That separation is
+ * carried by the vocabulary itself: a shape is "these filenames differ only here",
+ * a kind is "these files are the same sort of thing", and only the profiler agent
+ * can establish the second. Nothing here is named `kinds` or `axes`, and no type
+ * here is assignable to the profiler's output — a manifest an agent could copy
+ * into its submission would make its central judgement invisible.
+ */
+export {};

package/dist/prompts/sandbox/data-profiler.d.ts

@@ -1 +1 @@
-export declare const dataProfilerPrompt = "# Data Profiler Agent\n\nYou are a scientific data profiling and planning specialist. You thoroughly\ncharacterize datasets \u2014 structure, quality, experimental design, and analytical\npotential \u2014 so that downstream orchestration can plan a complete analysis.\n\nYour outputs guide automated analysis execution, so precision and completeness\nare essential. Be explicit about what the data CAN and CANNOT support.\n\nData files (count matrices, expression tables, large CSVs, genomic files) must be\nprocessed programmatically. Do not `read_file` data files \u2014 they will exceed\nyour context window. Preview structure with `head`/`wc -l` via\n`execute_command`, then write Python scripts for comprehensive profiling. R\nis available if a method has no adequate Python equivalent, but default to\nPython.\n\nSmall metadata or config files are fine to `read_file` directly. Paper\nPDFs, READMEs, and Word documents (see \"Document inputs\" below) need a\nparser \u2014 `pypdf` and `python-docx` are available.\n\nYour briefing lists this analysis's input files. Unlike every other sandbox\nagent, it carries no data orientation \u2014 you are the agent that produces one, so\nderiving the dataset's facts from the files themselves is your job here, not a\nre-derivation of work already done. To see the tree beyond the briefing's list,\ncall `list_files` with `path: \"data/inputs\"` \u2014 `path` is its only parameter.\nTo go deeper, call it again on a subdirectory it returned.\n\n## Stage 1: Orient \u2014 Subject, Source, Design\n\nBEFORE running format-specific profiling on individual data files, look\nacross ALL inputs to identify the experimental subject, the data source,\nand the high-level design. Without these, downstream planning cannot\nchoose the right tools (per-organism gene mappings, species-appropriate\nreferences, design-appropriate statistics).\n\nWhat to identify:\n\n1. **Organism + NCBI taxon ID** \u2014 REQUIRED for every dataset. Fill the\n   `organism` schema field with `{scientificName, taxonId, source,\n   confidence}`. Use `null` ONLY when no input identifies the organism;\n   never guess from gene-symbol patterns alone (HGNC symbols are widely\n   shared with orthologs and do NOT prove human).\n2. **Tissue / cell type / condition** \u2014 when applicable. Fill the\n   matching schema fields.\n3. **Public accessions** \u2014 GEO (`GSE`/`GSM`), SRA\n   (`SRP`/`SRR`/`SRX`), BioProject (`PRJNA`/`PRJEB`/`PRJDB`),\n   ArrayExpress (`E-MTAB-xxxx`), dbGaP (`phs`), EGA (`EGAS`/`EGAD`).\n   Collect into the `accessions` array.\n4. **High-level experimental design** \u2014 case/control, dose-response,\n   time-course, paired/longitudinal, group sizes. Goes into\n   `experimentalDesign`.\n\nWhere to look (in this order):\n\n- **Sample-sheet / metadata files** (`metadata.csv`, `samplesheet.tsv`,\n  `*_meta.csv`) \u2014 `read_file` for small (<1 MB) tabular files. Look for\n  columns named `organism`, `taxon`, `taxon_id`, `species`, `tissue`,\n  `cell_type`, `condition`, `disease`, `arm`, `treatment`,\n  `timepoint`. The first few data rows usually answer the subject\n  question outright.\n- **Document inputs** \u2014 paper PDFs, READMEs, methods documents,\n  manifests. See \"Document inputs\" under Format-Specific Profiling. The\n  abstract and methods section of a paper typically state organism,\n  tissue, and design in one paragraph.\n- **Filenames and folder structure** \u2014 accession prefixes\n  (`GSE...`, `SRR...`, `PRJNA...`), organism shorthand\n  (`human_`, `hg38`, `mm10`, `macaque_`, `cyno_`).\n- **Reference store** \u2014 `list_available_refs` labels catalogued files\n  with the organism they describe. Query it for a candidate organism as\n  corroboration only: a hit means reference data for that species is\n  staged here, not that your data is that species, and an empty store\n  is a normal state that proves nothing either way.\n\nHow to record what you find:\n\n- Set `source` honestly to the most-direct evidence\n  (`user-context` > `metadata` > `document` > `filename` >\n  `inferred`). `user-context` ranks highest because an explicit\n  user statement is direct evidence, not inference.\n- Set `confidence`: `high` for an explicit user statement, an\n  organism column, or a paper statement; `medium` for filename or\n  accession-prefix signals; `low` for inference from data content\n  alone.\n- If sources DISAGREE (paper says human, metadata column says mouse),\n  pick the most-trusted source, set `confidence` to `low`, and add the\n  conflict to `qualityAssessment.concerns`. Do not silently pick one.\n\nAfter Stage 1, run the format-specific profiling below for each data\nfile.\n\n## Format-Specific Profiling\n\nAfter identifying a file's format, apply the appropriate checks below.\nNot every check applies to every dataset \u2014 use judgement.\n\n### Count Matrices (`.counts`, `.csv`, `.tsv` with integer counts)\n- Library size distribution across samples\n- Gene detection rate (genes detected per sample)\n- Zero-inflation level (fraction of zeros in the matrix)\n- Count distribution (log-scale; check for expected dynamic range)\n- Replicate correlation (Pearson/Spearman between biological replicates)\n- Outlier sample detection (PCA, total counts far from median)\n\n### Single-Cell (`.mtx` + barcodes/features, `.h5ad`, `.loom`)\n- Cell and gene counts\n- Sparsity level\n- UMI count distribution and knee plot characteristics\n- Gene detection per cell\n- Mitochondrial gene fraction\n- Doublet-indicative metrics (unusually high UMI/gene counts)\n- Metadata completeness (cell annotations, batch labels, embeddings)\n- Layer availability (raw counts vs. normalized)\n\n### Normalized Expression (`.tpm`, `.fpkm`, normalized `.csv`/`.tsv`)\n- Expression distribution shape\n- Batch effects between sample groups (PCA separation by batch)\n- Coefficient of variation across replicates\n- Dynamic range assessment\n- Highly expressed gene concentration (top-N gene share of total)\n\n### Variants (`.vcf`, `.bcf`)\n- Variant count by type (SNP, indel, structural)\n- Quality score distribution\n- Transition/transversion ratio (Ti/Tv; exome ~2.8\u20133.0, genome ~2.0\u20132.1)\n- Allele frequency spectrum\n- Missing genotype rate per sample\n- Heterozygosity rates\n\n### Alignments (`.bam`, `.cram`)\n- Mapping rate and quality distribution\n- Coverage depth and uniformity\n- Insert size distribution (paired-end)\n- Duplicate rate\n- Strand bias\n\n### Sequence Data (`.fastq`, `.fasta`)\n- Read/sequence count and length distribution\n- GC content and bias\n- Quality score distribution (FASTQ: per-base and per-read)\n- N-base / ambiguous base content\n\n### Annotations (`.gff`, `.gtf`, `.bed`)\n- Feature type distribution and counts\n- Chromosome/contig coverage\n- Gene model statistics (exons per gene, transcript isoforms)\n- Attribute completeness\n\n### Chemical Structure Data (`.sdf`, `.mol`, CSV/TSV with SMILES)\nFor SDF/MOL files:\n- Molecule count and property fields present in the data block\n- 3D coordinate presence (2D vs 3D structures)\n- Structure validity: parse with RDKit `Chem.MolFromMolFile()` or\n  `Chem.SDMolSupplier()`, report percentage of valid molecules\n- Molecular weight distribution (median, min, max)\n- Heavy atom count range\n\nFor CSV/TSV with SMILES columns:\n- **Detection by name**: column named `smiles`, `canonical_smiles`, `SMILES`,\n  `Canonical_SMILES`, `mol`, `structure`, or `compound_smiles`\n- **Detection by content**: if no named column found, check string columns\n  where >80% of non-empty values parse as valid molecules via\n  `Chem.MolFromSmiles()`. Flag confidence as heuristic-based.\n- SMILES validity rate (percentage that parse successfully)\n\nActivity data detection:\n- Columns matching activity patterns: `IC50`, `EC50`, `Ki`, `Kd`, `pIC50`,\n  `% inhibition`, `standard_value`, `standard_type`\n- Report activity type(s) and value range (min, max, median in nM)\n\nChemical profiling metrics to report in the file's `metrics` map. Every\nvalue there must be a flat scalar (string, number, or boolean) \u2014 the schema\nrejects arrays and nested objects, so record a range as two scalar keys, not\nas a pair:\n- `molecule_count`, `valid_smiles_pct`, `mw_median`, `mw_min`, `mw_max`\n- `has_activity_data`, `activity_type`, `activity_min_nM`, `activity_max_nM`\n- `scaffold_count` (Murcko generic scaffolds via RDKit)\n- `pains_hit_pct` (PAINS filter hit rate via RDKit FilterCatalog)\n\nSet `domain: \"cheminformatics\"` when molecular structures are the primary\ndata type. Subtypes: `\"compound-screening\"`, `\"structure-activity\"`,\n`\"compound-library\"`, `\"molecular-properties\"`.\n\n### Clinical Trial Data (SDTM/ADaM/General Clinical)\n\nClinical trial datasets often follow CDISC standards. Detect and profile:\n\n**SDTM domain detection** \u2014 check column names for standard domain prefixes:\n- DM (Demographics): STUDYID, USUBJID, AGE, SEX, RACE, ARM, ACTARM\n- LB (Laboratory): LBTESTCD, LBORRES, LBORNRLO, LBORNRHI, LBSTRESU\n- AE (Adverse Events): AETERM, AESEV, AESER, AEREL, AESTDTC, AEENDTC\n- VS (Vital Signs): VSTESTCD, VSORRES, VSORRESU\n- EC/EX (Exposure): ECDOSE, ECROUTE, ECFREQ, ECDOSFRM\n- RS (Response): RSEVAL, RSCAT (RECIST), RSORRES\n- PC (Pharmacoconcentrations): PCTEST, PCORRES, PCSTRESU, PCTPTNUM\n\n**ADaM detection** \u2014 check for analysis-ready indicators:\n- ADSL: one-row-per-subject (SUBJID, TRT01P, TRT01A, SAFFL, ITTFL)\n- ADAE: one-row-per-AE with analysis flags\n- ADLB: one-row-per-lab-assessment with AVAL, BASE, CHG\n- ADTTE: time-to-event (AVAL=time, CNSR=censoring, PARAMCD=event)\n- ADRS: tumor response (PARAMCD=OVRLRESP, AVALC=CR/PR/SD/PD)\n\n**General clinical (non-CDISC):**\n- Patient ID columns (check for consistent IDs across files)\n- Treatment arm columns (randomization groups)\n- Timepoint/visit structure (baseline, on-treatment, follow-up)\n- Response classifications (if present)\n- Censoring indicators (for survival/time-to-event data)\n\n**Profile metrics for clinical data:**\n- Subject count per treatment arm\n- Visit completeness (% subjects with data at each visit)\n- Missing data rate per variable\n- Date range (first enrollment to last follow-up)\n- Adverse event severity distribution (if AE data)\n- Lab value ranges vs normal reference ranges (if LB data)\n- Response rate per arm (if response data)\n\n**Quality flags:**\n- Non-standard variable names when CDISC structure is otherwise present\n- Missing randomization/treatment arm information\n- Inconsistent subject IDs across data files\n- Missing baseline values for change-from-baseline analyses\n- Censoring indicator present but not documented\n\n### General Tabular Data\n- Row and column counts, data types per column\n- Missing value patterns and frequency\n- Outlier detection for numeric columns\n- Duplicate row detection\n- Encoding issues (mixed types in columns, non-UTF8 characters)\n\n### Document Inputs (`.pdf`, `.docx`, `README`, `.md`, `.txt`)\n\nDocuments commonly accompany datasets \u2014 published papers describing the\nexperiment, README files explaining the file layout, or methods\ndocuments. They are PRIMARY sources for the Stage 1 orient pass and must\nnot be skipped.\n\n- **PDFs**: parse with `pypdf` (`from pypdf import PdfReader`).\n  Extract text, then search for organism / species / taxon mentions,\n  accession patterns, and the methods/experimental-design paragraph.\n  Long papers \u2014 read the abstract and methods section, not the entire\n  body.\n- **Word documents**: `python-docx` (`from docx import Document`).\n- **READMEs / Markdown / plain text**: `read_file` directly.\n- For each document, register it in `files` with\n  `dataType: \"document\"` and a one-sentence `description` of what\n  context it provides. Do NOT report row/col counts for documents.\n\n## Quality Signals to Always Report\n\nRegardless of format, flag these when present:\n- **Batch structure** \u2014 are there batch labels? Do samples cluster by batch\n  in PCA rather than by biological condition?\n- **Sample imbalance** \u2014 uneven group sizes that limit statistical power\n- **Missing or malformed metadata** \u2014 sample annotations that are absent,\n  incomplete, or inconsistent with the data matrix\n- **Unexpected zeros or NAs** \u2014 patterns suggesting technical dropout vs.\n  true biological absence\n- **Normalization state** \u2014 is the data raw counts, normalized, or log-\n  transformed? Misidentifying this derails every downstream step.\n\n## Single-Cell Extended Profiling\n\n- For snRNA-seq data: also profile ribosomal gene fraction (pct_ribo), intronic read content, and ambient RNA contamination metrics.\n\n## Scripting Standards\n\n- Set random seeds for reproducibility in any sampling or statistical operations.\n\n## Submitting Results\n\nWhen profiling is complete, call the `submit_profile` tool with your\nstructured findings. This is the ONLY way to deliver results \u2014 do not\nreturn JSON in your message text. The tool validates your output against\nthe expected schema; if validation fails you will see the errors and can\nfix and re-submit.\n\nCall `submit_profile` exactly once, after all profiling work is done.\nInclude metadata for every input file.\n\n## Do NOT\n\n- Use `print()` for status messages \u2014 use the `logging` module.\n- Skip profiling of data quality indicators (missing values, outliers, distributions).\n- Skip Stage 1 orientation. Even if every input looks like a data file,\n  check filenames, headers, and any embedded metadata for organism /\n  accession signals.\n- Guess the organism. If no input identifies it, set `organism: null`\n  and explain in `analysisSummary`. Inferring \"human\" from gene symbols\n  alone is wrong \u2014 orthologs share symbols across species.\n- Skip document inputs (PDFs, READMEs, DOCX). They carry the subject and\n  design context and are the primary source for Stage 1.\n- Return profiling results as JSON in your message text. Always use the\n  `submit_profile` tool.\n";
+export declare const dataProfilerPrompt = "# Data Profiler Agent\n\nYou produce this dataset's **orientation record**: what the data IS, so that planning\ncan proceed. You do NOT perform quality control \u2014 that belongs to the analysis steps,\nwhich run on the data with a question in hand.\n\nThe rule that decides whether a check earns its place: it answers *what is this*, not\n*is this good*. Whether a matrix holds raw or normalised counts is identity, and you\nkeep it. The zero-inflation level of that matrix is a verdict on quality, and you do not.\n\nYour outputs guide automated analysis planning, so be explicit about what the data CAN\nand CANNOT support.\n\n## Orientation: the input scan\n\nYour briefing carries a deterministic scan of the input tree, produced before your first\nturn. It reports the tree's formats, the SHAPES its filenames form \u2014 sets of files whose\nnames differ only at marked positions \u2014 the distinct values each varying position takes,\nhow those positions co-occur, value overlap between shapes, and the files that share\nstructure with nothing else.\n\nRead it first. It is the orientation pass, so listing the tree yourself only rediscovers\nwhat you were handed.\n\nThe scan reports observations. It does not decide what the dataset is made of \u2014 that is\nyour judgement, and the next section is where you make it.\n\nFor exploration beyond the manifest:\n\n- `scan_inputs` with a path re-scans a subtree, when you want to look at one directory\n  more closely or check a grouping you are unsure about.\n- `list_files` with `path: \"data/inputs\"` lists the tree \u2014 `path` is its only\n  parameter, so recurse by calling it again on a subdirectory it returned.\n\nData files (count matrices, expression tables, large CSVs, genomic files) must be\nprocessed programmatically. Do not `read_file` data files \u2014 they will exceed your\ncontext window. Preview structure with `head`/`wc -l` via `execute_command`, or write\na Python script. R is available if a method has no adequate Python equivalent, but\ndefault to Python.\n\nSmall metadata or config files are fine to `read_file` directly. Paper PDFs, READMEs,\nand Word documents need a parser \u2014 `pypdf` and `python-docx` are available.\n\n## Stage 1: Identity \u2014 subject, source, design\n\nIdentify the experimental subject, the data source, and the high-level design. Without\nthese, downstream planning cannot choose the right tools (per-organism gene mappings,\nspecies-appropriate references, design-appropriate statistics).\n\nWhat to identify:\n\n1. **Organism + NCBI taxon ID** \u2014 REQUIRED for every dataset. Fill the `organism`\n   field with `{scientificName, taxonId, source, confidence}`. Use `null` ONLY when\n   no input identifies the organism; never guess from gene-symbol patterns alone (HGNC\n   symbols are widely shared with orthologs and do NOT prove human).\n2. **Tissue / cell type / condition** \u2014 when applicable. Fill the matching fields.\n3. **Public accessions** \u2014 GEO (`GSE`/`GSM`), SRA (`SRP`/`SRR`/`SRX`),\n   BioProject (`PRJNA`/`PRJEB`/`PRJDB`), ArrayExpress (`E-MTAB-xxxx`), dbGaP\n   (`phs`), EGA (`EGAS`/`EGAD`). Collect into `accessions`.\n4. **High-level experimental design** \u2014 case/control, dose-response, time-course,\n   paired/longitudinal, group sizes. Goes into `experimentalDesign`.\n\nThe scan's detected axes and their cardinalities are design evidence: 1171 \u00D7 3 \u00D7 2 across\nthree varying positions is a longitudinal design with replicates, stated in the filenames.\nRead them before you conclude the design is undocumented.\n\nWhere else to look (in this order):\n\n- **Sample-sheet / metadata files** (`metadata.csv`, `samplesheet.tsv`,\n  `*_meta.csv`) \u2014 `read_file` for small (<1 MB) tabular files. Look for columns named\n  `organism`, `taxon`, `taxon_id`, `species`, `tissue`, `cell_type`,\n  `condition`, `disease`, `arm`, `treatment`, `timepoint`. The first few data\n  rows usually answer the subject question outright.\n- **Document inputs** \u2014 paper PDFs, READMEs, methods documents. The abstract and methods\n  section of a paper typically state organism, tissue, and design in one paragraph.\n  Parse PDFs with `pypdf` (`from pypdf import PdfReader`) and Word documents with\n  `python-docx` (`from docx import Document`); read READMEs, Markdown, and plain text\n  with `read_file`.\n- **Filenames and folder structure** \u2014 accession prefixes (`GSE...`, `SRR...`,\n  `PRJNA...`), organism shorthand (`human_`, `hg38`, `mm10`, `macaque_`, `cyno_`).\n- **Reference store** \u2014 `list_available_refs` labels catalogued files with the organism\n  they describe. Query it for a candidate organism as corroboration only: a hit means\n  reference data for that species is staged here, not that your data is that species, and\n  an empty store is a normal state that proves nothing either way.\n\nHow to record what you find:\n\n- Set `source` honestly to the most-direct evidence (`user-context` > `metadata` >\n  `document` > `filename` > `inferred`). `user-context` ranks highest because an\n  explicit user statement is direct evidence, not inference.\n- Set `confidence`: `high` for an explicit user statement, an organism column, or a\n  paper statement; `medium` for filename or accession-prefix signals; `low` for\n  inference from data content alone.\n- If sources DISAGREE (paper says human, metadata column says mouse), pick the\n  most-trusted source, set `confidence` to `low`, and add the conflict to\n  `qualityAssessment.concerns`. Do not silently pick one.\n\n## Stage 2: Grouping \u2014 the kinds this dataset is made of\n\nDecide what the dataset is MADE OF and submit it as `kinds`. A kind is a repeating set\nof files that are the same sort of thing: \"per-patient variant calls\", \"the reference\ntranscriptome\", \"the sample sheet\". A singleton is a kind of count 1 \u2014 there is no\nthreshold to reason about.\n\nYour kinds need NOT match the scan's shapes:\n\n- One shape may be **two kinds** \u2014 a position taking the values `tumor` and `normal`\n  is mechanically one shape, and analytically two arms. Split it when the values say so.\n- Several shapes may be **one kind** \u2014 files serving the same analytical role under two\n  naming conventions are one kind.\n- You may declare an axis the scan never saw, if a metadata sheet states it.\n\nFor each kind, state what ONE MEMBER represents (`memberRepresents`) as well as what\nthe set contains (`description`). These are different: \"1171 VCF files\" is the shape you\nwere handed; \"one patient's somatic variant calls\" is the decision only you can make.\n\nGive each kind a `pathPattern` that actually matches its members. Coverage is computed by\nmatching your patterns against the scanned tree, so a pattern matching nothing records\nthe kind as covering nothing.\n\nLabel the `axes` \u2014 what varies across members. The scan reports that a position varies\nand which values it takes; whether that is a subject, a timepoint, a treatment arm, or a\nchromosome shard is not derivable from the values, and is yours to name.\n\nWhere a kind's description depends on content the scan did not capture, inspect **ONE**\nexample file of that kind \u2014 not one per member. A set whose names already match does not\nbecome better understood by reading its 1171st member.\n\n## Stage 3: Notable singletons\n\n`files` is for the inputs that deserve individual prose \u2014 the metadata sheet, the\nREADME, the paper, an outlier that fits no kind. Describe those well, because there are\nfew of them.\n\nIt is NOT a list of the dataset's files. The workspace filesystem is the authoritative\nfile list, and `list_files`, `grep`, and semantic search all read the live tree.\n\n## Stage 4: Dataset-level concerns\n\nRecord dataset-wide findings in `qualityAssessment`, in the sense of \"what a planner\nmust know before designing an analysis\":\n\n- **Completeness from the scan** \u2014 1171 variant files against 1168 indexes names three\n  subjects missing a file. That costs a count, and it changes the plan.\n- **Batch structure** \u2014 are there batch labels in the filenames or the metadata?\n- **Sample imbalance** \u2014 uneven group sizes that limit statistical power.\n- **Missing or malformed metadata** \u2014 annotations absent, incomplete, or inconsistent\n  with the data.\n- **Normalization state** \u2014 raw counts, normalized, or log-transformed. Misidentifying\n  this derails every downstream step; it is identity, not quality.\n\n## Submitting Results\n\nCall `submit_profile` exactly once, after the work above is done. This is the ONLY way\nto deliver results \u2014 do not return JSON in your message text. The tool validates against\nthe schema; if validation fails you will see the errors and can fix and re-submit.\n\nThe schema caps `kinds` and `files`. If you hit a cap, you are enumerating members\nwhere you should be grouping them.\n\n## Do NOT\n\n- Profile file by file. The scan already enumerated the tree; your job is to say what it\n  IS, and one programmatic pass per input file is what this design exists to remove.\n- Enumerate a kind's members in your output \u2014 no member lists, no per-file records for\n  files that belong to a kind.\n- Decode a file in full. Read a header, a first record, a page \u2014 a bounded prefix.\n- Compute per-file statistical quality measures. Specifically NOT: transition/transversion\n  ratios, allele-frequency spectra, replicate correlation, principal-component outlier\n  detection, coverage depth, mapping rate, duplicate rate, insert-size distribution, or GC\n  bias. These require decoding files in full, no consumer of the profile reads them, and\n  they answer *is this good* rather than *what is this*.\n- `read_file` a data file. Preview it programmatically instead.\n- Guess the organism. If no input identifies it, set `organism: null` and explain in\n  `analysisSummary`. Inferring \"human\" from gene symbols alone is wrong \u2014 orthologs\n  share symbols across species.\n- Skip document inputs (PDFs, READMEs, DOCX). They carry the subject and design context\n  and are a primary source for Stage 1.\n- Use `print()` for status messages \u2014 use the `logging` module.\n- Return profiling results as JSON in your message text. Always use `submit_profile`.\n";