dsh-molbio-tools 0.19.0 → 0.20.1

package/batch.mjs

@@ -0,0 +1,185 @@
+/**
+ * dsh-molbio-tools/batch.mjs
+ *
+ * Batch sequence-file analysis: parse every matching file in a workspace
+ * directory, compute the same summary for each, and emit one CSV/TSV table.
+ * Pure parsing and computation; file discovery and writes live in index.mjs and
+ * go through ctx.fs with the session sandbox policy.
+ *
+ * The point is the TABLE, not a second implementation of each analysis: GC/N
+ * come from lib.mjs's baseCounts, ORFs from translateFrames, enzyme sites from
+ * digest, and the GenBank/SnapGene parsers are the same ones the single-file
+ * tools use. A file that cannot be parsed becomes an error row instead of
+ * aborting the whole batch — one bad .dna in a folder of 60 must not hide the
+ * other 59.
+ */
+
+import { MolbioInputError, baseCounts, digest, normalizeSequence, translateFrames } from './lib.mjs';
+import { parseGenBank } from './genbank.mjs';
+import { parseSnapGeneBytes } from './snapgene.mjs';
+import { entryStats, parseFasta, parseFastq } from './seqio.mjs';
+
+export const BATCH_LIMITS = {
+  maxFiles: 500,
+  maxFileBytes: 50 * 1024 * 1024,
+  maxTotalBytes: 500 * 1024 * 1024,
+  maxOrfBases: 2_000_000,
+  maxEnzymes: 20,
+};
+
+const round2 = (value) => Math.round(value * 100) / 100;
+
+/** Detect the format from the extension, then from the content. */
+export function detectBatchFormat(path, text) {
+  const lower = String(path).toLowerCase();
+  if (lower.endsWith('.dna')) return 'snapgene';
+  if (/\.(gb|gbk|genbank)$/.test(lower)) return 'genbank';
+  if (/\.(fa|fasta|fna|fas|ffn)$/.test(lower)) return 'fasta';
+  if (/\.(fq|fastq)$/.test(lower)) return 'fastq';
+  const head = typeof text === 'string' ? text.trimStart() : '';
+  if (head.startsWith('>')) return 'fasta';
+  if (head.startsWith('@')) return 'fastq';
+  if (/^LOCUS\s/m.test(head)) return 'genbank';
+  return 'text';
+}
+
+/** Convert a simple basename glob (*, ?) to a RegExp. */
+export function globToRegExp(pattern) {
+  const source = String(pattern)
+    .replace(/[.+^$()|[\]{}]/g, '\$&')
+    .replaceAll('*', '.*')
+    .replaceAll('?', '.');
+  return new RegExp('^' + source + '$', 'i');
+}
+
+/** Parse one sequence file into its name/topology/sequence/features. */
+function parseOne({ path, format, text, bytes }) {
+  if (format === 'snapgene') {
+    if (bytes === undefined) throw new MolbioInputError('a .dna file needs binary bytes');
+    const parsed = parseSnapGeneBytes(bytes);
+    return {
+      name: parsed.name,
+      topology: parsed.topology === 'circular' ? 'circular' : 'linear',
+      sequence: parsed.sequence,
+      features: parsed.features.length,
+    };
+  }
+  if (format === 'genbank') {
+    const parsed = parseGenBank(String(text ?? ''));
+    return {
+      name: parsed.name ?? path,
+      topology: parsed.topology === 'circular' ? 'circular' : 'linear',
+      sequence: parsed.sequence,
+      features: parsed.features.length,
+    };
+  }
+  if (format === 'fasta') {
+    const entries = parseFasta(String(text ?? ''));
+    return {
+      name: entries[0]?.id ?? path,
+      topology: 'unknown',
+      sequence: entries.map((entry) => entry.sequence).join(''),
+      features: 0,
+      entries: entries.length,
+      fastaEntries: entries,
+    };
+  }
+  if (format === 'fastq') {
+    const entries = parseFastq(String(text ?? ''));
+    return {
+      name: entries[0]?.id ?? path,
+      topology: 'unknown',
+      sequence: entries.map((entry) => entry.sequence).join(''),
+      features: 0,
+      entries: entries.length,
+      fastqEntries: entries,
+    };
+  }
+  const sequence = normalizeSequence(String(text ?? ''), 'sequence');
+  return { name: path, topology: 'unknown', sequence, features: 0, entries: 1 };
+}
+
+/**
+ * Analyze one file.
+ *
+ * @param {{path: string, format?: string, text?: string, bytes?: Uint8Array,
+ *   enzymes?: string[], orfMinAa?: number, circular?: boolean, codonTable?: string}} input
+ * @returns {object} one table row; throws only for an unrecognizable file.
+ */
+export function analyzeSequenceFile(input) {
+  const { path, text, bytes } = input;
+  const format = input.format ?? detectBatchFormat(path, text);
+  const parsed = parseOne({ path, format, text, bytes });
+  const sequence = normalizeSequence(parsed.sequence, path + ' sequence');
+  const counts = baseCounts(sequence);
+  const totalBases = sequence.length;
+  const gcPercent = counts.gc + counts.at === 0 ? 0 : round2((counts.gc / (counts.gc + counts.at)) * 100);
+  const nPercent = totalBases === 0 ? 0 : round2((counts.n / totalBases) * 100);
+  const row = {
+    path,
+    format,
+    topology: parsed.topology,
+    entries: parsed.entries ?? 1,
+    total_bases: totalBases,
+    gc_percent: gcPercent,
+    n_percent: nPercent,
+    feature_count: parsed.features,
+    enzyme_sites: [],
+    error: '',
+  };
+  const enzymes = input.enzymes ?? [];
+  if (enzymes.length > 0) {
+    const circular = input.circular === true || parsed.topology === 'circular';
+    const result = digest(sequence, enzymes, circular);
+    row.enzyme_sites = result.map((entry, index) => ({
+      name: enzymes[index],
+      sites: entry.cut_positions.length,
+    }));
+  }
+  const orfMinAa = input.orfMinAa ?? 0;
+  if (orfMinAa > 0) {
+    if (totalBases > BATCH_LIMITS.maxOrfBases) {
+      row.orf_count = 0;
+      row.longest_orf_aa = 0;
+      row.orf_skipped = true;
+    } else {
+      const { orfs } = translateFrames(sequence, 'all', input.codonTable ?? 'standard', orfMinAa);
+      row.orf_count = orfs.length;
+      row.longest_orf_aa = orfs.length === 0 ? 0 : Math.max(...orfs.map((orf) => orf.length_aa));
+      row.orf_skipped = false;
+    }
+  }
+  if (format === 'fasta' || format === 'fastq') {
+    const stats = entryStats(parsed.fastaEntries ?? parsed.fastqEntries ?? [], format === 'fastq');
+    row.mean_length = stats.mean_length;
+    if (format === 'fastq' && stats.quality_mean !== undefined) row.quality_mean = stats.quality_mean;
+  }
+  return row;
+}
+
+/** Columns for the table, in a stable order. */
+export function batchColumns({ orfs, enzymes }) {
+  const columns = ['path', 'format', 'topology', 'entries', 'total_bases', 'gc_percent', 'n_percent', 'feature_count', 'mean_length'];
+  if (orfs) columns.push('orf_count', 'longest_orf_aa', 'orf_skipped');
+  if (enzymes.length > 0) columns.push(...enzymes.map((name) => name + '_sites'));
+  columns.push('error');
+  return columns;
+}
+
+function csvField(value, delimiter) {
+  const text = value === undefined || value === null ? '' : String(value);
+  return text.includes(delimiter) || /["\n\r]/.test(text) ? '"' + text.replaceAll('"', '""') + '"' : text;
+}
+
+/** Render rows as CSV (delimiter = ",") or TSV (delimiter = "\t"). */
+export function batchTable(rows, columns, delimiter = ',') {
+  const value = (row, column) => {
+    if (column.endsWith('_sites')) {
+      const name = column.slice(0, -'_sites'.length);
+      return row.enzyme_sites?.find((entry) => entry.name === name)?.sites;
+    }
+    return row[column];
+  };
+  const lines = rows.map((row) => columns.map((column) => csvField(value(row, column), delimiter)).join(delimiter));
+  return [columns.join(delimiter), ...lines].join('\n') + '\n';
+}

package/benchmark/README.md

@@ -76,11 +76,11 @@ Measured, and printed so the answer can be audited:
 
 ```
 $ touch crispr.mjs && node benchmark/run.mjs --list --changed
-1 of 56 task(s) selected — --tools molbio_grna_design --changed
+1 of 61 task(s) selected — --tools molbio_grna_design --changed
   (from --changed; 2 module(s) affected: crispr.mjs, index.mjs)
 
 $ touch lib.mjs && node benchmark/run.mjs --list --changed
-55 of 56 task(s) selected — --changed (27 module(s) affected: align.mjs, cloning.mjs, …)
+55 of 61 task(s) selected — --changed (27 module(s) affected: align.mjs, cloning.mjs, …)
 ```
 
 `lib.mjs` selecting nearly everything is the **correct** answer, not a failure to
@@ -102,7 +102,7 @@ you changed, so its task runs even if it is full-only.
 | `--tools a,b,c` / `primer_*` | the union | you changed a few |
 | `--changed --tools X` | the INTERSECTION | "of what I changed, test X" |
 | (default) | **core** — 14 tasks | a broad smoke of the catalog |
-| `--tier full` | all 56, 58/58 tools | before a release |
+| `--tier full` | all 61, 60/60 tools | before a release |
 
 The two `--tools` operators are stated because confusing them is the difference
 between testing what you changed and testing **less** than that: alone it unions,
@@ -125,7 +125,7 @@ widens to the full suite, with the reason printed.
 | a module **only** imported by other modules (`svgio.mjs`) | partial — its importers' tasks |
 | docs, changelog, `test/`, `benchmark/`, config | **`none`: nothing to run** |
 
-`none` is deliberately NOT `full`. A documentation commit that triggers 56 model
+`none` is deliberately NOT `full`. A documentation commit that triggers 61 model
 calls is how a `--changed` default gets switched off and the full suite goes back
 to running every time — the exact behaviour this option exists to prevent. It is
 only reachable with an explicit `--changed`, so everything else still fails safe,
@@ -134,7 +134,7 @@ and `test/benchmark-changed.mjs` asserts both directions.
 ### The part that still runs in full, on purpose
 
 A model run with `--changed` or `--tools` first executes the **whole** offline
-gate: all 56 tasks' `where: "tool"` expectations are recomputed against the shipped
+gate: all 61 tasks' `where: "tool"` expectations are recomputed against the shipped
 tools (no model, a few hundred milliseconds). That is what makes partial runs safe
 — the expected values are re-proven for the tasks you skipped, so "skipped because
 unchanged" is a checked claim, not a remembered one.
@@ -168,7 +168,7 @@ asserting whatever the tool happened to emit.
 
 Two guards make "someone forgot" fail loudly instead of quietly:
 
-- `test/benchmark-coverage.mjs` asserts **58/58 registered tools are covered by
+- `test/benchmark-coverage.mjs` asserts **60/60 registered tools are covered by
   at least one task**. Adding a tool without a task is a red build. It also
   asserts the fixtures' premises (the Golden Gate backbone is BsaI-free, the
   mutagenesis template still admits a primer pair), that a `covers` entry names a
@@ -209,7 +209,7 @@ recorded tool text, and it says so instead of silently scoring empty text).
 
 ## The headless profile (`benchmark/profile.mjs`)
 
-The model-facing way to get the 58 tools is the **Molecular Biology Lab** agent
+The model-facing way to get the 60 tools is the **Molecular Biology Lab** agent
 preset. The benchmark cannot use it, and the reason is structural rather than
 configurational — `dsh-headless/lib/index.js` refuses an adoption whose session
 records a preset:
@@ -368,6 +368,25 @@ These are recorded rather than scored, because each one is a question about the
 plugin or the harness, not about the model. The first is a **measured plugin
 defect**.
 
+### `molbio_extract_region` advertised `sequence`, but the resolver only read `vector`
+
+The 0.20.0 full run (61 tasks) failed `region-extract` on **arguments**: the model
+called `molbio_extract_region` with the parameter the schema advertises
+(`sequence`), and the tool answered "provide either vector (a sequence) or
+vector_path". The schema/reality split was the defect: `resolveVector` read only
+`args.vector`, so `sequence` was silently ignored. **Fixed in 0.20.0** by
+accepting every declared inline alias (`vector`/`sequence`/`reference`), with a
+smoke regression that calls the tool through the advertised `sequence` argument.
+
+The same run also showed four **assertion** failures rather than product defects:
+`plasmid-digest` (the model wrote `2,840`, the assertion `2840`),
+`protein-props` (`3,935` vs `3935`), `codon-optimize` (the model said
+`EcoRI-free`, the assertion demanded `no EcoRI`), and `chart-volcano` (the task
+asked for `de.svg`, the assertion pinned `volcano.svg`). `--replay` turns all four
+to PASS; the region-extract live rerun was blocked by the DSH sandbox approval
+for headless subprocesses, so the last measured score is **60/61**, with the
+underlying bug fixed and pinned by smoke.
+
 ### `molbio_design_primers` prints the forward/reverse roles reversed
 
 **Confirmed by direct inspection, not by reading a model's claim.**
@@ -548,7 +567,7 @@ implementation happens to return for one set of arguments.**
   and `args_ok` both true is a *reporting* finding, not a tool bug.
 - **Multimodal hand-off.** `attach_image` is exercised as an argument at most; no
   assertion inspects a decoded PNG (that is `svgpng.mjs` and `contract.mjs`).
-- **Most of the toolset is now covered, but shallowly.** 56 tasks touch all 58
+- **Most of the toolset is now covered, but shallowly.** 61 tasks touch all 60
   tools; most tools appear in exactly one task, and several appearances are a
   single argument check rather than a full workflow. Breadth is asserted by
   `test/benchmark-coverage.mjs`; depth is not, and cannot be.

package/benchmark/sequences.mjs

@@ -165,6 +165,13 @@ export const NAMED_SEQUENCES = {
   cds: FIXTURES.CDS,
   mutagenesis_template: FIXTURES.MUTAGENESIS_TEMPLATE,
   sanger_read: FIXTURES.SANGER_READ,
+  // v0.20.0: the family-geometry fixtures. The Cas12a target is a 5' TTTV PAM
+  // followed by a 23 nt protospacer; the Cas13a target is a 28 nt spacer with a
+  // 3' H PFS. Keeping them here means the instruction and the offline grader
+  // cannot diverge.
+  cas12a_target: 'AAAATTTAACGTACGTACGTACGTACGTACGCCCCTTTCGGGGGGGGGGG',
+  cas13_target: 'AAAAACGTACGTACGTACGTACGTACGTACGTACCCCCCCCCC',
+  silent_cds: 'ATGGCTGAATTCGGTTAAGCTTAA',
   /**
    * The three FASTQ reads the `fastq-qc` task pastes in.
    *

package/benchmark/tasks/02-primers.json

@@ -12,26 +12,54 @@
       "id": "primer-qc",
       "category": "primers",
       "tier": "full",
-      "covers": ["molbio_primer_check", "molbio_primer_tm"],
+      "covers": [
+        "molbio_primer_check",
+        "molbio_primer_tm"
+      ],
       "instruction": "I have these two primers already ordered and I want to know whether they are any good together before I run them. Report the Tm of the forward primer, and flag anything structurally worrying about the pair.\n\nForward: ATTACTCCTGCTCTTCCCATAC\nReverse: CACTACTCCTCTGTACGCAC",
       "timeout_ms": 240000,
-      "expect_tools": ["molbio_primer_check"],
+      "expect_tools": [
+        "molbio_primer_check"
+      ],
       "assertions": [
-        { "where": "tool", "pattern": "primer1: 22 bp, GC 45\\.45%", "note": "the forward primer's own metrics" },
-        { "where": "tool", "pattern": "primer2: 20 bp, GC 55%" },
-        { "where": "tool", "pattern": "self-any 7 \\(≤8\\)", "note": "the reverse primer's self-complementarity, right at the Primer3 limit" },
-        { "where": "tool", "pattern": "pair \\(dimer\\): score 14", "note": "the pair's dimer score" },
-        { "where": "answer", "pattern": "53\\.9|54\\b", "note": "the forward primer's Tm, 53.97 °C" }
+        {
+          "where": "tool",
+          "pattern": "primer1: 22 bp, GC 45\\.45%",
+          "note": "the forward primer's own metrics"
+        },
+        {
+          "where": "tool",
+          "pattern": "primer2: 20 bp, GC 55%"
+        },
+        {
+          "where": "tool",
+          "pattern": "self-any 7 \\(≤8\\)",
+          "note": "the reverse primer's self-complementarity, right at the Primer3 limit"
+        },
+        {
+          "where": "tool",
+          "pattern": "pair \\(dimer\\): score 14",
+          "note": "the pair's dimer score"
+        },
+        {
+          "where": "answer",
+          "pattern": "53\\.9|54\\b",
+          "note": "the forward primer's Tm, 53.97 °C"
+        }
       ]
     },
     {
       "id": "taqman-assay",
       "category": "primers",
       "tier": "full",
-      "covers": ["molbio_design_taqman"],
+      "covers": [
+        "molbio_design_taqman"
+      ],
       "instruction": "Design a TaqMan (5' nuclease) assay on this template. I want primers plus a hydrolysis probe between them, and I want the probe to sit at least a few degrees above the primers in Tm.\n\nTemplate:\n{template:400:7}",
       "timeout_ms": 300000,
-      "expect_tools": ["molbio_design_taqman"],
+      "expect_tools": [
+        "molbio_design_taqman"
+      ],
       "assertions": [
         {
           "where": "tool",
@@ -48,21 +76,39 @@
           "pattern": "vs the hotter primer",
           "note": "the probe-vs-primer Tm margin, which the tool reports per candidate"
         },
-        { "where": "answer", "pattern": "probe|Probe" }
+        {
+          "where": "answer",
+          "pattern": "probe|Probe"
+        }
       ]
     },
     {
       "id": "intron-qpcr",
       "category": "primers",
       "tier": "full",
-      "covers": ["molbio_design_intron_primers"],
+      "covers": [
+        "molbio_design_intron_primers"
+      ],
       "instruction": "I need a qPCR assay that will NOT amplify contaminating genomic DNA. Here is the genomic sequence and the two exon coordinates — design a pair where the forward primer spans the exon-exon junction. Tell me the spliced amplicon size and how far apart the primers sit on the genome.\n\n{seq:intron_genomic}\n\nExons: 1-400 and 701-1100 (1-based).",
       "timeout_ms": 300000,
-      "expect_tools": ["molbio_design_intron_primers"],
+      "expect_tools": [
+        "molbio_design_intron_primers"
+      ],
       "assertions": [
-        { "where": "tool", "pattern": "forward primer spans an exon-exon junction", "note": "the property that makes the assay genomic-DNA-proof" },
-        { "where": "tool", "pattern": "spliced amplicon 320-406 \\(87 bp\\), genomic span 387 bp" },
-        { "where": "tool", "pattern": "junction 12\\+6 bp", "note": "bases contributed on each side of the junction" },
+        {
+          "where": "tool",
+          "pattern": "forward primer spans an exon-exon junction",
+          "note": "the property that makes the assay genomic-DNA-proof"
+        },
+        {
+          "where": "tool",
+          "pattern": "spliced amplicon 320-406 \\(87 bp\\), genomic span 387 bp"
+        },
+        {
+          "where": "tool",
+          "pattern": "junction 12\\+6 bp",
+          "note": "bases contributed on each side of the junction"
+        },
         {
           "where": "answer",
           "pattern": "splic|junction|genomic|intron|87|387",
@@ -74,12 +120,19 @@
       "id": "multiplex-panel",
       "category": "primers",
       "tier": "full",
-      "covers": ["molbio_multiplex_check"],
-      "instruction": "I want to run these two primer pairs in the same multiplex tube. The first pair is on target A (template below), the second on target B. Check them against each other and tell me whether they interfere — dimers, unintended annealing, or bands I will not be able to tell apart.\n\nTarget A template:\n{template:400:7}\n\nPair A: forward ATTACTCCTGCTCTTCCCATAC (Tm 58.4), reverse CACTACTCCTCTGTACGCAC (Tm 58.9)\nPair B: forward GGGTTTCCCAAATTTGGGCCCA, reverse TTTGGGCCCAAATTTGGGAAAC\nTemplate B:\n{template:400:8}",
+      "covers": [
+        "molbio_multiplex_check"
+      ],
+      "instruction": "I want to run these two primer pairs in the same multiplex tube. The first pair is on target A (template below), the second on target B. Check them against each other and tell me whether they interfere — dimers, unintended annealing, or bands I will not be able to tell apart. Also give me the recommended annealing temperature for the panel and a starting primer concentration for each pair.\n\nTarget A template:\n{template:400:7}\n\nPair A: forward ATTACTCCTGCTCTTCCCATAC (Tm 58.4), reverse CACTACTCCTCTGTACGCAC (Tm 58.9)\nPair B: forward GGGTTTCCCAAATTTGGGCCCA, reverse TTTGGGCCCAAATTTGGGAAAC\nTemplate B:\n{template:400:8}",
       "timeout_ms": 300000,
-      "expect_tools": ["molbio_multiplex_check"],
+      "expect_tools": [
+        "molbio_multiplex_check"
+      ],
       "assertions": [
-        { "where": "tool", "pattern": "multiplex panel: 2 target\\(s\\) on 2 template\\(s\\), 4 primer\\(s\\)" },
+        {
+          "where": "tool",
+          "pattern": "multiplex panel: 2 target\\(s\\) on 2 template\\(s\\), 4 primer\\(s\\)"
+        },
         {
           "where": "tool",
           "pattern": "primer-primer interactions above the dimer threshold: [0-9]+",
@@ -94,6 +147,66 @@
           "where": "answer",
           "pattern": "dimer|interfere|conflict|anneal",
           "note": "the panel verdict. Note the measured behaviour this task must tolerate: a model that FIRST checks each pair on its own template often finds — correctly — that a pair as written does not amplify at all, and answers that instead. That is a better answer than 'no dimer conflict', not a worse one."
+        },
+        {
+          "where": "tool",
+          "pattern": "annealing temperature: [0-9.]+ °C \\(range [0-9.]+-[0-9.]+ °C\\)",
+          "note": "the panel-level annealing recommendation (lowest primer Tm minus the offset)"
+        },
+        {
+          "where": "tool",
+          "pattern": "primer concentration plan \\(nM\\):",
+          "note": "the starting primer-concentration plan"
+        },
+        {
+          "where": "answer",
+          "pattern": "anneal|temperature|concentration",
+          "note": "the model reports the new planning outputs, not only the dimer verdict"
+        }
+      ]
+    },
+    {
+      "id": "taqman-mgb",
+      "category": "primers",
+      "tier": "full",
+      "covers": [
+        "molbio_design_taqman"
+      ],
+      "instruction": "Design a TaqMan assay with an MGB probe on this template. Use an MGB probe, write the ordering sheet to mgb-order.csv, and tell me the probe length and Tm window the designer used, the top-ranked probe, and the 5'/3' modifications on its ordering row.\n\nTemplate:\n{template:400:7}",
+      "timeout_ms": 300000,
+      "expect_tools": [
+        "molbio_design_taqman"
+      ],
+      "assertions": [
+        {
+          "where": "tool",
+          "pattern": "Probe chemistry: TaqMan MGB",
+          "note": "the MGB chemistry is selected, not silently the standard one"
+        },
+        {
+          "where": "tool",
+          "pattern": "13-20 nt, Tm 48-60",
+          "note": "the MGB length/Tm window, on the unmodified NN scale"
+        },
+        {
+          "where": "tool",
+          "pattern": "P TTGCTACGCCTGT",
+          "note": "the top-ranked MGB probe"
+        },
+        {
+          "where": "tool",
+          "pattern": "ordering CSV \\([0-9]+ oligo row\\(s\\)\\) saved to",
+          "note": "the ordering sheet was written"
+        },
+        {
+          "where": "tool",
+          "pattern": "MGB probes: the reported probe Tm is the unmodified DNA NN estimate",
+          "note": "the tool refuses to fabricate a modified Tm"
+        },
+        {
+          "where": "answer",
+          "pattern": "MGB|unmodified|modif",
+          "note": "the model explains the MGB Tm caveat or the modifications"
         }
       ]
     }