dsh-molbio-tools 0.22.2 → 0.24.0

package/dotplot.mjs

@@ -0,0 +1,257 @@
+/**
+ * dsh-molbio-tools/dotplot.mjs
+ *
+ * Self-comparison dotplot by exact word matching: a dot at (i, j) means the
+ * k-mer starting at i equals the k-mer starting at j (direct repeat), and a red
+ * dot means it equals the reverse complement of the k-mer at j (inverted
+ * repeat). Pure computation; the SVG is binned into a fixed pixel grid so a
+ * repetitive sequence cannot produce a million-element document.
+ *
+ * Deliberately NOT a window/identity dotplot (EMBOSS dotmatcher-style): this is
+ * the k-mer view, which is exact, deterministic and needs no scoring matrix.
+ */
+import { MolbioInputError, normalizeSequence } from './lib.mjs';
+import { axis, linearScale, niceTicks, panel, rect, svgDocument, textRun } from './svgio.mjs';
+
+/** Accepted ranges and work caps. */
+export const DOTPLOT_LIMITS = {
+  minWord: 4,
+  maxWord: 12,
+  maxLength: 20_000,
+  maxPairs: 200_000,
+  maxCells: 60_000,
+  grid: 400,
+};
+
+const round = (value, digits = 3) => Math.round(value * 10 ** digits) / 10 ** digits;
+
+/** Reverse complement of one canonical k-mer (A/C/G/T only). */
+function rcWord(word) {
+  let out = '';
+  for (let index = word.length - 1; index >= 0; index--) {
+    const base = word[index];
+    out += base === 'A' ? 'T' : base === 'C' ? 'G' : base === 'G' ? 'C' : 'A';
+  }
+  return out;
+}
+
+/**
+ * Compute the self-comparison dotplot.
+ *
+ * @param {string} raw DNA sequence (IUPAC; ambiguous k-mers are skipped).
+ * @param {{word?: number, mode?: 'direct'|'inverted'|'both', grid?: number}} [options]
+ * @returns {{length: number, word: number, mode: string, direct_matches: number,
+ *   inverted_matches: number, direct_cells: number, inverted_cells: number,
+ *   ambiguous_bases: number, truncated: boolean, plot_truncated: boolean,
+ *   grid: number, directGrid: Set<number>, invertedGrid: Set<number>}}
+ */
+export function dotplot(raw, options = {}) {
+  const sequence = normalizeSequence(raw, 'sequence');
+  if (sequence.length > DOTPLOT_LIMITS.maxLength) {
+    throw new MolbioInputError('the sequence is ' + sequence.length + ' bp; the dotplot limit is ' + DOTPLOT_LIMITS.maxLength + ' bp');
+  }
+  const word = options.word ?? 8;
+  if (!Number.isInteger(word) || word < DOTPLOT_LIMITS.minWord || word > DOTPLOT_LIMITS.maxWord) {
+    throw new MolbioInputError('word must be an integer between ' + DOTPLOT_LIMITS.minWord + ' and ' + DOTPLOT_LIMITS.maxWord);
+  }
+  if (sequence.length < word * 2) {
+    throw new MolbioInputError('the sequence is ' + sequence.length + ' bp; at word ' + word + ' it needs at least ' + word * 2 + ' bp to show a repeat');
+  }
+  const mode = options.mode ?? 'both';
+  if (!['direct', 'inverted', 'both'].includes(mode)) {
+    throw new MolbioInputError('mode must be "direct", "inverted" or "both"');
+  }
+  const grid = options.grid ?? DOTPLOT_LIMITS.grid;
+  if (!Number.isInteger(grid) || grid < 50 || grid > 1000) {
+    throw new MolbioInputError('grid must be an integer between 50 and 1000');
+  }
+
+  const index = new Map();
+  let ambiguousBases = 0;
+  for (let start = 0; start + word <= sequence.length; start++) {
+    const wordText = sequence.slice(start, start + word);
+    let canonical = true;
+    for (const base of wordText) {
+      if (base !== 'A' && base !== 'C' && base !== 'G' && base !== 'T') {
+        canonical = false;
+        break;
+      }
+    }
+    if (!canonical) {
+      ambiguousBases++;
+      continue;
+    }
+    const list = index.get(wordText);
+    if (list === undefined) index.set(wordText, [start]);
+    else list.push(start);
+  }
+
+  const length = sequence.length;
+  const cellOf = (position) => Math.min(grid - 1, Math.floor((position / length) * grid));
+  const directGrid = new Set();
+  const invertedGrid = new Set();
+  let directMatches = 0;
+  let invertedMatches = 0;
+  let truncated = false;
+  let plotTruncated = false;
+  let work = 0;
+
+  const addCell = (set, x, y) => {
+    const key = x * 4096 + y;
+    if (set.has(key)) return;
+    // Bound the RENDERED cells, not just the match count: a dense plot must not
+    // produce a document with one <rect> per match.
+    if (directGrid.size + invertedGrid.size >= DOTPLOT_LIMITS.maxCells) {
+      plotTruncated = true;
+      return;
+    }
+    set.add(key);
+  };
+
+  outer: for (const [wordText, positions] of index) {
+    if (mode === 'direct' || mode === 'both') {
+      for (let a = 0; a < positions.length; a++) {
+        for (let b = 0; b < positions.length; b++) {
+          if (work >= DOTPLOT_LIMITS.maxPairs) {
+            truncated = true;
+            break outer;
+          }
+          work++;
+          directMatches++;
+          addCell(directGrid, cellOf(positions[a]), cellOf(positions[b]));
+        }
+      }
+    }
+    if (mode === 'inverted' || mode === 'both') {
+      const rc = rcWord(wordText);
+      if (rc === wordText) continue; // a palindromic word would double-count the same pairs
+      const rcPositions = index.get(rc);
+      if (rcPositions === undefined) continue;
+      for (const from of positions) {
+        for (const to of rcPositions) {
+          if (work >= DOTPLOT_LIMITS.maxPairs) {
+            truncated = true;
+            break outer;
+          }
+          work++;
+          invertedMatches++;
+          addCell(invertedGrid, cellOf(from), cellOf(to));
+        }
+      }
+    }
+  }
+
+  return {
+    length,
+    word,
+    mode,
+    direct_matches: directMatches,
+    inverted_matches: invertedMatches,
+    direct_cells: directGrid.size,
+    inverted_cells: invertedGrid.size,
+    ambiguous_bases: ambiguousBases,
+    truncated,
+    plot_truncated: plotTruncated,
+    grid,
+    directGrid,
+    invertedGrid,
+  };
+}
+
+/**
+ * Render a computed dotplot as an SVG document. The grid is binned: one
+ * `<rect>` per occupied cell, so the document size is bounded by the grid, not
+ * by the number of matches.
+ *
+ * @param {ReturnType<typeof dotplot>} data
+ * @param {{title?: string}} [options]
+ * @returns {string} SVG text.
+ */
+export function renderDotplot(data, options = {}) {
+  const width = 720;
+  // The x axis needs a title line BELOW its tick labels, and the legend needs a
+  // home INSIDE the panel. At 720 px the plot was 628 px square and the title /
+  // legend fell on (or outside) the bottom border; 760 px keeps the plot square
+  // (same 628 px) while giving both a row of their own. `bottom: 66` is what
+  // preserves that square: 728 (frame) - 34 (top) - 66 = 628.
+  const height = 760;
+  const frame = panel({
+    x: 16,
+    y: 16,
+    width: width - 32,
+    height: height - 32,
+    inset: { top: 34, right: 14, bottom: 66, left: 46 },
+    title: options.title ?? 'Sequence dotplot',
+    subtitle: data.length + ' bp · word ' + data.word + ' · ' + data.mode + ' · ' + data.direct_matches + ' direct / ' + data.inverted_matches + ' inverted match(es)'
+      + (data.truncated ? ' · scan capped' : '') + (data.ambiguous_bases > 0 ? ' · ' + data.ambiguous_bases + ' ambiguous k-mer(s) skipped' : ''),
+  });
+  const plot = frame.plot;
+  const x = linearScale([1, data.length], [plot.x, plot.x + plot.width]);
+  const y = linearScale([1, data.length], [plot.y + plot.height, plot.y]);
+  const cellWidth = plot.width / data.grid;
+  const cellHeight = plot.height / data.grid;
+  const parts = [frame.markup];
+  const drawCells = (set, fill, opacity) => {
+    const cells = [];
+    for (const key of set) {
+      const cx = Math.floor(key / 4096);
+      const cy = key - cx * 4096;
+      cells.push(rect({
+        x: round(plot.x + cx * cellWidth, 2),
+        y: round(plot.y + cy * cellHeight, 2),
+        width: Math.max(1, round(cellWidth, 2)),
+        height: Math.max(1, round(cellHeight, 2)),
+        fill,
+        fillOpacity: opacity,
+      }));
+    }
+    return cells.join('\n');
+  };
+  if (data.invertedGrid.size > 0) parts.push(drawCells(data.invertedGrid, '#cf222e', 0.55));
+  if (data.directGrid.size > 0) parts.push(drawCells(data.directGrid, '#24292f', 0.7));
+  parts.push(axis({
+    from: plot.x,
+    to: plot.x + plot.width,
+    at: plot.y + plot.height,
+    ticks: niceTicks([1, data.length], 6).map((tick) => x.of(tick)),
+    orientation: 'x',
+    tickLabel: (pixel) => String(Math.round(x.invert(pixel))),
+    title: 'position (bp)',
+  }));
+  parts.push(axis({
+    from: plot.y,
+    to: plot.y + plot.height,
+    at: plot.x,
+    ticks: niceTicks([1, data.length], 6).map((tick) => y.of(tick)),
+    orientation: 'y',
+    tickLabel: (pixel) => String(Math.round(y.invert(pixel))),
+    title: 'position (bp)',
+  }));
+  parts.push(legend(data));
+  return svgDocument({
+    width,
+    height,
+    title: options.title ?? 'Sequence dotplot',
+    description: 'Exact ' + data.word + '-mer self-comparison of a ' + data.length + ' bp sequence; black cells are direct repeats, red cells are inverted repeats.',
+    body: parts.join('\n'),
+  });
+}
+
+/** A compact legend; omitted when the corresponding mode produced no cells. */
+function legend(data) {
+  const entries = [];
+  // Only label a class that was actually computed AND has dots: in
+  // inverted-only mode a black "direct repeat" key would claim a measurement
+  // that never ran.
+  if (data.directGrid.size > 0) entries.push(['#24292f', 'direct repeat']);
+  if (data.invertedGrid.size > 0) entries.push(['#cf222e', 'inverted repeat']);
+  let x = 28;
+  const y = 730;
+  const parts = [];
+  for (const [color, label] of entries) {
+    parts.push(rect({ x, y: y - 8, width: 9, height: 9, fill: color }));
+    parts.push(textRun({ x: x + 13, y, text: label, size: 9, fill: '#57606a' }));
+    x += 130;
+  }
+  return parts.join('\n');
+}

package/fastq-qc.mjs

@@ -28,6 +28,11 @@
  */
 
 import { MolbioInputError } from './lib.mjs';
+import { ADAPTERS } from './contamination.mjs';
+
+// Re-exported for callers that used this module's adapter table directly. The
+// table now lives in contamination.mjs and is DNA-normalized there.
+export { ADAPTERS } from './contamination.mjs';
 import {
   axis,
   escapeXml,
@@ -63,15 +68,7 @@ export const FASTQ_QC_DEFAULTS = {
   duplication_bins: 10,
 };
 
-/** Public adapter/primer fragments used for the adapter-content screen. */
-export const ADAPTERS = [
-  { name: 'Illumina Universal', sequence: 'AGATCGGAAGAG' },
-  { name: 'Illumina Small RNA 3\'', sequence: 'TGGAATTCTCGG' },
-  { name: 'Illumina Small RNA 5\'', sequence: 'GUUCAGAGUUCUACAGUCCGACGAUC' },
-  { name: 'Nextera Transposase', sequence: 'CTGTCTCTTATA' },
-  { name: 'PolyA', sequence: 'AAAAAAAAAAAA' },
-  { name: 'PolyG', sequence: 'GGGGGGGGGGGG' },
-];
+/** Public adapter/primer fragments: see contamination.mjs (DNA-normalized). */
 
 /** Phred+33 decoding, with the offending read id in the message. */
 function phredOf(character) {

package/fastq-trim.mjs

@@ -0,0 +1,238 @@
+/**
+ * dsh-molbio-tools/fastq-trim.mjs
+ *
+ * Pure FASTQ trimming/filtering: 5' primer removal, 3' adapter removal, 3'
+ * sliding-window quality trimming, poly-G/poly-A tail trimming, and length/N
+ * filters. No file I/O and no quality re-encoding — the tool layer reads and
+ * writes through ctx.fs, while this module only transforms entry objects.
+ *
+ * The deliberately narrow scope is stated in the tool description: this is not
+ * a paired-end merger, not cutadapt's semiglobal aligner, and not a UMI
+ * pipeline. The algorithms here are the ones a bench user can reason about.
+ */
+import { MolbioInputError, normalizeSequence } from './lib.mjs';
+
+/** Defaults and hard bounds shared by the tool, tests and docs. */
+export const FASTQ_TRIM_DEFAULTS = {
+  minQuality: 20,
+  qualityWindow: 4,
+  polyMin: 8,
+  minLength: 0,
+  maxNPercent: 100,
+  phred: 33,
+  maxReads: 0,
+  maxReadsHard: 200000,
+};
+
+const PHRED_OFFSETS = [33, 64];
+
+function integerIn(value, name, min, max) {
+  if (!Number.isInteger(value) || value < min || value > max) {
+    throw new MolbioInputError(name + ' must be an integer between ' + min + ' and ' + max);
+  }
+  return value;
+}
+
+function numberIn(value, name, min, max) {
+  if (typeof value !== 'number' || !Number.isFinite(value) || value < min || value > max) {
+    throw new MolbioInputError(name + ' must be a number between ' + min + ' and ' + max);
+  }
+  return value;
+}
+
+/** Validate and normalize the options once per call. */
+export function resolveTrimOptions(options = {}) {
+  const minQuality = integerIn(options.minQuality ?? FASTQ_TRIM_DEFAULTS.minQuality, 'min_quality', 0, 60);
+  const qualityWindow = integerIn(options.qualityWindow ?? FASTQ_TRIM_DEFAULTS.qualityWindow, 'quality_window', 1, 100);
+  const polyMin = integerIn(options.polyMin ?? FASTQ_TRIM_DEFAULTS.polyMin, 'poly_min', 2, 100);
+  const minLength = integerIn(options.minLength ?? FASTQ_TRIM_DEFAULTS.minLength, 'min_length', 0, 1_000_000);
+  const maxNPercent = numberIn(options.maxNPercent ?? FASTQ_TRIM_DEFAULTS.maxNPercent, 'max_n_percent', 0, 100);
+  const phred = options.phred ?? FASTQ_TRIM_DEFAULTS.phred;
+  if (!PHRED_OFFSETS.includes(phred)) throw new MolbioInputError('phred must be 33 or 64');
+  const maxReads = integerIn(options.maxReads ?? FASTQ_TRIM_DEFAULTS.maxReads, 'max_reads', 0, FASTQ_TRIM_DEFAULTS.maxReadsHard);
+  const adapter = options.adapter === undefined || options.adapter === ''
+    ? ''
+    : normalizeSequence(options.adapter, 'adapter');
+  const primer = options.primer === undefined || options.primer === ''
+    ? ''
+    : normalizeSequence(options.primer, 'primer');
+  return {
+    minQuality,
+    qualityWindow,
+    polyMin,
+    minLength,
+    maxNPercent,
+    phred,
+    maxReads,
+    adapter,
+    primer,
+    trimPolyG: options.polyG === true,
+    trimPolyA: options.polyA === true,
+  };
+}
+
+/** 0-based exclusive end after 3' sliding-window quality trimming. */
+function qualityTrimEnd(sequence, quality, options) {
+  const length = sequence.length;
+  if (length === 0) return 0;
+  const window = Math.min(options.qualityWindow, length);
+  let end = length;
+  while (end >= window) {
+    let sum = 0;
+    for (let index = end - window; index < end; index++) {
+      const q = quality.charCodeAt(index) - options.phred;
+      if (q < 0 || q > 93) throw new MolbioInputError('FASTQ quality character outside Phred+' + options.phred + ' range at read position ' + (index + 1));
+      sum += q;
+    }
+    if (sum / window >= options.minQuality) break;
+    end--;
+  }
+  return end < window ? 0 : end;
+}
+
+/** Longest trailing run of one base, or 0. */
+function trailingRun(sequence, base) {
+  let length = 0;
+  for (let index = sequence.length - 1; index >= 0 && sequence[index] === base; index--) length++;
+  return length;
+}
+
+/** Count Ns (and any other non-ACGT base) in a sequence. */
+function ambiguousCount(sequence) {
+  let count = 0;
+  for (const base of sequence) if (!'ACGT'.includes(base)) count++;
+  return count;
+}
+
+/**
+ * Trim/filter one parsed FASTQ entry.
+ *
+ * @param {{id: string, description: string, sequence: string, quality: string}} entry
+ * @param {ReturnType<typeof resolveTrimOptions>} options
+ * @returns {{entry?: object, dropped?: 'length'|'n', trims: {quality:boolean, adapter:boolean, poly_g:boolean, poly_a:boolean, primer:boolean}}}
+ */
+export function trimRead(entry, options) {
+  if (typeof entry?.sequence !== 'string' || typeof entry?.quality !== 'string') throw new MolbioInputError('each read needs a sequence and a quality string');
+  if (entry.sequence.length !== entry.quality.length) throw new MolbioInputError('read ' + entry.id + ': sequence and quality lengths differ');
+  let sequence = entry.sequence;
+  let quality = entry.quality;
+  const trims = { quality: false, adapter: false, poly_g: false, poly_a: false, primer: false };
+
+  if (options.primer !== '' && sequence.startsWith(options.primer)) {
+    sequence = sequence.slice(options.primer.length);
+    quality = quality.slice(options.primer.length);
+    trims.primer = true;
+  }
+  if (options.adapter !== '') {
+    const at = sequence.indexOf(options.adapter);
+    if (at !== -1) {
+      sequence = sequence.slice(0, at);
+      quality = quality.slice(0, at);
+      trims.adapter = true;
+    }
+  }
+
+  const qualityEnd = qualityTrimEnd(sequence, quality, options);
+  if (qualityEnd < sequence.length) {
+    sequence = sequence.slice(0, qualityEnd);
+    quality = quality.slice(0, qualityEnd);
+    trims.quality = true;
+  }
+
+  if (options.trimPolyG) {
+    const run = trailingRun(sequence, 'G');
+    if (run >= options.polyMin) {
+      sequence = sequence.slice(0, sequence.length - run);
+      quality = quality.slice(0, quality.length - run);
+      trims.poly_g = true;
+    }
+  }
+  if (options.trimPolyA) {
+    const run = trailingRun(sequence, 'A');
+    if (run >= options.polyMin) {
+      sequence = sequence.slice(0, sequence.length - run);
+      quality = quality.slice(0, quality.length - run);
+      trims.poly_a = true;
+    }
+  }
+
+  if (sequence.length === 0) return { dropped: 'length', trims };
+  if (sequence.length < options.minLength) return { dropped: 'length', trims };
+  const nPercent = (ambiguousCount(sequence) / sequence.length) * 100;
+  if (nPercent > options.maxNPercent) return { dropped: 'n', trims };
+  return {
+    entry: {
+      id: entry.id,
+      description: entry.description ?? '',
+      sequence,
+      quality,
+    },
+    trims,
+  };
+}
+
+/**
+ * Trim/filter a set of parsed FASTQ entries.
+ *
+ * @param {Array<object>} entries
+ * @param {object} [options]
+ * @returns {{entries: Array<object>, stats: object}}
+ */
+export function trimFastq(entries, options = {}) {
+  if (!Array.isArray(entries) || entries.length === 0) throw new MolbioInputError('FASTQ trimming needs at least one read');
+  const resolved = resolveTrimOptions(options);
+  const limit = resolved.maxReads === 0 ? FASTQ_TRIM_DEFAULTS.maxReadsHard : resolved.maxReads;
+  const scanned = entries.slice(0, limit);
+  const kept = [];
+  const trimCounts = { quality: 0, adapter: 0, poly_g: 0, poly_a: 0, primer: 0 };
+  const dropCounts = { length: 0, n: 0 };
+  let basesIn = 0;
+  let basesOut = 0;
+  let trimmedReads = 0;
+  for (const entry of scanned) {
+    basesIn += entry.sequence.length;
+    const result = trimRead(entry, resolved);
+    if (result.entry === undefined) {
+      dropCounts[result.dropped]++;
+      continue;
+    }
+    kept.push(result.entry);
+    basesOut += result.entry.sequence.length;
+    let trimmed = false;
+    for (const [key, value] of Object.entries(result.trims)) {
+      if (value) {
+        trimCounts[key]++;
+        trimmed = true;
+      }
+    }
+    if (trimmed) trimmedReads++;
+  }
+  return {
+    entries: kept,
+    stats: {
+      total_reads: entries.length,
+      reads_in: scanned.length,
+      reads_out: kept.length,
+      bases_in: basesIn,
+      bases_out: basesOut,
+      trimmed_reads: trimmedReads,
+      dropped_reads: scanned.length - kept.length,
+      trim_counts: trimCounts,
+      drop_counts: dropCounts,
+      truncated: entries.length > scanned.length,
+      criteria: {
+        min_quality: resolved.minQuality,
+        quality_window: resolved.qualityWindow,
+        poly_min: resolved.polyMin,
+        min_length: resolved.minLength,
+        max_n_percent: resolved.maxNPercent,
+        phred: resolved.phred,
+        max_reads: resolved.maxReads,
+        adapter: resolved.adapter,
+        primer: resolved.primer,
+        poly_g: resolved.trimPolyG,
+        poly_a: resolved.trimPolyA,
+      },
+    },
+  };
+}