@jbrowse/cli 4.3.0 → 5.0.0-beta.2

package/dist/commands/make-density/density-generator.js

@@ -0,0 +1,180 @@
+import { createReadStream, existsSync, readFileSync } from 'node:fs';
+import { createInterface } from 'node:readline';
+import { createGunzip } from 'node:zlib';
+// Which parser a filename selects. BAM/CRAM/SAM are named so the refusal can
+// say what to do instead, rather than falling into the unknown-extension case.
+const formats = [
+    { regex: /\.gff3?(\.b?gz)?$/i, format: 'gff3' },
+    { regex: /\.gtf(\.b?gz)?$/i, format: 'gtf' },
+    { regex: /\.vcf(\.b?gz)?$/i, format: 'vcf' },
+    { regex: /\.bed(\.b?gz)?$/i, format: 'bed' },
+];
+const alignmentFormat = /\.(bam|cram|sam)$/i;
+export function densityFormat(file) {
+    const hit = formats.find(f => f.regex.test(file));
+    if (!hit) {
+        throw new Error(alignmentFormat.test(file)
+            ? `${file} is an alignment file, which make-density cannot read. Count coverage with a tool that reads BAM/CRAM (e.g. "bamCoverage" or "samtools depth" piped to bedGraphToBigWig) and pass the result with add-track --density.`
+            : `Cannot tell what kind of file ${file} is. make-density reads GFF3, GTF, BED and VCF, plain or bgzipped.`);
+    }
+    return hit.format;
+}
+/**
+ * The reference name and 0-based start of one record, or undefined for a line
+ * that contributes nothing: a header, a directive, a short line, or — in GFF3 —
+ * a child feature, whose start would count its parent's locus a second time.
+ * A GTF row says whether it is a `transcript`, the record its track draws;
+ * BED and VCF have no parent link to follow, so every record counts.
+ */
+function parseRecord(line, format) {
+    const columns = line.split('\t');
+    const refName = columns[0];
+    const startsHeader = line.startsWith('#') ||
+        line.startsWith('track ') ||
+        line.startsWith('browser ');
+    if (startsHeader || refName === undefined || refName === '') {
+        return undefined;
+    }
+    if (format === 'bed') {
+        const start = columns[1];
+        return start === undefined
+            ? undefined
+            : { refName, start: +start, transcript: false };
+    }
+    const start = columns[format === 'vcf' ? 1 : 3];
+    const attributes = columns[8];
+    const isChild = format === 'gff3' && attributes?.includes('Parent=');
+    return start === undefined || isChild
+        ? undefined
+        : {
+            refName,
+            start: +start - 1,
+            transcript: format === 'gtf' && columns[2] === 'transcript',
+        };
+}
+function tally(bins, key, bin) {
+    const existing = bins.get(key);
+    const perRef = existing === undefined ? new Map() : existing;
+    if (existing === undefined) {
+        bins.set(key, perRef);
+    }
+    const count = perRef.get(bin);
+    perRef.set(bin, count === undefined ? 1 : count + 1);
+}
+export async function countFeatureStarts({ file, format, binSize, chromSizes, }) {
+    // A GTF's transcripts are tallied apart from its exons and CDS rows: where a
+    // file has transcript rows they are the count, and a file with none (an
+    // exon-only GTF) counts every row.
+    const all = new Map();
+    const transcripts = new Map();
+    const unknownRefNames = new Set();
+    let records = 0;
+    let transcriptRecords = 0;
+    const source = createReadStream(file);
+    const lines = createInterface({
+        input: /\.b?gz$/i.test(file) ? source.pipe(createGunzip()) : source,
+        crlfDelay: Number.POSITIVE_INFINITY,
+    });
+    for await (const line of lines) {
+        // everything after a GFF3 ##FASTA directive is sequence, not features
+        if (line.trimEnd() === '##FASTA') {
+            break;
+        }
+        const record = parseRecord(line, format);
+        if (record && Number.isFinite(record.start)) {
+            records++;
+            transcriptRecords += record.transcript ? 1 : 0;
+            const length = chromSizes.get(record.refName);
+            if (length === undefined) {
+                unknownRefNames.add(record.refName);
+            }
+            else if (record.start >= 0 && record.start < length) {
+                const bin = Math.floor(record.start / binSize);
+                tally(all, record.refName, bin);
+                if (record.transcript) {
+                    tally(transcripts, record.refName, bin);
+                }
+            }
+        }
+    }
+    lines.close();
+    source.destroy();
+    return transcriptRecords > 0
+        ? { bins: transcripts, records: transcriptRecords, unknownRefNames }
+        : { bins: all, records, unknownRefNames };
+}
+/**
+ * bedGraph rows covering every base of every reference, in the order
+ * bedGraphToBigWig demands (`sort -k1,1 -k2,2n`): one row per counted bin,
+ * clipped to the reference's length, and one per run of empty bins between
+ * them. The empty runs are load-bearing: a bigWig's zoom levels average over
+ * the bases its rows cover, so a file that skipped them would read, zoomed out,
+ * as the mean over the bins that held something rather than over the span, and
+ * a gene desert beside a cluster would score the same.
+ */
+export function* bedGraphLines({ bins, chromSizes, binSize, }) {
+    const byName = [...chromSizes.entries()].sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0);
+    for (const [refName, length] of byName) {
+        const perRef = bins.get(refName);
+        const counted = perRef
+            ? [...perRef.entries()].sort(([a], [b]) => a - b)
+            : [];
+        let cursor = 0;
+        for (const [bin, count] of counted) {
+            const start = bin * binSize;
+            const end = Math.min(start + binSize, length);
+            if (start > cursor) {
+                yield `${refName}\t${cursor}\t${start}\t0\n`;
+            }
+            yield `${refName}\t${start}\t${end}\t${count}\n`;
+            cursor = end;
+        }
+        if (cursor < length) {
+            yield `${refName}\t${cursor}\t${length}\t0\n`;
+        }
+    }
+}
+export function chromSizesLines(chromSizes) {
+    return [...chromSizes.entries()]
+        .map(([refName, length]) => `${refName}\t${length}\n`)
+        .join('');
+}
+// a `.chrom.sizes` and a `.fai` both put the name in column 1 and the length in
+// column 2, so one parser reads either
+function parseChromSizes(text, source) {
+    const sizes = new Map();
+    for (const line of text.split('\n')) {
+        const [refName, length] = line.split('\t');
+        if (refName && length !== undefined && Number.isFinite(+length)) {
+            sizes.set(refName, +length);
+        }
+    }
+    if (sizes.size === 0) {
+        throw new Error(`No reference sequences found in ${source}`);
+    }
+    return sizes;
+}
+/**
+ * The reference lengths the bigWig header needs, from an explicit
+ * `--chrom-sizes` or from the `.fai` beside an `--assembly` FASTA.
+ */
+export function resolveChromSizes({ chromSizes, assembly, }) {
+    if (chromSizes !== undefined) {
+        return parseChromSizes(readFileSync(chromSizes, 'utf8'), chromSizes);
+    }
+    if (assembly !== undefined) {
+        const fai = `${assembly}.fai`;
+        if (!existsSync(fai)) {
+            throw new Error(`No index beside ${assembly}: expected ${fai}. Make one with "samtools faidx ${assembly}", or pass --chrom-sizes instead.`);
+        }
+        return parseChromSizes(readFileSync(fai, 'utf8'), fai);
+    }
+    throw new Error('Reference lengths are required: pass --chrom-sizes <file> or --assembly <fasta> (whose .fai is read)');
+}
+export function parseBinSize(bin) {
+    const binSize = +bin;
+    if (!Number.isInteger(binSize) || binSize < 1) {
+        throw new Error(`--bin must be a positive whole number of bp, got ${bin}`);
+    }
+    return binSize;
+}

package/dist/commands/make-density/index.js

@@ -0,0 +1,139 @@
+import { spawn } from 'node:child_process';
+import { createWriteStream, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { Readable } from 'node:stream';
+import { pipeline } from 'node:stream/promises';
+import { parseArgs } from 'node:util';
+import { printHelp, requirePositional } from '../../utils.js';
+import { describeExit, waitForProcessClose } from '../process-utils.js';
+import { densitySidecarPath } from '../shared/density.js';
+import { validateFileArgument, validateRequiredCommands, } from '../shared/validators.js';
+import { bedGraphLines, chromSizesLines, countFeatureStarts, densityFormat, parseBinSize, resolveChromSizes, } from './density-generator.js';
+export async function run(args) {
+    const options = {
+        help: {
+            type: 'boolean',
+            short: 'h',
+        },
+        bin: {
+            type: 'string',
+            default: '1000',
+            description: 'Width of each bin, in bp',
+        },
+        'chrom-sizes': {
+            type: 'string',
+            description: 'Two-column reference name and length table for the assembly the file is on',
+        },
+        assembly: {
+            type: 'string',
+            description: 'FASTA of the assembly the file is on; its .fai supplies the reference lengths, in place of --chrom-sizes',
+        },
+        out: {
+            type: 'string',
+            description: 'Where to write the bigWig. Defaults to <file>.density.bw beside the input, which is where add-track looks for it',
+        },
+    };
+    const { values: flags, positionals } = parseArgs({
+        args,
+        options,
+        allowPositionals: true,
+    });
+    const description = 'Counts feature starts per bin into a bigWig, the density sidecar a track draws where the region is too large to fetch its features';
+    const notes = "The sidecar is attached to a track through its adapter's densityAdapter " +
+        'slot, which BamAdapter, CramAdapter, HtsgetBamAdapter, Gff3TabixAdapter, ' +
+        'GtfTabixAdapter, BedTabixAdapter, BigBedAdapter, VcfTabixAdapter and ' +
+        'SplitVcfTabixAdapter declare. add-track attaches it on its own when the ' +
+        'default <file>.density.bw sits beside the track file, so building it before ' +
+        'adding the track is all it takes; --density attaches one by path or URL.\n\n' +
+        'GFF3 counts only top-level features — a line whose attributes carry no ' +
+        'Parent= — so a gene is one count rather than one per exon. GTF, BED and VCF ' +
+        'have no parent link to follow, so every record counts.\n\n' +
+        'BAM and CRAM are out of scope: read depth is not feature density, and the ' +
+        'tools that compute it already write bigWig. Point add-track --density at ' +
+        'what they produce.\n\n' +
+        'Requires bedGraphToBigWig (UCSC) on the PATH.';
+    const examples = [
+        '# writes genes.gff3.density.bw beside the input, in 1kb bins',
+        '$ jbrowse make-density genes.gff3.gz --chrom-sizes hg38.chrom.sizes',
+        '',
+        '# take the reference lengths from the assembly FASTA index instead',
+        '$ jbrowse make-density variants.vcf.gz --assembly hg38.fa',
+        '',
+        '# coarser bins, and a name of your own',
+        '$ jbrowse make-density genes.gff3.gz --assembly hg38.fa --bin 10000 --out genes.10kb.bw',
+    ];
+    const usage = 'jbrowse make-density <file> [options]';
+    if (flags.help) {
+        printHelp({
+            description,
+            examples,
+            notes,
+            usage,
+            options,
+        });
+        return;
+    }
+    const file = positionals[0];
+    requirePositional(file, 'file', usage);
+    // ahead of the bedGraphToBigWig check, so a file this command cannot read
+    // says so even where UCSC's tools are not installed
+    const format = densityFormat(file);
+    validateFileArgument(file, 'make-density', format);
+    const binSize = parseBinSize(flags.bin);
+    const chromSizes = resolveChromSizes({
+        chromSizes: flags['chrom-sizes'],
+        assembly: flags.assembly,
+    });
+    validateRequiredCommands(['bedGraphToBigWig']);
+    const outputFile = flags.out === undefined ? densitySidecarPath(file) : flags.out;
+    const { bins, records, unknownRefNames } = await countFeatureStarts({
+        file,
+        format,
+        binSize,
+        chromSizes,
+    });
+    if (unknownRefNames.size > 0) {
+        const shown = [...unknownRefNames].slice(0, 5).join(', ');
+        const rest = unknownRefNames.size - Math.min(unknownRefNames.size, 5);
+        console.warn(`Warning: skipped records on ${unknownRefNames.size} reference name(s) absent from the reference lengths (${shown}${rest > 0 ? `, and ${rest} more` : ''}). A chr-prefixed file against an unprefixed assembly looks exactly like this.`);
+    }
+    // A file that yielded nothing is refused rather than written as a flat
+    // band, which would read as "no features here" rather than "wrong file"
+    const binCount = [...bins.values()].reduce((sum, perRef) => sum + perRef.size, 0);
+    if (binCount === 0) {
+        throw new Error(`No feature starts found in ${file} on any reference in the given lengths (${records} record(s) read). Is this the right file and assembly?`);
+    }
+    const tmpDir = mkdtempSync(path.join(os.tmpdir(), 'jbrowse-density-'));
+    try {
+        const bedGraph = path.join(tmpDir, 'density.bedGraph');
+        // normalized rather than the caller's file, so a .fai (five columns) and a
+        // chrom.sizes with stray whitespace both reach bedGraphToBigWig as the two
+        // columns it reads
+        const sizes = path.join(tmpDir, 'reference.chrom.sizes');
+        writeFileSync(sizes, chromSizesLines(chromSizes));
+        await pipeline(Readable.from(bedGraphLines({ bins, chromSizes, binSize })), createWriteStream(bedGraph));
+        const child = spawn('bedGraphToBigWig', [bedGraph, sizes, outputFile], {
+            stdio: ['ignore', process.stdout, process.stderr],
+        });
+        const exit = await waitForProcessClose(child);
+        if (exit.code !== 0) {
+            throw new Error(`bedGraphToBigWig exited with ${describeExit(exit)}`);
+        }
+    }
+    finally {
+        rmSync(tmpDir, { recursive: true, force: true });
+    }
+    // The slot lives on the indexed adapters, so a plain text file has to be
+    // bgzipped and tabix-indexed before the track can carry the sidecar
+    const indexed = /\.b?gz$/i.test(file);
+    const preset = format === 'bed' ? 'bed' : format === 'vcf' ? 'vcf' : 'gff';
+    const trackFile = indexed ? file : `${file}.gz`;
+    const indexFirst = indexed
+        ? ''
+        : `  bgzip ${file} && tabix -p ${preset} ${trackFile}\n`;
+    const nextCommand = outputFile === densitySidecarPath(file)
+        ? `Next, add the track it summarizes — a sidecar under this name is picked up automatically:\n${indexFirst}  jbrowse add-track ${trackFile} --load copy`
+        : `Next, add the track it summarizes, naming this sidecar:\n${indexFirst}  jbrowse add-track ${trackFile} --load copy --density ${outputFile}`;
+    console.log(`Created ${outputFile}: ${records} feature start(s) in ${binCount} non-empty ${binSize}bp bin(s)\n\n${nextCommand}`);
+}

package/dist/commands/make-pif/index.js

@@ -1,7 +1,9 @@
-import { parseArgs } from 'util';
-import { createPIF, getOutputFilename, spawnSortProcess, waitForProcessClose, } from "./pif-generator.js";
-import { printHelp } from "../../utils.js";
-import { validateFileArgument, validateRequiredCommands, } from "../shared/validators.js";
+import fs from 'node:fs';
+import { parseArgs } from 'node:util';
+import { printHelp } from '../../utils.js';
+import { describeExit, waitForProcessClose } from '../process-utils.js';
+import { validateFileArgument, validateRequiredCommands, } from '../shared/validators.js';
+import { DEFAULT_BGZIP_THREADS, DEFAULT_COARSE_GAP, createPIF, getOutputFilename, missingPairs, spawnSortProcess, } from './pif-generator.js';
 export async function run(args) {
     const options = {
         help: {
@@ -16,6 +18,18 @@ export async function run(args) {
             type: 'boolean',
             description: 'Create a CSI index for the PIF file instead of TBI',
         },
+        coarse: {
+            type: 'string',
+            description: 'Accuracy bound (bp) of the coarse tier. A coarse row replaces its CIGAR with a coarse CIGAR (cr:Z:) that keeps every indel longer than half this length and folds the rest into runs, each run staying within this many bp of the real alignment. Defaults to 10000. The coarse tier (prefix T/Q) is emitted alongside the per-row CIGAR fine tier by default so whole-genome synteny views can auto-switch to it; pass --no-coarse to omit it.',
+        },
+        'no-coarse': {
+            type: 'boolean',
+            description: 'Do not emit the coarse tier; write only the per-row CIGAR fine tier.',
+        },
+        threads: {
+            type: 'string',
+            description: 'Compression threads for bgzip. Defaults to 4. Raise it on a machine with cores to spare, or set 1 to leave the rest of the machine alone.',
+        },
     };
     const { values: flags, positionals } = parseArgs({
         args,
@@ -24,14 +38,33 @@ export async function run(args) {
     });
     const description = 'creates pairwise indexed PAF (PIF), with bgzip and tabix';
     const examples = [
-        '$ jbrowse make-pif input.paf # creates input.pif.gz in same directory',
+        '# creates input.pif.gz and input.pif.gz.tbi in the same directory',
+        '$ jbrowse make-pif input.paf',
+        '',
+        '# specify the output file, also creates output.pif.gz.tbi',
+        '$ jbrowse make-pif input.paf --out output.pif.gz',
+        '',
+        '# use a CSI index for assemblies with chromosomes longer than ~512 Mb',
+        '$ jbrowse make-pif input.paf --csi',
+        '',
+        '# a looser coarse tier: runs within 50kb of the alignment, indels over 25kb kept',
+        '$ jbrowse make-pif input.paf --coarse 50000',
+        '',
+        '# emit only the per-row CIGAR fine tier, skipping the coarse tier',
+        '$ jbrowse make-pif input.paf --no-coarse',
         '',
-        '$ jbrowse make-pif input.paf --out output.pif.gz # specify output file, creates output.pif.gz.tbi also',
+        '# give bgzip more (or fewer) compression threads than the default 4',
+        '$ jbrowse make-pif input.paf --threads 8',
     ];
+    const notes = 'Use --csi for assemblies containing sequences longer than ~512 Mb. The ' +
+        'default TBI index cannot address coordinates beyond 2^29 (~536 Mb), so a ' +
+        'CSI index is required for large chromosomes (e.g. some plant and ' +
+        'amphibian genomes). Requires sh, sort, bgzip, and tabix on the PATH.';
     if (flags.help) {
         printHelp({
             description,
             examples,
+            notes,
             usage: 'jbrowse make-pif <file> [options]',
             options,
         });
@@ -39,11 +72,95 @@ export async function run(args) {
     }
     const file = positionals[0];
     validateFileArgument(file, 'make-pif', 'paf');
-    validateRequiredCommands(['sh', 'sort', 'grep', 'tabix', 'bgzip']);
-    const { out, csi = false } = flags;
+    validateRequiredCommands(['sh', 'sort', 'tabix', 'bgzip']);
+    const { out, csi = false, coarse, 'no-coarse': noCoarse = false, threads, } = flags;
     const outputFile = getOutputFilename(file, out);
-    const child = spawnSortProcess(outputFile, csi);
-    await createPIF(file, child.stdin);
+    // --no-coarse used to silently win over an explicit --coarse, so a run asking
+    // for both wrote a file with no coarse tier and said nothing about it
+    if (noCoarse && coarse !== undefined) {
+        throw new Error('--coarse and --no-coarse are mutually exclusive');
+    }
+    const coarseGap = noCoarse
+        ? undefined
+        : coarse === undefined
+            ? DEFAULT_COARSE_GAP
+            : +coarse;
+    // a whole number because the header states it as an `i` tag, and positive
+    // because a coarse row without a fold means "one run within the bound"
+    if (coarseGap !== undefined &&
+        (!Number.isInteger(coarseGap) || coarseGap <= 0)) {
+        throw new Error(`Invalid --coarse value: ${coarse} (must be a positive whole number of bp)`);
+    }
+    const bgzipThreads = threads === undefined ? DEFAULT_BGZIP_THREADS : +threads;
+    if (!Number.isInteger(bgzipThreads) || bgzipThreads < 1) {
+        throw new Error(`Invalid --threads value: ${threads}`);
+    }
+    const indexExt = csi ? 'csi' : 'tbi';
+    const indexFile = `${outputFile}.${indexExt}`;
+    // The pipeline writes under a temporary name, renamed into place only once
+    // the pass, the pipeline and the row census have all succeeded: a truncated
+    // .paf.gz or a non-PAF input used to leave a valid, indexed PIF of whatever
+    // preceded the failure, which a build script reads as done.
+    const partFile = `${outputFile}.part`;
+    const partIndex = `${partFile}.${indexExt}`;
+    const child = spawnSortProcess(partFile, csi, bgzipThreads);
+    const closed = waitForProcessClose(child);
+    const pass = await createPIF(file, child.stdin, coarseGap).then(stats => ({ stats }), (error) => ({
+        error: error instanceof Error ? error : new Error(String(error)),
+    }));
     child.stdin.end();
-    await waitForProcessClose(child);
+    // no SIGPIPE exemption here (unlike sort-gff/sort-bed): this pipeline's output
+    // is the .pif.gz file, so anything that killed it early left that file broken
+    const exit = await closed;
+    const discardPart = () => {
+        fs.rmSync(partFile, { force: true });
+        fs.rmSync(partIndex, { force: true });
+    };
+    if ('error' in pass) {
+        discardPart();
+        throw pass.error;
+    }
+    if (exit.code !== 0) {
+        discardPart();
+        throw new Error(`PIF sort/index pipeline exited with ${describeExit(exit)}`);
+    }
+    const { stats } = pass;
+    const { samples, rows, skipped } = stats;
+    if (rows === 0) {
+        discardPart();
+        throw new Error(`No valid PAF rows found in ${file ?? 'stdin'} (${skipped} line(s) had fewer than the 12 mandatory PAF columns). Is this a PAF file?`);
+    }
+    fs.renameSync(partFile, outputFile);
+    fs.renameSync(partIndex, indexFile);
+    if (skipped > 0) {
+        console.warn(`Warning: skipped ${skipped} of ${rows + skipped} line(s) with fewer than the 12 mandatory PAF columns`);
+    }
+    // An all-vs-all PAF that does not state every pair indexes perfectly well and
+    // then draws an empty synteny band for each pair the aligner never emitted,
+    // which reads as "no homology here" rather than as "never computed". This is
+    // the moment someone is looking at the file, and the census it needs was
+    // already taken on the pass above.
+    const missing = missingPairs(stats);
+    if (missing.length > 0) {
+        const shown = missing
+            .slice(0, 5)
+            .map(([a, b]) => `${a}<->${b}`)
+            .join(', ');
+        const rest = missing.length - Math.min(missing.length, 5);
+        console.warn(`Warning: this file has ${samples.size} samples but states only ` +
+            `${(samples.size * (samples.size - 1)) / 2 - missing.length} of their ` +
+            `${(samples.size * (samples.size - 1)) / 2} pairs. Synteny bands for the ` +
+            `missing pairs (${shown}${rest > 0 ? `, and ${rest} more` : ''}) will draw ` +
+            `empty. That is what a reference-anchored alignment looks like — only ` +
+            `pairs involving the reference are stated. Order the synteny rows so the ` +
+            `reference sits between the others, use a complete all-vs-all if the ` +
+            `project publishes one, or view a larger cohort as a multiple alignment ` +
+            `(MAF) rather than a stack of pairwise bands.`);
+    }
+    const nextCommand = samples.size > 0
+        ? 'Next, add it as an all-vs-all synteny track (PanSN names detected):\n' +
+            `  jbrowse add-track ${outputFile} --adapterType AllVsAllIndexedPAFAdapter -a ${[...samples].sort().join(',')} --load copy`
+        : 'Next, add it as a synteny track (set -a to your query,target assembly names):\n' +
+            `  jbrowse add-track ${outputFile} -a query,target --load copy`;
+    console.log(`Created ${outputFile} and ${indexFile}\n\n${nextCommand}`);
 }