@jbrowse/cli 5.0.0-beta.1 → 5.0.0-beta.3

package/dist/commands/make-density/index.js

@@ -0,0 +1,139 @@
+import { spawn } from 'node:child_process';
+import { createWriteStream, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { Readable } from 'node:stream';
+import { pipeline } from 'node:stream/promises';
+import { parseArgs } from 'node:util';
+import { printHelp, requirePositional } from '../../utils.js';
+import { describeExit, waitForProcessClose } from '../process-utils.js';
+import { densitySidecarPath } from '../shared/density.js';
+import { validateFileArgument, validateRequiredCommands, } from '../shared/validators.js';
+import { bedGraphLines, chromSizesLines, countFeatureStarts, densityFormat, parseBinSize, resolveChromSizes, } from './density-generator.js';
+export async function run(args) {
+    const options = {
+        help: {
+            type: 'boolean',
+            short: 'h',
+        },
+        bin: {
+            type: 'string',
+            default: '1000',
+            description: 'Width of each bin, in bp',
+        },
+        'chrom-sizes': {
+            type: 'string',
+            description: 'Two-column reference name and length table for the assembly the file is on',
+        },
+        assembly: {
+            type: 'string',
+            description: 'FASTA of the assembly the file is on; its .fai supplies the reference lengths, in place of --chrom-sizes',
+        },
+        out: {
+            type: 'string',
+            description: 'Where to write the bigWig. Defaults to <file>.density.bw beside the input, which is where add-track looks for it',
+        },
+    };
+    const { values: flags, positionals } = parseArgs({
+        args,
+        options,
+        allowPositionals: true,
+    });
+    const description = 'Counts feature starts per bin into a bigWig, the density sidecar a track draws where the region is too large to fetch its features';
+    const notes = "The sidecar is attached to a track through its adapter's densityAdapter " +
+        'slot, which BamAdapter, CramAdapter, HtsgetBamAdapter, Gff3TabixAdapter, ' +
+        'GtfTabixAdapter, BedTabixAdapter, BigBedAdapter, VcfTabixAdapter and ' +
+        'SplitVcfTabixAdapter declare. add-track attaches it on its own when the ' +
+        'default <file>.density.bw sits beside the track file, so building it before ' +
+        'adding the track is all it takes; --density attaches one by path or URL.\n\n' +
+        'GFF3 counts only top-level features — a line whose attributes carry no ' +
+        'Parent= — so a gene is one count rather than one per exon. GTF, BED and VCF ' +
+        'have no parent link to follow, so every record counts.\n\n' +
+        'BAM and CRAM are out of scope: read depth is not feature density, and the ' +
+        'tools that compute it already write bigWig. Point add-track --density at ' +
+        'what they produce.\n\n' +
+        'Requires bedGraphToBigWig (UCSC) on the PATH.';
+    const examples = [
+        '# writes genes.gff3.density.bw beside the input, in 1kb bins',
+        '$ jbrowse make-density genes.gff3.gz --chrom-sizes hg38.chrom.sizes',
+        '',
+        '# take the reference lengths from the assembly FASTA index instead',
+        '$ jbrowse make-density variants.vcf.gz --assembly hg38.fa',
+        '',
+        '# coarser bins, and a name of your own',
+        '$ jbrowse make-density genes.gff3.gz --assembly hg38.fa --bin 10000 --out genes.10kb.bw',
+    ];
+    const usage = 'jbrowse make-density <file> [options]';
+    if (flags.help) {
+        printHelp({
+            description,
+            examples,
+            notes,
+            usage,
+            options,
+        });
+        return;
+    }
+    const file = positionals[0];
+    requirePositional(file, 'file', usage);
+    // ahead of the bedGraphToBigWig check, so a file this command cannot read
+    // says so even where UCSC's tools are not installed
+    const format = densityFormat(file);
+    validateFileArgument(file, 'make-density', format);
+    const binSize = parseBinSize(flags.bin);
+    const chromSizes = resolveChromSizes({
+        chromSizes: flags['chrom-sizes'],
+        assembly: flags.assembly,
+    });
+    validateRequiredCommands(['bedGraphToBigWig']);
+    const outputFile = flags.out === undefined ? densitySidecarPath(file) : flags.out;
+    const { bins, records, unknownRefNames } = await countFeatureStarts({
+        file,
+        format,
+        binSize,
+        chromSizes,
+    });
+    if (unknownRefNames.size > 0) {
+        const shown = [...unknownRefNames].slice(0, 5).join(', ');
+        const rest = unknownRefNames.size - Math.min(unknownRefNames.size, 5);
+        console.warn(`Warning: skipped records on ${unknownRefNames.size} reference name(s) absent from the reference lengths (${shown}${rest > 0 ? `, and ${rest} more` : ''}). A chr-prefixed file against an unprefixed assembly looks exactly like this.`);
+    }
+    // A file that yielded nothing is refused rather than written as a flat
+    // band, which would read as "no features here" rather than "wrong file"
+    const binCount = [...bins.values()].reduce((sum, perRef) => sum + perRef.size, 0);
+    if (binCount === 0) {
+        throw new Error(`No feature starts found in ${file} on any reference in the given lengths (${records} record(s) read). Is this the right file and assembly?`);
+    }
+    const tmpDir = mkdtempSync(path.join(os.tmpdir(), 'jbrowse-density-'));
+    try {
+        const bedGraph = path.join(tmpDir, 'density.bedGraph');
+        // normalized rather than the caller's file, so a .fai (five columns) and a
+        // chrom.sizes with stray whitespace both reach bedGraphToBigWig as the two
+        // columns it reads
+        const sizes = path.join(tmpDir, 'reference.chrom.sizes');
+        writeFileSync(sizes, chromSizesLines(chromSizes));
+        await pipeline(Readable.from(bedGraphLines({ bins, chromSizes, binSize })), createWriteStream(bedGraph));
+        const child = spawn('bedGraphToBigWig', [bedGraph, sizes, outputFile], {
+            stdio: ['ignore', process.stdout, process.stderr],
+        });
+        const exit = await waitForProcessClose(child);
+        if (exit.code !== 0) {
+            throw new Error(`bedGraphToBigWig exited with ${describeExit(exit)}`);
+        }
+    }
+    finally {
+        rmSync(tmpDir, { recursive: true, force: true });
+    }
+    // The slot lives on the indexed adapters, so a plain text file has to be
+    // bgzipped and tabix-indexed before the track can carry the sidecar
+    const indexed = /\.b?gz$/i.test(file);
+    const preset = format === 'bed' ? 'bed' : format === 'vcf' ? 'vcf' : 'gff';
+    const trackFile = indexed ? file : `${file}.gz`;
+    const indexFirst = indexed
+        ? ''
+        : `  bgzip ${file} && tabix -p ${preset} ${trackFile}\n`;
+    const nextCommand = outputFile === densitySidecarPath(file)
+        ? `Next, add the track it summarizes — a sidecar under this name is picked up automatically:\n${indexFirst}  jbrowse add-track ${trackFile} --load copy`
+        : `Next, add the track it summarizes, naming this sidecar:\n${indexFirst}  jbrowse add-track ${trackFile} --load copy --density ${outputFile}`;
+    console.log(`Created ${outputFile}: ${records} feature start(s) in ${binCount} non-empty ${binSize}bp bin(s)\n\n${nextCommand}`);
+}

package/dist/commands/make-pif/index.js

@@ -1,8 +1,9 @@
+import fs from 'node:fs';
 import { parseArgs } from 'node:util';
 import { printHelp } from '../../utils.js';
 import { describeExit, waitForProcessClose } from '../process-utils.js';
 import { validateFileArgument, validateRequiredCommands, } from '../shared/validators.js';
-import { DEFAULT_COARSE_SPLIT_GAP, createPIF, getOutputFilename, missingPairs, spawnSortProcess, } from './pif-generator.js';
+import { DEFAULT_BGZIP_THREADS, DEFAULT_COARSE_GAP, createPIF, getOutputFilename, missingPairs, spawnSortProcess, } from './pif-generator.js';
 export async function run(args) {
     const options = {
         help: {
@@ -19,11 +20,15 @@ export async function run(args) {
         },
         coarse: {
             type: 'string',
-            description: 'Minimum insertion/deletion length (bp) at which a coarse-tier row is split into multiple pieces so each row stays tight — 0 emits an unsplit coarse tier. Defaults to 10000. The no-CIGAR coarse tier (prefix T/Q) is emitted alongside the per-row CIGAR fine tier by default so whole-genome synteny views can auto-switch to it; pass --no-coarse to omit it.',
+            description: 'Accuracy bound (bp) of the coarse tier. A coarse row replaces its CIGAR with a coarse CIGAR (cr:Z:) that keeps every indel longer than half this length and folds the rest into runs, each run staying within this many bp of the real alignment. Defaults to 10000. The coarse tier (prefix T/Q) is emitted alongside the per-row CIGAR fine tier by default so whole-genome synteny views can auto-switch to it; pass --no-coarse to omit it.',
         },
         'no-coarse': {
             type: 'boolean',
-            description: 'Do not emit the coarse no-CIGAR tier; write only the per-row CIGAR fine tier.',
+            description: 'Do not emit the coarse tier; write only the per-row CIGAR fine tier.',
+        },
+        threads: {
+            type: 'string',
+            description: 'Compression threads for bgzip. Defaults to 4. Raise it on a machine with cores to spare, or set 1 to leave the rest of the machine alone.',
         },
     };
     const { values: flags, positionals } = parseArgs({
@@ -42,11 +47,14 @@ export async function run(args) {
         '# use a CSI index for assemblies with chromosomes longer than ~512 Mb',
         '$ jbrowse make-pif input.paf --csi',
         '',
-        '# emit an unsplit coarse tier (alongside the fine tier)',
-        '$ jbrowse make-pif input.paf --coarse 0',
+        '# a looser coarse tier: runs within 50kb of the alignment, indels over 25kb kept',
+        '$ jbrowse make-pif input.paf --coarse 50000',
         '',
         '# emit only the per-row CIGAR fine tier, skipping the coarse tier',
         '$ jbrowse make-pif input.paf --no-coarse',
+        '',
+        '# give bgzip more (or fewer) compression threads than the default 4',
+        '$ jbrowse make-pif input.paf --threads 8',
     ];
     const notes = 'Use --csi for assemblies containing sequences longer than ~512 Mb. The ' +
         'default TBI index cannot address coordinates beyond 2^29 (~536 Mb), so a ' +
@@ -65,42 +73,65 @@ export async function run(args) {
     const file = positionals[0];
     validateFileArgument(file, 'make-pif', 'paf');
     validateRequiredCommands(['sh', 'sort', 'tabix', 'bgzip']);
-    const { out, csi = false, coarse, 'no-coarse': noCoarse = false } = flags;
+    const { out, csi = false, coarse, 'no-coarse': noCoarse = false, threads, } = flags;
     const outputFile = getOutputFilename(file, out);
     // --no-coarse used to silently win over an explicit --coarse, so a run asking
     // for both wrote a file with no coarse tier and said nothing about it
     if (noCoarse && coarse !== undefined) {
         throw new Error('--coarse and --no-coarse are mutually exclusive');
     }
-    const coarseSplitGap = noCoarse
+    const coarseGap = noCoarse
         ? undefined
         : coarse === undefined
-            ? DEFAULT_COARSE_SPLIT_GAP
+            ? DEFAULT_COARSE_GAP
             : +coarse;
-    if (coarseSplitGap !== undefined &&
-        (!Number.isFinite(coarseSplitGap) || coarseSplitGap < 0)) {
-        throw new Error(`Invalid --coarse value: ${coarse}`);
+    // a whole number because the header states it as an `i` tag, and positive
+    // because a coarse row without a fold means "one run within the bound"
+    if (coarseGap !== undefined &&
+        (!Number.isInteger(coarseGap) || coarseGap <= 0)) {
+        throw new Error(`Invalid --coarse value: ${coarse} (must be a positive whole number of bp)`);
     }
-    const child = spawnSortProcess(outputFile, csi);
-    const stdin = child.stdin;
-    // end stdin even if createPIF throws, otherwise the spawned sort/index child
-    // is left running with an open stdin
-    const stats = await createPIF(file, stdin, coarseSplitGap).finally(() => {
-        stdin.end();
-    });
-    const { samples, rows, skipped } = stats;
+    const bgzipThreads = threads === undefined ? DEFAULT_BGZIP_THREADS : +threads;
+    if (!Number.isInteger(bgzipThreads) || bgzipThreads < 1) {
+        throw new Error(`Invalid --threads value: ${threads}`);
+    }
+    const indexExt = csi ? 'csi' : 'tbi';
+    const indexFile = `${outputFile}.${indexExt}`;
+    // The pipeline writes under a temporary name, renamed into place only once
+    // the pass, the pipeline and the row census have all succeeded: a truncated
+    // .paf.gz or a non-PAF input used to leave a valid, indexed PIF of whatever
+    // preceded the failure, which a build script reads as done.
+    const partFile = `${outputFile}.part`;
+    const partIndex = `${partFile}.${indexExt}`;
+    const child = spawnSortProcess(partFile, csi, bgzipThreads);
+    const closed = waitForProcessClose(child);
+    const pass = await createPIF(file, child.stdin, coarseGap).then(stats => ({ stats }), (error) => ({
+        error: error instanceof Error ? error : new Error(String(error)),
+    }));
+    child.stdin.end();
     // no SIGPIPE exemption here (unlike sort-gff/sort-bed): this pipeline's output
     // is the .pif.gz file, so anything that killed it early left that file broken
-    const exit = await waitForProcessClose(child);
+    const exit = await closed;
+    const discardPart = () => {
+        fs.rmSync(partFile, { force: true });
+        fs.rmSync(partIndex, { force: true });
+    };
+    if ('error' in pass) {
+        discardPart();
+        throw pass.error;
+    }
     if (exit.code !== 0) {
+        discardPart();
         throw new Error(`PIF sort/index pipeline exited with ${describeExit(exit)}`);
     }
-    // A file that yielded nothing is almost always the wrong file, not an empty
-    // alignment: without this the command wrote a valid, indexed, empty PIF and
-    // printed the add-track suggestion for it
+    const { stats } = pass;
+    const { samples, rows, skipped } = stats;
     if (rows === 0) {
+        discardPart();
         throw new Error(`No valid PAF rows found in ${file ?? 'stdin'} (${skipped} line(s) had fewer than the 12 mandatory PAF columns). Is this a PAF file?`);
     }
+    fs.renameSync(partFile, outputFile);
+    fs.renameSync(partIndex, indexFile);
     if (skipped > 0) {
         console.warn(`Warning: skipped ${skipped} of ${rows + skipped} line(s) with fewer than the 12 mandatory PAF columns`);
     }
@@ -126,10 +157,9 @@ export async function run(args) {
             `project publishes one, or view a larger cohort as a multiple alignment ` +
             `(MAF) rather than a stack of pairwise bands.`);
     }
-    const indexFile = `${outputFile}.${csi ? 'csi' : 'tbi'}`;
     const nextCommand = samples.size > 0
         ? 'Next, add it as an all-vs-all synteny track (PanSN names detected):\n' +
-            `  jbrowse add-track ${outputFile} --adapterType AllVsAllIndexedPAFAdapter -a ${[...samples].sort().join(',')} --load copy`
+            `  jbrowse add-track ${outputFile} --adapterType MultiGenomeIndexedPAFAdapter -a ${[...samples].sort().join(',')} --load copy`
         : 'Next, add it as a synteny track (set -a to your query,target assembly names):\n' +
             `  jbrowse add-track ${outputFile} -a query,target --load copy`;
     console.log(`Created ${outputFile} and ${indexFile}\n\n${nextCommand}`);

package/dist/commands/make-pif/pif-generator.js

@@ -4,14 +4,29 @@ import path from 'node:path';
 import { Transform } from 'node:stream';
 import { pipeline } from 'node:stream/promises';
 import { createGunzip } from 'node:zlib';
-import { csToCigar, flipCigar, pafIdentity, swapIndelCigar, } from '@jbrowse/cigar-utils';
-import { splitCigarOnLargeGaps } from './cigar-utils.js';
-// Default split gap (bp) for the coarse tier. A row is broken into multiple
-// coarse pieces wherever a CIGAR insertion/deletion is at least this long, so
-// each coarse row's bounding box stays tight. 10kb matches the adapter's
-// default coarseBpPerPxThreshold (~1px at the zoom where the coarse tier is
-// served), so smaller gaps that would be sub-pixel there don't fragment rows.
-export const DEFAULT_COARSE_SPLIT_GAP = 10_000;
+import { coarsenCigar, csToCigar, flipCigar, flipCoarseCigar, swapCoarseCigar, swapIndelCigar, } from '@jbrowse/cigar-utils';
+// Default gap (bp) of the coarse tier's coarse CIGAR: how far a straight line
+// across one of its runs may be from the alignment's real path. Indels longer
+// than half of it keep their letter; shorter ones fold into the runs. 10kb
+// matches the adapter's default coarseBpPerPxThreshold, so at the zoom where the
+// tier is served the bound is ~1px and a folded indel was sub-pixel anyway.
+export const DEFAULT_COARSE_GAP = 10_000;
+/**
+ * The one meta line a PIF carries, sorted first by the C-locale sort and kept
+ * by tabix as a header. It states the format generation, the tiers written,
+ * the coarse tier's accuracy bound (`--coarse`), and whether every input row
+ * had a CIGAR — the facts a reader cannot recover from the rows.
+ */
+export function pifHeader(coarseGap, stats) {
+    const cigars = stats.cigarRows === 0
+        ? 'none'
+        : stats.cigarRows === stats.rows && stats.unboundedRows === 0
+            ? 'all'
+            : 'some';
+    const tiers = coarseGap === undefined ? 'fine' : 'fine,coarse';
+    const bound = coarseGap === undefined ? '' : `\tcoarse:i:${coarseGap}`;
+    return `#pif\tversion:i:1\ttiers:Z:${tiers}${bound}\tcigars:Z:${cigars}\n`;
+}
 function panSNSample(refName) {
     const i = refName.indexOf('#');
     return i === -1 ? undefined : refName.slice(0, i);
@@ -47,34 +62,33 @@ export function missingPairs({ samples, pairs }) {
     }
     return out;
 }
-function coarsePieces({ cigar, strand, tstart, tend, qstart, qend, numMatches, blockLen, splitGap, }) {
-    const whole = { tstart, tend, qstart, qend, numMatches, blockLen };
-    const split = cigar && splitGap > 0
-        ? splitCigarOnLargeGaps({
-            cigar,
-            strand,
-            tstart,
-            tend,
-            qstart,
-            qend,
-            splitGap,
-        })
-        : undefined;
-    // The walk's reconstruction is taken only when it CLOSED on the row's own far
-    // corner (see `CigarSplit.closed`) — otherwise the columns are what the fine
-    // tier draws and the coarse row must not drift off them.
-    //
-    // Closure rather than a piece count: `segments.length < 2` read as "nothing to
-    // split on", but a large gap at either END also yields a single piece, a
-    // genuine tightening that was being thrown away for the loose box. It is the
-    // ordinary case below the default `--coarse`. The count also never noticed a
-    // MULTI-piece walk that failed to close, which is the case it was written for.
-    return split?.closed && split.segments.length > 0
-        ? split.segments.map(seg => ({
-            ...seg,
-            numMatches: blockLen > 0 ? Math.round((numMatches * seg.blockLen) / blockLen) : 0,
-        }))
-        : [whole];
+// The coarse row's `cr:Z:` value, or nothing, and whether withholding it left
+// the row making a claim it cannot keep.
+//
+// `ops` is nothing when the row has no CIGAR, when the fold is a single run
+// (the coordinate columns already say it all), and when the walk does not close
+// on the row's own far corner — clipping ops, a hand-made cg, a cs whose spans
+// don't add up — since the columns are what the fine tier draws and the coarse
+// row must not disagree with them. A fold of several runs with no kept indel is
+// still written: the runs are where a lopsided cluster of small indels bends
+// the path, which a straight ribbon across the row would miss by up to the
+// whole cluster.
+//
+// `unbounded` separates the last of those from the rest. The first two leave a
+// tagless row a reader may take as one run within `--coarse`, and it is one; a
+// walk that misses its own corner leaves a row that reads the same way and is
+// not, so the header has to stop saying `cigars:Z:all` on its account.
+function coarseFold({ cigar, coarseGap, ownLen, mateLen, }) {
+    const coarse = cigar === undefined ? undefined : coarsenCigar(cigar, coarseGap);
+    const closed = coarse !== undefined &&
+        coarse.ownLen === ownLen &&
+        coarse.mateLen === mateLen;
+    return {
+        ops: closed && (coarse.gapCount > 0 || coarse.opCount > 1)
+            ? coarse.ops
+            : undefined,
+        unbounded: cigar !== undefined && !closed,
+    };
 }
 // A PIF row carries exactly ONE alignment string — `cg:Z:`, in the orientation
 // of the perspective it is indexed under. A minimap2 `cs:Z:` is folded into that
@@ -102,34 +116,11 @@ function foldCsIntoCg(tags) {
     const rewritten = tags.flatMap((f, i) => i === csIdx ? [folded] : f.startsWith('cg:Z') ? [] : [f]);
     return { tags: rewritten, cigarIdx: rewritten.indexOf(folded) };
 }
-// Coarse identity must match the fine tier exactly, or coloring jumps at the
-// zoom where the view switches tiers. So it is written off the SAME function the
-// adapters read the fine tier with (de:f: tag, then odgi's id:f:, then
-// num_matches/block_len) rather than a private copy of that chain — a copy is
-// what previously skipped the id:f: rung. A CIGAR recompute is deliberately not
-// in the chain at all: a cg (M-style) CIGAR folds mismatches into M, so it would
-// report ~0 divergence (spurious 100% identity) for a divergent alignment.
-//
-// When the row's own de:f: is what the reader lands on, that string is passed
-// through byte-for-byte rather than recomputed: a round trip through toFixed(6)
-// would silently truncate a 7-decimal tag and drift the two tiers apart by the
-// exact mechanism this is guarding against.
-function coarseDivergence(tags, numMatches, blockLen) {
-    const rawDe = tags.find(f => f.startsWith('de:f:'))?.slice(5);
-    const identity = pafIdentity({
-        de: rawDe,
-        id: tags.find(f => f.startsWith('id:f:'))?.slice(5),
-        numMatches,
-        blockLen,
-    });
-    return rawDe !== undefined && 1 - +rawDe === identity
-        ? rawDe
-        : (1 - identity).toFixed(6);
-}
 function pifRow(fields) {
     return `${fields.join('\t')}\n`;
 }
-function processLine(line, coarseSplitGap, stats) {
+function processLine(rawLine, coarseGap, stats) {
+    const line = rawLine.endsWith('\r') ? rawLine.slice(0, -1) : rawLine;
     if (line.startsWith('#')) {
         return '';
     }
@@ -144,8 +135,14 @@ function processLine(line, coarseSplitGap, stats) {
     // rest[0]=num_matches, rest[1]=block_len, rest[2]=mapq, rest[3+]=optional tags
     const [c1, l1, s1, e1, strand, c2, l2, s2, e2, ...rest] = parts;
     addPanSNPair(stats, c1, c2);
-    const { tags, cigarIdx } = foldCsIntoCg(rest);
+    // an incoming cr:Z: (a PIF turned back into PAF, or a tool that adopted the
+    // tag) is dropped from both tiers: the fine tier never carries one, and the
+    // coarse tier writes its own below
+    const { tags, cigarIdx } = foldCsIntoCg(rest.filter(f => !f.startsWith('cr:Z:')));
     const cigar = cigarIdx === -1 ? undefined : tags[cigarIdx].slice(5);
+    if (cigar !== undefined) {
+        stats.cigarRows++;
+    }
     // the t-row keeps the CIGAR as PAF spelled it (target perspective); the q-row
     // re-orients it for the query perspective it is indexed under
     const queryTags = cigar === undefined
@@ -155,73 +152,33 @@ function processLine(line, coarseSplitGap, stats) {
             : f);
     const fineRows = pifRow([`t${c2}`, l2, s2, e2, strand, c1, l1, s1, e1, ...tags]) +
         pifRow([`q${c1}`, l1, s1, e1, strand, c2, l2, s2, e2, ...queryTags]);
-    if (coarseSplitGap === undefined) {
+    if (coarseGap === undefined) {
         return fineRows;
     }
-    const numMatches = +tags[0];
-    const blockLen = +tags[1];
-    const mapq = tags[2];
-    // Every optional tag except the alignment string itself rides along, so a
-    // click on a coarse ribbon shows the same attributes as the same alignment
-    // does zoomed in (rustybam's tags, minimap2's tp/cm/s1, odgi's id). Only the
-    // CIGAR is dropped — dropping it is the entire point of the tier — and de:f:,
-    // which is rewritten below. No `cs:Z:` can reach here: it was folded into the
-    // cg above, since a PIF row carries one alignment string.
-    const passthrough = tags
-        .slice(3)
-        .filter(f => !f.startsWith('cg:Z:') && !f.startsWith('de:f:'));
-    const de = coarseDivergence(tags, numMatches, blockLen);
+    // the coarse row is the fine row with its CIGAR replaced by the fold and
+    // every other column and tag verbatim, so a click and the identity coloring
+    // read the same on both tiers
+    const coarse = cigarIdx === -1 ? tags : tags.filter((_, i) => i !== cigarIdx);
+    // the T row's own axis is the target, as the PAF CIGAR is written; the Q row
+    // re-orients it for the query, the way the fine tier's cg is
+    const { ops: cr, unbounded } = coarseFold({
+        cigar,
+        coarseGap,
+        ownLen: +e2 - +s2,
+        mateLen: +e1 - +s1,
+    });
+    if (unbounded) {
+        stats.unboundedRows++;
+    }
+    const tCr = cr === undefined ? [] : [`cr:Z:${cr}`];
+    const qCr = cr === undefined
+        ? []
+        : [`cr:Z:${strand === '-' ? flipCoarseCigar(cr) : swapCoarseCigar(cr)}`];
     return (fineRows +
-        coarsePieces({
-            cigar,
-            strand: strand,
-            tstart: +s2,
-            tend: +e2,
-            qstart: +s1,
-            qend: +e1,
-            numMatches,
-            blockLen,
-            splitGap: coarseSplitGap,
-        })
-            .flatMap(piece => {
-            const tail = [
-                piece.numMatches,
-                piece.blockLen,
-                mapq,
-                ...passthrough,
-                `de:f:${de}`,
-            ];
-            const { tstart, tend, qstart, qend } = piece;
-            return [
-                pifRow([
-                    `T${c2}`,
-                    l2,
-                    tstart,
-                    tend,
-                    strand,
-                    c1,
-                    l1,
-                    qstart,
-                    qend,
-                    ...tail,
-                ]),
-                pifRow([
-                    `Q${c1}`,
-                    l1,
-                    qstart,
-                    qend,
-                    strand,
-                    c2,
-                    l2,
-                    tstart,
-                    tend,
-                    ...tail,
-                ]),
-            ];
-        })
-            .join(''));
+        pifRow([`T${c2}`, l2, s2, e2, strand, c1, l1, s1, e1, ...coarse, ...tCr]) +
+        pifRow([`Q${c1}`, l1, s1, e1, strand, c2, l2, s2, e2, ...coarse, ...qCr]));
 }
-function makePifTransform(coarseSplitGap, stats) {
+function makePifTransform(coarseGap, stats) {
     let tail = '';
     return new Transform({
         transform(chunk, _enc, callback) {
@@ -237,25 +194,31 @@ function makePifTransform(coarseSplitGap, stats) {
                 .slice(0, lastNl)
                 .split('\n')
                 .filter(Boolean)
-                .map(l => processLine(l, coarseSplitGap, stats))
+                .map(l => processLine(l, coarseGap, stats))
                 .join(''));
         },
         flush(callback) {
-            callback(null, tail ? processLine(tail, coarseSplitGap, stats) : '');
+            // the header goes out last and sorts first: the counts it states are
+            // only known once every row has been seen, and `#` precedes every tier
+            // letter in the C locale
+            callback(null, (tail ? processLine(tail, coarseGap, stats) : '') +
+                pifHeader(coarseGap, stats));
         },
     });
 }
 // resolves to what the pass observed: the PanSN samples (empty for a pairwise
 // PAF), so the caller can suggest the right adapter and assembly names, and the
 // row/skip counts so it can warn on a file that was not PAF at all
-export async function createPIF(filename, stream, coarseSplitGap) {
+export async function createPIF(filename, stream, coarseGap) {
     const stats = {
         samples: new Set(),
         pairs: new Set(),
         rows: 0,
+        cigarRows: 0,
+        unboundedRows: 0,
         skipped: 0,
     };
-    const transform = makePifTransform(coarseSplitGap, stats);
+    const transform = makePifTransform(coarseGap, stats);
     if (filename) {
         const source = createReadStream(filename);
         await (/\.b?gz$/i.test(filename)
@@ -267,15 +230,24 @@ export async function createPIF(filename, stream, coarseSplitGap) {
     }
     return stats;
 }
-export function spawnSortProcess(outputFile, useCsi) {
-    const sortCmd = `sort -t"\`printf '\\t'\`" -k1,1 -k3,3n`;
-    const bgzipCommand = `bgzip > "$1"`;
+// bgzip is the only stage of this pipeline that cannot use more than one core on
+// its own: sort parallelizes itself and tabix is I/O bound. 4 is enough to take
+// compression off the critical path on the files this command is pointed at
+// without claiming a whole machine.
+export const DEFAULT_BGZIP_THREADS = 4;
+export function spawnSortProcess(outputFile, useCsi, threads = DEFAULT_BGZIP_THREADS) {
+    // -s: tabix wants the rows ordered by (refName, start) and by nothing else,
+    // so the whole-line comparison a non-stable sort falls back to on a tie is
+    // comparing multi-kb CIGARs to pick an order no reader can observe
+    const sortCmd = `sort -s -t"\`printf '\\t'\`" -k1,1 -k3,3n`;
+    const bgzipCommand = `bgzip -@ ${threads} > "$1"`;
     const tabixCommand = `tabix ${useCsi ? '-C ' : ''}-s1 -b3 -e4 -0 "$1"`;
     // `&&` (not `;`) so a bgzip failure aborts before tabix runs and propagates
     // as the pipeline's exit code. The output path is passed as the shell's "$1"
     // positional rather than interpolated into the command string, so a path
     // with shell metacharacters (`"`, `$(...)`, backticks) can't break out and
-    // execute — same technique as sort-utils.ts. useCsi is a fixed literal.
+    // execute — same technique as sort-utils.ts. useCsi is a fixed literal, and
+    // the caller has already checked threads is a positive integer.
     const fullCommand = `${sortCmd} | ${bgzipCommand} && ${tabixCommand}`;
     return spawn('sh', ['-c', fullCommand, 'sh', outputFile], {
         env: { ...process.env, LC_ALL: 'C' },

package/dist/commands/shared/density.js

@@ -0,0 +1,12 @@
+/**
+ * Where the density sidecar for a data file lives: `<file>.density.bw`, with a
+ * `.gz`/`.bgz` stripped first so `genes.gff3.gz` yields `genes.gff3.density.bw`
+ * rather than a name carrying the compression of a file it is not compressed
+ * like.
+ *
+ * One rule, two commands: `make-density` writes here by default and `add-track`
+ * probes here, so a sidecar built with no `--out` is the one the track picks up.
+ */
+export function densitySidecarPath(file) {
+    return `${file.replace(/\.b?gz$/i, '')}.density.bw`;
+}

package/dist/commands/text-index/adapter-utils.js

@@ -12,10 +12,6 @@ export function createTrixAdapter(name, assemblyNames, idSuffix = 'index') {
             uri: uris.ixx,
             locationType: 'UriLocation',
         },
-        metaFilePath: {
-            uri: uris.meta,
-            locationType: 'UriLocation',
-        },
         assemblyNames,
     };
 }