@jbrowse/cli 5.0.0-beta.1 → 5.0.0-beta.2

package/dist/commands/make-pif/pif-generator.js

@@ -4,14 +4,29 @@ import path from 'node:path';
 import { Transform } from 'node:stream';
 import { pipeline } from 'node:stream/promises';
 import { createGunzip } from 'node:zlib';
-import { csToCigar, flipCigar, pafIdentity, swapIndelCigar, } from '@jbrowse/cigar-utils';
-import { splitCigarOnLargeGaps } from './cigar-utils.js';
-// Default split gap (bp) for the coarse tier. A row is broken into multiple
-// coarse pieces wherever a CIGAR insertion/deletion is at least this long, so
-// each coarse row's bounding box stays tight. 10kb matches the adapter's
-// default coarseBpPerPxThreshold (~1px at the zoom where the coarse tier is
-// served), so smaller gaps that would be sub-pixel there don't fragment rows.
-export const DEFAULT_COARSE_SPLIT_GAP = 10_000;
+import { coarsenCigar, csToCigar, flipCigar, flipCoarseCigar, swapCoarseCigar, swapIndelCigar, } from '@jbrowse/cigar-utils';
+// Default gap (bp) of the coarse tier's coarse CIGAR: how far a straight line
+// across one of its runs may be from the alignment's real path. Indels longer
+// than half of it keep their letter; shorter ones fold into the runs. 10kb
+// matches the adapter's default coarseBpPerPxThreshold, so at the zoom where the
+// tier is served the bound is ~1px and a folded indel was sub-pixel anyway.
+export const DEFAULT_COARSE_GAP = 10_000;
+/**
+ * The one meta line a PIF carries, sorted first by the C-locale sort and kept
+ * by tabix as a header. It states the format generation, the tiers written,
+ * the coarse tier's accuracy bound (`--coarse`), and whether every input row
+ * had a CIGAR — the facts a reader cannot recover from the rows.
+ */
+export function pifHeader(coarseGap, stats) {
+    const cigars = stats.cigarRows === 0
+        ? 'none'
+        : stats.cigarRows === stats.rows && stats.unboundedRows === 0
+            ? 'all'
+            : 'some';
+    const tiers = coarseGap === undefined ? 'fine' : 'fine,coarse';
+    const bound = coarseGap === undefined ? '' : `\tcoarse:i:${coarseGap}`;
+    return `#pif\tversion:i:1\ttiers:Z:${tiers}${bound}\tcigars:Z:${cigars}\n`;
+}
 function panSNSample(refName) {
     const i = refName.indexOf('#');
     return i === -1 ? undefined : refName.slice(0, i);
@@ -47,34 +62,33 @@ export function missingPairs({ samples, pairs }) {
     }
     return out;
 }
-function coarsePieces({ cigar, strand, tstart, tend, qstart, qend, numMatches, blockLen, splitGap, }) {
-    const whole = { tstart, tend, qstart, qend, numMatches, blockLen };
-    const split = cigar && splitGap > 0
-        ? splitCigarOnLargeGaps({
-            cigar,
-            strand,
-            tstart,
-            tend,
-            qstart,
-            qend,
-            splitGap,
-        })
-        : undefined;
-    // The walk's reconstruction is taken only when it CLOSED on the row's own far
-    // corner (see `CigarSplit.closed`) — otherwise the columns are what the fine
-    // tier draws and the coarse row must not drift off them.
-    //
-    // Closure rather than a piece count: `segments.length < 2` read as "nothing to
-    // split on", but a large gap at either END also yields a single piece, a
-    // genuine tightening that was being thrown away for the loose box. It is the
-    // ordinary case below the default `--coarse`. The count also never noticed a
-    // MULTI-piece walk that failed to close, which is the case it was written for.
-    return split?.closed && split.segments.length > 0
-        ? split.segments.map(seg => ({
-            ...seg,
-            numMatches: blockLen > 0 ? Math.round((numMatches * seg.blockLen) / blockLen) : 0,
-        }))
-        : [whole];
+// The coarse row's `cr:Z:` value, or nothing, and whether withholding it left
+// the row making a claim it cannot keep.
+//
+// `ops` is nothing when the row has no CIGAR, when the fold is a single run
+// (the coordinate columns already say it all), and when the walk does not close
+// on the row's own far corner — clipping ops, a hand-made cg, a cs whose spans
+// don't add up — since the columns are what the fine tier draws and the coarse
+// row must not disagree with them. A fold of several runs with no kept indel is
+// still written: the runs are where a lopsided cluster of small indels bends
+// the path, which a straight ribbon across the row would miss by up to the
+// whole cluster.
+//
+// `unbounded` separates the last of those from the rest. The first two leave a
+// tagless row a reader may take as one run within `--coarse`, and it is one; a
+// walk that misses its own corner leaves a row that reads the same way and is
+// not, so the header has to stop saying `cigars:Z:all` on its account.
+function coarseFold({ cigar, coarseGap, ownLen, mateLen, }) {
+    const coarse = cigar === undefined ? undefined : coarsenCigar(cigar, coarseGap);
+    const closed = coarse !== undefined &&
+        coarse.ownLen === ownLen &&
+        coarse.mateLen === mateLen;
+    return {
+        ops: closed && (coarse.gapCount > 0 || coarse.opCount > 1)
+            ? coarse.ops
+            : undefined,
+        unbounded: cigar !== undefined && !closed,
+    };
 }
 // A PIF row carries exactly ONE alignment string — `cg:Z:`, in the orientation
 // of the perspective it is indexed under. A minimap2 `cs:Z:` is folded into that
@@ -102,34 +116,11 @@ function foldCsIntoCg(tags) {
     const rewritten = tags.flatMap((f, i) => i === csIdx ? [folded] : f.startsWith('cg:Z') ? [] : [f]);
     return { tags: rewritten, cigarIdx: rewritten.indexOf(folded) };
 }
-// Coarse identity must match the fine tier exactly, or coloring jumps at the
-// zoom where the view switches tiers. So it is written off the SAME function the
-// adapters read the fine tier with (de:f: tag, then odgi's id:f:, then
-// num_matches/block_len) rather than a private copy of that chain — a copy is
-// what previously skipped the id:f: rung. A CIGAR recompute is deliberately not
-// in the chain at all: a cg (M-style) CIGAR folds mismatches into M, so it would
-// report ~0 divergence (spurious 100% identity) for a divergent alignment.
-//
-// When the row's own de:f: is what the reader lands on, that string is passed
-// through byte-for-byte rather than recomputed: a round trip through toFixed(6)
-// would silently truncate a 7-decimal tag and drift the two tiers apart by the
-// exact mechanism this is guarding against.
-function coarseDivergence(tags, numMatches, blockLen) {
-    const rawDe = tags.find(f => f.startsWith('de:f:'))?.slice(5);
-    const identity = pafIdentity({
-        de: rawDe,
-        id: tags.find(f => f.startsWith('id:f:'))?.slice(5),
-        numMatches,
-        blockLen,
-    });
-    return rawDe !== undefined && 1 - +rawDe === identity
-        ? rawDe
-        : (1 - identity).toFixed(6);
-}
 function pifRow(fields) {
     return `${fields.join('\t')}\n`;
 }
-function processLine(line, coarseSplitGap, stats) {
+function processLine(rawLine, coarseGap, stats) {
+    const line = rawLine.endsWith('\r') ? rawLine.slice(0, -1) : rawLine;
     if (line.startsWith('#')) {
         return '';
     }
@@ -144,8 +135,14 @@ function processLine(line, coarseSplitGap, stats) {
     // rest[0]=num_matches, rest[1]=block_len, rest[2]=mapq, rest[3+]=optional tags
     const [c1, l1, s1, e1, strand, c2, l2, s2, e2, ...rest] = parts;
     addPanSNPair(stats, c1, c2);
-    const { tags, cigarIdx } = foldCsIntoCg(rest);
+    // an incoming cr:Z: (a PIF turned back into PAF, or a tool that adopted the
+    // tag) is dropped from both tiers: the fine tier never carries one, and the
+    // coarse tier writes its own below
+    const { tags, cigarIdx } = foldCsIntoCg(rest.filter(f => !f.startsWith('cr:Z:')));
     const cigar = cigarIdx === -1 ? undefined : tags[cigarIdx].slice(5);
+    if (cigar !== undefined) {
+        stats.cigarRows++;
+    }
     // the t-row keeps the CIGAR as PAF spelled it (target perspective); the q-row
     // re-orients it for the query perspective it is indexed under
     const queryTags = cigar === undefined
@@ -155,73 +152,33 @@ function processLine(line, coarseSplitGap, stats) {
             : f);
     const fineRows = pifRow([`t${c2}`, l2, s2, e2, strand, c1, l1, s1, e1, ...tags]) +
         pifRow([`q${c1}`, l1, s1, e1, strand, c2, l2, s2, e2, ...queryTags]);
-    if (coarseSplitGap === undefined) {
+    if (coarseGap === undefined) {
         return fineRows;
     }
-    const numMatches = +tags[0];
-    const blockLen = +tags[1];
-    const mapq = tags[2];
-    // Every optional tag except the alignment string itself rides along, so a
-    // click on a coarse ribbon shows the same attributes as the same alignment
-    // does zoomed in (rustybam's tags, minimap2's tp/cm/s1, odgi's id). Only the
-    // CIGAR is dropped — dropping it is the entire point of the tier — and de:f:,
-    // which is rewritten below. No `cs:Z:` can reach here: it was folded into the
-    // cg above, since a PIF row carries one alignment string.
-    const passthrough = tags
-        .slice(3)
-        .filter(f => !f.startsWith('cg:Z:') && !f.startsWith('de:f:'));
-    const de = coarseDivergence(tags, numMatches, blockLen);
+    // the coarse row is the fine row with its CIGAR replaced by the fold and
+    // every other column and tag verbatim, so a click and the identity coloring
+    // read the same on both tiers
+    const coarse = cigarIdx === -1 ? tags : tags.filter((_, i) => i !== cigarIdx);
+    // the T row's own axis is the target, as the PAF CIGAR is written; the Q row
+    // re-orients it for the query, the way the fine tier's cg is
+    const { ops: cr, unbounded } = coarseFold({
+        cigar,
+        coarseGap,
+        ownLen: +e2 - +s2,
+        mateLen: +e1 - +s1,
+    });
+    if (unbounded) {
+        stats.unboundedRows++;
+    }
+    const tCr = cr === undefined ? [] : [`cr:Z:${cr}`];
+    const qCr = cr === undefined
+        ? []
+        : [`cr:Z:${strand === '-' ? flipCoarseCigar(cr) : swapCoarseCigar(cr)}`];
     return (fineRows +
-        coarsePieces({
-            cigar,
-            strand: strand,
-            tstart: +s2,
-            tend: +e2,
-            qstart: +s1,
-            qend: +e1,
-            numMatches,
-            blockLen,
-            splitGap: coarseSplitGap,
-        })
-            .flatMap(piece => {
-            const tail = [
-                piece.numMatches,
-                piece.blockLen,
-                mapq,
-                ...passthrough,
-                `de:f:${de}`,
-            ];
-            const { tstart, tend, qstart, qend } = piece;
-            return [
-                pifRow([
-                    `T${c2}`,
-                    l2,
-                    tstart,
-                    tend,
-                    strand,
-                    c1,
-                    l1,
-                    qstart,
-                    qend,
-                    ...tail,
-                ]),
-                pifRow([
-                    `Q${c1}`,
-                    l1,
-                    qstart,
-                    qend,
-                    strand,
-                    c2,
-                    l2,
-                    tstart,
-                    tend,
-                    ...tail,
-                ]),
-            ];
-        })
-            .join(''));
+        pifRow([`T${c2}`, l2, s2, e2, strand, c1, l1, s1, e1, ...coarse, ...tCr]) +
+        pifRow([`Q${c1}`, l1, s1, e1, strand, c2, l2, s2, e2, ...coarse, ...qCr]));
 }
-function makePifTransform(coarseSplitGap, stats) {
+function makePifTransform(coarseGap, stats) {
     let tail = '';
     return new Transform({
         transform(chunk, _enc, callback) {
@@ -237,25 +194,31 @@ function makePifTransform(coarseSplitGap, stats) {
                 .slice(0, lastNl)
                 .split('\n')
                 .filter(Boolean)
-                .map(l => processLine(l, coarseSplitGap, stats))
+                .map(l => processLine(l, coarseGap, stats))
                 .join(''));
         },
         flush(callback) {
-            callback(null, tail ? processLine(tail, coarseSplitGap, stats) : '');
+            // the header goes out last and sorts first: the counts it states are
+            // only known once every row has been seen, and `#` precedes every tier
+            // letter in the C locale
+            callback(null, (tail ? processLine(tail, coarseGap, stats) : '') +
+                pifHeader(coarseGap, stats));
         },
     });
 }
 // resolves to what the pass observed: the PanSN samples (empty for a pairwise
 // PAF), so the caller can suggest the right adapter and assembly names, and the
 // row/skip counts so it can warn on a file that was not PAF at all
-export async function createPIF(filename, stream, coarseSplitGap) {
+export async function createPIF(filename, stream, coarseGap) {
     const stats = {
         samples: new Set(),
         pairs: new Set(),
         rows: 0,
+        cigarRows: 0,
+        unboundedRows: 0,
         skipped: 0,
     };
-    const transform = makePifTransform(coarseSplitGap, stats);
+    const transform = makePifTransform(coarseGap, stats);
     if (filename) {
         const source = createReadStream(filename);
         await (/\.b?gz$/i.test(filename)
@@ -267,15 +230,24 @@ export async function createPIF(filename, stream, coarseSplitGap) {
     }
     return stats;
 }
-export function spawnSortProcess(outputFile, useCsi) {
-    const sortCmd = `sort -t"\`printf '\\t'\`" -k1,1 -k3,3n`;
-    const bgzipCommand = `bgzip > "$1"`;
+// bgzip is the only stage of this pipeline that cannot use more than one core on
+// its own: sort parallelizes itself and tabix is I/O bound. 4 is enough to take
+// compression off the critical path on the files this command is pointed at
+// without claiming a whole machine.
+export const DEFAULT_BGZIP_THREADS = 4;
+export function spawnSortProcess(outputFile, useCsi, threads = DEFAULT_BGZIP_THREADS) {
+    // -s: tabix wants the rows ordered by (refName, start) and by nothing else,
+    // so the whole-line comparison a non-stable sort falls back to on a tie is
+    // comparing multi-kb CIGARs to pick an order no reader can observe
+    const sortCmd = `sort -s -t"\`printf '\\t'\`" -k1,1 -k3,3n`;
+    const bgzipCommand = `bgzip -@ ${threads} > "$1"`;
     const tabixCommand = `tabix ${useCsi ? '-C ' : ''}-s1 -b3 -e4 -0 "$1"`;
     // `&&` (not `;`) so a bgzip failure aborts before tabix runs and propagates
     // as the pipeline's exit code. The output path is passed as the shell's "$1"
     // positional rather than interpolated into the command string, so a path
     // with shell metacharacters (`"`, `$(...)`, backticks) can't break out and
-    // execute — same technique as sort-utils.ts. useCsi is a fixed literal.
+    // execute — same technique as sort-utils.ts. useCsi is a fixed literal, and
+    // the caller has already checked threads is a positive integer.
     const fullCommand = `${sortCmd} | ${bgzipCommand} && ${tabixCommand}`;
     return spawn('sh', ['-c', fullCommand, 'sh', outputFile], {
         env: { ...process.env, LC_ALL: 'C' },

package/dist/commands/shared/density.js

@@ -0,0 +1,12 @@
+/**
+ * Where the density sidecar for a data file lives: `<file>.density.bw`, with a
+ * `.gz`/`.bgz` stripped first so `genes.gff3.gz` yields `genes.gff3.density.bw`
+ * rather than a name carrying the compression of a file it is not compressed
+ * like.
+ *
+ * One rule, two commands: `make-density` writes here by default and `add-track`
+ * probes here, so a sidecar built with no `--out` is the one the track picks up.
+ */
+export function densitySidecarPath(file) {
+    return `${file.replace(/\.b?gz$/i, '')}.density.bw`;
+}