@jbrowse/cli 5.0.0-beta.1 → 5.0.0-beta.11

package/dist/commands/make-pif/pif-generator.js

@@ -4,14 +4,33 @@ import path from 'node:path';
 import { Transform } from 'node:stream';
 import { pipeline } from 'node:stream/promises';
 import { createGunzip } from 'node:zlib';
-import { csToCigar, flipCigar, pafIdentity, swapIndelCigar, } from '@jbrowse/cigar-utils';
-import { splitCigarOnLargeGaps } from './cigar-utils.js';
-// Default split gap (bp) for the coarse tier. A row is broken into multiple
-// coarse pieces wherever a CIGAR insertion/deletion is at least this long, so
-// each coarse row's bounding box stays tight. 10kb matches the adapter's
-// default coarseBpPerPxThreshold (~1px at the zoom where the coarse tier is
-// served), so smaller gaps that would be sub-pixel there don't fragment rows.
-export const DEFAULT_COARSE_SPLIT_GAP = 10_000;
+import { coarsenCigar, csToCigar, flipCigar, flipCoarseCigar, swapCoarseCigar, swapIndelCigar, } from '@jbrowse/cigar-utils';
+import { version } from '../../version.js';
+// Default gap (bp) of the coarse tier's coarse CIGAR: how far a straight line
+// across one of its runs may be from the alignment's real path. Indels longer
+// than half of it keep their letter; shorter ones fold into the runs. 10kb
+// matches the adapter's default coarseBpPerPxThreshold, so at the zoom where the
+// tier is served the bound is ~1px and a folded indel was sub-pixel anyway.
+export const DEFAULT_COARSE_GAP = 10_000;
+// Generation 2 (2026-09): every row carries `pi:i:`, the input row's index,
+// and the header names the writer.
+export const PIF_FORMAT_VERSION = 2;
+/**
+ * The one meta line a PIF carries, sorted first by the C-locale sort and kept
+ * by tabix as a header. It states the format generation, the writer, the
+ * tiers written, the coarse tier's accuracy bound (`--coarse`), and whether
+ * every input row had a CIGAR — the facts a reader cannot recover from the rows.
+ */
+export function pifHeader(coarseGap, stats) {
+    const cigars = stats.cigarRows === 0
+        ? 'none'
+        : stats.cigarRows === stats.rows
+            ? 'all'
+            : 'some';
+    const tiers = coarseGap === undefined ? 'fine' : 'fine,coarse';
+    const bound = coarseGap === undefined ? '' : `\tcoarse:i:${coarseGap}`;
+    return `#pif\tversion:i:${PIF_FORMAT_VERSION}\twriter:Z:jbrowse-cli/${version}\ttiers:Z:${tiers}${bound}\tcigars:Z:${cigars}\n`;
+}
 function panSNSample(refName) {
     const i = refName.indexOf('#');
     return i === -1 ? undefined : refName.slice(0, i);
@@ -47,34 +66,23 @@ export function missingPairs({ samples, pairs }) {
     }
     return out;
 }
-function coarsePieces({ cigar, strand, tstart, tend, qstart, qend, numMatches, blockLen, splitGap, }) {
-    const whole = { tstart, tend, qstart, qend, numMatches, blockLen };
-    const split = cigar && splitGap > 0
-        ? splitCigarOnLargeGaps({
-            cigar,
-            strand,
-            tstart,
-            tend,
-            qstart,
-            qend,
-            splitGap,
-        })
+// The coarse row's `cr:Z:` value, or nothing when the row has no CIGAR or the
+// fold is a single run that closes on the coordinate columns, which then say it
+// all. A fold of several runs with no kept indel is still written: a lopsided
+// cluster of small indels bends the path by their sum. So is a fold whose walk
+// misses the row's own far corner (clipping ops, a hand-made cg, a cs whose
+// spans don't add up): the fine tier walks that CIGAR from the start corner
+// without checking, and the coarse tier draws the same path only if it walks
+// the same fold.
+function coarseFold({ cigar, coarseGap, ownLen, mateLen, }) {
+    if (cigar === undefined) {
+        return undefined;
+    }
+    const coarse = coarsenCigar(cigar, coarseGap);
+    const closed = coarse.ownLen === ownLen && coarse.mateLen === mateLen;
+    return !closed || coarse.gapCount > 0 || coarse.opCount > 1
+        ? coarse.ops
         : undefined;
-    // The walk's reconstruction is taken only when it CLOSED on the row's own far
-    // corner (see `CigarSplit.closed`) — otherwise the columns are what the fine
-    // tier draws and the coarse row must not drift off them.
-    //
-    // Closure rather than a piece count: `segments.length < 2` read as "nothing to
-    // split on", but a large gap at either END also yields a single piece, a
-    // genuine tightening that was being thrown away for the loose box. It is the
-    // ordinary case below the default `--coarse`. The count also never noticed a
-    // MULTI-piece walk that failed to close, which is the case it was written for.
-    return split?.closed && split.segments.length > 0
-        ? split.segments.map(seg => ({
-            ...seg,
-            numMatches: blockLen > 0 ? Math.round((numMatches * seg.blockLen) / blockLen) : 0,
-        }))
-        : [whole];
 }
 // A PIF row carries exactly ONE alignment string — `cg:Z:`, in the orientation
 // of the perspective it is indexed under. A minimap2 `cs:Z:` is folded into that
@@ -102,34 +110,11 @@ function foldCsIntoCg(tags) {
     const rewritten = tags.flatMap((f, i) => i === csIdx ? [folded] : f.startsWith('cg:Z') ? [] : [f]);
     return { tags: rewritten, cigarIdx: rewritten.indexOf(folded) };
 }
-// Coarse identity must match the fine tier exactly, or coloring jumps at the
-// zoom where the view switches tiers. So it is written off the SAME function the
-// adapters read the fine tier with (de:f: tag, then odgi's id:f:, then
-// num_matches/block_len) rather than a private copy of that chain — a copy is
-// what previously skipped the id:f: rung. A CIGAR recompute is deliberately not
-// in the chain at all: a cg (M-style) CIGAR folds mismatches into M, so it would
-// report ~0 divergence (spurious 100% identity) for a divergent alignment.
-//
-// When the row's own de:f: is what the reader lands on, that string is passed
-// through byte-for-byte rather than recomputed: a round trip through toFixed(6)
-// would silently truncate a 7-decimal tag and drift the two tiers apart by the
-// exact mechanism this is guarding against.
-function coarseDivergence(tags, numMatches, blockLen) {
-    const rawDe = tags.find(f => f.startsWith('de:f:'))?.slice(5);
-    const identity = pafIdentity({
-        de: rawDe,
-        id: tags.find(f => f.startsWith('id:f:'))?.slice(5),
-        numMatches,
-        blockLen,
-    });
-    return rawDe !== undefined && 1 - +rawDe === identity
-        ? rawDe
-        : (1 - identity).toFixed(6);
-}
 function pifRow(fields) {
     return `${fields.join('\t')}\n`;
 }
-function processLine(line, coarseSplitGap, stats) {
+function processLine(rawLine, coarseGap, stats) {
+    const line = rawLine.endsWith('\r') ? rawLine.slice(0, -1) : rawLine;
     if (line.startsWith('#')) {
         return '';
     }
@@ -144,8 +129,18 @@ function processLine(line, coarseSplitGap, stats) {
     // rest[0]=num_matches, rest[1]=block_len, rest[2]=mapq, rest[3+]=optional tags
     const [c1, l1, s1, e1, strand, c2, l2, s2, e2, ...rest] = parts;
     addPanSNPair(stats, c1, c2);
-    const { tags, cigarIdx } = foldCsIntoCg(rest);
+    // an incoming cr:Z: or pi:i: (a PIF turned back into PAF, or a tool that
+    // adopted the tags) is dropped: the coarse tier writes its own fold below,
+    // and every row of this alignment gets this pass's row index as `pi`, the
+    // one id its four rows share across perspectives and tiers
+    const { tags, cigarIdx } = foldCsIntoCg([
+        ...rest.filter(f => !f.startsWith('cr:') && !f.startsWith('pi:')),
+        `pi:i:${stats.rows - 1}`,
+    ]);
     const cigar = cigarIdx === -1 ? undefined : tags[cigarIdx].slice(5);
+    if (cigar !== undefined) {
+        stats.cigarRows++;
+    }
     // the t-row keeps the CIGAR as PAF spelled it (target perspective); the q-row
     // re-orients it for the query perspective it is indexed under
     const queryTags = cigar === undefined
@@ -155,73 +150,30 @@ function processLine(line, coarseSplitGap, stats) {
             : f);
     const fineRows = pifRow([`t${c2}`, l2, s2, e2, strand, c1, l1, s1, e1, ...tags]) +
         pifRow([`q${c1}`, l1, s1, e1, strand, c2, l2, s2, e2, ...queryTags]);
-    if (coarseSplitGap === undefined) {
+    if (coarseGap === undefined) {
         return fineRows;
     }
-    const numMatches = +tags[0];
-    const blockLen = +tags[1];
-    const mapq = tags[2];
-    // Every optional tag except the alignment string itself rides along, so a
-    // click on a coarse ribbon shows the same attributes as the same alignment
-    // does zoomed in (rustybam's tags, minimap2's tp/cm/s1, odgi's id). Only the
-    // CIGAR is dropped — dropping it is the entire point of the tier — and de:f:,
-    // which is rewritten below. No `cs:Z:` can reach here: it was folded into the
-    // cg above, since a PIF row carries one alignment string.
-    const passthrough = tags
-        .slice(3)
-        .filter(f => !f.startsWith('cg:Z:') && !f.startsWith('de:f:'));
-    const de = coarseDivergence(tags, numMatches, blockLen);
+    // the coarse row is the fine row with its CIGAR replaced by the fold and
+    // every other column and tag verbatim, so a click and the identity coloring
+    // read the same on both tiers
+    const coarse = cigarIdx === -1 ? tags : tags.filter((_, i) => i !== cigarIdx);
+    // the T row's own axis is the target, as the PAF CIGAR is written; the Q row
+    // re-orients it for the query, the way the fine tier's cg is
+    const cr = coarseFold({
+        cigar,
+        coarseGap,
+        ownLen: +e2 - +s2,
+        mateLen: +e1 - +s1,
+    });
+    const tCr = cr === undefined ? [] : [`cr:Z:${cr}`];
+    const qCr = cr === undefined
+        ? []
+        : [`cr:Z:${strand === '-' ? flipCoarseCigar(cr) : swapCoarseCigar(cr)}`];
     return (fineRows +
-        coarsePieces({
-            cigar,
-            strand: strand,
-            tstart: +s2,
-            tend: +e2,
-            qstart: +s1,
-            qend: +e1,
-            numMatches,
-            blockLen,
-            splitGap: coarseSplitGap,
-        })
-            .flatMap(piece => {
-            const tail = [
-                piece.numMatches,
-                piece.blockLen,
-                mapq,
-                ...passthrough,
-                `de:f:${de}`,
-            ];
-            const { tstart, tend, qstart, qend } = piece;
-            return [
-                pifRow([
-                    `T${c2}`,
-                    l2,
-                    tstart,
-                    tend,
-                    strand,
-                    c1,
-                    l1,
-                    qstart,
-                    qend,
-                    ...tail,
-                ]),
-                pifRow([
-                    `Q${c1}`,
-                    l1,
-                    qstart,
-                    qend,
-                    strand,
-                    c2,
-                    l2,
-                    tstart,
-                    tend,
-                    ...tail,
-                ]),
-            ];
-        })
-            .join(''));
+        pifRow([`T${c2}`, l2, s2, e2, strand, c1, l1, s1, e1, ...coarse, ...tCr]) +
+        pifRow([`Q${c1}`, l1, s1, e1, strand, c2, l2, s2, e2, ...coarse, ...qCr]));
 }
-function makePifTransform(coarseSplitGap, stats) {
+function makePifTransform(coarseGap, stats) {
     let tail = '';
     return new Transform({
         transform(chunk, _enc, callback) {
@@ -237,25 +189,30 @@ function makePifTransform(coarseSplitGap, stats) {
                 .slice(0, lastNl)
                 .split('\n')
                 .filter(Boolean)
-                .map(l => processLine(l, coarseSplitGap, stats))
+                .map(l => processLine(l, coarseGap, stats))
                 .join(''));
         },
         flush(callback) {
-            callback(null, tail ? processLine(tail, coarseSplitGap, stats) : '');
+            // the header goes out last and sorts first: the counts it states are
+            // only known once every row has been seen, and `#` precedes every tier
+            // letter in the C locale
+            callback(null, (tail ? processLine(tail, coarseGap, stats) : '') +
+                pifHeader(coarseGap, stats));
         },
     });
 }
 // resolves to what the pass observed: the PanSN samples (empty for a pairwise
 // PAF), so the caller can suggest the right adapter and assembly names, and the
 // row/skip counts so it can warn on a file that was not PAF at all
-export async function createPIF(filename, stream, coarseSplitGap) {
+export async function createPIF(filename, stream, coarseGap) {
     const stats = {
         samples: new Set(),
         pairs: new Set(),
         rows: 0,
+        cigarRows: 0,
         skipped: 0,
     };
-    const transform = makePifTransform(coarseSplitGap, stats);
+    const transform = makePifTransform(coarseGap, stats);
     if (filename) {
         const source = createReadStream(filename);
         await (/\.b?gz$/i.test(filename)
@@ -267,15 +224,24 @@ export async function createPIF(filename, stream, coarseSplitGap) {
     }
     return stats;
 }
-export function spawnSortProcess(outputFile, useCsi) {
-    const sortCmd = `sort -t"\`printf '\\t'\`" -k1,1 -k3,3n`;
-    const bgzipCommand = `bgzip > "$1"`;
+// bgzip is the only stage of this pipeline that cannot use more than one core on
+// its own: sort parallelizes itself and tabix is I/O bound. 4 is enough to take
+// compression off the critical path on the files this command is pointed at
+// without claiming a whole machine.
+export const DEFAULT_BGZIP_THREADS = 4;
+export function spawnSortProcess(outputFile, useCsi, threads = DEFAULT_BGZIP_THREADS) {
+    // -s: tabix wants the rows ordered by (refName, start) and by nothing else,
+    // so the whole-line comparison a non-stable sort falls back to on a tie is
+    // comparing multi-kb CIGARs to pick an order no reader can observe
+    const sortCmd = `sort -s -t"\`printf '\\t'\`" -k1,1 -k3,3n`;
+    const bgzipCommand = `bgzip -@ ${threads} > "$1"`;
     const tabixCommand = `tabix ${useCsi ? '-C ' : ''}-s1 -b3 -e4 -0 "$1"`;
     // `&&` (not `;`) so a bgzip failure aborts before tabix runs and propagates
     // as the pipeline's exit code. The output path is passed as the shell's "$1"
     // positional rather than interpolated into the command string, so a path
     // with shell metacharacters (`"`, `$(...)`, backticks) can't break out and
-    // execute — same technique as sort-utils.ts. useCsi is a fixed literal.
+    // execute — same technique as sort-utils.ts. useCsi is a fixed literal, and
+    // the caller has already checked threads is a positive integer.
     const fullCommand = `${sortCmd} | ${bgzipCommand} && ${tabixCommand}`;
     return spawn('sh', ['-c', fullCommand, 'sh', outputFile], {
         env: { ...process.env, LC_ALL: 'C' },

package/dist/commands/shared/density.js

@@ -0,0 +1,12 @@
+/**
+ * Where the density sidecar for a data file lives: `<file>.density.bw`, with a
+ * `.gz`/`.bgz` stripped first so `genes.gff3.gz` yields `genes.gff3.density.bw`
+ * rather than a name carrying the compression of a file it is not compressed
+ * like.
+ *
+ * One rule, two commands: `make-density` writes here by default and `add-track`
+ * probes here, so a sidecar built with no `--out` is the one the track picks up.
+ */
+export function densitySidecarPath(file) {
+    return `${file.replace(/\.b?gz$/i, '')}.density.bw`;
+}

package/dist/commands/text-index/adapter-utils.js

@@ -1,9 +1,8 @@
-import { sanitizeForFilename, trixFileUris } from '@jbrowse/text-indexing-core';
-export function createTrixAdapter(name, assemblyNames, idSuffix = 'index') {
+import { trixFileUris } from '@jbrowse/text-indexing-core';
+export function createTrixAdapter(name, assemblyNames) {
     const uris = trixFileUris(name);
     return {
         type: 'TrixTextSearchAdapter',
-        textSearchAdapterId: `${sanitizeForFilename(name)}-${idSuffix}`,
         ixFilePath: {
             uri: uris.ix,
             locationType: 'UriLocation',
@@ -12,10 +11,21 @@ export function createTrixAdapter(name, assemblyNames, idSuffix = 'index') {
             uri: uris.ixx,
             locationType: 'UriLocation',
         },
-        metaFilePath: {
-            uri: uris.meta,
-            locationType: 'UriLocation',
-        },
         assemblyNames,
     };
 }
+// an aggregate entry is the index its files are, so a re-index of the same
+// assembly finds the entry pointing at them however it was written, `.ix`
+// string and `{ uri }` included; an entry with no trix file, a JBrowse 1
+// index, is only ever itself
+export function indexFileOf(entry) {
+    if (typeof entry === 'string') {
+        return entry;
+    }
+    const { ixFilePath, uri } = entry;
+    return ixFilePath
+        ? 'uri' in ixFilePath
+            ? ixFilePath.uri
+            : ixFilePath.localPath
+        : (uri ?? entry);
+}

package/dist/commands/text-index/aggregate.js

@@ -1,5 +1,5 @@
 import { parseCommaSeparatedString, writeJsonFile } from '../../utils.js';
-import { createTrixAdapter } from './adapter-utils.js';
+import { createTrixAdapter, indexFileOf } from './adapter-utils.js';
 import { formatDryRun, getAssemblyNames, getTrackConfigs, loadConfigForIndexing, prepareIndexDriverFlags, } from './config-utils.js';
 import { indexDriver } from './indexing-utils.js';
 export async function aggregateIndex(flags) {
@@ -22,7 +22,7 @@ export async function aggregateIndex(flags) {
         }
         const trixConf = createTrixAdapter(asm, [asm]);
         if (!force &&
-            existing.some(x => x.textSearchAdapterId === trixConf.textSearchAdapterId)) {
+            existing.some(x => indexFileOf(x) === indexFileOf(trixConf))) {
             console.log(`Note: ${asm} has already been indexed with this configuration, use --force to overwrite this assembly. Skipping for now`);
             continue;
         }
@@ -44,11 +44,11 @@ export async function aggregateIndex(flags) {
     if (!dryrun) {
         await writeJsonFile(configPath, {
             ...config,
-            // upsert by adapter id: a Map keyed on it keeps the first occurrence's
+            // upsert by index file: a Map keyed on it keeps the first occurrence's
             // position while the last value wins, so re-indexing an assembly replaces
-            // its adapter in place and a new one lands at the end
+            // its entry in place and a new one lands at the end
             aggregateTextSearchAdapters: [
-                ...new Map([...existing, ...written].map(a => [a.textSearchAdapterId, a])).values(),
+                ...new Map([...existing, ...written].map(a => [indexFileOf(a), a])).values(),
             ],
         });
     }

package/dist/commands/text-index/command.js

@@ -90,7 +90,9 @@ export async function run(args) {
         `Only tracks with an indexable adapter type (${Object.keys(indexableAdapters).sort().join(', ')}) are indexed; ` +
         'tracks with other adapter types are skipped automatically.\n\n' +
         'GTF has no Name/ID attributes, so the default --attributes also match ' +
-        'their GTF spellings (gene_name, transcript_name, gene_id, transcript_id).' +
+        'their GTF spellings (gene_name, transcript_name, gene_id, transcript_id). ' +
+        'A GFF3 row with no Name, as in GENCODE, is named by its own gene_name or ' +
+        'transcript_name.' +
         '\n\n' +
         '--exclude names types not to index; --include names the only types to ' +
         'index. Reach for --include when the file draws from a vocabulary you do ' +

package/dist/commands/text-index/config-utils.js

@@ -13,13 +13,13 @@ function validatePrefixSize(value) {
 }
 export function prepareIndexDriverFlags(flags) {
     return {
-        attributes: parseCommaSeparatedString(flags.attributes),
-        typesToExclude: parseCommaSeparatedString(flags.exclude),
-        // undefined rather than [] when the flag is absent, because an empty allow
-        // list has to mean "no allow list" and not "index nothing"
-        typesToInclude: flags.include
-            ? parseCommaSeparatedString(flags.include)
-            : undefined,
+        policy: {
+            attributes: parseCommaSeparatedString(flags.attributes),
+            exclude: parseCommaSeparatedString(flags.exclude),
+            include: flags.include
+                ? parseCommaSeparatedString(flags.include)
+                : undefined,
+        },
         quiet: flags.quiet ?? false,
         prefixSize: validatePrefixSize(flags.prefixSize),
     };

package/dist/commands/text-index/indexing-utils.js

@@ -1,44 +1,30 @@
-import fs from 'node:fs';
-import path from 'node:path';
-import { Readable } from 'node:stream';
-import { TRIX_DIR, generateMeta, guessAdapterFromFileName, indexFiles, trixFilePaths, } from '@jbrowse/text-indexing-core';
+import { guessAdapterFromFileName, writeTrixIndex, } from '@jbrowse/text-indexing-core';
 import { Presets, SingleBar } from 'cli-progress';
-import { ixIxxStream } from 'ixixx';
 import { supported } from '../../types/common.js';
-// every index artifact lands in <outLocation>/trix, so the directory is created
-// here rather than by each caller: --dryrun goes nowhere near indexDriver and so
-// no longer leaves an empty trix/ behind
-function ensureTrixDir(outLocation) {
-    fs.mkdirSync(path.join(outLocation, TRIX_DIR), { recursive: true });
+function bar(format) {
+    return new SingleBar({ format, etaBuffer: 2000 }, Presets.shades_classic);
 }
-async function runIxIxx({ readStream, outLocation, name, prefixSize, quiet, }) {
-    const progressBar = new SingleBar({ format: '{bar} Sorting and writing index...', etaBuffer: 2000 }, Presets.shades_classic);
-    if (!quiet) {
-        progressBar.start(1, 0);
-    }
-    const { ix, ixx } = trixFilePaths(outLocation, name);
-    await ixIxxStream(readStream, ix, ixx, prefixSize);
-    if (!quiet) {
-        progressBar.update(1);
-        progressBar.stop();
-    }
-}
-export async function indexDriver({ trackConfigs, attributes, outLocation, name, quiet, typesToExclude, typesToInclude, assemblyNames, prefixSize, }) {
-    ensureTrixDir(outLocation);
-    const readStream = Readable.from(indexFiles({
+export async function indexDriver({ trackConfigs, policy, outLocation, name, quiet, assemblyNames, prefixSize, }) {
+    let sortBar;
+    await writeTrixIndex({
         tracks: trackConfigs,
-        attributesToIndex: attributes,
         outDir: outLocation,
-        featureTypesToExclude: typesToExclude,
-        featureTypesToInclude: typesToInclude,
+        name,
+        policy,
+        assemblyNames,
+        prefixSize,
+        onSort: quiet
+            ? undefined
+            : () => {
+                sortBar = bar('{bar} Sorting and writing index...');
+                sortBar.start(1, 0);
+            },
         makeProgress: quiet
             ? undefined
             : trackId => {
-                const progressBar = new SingleBar({
-                    // eslint-disable-next-line unicorn/no-incorrect-template-string-interpolation -- {bar}/{percentage}/{eta} are cli-progress format tokens, not JS interpolation
-                    format: `{bar} ${trackId} {percentage}% | ETA: {eta}s`,
-                    etaBuffer: 2000,
-                }, Presets.shades_classic);
+                const progressBar = bar(
+                // eslint-disable-next-line unicorn/no-incorrect-template-string-interpolation -- {bar}/{percentage}/{eta} are cli-progress format tokens, not JS interpolation
+                `{bar} ${trackId} {percentage}% | ETA: {eta}s`);
                 return {
                     onStart: totalBytes => {
                         progressBar.start(totalBytes, 0);
@@ -51,23 +37,9 @@ export async function indexDriver({ trackConfigs, attributes, outLocation, name,
                     },
                 };
             },
-    }));
-    await runIxIxx({
-        readStream,
-        outLocation,
-        name,
-        prefixSize,
-        quiet,
-    });
-    generateMeta({
-        configs: trackConfigs,
-        attributesToIndex: attributes,
-        outDir: outLocation,
-        name,
-        featureTypesToExclude: typesToExclude,
-        featureTypesToInclude: typesToInclude,
-        assemblyNames,
     });
+    sortBar?.update(1);
+    sortBar?.stop();
 }
 export function prepareFileTrackConfigs(files, fileIds) {
     return files.map((file, i) => {