@jbrowse/cli 4.3.0 → 5.0.0-beta.1

package/dist/commands/text-index/config-utils.js

@@ -1,114 +1,91 @@
-import fs from 'fs';
-import path from 'path';
-import { supported } from "../../types/common.js";
-/**
- * Parses a comma-separated string into an array of trimmed, non-empty strings
- */
-export function parseCommaSeparatedString(value) {
-    return (value
-        ?.split(',')
-        .map(s => s.trim())
-        .filter(Boolean) ?? []);
-}
-/**
- * Validates and parses a prefix size value
- */
-export function validatePrefixSize(value) {
+import path from 'node:path';
+import { supported } from '../../types/common.js';
+import { parseCommaSeparatedString, readConfigFile, resolveConfigPath, } from '../../utils.js';
+function validatePrefixSize(value) {
     if (!value) {
         return undefined;
     }
     const parsed = typeof value === 'number' ? value : parseInt(value, 10);
-    if (isNaN(parsed) || parsed < 0) {
+    if (Number.isNaN(parsed) || parsed <= 0) {
         throw new Error(`Invalid prefixSize: "${value}". Must be a positive number.`);
     }
     return parsed;
 }
-/**
- * Prepares common indexDriver parameters from raw flag values
- */
 export function prepareIndexDriverFlags(flags) {
     return {
         attributes: parseCommaSeparatedString(flags.attributes),
         typesToExclude: parseCommaSeparatedString(flags.exclude),
+        // undefined rather than [] when the flag is absent, because an empty allow
+        // list has to mean "no allow list" and not "index nothing"
+        typesToInclude: flags.include
+            ? parseCommaSeparatedString(flags.include)
+            : undefined,
         quiet: flags.quiet ?? false,
         prefixSize: validatePrefixSize(flags.prefixSize),
     };
 }
-export function readConf(configPath) {
-    return JSON.parse(fs.readFileSync(configPath, 'utf8'));
-}
-export function writeConf(obj, configPath) {
-    fs.writeFileSync(configPath, JSON.stringify(obj, null, 2));
-}
-/**
- * Loads config and prepares output location for indexing
- */
-export async function loadConfigForIndexing(target, out, resolveConfigPath) {
+export async function loadConfigForIndexing(target, out) {
     const configPath = await resolveConfigPath(target, out);
-    const outLocation = path.dirname(configPath);
-    const config = readConf(configPath);
-    ensureTrixDir(outLocation);
-    return { config, configPath, outLocation };
-}
-export function ensureTrixDir(outLocation) {
-    const trixDir = path.join(outLocation, 'trix');
-    if (!fs.existsSync(trixDir)) {
-        fs.mkdirSync(trixDir, { recursive: true });
-    }
-    return trixDir;
-}
-/**
- * Extracts all assembly names from a config object
- * Handles both single assembly (config.assembly) and multiple assemblies (config.assemblies)
- */
-function extractAssemblyNamesFromConfig(config) {
-    if (config.assemblies) {
-        return config.assemblies.map(a => a.name);
-    }
-    if (config.assembly) {
-        return [config.assembly.name];
-    }
-    return [];
+    // readConfigFile, not a bare readFileSync: run from the wrong directory, this
+    // is the error the user sees, and it should name the config.json it looked for
+    const config = await readConfigFile(configPath);
+    return { config, configPath, outLocation: path.dirname(configPath) };
 }
 export function getAssemblyNames(config, assemblies) {
     const asms = assemblies
         ? parseCommaSeparatedString(assemblies)
-        : extractAssemblyNamesFromConfig(config);
+        : (config.assemblies?.map(a => a.name) ??
+            (config.assembly ? [config.assembly.name] : []));
     if (!asms.length) {
         throw new Error('No assemblies found');
     }
     return asms;
 }
-export function getTrackConfigs(config, trackIds, assemblyName, excludeTrackIds) {
-    const { tracks } = config;
-    if (!tracks) {
-        return [];
+export function formatDryRun(trackConfigs) {
+    return trackConfigs.map(t => `${t.trackId}\t${t.adapter?.type}`).join('\n');
+}
+// why a track was left out, or undefined if it is indexable
+function skipReason(track, assemblyName, excludeSet) {
+    if (assemblyName && !track.assemblyNames.includes(assemblyName)) {
+        return `not in assembly '${assemblyName}'`;
     }
-    const trackIdsToIndex = trackIds?.length
-        ? trackIds
-        : tracks.map(track => track.trackId);
-    const excludeSet = new Set(excludeTrackIds || []);
-    return trackIdsToIndex
-        .map(trackId => {
-        const currentTrack = tracks.find(t => trackId === t.trackId);
-        if (!currentTrack) {
-            throw new Error(`Track not found in config.json for trackId ${trackId}, please add track configuration before indexing.`);
-        }
-        return currentTrack;
-    })
-        .filter(track => {
-        if (excludeSet.has(track.trackId)) {
-            // console.log(`Skipping ${track.trackId}: excluded via --exclude-tracks`)
-            return false;
-        }
-        if (!supported(track.adapter?.type)) {
-            return false;
-        }
-        if (assemblyName && !track.assemblyNames.includes(assemblyName)) {
-            console.log(`Skipping ${track.trackId}: not in assembly '${assemblyName}'`);
-            return false;
+    else if (excludeSet.has(track.trackId)) {
+        return 'excluded with --excludeTracks';
+    }
+    else if (track.metadata?.skipTextIndex) {
+        return 'metadata.skipTextIndex is set';
+    }
+    else if (!supported(track.adapter?.type)) {
+        return `adapter type ${track.adapter?.type} is not indexable`;
+    }
+    else {
+        return undefined;
+    }
+}
+export function getTrackConfigs(config, trackIds, assemblyName, excludeTrackIds) {
+    const tracks = config.tracks ?? [];
+    // when specific trackIds are requested every one must exist; otherwise
+    // consider all tracks in the config
+    const requested = trackIds?.length
+        ? trackIds.map(trackId => {
+            const track = tracks.find(t => t.trackId === trackId);
+            if (track) {
+                return track;
+            }
+            else {
+                throw new Error(`Track not found in config.json for trackId ${trackId}, please add track configuration before indexing.`);
+            }
+        })
+        : tracks;
+    const excludeSet = new Set(excludeTrackIds);
+    return requested.filter(track => {
+        const reason = skipReason(track, assemblyName, excludeSet);
+        // only narrate skips for tracks the user named: sweeping a multi-assembly
+        // config otherwise prints a line per track per assembly
+        if (reason && trackIds?.length) {
+            console.log(`Skipping ${track.trackId}: ${reason}`);
         }
-        return true;
+        return !reason;
     });
 }
 export { sanitizeForFilename as sanitizeNameForPath } from '@jbrowse/text-indexing-core';

package/dist/commands/text-index/file-list.js

@@ -1,22 +1,36 @@
-import path from 'path';
-import { ensureTrixDir, prepareIndexDriverFlags, sanitizeNameForPath, } from "./config-utils.js";
-import { indexDriver, prepareFileTrackConfigs } from "./indexing-utils.js";
-import { validateFileInput } from "./validators.js";
+import path from 'node:path';
+import { formatDryRun, prepareIndexDriverFlags, sanitizeNameForPath, } from './config-utils.js';
+import { indexDriver, prepareFileTrackConfigs } from './indexing-utils.js';
+import { validateFileInput } from './validators.js';
 export async function indexFileList(flags) {
-    const { out, target, fileId, file, attributes, quiet, exclude, prefixSize } = flags;
+    const { out, target, fileId, file, attributes, quiet, exclude, include, prefixSize, dryrun, } = flags;
     validateFileInput(file);
-    const outFlag = target || out || '.';
-    ensureTrixDir(outFlag);
+    // --out/--target names either the install directory or its config.json, as
+    // every other command accepts; the trix output goes beside the config, and
+    // pointing at the config.json used to mkdir config.json/trix and fail ENOTDIR
+    const outArg = target || out || '.';
+    const outLocation = outArg.endsWith('.json') ? path.dirname(outArg) : outArg;
     const trackConfigs = prepareFileTrackConfigs(file, fileId);
-    const name = trackConfigs.length > 1
-        ? 'aggregate'
-        : sanitizeNameForPath(path.basename(file[0]));
-    await indexDriver({
-        trackConfigs,
-        outLocation: outFlag,
-        name,
-        assemblyNames: [],
-        ...prepareIndexDriverFlags({ attributes, exclude, quiet, prefixSize }),
-    });
-    console.log('Successfully created index for these files. See https://jbrowse.org/storybook/lgv/main/?path=/story/text-searching--page for info about usage');
+    if (dryrun) {
+        console.log(formatDryRun(trackConfigs));
+    }
+    else {
+        const name = trackConfigs.length > 1
+            ? 'aggregate'
+            : sanitizeNameForPath(path.basename(file[0]));
+        await indexDriver({
+            trackConfigs,
+            outLocation,
+            name,
+            assemblyNames: [],
+            ...prepareIndexDriverFlags({
+                attributes,
+                exclude,
+                include,
+                quiet,
+                prefixSize,
+            }),
+        });
+        console.log('Successfully created index for these files. See https://jbrowse.org/storybook/lgv/with-aggregate-text-searching/ for info about usage');
+    }
 }

package/dist/commands/text-index/index.js

@@ -1,4 +1,4 @@
-export { aggregateIndex } from "./aggregate.js";
-export { perTrackIndex } from "./per-track.js";
-export { indexFileList } from "./file-list.js";
-export { run } from "./command.js";
+export { aggregateIndex } from './aggregate.js';
+export { perTrackIndex } from './per-track.js';
+export { indexFileList } from './file-list.js';
+export { run } from './command.js';

package/dist/commands/text-index/indexing-utils.js

@@ -1,85 +1,56 @@
-import path from 'path';
-import { Readable } from 'stream';
-import { generateMeta, guessAdapterFromFileName, indexGff3, indexVcf, } from '@jbrowse/text-indexing-core';
+import fs from 'node:fs';
+import path from 'node:path';
+import { Readable } from 'node:stream';
+import { TRIX_DIR, generateMeta, guessAdapterFromFileName, indexFiles, trixFilePaths, } from '@jbrowse/text-indexing-core';
 import { Presets, SingleBar } from 'cli-progress';
 import { ixIxxStream } from 'ixixx';
-import { getAdapterLocation, getLoc } from "./adapter-utils.js";
-import { supported } from "../../types/common.js";
-export async function runIxIxx({ readStream, outLocation, name, prefixSize, quiet, }) {
+import { supported } from '../../types/common.js';
+// every index artifact lands in <outLocation>/trix, so the directory is created
+// here rather than by each caller: --dryrun goes nowhere near indexDriver and so
+// no longer leaves an empty trix/ behind
+function ensureTrixDir(outLocation) {
+    fs.mkdirSync(path.join(outLocation, TRIX_DIR), { recursive: true });
+}
+async function runIxIxx({ readStream, outLocation, name, prefixSize, quiet, }) {
     const progressBar = new SingleBar({ format: '{bar} Sorting and writing index...', etaBuffer: 2000 }, Presets.shades_classic);
     if (!quiet) {
         progressBar.start(1, 0);
     }
-    const ixPath = path.join(outLocation, 'trix', `${name}.ix`);
-    const ixxPath = path.join(outLocation, 'trix', `${name}.ixx`);
-    await ixIxxStream(readStream, ixPath, ixxPath, prefixSize);
+    const { ix, ixx } = trixFilePaths(outLocation, name);
+    await ixIxxStream(readStream, ix, ixx, prefixSize);
     if (!quiet) {
         progressBar.update(1);
         progressBar.stop();
     }
 }
-export async function* indexFiles({ trackConfigs, attributes, outLocation, quiet, typesToExclude, }) {
-    for (const config of trackConfigs) {
-        const { adapter, textSearching, trackId } = config;
-        const { type } = adapter || {};
-        const { indexingFeatureTypesToExclude = typesToExclude, indexingAttributes = attributes, } = textSearching || {};
-        const loc = getAdapterLocation(adapter);
-        if (!loc) {
-            continue;
-        }
-        const progressBar = new SingleBar({
-            format: `{bar} ${trackId} {percentage}% | ETA: {eta}s`,
-            etaBuffer: 2000,
-        }, Presets.shades_classic);
-        if (type === 'Gff3TabixAdapter' || type === 'Gff3Adapter') {
-            yield* indexGff3({
-                config,
-                attributesToIndex: indexingAttributes,
-                inLocation: getLoc(loc),
-                outDir: outLocation,
-                featureTypesToExclude: indexingFeatureTypesToExclude,
-                onStart: totalBytes => {
-                    if (!quiet) {
-                        progressBar.start(totalBytes, 0);
-                    }
-                },
-                onUpdate: receivedBytes => {
-                    if (!quiet) {
-                        progressBar.update(receivedBytes);
-                    }
-                },
-            });
-        }
-        else if (type === 'VcfTabixAdapter' || type === 'VcfAdapter') {
-            yield* indexVcf({
-                config,
-                attributesToIndex: indexingAttributes,
-                inLocation: getLoc(loc),
-                outDir: outLocation,
-                onStart: totalBytes => {
-                    if (!quiet) {
+export async function indexDriver({ trackConfigs, attributes, outLocation, name, quiet, typesToExclude, typesToInclude, assemblyNames, prefixSize, }) {
+    ensureTrixDir(outLocation);
+    const readStream = Readable.from(indexFiles({
+        tracks: trackConfigs,
+        attributesToIndex: attributes,
+        outDir: outLocation,
+        featureTypesToExclude: typesToExclude,
+        featureTypesToInclude: typesToInclude,
+        makeProgress: quiet
+            ? undefined
+            : trackId => {
+                const progressBar = new SingleBar({
+                    // eslint-disable-next-line unicorn/no-incorrect-template-string-interpolation -- {bar}/{percentage}/{eta} are cli-progress format tokens, not JS interpolation
+                    format: `{bar} ${trackId} {percentage}% | ETA: {eta}s`,
+                    etaBuffer: 2000,
+                }, Presets.shades_classic);
+                return {
+                    onStart: totalBytes => {
                         progressBar.start(totalBytes, 0);
-                    }
-                },
-                onUpdate: receivedBytes => {
-                    if (!quiet) {
+                    },
+                    onUpdate: receivedBytes => {
                         progressBar.update(receivedBytes);
-                    }
-                },
-            });
-        }
-        if (!quiet) {
-            progressBar.stop();
-        }
-    }
-}
-export async function indexDriver({ trackConfigs, attributes, outLocation, name, quiet, typesToExclude, assemblyNames, prefixSize, }) {
-    const readStream = Readable.from(indexFiles({
-        trackConfigs,
-        attributes,
-        outLocation,
-        quiet,
-        typesToExclude,
+                    },
+                    onDone: () => {
+                        progressBar.stop();
+                    },
+                };
+            },
     }));
     await runIxIxx({
         readStream,
@@ -94,17 +65,22 @@ export async function indexDriver({ trackConfigs, attributes, outLocation, name,
         outDir: outLocation,
         name,
         featureTypesToExclude: typesToExclude,
+        featureTypesToInclude: typesToInclude,
         assemblyNames,
     });
 }
 export function prepareFileTrackConfigs(files, fileIds) {
-    return files
-        .map((file, i) => {
+    return files.map((file, i) => {
         const config = guessAdapterFromFileName(file);
         if (fileIds?.[i]) {
             config.trackId = fileIds[i];
         }
+        // throw rather than silently filtering: a user who named the file with
+        // --file should be told it can't be indexed instead of getting a misleading
+        // "successfully created index" with no output
+        if (!supported(config.adapter?.type)) {
+            throw new Error(`Cannot text-index ${file}: adapter type ${config.adapter?.type} is not indexable (only GFF3, GTF and VCF are supported)`);
+        }
         return config;
-    })
-        .filter(fileConfig => supported(fileConfig.adapter?.type));
+    });
 }

package/dist/commands/text-index/per-track.js

@@ -1,18 +1,22 @@
-import { createTrixAdapter } from "./adapter-utils.js";
-import { getTrackConfigs, loadConfigForIndexing, parseCommaSeparatedString, prepareIndexDriverFlags, writeConf, } from "./config-utils.js";
-import { indexDriver } from "./indexing-utils.js";
-import { validateAssembliesForPerTrack } from "./validators.js";
-import { resolveConfigPath } from "../../utils.js";
+import { parseCommaSeparatedString, writeJsonFile } from '../../utils.js';
+import { createTrixAdapter } from './adapter-utils.js';
+import { formatDryRun, getTrackConfigs, loadConfigForIndexing, prepareIndexDriverFlags, } from './config-utils.js';
+import { indexDriver } from './indexing-utils.js';
+import { validateAssembliesForPerTrack, validateTrackConfigs, } from './validators.js';
 export async function perTrackIndex(flags) {
-    const { out, target, tracks, excludeTracks, assemblies, attributes, quiet, force, exclude, prefixSize, } = flags;
-    const { config, configPath, outLocation } = await loadConfigForIndexing(target, out, resolveConfigPath);
-    const configTracks = config.tracks || [];
+    const { out, target, tracks, excludeTracks, assemblies, attributes, quiet, force, exclude, include, prefixSize, dryrun, } = flags;
     validateAssembliesForPerTrack(assemblies);
+    const { config, configPath, outLocation } = await loadConfigForIndexing(target, out);
+    const configTracks = config.tracks ?? [];
     const confs = getTrackConfigs(config, parseCommaSeparatedString(tracks), undefined, parseCommaSeparatedString(excludeTracks));
-    if (!confs.length) {
-        throw new Error('Tracks not found in config.json, please add track configurations before indexing.');
+    validateTrackConfigs(confs);
+    if (dryrun) {
+        console.log(formatDryRun(confs));
+        return;
     }
-    let hasChanges = false;
+    // sequential, not Promise.all: each track's indexer owns the progress bar and
+    // streams a whole file through ixIxx
+    const indexed = [];
     for (const trackConfig of confs) {
         const { textSearching, trackId, assemblyNames } = trackConfig;
         if (textSearching?.textSearchAdapter && !force) {
@@ -25,27 +29,36 @@ export async function perTrackIndex(flags) {
             outLocation,
             name: trackId,
             assemblyNames,
-            ...prepareIndexDriverFlags({ attributes, exclude, quiet, prefixSize }),
+            ...prepareIndexDriverFlags({
+                attributes,
+                exclude,
+                include,
+                quiet,
+                prefixSize,
+            }),
         });
-        if (!textSearching?.textSearchAdapter) {
-            // modifies track with new text search adapter
-            const index = configTracks.findIndex(track => trackId === track.trackId);
-            if (index !== -1) {
-                configTracks[index] = {
-                    ...trackConfig,
-                    textSearching: {
-                        ...textSearching,
-                        textSearchAdapter: createTrixAdapter(trackId, assemblyNames),
-                    },
-                };
-                hasChanges = true;
-            }
-            else {
-                console.warn(`Warning: can't find trackId ${trackId}`);
-            }
-        }
+        indexed.push(trackConfig);
     }
-    if (hasChanges) {
-        writeConf({ ...config, tracks: configTracks }, configPath);
+    // a track that already carried an adapter keeps it — re-indexing under --force
+    // refreshes the trix files the existing adapter already points at. The rest
+    // gain one naming the files just written. getTrackConfigs returns elements of
+    // config.tracks, so every indexed track is somewhere in configTracks.
+    const newAdapters = new Map(indexed
+        .filter(track => !track.textSearching?.textSearchAdapter)
+        .map(track => [
+        track.trackId,
+        createTrixAdapter(track.trackId, track.assemblyNames),
+    ]));
+    const updatedTracks = configTracks.map(track => {
+        const textSearchAdapter = newAdapters.get(track.trackId);
+        return textSearchAdapter
+            ? {
+                ...track,
+                textSearching: { ...track.textSearching, textSearchAdapter },
+            }
+            : track;
+    });
+    if (updatedTracks.some((track, i) => track !== configTracks[i])) {
+        await writeJsonFile(configPath, { ...config, tracks: updatedTracks });
     }
 }

package/dist/commands/text-index/validators.js

@@ -4,7 +4,7 @@ export function validateTrackConfigs(tracks) {
     }
 }
 export function validateFileInput(file) {
-    if (!file) {
+    if (!file?.length) {
         throw new Error('Cannot index file list without files');
     }
 }

package/dist/commands/upgrade.js

@@ -1,9 +1,7 @@
-import fs from 'fs';
-import path from 'path';
-import { parseArgs } from 'util';
-import decompress from 'decompress';
-import fetch from "../cliFetch.js";
-import { fetchGithubVersions, getBranch, getLatest, getTag, printHelp, } from "../utils.js";
+import fs from 'node:fs';
+import path from 'node:path';
+import { parseArgs } from 'node:util';
+import { downloadRelease, extractZip, printHelp, printVersions, } from '../utils.js';
 const description = 'Upgrades JBrowse 2 to latest version';
 const examples = [
     '# Upgrades current directory to latest jbrowse release',
@@ -65,8 +63,6 @@ export async function run(args) {
         args,
         allowPositionals: true,
     });
-    const argsPath = positionals[0];
-    const { clean, listVersions, tag, url, branch, nightly } = runFlags;
     if (runFlags.help) {
         printHelp({
             options,
@@ -76,41 +72,33 @@ export async function run(args) {
         });
         return;
     }
+    const { clean, listVersions, tag, url, branch, nightly } = runFlags;
     if (listVersions) {
-        const versions = (await fetchGithubVersions()).map(v => v.tag_name);
-        console.log(`All JBrowse versions:\n${versions.join('\n')}`);
-        process.exit(0);
-    }
-    if (!argsPath) {
-        throw new Error('No directory supplied');
+        await printVersions();
+        return;
     }
+    // localPath is optional: `jbrowse upgrade` upgrades the install in the
+    // current directory, as the examples and docs advertise
+    const argsPath = positionals[0] ?? '.';
     if (!fs.existsSync(path.join(argsPath, 'manifest.json'))) {
         throw new Error(`No manifest.json found in this directory, are you sure it is an
         existing jbrowse 2 installation?`);
     }
-    const locationUrl = url ||
-        (nightly ? await getBranch('main') : '') ||
-        (branch ? await getBranch(branch) : '') ||
-        (tag ? await getTag(tag) : await getLatest());
-    console.log(`Fetching ${locationUrl}...`);
-    const response = await fetch(locationUrl);
-    if (!response.ok) {
-        throw new Error(`HTTP ${response.status} fetching ${locationUrl}: ${response.statusText}`);
-    }
-    const type = response.headers.get('content-type');
-    if (url &&
-        type !== 'application/zip' &&
-        type !== 'application/octet-stream') {
-        throw new Error('The URL provided does not seem to be a JBrowse installation URL');
-    }
+    const { locationUrl, archive } = await downloadRelease({
+        url,
+        nightly,
+        branch,
+        tag,
+    });
     if (clean) {
         fs.rmSync(path.join(argsPath, 'static'), { recursive: true, force: true });
-        for (const f of fs
+        const workerFiles = fs
             .readdirSync(argsPath)
-            .filter(f => f.includes('worker.js'))) {
+            .filter(f => f.includes('worker.js'));
+        for (const f of workerFiles) {
             fs.unlinkSync(path.join(argsPath, f));
         }
     }
-    await decompress(Buffer.from(await response.arrayBuffer()), argsPath);
+    await extractZip(archive, argsPath);
     console.log(`Unpacked ${locationUrl} at ${argsPath}`);
 }