@jbrowse/cli 4.3.0 → 5.0.0-beta.2

package/dist/commands/text-index/indexing-utils.js

@@ -1,85 +1,56 @@
-import path from 'path';
-import { Readable } from 'stream';
-import { generateMeta, guessAdapterFromFileName, indexGff3, indexVcf, } from '@jbrowse/text-indexing-core';
+import fs from 'node:fs';
+import path from 'node:path';
+import { Readable } from 'node:stream';
+import { TRIX_DIR, generateMeta, guessAdapterFromFileName, indexFiles, trixFilePaths, } from '@jbrowse/text-indexing-core';
 import { Presets, SingleBar } from 'cli-progress';
 import { ixIxxStream } from 'ixixx';
-import { getAdapterLocation, getLoc } from "./adapter-utils.js";
-import { supported } from "../../types/common.js";
-export async function runIxIxx({ readStream, outLocation, name, prefixSize, quiet, }) {
+import { supported } from '../../types/common.js';
+// every index artifact lands in <outLocation>/trix, so the directory is created
+// here rather than by each caller: --dryrun goes nowhere near indexDriver and so
+// no longer leaves an empty trix/ behind
+function ensureTrixDir(outLocation) {
+    fs.mkdirSync(path.join(outLocation, TRIX_DIR), { recursive: true });
+}
+async function runIxIxx({ readStream, outLocation, name, prefixSize, quiet, }) {
     const progressBar = new SingleBar({ format: '{bar} Sorting and writing index...', etaBuffer: 2000 }, Presets.shades_classic);
     if (!quiet) {
         progressBar.start(1, 0);
     }
-    const ixPath = path.join(outLocation, 'trix', `${name}.ix`);
-    const ixxPath = path.join(outLocation, 'trix', `${name}.ixx`);
-    await ixIxxStream(readStream, ixPath, ixxPath, prefixSize);
+    const { ix, ixx } = trixFilePaths(outLocation, name);
+    await ixIxxStream(readStream, ix, ixx, prefixSize);
     if (!quiet) {
         progressBar.update(1);
         progressBar.stop();
     }
 }
-export async function* indexFiles({ trackConfigs, attributes, outLocation, quiet, typesToExclude, }) {
-    for (const config of trackConfigs) {
-        const { adapter, textSearching, trackId } = config;
-        const { type } = adapter || {};
-        const { indexingFeatureTypesToExclude = typesToExclude, indexingAttributes = attributes, } = textSearching || {};
-        const loc = getAdapterLocation(adapter);
-        if (!loc) {
-            continue;
-        }
-        const progressBar = new SingleBar({
-            format: `{bar} ${trackId} {percentage}% | ETA: {eta}s`,
-            etaBuffer: 2000,
-        }, Presets.shades_classic);
-        if (type === 'Gff3TabixAdapter' || type === 'Gff3Adapter') {
-            yield* indexGff3({
-                config,
-                attributesToIndex: indexingAttributes,
-                inLocation: getLoc(loc),
-                outDir: outLocation,
-                featureTypesToExclude: indexingFeatureTypesToExclude,
-                onStart: totalBytes => {
-                    if (!quiet) {
-                        progressBar.start(totalBytes, 0);
-                    }
-                },
-                onUpdate: receivedBytes => {
-                    if (!quiet) {
-                        progressBar.update(receivedBytes);
-                    }
-                },
-            });
-        }
-        else if (type === 'VcfTabixAdapter' || type === 'VcfAdapter') {
-            yield* indexVcf({
-                config,
-                attributesToIndex: indexingAttributes,
-                inLocation: getLoc(loc),
-                outDir: outLocation,
-                onStart: totalBytes => {
-                    if (!quiet) {
+export async function indexDriver({ trackConfigs, attributes, outLocation, name, quiet, typesToExclude, typesToInclude, assemblyNames, prefixSize, }) {
+    ensureTrixDir(outLocation);
+    const readStream = Readable.from(indexFiles({
+        tracks: trackConfigs,
+        attributesToIndex: attributes,
+        outDir: outLocation,
+        featureTypesToExclude: typesToExclude,
+        featureTypesToInclude: typesToInclude,
+        makeProgress: quiet
+            ? undefined
+            : trackId => {
+                const progressBar = new SingleBar({
+                    // eslint-disable-next-line unicorn/no-incorrect-template-string-interpolation -- {bar}/{percentage}/{eta} are cli-progress format tokens, not JS interpolation
+                    format: `{bar} ${trackId} {percentage}% | ETA: {eta}s`,
+                    etaBuffer: 2000,
+                }, Presets.shades_classic);
+                return {
+                    onStart: totalBytes => {
                         progressBar.start(totalBytes, 0);
-                    }
-                },
-                onUpdate: receivedBytes => {
-                    if (!quiet) {
+                    },
+                    onUpdate: receivedBytes => {
                         progressBar.update(receivedBytes);
-                    }
-                },
-            });
-        }
-        if (!quiet) {
-            progressBar.stop();
-        }
-    }
-}
-export async function indexDriver({ trackConfigs, attributes, outLocation, name, quiet, typesToExclude, assemblyNames, prefixSize, }) {
-    const readStream = Readable.from(indexFiles({
-        trackConfigs,
-        attributes,
-        outLocation,
-        quiet,
-        typesToExclude,
+                    },
+                    onDone: () => {
+                        progressBar.stop();
+                    },
+                };
+            },
     }));
     await runIxIxx({
         readStream,
@@ -94,17 +65,22 @@ export async function indexDriver({ trackConfigs, attributes, outLocation, name,
         outDir: outLocation,
         name,
         featureTypesToExclude: typesToExclude,
+        featureTypesToInclude: typesToInclude,
         assemblyNames,
     });
 }
 export function prepareFileTrackConfigs(files, fileIds) {
-    return files
-        .map((file, i) => {
+    return files.map((file, i) => {
         const config = guessAdapterFromFileName(file);
         if (fileIds?.[i]) {
             config.trackId = fileIds[i];
         }
+        // throw rather than silently filtering: a user who named the file with
+        // --file should be told it can't be indexed instead of getting a misleading
+        // "successfully created index" with no output
+        if (!supported(config.adapter?.type)) {
+            throw new Error(`Cannot text-index ${file}: adapter type ${config.adapter?.type} is not indexable (only GFF3, GTF and VCF are supported)`);
+        }
         return config;
-    })
-        .filter(fileConfig => supported(fileConfig.adapter?.type));
+    });
 }

package/dist/commands/text-index/per-track.js

@@ -1,18 +1,22 @@
-import { createTrixAdapter } from "./adapter-utils.js";
-import { getTrackConfigs, loadConfigForIndexing, parseCommaSeparatedString, prepareIndexDriverFlags, writeConf, } from "./config-utils.js";
-import { indexDriver } from "./indexing-utils.js";
-import { validateAssembliesForPerTrack } from "./validators.js";
-import { resolveConfigPath } from "../../utils.js";
+import { parseCommaSeparatedString, writeJsonFile } from '../../utils.js';
+import { createTrixAdapter } from './adapter-utils.js';
+import { formatDryRun, getTrackConfigs, loadConfigForIndexing, prepareIndexDriverFlags, } from './config-utils.js';
+import { indexDriver } from './indexing-utils.js';
+import { validateAssembliesForPerTrack, validateTrackConfigs, } from './validators.js';
 export async function perTrackIndex(flags) {
-    const { out, target, tracks, excludeTracks, assemblies, attributes, quiet, force, exclude, prefixSize, } = flags;
-    const { config, configPath, outLocation } = await loadConfigForIndexing(target, out, resolveConfigPath);
-    const configTracks = config.tracks || [];
+    const { out, target, tracks, excludeTracks, assemblies, attributes, quiet, force, exclude, include, prefixSize, dryrun, } = flags;
     validateAssembliesForPerTrack(assemblies);
+    const { config, configPath, outLocation } = await loadConfigForIndexing(target, out);
+    const configTracks = config.tracks ?? [];
     const confs = getTrackConfigs(config, parseCommaSeparatedString(tracks), undefined, parseCommaSeparatedString(excludeTracks));
-    if (!confs.length) {
-        throw new Error('Tracks not found in config.json, please add track configurations before indexing.');
+    validateTrackConfigs(confs);
+    if (dryrun) {
+        console.log(formatDryRun(confs));
+        return;
     }
-    let hasChanges = false;
+    // sequential, not Promise.all: each track's indexer owns the progress bar and
+    // streams a whole file through ixIxx
+    const indexed = [];
     for (const trackConfig of confs) {
         const { textSearching, trackId, assemblyNames } = trackConfig;
         if (textSearching?.textSearchAdapter && !force) {
@@ -25,27 +29,36 @@ export async function perTrackIndex(flags) {
             outLocation,
             name: trackId,
             assemblyNames,
-            ...prepareIndexDriverFlags({ attributes, exclude, quiet, prefixSize }),
+            ...prepareIndexDriverFlags({
+                attributes,
+                exclude,
+                include,
+                quiet,
+                prefixSize,
+            }),
         });
-        if (!textSearching?.textSearchAdapter) {
-            // modifies track with new text search adapter
-            const index = configTracks.findIndex(track => trackId === track.trackId);
-            if (index !== -1) {
-                configTracks[index] = {
-                    ...trackConfig,
-                    textSearching: {
-                        ...textSearching,
-                        textSearchAdapter: createTrixAdapter(trackId, assemblyNames),
-                    },
-                };
-                hasChanges = true;
-            }
-            else {
-                console.warn(`Warning: can't find trackId ${trackId}`);
-            }
-        }
+        indexed.push(trackConfig);
     }
-    if (hasChanges) {
-        writeConf({ ...config, tracks: configTracks }, configPath);
+    // a track that already carried an adapter keeps it — re-indexing under --force
+    // refreshes the trix files the existing adapter already points at. The rest
+    // gain one naming the files just written. getTrackConfigs returns elements of
+    // config.tracks, so every indexed track is somewhere in configTracks.
+    const newAdapters = new Map(indexed
+        .filter(track => !track.textSearching?.textSearchAdapter)
+        .map(track => [
+        track.trackId,
+        createTrixAdapter(track.trackId, track.assemblyNames),
+    ]));
+    const updatedTracks = configTracks.map(track => {
+        const textSearchAdapter = newAdapters.get(track.trackId);
+        return textSearchAdapter
+            ? {
+                ...track,
+                textSearching: { ...track.textSearching, textSearchAdapter },
+            }
+            : track;
+    });
+    if (updatedTracks.some((track, i) => track !== configTracks[i])) {
+        await writeJsonFile(configPath, { ...config, tracks: updatedTracks });
     }
 }

package/dist/commands/text-index/validators.js

@@ -4,7 +4,7 @@ export function validateTrackConfigs(tracks) {
     }
 }
 export function validateFileInput(file) {
-    if (!file) {
+    if (!file?.length) {
         throw new Error('Cannot index file list without files');
     }
 }

package/dist/commands/upgrade.js

@@ -1,9 +1,7 @@
-import fs from 'fs';
-import path from 'path';
-import { parseArgs } from 'util';
-import decompress from 'decompress';
-import fetch from "../cliFetch.js";
-import { fetchGithubVersions, getBranch, getLatest, getTag, printHelp, } from "../utils.js";
+import fs from 'node:fs';
+import path from 'node:path';
+import { parseArgs } from 'node:util';
+import { downloadRelease, extractZip, printHelp, printVersions, } from '../utils.js';
 const description = 'Upgrades JBrowse 2 to latest version';
 const examples = [
     '# Upgrades current directory to latest jbrowse release',
@@ -65,8 +63,6 @@ export async function run(args) {
         args,
         allowPositionals: true,
     });
-    const argsPath = positionals[0];
-    const { clean, listVersions, tag, url, branch, nightly } = runFlags;
     if (runFlags.help) {
         printHelp({
             options,
@@ -76,41 +72,33 @@ export async function run(args) {
         });
         return;
     }
+    const { clean, listVersions, tag, url, branch, nightly } = runFlags;
     if (listVersions) {
-        const versions = (await fetchGithubVersions()).map(v => v.tag_name);
-        console.log(`All JBrowse versions:\n${versions.join('\n')}`);
-        process.exit(0);
-    }
-    if (!argsPath) {
-        throw new Error('No directory supplied');
+        await printVersions();
+        return;
     }
+    // localPath is optional: `jbrowse upgrade` upgrades the install in the
+    // current directory, as the examples and docs advertise
+    const argsPath = positionals[0] ?? '.';
     if (!fs.existsSync(path.join(argsPath, 'manifest.json'))) {
         throw new Error(`No manifest.json found in this directory, are you sure it is an
         existing jbrowse 2 installation?`);
     }
-    const locationUrl = url ||
-        (nightly ? await getBranch('main') : '') ||
-        (branch ? await getBranch(branch) : '') ||
-        (tag ? await getTag(tag) : await getLatest());
-    console.log(`Fetching ${locationUrl}...`);
-    const response = await fetch(locationUrl);
-    if (!response.ok) {
-        throw new Error(`HTTP ${response.status} fetching ${locationUrl}: ${response.statusText}`);
-    }
-    const type = response.headers.get('content-type');
-    if (url &&
-        type !== 'application/zip' &&
-        type !== 'application/octet-stream') {
-        throw new Error('The URL provided does not seem to be a JBrowse installation URL');
-    }
+    const { locationUrl, archive } = await downloadRelease({
+        url,
+        nightly,
+        branch,
+        tag,
+    });
     if (clean) {
         fs.rmSync(path.join(argsPath, 'static'), { recursive: true, force: true });
-        for (const f of fs
+        const workerFiles = fs
             .readdirSync(argsPath)
-            .filter(f => f.includes('worker.js'))) {
+            .filter(f => f.includes('worker.js'));
+        for (const f of workerFiles) {
             fs.unlinkSync(path.join(argsPath, f));
         }
     }
-    await decompress(Buffer.from(await response.arrayBuffer()), argsPath);
+    await extractZip(archive, argsPath);
     console.log(`Unpacked ${locationUrl} at ${argsPath}`);
 }