@jbrowse/cli 4.3.0 → 5.0.0-beta.10

package/dist/commands/add-track-utils/adapter-utils.js

@@ -1,302 +1,203 @@
-export function makeLocationProtocol(protocol) {
-    return (location) => {
-        if (protocol === 'uri') {
+import fs from 'node:fs';
+import { adapterTypesToTrackTypeMap, formats, matchFormat, resolveIndexType, sidecarCandidateNames, trackTypeForAdapter, } from '@jbrowse/add-track-core';
+import { isURL } from '../../types/common.js';
+import { densitySidecarPath } from '../shared/density.js';
+import { configManifest } from '../validate/configManifest.generated.js';
+function hasLocField(spec) {
+    return 'locField' in spec;
+}
+const adapterTypeToSpec = {};
+for (const { spec } of formats) {
+    if (hasLocField(spec) && !adapterTypeToSpec[spec.adapterType]) {
+        adapterTypeToSpec[spec.adapterType] = spec;
+    }
+}
+/**
+ * The sidecar actually sitting beside `location`, or the conventional name when
+ * none of the spellings is there — so a missing index still reports the file
+ * that was expected, and `--load` still names it.
+ *
+ * Probed with `existsSync`, which is false for every candidate of a URL and so
+ * leaves a remote track with exactly the conventional guess it had before this
+ * existed. A remote `.csi` still wants `--indexFile`.
+ */
+export function siblingSidecar(location, suffix) {
+    const candidates = sidecarCandidateNames(location, suffix);
+    return candidates.find(c => fs.existsSync(c)) ?? candidates[0];
+}
+// The adapters that declare a `densityAdapter` slot, read off the generated
+// manifest `jbrowse validate` uses. The hand-written list missed `GWASAdapter`,
+// which inherits the slot through `bedTabixConfigSchema` instead of spreading
+// `densityAdapterConfigSchemaFields`, so `--density` on a Pan-UKBB file was
+// refused by a message that listed nine adapters and not the tenth.
+const densityAdapterTypes = new Set(Object.entries(configManifest.adapters)
+    .filter(([, entry]) => entry.slots.some(s => s.name === 'densityAdapter'))
+    .map(([name]) => name)
+    .sort());
+/**
+ * The adapter with its density sidecar attached, and the local file to load
+ * beside the config when there is one.
+ *
+ * An explicit `--density` is taken as given, since a URL cannot be probed;
+ * otherwise the conventional `<file>.density.bw` counts only when it is
+ * actually there, the way `siblingSidecar` probes an index.
+ */
+export function withDensityAdapter({ adapter, location, density, makeLocation, }) {
+    const supported = densityAdapterTypes.has(adapter.type);
+    if (density !== undefined && !supported) {
+        throw new Error(`--density has no slot on ${adapter.type}. The adapters carrying a density sidecar are: ${[...densityAdapterTypes].join(', ')}`);
+    }
+    const probed = location !== undefined && fs.existsSync(densitySidecarPath(location))
+        ? densitySidecarPath(location)
+        : undefined;
+    if (probed !== undefined && !supported) {
+        console.warn(`Warning: ${probed} sits beside the track file but ${adapter.type} has no densityAdapter slot, so it is not attached. The adapters carrying one are indexed: bgzip and tabix the file, then add it.`);
+    }
+    const file = density === undefined ? (supported ? probed : undefined) : density;
+    return file === undefined
+        ? { adapter }
+        : {
+            adapter: {
+                ...adapter,
+                densityAdapter: {
+                    type: 'BigWigAdapter',
+                    bigWigLocation: makeLocation(file),
+                },
+            },
+            file: isURL(file) ? undefined : file,
+        };
+}
+// builds the adapter object and the list of source files for a spec in one
+// place, so the config it writes and the files add-track copies can never drift
+function buildFromSpec(spec, { location, index, bed1, bed2, makeLocation }) {
+    switch (spec.kind) {
+        case 'single':
+            return {
+                adapter: {
+                    type: spec.adapterType,
+                    [spec.locField]: makeLocation(location),
+                },
+                files: [location],
+            };
+        case 'indexed': {
+            const idx = index || siblingSidecar(location, spec.suffix);
             return {
-                uri: location,
-                locationType: 'UriLocation',
+                adapter: {
+                    type: spec.adapterType,
+                    [spec.locField]: makeLocation(location),
+                    index: {
+                        location: makeLocation(idx),
+                        // the type follows the file that was CHOSEN, not the one the user
+                        // typed, so a detected `.csi` is opened as one
+                        indexType: resolveIndexType(idx, spec.indexType),
+                    },
+                },
+                files: [location, idx],
             };
         }
-        if (protocol === 'localPath') {
+        case 'sidecar': {
+            const sidecars = spec.sidecars.map(s => ({
+                field: s.field,
+                path: s.fromIndex && index ? index : siblingSidecar(location, s.suffix),
+            }));
             return {
-                localPath: location,
-                locationType: 'LocalPathLocation',
+                adapter: {
+                    type: spec.adapterType,
+                    [spec.locField]: makeLocation(location),
+                    ...Object.fromEntries(sidecars.map(s => [s.field, makeLocation(s.path)])),
+                },
+                files: [location, ...sidecars.map(s => s.path)],
             };
         }
+        case 'anchors':
+            return {
+                adapter: {
+                    type: spec.adapterType,
+                    [spec.locField]: makeLocation(location),
+                    bed1Location: bed1 ? makeLocation(bed1) : undefined,
+                    bed2Location: bed2 ? makeLocation(bed2) : undefined,
+                },
+                files: [location, bed1, bed2],
+            };
+        case 'unsupported':
+            return { adapter: { type: 'UNSUPPORTED' }, files: [] };
+    }
+}
+export function makeLocationProtocol(protocol) {
+    return (location) => {
+        if (protocol === 'uri') {
+            return { uri: location, locationType: 'UriLocation' };
+        }
+        if (protocol === 'localPath') {
+            return { localPath: location, locationType: 'LocalPathLocation' };
+        }
         throw new Error(`invalid protocol ${protocol}`);
     };
 }
-export function guessFileNames({ location, index, bed1, bed2, }) {
-    if (/\.anchors(\.simple)?$/i.test(location)) {
-        return {
-            file: location,
-            bed1: bed1,
-            bed2: bed2,
-        };
-    }
-    else if (/\.bam$/i.test(location)) {
-        return {
-            file: location,
-            index: index || `${location}.bai`,
-        };
-    }
-    else if (/\.cram$/i.test(location)) {
-        return {
-            file: location,
-            index: index || `${location}.crai`,
-        };
-    }
-    else if (/\.gff3?\.b?gz$/i.test(location) ||
-        /\.vcf\.b?gz$/i.test(location) ||
-        /\.bed\.b?gz$/i.test(location) ||
-        /\.bedmethyl\.gz$/i.test(location) ||
-        /\.pif\.b?gz$/i.test(location)) {
-        return {
-            file: location,
-            index: index || `${location}.tbi`,
-        };
-    }
-    else if (/\.(fa|fasta|fas|fna|mfa)$/i.test(location)) {
-        return {
-            file: location,
-            index: index || `${location}.fai`,
-        };
-    }
-    else if (/\.(fa|fasta|fas|fna|mfa)\.b?gz$/i.test(location)) {
-        return {
-            file: location,
-            index: `${location}.fai`,
-            index2: `${location}.gzi`,
-        };
-    }
-    else if (/\.2bit$/i.test(location) ||
-        /\.bedpe(\.gz)?$/i.test(location) ||
-        /\/trackData.jsonz?$/i.test(location) ||
-        /\/sparql$/i.test(location) ||
-        /\.out(\.gz)?$/i.test(location) ||
-        /\.paf(\.gz)?$/i.test(location) ||
-        /\.delta(\.gz)?$/i.test(location) ||
-        /\.bed$/i.test(location) ||
-        /\.(bw|bigwig)$/i.test(location) ||
-        /\.(bb|bigbed)$/i.test(location) ||
-        /\.vcf$/i.test(location) ||
-        /\.gtf$/i.test(location) ||
-        /\.gff3?$/i.test(location) ||
-        /\.chain(\.gz)?$/i.test(location) ||
-        /\.hic$/i.test(location)) {
-        return {
-            file: location,
-        };
-    }
-    return {};
+/**
+ * The bare filename the format table matches against — the same thing
+ * `@jbrowse/core`'s `getFileName` hands the guesser chain, so a path and a URL
+ * with a presigned query string both reduce to what the regexes expect.
+ */
+export function fileNameOf(location) {
+    return (location.replaceAll('\\', '/').split('/').at(-1)?.split(/[?#]/)[0] ?? '');
 }
-export function guessAdapter({ location, protocol, index, bed1, bed2, }) {
-    const makeLocation = makeLocationProtocol(protocol);
-    if (/\.bam$/i.test(location)) {
-        return {
-            type: 'BamAdapter',
-            bamLocation: makeLocation(location),
-            index: {
-                location: makeLocation(index || `${location}.bai`),
-                indexType: index?.toUpperCase().endsWith('CSI') ? 'CSI' : 'BAI',
-            },
-        };
-    }
-    else if (/\.cram$/i.test(location)) {
-        return {
-            type: 'CramAdapter',
-            cramLocation: makeLocation(location),
-            craiLocation: makeLocation(`${location}.crai`),
-        };
-    }
-    else if (/\.gff3?$/i.test(location)) {
-        return {
-            type: 'Gff3Adapter',
-            gffLocation: makeLocation(location),
-        };
-    }
-    else if (/\.gff3?\.b?gz$/i.test(location)) {
-        return {
-            type: 'Gff3TabixAdapter',
-            gffGzLocation: makeLocation(location),
-            index: {
-                location: makeLocation(index || `${location}.tbi`),
-                indexType: index?.toUpperCase().endsWith('CSI') ? 'CSI' : 'TBI',
-            },
-        };
-    }
-    else if (/\.gtf$/i.test(location)) {
-        return {
-            type: 'GtfAdapter',
-            gtfLocation: makeLocation(location),
-        };
-    }
-    else if (/\.vcf$/i.test(location)) {
-        return {
-            type: 'VcfAdapter',
-            vcfLocation: makeLocation(location),
-        };
-    }
-    else if (/\.vcf\.b?gz$/i.test(location)) {
-        return {
-            type: 'VcfTabixAdapter',
-            vcfGzLocation: makeLocation(location),
-            index: {
-                location: makeLocation(index || `${location}.tbi`),
-                indexType: index?.toUpperCase().endsWith('CSI') ? 'CSI' : 'TBI',
-            },
-        };
-    }
-    else if (/\.vcf\.idx$/i.test(location)) {
-        return {
-            type: 'UNSUPPORTED',
-        };
-    }
-    else if (/\.bedpe(.gz)?$/i.test(location)) {
-        return {
-            type: 'BedpeAdapter',
-            bedpeLocation: makeLocation(location),
-        };
-    }
-    else if (/\.bed$/i.test(location)) {
-        return {
-            type: 'BedAdapter',
-            bedLocation: makeLocation(location),
-        };
-    }
-    else if (/\.pif\.b?gz$/i.test(location)) {
-        return {
-            type: 'PairwiseIndexedPAFAdapter',
-            pifGzLocation: makeLocation(location),
-            index: {
-                location: makeLocation(index || `${location}.tbi`),
-                indexType: index?.toUpperCase().endsWith('CSI') ? 'CSI' : 'TBI',
-            },
-        };
-    }
-    else if (/\.bedmethyl\.gz$/i.test(location) ||
-        /\.bed\.b?gz$/i.test(location)) {
-        return {
-            type: 'BedTabixAdapter',
-            bedGzLocation: makeLocation(location),
-            index: {
-                location: makeLocation(index || `${location}.tbi`),
-                indexType: index?.toUpperCase().endsWith('CSI') ? 'CSI' : 'TBI',
-            },
-        };
-    }
-    else if (/\.(bb|bigbed)$/i.test(location)) {
-        return {
-            type: 'BigBedAdapter',
-            bigBedLocation: makeLocation(location),
-        };
-    }
-    else if (/\.(bw|bigwig)$/i.test(location)) {
-        return {
-            type: 'BigWigAdapter',
-            bigWigLocation: makeLocation(location),
-        };
-    }
-    else if (/\.(fa|fasta|fna|mfa)$/i.test(location)) {
-        return {
-            type: 'IndexedFastaAdapter',
-            fastaLocation: makeLocation(location),
-            faiLocation: makeLocation(index || `${location}.fai`),
-        };
-    }
-    else if (/\.(fa|fasta|fna|mfa)\.b?gz$/i.test(location)) {
-        return {
-            type: 'BgzipFastaAdapter',
-            fastaLocation: makeLocation(location),
-            faiLocation: makeLocation(`${location}.fai`),
-            gziLocation: makeLocation(`${location}.gzi`),
-        };
-    }
-    else if (/\.2bit$/i.test(location)) {
-        return {
-            type: 'TwoBitAdapter',
-            twoBitLocation: makeLocation(location),
-        };
-    }
-    else if (/\.sizes$/i.test(location)) {
-        return {
-            type: 'UNSUPPORTED',
-        };
-    }
-    else if (/\/trackData.jsonz?$/i.test(location)) {
-        return {
-            type: 'NCListAdapter',
-            rootUrlTemplate: makeLocation(location),
-        };
-    }
-    else if (/\/sparql$/i.test(location)) {
-        return {
-            type: 'SPARQLAdapter',
-            endpoint: location,
-        };
-    }
-    else if (/\.hic$/i.test(location)) {
-        return {
-            type: 'HicAdapter',
-            hicLocation: makeLocation(location),
-        };
-    }
-    else if (/\.paf(.gz)?$/i.test(location)) {
-        return {
-            type: 'PAFAdapter',
-            pafLocation: makeLocation(location),
-        };
-    }
-    else if (/\.out(.gz)?$/i.test(location)) {
-        return {
-            type: 'MashMapAdapter',
-            outLocation: makeLocation(location),
-        };
-    }
-    else if (/\.chain(.gz)?$/i.test(location)) {
-        return {
-            type: 'ChainAdapter',
-            chainLocation: makeLocation(location),
-        };
-    }
-    else if (/\.delta(.gz)?$/i.test(location)) {
-        return {
-            type: 'DeltaAdapter',
-            deltaLocation: makeLocation(location),
-        };
-    }
-    else if (/\.anchors(\.gz)?$/i.test(location)) {
-        return {
-            type: 'MCScanAnchorsAdapter',
-            mcscanAnchorsLocation: makeLocation(location),
-            bed1Location: bed1 ? makeLocation(bed1) : undefined,
-            bed2Location: bed2 ? makeLocation(bed2) : undefined,
-        };
+// resolves the file-layout spec that both the adapter config and the copied
+// file set derive from, honoring an explicit --adapterType. `typeOverride` is
+// the label to stamp on the adapter when the extension's layout is reused under
+// a different adapter type.
+function resolveSpec(location, adapterType) {
+    const spec = matchFormat(fileNameOf(location))?.spec;
+    if (adapterType) {
+        const known = adapterTypeToSpec[adapterType];
+        if (known) {
+            // explicit --adapterType resolves back to its known file layout
+            return { spec: known };
+        }
+        else if (spec) {
+            // unknown adapter type on a recognized extension: reuse that extension's
+            // file layout (location field + any sidecar/index files) under the new
+            // type name, so e.g. --adapterType MyPAFAdapter on a .paf keeps its
+            // pafLocation instead of dropping the file. Override with --config if the
+            // custom adapter uses a different field.
+            return { spec, typeOverride: adapterType };
+        }
+        else {
+            // custom adapter type on an unrecognized extension: no layout to reuse
+            return { typeOverride: adapterType };
+        }
     }
-    else if (/\.anchors\.simple(\.gz)?$/i.test(location)) {
-        return {
-            type: 'MCScanSimpleAnchorsAdapter',
-            mcscanSimpleAnchorsLocation: makeLocation(location),
-            bed1Location: bed1 ? makeLocation(bed1) : undefined,
-            bed2Location: bed2 ? makeLocation(bed2) : undefined,
-        };
+    else {
+        return { spec };
     }
-    return {
-        type: 'UNKNOWN',
-    };
 }
-export const adapterTypesToTrackTypeMap = {
-    BamAdapter: 'AlignmentsTrack',
-    CramAdapter: 'AlignmentsTrack',
-    BgzipFastaAdapter: 'ReferenceSequenceTrack',
-    BigWigAdapter: 'QuantitativeTrack',
-    BedGraphAdapter: 'QuantitativeTrack',
-    BedGraphTabixAdapter: 'QuantitativeTrack',
-    IndexedFastaAdapter: 'ReferenceSequenceTrack',
-    TwoBitAdapter: 'ReferenceSequenceTrack',
-    VcfTabixAdapter: 'VariantTrack',
-    VcfAdapter: 'VariantTrack',
-    BedpeAdapter: 'VariantTrack',
-    BedAdapter: 'FeatureTrack',
-    BedTabixAdapter: 'FeatureTrack',
-    HicAdapter: 'HicTrack',
-    PAFAdapter: 'SyntenyTrack',
-    DeltaAdapter: 'SyntenyTrack',
-    ChainAdapter: 'SyntenyTrack',
-    MashMapAdapter: 'SyntenyTrack',
-    PairwiseIndexedPAFAdapter: 'SyntenyTrack',
-    MCScanAnchorsAdapter: 'SyntenyTrack',
-    MCScanSimpleAnchorsAdapter: 'SyntenyTrack',
-};
-export function guessTrackType(adapterType) {
-    return adapterTypesToTrackTypeMap[adapterType] || 'FeatureTrack';
+// derives the track adapter and the raw source files together from one spec, so
+// the config written and the files add-track copies can never drift. mapLocation
+// turns a raw source path into the location the adapter stores (relative path +
+// protocol wrapper); the returned files stay raw for the copy step.
+export function guessTrack({ location, index, bed1, bed2, adapterType, mapLocation, }) {
+    const { spec, typeOverride } = resolveSpec(location, adapterType);
+    if (spec) {
+        const { adapter, files } = buildFromSpec(spec, {
+            location,
+            index,
+            bed1,
+            bed2,
+            makeLocation: mapLocation,
+        });
+        return {
+            adapter: typeOverride ? { ...adapter, type: typeOverride } : adapter,
+            files,
+        };
+    }
+    return { adapter: { type: typeOverride ?? 'UNKNOWN' }, files: [] };
+}
+export function guessTrackType(adapterType, location) {
+    return (trackTypeForAdapter(adapterType, location && fileNameOf(location)) ||
+        'FeatureTrack');
 }
+// the synteny adapters are exactly those mapping to a SyntenyTrack, so derive
+// the set instead of maintaining a second hand-written list that can drift
+export const syntenyAdapterTypes = new Set(Object.entries(adapterTypesToTrackTypeMap)
+    .filter(([, trackType]) => trackType === 'SyntenyTrack')
+    .map(([adapterType]) => adapterType));

package/dist/commands/add-track-utils/file-operations.js

@@ -1,21 +1,23 @@
-import fs from 'fs';
-import path from 'path';
+import fs from 'node:fs';
+import path from 'node:path';
+import { ignoreNotFound } from '../../utils.js';
 const { copyFile, rename, symlink, unlink } = fs.promises;
 const { COPYFILE_EXCL } = fs.constants;
-export async function loadFile({ src, destDir, mode, subDir = '', force = false, }) {
+async function loadFile({ src, destDir, mode, subDir = '', force = false, }) {
     if (mode === 'inPlace') {
         return;
     }
     const dest = path.join(destDir, subDir, path.basename(src));
+    // the file already sits where the config wants it (add-track run from inside
+    // the install dir), so there is nothing to copy/move/link. Bailing before the
+    // force-unlink matters: dest and src are the same file, so unlinking it
+    // deleted the user's data and left the copy with nothing to read. A source
+    // that isn't there still has to fail, so fall through to get its ENOENT.
+    if (path.resolve(src) === path.resolve(dest) && fs.existsSync(src)) {
+        return;
+    }
     if (force) {
-        try {
-            await unlink(dest);
-        }
-        catch (e) {
-            if (e && typeof e === 'object' && 'code' in e && e.code !== 'ENOENT') {
-                throw e;
-            }
-        }
+        await ignoreNotFound(unlink(dest));
     }
     if (mode === 'copy') {
         return copyFile(src, dest, force ? 0 : COPYFILE_EXCL);
@@ -28,3 +30,17 @@ export async function loadFile({ src, destDir, mode, subDir = '', force = false,
     }
     return undefined;
 }
+// load a set of source files into the config directory, skipping the operation
+// entirely when no load mode is given (e.g. a URL track). undefined entries
+// (optional index/bed files) are ignored
+export async function loadFiles({ files, destDir, mode, subDir, force, }) {
+    // inPlace references files where they sit, so it needs no destination dir
+    if (mode && mode !== 'inPlace') {
+        if (subDir) {
+            fs.mkdirSync(path.join(destDir, subDir), { recursive: true });
+        }
+        await Promise.all(files
+            .filter((f) => !!f)
+            .map(src => loadFile({ src, destDir, mode, subDir, force })));
+    }
+}

package/dist/commands/add-track-utils/multiwig.js

@@ -0,0 +1,47 @@
+import fs from 'node:fs';
+import { isURL } from '../../types/common.js';
+// Parse the --multiwig argument into a subadapter list, mirroring @jbrowse/img's
+// --multiwig: either a comma-separated list of BigWig files/URLs, or a `.json`
+// file holding an array of BigWig locations or full subadapter objects (each a
+// BigWigAdapter config carrying its own name/color/group).
+//
+// A string entry becomes an explicit `{ type: 'BigWigAdapter', bigWigLocation }`
+// subadapter — NOT an entry in the adapter's `bigWigs` string-array shorthand.
+// The shorthand can only carry bare URIs and does not resolve a relative path
+// against the config's baseUri, so a relative local file 404s there. Building the
+// location object ourselves runs it through the same mapLocation as every other
+// track file, so `--load copy --subDir bw` yields `{ uri: 'bw/a.bw' }` that
+// resolves relative to config.json exactly like a single-file track.
+//
+// Object entries are full subadapter configs, so they pass through unchanged
+// (keeping their per-row metadata) and own their own locations — use URLs or
+// config-relative uris there, since --load only copies the string entries.
+export function buildMultiWiggle({ sources, mapLocation, }) {
+    const files = [];
+    const subadapters = parseSources(sources).map(entry => {
+        if (typeof entry === 'string') {
+            if (!isURL(entry)) {
+                files.push(entry);
+            }
+            return { type: 'BigWigAdapter', bigWigLocation: mapLocation(entry) };
+        }
+        return entry;
+    });
+    return {
+        adapter: { type: 'MultiWiggleAdapter', subadapters },
+        files,
+    };
+}
+function parseSources(sources) {
+    if (sources.toLowerCase().endsWith('.json')) {
+        const data = JSON.parse(fs.readFileSync(sources, 'utf8'));
+        if (!Array.isArray(data)) {
+            throw new Error(`${sources}: expected a JSON array of BigWig URLs or subadapter objects`);
+        }
+        return data;
+    }
+    return sources
+        .split(',')
+        .map(s => s.trim())
+        .filter(Boolean);
+}