@jbrowse/cli 4.3.0 → 5.0.0-beta.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +451 -72
- package/dist/base.js +0 -3
- package/dist/bin.js +2 -1
- package/dist/cliFetch.js +3 -0
- package/dist/commands/add-assembly/index.js +46 -19
- package/dist/commands/add-assembly/utils.js +204 -290
- package/dist/commands/add-connection.js +73 -79
- package/dist/commands/add-track-json.js +24 -12
- package/dist/commands/add-track-utils/adapter-utils.js +189 -288
- package/dist/commands/add-track-utils/file-operations.js +27 -11
- package/dist/commands/add-track-utils/multiwig.js +47 -0
- package/dist/commands/add-track-utils/track-config.js +66 -25
- package/dist/commands/add-track-utils/validators.js +36 -20
- package/dist/commands/add-track.js +200 -62
- package/dist/commands/admin-server/index.js +39 -50
- package/dist/commands/admin-server/utils.js +238 -126
- package/dist/commands/create.js +18 -34
- package/dist/commands/make-density/density-generator.js +180 -0
- package/dist/commands/make-density/index.js +139 -0
- package/dist/commands/make-pif/index.js +133 -11
- package/dist/commands/make-pif/pif-generator.js +243 -43
- package/dist/commands/process-utils.js +13 -2
- package/dist/commands/remove-track.js +20 -13
- package/dist/commands/set-default-session.js +35 -31
- package/dist/commands/shared/config-operations.js +3 -10
- package/dist/commands/shared/density.js +12 -0
- package/dist/commands/shared/sort-utils.js +67 -10
- package/dist/commands/shared/validators.js +38 -3
- package/dist/commands/sort-bed.js +2 -33
- package/dist/commands/sort-gff.js +2 -33
- package/dist/commands/text-index/adapter-utils.js +19 -26
- package/dist/commands/text-index/aggregate.js +40 -34
- package/dist/commands/text-index/command.js +37 -7
- package/dist/commands/text-index/config-utils.js +62 -85
- package/dist/commands/text-index/file-list.js +32 -18
- package/dist/commands/text-index/index.js +4 -4
- package/dist/commands/text-index/indexing-utils.js +44 -96
- package/dist/commands/text-index/per-track.js +44 -31
- package/dist/commands/text-index/validators.js +1 -1
- package/dist/commands/upgrade.js +20 -32
- package/dist/commands/validate/configManifest.generated.js +7613 -0
- package/dist/commands/validate/configSchema.generated.js +20400 -0
- package/dist/commands/validate/displayDefaultKeys.js +28 -0
- package/dist/commands/validate/index.js +113 -0
- package/dist/commands/validate/liftConfig.js +34 -0
- package/dist/commands/validate/markRules/aggregateFieldName.js +9 -0
- package/dist/commands/validate/markRules/autoBin.js +27 -0
- package/dist/commands/validate/markRules/colorScale.js +244 -0
- package/dist/commands/validate/markRules/cssColorsLevel4.js +162 -0
- package/dist/commands/validate/markRules/jexlStrings.js +23 -0
- package/dist/commands/validate/markRules/markProblems.js +532 -0
- package/dist/commands/validate/markRules/markRuleFacts.js +6 -0
- package/dist/commands/validate/markRules/markSpecs.js +96 -0
- package/dist/commands/validate/markRules/markVocabulary.js +57 -0
- package/dist/commands/validate/markRules/shapeNameList.js +7 -0
- package/dist/commands/validate/markRules/stepChannels.js +55 -0
- package/dist/commands/validate/schemaValidate.js +425 -0
- package/dist/commands/validate/suggest.js +35 -0
- package/dist/commands/validate/types.js +6 -0
- package/dist/commands/validate/validateConfig.js +502 -0
- package/dist/index.js +104 -50
- package/dist/types/common.js +3 -27
- package/dist/utils.js +309 -128
- package/dist/version.js +1 -1
- package/package.json +8 -14
- package/bin/run +0 -4
- package/bundle/index.js +0 -3686
- package/dist/commands/make-pif/cigar-utils.js +0 -24
- package/dist/commands/make-pif/file-utils.js +0 -30
- package/dist/commands/track-utils.js +0 -74
- package/dist/util.js +0 -35
|
@@ -1,14 +1,49 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { execFileSync } from 'node:child_process';
|
|
2
|
+
import { accessSync, constants, statSync } from 'node:fs';
|
|
2
3
|
export function validateFileArgument(file, commandName, fileType) {
|
|
3
|
-
if (
|
|
4
|
+
if (file) {
|
|
5
|
+
// The sort pipeline's exit status is `sort`'s, which succeeds on empty
|
|
6
|
+
// input, so a missing/unreadable file would otherwise "succeed" with empty
|
|
7
|
+
// output. Fail loudly here instead.
|
|
8
|
+
if (!statSync(file, { throwIfNoEntry: false })?.isFile()) {
|
|
9
|
+
throw new Error(`Input file does not exist or is not a regular file: ${file}`);
|
|
10
|
+
}
|
|
11
|
+
try {
|
|
12
|
+
accessSync(file, constants.R_OK);
|
|
13
|
+
}
|
|
14
|
+
catch {
|
|
15
|
+
throw new Error(`Input file is not readable: ${file}`);
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
else if (process.stdin.isTTY) {
|
|
4
19
|
throw new Error(`Missing required argument: file\n` +
|
|
5
20
|
`Usage: jbrowse ${commandName} <file>\n` +
|
|
6
21
|
` OR pipe data via stdin: cat file.${fileType} | jbrowse ${commandName}`);
|
|
7
22
|
}
|
|
8
23
|
}
|
|
24
|
+
// Replaces command-exists: is `command` runnable? On POSIX, `command -v` (a
|
|
25
|
+
// shell builtin, so invoked via `sh`) resolves builtins, PATH entries, and
|
|
26
|
+
// absolute paths; on Windows, `where` does the same. The name is passed as an
|
|
27
|
+
// argv item, never interpolated into the shell, so it can't be injected.
|
|
28
|
+
function commandExistsSync(command) {
|
|
29
|
+
try {
|
|
30
|
+
if (process.platform === 'win32') {
|
|
31
|
+
execFileSync('where', [command], { stdio: 'ignore' });
|
|
32
|
+
}
|
|
33
|
+
else {
|
|
34
|
+
execFileSync('sh', ['-c', 'command -v "$1"', 'sh', command], {
|
|
35
|
+
stdio: 'ignore',
|
|
36
|
+
});
|
|
37
|
+
}
|
|
38
|
+
return true;
|
|
39
|
+
}
|
|
40
|
+
catch {
|
|
41
|
+
return false;
|
|
42
|
+
}
|
|
43
|
+
}
|
|
9
44
|
export function validateRequiredCommands(requiredCommands) {
|
|
10
45
|
const missingCommands = requiredCommands.filter(cmd => !commandExistsSync(cmd));
|
|
11
46
|
if (missingCommands.length > 0) {
|
|
12
|
-
throw new Error(`Unable to run command, requires unix type environment with: ${requiredCommands.join(', ')}`);
|
|
47
|
+
throw new Error(`Unable to run command, requires a unix type environment with the following available: ${requiredCommands.join(', ')}. Missing: ${missingCommands.join(', ')}`);
|
|
13
48
|
}
|
|
14
49
|
}
|
|
@@ -1,35 +1,4 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { printHelp } from "../utils.js";
|
|
3
|
-
import { waitForProcessClose } from "./process-utils.js";
|
|
4
|
-
import { BED_CONFIG, spawnSortProcess } from "./shared/sort-utils.js";
|
|
5
|
-
import { validateFileArgument, validateRequiredCommands, } from "./shared/validators.js";
|
|
1
|
+
import { BED_CONFIG, runSort } from './shared/sort-utils.js';
|
|
6
2
|
export async function run(args) {
|
|
7
|
-
|
|
8
|
-
help: {
|
|
9
|
-
type: 'boolean',
|
|
10
|
-
short: 'h',
|
|
11
|
-
},
|
|
12
|
-
};
|
|
13
|
-
const { values: flags, positionals } = parseArgs({
|
|
14
|
-
args,
|
|
15
|
-
options,
|
|
16
|
-
allowPositionals: true,
|
|
17
|
-
});
|
|
18
|
-
if (flags.help) {
|
|
19
|
-
printHelp({
|
|
20
|
-
description: BED_CONFIG.description,
|
|
21
|
-
examples: BED_CONFIG.examples,
|
|
22
|
-
usage: 'jbrowse sort-bed [file] [options]',
|
|
23
|
-
options,
|
|
24
|
-
});
|
|
25
|
-
return;
|
|
26
|
-
}
|
|
27
|
-
const file = positionals[0];
|
|
28
|
-
validateFileArgument(file, 'sort-bed', 'bed');
|
|
29
|
-
validateRequiredCommands(['sh', 'sort', 'grep']);
|
|
30
|
-
const child = spawnSortProcess(file, BED_CONFIG.sortColumn);
|
|
31
|
-
const exitCode = await waitForProcessClose(child);
|
|
32
|
-
if (exitCode !== 0) {
|
|
33
|
-
throw new Error(`Sort process exited with code ${exitCode}`);
|
|
34
|
-
}
|
|
3
|
+
return runSort(BED_CONFIG, 'sort-bed', args);
|
|
35
4
|
}
|
|
@@ -1,35 +1,4 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { printHelp } from "../utils.js";
|
|
3
|
-
import { waitForProcessClose } from "./process-utils.js";
|
|
4
|
-
import { GFF_CONFIG, spawnSortProcess } from "./shared/sort-utils.js";
|
|
5
|
-
import { validateFileArgument, validateRequiredCommands, } from "./shared/validators.js";
|
|
1
|
+
import { GFF_CONFIG, runSort } from './shared/sort-utils.js';
|
|
6
2
|
export async function run(args) {
|
|
7
|
-
|
|
8
|
-
help: {
|
|
9
|
-
type: 'boolean',
|
|
10
|
-
short: 'h',
|
|
11
|
-
},
|
|
12
|
-
};
|
|
13
|
-
const { values: flags, positionals } = parseArgs({
|
|
14
|
-
args,
|
|
15
|
-
options,
|
|
16
|
-
allowPositionals: true,
|
|
17
|
-
});
|
|
18
|
-
if (flags.help) {
|
|
19
|
-
printHelp({
|
|
20
|
-
description: GFF_CONFIG.description,
|
|
21
|
-
examples: GFF_CONFIG.examples,
|
|
22
|
-
usage: 'jbrowse sort-gff [file] [options]',
|
|
23
|
-
options,
|
|
24
|
-
});
|
|
25
|
-
return;
|
|
26
|
-
}
|
|
27
|
-
const file = positionals[0];
|
|
28
|
-
validateFileArgument(file, 'sort-gff', 'gff');
|
|
29
|
-
validateRequiredCommands(['sh', 'sort', 'grep']);
|
|
30
|
-
const child = spawnSortProcess(file, GFF_CONFIG.sortColumn);
|
|
31
|
-
const exitCode = await waitForProcessClose(child);
|
|
32
|
-
if (exitCode !== 0) {
|
|
33
|
-
throw new Error(`Sort process exited with code ${exitCode}`);
|
|
34
|
-
}
|
|
3
|
+
return runSort(GFF_CONFIG, 'sort-gff', args);
|
|
35
4
|
}
|
|
@@ -1,38 +1,31 @@
|
|
|
1
|
-
import {
|
|
2
|
-
export function
|
|
3
|
-
|
|
4
|
-
}
|
|
5
|
-
export function createTrixAdapter(name, assemblyNames, idSuffix = 'index') {
|
|
6
|
-
const safeName = sanitizeNameForPath(name);
|
|
1
|
+
import { trixFileUris } from '@jbrowse/text-indexing-core';
|
|
2
|
+
export function createTrixAdapter(name, assemblyNames) {
|
|
3
|
+
const uris = trixFileUris(name);
|
|
7
4
|
return {
|
|
8
5
|
type: 'TrixTextSearchAdapter',
|
|
9
|
-
textSearchAdapterId: `${safeName}-${idSuffix}`,
|
|
10
6
|
ixFilePath: {
|
|
11
|
-
uri:
|
|
7
|
+
uri: uris.ix,
|
|
12
8
|
locationType: 'UriLocation',
|
|
13
9
|
},
|
|
14
10
|
ixxFilePath: {
|
|
15
|
-
uri:
|
|
16
|
-
locationType: 'UriLocation',
|
|
17
|
-
},
|
|
18
|
-
metaFilePath: {
|
|
19
|
-
uri: `trix/${safeName}_meta.json`,
|
|
11
|
+
uri: uris.ixx,
|
|
20
12
|
locationType: 'UriLocation',
|
|
21
13
|
},
|
|
22
14
|
assemblyNames,
|
|
23
15
|
};
|
|
24
16
|
}
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
17
|
+
// an aggregate entry is the index its files are, so a re-index of the same
|
|
18
|
+
// assembly finds the entry pointing at them however it was written, `.ix`
|
|
19
|
+
// string and `{ uri }` included; an entry with no trix file, a JBrowse 1
|
|
20
|
+
// index, is only ever itself
|
|
21
|
+
export function indexFileOf(entry) {
|
|
22
|
+
if (typeof entry === 'string') {
|
|
23
|
+
return entry;
|
|
24
|
+
}
|
|
25
|
+
const { ixFilePath, uri } = entry;
|
|
26
|
+
return ixFilePath
|
|
27
|
+
? 'uri' in ixFilePath
|
|
28
|
+
? ixFilePath.uri
|
|
29
|
+
: ixFilePath.localPath
|
|
30
|
+
: (uri ?? entry);
|
|
38
31
|
}
|
|
@@ -1,13 +1,15 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
1
|
+
import { parseCommaSeparatedString, writeJsonFile } from '../../utils.js';
|
|
2
|
+
import { createTrixAdapter, indexFileOf } from './adapter-utils.js';
|
|
3
|
+
import { formatDryRun, getAssemblyNames, getTrackConfigs, loadConfigForIndexing, prepareIndexDriverFlags, } from './config-utils.js';
|
|
4
|
+
import { indexDriver } from './indexing-utils.js';
|
|
5
5
|
export async function aggregateIndex(flags) {
|
|
6
|
-
const { out, target, tracks, excludeTracks, assemblies, attributes, quiet, force, exclude, dryrun, prefixSize, } = flags;
|
|
7
|
-
const { config, configPath, outLocation } = await loadConfigForIndexing(target, out
|
|
8
|
-
const
|
|
9
|
-
|
|
10
|
-
|
|
6
|
+
const { out, target, tracks, excludeTracks, assemblies, attributes, quiet, force, exclude, include, dryrun, prefixSize, } = flags;
|
|
7
|
+
const { config, configPath, outLocation } = await loadConfigForIndexing(target, out);
|
|
8
|
+
const existing = config.aggregateTextSearchAdapters ?? [];
|
|
9
|
+
// sequential, not Promise.all: each assembly's indexer owns the progress bar
|
|
10
|
+
// and streams whole files through ixIxx
|
|
11
|
+
const written = [];
|
|
12
|
+
for (const asm of getAssemblyNames(config, assemblies)) {
|
|
11
13
|
const trackConfigs = getTrackConfigs(config, parseCommaSeparatedString(tracks), asm, parseCommaSeparatedString(excludeTracks));
|
|
12
14
|
if (!trackConfigs.length) {
|
|
13
15
|
console.log(`Indexing assembly ${asm}...(no tracks found)...`);
|
|
@@ -15,35 +17,39 @@ export async function aggregateIndex(flags) {
|
|
|
15
17
|
}
|
|
16
18
|
console.log(`Indexing assembly ${asm}...`);
|
|
17
19
|
if (dryrun) {
|
|
18
|
-
console.log(trackConfigs
|
|
20
|
+
console.log(formatDryRun(trackConfigs));
|
|
21
|
+
continue;
|
|
19
22
|
}
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
continue;
|
|
26
|
-
}
|
|
27
|
-
await indexDriver({
|
|
28
|
-
trackConfigs,
|
|
29
|
-
outLocation,
|
|
30
|
-
name: asm,
|
|
31
|
-
assemblyNames: [asm],
|
|
32
|
-
...prepareIndexDriverFlags({ attributes, exclude, quiet, prefixSize }),
|
|
33
|
-
});
|
|
34
|
-
const trixConf = createTrixAdapter(asm, [asm]);
|
|
35
|
-
if (idx === -1) {
|
|
36
|
-
aggregateTextSearchAdapters.push(trixConf);
|
|
37
|
-
}
|
|
38
|
-
else {
|
|
39
|
-
aggregateTextSearchAdapters[idx] = trixConf;
|
|
40
|
-
}
|
|
23
|
+
const trixConf = createTrixAdapter(asm, [asm]);
|
|
24
|
+
if (!force &&
|
|
25
|
+
existing.some(x => indexFileOf(x) === indexFileOf(trixConf))) {
|
|
26
|
+
console.log(`Note: ${asm} has already been indexed with this configuration, use --force to overwrite this assembly. Skipping for now`);
|
|
27
|
+
continue;
|
|
41
28
|
}
|
|
29
|
+
await indexDriver({
|
|
30
|
+
trackConfigs,
|
|
31
|
+
outLocation,
|
|
32
|
+
name: asm,
|
|
33
|
+
assemblyNames: [asm],
|
|
34
|
+
...prepareIndexDriverFlags({
|
|
35
|
+
attributes,
|
|
36
|
+
exclude,
|
|
37
|
+
include,
|
|
38
|
+
quiet,
|
|
39
|
+
prefixSize,
|
|
40
|
+
}),
|
|
41
|
+
});
|
|
42
|
+
written.push(trixConf);
|
|
42
43
|
}
|
|
43
44
|
if (!dryrun) {
|
|
44
|
-
|
|
45
|
+
await writeJsonFile(configPath, {
|
|
45
46
|
...config,
|
|
46
|
-
|
|
47
|
-
|
|
47
|
+
// upsert by index file: a Map keyed on it keeps the first occurrence's
|
|
48
|
+
// position while the last value wins, so re-indexing an assembly replaces
|
|
49
|
+
// its entry in place and a new one lands at the end
|
|
50
|
+
aggregateTextSearchAdapters: [
|
|
51
|
+
...new Map([...existing, ...written].map(a => [indexFileOf(a), a])).values(),
|
|
52
|
+
],
|
|
53
|
+
});
|
|
48
54
|
}
|
|
49
55
|
}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
|
-
import { parseArgs } from 'util';
|
|
2
|
-
import {
|
|
3
|
-
import { printHelp } from
|
|
1
|
+
import { parseArgs } from 'node:util';
|
|
2
|
+
import { defaultAttributesToIndex, indexableAdapters, } from '@jbrowse/text-indexing-core';
|
|
3
|
+
import { printHelp } from '../../utils.js';
|
|
4
|
+
import { aggregateIndex, indexFileList, perTrackIndex } from './index.js';
|
|
4
5
|
export async function run(args) {
|
|
5
6
|
const options = {
|
|
6
7
|
help: {
|
|
@@ -14,7 +15,7 @@ export async function run(args) {
|
|
|
14
15
|
},
|
|
15
16
|
excludeTracks: {
|
|
16
17
|
type: 'string',
|
|
17
|
-
description: 'Specific tracks to exclude from indexing, formatted as comma separated trackIds',
|
|
18
|
+
description: 'Specific tracks to exclude from indexing, formatted as comma separated trackIds. To exclude a track permanently, set metadata.skipTextIndex on it in config.json instead (see Notes)',
|
|
18
19
|
},
|
|
19
20
|
target: {
|
|
20
21
|
type: 'string',
|
|
@@ -24,7 +25,7 @@ export async function run(args) {
|
|
|
24
25
|
attributes: {
|
|
25
26
|
type: 'string',
|
|
26
27
|
description: 'Comma separated list of attributes to index',
|
|
27
|
-
default: '
|
|
28
|
+
default: defaultAttributesToIndex.join(','),
|
|
28
29
|
},
|
|
29
30
|
assemblies: {
|
|
30
31
|
type: 'string',
|
|
@@ -49,9 +50,13 @@ export async function run(args) {
|
|
|
49
50
|
},
|
|
50
51
|
exclude: {
|
|
51
52
|
type: 'string',
|
|
52
|
-
description: '
|
|
53
|
+
description: 'Comma separated list of feature types to exclude from indexing',
|
|
53
54
|
default: 'CDS,exon',
|
|
54
55
|
},
|
|
56
|
+
include: {
|
|
57
|
+
type: 'string',
|
|
58
|
+
description: 'Comma separated list of feature types to index, dropping every other type. Unset by default, which indexes every type --exclude does not name. GFF3 only',
|
|
59
|
+
},
|
|
55
60
|
prefixSize: {
|
|
56
61
|
type: 'string',
|
|
57
62
|
description: 'Specify the prefix size for the ixx index. We attempt to automatically calculate this, but you can manually specify this too. If many genes have similar gene IDs e.g. Z000000001, Z000000002 the prefix size should be larger so that they get split into different bins',
|
|
@@ -76,6 +81,30 @@ export async function run(args) {
|
|
|
76
81
|
options,
|
|
77
82
|
});
|
|
78
83
|
const description = 'Make a text-indexing file for any given track(s).';
|
|
84
|
+
const notes = 'Individual tracks in config.json can be permanently excluded from ' +
|
|
85
|
+
'indexing by setting "metadata": { "skipTextIndex": true } on the track. ' +
|
|
86
|
+
'Such tracks are skipped even when indexing all tracks or a whole ' +
|
|
87
|
+
'assembly, so you do not have to pass --excludeTracks on every run.\n\n' +
|
|
88
|
+
// read off the table rather than restated, because the hand-written version
|
|
89
|
+
// of this sentence outlived GtfTabixAdapter's absence from it
|
|
90
|
+
`Only tracks with an indexable adapter type (${Object.keys(indexableAdapters).sort().join(', ')}) are indexed; ` +
|
|
91
|
+
'tracks with other adapter types are skipped automatically.\n\n' +
|
|
92
|
+
'GTF has no Name/ID attributes, so the default --attributes also match ' +
|
|
93
|
+
'their GTF spellings (gene_name, transcript_name, gene_id, transcript_id). ' +
|
|
94
|
+
'A GFF3 row with no Name, as in GENCODE, is named by its own gene_name or ' +
|
|
95
|
+
'transcript_name.' +
|
|
96
|
+
'\n\n' +
|
|
97
|
+
'--exclude names types not to index; --include names the only types to ' +
|
|
98
|
+
'index. Reach for --include when the file draws from a vocabulary you do ' +
|
|
99
|
+
'not control: an NCBI RefSeq GFF3 uses 115 feature types, 80 of them leaf ' +
|
|
100
|
+
'records with no name to search (a match is labelled with a bare UUID, a ' +
|
|
101
|
+
'cDNA_match with an MD5, every biological_region with the string ' +
|
|
102
|
+
'"biological region"), so a deny list leaks whichever type is added next ' +
|
|
103
|
+
'while the allow list — gene, pseudogene and the transcript types — does ' +
|
|
104
|
+
'not grow. Both may be given: --include admits, --exclude then narrows. ' +
|
|
105
|
+
'Either can also be set per track in config.json as ' +
|
|
106
|
+
'textSearching.indexingFeatureTypesToInclude / ' +
|
|
107
|
+
'indexingFeatureTypesToExclude, which takes precedence over the flag.';
|
|
79
108
|
const examples = [
|
|
80
109
|
"# indexes all tracks that it can find in the current directory's config.json",
|
|
81
110
|
'$ jbrowse text-index',
|
|
@@ -84,7 +113,7 @@ export async function run(args) {
|
|
|
84
113
|
'$ jbrowse text-index --tracks=track1,track2,track3',
|
|
85
114
|
'',
|
|
86
115
|
'# indexes all tracks except specific trackIds',
|
|
87
|
-
'$ jbrowse text-index --
|
|
116
|
+
'$ jbrowse text-index --excludeTracks=track1,track2,track3',
|
|
88
117
|
'',
|
|
89
118
|
"# indexes all tracks in a directory's config.json or in a specific config file",
|
|
90
119
|
'$ jbrowse text-index --out /path/to/jb2/',
|
|
@@ -99,6 +128,7 @@ export async function run(args) {
|
|
|
99
128
|
printHelp({
|
|
100
129
|
description,
|
|
101
130
|
examples,
|
|
131
|
+
notes,
|
|
102
132
|
usage: 'jbrowse text-index [options]',
|
|
103
133
|
options,
|
|
104
134
|
});
|
|
@@ -1,114 +1,91 @@
|
|
|
1
|
-
import
|
|
2
|
-
import
|
|
3
|
-
import {
|
|
4
|
-
|
|
5
|
-
* Parses a comma-separated string into an array of trimmed, non-empty strings
|
|
6
|
-
*/
|
|
7
|
-
export function parseCommaSeparatedString(value) {
|
|
8
|
-
return (value
|
|
9
|
-
?.split(',')
|
|
10
|
-
.map(s => s.trim())
|
|
11
|
-
.filter(Boolean) ?? []);
|
|
12
|
-
}
|
|
13
|
-
/**
|
|
14
|
-
* Validates and parses a prefix size value
|
|
15
|
-
*/
|
|
16
|
-
export function validatePrefixSize(value) {
|
|
1
|
+
import path from 'node:path';
|
|
2
|
+
import { supported } from '../../types/common.js';
|
|
3
|
+
import { parseCommaSeparatedString, readConfigFile, resolveConfigPath, } from '../../utils.js';
|
|
4
|
+
function validatePrefixSize(value) {
|
|
17
5
|
if (!value) {
|
|
18
6
|
return undefined;
|
|
19
7
|
}
|
|
20
8
|
const parsed = typeof value === 'number' ? value : parseInt(value, 10);
|
|
21
|
-
if (isNaN(parsed) || parsed
|
|
9
|
+
if (Number.isNaN(parsed) || parsed <= 0) {
|
|
22
10
|
throw new Error(`Invalid prefixSize: "${value}". Must be a positive number.`);
|
|
23
11
|
}
|
|
24
12
|
return parsed;
|
|
25
13
|
}
|
|
26
|
-
/**
|
|
27
|
-
* Prepares common indexDriver parameters from raw flag values
|
|
28
|
-
*/
|
|
29
14
|
export function prepareIndexDriverFlags(flags) {
|
|
30
15
|
return {
|
|
31
|
-
|
|
32
|
-
|
|
16
|
+
policy: {
|
|
17
|
+
attributes: parseCommaSeparatedString(flags.attributes),
|
|
18
|
+
exclude: parseCommaSeparatedString(flags.exclude),
|
|
19
|
+
include: flags.include
|
|
20
|
+
? parseCommaSeparatedString(flags.include)
|
|
21
|
+
: undefined,
|
|
22
|
+
},
|
|
33
23
|
quiet: flags.quiet ?? false,
|
|
34
24
|
prefixSize: validatePrefixSize(flags.prefixSize),
|
|
35
25
|
};
|
|
36
26
|
}
|
|
37
|
-
export function
|
|
38
|
-
return JSON.parse(fs.readFileSync(configPath, 'utf8'));
|
|
39
|
-
}
|
|
40
|
-
export function writeConf(obj, configPath) {
|
|
41
|
-
fs.writeFileSync(configPath, JSON.stringify(obj, null, 2));
|
|
42
|
-
}
|
|
43
|
-
/**
|
|
44
|
-
* Loads config and prepares output location for indexing
|
|
45
|
-
*/
|
|
46
|
-
export async function loadConfigForIndexing(target, out, resolveConfigPath) {
|
|
27
|
+
export async function loadConfigForIndexing(target, out) {
|
|
47
28
|
const configPath = await resolveConfigPath(target, out);
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
return { config, configPath, outLocation };
|
|
52
|
-
}
|
|
53
|
-
export function ensureTrixDir(outLocation) {
|
|
54
|
-
const trixDir = path.join(outLocation, 'trix');
|
|
55
|
-
if (!fs.existsSync(trixDir)) {
|
|
56
|
-
fs.mkdirSync(trixDir, { recursive: true });
|
|
57
|
-
}
|
|
58
|
-
return trixDir;
|
|
59
|
-
}
|
|
60
|
-
/**
|
|
61
|
-
* Extracts all assembly names from a config object
|
|
62
|
-
* Handles both single assembly (config.assembly) and multiple assemblies (config.assemblies)
|
|
63
|
-
*/
|
|
64
|
-
function extractAssemblyNamesFromConfig(config) {
|
|
65
|
-
if (config.assemblies) {
|
|
66
|
-
return config.assemblies.map(a => a.name);
|
|
67
|
-
}
|
|
68
|
-
if (config.assembly) {
|
|
69
|
-
return [config.assembly.name];
|
|
70
|
-
}
|
|
71
|
-
return [];
|
|
29
|
+
// readConfigFile, not a bare readFileSync: run from the wrong directory, this
|
|
30
|
+
// is the error the user sees, and it should name the config.json it looked for
|
|
31
|
+
const config = await readConfigFile(configPath);
|
|
32
|
+
return { config, configPath, outLocation: path.dirname(configPath) };
|
|
72
33
|
}
|
|
73
34
|
export function getAssemblyNames(config, assemblies) {
|
|
74
35
|
const asms = assemblies
|
|
75
36
|
? parseCommaSeparatedString(assemblies)
|
|
76
|
-
:
|
|
37
|
+
: (config.assemblies?.map(a => a.name) ??
|
|
38
|
+
(config.assembly ? [config.assembly.name] : []));
|
|
77
39
|
if (!asms.length) {
|
|
78
40
|
throw new Error('No assemblies found');
|
|
79
41
|
}
|
|
80
42
|
return asms;
|
|
81
43
|
}
|
|
82
|
-
export function
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
44
|
+
export function formatDryRun(trackConfigs) {
|
|
45
|
+
return trackConfigs.map(t => `${t.trackId}\t${t.adapter?.type}`).join('\n');
|
|
46
|
+
}
|
|
47
|
+
// why a track was left out, or undefined if it is indexable
|
|
48
|
+
function skipReason(track, assemblyName, excludeSet) {
|
|
49
|
+
if (assemblyName && !track.assemblyNames.includes(assemblyName)) {
|
|
50
|
+
return `not in assembly '${assemblyName}'`;
|
|
86
51
|
}
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
return
|
|
98
|
-
}
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
52
|
+
else if (excludeSet.has(track.trackId)) {
|
|
53
|
+
return 'excluded with --excludeTracks';
|
|
54
|
+
}
|
|
55
|
+
else if (track.metadata?.skipTextIndex) {
|
|
56
|
+
return 'metadata.skipTextIndex is set';
|
|
57
|
+
}
|
|
58
|
+
else if (!supported(track.adapter?.type)) {
|
|
59
|
+
return `adapter type ${track.adapter?.type} is not indexable`;
|
|
60
|
+
}
|
|
61
|
+
else {
|
|
62
|
+
return undefined;
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
export function getTrackConfigs(config, trackIds, assemblyName, excludeTrackIds) {
|
|
66
|
+
const tracks = config.tracks ?? [];
|
|
67
|
+
// when specific trackIds are requested every one must exist; otherwise
|
|
68
|
+
// consider all tracks in the config
|
|
69
|
+
const requested = trackIds?.length
|
|
70
|
+
? trackIds.map(trackId => {
|
|
71
|
+
const track = tracks.find(t => t.trackId === trackId);
|
|
72
|
+
if (track) {
|
|
73
|
+
return track;
|
|
74
|
+
}
|
|
75
|
+
else {
|
|
76
|
+
throw new Error(`Track not found in config.json for trackId ${trackId}, please add track configuration before indexing.`);
|
|
77
|
+
}
|
|
78
|
+
})
|
|
79
|
+
: tracks;
|
|
80
|
+
const excludeSet = new Set(excludeTrackIds);
|
|
81
|
+
return requested.filter(track => {
|
|
82
|
+
const reason = skipReason(track, assemblyName, excludeSet);
|
|
83
|
+
// only narrate skips for tracks the user named: sweeping a multi-assembly
|
|
84
|
+
// config otherwise prints a line per track per assembly
|
|
85
|
+
if (reason && trackIds?.length) {
|
|
86
|
+
console.log(`Skipping ${track.trackId}: ${reason}`);
|
|
110
87
|
}
|
|
111
|
-
return
|
|
88
|
+
return !reason;
|
|
112
89
|
});
|
|
113
90
|
}
|
|
114
91
|
export { sanitizeForFilename as sanitizeNameForPath } from '@jbrowse/text-indexing-core';
|
|
@@ -1,22 +1,36 @@
|
|
|
1
|
-
import path from 'path';
|
|
2
|
-
import {
|
|
3
|
-
import { indexDriver, prepareFileTrackConfigs } from
|
|
4
|
-
import { validateFileInput } from
|
|
1
|
+
import path from 'node:path';
|
|
2
|
+
import { formatDryRun, prepareIndexDriverFlags, sanitizeNameForPath, } from './config-utils.js';
|
|
3
|
+
import { indexDriver, prepareFileTrackConfigs } from './indexing-utils.js';
|
|
4
|
+
import { validateFileInput } from './validators.js';
|
|
5
5
|
export async function indexFileList(flags) {
|
|
6
|
-
const { out, target, fileId, file, attributes, quiet, exclude, prefixSize } = flags;
|
|
6
|
+
const { out, target, fileId, file, attributes, quiet, exclude, include, prefixSize, dryrun, } = flags;
|
|
7
7
|
validateFileInput(file);
|
|
8
|
-
|
|
9
|
-
|
|
8
|
+
// --out/--target names either the install directory or its config.json, as
|
|
9
|
+
// every other command accepts; the trix output goes beside the config, and
|
|
10
|
+
// pointing at the config.json used to mkdir config.json/trix and fail ENOTDIR
|
|
11
|
+
const outArg = target || out || '.';
|
|
12
|
+
const outLocation = outArg.endsWith('.json') ? path.dirname(outArg) : outArg;
|
|
10
13
|
const trackConfigs = prepareFileTrackConfigs(file, fileId);
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
trackConfigs
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
14
|
+
if (dryrun) {
|
|
15
|
+
console.log(formatDryRun(trackConfigs));
|
|
16
|
+
}
|
|
17
|
+
else {
|
|
18
|
+
const name = trackConfigs.length > 1
|
|
19
|
+
? 'aggregate'
|
|
20
|
+
: sanitizeNameForPath(path.basename(file[0]));
|
|
21
|
+
await indexDriver({
|
|
22
|
+
trackConfigs,
|
|
23
|
+
outLocation,
|
|
24
|
+
name,
|
|
25
|
+
assemblyNames: [],
|
|
26
|
+
...prepareIndexDriverFlags({
|
|
27
|
+
attributes,
|
|
28
|
+
exclude,
|
|
29
|
+
include,
|
|
30
|
+
quiet,
|
|
31
|
+
prefixSize,
|
|
32
|
+
}),
|
|
33
|
+
});
|
|
34
|
+
console.log('Successfully created index for these files. See https://jbrowse.org/storybook/lgv/with-aggregate-text-searching/ for info about usage');
|
|
35
|
+
}
|
|
22
36
|
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
export { aggregateIndex } from
|
|
2
|
-
export { perTrackIndex } from
|
|
3
|
-
export { indexFileList } from
|
|
4
|
-
export { run } from
|
|
1
|
+
export { aggregateIndex } from './aggregate.js';
|
|
2
|
+
export { perTrackIndex } from './per-track.js';
|
|
3
|
+
export { indexFileList } from './file-list.js';
|
|
4
|
+
export { run } from './command.js';
|