@jbrowse/cli 4.3.0 → 5.0.0-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +331 -57
- package/bin/run +1 -1
- package/bundle/index.js +457 -2574
- package/dist/base.js +0 -3
- package/dist/bin.js +2 -1
- package/dist/cliFetch.js +3 -0
- package/dist/commands/add-assembly/index.js +46 -19
- package/dist/commands/add-assembly/utils.js +204 -290
- package/dist/commands/add-connection.js +73 -79
- package/dist/commands/add-track-json.js +24 -12
- package/dist/commands/add-track-utils/adapter-utils.js +173 -288
- package/dist/commands/add-track-utils/file-operations.js +27 -11
- package/dist/commands/add-track-utils/multiwig.js +47 -0
- package/dist/commands/add-track-utils/track-config.js +68 -25
- package/dist/commands/add-track-utils/validators.js +36 -20
- package/dist/commands/add-track.js +178 -62
- package/dist/commands/admin-server/index.js +30 -49
- package/dist/commands/admin-server/utils.js +235 -125
- package/dist/commands/create.js +18 -34
- package/dist/commands/make-pif/cigar-utils.js +62 -19
- package/dist/commands/make-pif/index.js +98 -11
- package/dist/commands/make-pif/pif-generator.js +276 -42
- package/dist/commands/process-utils.js +13 -2
- package/dist/commands/remove-track.js +20 -13
- package/dist/commands/set-default-session.js +35 -31
- package/dist/commands/shared/config-operations.js +3 -10
- package/dist/commands/shared/sort-utils.js +67 -10
- package/dist/commands/shared/validators.js +38 -3
- package/dist/commands/sort-bed.js +2 -33
- package/dist/commands/sort-gff.js +2 -33
- package/dist/commands/text-index/adapter-utils.js +6 -23
- package/dist/commands/text-index/aggregate.js +40 -34
- package/dist/commands/text-index/command.js +35 -7
- package/dist/commands/text-index/config-utils.js +60 -83
- package/dist/commands/text-index/file-list.js +32 -18
- package/dist/commands/text-index/index.js +4 -4
- package/dist/commands/text-index/indexing-utils.js +49 -73
- package/dist/commands/text-index/per-track.js +44 -31
- package/dist/commands/text-index/validators.js +1 -1
- package/dist/commands/upgrade.js +20 -32
- package/dist/commands/validate/configManifest.generated.js +5394 -0
- package/dist/commands/validate/displayDefaultKeys.js +28 -0
- package/dist/commands/validate/index.js +97 -0
- package/dist/commands/validate/suggest.js +35 -0
- package/dist/commands/validate/types.js +6 -0
- package/dist/commands/validate/validateConfig.js +457 -0
- package/dist/index.js +99 -50
- package/dist/types/common.js +3 -27
- package/dist/utils.js +309 -128
- package/dist/version.js +1 -1
- package/package.json +8 -10
- package/dist/commands/make-pif/file-utils.js +0 -30
- package/dist/commands/track-utils.js +0 -74
- package/dist/util.js +0 -35
|
@@ -1,7 +1,5 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import parseJSON from 'json-parse-better-errors';
|
|
4
|
-
import { printHelp, readJsonFile, resolveConfigPath, writeJsonFile, } from "../utils.js";
|
|
1
|
+
import { parseArgs } from 'node:util';
|
|
2
|
+
import { isRecord, printHelp, readConfigFile, readJsonFile, resolveConfigPath, writeJsonFile, } from '../utils.js';
|
|
5
3
|
const description = 'Set a default session with views and tracks';
|
|
6
4
|
const examples = [
|
|
7
5
|
'# set default session for the config.json in your current directory',
|
|
@@ -10,20 +8,28 @@ const examples = [
|
|
|
10
8
|
'# make session.json the defaultSession on the specified target config.json file',
|
|
11
9
|
'$ jbrowse set-default-session --target /path/to/jb2/installation/config.json --session session.json',
|
|
12
10
|
'',
|
|
13
|
-
'#
|
|
14
|
-
'$ jbrowse set-default-session --
|
|
11
|
+
'# override the name stored in the session file',
|
|
12
|
+
'$ jbrowse set-default-session --session session.json --name "My default view"',
|
|
13
|
+
'',
|
|
14
|
+
'# pipe a session in from another program',
|
|
15
|
+
`$ jq '.defaultSession' other/config.json | jbrowse set-default-session --session -`,
|
|
16
|
+
'',
|
|
17
|
+
'# print the current default session',
|
|
18
|
+
'$ jbrowse set-default-session --currentSession',
|
|
19
|
+
'',
|
|
20
|
+
'# remove the existing default session',
|
|
21
|
+
'$ jbrowse set-default-session --delete',
|
|
15
22
|
];
|
|
16
23
|
const options = {
|
|
17
24
|
session: {
|
|
18
25
|
type: 'string',
|
|
19
26
|
short: 's',
|
|
20
|
-
description: 'set path to a file containing session in json format (required, unless using delete/currentSession flags)',
|
|
27
|
+
description: 'set path to a file containing session in json format, or "-" to read it from stdin (required, unless using delete/currentSession flags)',
|
|
21
28
|
},
|
|
22
29
|
name: {
|
|
23
30
|
type: 'string',
|
|
24
31
|
short: 'n',
|
|
25
|
-
description: 'Give a name for the default session',
|
|
26
|
-
default: 'New Default Session',
|
|
32
|
+
description: 'Give a name for the default session (overrides any name in the session file; defaults to "New Default Session")',
|
|
27
33
|
},
|
|
28
34
|
currentSession: {
|
|
29
35
|
type: 'boolean',
|
|
@@ -47,14 +53,14 @@ export async function run(args) {
|
|
|
47
53
|
printHelp({
|
|
48
54
|
description,
|
|
49
55
|
examples,
|
|
50
|
-
usage: 'jbrowse
|
|
56
|
+
usage: 'jbrowse set-default-session [options]',
|
|
51
57
|
options,
|
|
52
58
|
});
|
|
53
59
|
return;
|
|
54
60
|
}
|
|
55
|
-
const { session, currentSession, delete: deleteDefaultSession } = runFlags;
|
|
61
|
+
const { session, name, currentSession, delete: deleteDefaultSession, } = runFlags;
|
|
56
62
|
const target = await resolveConfigPath(runFlags.target, runFlags.out);
|
|
57
|
-
const configContents = await
|
|
63
|
+
const configContents = await readConfigFile(target);
|
|
58
64
|
if (deleteDefaultSession) {
|
|
59
65
|
configContents.defaultSession = undefined;
|
|
60
66
|
await writeJsonFile(target, configContents);
|
|
@@ -65,30 +71,28 @@ export async function run(args) {
|
|
|
65
71
|
else if (!session) {
|
|
66
72
|
throw new Error('Please provide a --session file');
|
|
67
73
|
}
|
|
68
|
-
else
|
|
74
|
+
else {
|
|
75
|
+
const fileSession = await readDefaultSessionFile(session);
|
|
69
76
|
await writeJsonFile(target, {
|
|
70
77
|
...configContents,
|
|
71
|
-
|
|
78
|
+
// precedence: explicit --name > the session file's own name > fallback
|
|
79
|
+
defaultSession: {
|
|
80
|
+
name: 'New Default Session',
|
|
81
|
+
...fileSession,
|
|
82
|
+
...(name !== undefined ? { name } : {}),
|
|
83
|
+
},
|
|
72
84
|
});
|
|
73
85
|
}
|
|
74
86
|
}
|
|
75
87
|
async function readDefaultSessionFile(defaultSessionFile) {
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
throw new Error(
|
|
84
|
-
}
|
|
85
|
-
try {
|
|
86
|
-
const session = parseJSON(defaultSessionJson);
|
|
87
|
-
// return top-level "session" if it exists, such as in files created by
|
|
88
|
-
// "File -> Export session"
|
|
89
|
-
return session.session || session;
|
|
90
|
-
}
|
|
91
|
-
catch (error) {
|
|
92
|
-
throw new Error('Could not parse the given default session file');
|
|
88
|
+
const contents = await readJsonFile(defaultSessionFile);
|
|
89
|
+
// unwrap the top-level "session" if it exists, such as in files created by
|
|
90
|
+
// "File -> Export session"
|
|
91
|
+
const session = contents.session ?? contents;
|
|
92
|
+
// spreading a string or an array here silently wrote a defaultSession of
|
|
93
|
+
// numeric keys, so say what is wrong with the file instead
|
|
94
|
+
if (!isRecord(session)) {
|
|
95
|
+
throw new Error(`${defaultSessionFile} does not contain a session object${contents.session === undefined ? '' : ' under its "session" key'}`);
|
|
93
96
|
}
|
|
97
|
+
return session;
|
|
94
98
|
}
|
|
@@ -1,14 +1,10 @@
|
|
|
1
|
-
import { debug, writeJsonFile } from
|
|
2
|
-
|
|
3
|
-
* Generic function to find and update or add an item to a config array
|
|
4
|
-
* Returns the updated config and whether an item was overwritten
|
|
5
|
-
*/
|
|
6
|
-
export function findAndUpdateOrAdd({ items, newItem, idField, getId, allowOverwrite, itemType, }) {
|
|
1
|
+
import { debug, writeJsonFile } from '../../utils.js';
|
|
2
|
+
export function findAndUpdateOrAdd({ items, newItem, idField, getId, force, itemType, }) {
|
|
7
3
|
const newId = getId(newItem);
|
|
8
4
|
const idx = items.findIndex(item => getId(item) === newId);
|
|
9
5
|
if (idx !== -1) {
|
|
10
6
|
debug(`Found existing ${itemType} ${newId} in configuration`);
|
|
11
|
-
if (
|
|
7
|
+
if (force) {
|
|
12
8
|
debug(`Overwriting ${itemType} ${newId} in configuration`);
|
|
13
9
|
const updatedItems = [...items];
|
|
14
10
|
updatedItems[idx] = newItem;
|
|
@@ -22,9 +18,6 @@ export function findAndUpdateOrAdd({ items, newItem, idField, getId, allowOverwr
|
|
|
22
18
|
return { updatedItems: [...items, newItem], wasOverwritten: false };
|
|
23
19
|
}
|
|
24
20
|
}
|
|
25
|
-
/**
|
|
26
|
-
* Saves a config file and reports the result to the user
|
|
27
|
-
*/
|
|
28
21
|
export async function saveConfigAndReport({ config, target, itemType, itemName, itemId, wasOverwritten, }) {
|
|
29
22
|
debug(`Writing configuration to file ${target}`);
|
|
30
23
|
await writeJsonFile(target, config);
|
|
@@ -1,7 +1,11 @@
|
|
|
1
|
-
import { spawn } from 'child_process';
|
|
2
|
-
import
|
|
1
|
+
import { spawn } from 'node:child_process';
|
|
2
|
+
import { parseArgs } from 'node:util';
|
|
3
|
+
import { fileSync } from 'tmp';
|
|
4
|
+
import { printHelp } from '../../utils.js';
|
|
5
|
+
import { describeExit, diedOfSigpipe, waitForProcessClose, } from '../process-utils.js';
|
|
6
|
+
import { validateFileArgument, validateRequiredCommands } from './validators.js';
|
|
3
7
|
export const BED_CONFIG = {
|
|
4
|
-
description: '
|
|
8
|
+
description: 'Sort a BED file for tabix. It is `sort -k1,1 -k2,2n` with LC_ALL=C and a tab separator, plus every line starting with # kept at the top rather than sorted into the data. Takes a file, or the same data on stdin',
|
|
5
9
|
examples: [
|
|
6
10
|
'# sort bed and pipe to bgzip',
|
|
7
11
|
'$ jbrowse sort-bed input.bed | bgzip > sorted.bed.gz',
|
|
@@ -13,7 +17,7 @@ export const BED_CONFIG = {
|
|
|
13
17
|
fileType: 'bed',
|
|
14
18
|
};
|
|
15
19
|
export const GFF_CONFIG = {
|
|
16
|
-
description: '
|
|
20
|
+
description: 'Sort a GFF or GTF for tabix — the two share a refname/start column layout. It is `sort -k1,1 -k4,4n` with LC_ALL=C and a tab separator, plus every line starting with # kept at the top rather than sorted into the data. Takes a file, or the same data on stdin',
|
|
17
21
|
examples: [
|
|
18
22
|
'# sort gff and pipe to bgzip',
|
|
19
23
|
'$ jbrowse sort-gff input.gff | bgzip > sorted.gff.gz',
|
|
@@ -21,6 +25,10 @@ export const GFF_CONFIG = {
|
|
|
21
25
|
'',
|
|
22
26
|
'# sort gff from stdin',
|
|
23
27
|
'$ cat input.gff | jbrowse sort-gff | bgzip > sorted.gff.gz',
|
|
28
|
+
'',
|
|
29
|
+
'# also works on GTF',
|
|
30
|
+
'$ jbrowse sort-gff input.gtf | bgzip > sorted.gtf.gz',
|
|
31
|
+
'$ tabix -p gff sorted.gtf.gz',
|
|
24
32
|
],
|
|
25
33
|
sortColumn: 4,
|
|
26
34
|
fileType: 'gff',
|
|
@@ -31,20 +39,69 @@ function getMinimalEnvironment() {
|
|
|
31
39
|
LC_ALL: 'C',
|
|
32
40
|
};
|
|
33
41
|
}
|
|
42
|
+
// The file path is passed to the shell as the positional argument "$1" rather
|
|
43
|
+
// than interpolated into the command string, so a path containing shell
|
|
44
|
+
// metacharacters (`"`, `$(...)`, backticks) cannot break out and execute. Only
|
|
45
|
+
// sortColumn, an integer from the BED/GFF configs, is interpolated.
|
|
46
|
+
function sortPipeline(sortColumn) {
|
|
47
|
+
return `grep "^#" "$1"; grep -v "^#" "$1" | sort -t"\`printf '\\t'\`" -k1,1 -k${sortColumn},${sortColumn}n`;
|
|
48
|
+
}
|
|
34
49
|
function createSortCommandForStdin(sortColumn) {
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
50
|
+
// discardDescriptor: we only want the path — the shell below opens the file
|
|
51
|
+
// itself, so tmp's own handle would just be a descriptor left open for the
|
|
52
|
+
// life of the process
|
|
53
|
+
const tmpFile = fileSync({
|
|
54
|
+
prefix: 'jbrowse-sort',
|
|
55
|
+
discardDescriptor: true,
|
|
56
|
+
}).name;
|
|
57
|
+
// trap on EXIT removes the temp file whether or not the sort pipeline
|
|
58
|
+
// succeeds (a bare `&& rm` leaks the file when the pipeline fails)
|
|
59
|
+
return {
|
|
60
|
+
command: `trap 'rm -f "$1"' EXIT; cat > "$1" && (${sortPipeline(sortColumn)})`,
|
|
61
|
+
pathArg: tmpFile,
|
|
62
|
+
};
|
|
38
63
|
}
|
|
39
64
|
function createSortCommandForFile(file, sortColumn) {
|
|
40
|
-
return `(
|
|
65
|
+
return { command: `(${sortPipeline(sortColumn)})`, pathArg: file };
|
|
41
66
|
}
|
|
42
67
|
export function spawnSortProcess(file, sortColumn) {
|
|
43
|
-
const command = file
|
|
68
|
+
const { command, pathArg } = file
|
|
44
69
|
? createSortCommandForFile(file, sortColumn)
|
|
45
70
|
: createSortCommandForStdin(sortColumn);
|
|
46
|
-
|
|
71
|
+
// 'sh' becomes $0, pathArg becomes $1 inside the command
|
|
72
|
+
return spawn('sh', ['-c', command, 'sh', pathArg], {
|
|
47
73
|
env: getMinimalEnvironment(),
|
|
48
74
|
stdio: 'inherit',
|
|
49
75
|
});
|
|
50
76
|
}
|
|
77
|
+
export async function runSort(config, commandName, args) {
|
|
78
|
+
const options = { help: { type: 'boolean', short: 'h' } };
|
|
79
|
+
const { values: flags, positionals } = parseArgs({
|
|
80
|
+
args,
|
|
81
|
+
options,
|
|
82
|
+
allowPositionals: true,
|
|
83
|
+
});
|
|
84
|
+
if (flags.help) {
|
|
85
|
+
printHelp({
|
|
86
|
+
description: config.description,
|
|
87
|
+
examples: config.examples,
|
|
88
|
+
usage: `jbrowse ${commandName} [file] [options]`,
|
|
89
|
+
options,
|
|
90
|
+
});
|
|
91
|
+
return;
|
|
92
|
+
}
|
|
93
|
+
const file = positionals[0];
|
|
94
|
+
validateFileArgument(file, commandName, config.fileType);
|
|
95
|
+
validateRequiredCommands(['sh', 'sort', 'grep']);
|
|
96
|
+
const child = spawnSortProcess(file, config.sortColumn);
|
|
97
|
+
const exit = await waitForProcessClose(child);
|
|
98
|
+
// These commands exist to be piped (`jbrowse sort-bed in.bed | bgzip > …`), so
|
|
99
|
+
// a consumer that stops reading early — `| head`, `| grep -q` — must not be
|
|
100
|
+
// reported as a sort failure. It used to exit 1 with "exited with code 141".
|
|
101
|
+
if (diedOfSigpipe(exit)) {
|
|
102
|
+
return;
|
|
103
|
+
}
|
|
104
|
+
if (exit.code !== 0) {
|
|
105
|
+
throw new Error(`Sort process exited with ${describeExit(exit)}`);
|
|
106
|
+
}
|
|
107
|
+
}
|
|
@@ -1,14 +1,49 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { execFileSync } from 'node:child_process';
|
|
2
|
+
import { accessSync, constants, statSync } from 'node:fs';
|
|
2
3
|
export function validateFileArgument(file, commandName, fileType) {
|
|
3
|
-
if (
|
|
4
|
+
if (file) {
|
|
5
|
+
// The sort pipeline's exit status is `sort`'s, which succeeds on empty
|
|
6
|
+
// input, so a missing/unreadable file would otherwise "succeed" with empty
|
|
7
|
+
// output. Fail loudly here instead.
|
|
8
|
+
if (!statSync(file, { throwIfNoEntry: false })?.isFile()) {
|
|
9
|
+
throw new Error(`Input file does not exist or is not a regular file: ${file}`);
|
|
10
|
+
}
|
|
11
|
+
try {
|
|
12
|
+
accessSync(file, constants.R_OK);
|
|
13
|
+
}
|
|
14
|
+
catch {
|
|
15
|
+
throw new Error(`Input file is not readable: ${file}`);
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
else if (process.stdin.isTTY) {
|
|
4
19
|
throw new Error(`Missing required argument: file\n` +
|
|
5
20
|
`Usage: jbrowse ${commandName} <file>\n` +
|
|
6
21
|
` OR pipe data via stdin: cat file.${fileType} | jbrowse ${commandName}`);
|
|
7
22
|
}
|
|
8
23
|
}
|
|
24
|
+
// Replaces command-exists: is `command` runnable? On POSIX, `command -v` (a
|
|
25
|
+
// shell builtin, so invoked via `sh`) resolves builtins, PATH entries, and
|
|
26
|
+
// absolute paths; on Windows, `where` does the same. The name is passed as an
|
|
27
|
+
// argv item, never interpolated into the shell, so it can't be injected.
|
|
28
|
+
function commandExistsSync(command) {
|
|
29
|
+
try {
|
|
30
|
+
if (process.platform === 'win32') {
|
|
31
|
+
execFileSync('where', [command], { stdio: 'ignore' });
|
|
32
|
+
}
|
|
33
|
+
else {
|
|
34
|
+
execFileSync('sh', ['-c', 'command -v "$1"', 'sh', command], {
|
|
35
|
+
stdio: 'ignore',
|
|
36
|
+
});
|
|
37
|
+
}
|
|
38
|
+
return true;
|
|
39
|
+
}
|
|
40
|
+
catch {
|
|
41
|
+
return false;
|
|
42
|
+
}
|
|
43
|
+
}
|
|
9
44
|
export function validateRequiredCommands(requiredCommands) {
|
|
10
45
|
const missingCommands = requiredCommands.filter(cmd => !commandExistsSync(cmd));
|
|
11
46
|
if (missingCommands.length > 0) {
|
|
12
|
-
throw new Error(`Unable to run command, requires unix type environment with: ${requiredCommands.join(', ')}`);
|
|
47
|
+
throw new Error(`Unable to run command, requires a unix type environment with the following available: ${requiredCommands.join(', ')}. Missing: ${missingCommands.join(', ')}`);
|
|
13
48
|
}
|
|
14
49
|
}
|
|
@@ -1,35 +1,4 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { printHelp } from "../utils.js";
|
|
3
|
-
import { waitForProcessClose } from "./process-utils.js";
|
|
4
|
-
import { BED_CONFIG, spawnSortProcess } from "./shared/sort-utils.js";
|
|
5
|
-
import { validateFileArgument, validateRequiredCommands, } from "./shared/validators.js";
|
|
1
|
+
import { BED_CONFIG, runSort } from './shared/sort-utils.js';
|
|
6
2
|
export async function run(args) {
|
|
7
|
-
|
|
8
|
-
help: {
|
|
9
|
-
type: 'boolean',
|
|
10
|
-
short: 'h',
|
|
11
|
-
},
|
|
12
|
-
};
|
|
13
|
-
const { values: flags, positionals } = parseArgs({
|
|
14
|
-
args,
|
|
15
|
-
options,
|
|
16
|
-
allowPositionals: true,
|
|
17
|
-
});
|
|
18
|
-
if (flags.help) {
|
|
19
|
-
printHelp({
|
|
20
|
-
description: BED_CONFIG.description,
|
|
21
|
-
examples: BED_CONFIG.examples,
|
|
22
|
-
usage: 'jbrowse sort-bed [file] [options]',
|
|
23
|
-
options,
|
|
24
|
-
});
|
|
25
|
-
return;
|
|
26
|
-
}
|
|
27
|
-
const file = positionals[0];
|
|
28
|
-
validateFileArgument(file, 'sort-bed', 'bed');
|
|
29
|
-
validateRequiredCommands(['sh', 'sort', 'grep']);
|
|
30
|
-
const child = spawnSortProcess(file, BED_CONFIG.sortColumn);
|
|
31
|
-
const exitCode = await waitForProcessClose(child);
|
|
32
|
-
if (exitCode !== 0) {
|
|
33
|
-
throw new Error(`Sort process exited with code ${exitCode}`);
|
|
34
|
-
}
|
|
3
|
+
return runSort(BED_CONFIG, 'sort-bed', args);
|
|
35
4
|
}
|
|
@@ -1,35 +1,4 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { printHelp } from "../utils.js";
|
|
3
|
-
import { waitForProcessClose } from "./process-utils.js";
|
|
4
|
-
import { GFF_CONFIG, spawnSortProcess } from "./shared/sort-utils.js";
|
|
5
|
-
import { validateFileArgument, validateRequiredCommands, } from "./shared/validators.js";
|
|
1
|
+
import { GFF_CONFIG, runSort } from './shared/sort-utils.js';
|
|
6
2
|
export async function run(args) {
|
|
7
|
-
|
|
8
|
-
help: {
|
|
9
|
-
type: 'boolean',
|
|
10
|
-
short: 'h',
|
|
11
|
-
},
|
|
12
|
-
};
|
|
13
|
-
const { values: flags, positionals } = parseArgs({
|
|
14
|
-
args,
|
|
15
|
-
options,
|
|
16
|
-
allowPositionals: true,
|
|
17
|
-
});
|
|
18
|
-
if (flags.help) {
|
|
19
|
-
printHelp({
|
|
20
|
-
description: GFF_CONFIG.description,
|
|
21
|
-
examples: GFF_CONFIG.examples,
|
|
22
|
-
usage: 'jbrowse sort-gff [file] [options]',
|
|
23
|
-
options,
|
|
24
|
-
});
|
|
25
|
-
return;
|
|
26
|
-
}
|
|
27
|
-
const file = positionals[0];
|
|
28
|
-
validateFileArgument(file, 'sort-gff', 'gff');
|
|
29
|
-
validateRequiredCommands(['sh', 'sort', 'grep']);
|
|
30
|
-
const child = spawnSortProcess(file, GFF_CONFIG.sortColumn);
|
|
31
|
-
const exitCode = await waitForProcessClose(child);
|
|
32
|
-
if (exitCode !== 0) {
|
|
33
|
-
throw new Error(`Sort process exited with code ${exitCode}`);
|
|
34
|
-
}
|
|
3
|
+
return runSort(GFF_CONFIG, 'sort-gff', args);
|
|
35
4
|
}
|
|
@@ -1,38 +1,21 @@
|
|
|
1
|
-
import {
|
|
2
|
-
export function getLoc(elt) {
|
|
3
|
-
return elt.locationType === 'LocalPathLocation' ? elt.localPath : elt.uri;
|
|
4
|
-
}
|
|
1
|
+
import { sanitizeForFilename, trixFileUris } from '@jbrowse/text-indexing-core';
|
|
5
2
|
export function createTrixAdapter(name, assemblyNames, idSuffix = 'index') {
|
|
6
|
-
const
|
|
3
|
+
const uris = trixFileUris(name);
|
|
7
4
|
return {
|
|
8
5
|
type: 'TrixTextSearchAdapter',
|
|
9
|
-
textSearchAdapterId: `${
|
|
6
|
+
textSearchAdapterId: `${sanitizeForFilename(name)}-${idSuffix}`,
|
|
10
7
|
ixFilePath: {
|
|
11
|
-
uri:
|
|
8
|
+
uri: uris.ix,
|
|
12
9
|
locationType: 'UriLocation',
|
|
13
10
|
},
|
|
14
11
|
ixxFilePath: {
|
|
15
|
-
uri:
|
|
12
|
+
uri: uris.ixx,
|
|
16
13
|
locationType: 'UriLocation',
|
|
17
14
|
},
|
|
18
15
|
metaFilePath: {
|
|
19
|
-
uri:
|
|
16
|
+
uri: uris.meta,
|
|
20
17
|
locationType: 'UriLocation',
|
|
21
18
|
},
|
|
22
19
|
assemblyNames,
|
|
23
20
|
};
|
|
24
21
|
}
|
|
25
|
-
const ADAPTER_LOCATION_KEYS = {
|
|
26
|
-
Gff3TabixAdapter: 'gffGzLocation',
|
|
27
|
-
Gff3Adapter: 'gffLocation',
|
|
28
|
-
VcfAdapter: 'vcfLocation',
|
|
29
|
-
VcfTabixAdapter: 'vcfGzLocation',
|
|
30
|
-
};
|
|
31
|
-
export function getAdapterLocation(adapter) {
|
|
32
|
-
const key = ADAPTER_LOCATION_KEYS[adapter?.type ?? ''];
|
|
33
|
-
return key
|
|
34
|
-
? // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
|
|
35
|
-
(adapter?.[key] ??
|
|
36
|
-
adapter)
|
|
37
|
-
: undefined;
|
|
38
|
-
}
|
|
@@ -1,13 +1,15 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
1
|
+
import { parseCommaSeparatedString, writeJsonFile } from '../../utils.js';
|
|
2
|
+
import { createTrixAdapter } from './adapter-utils.js';
|
|
3
|
+
import { formatDryRun, getAssemblyNames, getTrackConfigs, loadConfigForIndexing, prepareIndexDriverFlags, } from './config-utils.js';
|
|
4
|
+
import { indexDriver } from './indexing-utils.js';
|
|
5
5
|
export async function aggregateIndex(flags) {
|
|
6
|
-
const { out, target, tracks, excludeTracks, assemblies, attributes, quiet, force, exclude, dryrun, prefixSize, } = flags;
|
|
7
|
-
const { config, configPath, outLocation } = await loadConfigForIndexing(target, out
|
|
8
|
-
const
|
|
9
|
-
|
|
10
|
-
|
|
6
|
+
const { out, target, tracks, excludeTracks, assemblies, attributes, quiet, force, exclude, include, dryrun, prefixSize, } = flags;
|
|
7
|
+
const { config, configPath, outLocation } = await loadConfigForIndexing(target, out);
|
|
8
|
+
const existing = config.aggregateTextSearchAdapters ?? [];
|
|
9
|
+
// sequential, not Promise.all: each assembly's indexer owns the progress bar
|
|
10
|
+
// and streams whole files through ixIxx
|
|
11
|
+
const written = [];
|
|
12
|
+
for (const asm of getAssemblyNames(config, assemblies)) {
|
|
11
13
|
const trackConfigs = getTrackConfigs(config, parseCommaSeparatedString(tracks), asm, parseCommaSeparatedString(excludeTracks));
|
|
12
14
|
if (!trackConfigs.length) {
|
|
13
15
|
console.log(`Indexing assembly ${asm}...(no tracks found)...`);
|
|
@@ -15,35 +17,39 @@ export async function aggregateIndex(flags) {
|
|
|
15
17
|
}
|
|
16
18
|
console.log(`Indexing assembly ${asm}...`);
|
|
17
19
|
if (dryrun) {
|
|
18
|
-
console.log(trackConfigs
|
|
20
|
+
console.log(formatDryRun(trackConfigs));
|
|
21
|
+
continue;
|
|
19
22
|
}
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
continue;
|
|
26
|
-
}
|
|
27
|
-
await indexDriver({
|
|
28
|
-
trackConfigs,
|
|
29
|
-
outLocation,
|
|
30
|
-
name: asm,
|
|
31
|
-
assemblyNames: [asm],
|
|
32
|
-
...prepareIndexDriverFlags({ attributes, exclude, quiet, prefixSize }),
|
|
33
|
-
});
|
|
34
|
-
const trixConf = createTrixAdapter(asm, [asm]);
|
|
35
|
-
if (idx === -1) {
|
|
36
|
-
aggregateTextSearchAdapters.push(trixConf);
|
|
37
|
-
}
|
|
38
|
-
else {
|
|
39
|
-
aggregateTextSearchAdapters[idx] = trixConf;
|
|
40
|
-
}
|
|
23
|
+
const trixConf = createTrixAdapter(asm, [asm]);
|
|
24
|
+
if (!force &&
|
|
25
|
+
existing.some(x => x.textSearchAdapterId === trixConf.textSearchAdapterId)) {
|
|
26
|
+
console.log(`Note: ${asm} has already been indexed with this configuration, use --force to overwrite this assembly. Skipping for now`);
|
|
27
|
+
continue;
|
|
41
28
|
}
|
|
29
|
+
await indexDriver({
|
|
30
|
+
trackConfigs,
|
|
31
|
+
outLocation,
|
|
32
|
+
name: asm,
|
|
33
|
+
assemblyNames: [asm],
|
|
34
|
+
...prepareIndexDriverFlags({
|
|
35
|
+
attributes,
|
|
36
|
+
exclude,
|
|
37
|
+
include,
|
|
38
|
+
quiet,
|
|
39
|
+
prefixSize,
|
|
40
|
+
}),
|
|
41
|
+
});
|
|
42
|
+
written.push(trixConf);
|
|
42
43
|
}
|
|
43
44
|
if (!dryrun) {
|
|
44
|
-
|
|
45
|
+
await writeJsonFile(configPath, {
|
|
45
46
|
...config,
|
|
46
|
-
|
|
47
|
-
|
|
47
|
+
// upsert by adapter id: a Map keyed on it keeps the first occurrence's
|
|
48
|
+
// position while the last value wins, so re-indexing an assembly replaces
|
|
49
|
+
// its adapter in place and a new one lands at the end
|
|
50
|
+
aggregateTextSearchAdapters: [
|
|
51
|
+
...new Map([...existing, ...written].map(a => [a.textSearchAdapterId, a])).values(),
|
|
52
|
+
],
|
|
53
|
+
});
|
|
48
54
|
}
|
|
49
55
|
}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
|
-
import { parseArgs } from 'util';
|
|
2
|
-
import {
|
|
3
|
-
import { printHelp } from
|
|
1
|
+
import { parseArgs } from 'node:util';
|
|
2
|
+
import { defaultAttributesToIndex, indexableAdapters, } from '@jbrowse/text-indexing-core';
|
|
3
|
+
import { printHelp } from '../../utils.js';
|
|
4
|
+
import { aggregateIndex, indexFileList, perTrackIndex } from './index.js';
|
|
4
5
|
export async function run(args) {
|
|
5
6
|
const options = {
|
|
6
7
|
help: {
|
|
@@ -14,7 +15,7 @@ export async function run(args) {
|
|
|
14
15
|
},
|
|
15
16
|
excludeTracks: {
|
|
16
17
|
type: 'string',
|
|
17
|
-
description: 'Specific tracks to exclude from indexing, formatted as comma separated trackIds',
|
|
18
|
+
description: 'Specific tracks to exclude from indexing, formatted as comma separated trackIds. To exclude a track permanently, set metadata.skipTextIndex on it in config.json instead (see Notes)',
|
|
18
19
|
},
|
|
19
20
|
target: {
|
|
20
21
|
type: 'string',
|
|
@@ -24,7 +25,7 @@ export async function run(args) {
|
|
|
24
25
|
attributes: {
|
|
25
26
|
type: 'string',
|
|
26
27
|
description: 'Comma separated list of attributes to index',
|
|
27
|
-
default: '
|
|
28
|
+
default: defaultAttributesToIndex.join(','),
|
|
28
29
|
},
|
|
29
30
|
assemblies: {
|
|
30
31
|
type: 'string',
|
|
@@ -49,9 +50,13 @@ export async function run(args) {
|
|
|
49
50
|
},
|
|
50
51
|
exclude: {
|
|
51
52
|
type: 'string',
|
|
52
|
-
description: '
|
|
53
|
+
description: 'Comma separated list of feature types to exclude from indexing',
|
|
53
54
|
default: 'CDS,exon',
|
|
54
55
|
},
|
|
56
|
+
include: {
|
|
57
|
+
type: 'string',
|
|
58
|
+
description: 'Comma separated list of feature types to index, dropping every other type. Unset by default, which indexes every type --exclude does not name. GFF3 only',
|
|
59
|
+
},
|
|
55
60
|
prefixSize: {
|
|
56
61
|
type: 'string',
|
|
57
62
|
description: 'Specify the prefix size for the ixx index. We attempt to automatically calculate this, but you can manually specify this too. If many genes have similar gene IDs e.g. Z000000001, Z000000002 the prefix size should be larger so that they get split into different bins',
|
|
@@ -76,6 +81,28 @@ export async function run(args) {
|
|
|
76
81
|
options,
|
|
77
82
|
});
|
|
78
83
|
const description = 'Make a text-indexing file for any given track(s).';
|
|
84
|
+
const notes = 'Individual tracks in config.json can be permanently excluded from ' +
|
|
85
|
+
'indexing by setting "metadata": { "skipTextIndex": true } on the track. ' +
|
|
86
|
+
'Such tracks are skipped even when indexing all tracks or a whole ' +
|
|
87
|
+
'assembly, so you do not have to pass --excludeTracks on every run.\n\n' +
|
|
88
|
+
// read off the table rather than restated, because the hand-written version
|
|
89
|
+
// of this sentence outlived GtfTabixAdapter's absence from it
|
|
90
|
+
`Only tracks with an indexable adapter type (${Object.keys(indexableAdapters).sort().join(', ')}) are indexed; ` +
|
|
91
|
+
'tracks with other adapter types are skipped automatically.\n\n' +
|
|
92
|
+
'GTF has no Name/ID attributes, so the default --attributes also match ' +
|
|
93
|
+
'their GTF spellings (gene_name, transcript_name, gene_id, transcript_id).' +
|
|
94
|
+
'\n\n' +
|
|
95
|
+
'--exclude names types not to index; --include names the only types to ' +
|
|
96
|
+
'index. Reach for --include when the file draws from a vocabulary you do ' +
|
|
97
|
+
'not control: an NCBI RefSeq GFF3 uses 115 feature types, 80 of them leaf ' +
|
|
98
|
+
'records with no name to search (a match is labelled with a bare UUID, a ' +
|
|
99
|
+
'cDNA_match with an MD5, every biological_region with the string ' +
|
|
100
|
+
'"biological region"), so a deny list leaks whichever type is added next ' +
|
|
101
|
+
'while the allow list — gene, pseudogene and the transcript types — does ' +
|
|
102
|
+
'not grow. Both may be given: --include admits, --exclude then narrows. ' +
|
|
103
|
+
'Either can also be set per track in config.json as ' +
|
|
104
|
+
'textSearching.indexingFeatureTypesToInclude / ' +
|
|
105
|
+
'indexingFeatureTypesToExclude, which takes precedence over the flag.';
|
|
79
106
|
const examples = [
|
|
80
107
|
"# indexes all tracks that it can find in the current directory's config.json",
|
|
81
108
|
'$ jbrowse text-index',
|
|
@@ -84,7 +111,7 @@ export async function run(args) {
|
|
|
84
111
|
'$ jbrowse text-index --tracks=track1,track2,track3',
|
|
85
112
|
'',
|
|
86
113
|
'# indexes all tracks except specific trackIds',
|
|
87
|
-
'$ jbrowse text-index --
|
|
114
|
+
'$ jbrowse text-index --excludeTracks=track1,track2,track3',
|
|
88
115
|
'',
|
|
89
116
|
"# indexes all tracks in a directory's config.json or in a specific config file",
|
|
90
117
|
'$ jbrowse text-index --out /path/to/jb2/',
|
|
@@ -99,6 +126,7 @@ export async function run(args) {
|
|
|
99
126
|
printHelp({
|
|
100
127
|
description,
|
|
101
128
|
examples,
|
|
129
|
+
notes,
|
|
102
130
|
usage: 'jbrowse text-index [options]',
|
|
103
131
|
options,
|
|
104
132
|
});
|