miga-base 0.7.3.0 → 0.7.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/miga/cli.rb +10 -8
- data/lib/miga/cli/action.rb +2 -3
- data/lib/miga/cli/action/about.rb +5 -6
- data/lib/miga/cli/action/add.rb +18 -12
- data/lib/miga/cli/action/add_result.rb +2 -3
- data/lib/miga/cli/action/archive.rb +1 -2
- data/lib/miga/cli/action/classify_wf.rb +8 -6
- data/lib/miga/cli/action/console.rb +0 -1
- data/lib/miga/cli/action/daemon.rb +7 -7
- data/lib/miga/cli/action/date.rb +0 -1
- data/lib/miga/cli/action/derep_wf.rb +5 -4
- data/lib/miga/cli/action/doctor.rb +71 -82
- data/lib/miga/cli/action/doctor/base.rb +102 -0
- data/lib/miga/cli/action/edit.rb +14 -2
- data/lib/miga/cli/action/files.rb +8 -8
- data/lib/miga/cli/action/find.rb +5 -6
- data/lib/miga/cli/action/generic.rb +7 -7
- data/lib/miga/cli/action/get.rb +20 -17
- data/lib/miga/cli/action/get_db.rb +8 -2
- data/lib/miga/cli/action/index_wf.rb +1 -1
- data/lib/miga/cli/action/init.rb +53 -41
- data/lib/miga/cli/action/init/daemon_helper.rb +65 -43
- data/lib/miga/cli/action/lair.rb +7 -7
- data/lib/miga/cli/action/ln.rb +6 -6
- data/lib/miga/cli/action/ls.rb +1 -2
- data/lib/miga/cli/action/ncbi_get.rb +11 -3
- data/lib/miga/cli/action/new.rb +4 -4
- data/lib/miga/cli/action/next_step.rb +0 -1
- data/lib/miga/cli/action/preproc_wf.rb +3 -3
- data/lib/miga/cli/action/quality_wf.rb +1 -1
- data/lib/miga/cli/action/rm.rb +2 -3
- data/lib/miga/cli/action/run.rb +8 -8
- data/lib/miga/cli/action/stats.rb +8 -4
- data/lib/miga/cli/action/summary.rb +7 -6
- data/lib/miga/cli/action/tax_dist.rb +8 -4
- data/lib/miga/cli/action/tax_index.rb +3 -4
- data/lib/miga/cli/action/tax_set.rb +7 -6
- data/lib/miga/cli/action/tax_test.rb +6 -5
- data/lib/miga/cli/action/wf.rb +21 -19
- data/lib/miga/cli/base.rb +34 -32
- data/lib/miga/cli/objects_helper.rb +27 -18
- data/lib/miga/cli/opt_helper.rb +3 -2
- data/lib/miga/common.rb +2 -5
- data/lib/miga/common/base.rb +15 -16
- data/lib/miga/common/format.rb +8 -5
- data/lib/miga/common/hooks.rb +1 -4
- data/lib/miga/common/path.rb +4 -9
- data/lib/miga/common/with_daemon.rb +6 -3
- data/lib/miga/common/with_daemon_class.rb +3 -2
- data/lib/miga/common/with_result.rb +2 -1
- data/lib/miga/daemon.rb +93 -44
- data/lib/miga/daemon/base.rb +30 -11
- data/lib/miga/dataset.rb +47 -37
- data/lib/miga/dataset/base.rb +52 -37
- data/lib/miga/dataset/hooks.rb +3 -4
- data/lib/miga/dataset/result.rb +17 -1
- data/lib/miga/dataset/status.rb +6 -5
- data/lib/miga/json.rb +5 -7
- data/lib/miga/lair.rb +4 -0
- data/lib/miga/metadata.rb +4 -3
- data/lib/miga/project.rb +29 -20
- data/lib/miga/project/base.rb +52 -37
- data/lib/miga/project/dataset.rb +33 -26
- data/lib/miga/project/hooks.rb +0 -3
- data/lib/miga/project/result.rb +14 -5
- data/lib/miga/remote_dataset.rb +85 -72
- data/lib/miga/remote_dataset/base.rb +11 -13
- data/lib/miga/remote_dataset/download.rb +34 -12
- data/lib/miga/result.rb +34 -25
- data/lib/miga/result/base.rb +0 -2
- data/lib/miga/result/dates.rb +1 -3
- data/lib/miga/result/source.rb +15 -16
- data/lib/miga/result/stats.rb +37 -27
- data/lib/miga/tax_dist.rb +6 -4
- data/lib/miga/tax_index.rb +17 -17
- data/lib/miga/taxonomy.rb +6 -1
- data/lib/miga/taxonomy/base.rb +19 -15
- data/lib/miga/version.rb +19 -16
- data/scripts/project_stats.bash +3 -0
- data/scripts/stats.bash +1 -1
- data/test/common_test.rb +3 -11
- data/test/daemon_helper.rb +38 -0
- data/test/daemon_test.rb +91 -99
- data/test/dataset_test.rb +63 -59
- data/test/format_test.rb +3 -11
- data/test/hook_test.rb +50 -55
- data/test/json_test.rb +7 -8
- data/test/lair_test.rb +22 -28
- data/test/metadata_test.rb +6 -14
- data/test/project_test.rb +33 -40
- data/test/remote_dataset_test.rb +26 -32
- data/test/result_stats_test.rb +17 -27
- data/test/result_test.rb +41 -34
- data/test/tax_dist_test.rb +2 -4
- data/test/tax_index_test.rb +4 -10
- data/test/taxonomy_test.rb +7 -9
- data/test/test_helper.rb +42 -1
- data/test/with_daemon_test.rb +14 -22
- data/utils/adapters.fa +13 -0
- data/utils/cleanup-databases.rb +6 -5
- data/utils/distance/base.rb +0 -1
- data/utils/distance/commands.rb +19 -12
- data/utils/distance/database.rb +25 -21
- data/utils/distance/pipeline.rb +16 -10
- data/utils/distance/runner.rb +19 -13
- data/utils/distance/temporal.rb +7 -4
- data/utils/distances.rb +1 -1
- data/utils/domain-ess-genes.rb +7 -7
- data/utils/index_metadata.rb +5 -4
- data/utils/mytaxa_scan.rb +18 -16
- data/utils/representatives.rb +5 -4
- data/utils/requirements.txt +1 -1
- data/utils/subclade/base.rb +0 -1
- data/utils/subclade/pipeline.rb +7 -6
- data/utils/subclade/runner.rb +9 -9
- data/utils/subclade/temporal.rb +0 -2
- data/utils/subclades-compile.rb +39 -37
- data/utils/subclades.rb +1 -1
- metadata +6 -4
@@ -0,0 +1,102 @@
|
|
1
|
+
require 'miga/cli/action'
|
2
|
+
require 'sqlite3'
|
3
|
+
|
4
|
+
class MiGA::Cli::Action::Doctor < MiGA::Cli::Action
|
5
|
+
end
|
6
|
+
|
7
|
+
module MiGA::Cli::Action::Doctor::Base
|
8
|
+
##
|
9
|
+
# Check the database in +db_file+ maintains integrity for the
|
10
|
+
# tables saving +metric+ (:ani or :aai) and call +blk+ if the
|
11
|
+
# file is corrupt or doesn't contain the expected structure
|
12
|
+
def check_sqlite3_database(db_file, metric, &blk)
|
13
|
+
SQLite3::Database.new(db_file) do |conn|
|
14
|
+
conn.execute("select count(*) from #{metric}").first
|
15
|
+
end
|
16
|
+
rescue SQLite3::SQLException, SQLite3::CorruptException
|
17
|
+
blk.call
|
18
|
+
end
|
19
|
+
|
20
|
+
def each_database_file(dataset, &blk)
|
21
|
+
ref_db = { '01.haai' => :aai, '02.aai' => :aai, '03.ani' => :ani }
|
22
|
+
qry_db = { '.haai.db' => :aai, '.aai.db' => :aai, '.ani.db' => :ani }
|
23
|
+
base = File.join(dataset.project.path, 'data', '09.distances')
|
24
|
+
result = :distances
|
25
|
+
if dataset.ref?
|
26
|
+
file_db = "#{dataset.name}.db"
|
27
|
+
ref_db.each do |dir, metric|
|
28
|
+
file = File.join(base, dir, file_db)
|
29
|
+
blk[file, metric, result] if File.exist? file
|
30
|
+
end
|
31
|
+
base = File.join(base, '05.taxonomy')
|
32
|
+
result = :taxonomy
|
33
|
+
end
|
34
|
+
qry_db.each do |ext, metric|
|
35
|
+
file = File.join(base, "#{dataset.name}#{ext}")
|
36
|
+
blk[file, metric, result] if File.exist? file
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
##
|
41
|
+
# Scans the all-vs-all matrix registered in +res+ (MiGA::Result) in search of
|
42
|
+
# pairs where one or both datasets are missing or inactive in the project +p+
|
43
|
+
# (MiGA::Project), and report progress through +cli+ (MiGA::Cli).
|
44
|
+
# Returns an Array with two arrays: the first a list of dataset names that are
|
45
|
+
# no longer registered in the project or are currently inactive, and the
|
46
|
+
# second a list of dataset names that have registered pairs with the first
|
47
|
+
# list, and therefore the databases need to be cleaned.
|
48
|
+
# This is a subtask of +check_dist+
|
49
|
+
def check_dist_eval(cli, p, res)
|
50
|
+
notok = {}
|
51
|
+
fix = {}
|
52
|
+
Zlib::GzipReader.open(res.file_path(:matrix)) do |fh|
|
53
|
+
lineno = 0
|
54
|
+
fh.each_line do |ln|
|
55
|
+
next if (lineno += 1) == 1
|
56
|
+
|
57
|
+
r = ln.split("\t")
|
58
|
+
next unless [1, 2].map { |i| p.dataset(r[i]).nil? }.any?
|
59
|
+
|
60
|
+
[1, 2].each do |i|
|
61
|
+
if p.dataset(r[i]).nil? || !p.dataset(r[i]).active?
|
62
|
+
notok[r[i]] = true
|
63
|
+
else
|
64
|
+
fix[r[i]] = true
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
68
|
+
end
|
69
|
+
[notok.keys, fix.keys]
|
70
|
+
end
|
71
|
+
|
72
|
+
##
|
73
|
+
# Cleanup distance databases for datasets names in +fix+ (Array: String)
|
74
|
+
# from project +p+ (MiGA::Project), and report through +cli+ (MiGA::Cli).
|
75
|
+
# This is a subtask of +check_dist+
|
76
|
+
def check_dist_fix(cli, p, fix)
|
77
|
+
return if fix.empty?
|
78
|
+
|
79
|
+
cli.say("- Fixing #{fix.size} datasets")
|
80
|
+
fix.each do |d_n|
|
81
|
+
cli.say " > Fixing #{d_n}."
|
82
|
+
p.dataset(d_n).cleanup_distances!
|
83
|
+
end
|
84
|
+
end
|
85
|
+
|
86
|
+
##
|
87
|
+
# Recompute +res+ (MiGA::Result) if +notok+ (Array: String) has any dataset
|
88
|
+
# names registered, and report through +cli+ (MiGA::Cli).
|
89
|
+
# This is a subtask of +check_dist+
|
90
|
+
def check_dist_recompute(cli, res, notok)
|
91
|
+
return if notok.empty?
|
92
|
+
|
93
|
+
cli.say '- Unregistered datasets detected: '
|
94
|
+
if notok.size <= 5
|
95
|
+
notok.each { |i| cli.say " > #{i}" }
|
96
|
+
else
|
97
|
+
cli.say " > #{notok.size}, including #{notok.first}"
|
98
|
+
end
|
99
|
+
cli.say '- Removing tables, recompute'
|
100
|
+
res.remove!
|
101
|
+
end
|
102
|
+
end
|
data/lib/miga/cli/action/edit.rb
CHANGED
@@ -4,7 +4,6 @@
|
|
4
4
|
require 'miga/cli/action'
|
5
5
|
|
6
6
|
class MiGA::Cli::Action::Edit < MiGA::Cli::Action
|
7
|
-
|
8
7
|
def parse_cli
|
9
8
|
cli.parse do |opt|
|
10
9
|
cli.opt_object(opt, [:project, :dataset_opt])
|
@@ -12,12 +11,25 @@ class MiGA::Cli::Action::Edit < MiGA::Cli::Action
|
|
12
11
|
'-m', '--metadata STRING',
|
13
12
|
'Metadata as key-value pairs separated by = and delimited by comma',
|
14
13
|
'Values are saved as strings except for booleans (true / false) or nil'
|
15
|
-
|
14
|
+
) { |v| cli[:metadata] = v }
|
15
|
+
opt.on(
|
16
|
+
'--activate',
|
17
|
+
'Activate dataset; requires -D'
|
18
|
+
) { |v| cli[:activate] = v }
|
19
|
+
opt.on(
|
20
|
+
'--inactivate',
|
21
|
+
'Inactivate dataset; requires -D'
|
22
|
+
) { |v| cli[:activate] = !v }
|
16
23
|
end
|
17
24
|
end
|
18
25
|
|
19
26
|
def perform
|
20
27
|
obj = cli.load_project_or_dataset
|
28
|
+
unless cli[:activate].nil?
|
29
|
+
cli.ensure_par({ dataset: '-D' },
|
30
|
+
'%<name>s is mandatory with --[in-]activate: please provide %<flag>s')
|
31
|
+
cli[:activate] ? obj.activate! : obj.inactivate!
|
32
|
+
end
|
21
33
|
cli.add_metadata(obj)
|
22
34
|
obj.save
|
23
35
|
end
|
@@ -4,27 +4,27 @@
|
|
4
4
|
require 'miga/cli/action'
|
5
5
|
|
6
6
|
class MiGA::Cli::Action::Files < MiGA::Cli::Action
|
7
|
-
|
8
7
|
def parse_cli
|
9
|
-
cli.defaults = {details: false, json: true}
|
8
|
+
cli.defaults = { details: false, json: true }
|
10
9
|
cli.parse do |opt|
|
11
10
|
cli.opt_object(opt, [:project, :dataset_opt])
|
12
11
|
opt.on(
|
13
12
|
'-i', '--info',
|
14
13
|
'Print additional details for each file'
|
15
|
-
|
16
|
-
opt.on(
|
14
|
+
) { |v| cli[:details] = v }
|
15
|
+
opt.on(
|
16
|
+
'--[no-]json',
|
17
17
|
'Include (or not) JSON files containing results metadata',
|
18
18
|
'JSON files are included by default'
|
19
|
-
|
19
|
+
) { |v| cli[:json] = v }
|
20
20
|
end
|
21
21
|
end
|
22
22
|
|
23
23
|
def perform
|
24
24
|
cli.load_project_or_dataset.each_result do |sym, res|
|
25
|
-
cli.puts "#{
|
26
|
-
res.each_file do |k,f|
|
27
|
-
cli.puts "#{
|
25
|
+
cli.puts "#{"#{sym}\tjson\t" if cli[:details]}#{res.path}" if cli[:json]
|
26
|
+
res.each_file do |k, f|
|
27
|
+
cli.puts "#{"#{sym}\t#{k}\t" if cli[:details]}#{res.dir}/#{f}"
|
28
28
|
end
|
29
29
|
end
|
30
30
|
end
|
data/lib/miga/cli/action/find.rb
CHANGED
@@ -4,29 +4,28 @@
|
|
4
4
|
require 'miga/cli/action'
|
5
5
|
|
6
6
|
class MiGA::Cli::Action::Find < MiGA::Cli::Action
|
7
|
-
|
8
7
|
def parse_cli
|
9
|
-
cli.defaults = {add: false, ref: true}
|
8
|
+
cli.defaults = { add: false, ref: true }
|
10
9
|
cli.parse do |opt|
|
11
10
|
cli.opt_object(opt, [:project, :dataset_type])
|
12
11
|
opt.on(
|
13
12
|
'-a', '--add',
|
14
13
|
'Register the datasets found',
|
15
14
|
'By default, only lists them (dry run)'
|
16
|
-
|
15
|
+
) { |v| cli[:add] = v }
|
17
16
|
opt.on(
|
18
17
|
'-q', '--query',
|
19
18
|
'Register datasets as query'
|
20
|
-
|
19
|
+
) { |v| cli[:ref] = !v }
|
21
20
|
opt.on(
|
22
21
|
'-u', '--user STRING',
|
23
22
|
'Owner of the dataset.'
|
24
|
-
|
23
|
+
) { |v| cli[:user] = v }
|
25
24
|
opt.on(
|
26
25
|
'-m', '--metadata STRING',
|
27
26
|
'Metadata as key-value pairs separated by = and delimited by comma',
|
28
27
|
'Values are saved as strings except for booleans (true / false) or nil'
|
29
|
-
|
28
|
+
) { |v| cli[:metadata] = v }
|
30
29
|
end
|
31
30
|
end
|
32
31
|
|
@@ -4,31 +4,31 @@
|
|
4
4
|
require 'miga/cli/action'
|
5
5
|
|
6
6
|
class MiGA::Cli::Action::Generic < MiGA::Cli::Action
|
7
|
-
|
8
7
|
def parse_cli
|
9
8
|
cli.opt_common = false
|
10
9
|
cli.parse do |opt|
|
11
|
-
descriptions = cli.class.TASK_DESC.keep_if { |k,v| k != :generic }
|
10
|
+
descriptions = cli.class.TASK_DESC.keep_if { |k, v| k != :generic }
|
12
11
|
opt.separator MiGA::MiGA.tabulate(
|
13
|
-
[:action, :description], descriptions
|
12
|
+
[:action, :description], descriptions
|
13
|
+
).join("\n")
|
14
14
|
opt.separator ''
|
15
15
|
opt.separator 'generic options:'
|
16
16
|
opt.on(
|
17
17
|
'-h', '--help',
|
18
18
|
'Display this screen'
|
19
|
-
|
19
|
+
) { puts opt; exit }
|
20
20
|
opt.on(
|
21
21
|
'-v', '--version',
|
22
22
|
'Show MiGA version'
|
23
|
-
|
23
|
+
) { puts MiGA::MiGA.VERSION; exit }
|
24
24
|
opt.on(
|
25
25
|
'-V', '--long-version',
|
26
26
|
'Show complete MiGA version'
|
27
|
-
|
27
|
+
) { |v| puts MiGA::MiGA.LONG_VERSION; exit }
|
28
28
|
opt.on(
|
29
29
|
'-C', '--citation',
|
30
30
|
'How to cite MiGA'
|
31
|
-
|
31
|
+
) { |v| puts MiGA::MiGA.CITATION; exit }
|
32
32
|
end
|
33
33
|
end
|
34
34
|
|
data/lib/miga/cli/action/get.rb
CHANGED
@@ -5,24 +5,23 @@ require 'miga/cli/action'
|
|
5
5
|
require 'miga/remote_dataset'
|
6
6
|
|
7
7
|
class MiGA::Cli::Action::Get < MiGA::Cli::Action
|
8
|
-
|
9
8
|
def parse_cli
|
10
|
-
cli.defaults = {query: false, universe: :ncbi, db: :nuccore,
|
11
|
-
|
9
|
+
cli.defaults = { query: false, universe: :ncbi, db: :nuccore,
|
10
|
+
get_md: false, only_md: false }
|
12
11
|
cli.parse do |opt|
|
13
12
|
cli.opt_object(opt, [:project, :dataset, :dataset_type])
|
14
13
|
opt.on(
|
15
14
|
'-I', '--ids ID1,ID2,...', Array,
|
16
15
|
'(Mandatory unless -F) IDs in the remote database separated by commas'
|
17
|
-
|
16
|
+
) { |v| cli[:ids] = v }
|
18
17
|
opt.on(
|
19
18
|
'-U', '--universe STRING',
|
20
19
|
"Universe of the remote database. By default: #{cli[:universe]}"
|
21
|
-
|
20
|
+
) { |v| cli[:universe] = v.to_sym }
|
22
21
|
opt.on(
|
23
22
|
'--db STRING',
|
24
23
|
"Name of the remote database. By default: #{cli[:db]}"
|
25
|
-
|
24
|
+
) { |v| cli[:db] = v.to_sym }
|
26
25
|
opt.on(
|
27
26
|
'-F', '--file PATH',
|
28
27
|
'Tab-delimited file (with header) listing the datasets to download',
|
@@ -30,39 +29,40 @@ class MiGA::Cli::Action::Get < MiGA::Cli::Action
|
|
30
29
|
'including: dataset, ids, universe, db, metadata',
|
31
30
|
'For flags without value (like query) use true/false',
|
32
31
|
'Unsupported values are: project, file, verbose, help, and debug'
|
33
|
-
|
32
|
+
) { |v| cli[:file] = v }
|
34
33
|
opt.on(
|
35
34
|
'-q', '--query',
|
36
35
|
'Register the dataset as a query, not a reference dataset'
|
37
|
-
|
38
|
-
opt.on(
|
36
|
+
) { |v| cli[:query] = v }
|
37
|
+
opt.on(
|
38
|
+
'--ignore-dup',
|
39
39
|
'Ignore datasets that already exist'
|
40
|
-
|
40
|
+
) { |v| cli[:ignore_dup] = v }
|
41
41
|
opt.on(
|
42
42
|
'-d', '--description STRING',
|
43
43
|
'Description of the dataset'
|
44
|
-
|
44
|
+
) { |v| cli[:description] = v }
|
45
45
|
opt.on(
|
46
46
|
'-c', '--comments STRING',
|
47
47
|
'Comments on the dataset'
|
48
|
-
|
48
|
+
) { |v| cli[:comments] = v }
|
49
49
|
opt.on(
|
50
50
|
'-m', '--metadata STRING',
|
51
51
|
'Metadata as key-value pairs separated by = and delimited by comma',
|
52
52
|
'Values are saved as strings except for booleans (true / false) or nil'
|
53
|
-
|
53
|
+
) { |v| cli[:metadata] = v }
|
54
54
|
opt.on(
|
55
55
|
'--get-metadata',
|
56
56
|
'Only download and update metadata for existing datasets'
|
57
|
-
|
57
|
+
) { |v| cli[:get_md] = v }
|
58
58
|
opt.on(
|
59
59
|
'--only-metadata',
|
60
60
|
'Create datasets without input data but retrieve all metadata'
|
61
|
-
|
61
|
+
) { |v| cli[:only_md] = v }
|
62
62
|
opt.on(
|
63
63
|
'--api-key STRING',
|
64
64
|
'API key for the given universe'
|
65
|
-
|
65
|
+
) { |v| cli[:api_key] = v }
|
66
66
|
end
|
67
67
|
end
|
68
68
|
|
@@ -72,6 +72,7 @@ class MiGA::Cli::Action::Get < MiGA::Cli::Action
|
|
72
72
|
glob.each do |sub_cli|
|
73
73
|
rd = create_remote_dataset(sub_cli)
|
74
74
|
next if rd.nil?
|
75
|
+
|
75
76
|
if sub_cli[:get_md]
|
76
77
|
update_metadata(sub_cli, p, rd)
|
77
78
|
else
|
@@ -84,13 +85,14 @@ class MiGA::Cli::Action::Get < MiGA::Cli::Action
|
|
84
85
|
|
85
86
|
def get_sub_cli
|
86
87
|
return [cli] if cli[:file].nil?
|
88
|
+
|
87
89
|
glob = []
|
88
90
|
File.open(cli[:file], 'r') do |fh|
|
89
91
|
h = nil
|
90
92
|
fh.each do |ln|
|
91
93
|
r = ln.chomp.split(/\t/)
|
92
94
|
if h.nil?
|
93
|
-
|
95
|
+
h = r
|
94
96
|
else
|
95
97
|
argv_i = [self.name]
|
96
98
|
h.each_with_index do |field, k|
|
@@ -132,6 +134,7 @@ class MiGA::Cli::Action::Get < MiGA::Cli::Action
|
|
132
134
|
sub_cli.say 'Updating dataset'
|
133
135
|
d = p.dataset(sub_cli[:dataset])
|
134
136
|
return if d.nil?
|
137
|
+
|
135
138
|
md = sub_cli.add_metadata(d).metadata.data
|
136
139
|
rd.update_metadata(d, md)
|
137
140
|
end
|
@@ -7,7 +7,6 @@ require 'digest/md5'
|
|
7
7
|
require 'open-uri'
|
8
8
|
|
9
9
|
class MiGA::Cli::Action::GetDb < MiGA::Cli::Action
|
10
|
-
|
11
10
|
def parse_cli
|
12
11
|
cli.defaults = {
|
13
12
|
database: :recommended,
|
@@ -81,6 +80,7 @@ class MiGA::Cli::Action::GetDb < MiGA::Cli::Action
|
|
81
80
|
cli.say "Connecting to '#{cli[:host]}'"
|
82
81
|
uri = URI.parse(cli[:host])
|
83
82
|
raise 'Only FTP hosts are supported' unless uri.scheme == 'ftp'
|
83
|
+
|
84
84
|
ftp = Net::FTP.new(uri.host)
|
85
85
|
ftp.passive = true
|
86
86
|
ftp.login
|
@@ -113,11 +113,13 @@ class MiGA::Cli::Action::GetDb < MiGA::Cli::Action
|
|
113
113
|
[:recommended, :test].each do |n|
|
114
114
|
if cli[:database] == n
|
115
115
|
raise "This host has no #{n} database" if manif[n].nil?
|
116
|
+
|
116
117
|
cli[:database] = manif[n].to_sym
|
117
118
|
end
|
118
119
|
end
|
119
120
|
db = manif[:databases][cli[:database]]
|
120
121
|
raise 'Cannot find database in this host' if db.nil?
|
122
|
+
|
121
123
|
db
|
122
124
|
end
|
123
125
|
|
@@ -127,12 +129,14 @@ class MiGA::Cli::Action::GetDb < MiGA::Cli::Action
|
|
127
129
|
end
|
128
130
|
ver = db[:versions][cli[:version]]
|
129
131
|
raise 'Cannot find database version' if ver.nil?
|
132
|
+
|
130
133
|
cli.puts "# Database size: #{version_size(ver)}"
|
131
134
|
ver
|
132
135
|
end
|
133
136
|
|
134
137
|
def list_databases(manif)
|
135
138
|
return false unless cli[:list_databases]
|
139
|
+
|
136
140
|
cli.puts "# Recommended database: #{manif[:recommended]}"
|
137
141
|
cli.puts ''
|
138
142
|
cli.table(
|
@@ -146,6 +150,7 @@ class MiGA::Cli::Action::GetDb < MiGA::Cli::Action
|
|
146
150
|
|
147
151
|
def list_versions(db)
|
148
152
|
return false unless cli[:list_versions]
|
153
|
+
|
149
154
|
cli.puts "# Database: #{cli[:database]}"
|
150
155
|
cli.puts ''
|
151
156
|
cli.table(
|
@@ -159,6 +164,7 @@ class MiGA::Cli::Action::GetDb < MiGA::Cli::Action
|
|
159
164
|
|
160
165
|
def check_target
|
161
166
|
return false if cli[:overwrite]
|
167
|
+
|
162
168
|
file = File.expand_path(cli[:database], cli[:local])
|
163
169
|
if Dir.exist? file
|
164
170
|
warn "The target directory already exists: #{file}"
|
@@ -201,7 +207,7 @@ class MiGA::Cli::Action::GetDb < MiGA::Cli::Action
|
|
201
207
|
reg[:databases][cli[:database]] ||= {}
|
202
208
|
reg[:databases][cli[:database]][:manif_last_update] = manif[:last_update]
|
203
209
|
reg[:databases][cli[:database]][:manif_host] = manif[:host]
|
204
|
-
db.each { |k,v| reg[:databases][cli[:database]][k] = v }
|
210
|
+
db.each { |k, v| reg[:databases][cli[:database]][k] = v }
|
205
211
|
reg[:databases][cli[:database]][:local_version] = ver
|
206
212
|
MiGA::Json.generate(reg, local_manif)
|
207
213
|
end
|
@@ -17,7 +17,7 @@ class MiGA::Cli::Action::IndexWf < MiGA::Cli::Action
|
|
17
17
|
) { |v| cli[:mytaxa] = v }
|
18
18
|
opts_for_wf_distances(opt)
|
19
19
|
opts_for_wf(opt, 'Input genome assemblies (nucleotides, FastA)',
|
20
|
-
|
20
|
+
cleanup: false, project_type: true)
|
21
21
|
end
|
22
22
|
end
|
23
23
|
|
data/lib/miga/cli/action/init.rb
CHANGED
@@ -7,44 +7,44 @@ require 'shellwords'
|
|
7
7
|
class MiGA::Cli::Action::Init < MiGA::Cli::Action
|
8
8
|
require 'miga/cli/action/init/daemon_helper'
|
9
9
|
include MiGA::Cli::Action::Init::DaemonHelper
|
10
|
-
|
10
|
+
|
11
11
|
def parse_cli
|
12
12
|
cli.interactive = true
|
13
13
|
cli.defaults = { mytaxa: nil,
|
14
|
-
|
15
|
-
|
14
|
+
config: File.expand_path('.miga_modules', ENV['HOME']),
|
15
|
+
ask: false, auto: false, dtype: :bash }
|
16
16
|
cli.parse do |opt|
|
17
17
|
opt.on(
|
18
18
|
'-c', '--config PATH',
|
19
19
|
'Path to the Bash configuration file',
|
20
20
|
"By default: #{cli[:config]}"
|
21
|
-
){ |v| cli[:config] = v }
|
21
|
+
) { |v| cli[:config] = v }
|
22
22
|
opt.on(
|
23
23
|
'--[no-]mytaxa',
|
24
24
|
'Should I try setting up MyTaxa its dependencies?',
|
25
25
|
'By default: interactive (true if --auto)'
|
26
|
-
){ |v| cli[:mytaxa] = v }
|
26
|
+
) { |v| cli[:mytaxa] = v }
|
27
27
|
opt.on(
|
28
28
|
'--daemon-type STRING',
|
29
29
|
'Type of daemon launcher, one of: bash, ssh, qsub, msub, slurm',
|
30
30
|
"By default: interactive (#{cli[:dtype]} if --auto)"
|
31
|
-
){ |v| cli[:dtype] = v.to_sym }
|
31
|
+
) { |v| cli[:dtype] = v.to_sym }
|
32
32
|
opt.on(
|
33
33
|
'--ask-all',
|
34
34
|
'Ask for the location of all software',
|
35
35
|
'By default, only the locations missing in PATH are requested'
|
36
|
-
){ |v| cli[:ask] = v }
|
36
|
+
) { |v| cli[:ask] = v }
|
37
37
|
end
|
38
38
|
end
|
39
39
|
|
40
40
|
def perform
|
41
|
-
cli.puts
|
42
|
-
===[ Welcome to MiGA, the Microbial Genome Atlas ]===
|
41
|
+
cli.puts <<~BANNER
|
42
|
+
===[ Welcome to MiGA, the Microbial Genome Atlas ]===
|
43
43
|
|
44
|
-
I'm the initialization script, and I'll sniff around your computer to
|
45
|
-
make sure you have all the requirements for MiGA data processing.
|
44
|
+
I'm the initialization script, and I'll sniff around your computer to
|
45
|
+
make sure you have all the requirements for MiGA data processing.
|
46
46
|
|
47
|
-
BANNER
|
47
|
+
BANNER
|
48
48
|
list_requirements
|
49
49
|
rc_fh = open_rc_file
|
50
50
|
check_configuration_script rc_fh
|
@@ -66,8 +66,10 @@ BANNER
|
|
66
66
|
end
|
67
67
|
|
68
68
|
def run_r_cmd(cli, paths, cmd)
|
69
|
-
run_cmd(
|
70
|
-
|
69
|
+
run_cmd(
|
70
|
+
cli,
|
71
|
+
"echo #{cmd.shellescape} | #{paths['R'].shellescape} --vanilla -q 2>&1"
|
72
|
+
)
|
71
73
|
end
|
72
74
|
|
73
75
|
def test_r_package(cli, paths, pkg)
|
@@ -81,24 +83,30 @@ BANNER
|
|
81
83
|
end
|
82
84
|
|
83
85
|
def test_ruby_gem(cli, paths, pkg)
|
84
|
-
run_cmd(
|
85
|
-
|
86
|
+
run_cmd(
|
87
|
+
cli,
|
88
|
+
"#{paths['ruby'].shellescape} -r #{pkg.shellescape} -e '' 2>/dev/null"
|
89
|
+
)
|
86
90
|
$?.success?
|
87
91
|
end
|
88
92
|
|
89
93
|
def install_ruby_gem(cli, paths, pkg)
|
90
94
|
gem_cmd = "Gem::GemRunner.new.run %w(install --user #{pkg})"
|
91
|
-
run_cmd(
|
95
|
+
run_cmd(
|
96
|
+
cli,
|
97
|
+
"#{paths['ruby'].shellescape} \
|
92
98
|
-r rubygems -r rubygems/gem_runner \
|
93
|
-
-e #{gem_cmd.shellescape} 2>&1"
|
99
|
+
-e #{gem_cmd.shellescape} 2>&1"
|
100
|
+
)
|
94
101
|
end
|
95
102
|
|
96
103
|
def list_requirements
|
97
104
|
if cli.ask_user(
|
98
|
-
|
99
|
-
|
105
|
+
'Would you like to see all the requirements before starting?',
|
106
|
+
'no', %w(yes no)
|
107
|
+
) == 'yes'
|
100
108
|
cli.puts ''
|
101
|
-
req_path = File.
|
109
|
+
req_path = File.join(MiGA.root_path, 'utils', 'requirements.txt')
|
102
110
|
File.open(req_path, 'r') do |fh|
|
103
111
|
fh.each_line { |ln| cli.puts ln }
|
104
112
|
end
|
@@ -112,26 +120,28 @@ BANNER
|
|
112
120
|
rc_path = File.expand_path('.miga_rc', ENV['HOME'])
|
113
121
|
if File.exist? rc_path
|
114
122
|
if cli.ask_user(
|
115
|
-
|
116
|
-
|
123
|
+
'I found a previous configuration. Do you want to continue?',
|
124
|
+
'yes', %w(yes no)
|
125
|
+
) == 'no'
|
117
126
|
cli.puts 'OK, see you soon!'
|
118
127
|
exit(0)
|
119
128
|
end
|
120
129
|
end
|
121
130
|
rc_fh = File.open(rc_path, 'w')
|
122
|
-
rc_fh.puts
|
123
|
-
#!/bin/bash
|
124
|
-
# `miga init` made this on #{Time.now}
|
131
|
+
rc_fh.puts <<~BASH
|
132
|
+
#!/bin/bash
|
133
|
+
# `miga init` made this on #{Time.now}
|
125
134
|
|
126
|
-
BASH
|
135
|
+
BASH
|
127
136
|
rc_fh
|
128
137
|
end
|
129
138
|
|
130
139
|
def check_configuration_script(rc_fh)
|
131
140
|
unless File.exist? cli[:config]
|
132
141
|
cli[:config] = cli.ask_user(
|
133
|
-
|
134
|
-
|
142
|
+
'Is there a script I need to load at startup?',
|
143
|
+
cli[:config]
|
144
|
+
)
|
135
145
|
end
|
136
146
|
if File.exist? cli[:config]
|
137
147
|
cli[:config] = File.expand_path(cli[:config])
|
@@ -154,8 +164,10 @@ BASH
|
|
154
164
|
File.open(req_path, 'r') do |fh|
|
155
165
|
fh.each_line do |ln|
|
156
166
|
next if $. < 3
|
167
|
+
|
157
168
|
r = ln.chomp.split(/\t+/)
|
158
169
|
next if r[0] =~ /\(opt\)$/ && !cli[:mytaxa]
|
170
|
+
|
159
171
|
cli.print "Testing #{r[0]}#{" (#{r[3]})" if r[3]}... "
|
160
172
|
path = find_software(r[1])
|
161
173
|
paths[r[1]] = File.expand_path(r[1], path).shellescape
|
@@ -200,18 +212,18 @@ BASH
|
|
200
212
|
def check_additional_files(paths)
|
201
213
|
if cli[:mytaxa]
|
202
214
|
cli.puts 'Looking for MyTaxa databases:'
|
203
|
-
mt = File.dirname paths[
|
215
|
+
mt = File.dirname paths['MyTaxa']
|
204
216
|
cli.print 'Looking for scores... '
|
205
217
|
unless Dir.exist?(File.expand_path('db', mt))
|
206
|
-
cli.puts "no
|
218
|
+
cli.puts "no\nExecute 'python2 #{mt}/utils/download_db.py'"
|
207
219
|
exit(1)
|
208
220
|
end
|
209
|
-
cli.puts 'yes
|
221
|
+
cli.puts 'yes'
|
210
222
|
cli.print 'Looking for diamond db... '
|
211
223
|
unless File.exist?(File.expand_path('AllGenomes.faa.dmnd', mt))
|
212
|
-
cli.puts "no
|
224
|
+
cli.puts "no\nDownload " \
|
213
225
|
"'http://enve-omics.ce.gatech.edu/data/public_mytaxa/" \
|
214
|
-
"AllGenomes.faa.dmnd' into #{mt}
|
226
|
+
"AllGenomes.faa.dmnd' into #{mt}"
|
215
227
|
exit(1)
|
216
228
|
end
|
217
229
|
cli.puts ''
|
@@ -223,7 +235,7 @@ BASH
|
|
223
235
|
%w(ape cluster vegan).each do |pkg|
|
224
236
|
cli.print "Testing #{pkg}... "
|
225
237
|
if test_r_package(cli, paths, pkg)
|
226
|
-
cli.puts 'yes
|
238
|
+
cli.puts 'yes'
|
227
239
|
else
|
228
240
|
cli.puts 'no, installing'
|
229
241
|
cli.print '' + install_r_package(cli, paths, pkg)
|
@@ -240,7 +252,7 @@ BASH
|
|
240
252
|
%w(sqlite3 daemons json).each do |pkg|
|
241
253
|
cli.print "Testing #{pkg}... "
|
242
254
|
if test_ruby_gem(cli, paths, pkg)
|
243
|
-
cli.puts 'yes
|
255
|
+
cli.puts 'yes'
|
244
256
|
else
|
245
257
|
cli.puts 'no, installing'
|
246
258
|
# This hackey mess is meant to ensure the test and installation are done
|
@@ -256,13 +268,13 @@ BASH
|
|
256
268
|
end
|
257
269
|
|
258
270
|
def close_rc_file(rc_fh)
|
259
|
-
rc_fh.puts
|
271
|
+
rc_fh.puts <<~FOOT
|
260
272
|
|
261
|
-
MIGA_CONFIG_VERSION='#{MiGA::MiGA.VERSION}'
|
262
|
-
MIGA_CONFIG_LONGVERSION='#{MiGA::MiGA.LONG_VERSION}'
|
263
|
-
MIGA_CONFIG_DATE='#{Time.now}'
|
273
|
+
MIGA_CONFIG_VERSION='#{MiGA::MiGA.VERSION}'
|
274
|
+
MIGA_CONFIG_LONGVERSION='#{MiGA::MiGA.LONG_VERSION}'
|
275
|
+
MIGA_CONFIG_DATE='#{Time.now}'
|
264
276
|
|
265
|
-
FOOT
|
277
|
+
FOOT
|
266
278
|
rc_fh.close
|
267
279
|
end
|
268
280
|
end
|