miga-base 0.7.3.0 → 0.7.7.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/miga/cli.rb +10 -8
- data/lib/miga/cli/action.rb +2 -3
- data/lib/miga/cli/action/about.rb +5 -6
- data/lib/miga/cli/action/add.rb +18 -12
- data/lib/miga/cli/action/add_result.rb +2 -3
- data/lib/miga/cli/action/archive.rb +1 -2
- data/lib/miga/cli/action/classify_wf.rb +8 -6
- data/lib/miga/cli/action/console.rb +0 -1
- data/lib/miga/cli/action/daemon.rb +7 -7
- data/lib/miga/cli/action/date.rb +0 -1
- data/lib/miga/cli/action/derep_wf.rb +5 -4
- data/lib/miga/cli/action/doctor.rb +71 -82
- data/lib/miga/cli/action/doctor/base.rb +102 -0
- data/lib/miga/cli/action/edit.rb +14 -2
- data/lib/miga/cli/action/files.rb +8 -8
- data/lib/miga/cli/action/find.rb +5 -6
- data/lib/miga/cli/action/generic.rb +7 -7
- data/lib/miga/cli/action/get.rb +20 -17
- data/lib/miga/cli/action/get_db.rb +8 -2
- data/lib/miga/cli/action/index_wf.rb +1 -1
- data/lib/miga/cli/action/init.rb +53 -41
- data/lib/miga/cli/action/init/daemon_helper.rb +65 -43
- data/lib/miga/cli/action/lair.rb +7 -7
- data/lib/miga/cli/action/ln.rb +6 -6
- data/lib/miga/cli/action/ls.rb +1 -2
- data/lib/miga/cli/action/ncbi_get.rb +11 -3
- data/lib/miga/cli/action/new.rb +4 -4
- data/lib/miga/cli/action/next_step.rb +0 -1
- data/lib/miga/cli/action/preproc_wf.rb +3 -3
- data/lib/miga/cli/action/quality_wf.rb +1 -1
- data/lib/miga/cli/action/rm.rb +2 -3
- data/lib/miga/cli/action/run.rb +8 -8
- data/lib/miga/cli/action/stats.rb +8 -4
- data/lib/miga/cli/action/summary.rb +7 -6
- data/lib/miga/cli/action/tax_dist.rb +8 -4
- data/lib/miga/cli/action/tax_index.rb +3 -4
- data/lib/miga/cli/action/tax_set.rb +7 -6
- data/lib/miga/cli/action/tax_test.rb +6 -5
- data/lib/miga/cli/action/wf.rb +21 -19
- data/lib/miga/cli/base.rb +34 -32
- data/lib/miga/cli/objects_helper.rb +27 -18
- data/lib/miga/cli/opt_helper.rb +3 -2
- data/lib/miga/common.rb +2 -5
- data/lib/miga/common/base.rb +15 -16
- data/lib/miga/common/format.rb +8 -5
- data/lib/miga/common/hooks.rb +1 -4
- data/lib/miga/common/path.rb +4 -9
- data/lib/miga/common/with_daemon.rb +6 -3
- data/lib/miga/common/with_daemon_class.rb +3 -2
- data/lib/miga/common/with_result.rb +2 -1
- data/lib/miga/daemon.rb +93 -44
- data/lib/miga/daemon/base.rb +30 -11
- data/lib/miga/dataset.rb +47 -37
- data/lib/miga/dataset/base.rb +52 -37
- data/lib/miga/dataset/hooks.rb +3 -4
- data/lib/miga/dataset/result.rb +17 -1
- data/lib/miga/dataset/status.rb +6 -5
- data/lib/miga/json.rb +5 -7
- data/lib/miga/lair.rb +4 -0
- data/lib/miga/metadata.rb +4 -3
- data/lib/miga/project.rb +29 -20
- data/lib/miga/project/base.rb +52 -37
- data/lib/miga/project/dataset.rb +33 -26
- data/lib/miga/project/hooks.rb +0 -3
- data/lib/miga/project/result.rb +14 -5
- data/lib/miga/remote_dataset.rb +85 -72
- data/lib/miga/remote_dataset/base.rb +11 -13
- data/lib/miga/remote_dataset/download.rb +34 -12
- data/lib/miga/result.rb +34 -25
- data/lib/miga/result/base.rb +0 -2
- data/lib/miga/result/dates.rb +1 -3
- data/lib/miga/result/source.rb +15 -16
- data/lib/miga/result/stats.rb +37 -27
- data/lib/miga/tax_dist.rb +6 -4
- data/lib/miga/tax_index.rb +17 -17
- data/lib/miga/taxonomy.rb +6 -1
- data/lib/miga/taxonomy/base.rb +19 -15
- data/lib/miga/version.rb +19 -16
- data/scripts/project_stats.bash +3 -0
- data/scripts/stats.bash +1 -1
- data/test/common_test.rb +3 -11
- data/test/daemon_helper.rb +38 -0
- data/test/daemon_test.rb +91 -99
- data/test/dataset_test.rb +63 -59
- data/test/format_test.rb +3 -11
- data/test/hook_test.rb +50 -55
- data/test/json_test.rb +7 -8
- data/test/lair_test.rb +22 -28
- data/test/metadata_test.rb +6 -14
- data/test/project_test.rb +33 -40
- data/test/remote_dataset_test.rb +26 -32
- data/test/result_stats_test.rb +17 -27
- data/test/result_test.rb +41 -34
- data/test/tax_dist_test.rb +2 -4
- data/test/tax_index_test.rb +4 -10
- data/test/taxonomy_test.rb +7 -9
- data/test/test_helper.rb +42 -1
- data/test/with_daemon_test.rb +14 -22
- data/utils/adapters.fa +13 -0
- data/utils/cleanup-databases.rb +6 -5
- data/utils/distance/base.rb +0 -1
- data/utils/distance/commands.rb +19 -12
- data/utils/distance/database.rb +25 -21
- data/utils/distance/pipeline.rb +16 -10
- data/utils/distance/runner.rb +19 -13
- data/utils/distance/temporal.rb +7 -4
- data/utils/distances.rb +1 -1
- data/utils/domain-ess-genes.rb +7 -7
- data/utils/index_metadata.rb +5 -4
- data/utils/mytaxa_scan.rb +18 -16
- data/utils/representatives.rb +5 -4
- data/utils/requirements.txt +1 -1
- data/utils/subclade/base.rb +0 -1
- data/utils/subclade/pipeline.rb +7 -6
- data/utils/subclade/runner.rb +9 -9
- data/utils/subclade/temporal.rb +0 -2
- data/utils/subclades-compile.rb +39 -37
- data/utils/subclades.rb +1 -1
- metadata +6 -4
@@ -0,0 +1,102 @@
|
|
1
|
+
require 'miga/cli/action'
|
2
|
+
require 'sqlite3'
|
3
|
+
|
4
|
+
class MiGA::Cli::Action::Doctor < MiGA::Cli::Action
|
5
|
+
end
|
6
|
+
|
7
|
+
module MiGA::Cli::Action::Doctor::Base
|
8
|
+
##
|
9
|
+
# Check the database in +db_file+ maintains integrity for the
|
10
|
+
# tables saving +metric+ (:ani or :aai) and call +blk+ if the
|
11
|
+
# file is corrupt or doesn't contain the expected structure
|
12
|
+
def check_sqlite3_database(db_file, metric, &blk)
|
13
|
+
SQLite3::Database.new(db_file) do |conn|
|
14
|
+
conn.execute("select count(*) from #{metric}").first
|
15
|
+
end
|
16
|
+
rescue SQLite3::SQLException, SQLite3::CorruptException
|
17
|
+
blk.call
|
18
|
+
end
|
19
|
+
|
20
|
+
def each_database_file(dataset, &blk)
|
21
|
+
ref_db = { '01.haai' => :aai, '02.aai' => :aai, '03.ani' => :ani }
|
22
|
+
qry_db = { '.haai.db' => :aai, '.aai.db' => :aai, '.ani.db' => :ani }
|
23
|
+
base = File.join(dataset.project.path, 'data', '09.distances')
|
24
|
+
result = :distances
|
25
|
+
if dataset.ref?
|
26
|
+
file_db = "#{dataset.name}.db"
|
27
|
+
ref_db.each do |dir, metric|
|
28
|
+
file = File.join(base, dir, file_db)
|
29
|
+
blk[file, metric, result] if File.exist? file
|
30
|
+
end
|
31
|
+
base = File.join(base, '05.taxonomy')
|
32
|
+
result = :taxonomy
|
33
|
+
end
|
34
|
+
qry_db.each do |ext, metric|
|
35
|
+
file = File.join(base, "#{dataset.name}#{ext}")
|
36
|
+
blk[file, metric, result] if File.exist? file
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
##
|
41
|
+
# Scans the all-vs-all matrix registered in +res+ (MiGA::Result) in search of
|
42
|
+
# pairs where one or both datasets are missing or inactive in the project +p+
|
43
|
+
# (MiGA::Project), and report progress through +cli+ (MiGA::Cli).
|
44
|
+
# Returns an Array with two arrays: the first a list of dataset names that are
|
45
|
+
# no longer registered in the project or are currently inactive, and the
|
46
|
+
# second a list of dataset names that have registered pairs with the first
|
47
|
+
# list, and therefore the databases need to be cleaned.
|
48
|
+
# This is a subtask of +check_dist+
|
49
|
+
def check_dist_eval(cli, p, res)
|
50
|
+
notok = {}
|
51
|
+
fix = {}
|
52
|
+
Zlib::GzipReader.open(res.file_path(:matrix)) do |fh|
|
53
|
+
lineno = 0
|
54
|
+
fh.each_line do |ln|
|
55
|
+
next if (lineno += 1) == 1
|
56
|
+
|
57
|
+
r = ln.split("\t")
|
58
|
+
next unless [1, 2].map { |i| p.dataset(r[i]).nil? }.any?
|
59
|
+
|
60
|
+
[1, 2].each do |i|
|
61
|
+
if p.dataset(r[i]).nil? || !p.dataset(r[i]).active?
|
62
|
+
notok[r[i]] = true
|
63
|
+
else
|
64
|
+
fix[r[i]] = true
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
68
|
+
end
|
69
|
+
[notok.keys, fix.keys]
|
70
|
+
end
|
71
|
+
|
72
|
+
##
|
73
|
+
# Cleanup distance databases for datasets names in +fix+ (Array: String)
|
74
|
+
# from project +p+ (MiGA::Project), and report through +cli+ (MiGA::Cli).
|
75
|
+
# This is a subtask of +check_dist+
|
76
|
+
def check_dist_fix(cli, p, fix)
|
77
|
+
return if fix.empty?
|
78
|
+
|
79
|
+
cli.say("- Fixing #{fix.size} datasets")
|
80
|
+
fix.each do |d_n|
|
81
|
+
cli.say " > Fixing #{d_n}."
|
82
|
+
p.dataset(d_n).cleanup_distances!
|
83
|
+
end
|
84
|
+
end
|
85
|
+
|
86
|
+
##
|
87
|
+
# Recompute +res+ (MiGA::Result) if +notok+ (Array: String) has any dataset
|
88
|
+
# names registered, and report through +cli+ (MiGA::Cli).
|
89
|
+
# This is a subtask of +check_dist+
|
90
|
+
def check_dist_recompute(cli, res, notok)
|
91
|
+
return if notok.empty?
|
92
|
+
|
93
|
+
cli.say '- Unregistered datasets detected: '
|
94
|
+
if notok.size <= 5
|
95
|
+
notok.each { |i| cli.say " > #{i}" }
|
96
|
+
else
|
97
|
+
cli.say " > #{notok.size}, including #{notok.first}"
|
98
|
+
end
|
99
|
+
cli.say '- Removing tables, recompute'
|
100
|
+
res.remove!
|
101
|
+
end
|
102
|
+
end
|
data/lib/miga/cli/action/edit.rb
CHANGED
@@ -4,7 +4,6 @@
|
|
4
4
|
require 'miga/cli/action'
|
5
5
|
|
6
6
|
class MiGA::Cli::Action::Edit < MiGA::Cli::Action
|
7
|
-
|
8
7
|
def parse_cli
|
9
8
|
cli.parse do |opt|
|
10
9
|
cli.opt_object(opt, [:project, :dataset_opt])
|
@@ -12,12 +11,25 @@ class MiGA::Cli::Action::Edit < MiGA::Cli::Action
|
|
12
11
|
'-m', '--metadata STRING',
|
13
12
|
'Metadata as key-value pairs separated by = and delimited by comma',
|
14
13
|
'Values are saved as strings except for booleans (true / false) or nil'
|
15
|
-
|
14
|
+
) { |v| cli[:metadata] = v }
|
15
|
+
opt.on(
|
16
|
+
'--activate',
|
17
|
+
'Activate dataset; requires -D'
|
18
|
+
) { |v| cli[:activate] = v }
|
19
|
+
opt.on(
|
20
|
+
'--inactivate',
|
21
|
+
'Inactivate dataset; requires -D'
|
22
|
+
) { |v| cli[:activate] = !v }
|
16
23
|
end
|
17
24
|
end
|
18
25
|
|
19
26
|
def perform
|
20
27
|
obj = cli.load_project_or_dataset
|
28
|
+
unless cli[:activate].nil?
|
29
|
+
cli.ensure_par({ dataset: '-D' },
|
30
|
+
'%<name>s is mandatory with --[in-]activate: please provide %<flag>s')
|
31
|
+
cli[:activate] ? obj.activate! : obj.inactivate!
|
32
|
+
end
|
21
33
|
cli.add_metadata(obj)
|
22
34
|
obj.save
|
23
35
|
end
|
@@ -4,27 +4,27 @@
|
|
4
4
|
require 'miga/cli/action'
|
5
5
|
|
6
6
|
class MiGA::Cli::Action::Files < MiGA::Cli::Action
|
7
|
-
|
8
7
|
def parse_cli
|
9
|
-
cli.defaults = {details: false, json: true}
|
8
|
+
cli.defaults = { details: false, json: true }
|
10
9
|
cli.parse do |opt|
|
11
10
|
cli.opt_object(opt, [:project, :dataset_opt])
|
12
11
|
opt.on(
|
13
12
|
'-i', '--info',
|
14
13
|
'Print additional details for each file'
|
15
|
-
|
16
|
-
opt.on(
|
14
|
+
) { |v| cli[:details] = v }
|
15
|
+
opt.on(
|
16
|
+
'--[no-]json',
|
17
17
|
'Include (or not) JSON files containing results metadata',
|
18
18
|
'JSON files are included by default'
|
19
|
-
|
19
|
+
) { |v| cli[:json] = v }
|
20
20
|
end
|
21
21
|
end
|
22
22
|
|
23
23
|
def perform
|
24
24
|
cli.load_project_or_dataset.each_result do |sym, res|
|
25
|
-
cli.puts "#{
|
26
|
-
res.each_file do |k,f|
|
27
|
-
cli.puts "#{
|
25
|
+
cli.puts "#{"#{sym}\tjson\t" if cli[:details]}#{res.path}" if cli[:json]
|
26
|
+
res.each_file do |k, f|
|
27
|
+
cli.puts "#{"#{sym}\t#{k}\t" if cli[:details]}#{res.dir}/#{f}"
|
28
28
|
end
|
29
29
|
end
|
30
30
|
end
|
data/lib/miga/cli/action/find.rb
CHANGED
@@ -4,29 +4,28 @@
|
|
4
4
|
require 'miga/cli/action'
|
5
5
|
|
6
6
|
class MiGA::Cli::Action::Find < MiGA::Cli::Action
|
7
|
-
|
8
7
|
def parse_cli
|
9
|
-
cli.defaults = {add: false, ref: true}
|
8
|
+
cli.defaults = { add: false, ref: true }
|
10
9
|
cli.parse do |opt|
|
11
10
|
cli.opt_object(opt, [:project, :dataset_type])
|
12
11
|
opt.on(
|
13
12
|
'-a', '--add',
|
14
13
|
'Register the datasets found',
|
15
14
|
'By default, only lists them (dry run)'
|
16
|
-
|
15
|
+
) { |v| cli[:add] = v }
|
17
16
|
opt.on(
|
18
17
|
'-q', '--query',
|
19
18
|
'Register datasets as query'
|
20
|
-
|
19
|
+
) { |v| cli[:ref] = !v }
|
21
20
|
opt.on(
|
22
21
|
'-u', '--user STRING',
|
23
22
|
'Owner of the dataset.'
|
24
|
-
|
23
|
+
) { |v| cli[:user] = v }
|
25
24
|
opt.on(
|
26
25
|
'-m', '--metadata STRING',
|
27
26
|
'Metadata as key-value pairs separated by = and delimited by comma',
|
28
27
|
'Values are saved as strings except for booleans (true / false) or nil'
|
29
|
-
|
28
|
+
) { |v| cli[:metadata] = v }
|
30
29
|
end
|
31
30
|
end
|
32
31
|
|
@@ -4,31 +4,31 @@
|
|
4
4
|
require 'miga/cli/action'
|
5
5
|
|
6
6
|
class MiGA::Cli::Action::Generic < MiGA::Cli::Action
|
7
|
-
|
8
7
|
def parse_cli
|
9
8
|
cli.opt_common = false
|
10
9
|
cli.parse do |opt|
|
11
|
-
descriptions = cli.class.TASK_DESC.keep_if { |k,v| k != :generic }
|
10
|
+
descriptions = cli.class.TASK_DESC.keep_if { |k, v| k != :generic }
|
12
11
|
opt.separator MiGA::MiGA.tabulate(
|
13
|
-
[:action, :description], descriptions
|
12
|
+
[:action, :description], descriptions
|
13
|
+
).join("\n")
|
14
14
|
opt.separator ''
|
15
15
|
opt.separator 'generic options:'
|
16
16
|
opt.on(
|
17
17
|
'-h', '--help',
|
18
18
|
'Display this screen'
|
19
|
-
|
19
|
+
) { puts opt; exit }
|
20
20
|
opt.on(
|
21
21
|
'-v', '--version',
|
22
22
|
'Show MiGA version'
|
23
|
-
|
23
|
+
) { puts MiGA::MiGA.VERSION; exit }
|
24
24
|
opt.on(
|
25
25
|
'-V', '--long-version',
|
26
26
|
'Show complete MiGA version'
|
27
|
-
|
27
|
+
) { |v| puts MiGA::MiGA.LONG_VERSION; exit }
|
28
28
|
opt.on(
|
29
29
|
'-C', '--citation',
|
30
30
|
'How to cite MiGA'
|
31
|
-
|
31
|
+
) { |v| puts MiGA::MiGA.CITATION; exit }
|
32
32
|
end
|
33
33
|
end
|
34
34
|
|
data/lib/miga/cli/action/get.rb
CHANGED
@@ -5,24 +5,23 @@ require 'miga/cli/action'
|
|
5
5
|
require 'miga/remote_dataset'
|
6
6
|
|
7
7
|
class MiGA::Cli::Action::Get < MiGA::Cli::Action
|
8
|
-
|
9
8
|
def parse_cli
|
10
|
-
cli.defaults = {query: false, universe: :ncbi, db: :nuccore,
|
11
|
-
|
9
|
+
cli.defaults = { query: false, universe: :ncbi, db: :nuccore,
|
10
|
+
get_md: false, only_md: false }
|
12
11
|
cli.parse do |opt|
|
13
12
|
cli.opt_object(opt, [:project, :dataset, :dataset_type])
|
14
13
|
opt.on(
|
15
14
|
'-I', '--ids ID1,ID2,...', Array,
|
16
15
|
'(Mandatory unless -F) IDs in the remote database separated by commas'
|
17
|
-
|
16
|
+
) { |v| cli[:ids] = v }
|
18
17
|
opt.on(
|
19
18
|
'-U', '--universe STRING',
|
20
19
|
"Universe of the remote database. By default: #{cli[:universe]}"
|
21
|
-
|
20
|
+
) { |v| cli[:universe] = v.to_sym }
|
22
21
|
opt.on(
|
23
22
|
'--db STRING',
|
24
23
|
"Name of the remote database. By default: #{cli[:db]}"
|
25
|
-
|
24
|
+
) { |v| cli[:db] = v.to_sym }
|
26
25
|
opt.on(
|
27
26
|
'-F', '--file PATH',
|
28
27
|
'Tab-delimited file (with header) listing the datasets to download',
|
@@ -30,39 +29,40 @@ class MiGA::Cli::Action::Get < MiGA::Cli::Action
|
|
30
29
|
'including: dataset, ids, universe, db, metadata',
|
31
30
|
'For flags without value (like query) use true/false',
|
32
31
|
'Unsupported values are: project, file, verbose, help, and debug'
|
33
|
-
|
32
|
+
) { |v| cli[:file] = v }
|
34
33
|
opt.on(
|
35
34
|
'-q', '--query',
|
36
35
|
'Register the dataset as a query, not a reference dataset'
|
37
|
-
|
38
|
-
opt.on(
|
36
|
+
) { |v| cli[:query] = v }
|
37
|
+
opt.on(
|
38
|
+
'--ignore-dup',
|
39
39
|
'Ignore datasets that already exist'
|
40
|
-
|
40
|
+
) { |v| cli[:ignore_dup] = v }
|
41
41
|
opt.on(
|
42
42
|
'-d', '--description STRING',
|
43
43
|
'Description of the dataset'
|
44
|
-
|
44
|
+
) { |v| cli[:description] = v }
|
45
45
|
opt.on(
|
46
46
|
'-c', '--comments STRING',
|
47
47
|
'Comments on the dataset'
|
48
|
-
|
48
|
+
) { |v| cli[:comments] = v }
|
49
49
|
opt.on(
|
50
50
|
'-m', '--metadata STRING',
|
51
51
|
'Metadata as key-value pairs separated by = and delimited by comma',
|
52
52
|
'Values are saved as strings except for booleans (true / false) or nil'
|
53
|
-
|
53
|
+
) { |v| cli[:metadata] = v }
|
54
54
|
opt.on(
|
55
55
|
'--get-metadata',
|
56
56
|
'Only download and update metadata for existing datasets'
|
57
|
-
|
57
|
+
) { |v| cli[:get_md] = v }
|
58
58
|
opt.on(
|
59
59
|
'--only-metadata',
|
60
60
|
'Create datasets without input data but retrieve all metadata'
|
61
|
-
|
61
|
+
) { |v| cli[:only_md] = v }
|
62
62
|
opt.on(
|
63
63
|
'--api-key STRING',
|
64
64
|
'API key for the given universe'
|
65
|
-
|
65
|
+
) { |v| cli[:api_key] = v }
|
66
66
|
end
|
67
67
|
end
|
68
68
|
|
@@ -72,6 +72,7 @@ class MiGA::Cli::Action::Get < MiGA::Cli::Action
|
|
72
72
|
glob.each do |sub_cli|
|
73
73
|
rd = create_remote_dataset(sub_cli)
|
74
74
|
next if rd.nil?
|
75
|
+
|
75
76
|
if sub_cli[:get_md]
|
76
77
|
update_metadata(sub_cli, p, rd)
|
77
78
|
else
|
@@ -84,13 +85,14 @@ class MiGA::Cli::Action::Get < MiGA::Cli::Action
|
|
84
85
|
|
85
86
|
def get_sub_cli
|
86
87
|
return [cli] if cli[:file].nil?
|
88
|
+
|
87
89
|
glob = []
|
88
90
|
File.open(cli[:file], 'r') do |fh|
|
89
91
|
h = nil
|
90
92
|
fh.each do |ln|
|
91
93
|
r = ln.chomp.split(/\t/)
|
92
94
|
if h.nil?
|
93
|
-
|
95
|
+
h = r
|
94
96
|
else
|
95
97
|
argv_i = [self.name]
|
96
98
|
h.each_with_index do |field, k|
|
@@ -132,6 +134,7 @@ class MiGA::Cli::Action::Get < MiGA::Cli::Action
|
|
132
134
|
sub_cli.say 'Updating dataset'
|
133
135
|
d = p.dataset(sub_cli[:dataset])
|
134
136
|
return if d.nil?
|
137
|
+
|
135
138
|
md = sub_cli.add_metadata(d).metadata.data
|
136
139
|
rd.update_metadata(d, md)
|
137
140
|
end
|
@@ -7,7 +7,6 @@ require 'digest/md5'
|
|
7
7
|
require 'open-uri'
|
8
8
|
|
9
9
|
class MiGA::Cli::Action::GetDb < MiGA::Cli::Action
|
10
|
-
|
11
10
|
def parse_cli
|
12
11
|
cli.defaults = {
|
13
12
|
database: :recommended,
|
@@ -81,6 +80,7 @@ class MiGA::Cli::Action::GetDb < MiGA::Cli::Action
|
|
81
80
|
cli.say "Connecting to '#{cli[:host]}'"
|
82
81
|
uri = URI.parse(cli[:host])
|
83
82
|
raise 'Only FTP hosts are supported' unless uri.scheme == 'ftp'
|
83
|
+
|
84
84
|
ftp = Net::FTP.new(uri.host)
|
85
85
|
ftp.passive = true
|
86
86
|
ftp.login
|
@@ -113,11 +113,13 @@ class MiGA::Cli::Action::GetDb < MiGA::Cli::Action
|
|
113
113
|
[:recommended, :test].each do |n|
|
114
114
|
if cli[:database] == n
|
115
115
|
raise "This host has no #{n} database" if manif[n].nil?
|
116
|
+
|
116
117
|
cli[:database] = manif[n].to_sym
|
117
118
|
end
|
118
119
|
end
|
119
120
|
db = manif[:databases][cli[:database]]
|
120
121
|
raise 'Cannot find database in this host' if db.nil?
|
122
|
+
|
121
123
|
db
|
122
124
|
end
|
123
125
|
|
@@ -127,12 +129,14 @@ class MiGA::Cli::Action::GetDb < MiGA::Cli::Action
|
|
127
129
|
end
|
128
130
|
ver = db[:versions][cli[:version]]
|
129
131
|
raise 'Cannot find database version' if ver.nil?
|
132
|
+
|
130
133
|
cli.puts "# Database size: #{version_size(ver)}"
|
131
134
|
ver
|
132
135
|
end
|
133
136
|
|
134
137
|
def list_databases(manif)
|
135
138
|
return false unless cli[:list_databases]
|
139
|
+
|
136
140
|
cli.puts "# Recommended database: #{manif[:recommended]}"
|
137
141
|
cli.puts ''
|
138
142
|
cli.table(
|
@@ -146,6 +150,7 @@ class MiGA::Cli::Action::GetDb < MiGA::Cli::Action
|
|
146
150
|
|
147
151
|
def list_versions(db)
|
148
152
|
return false unless cli[:list_versions]
|
153
|
+
|
149
154
|
cli.puts "# Database: #{cli[:database]}"
|
150
155
|
cli.puts ''
|
151
156
|
cli.table(
|
@@ -159,6 +164,7 @@ class MiGA::Cli::Action::GetDb < MiGA::Cli::Action
|
|
159
164
|
|
160
165
|
def check_target
|
161
166
|
return false if cli[:overwrite]
|
167
|
+
|
162
168
|
file = File.expand_path(cli[:database], cli[:local])
|
163
169
|
if Dir.exist? file
|
164
170
|
warn "The target directory already exists: #{file}"
|
@@ -201,7 +207,7 @@ class MiGA::Cli::Action::GetDb < MiGA::Cli::Action
|
|
201
207
|
reg[:databases][cli[:database]] ||= {}
|
202
208
|
reg[:databases][cli[:database]][:manif_last_update] = manif[:last_update]
|
203
209
|
reg[:databases][cli[:database]][:manif_host] = manif[:host]
|
204
|
-
db.each { |k,v| reg[:databases][cli[:database]][k] = v }
|
210
|
+
db.each { |k, v| reg[:databases][cli[:database]][k] = v }
|
205
211
|
reg[:databases][cli[:database]][:local_version] = ver
|
206
212
|
MiGA::Json.generate(reg, local_manif)
|
207
213
|
end
|
@@ -17,7 +17,7 @@ class MiGA::Cli::Action::IndexWf < MiGA::Cli::Action
|
|
17
17
|
) { |v| cli[:mytaxa] = v }
|
18
18
|
opts_for_wf_distances(opt)
|
19
19
|
opts_for_wf(opt, 'Input genome assemblies (nucleotides, FastA)',
|
20
|
-
|
20
|
+
cleanup: false, project_type: true)
|
21
21
|
end
|
22
22
|
end
|
23
23
|
|
data/lib/miga/cli/action/init.rb
CHANGED
@@ -7,44 +7,44 @@ require 'shellwords'
|
|
7
7
|
class MiGA::Cli::Action::Init < MiGA::Cli::Action
|
8
8
|
require 'miga/cli/action/init/daemon_helper'
|
9
9
|
include MiGA::Cli::Action::Init::DaemonHelper
|
10
|
-
|
10
|
+
|
11
11
|
def parse_cli
|
12
12
|
cli.interactive = true
|
13
13
|
cli.defaults = { mytaxa: nil,
|
14
|
-
|
15
|
-
|
14
|
+
config: File.expand_path('.miga_modules', ENV['HOME']),
|
15
|
+
ask: false, auto: false, dtype: :bash }
|
16
16
|
cli.parse do |opt|
|
17
17
|
opt.on(
|
18
18
|
'-c', '--config PATH',
|
19
19
|
'Path to the Bash configuration file',
|
20
20
|
"By default: #{cli[:config]}"
|
21
|
-
){ |v| cli[:config] = v }
|
21
|
+
) { |v| cli[:config] = v }
|
22
22
|
opt.on(
|
23
23
|
'--[no-]mytaxa',
|
24
24
|
'Should I try setting up MyTaxa its dependencies?',
|
25
25
|
'By default: interactive (true if --auto)'
|
26
|
-
){ |v| cli[:mytaxa] = v }
|
26
|
+
) { |v| cli[:mytaxa] = v }
|
27
27
|
opt.on(
|
28
28
|
'--daemon-type STRING',
|
29
29
|
'Type of daemon launcher, one of: bash, ssh, qsub, msub, slurm',
|
30
30
|
"By default: interactive (#{cli[:dtype]} if --auto)"
|
31
|
-
){ |v| cli[:dtype] = v.to_sym }
|
31
|
+
) { |v| cli[:dtype] = v.to_sym }
|
32
32
|
opt.on(
|
33
33
|
'--ask-all',
|
34
34
|
'Ask for the location of all software',
|
35
35
|
'By default, only the locations missing in PATH are requested'
|
36
|
-
){ |v| cli[:ask] = v }
|
36
|
+
) { |v| cli[:ask] = v }
|
37
37
|
end
|
38
38
|
end
|
39
39
|
|
40
40
|
def perform
|
41
|
-
cli.puts
|
42
|
-
===[ Welcome to MiGA, the Microbial Genome Atlas ]===
|
41
|
+
cli.puts <<~BANNER
|
42
|
+
===[ Welcome to MiGA, the Microbial Genome Atlas ]===
|
43
43
|
|
44
|
-
I'm the initialization script, and I'll sniff around your computer to
|
45
|
-
make sure you have all the requirements for MiGA data processing.
|
44
|
+
I'm the initialization script, and I'll sniff around your computer to
|
45
|
+
make sure you have all the requirements for MiGA data processing.
|
46
46
|
|
47
|
-
BANNER
|
47
|
+
BANNER
|
48
48
|
list_requirements
|
49
49
|
rc_fh = open_rc_file
|
50
50
|
check_configuration_script rc_fh
|
@@ -66,8 +66,10 @@ BANNER
|
|
66
66
|
end
|
67
67
|
|
68
68
|
def run_r_cmd(cli, paths, cmd)
|
69
|
-
run_cmd(
|
70
|
-
|
69
|
+
run_cmd(
|
70
|
+
cli,
|
71
|
+
"echo #{cmd.shellescape} | #{paths['R'].shellescape} --vanilla -q 2>&1"
|
72
|
+
)
|
71
73
|
end
|
72
74
|
|
73
75
|
def test_r_package(cli, paths, pkg)
|
@@ -81,24 +83,30 @@ BANNER
|
|
81
83
|
end
|
82
84
|
|
83
85
|
def test_ruby_gem(cli, paths, pkg)
|
84
|
-
run_cmd(
|
85
|
-
|
86
|
+
run_cmd(
|
87
|
+
cli,
|
88
|
+
"#{paths['ruby'].shellescape} -r #{pkg.shellescape} -e '' 2>/dev/null"
|
89
|
+
)
|
86
90
|
$?.success?
|
87
91
|
end
|
88
92
|
|
89
93
|
def install_ruby_gem(cli, paths, pkg)
|
90
94
|
gem_cmd = "Gem::GemRunner.new.run %w(install --user #{pkg})"
|
91
|
-
run_cmd(
|
95
|
+
run_cmd(
|
96
|
+
cli,
|
97
|
+
"#{paths['ruby'].shellescape} \
|
92
98
|
-r rubygems -r rubygems/gem_runner \
|
93
|
-
-e #{gem_cmd.shellescape} 2>&1"
|
99
|
+
-e #{gem_cmd.shellescape} 2>&1"
|
100
|
+
)
|
94
101
|
end
|
95
102
|
|
96
103
|
def list_requirements
|
97
104
|
if cli.ask_user(
|
98
|
-
|
99
|
-
|
105
|
+
'Would you like to see all the requirements before starting?',
|
106
|
+
'no', %w(yes no)
|
107
|
+
) == 'yes'
|
100
108
|
cli.puts ''
|
101
|
-
req_path = File.
|
109
|
+
req_path = File.join(MiGA.root_path, 'utils', 'requirements.txt')
|
102
110
|
File.open(req_path, 'r') do |fh|
|
103
111
|
fh.each_line { |ln| cli.puts ln }
|
104
112
|
end
|
@@ -112,26 +120,28 @@ BANNER
|
|
112
120
|
rc_path = File.expand_path('.miga_rc', ENV['HOME'])
|
113
121
|
if File.exist? rc_path
|
114
122
|
if cli.ask_user(
|
115
|
-
|
116
|
-
|
123
|
+
'I found a previous configuration. Do you want to continue?',
|
124
|
+
'yes', %w(yes no)
|
125
|
+
) == 'no'
|
117
126
|
cli.puts 'OK, see you soon!'
|
118
127
|
exit(0)
|
119
128
|
end
|
120
129
|
end
|
121
130
|
rc_fh = File.open(rc_path, 'w')
|
122
|
-
rc_fh.puts
|
123
|
-
#!/bin/bash
|
124
|
-
# `miga init` made this on #{Time.now}
|
131
|
+
rc_fh.puts <<~BASH
|
132
|
+
#!/bin/bash
|
133
|
+
# `miga init` made this on #{Time.now}
|
125
134
|
|
126
|
-
BASH
|
135
|
+
BASH
|
127
136
|
rc_fh
|
128
137
|
end
|
129
138
|
|
130
139
|
def check_configuration_script(rc_fh)
|
131
140
|
unless File.exist? cli[:config]
|
132
141
|
cli[:config] = cli.ask_user(
|
133
|
-
|
134
|
-
|
142
|
+
'Is there a script I need to load at startup?',
|
143
|
+
cli[:config]
|
144
|
+
)
|
135
145
|
end
|
136
146
|
if File.exist? cli[:config]
|
137
147
|
cli[:config] = File.expand_path(cli[:config])
|
@@ -154,8 +164,10 @@ BASH
|
|
154
164
|
File.open(req_path, 'r') do |fh|
|
155
165
|
fh.each_line do |ln|
|
156
166
|
next if $. < 3
|
167
|
+
|
157
168
|
r = ln.chomp.split(/\t+/)
|
158
169
|
next if r[0] =~ /\(opt\)$/ && !cli[:mytaxa]
|
170
|
+
|
159
171
|
cli.print "Testing #{r[0]}#{" (#{r[3]})" if r[3]}... "
|
160
172
|
path = find_software(r[1])
|
161
173
|
paths[r[1]] = File.expand_path(r[1], path).shellescape
|
@@ -200,18 +212,18 @@ BASH
|
|
200
212
|
def check_additional_files(paths)
|
201
213
|
if cli[:mytaxa]
|
202
214
|
cli.puts 'Looking for MyTaxa databases:'
|
203
|
-
mt = File.dirname paths[
|
215
|
+
mt = File.dirname paths['MyTaxa']
|
204
216
|
cli.print 'Looking for scores... '
|
205
217
|
unless Dir.exist?(File.expand_path('db', mt))
|
206
|
-
cli.puts "no
|
218
|
+
cli.puts "no\nExecute 'python2 #{mt}/utils/download_db.py'"
|
207
219
|
exit(1)
|
208
220
|
end
|
209
|
-
cli.puts 'yes
|
221
|
+
cli.puts 'yes'
|
210
222
|
cli.print 'Looking for diamond db... '
|
211
223
|
unless File.exist?(File.expand_path('AllGenomes.faa.dmnd', mt))
|
212
|
-
cli.puts "no
|
224
|
+
cli.puts "no\nDownload " \
|
213
225
|
"'http://enve-omics.ce.gatech.edu/data/public_mytaxa/" \
|
214
|
-
"AllGenomes.faa.dmnd' into #{mt}
|
226
|
+
"AllGenomes.faa.dmnd' into #{mt}"
|
215
227
|
exit(1)
|
216
228
|
end
|
217
229
|
cli.puts ''
|
@@ -223,7 +235,7 @@ BASH
|
|
223
235
|
%w(ape cluster vegan).each do |pkg|
|
224
236
|
cli.print "Testing #{pkg}... "
|
225
237
|
if test_r_package(cli, paths, pkg)
|
226
|
-
cli.puts 'yes
|
238
|
+
cli.puts 'yes'
|
227
239
|
else
|
228
240
|
cli.puts 'no, installing'
|
229
241
|
cli.print '' + install_r_package(cli, paths, pkg)
|
@@ -240,7 +252,7 @@ BASH
|
|
240
252
|
%w(sqlite3 daemons json).each do |pkg|
|
241
253
|
cli.print "Testing #{pkg}... "
|
242
254
|
if test_ruby_gem(cli, paths, pkg)
|
243
|
-
cli.puts 'yes
|
255
|
+
cli.puts 'yes'
|
244
256
|
else
|
245
257
|
cli.puts 'no, installing'
|
246
258
|
# This hackey mess is meant to ensure the test and installation are done
|
@@ -256,13 +268,13 @@ BASH
|
|
256
268
|
end
|
257
269
|
|
258
270
|
def close_rc_file(rc_fh)
|
259
|
-
rc_fh.puts
|
271
|
+
rc_fh.puts <<~FOOT
|
260
272
|
|
261
|
-
MIGA_CONFIG_VERSION='#{MiGA::MiGA.VERSION}'
|
262
|
-
MIGA_CONFIG_LONGVERSION='#{MiGA::MiGA.LONG_VERSION}'
|
263
|
-
MIGA_CONFIG_DATE='#{Time.now}'
|
273
|
+
MIGA_CONFIG_VERSION='#{MiGA::MiGA.VERSION}'
|
274
|
+
MIGA_CONFIG_LONGVERSION='#{MiGA::MiGA.LONG_VERSION}'
|
275
|
+
MIGA_CONFIG_DATE='#{Time.now}'
|
264
276
|
|
265
|
-
FOOT
|
277
|
+
FOOT
|
266
278
|
rc_fh.close
|
267
279
|
end
|
268
280
|
end
|