miga-base 0.7.3.1 → 0.7.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/miga/cli/action/doctor/base.rb +79 -0
- data/lib/miga/cli/action/doctor.rb +54 -73
- data/lib/miga/cli/action/edit.rb +14 -1
- data/lib/miga/daemon/base.rb +30 -9
- data/lib/miga/daemon.rb +44 -11
- data/lib/miga/dataset/status.rb +6 -5
- data/lib/miga/project/dataset.rb +6 -13
- data/lib/miga/version.rb +2 -2
- data/test/daemon_test.rb +22 -2
- data/test/project_test.rb +0 -1
- data/test/tax_dist_test.rb +2 -2
- data/utils/index_metadata.rb +2 -3
- metadata +3 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 917085ea3480e143d61e0ce468cf041cd6ad4f7b45b4231c7239037a44bd1879
|
4
|
+
data.tar.gz: 9b6751862e49aee3980133dc16d85d51d973111ad5db9eb15dcbb369972e548d
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 3bd3c871edb8ffe3cb71e697d56cd9108501073db882b9ea61c58a243bfee6c31040a2197211f4c97b963dccd629f0639b4e3894b8a5173db77c3a3ee4088065
|
7
|
+
data.tar.gz: c14f8d8b194e8691a194a5c99ebb06720c079808ce41a687c7d9106cb1e49d8baaf0e52aee3bf55bd750534ea85121fa9dab820419b42fb1b2927e71fe72a467
|
@@ -0,0 +1,79 @@
|
|
1
|
+
|
2
|
+
require 'miga/cli/action'
|
3
|
+
require 'sqlite3'
|
4
|
+
|
5
|
+
class MiGA::Cli::Action::Doctor < MiGA::Cli::Action
|
6
|
+
end
|
7
|
+
|
8
|
+
module MiGA::Cli::Action::Doctor::Base
|
9
|
+
##
|
10
|
+
# Check the database in +db_file+ maintains integrity for the
|
11
|
+
# tables saving +metric+ (:ani or :aai)
|
12
|
+
def check_sqlite3_database(db_file, metric)
|
13
|
+
SQLite3::Database.new(db_file) do |conn|
|
14
|
+
conn.execute("select count(*) from #{metric}").first
|
15
|
+
end
|
16
|
+
rescue SQLite3::SQLException
|
17
|
+
yield
|
18
|
+
end
|
19
|
+
|
20
|
+
##
|
21
|
+
# Scans the all-vs-all matrix registered in +res+ (MiGA::Result) in search of
|
22
|
+
# pairs where one or both datasets are missing or inactive in the project +p+
|
23
|
+
# (MiGA::Project), and report progress through +cli+ (MiGA::Cli).
|
24
|
+
# Returns an Array with two arrays: the first a list of dataset names that are
|
25
|
+
# no longer registered in the project or are currently inactive, and the
|
26
|
+
# second a list of dataset names that have registered pairs with the first
|
27
|
+
# list, and therefore the databases need to be cleaned.
|
28
|
+
# This is a subtask of +check_dist+
|
29
|
+
def check_dist_eval(cli, p, res)
|
30
|
+
notok = {}
|
31
|
+
fix = {}
|
32
|
+
Zlib::GzipReader.open(res.file_path(:matrix)) do |fh|
|
33
|
+
lineno = 0
|
34
|
+
fh.each_line do |ln|
|
35
|
+
next if (lineno += 1) == 1
|
36
|
+
r = ln.split("\t")
|
37
|
+
next unless [1, 2].map { |i| p.dataset(r[i]).nil? }.any?
|
38
|
+
[1, 2].each do |i|
|
39
|
+
if p.dataset(r[i]).nil? || !p.dataset(r[i]).active?
|
40
|
+
notok[r[i]] = true
|
41
|
+
else
|
42
|
+
fix[r[i]] = true
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
[notok.keys, fix.keys]
|
48
|
+
end
|
49
|
+
|
50
|
+
##
|
51
|
+
# Cleanup distance databases for datasets names in +fix+ (Array: String)
|
52
|
+
# from project +p+ (MiGA::Project), and report through +cli+ (MiGA::Cli).
|
53
|
+
# This is a subtask of +check_dist+
|
54
|
+
def check_dist_fix(cli, p, fix)
|
55
|
+
return if fix.empty?
|
56
|
+
cli.say("- Fixing #{fix.size} datasets")
|
57
|
+
fix.each do |d_n|
|
58
|
+
cli.say " > Fixing #{d_n}."
|
59
|
+
p.dataset(d_n).cleanup_distances!
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
##
|
64
|
+
# Recompute +res+ (MiGA::Result) if +notok+ (Array: String) has any dataset
|
65
|
+
# names registered, and report through +cli+ (MiGA::Cli).
|
66
|
+
# This is a subtask of +check_dist+
|
67
|
+
def check_dist_recompute(cli, res, notok)
|
68
|
+
return if notok.empty?
|
69
|
+
cli.say '- Unregistered datasets detected: '
|
70
|
+
if notok.size <= 5
|
71
|
+
notok.each { |i| cli.say " > #{i}" }
|
72
|
+
else
|
73
|
+
cli.say " > #{notok.size}, including #{notok.first}"
|
74
|
+
end
|
75
|
+
cli.say '- Removing tables, recompute'
|
76
|
+
res.remove!
|
77
|
+
end
|
78
|
+
end
|
79
|
+
|
@@ -1,15 +1,15 @@
|
|
1
1
|
# @package MiGA
|
2
2
|
# @license Artistic-2.0
|
3
3
|
|
4
|
-
require 'miga/cli/action'
|
5
|
-
require 'sqlite3'
|
4
|
+
require 'miga/cli/action/doctor/base'
|
6
5
|
|
7
6
|
class MiGA::Cli::Action::Doctor < MiGA::Cli::Action
|
7
|
+
include MiGA::Cli::Action::Doctor::Base
|
8
8
|
|
9
9
|
def parse_cli
|
10
|
-
@@OPERATIONS.keys.
|
10
|
+
cli.defaults = Hash[@@OPERATIONS.keys.map { |i| [i, true] }]
|
11
11
|
cli.parse do |opt|
|
12
|
-
operation_n = Hash[@@OPERATIONS.map { |k,v| [v[0], k] }]
|
12
|
+
operation_n = Hash[@@OPERATIONS.map { |k, v| [v[0], k] }]
|
13
13
|
cli.opt_object(opt, [:project])
|
14
14
|
opt.on(
|
15
15
|
'--ignore TASK1,TASK2', Array,
|
@@ -21,20 +21,12 @@ class MiGA::Cli::Action::Doctor < MiGA::Cli::Action
|
|
21
21
|
'Perform only the specified task (see --ignore)'
|
22
22
|
) do |v|
|
23
23
|
op_k = @@OPERATIONS.find { |_, i| i[0] == v.downcase }.first
|
24
|
-
@@OPERATIONS.
|
24
|
+
@@OPERATIONS.each_key { |i| cli[i] = false }
|
25
25
|
cli[op_k] = true
|
26
26
|
end
|
27
27
|
end
|
28
28
|
end
|
29
29
|
|
30
|
-
def check_sqlite3_database(db_file, metric)
|
31
|
-
SQLite3::Database.new(db_file) do |conn|
|
32
|
-
conn.execute("select count(*) from #{metric}").first
|
33
|
-
end
|
34
|
-
rescue SQLite3::SQLException
|
35
|
-
yield
|
36
|
-
end
|
37
|
-
|
38
30
|
def perform
|
39
31
|
p = cli.load_project
|
40
32
|
@@OPERATIONS.keys.each do |k|
|
@@ -43,7 +35,8 @@ class MiGA::Cli::Action::Doctor < MiGA::Cli::Action
|
|
43
35
|
end
|
44
36
|
|
45
37
|
@@OPERATIONS = {
|
46
|
-
|
38
|
+
status: ['status', 'Update metadata status of all datasets'],
|
39
|
+
db: ['databases', 'Check integrity of database files'],
|
47
40
|
dist: ['distances', 'Check distance summary tables'],
|
48
41
|
files: ['files', 'Check for outdated files'],
|
49
42
|
cds: ['cds', 'Check for gzipped genes and proteins'],
|
@@ -52,34 +45,52 @@ class MiGA::Cli::Action::Doctor < MiGA::Cli::Action
|
|
52
45
|
start: ['start', 'Check for lingering .start files'],
|
53
46
|
tax: ['taxonomy', 'Check for taxonomy consistency (not yet implemented)']
|
54
47
|
}
|
48
|
+
|
55
49
|
class << self
|
50
|
+
##
|
51
|
+
# All supported operations
|
56
52
|
def OPERATIONS
|
57
53
|
@@OPERATIONS
|
58
54
|
end
|
59
55
|
end
|
60
56
|
|
57
|
+
##
|
58
|
+
# Perform status operation with MiGA::Cli +cli+
|
59
|
+
def check_status(cli)
|
60
|
+
cli.say 'Updating metadata status'
|
61
|
+
cli.load_project.each_dataset { |d| d.recalculate_status }
|
62
|
+
end
|
63
|
+
|
64
|
+
##
|
65
|
+
# Perform databases operation with MiGA::Cli +cli+
|
61
66
|
def check_db(cli)
|
62
|
-
cli.say 'Checking databases
|
67
|
+
cli.say 'Checking integrity of databases'
|
63
68
|
cli.load_project.each_dataset do |d|
|
64
|
-
[
|
69
|
+
%i[distances taxonomy].each do |r_key|
|
65
70
|
r = d.result(r_key) or next
|
66
|
-
{haai_db: :aai, aai_db: :aai, ani_db: :ani}.each do |db_key, metric|
|
71
|
+
{ haai_db: :aai, aai_db: :aai, ani_db: :ani }.each do |db_key, metric|
|
67
72
|
db_file = r.file_path(db_key) or next
|
68
73
|
check_sqlite3_database(db_file, metric) do
|
69
|
-
cli.say(
|
70
|
-
" > Removing #{db_key} #{r_key} table for #{d.name}")
|
74
|
+
cli.say(" > Removing #{db_key} #{r_key} table for #{d.name}")
|
71
75
|
[db_file, r.path(:done), r.path].each do |f|
|
72
76
|
File.unlink(f) if File.exist? f
|
73
|
-
end
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
77
|
+
end
|
78
|
+
# /each |f|
|
79
|
+
end
|
80
|
+
# /check_sqlite3_database
|
81
|
+
end
|
82
|
+
# /each |db_key, metric|
|
83
|
+
end
|
84
|
+
# /each |r_key|
|
85
|
+
end
|
86
|
+
# /each |d|
|
78
87
|
end
|
79
88
|
|
89
|
+
##
|
90
|
+
# Perform distances operation with MiGA::Cli +cli+
|
80
91
|
def check_dist(cli)
|
81
92
|
p = cli.load_project
|
82
|
-
[
|
93
|
+
%i[ani aai].each do |dist|
|
83
94
|
res = p.result("#{dist}_distances")
|
84
95
|
next if res.nil?
|
85
96
|
cli.say "Checking #{dist} table for consistent datasets"
|
@@ -89,6 +100,8 @@ class MiGA::Cli::Action::Doctor < MiGA::Cli::Action
|
|
89
100
|
end
|
90
101
|
end
|
91
102
|
|
103
|
+
##
|
104
|
+
# Perform files operation with MiGA::Cli +cli+
|
92
105
|
def check_files(cli)
|
93
106
|
cli.say 'Looking for outdated files in results'
|
94
107
|
p = cli.load_project
|
@@ -110,12 +123,14 @@ class MiGA::Cli::Action::Doctor < MiGA::Cli::Action
|
|
110
123
|
end
|
111
124
|
end
|
112
125
|
|
126
|
+
##
|
127
|
+
# Perform cds operation with MiGA::Cli +cli+
|
113
128
|
def check_cds(cli)
|
114
129
|
cli.say 'Looking for unzipped genes or proteins'
|
115
130
|
cli.load_project.each_dataset do |d|
|
116
131
|
res = d.result(:cds) or next
|
117
132
|
changed = false
|
118
|
-
[
|
133
|
+
%i[genes proteins gff3 gff2 tab].each do |f|
|
119
134
|
file = res.file_path(f) or next
|
120
135
|
if file !~ /\.gz/
|
121
136
|
cli.say " > Gzipping #{d.name} #{f}"
|
@@ -131,6 +146,8 @@ class MiGA::Cli::Action::Doctor < MiGA::Cli::Action
|
|
131
146
|
end
|
132
147
|
end
|
133
148
|
|
149
|
+
##
|
150
|
+
# Perform essential-genes operation with MiGA::Cli +cli+
|
134
151
|
def check_ess(cli)
|
135
152
|
cli.say 'Looking for unarchived essential genes'
|
136
153
|
cli.load_project.each_dataset do |d|
|
@@ -150,6 +167,8 @@ class MiGA::Cli::Action::Doctor < MiGA::Cli::Action
|
|
150
167
|
end
|
151
168
|
end
|
152
169
|
|
170
|
+
##
|
171
|
+
# Perform mytaxa-scan operation with MiGA::Cli +cli+
|
153
172
|
def check_mts(cli)
|
154
173
|
cli.say 'Looking for unarchived MyTaxa Scan runs'
|
155
174
|
cli.load_project.each_dataset do |d|
|
@@ -166,8 +185,8 @@ class MiGA::Cli::Action::Doctor < MiGA::Cli::Action
|
|
166
185
|
end
|
167
186
|
fix = true
|
168
187
|
end
|
169
|
-
%
|
170
|
-
file = res.file_path(ext
|
188
|
+
%i[blast mytaxain wintax gene_ids region_ids].each do |ext|
|
189
|
+
file = res.file_path(ext)
|
171
190
|
unless file.nil?
|
172
191
|
FileUtils.rm(file) if File.exist? file
|
173
192
|
fix = true
|
@@ -180,6 +199,8 @@ class MiGA::Cli::Action::Doctor < MiGA::Cli::Action
|
|
180
199
|
end
|
181
200
|
end
|
182
201
|
|
202
|
+
##
|
203
|
+
# Perform start operation with MiGA::Cli +cli+
|
183
204
|
def check_start(cli)
|
184
205
|
cli.say 'Looking for legacy .start files lingering'
|
185
206
|
cli.load_project.each_dataset do |d|
|
@@ -192,52 +213,12 @@ class MiGA::Cli::Action::Doctor < MiGA::Cli::Action
|
|
192
213
|
end
|
193
214
|
end
|
194
215
|
|
216
|
+
##
|
217
|
+
# Perform taxonomy operation with MiGA::Cli +cli+
|
195
218
|
def check_tax(cli)
|
196
|
-
#cli.say 'o Checking for taxonomy/distances consistency'
|
219
|
+
# cli.say 'o Checking for taxonomy/distances consistency'
|
197
220
|
# TODO: Find 95%ANI clusters with entries from different species
|
198
|
-
|
199
|
-
|
200
|
-
private
|
201
|
-
|
202
|
-
def check_dist_eval(cli, p, res)
|
203
|
-
notok = {}
|
204
|
-
fix = {}
|
205
|
-
Zlib::GzipReader.open(res.file_path(:matrix)) do |fh|
|
206
|
-
lineno = 0
|
207
|
-
fh.each_line do |ln|
|
208
|
-
next if (lineno += 1) == 1
|
209
|
-
r = ln.split("\t")
|
210
|
-
next unless [1, 2].map { |i| p.dataset(r[i]).nil? }.any?
|
211
|
-
[1, 2].each do |i|
|
212
|
-
if p.dataset(r[i]).nil?
|
213
|
-
notok[r[i]] = true
|
214
|
-
else
|
215
|
-
fix[r[i]] = true
|
216
|
-
end
|
217
|
-
end
|
218
|
-
end
|
219
|
-
end
|
220
|
-
[notok, fix]
|
221
|
-
end
|
222
|
-
|
223
|
-
def check_dist_fix(cli, p, fix)
|
224
|
-
return if fix.empty?
|
225
|
-
cli.say("- Fixing #{fix.size} datasets")
|
226
|
-
fix.keys.each do |d_n|
|
227
|
-
cli.say " > Fixing #{d_n}."
|
228
|
-
p.dataset(d_n).cleanup_distances!
|
229
|
-
end
|
230
|
-
end
|
231
|
-
|
232
|
-
def check_dist_recompute(cli, p, notok)
|
233
|
-
return if notok.empty?
|
234
|
-
cli.say '- Unregistered datasets detected: '
|
235
|
-
if notok.size <= 5
|
236
|
-
notok.keys.each { |i| cli.say " > #{i}" }
|
237
|
-
else
|
238
|
-
cli.say " > #{notok.size}, including #{notok.keys.first}"
|
239
|
-
end
|
240
|
-
cli.say '- Removing tables, recompute'
|
241
|
-
res.remove!
|
221
|
+
# TODO: Find different 95%ANI clusters with genomes from the same species
|
222
|
+
# TODO: Find AAI values too high or too low for each LCA rank
|
242
223
|
end
|
243
224
|
end
|
data/lib/miga/cli/action/edit.rb
CHANGED
@@ -12,12 +12,25 @@ class MiGA::Cli::Action::Edit < MiGA::Cli::Action
|
|
12
12
|
'-m', '--metadata STRING',
|
13
13
|
'Metadata as key-value pairs separated by = and delimited by comma',
|
14
14
|
'Values are saved as strings except for booleans (true / false) or nil'
|
15
|
-
|
15
|
+
) { |v| cli[:metadata] = v }
|
16
|
+
opt.on(
|
17
|
+
'--activate',
|
18
|
+
'Activate dataset; requires -D'
|
19
|
+
) { |v| cli[:activate] = v }
|
20
|
+
opt.on(
|
21
|
+
'--inactivate',
|
22
|
+
'Inactivate dataset; requires -D'
|
23
|
+
) { |v| cli[:activate] = !v }
|
16
24
|
end
|
17
25
|
end
|
18
26
|
|
19
27
|
def perform
|
20
28
|
obj = cli.load_project_or_dataset
|
29
|
+
unless cli[:activate].nil?
|
30
|
+
cli.ensure_par({ dataset: '-D' },
|
31
|
+
'%<name>s is mandatory with --[in-]activate: please provide %<flag>s')
|
32
|
+
cli[:activate] ? obj.activate! : obj.inactivate!
|
33
|
+
end
|
21
34
|
cli.add_metadata(obj)
|
22
35
|
obj.save
|
23
36
|
end
|
data/lib/miga/daemon/base.rb
CHANGED
@@ -14,19 +14,21 @@ module MiGA::Daemon::Base
|
|
14
14
|
k = k.to_sym
|
15
15
|
unless v.nil?
|
16
16
|
case k
|
17
|
-
when :latency, :maxjobs, :ppn, :format_version
|
17
|
+
when :latency, :maxjobs, :ppn, :format_version, :verbosity
|
18
18
|
v = v.to_i
|
19
|
+
if !force && v == 0 && k != :verbosity
|
20
|
+
raise "Daemon's #{k} cannot be set to zero"
|
21
|
+
end
|
19
22
|
when :shutdown_when_done
|
20
23
|
v = !!v
|
21
24
|
when :nodelist
|
22
25
|
if v =~ /^\$/
|
23
|
-
vv = ENV[v.sub('$','')] or raise "Unset environment variable: #{v}"
|
26
|
+
vv = ENV[v.sub('$', '')] or raise "Unset environment variable: #{v}"
|
24
27
|
v = vv
|
25
28
|
end
|
26
29
|
say "Reading node list: #{v}"
|
27
30
|
v = File.readlines(v).map(&:chomp)
|
28
31
|
end
|
29
|
-
raise "Daemon's #{k} cannot be set to zero." if !force and v == 0
|
30
32
|
@runopts[k] = v
|
31
33
|
end
|
32
34
|
@runopts[k]
|
@@ -34,24 +36,43 @@ module MiGA::Daemon::Base
|
|
34
36
|
|
35
37
|
##
|
36
38
|
# Returns Integer indicating the number of seconds to sleep between checks
|
37
|
-
def latency
|
39
|
+
def latency
|
40
|
+
runopts(:latency)
|
41
|
+
end
|
38
42
|
|
39
43
|
##
|
40
44
|
# Returns Integer indicating the maximum number of concurrent jobs to run
|
41
|
-
def maxjobs
|
45
|
+
def maxjobs
|
46
|
+
runopts(:maxjobs)
|
47
|
+
end
|
42
48
|
|
43
49
|
##
|
44
50
|
# Returns the path to the list of execution hostnames
|
45
|
-
def nodelist
|
51
|
+
def nodelist
|
52
|
+
runopts(:nodelist)
|
53
|
+
end
|
46
54
|
|
47
55
|
##
|
48
56
|
# Returns Integer indicating the number of CPUs per job
|
49
|
-
def ppn
|
57
|
+
def ppn
|
58
|
+
runopts(:ppn)
|
59
|
+
end
|
50
60
|
|
51
61
|
##
|
52
62
|
# Returns Boolean indicating if the daemon should shutdown when processing is
|
53
63
|
# complete
|
54
|
-
def shutdown_when_done?
|
64
|
+
def shutdown_when_done?
|
65
|
+
!!runopts(:shutdown_when_done)
|
66
|
+
end
|
55
67
|
|
68
|
+
##
|
69
|
+
# Returns the level of verbosity for the daemon as an Integer, or 1 if unset.
|
70
|
+
# Verbosity levels are:
|
71
|
+
# 0: No output
|
72
|
+
# 1: General daemon and job information
|
73
|
+
# 2: Same, and indicate when each task is performed (even if nothing happens)
|
74
|
+
# 3: Same, and indicate when each loop begins and ends
|
75
|
+
def verbosity
|
76
|
+
runopts(:verbosity) || 1
|
77
|
+
end
|
56
78
|
end
|
57
|
-
|
data/lib/miga/daemon.rb
CHANGED
@@ -45,8 +45,9 @@ class MiGA::Daemon < MiGA::MiGA
|
|
45
45
|
@project = project
|
46
46
|
@runopts = {}
|
47
47
|
json ||= File.join(project.path, 'daemon/daemon.json')
|
48
|
+
default_json = File.expand_path('.miga_daemon.json', ENV['MIGA_HOME'])
|
48
49
|
MiGA::Json.parse(
|
49
|
-
json, default: File.
|
50
|
+
json, default: File.exist?(default_json) ? default_json : nil
|
50
51
|
).each { |k,v| runopts(k, v) }
|
51
52
|
update_format_0
|
52
53
|
@jobs_to_run = []
|
@@ -77,29 +78,54 @@ class MiGA::Daemon < MiGA::MiGA
|
|
77
78
|
end
|
78
79
|
|
79
80
|
##
|
80
|
-
# Run one loop step. Returns a Boolean indicating if the loop should continue
|
81
|
+
# Run one loop step. Returns a Boolean indicating if the loop should continue
|
81
82
|
def daemon_loop
|
82
|
-
|
83
|
+
l_say(3, 'Daemon loop start')
|
84
|
+
reload_project
|
83
85
|
check_datasets
|
84
86
|
check_project
|
85
|
-
if shutdown_when_done?
|
86
|
-
say 'Nothing else to do, shutting down
|
87
|
+
if shutdown_when_done? && jobs_running.size + jobs_to_run.size == 0
|
88
|
+
say 'Nothing else to do, shutting down'
|
87
89
|
return false
|
88
90
|
end
|
89
91
|
flush!
|
90
|
-
if loop_i
|
91
|
-
say 'Probing running jobs'
|
92
|
-
@loop_i = 0
|
92
|
+
if loop_i % 12 == 0
|
93
93
|
purge!
|
94
|
+
recalculate_status!
|
94
95
|
end
|
95
|
-
|
96
|
+
save_status
|
96
97
|
sleep(latency)
|
98
|
+
l_say(3, 'Daemon loop end')
|
97
99
|
true
|
98
100
|
end
|
99
101
|
|
102
|
+
def recalculate_status!
|
103
|
+
project.each_dataset(&:recalculate_status)
|
104
|
+
end
|
105
|
+
|
106
|
+
##
|
107
|
+
# Send +msg+ to +say+ as long as +level+ is at most +verbosity+
|
108
|
+
def l_say(level, *msg)
|
109
|
+
say(*msg) if verbosity >= level
|
110
|
+
end
|
111
|
+
|
112
|
+
##
|
113
|
+
# Same as +l_say+ with +level = 1+
|
114
|
+
def say(*msg)
|
115
|
+
super(*msg) if verbosity >= 1
|
116
|
+
end
|
117
|
+
|
118
|
+
##
|
119
|
+
# Reload the project's metadata
|
120
|
+
def reload_project
|
121
|
+
l_say(2, 'Reloading project')
|
122
|
+
project.load
|
123
|
+
end
|
124
|
+
|
100
125
|
##
|
101
126
|
# Report status in a JSON file.
|
102
|
-
def
|
127
|
+
def save_status
|
128
|
+
l_say(2, 'Saving current status')
|
103
129
|
MiGA::Json.generate(
|
104
130
|
{ jobs_running: @jobs_running, jobs_to_run: @jobs_to_run },
|
105
131
|
File.join(daemon_home, 'status.json')
|
@@ -134,7 +160,9 @@ class MiGA::Daemon < MiGA::MiGA
|
|
134
160
|
##
|
135
161
|
# Traverse datasets
|
136
162
|
def check_datasets
|
137
|
-
|
163
|
+
l_say(2, 'Checking datasets')
|
164
|
+
project.each_dataset do |ds|
|
165
|
+
next unless ds.status == :incomplete
|
138
166
|
to_run = ds.next_preprocessing(false)
|
139
167
|
queue_job(:d, ds) unless to_run.nil?
|
140
168
|
end
|
@@ -144,6 +172,7 @@ class MiGA::Daemon < MiGA::MiGA
|
|
144
172
|
# Check if all reference datasets are pre-processed. If yes, check the
|
145
173
|
# project-level tasks
|
146
174
|
def check_project
|
175
|
+
l_say(2, 'Checking project')
|
147
176
|
return if project.dataset_names.empty?
|
148
177
|
return unless project.done_preprocessing?(false)
|
149
178
|
to_run = project.next_task(nil, false)
|
@@ -199,6 +228,7 @@ class MiGA::Daemon < MiGA::MiGA
|
|
199
228
|
# possible respecting #maxjobs or #nodelist (if set).
|
200
229
|
def flush!
|
201
230
|
# Check for finished jobs
|
231
|
+
l_say(2, 'Checking for finished jobs')
|
202
232
|
@jobs_running.select! do |job|
|
203
233
|
ongoing = case job[:job].to_s
|
204
234
|
when 'd'
|
@@ -211,8 +241,10 @@ class MiGA::Daemon < MiGA::MiGA
|
|
211
241
|
say "Completed pid:#{job[:pid]} for #{job[:task_name]}" unless ongoing
|
212
242
|
ongoing
|
213
243
|
end
|
244
|
+
|
214
245
|
# Avoid single datasets hogging resources
|
215
246
|
@jobs_to_run.rotate! rand(jobs_to_run.size)
|
247
|
+
|
216
248
|
# Launch as many +jobs_to_run+ as possible
|
217
249
|
while hostk = next_host
|
218
250
|
break if jobs_to_run.empty?
|
@@ -233,6 +265,7 @@ class MiGA::Daemon < MiGA::MiGA
|
|
233
265
|
##
|
234
266
|
# Remove dead jobs.
|
235
267
|
def purge!
|
268
|
+
say 'Probing running jobs'
|
236
269
|
@jobs_running.select! do |job|
|
237
270
|
`#{runopts(:alive).miga_variables(pid: job[:pid])}`.chomp.to_i == 1
|
238
271
|
end
|
data/lib/miga/dataset/status.rb
CHANGED
@@ -4,7 +4,7 @@ module MiGA::Dataset::Status
|
|
4
4
|
##
|
5
5
|
# Returns the status of the dataset. If the status is not yet defined,
|
6
6
|
# it recalculates it and, if +save+ is true, saves it in metadata.
|
7
|
-
# Return
|
7
|
+
# Return symbols are:
|
8
8
|
# - +:inactive+ The dataset is currently inactive
|
9
9
|
# - +:incomplete+ The dataset is not yet fully processed
|
10
10
|
# - +:complete+ The dataset is fully processed
|
@@ -14,12 +14,13 @@ module MiGA::Dataset::Status
|
|
14
14
|
end
|
15
15
|
|
16
16
|
##
|
17
|
-
# Identify the current status
|
18
|
-
#
|
17
|
+
# Identify the current status and save it if +save+ and the status changed.
|
18
|
+
# Return symbols are the same as +status+.
|
19
19
|
def recalculate_status(save = true)
|
20
|
+
old_status = metadata[:status]
|
20
21
|
metadata[:status] =
|
21
|
-
!active? ?
|
22
|
-
self.save if save
|
22
|
+
!active? ? 'inactive' : done_preprocessing? ? 'complete' : 'incomplete'
|
23
|
+
self.save if save && (old_status.nil? || old_status != metadata[:status])
|
23
24
|
metadata[:status].to_sym
|
24
25
|
end
|
25
26
|
end
|
data/lib/miga/project/dataset.rb
CHANGED
@@ -35,16 +35,12 @@ module MiGA::Project::Dataset
|
|
35
35
|
end
|
36
36
|
|
37
37
|
##
|
38
|
-
# Iterate through datasets
|
39
|
-
# If one, the dataset MiGA::Dataset object is passed. If two, the name and
|
40
|
-
# the dataset object are passed.
|
38
|
+
# Iterate through datasets (MiGA::Dataset)
|
41
39
|
def each_dataset(&blk)
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
blk.call(name, dataset(name))
|
47
|
-
end
|
40
|
+
if block_given?
|
41
|
+
metadata[:datasets].each { |name| blk.call(dataset(name)) }
|
42
|
+
else
|
43
|
+
to_enum(:each_dataset)
|
48
44
|
end
|
49
45
|
end
|
50
46
|
|
@@ -140,9 +136,7 @@ module MiGA::Project::Dataset
|
|
140
136
|
# - 2: To do.
|
141
137
|
def profile_datasets_advance
|
142
138
|
advance = []
|
143
|
-
|
144
|
-
advance << ds_adv
|
145
|
-
end
|
139
|
+
each_dataset_profile_advance { |adv| advance << adv }
|
146
140
|
advance
|
147
141
|
end
|
148
142
|
|
@@ -154,4 +148,3 @@ module MiGA::Project::Dataset
|
|
154
148
|
end
|
155
149
|
|
156
150
|
end
|
157
|
-
|
data/lib/miga/version.rb
CHANGED
@@ -10,7 +10,7 @@ module MiGA
|
|
10
10
|
# - Float representing the major.minor version.
|
11
11
|
# - Integer representing gem releases of the current version.
|
12
12
|
# - Integer representing minor changes that require new version number.
|
13
|
-
VERSION = [0.7,
|
13
|
+
VERSION = [0.7, 4, 0]
|
14
14
|
|
15
15
|
##
|
16
16
|
# Nickname for the current major.minor version.
|
@@ -18,7 +18,7 @@ module MiGA
|
|
18
18
|
|
19
19
|
##
|
20
20
|
# Date of the current gem release.
|
21
|
-
VERSION_DATE = Date.new(2020, 4,
|
21
|
+
VERSION_DATE = Date.new(2020, 4, 28)
|
22
22
|
|
23
23
|
##
|
24
24
|
# Reference of MiGA.
|
data/test/daemon_test.rb
CHANGED
@@ -88,7 +88,7 @@ class DaemonTest < Test::Unit::TestCase
|
|
88
88
|
assert_equal(11, d.loop_i)
|
89
89
|
out = capture_stderr { d.in_loop }.string
|
90
90
|
assert_match(/Probing running jobs/, out)
|
91
|
-
assert_equal(
|
91
|
+
assert_equal(12, d.loop_i)
|
92
92
|
end
|
93
93
|
|
94
94
|
def test_start
|
@@ -109,6 +109,7 @@ class DaemonTest < Test::Unit::TestCase
|
|
109
109
|
assert_match(/Sending termination message/, out)
|
110
110
|
assert_path_not_exist(d.pid_file)
|
111
111
|
assert_path_exist(d.output_file)
|
112
|
+
assert_equal(1, d.verbosity)
|
112
113
|
l = File.readlines(d.output_file)
|
113
114
|
{
|
114
115
|
0 => /-{20}\n/,
|
@@ -312,6 +313,26 @@ class DaemonTest < Test::Unit::TestCase
|
|
312
313
|
assert_equal(1, d1.jobs_to_run.size)
|
313
314
|
end
|
314
315
|
|
316
|
+
def test_verbosity
|
317
|
+
d1 = $d1
|
318
|
+
d1.runopts(:verbosity, 0)
|
319
|
+
out = capture_stderr { d1.in_loop }.string
|
320
|
+
assert_empty(out)
|
321
|
+
|
322
|
+
d1.runopts(:verbosity, 1)
|
323
|
+
helper_datasets_with_results.first.inactivate!
|
324
|
+
out = capture_stderr { d1.check_project }
|
325
|
+
assert_match(/Queueing miga-project:p/, out.string)
|
326
|
+
|
327
|
+
d1.runopts(:verbosity, 2)
|
328
|
+
out = capture_stderr { d1.in_loop }.string
|
329
|
+
assert_match(/Reloading project/, out)
|
330
|
+
|
331
|
+
d1.runopts(:verbosity, 3)
|
332
|
+
out = capture_stderr { d1.in_loop }.string
|
333
|
+
assert_match(/Daemon loop start/, out)
|
334
|
+
end
|
335
|
+
|
315
336
|
def helper_daemon_launch_job
|
316
337
|
omit_if($jruby_tests, 'JRuby doesn\'t implement fork.')
|
317
338
|
d1 = $d1
|
@@ -324,5 +345,4 @@ class DaemonTest < Test::Unit::TestCase
|
|
324
345
|
assert_equal(0, d1.jobs_to_run.size, 'There should be nothing running')
|
325
346
|
assert_equal(1, d1.jobs_running.size, 'There should be one job running')
|
326
347
|
end
|
327
|
-
|
328
348
|
end
|
data/test/project_test.rb
CHANGED
@@ -58,7 +58,6 @@ class ProjectTest < Test::Unit::TestCase
|
|
58
58
|
assert_equal([d], p.datasets)
|
59
59
|
assert_equal(['d1'], p.dataset_names)
|
60
60
|
p.each_dataset { |ds| assert_equal(d, ds) }
|
61
|
-
p.each_dataset { |n, ds| assert_equal(n, ds.name) }
|
62
61
|
dr = p.unlink_dataset('d1')
|
63
62
|
assert_equal(d, dr)
|
64
63
|
assert_equal([], p.datasets)
|
data/test/tax_dist_test.rb
CHANGED
@@ -40,12 +40,12 @@ class TaxDistTest < Test::Unit::TestCase
|
|
40
40
|
def test_aai_taxtest
|
41
41
|
distant_intax = MiGA::TaxDist.aai_taxtest(35.0, :intax, engine: :diamond)
|
42
42
|
assert_equal(:root, distant_intax[:most_likely][0])
|
43
|
-
|
43
|
+
assert_equal(:d, distant_intax[:probably][0])
|
44
44
|
assert_nil(distant_intax[:possibly_even])
|
45
45
|
|
46
46
|
distant_intax = MiGA::TaxDist.aai_taxtest(35.0, :intax, engine: :blast)
|
47
47
|
assert_equal(:root, distant_intax[:most_likely][0])
|
48
|
-
|
48
|
+
assert_equal(:d, distant_intax[:probably][0])
|
49
49
|
assert_nil(distant_intax[:possibly_even])
|
50
50
|
|
51
51
|
close_intax = MiGA::TaxDist.aai_taxtest(99.0, :intax, engine: :diamond)
|
data/utils/index_metadata.rb
CHANGED
@@ -16,14 +16,13 @@ def searchable(db, d, k, v)
|
|
16
16
|
d.name, k.to_s, " #{v.to_s.downcase.gsub(/[^A-Za-z0-9\-]+/, ' ')} "
|
17
17
|
end
|
18
18
|
|
19
|
-
p.each_dataset do |
|
19
|
+
p.each_dataset do |d|
|
20
20
|
next unless d.is_ref?
|
21
21
|
next unless d.is_active?
|
22
22
|
searchable(db, d, :name, d.name)
|
23
23
|
d.metadata.each do |k, v|
|
24
24
|
next if [:created, :updated].include? k
|
25
|
-
v = v.sorted_ranks.map{ |r| r[1] }.join(
|
25
|
+
v = v.sorted_ranks.map{ |r| r[1] }.join(' ') if k == :tax
|
26
26
|
searchable(db, d, k, v)
|
27
27
|
end
|
28
28
|
end
|
29
|
-
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: miga-base
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.7.
|
4
|
+
version: 0.7.4.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Luis M. Rodriguez-R
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-04-
|
11
|
+
date: 2020-04-28 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: daemons
|
@@ -124,6 +124,7 @@ files:
|
|
124
124
|
- lib/miga/cli/action/date.rb
|
125
125
|
- lib/miga/cli/action/derep_wf.rb
|
126
126
|
- lib/miga/cli/action/doctor.rb
|
127
|
+
- lib/miga/cli/action/doctor/base.rb
|
127
128
|
- lib/miga/cli/action/edit.rb
|
128
129
|
- lib/miga/cli/action/files.rb
|
129
130
|
- lib/miga/cli/action/find.rb
|