miga-base 0.7.18.4 → 0.7.22.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/miga/cli/action/add_result.rb +1 -1
- data/lib/miga/cli/action/lair.rb +12 -5
- data/lib/miga/cli/action/next_step.rb +3 -11
- data/lib/miga/common/with_result.rb +39 -1
- data/lib/miga/daemon.rb +24 -14
- data/lib/miga/daemon/base.rb +1 -1
- data/lib/miga/dataset.rb +2 -0
- data/lib/miga/dataset/result.rb +2 -13
- data/lib/miga/lair.rb +4 -2
- data/lib/miga/project.rb +17 -2
- data/lib/miga/project/base.rb +1 -4
- data/lib/miga/project/dataset.rb +3 -1
- data/lib/miga/project/result.rb +5 -11
- data/lib/miga/result.rb +17 -1
- data/lib/miga/tax_index.rb +20 -12
- data/lib/miga/version.rb +2 -2
- data/scripts/aai_distances.bash +4 -5
- data/scripts/ani_distances.bash +4 -5
- data/scripts/clade_finding.bash +3 -4
- data/scripts/haai_distances.bash +4 -5
- data/scripts/miga.bash +18 -0
- data/scripts/ogs.bash +3 -4
- data/scripts/p.bash +1 -1
- data/scripts/project_stats.bash +2 -5
- data/scripts/subclades.bash +4 -5
- data/test/common_test.rb +30 -0
- data/test/daemon_test.rb +17 -1
- data/test/hook_test.rb +2 -2
- data/test/tax_index_test.rb +26 -0
- metadata +6 -6
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 02ebfc11d56c4647f2ad4aac749d507f5ff142840bc67efec81eac54c4514b13
|
4
|
+
data.tar.gz: cdc1c8d3433ea6bb2fea77580645d9069b0311ec0fe2a6ca0ad8bae02553a4ea
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: a19fa111709a73948496269ad583773ecc26aaf3e6285f38241614fc33aff509d4d379b5a0bf752b40e59f388d22432440eba977946afdfa59fda2a39f82b0ba
|
7
|
+
data.tar.gz: 7cbafa4d27b1516de8e6cc1c17bc5044fcba085adabb4c9c20d3c5cce6b1b3155aa53e90b4cb2afd86e43d72b63dba32c06e2317651fbcecac63837b3b31b383
|
@@ -20,6 +20,6 @@ class MiGA::Cli::Action::AddResult < MiGA::Cli::Action
|
|
20
20
|
obj = cli.load_project_or_dataset
|
21
21
|
cli.say "Registering result: #{cli[:result]}"
|
22
22
|
r = obj.add_result(cli[:result], true, force: cli[:force])
|
23
|
-
raise
|
23
|
+
raise 'Cannot add result, incomplete expected files' if r.nil?
|
24
24
|
end
|
25
25
|
end
|
data/lib/miga/cli/action/lair.rb
CHANGED
@@ -25,6 +25,10 @@ class MiGA::Cli::Action::Lair < MiGA::Cli::Action
|
|
25
25
|
'-p', '--path PATH',
|
26
26
|
'(Mandatory) Path to the directory where the MiGA projects are located'
|
27
27
|
) { |v| cli[:path] = v }
|
28
|
+
opt.on(
|
29
|
+
'--exclude NAME1,NAME2,NAME3', Array,
|
30
|
+
'Exclude these projects (identified by name) from the lair'
|
31
|
+
) { |v| cli[:exclude] = v }
|
28
32
|
opt.on(
|
29
33
|
'--json PATH',
|
30
34
|
'Path to a custom daemon definition in json format'
|
@@ -80,19 +84,22 @@ class MiGA::Cli::Action::Lair < MiGA::Cli::Action
|
|
80
84
|
|
81
85
|
def perform
|
82
86
|
cli.ensure_par(path: '-p')
|
87
|
+
k_opts = %i[
|
88
|
+
json latency wait_for keep_inactive trust_timestamp name dry exclude
|
89
|
+
]
|
90
|
+
opts = Hash[k_opts.map { |k| [k, cli[k]] }]
|
91
|
+
lair = MiGA::Lair.new(cli[:path], opts)
|
92
|
+
|
83
93
|
case cli.operation.to_sym
|
84
94
|
when :terminate
|
85
|
-
|
95
|
+
lair.terminate_daemons
|
86
96
|
when :list
|
87
97
|
o = []
|
88
|
-
|
98
|
+
lair.each_daemon do |d|
|
89
99
|
o << [d.daemon_name, d.class, d.daemon_home, d.active?, d.last_alive]
|
90
100
|
end
|
91
101
|
cli.table(%w[name class path active last_alive], o)
|
92
102
|
else
|
93
|
-
k_opts = %i[json latency wait_for keep_inactive trust_timestamp name dry]
|
94
|
-
opts = Hash[k_opts.map { |k| [k, cli[k]] }]
|
95
|
-
lair = MiGA::Lair.new(cli[:path], opts)
|
96
103
|
lair.daemon(cli.operation, cli[:daemon_opts])
|
97
104
|
end
|
98
105
|
end
|
@@ -11,16 +11,8 @@ class MiGA::Cli::Action::NextStep < MiGA::Cli::Action
|
|
11
11
|
end
|
12
12
|
|
13
13
|
def perform
|
14
|
-
|
15
|
-
n =
|
16
|
-
|
17
|
-
n = p.next_distances(false)
|
18
|
-
n ||= p.next_inclade(false)
|
19
|
-
else
|
20
|
-
d = cli.load_dataset
|
21
|
-
n = d.next_preprocessing if d.is_active?
|
22
|
-
end
|
23
|
-
n ||= '?'
|
24
|
-
cli.puts n
|
14
|
+
obj = cli.load_project_or_dataset
|
15
|
+
n = obj.next_task
|
16
|
+
cli.puts(n || '?')
|
25
17
|
end
|
26
18
|
end
|
@@ -1,6 +1,8 @@
|
|
1
1
|
##
|
2
2
|
# Helper module including specific functions to handle objects that
|
3
|
-
# have results.
|
3
|
+
# have results. The class including this module must implement methods
|
4
|
+
# +.RESULT_DIRS+, +#ignore_task?+, +#metadata+, +#project+,
|
5
|
+
# and +#inactivate!+.
|
4
6
|
module MiGA::Common::WithResult
|
5
7
|
##
|
6
8
|
# Result directories as a Hash
|
@@ -68,4 +70,40 @@ module MiGA::Common::WithResult
|
|
68
70
|
def get_result(task)
|
69
71
|
add_result(task, false)
|
70
72
|
end
|
73
|
+
|
74
|
+
##
|
75
|
+
# Get the next task from +tasks+, saving intermediate results if +save+.
|
76
|
+
# If +tasks+ is +nil+ (default), it uses the entire list of tasks.
|
77
|
+
# Returns a Symbol.
|
78
|
+
def next_task(tasks = nil, save = false)
|
79
|
+
tasks ||= result_dirs.keys
|
80
|
+
tasks.find do |t|
|
81
|
+
if ignore_task?(t)
|
82
|
+
# Do not run if this step is to be ignored
|
83
|
+
false
|
84
|
+
else
|
85
|
+
res = add_result(t, save)
|
86
|
+
if res.nil?
|
87
|
+
# Run if the step has not been calculated,
|
88
|
+
# unless too many attempts were already made
|
89
|
+
if (metadata["_try_#{t}"] || 0) > (project.metadata[:max_try] || 10)
|
90
|
+
inactivate! "Too many errors in step #{t}"
|
91
|
+
false
|
92
|
+
else
|
93
|
+
true
|
94
|
+
end
|
95
|
+
else
|
96
|
+
# Run if the step is ready but has to be recalculated
|
97
|
+
res.recalculate? ? true : false
|
98
|
+
end
|
99
|
+
end
|
100
|
+
end
|
101
|
+
end
|
102
|
+
|
103
|
+
##
|
104
|
+
# Mark all results for recalculation
|
105
|
+
def recalculate_tasks(reason = nil)
|
106
|
+
each_result { |res| res.recalculate!(reason).save }
|
107
|
+
end
|
108
|
+
|
71
109
|
end
|
data/lib/miga/daemon.rb
CHANGED
@@ -103,7 +103,7 @@ class MiGA::Daemon < MiGA::MiGA
|
|
103
103
|
##
|
104
104
|
# Queue maintenance tasks as an analysis job
|
105
105
|
def queue_maintenance
|
106
|
-
return if bypass_maintenance?
|
106
|
+
return if bypass_maintenance? || shutdown_when_done?
|
107
107
|
|
108
108
|
say 'Queueing maintenance tasks'
|
109
109
|
queue_job(:maintenance)
|
@@ -214,29 +214,38 @@ class MiGA::Daemon < MiGA::MiGA
|
|
214
214
|
return nil unless get_job(job, ds).nil?
|
215
215
|
|
216
216
|
ds_name = (ds.nil? ? 'miga-project' : ds.name)
|
217
|
-
|
217
|
+
task_name = "#{project.metadata[:name][0..9]}:#{job}:#{ds_name}"
|
218
|
+
to_run = { ds: ds, ds_name: ds_name, job: job, task_name: task_name }
|
219
|
+
say 'Queueing %s:%s' % [to_run[:ds_name], to_run[:job]]
|
220
|
+
@jobs_to_run << to_run
|
221
|
+
end
|
222
|
+
|
223
|
+
##
|
224
|
+
# Construct the command for the given job definition with current
|
225
|
+
# daemon settings
|
226
|
+
def job_cmd(to_run)
|
218
227
|
vars = {
|
219
228
|
'PROJECT' => project.path,
|
220
229
|
'RUNTYPE' => runopts(:type),
|
221
230
|
'CORES' => ppn,
|
222
231
|
'MIGA' => MiGA::MiGA.root_path
|
223
232
|
}
|
224
|
-
vars['DATASET'] = ds.name unless ds.nil?
|
225
|
-
log_dir = File.expand_path("daemon/#{job}", project.path)
|
226
|
-
|
227
|
-
|
228
|
-
|
229
|
-
|
230
|
-
|
233
|
+
vars['DATASET'] = to_run[:ds].name unless to_run[:ds].nil?
|
234
|
+
log_dir = File.expand_path("daemon/#{to_run[:job]}", project.path)
|
235
|
+
FileUtils.mkdir_p(log_dir)
|
236
|
+
var_hsh = {
|
237
|
+
script: MiGA::MiGA.script_path(
|
238
|
+
to_run[:job], miga: vars['MIGA'], project: project
|
239
|
+
),
|
231
240
|
vars: vars.map do |k, v|
|
232
241
|
runopts(:var).miga_variables(key: k, value: v)
|
233
242
|
end.join(runopts(:varsep)),
|
234
243
|
cpus: ppn,
|
235
|
-
log: File.
|
236
|
-
task_name: task_name,
|
237
|
-
miga: File.
|
238
|
-
|
239
|
-
|
244
|
+
log: File.join(log_dir, "#{to_run[:ds_name]}.log"),
|
245
|
+
task_name: to_run[:task_name],
|
246
|
+
miga: File.join(MiGA::MiGA.root_path, 'bin/miga').shellescape
|
247
|
+
}
|
248
|
+
runopts(:cmd).miga_variables(var_hsh)
|
240
249
|
end
|
241
250
|
|
242
251
|
##
|
@@ -312,6 +321,7 @@ class MiGA::Daemon < MiGA::MiGA
|
|
312
321
|
# Launch the job described by Hash +job+ to +hostk+-th host
|
313
322
|
def launch_job(job, hostk = nil)
|
314
323
|
# Execute job
|
324
|
+
job[:cmd] = job_cmd(job)
|
315
325
|
case runopts(:type)
|
316
326
|
when 'ssh'
|
317
327
|
# Remote job
|
data/lib/miga/daemon/base.rb
CHANGED
data/lib/miga/dataset.rb
CHANGED
@@ -107,6 +107,7 @@ class MiGA::Dataset < MiGA::MiGA
|
|
107
107
|
metadata[:warn] = "Inactive: #{reason}" unless reason.nil?
|
108
108
|
metadata[:inactive] = true
|
109
109
|
metadata.save
|
110
|
+
project.recalculate_tasks('Reference dataset inactivated') if ref?
|
110
111
|
pull_hook :on_inactivate
|
111
112
|
end
|
112
113
|
|
@@ -116,6 +117,7 @@ class MiGA::Dataset < MiGA::MiGA
|
|
116
117
|
metadata[:inactive] = nil
|
117
118
|
metadata[:warn] = nil if metadata[:warn] && metadata[:warn] =~ /^Inactive: /
|
118
119
|
metadata.save
|
120
|
+
project.recalculate_tasks('Reference dataset activated') if ref?
|
119
121
|
pull_hook :on_activate
|
120
122
|
end
|
121
123
|
|
data/lib/miga/dataset/result.rb
CHANGED
@@ -77,19 +77,8 @@ module MiGA::Dataset::Result
|
|
77
77
|
# Returns the key symbol of the next task that needs to be executed or nil.
|
78
78
|
# Passes +save+ to #add_result.
|
79
79
|
def next_preprocessing(save = false)
|
80
|
-
|
81
|
-
|
82
|
-
if ignore_task? t
|
83
|
-
false
|
84
|
-
elsif add_result(t, save).nil?
|
85
|
-
if (metadata["_try_#{t}"] || 0) > (project.metadata[:max_try] || 10)
|
86
|
-
inactivate! "Too many errors in step #{t}"
|
87
|
-
false
|
88
|
-
else
|
89
|
-
true
|
90
|
-
end
|
91
|
-
end
|
92
|
-
end
|
80
|
+
first_preprocessing(save) if save
|
81
|
+
next_task(nil, save)
|
93
82
|
end
|
94
83
|
|
95
84
|
##
|
data/lib/miga/lair.rb
CHANGED
@@ -31,6 +31,7 @@ class MiGA::Lair < MiGA::MiGA
|
|
31
31
|
# by default: true
|
32
32
|
# - dry: Only report when daemons would be launched, but don't actually launch
|
33
33
|
# them
|
34
|
+
# - exclude: Array of project names to be excluded from the lair
|
34
35
|
def initialize(path, opts = {})
|
35
36
|
@path = File.expand_path(path)
|
36
37
|
@options = opts
|
@@ -41,7 +42,8 @@ class MiGA::Lair < MiGA::MiGA
|
|
41
42
|
keep_inactive: false,
|
42
43
|
trust_timestamp: true,
|
43
44
|
name: File.basename(@path),
|
44
|
-
dry: false
|
45
|
+
dry: false,
|
46
|
+
exclude: []
|
45
47
|
}.each { |k, v| @options[k] = v if @options[k].nil? }
|
46
48
|
end
|
47
49
|
|
@@ -109,7 +111,7 @@ class MiGA::Lair < MiGA::MiGA
|
|
109
111
|
project = MiGA::Project.load(f)
|
110
112
|
raise "Cannot load project: #{f}" if project.nil?
|
111
113
|
|
112
|
-
yield(project)
|
114
|
+
yield(project) unless options[:exclude].include?(project.name)
|
113
115
|
elsif Dir.exist? f
|
114
116
|
each_project(f) { |p| yield(p) }
|
115
117
|
end
|
data/lib/miga/project.rb
CHANGED
@@ -97,13 +97,28 @@ class MiGA::Project < MiGA::MiGA
|
|
97
97
|
|
98
98
|
##
|
99
99
|
# Is this a clade project?
|
100
|
-
def
|
100
|
+
def clade?
|
101
101
|
type == :clade
|
102
102
|
end
|
103
103
|
|
104
|
+
##
|
105
|
+
# Same as active? For backward compatibility
|
106
|
+
alias is_clade? clade?
|
107
|
+
|
104
108
|
##
|
105
109
|
# Is this a project for multi-organism datasets?
|
106
|
-
def
|
110
|
+
def multi?
|
107
111
|
@@KNOWN_TYPES[type][:multi]
|
108
112
|
end
|
113
|
+
|
114
|
+
##
|
115
|
+
# Same as multi? For backward compatibility
|
116
|
+
alias is_multi? multi?
|
117
|
+
|
118
|
+
##
|
119
|
+
# Is this project active? Currently a dummy function, returns
|
120
|
+
# always true.
|
121
|
+
def active?
|
122
|
+
true
|
123
|
+
end
|
109
124
|
end
|
data/lib/miga/project/base.rb
CHANGED
@@ -50,14 +50,11 @@ module MiGA::Project::Base
|
|
50
50
|
07.annotation/01.function/01.essential
|
51
51
|
07.annotation/01.function/02.ssu
|
52
52
|
07.annotation/02.taxonomy/01.mytaxa
|
53
|
-
07.annotation/03.qa 07.annotation/03.qa/
|
54
|
-
07.annotation/03.qa/02.mytaxa_scan
|
53
|
+
07.annotation/03.qa 07.annotation/03.qa/02.mytaxa_scan
|
55
54
|
08.mapping 08.mapping/01.read-ctg 08.mapping/02.read-gene
|
56
55
|
09.distances 09.distances/01.haai 09.distances/02.aai
|
57
56
|
09.distances/03.ani 09.distances/04.ssu 09.distances/05.taxonomy
|
58
57
|
10.clades 10.clades/01.find 10.clades/02.ani 10.clades/03.ogs
|
59
|
-
10.clades/04.phylogeny 10.clades/04.phylogeny/01.essential
|
60
|
-
10.clades/04.phylogeny/02.core 10.clades/05.metadata
|
61
58
|
90.stats
|
62
59
|
]
|
63
60
|
|
data/lib/miga/project/dataset.rb
CHANGED
@@ -48,10 +48,11 @@ module MiGA::Project::Dataset
|
|
48
48
|
# Add dataset identified by +name+ and return MiGA::Dataset.
|
49
49
|
def add_dataset(name)
|
50
50
|
unless metadata[:datasets].include? name
|
51
|
-
MiGA::Dataset.new(self, name)
|
51
|
+
d = MiGA::Dataset.new(self, name)
|
52
52
|
@metadata[:datasets] << name
|
53
53
|
@dataset_names_hash = nil # Ensure loading even if +do_not_save+ is true
|
54
54
|
save
|
55
|
+
recalculate_tasks('New reference dataset added') if d.ref? && d.active?
|
55
56
|
pull_hook(:on_add_dataset, name)
|
56
57
|
end
|
57
58
|
dataset(name)
|
@@ -65,6 +66,7 @@ module MiGA::Project::Dataset
|
|
65
66
|
|
66
67
|
self.metadata[:datasets].delete(name)
|
67
68
|
save
|
69
|
+
recalculate_tasks('Reference dataset unlinked') if d.ref? && d.active?
|
68
70
|
pull_hook(:on_unlink_dataset, name)
|
69
71
|
d
|
70
72
|
end
|
data/lib/miga/project/result.rb
CHANGED
@@ -23,6 +23,11 @@ module MiGA::Project::Result
|
|
23
23
|
self
|
24
24
|
end
|
25
25
|
|
26
|
+
##
|
27
|
+
# Do nothing, only to comply with MiGA::Common::WithResult
|
28
|
+
def inactivate!(reason = nil)
|
29
|
+
end
|
30
|
+
|
26
31
|
##
|
27
32
|
# Is this +task+ to be bypassed?
|
28
33
|
def ignore_task?(task)
|
@@ -45,17 +50,6 @@ module MiGA::Project::Result
|
|
45
50
|
next_task(@@INCLADE_TASKS, save)
|
46
51
|
end
|
47
52
|
|
48
|
-
##
|
49
|
-
# Get the next task from +tasks+, saving intermediate results if +save+.
|
50
|
-
# If +tasks+ is +nil+ (default), it uses the entire list of tasks.
|
51
|
-
# Returns a Symbol.
|
52
|
-
def next_task(tasks = nil, save = true)
|
53
|
-
tasks ||= @@DISTANCE_TASKS + @@INCLADE_TASKS
|
54
|
-
tasks.find do |t|
|
55
|
-
ignore_task?(t) ? false : add_result(t, save).nil?
|
56
|
-
end
|
57
|
-
end
|
58
|
-
|
59
53
|
private
|
60
54
|
|
61
55
|
##
|
data/lib/miga/result.rb
CHANGED
@@ -59,9 +59,25 @@ class MiGA::Result < MiGA::MiGA
|
|
59
59
|
end
|
60
60
|
|
61
61
|
##
|
62
|
-
# Register the result as cleaned
|
62
|
+
# Register the result as cleaned, returns self
|
63
63
|
def clean!
|
64
64
|
self[:clean] = true
|
65
|
+
self
|
66
|
+
end
|
67
|
+
|
68
|
+
##
|
69
|
+
# Is the result marked to be recalculated? Returns Boolean
|
70
|
+
def recalculate?
|
71
|
+
!!self[:recalculate]
|
72
|
+
end
|
73
|
+
|
74
|
+
##
|
75
|
+
# Mark the result to be recalculated, returns self
|
76
|
+
def recalculate!(reason = nil)
|
77
|
+
self[:recalculate] = true
|
78
|
+
self[:recalculate_why] = reason
|
79
|
+
self[:recalculate_when] = Time.now.to_s
|
80
|
+
self
|
65
81
|
end
|
66
82
|
|
67
83
|
##
|
data/lib/miga/tax_index.rb
CHANGED
@@ -17,7 +17,7 @@ class MiGA::TaxIndex < MiGA::MiGA
|
|
17
17
|
##
|
18
18
|
# Initialize an empty MiGA::TaxIndex
|
19
19
|
def initialize
|
20
|
-
@root = MiGA::TaxIndexTaxon.new
|
20
|
+
@root = MiGA::TaxIndexTaxon.new(:root, 'biota')
|
21
21
|
@datasets = []
|
22
22
|
end
|
23
23
|
|
@@ -37,21 +37,22 @@ class MiGA::TaxIndex < MiGA::MiGA
|
|
37
37
|
end
|
38
38
|
|
39
39
|
##
|
40
|
-
# Finds all the taxa in the collection at the +rank+ taxonomic rank
|
40
|
+
# Finds all the taxa in the collection at the +rank+ taxonomic rank
|
41
41
|
def taxa_by_rank(rank)
|
42
42
|
rank = MiGA::Taxonomy.normalize_rank(rank)
|
43
43
|
taxa = [@root]
|
44
44
|
select = []
|
45
45
|
loop do
|
46
46
|
new_taxa = []
|
47
|
-
taxa.map
|
47
|
+
taxa.map(&:children).flatten.each do |ch|
|
48
48
|
if ch.rank == rank
|
49
49
|
select << ch
|
50
|
-
elsif
|
50
|
+
elsif !ch.children.empty?
|
51
51
|
new_taxa << ch
|
52
52
|
end
|
53
53
|
end
|
54
|
-
|
54
|
+
taxa = new_taxa
|
55
|
+
break if taxa.empty?
|
55
56
|
end
|
56
57
|
select
|
57
58
|
end
|
@@ -60,7 +61,8 @@ class MiGA::TaxIndex < MiGA::MiGA
|
|
60
61
|
# Generate JSON String for the index.
|
61
62
|
def to_json
|
62
63
|
MiGA::Json.generate(
|
63
|
-
|
64
|
+
root: root.to_hash,
|
65
|
+
datasets: datasets.map(&:name)
|
64
66
|
)
|
65
67
|
end
|
66
68
|
|
@@ -119,27 +121,33 @@ class MiGA::TaxIndexTaxon < MiGA::MiGA
|
|
119
121
|
##
|
120
122
|
# Get the number of datasets in the taxon (including children).
|
121
123
|
def datasets_count
|
122
|
-
children.map
|
124
|
+
children.map(&:datasets_count).reduce(datasets.size, :+)
|
123
125
|
end
|
124
126
|
|
125
127
|
##
|
126
128
|
# Get all the datasets in the taxon (including children).
|
127
129
|
def all_datasets
|
128
|
-
children.map
|
130
|
+
children.map(&:datasets).reduce(datasets, :+)
|
129
131
|
end
|
130
132
|
|
131
133
|
##
|
132
134
|
# JSON String of the taxon.
|
133
135
|
def to_json(*a)
|
134
|
-
{
|
135
|
-
|
136
|
+
{
|
137
|
+
str: tax_str,
|
138
|
+
datasets: datasets.map(&:name),
|
139
|
+
children: children
|
140
|
+
}.to_json(a)
|
136
141
|
end
|
137
142
|
|
138
143
|
##
|
139
144
|
# Hash representation of the taxon.
|
140
145
|
def to_hash
|
141
|
-
{
|
142
|
-
|
146
|
+
{
|
147
|
+
str: tax_str,
|
148
|
+
datasets: datasets.map(&:name),
|
149
|
+
children: children.map(&:to_hash)
|
150
|
+
}
|
143
151
|
end
|
144
152
|
|
145
153
|
##
|
data/lib/miga/version.rb
CHANGED
@@ -8,7 +8,7 @@ module MiGA
|
|
8
8
|
# - Float representing the major.minor version.
|
9
9
|
# - Integer representing gem releases of the current version.
|
10
10
|
# - Integer representing minor changes that require new version number.
|
11
|
-
VERSION = [0.7,
|
11
|
+
VERSION = [0.7, 22, 0]
|
12
12
|
|
13
13
|
##
|
14
14
|
# Nickname for the current major.minor version.
|
@@ -16,7 +16,7 @@ module MiGA
|
|
16
16
|
|
17
17
|
##
|
18
18
|
# Date of the current gem release.
|
19
|
-
VERSION_DATE = Date.new(2021, 1,
|
19
|
+
VERSION_DATE = Date.new(2021, 1, 23)
|
20
20
|
|
21
21
|
##
|
22
22
|
# Reference of MiGA.
|
data/scripts/aai_distances.bash
CHANGED
@@ -4,10 +4,10 @@ set -e
|
|
4
4
|
SCRIPT="aai_distances"
|
5
5
|
# shellcheck source=scripts/miga.bash
|
6
6
|
. "$MIGA/scripts/miga.bash" || exit 1
|
7
|
-
|
7
|
+
DIR="$PROJECT/data/09.distances/02.aai"
|
8
8
|
|
9
9
|
# Initialize
|
10
|
-
|
10
|
+
miga_start_project_step "$DIR"
|
11
11
|
|
12
12
|
echo -n "" > miga-project.log
|
13
13
|
DS=$(miga ls -P "$PROJECT" --ref --no-multi --active)
|
@@ -19,7 +19,7 @@ rm -f miga-project.txt
|
|
19
19
|
for i in $DS ; do
|
20
20
|
echo "SELECT CASE WHEN omega!=0 THEN 'AAI' ELSE 'hAAI_AAI' END," \
|
21
21
|
" seq1, seq2, aai, sd, n, omega from aai;" \
|
22
|
-
| sqlite3 "$i.db" | tr "\\|" "\\t"
|
22
|
+
| sqlite3 "$DIR/$i.db" | tr "\\|" "\\t"
|
23
23
|
echo "$i" >> miga-project.log
|
24
24
|
done
|
25
25
|
) | gzip -9c > miga-project.txt.gz
|
@@ -39,5 +39,4 @@ if(sum(aai[,'a'] != aai[,'b']) > 0){
|
|
39
39
|
" | R --vanilla
|
40
40
|
|
41
41
|
# Finalize
|
42
|
-
|
43
|
-
miga add_result -P "$PROJECT" -r "$SCRIPT" -f
|
42
|
+
miga_end_project_step "$DIR"
|
data/scripts/ani_distances.bash
CHANGED
@@ -4,10 +4,10 @@ set -e
|
|
4
4
|
SCRIPT="ani_distances"
|
5
5
|
# shellcheck source=scripts/miga.bash
|
6
6
|
. "$MIGA/scripts/miga.bash" || exit 1
|
7
|
-
|
7
|
+
DIR="$PROJECT/data/09.distances/03.ani"
|
8
8
|
|
9
9
|
# Initialize
|
10
|
-
|
10
|
+
miga_start_project_step "$DIR"
|
11
11
|
|
12
12
|
echo -n "" > miga-project.log
|
13
13
|
DS=$(miga ls -P "$PROJECT" --ref --no-multi --active)
|
@@ -18,7 +18,7 @@ rm -f miga-project.txt
|
|
18
18
|
echo "metric a b value sd n omega" | tr " " "\\t"
|
19
19
|
for i in $DS ; do
|
20
20
|
echo "SELECT 'ANI', seq1, seq2, ani, sd, n, omega from ani ;" \
|
21
|
-
| sqlite3 "$i.db" | tr "\\|" "\\t"
|
21
|
+
| sqlite3 "$DIR/$i.db" | tr "\\|" "\\t"
|
22
22
|
echo "$i" >> miga-project.log
|
23
23
|
done
|
24
24
|
) | gzip -9c > miga-project.txt.gz
|
@@ -38,5 +38,4 @@ if(sum(ani[,'a'] != ani[,'b']) > 0){
|
|
38
38
|
" | R --vanilla
|
39
39
|
|
40
40
|
# Finalize
|
41
|
-
|
42
|
-
miga add_result -P "$PROJECT" -r "$SCRIPT" -f
|
41
|
+
miga_end_project_step "$DIR"
|
data/scripts/clade_finding.bash
CHANGED
@@ -4,14 +4,13 @@ set -e
|
|
4
4
|
SCRIPT="clade_finding"
|
5
5
|
# shellcheck source=scripts/miga.bash
|
6
6
|
. "$MIGA/scripts/miga.bash" || exit 1
|
7
|
-
|
7
|
+
DIR="$PROJECT/data/10.clades/01.find"
|
8
8
|
|
9
9
|
# Initialize
|
10
|
-
|
10
|
+
miga_start_project_step "$DIR"
|
11
11
|
|
12
12
|
# Run
|
13
13
|
ruby -I "$MIGA/lib" "$MIGA/utils/subclades.rb" "$PROJECT" "$SCRIPT"
|
14
14
|
|
15
15
|
# Finalize
|
16
|
-
|
17
|
-
miga add_result -P "$PROJECT" -r "$SCRIPT" -f
|
16
|
+
miga_end_project_step "$DIR"
|
data/scripts/haai_distances.bash
CHANGED
@@ -4,10 +4,10 @@ set -e
|
|
4
4
|
SCRIPT="haai_distances"
|
5
5
|
# shellcheck source=scripts/miga.bash
|
6
6
|
. "$MIGA/scripts/miga.bash" || exit 1
|
7
|
-
|
7
|
+
DIR="$PROJECT/data/09.distances/01.haai"
|
8
8
|
|
9
9
|
# Initialize
|
10
|
-
|
10
|
+
miga_start_project_step "$DIR"
|
11
11
|
|
12
12
|
# Cleanup databases
|
13
13
|
ruby -I "$MIGA/lib" "$MIGA/utils/cleanup-databases.rb" "$PROJECT" "$CORES"
|
@@ -22,7 +22,7 @@ rm -f miga-project.txt
|
|
22
22
|
echo "metric a b value sd n omega" | tr " " "\\t"
|
23
23
|
for i in $DS ; do
|
24
24
|
echo "SELECT 'hAAI', seq1, seq2, aai, sd, n, omega from aai ;" \
|
25
|
-
| sqlite3 "$i.db" | tr "\\|" "\\t"
|
25
|
+
| sqlite3 "$DIR/$i.db" | tr "\\|" "\\t"
|
26
26
|
echo "$i" >> miga-project.log
|
27
27
|
done
|
28
28
|
) | gzip -9c > miga-project.txt.gz
|
@@ -42,5 +42,4 @@ if(sum(haai[,'a'] != haai[,'b']) > 0){
|
|
42
42
|
" | R --vanilla
|
43
43
|
|
44
44
|
# Finalize
|
45
|
-
|
46
|
-
miga add_result -P "$PROJECT" -r "$SCRIPT" -f
|
45
|
+
miga_end_project_step "$DIR"
|
data/scripts/miga.bash
CHANGED
@@ -9,6 +9,24 @@ SCRIPT=${SCRIPT:-$(basename "$0" .bash)}
|
|
9
9
|
|
10
10
|
function exists { [[ -e "$1" ]] ; }
|
11
11
|
function fx_exists { [[ $(type -t "$1") == "function" ]] ; }
|
12
|
+
function miga_start_project_step {
|
13
|
+
local dir="$1"
|
14
|
+
local dir_r="${dir}.running"
|
15
|
+
mkdir -p "$dir"
|
16
|
+
mkdir -p "$dir_r"
|
17
|
+
cd "$dir_r"
|
18
|
+
miga date > "miga-project.start"
|
19
|
+
}
|
20
|
+
function miga_end_project_step {
|
21
|
+
local dir="$1"
|
22
|
+
local dir_r="${dir}.running"
|
23
|
+
cd "$dir"
|
24
|
+
rm -rf miga-project.*
|
25
|
+
mv "$dir_r"/miga-project.* .
|
26
|
+
rmdir "$dir_r" &>/dev/null || true
|
27
|
+
miga date > "miga-project.done"
|
28
|
+
miga add_result -P "$PROJECT" -r "$SCRIPT" -f
|
29
|
+
}
|
12
30
|
|
13
31
|
if [[ "$SCRIPT" != "d" && "$SCRIPT" != "p" ]] ; then
|
14
32
|
echo ""
|
data/scripts/ogs.bash
CHANGED
@@ -4,10 +4,10 @@ set -e
|
|
4
4
|
SCRIPT="ogs"
|
5
5
|
# shellcheck source=scripts/miga.bash
|
6
6
|
. "$MIGA/scripts/miga.bash" || exit 1
|
7
|
-
|
7
|
+
DIR="$PROJECT/data/10.clades/03.ogs"
|
8
8
|
|
9
9
|
# Initialize
|
10
|
-
|
10
|
+
miga_start_project_step "$DIR"
|
11
11
|
|
12
12
|
DS=$(miga ls -P "$PROJECT" --ref --no-multi)
|
13
13
|
|
@@ -51,5 +51,4 @@ else
|
|
51
51
|
fi
|
52
52
|
|
53
53
|
# Finalize
|
54
|
-
|
55
|
-
miga add_result -P "$PROJECT" -r "$SCRIPT" -f
|
54
|
+
miga_end_project_step "$DIR"
|
data/scripts/p.bash
CHANGED
data/scripts/project_stats.bash
CHANGED
@@ -5,11 +5,9 @@ SCRIPT="project_stats"
|
|
5
5
|
# shellcheck source=scripts/miga.bash
|
6
6
|
. "$MIGA/scripts/miga.bash" || exit 1
|
7
7
|
DIR="$PROJECT/data/90.stats"
|
8
|
-
[[ -d "$DIR" ]] || mkdir -p "$DIR"
|
9
|
-
cd "$DIR"
|
10
8
|
|
11
9
|
# Initialize
|
12
|
-
|
10
|
+
miga_start_project_step "$DIR"
|
13
11
|
|
14
12
|
# Execute doctor
|
15
13
|
echo "# Doctor"
|
@@ -25,5 +23,4 @@ ruby -I "$MIGA/lib" \
|
|
25
23
|
"$MIGA/utils/index_metadata.rb" "$PROJECT" "miga-project.metadata.db"
|
26
24
|
|
27
25
|
# Finalize
|
28
|
-
|
29
|
-
miga add_result -P "$PROJECT" -r "$SCRIPT" -f
|
26
|
+
miga_end_project_step "$DIR"
|
data/scripts/subclades.bash
CHANGED
@@ -4,14 +4,13 @@ set -e
|
|
4
4
|
SCRIPT="subclades"
|
5
5
|
# shellcheck source=scripts/miga.bash
|
6
6
|
. "$MIGA/scripts/miga.bash" || exit 1
|
7
|
-
|
7
|
+
DIR="$PROJECT/data/10.clades/02.ani"
|
8
8
|
|
9
9
|
# Initialize
|
10
|
-
|
10
|
+
miga_start_project_step "$DIR"
|
11
11
|
|
12
|
-
# Run
|
12
|
+
# Run
|
13
13
|
ruby -I "$MIGA/lib" "$MIGA/utils/subclades.rb" "$PROJECT" "$SCRIPT"
|
14
14
|
|
15
15
|
# Finalize
|
16
|
-
|
17
|
-
miga add_result -P "$PROJECT" -r "$SCRIPT" -f
|
16
|
+
miga_end_project_step "$DIR"
|
data/test/common_test.rb
CHANGED
@@ -72,4 +72,34 @@ class CommonTest < Test::Unit::TestCase
|
|
72
72
|
assert_not_predicate('C3-PO', :miga_name?)
|
73
73
|
assert_equal("123\n1\n", '1231'.wrap_width(3))
|
74
74
|
end
|
75
|
+
|
76
|
+
def test_advance
|
77
|
+
m = MiGA::MiGA.new
|
78
|
+
|
79
|
+
# Check advance when missing total
|
80
|
+
o = capture_stderr { m.advance('x', 0) }.string
|
81
|
+
assert_match(/\] x *\r/, o)
|
82
|
+
|
83
|
+
# Initialize advance
|
84
|
+
o = capture_stderr { m.advance('x', 0, 10) }.string
|
85
|
+
assert_match(/\] x 0\.0% \(0\/10\) *\r/, o)
|
86
|
+
|
87
|
+
# Insufficient data for prediction
|
88
|
+
sleep(1)
|
89
|
+
o = capture_stderr { m.advance('x', 1, 10) }.string
|
90
|
+
assert_match(/\] x 10\.0% \(1\/10\) *\r/, o)
|
91
|
+
|
92
|
+
# Predict time
|
93
|
+
sleep(1)
|
94
|
+
o = capture_stderr { m.advance('x', 2, 10) }.string
|
95
|
+
assert_match(/\] x 20\.0% \(2\/10\) [7-9]s left *\r/, o)
|
96
|
+
end
|
97
|
+
|
98
|
+
def test_num_suffix
|
99
|
+
m = MiGA::MiGA.new
|
100
|
+
assert_equal('12', m.num_suffix(12))
|
101
|
+
assert_equal('1.5K', m.num_suffix(1.5e3))
|
102
|
+
assert_equal('1.0M', m.num_suffix(1024**2 + 1, true))
|
103
|
+
assert_equal('1.1G', m.num_suffix(1024**3))
|
104
|
+
end
|
75
105
|
end
|
data/test/daemon_test.rb
CHANGED
@@ -48,7 +48,10 @@ class DaemonTest < Test::Unit::TestCase
|
|
48
48
|
end
|
49
49
|
assert_match(/Queueing #{ds.name}:d/, out.string)
|
50
50
|
assert_equal(1, d.jobs_to_run.size)
|
51
|
-
assert_equal(
|
51
|
+
assert_equal(
|
52
|
+
'echo project0:d:ds1 >/dev/null',
|
53
|
+
d.job_cmd(d.jobs_to_run.first)
|
54
|
+
)
|
52
55
|
assert_equal(d.jobs_to_run.first, d.get_job(:d, ds))
|
53
56
|
end
|
54
57
|
|
@@ -335,4 +338,17 @@ class DaemonTest < Test::Unit::TestCase
|
|
335
338
|
capture_stderr { d.in_loop }
|
336
339
|
assert_equal([], d.jobs_running)
|
337
340
|
end
|
341
|
+
|
342
|
+
def test_show_log
|
343
|
+
d = daemon
|
344
|
+
assert(d.show_log?)
|
345
|
+
d.runopts(:show_log, false)
|
346
|
+
assert(!d.show_log?)
|
347
|
+
d.show_log!
|
348
|
+
assert(d.show_log?)
|
349
|
+
assert_equal($stderr, d.logfh)
|
350
|
+
d.show_summary!
|
351
|
+
assert(!d.show_log?)
|
352
|
+
assert_not_equal($stderr, d.logfh)
|
353
|
+
end
|
338
354
|
end
|
data/test/hook_test.rb
CHANGED
@@ -107,10 +107,10 @@ class HookTest < Test::Unit::TestCase
|
|
107
107
|
File.join(project.path, 'data', '90.stats', "miga-project.#{ext}")
|
108
108
|
)
|
109
109
|
end
|
110
|
-
assert_equal(:project_stats, project.next_task
|
110
|
+
assert_equal(:project_stats, project.next_task)
|
111
111
|
assert_equal(:test, $res)
|
112
112
|
assert_equal(1, $counter)
|
113
|
-
assert_equal(:haai_distances, project.next_task)
|
113
|
+
assert_equal(:haai_distances, project.next_task(nil, true))
|
114
114
|
assert_equal(:project_stats, $res)
|
115
115
|
assert_equal(2, $counter)
|
116
116
|
end
|
data/test/tax_index_test.rb
CHANGED
@@ -35,4 +35,30 @@ class TaxIndexTest < Test::Unit::TestCase
|
|
35
35
|
ti = MiGA::TaxIndex.new
|
36
36
|
assert_equal("root:biota: 0\n", ti.to_tab)
|
37
37
|
end
|
38
|
+
|
39
|
+
def test_tax_index_taxon
|
40
|
+
taxon = MiGA::TaxIndexTaxon.new(:root, 'biota')
|
41
|
+
assert_empty(taxon.datasets)
|
42
|
+
assert_equal('root:biota', taxon.to_hash[:str])
|
43
|
+
assert_equal(
|
44
|
+
'{"str":"root:biota","datasets":[],"children":[]}',
|
45
|
+
taxon.to_json
|
46
|
+
)
|
47
|
+
assert_empty(taxon.all_datasets)
|
48
|
+
assert_equal(0, taxon.datasets_count)
|
49
|
+
end
|
50
|
+
|
51
|
+
def test_taxa_by_rank
|
52
|
+
initialize_miga_home
|
53
|
+
d = dataset
|
54
|
+
d.metadata[:tax] = MiGA::Taxonomy.new('k:Fantasia g:Unicornia')
|
55
|
+
|
56
|
+
# Empty
|
57
|
+
ti = MiGA::TaxIndex.new
|
58
|
+
assert_empty(ti.taxa_by_rank(:genus))
|
59
|
+
|
60
|
+
# Unicorn
|
61
|
+
ti << d
|
62
|
+
assert_equal('Unicornia', ti.taxa_by_rank(:genus).first.name)
|
63
|
+
end
|
38
64
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: miga-base
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.7.
|
4
|
+
version: 0.7.22.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Luis M. Rodriguez-R
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2021-01-
|
11
|
+
date: 2021-01-23 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: daemons
|
@@ -542,7 +542,7 @@ homepage: http://enve-omics.ce.gatech.edu/miga
|
|
542
542
|
licenses:
|
543
543
|
- Artistic-2.0
|
544
544
|
metadata: {}
|
545
|
-
post_install_message:
|
545
|
+
post_install_message:
|
546
546
|
rdoc_options:
|
547
547
|
- lib
|
548
548
|
- README.md
|
@@ -563,8 +563,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
563
563
|
- !ruby/object:Gem::Version
|
564
564
|
version: '0'
|
565
565
|
requirements: []
|
566
|
-
rubygems_version: 3.
|
567
|
-
signing_key:
|
566
|
+
rubygems_version: 3.0.3
|
567
|
+
signing_key:
|
568
568
|
specification_version: 4
|
569
569
|
summary: MiGA
|
570
570
|
test_files: []
|