miga-base 0.7.4.0 → 0.7.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/miga/cli.rb +10 -8
- data/lib/miga/cli/action.rb +2 -3
- data/lib/miga/cli/action/about.rb +5 -6
- data/lib/miga/cli/action/add.rb +18 -12
- data/lib/miga/cli/action/add_result.rb +2 -3
- data/lib/miga/cli/action/archive.rb +1 -2
- data/lib/miga/cli/action/classify_wf.rb +8 -6
- data/lib/miga/cli/action/console.rb +0 -1
- data/lib/miga/cli/action/daemon.rb +7 -7
- data/lib/miga/cli/action/date.rb +0 -1
- data/lib/miga/cli/action/derep_wf.rb +5 -4
- data/lib/miga/cli/action/doctor.rb +28 -20
- data/lib/miga/cli/action/doctor/base.rb +29 -6
- data/lib/miga/cli/action/edit.rb +1 -2
- data/lib/miga/cli/action/files.rb +8 -8
- data/lib/miga/cli/action/find.rb +5 -6
- data/lib/miga/cli/action/generic.rb +7 -7
- data/lib/miga/cli/action/get.rb +20 -17
- data/lib/miga/cli/action/get_db.rb +8 -2
- data/lib/miga/cli/action/index_wf.rb +1 -1
- data/lib/miga/cli/action/init.rb +34 -29
- data/lib/miga/cli/action/init/daemon_helper.rb +65 -43
- data/lib/miga/cli/action/lair.rb +7 -7
- data/lib/miga/cli/action/ln.rb +6 -6
- data/lib/miga/cli/action/ls.rb +1 -2
- data/lib/miga/cli/action/ncbi_get.rb +11 -3
- data/lib/miga/cli/action/new.rb +4 -4
- data/lib/miga/cli/action/next_step.rb +0 -1
- data/lib/miga/cli/action/preproc_wf.rb +3 -3
- data/lib/miga/cli/action/quality_wf.rb +1 -1
- data/lib/miga/cli/action/rm.rb +2 -3
- data/lib/miga/cli/action/run.rb +8 -8
- data/lib/miga/cli/action/stats.rb +3 -3
- data/lib/miga/cli/action/summary.rb +7 -6
- data/lib/miga/cli/action/tax_dist.rb +8 -4
- data/lib/miga/cli/action/tax_index.rb +3 -4
- data/lib/miga/cli/action/tax_set.rb +7 -6
- data/lib/miga/cli/action/tax_test.rb +6 -5
- data/lib/miga/cli/action/wf.rb +21 -19
- data/lib/miga/cli/base.rb +34 -32
- data/lib/miga/cli/objects_helper.rb +24 -17
- data/lib/miga/cli/opt_helper.rb +3 -2
- data/lib/miga/common.rb +2 -5
- data/lib/miga/common/base.rb +15 -16
- data/lib/miga/common/format.rb +8 -5
- data/lib/miga/common/hooks.rb +1 -4
- data/lib/miga/common/path.rb +4 -9
- data/lib/miga/common/with_daemon.rb +5 -2
- data/lib/miga/common/with_daemon_class.rb +1 -1
- data/lib/miga/common/with_result.rb +2 -1
- data/lib/miga/daemon.rb +51 -35
- data/lib/miga/daemon/base.rb +0 -2
- data/lib/miga/dataset.rb +47 -37
- data/lib/miga/dataset/base.rb +52 -37
- data/lib/miga/dataset/hooks.rb +3 -4
- data/lib/miga/dataset/result.rb +17 -1
- data/lib/miga/json.rb +5 -7
- data/lib/miga/lair.rb +4 -0
- data/lib/miga/metadata.rb +4 -3
- data/lib/miga/project.rb +29 -20
- data/lib/miga/project/base.rb +52 -37
- data/lib/miga/project/dataset.rb +27 -13
- data/lib/miga/project/hooks.rb +0 -3
- data/lib/miga/project/result.rb +14 -5
- data/lib/miga/remote_dataset.rb +85 -72
- data/lib/miga/remote_dataset/base.rb +11 -13
- data/lib/miga/remote_dataset/download.rb +33 -12
- data/lib/miga/result.rb +34 -25
- data/lib/miga/result/base.rb +0 -2
- data/lib/miga/result/dates.rb +1 -3
- data/lib/miga/result/source.rb +15 -16
- data/lib/miga/result/stats.rb +36 -25
- data/lib/miga/tax_dist.rb +6 -3
- data/lib/miga/tax_index.rb +17 -17
- data/lib/miga/taxonomy.rb +6 -1
- data/lib/miga/taxonomy/base.rb +19 -15
- data/lib/miga/version.rb +19 -16
- data/test/common_test.rb +3 -11
- data/test/daemon_helper.rb +38 -0
- data/test/daemon_test.rb +73 -101
- data/test/dataset_test.rb +58 -59
- data/test/format_test.rb +3 -11
- data/test/hook_test.rb +50 -55
- data/test/json_test.rb +7 -8
- data/test/lair_test.rb +22 -28
- data/test/metadata_test.rb +6 -14
- data/test/project_test.rb +33 -39
- data/test/remote_dataset_test.rb +20 -28
- data/test/result_stats_test.rb +17 -27
- data/test/result_test.rb +41 -34
- data/test/tax_dist_test.rb +0 -2
- data/test/tax_index_test.rb +4 -10
- data/test/taxonomy_test.rb +7 -9
- data/test/test_helper.rb +42 -1
- data/test/with_daemon_test.rb +14 -22
- data/utils/cleanup-databases.rb +6 -5
- data/utils/distance/base.rb +0 -1
- data/utils/distance/commands.rb +19 -12
- data/utils/distance/database.rb +24 -21
- data/utils/distance/pipeline.rb +12 -9
- data/utils/distance/runner.rb +14 -13
- data/utils/distance/temporal.rb +1 -3
- data/utils/distances.rb +1 -1
- data/utils/domain-ess-genes.rb +7 -7
- data/utils/index_metadata.rb +4 -2
- data/utils/mytaxa_scan.rb +18 -16
- data/utils/representatives.rb +5 -4
- data/utils/requirements.txt +1 -1
- data/utils/subclade/base.rb +0 -1
- data/utils/subclade/pipeline.rb +7 -6
- data/utils/subclade/runner.rb +9 -9
- data/utils/subclade/temporal.rb +0 -2
- data/utils/subclades-compile.rb +39 -37
- data/utils/subclades.rb +1 -1
- metadata +3 -2
@@ -1,15 +1,17 @@
|
|
1
1
|
# @package MiGA
|
2
2
|
# @license Artistic-2.0
|
3
3
|
|
4
|
-
module MiGA::Cli::ObjectsHelper
|
4
|
+
module MiGA::Cli::ObjectsHelper
|
5
5
|
##
|
6
6
|
# Get the project defined in the CLI by parameter +name+ and +flag+
|
7
7
|
def load_project(name = :project, flag = '-P')
|
8
8
|
return @objects[name] unless @objects[name].nil?
|
9
|
+
|
9
10
|
ensure_par(name => flag)
|
10
11
|
say "Loading project: #{self[name]}"
|
11
12
|
@objects[name] = MiGA::Project.load(self[name])
|
12
13
|
raise "Cannot load project: #{self[name]}" if @objects[name].nil?
|
14
|
+
|
13
15
|
@objects[name]
|
14
16
|
end
|
15
17
|
|
@@ -23,6 +25,7 @@ module MiGA::Cli::ObjectsHelper
|
|
23
25
|
end
|
24
26
|
d = load_project.dataset(name)
|
25
27
|
raise "Cannot load dataset: #{self[:dataset]}" if !silent && d.nil?
|
28
|
+
|
26
29
|
return d
|
27
30
|
end
|
28
31
|
|
@@ -37,16 +40,18 @@ module MiGA::Cli::ObjectsHelper
|
|
37
40
|
# If +silent=true+, it allows failures silently
|
38
41
|
def load_and_filter_datasets(silent = false)
|
39
42
|
return @objects[:filtered_datasets] unless @objects[:filtered_datasets].nil?
|
43
|
+
|
40
44
|
say 'Listing datasets'
|
41
|
-
ds =
|
42
|
-
[
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
45
|
+
ds =
|
46
|
+
if !self[:dataset].nil?
|
47
|
+
[load_dataset(silent)].compact
|
48
|
+
elsif !self[:ds_list].nil?
|
49
|
+
File.readlines(self[:ds_list]).map do |i|
|
50
|
+
load_dataset(silent, i.chomp)
|
51
|
+
end.compact
|
52
|
+
else
|
53
|
+
load_project.datasets
|
54
|
+
end
|
50
55
|
k = 0
|
51
56
|
n = ds.size
|
52
57
|
ds.select! do |d|
|
@@ -61,31 +66,34 @@ module MiGA::Cli::ObjectsHelper
|
|
61
66
|
o
|
62
67
|
end
|
63
68
|
say ''
|
64
|
-
ds = ds.values_at(self[:dataset_k]-1) unless self[:dataset_k].nil?
|
69
|
+
ds = ds.values_at(self[:dataset_k] - 1) unless self[:dataset_k].nil?
|
65
70
|
@objects[:filtered_datasets] = ds
|
66
71
|
end
|
67
72
|
|
68
73
|
def load_result
|
69
74
|
return @objects[:result] unless @objects[:result].nil?
|
75
|
+
|
70
76
|
ensure_par(result: '-r')
|
71
77
|
obj = load_project_or_dataset
|
72
78
|
if obj.class.RESULT_DIRS[self[:result]].nil?
|
73
|
-
klass = obj.class.to_s.gsub(/.*::/,'')
|
79
|
+
klass = obj.class.to_s.gsub(/.*::/, '')
|
74
80
|
raise "Unsupported result for #{klass}: #{self[:result]}"
|
75
81
|
end
|
76
82
|
r = obj.add_result(self[:result], false)
|
77
83
|
raise "Cannot load result: #{self[:result]}" if r.nil?
|
84
|
+
|
78
85
|
@objects[:result] = r
|
79
86
|
end
|
80
87
|
|
81
88
|
def add_metadata(obj, cli = self)
|
82
89
|
raise "Unsupported object: #{obj.class}" unless obj.respond_to? :metadata
|
90
|
+
|
83
91
|
cli[:metadata].split(',').each do |pair|
|
84
|
-
(k,v) = pair.split('=')
|
92
|
+
(k, v) = pair.split('=')
|
85
93
|
case v
|
86
|
-
|
87
|
-
|
88
|
-
|
94
|
+
when 'true'; v = true
|
95
|
+
when 'false'; v = false
|
96
|
+
when 'nil'; v = nil
|
89
97
|
end
|
90
98
|
if k == '_step'
|
91
99
|
obj.metadata["_try_#{v}"] ||= 0
|
@@ -99,4 +107,3 @@ module MiGA::Cli::ObjectsHelper
|
|
99
107
|
obj
|
100
108
|
end
|
101
109
|
end
|
102
|
-
|
data/lib/miga/cli/opt_helper.rb
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
# @package MiGA
|
2
2
|
# @license Artistic-2.0
|
3
3
|
|
4
|
-
module MiGA::Cli::OptHelper
|
4
|
+
module MiGA::Cli::OptHelper
|
5
5
|
##
|
6
6
|
# Send MiGA's banner to OptionParser +opt+
|
7
7
|
def banner(opt)
|
@@ -19,6 +19,7 @@ module MiGA::Cli::OptHelper
|
|
19
19
|
# Executes only once, unless +#opt_common = true+ is passed between calls
|
20
20
|
def opt_common(opt)
|
21
21
|
return unless @opt_common
|
22
|
+
|
22
23
|
if interactive
|
23
24
|
opt.on(
|
24
25
|
'--auto',
|
@@ -159,6 +160,6 @@ module MiGA::Cli::OptHelper
|
|
159
160
|
# If +sym+ is nil, +flag+ is used as Symbol
|
160
161
|
def opt_flag(opt, flag, description, sym = nil)
|
161
162
|
sym = flag.to_sym if sym.nil?
|
162
|
-
opt.on("--#{flag.to_s.gsub('_','-')}", description) { |v| self[sym] = v }
|
163
|
+
opt.on("--#{flag.to_s.gsub('_', '-')}", description) { |v| self[sym] = v }
|
163
164
|
end
|
164
165
|
end
|
data/lib/miga/common.rb
CHANGED
@@ -11,12 +11,11 @@ require 'miga/common/format'
|
|
11
11
|
# Generic class used to handle system-wide information and methods, and parent
|
12
12
|
# of all other MiGA::* classes.
|
13
13
|
class MiGA::MiGA
|
14
|
-
|
15
14
|
include MiGA::Common
|
16
|
-
|
15
|
+
|
17
16
|
extend MiGA::Common::Path
|
18
17
|
extend MiGA::Common::Format
|
19
|
-
|
18
|
+
|
20
19
|
ENV['MIGA_HOME'] ||= ENV['HOME']
|
21
20
|
|
22
21
|
##
|
@@ -45,6 +44,4 @@ class MiGA::MiGA
|
|
45
44
|
io = par.first.is_a?(IO) ? par.shift : $stderr
|
46
45
|
io.puts(*par.map { |i| "[#{Time.now}] #{i}" })
|
47
46
|
end
|
48
|
-
|
49
47
|
end
|
50
|
-
|
data/lib/miga/common/base.rb
CHANGED
@@ -1,42 +1,43 @@
|
|
1
|
-
|
2
1
|
class MiGA::MiGA
|
3
|
-
|
4
2
|
# Class-level
|
5
3
|
class << self
|
6
4
|
##
|
7
|
-
# Turn on debugging
|
8
|
-
def DEBUG_ON
|
5
|
+
# Turn on debugging
|
6
|
+
def DEBUG_ON
|
7
|
+
@@DEBUG = true
|
8
|
+
end
|
9
9
|
|
10
10
|
##
|
11
|
-
# Turn off debugging
|
12
|
-
def DEBUG_OFF
|
11
|
+
# Turn off debugging
|
12
|
+
def DEBUG_OFF
|
13
|
+
@@DEBUG = false
|
14
|
+
end
|
13
15
|
|
14
16
|
##
|
15
|
-
# Turn on debug tracing (and debugging)
|
17
|
+
# Turn on debug tracing (and debugging)
|
16
18
|
def DEBUG_TRACE_ON
|
17
|
-
@@DEBUG_TRACE=true
|
19
|
+
@@DEBUG_TRACE = true
|
18
20
|
DEBUG_ON()
|
19
21
|
end
|
20
22
|
|
21
23
|
##
|
22
|
-
# Turn off debug tracing (but not debugging)
|
24
|
+
# Turn off debug tracing (but not debugging)
|
23
25
|
def DEBUG_TRACE_OFF
|
24
|
-
@@DEBUG_TRACE=false
|
26
|
+
@@DEBUG_TRACE = false
|
25
27
|
end
|
26
28
|
|
27
29
|
##
|
28
|
-
# Send debug message
|
30
|
+
# Send debug message
|
29
31
|
def DEBUG(*args)
|
30
32
|
$stderr.puts(*args) if @@DEBUG
|
31
33
|
$stderr.puts(
|
32
|
-
|
34
|
+
caller.map { |v| v.gsub(/^/, ' ') }.join("\n")
|
35
|
+
) if @@DEBUG_TRACE
|
33
36
|
end
|
34
37
|
end
|
35
|
-
|
36
38
|
end
|
37
39
|
|
38
40
|
module MiGA::Common
|
39
|
-
|
40
41
|
##
|
41
42
|
# Should debugging information be reported?
|
42
43
|
@@DEBUG = false
|
@@ -44,6 +45,4 @@ module MiGA::Common
|
|
44
45
|
##
|
45
46
|
# Should the trace of debugging information be reported?
|
46
47
|
@@DEBUG_TRACE = false
|
47
|
-
|
48
48
|
end
|
49
|
-
|
data/lib/miga/common/format.rb
CHANGED
@@ -1,4 +1,3 @@
|
|
1
|
-
|
2
1
|
require 'tempfile'
|
3
2
|
require 'zlib'
|
4
3
|
|
@@ -69,30 +68,34 @@ module MiGA::Common::Format
|
|
69
68
|
# a FastA or FastQ file (supports gzipped files). The +format+ must be a
|
70
69
|
# Symbol, one of +:fasta+ or +:fastq+. Additional estimations can be
|
71
70
|
# controlled via the +opts+ Hash. Supported options include:
|
72
|
-
# - +:n50+: If true, it also returns the N50 and the median (in bp)
|
73
|
-
# -
|
71
|
+
# - +:n50+: If true, it also returns the N50 and the median (in bp)
|
72
|
+
# - +:gc+: If true, it also returns the G+C content (in %)
|
73
|
+
# - +:x+: If true, it also returns the undetermined bases content (in %)
|
74
74
|
def seqs_length(file, format, opts = {})
|
75
75
|
fh = file =~ /\.gz/ ? Zlib::GzipReader.open(file) : File.open(file, 'r')
|
76
76
|
l = []
|
77
77
|
gc = 0
|
78
|
+
xn = 0
|
78
79
|
i = 0 # <- Zlib::GzipReader doesn't set `$.`
|
79
80
|
fh.each_line do |ln|
|
80
81
|
i += 1
|
81
82
|
if (format == :fasta and ln =~ /^>/) or
|
82
|
-
|
83
|
+
(format == :fastq and (i % 4) == 1)
|
83
84
|
l << 0
|
84
85
|
elsif format == :fasta or (i % 4) == 2
|
85
86
|
l[l.size - 1] += ln.chomp.size
|
86
87
|
gc += ln.scan(/[GCgc]/).count if opts[:gc]
|
88
|
+
xn += ln.scan(/[XNxn]/).count if opts[:x]
|
87
89
|
end
|
88
90
|
end
|
89
91
|
fh.close
|
90
92
|
|
91
|
-
o = { n: l.size, tot: l.inject(:+) }
|
93
|
+
o = { n: l.size, tot: l.inject(:+), max: l.max }
|
92
94
|
o[:avg] = o[:tot].to_f / l.size
|
93
95
|
o[:var] = l.map { |a| a**2 }.inject(:+).to_f / l.size - o[:avg]**2
|
94
96
|
o[:sd] = Math.sqrt o[:var]
|
95
97
|
o[:gc] = 100.0 * gc / o[:tot] if opts[:gc]
|
98
|
+
o[:x] = 100.0 * xn / o[:tot] if opts[:x]
|
96
99
|
if opts[:n50]
|
97
100
|
l.sort!
|
98
101
|
thr = o[:tot] / 2
|
data/lib/miga/common/hooks.rb
CHANGED
@@ -1,8 +1,6 @@
|
|
1
|
-
|
2
1
|
##
|
3
2
|
# Helper module including specific functions to handle dataset hooks.
|
4
3
|
module MiGA::Common::Hooks
|
5
|
-
|
6
4
|
##
|
7
5
|
# Call the hook with symbol +event+ and any parameters +event_args+
|
8
6
|
def pull_hook(event, *event_args)
|
@@ -12,7 +10,7 @@ module MiGA::Common::Hooks
|
|
12
10
|
event_queue.each do |i|
|
13
11
|
action = i.first
|
14
12
|
hook_name = :"hook_#{action}"
|
15
|
-
hook_args = i[1
|
13
|
+
hook_args = i[1..-1]
|
16
14
|
if respond_to? hook_name
|
17
15
|
MiGA::MiGA.DEBUG "Hook: #{self.class}(#{event} > #{action})"
|
18
16
|
self.send(hook_name, hook_args, event_args)
|
@@ -45,5 +43,4 @@ module MiGA::Common::Hooks
|
|
45
43
|
def hook_run_lambda(hook_args, event_args)
|
46
44
|
hook_args.first[*event_args]
|
47
45
|
end
|
48
|
-
|
49
46
|
end
|
data/lib/miga/common/path.rb
CHANGED
@@ -1,6 +1,4 @@
|
|
1
|
-
|
2
1
|
module MiGA::Common::Path
|
3
|
-
|
4
2
|
##
|
5
3
|
# Root path to MiGA (as estimated from the location of the current file).
|
6
4
|
def root_path
|
@@ -11,30 +9,27 @@ module MiGA::Common::Path
|
|
11
9
|
# Path to a script to be executed for +task+. Supported +opts+ are:
|
12
10
|
# - +:miga+ Path to the MiGA home to use. If not passed, the home of the
|
13
11
|
# library is used).
|
14
|
-
def script_path(task, opts={})
|
12
|
+
def script_path(task, opts = {})
|
15
13
|
opts[:miga] ||= root_path
|
16
14
|
File.expand_path("scripts/#{task}.bash", opts[:miga])
|
17
15
|
end
|
18
|
-
|
19
16
|
end
|
20
17
|
|
21
18
|
##
|
22
19
|
# MiGA extensions to the File class.
|
23
20
|
class File
|
24
|
-
|
25
21
|
##
|
26
22
|
# Method to transfer a file from +old_name+ to +new_name+, using a +method+
|
27
23
|
# that can be one of :symlink for File#symlink, :hardlink for File#link, or
|
28
24
|
# :copy for FileUtils#cp_r.
|
29
25
|
def self.generic_transfer(old_name, new_name, method)
|
30
26
|
return nil if exist? new_name
|
31
|
-
|
27
|
+
|
28
|
+
if (method == :copy)
|
32
29
|
FileUtils.cp_r(old_name, new_name)
|
33
30
|
else
|
34
|
-
method
|
31
|
+
method = :link if method == :hardlink
|
35
32
|
File.send(method, old_name, new_name)
|
36
33
|
end
|
37
34
|
end
|
38
|
-
|
39
35
|
end
|
40
|
-
|
@@ -1,4 +1,3 @@
|
|
1
|
-
|
2
1
|
require 'daemons'
|
3
2
|
require 'miga/common/with_daemon_class'
|
4
3
|
|
@@ -23,7 +22,7 @@ module MiGA::Common::WithDaemon
|
|
23
22
|
def output_file
|
24
23
|
File.join(daemon_home, "#{daemon_name}.output")
|
25
24
|
end
|
26
|
-
|
25
|
+
|
27
26
|
def terminate_file
|
28
27
|
File.join(daemon_home, 'terminate-daemon')
|
29
28
|
end
|
@@ -46,6 +45,7 @@ module MiGA::Common::WithDaemon
|
|
46
45
|
# Is the daemon active?
|
47
46
|
def active?
|
48
47
|
return false unless File.exist? alive_file
|
48
|
+
|
49
49
|
(last_alive || Time.new(0)) > Time.now - 60
|
50
50
|
end
|
51
51
|
|
@@ -74,8 +74,10 @@ module MiGA::Common::WithDaemon
|
|
74
74
|
i += 1
|
75
75
|
return :no_home unless Dir.exist? daemon_home
|
76
76
|
return :no_process_alive unless process_alive? pid
|
77
|
+
|
77
78
|
write_alive_file if i % 30 == 0
|
78
79
|
return :termination_file if termination_file? pid
|
80
|
+
|
79
81
|
sleep(1)
|
80
82
|
end
|
81
83
|
end
|
@@ -100,6 +102,7 @@ module MiGA::Common::WithDaemon
|
|
100
102
|
# if it does. Do not kill any process if +pid+ is +nil+
|
101
103
|
def termination_file?(pid)
|
102
104
|
return false unless File.exist? terminate_file
|
105
|
+
|
103
106
|
say 'Found termination file, terminating'
|
104
107
|
File.unlink(terminate_file)
|
105
108
|
terminate
|
@@ -1,4 +1,3 @@
|
|
1
|
-
|
2
1
|
##
|
3
2
|
# Helper module with specific class-level functions to be used with
|
4
3
|
# +include MiGA::Common::WithDaemon+.
|
@@ -28,6 +27,7 @@ module MiGA::Common::WithDaemonClass
|
|
28
27
|
f = terminated_file(path) unless File.exist? f
|
29
28
|
c = File.read(f)
|
30
29
|
return nil if c.nil? || c.empty?
|
30
|
+
|
31
31
|
Time.parse(c)
|
32
32
|
rescue Errno::ENOENT
|
33
33
|
return nil
|
@@ -1,4 +1,3 @@
|
|
1
|
-
|
2
1
|
##
|
3
2
|
# Helper module including specific functions to handle objects that
|
4
3
|
# have results.
|
@@ -24,6 +23,7 @@ module MiGA::Common::WithResult
|
|
24
23
|
def add_result(task, save = true, opts = {})
|
25
24
|
task = task.to_sym
|
26
25
|
return nil if result_dirs[task].nil?
|
26
|
+
|
27
27
|
base = File.join(
|
28
28
|
project.path, "data/#{result_dirs[task]}/#{result_base}"
|
29
29
|
)
|
@@ -44,6 +44,7 @@ module MiGA::Common::WithResult
|
|
44
44
|
def result(task)
|
45
45
|
task = task.to_sym
|
46
46
|
return nil if result_dirs[task].nil?
|
47
|
+
|
47
48
|
MiGA::Result.load(
|
48
49
|
"#{project.path}/data/#{result_dirs[task]}/#{result_base}.json"
|
49
50
|
)
|
data/lib/miga/daemon.rb
CHANGED
@@ -8,7 +8,6 @@ require 'miga/daemon/base'
|
|
8
8
|
##
|
9
9
|
# MiGA Daemons handling job submissions.
|
10
10
|
class MiGA::Daemon < MiGA::MiGA
|
11
|
-
|
12
11
|
include MiGA::Daemon::Base
|
13
12
|
include MiGA::Common::WithDaemon
|
14
13
|
extend MiGA::Common::WithDaemonClass
|
@@ -19,6 +18,7 @@ class MiGA::Daemon < MiGA::MiGA
|
|
19
18
|
# full path to the project's 'daemon' folder
|
20
19
|
def daemon_home(project)
|
21
20
|
return project if project.is_a? String
|
21
|
+
|
22
22
|
File.join(project.path, 'daemon')
|
23
23
|
end
|
24
24
|
end
|
@@ -48,7 +48,7 @@ class MiGA::Daemon < MiGA::MiGA
|
|
48
48
|
default_json = File.expand_path('.miga_daemon.json', ENV['MIGA_HOME'])
|
49
49
|
MiGA::Json.parse(
|
50
50
|
json, default: File.exist?(default_json) ? default_json : nil
|
51
|
-
).each { |k,v| runopts(k, v) }
|
51
|
+
).each { |k, v| runopts(k, v) }
|
52
52
|
update_format_0
|
53
53
|
@jobs_to_run = []
|
54
54
|
@jobs_running = []
|
@@ -82,14 +82,13 @@ class MiGA::Daemon < MiGA::MiGA
|
|
82
82
|
def daemon_loop
|
83
83
|
l_say(3, 'Daemon loop start')
|
84
84
|
reload_project
|
85
|
-
check_datasets
|
86
|
-
|
87
|
-
if shutdown_when_done? && jobs_running.size + jobs_to_run.size == 0
|
85
|
+
check_datasets or check_project
|
86
|
+
if shutdown_when_done? && (jobs_running.size + jobs_to_run.size).zero?
|
88
87
|
say 'Nothing else to do, shutting down'
|
89
88
|
return false
|
90
89
|
end
|
91
90
|
flush!
|
92
|
-
if loop_i % 12
|
91
|
+
if (loop_i % 12).zero?
|
93
92
|
purge!
|
94
93
|
recalculate_status!
|
95
94
|
end
|
@@ -137,12 +136,13 @@ class MiGA::Daemon < MiGA::MiGA
|
|
137
136
|
def load_status
|
138
137
|
f_path = File.join(daemon_home, 'status.json')
|
139
138
|
return unless File.size? f_path
|
139
|
+
|
140
140
|
say 'Loading previous status in daemon/status.json:'
|
141
141
|
status = MiGA::Json.parse(f_path)
|
142
|
-
status.
|
142
|
+
status.each_key do |i|
|
143
143
|
status[i].map! do |j|
|
144
144
|
j.tap do |k|
|
145
|
-
unless k[:ds].nil?
|
145
|
+
unless k[:ds].nil? || k[:ds_name] == 'miga-project'
|
146
146
|
k[:ds] = project.dataset(k[:ds_name])
|
147
147
|
end
|
148
148
|
k[:job] = k[:job].to_sym unless k[:job].nil?
|
@@ -158,14 +158,19 @@ class MiGA::Daemon < MiGA::MiGA
|
|
158
158
|
end
|
159
159
|
|
160
160
|
##
|
161
|
-
# Traverse datasets
|
161
|
+
# Traverse datasets, and returns boolean indicating if at any datasets
|
162
|
+
# are incomplete
|
162
163
|
def check_datasets
|
163
164
|
l_say(2, 'Checking datasets')
|
165
|
+
o = false
|
164
166
|
project.each_dataset do |ds|
|
165
167
|
next unless ds.status == :incomplete
|
166
|
-
|
167
|
-
|
168
|
+
next if ds.next_preprocessing(false).nil?
|
169
|
+
|
170
|
+
o = true
|
171
|
+
queue_job(:d, ds)
|
168
172
|
end
|
173
|
+
o
|
169
174
|
end
|
170
175
|
|
171
176
|
##
|
@@ -173,8 +178,14 @@ class MiGA::Daemon < MiGA::MiGA
|
|
173
178
|
# project-level tasks
|
174
179
|
def check_project
|
175
180
|
l_say(2, 'Checking project')
|
181
|
+
|
182
|
+
# Ignore task if the project has no datasets
|
176
183
|
return if project.dataset_names.empty?
|
184
|
+
|
185
|
+
# Double-check if all datasets are ready
|
177
186
|
return unless project.done_preprocessing?(false)
|
187
|
+
|
188
|
+
# Queue project-level job
|
178
189
|
to_run = project.next_task(nil, false)
|
179
190
|
queue_job(:p) unless to_run.nil?
|
180
191
|
end
|
@@ -185,13 +196,14 @@ class MiGA::Daemon < MiGA::MiGA
|
|
185
196
|
# scheduler (or to bash or ssh) see #flush!
|
186
197
|
def queue_job(job, ds = nil)
|
187
198
|
return nil unless get_job(job, ds).nil?
|
199
|
+
|
188
200
|
ds_name = (ds.nil? ? 'miga-project' : ds.name)
|
189
201
|
say 'Queueing %s:%s' % [ds_name, job]
|
190
202
|
vars = {
|
191
203
|
'PROJECT' => project.path,
|
192
204
|
'RUNTYPE' => runopts(:type),
|
193
|
-
'CORES'
|
194
|
-
'MIGA'
|
205
|
+
'CORES' => ppn,
|
206
|
+
'MIGA' => MiGA::MiGA.root_path
|
195
207
|
}
|
196
208
|
vars['DATASET'] = ds.name unless ds.nil?
|
197
209
|
log_dir = File.expand_path("daemon/#{job}", project.path)
|
@@ -199,9 +211,10 @@ class MiGA::Daemon < MiGA::MiGA
|
|
199
211
|
task_name = "#{project.metadata[:name][0..9]}:#{job}:#{ds_name}"
|
200
212
|
to_run = { ds: ds, ds_name: ds_name, job: job, task_name: task_name }
|
201
213
|
to_run[:cmd] = runopts(:cmd).miga_variables(
|
202
|
-
script: MiGA::MiGA.script_path(job, miga:vars['MIGA'], project: project),
|
203
|
-
vars: vars.map
|
204
|
-
|
214
|
+
script: MiGA::MiGA.script_path(job, miga: vars['MIGA'], project: project),
|
215
|
+
vars: vars.map do |k, v|
|
216
|
+
runopts(:var).miga_variables(key: k, value: v)
|
217
|
+
end.join(runopts(:varsep)),
|
205
218
|
cpus: ppn,
|
206
219
|
log: File.expand_path("#{ds_name}.log", log_dir),
|
207
220
|
task_name: task_name,
|
@@ -216,9 +229,9 @@ class MiGA::Daemon < MiGA::MiGA
|
|
216
229
|
def get_job(job, ds = nil)
|
217
230
|
(jobs_to_run + jobs_running).find do |j|
|
218
231
|
if ds.nil?
|
219
|
-
j[:ds].nil?
|
232
|
+
j[:ds].nil? && j[:job] == job
|
220
233
|
else
|
221
|
-
|
234
|
+
!j[:ds].nil? && j[:ds].name == ds.name && j[:job] == job
|
222
235
|
end
|
223
236
|
end
|
224
237
|
end
|
@@ -230,14 +243,15 @@ class MiGA::Daemon < MiGA::MiGA
|
|
230
243
|
# Check for finished jobs
|
231
244
|
l_say(2, 'Checking for finished jobs')
|
232
245
|
@jobs_running.select! do |job|
|
233
|
-
ongoing =
|
234
|
-
|
235
|
-
|
236
|
-
|
237
|
-
|
238
|
-
|
239
|
-
|
240
|
-
|
246
|
+
ongoing =
|
247
|
+
case job[:job].to_s
|
248
|
+
when 'd'
|
249
|
+
!job[:ds].nil? && !job[:ds].next_preprocessing(false).nil?
|
250
|
+
when 'p'
|
251
|
+
!project.next_task(nil, false).nil?
|
252
|
+
else
|
253
|
+
(job[:ds].nil? ? project : job[:ds]).add_result(job[:job], false).nil?
|
254
|
+
end
|
241
255
|
say "Completed pid:#{job[:pid]} for #{job[:task_name]}" unless ongoing
|
242
256
|
ongoing
|
243
257
|
end
|
@@ -246,8 +260,9 @@ class MiGA::Daemon < MiGA::MiGA
|
|
246
260
|
@jobs_to_run.rotate! rand(jobs_to_run.size)
|
247
261
|
|
248
262
|
# Launch as many +jobs_to_run+ as possible
|
249
|
-
while hostk = next_host
|
263
|
+
while (hostk = next_host)
|
250
264
|
break if jobs_to_run.empty?
|
265
|
+
|
251
266
|
launch_job(@jobs_to_run.shift, hostk)
|
252
267
|
end
|
253
268
|
end
|
@@ -257,7 +272,8 @@ class MiGA::Daemon < MiGA::MiGA
|
|
257
272
|
# In any other daemons, returns true as long as #maxjobs is not reached
|
258
273
|
def next_host
|
259
274
|
return jobs_running.size < maxjobs if runopts(:type) != 'ssh'
|
260
|
-
|
275
|
+
|
276
|
+
allk = (0..nodelist.size - 1).to_a
|
261
277
|
busyk = jobs_running.map { |k| k[:hostk] }
|
262
278
|
(allk - busyk).first
|
263
279
|
end
|
@@ -298,9 +314,8 @@ class MiGA::Daemon < MiGA::MiGA
|
|
298
314
|
say "Unsuccessful #{job[:task_name]}, rescheduling"
|
299
315
|
else
|
300
316
|
@jobs_running << job
|
301
|
-
|
302
|
-
|
303
|
-
} for #{job[:task_name]}"
|
317
|
+
job_host = " to #{job[:hostk]}:#{nodelist[job[:hostk]]}" if job[:hostk]
|
318
|
+
say "Spawned pid:#{job[:pid]}#{job_host} for #{job[:task_name]}"
|
304
319
|
end
|
305
320
|
end
|
306
321
|
|
@@ -312,10 +327,11 @@ class MiGA::Daemon < MiGA::MiGA
|
|
312
327
|
var: %w[key value],
|
313
328
|
alive: %w[pid],
|
314
329
|
kill: %w[pid]
|
315
|
-
}.each do |k,v|
|
316
|
-
runopts(
|
317
|
-
|
318
|
-
|
330
|
+
}.each do |k, v|
|
331
|
+
if !runopts(k).nil? && runopts(k) =~ /%(\d+\$)?[ds]/
|
332
|
+
runopts(k,
|
333
|
+
runopts(k).gsub(/%(\d+\$)?d/, '%\\1s') % v.map { |i| "{{#{i}}}" })
|
334
|
+
end
|
319
335
|
end
|
320
336
|
runopts(:format_version, 1)
|
321
337
|
end
|