miga-base 0.7.4.0 → 0.7.9.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (119) hide show
  1. checksums.yaml +4 -4
  2. data/lib/miga/cli.rb +10 -8
  3. data/lib/miga/cli/action.rb +2 -3
  4. data/lib/miga/cli/action/about.rb +5 -6
  5. data/lib/miga/cli/action/add.rb +18 -12
  6. data/lib/miga/cli/action/add_result.rb +2 -3
  7. data/lib/miga/cli/action/archive.rb +1 -2
  8. data/lib/miga/cli/action/classify_wf.rb +8 -6
  9. data/lib/miga/cli/action/console.rb +0 -1
  10. data/lib/miga/cli/action/daemon.rb +7 -7
  11. data/lib/miga/cli/action/date.rb +0 -1
  12. data/lib/miga/cli/action/derep_wf.rb +5 -4
  13. data/lib/miga/cli/action/doctor.rb +28 -20
  14. data/lib/miga/cli/action/doctor/base.rb +29 -6
  15. data/lib/miga/cli/action/edit.rb +1 -2
  16. data/lib/miga/cli/action/files.rb +8 -8
  17. data/lib/miga/cli/action/find.rb +5 -6
  18. data/lib/miga/cli/action/generic.rb +7 -7
  19. data/lib/miga/cli/action/get.rb +20 -17
  20. data/lib/miga/cli/action/get_db.rb +8 -2
  21. data/lib/miga/cli/action/index_wf.rb +1 -1
  22. data/lib/miga/cli/action/init.rb +53 -41
  23. data/lib/miga/cli/action/init/daemon_helper.rb +65 -43
  24. data/lib/miga/cli/action/lair.rb +7 -7
  25. data/lib/miga/cli/action/ln.rb +6 -6
  26. data/lib/miga/cli/action/ls.rb +1 -2
  27. data/lib/miga/cli/action/ncbi_get.rb +11 -3
  28. data/lib/miga/cli/action/new.rb +4 -4
  29. data/lib/miga/cli/action/next_step.rb +0 -1
  30. data/lib/miga/cli/action/preproc_wf.rb +3 -3
  31. data/lib/miga/cli/action/quality_wf.rb +1 -1
  32. data/lib/miga/cli/action/rm.rb +2 -3
  33. data/lib/miga/cli/action/run.rb +8 -8
  34. data/lib/miga/cli/action/stats.rb +9 -5
  35. data/lib/miga/cli/action/summary.rb +13 -7
  36. data/lib/miga/cli/action/tax_dist.rb +8 -4
  37. data/lib/miga/cli/action/tax_index.rb +3 -4
  38. data/lib/miga/cli/action/tax_set.rb +7 -6
  39. data/lib/miga/cli/action/tax_test.rb +6 -5
  40. data/lib/miga/cli/action/wf.rb +25 -19
  41. data/lib/miga/cli/base.rb +34 -32
  42. data/lib/miga/cli/objects_helper.rb +27 -18
  43. data/lib/miga/cli/opt_helper.rb +3 -2
  44. data/lib/miga/common.rb +2 -5
  45. data/lib/miga/common/base.rb +15 -16
  46. data/lib/miga/common/format.rb +11 -6
  47. data/lib/miga/common/hooks.rb +1 -4
  48. data/lib/miga/common/path.rb +4 -9
  49. data/lib/miga/common/with_daemon.rb +5 -2
  50. data/lib/miga/common/with_daemon_class.rb +1 -1
  51. data/lib/miga/common/with_result.rb +2 -1
  52. data/lib/miga/daemon.rb +51 -35
  53. data/lib/miga/daemon/base.rb +0 -2
  54. data/lib/miga/dataset.rb +47 -37
  55. data/lib/miga/dataset/base.rb +52 -37
  56. data/lib/miga/dataset/hooks.rb +3 -4
  57. data/lib/miga/dataset/result.rb +17 -1
  58. data/lib/miga/json.rb +5 -7
  59. data/lib/miga/lair.rb +4 -0
  60. data/lib/miga/metadata.rb +4 -3
  61. data/lib/miga/project.rb +29 -20
  62. data/lib/miga/project/base.rb +52 -37
  63. data/lib/miga/project/dataset.rb +27 -13
  64. data/lib/miga/project/hooks.rb +0 -3
  65. data/lib/miga/project/result.rb +14 -5
  66. data/lib/miga/remote_dataset.rb +85 -72
  67. data/lib/miga/remote_dataset/base.rb +11 -13
  68. data/lib/miga/remote_dataset/download.rb +34 -12
  69. data/lib/miga/result.rb +48 -53
  70. data/lib/miga/result/base.rb +0 -2
  71. data/lib/miga/result/dates.rb +1 -3
  72. data/lib/miga/result/source.rb +15 -16
  73. data/lib/miga/result/stats.rb +37 -27
  74. data/lib/miga/tax_dist.rb +6 -3
  75. data/lib/miga/tax_index.rb +17 -17
  76. data/lib/miga/taxonomy.rb +6 -1
  77. data/lib/miga/taxonomy/base.rb +19 -15
  78. data/lib/miga/version.rb +19 -16
  79. data/scripts/project_stats.bash +3 -0
  80. data/scripts/stats.bash +1 -1
  81. data/test/common_test.rb +3 -11
  82. data/test/daemon_helper.rb +38 -0
  83. data/test/daemon_test.rb +73 -101
  84. data/test/dataset_test.rb +63 -59
  85. data/test/format_test.rb +3 -11
  86. data/test/hook_test.rb +50 -55
  87. data/test/json_test.rb +7 -8
  88. data/test/lair_test.rb +22 -28
  89. data/test/metadata_test.rb +6 -14
  90. data/test/project_test.rb +33 -39
  91. data/test/remote_dataset_test.rb +26 -32
  92. data/test/result_stats_test.rb +17 -27
  93. data/test/result_test.rb +41 -34
  94. data/test/tax_dist_test.rb +0 -2
  95. data/test/tax_index_test.rb +4 -10
  96. data/test/taxonomy_test.rb +7 -9
  97. data/test/test_helper.rb +42 -1
  98. data/test/with_daemon_test.rb +14 -22
  99. data/utils/adapters.fa +13 -0
  100. data/utils/cleanup-databases.rb +6 -5
  101. data/utils/distance/base.rb +0 -1
  102. data/utils/distance/commands.rb +19 -12
  103. data/utils/distance/database.rb +24 -21
  104. data/utils/distance/pipeline.rb +23 -10
  105. data/utils/distance/runner.rb +20 -16
  106. data/utils/distance/temporal.rb +1 -3
  107. data/utils/distances.rb +1 -1
  108. data/utils/domain-ess-genes.rb +7 -7
  109. data/utils/index_metadata.rb +4 -2
  110. data/utils/mytaxa_scan.rb +18 -16
  111. data/utils/representatives.rb +5 -4
  112. data/utils/requirements.txt +1 -1
  113. data/utils/subclade/base.rb +0 -1
  114. data/utils/subclade/pipeline.rb +7 -6
  115. data/utils/subclade/runner.rb +9 -9
  116. data/utils/subclade/temporal.rb +0 -2
  117. data/utils/subclades-compile.rb +39 -37
  118. data/utils/subclades.rb +1 -1
  119. metadata +5 -4
@@ -1,15 +1,17 @@
1
1
  # @package MiGA
2
2
  # @license Artistic-2.0
3
3
 
4
- module MiGA::Cli::ObjectsHelper
4
+ module MiGA::Cli::ObjectsHelper
5
5
  ##
6
6
  # Get the project defined in the CLI by parameter +name+ and +flag+
7
7
  def load_project(name = :project, flag = '-P')
8
8
  return @objects[name] unless @objects[name].nil?
9
+
9
10
  ensure_par(name => flag)
10
11
  say "Loading project: #{self[name]}"
11
12
  @objects[name] = MiGA::Project.load(self[name])
12
13
  raise "Cannot load project: #{self[name]}" if @objects[name].nil?
14
+
13
15
  @objects[name]
14
16
  end
15
17
 
@@ -23,6 +25,7 @@ module MiGA::Cli::ObjectsHelper
23
25
  end
24
26
  d = load_project.dataset(name)
25
27
  raise "Cannot load dataset: #{self[:dataset]}" if !silent && d.nil?
28
+
26
29
  return d
27
30
  end
28
31
 
@@ -37,16 +40,18 @@ module MiGA::Cli::ObjectsHelper
37
40
  # If +silent=true+, it allows failures silently
38
41
  def load_and_filter_datasets(silent = false)
39
42
  return @objects[:filtered_datasets] unless @objects[:filtered_datasets].nil?
43
+
40
44
  say 'Listing datasets'
41
- ds = if ! self[:dataset].nil?
42
- [load_dataset(silent)].compact
43
- elsif ! self[:ds_list].nil?
44
- File.readlines(self[:ds_list]).map do |i|
45
- load_dataset(silent, i.chomp)
46
- end.compact
47
- else
48
- load_project.datasets
49
- end
45
+ ds =
46
+ if !self[:dataset].nil?
47
+ [load_dataset(silent)].compact
48
+ elsif !self[:ds_list].nil?
49
+ File.readlines(self[:ds_list]).map do |i|
50
+ load_dataset(silent, i.chomp)
51
+ end.compact
52
+ else
53
+ load_project.datasets
54
+ end
50
55
  k = 0
51
56
  n = ds.size
52
57
  ds.select! do |d|
@@ -61,31 +66,36 @@ module MiGA::Cli::ObjectsHelper
61
66
  o
62
67
  end
63
68
  say ''
64
- ds = ds.values_at(self[:dataset_k]-1) unless self[:dataset_k].nil?
69
+ ds = ds.values_at(self[:dataset_k] - 1) unless self[:dataset_k].nil?
65
70
  @objects[:filtered_datasets] = ds
66
71
  end
67
72
 
68
73
  def load_result
69
74
  return @objects[:result] unless @objects[:result].nil?
75
+
70
76
  ensure_par(result: '-r')
71
77
  obj = load_project_or_dataset
72
78
  if obj.class.RESULT_DIRS[self[:result]].nil?
73
- klass = obj.class.to_s.gsub(/.*::/,'')
79
+ klass = obj.class.to_s.gsub(/.*::/, '')
74
80
  raise "Unsupported result for #{klass}: #{self[:result]}"
75
81
  end
76
82
  r = obj.add_result(self[:result], false)
77
- raise "Cannot load result: #{self[:result]}" if r.nil?
83
+ if r.nil? && !self[:ignore_result_empty]
84
+ raise "Cannot load result: #{self[:result]}"
85
+ end
86
+
78
87
  @objects[:result] = r
79
88
  end
80
89
 
81
90
  def add_metadata(obj, cli = self)
82
91
  raise "Unsupported object: #{obj.class}" unless obj.respond_to? :metadata
92
+
83
93
  cli[:metadata].split(',').each do |pair|
84
- (k,v) = pair.split('=')
94
+ (k, v) = pair.split('=')
85
95
  case v
86
- when 'true'; v = true
87
- when 'false'; v = false
88
- when 'nil'; v = nil
96
+ when 'true'; v = true
97
+ when 'false'; v = false
98
+ when 'nil'; v = nil
89
99
  end
90
100
  if k == '_step'
91
101
  obj.metadata["_try_#{v}"] ||= 0
@@ -99,4 +109,3 @@ module MiGA::Cli::ObjectsHelper
99
109
  obj
100
110
  end
101
111
  end
102
-
@@ -1,7 +1,7 @@
1
1
  # @package MiGA
2
2
  # @license Artistic-2.0
3
3
 
4
- module MiGA::Cli::OptHelper
4
+ module MiGA::Cli::OptHelper
5
5
  ##
6
6
  # Send MiGA's banner to OptionParser +opt+
7
7
  def banner(opt)
@@ -19,6 +19,7 @@ module MiGA::Cli::OptHelper
19
19
  # Executes only once, unless +#opt_common = true+ is passed between calls
20
20
  def opt_common(opt)
21
21
  return unless @opt_common
22
+
22
23
  if interactive
23
24
  opt.on(
24
25
  '--auto',
@@ -159,6 +160,6 @@ module MiGA::Cli::OptHelper
159
160
  # If +sym+ is nil, +flag+ is used as Symbol
160
161
  def opt_flag(opt, flag, description, sym = nil)
161
162
  sym = flag.to_sym if sym.nil?
162
- opt.on("--#{flag.to_s.gsub('_','-')}", description) { |v| self[sym] = v }
163
+ opt.on("--#{flag.to_s.gsub('_', '-')}", description) { |v| self[sym] = v }
163
164
  end
164
165
  end
@@ -11,12 +11,11 @@ require 'miga/common/format'
11
11
  # Generic class used to handle system-wide information and methods, and parent
12
12
  # of all other MiGA::* classes.
13
13
  class MiGA::MiGA
14
-
15
14
  include MiGA::Common
16
-
15
+
17
16
  extend MiGA::Common::Path
18
17
  extend MiGA::Common::Format
19
-
18
+
20
19
  ENV['MIGA_HOME'] ||= ENV['HOME']
21
20
 
22
21
  ##
@@ -45,6 +44,4 @@ class MiGA::MiGA
45
44
  io = par.first.is_a?(IO) ? par.shift : $stderr
46
45
  io.puts(*par.map { |i| "[#{Time.now}] #{i}" })
47
46
  end
48
-
49
47
  end
50
-
@@ -1,42 +1,43 @@
1
-
2
1
  class MiGA::MiGA
3
-
4
2
  # Class-level
5
3
  class << self
6
4
  ##
7
- # Turn on debugging.
8
- def DEBUG_ON ; @@DEBUG=true end
5
+ # Turn on debugging
6
+ def DEBUG_ON
7
+ @@DEBUG = true
8
+ end
9
9
 
10
10
  ##
11
- # Turn off debugging.
12
- def DEBUG_OFF ; @@DEBUG=false end
11
+ # Turn off debugging
12
+ def DEBUG_OFF
13
+ @@DEBUG = false
14
+ end
13
15
 
14
16
  ##
15
- # Turn on debug tracing (and debugging).
17
+ # Turn on debug tracing (and debugging)
16
18
  def DEBUG_TRACE_ON
17
- @@DEBUG_TRACE=true
19
+ @@DEBUG_TRACE = true
18
20
  DEBUG_ON()
19
21
  end
20
22
 
21
23
  ##
22
- # Turn off debug tracing (but not debugging).
24
+ # Turn off debug tracing (but not debugging)
23
25
  def DEBUG_TRACE_OFF
24
- @@DEBUG_TRACE=false
26
+ @@DEBUG_TRACE = false
25
27
  end
26
28
 
27
29
  ##
28
- # Send debug message.
30
+ # Send debug message
29
31
  def DEBUG(*args)
30
32
  $stderr.puts(*args) if @@DEBUG
31
33
  $stderr.puts(
32
- caller.map{ |v| v.gsub(/^/,' ') }.join("\n") ) if @@DEBUG_TRACE
34
+ caller.map { |v| v.gsub(/^/, ' ') }.join("\n")
35
+ ) if @@DEBUG_TRACE
33
36
  end
34
37
  end
35
-
36
38
  end
37
39
 
38
40
  module MiGA::Common
39
-
40
41
  ##
41
42
  # Should debugging information be reported?
42
43
  @@DEBUG = false
@@ -44,6 +45,4 @@ module MiGA::Common
44
45
  ##
45
46
  # Should the trace of debugging information be reported?
46
47
  @@DEBUG_TRACE = false
47
-
48
48
  end
49
-
@@ -1,4 +1,3 @@
1
-
2
1
  require 'tempfile'
3
2
  require 'zlib'
4
3
 
@@ -69,30 +68,35 @@ module MiGA::Common::Format
69
68
  # a FastA or FastQ file (supports gzipped files). The +format+ must be a
70
69
  # Symbol, one of +:fasta+ or +:fastq+. Additional estimations can be
71
70
  # controlled via the +opts+ Hash. Supported options include:
72
- # - +:n50+: If true, it also returns the N50 and the median (in bp).
73
- # - +gc+: If true, it also returns the G+C content (in %).
71
+ # - +:n50+: If true, it also returns the N50 and the median (in bp)
72
+ # - +:gc+: If true, it also returns the G+C content (in %)
73
+ # - +:x+: If true, it also returns the undetermined bases content (in %)
74
74
  def seqs_length(file, format, opts = {})
75
75
  fh = file =~ /\.gz/ ? Zlib::GzipReader.open(file) : File.open(file, 'r')
76
76
  l = []
77
77
  gc = 0
78
+ xn = 0
78
79
  i = 0 # <- Zlib::GzipReader doesn't set `$.`
79
80
  fh.each_line do |ln|
80
81
  i += 1
81
82
  if (format == :fasta and ln =~ /^>/) or
82
- (format == :fastq and (i % 4) == 1)
83
+ (format == :fastq and (i % 4) == 1)
83
84
  l << 0
84
85
  elsif format == :fasta or (i % 4) == 2
85
86
  l[l.size - 1] += ln.chomp.size
86
87
  gc += ln.scan(/[GCgc]/).count if opts[:gc]
88
+ xn += ln.scan(/[XNxn]/).count if opts[:x]
87
89
  end
88
90
  end
89
91
  fh.close
90
92
 
91
- o = { n: l.size, tot: l.inject(:+) }
93
+ o = { n: l.size, tot: l.inject(0, :+), max: l.max }
94
+ return o if o[:tot].zero?
92
95
  o[:avg] = o[:tot].to_f / l.size
93
96
  o[:var] = l.map { |a| a**2 }.inject(:+).to_f / l.size - o[:avg]**2
94
97
  o[:sd] = Math.sqrt o[:var]
95
98
  o[:gc] = 100.0 * gc / o[:tot] if opts[:gc]
99
+ o[:x] = 100.0 * xn / o[:tot] if opts[:x]
96
100
  if opts[:n50]
97
101
  l.sort!
98
102
  thr = o[:tot] / 2
@@ -103,7 +107,8 @@ module MiGA::Common::Format
103
107
  break if pos >= thr
104
108
  end
105
109
  o[:med] = o[:n].even? ?
106
- 0.5 * l[o[:n] / 2 - 1, 2].inject(:+) : l[(o[:n] - 1) / 2]
110
+ 0.5 * l[o[:n] / 2 - 1, 2].inject(:+) :
111
+ l[(o[:n] - 1) / 2]
107
112
  end
108
113
  o
109
114
  end
@@ -1,8 +1,6 @@
1
-
2
1
  ##
3
2
  # Helper module including specific functions to handle dataset hooks.
4
3
  module MiGA::Common::Hooks
5
-
6
4
  ##
7
5
  # Call the hook with symbol +event+ and any parameters +event_args+
8
6
  def pull_hook(event, *event_args)
@@ -12,7 +10,7 @@ module MiGA::Common::Hooks
12
10
  event_queue.each do |i|
13
11
  action = i.first
14
12
  hook_name = :"hook_#{action}"
15
- hook_args = i[1 .. -1]
13
+ hook_args = i[1..-1]
16
14
  if respond_to? hook_name
17
15
  MiGA::MiGA.DEBUG "Hook: #{self.class}(#{event} > #{action})"
18
16
  self.send(hook_name, hook_args, event_args)
@@ -45,5 +43,4 @@ module MiGA::Common::Hooks
45
43
  def hook_run_lambda(hook_args, event_args)
46
44
  hook_args.first[*event_args]
47
45
  end
48
-
49
46
  end
@@ -1,6 +1,4 @@
1
-
2
1
  module MiGA::Common::Path
3
-
4
2
  ##
5
3
  # Root path to MiGA (as estimated from the location of the current file).
6
4
  def root_path
@@ -11,30 +9,27 @@ module MiGA::Common::Path
11
9
  # Path to a script to be executed for +task+. Supported +opts+ are:
12
10
  # - +:miga+ Path to the MiGA home to use. If not passed, the home of the
13
11
  # library is used).
14
- def script_path(task, opts={})
12
+ def script_path(task, opts = {})
15
13
  opts[:miga] ||= root_path
16
14
  File.expand_path("scripts/#{task}.bash", opts[:miga])
17
15
  end
18
-
19
16
  end
20
17
 
21
18
  ##
22
19
  # MiGA extensions to the File class.
23
20
  class File
24
-
25
21
  ##
26
22
  # Method to transfer a file from +old_name+ to +new_name+, using a +method+
27
23
  # that can be one of :symlink for File#symlink, :hardlink for File#link, or
28
24
  # :copy for FileUtils#cp_r.
29
25
  def self.generic_transfer(old_name, new_name, method)
30
26
  return nil if exist? new_name
31
- if(method==:copy)
27
+
28
+ if (method == :copy)
32
29
  FileUtils.cp_r(old_name, new_name)
33
30
  else
34
- method=:link if method==:hardlink
31
+ method = :link if method == :hardlink
35
32
  File.send(method, old_name, new_name)
36
33
  end
37
34
  end
38
-
39
35
  end
40
-
@@ -1,4 +1,3 @@
1
-
2
1
  require 'daemons'
3
2
  require 'miga/common/with_daemon_class'
4
3
 
@@ -23,7 +22,7 @@ module MiGA::Common::WithDaemon
23
22
  def output_file
24
23
  File.join(daemon_home, "#{daemon_name}.output")
25
24
  end
26
-
25
+
27
26
  def terminate_file
28
27
  File.join(daemon_home, 'terminate-daemon')
29
28
  end
@@ -46,6 +45,7 @@ module MiGA::Common::WithDaemon
46
45
  # Is the daemon active?
47
46
  def active?
48
47
  return false unless File.exist? alive_file
48
+
49
49
  (last_alive || Time.new(0)) > Time.now - 60
50
50
  end
51
51
 
@@ -74,8 +74,10 @@ module MiGA::Common::WithDaemon
74
74
  i += 1
75
75
  return :no_home unless Dir.exist? daemon_home
76
76
  return :no_process_alive unless process_alive? pid
77
+
77
78
  write_alive_file if i % 30 == 0
78
79
  return :termination_file if termination_file? pid
80
+
79
81
  sleep(1)
80
82
  end
81
83
  end
@@ -100,6 +102,7 @@ module MiGA::Common::WithDaemon
100
102
  # if it does. Do not kill any process if +pid+ is +nil+
101
103
  def termination_file?(pid)
102
104
  return false unless File.exist? terminate_file
105
+
103
106
  say 'Found termination file, terminating'
104
107
  File.unlink(terminate_file)
105
108
  terminate
@@ -1,4 +1,3 @@
1
-
2
1
  ##
3
2
  # Helper module with specific class-level functions to be used with
4
3
  # +include MiGA::Common::WithDaemon+.
@@ -28,6 +27,7 @@ module MiGA::Common::WithDaemonClass
28
27
  f = terminated_file(path) unless File.exist? f
29
28
  c = File.read(f)
30
29
  return nil if c.nil? || c.empty?
30
+
31
31
  Time.parse(c)
32
32
  rescue Errno::ENOENT
33
33
  return nil
@@ -1,4 +1,3 @@
1
-
2
1
  ##
3
2
  # Helper module including specific functions to handle objects that
4
3
  # have results.
@@ -24,6 +23,7 @@ module MiGA::Common::WithResult
24
23
  def add_result(task, save = true, opts = {})
25
24
  task = task.to_sym
26
25
  return nil if result_dirs[task].nil?
26
+
27
27
  base = File.join(
28
28
  project.path, "data/#{result_dirs[task]}/#{result_base}"
29
29
  )
@@ -44,6 +44,7 @@ module MiGA::Common::WithResult
44
44
  def result(task)
45
45
  task = task.to_sym
46
46
  return nil if result_dirs[task].nil?
47
+
47
48
  MiGA::Result.load(
48
49
  "#{project.path}/data/#{result_dirs[task]}/#{result_base}.json"
49
50
  )
@@ -8,7 +8,6 @@ require 'miga/daemon/base'
8
8
  ##
9
9
  # MiGA Daemons handling job submissions.
10
10
  class MiGA::Daemon < MiGA::MiGA
11
-
12
11
  include MiGA::Daemon::Base
13
12
  include MiGA::Common::WithDaemon
14
13
  extend MiGA::Common::WithDaemonClass
@@ -19,6 +18,7 @@ class MiGA::Daemon < MiGA::MiGA
19
18
  # full path to the project's 'daemon' folder
20
19
  def daemon_home(project)
21
20
  return project if project.is_a? String
21
+
22
22
  File.join(project.path, 'daemon')
23
23
  end
24
24
  end
@@ -48,7 +48,7 @@ class MiGA::Daemon < MiGA::MiGA
48
48
  default_json = File.expand_path('.miga_daemon.json', ENV['MIGA_HOME'])
49
49
  MiGA::Json.parse(
50
50
  json, default: File.exist?(default_json) ? default_json : nil
51
- ).each { |k,v| runopts(k, v) }
51
+ ).each { |k, v| runopts(k, v) }
52
52
  update_format_0
53
53
  @jobs_to_run = []
54
54
  @jobs_running = []
@@ -82,14 +82,13 @@ class MiGA::Daemon < MiGA::MiGA
82
82
  def daemon_loop
83
83
  l_say(3, 'Daemon loop start')
84
84
  reload_project
85
- check_datasets
86
- check_project
87
- if shutdown_when_done? && jobs_running.size + jobs_to_run.size == 0
85
+ check_datasets or check_project
86
+ if shutdown_when_done? && (jobs_running.size + jobs_to_run.size).zero?
88
87
  say 'Nothing else to do, shutting down'
89
88
  return false
90
89
  end
91
90
  flush!
92
- if loop_i % 12 == 0
91
+ if (loop_i % 12).zero?
93
92
  purge!
94
93
  recalculate_status!
95
94
  end
@@ -137,12 +136,13 @@ class MiGA::Daemon < MiGA::MiGA
137
136
  def load_status
138
137
  f_path = File.join(daemon_home, 'status.json')
139
138
  return unless File.size? f_path
139
+
140
140
  say 'Loading previous status in daemon/status.json:'
141
141
  status = MiGA::Json.parse(f_path)
142
- status.keys.each do |i|
142
+ status.each_key do |i|
143
143
  status[i].map! do |j|
144
144
  j.tap do |k|
145
- unless k[:ds].nil? or k[:ds_name] == 'miga-project'
145
+ unless k[:ds].nil? || k[:ds_name] == 'miga-project'
146
146
  k[:ds] = project.dataset(k[:ds_name])
147
147
  end
148
148
  k[:job] = k[:job].to_sym unless k[:job].nil?
@@ -158,14 +158,19 @@ class MiGA::Daemon < MiGA::MiGA
158
158
  end
159
159
 
160
160
  ##
161
- # Traverse datasets
161
+ # Traverse datasets, and returns boolean indicating if at any datasets
162
+ # are incomplete
162
163
  def check_datasets
163
164
  l_say(2, 'Checking datasets')
165
+ o = false
164
166
  project.each_dataset do |ds|
165
167
  next unless ds.status == :incomplete
166
- to_run = ds.next_preprocessing(false)
167
- queue_job(:d, ds) unless to_run.nil?
168
+ next if ds.next_preprocessing(false).nil?
169
+
170
+ o = true
171
+ queue_job(:d, ds)
168
172
  end
173
+ o
169
174
  end
170
175
 
171
176
  ##
@@ -173,8 +178,14 @@ class MiGA::Daemon < MiGA::MiGA
173
178
  # project-level tasks
174
179
  def check_project
175
180
  l_say(2, 'Checking project')
181
+
182
+ # Ignore task if the project has no datasets
176
183
  return if project.dataset_names.empty?
184
+
185
+ # Double-check if all datasets are ready
177
186
  return unless project.done_preprocessing?(false)
187
+
188
+ # Queue project-level job
178
189
  to_run = project.next_task(nil, false)
179
190
  queue_job(:p) unless to_run.nil?
180
191
  end
@@ -185,13 +196,14 @@ class MiGA::Daemon < MiGA::MiGA
185
196
  # scheduler (or to bash or ssh) see #flush!
186
197
  def queue_job(job, ds = nil)
187
198
  return nil unless get_job(job, ds).nil?
199
+
188
200
  ds_name = (ds.nil? ? 'miga-project' : ds.name)
189
201
  say 'Queueing %s:%s' % [ds_name, job]
190
202
  vars = {
191
203
  'PROJECT' => project.path,
192
204
  'RUNTYPE' => runopts(:type),
193
- 'CORES' => ppn,
194
- 'MIGA' => MiGA::MiGA.root_path
205
+ 'CORES' => ppn,
206
+ 'MIGA' => MiGA::MiGA.root_path
195
207
  }
196
208
  vars['DATASET'] = ds.name unless ds.nil?
197
209
  log_dir = File.expand_path("daemon/#{job}", project.path)
@@ -199,9 +211,10 @@ class MiGA::Daemon < MiGA::MiGA
199
211
  task_name = "#{project.metadata[:name][0..9]}:#{job}:#{ds_name}"
200
212
  to_run = { ds: ds, ds_name: ds_name, job: job, task_name: task_name }
201
213
  to_run[:cmd] = runopts(:cmd).miga_variables(
202
- script: MiGA::MiGA.script_path(job, miga:vars['MIGA'], project: project),
203
- vars: vars.map { |k, v|
204
- runopts(:var).miga_variables(key: k, value: v) }.join(runopts(:varsep)),
214
+ script: MiGA::MiGA.script_path(job, miga: vars['MIGA'], project: project),
215
+ vars: vars.map do |k, v|
216
+ runopts(:var).miga_variables(key: k, value: v)
217
+ end.join(runopts(:varsep)),
205
218
  cpus: ppn,
206
219
  log: File.expand_path("#{ds_name}.log", log_dir),
207
220
  task_name: task_name,
@@ -216,9 +229,9 @@ class MiGA::Daemon < MiGA::MiGA
216
229
  def get_job(job, ds = nil)
217
230
  (jobs_to_run + jobs_running).find do |j|
218
231
  if ds.nil?
219
- j[:ds].nil? and j[:job] == job
232
+ j[:ds].nil? && j[:job] == job
220
233
  else
221
- (! j[:ds].nil?) and j[:ds].name == ds.name and j[:job] == job
234
+ !j[:ds].nil? && j[:ds].name == ds.name && j[:job] == job
222
235
  end
223
236
  end
224
237
  end
@@ -230,14 +243,15 @@ class MiGA::Daemon < MiGA::MiGA
230
243
  # Check for finished jobs
231
244
  l_say(2, 'Checking for finished jobs')
232
245
  @jobs_running.select! do |job|
233
- ongoing = case job[:job].to_s
234
- when 'd'
235
- !job[:ds].nil? && !job[:ds].next_preprocessing(false).nil?
236
- when 'p'
237
- !project.next_task(nil, false).nil?
238
- else
239
- (job[:ds].nil? ? project : job[:ds]).add_result(job[:job], false).nil?
240
- end
246
+ ongoing =
247
+ case job[:job].to_s
248
+ when 'd'
249
+ !job[:ds].nil? && !job[:ds].next_preprocessing(false).nil?
250
+ when 'p'
251
+ !project.next_task(nil, false).nil?
252
+ else
253
+ (job[:ds].nil? ? project : job[:ds]).add_result(job[:job], false).nil?
254
+ end
241
255
  say "Completed pid:#{job[:pid]} for #{job[:task_name]}" unless ongoing
242
256
  ongoing
243
257
  end
@@ -246,8 +260,9 @@ class MiGA::Daemon < MiGA::MiGA
246
260
  @jobs_to_run.rotate! rand(jobs_to_run.size)
247
261
 
248
262
  # Launch as many +jobs_to_run+ as possible
249
- while hostk = next_host
263
+ while (hostk = next_host)
250
264
  break if jobs_to_run.empty?
265
+
251
266
  launch_job(@jobs_to_run.shift, hostk)
252
267
  end
253
268
  end
@@ -257,7 +272,8 @@ class MiGA::Daemon < MiGA::MiGA
257
272
  # In any other daemons, returns true as long as #maxjobs is not reached
258
273
  def next_host
259
274
  return jobs_running.size < maxjobs if runopts(:type) != 'ssh'
260
- allk = (0 .. nodelist.size-1).to_a
275
+
276
+ allk = (0..nodelist.size - 1).to_a
261
277
  busyk = jobs_running.map { |k| k[:hostk] }
262
278
  (allk - busyk).first
263
279
  end
@@ -298,9 +314,8 @@ class MiGA::Daemon < MiGA::MiGA
298
314
  say "Unsuccessful #{job[:task_name]}, rescheduling"
299
315
  else
300
316
  @jobs_running << job
301
- say "Spawned pid:#{job[:pid]}#{
302
- " to #{job[:hostk]}:#{nodelist[job[:hostk]]}" if job[:hostk]
303
- } for #{job[:task_name]}"
317
+ job_host = " to #{job[:hostk]}:#{nodelist[job[:hostk]]}" if job[:hostk]
318
+ say "Spawned pid:#{job[:pid]}#{job_host} for #{job[:task_name]}"
304
319
  end
305
320
  end
306
321
 
@@ -312,10 +327,11 @@ class MiGA::Daemon < MiGA::MiGA
312
327
  var: %w[key value],
313
328
  alive: %w[pid],
314
329
  kill: %w[pid]
315
- }.each do |k,v|
316
- runopts(
317
- k, runopts(k).gsub(/%(\d+\$)?d/, '%\\1s') % v.map{ |i| "{{#{i}}}" }
318
- ) if !runopts(k).nil? && runopts(k) =~ /%(\d+\$)?[ds]/
330
+ }.each do |k, v|
331
+ if !runopts(k).nil? && runopts(k) =~ /%(\d+\$)?[ds]/
332
+ runopts(k,
333
+ runopts(k).gsub(/%(\d+\$)?d/, '%\\1s') % v.map { |i| "{{#{i}}}" })
334
+ end
319
335
  end
320
336
  runopts(:format_version, 1)
321
337
  end