miga-base 0.7.4.0 → 0.7.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (116) hide show
  1. checksums.yaml +4 -4
  2. data/lib/miga/cli.rb +10 -8
  3. data/lib/miga/cli/action.rb +2 -3
  4. data/lib/miga/cli/action/about.rb +5 -6
  5. data/lib/miga/cli/action/add.rb +18 -12
  6. data/lib/miga/cli/action/add_result.rb +2 -3
  7. data/lib/miga/cli/action/archive.rb +1 -2
  8. data/lib/miga/cli/action/classify_wf.rb +8 -6
  9. data/lib/miga/cli/action/console.rb +0 -1
  10. data/lib/miga/cli/action/daemon.rb +7 -7
  11. data/lib/miga/cli/action/date.rb +0 -1
  12. data/lib/miga/cli/action/derep_wf.rb +5 -4
  13. data/lib/miga/cli/action/doctor.rb +28 -20
  14. data/lib/miga/cli/action/doctor/base.rb +29 -6
  15. data/lib/miga/cli/action/edit.rb +1 -2
  16. data/lib/miga/cli/action/files.rb +8 -8
  17. data/lib/miga/cli/action/find.rb +5 -6
  18. data/lib/miga/cli/action/generic.rb +7 -7
  19. data/lib/miga/cli/action/get.rb +20 -17
  20. data/lib/miga/cli/action/get_db.rb +8 -2
  21. data/lib/miga/cli/action/index_wf.rb +1 -1
  22. data/lib/miga/cli/action/init.rb +34 -29
  23. data/lib/miga/cli/action/init/daemon_helper.rb +65 -43
  24. data/lib/miga/cli/action/lair.rb +7 -7
  25. data/lib/miga/cli/action/ln.rb +6 -6
  26. data/lib/miga/cli/action/ls.rb +1 -2
  27. data/lib/miga/cli/action/ncbi_get.rb +11 -3
  28. data/lib/miga/cli/action/new.rb +4 -4
  29. data/lib/miga/cli/action/next_step.rb +0 -1
  30. data/lib/miga/cli/action/preproc_wf.rb +3 -3
  31. data/lib/miga/cli/action/quality_wf.rb +1 -1
  32. data/lib/miga/cli/action/rm.rb +2 -3
  33. data/lib/miga/cli/action/run.rb +8 -8
  34. data/lib/miga/cli/action/stats.rb +3 -3
  35. data/lib/miga/cli/action/summary.rb +7 -6
  36. data/lib/miga/cli/action/tax_dist.rb +8 -4
  37. data/lib/miga/cli/action/tax_index.rb +3 -4
  38. data/lib/miga/cli/action/tax_set.rb +7 -6
  39. data/lib/miga/cli/action/tax_test.rb +6 -5
  40. data/lib/miga/cli/action/wf.rb +21 -19
  41. data/lib/miga/cli/base.rb +34 -32
  42. data/lib/miga/cli/objects_helper.rb +24 -17
  43. data/lib/miga/cli/opt_helper.rb +3 -2
  44. data/lib/miga/common.rb +2 -5
  45. data/lib/miga/common/base.rb +15 -16
  46. data/lib/miga/common/format.rb +8 -5
  47. data/lib/miga/common/hooks.rb +1 -4
  48. data/lib/miga/common/path.rb +4 -9
  49. data/lib/miga/common/with_daemon.rb +5 -2
  50. data/lib/miga/common/with_daemon_class.rb +1 -1
  51. data/lib/miga/common/with_result.rb +2 -1
  52. data/lib/miga/daemon.rb +51 -35
  53. data/lib/miga/daemon/base.rb +0 -2
  54. data/lib/miga/dataset.rb +47 -37
  55. data/lib/miga/dataset/base.rb +52 -37
  56. data/lib/miga/dataset/hooks.rb +3 -4
  57. data/lib/miga/dataset/result.rb +17 -1
  58. data/lib/miga/json.rb +5 -7
  59. data/lib/miga/lair.rb +4 -0
  60. data/lib/miga/metadata.rb +4 -3
  61. data/lib/miga/project.rb +29 -20
  62. data/lib/miga/project/base.rb +52 -37
  63. data/lib/miga/project/dataset.rb +27 -13
  64. data/lib/miga/project/hooks.rb +0 -3
  65. data/lib/miga/project/result.rb +14 -5
  66. data/lib/miga/remote_dataset.rb +85 -72
  67. data/lib/miga/remote_dataset/base.rb +11 -13
  68. data/lib/miga/remote_dataset/download.rb +33 -12
  69. data/lib/miga/result.rb +34 -25
  70. data/lib/miga/result/base.rb +0 -2
  71. data/lib/miga/result/dates.rb +1 -3
  72. data/lib/miga/result/source.rb +15 -16
  73. data/lib/miga/result/stats.rb +36 -25
  74. data/lib/miga/tax_dist.rb +6 -3
  75. data/lib/miga/tax_index.rb +17 -17
  76. data/lib/miga/taxonomy.rb +6 -1
  77. data/lib/miga/taxonomy/base.rb +19 -15
  78. data/lib/miga/version.rb +19 -16
  79. data/test/common_test.rb +3 -11
  80. data/test/daemon_helper.rb +38 -0
  81. data/test/daemon_test.rb +73 -101
  82. data/test/dataset_test.rb +58 -59
  83. data/test/format_test.rb +3 -11
  84. data/test/hook_test.rb +50 -55
  85. data/test/json_test.rb +7 -8
  86. data/test/lair_test.rb +22 -28
  87. data/test/metadata_test.rb +6 -14
  88. data/test/project_test.rb +33 -39
  89. data/test/remote_dataset_test.rb +20 -28
  90. data/test/result_stats_test.rb +17 -27
  91. data/test/result_test.rb +41 -34
  92. data/test/tax_dist_test.rb +0 -2
  93. data/test/tax_index_test.rb +4 -10
  94. data/test/taxonomy_test.rb +7 -9
  95. data/test/test_helper.rb +42 -1
  96. data/test/with_daemon_test.rb +14 -22
  97. data/utils/cleanup-databases.rb +6 -5
  98. data/utils/distance/base.rb +0 -1
  99. data/utils/distance/commands.rb +19 -12
  100. data/utils/distance/database.rb +24 -21
  101. data/utils/distance/pipeline.rb +12 -9
  102. data/utils/distance/runner.rb +14 -13
  103. data/utils/distance/temporal.rb +1 -3
  104. data/utils/distances.rb +1 -1
  105. data/utils/domain-ess-genes.rb +7 -7
  106. data/utils/index_metadata.rb +4 -2
  107. data/utils/mytaxa_scan.rb +18 -16
  108. data/utils/representatives.rb +5 -4
  109. data/utils/requirements.txt +1 -1
  110. data/utils/subclade/base.rb +0 -1
  111. data/utils/subclade/pipeline.rb +7 -6
  112. data/utils/subclade/runner.rb +9 -9
  113. data/utils/subclade/temporal.rb +0 -2
  114. data/utils/subclades-compile.rb +39 -37
  115. data/utils/subclades.rb +1 -1
  116. metadata +3 -2
@@ -1,15 +1,17 @@
1
1
  # @package MiGA
2
2
  # @license Artistic-2.0
3
3
 
4
- module MiGA::Cli::ObjectsHelper
4
+ module MiGA::Cli::ObjectsHelper
5
5
  ##
6
6
  # Get the project defined in the CLI by parameter +name+ and +flag+
7
7
  def load_project(name = :project, flag = '-P')
8
8
  return @objects[name] unless @objects[name].nil?
9
+
9
10
  ensure_par(name => flag)
10
11
  say "Loading project: #{self[name]}"
11
12
  @objects[name] = MiGA::Project.load(self[name])
12
13
  raise "Cannot load project: #{self[name]}" if @objects[name].nil?
14
+
13
15
  @objects[name]
14
16
  end
15
17
 
@@ -23,6 +25,7 @@ module MiGA::Cli::ObjectsHelper
23
25
  end
24
26
  d = load_project.dataset(name)
25
27
  raise "Cannot load dataset: #{self[:dataset]}" if !silent && d.nil?
28
+
26
29
  return d
27
30
  end
28
31
 
@@ -37,16 +40,18 @@ module MiGA::Cli::ObjectsHelper
37
40
  # If +silent=true+, it allows failures silently
38
41
  def load_and_filter_datasets(silent = false)
39
42
  return @objects[:filtered_datasets] unless @objects[:filtered_datasets].nil?
43
+
40
44
  say 'Listing datasets'
41
- ds = if ! self[:dataset].nil?
42
- [load_dataset(silent)].compact
43
- elsif ! self[:ds_list].nil?
44
- File.readlines(self[:ds_list]).map do |i|
45
- load_dataset(silent, i.chomp)
46
- end.compact
47
- else
48
- load_project.datasets
49
- end
45
+ ds =
46
+ if !self[:dataset].nil?
47
+ [load_dataset(silent)].compact
48
+ elsif !self[:ds_list].nil?
49
+ File.readlines(self[:ds_list]).map do |i|
50
+ load_dataset(silent, i.chomp)
51
+ end.compact
52
+ else
53
+ load_project.datasets
54
+ end
50
55
  k = 0
51
56
  n = ds.size
52
57
  ds.select! do |d|
@@ -61,31 +66,34 @@ module MiGA::Cli::ObjectsHelper
61
66
  o
62
67
  end
63
68
  say ''
64
- ds = ds.values_at(self[:dataset_k]-1) unless self[:dataset_k].nil?
69
+ ds = ds.values_at(self[:dataset_k] - 1) unless self[:dataset_k].nil?
65
70
  @objects[:filtered_datasets] = ds
66
71
  end
67
72
 
68
73
  def load_result
69
74
  return @objects[:result] unless @objects[:result].nil?
75
+
70
76
  ensure_par(result: '-r')
71
77
  obj = load_project_or_dataset
72
78
  if obj.class.RESULT_DIRS[self[:result]].nil?
73
- klass = obj.class.to_s.gsub(/.*::/,'')
79
+ klass = obj.class.to_s.gsub(/.*::/, '')
74
80
  raise "Unsupported result for #{klass}: #{self[:result]}"
75
81
  end
76
82
  r = obj.add_result(self[:result], false)
77
83
  raise "Cannot load result: #{self[:result]}" if r.nil?
84
+
78
85
  @objects[:result] = r
79
86
  end
80
87
 
81
88
  def add_metadata(obj, cli = self)
82
89
  raise "Unsupported object: #{obj.class}" unless obj.respond_to? :metadata
90
+
83
91
  cli[:metadata].split(',').each do |pair|
84
- (k,v) = pair.split('=')
92
+ (k, v) = pair.split('=')
85
93
  case v
86
- when 'true'; v = true
87
- when 'false'; v = false
88
- when 'nil'; v = nil
94
+ when 'true'; v = true
95
+ when 'false'; v = false
96
+ when 'nil'; v = nil
89
97
  end
90
98
  if k == '_step'
91
99
  obj.metadata["_try_#{v}"] ||= 0
@@ -99,4 +107,3 @@ module MiGA::Cli::ObjectsHelper
99
107
  obj
100
108
  end
101
109
  end
102
-
@@ -1,7 +1,7 @@
1
1
  # @package MiGA
2
2
  # @license Artistic-2.0
3
3
 
4
- module MiGA::Cli::OptHelper
4
+ module MiGA::Cli::OptHelper
5
5
  ##
6
6
  # Send MiGA's banner to OptionParser +opt+
7
7
  def banner(opt)
@@ -19,6 +19,7 @@ module MiGA::Cli::OptHelper
19
19
  # Executes only once, unless +#opt_common = true+ is passed between calls
20
20
  def opt_common(opt)
21
21
  return unless @opt_common
22
+
22
23
  if interactive
23
24
  opt.on(
24
25
  '--auto',
@@ -159,6 +160,6 @@ module MiGA::Cli::OptHelper
159
160
  # If +sym+ is nil, +flag+ is used as Symbol
160
161
  def opt_flag(opt, flag, description, sym = nil)
161
162
  sym = flag.to_sym if sym.nil?
162
- opt.on("--#{flag.to_s.gsub('_','-')}", description) { |v| self[sym] = v }
163
+ opt.on("--#{flag.to_s.gsub('_', '-')}", description) { |v| self[sym] = v }
163
164
  end
164
165
  end
data/lib/miga/common.rb CHANGED
@@ -11,12 +11,11 @@ require 'miga/common/format'
11
11
  # Generic class used to handle system-wide information and methods, and parent
12
12
  # of all other MiGA::* classes.
13
13
  class MiGA::MiGA
14
-
15
14
  include MiGA::Common
16
-
15
+
17
16
  extend MiGA::Common::Path
18
17
  extend MiGA::Common::Format
19
-
18
+
20
19
  ENV['MIGA_HOME'] ||= ENV['HOME']
21
20
 
22
21
  ##
@@ -45,6 +44,4 @@ class MiGA::MiGA
45
44
  io = par.first.is_a?(IO) ? par.shift : $stderr
46
45
  io.puts(*par.map { |i| "[#{Time.now}] #{i}" })
47
46
  end
48
-
49
47
  end
50
-
@@ -1,42 +1,43 @@
1
-
2
1
  class MiGA::MiGA
3
-
4
2
  # Class-level
5
3
  class << self
6
4
  ##
7
- # Turn on debugging.
8
- def DEBUG_ON ; @@DEBUG=true end
5
+ # Turn on debugging
6
+ def DEBUG_ON
7
+ @@DEBUG = true
8
+ end
9
9
 
10
10
  ##
11
- # Turn off debugging.
12
- def DEBUG_OFF ; @@DEBUG=false end
11
+ # Turn off debugging
12
+ def DEBUG_OFF
13
+ @@DEBUG = false
14
+ end
13
15
 
14
16
  ##
15
- # Turn on debug tracing (and debugging).
17
+ # Turn on debug tracing (and debugging)
16
18
  def DEBUG_TRACE_ON
17
- @@DEBUG_TRACE=true
19
+ @@DEBUG_TRACE = true
18
20
  DEBUG_ON()
19
21
  end
20
22
 
21
23
  ##
22
- # Turn off debug tracing (but not debugging).
24
+ # Turn off debug tracing (but not debugging)
23
25
  def DEBUG_TRACE_OFF
24
- @@DEBUG_TRACE=false
26
+ @@DEBUG_TRACE = false
25
27
  end
26
28
 
27
29
  ##
28
- # Send debug message.
30
+ # Send debug message
29
31
  def DEBUG(*args)
30
32
  $stderr.puts(*args) if @@DEBUG
31
33
  $stderr.puts(
32
- caller.map{ |v| v.gsub(/^/,' ') }.join("\n") ) if @@DEBUG_TRACE
34
+ caller.map { |v| v.gsub(/^/, ' ') }.join("\n")
35
+ ) if @@DEBUG_TRACE
33
36
  end
34
37
  end
35
-
36
38
  end
37
39
 
38
40
  module MiGA::Common
39
-
40
41
  ##
41
42
  # Should debugging information be reported?
42
43
  @@DEBUG = false
@@ -44,6 +45,4 @@ module MiGA::Common
44
45
  ##
45
46
  # Should the trace of debugging information be reported?
46
47
  @@DEBUG_TRACE = false
47
-
48
48
  end
49
-
@@ -1,4 +1,3 @@
1
-
2
1
  require 'tempfile'
3
2
  require 'zlib'
4
3
 
@@ -69,30 +68,34 @@ module MiGA::Common::Format
69
68
  # a FastA or FastQ file (supports gzipped files). The +format+ must be a
70
69
  # Symbol, one of +:fasta+ or +:fastq+. Additional estimations can be
71
70
  # controlled via the +opts+ Hash. Supported options include:
72
- # - +:n50+: If true, it also returns the N50 and the median (in bp).
73
- # - +gc+: If true, it also returns the G+C content (in %).
71
+ # - +:n50+: If true, it also returns the N50 and the median (in bp)
72
+ # - +:gc+: If true, it also returns the G+C content (in %)
73
+ # - +:x+: If true, it also returns the undetermined bases content (in %)
74
74
  def seqs_length(file, format, opts = {})
75
75
  fh = file =~ /\.gz/ ? Zlib::GzipReader.open(file) : File.open(file, 'r')
76
76
  l = []
77
77
  gc = 0
78
+ xn = 0
78
79
  i = 0 # <- Zlib::GzipReader doesn't set `$.`
79
80
  fh.each_line do |ln|
80
81
  i += 1
81
82
  if (format == :fasta and ln =~ /^>/) or
82
- (format == :fastq and (i % 4) == 1)
83
+ (format == :fastq and (i % 4) == 1)
83
84
  l << 0
84
85
  elsif format == :fasta or (i % 4) == 2
85
86
  l[l.size - 1] += ln.chomp.size
86
87
  gc += ln.scan(/[GCgc]/).count if opts[:gc]
88
+ xn += ln.scan(/[XNxn]/).count if opts[:x]
87
89
  end
88
90
  end
89
91
  fh.close
90
92
 
91
- o = { n: l.size, tot: l.inject(:+) }
93
+ o = { n: l.size, tot: l.inject(:+), max: l.max }
92
94
  o[:avg] = o[:tot].to_f / l.size
93
95
  o[:var] = l.map { |a| a**2 }.inject(:+).to_f / l.size - o[:avg]**2
94
96
  o[:sd] = Math.sqrt o[:var]
95
97
  o[:gc] = 100.0 * gc / o[:tot] if opts[:gc]
98
+ o[:x] = 100.0 * xn / o[:tot] if opts[:x]
96
99
  if opts[:n50]
97
100
  l.sort!
98
101
  thr = o[:tot] / 2
@@ -1,8 +1,6 @@
1
-
2
1
  ##
3
2
  # Helper module including specific functions to handle dataset hooks.
4
3
  module MiGA::Common::Hooks
5
-
6
4
  ##
7
5
  # Call the hook with symbol +event+ and any parameters +event_args+
8
6
  def pull_hook(event, *event_args)
@@ -12,7 +10,7 @@ module MiGA::Common::Hooks
12
10
  event_queue.each do |i|
13
11
  action = i.first
14
12
  hook_name = :"hook_#{action}"
15
- hook_args = i[1 .. -1]
13
+ hook_args = i[1..-1]
16
14
  if respond_to? hook_name
17
15
  MiGA::MiGA.DEBUG "Hook: #{self.class}(#{event} > #{action})"
18
16
  self.send(hook_name, hook_args, event_args)
@@ -45,5 +43,4 @@ module MiGA::Common::Hooks
45
43
  def hook_run_lambda(hook_args, event_args)
46
44
  hook_args.first[*event_args]
47
45
  end
48
-
49
46
  end
@@ -1,6 +1,4 @@
1
-
2
1
  module MiGA::Common::Path
3
-
4
2
  ##
5
3
  # Root path to MiGA (as estimated from the location of the current file).
6
4
  def root_path
@@ -11,30 +9,27 @@ module MiGA::Common::Path
11
9
  # Path to a script to be executed for +task+. Supported +opts+ are:
12
10
  # - +:miga+ Path to the MiGA home to use. If not passed, the home of the
13
11
  # library is used).
14
- def script_path(task, opts={})
12
+ def script_path(task, opts = {})
15
13
  opts[:miga] ||= root_path
16
14
  File.expand_path("scripts/#{task}.bash", opts[:miga])
17
15
  end
18
-
19
16
  end
20
17
 
21
18
  ##
22
19
  # MiGA extensions to the File class.
23
20
  class File
24
-
25
21
  ##
26
22
  # Method to transfer a file from +old_name+ to +new_name+, using a +method+
27
23
  # that can be one of :symlink for File#symlink, :hardlink for File#link, or
28
24
  # :copy for FileUtils#cp_r.
29
25
  def self.generic_transfer(old_name, new_name, method)
30
26
  return nil if exist? new_name
31
- if(method==:copy)
27
+
28
+ if (method == :copy)
32
29
  FileUtils.cp_r(old_name, new_name)
33
30
  else
34
- method=:link if method==:hardlink
31
+ method = :link if method == :hardlink
35
32
  File.send(method, old_name, new_name)
36
33
  end
37
34
  end
38
-
39
35
  end
40
-
@@ -1,4 +1,3 @@
1
-
2
1
  require 'daemons'
3
2
  require 'miga/common/with_daemon_class'
4
3
 
@@ -23,7 +22,7 @@ module MiGA::Common::WithDaemon
23
22
  def output_file
24
23
  File.join(daemon_home, "#{daemon_name}.output")
25
24
  end
26
-
25
+
27
26
  def terminate_file
28
27
  File.join(daemon_home, 'terminate-daemon')
29
28
  end
@@ -46,6 +45,7 @@ module MiGA::Common::WithDaemon
46
45
  # Is the daemon active?
47
46
  def active?
48
47
  return false unless File.exist? alive_file
48
+
49
49
  (last_alive || Time.new(0)) > Time.now - 60
50
50
  end
51
51
 
@@ -74,8 +74,10 @@ module MiGA::Common::WithDaemon
74
74
  i += 1
75
75
  return :no_home unless Dir.exist? daemon_home
76
76
  return :no_process_alive unless process_alive? pid
77
+
77
78
  write_alive_file if i % 30 == 0
78
79
  return :termination_file if termination_file? pid
80
+
79
81
  sleep(1)
80
82
  end
81
83
  end
@@ -100,6 +102,7 @@ module MiGA::Common::WithDaemon
100
102
  # if it does. Do not kill any process if +pid+ is +nil+
101
103
  def termination_file?(pid)
102
104
  return false unless File.exist? terminate_file
105
+
103
106
  say 'Found termination file, terminating'
104
107
  File.unlink(terminate_file)
105
108
  terminate
@@ -1,4 +1,3 @@
1
-
2
1
  ##
3
2
  # Helper module with specific class-level functions to be used with
4
3
  # +include MiGA::Common::WithDaemon+.
@@ -28,6 +27,7 @@ module MiGA::Common::WithDaemonClass
28
27
  f = terminated_file(path) unless File.exist? f
29
28
  c = File.read(f)
30
29
  return nil if c.nil? || c.empty?
30
+
31
31
  Time.parse(c)
32
32
  rescue Errno::ENOENT
33
33
  return nil
@@ -1,4 +1,3 @@
1
-
2
1
  ##
3
2
  # Helper module including specific functions to handle objects that
4
3
  # have results.
@@ -24,6 +23,7 @@ module MiGA::Common::WithResult
24
23
  def add_result(task, save = true, opts = {})
25
24
  task = task.to_sym
26
25
  return nil if result_dirs[task].nil?
26
+
27
27
  base = File.join(
28
28
  project.path, "data/#{result_dirs[task]}/#{result_base}"
29
29
  )
@@ -44,6 +44,7 @@ module MiGA::Common::WithResult
44
44
  def result(task)
45
45
  task = task.to_sym
46
46
  return nil if result_dirs[task].nil?
47
+
47
48
  MiGA::Result.load(
48
49
  "#{project.path}/data/#{result_dirs[task]}/#{result_base}.json"
49
50
  )
data/lib/miga/daemon.rb CHANGED
@@ -8,7 +8,6 @@ require 'miga/daemon/base'
8
8
  ##
9
9
  # MiGA Daemons handling job submissions.
10
10
  class MiGA::Daemon < MiGA::MiGA
11
-
12
11
  include MiGA::Daemon::Base
13
12
  include MiGA::Common::WithDaemon
14
13
  extend MiGA::Common::WithDaemonClass
@@ -19,6 +18,7 @@ class MiGA::Daemon < MiGA::MiGA
19
18
  # full path to the project's 'daemon' folder
20
19
  def daemon_home(project)
21
20
  return project if project.is_a? String
21
+
22
22
  File.join(project.path, 'daemon')
23
23
  end
24
24
  end
@@ -48,7 +48,7 @@ class MiGA::Daemon < MiGA::MiGA
48
48
  default_json = File.expand_path('.miga_daemon.json', ENV['MIGA_HOME'])
49
49
  MiGA::Json.parse(
50
50
  json, default: File.exist?(default_json) ? default_json : nil
51
- ).each { |k,v| runopts(k, v) }
51
+ ).each { |k, v| runopts(k, v) }
52
52
  update_format_0
53
53
  @jobs_to_run = []
54
54
  @jobs_running = []
@@ -82,14 +82,13 @@ class MiGA::Daemon < MiGA::MiGA
82
82
  def daemon_loop
83
83
  l_say(3, 'Daemon loop start')
84
84
  reload_project
85
- check_datasets
86
- check_project
87
- if shutdown_when_done? && jobs_running.size + jobs_to_run.size == 0
85
+ check_datasets or check_project
86
+ if shutdown_when_done? && (jobs_running.size + jobs_to_run.size).zero?
88
87
  say 'Nothing else to do, shutting down'
89
88
  return false
90
89
  end
91
90
  flush!
92
- if loop_i % 12 == 0
91
+ if (loop_i % 12).zero?
93
92
  purge!
94
93
  recalculate_status!
95
94
  end
@@ -137,12 +136,13 @@ class MiGA::Daemon < MiGA::MiGA
137
136
  def load_status
138
137
  f_path = File.join(daemon_home, 'status.json')
139
138
  return unless File.size? f_path
139
+
140
140
  say 'Loading previous status in daemon/status.json:'
141
141
  status = MiGA::Json.parse(f_path)
142
- status.keys.each do |i|
142
+ status.each_key do |i|
143
143
  status[i].map! do |j|
144
144
  j.tap do |k|
145
- unless k[:ds].nil? or k[:ds_name] == 'miga-project'
145
+ unless k[:ds].nil? || k[:ds_name] == 'miga-project'
146
146
  k[:ds] = project.dataset(k[:ds_name])
147
147
  end
148
148
  k[:job] = k[:job].to_sym unless k[:job].nil?
@@ -158,14 +158,19 @@ class MiGA::Daemon < MiGA::MiGA
158
158
  end
159
159
 
160
160
  ##
161
- # Traverse datasets
161
+ # Traverse datasets, and returns boolean indicating if at any datasets
162
+ # are incomplete
162
163
  def check_datasets
163
164
  l_say(2, 'Checking datasets')
165
+ o = false
164
166
  project.each_dataset do |ds|
165
167
  next unless ds.status == :incomplete
166
- to_run = ds.next_preprocessing(false)
167
- queue_job(:d, ds) unless to_run.nil?
168
+ next if ds.next_preprocessing(false).nil?
169
+
170
+ o = true
171
+ queue_job(:d, ds)
168
172
  end
173
+ o
169
174
  end
170
175
 
171
176
  ##
@@ -173,8 +178,14 @@ class MiGA::Daemon < MiGA::MiGA
173
178
  # project-level tasks
174
179
  def check_project
175
180
  l_say(2, 'Checking project')
181
+
182
+ # Ignore task if the project has no datasets
176
183
  return if project.dataset_names.empty?
184
+
185
+ # Double-check if all datasets are ready
177
186
  return unless project.done_preprocessing?(false)
187
+
188
+ # Queue project-level job
178
189
  to_run = project.next_task(nil, false)
179
190
  queue_job(:p) unless to_run.nil?
180
191
  end
@@ -185,13 +196,14 @@ class MiGA::Daemon < MiGA::MiGA
185
196
  # scheduler (or to bash or ssh) see #flush!
186
197
  def queue_job(job, ds = nil)
187
198
  return nil unless get_job(job, ds).nil?
199
+
188
200
  ds_name = (ds.nil? ? 'miga-project' : ds.name)
189
201
  say 'Queueing %s:%s' % [ds_name, job]
190
202
  vars = {
191
203
  'PROJECT' => project.path,
192
204
  'RUNTYPE' => runopts(:type),
193
- 'CORES' => ppn,
194
- 'MIGA' => MiGA::MiGA.root_path
205
+ 'CORES' => ppn,
206
+ 'MIGA' => MiGA::MiGA.root_path
195
207
  }
196
208
  vars['DATASET'] = ds.name unless ds.nil?
197
209
  log_dir = File.expand_path("daemon/#{job}", project.path)
@@ -199,9 +211,10 @@ class MiGA::Daemon < MiGA::MiGA
199
211
  task_name = "#{project.metadata[:name][0..9]}:#{job}:#{ds_name}"
200
212
  to_run = { ds: ds, ds_name: ds_name, job: job, task_name: task_name }
201
213
  to_run[:cmd] = runopts(:cmd).miga_variables(
202
- script: MiGA::MiGA.script_path(job, miga:vars['MIGA'], project: project),
203
- vars: vars.map { |k, v|
204
- runopts(:var).miga_variables(key: k, value: v) }.join(runopts(:varsep)),
214
+ script: MiGA::MiGA.script_path(job, miga: vars['MIGA'], project: project),
215
+ vars: vars.map do |k, v|
216
+ runopts(:var).miga_variables(key: k, value: v)
217
+ end.join(runopts(:varsep)),
205
218
  cpus: ppn,
206
219
  log: File.expand_path("#{ds_name}.log", log_dir),
207
220
  task_name: task_name,
@@ -216,9 +229,9 @@ class MiGA::Daemon < MiGA::MiGA
216
229
  def get_job(job, ds = nil)
217
230
  (jobs_to_run + jobs_running).find do |j|
218
231
  if ds.nil?
219
- j[:ds].nil? and j[:job] == job
232
+ j[:ds].nil? && j[:job] == job
220
233
  else
221
- (! j[:ds].nil?) and j[:ds].name == ds.name and j[:job] == job
234
+ !j[:ds].nil? && j[:ds].name == ds.name && j[:job] == job
222
235
  end
223
236
  end
224
237
  end
@@ -230,14 +243,15 @@ class MiGA::Daemon < MiGA::MiGA
230
243
  # Check for finished jobs
231
244
  l_say(2, 'Checking for finished jobs')
232
245
  @jobs_running.select! do |job|
233
- ongoing = case job[:job].to_s
234
- when 'd'
235
- !job[:ds].nil? && !job[:ds].next_preprocessing(false).nil?
236
- when 'p'
237
- !project.next_task(nil, false).nil?
238
- else
239
- (job[:ds].nil? ? project : job[:ds]).add_result(job[:job], false).nil?
240
- end
246
+ ongoing =
247
+ case job[:job].to_s
248
+ when 'd'
249
+ !job[:ds].nil? && !job[:ds].next_preprocessing(false).nil?
250
+ when 'p'
251
+ !project.next_task(nil, false).nil?
252
+ else
253
+ (job[:ds].nil? ? project : job[:ds]).add_result(job[:job], false).nil?
254
+ end
241
255
  say "Completed pid:#{job[:pid]} for #{job[:task_name]}" unless ongoing
242
256
  ongoing
243
257
  end
@@ -246,8 +260,9 @@ class MiGA::Daemon < MiGA::MiGA
246
260
  @jobs_to_run.rotate! rand(jobs_to_run.size)
247
261
 
248
262
  # Launch as many +jobs_to_run+ as possible
249
- while hostk = next_host
263
+ while (hostk = next_host)
250
264
  break if jobs_to_run.empty?
265
+
251
266
  launch_job(@jobs_to_run.shift, hostk)
252
267
  end
253
268
  end
@@ -257,7 +272,8 @@ class MiGA::Daemon < MiGA::MiGA
257
272
  # In any other daemons, returns true as long as #maxjobs is not reached
258
273
  def next_host
259
274
  return jobs_running.size < maxjobs if runopts(:type) != 'ssh'
260
- allk = (0 .. nodelist.size-1).to_a
275
+
276
+ allk = (0..nodelist.size - 1).to_a
261
277
  busyk = jobs_running.map { |k| k[:hostk] }
262
278
  (allk - busyk).first
263
279
  end
@@ -298,9 +314,8 @@ class MiGA::Daemon < MiGA::MiGA
298
314
  say "Unsuccessful #{job[:task_name]}, rescheduling"
299
315
  else
300
316
  @jobs_running << job
301
- say "Spawned pid:#{job[:pid]}#{
302
- " to #{job[:hostk]}:#{nodelist[job[:hostk]]}" if job[:hostk]
303
- } for #{job[:task_name]}"
317
+ job_host = " to #{job[:hostk]}:#{nodelist[job[:hostk]]}" if job[:hostk]
318
+ say "Spawned pid:#{job[:pid]}#{job_host} for #{job[:task_name]}"
304
319
  end
305
320
  end
306
321
 
@@ -312,10 +327,11 @@ class MiGA::Daemon < MiGA::MiGA
312
327
  var: %w[key value],
313
328
  alive: %w[pid],
314
329
  kill: %w[pid]
315
- }.each do |k,v|
316
- runopts(
317
- k, runopts(k).gsub(/%(\d+\$)?d/, '%\\1s') % v.map{ |i| "{{#{i}}}" }
318
- ) if !runopts(k).nil? && runopts(k) =~ /%(\d+\$)?[ds]/
330
+ }.each do |k, v|
331
+ if !runopts(k).nil? && runopts(k) =~ /%(\d+\$)?[ds]/
332
+ runopts(k,
333
+ runopts(k).gsub(/%(\d+\$)?d/, '%\\1s') % v.map { |i| "{{#{i}}}" })
334
+ end
319
335
  end
320
336
  runopts(:format_version, 1)
321
337
  end