miga-base 0.7.3.0 → 0.7.7.0

data/lib/miga/cli/objects_helper.rb

@@ -1,15 +1,17 @@
 # @package MiGA
 # @license Artistic-2.0
 
-module MiGA::Cli::ObjectsHelper 
+module MiGA::Cli::ObjectsHelper
   ##
   # Get the project defined in the CLI by parameter +name+ and +flag+
   def load_project(name = :project, flag = '-P')
     return @objects[name] unless @objects[name].nil?
+
     ensure_par(name => flag)
     say "Loading project: #{self[name]}"
     @objects[name] = MiGA::Project.load(self[name])
     raise "Cannot load project: #{self[name]}" if @objects[name].nil?
+
     @objects[name]
   end
 
@@ -23,6 +25,7 @@ module MiGA::Cli::ObjectsHelper
     end
     d = load_project.dataset(name)
     raise "Cannot load dataset: #{self[:dataset]}" if !silent && d.nil?
+
     return d
   end
 
@@ -37,16 +40,18 @@ module MiGA::Cli::ObjectsHelper
   # If +silent=true+, it allows failures silently
   def load_and_filter_datasets(silent = false)
     return @objects[:filtered_datasets] unless @objects[:filtered_datasets].nil?
+
     say 'Listing datasets'
-    ds = if ! self[:dataset].nil?
-      [load_dataset(silent)].compact
-    elsif ! self[:ds_list].nil?
-      File.readlines(self[:ds_list]).map do |i|
-        load_dataset(silent, i.chomp)
-      end.compact
-    else
-      load_project.datasets
-    end
+    ds =
+      if !self[:dataset].nil?
+        [load_dataset(silent)].compact
+      elsif !self[:ds_list].nil?
+        File.readlines(self[:ds_list]).map do |i|
+          load_dataset(silent, i.chomp)
+        end.compact
+      else
+        load_project.datasets
+      end
     k = 0
     n = ds.size
     ds.select! do |d|
@@ -61,31 +66,36 @@ module MiGA::Cli::ObjectsHelper
       o
     end
     say ''
-    ds = ds.values_at(self[:dataset_k]-1) unless self[:dataset_k].nil?
+    ds = ds.values_at(self[:dataset_k] - 1) unless self[:dataset_k].nil?
     @objects[:filtered_datasets] = ds
   end
 
   def load_result
     return @objects[:result] unless @objects[:result].nil?
+
     ensure_par(result: '-r')
     obj = load_project_or_dataset
     if obj.class.RESULT_DIRS[self[:result]].nil?
-      klass = obj.class.to_s.gsub(/.*::/,'')
+      klass = obj.class.to_s.gsub(/.*::/, '')
       raise "Unsupported result for #{klass}: #{self[:result]}"
     end
     r = obj.add_result(self[:result], false)
-    raise "Cannot load result: #{self[:result]}" if r.nil?
+    if r.nil? && !self[:ignore_result_empty]
+      raise "Cannot load result: #{self[:result]}"
+    end
+
     @objects[:result] = r
   end
 
   def add_metadata(obj, cli = self)
     raise "Unsupported object: #{obj.class}" unless obj.respond_to? :metadata
+
     cli[:metadata].split(',').each do |pair|
-      (k,v) = pair.split('=')
+      (k, v) = pair.split('=')
       case v
-        when 'true';  v = true
-        when 'false'; v = false
-        when 'nil';   v = nil
+      when 'true';  v = true
+      when 'false'; v = false
+      when 'nil';   v = nil
       end
       if k == '_step'
         obj.metadata["_try_#{v}"] ||= 0
@@ -99,4 +109,3 @@ module MiGA::Cli::ObjectsHelper
     obj
   end
 end
-

data/lib/miga/cli/opt_helper.rb

@@ -1,7 +1,7 @@
 # @package MiGA
 # @license Artistic-2.0
 
-module MiGA::Cli::OptHelper 
+module MiGA::Cli::OptHelper
   ##
   # Send MiGA's banner to OptionParser +opt+
   def banner(opt)
@@ -19,6 +19,7 @@ module MiGA::Cli::OptHelper
   # Executes only once, unless +#opt_common = true+ is passed between calls
   def opt_common(opt)
     return unless @opt_common
+
     if interactive
       opt.on(
         '--auto',
@@ -159,6 +160,6 @@ module MiGA::Cli::OptHelper
   # If +sym+ is nil, +flag+ is used as Symbol
   def opt_flag(opt, flag, description, sym = nil)
     sym = flag.to_sym if sym.nil?
-    opt.on("--#{flag.to_s.gsub('_','-')}", description) { |v| self[sym] = v }
+    opt.on("--#{flag.to_s.gsub('_', '-')}", description) { |v| self[sym] = v }
   end
 end

data/lib/miga/common.rb

@@ -11,12 +11,11 @@ require 'miga/common/format'
 # Generic class used to handle system-wide information and methods, and parent
 # of all other MiGA::* classes.
 class MiGA::MiGA
-
   include MiGA::Common
-  
+
   extend MiGA::Common::Path
   extend MiGA::Common::Format
-  
+
   ENV['MIGA_HOME'] ||= ENV['HOME']
 
   ##
@@ -45,6 +44,4 @@ class MiGA::MiGA
     io = par.first.is_a?(IO) ? par.shift : $stderr
     io.puts(*par.map { |i| "[#{Time.now}] #{i}" })
   end
-
 end
-

data/lib/miga/common/base.rb

@@ -1,42 +1,43 @@
-
 class MiGA::MiGA
-  
   # Class-level
   class << self
     ##
-    # Turn on debugging.
-    def DEBUG_ON ; @@DEBUG=true end
+    # Turn on debugging
+    def DEBUG_ON
+      @@DEBUG = true
+    end
 
     ##
-    # Turn off debugging.
-    def DEBUG_OFF ; @@DEBUG=false end
+    # Turn off debugging
+    def DEBUG_OFF
+      @@DEBUG = false
+    end
 
     ##
-    # Turn on debug tracing (and debugging).
+    # Turn on debug tracing (and debugging)
     def DEBUG_TRACE_ON
-      @@DEBUG_TRACE=true
+      @@DEBUG_TRACE = true
       DEBUG_ON()
     end
 
     ##
-    # Turn off debug tracing (but not debugging).
+    # Turn off debug tracing (but not debugging)
     def DEBUG_TRACE_OFF
-      @@DEBUG_TRACE=false
+      @@DEBUG_TRACE = false
     end
 
     ##
-    # Send debug message.
+    # Send debug message
     def DEBUG(*args)
       $stderr.puts(*args) if @@DEBUG
       $stderr.puts(
-            caller.map{ |v| v.gsub(/^/,'     ') }.join("\n") ) if @@DEBUG_TRACE
+        caller.map { |v| v.gsub(/^/, '     ') }.join("\n")
+      ) if @@DEBUG_TRACE
     end
   end
-
 end
 
 module MiGA::Common
-
   ##
   # Should debugging information be reported?
   @@DEBUG = false
@@ -44,6 +45,4 @@ module MiGA::Common
   ##
   # Should the trace of debugging information be reported?
   @@DEBUG_TRACE = false
-
 end
-

data/lib/miga/common/format.rb

@@ -1,4 +1,3 @@
-
 require 'tempfile'
 require 'zlib'
 
@@ -69,30 +68,34 @@ module MiGA::Common::Format
   # a FastA or FastQ file (supports gzipped files). The +format+ must be a
   # Symbol, one of +:fasta+ or +:fastq+. Additional estimations can be
   # controlled via the +opts+ Hash. Supported options include:
-  # - +:n50+: If true, it also returns the N50 and the median (in bp).
-  # - +gc+: If true, it also returns the G+C content (in %).
+  # - +:n50+: If true, it also returns the N50 and the median (in bp)
+  # - +:gc+: If true, it also returns the G+C content (in %)
+  # - +:x+: If true, it also returns the undetermined bases content (in %)
   def seqs_length(file, format, opts = {})
     fh = file =~ /\.gz/ ? Zlib::GzipReader.open(file) : File.open(file, 'r')
     l = []
     gc = 0
+    xn = 0
     i = 0 # <- Zlib::GzipReader doesn't set `$.`
     fh.each_line do |ln|
       i += 1
       if (format == :fasta and ln =~ /^>/) or
-            (format == :fastq and (i % 4) == 1)
+         (format == :fastq and (i % 4) == 1)
         l << 0
       elsif format == :fasta or (i % 4) == 2
         l[l.size - 1] += ln.chomp.size
         gc += ln.scan(/[GCgc]/).count if opts[:gc]
+        xn += ln.scan(/[XNxn]/).count if opts[:x]
       end
     end
     fh.close
 
-    o = { n: l.size, tot: l.inject(:+) }
+    o = { n: l.size, tot: l.inject(:+), max: l.max }
     o[:avg] = o[:tot].to_f / l.size
     o[:var] = l.map { |a| a**2 }.inject(:+).to_f / l.size - o[:avg]**2
     o[:sd]  = Math.sqrt o[:var]
     o[:gc]  = 100.0 * gc / o[:tot] if opts[:gc]
+    o[:x]   = 100.0 * xn / o[:tot] if opts[:x]
     if opts[:n50]
       l.sort!
       thr = o[:tot] / 2

data/lib/miga/common/hooks.rb

@@ -1,8 +1,6 @@
-
 ##
 # Helper module including specific functions to handle dataset hooks.
 module MiGA::Common::Hooks
-
   ##
   # Call the hook with symbol +event+ and any parameters +event_args+
   def pull_hook(event, *event_args)
@@ -12,7 +10,7 @@ module MiGA::Common::Hooks
     event_queue.each do |i|
       action = i.first
       hook_name = :"hook_#{action}"
-      hook_args = i[1 .. -1]
+      hook_args = i[1..-1]
       if respond_to? hook_name
         MiGA::MiGA.DEBUG "Hook: #{self.class}(#{event} > #{action})"
         self.send(hook_name, hook_args, event_args)
@@ -45,5 +43,4 @@ module MiGA::Common::Hooks
   def hook_run_lambda(hook_args, event_args)
     hook_args.first[*event_args]
   end
-
 end

data/lib/miga/common/path.rb

@@ -1,6 +1,4 @@
-
 module MiGA::Common::Path
-
   ##
   # Root path to MiGA (as estimated from the location of the current file).
   def root_path
@@ -11,30 +9,27 @@ module MiGA::Common::Path
   # Path to a script to be executed for +task+. Supported +opts+ are:
   # - +:miga+ Path to the MiGA home to use. If not passed, the home of the
   #   library is used).
-  def script_path(task, opts={})
+  def script_path(task, opts = {})
     opts[:miga] ||= root_path
     File.expand_path("scripts/#{task}.bash", opts[:miga])
   end
-
 end
 
 ##
 # MiGA extensions to the File class.
 class File
-
   ##
   # Method to transfer a file from +old_name+ to +new_name+, using a +method+
   # that can be one of :symlink for File#symlink, :hardlink for File#link, or
   # :copy for FileUtils#cp_r.
   def self.generic_transfer(old_name, new_name, method)
     return nil if exist? new_name
-    if(method==:copy)
+
+    if (method == :copy)
       FileUtils.cp_r(old_name, new_name)
     else
-      method=:link if method==:hardlink
+      method = :link if method == :hardlink
       File.send(method, old_name, new_name)
     end
   end
-
 end
-

data/lib/miga/common/with_daemon.rb

@@ -1,4 +1,3 @@
-
 require 'daemons'
 require 'miga/common/with_daemon_class'
 
@@ -23,7 +22,7 @@ module MiGA::Common::WithDaemon
   def output_file
     File.join(daemon_home, "#{daemon_name}.output")
   end
-  
+
   def terminate_file
     File.join(daemon_home, 'terminate-daemon')
   end
@@ -46,7 +45,8 @@ module MiGA::Common::WithDaemon
   # Is the daemon active?
   def active?
     return false unless File.exist? alive_file
-    last_alive > Time.now - 60
+
+    (last_alive || Time.new(0)) > Time.now - 60
   end
 
   ##
@@ -74,8 +74,10 @@ module MiGA::Common::WithDaemon
       i += 1
       return :no_home unless Dir.exist? daemon_home
       return :no_process_alive unless process_alive? pid
+
       write_alive_file if i % 30 == 0
       return :termination_file if termination_file? pid
+
       sleep(1)
     end
   end
@@ -100,6 +102,7 @@ module MiGA::Common::WithDaemon
   # if it does. Do not kill any process if +pid+ is +nil+
   def termination_file?(pid)
     return false unless File.exist? terminate_file
+
     say 'Found termination file, terminating'
     File.unlink(terminate_file)
     terminate

data/lib/miga/common/with_daemon_class.rb

@@ -1,4 +1,3 @@
-
 ##
 # Helper module with specific class-level functions to be used with
 # +include MiGA::Common::WithDaemon+.
@@ -26,9 +25,11 @@ module MiGA::Common::WithDaemonClass
   def last_alive(path)
     f = alive_file(path)
     f = terminated_file(path) unless File.exist? f
-    return nil unless File.exist? f
     c = File.read(f)
     return nil if c.nil? || c.empty?
+
     Time.parse(c)
+  rescue Errno::ENOENT
+    return nil
   end
 end

data/lib/miga/common/with_result.rb

@@ -1,4 +1,3 @@
-
 ##
 # Helper module including specific functions to handle objects that
 # have results.
@@ -24,6 +23,7 @@ module MiGA::Common::WithResult
   def add_result(task, save = true, opts = {})
     task = task.to_sym
     return nil if result_dirs[task].nil?
+
     base = File.join(
       project.path, "data/#{result_dirs[task]}/#{result_base}"
     )
@@ -44,6 +44,7 @@ module MiGA::Common::WithResult
   def result(task)
     task = task.to_sym
     return nil if result_dirs[task].nil?
+
     MiGA::Result.load(
       "#{project.path}/data/#{result_dirs[task]}/#{result_base}.json"
     )

data/lib/miga/daemon.rb

@@ -8,7 +8,6 @@ require 'miga/daemon/base'
 ##
 # MiGA Daemons handling job submissions.
 class MiGA::Daemon < MiGA::MiGA
-
   include MiGA::Daemon::Base
   include MiGA::Common::WithDaemon
   extend MiGA::Common::WithDaemonClass
@@ -19,6 +18,7 @@ class MiGA::Daemon < MiGA::MiGA
     # full path to the project's 'daemon' folder
     def daemon_home(project)
       return project if project.is_a? String
+
       File.join(project.path, 'daemon')
     end
   end
@@ -45,9 +45,10 @@ class MiGA::Daemon < MiGA::MiGA
     @project = project
     @runopts = {}
     json ||= File.join(project.path, 'daemon/daemon.json')
+    default_json = File.expand_path('.miga_daemon.json', ENV['MIGA_HOME'])
     MiGA::Json.parse(
-      json, default: File.expand_path('.miga_daemon.json', ENV['MIGA_HOME'])
-    ).each { |k,v| runopts(k, v) }
+      json, default: File.exist?(default_json) ? default_json : nil
+    ).each { |k, v| runopts(k, v) }
     update_format_0
     @jobs_to_run = []
     @jobs_running = []
@@ -77,29 +78,53 @@ class MiGA::Daemon < MiGA::MiGA
   end
 
   ##
-  # Run one loop step. Returns a Boolean indicating if the loop should continue.
+  # Run one loop step. Returns a Boolean indicating if the loop should continue
   def daemon_loop
-    project.load
-    check_datasets
-    check_project
-    if shutdown_when_done? and jobs_running.size + jobs_to_run.size == 0
-      say 'Nothing else to do, shutting down.'
+    l_say(3, 'Daemon loop start')
+    reload_project
+    check_datasets or check_project
+    if shutdown_when_done? && (jobs_running.size + jobs_to_run.size).zero?
+      say 'Nothing else to do, shutting down'
       return false
     end
     flush!
-    if loop_i >= 12
-      say 'Probing running jobs'
-      @loop_i = 0
+    if (loop_i % 12).zero?
       purge!
+      recalculate_status!
     end
-    report_status
+    save_status
     sleep(latency)
+    l_say(3, 'Daemon loop end')
     true
   end
 
+  def recalculate_status!
+    project.each_dataset(&:recalculate_status)
+  end
+
+  ##
+  # Send +msg+ to +say+ as long as +level+ is at most +verbosity+
+  def l_say(level, *msg)
+    say(*msg) if verbosity >= level
+  end
+
+  ##
+  # Same as +l_say+ with +level = 1+
+  def say(*msg)
+    super(*msg) if verbosity >= 1
+  end
+
+  ##
+  # Reload the project's metadata
+  def reload_project
+    l_say(2, 'Reloading project')
+    project.load
+  end
+
   ##
   # Report status in a JSON file.
-  def report_status
+  def save_status
+    l_say(2, 'Saving current status')
     MiGA::Json.generate(
       { jobs_running: @jobs_running, jobs_to_run: @jobs_to_run },
       File.join(daemon_home, 'status.json')
@@ -111,12 +136,13 @@ class MiGA::Daemon < MiGA::MiGA
   def load_status
     f_path = File.join(daemon_home, 'status.json')
     return unless File.size? f_path
+
     say 'Loading previous status in daemon/status.json:'
     status = MiGA::Json.parse(f_path)
-    status.keys.each do |i|
+    status.each_key do |i|
       status[i].map! do |j|
         j.tap do |k|
-          unless k[:ds].nil? or k[:ds_name] == 'miga-project'
+          unless k[:ds].nil? || k[:ds_name] == 'miga-project'
             k[:ds] = project.dataset(k[:ds_name])
           end
           k[:job] = k[:job].to_sym unless k[:job].nil?
@@ -132,20 +158,34 @@ class MiGA::Daemon < MiGA::MiGA
   end
 
   ##
-  # Traverse datasets
+  # Traverse datasets, and returns boolean indicating if at any datasets
+  # are incomplete
   def check_datasets
-    project.each_dataset do |n, ds|
-      to_run = ds.next_preprocessing(false)
-      queue_job(:d, ds) unless to_run.nil?
+    l_say(2, 'Checking datasets')
+    o = false
+    project.each_dataset do |ds|
+      next unless ds.status == :incomplete
+      next if ds.next_preprocessing(false).nil?
+
+      o = true
+      queue_job(:d, ds)
     end
+    o
   end
 
   ##
   # Check if all reference datasets are pre-processed. If yes, check the
   # project-level tasks
   def check_project
+    l_say(2, 'Checking project')
+
+    # Ignore task if the project has no datasets
     return if project.dataset_names.empty?
+
+    # Double-check if all datasets are ready
     return unless project.done_preprocessing?(false)
+
+    # Queue project-level job
     to_run = project.next_task(nil, false)
     queue_job(:p) unless to_run.nil?
   end
@@ -156,13 +196,14 @@ class MiGA::Daemon < MiGA::MiGA
   # scheduler (or to bash or ssh) see #flush!
   def queue_job(job, ds = nil)
     return nil unless get_job(job, ds).nil?
+
     ds_name = (ds.nil? ? 'miga-project' : ds.name)
     say 'Queueing %s:%s' % [ds_name, job]
     vars = {
       'PROJECT' => project.path,
       'RUNTYPE' => runopts(:type),
-      'CORES'   => ppn,
-      'MIGA'    => MiGA::MiGA.root_path
+      'CORES' => ppn,
+      'MIGA' => MiGA::MiGA.root_path
     }
     vars['DATASET'] = ds.name unless ds.nil?
     log_dir = File.expand_path("daemon/#{job}", project.path)
@@ -170,9 +211,10 @@ class MiGA::Daemon < MiGA::MiGA
     task_name = "#{project.metadata[:name][0..9]}:#{job}:#{ds_name}"
     to_run = { ds: ds, ds_name: ds_name, job: job, task_name: task_name }
     to_run[:cmd] = runopts(:cmd).miga_variables(
-      script: MiGA::MiGA.script_path(job, miga:vars['MIGA'], project: project),
-      vars: vars.map { |k, v|
-        runopts(:var).miga_variables(key: k, value: v) }.join(runopts(:varsep)),
+      script: MiGA::MiGA.script_path(job, miga: vars['MIGA'], project: project),
+      vars: vars.map do |k, v|
+              runopts(:var).miga_variables(key: k, value: v)
+            end.join(runopts(:varsep)),
       cpus: ppn,
       log: File.expand_path("#{ds_name}.log", log_dir),
       task_name: task_name,
@@ -187,9 +229,9 @@ class MiGA::Daemon < MiGA::MiGA
   def get_job(job, ds = nil)
     (jobs_to_run + jobs_running).find do |j|
       if ds.nil?
-        j[:ds].nil? and j[:job] == job
+        j[:ds].nil? && j[:job] == job
       else
-        (! j[:ds].nil?) and j[:ds].name == ds.name and j[:job] == job
+        !j[:ds].nil? && j[:ds].name == ds.name && j[:job] == job
       end
     end
   end
@@ -199,23 +241,28 @@ class MiGA::Daemon < MiGA::MiGA
   # possible respecting #maxjobs or #nodelist (if set).
   def flush!
     # Check for finished jobs
+    l_say(2, 'Checking for finished jobs')
     @jobs_running.select! do |job|
-      ongoing = case job[:job].to_s
-      when 'd'
-        !job[:ds].nil? && !job[:ds].next_preprocessing(false).nil?
-      when 'p'
-        !project.next_task(nil, false).nil?
-      else
-        (job[:ds].nil? ? project : job[:ds]).add_result(job[:job], false).nil?
-      end
+      ongoing =
+        case job[:job].to_s
+        when 'd'
+          !job[:ds].nil? && !job[:ds].next_preprocessing(false).nil?
+        when 'p'
+          !project.next_task(nil, false).nil?
+        else
+          (job[:ds].nil? ? project : job[:ds]).add_result(job[:job], false).nil?
+        end
       say "Completed pid:#{job[:pid]} for #{job[:task_name]}" unless ongoing
       ongoing
     end
+
     # Avoid single datasets hogging resources
     @jobs_to_run.rotate! rand(jobs_to_run.size)
+
     # Launch as many +jobs_to_run+ as possible
-    while hostk = next_host
+    while (hostk = next_host)
       break if jobs_to_run.empty?
+
       launch_job(@jobs_to_run.shift, hostk)
     end
   end
@@ -225,7 +272,8 @@ class MiGA::Daemon < MiGA::MiGA
   # In any other daemons, returns true as long as #maxjobs is not reached
   def next_host
     return jobs_running.size < maxjobs if runopts(:type) != 'ssh'
-    allk = (0 .. nodelist.size-1).to_a
+
+    allk = (0..nodelist.size - 1).to_a
     busyk = jobs_running.map { |k| k[:hostk] }
     (allk - busyk).first
   end
@@ -233,6 +281,7 @@ class MiGA::Daemon < MiGA::MiGA
   ##
   # Remove dead jobs.
   def purge!
+    say 'Probing running jobs'
     @jobs_running.select! do |job|
       `#{runopts(:alive).miga_variables(pid: job[:pid])}`.chomp.to_i == 1
     end
@@ -265,9 +314,8 @@ class MiGA::Daemon < MiGA::MiGA
       say "Unsuccessful #{job[:task_name]}, rescheduling"
     else
       @jobs_running << job
-      say "Spawned pid:#{job[:pid]}#{
-            " to #{job[:hostk]}:#{nodelist[job[:hostk]]}" if job[:hostk]
-          } for #{job[:task_name]}"
+      job_host = " to #{job[:hostk]}:#{nodelist[job[:hostk]]}" if job[:hostk]
+      say "Spawned pid:#{job[:pid]}#{job_host} for #{job[:task_name]}"
     end
   end
 
@@ -279,10 +327,11 @@ class MiGA::Daemon < MiGA::MiGA
       var: %w[key value],
       alive: %w[pid],
       kill: %w[pid]
-    }.each do |k,v|
-      runopts(
-        k, runopts(k).gsub(/%(\d+\$)?d/, '%\\1s') % v.map{ |i| "{{#{i}}}" }
-      ) if !runopts(k).nil? && runopts(k) =~ /%(\d+\$)?[ds]/
+    }.each do |k, v|
+      if !runopts(k).nil? && runopts(k) =~ /%(\d+\$)?[ds]/
+        runopts(k,
+                runopts(k).gsub(/%(\d+\$)?d/, '%\\1s') % v.map { |i| "{{#{i}}}" })
+      end
     end
     runopts(:format_version, 1)
   end