miga-base 0.5.10.0 → 0.6.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a3fc33951911397d8c6305f0bf893cbc373c7ca09f55531c95adf15439e9a938
-  data.tar.gz: e1fd8f3d96c50770d2dc029fd9285ff79e536df888097d89705087d3a2c9a5d0
+  metadata.gz: 13b2ec821a69376491a0a3a31da7ccf717d8aafd1dc7753acaec36f3b7aa42c4
+  data.tar.gz: f6d513971e2c06c3f299bc5bf680682c533b380ff4340443f789446a6835b54f
 SHA512:
-  metadata.gz: be1241558fc879d2c6e5e6c2cb36432c004b7b4264116d3a2ff9ddc1cf5354efdfe001362a23145a6cc341ee9133d00e181184624a6adf4e40199e0d22b6ffb9
-  data.tar.gz: f2f14e380a596510f7d2803cac7c41069bbe0a9e2b5d1fd565d57daebbaebfdcd9bf71d1065fb92dfca63e278ff9647cc032abeaea3302ef79c790f481b90bfd
+  metadata.gz: 5a4de9af15f63372d98b64cf1604e64fc576c3809f435dc543acb98cab9779155998e5f1b382af856e5d4760faba0d9f606f614654afdad10659b764c9c6a17f
+  data.tar.gz: bd01962ae2c1f75ec3768d2515bab711191c4a2b256996c51a69bcbdce5f87b06215816a6368fad0e399d69c09ec7992f97b9868e8670f3866bc1fdcb29d6c41

data/Rakefile

@@ -1,18 +1,28 @@
-require "rake/testtask"
+require 'rake/testtask'
 
-SOURCES = FileList["lib/**/*.rb"]
+SOURCES = FileList['lib/**/*.rb']
 
-desc "Default Task"
-task :default => "test:all"
+desc 'Default Task'
+task :default => 'test:all'
 
-desc "Default tests"
-task :test => "test:all"
+desc 'Default tests'
+task :test => 'test:all'
 
 namespace :test do
-  desc "All tests"
+  desc 'All tests'
   Rake::TestTask.new(:all) do |t|
-    t.libs << "test"
-    t.pattern = "test/*_test.rb"
+    t.libs << 'test'
+    t.pattern = 'test/*_test.rb'
     t.verbose = true
   end
+
+  FileList['test/*_test.rb'].each do |i|
+    b = File.basename(i, '_test.rb')
+    desc "Test #{b}"
+    Rake::TestTask.new(:"#{b}") do |t|
+      t.libs << 'test'
+      t.pattern = i
+      t.verbose = true
+    end
+  end
 end

data/lib/miga/cli/action/run.rb

@@ -31,33 +31,42 @@ class MiGA::Cli::Action::Run < MiGA::Cli::Action
   end
 
   def perform
+    # Load environment variables if requested (typically by the daemon)
     if cli[:env]
       cli[:project] ||= ENV['PROJECT']
       cli[:dataset] ||= ENV['DATASET']
       cli[:thr] ||= ENV['CORES'].to_i unless ENV['CORES'].nil?
       cli[:result] = File.basename(cli[:result].to_s, '.bash').to_sym
     end
-    virtual_task = false
-    miga = MiGA.root_path
+    
+    # Unset dataset if the requested result is for projects
+    if (MiGA::Project.RESULT_DIRS.keys + [:p]).include? cli[:result]
+      cli[:dataset] = nil
+    end
+    
+    # Load project
     p = cli.load_project
     
+    # Prepare command
+    miga = MiGA.root_path
     cmd = ["PROJECT=#{p.path.shellescape}",
       "RUNTYPE=#{cli[:remote] ? 'ssh' : 'bash'}",
       "MIGA=#{miga.shellescape}", "CORES=#{cli[:thr]}"]
     obj = cli.load_project_or_dataset
     klass = obj.class
-    virtual_task = true if [:p, :d].include? cli[:result]
+    virtual_task = [:p, :d].include?(cli[:result])
     cmd << "DATASET=#{obj.name.shellescape}" if obj.is_a? MiGA::Dataset
     if klass.RESULT_DIRS[cli[:result]].nil? and not virtual_task
       raise "Unsupported #{klass.to_s.sub(/.*::/, '')} result: #{cli[:result]}."
     end
     cmd << MiGA.script_path(cli[:result], miga: miga, project: p).shellescape
     if cli[:remote]
-      #cmd.unshift '.', '/etc/profile', ';'
       cmd = ['ssh', '-t', '-t', cli[:remote].shellescape,
         cmd.join(' ').shellescape]
     end
     cmd << ['>', cli[:log].shellescape, '2>&1'] if cli[:log]
+    
+    # Launch
     pid = spawn cmd.join(' ')
     Process.wait pid
   end

data/lib/miga/cli/action/stats.rb

@@ -30,12 +30,7 @@ class MiGA::Cli::Action::Stats < MiGA::Cli::Action
     r = cli.load_result
     if cli[:compute]
       cli.say 'Computing statistics'
-      method = :"compute_#{cli[:result]}"
-      stats = self.respond_to?(method, true) ? send(method, r) : nil
-      unless stats.nil?
-        r[:stats] = stats
-        r.save
-      end
+      r.compute_stats
     end
     if cli[:key].nil?
       r[:stats].each do |k, v|
@@ -47,143 +42,4 @@ class MiGA::Cli::Action::Stats < MiGA::Cli::Action
       puts v.is_a?(Array) ? v.first : v
     end
   end
-
-  private
-
-  def compute_raw_reads(r)
-    stats = {}
-    if r[:files][:pair1].nil?
-      s = MiGA.seqs_length(r.file_path(:single), :fastq, gc: true)
-      stats = {
-        reads: s[:n],
-        length_average: [s[:avg], 'bp'],
-        length_standard_deviation: [s[:sd], 'bp'],
-        g_c_content: [s[:gc], '%']}
-    else
-      s1 = MiGA.seqs_length(r.file_path(:pair1), :fastq, gc: true)
-      s2 = MiGA.seqs_length(r.file_path(:pair2), :fastq, gc: true)
-      stats = {
-        read_pairs: s1[:n],
-        forward_length_average: [s1[:avg], 'bp'],
-        forward_length_standard_deviation: [s1[:sd], 'bp'],
-        forward_g_c_content: [s1[:gc], '%'],
-        reverse_length_average: [s2[:avg], 'bp'],
-        reverse_length_standard_deviation: [s2[:sd], 'bp'],
-        reverse_g_c_content: [s2[:gc], '%']}
-    end
-    stats
-  end
-
-  def compute_trimmed_fasta(r)
-    f = r[:files][:coupled].nil? ? r.file_path(:single) : r.file_path(:coupled)
-    s = MiGA.seqs_length(f, :fasta, gc: true)
-    {
-      reads: s[:n],
-      length_average: [s[:avg], 'bp'],
-      length_standard_deviation: [s[:sd], 'bp'],
-      g_c_content: [s[:gc], '%']
-    }
-  end
-
-  def compute_assembly(r)
-    s = MiGA.seqs_length(r.file_path(:largecontigs), :fasta,
-      n50: true, gc: true)
-    {
-      contigs: s[:n],
-      n50: [s[:n50], 'bp'],
-      total_length: [s[:tot], 'bp'],
-      g_c_content: [s[:gc], '%']
-    }
-  end
-
-  def compute_cds(r)
-    s = MiGA.seqs_length(r.file_path(:proteins), :fasta)
-    stats = {
-      predicted_proteins: s[:n],
-      average_length: [s[:avg], 'aa']}
-    asm = cli.load_dataset.add_result(:assembly, false)
-    unless asm.nil? or asm[:stats][:total_length].nil?
-      stats[:coding_density] =
-        [300.0 * s[:tot] / asm[:stats][:total_length][0], '%']
-    end
-    stats
-  end
-
-  def compute_essential_genes(r)
-    stats = {}
-    d = cli.load_dataset
-    if d.is_multi?
-      stats = {median_copies: 0, mean_copies: 0}
-      File.open(r.file_path(:report), 'r') do |fh|
-        fh.each_line do |ln|
-          if /^! (Mean|Median) number of copies per model: (.*)\./.match(ln)
-            stats["#{$1.downcase}_copies".to_sym] = $2.to_f
-          end
-        end
-      end
-    else
-      # Fix estimate by domain
-      if !(tax = d.metadata[:tax]).nil? &&
-            %w[Archaea Bacteria].include?(tax[:d]) &&
-            r.file_path(:raw_report).nil?
-        scr = "#{MiGA.root_path}/utils/domain-ess-genes.rb"
-        rep = r.file_path(:report)
-        rc_p = File.expand_path('.miga_rc', ENV['HOME'])
-        rc = File.exist?(rc_p) ? ". '#{rc_p}' && " : ''
-        $stderr.print `#{rc} ruby '#{scr}' \
-          '#{rep}' '#{rep}.domain' '#{tax[:d][0]}'`
-        r.add_file(:raw_report, "#{d.name}.ess/log")
-        r.add_file(:report, "#{d.name}.ess/log.domain")
-      end
-      # Extract/compute quality values
-      stats = {completeness: [0.0, '%'], contamination: [0.0, '%']}
-      File.open(r.file_path(:report), 'r') do |fh|
-        fh.each_line do |ln|
-          if /^! (Completeness|Contamination): (.*)%/.match(ln)
-            stats[$1.downcase.to_sym][0] = $2.to_f
-          end
-        end
-      end
-      stats[:quality] = stats[:completeness][0] - stats[:contamination][0] * 5
-      d.metadata[:quality] = case stats[:quality]
-        when 80..100 ; :excellent
-        when 50..80  ; :high
-        when 20..50  ; :intermediate
-        else         ; :low
-      end
-      d.save
-    end
-    stats
-  end
-
-  def compute_ssu(r)
-    stats = {ssu: 0, complete_ssu: 0}
-    Zlib::GzipReader.open(r.file_path(:gff)) do |fh|
-      fh.each_line do |ln|
-        next if ln =~ /^#/
-        rl = ln.chomp.split("\t")
-        len = (rl[4].to_i - rl[3].to_i).abs + 1
-        stats[:max_length] = [stats[:max_length] || 0, len].max
-        stats[:ssu] += 1
-        stats[:complete_ssu] += 1 unless rl[8] =~ /\(partial\)/
-      end
-    end
-    stats
-  end
-
-  def compute_taxonomy(r)
-    stats = {}
-    File.open(r.file_path(:intax_test), 'r') do |fh|
-      fh.gets.chomp =~ /Closest relative: (\S+) with AAI: (\S+)\.?/
-      stats[:closest_relative] = $1
-      stats[:aai] = [$2.to_f, '%']
-      3.times { fh.gets }
-      fh.each_line do |ln|
-        row = ln.chomp.gsub(/^\s*/,'').split(/\s+/)
-        break if row.empty?
-        stats[:"#{row[0]}_pvalue"] = row[2].to_f unless row[0] == 'root'
-      end
-    end
-    stats
-  end
 end

data/lib/miga/cli/base.rb

@@ -14,7 +14,6 @@ module MiGA::Cli::Base
     # Projects
     new:      'Creates an empty MiGA project',
     about:    'Displays information about a MiGA project',
-    plugins:  'Lists or (un)installs plugins in a MiGA project',
     doctor:   'Performs consistency checks on a MiGA project',
     get_db:   'Downloads a pre-indexed database',
     # Datasets

data/lib/miga/common/hooks.rb

@@ -0,0 +1,49 @@
+
+##
+# Helper module including specific functions to handle dataset hooks.
+module MiGA::Common::Hooks
+
+  ##
+  # Call the hook with symbol +event+ and any parameters +event_args+
+  def pull_hook(event, *event_args)
+    event = event.to_sym
+    event_queue = (hooks[event] || [])
+    event_queue += (metadata[event] || []) if respond_to? :metadata
+    event_queue.each do |i|
+      action = i.first
+      hook_name = :"hook_#{action}"
+      hook_args = i[1 .. -1]
+      if respond_to? hook_name
+        MiGA::MiGA.DEBUG "Hook: #{self.class}(#{event} > #{action})"
+        self.send(hook_name, hook_args, event_args)
+      else
+        raise "Cannot find action #{action} elicited by #{self.class}(#{event})"
+      end
+    end
+  end
+
+  ##
+  # Whenever +event+ occurs, launch +action+ with parameters +args+.
+  def add_hook(event, action, *args)
+    (hooks[event] ||= []) << [action, *args]
+  end
+
+  ##
+  # Get the stack of hooks
+  def hooks
+    @_hooks ||= default_hooks
+  end
+
+  ##
+  # Default object's hooks
+  def default_hooks
+    {}
+  end
+
+  ##
+  # Run the function defined in the first hook argument
+  def hook_run_lambda(hook_args, event_args)
+    hook_args.first[*event_args]
+  end
+
+end

data/lib/miga/common/path.rb

@@ -11,17 +11,8 @@ module MiGA::Common::Path
   # Path to a script to be executed for +task+. Supported +opts+ are:
   # - +:miga+ Path to the MiGA home to use. If not passed, the home of the
   #   library is used).
-  # - +:project+ MiGA::Project object to check within plugins. If not passed,
-  #   only core scripts are supported.
   def script_path(task, opts={})
     opts[:miga] ||= root_path
-    unless opts[:project].nil?
-      opts[:project].plugins.each do |pl|
-        if File.exist? File.expand_path("scripts/#{task}.bash", pl)
-          opts[:miga] = pl
-        end
-      end
-    end
     File.expand_path("scripts/#{task}.bash", opts[:miga])
   end
 

data/lib/miga/daemon.rb

@@ -146,8 +146,7 @@ class MiGA::Daemon < MiGA::MiGA
   def check_project
     return if project.dataset_names.empty?
     return unless project.done_preprocessing?(false)
-    to_run = project.next_distances(true)
-    to_run = project.next_inclade(true) if to_run.nil?
+    to_run = project.next_task(nil, false)
     queue_job(:p) unless to_run.nil?
   end
 

data/lib/miga/dataset.rb

@@ -3,6 +3,7 @@
 
 require 'miga/metadata'
 require 'miga/dataset/result'
+require 'miga/dataset/hooks'
 require 'sqlite3'
 
 ##
@@ -10,6 +11,7 @@ require 'sqlite3'
 class MiGA::Dataset < MiGA::MiGA
   
   include MiGA::Dataset::Result
+  include MiGA::Dataset::Hooks
   
   # Class-level
   class << self
@@ -56,6 +58,7 @@ class MiGA::Dataset < MiGA::MiGA
       metadata
     ]
     save unless File.exist? @metadata_future[0]
+    pull_hook :on_load
   end
 
   ##
@@ -69,6 +72,7 @@ class MiGA::Dataset < MiGA::MiGA
   def save
     MiGA.DEBUG "Dataset.metadata: #{metadata.data}"
     metadata.save
+    pull_hook :on_save
   end
   
   ##
@@ -81,6 +85,7 @@ class MiGA::Dataset < MiGA::MiGA
   def remove!
     self.results.each{ |r| r.remove! }
     self.metadata.remove!
+    pull_hook :on_remove
   end
 
   ##
@@ -88,6 +93,7 @@ class MiGA::Dataset < MiGA::MiGA
   def inactivate!
     self.metadata[:inactive] = true
     self.metadata.save
+    pull_hook :on_inactivate
   end
 
   ##
@@ -95,6 +101,7 @@ class MiGA::Dataset < MiGA::MiGA
   def activate!
     self.metadata[:inactive] = nil
     self.metadata.save
+    pull_hook :on_activate
   end
   
   ##

data/lib/miga/dataset/hooks.rb

@@ -0,0 +1,72 @@
+
+require 'miga/common/hooks'
+
+##
+# Helper module including specific functions to handle dataset hooks.
+# Supported events:
+# - on_load(): When loaded
+# - on_save(): When saved
+# - on_remove(): When removed
+# - on_inactivate(): When inactivated
+# - on_activate(): When activated
+# - on_result_ready(result): When any result is ready, with key +result+
+# - on_result_ready_{result}(): When +result+ is ready
+# - on_preprocessing_ready(): When preprocessing is complete
+# Supported hooks:
+# - run_lambda(lambda, args...)
+# - clear_run_counts()
+# - run_cmd(cmd)
+# Internal hooks:
+# - _pull_preprocessing_ready_hooks()
+# - _pull_result_hooks()
+module MiGA::Dataset::Hooks 
+
+  include MiGA::Common::Hooks
+
+  def default_hooks
+    {
+      on_preprocessing_ready: [[:clear_run_counts]],
+      on_result_ready: [
+        [:_pull_result_hooks],
+        [:_pull_preprocessing_ready_hooks]
+      ]
+    }
+  end
+
+  ##
+  # Clear metadata from run counts
+  def hook_clear_run_counts(_hook_args, _event_args)
+    metadata.data.keys
+      .select { |k| k.to_s =~ /^_try_/ }
+      .each { |k| metadata[k] = nil }
+    save
+  end
+
+  ##
+  # Run +cmd+ in the command-line with {{variables}}: dataset, project, miga,
+  # object (as defined for the event, if any)
+  # - +hook_args+: +[cmd]+
+  # - +event_args+: +[object (optional)]+
+  def hook_run_cmd(hook_args, event_args)
+    Process.wait(
+      spawn hook_args.first.miga_variables(
+        dataset: name, project: project.path, miga: MiGA::MiGA.root_path,
+        object: event_args.first
+      )
+    )
+  end
+
+  ##
+  # Pull :dataset_ready hook if preprocessing is complete
+  def hook__pull_preprocessing_ready_hooks(_hook_args, _event_args)
+    pull_hook(:on_preprocessing_ready) if done_preprocessing?
+  end
+
+  ##
+  # Dataset Action :pull_result_hooks([], [res])
+  # Pull the hook specific to the type of result
+  def hook__pull_result_hooks(_hook_args, event_args)
+    pull_hook(:"on_result_ready_#{event_args.first}", *event_args)
+  end
+
+end