miga-base 0.7.22.0 → 0.7.25.0

data/lib/miga/cli/base.rb

@@ -34,6 +34,7 @@ module MiGA::Cli::Base
     next_step: 'Return the next task to run in a dataset or project',
     # Objects (Datasets or Projects)
     edit: 'Edit the metadata of a dataset or project',
+    option: 'Get or set options of a dataset or project',
     # System
     init: 'Initialize MiGA to process new projects',
     daemon: 'Control the daemon of a MiGA project',
@@ -87,19 +88,30 @@ module MiGA::Cli::Base
   @@EXECS = @@TASK_DESC.keys
 
   @@FILE_REGEXP =
-    %r{^(?:.*/)?(.+?)(\.[A-Z]*([12]|Reads|Contigs))?(\.f[nastq]+)?$}i
+    %r{^(?:.*/)?(.+?)(\.[A-Z]*(Reads|Contigs))?(\.f[nastq]+)?(\.gz)?$}i
+
+  @@PAIRED_FILE_REGEXP =
+    %r{^(?:.*/)?(.+?)(\.[A-Z]*([12]|Reads))?(\.f[nastq]+)?(\.gz)?$}i
 end
 
 class MiGA::Cli < MiGA::MiGA
   include MiGA::Cli::Base
 
   class << self
-    def TASK_DESC; @@TASK_DESC end
+    def TASK_DESC
+      @@TASK_DESC
+    end
 
-    def TASK_ALIAS; @@TASK_ALIAS end
+    def TASK_ALIAS
+      @@TASK_ALIAS
+    end
 
-    def EXECS; @@EXECS end
+    def EXECS
+      @@EXECS
+    end
 
-    def FILE_REGEXP; @@FILE_REGEXP end
+    def FILE_REGEXP(paired = false)
+      paired ? @@PAIRED_FILE_REGEXP : @@FILE_REGEXP
+    end
   end
 end

data/lib/miga/cli/objects_helper.rb

@@ -57,12 +57,12 @@ module MiGA::Cli::ObjectsHelper
     ds.select! do |d|
       advance('Datasets:', k += 1, n, false)
       o = true
-      o &&= (d.is_ref? == self[:ref]) unless self[:ref].nil?
-      o &&= (d.is_active? == self[:active]) unless self[:active].nil?
-      o &&= (self[:multi] ? d.is_multi? :
-            d.is_nonmulti?) unless self[:multi].nil?
-      o &&= (not d.metadata[:tax].nil?) &&
-            d.metadata[:tax].in?(self[:taxonomy]) unless self[:taxonomy].nil?
+      o &&= (d.ref? == self[:ref]) unless self[:ref].nil?
+      o &&= (d.active? == self[:active]) unless self[:active].nil?
+      o &&= (self[:multi] ? d.multi? : d.nonmulti?) unless self[:multi].nil?
+      unless self[:taxonomy].nil?
+        o &&= !d.metadata[:tax].nil? && d.metadata[:tax].in?(self[:taxonomy])
+      end
       o
     end
     say ''
@@ -90,22 +90,27 @@ module MiGA::Cli::ObjectsHelper
   def add_metadata(obj, cli = self)
     raise "Unsupported object: #{obj.class}" unless obj.respond_to? :metadata
 
-    cli[:metadata].split(',').each do |pair|
+    (cli[:metadata] || '').split(',').each do |pair|
       (k, v) = pair.split('=')
-      case v
-      when 'true';  v = true
-      when 'false'; v = false
-      when 'nil';   v = nil
-      end
-      if k == '_step'
-        obj.metadata["_try_#{v}"] ||= 0
-        obj.metadata["_try_#{v}"]  += 1
+      if obj.option?(k)
+        obj.set_option(k, v, true)
+      else
+        case v
+        when 'true';  v = true
+        when 'false'; v = false
+        when 'nil';   v = nil
+        end
+        if k == '_step'
+          obj.metadata["_try_#{v}"] ||= 0
+          obj.metadata["_try_#{v}"]  += 1
+        end
+        obj.metadata[k] = v
       end
-      obj.metadata[k] = v
-    end unless cli[:metadata].nil?
-    [:type, :name, :user, :description, :comments].each do |k|
+    end
+    %i[type name user description comments].each do |k|
       obj.metadata[k] = cli[k] unless cli[k].nil?
     end
+    obj.save
     obj
   end
 end

data/lib/miga/common.rb

@@ -6,6 +6,7 @@ require 'miga/json'
 require 'miga/common/base'
 require 'miga/common/path'
 require 'miga/common/format'
+require 'miga/common/net'
 require 'stringio'
 
 ##
@@ -16,13 +17,14 @@ class MiGA::MiGA
 
   extend MiGA::Common::Path
   extend MiGA::Common::Format
+  extend MiGA::Common::Net
 
   ENV['MIGA_HOME'] ||= ENV['HOME']
 
   ##
   # Has MiGA been initialized?
   def self.initialized?
-    File.exist?(File.expand_path('.miga_rc', ENV['MIGA_HOME'])) and
+    File.exist?(File.expand_path('.miga_rc', ENV['MIGA_HOME'])) &&
       File.exist?(File.expand_path('.miga_daemon.json', ENV['MIGA_HOME']))
   end
 
@@ -64,7 +66,7 @@ class MiGA::MiGA
     adv_n = n - @_advance_time[:n]
     unless total.nil? || @_advance_time[:last].nil? || adv_n <= 0
       if adv_n.to_f/n > 0.001
-        this_time = Time.now - @_advance_time[:last]
+        this_time = (Time.now - @_advance_time[:last]).to_f
         this_avg = this_time / adv_n
         @_advance_time[:avg] ||= this_avg
         @_advance_time[:avg] = 0.9 * @_advance_time[:avg] + 0.1 * this_avg

data/lib/miga/common/net.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+require 'net/ftp'
+require 'open-uri'
+require 'fileutils'
+
+Net::FTP.send(:remove_const, 'FTP_PORT') # just to avoid warnings
+Net::FTP.const_set('FTP_PORT', 21)
+
+##
+# General web-access functions shared throughout MiGA.
+module MiGA::Common::Net
+  ##
+  # Returns the URL of the host +name+ (Symbol)
+  def known_hosts(name)
+    case name.to_sym
+    when :miga_online_ftp
+      'ftp://microbial-genomes.org//' # <- // to simplify chdir in connection
+    when :miga_db
+      'ftp://microbial-genomes.org/db'
+    when :miga_dist
+      'ftp://microbial-genomes.org/dist'
+    else
+      raise "Unrecognized server name: #{host}"
+    end
+  end
+
+  ##
+  # Connect to an FTP +host+ (String) or a known host name (Symbol, see
+  # +.known_hosts+)
+  def remote_connection(host)
+    host = known_hosts(host) if host.is_a?(Symbol)
+    uri = URI.parse(host)
+    raise 'Only FTP hosts are currently supported' unless uri.scheme == 'ftp'
+
+    ftp = Net::FTP.new(uri.host)
+    ftp.passive = true
+    ftp.login
+    ftp.chdir(uri.path)
+    ftp
+  end
+
+  ##
+  # Download a file via FTP using the +connection+ (returned by
+  # +.remote_connection+) with remote name +file+ into local +target+.
+  #
+  # Alternatively, +connection+ can simply be the host (String) or a recognized
+  # Symbol (see +.remote_connection+), in which case the function opens the
+  # connection automatically
+  #
+  # Reports progress to the function block with two arguments: the
+  # currently transferred size and the total file size
+  def download_file_ftp(connection, file, target)
+    # Open connection unless passed
+    close_conn = false
+    if connection.is_a?(String) || connection.is_a?(Symbol)
+      connection = remote_connection(connection)
+      close_conn = true
+    end
+
+    # Prepare download
+    FileUtils.mkdir_p(File.dirname(target))
+    filesize = connection.size(file)
+    transferred = 0
+
+    # Get in chunks of 1KiB
+    connection.getbinaryfile(file, target, 1024) do |data|
+      yield(transferred += data.size, filesize) if block_given?
+    end
+
+    # Close connection if automatically opened
+    connection.close if close_conn
+  end
+end

data/lib/miga/common/with_option.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+##
+# Helper module including specific functions to handle objects that
+# have configurable options. The class including this module must implement
+# the methods +.OPTIONS+, +#metadata+, and +#save+.
+module MiGA::Common::WithOption
+  def option(key)
+    assert_has_option(key)
+    opt = option_by_metadata(key)
+    value = opt.nil? ? option_by_default(key) : opt
+    value = value[self] if value.is_a?(Proc)
+    value
+  end
+
+  def set_option(key, value, from_string = false)
+    metadata[key] = assert_valid_option_value(key, value, from_string)
+    save
+    option(key)
+  end
+
+  def all_options
+    Hash[self.class.OPTIONS.each_key.map { |key| [key, option(key)] }]
+  end
+
+  def option?(key)
+    !self.class.OPTIONS[key.to_sym].nil?
+  end
+
+  def option_by_metadata(key)
+    metadata[key]
+  end
+
+  def option_by_default(key)
+    self.class.OPTIONS[key.to_sym][:default]
+  end
+
+  def assert_has_option(key)
+    opt = self.class.OPTIONS[key.to_sym]
+    raise "Unrecognized option: #{key}" if opt.nil?
+    opt
+  end
+
+  def assert_valid_option_value(key, value, from_string = false)
+    opt = assert_has_option(key)
+    value = option_from_string(key, value) if from_string
+
+    # nil is always valid, and so are supported tokens
+    return value if value.nil? || opt[:tokens]&.include?(value)
+
+    if opt[:type] && !value.is_a?(opt[:type])
+      raise "Invalid value type for #{key}: #{value.class}, not #{opt[:type]}"
+    end
+
+    if opt[:in] && !opt[:in].include?(value)
+      raise "Value out of range for #{key}: #{value}, not in #{opt[:in]}"
+    end
+
+    value
+  end
+
+  def option_from_string(key, value)
+    opt = assert_has_option(key)
+
+    if ['', 'nil'].include?(value)
+      nil
+    elsif opt[:tokens]&.include?(value)
+      value
+    elsif opt[:type]&.equal?(Float)
+      raise "Not a float: #{value}" unless value =~ /^-?\.?\d/
+      value.to_f
+    elsif opt[:type]&.equal?(Integer)
+      raise "Not an integer: #{value}" unless value =~ /^-?\d/
+      value.to_i
+    elsif opt[:in]&.include?(true) && value == 'true'
+      true
+    elsif opt[:in]&.include?(false) && value == 'false'
+      false
+    else
+      value
+    end
+  end
+end

data/lib/miga/common/with_result.rb

@@ -86,7 +86,8 @@ module MiGA::Common::WithResult
         if res.nil?
           # Run if the step has not been calculated,
           # unless too many attempts were already made
-          if (metadata["_try_#{t}"] || 0) > (project.metadata[:max_try] || 10)
+          cur_try = metadata["_try_#{t}"] || 0
+          if cur_try > project.option(:max_try)
             inactivate! "Too many errors in step #{t}"
             false
           else
@@ -103,7 +104,7 @@ module MiGA::Common::WithResult
   ##
   # Mark all results for recalculation
   def recalculate_tasks(reason = nil)
-    each_result { |res| res.recalculate!(reason).save }
+    each_result { |_k, res| res.recalculate!(reason).save }
   end
 
 end

data/lib/miga/dataset/base.rb

@@ -1,7 +1,10 @@
-# @package MiGA
-# @license Artistic-2.0
+# frozen_string_literal: true
+
+require 'miga/common/with_option'
 
 class MiGA::Dataset < MiGA::MiGA
+  include MiGA::Common::WithOption
+
   # Class-level
   class << self
     def RESULT_DIRS
@@ -15,6 +18,10 @@ class MiGA::Dataset < MiGA::MiGA
     def PREPROCESSING_TASKS
       @@PREPROCESSING_TASKS
     end
+
+    def OPTIONS
+      @@OPTIONS
+    end
   end
 end
 
@@ -85,4 +92,15 @@ module MiGA::Dataset::Base
   # tasks are ignored for single-organism datasets or for unknwon types.
   @@ONLY_MULTI_TASKS = [:mytaxa]
   @@_ONLY_MULTI_TASKS_H = Hash[@@ONLY_MULTI_TASKS.map { |i| [i, true] }]
+
+  ##
+  # Options supported by datasets
+  @@OPTIONS = {
+    db_project: {
+      desc: 'Project to use as database', type: String
+    },
+    dist_req: {
+      desc: 'Run distances against these datasets', type: Array, default: []
+    }
+  }
 end

data/lib/miga/dataset/result.rb

@@ -50,7 +50,7 @@ module MiGA::Dataset::Result
       :upstream
     elsif !metadata["run_#{task}"].nil?
       metadata["run_#{task}"] ? :execute : :force
-    elsif task == :taxonomy && project.metadata[:ref_project].nil?
+    elsif task == :taxonomy && project.option(:ref_project).nil?
       :project
     elsif @@_EXCLUDE_NOREF_TASKS_H[task] && !ref?
       :noref
@@ -276,7 +276,8 @@ module MiGA::Dataset::Result
       ess_genes: '.ess.faa',
       collection: '.ess',
       report: '.ess/log',
-      alignments: '.ess/proteins.aln'
+      alignments: '.ess/proteins.aln',
+      fastaai_index: '.faix.db.gz'
     )
   end
 
@@ -290,7 +291,8 @@ module MiGA::Dataset::Result
       MiGA::Result.new("#{base}.json"), name,
       longest_ssu_gene: '.ssu.fa',
       gff: '.ssu.gff',
-      all_ssu_genes: '.ssu.all.fa'
+      all_ssu_genes: '.ssu.all.fa',
+      classification: '.rdp.tsv'
     )
     opts[:is_clean] ||= false
     r.clean! if opts[:is_clean]

data/lib/miga/metadata.rb

@@ -56,24 +56,20 @@ class MiGA::Metadata < MiGA::MiGA
   ##
   # Save the metadata into #path
   def save
-    MiGA.DEBUG "Metadata.save #{path}"
+    return if self[:never_save]
+
+    MiGA::MiGA.DEBUG "Metadata.save #{path}"
     self[:updated] = Time.now.to_s
     json = to_json
-    sleeper = 0.0
-    slept = 0
-    while File.exist?(lock_file)
-      MiGA::MiGA.DEBUG "Waiting for lock: #{lock_file}"
-      sleeper += 0.1 if sleeper <= 10.0
-      sleep(sleeper.to_i)
-      slept += sleeper.to_i
-      raise "Lock detected for over 10 minutes: #{lock_file}" if slept > 600
-    end
-    FileUtils.touch lock_file
+    wait_for_lock
+    FileUtils.touch(lock_file)
     ofh = File.open("#{path}.tmp", 'w')
     ofh.puts json
     ofh.close
-    raise "Lock-racing detected for #{path}" unless
-      File.exist?("#{path}.tmp") and File.exist?(lock_file)
+
+    unless File.exist?("#{path}.tmp") && File.exist?(lock_file)
+      raise "Lock-racing detected for #{path}"
+    end
 
     File.rename("#{path}.tmp", path)
     File.unlink(lock_file)
@@ -154,4 +150,20 @@ class MiGA::Metadata < MiGA::MiGA
   def to_json
     MiGA::Json.generate(data)
   end
+
+  private
+
+  ##
+  # Wait for the lock to go away
+  def wait_for_lock
+    sleeper = 0.0
+    slept = 0.0
+    while File.exist?(lock_file)
+      MiGA::MiGA.DEBUG "Waiting for lock: #{lock_file}"
+      sleeper += 0.1 if sleeper <= 10.0
+      sleep(sleeper)
+      slept += sleeper
+      raise "Lock detected for over 10 minutes: #{lock_file}" if slept > 600
+    end
+  end
 end

data/lib/miga/project/base.rb

@@ -1,7 +1,10 @@
-# @package MiGA
-# @license Artistic-2.0
+# frozen_string_literal: true
+
+require 'miga/common/with_option'
 
 class MiGA::Project < MiGA::MiGA
+  include MiGA::Common::WithOption
+
   class << self
     ##
     # Does the project at +path+ exist?
@@ -33,6 +36,10 @@ class MiGA::Project < MiGA::MiGA
     def RESULT_DIRS
       @@RESULT_DIRS
     end
+
+    def OPTIONS
+      @@OPTIONS
+    end
   end
 end
 
@@ -108,4 +115,77 @@ module MiGA::Project::Base
   ##
   # Project-wide tasks for :clade projects
   @@INCLADE_TASKS = [:subclades, :ogs]
+
+  ##
+  # Options supported by projects
+  @@OPTIONS = {
+    ref_project: {
+      desc: 'Project with reference taxonomy', type: String
+    },
+    db_proj_dir: {
+      desc: 'Directory containing database projects', type: String
+    },
+    tax_pvalue: {
+      desc: 'Maximum p-value to transfer taxonomy', default: 0.05, type: Float,
+      in: 0.0..1.0
+    },
+    haai_p: {
+      desc: 'Value of aai.rb -p on hAAI', type: String,
+      default: proc { |project| project.clade? ? 'no' : 'blast+' },
+      in: %w[fastaai blast+ blast blat diamond no]
+    },
+    aai_p: {
+      desc: 'Value of aai.rb -p on AAI', default: 'blast+', type: String,
+      in: %w[blast+ blast blat diamond]
+    },
+    ani_p: {
+      desc: 'Value of ani.rb -p on ANI', default: 'blast+', type: String,
+      in: %w[blast+ blast blat fastani]
+    },
+    max_try: {
+      desc: 'Maximum number of task attempts', default: 10, type: Integer,
+      in: (0..1000)
+    },
+    aai_save_rbm: {
+      desc: 'Should RBMs be saved for OGS analysis?',
+      default: proc { |project| project.clade? },
+      in: [true, false]
+    },
+    ogs_identity: {
+      desc: 'Min RBM identity for OGS', default: 80.0, type: Float,
+      in: (0.0..100.0)
+    },
+    clean_ogs: {
+      desc: 'If false, keeps ABC files (clades only)', default: true,
+      in: [true, false]
+    },
+    run_clades: {
+      desc: 'Should clades be estimated from distances?', default: true,
+      in: [true, false]
+    },
+    gsp_ani: {
+      desc: 'ANI limit to propose gsp clades', default: 95.0, type: Float,
+      in: (0.0..100.0)
+    },
+    gsp_aai: {
+      desc: 'AAI limit to propose gsp clades', default: 90.0, type: Float,
+      in: (0.0..100.0)
+    },
+    gsp_metric: {
+      desc: 'Metric to propose clades', default: 'ani', type: String,
+      in: %w[ani aai]
+    },
+    ess_coll: {
+      desc: 'Collection of essential genes to use', default: 'dupont_2012',
+      type: String, in: %w[dupont_2012 lee_2019]
+    },
+    min_qual: {
+      desc: 'Minimum genome quality', default: 25.0, type: Float,
+      in: -Float::INFINITY..100.0, tokens: %w[no]
+    },
+    distances_checkpoint: {
+      desc: 'Number of comparisons before storing data', default: 10,
+      type: Integer, in: 1...Float::INFINITY
+    }
+  }
 end