miga-base 0.7.23.0 → 0.7.25.3

data/lib/miga/dataset/base.rb

@@ -1,7 +1,10 @@
-# @package MiGA
-# @license Artistic-2.0
+# frozen_string_literal: true
+
+require 'miga/common/with_option'
 
 class MiGA::Dataset < MiGA::MiGA
+  include MiGA::Common::WithOption
+
   # Class-level
   class << self
     def RESULT_DIRS
@@ -15,6 +18,10 @@ class MiGA::Dataset < MiGA::MiGA
     def PREPROCESSING_TASKS
       @@PREPROCESSING_TASKS
     end
+
+    def OPTIONS
+      @@OPTIONS
+    end
   end
 end
 
@@ -85,4 +92,15 @@ module MiGA::Dataset::Base
   # tasks are ignored for single-organism datasets or for unknwon types.
   @@ONLY_MULTI_TASKS = [:mytaxa]
   @@_ONLY_MULTI_TASKS_H = Hash[@@ONLY_MULTI_TASKS.map { |i| [i, true] }]
+
+  ##
+  # Options supported by datasets
+  @@OPTIONS = {
+    db_project: {
+      desc: 'Project to use as database', type: String
+    },
+    dist_req: {
+      desc: 'Run distances against these datasets', type: Array, default: []
+    }
+  }
 end

data/lib/miga/dataset/result.rb

@@ -50,7 +50,7 @@ module MiGA::Dataset::Result
       :upstream
     elsif !metadata["run_#{task}"].nil?
       metadata["run_#{task}"] ? :execute : :force
-    elsif task == :taxonomy && project.metadata[:ref_project].nil?
+    elsif task == :taxonomy && project.option(:ref_project).nil?
       :project
     elsif @@_EXCLUDE_NOREF_TASKS_H[task] && !ref?
       :noref
@@ -276,7 +276,8 @@ module MiGA::Dataset::Result
       ess_genes: '.ess.faa',
       collection: '.ess',
       report: '.ess/log',
-      alignments: '.ess/proteins.aln'
+      alignments: '.ess/proteins.aln',
+      fastaai_index: '.faix.db.gz'
     )
   end
 

data/lib/miga/metadata.rb

@@ -56,24 +56,20 @@ class MiGA::Metadata < MiGA::MiGA
   ##
   # Save the metadata into #path
   def save
-    MiGA.DEBUG "Metadata.save #{path}"
+    return if self[:never_save]
+
+    MiGA::MiGA.DEBUG "Metadata.save #{path}"
     self[:updated] = Time.now.to_s
     json = to_json
-    sleeper = 0.0
-    slept = 0
-    while File.exist?(lock_file)
-      MiGA::MiGA.DEBUG "Waiting for lock: #{lock_file}"
-      sleeper += 0.1 if sleeper <= 10.0
-      sleep(sleeper.to_i)
-      slept += sleeper.to_i
-      raise "Lock detected for over 10 minutes: #{lock_file}" if slept > 600
-    end
-    FileUtils.touch lock_file
+    wait_for_lock
+    FileUtils.touch(lock_file)
     ofh = File.open("#{path}.tmp", 'w')
     ofh.puts json
     ofh.close
-    raise "Lock-racing detected for #{path}" unless
-      File.exist?("#{path}.tmp") and File.exist?(lock_file)
+
+    unless File.exist?("#{path}.tmp") && File.exist?(lock_file)
+      raise "Lock-racing detected for #{path}"
+    end
 
     File.rename("#{path}.tmp", path)
     File.unlink(lock_file)
@@ -154,4 +150,20 @@ class MiGA::Metadata < MiGA::MiGA
   def to_json
     MiGA::Json.generate(data)
   end
+
+  private
+
+  ##
+  # Wait for the lock to go away
+  def wait_for_lock
+    sleeper = 0.0
+    slept = 0.0
+    while File.exist?(lock_file)
+      MiGA::MiGA.DEBUG "Waiting for lock: #{lock_file}"
+      sleeper += 0.1 if sleeper <= 10.0
+      sleep(sleeper)
+      slept += sleeper
+      raise "Lock detected for over 10 minutes: #{lock_file}" if slept > 600
+    end
+  end
 end

data/lib/miga/project/base.rb

@@ -1,7 +1,10 @@
-# @package MiGA
-# @license Artistic-2.0
+# frozen_string_literal: true
+
+require 'miga/common/with_option'
 
 class MiGA::Project < MiGA::MiGA
+  include MiGA::Common::WithOption
+
   class << self
     ##
     # Does the project at +path+ exist?
@@ -33,6 +36,10 @@ class MiGA::Project < MiGA::MiGA
     def RESULT_DIRS
       @@RESULT_DIRS
     end
+
+    def OPTIONS
+      @@OPTIONS
+    end
   end
 end
 
@@ -108,4 +115,77 @@ module MiGA::Project::Base
   ##
   # Project-wide tasks for :clade projects
   @@INCLADE_TASKS = [:subclades, :ogs]
+
+  ##
+  # Options supported by projects
+  @@OPTIONS = {
+    ref_project: {
+      desc: 'Project with reference taxonomy', type: String
+    },
+    db_proj_dir: {
+      desc: 'Directory containing database projects', type: String
+    },
+    tax_pvalue: {
+      desc: 'Maximum p-value to transfer taxonomy', default: 0.05, type: Float,
+      in: 0.0..1.0
+    },
+    haai_p: {
+      desc: 'Value of aai.rb -p on hAAI', type: String,
+      default: proc { |project| project.clade? ? 'no' : 'blast+' },
+      in: %w[fastaai blast+ blast blat diamond no]
+    },
+    aai_p: {
+      desc: 'Value of aai.rb -p on AAI', default: 'blast+', type: String,
+      in: %w[blast+ blast blat diamond]
+    },
+    ani_p: {
+      desc: 'Value of ani.rb -p on ANI', default: 'blast+', type: String,
+      in: %w[blast+ blast blat fastani]
+    },
+    max_try: {
+      desc: 'Maximum number of task attempts', default: 10, type: Integer,
+      in: (0..1000)
+    },
+    aai_save_rbm: {
+      desc: 'Should RBMs be saved for OGS analysis?',
+      default: proc { |project| project.clade? },
+      in: [true, false]
+    },
+    ogs_identity: {
+      desc: 'Min RBM identity for OGS', default: 80.0, type: Float,
+      in: (0.0..100.0)
+    },
+    clean_ogs: {
+      desc: 'If false, keeps ABC files (clades only)', default: true,
+      in: [true, false]
+    },
+    run_clades: {
+      desc: 'Should clades be estimated from distances?', default: true,
+      in: [true, false]
+    },
+    gsp_ani: {
+      desc: 'ANI limit to propose gsp clades', default: 95.0, type: Float,
+      in: (0.0..100.0)
+    },
+    gsp_aai: {
+      desc: 'AAI limit to propose gsp clades', default: 90.0, type: Float,
+      in: (0.0..100.0)
+    },
+    gsp_metric: {
+      desc: 'Metric to propose clades', default: 'ani', type: String,
+      in: %w[ani aai]
+    },
+    ess_coll: {
+      desc: 'Collection of essential genes to use', default: 'dupont_2012',
+      type: String, in: %w[dupont_2012 lee_2019]
+    },
+    min_qual: {
+      desc: 'Minimum genome quality', default: 25.0, type: Float,
+      in: -Float::INFINITY..100.0, tokens: %w[no]
+    },
+    distances_checkpoint: {
+      desc: 'Number of comparisons before storing data', default: 10,
+      type: Integer, in: 1...Float::INFINITY
+    }
+  }
 end

data/lib/miga/project/result.rb

@@ -31,9 +31,9 @@ module MiGA::Project::Result
   ##
   # Is this +task+ to be bypassed?
   def ignore_task?(task)
-    metadata["run_#{task}"] == false ||
-      (!is_clade? && @@INCLADE_TASKS.include?(task) &&
-        metadata["run_#{task}"] != true)
+    return true if metadata["run_#{task}"] == false
+
+    !clade? && @@INCLADE_TASKS.include?(task) && metadata["run_#{task}"] != true
   end
 
   ##
@@ -74,7 +74,7 @@ module MiGA::Project::Result
       return r
     end
     return nil unless result_files_exist?(base, %w[.proposed-clades])
-    unless is_clade? ||
+    unless clade? ||
            result_files_exist?(
              base, %w[.pdf .classif .medoids .class.tsv .class.nwk]
            )

data/lib/miga/result.rb

@@ -1,5 +1,4 @@
-# @package MiGA
-# @license Artistic-2.0
+# frozen_string_literal: true
 
 require 'miga/result/dates'
 require 'miga/result/source'
@@ -29,13 +28,17 @@ class MiGA::Result < MiGA::MiGA
       MiGA::Result.new(path)
     end
 
-    def create(path, force = false, &blk)
+    ##
+    # Check if +path+ describes a result and otherwise create
+    # it using the passed block. If +force+, ignore existing
+    # JSON in +path+ if any.
+    def create(path, force = false)
       FileUtils.rm(path) if force && File.exist?(path)
-      r_pre = self.load(path)
+      r_pre = load(path)
       return r_pre unless r_pre.nil?
 
       yield
-      self.load(path)
+      load(path)
     end
   end
 
@@ -49,7 +52,7 @@ class MiGA::Result < MiGA::MiGA
   # Load or create the MiGA::Result described by the JSON file +path+
   def initialize(path)
     @path = File.absolute_path(path)
-    MiGA::Result.exist?(@path) ? self.load : create
+    MiGA::Result.exist?(@path) ? load : create
   end
 
   ##
@@ -162,7 +165,7 @@ class MiGA::Result < MiGA::MiGA
       File.unlink s
     end
     MiGA::Json.generate(data, path)
-    self.load
+    load
   end
 
   ##
@@ -182,10 +185,9 @@ class MiGA::Result < MiGA::MiGA
   # Unlink result by removing the .done and .start timestamps and the
   # .json descriptor, but don't remove any other associated files
   def unlink
-    %i(start done).each do |i|
-      f = path(i) and File.exists?(f) and File.unlink(f)
+    %i[start done json].each do |i|
+      f = path(i) and File.exist?(f) and File.unlink(f)
     end
-    File.unlink path
   end
 
   ##
@@ -201,13 +203,14 @@ class MiGA::Result < MiGA::MiGA
 
     @data[:files] ||= {}
     self[:files].each do |k, files|
-      files = [files] unless files.kind_of? Array
+      files = [files] unless files.is_a? Array
       files.each do |file|
         case blk.arity
-        when 1; blk.call(file)
-        when 2; blk.call(k, file)
-        when 3; blk.call(k, file, File.expand_path(file, dir))
-        else; raise "Wrong number of arguments: #{blk.arity} for 1..3"
+        when 1 then blk.call(file)
+        when 2 then blk.call(k, file)
+        when 3 then blk.call(k, file, File.expand_path(file, dir))
+        else
+          raise "Wrong number of arguments: #{blk.arity} for 1..3"
         end
       end
     end

data/lib/miga/result/stats.rb

@@ -118,7 +118,7 @@ module MiGA::Result::Stats
 
   def compute_stats_essential_genes
     stats = {}
-    if source.is_multi?
+    if source.multi?
       stats = { median_copies: 0, mean_copies: 0 }
       File.open(file_path(:report), 'r') do |fh|
         fh.each_line do |ln|
@@ -151,7 +151,7 @@ module MiGA::Result::Stats
       source.save
 
       # Inactivate low-quality datasets
-      min_qual = (project.metadata[:min_qual] || 25)
+      min_qual = project.option(:min_qual)
       if min_qual != 'no' && stats[:quality] < min_qual
         source.inactivate! 'Low quality genome'
       end

data/lib/miga/version.rb

@@ -10,7 +10,7 @@ module MiGA
   # - Float representing the major.minor version.
   # - Integer representing gem releases of the current version.
   # - Integer representing minor changes that require new version number.
-  VERSION = [0.7, 23, 0].freeze
+  VERSION = [0.7, 25, 3].freeze
 
   ##
   # Nickname for the current major.minor version.
@@ -18,7 +18,7 @@ module MiGA
 
   ##
   # Date of the current gem release.
-  VERSION_DATE = Date.new(2021, 2, 9)
+  VERSION_DATE = Date.new(2021, 2, 26)
 
   ##
   # Reference of MiGA.

data/scripts/essential_genes.bash

@@ -24,18 +24,33 @@ fi
 mkdir "${DATASET}.ess"
 TYPE=$(miga ls -P "$PROJECT" -D "$DATASET" \
   --metadata "type" | awk '{print $2}')
-COLL=$(miga about -P "$PROJECT" -m ess_coll)
-[[ "$COLL" == "?" ]] && COLL=dupont_2012
+COLL=$(miga option -P "$PROJECT" --key ess_coll)
 if [[ "$TYPE" == "metagenome" || "$TYPE" == "virome" ]] ; then
   FLAGS="--metagenome"
 else
-  FLAGS="--alignments ${DATASET}.ess/proteins.aln"
+  FLAGS=""
 fi
 HMM.essential.rb \
   -i "$FAA" -o "${DATASET}.ess.faa" -m "${DATASET}.ess/" \
   -t "$CORES" -r "$DATASET" --collection "$COLL" $FLAGS \
   > "${DATASET}.ess/log"
 
+# Index for FastAAI
+NOMULTI=$(miga list_datasets -P "$PROJECT" -D "$DATASET" --no-multi \
+            | wc -l | awk '{print $1}')
+if [[ "$NOMULTI" -eq "1" ]] ; then
+  if [[ "$FAA" == *.gz ]] ; then
+    gzip -cd "$FAA" > "${DATASET}.faix"
+  else
+    cp "$FAA" "${DATASET}.faix"
+  fi
+  FastAAI --qp "${DATASET}.faix" --output "${DATASET}.faix" \
+    --ext ".faix" --index --input-paths --all-vs-all --threads "$CORES"
+  rm "${DATASET}.faix"
+  rm "${DATASET}.faix.hmm"
+  rm "${DATASET}.faix.hmm.filt"
+fi
+
 # Reduce files
 if exists "$DATASET".ess/*.faa ; then
   ( cd "${DATASET}.ess" \

data/scripts/miga.bash

@@ -1,12 +1,17 @@
 #!/bin/bash
+
+# Setup environment
 set -e
-#MIGA=${MIGA:-$(cd "$(dirname "$0")/.."; pwd)}
 MIGA_HOME=${MIGA_HOME:-"$HOME"}
+SCRIPT=${SCRIPT:-$(basename "$0" .bash)}
 # shellcheck source=/dev/null
 . "$MIGA_HOME/.miga_rc"
+
+# Ensure submodules are first in PATH
 export PATH="$MIGA/bin:$MIGA/utils/enveomics/Scripts:$PATH"
-SCRIPT=${SCRIPT:-$(basename "$0" .bash)}
+export PATH="$MIGA/utils/FastAAI/FastAAI:$PATH"
 
+# Ancillary functions
 function exists { [[ -e "$1" ]] ; }
 function fx_exists { [[ $(type -t "$1") == "function" ]] ; }
 function miga_start_project_step {
@@ -28,6 +33,7 @@ function miga_end_project_step {
   miga add_result -P "$PROJECT" -r "$SCRIPT" -f
 }
 
+# Environment header
 if [[ "$SCRIPT" != "d" && "$SCRIPT" != "p" ]] ; then
   echo ""
   echo "######[ $SCRIPT ]######"

data/scripts/ogs.bash

@@ -12,8 +12,7 @@ miga_start_project_step "$DIR"
 DS=$(miga ls -P "$PROJECT" --ref --no-multi)
 
 if [[ -n $DS ]] ; then
-  MIN_ID=$(miga about -P "$PROJECT" -m ogs_identity)
-  [[ $MIN_ID == "?" ]] && MIN_ID=80
+  MIN_ID=$(miga option -P "$PROJECT" --key ogs_identity)
   if [[ ! -s miga-project.ogs ]] ; then
     # Extract RBMs
     if [[ ! -s miga-project.abc ]] ; then
@@ -34,7 +33,7 @@ if [[ -n $DS ]] ; then
 
     # Estimate OGs and Clean RBMs
     ogs.mcl.rb -o miga-project.ogs --abc miga-project.abc -t "$CORES"
-    if [[ $(miga about -P "$PROJECT" -m clean_ogs) == "false" ]] ; then
+    if [[ $(miga option -P "$PROJECT" --key clean_ogs) == "false" ]] ; then
       gzip -9 miga-project.abc
     else
       rm miga-project.abc

data/test/dataset_test.rb

@@ -29,7 +29,7 @@ class DatasetTest < Test::Unit::TestCase
     assert_raise { MiGA::Dataset.new(project, 'dataset-1') }
     assert_equal(project, dataset.project)
     assert_equal('dataset0', dataset.name)
-    assert_predicate(dataset, :is_ref?)
+    assert_predicate(dataset, :ref?)
     assert_equal(MiGA::Metadata, dataset.metadata.class)
     assert_equal(:incomplete, dataset.status)
   end
@@ -38,14 +38,14 @@ class DatasetTest < Test::Unit::TestCase
     d2 = project.add_dataset('ds_save')
     assert_respond_to(d2, :save)
     d2.save
-    assert_not_predicate(d2, :is_multi?)
-    assert_not_predicate(d2, :is_nonmulti?)
+    assert_not_predicate(d2, :multi?)
+    assert_not_predicate(d2, :nonmulti?)
     assert_nil(d2.metadata[:type])
     d2.metadata[:type] = :metagenome
     d2.save
     assert_equal(:metagenome, d2.metadata[:type])
-    assert_predicate(d2, :is_multi?)
-    assert_not_predicate(d2, :is_nonmulti?)
+    assert_predicate(d2, :multi?)
+    assert_not_predicate(d2, :nonmulti?)
   end
 
   def test_remove