miga-base 0.7.21.0 → 0.7.24.0

data/scripts/aai_distances.bash

@@ -19,7 +19,7 @@ rm -f miga-project.txt
   for i in $DS ; do
     echo "SELECT CASE WHEN omega!=0 THEN 'AAI' ELSE 'hAAI_AAI' END," \
       " seq1, seq2, aai, sd, n, omega from aai;" \
-      | sqlite3 "$i.db" | tr "\\|" "\\t"
+      | sqlite3 "$DIR/$i.db" | tr "\\|" "\\t"
     echo "$i" >> miga-project.log
   done
 ) | gzip -9c > miga-project.txt.gz

data/scripts/ani_distances.bash

@@ -18,7 +18,7 @@ rm -f miga-project.txt
   echo "metric a b value sd n omega" | tr " " "\\t"
   for i in $DS ; do
     echo "SELECT 'ANI', seq1, seq2, ani, sd, n, omega from ani ;" \
-      | sqlite3 "$i.db" | tr "\\|" "\\t"
+      | sqlite3 "$DIR/$i.db" | tr "\\|" "\\t"
     echo "$i" >> miga-project.log
   done
 ) | gzip -9c > miga-project.txt.gz

data/scripts/essential_genes.bash

@@ -24,8 +24,7 @@ fi
 mkdir "${DATASET}.ess"
 TYPE=$(miga ls -P "$PROJECT" -D "$DATASET" \
   --metadata "type" | awk '{print $2}')
-COLL=$(miga about -P "$PROJECT" -m ess_coll)
-[[ "$COLL" == "?" ]] && COLL=dupont_2012
+COLL=$(miga option -P "$PROJECT" --key ess_coll)
 if [[ "$TYPE" == "metagenome" || "$TYPE" == "virome" ]] ; then
   FLAGS="--metagenome"
 else

data/scripts/haai_distances.bash

@@ -22,7 +22,7 @@ rm -f miga-project.txt
   echo "metric a b value sd n omega" | tr " " "\\t"
   for i in $DS ; do
     echo "SELECT 'hAAI', seq1, seq2, aai, sd, n, omega from aai ;" \
-      | sqlite3 "$i.db" | tr "\\|" "\\t"
+      | sqlite3 "$DIR/$i.db" | tr "\\|" "\\t"
     echo "$i" >> miga-project.log
   done
 ) | gzip -9c > miga-project.txt.gz

data/scripts/mytaxa.bash

@@ -14,15 +14,16 @@ if [[ "$MIGA_MYTAXA" == "no" ]] ; then
   echo "This system doesn't currently support MyTaxa." \
     > "$DATASET.nomytaxa.txt"
 else
-  MT=$(dirname -- "$(which MyTaxa)")
-
   # Check type of dataset
   MULTI=$(miga list_datasets -P "$PROJECT" -D "$DATASET" --multi \
     | wc -l | awk '{print $1}')
   if [[ "$MULTI" -eq "1" ]] ; then
     # Check requirements
-    if [[ ! -e "$MT/AllGenomes.faa.dmnd" ]] ; then
-      echo "Cannot locate the database: $MT/AllGenomes.faa.dmnd:" \
+    MT=$(dirname -- "$(which MyTaxa)")
+    DB="$MIGA_HOME/.miga_db/AllGenomes.faa.dmnd"
+    [[ -e "$DB" ]] || DB="$MT/AllGenomes.faa.dmnd"
+    if [[ ! -e "$DB" ]] ; then
+      echo "Cannot locate the database: AllGenomes.faa.dmnd:" \
             "no such file or directory" >&2
       exit 1
     fi
@@ -40,7 +41,7 @@ else
     # Execute search
     FAA="../../../06.cds/$DATASET.faa"
     [[ -s "$FAA" ]] || FAA="${FAA}.gz"
-    diamond blastp -q "$FAA" -d "$MT/AllGenomes.faa" \
+    diamond blastp -q "$FAA" -d "$DB" \
       -a "$DATASET.daa" -k 5 -p "$CORES" --min-score 60
     diamond view -a "$DATASET.daa" -o "$DATASET.blast"
 

data/scripts/mytaxa_scan.bash

@@ -5,7 +5,6 @@ SCRIPT="mytaxa_scan"
 # shellcheck source=scripts/miga.bash
 . "$MIGA/scripts/miga.bash" || exit 1
 DIR="$PROJECT/data/07.annotation/03.qa/02.mytaxa_scan"
-[[ -d "$DIR" ]] || mkdir -p "$DIR"
 cd "$DIR"
 
 # Initialize
@@ -14,17 +13,16 @@ if [[ "$MIGA_MYTAXA" == "no" ]] ; then
   echo "This system doesn't currently support MyTaxa." \
     > "$DATASET.nomytaxa.txt"
 else
-  MT=$(dirname -- "$(which MyTaxa)")
-  TMPDIR=$(mktemp -d /tmp/MiGA.XXXXXXXXXXXX)
-  trap "rm -rf '$TMPDIR'; exit" SIGHUP SIGINT SIGTERM
-
   # Check type of dataset
   NOMULTI=$(miga list_datasets -P "$PROJECT" -D "$DATASET" --no-multi \
     | wc -l | awk '{print $1}')
   if [[ "$NOMULTI" -eq "1" ]] ; then
     # Check requirements
+    MT=$(dirname -- "$(which MyTaxa)")
+    DB="$MIGA_HOME/.miga_db/AllGenomes.faa.dmnd"
+    [[ -e "$DB" ]] || DB="$MT/AllGenomes.faa.dmnd"
     if [[ ! -e "$MT/AllGenomes.faa.dmnd" ]] ; then
-      echo "Cannot locate the database: $MT/AllGenomes.faa.dmnd:" \
+      echo "Cannot locate the database: AllGenomes.faa.dmnd:" \
             "no such file or directory" >&2
       exit 1
     fi
@@ -39,13 +37,16 @@ else
       exit 1
     fi
      
+    TMPDIR=$(mktemp -d /tmp/MiGA.XXXXXXXXXXXX)
+    trap "rm -rf '$TMPDIR'; exit" SIGHUP SIGINT SIGTERM
+
     FAA="../../../06.cds/$DATASET.faa"
     [[ -s "$FAA" ]] || FAA="${FAA}.gz"
     if [[ ! -s "$DATASET.mytaxa" ]] ; then
       # Execute search
       if [[ ! -s "$DATASET.blast" ]] ; then
         diamond blastp -q "$FAA" -a "$DATASET.daa" -t "$TMPDIR" \
-          -d "$MT/AllGenomes.faa" -k 5 -p "$CORES" --min-score 60
+          -d "$DB" -k 5 -p "$CORES" --min-score 60
         diamond view -a "$DATASET.daa" -o "$DATASET.blast" -t "$TMPDIR"
       fi
 

data/scripts/ogs.bash

@@ -12,8 +12,7 @@ miga_start_project_step "$DIR"
 DS=$(miga ls -P "$PROJECT" --ref --no-multi)
 
 if [[ -n $DS ]] ; then
-  MIN_ID=$(miga about -P "$PROJECT" -m ogs_identity)
-  [[ $MIN_ID == "?" ]] && MIN_ID=80
+  MIN_ID=$(miga option -P "$PROJECT" --key ogs_identity)
   if [[ ! -s miga-project.ogs ]] ; then
     # Extract RBMs
     if [[ ! -s miga-project.abc ]] ; then
@@ -34,7 +33,7 @@ if [[ -n $DS ]] ; then
 
     # Estimate OGs and Clean RBMs
     ogs.mcl.rb -o miga-project.ogs --abc miga-project.abc -t "$CORES"
-    if [[ $(miga about -P "$PROJECT" -m clean_ogs) == "false" ]] ; then
+    if [[ $(miga option -P "$PROJECT" --key clean_ogs) == "false" ]] ; then
       gzip -9 miga-project.abc
     else
       rm miga-project.abc

data/scripts/ssu.bash

@@ -16,6 +16,7 @@ if [[ -s $fa ]] ; then
   # Run barrnap
   barrnap --quiet --threads "$CORES" "$fa" | grep "^##gff\\|;product=16S " \
     > "$DATASET.ssu.gff"
+
   # Extract
   bedtools getfasta -s "-fi" "$fa" -bed "$DATASET.ssu.gff" \
     -fo "$DATASET.ssu.all.fa"
@@ -24,9 +25,22 @@ if [[ -s $fa ]] ; then
   FastA.filter.pl "$DATASET.ssu.fa.id" "$DATASET.ssu.all.fa" > "$DATASET.ssu.fa"
   rm "$DATASET.ssu.fa.id"
   [[ -e "$fa.fai" ]] && rm "$fa.fai"
+
+  # RDP classifier
+  if [[ "$MIGA_RDP" == "yes" && -s "$DATASET.ssu.all.fa" ]] ; then
+    java -jar "$MIGA_HOME/.miga_db/classifier.jar" classify \
+      -c 0.8 -f fixrank -g 16srrna -o "$DATASET.rdp.tsv" \
+      "$DATASET.ssu.all.fa"
+    echo "# Version: $(perl -pe 's/.*://' \
+          < "$MIGA_HOME/.miga_db/classifier.version.txt" \
+          | grep . | paste - - | perl -pe 's/\t/; /')" \
+      >> "$DATASET.rdp.tsv"
+  fi
+
   # Gzip
-  gzip -9 -f "$DATASET.ssu.gff"
-  gzip -9 -f "$DATASET.ssu.all.fa"
+  for x in ssu.gff ssu.all.fa rdp.tsv ; do
+    [[ -e "${DATASET}.${x}" ]] && gzip -9 -f "${DATASET}.${x}"
+  done
 fi
 
 # Finalize

data/test/dataset_test.rb

@@ -29,7 +29,7 @@ class DatasetTest < Test::Unit::TestCase
     assert_raise { MiGA::Dataset.new(project, 'dataset-1') }
     assert_equal(project, dataset.project)
     assert_equal('dataset0', dataset.name)
-    assert_predicate(dataset, :is_ref?)
+    assert_predicate(dataset, :ref?)
     assert_equal(MiGA::Metadata, dataset.metadata.class)
     assert_equal(:incomplete, dataset.status)
   end
@@ -38,14 +38,14 @@ class DatasetTest < Test::Unit::TestCase
     d2 = project.add_dataset('ds_save')
     assert_respond_to(d2, :save)
     d2.save
-    assert_not_predicate(d2, :is_multi?)
-    assert_not_predicate(d2, :is_nonmulti?)
+    assert_not_predicate(d2, :multi?)
+    assert_not_predicate(d2, :nonmulti?)
     assert_nil(d2.metadata[:type])
     d2.metadata[:type] = :metagenome
     d2.save
     assert_equal(:metagenome, d2.metadata[:type])
-    assert_predicate(d2, :is_multi?)
-    assert_not_predicate(d2, :is_nonmulti?)
+    assert_predicate(d2, :multi?)
+    assert_not_predicate(d2, :nonmulti?)
   end
 
   def test_remove

data/test/net_test.rb

@@ -0,0 +1,34 @@
+require 'test_helper'
+
+class FormatTest < Test::Unit::TestCase
+  include TestHelper
+
+  def test_known_hosts
+    m = MiGA::MiGA
+    assert_not_nil(m.known_hosts(:miga_db))
+    assert_not_nil(m.known_hosts('miga_db'))
+    assert_not_nil(m.known_hosts(:miga_dist))
+    assert_raise { m.known_kosts(:not_a_host) }
+  end
+
+  def test_remote_connection
+    declare_remote_access
+    m = MiGA::MiGA
+    assert_raise { m.remote_connection(:bad_descriptor) }
+    assert_raise { m.remote_connection('http://microbial-genomes.org/') }
+    c = m.remote_connection(:miga_db)
+    assert_equal(Net::FTP, c.class)
+    c.close
+  end
+
+  def test_download_file_ftp
+    declare_remote_access
+    m = MiGA::MiGA
+    f = tmpfile('t/test.txt')
+    d = File.dirname(f)
+    assert(!Dir.exist?(d))
+    m.download_file_ftp(:miga_online_ftp, 'test.txt', f)
+    assert(Dir.exist?(d))
+    assert_equal('miga', File.read(f).chomp)
+  end
+end

data/test/with_option_test.rb

@@ -0,0 +1,115 @@
+# frozen_string_literal: true
+
+require 'test_helper'
+require 'miga/common/with_option'
+
+class WithDaemonTest < Test::Unit::TestCase
+  include TestHelper
+
+  class TestWithOption < MiGA::MiGA
+    include MiGA::Common::WithOption
+
+    attr_reader :metadata, :saved
+
+    def initialize
+      @metadata = { range: 0.9 }
+      @saved = false
+    end
+
+    def self.OPTIONS
+      {
+        empty: {},
+        float: { type: Float },
+        range: { default: 1.0, in: -5.5..5.5, type: Float },
+        default: { default: 9, type: Integer },
+        token: { type: Integer, tokens: %w[yes no 0] },
+        proc: { default: proc { Date.today } },
+        bool: { in: [true, false] }
+      }
+    end
+
+    def save
+      @saved = true
+    end
+  end
+
+  def test_with_option
+    o = TestWithOption.new
+    assert_respond_to(o, :option)
+    assert_equal(1, o.metadata.size)
+  end
+
+  def test_option
+    o = TestWithOption.new
+    assert_equal(9, o.option(:default))
+    assert_nil(o.option(:bool))
+    assert_raise { o.option(:not_an_option) }
+    assert_nil(o.option(:empty))
+  end
+
+  def test_set_bool
+    o = TestWithOption.new
+    assert_nil(o.option(:bool))
+    assert(!o.saved)
+    assert_raise { o.set_option(:bool, 'true') }
+    assert_nil(o.option(:bool))
+    assert(!o.saved)
+    assert_equal(true, o.set_option(:bool, 'true', true))
+    assert(o.saved)
+    assert_equal(false, o.set_option(:bool, false))
+    assert_equal(false, o.set_option(:bool, 'false', true))
+    assert_nil(o.set_option(:bool, nil))
+  end
+
+  def test_set_empty
+    o = TestWithOption.new
+    assert_nil(o.option(:empty))
+    assert_equal('a', o.set_option(:empty, 'a'))
+    assert_equal('1', o.set_option(:empty, '1', true))
+  end
+
+  def test_all_options
+    o = TestWithOption.new
+    assert(o.all_options.is_a?(Hash))
+    assert_include(o.all_options.keys, :bool)
+    assert_nil(o.all_options[:bool])
+  end
+
+  def test_option?
+    o = TestWithOption.new
+    assert(o.option?(:range))
+    assert(!o.option?(:not_an_option))
+  end
+
+  def test_option_metadata
+    o = TestWithOption.new
+    assert_equal(0.9, o.option(:range))
+    assert_equal(1.0, o.set_option(:range, nil))
+    assert_equal(2.0, o.set_option(:range, 2.0))
+    assert_equal(3.0, o.set_option(:range, '3', true))
+  end
+
+  def test_option_range
+    o = TestWithOption.new
+    assert_raise { o.set_option(:range, 9.0) }
+    assert_raise { o.set_option(:range, 3) }
+    assert_raise { o.set_option(:range, true) }
+  end
+
+  def test_option_proc
+    o = TestWithOption.new
+    assert(o.option(:proc).is_a?(Date))
+    assert(o.set_option(:proc, 1).is_a?(Integer))
+    assert(o.set_option(:proc, nil).is_a?(Date))
+  end
+
+  def test_token
+    o = TestWithOption.new
+    assert_nil(o.option(:token))
+    assert_equal(1, o.set_option(:token, 1))
+    assert_equal(-2, o.set_option(:token, '-2', true))
+    assert_equal('yes', o.set_option(:token, 'yes'))
+    assert_equal('0', o.set_option(:token, '0', true))
+    assert_raise { o.set_option(:token, 'maybe') }
+  end
+end

data/utils/cleanup-databases.rb

@@ -15,11 +15,11 @@ m.say 'Cleaning Databases'
 (0..thr - 1).each do |t|
   fork do
     dsn.each_with_index do |i, idx|
-      m.advance('Dataset:', dsn.size, idx + 1) if t == 0
+      m.advance('Dataset:', idx + 1, dsn.size) if t == 0
       next unless (idx % thr) == t
 
       d = p.dataset(i)
-      next unless d.is_ref? and d.is_active?
+      next unless d.ref? && d.active?
 
       d.cleanup_distances!
     end
@@ -28,4 +28,3 @@ end
 Process.waitall
 m.advance('Dataset:', dsn.size, dsn.size)
 m.say
-

data/utils/distance/commands.rb

@@ -9,7 +9,7 @@ module MiGA::DistanceRunner::Commands
     return y unless y.nil? || y.zero?
 
     # Try hAAI (except in clade projects)
-    unless @ref_project.is_clade?
+    unless @ref_project.clade?
       y = haai(target)
       return y unless y.nil? || y.zero?
     end
@@ -75,7 +75,7 @@ module MiGA::DistanceRunner::Commands
     v = `aai.rb -1 "#{f1}" -2 "#{f2}" -S "#{db}" \
           --name1 "#{n1}" --name2 "#{n2}" \
           -t "#{o[:thr]}" -a --lookup-first "--#{o[:aai_save_rbm]}" \
-          -p "#{o[:aai_p] || 'blast+'}"`.chomp
+          -p "#{o[:aai_p]}"`.chomp
     (v.nil? || v.empty?) ? 0 : v.to_f
   end
 

data/utils/distance/database.rb

@@ -66,7 +66,7 @@ module MiGA::DistanceRunner::Database
     return y unless y.nil? || y.zero?
 
     # Check if self.dataset <- target is done (another thread)
-    if dataset.is_ref? && project.path == ref_project.path
+    if dataset.ref? && project.path == ref_project.path
       y = data_from_db(
         target.name, dataset.name, ref_db(metric, target.name), metric
       )

data/utils/distance/pipeline.rb

@@ -31,10 +31,8 @@ module MiGA::DistanceRunner::Pipeline
 
   # Run distances against datasets listed in metadata's +:dist_req+
   def distances_by_request(metric)
-    return unless dataset.metadata[:dist_req]
-
     $stderr.puts 'Running distances by request'
-    dataset.metadata[:dist_req].each do |target|
+    dataset.option(:dist_req).each do |target|
       ds = ref_project.dataset(target) and send(metric, ds)
     end
   end
@@ -112,7 +110,7 @@ module MiGA::DistanceRunner::Pipeline
     $stderr.puts 'Transferring taxonomy'
     return if tax.nil?
 
-    pval = (project.metadata[:tax_pvalue] || 0.05).to_f
+    pval = project.option(:tax_pvalue)
     tax_a = tax
             .select { |i| i[1] != '?' && i[2] <= pval }
             .map { |i| i[0, 2].join(':') }

data/utils/distance/runner.rb

@@ -21,47 +21,39 @@ class MiGA::DistanceRunner
     @home = File.expand_path('data/09.distances', project.path)
 
     # Default opts
-    if project.metadata[:aai_save_rbm] == false
-      @opts[:aai_save_rbm] ||= 'no-save-rbm'
-    end
-    @opts[:aai_save_rbm] ||= ENV.fetch('MIGA_AAI_SAVE_RBM') do
-      project.is_clade? ? 'save-rbm' : 'no-save-rbm'
-    end
+    @opts[:aai_save_rbm] =
+      project.option(:aai_save_rbm) ? 'save-rbm' : 'no-save-rbm'
     @opts[:thr] ||= ENV.fetch('CORES') { 2 }.to_i
-    if opts[:run_taxonomy] and project.metadata[:ref_project]
-      ref_path = project.metadata[:ref_project]
+    if opts[:run_taxonomy] && project.option(:ref_project)
+      ref_path = project.option(:ref_project)
       @home = File.expand_path('05.taxonomy', @home)
       @ref_project = MiGA::Project.load(ref_path)
       raise "Cannot load reference project: #{ref_path}" if @ref_project.nil?
-    elsif !opts[:run_taxonomy] and dataset.metadata[:db_project]
-      ref_path = dataset.metadata[:db_project]
-      if project.metadata[:db_proj_dir]
-        ref_path = File.expand_path(ref_path, project.metadata[:db_proj_dir])
+    elsif !opts[:run_taxonomy] && dataset.option(:db_project)
+      ref_path = dataset.option(:db_project)
+      if project.option(:db_proj_dir)
+        ref_path = File.expand_path(ref_path, project.option(:db_proj_dir))
       end
       @ref_project = MiGA::Project.load(ref_path)
       raise "Cannot load reference project: #{ref_path}" if @ref_project.nil?
     else
       @ref_project = project
     end
-    [:haai_p, :aai_p, :ani_p, :distances_checkpoint].each do |m|
-      @opts[m] ||= ref_project.metadata[m]
+    %i[haai_p aai_p ani_p distances_checkpoint].each do |m|
+      @opts[m] ||= ref_project.option(m)
     end
-    @opts[:aai_p] ||= 'blast+'
-    @opts[:ani_p] ||= 'blast+'
-    @opts[:distances_checkpoint] ||= 10
-    @opts[:distances_checkpoint] = @opts[:distances_checkpoint].to_i
     $stderr.puts "Options: #{opts}"
   end
 
   # Launch the appropriate analysis
   def go!
     $stderr.puts "Launching analysis"
-    return if dataset.is_multi?
+    return if dataset.multi?
 
     Dir.mktmpdir do |tmp_dir|
       @tmp = tmp_dir
       create_temporals
-      opts[:run_taxonomy] ? go_taxonomy! : dataset.is_ref? ? go_ref! : go_query!
+      opts[:run_taxonomy] ? go_taxonomy! : dataset.ref? ? go_ref! : go_query!
     end
   end
 
@@ -73,7 +65,7 @@ class MiGA::DistanceRunner
 
     # first-come-first-serve traverse
     ref_project.each_dataset do |ds|
-      next if !ds.is_ref? or ds.is_multi? or ds.result(:essential_genes).nil?
+      next if !ds.ref? or ds.multi? or ds.result(:essential_genes).nil?
 
       puts "[ #{Time.now} ] #{ds.name}"
       ani_after_aai(ds)
@@ -88,7 +80,7 @@ class MiGA::DistanceRunner
   def go_query!
     $stderr.puts 'Launching analysis for query dataset'
     # Check if project is ready
-    tsk = ref_project.is_clade? ? [:subclades, :ani] : [:clade_finding, :aai]
+    tsk = ref_project.clade? ? [:subclades, :ani] : [:clade_finding, :aai]
     res = ref_project.result(tsk[0])
     return if res.nil?
 
@@ -137,7 +129,7 @@ class MiGA::DistanceRunner
   # Launch analysis for taxonomy jobs
   def go_taxonomy!
     $stderr.puts 'Launching taxonomy analysis'
-    return unless project.metadata[:ref_project]
+    return unless project.option(:ref_project)
 
     go_query! # <- yeah, it's actually the same, just different ref_project
   end