miga-base 0.3.0.7 → 0.3.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4132f066f463f3967517e7e278b765087a22bb89
-  data.tar.gz: 78fcfd7accea31d041991f26944d701cc6b31fef
+  metadata.gz: 220a35c60112758e6f1f2226c9978774db5b6de0
+  data.tar.gz: e08ef0fb54d966b76376f2c87284f532e3c3fa9b
 SHA512:
-  metadata.gz: b7446a3f81368c04f4e8cdec66abe28eb7eeb0c9f00d046dee1b4882007c8f4f2325052f466069407a17a0d65b1e6b1cc274fa10f2a3c5dabf7684a3f34e6852
-  data.tar.gz: 1c47807eb8e3a97b2cd464495a0fd022f6c8f1aa86bfd8ed6dc018d376f63994dded3616467df4528b9286a057f0ac4e219313f76fa03542ad143925403c877d
+  metadata.gz: 3ab68a7fefcd05049dfb9939adade6efe490171afff1039650d49ba53ba8b55ad6be54c334b6d2ab0188c2ecc469d161455e1ea3eca0e1dab60063a686704ef5
+  data.tar.gz: 03c15a94148521c562dab2b26c377666d59e6c2720702b12c646ad900247089ffb28aef9706e5b299e80eca1a38addfdb6afb78c0a2bce91a28cd401a15aa461

data/actions/stats.rb

@@ -69,15 +69,42 @@ if o[:compute]
     s = `FastA.length.pl '#{f}' | #{scr}`.chomp.split(" ")
     stats = {predicted_proteins: s[0].to_i, average_length: [s[1].to_f, "aa"]}
   when :essential_genes
-    stats = {completeness:[0.0,"%"], contamination:[0.0,"%"]}
-    File.open(r.file_path(:report), "r") do |fh|
-      fh.each_line do |ln|
-        if /^! (Completeness|Contamination): (.*)%/.match(ln)
-          stats[$1.downcase.to_sym][0] = $2.to_f
+    if d.is_multi?
+      stats = {median_copies:0, mean_copies:0}
+      File.open(r.file_path(:report), "r") do |fh|
+        fh.each_line do |ln|
+          if /^! (Mean|Median) number of copies per model: (.*)\./.match(ln)
+            stats["#{$1.downcase}_copies".to_sym] = $2.to_f
+          end
         end
       end
+    else
+      # Fix estimate for Archaea
+      if not d.metadata[:tax].nil? and
+            d.metadata[:tax].is_in? MiGA::Taxonomy.new("d:Archaea") and
+            r.file_path(:bac_report).nil?
+        scr = "#{MiGA::MiGA.root_path}/utils/arch-ess-genes.rb"
+        rep = r.file_path(:report)
+        $stderr.print `ruby '#{scr}' '#{rep}' '#{rep}.archaea'`
+        r.add_file(:bac_report, "#{d.name}.ess/log")
+        r.add_file(:report, "#{d.name}.ess/log.archaea")
+      end
+      # Extract/compute quality values
+      stats = {completeness:[0.0,"%"], contamination:[0.0,"%"]}
+      File.open(r.file_path(:report), "r") do |fh|
+        fh.each_line do |ln|
+          if /^! (Completeness|Contamination): (.*)%/.match(ln)
+            stats[$1.downcase.to_sym][0] = $2.to_f
+          end
+        end
+      end
+      stats[:quality] = stats[:completeness][0] - stats[:contamination][0]*5
+      q_range = stats[:quality] > 80.0 ? :excellent :
+        stats[:quality] > 50.0 ? :high :
+        stats[:quality] > 20.0 ? :intermediate : :low
+      d.metadata[:quality_interval] = q_range
+      d.save
     end
-    stats[:quality] = stats[:completeness][0] - stats[:contamination][0]*5
   when :distances
     d.cleanup_distances! unless d.nil?
   else

data/actions/tax_test.rb

@@ -5,10 +5,13 @@
 
 require "miga/tax_dist"
 
-o = {q:true, test:"both"}
+o = {q:true, test:"both", ref_project:false}
 OptionParser.new do |opt|
   opt_banner(opt)
   opt_object(opt, o, [:project, :dataset])
+  opt.on("--ref-project",
+    "Use the taxonomy from the reference project, not the current project."
+    ){ |v| o[:ref_project]=v }
   opt.on("-t", "--test STRING",
     "Test to perform. Supported values: intax, novel, both."
     ){ |v| o[:test]=v.downcase }
@@ -26,7 +29,7 @@ $stderr.puts "Loading dataset." unless o[:q]
 ds = p.dataset(o[:dataset])
 
 $stderr.puts "Finding closest relative." unless o[:q]
-cr = ds.closest_relatives(1)
+cr = ds.closest_relatives(1, o[:ref_project])
 
 if cr.nil? or cr.empty?
   raise "This action is not supported for the project or dataset." if cr.nil?
@@ -35,7 +38,15 @@ else
   $stderr.puts "Querying probability distributions." unless o[:q]
   cr = cr[0]
   puts "Closest relative: #{cr[0]} with AAI: #{cr[1]}."
-  tax = p.dataset(cr[0]).metadata[:tax]
+  if o[:ref_project]
+    ref = p.metadata[:ref_project]
+    raise "--ref-project requested, but no reference project has been set." if ref.nil?
+    q = MiGA::Project.load(ref)
+    raise "--ref-project requested, but reference project doesn't exist." if q.nil?
+    tax = q.dataset(cr[0]).metadata[:tax]
+  else
+    tax = p.dataset(cr[0]).metadata[:tax]
+  end
   tax ||= {}
   
   if %w[intax both].include? o[:test]

data/lib/miga/dataset.rb

@@ -28,7 +28,7 @@ class MiGA::Dataset < MiGA::MiGA
     mytaxa: "07.annotation/02.taxonomy/01.mytaxa",
     mytaxa_scan: "07.annotation/03.qa/02.mytaxa_scan",
     # Distances (for single-species datasets)
-    distances: "09.distances",
+    distances: "09.distances", taxonomy: "09.distances/05.taxonomy",
     # General statistics
     stats: "90.stats"
   }
@@ -52,17 +52,17 @@ class MiGA::Dataset < MiGA::MiGA
   def self.PREPROCESSING_TASKS ; @@PREPROCESSING_TASKS ; end
   @@PREPROCESSING_TASKS = [:raw_reads, :trimmed_reads, :read_quality,
     :trimmed_fasta, :assembly, :cds, :essential_genes, :ssu, :mytaxa,
-    :mytaxa_scan, :distances, :stats]
+    :mytaxa_scan, :distances, :taxonomy, :stats]
   
   ##
   # Tasks to be excluded from query datasets.
-  @@EXCLUDE_NOREF_TASKS = [:mytaxa_scan]
+  @@EXCLUDE_NOREF_TASKS = [:mytaxa_scan, :taxonomy]
   @@_EXCLUDE_NOREF_TASKS_H = Hash[@@EXCLUDE_NOREF_TASKS.map{ |i| [i,true] }]
   
   ##
   # Tasks to be executed only in datasets that are not multi-organism. These
   # tasks are ignored for multi-organism datasets or for unknown types.
-  @@ONLY_NONMULTI_TASKS = [:mytaxa_scan, :distances]
+  @@ONLY_NONMULTI_TASKS = [:mytaxa_scan, :distances, :taxonomy]
   @@_ONLY_NONMULTI_TASKS_H = Hash[@@ONLY_NONMULTI_TASKS.map{ |i| [i,true] }]
 
   ##
@@ -237,6 +237,7 @@ class MiGA::Dataset < MiGA::MiGA
   # Should I ignore +task+ for this dataset?
   def ignore_task?(task)
     return !metadata["run_#{task}"] unless metadata["run_#{task}"].nil?
+    return true if task==:taxonomy and project.metadata[:ref_project].nil?
     pattern = [true, false]
     ( [@@_EXCLUDE_NOREF_TASKS_H[task], is_ref?     ]==pattern or
       [@@_ONLY_MULTI_TASKS_H[task],    is_multi?   ]==pattern or
@@ -271,14 +272,15 @@ class MiGA::Dataset < MiGA::MiGA
   end
   
   ##
-  # Returns an Array of duples (Arrays) sorted by AAI:
+  # Returns an Array of +how_many+ duples (Arrays) sorted by AAI:
   # - +0+: A String with the name(s) of the reference dataset.
   # - +1+: A Float with the AAI.
-  # This function is currently only supported for query datasets. It returns
+  # This function is currently only supported for query datasets when +ref_project+ is false
+  # (default), and only for reference dataset when +ref_project+ is true. It returns
   # +nil+ if this analysis is not supported.
-  def closest_relatives(how_many=1)
-    return nil if is_ref? or project.is_multi?
-    r = result :distances
+  def closest_relatives(how_many=1, ref_project=false)
+    return nil if (is_ref? != ref_project) or is_multi?
+    r = result(ref_project ? :taxonomy : :distances)
     return nil if r.nil?
     db = SQLite3::Database.new(r.file_path :aai_db)
     db.execute("SELECT seq2, aai FROM aai WHERE seq2 != ? " +

data/lib/miga/dataset_result.rb

@@ -188,6 +188,13 @@ module MiGA::DatasetResult
       end
     end
 
+    ##
+    # Add result type +:taxonomy+ at +base+ (no +_opts+ supported).
+    def add_result_taxonomy(base, _opts)
+      r = add_result_distances_nonref(base)
+      add_files_to_ds_result(r, name, intax_test:".intax.txt")
+    end
+
     ##
     # Add result type +:stats+ at +base+ (no +_opts+ supported).
     def add_result_stats(base, _opts)

data/lib/miga/project.rb

@@ -29,7 +29,7 @@ class MiGA::Project < MiGA::MiGA
     07.annotation/03.qa/02.mytaxa_scan
     08.mapping 08.mapping/01.read-ctg 08.mapping/02.read-gene
     09.distances 09.distances/01.haai 09.distances/02.aai
-    09.distances/03.ani 09.distances/04.ssu
+    09.distances/03.ani 09.distances/04.ssu 09.distances/05.taxonomy
     10.clades 10.clades/01.find 10.clades/02.ani 10.clades/03.ogs
     10.clades/04.phylogeny 10.clades/04.phylogeny/01.essential
     10.clades/04.phylogeny/02.core 10.clades/05.metadata

data/lib/miga/result.rb

@@ -112,6 +112,7 @@ class MiGA::Result < MiGA::MiGA
   # Load (or reload) result data in the JSON file #path.
   def load
     json = File.read(path)
+    raise "Impossible to load result, empty descriptor: #{path}." if json.empty?
     @data = JSON.parse(json, {:symbolize_names=>true})
     @data[:files] ||= {}
     @results = (self[:results] || []).map{ |rs| MiGA::Result.new rs }

data/lib/miga/taxonomy.rb

@@ -133,7 +133,7 @@ class MiGA::Taxonomy < MiGA::MiGA
   ##
   # Generate cannonical String for the taxonomy.
   def to_s
-    sorted_ranks.map{ |r| "#{r[0]}:#{r[1].gsub(/\s/,"_")}" }.join(" ")
+    sorted_ranks.map{ |r| "#{r[0]}:#{r[1].gsub(/[\s:]/,"_")}" }.join(" ")
   end
   
   ##

data/lib/miga/version.rb

@@ -10,7 +10,7 @@ module MiGA
   # - Float representing the major.minor version.
   # - Integer representing gem releases of the current version.
   # - Integer representing minor changes that require new version number.
-  VERSION = [0.3, 0, 7]
+  VERSION = [0.3, 1, 0]
   
   ##
   # Nickname for the current major.minor version.
@@ -18,7 +18,7 @@ module MiGA
   
   ##
   # Date of the current gem release.
-  VERSION_DATE = Date.new(2017, 8, 25)
+  VERSION_DATE = Date.new(2017, 9, 14)
   
   ##
   # Reference of MiGA.

data/scripts/_distances_noref_nomulti.bash

@@ -28,32 +28,35 @@ fx_exists miga-noref_haai_or_aai || function miga-noref_haai_or_aai {
   local Q=$1
   local S=$2
   [[ -s $TMPDIR/$Q.faa ]] \
-    || cp "../06.cds/$Q.faa" "$TMPDIR/$Q.faa"
-  miga-haai_or_aai "$ESS/$Q.ess.faa" "$ESS/$S.ess.faa" "$TMPDIR/$Q.haai.db" \
-    "$TMPDIR/$Q.faa" "../06.cds/$S.faa" "$TMPDIR/$Q.aai.db" "$CORES"
+    || cp "$PROJECT/data/06.cds/$Q.faa" "$TMPDIR/$Q.faa"
+  miga-haai_or_aai "$PROJECT/$ESS/$Q.ess.faa" "$S_PROJ/$ESS/$S.ess.faa" \
+    "$TMPDIR/$Q.haai.db" "$TMPDIR/$Q.faa" "$S_PROJ/data/06.cds/$S.faa" \
+    "$TMPDIR/$Q.aai.db" "$CORES"
 }
 
 fx_exists miga-noref_ani || function miga-noref_ani {
   local Q=$1
   local S=$2
   [[ -s "$TMPDIR/$Q.LargeContigs.fna" ]] \
-    || cp "../05.assembly/$Q.LargeContigs.fna" "$TMPDIR/$Q.LargeContigs.fna"
-  miga-ani "$TMPDIR/$Q.LargeContigs.fna" "../05.assembly/$S.LargeContigs.fna" \
+    || cp "$PROJECT/data/05.assembly/$Q.LargeContigs.fna" \
+          "$TMPDIR/$Q.LargeContigs.fna"
+  miga-ani "$TMPDIR/$Q.LargeContigs.fna" \
+    "$S_PROJ/data/05.assembly/$S.LargeContigs.fna" \
     "$CORES" "$TMPDIR/$Q.ani.db"
 }
 
 # Calculate the classification-informed AAI/ANI traverse (if not classified)
-ESS="../07.annotation/01.function/01.essential"
-if [[ $(miga project_info -P "$PROJECT" -m type) != "clade" ]] ; then
+ESS="data/07.annotation/01.function/01.essential"
+if [[ $(miga about -P "$S_PROJ" -m type) != "clade" ]] ; then
   # Classify aai-clade (if project type is not clade)
-  CLADES="../10.clades/01.find"
+  CLADES="$S_PROJ/data/10.clades/01.find"
   METRIC="aai"
-  REF_TABLE="02.aai/miga-project.txt.gz"
+  REF_TABLE="$S_PROJ/data/09.distances/02.aai/miga-project.txt.gz"
 else
   # Classify ani-clade (if project type is clade)
-  CLADES="../10.clades/02.ani"
+  CLADES="$S_PROJ/data/10.clades/02.ani"
   METRIC="ani"
-  REF_TABLE="03.ani/miga-project.txt.gz"
+  REF_TABLE="$S_PROJ/data/09.distances/03.ani/miga-project.txt.gz"
 fi
 
 CLASSIF="."
@@ -125,3 +128,22 @@ if [[ -s "${DATASET}.${METRIC}.db" ]] ; then
   "$MIGA/utils/ref-tree.R" "${DATASET}.txt" "$DATASET" "$DATASET"
   rm "$DATASET".tmp[012] "${DATASET}.txt"
 fi
+
+# Test taxonomy
+(
+  trap 'rm "$DATASET.json" "$DATASET.done"' EXIT
+  FLAGS=""
+  [[ "$PROJECT" == "$S_PROJ" ]] || FLAGS="--ref-project"
+  miga date > "$DATASET.done"
+  miga add_result -P "$PROJECT" -D "$DATASET" -r "$SCRIPT"
+  miga tax_test -P "$PROJECT" -D "$DATASET" -t intax \
+    $FLAGS > "$DATASET.intax.txt"
+)
+
+# Transfer taxonomy
+TAX_PVALUE=$(miga about -P "$PROJECT" -m tax_pvalue)
+[[ "$TAX_PVALUE" == "?" ]] && TAX_PVALUE="0.05"
+NEW_TAX=$(tail -n +6 "$DATASET.intax.txt" | head -n -3 \
+  | awk '$3<'$TAX_PVALUE'{print $1":"$2}' | grep -v "?" \
+  | tr "\\n" ' ' | perl -pe 's/ *$//')
+miga tax_set -P "$PROJECT" -D "$DATASET" -s "$NEW_TAX"

data/scripts/distances.bash

@@ -14,9 +14,9 @@ TMPDIR=$(mktemp -d /tmp/MiGA.XXXXXXXXXXXX)
 trap "rm -rf '$TMPDIR'; exit" SIGHUP SIGINT SIGTERM
 
 # Check type of dataset
-NOMULTI=$(miga list_datasets -P "$PROJECT" -D "$DATASET" --no-multi \
+NOMULTI=$(miga ls -P "$PROJECT" -D "$DATASET" --no-multi \
   | wc -l | awk '{print $1}')
-REF=$(miga list_datasets -P "$PROJECT" -D "$DATASET" --ref \
+REF=$(miga ls -P "$PROJECT" -D "$DATASET" --ref \
   | wc -l | awk '{print $1}')
 
 # Call submodules
@@ -26,6 +26,7 @@ if [[ "$NOMULTI" -eq "1" && "$REF" -eq "1" ]] ; then
   # shellcheck source=scripts/_distances_ref_nomulti.bash
   source "$MIGA/scripts/_distances_ref_nomulti.bash"
 elif [[ "$NOMULTI" -eq "1" ]] ; then
+  S_PROJ=$PROJECT
   # shellcheck source=scripts/_distances_noref_nomulti.bash
   source "$MIGA/scripts/_distances_noref_nomulti.bash"
 fi

data/scripts/taxonomy.bash

@@ -0,0 +1,40 @@
+#!/bin/bash
+# Available variables: $PROJECT, $RUNTYPE, $MIGA, $CORES, $DATASET
+set -e
+SCRIPT="taxonomy"
+echo "MiGA: $MIGA"
+echo "Project: $PROJECT"
+# shellcheck source=scripts/miga.bash
+source "$MIGA/scripts/miga.bash" || exit 1
+DIR="$PROJECT/data/09.distances/05.taxonomy"
+[[ -d "$DIR" ]] || mkdir -p "$DIR"
+cd "$DIR"
+
+# Initialize
+miga date > "$DATASET.start"
+
+# Check if there is a reference project
+S_PROJ=$(miga about -P "$PROJECT" -m ref_project)
+
+if [[ "$S_PROJ" != "?" ]] ; then
+
+  # Check type of dataset
+  NOMULTI=$(miga ls -P "$PROJECT" -D "$DATASET" --no-multi \
+    | wc -l | awk '{print $1}')
+
+  if [[ "$NOMULTI" -eq "1" ]] ; then
+    # Call submodules
+    TMPDIR=$(mktemp -d /tmp/MiGA.XXXXXXXXXXXX)
+    trap "rm -rf '$TMPDIR'; exit" SIGHUP SIGINT SIGTERM
+    # shellcheck source=scripts/_distances_functions.bash
+    source "$MIGA/scripts/_distances_functions.bash"
+    # shellcheck source=scripts/_distances_noref_nomulti.bash
+    source "$MIGA/scripts/_distances_noref_nomulti.bash"
+    rm -R "$TMPDIR"
+  fi
+
+fi
+
+# Finalize
+miga date > "$DATASET.done"
+miga add_result -P "$PROJECT" -D "$DATASET" -r "$SCRIPT"

data/utils/arch-ess-genes.rb

@@ -0,0 +1,57 @@
+#!/usr/bin/env ruby
+
+esslog = ARGV.shift
+outlog = ARGV.shift
+l_all = `HMM.essential.rb -l -q`.chomp.split("\n").map{ |i| i.gsub(/\t.*/,"") }
+n_arc = Hash[
+  `HMM.essential.rb -l -q -A`.chomp.split("\n").map{ |i| i.split("\t") }
+]
+l_arc = n_arc.keys
+
+def quality(hsh)
+  q = {}
+  q[:found] = hsh.values.map{ |i| i==0 ? 0 : 1 }.inject(:+)
+  q[:multi] = hsh.values.map{ |i| i==0 ? 0 : i-1 }.inject(:+)
+  q[:cmp] = 100.0*q[:found].to_f/hsh.size
+  q[:cnt] = 100.0*q[:multi].to_f/hsh.size
+  q
+end
+
+cnt_ref = {}
+l_all.each{ |i| cnt_ref[i] = 1 }
+
+at = :header
+File.open(esslog, "r") do |fh|
+  fh.each_line do |ln|
+    v = ln.chomp.gsub(/^! +/, "")
+    if v=="Multiple copies: "
+      at = :multi
+    elsif v=="Missing genes: "
+      at = :missing
+    elsif at==:multi
+      v =~ /^(\d+) (\S+): .*/ or raise "Unexpected multi-copies format: #{v}"
+      cnt_ref[$2] = $1.to_i
+    elsif at==:missing
+      v =~ /^(\S+): .*/ or raise "Unexpected missing format: #{v}"
+      cnt_ref[$1] = 0
+    end
+  end
+end
+
+cnt_arc = {}
+l_arc.each{ |i| cnt_arc[i] = cnt_ref[i] }
+
+q = quality(cnt_arc)
+File.open(outlog, "w") do |ofh|
+  ofh.puts "! Essential genes found: #{q[:found]}/#{cnt_arc.size}."
+  ofh.puts "! Completeness: #{q[:cmp].round(1)}%."
+  ofh.puts "! Contamination: #{q[:cnt].round(1)}%."
+  if q[:multi] > 0
+    ofh.puts "! Multiple copies: "
+    cnt_arc.each{ |k,v| ofh.puts "!   #{v} #{k}: #{n_arc[k]}." if v>1 }
+  end
+  if q[:found] < cnt_arc.size
+    ofh.puts "! Missing genes: "
+    cnt_arc.each{ |k,v| ofh.puts "!   #{k}: #{n_arc[k]}." if v==0 }
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: miga-base
 version: !ruby/object:Gem::Version
-  version: 0.3.0.7
+  version: 0.3.1.0
 platform: ruby
 authors:
 - Luis M. Rodriguez-R
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-08-25 00:00:00.000000000 Z
+date: 2017-09-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client
@@ -160,9 +160,11 @@ files:
 - scripts/ssu.bash
 - scripts/stats.bash
 - scripts/subclades.bash
+- scripts/taxonomy.bash
 - scripts/trimmed_fasta.bash
 - scripts/trimmed_reads.bash
 - utils/adapters.fa
+- utils/arch-ess-genes.rb
 - utils/core-pan-plot.R
 - utils/enveomics/build_enveomics_r.bash
 - utils/enveomics/enveomics.R/data/growth.curves.rda