miga-base 0.2.0.6 → 0.2.0.7

data/lib/miga/tax_index.rb

@@ -1,70 +1,127 @@
-
 # @package MiGA
-# @author Luis M. Rodriguez-R <lmrodriguezr at gmail dot com>
-# @license artistic license 2.0
-# @update Jul-09-2015
-#
-
-require 'miga/taxonomy'
-
-module MiGA
-   class TaxIndex
-      # Instance
-      attr_reader :datasets, :root
-      def initialize()
-	 @root = TaxIndexTaxon.new :root, "biota"
-	 @datasets = []
-      end
-      def <<(dataset)
-         return nil if dataset.metadata[:tax].nil?
-	 taxon = @root
-	 Taxonomy.KNOWN_RANKS.each { |rank| taxon = taxon.add_child(rank, dataset.metadata[:tax][rank]) }
-	 taxon.add_dataset dataset
-	 @datasets << dataset
-      end
-      def to_json
-	 JSON.pretty_generate({ root:root.to_hash, datasets:datasets.map{|d| d.name} })
-      end
-      def to_tab(unknown=false) ; root.to_tab(unknown) ; end
-   end
-   class TaxIndexTaxon
-      # Instance
-      attr_reader :rank, :name,:children, :datasets
-      def initialize(rank, name)
-	 @rank = rank.to_sym
-	 @name = (name.nil? ? nil : name.miga_name)
-	 @children = []
-	 @datasets = []
-      end
-      def tax_str ; "#{rank}:#{name.nil? ? "?" : name}" ; end
-      def add_child(rank, name)
-	 rank = rank.to_sym
-	 name = name.miga_name unless name.nil?
-         child = children.find{ |it| it.rank==rank and it.name==name }
-	 if child.nil?
-	    child = TaxIndexTaxon.new(rank, name)
-	    @children << child
-	 end
-	 child
-      end
-      def add_dataset(dataset) @datasets << dataset ; end
-      def datasets_count
-	 datasets.size + children.map{ |it| it.datasets_count }.reduce(0, :+)
-      end
-      def to_json(*a)
-	 { str:tax_str, datasets:datasets.map{|d| d.name}, children:children }.to_json(a)
-      end
-      def to_hash
-         { str:tax_str, datasets:datasets.map{|d| d.name}, children:children.map{ |it| it.to_hash } }
-      end
-      def to_tab(unknown, indent=0)
-	 o = ""
-	 o = (" " * indent) + tax_str + ": " + datasets_count.to_s + "\n" if unknown or not datasets.empty? or not name.nil?
-	 indent += 2
-	 datasets.each{ |ds| o += (" " * indent) + "# " + ds.name + "\n" }
-	 children.each{ |it| o += it.to_tab(unknown, indent) }
-	 o
-      end
-   end
+# @license Artistic-2.0
+
+require "miga/taxonomy"
+
+##
+# Indexing methods based on taxonomy.
+class MiGA::TaxIndex < MiGA::MiGA
+  
+  # Instance-level
+  
+  ##
+  # Datasets in the index.
+  attr_reader :datasets
+  # Taxonomy root.
+  attr_reader :root
+
+  ##
+  # Initialize an empty MiGA::TaxIndex
+  def initialize
+    @root = MiGA::TaxIndexTaxon.new :root, "biota"
+    @datasets = []
+  end
+
+  ##
+  # Index +dataset+, a MiGA::Dataset object.
+  def <<(dataset)
+    return nil if dataset.metadata[:tax].nil?
+    taxon = @root
+    MiGA::Taxonomy.KNOWN_RANKS.each do |rank|
+      taxon = taxon.add_child(rank, dataset.metadata[:tax][rank])
+    end
+    taxon.add_dataset dataset
+    @datasets << dataset
+  end
+
+  ##
+  # Generate JSON String for the index.
+  def to_json
+    JSON.pretty_generate({ root:root.to_hash,
+      datasets:datasets.map{ |d| d.name } })
+  end
+
+  ##
+  # Generate tabular String for the index.
+  def to_tab(unknown=false) ; root.to_tab(unknown) ; end
 end
 
+##
+# Helper class for MiGA::TaxIndex.
+class MiGA::TaxIndexTaxon < MiGA::MiGA
+  
+  # Instance-level
+  
+  ##
+  # Rank of the taxon.
+  attr_reader :rank
+  # Name of the taxon.
+  attr_reader :name
+  # Children of the taxon.
+  attr_reader :children
+  # Datasets directly classified at the taxon (not at children).
+  attr_reader :datasets
+
+  ##
+  # Initalize taxon at +rank+ with +name+.
+  def initialize(rank, name)
+    @rank = rank.to_sym
+    @name = (name.nil? ? nil : name.miga_name)
+    @children = []
+    @datasets = []
+  end
+
+  ##
+  # String representation of the taxon.
+  def tax_str ; "#{rank}:#{name.nil? ? "?" : name}" ; end
+
+  ##
+  # Add child at +rank+ with +name+.
+  def add_child(rank, name)
+    rank = rank.to_sym
+    name = name.miga_name unless name.nil?
+    child = children.find{ |it| it.rank==rank and it.name==name }
+    if child.nil?
+      child = MiGA::TaxIndexTaxon.new(rank, name)
+      @children << child
+    end
+    child
+  end
+
+  ##
+  # Add dataset at the current taxon (not children).
+  def add_dataset(dataset) @datasets << dataset ; end
+
+  ##
+  # Get the number of datasets in the taxon (including children).
+  def datasets_count
+    datasets.size + children.map{ |it| it.datasets_count }.reduce(0, :+)
+  end
+
+  ##
+  # JSON String of the taxon.
+  def to_json(*a)
+    { str:tax_str, datasets:datasets.map{|d| d.name},
+      children:children }.to_json(a)
+  end
+
+  ##
+  # Hash representation of the taxon.
+  def to_hash
+    { str:tax_str, datasets:datasets.map{|d| d.name},
+      children:children.map{ |it| it.to_hash } }
+  end
+
+  ##
+  # Tabular String of the taxon.
+  def to_tab(unknown, indent=0)
+    o = ""
+    o = (" " * indent) + tax_str + ": " + datasets_count.to_s + "\n" if
+      unknown or not datasets.empty? or not name.nil?
+    indent += 2
+    datasets.each{ |ds| o += (" " * indent) + "# " + ds.name + "\n" }
+    children.each{ |it| o += it.to_tab(unknown, indent) }
+    o
+  end
+
+end

data/lib/miga/taxonomy.rb

@@ -1,107 +1,136 @@
-#
 # @package MiGA
-# @author  Luis M. Rodriguez-R <lmrodriguezr at gmail dot com>
-# @license artistic license 2.0
-# @update  Oct-05-2015
-#
+# @license Artistic-2.0
 
-module MiGA
-   class Taxonomy
-      # Class
-      # Cannonical ranks
-      @@KNOWN_RANKS = %w{ns d k p c o f g s ssp str ds}.map{|r| r.to_sym}
-      # Synonms for cannonical ranks
-      @@RANK_SYNONYMS = {
-	 "namespace"=>"ns",
-	 "domain"=>"d","superkingdom"=>"d",
-	 "kingdom"=>"k",
-	 "phylum"=>"p",
-	 "class"=>"c",
-	 "order"=>"o",
-	 "family"=>"f",
-	 "genus"=>"g",
-	 "species"=>"s","sp"=>"s",
-	 "subspecies"=>"ssp",
-	 "strain"=>"str","isolate"=>"str","culture"=>"str",
-	 "dataset"=>"ds","organism"=>"ds","genome"=>"ds","specimen"=>"ds"
-      }
-      def self.KNOWN_RANKS() @@KNOWN_RANKS ; end
-      def self.json_create(o) new(o["str"]) ; end
-      def self.normalize_rank(rank)
-	 rank = rank.to_s.downcase
-	 return nil if rank=="no rank"
-	 rank = @@RANK_SYNONYMS[rank] unless @@RANK_SYNONYMS[rank].nil?
-	 rank = rank.to_sym
-	 return nil unless @@KNOWN_RANKS.include? rank
-	 rank
-      end
-      # Instance
-      attr_reader :ranks
-      def initialize(str, ranks=nil)
-	 @ranks = {}
-	 if ranks.nil?
-	    if str.is_a? Array or str.is_a? Hash
-	       self << str
-	    else
-	       (str + " ").scan(/([A-Za-z]+):([^:]*)( )/) do |r,n,s|
-		  self << {r=>n}
-	       end
-	    end
-	 else
-	    ranks = ranks.split(/\s+/) unless ranks.is_a? Array
-	    str = str.split(/\s/) unless str.is_a? Array
-	    raise "Unequal number of ranks (#{ranks.size}) " +
-	       "and names (#{str.size}):#{ranks} => #{str}" unless
-	       ranks.size==str.size
-	    (0 .. str.size).each{ |i| self << "#{ranks[i]}:#{str[i]}" }
-	 end
-      end
-      
-      def <<(value)
-	 if value.is_a? Array
-	    value.each{ |v| self << v }
-	 elsif value.is_a? String
-	    (rank,name) = value.split /:/
-	    self << { rank => name }
-	 elsif value.is_a? Hash
-	    value.each_pair do |rank, name|
-	       next if name.nil? or name == ""
-	       @ranks[ Taxonomy.normalize_rank rank ] = name.gsub(/_/," ")
-	    end
-	 else
-	    raise "Unsupported class '#{value.class.name}'."
-	 end
-      end
-      
-      def [](rank) @ranks[ rank.to_sym ] ; end
-      
-      ### Evaluates if the loaded taxonomy includes `taxon`. It assumes that
-      ### `taxon` only has one informative rank. The evaluation is
-      ### case-insensitive.
-      def is_in? taxon
-	 r = taxon.ranks.keys.first
-	 return false if self[ r ].nil?
-	 self[ r ].downcase == taxon[ r ].downcase
-      end
-      
-      ### Sorted list of ranks, as two-entry arrays
-      def sorted_ranks
-	 @@KNOWN_RANKS.map do |r|
-	    ranks[r].nil? ? nil : [r, ranks[r]]
-	 end.compact
-      end
+##
+# Taxonomic classifications in MiGA.
+class MiGA::Taxonomy < MiGA::MiGA
+  # Class-level
+
+  ##
+  # Cannonical ranks.
+  def self.KNOWN_RANKS() @@KNOWN_RANKS ; end
+  @@KNOWN_RANKS = %w{ns d k p c o f g s ssp str ds}.map{|r| r.to_sym}
+
+  ##
+  # Synonms for cannonical ranks.
+  @@RANK_SYNONYMS = {
+    "namespace"=>"ns",
+    "domain"=>"d","superkingdom"=>"d",
+    "kingdom"=>"k",
+    "phylum"=>"p",
+    "class"=>"c",
+    "order"=>"o",
+    "family"=>"f",
+    "genus"=>"g",
+    "species"=>"s","sp"=>"s",
+    "subspecies"=>"ssp",
+    "strain"=>"str","isolate"=>"str","culture"=>"str",
+    "dataset"=>"ds","organism"=>"ds","genome"=>"ds","specimen"=>"ds"
+  }
 
-      def highest; sorted_ranks.first ; end
+  ##
+  # Initialize from JSON-derived Hash +o+.
+  def self.json_create(o) new(o["str"]) ; end
 
-      def lowest; sorted_ranks.last ; end
-      
-      def to_s
-	 sorted_ranks.map{ |r| "#{r[0].to_s}:#{r[1].gsub(/\s/,"_")}" }.join(" ")
+  ##
+  # Returns cannonical rank (Symbol) for the +rank+ String.
+  def self.normalize_rank(rank)
+    rank = rank.to_s.downcase
+    return nil if rank=="no rank"
+    rank = @@RANK_SYNONYMS[rank] unless @@RANK_SYNONYMS[rank].nil?
+    rank = rank.to_sym
+    return nil unless @@KNOWN_RANKS.include? rank
+    rank
+  end
+
+  # Instance-level
+
+  ##
+  # Taxonomic hierarchy Hash.
+  attr_reader :ranks
+
+  ##
+  # Create MiGA::Taxonomy from String or Array +str+. The string is a series of
+  # space-delimited entries, the array is a vector of entries. Each entry can be
+  # either a rank:value pair (if +ranks+ is nil), or just values in the same
+  # order as ther ranks in +ranks+. Alternatively, +str+ as a Hash with rank =>
+  # value pairs is also supported.
+  def initialize(str, ranks=nil)
+    @ranks = {}
+    if ranks.nil?
+      case str when Array, Hash
+        self << str
+      else
+        "#{str} ".scan(/([A-Za-z]+):([^:]*)( )/){ |r,n,_| self << {r=>n} }
       end
-      
-      def to_json(*a)
-	 { JSON.create_id => self.class.name, "str" => self.to_s }.to_json(*a)
+    else
+      ranks = ranks.split(/\s+/) unless ranks.is_a? Array
+      str = str.split(/\s/) unless str.is_a? Array
+      raise "Unequal number of ranks (#{ranks.size}) " +
+        "and names (#{str.size}):#{ranks} => #{str}" unless
+        ranks.size==str.size
+      (0 .. str.size).each{ |i| self << "#{ranks[i]}:#{str[i]}" }
+    end
+  end
+  
+  ##
+  # Add +value+ to the hierarchy, that can be an Array, a String, or a Hash, as
+  # described in #initialize.
+  def <<(value)
+    if value.is_a? Array
+      value.each{ |v| self << v }
+    elsif value.is_a? String
+      (rank, name) = value.split(/:/)
+      self << { rank => name }
+    elsif value.is_a? Hash
+      value.each_pair do |rank_i, name_i|
+        next if name_i.nil? or name_i == ""
+        @ranks[ Taxonomy.normalize_rank rank_i ] = name_i.tr("_"," ")
       end
-   end
-end
+    else
+      raise "Unsupported class: #{value.class.name}."
+    end
+  end
+  
+  ##
+  # Get +rank+ value.
+  def [](rank) @ranks[ rank.to_sym ] ; end
+  
+  ##
+  # Evaluates if the loaded taxonomy includes +taxon+. It assumes that +taxon+
+  # only has one informative rank. The evaluation is case-insensitive.
+  def is_in? taxon
+    r = taxon.ranks.keys.first
+    return false if self[ r ].nil?
+    self[ r ].downcase == taxon[ r ].downcase
+  end
+
+  ##
+  # Sorted list of ranks, as an Array of two-entry Arrays (rank and value).
+  def sorted_ranks
+    @@KNOWN_RANKS.map do |r|
+      ranks[r].nil? ? nil : [r, ranks[r]]
+    end.compact
+  end
+  
+  ##
+  # Get the most general rank as a two-entry Array (rank and value).
+  def highest; sorted_ranks.first ; end
 
+  ##
+  # Get the most specific rank as a two-entry Array (rank and value).
+  def lowest; sorted_ranks.last ; end
+  
+  ##
+  # Generate cannonical String for the taxonomy.
+  def to_s
+    sorted_ranks.map{ |r| "#{r[0]}:#{r[1].gsub(/\s/,"_")}" }.join(" ")
+  end
+  
+  ##
+  # Generate JSON-formated String representing the taxonomy.
+  def to_json(*a)
+    { JSON.create_id => self.class.name, "str" => self.to_s }.to_json(*a)
+  end
+  
+end

data/lib/miga/version.rb

@@ -0,0 +1,57 @@
+
+require "date"
+
+##
+# High-level minimal requirements for the MiGA::MiGA class.
+module MiGA
+  
+  ##
+  # Current version of MiGA. An Array with three values:
+  # - Float representing the major.minor version.
+  # - Integer representing gem releases of the current version.
+  # - Integer representing minor changes that require new version number.
+  VERSION = [0.2, 0, 7]
+  
+  ##
+  # Nickname for the current major.minor version.
+  VERSION_NAME = "pochoir"
+  
+  ##
+  # Date of the current gem release.
+  VERSION_DATE = Date.new(2016, 04, 13)
+  
+  ##
+  # Reference of MiGA.
+  CITATION = "Rodriguez-R et al, in preparation. Microbial Genomes Atlas: " +
+    "Standardizing genomic and metagenomic analyses for Archaea and Bacteria."
+
+end
+
+class MiGA::MiGA
+  
+  include MiGA
+  
+  ##
+  # Major.minor version as Float.
+  def self.VERSION ; VERSION[0] ; end
+
+  ##
+  # Complete version as string.
+  def self.FULL_VERSION ; VERSION.join(".") ; end
+
+  ##
+  # Complete version with nickname and date as string.
+  def self.LONG_VERSION
+    "MiGA " + VERSION.join(".") + " - " + VERSION_NAME + " - " +
+      VERSION_DATE.to_s
+  end
+
+  ##
+  # Date of the current gem release.
+  def self.VERSION_DATE ; VERSION_DATE ; end
+
+  ##
+  # Reference of MiGA.
+  def self.CITATION ; CITATION ; end
+
+end

data/lib/miga.rb

@@ -1,83 +1,8 @@
-#
 # @package MiGA
-# @author  Luis M. Rodriguez-R <lmrodriguezr at gmail dot com>
-# @license artistic license 2.0
-#
+# @license Artistic-2.0
 
-require "date"
 require "json"
 require "fileutils"
+require "miga/common"
 require "miga/project"
 require "miga/taxonomy"
-
-module MiGA
-   VERSION = [0.2, 0, 6]
-   VERSION_NAME = "pochoir"
-   VERSION_DATE = Date.new(2015, 12, 07)
-   CITATION = "Rodriguez-R et al, in preparation. Microbial Genomes Atlas: " +
-      "Standardizing genomic and metagenomic analyses for Archaea and Bacteria."
-   class MiGA
-      @@DEBUG = false
-      @@DEBUG_TRACE = false
-      def self.DEBUG_ON() @@DEBUG=true end
-      def self.DEBUG_OFF() @@DEBUG=false end
-      def self.DEBUG_TRACE_ON
-	 @@DEBUG_TRACE=true
-	 self.DEBUG_ON
-      end
-      def self.DEBUG_TRACE_OFF
-	 @@DEBUG_TRACE=false
-	 self.DEBUG_OFF
-      end
-      def self.DEBUG *args
-	 $stderr.puts(*args) if @@DEBUG
-	 $stderr.puts caller.map{|v| v.gsub(/^/,"    ")}.join("\n") if
-	    @@DEBUG_TRACE
-      end
-      def self.VERSION ; VERSION[0] ; end
-      def self.FULL_VERSION ; VERSION.join(".") ; end
-      def self.LONG_VERSION
-	 "MiGA " + VERSION.join(".") + " - " + VERSION_NAME + " - " +
-	    VERSION_DATE.to_s
-      end
-      def self.VERSION_DATE ; VERSION_DATE ; end
-      def self.CITATION ; CITATION ; end
-   end
-end
-
-class File
-   def self.unlink_r(path)
-      if Dir.exists? path
-	 unless File.symlink? path
-	    Dir.entries(path).reject{|f| f =~ /^\.\.?$/}.each do |f|
-	       File.unlink_r path + "/" + f
-	    end
-	 end
-	 Dir.unlink path
-      elsif File.exists? path
-	 File.unlink path
-      else
-	 raise "Cannot find file: #{path}"
-      end
-   end
-   def self.generic_transfer(old_name, new_name, method)
-      return nil if exist? new_name
-      case method
-      when :symlink
-	 File.symlink(old_name, new_name)
-      when :hardlink
-	 File.link(old_name, new_name)
-      when :copy
-	 FileUtils.cp_r(old_name, new_name)
-      else
-	 raise "Unknown transfer method: #{method}."
-      end
-   end
-end
-
-class String
-   def miga_name ; gsub /[^A-Za-z0-9_]/, "_" ; end
-   def miga_name? ; not(self !~ /^[A-Za-z0-9_]+$/) ; end
-   def unmiga_name ; gsub /_/, " " ; end
-end
-

data/scripts/_distances_noref_nomulti.bash

@@ -2,6 +2,8 @@
 # Available variables: $PROJECT, $DATASET, $RUNTYPE, $MIGA, $CORES, $TMPDIR,
 # 			$NOMULTI, $REF
 
+set -e
+
 # Deal with previous runs (if any)
 exists $DATASET.a[an]i.db && cp $DATASET.a[an]i.db $TMPDIR
 exists $DATASET.a[an]i.9[05] && rm $DATASET.a[an]i.9[05]

data/scripts/_distances_ref_nomulti.bash

@@ -2,6 +2,8 @@
 # Available variables: $PROJECT, $DATASET, $RUNTYPE, $MIGA, $CORES, $TMPDIR,
 # 			$NOMULTI, $REF
 
+set -e
+
 function checkpoint_n {
    if [[ $N -eq 10 ]] ; then
       for t in 01.haai 02.aai 03.ani ; do

data/scripts/aai_distances.bash

@@ -1,5 +1,6 @@
 #!/bin/bash
 # Available variables: $PROJECT, $RUNTYPE, $MIGA, $CORES
+set -e
 echo "MiGA: $MIGA"
 echo "Project: $PROJECT"
 source "$MIGA/scripts/miga.bash" || exit 1

data/scripts/ani_distances.bash

@@ -1,5 +1,6 @@
 #!/bin/bash
 # Available variables: $PROJECT, $RUNTYPE, $MIGA, $CORES
+set -e
 echo "MiGA: $MIGA"
 echo "Project: $PROJECT"
 source "$MIGA/scripts/miga.bash" || exit 1

data/scripts/assembly.bash

@@ -1,5 +1,6 @@
 #!/bin/bash
 # Available variables: $PROJECT, $RUNTYPE, $MIGA, $CORES
+set -e
 echo "MiGA: $MIGA"
 echo "Project: $PROJECT"
 source "$MIGA/scripts/miga.bash" || exit 1

data/scripts/cds.bash

@@ -1,5 +1,6 @@
 #!/bin/bash
 # Available variables: $PROJECT, $RUNTYPE, $MIGA, $CORES
+set -e
 echo "MiGA: $MIGA"
 echo "Project: $PROJECT"
 source "$MIGA/scripts/miga.bash" || exit 1

data/scripts/clade_finding.bash

@@ -1,5 +1,6 @@
 #!/bin/bash
 # Available variables: $PROJECT, $RUNTYPE, $MIGA, $CORES
+set -e
 echo "MiGA: $MIGA"
 echo "Project: $PROJECT"
 source "$MIGA/scripts/miga.bash" || exit 1
@@ -11,16 +12,31 @@ date "+%Y-%m-%d %H:%M:%S %z" > "miga-project.start"
 # Markov-cluster genomes by ANI
 gunzip -c ../../09.distances/03.ani/miga-project.txt.gz | tail -n+2 \
    | awk -F"\\t" '{print $2"'"\\t"'"$3"'"\\t"'"$4}' > genome-genome.aai90.rbm
-ogs.mcl.rb -d . -o miga-project.ani-clades -t "$CORES" -i \
+ogs.mcl.rb -d . -o miga-project.aai90-clades -t "$CORES" -i \
    -f "(\\S+)-(\\S+)\\.aai90\\.rbm"
+rm genome-genome.aai90.rbm
 cat genome-genome.aai90.rbm | awk -F"\\t" '$3>=95' > genome-genome.ani95.rbm
 ogs.mcl.rb -d . -o miga-project.ani95-clades -t "$CORES" -b \
    -f "(\\S+)-(\\S+)\\.ani95\\.rbm"
+rm genome-genome.ani95.rbm
 
 # Propose clade projects
 cat miga-project.ani95-clades | tail -n +2 | tr "," "\\t" | awk 'NF >= 5' \
    > miga-project.proposed-clades
 
+# Run R code
+echo "
+source('$MIGA/utils/subclades.R');
+subclades('../../09.distances/02.aai/miga-project.txt.gz',
+   'miga-project', $CORES);
+" | R --vanilla
+mv miga-project.ani.nwk miga-project.aai.nwk
+
+# Compile
+ruby "$MIGA/utils/subclades-compile.rb" . \
+   >  miga-project.class.tsv \
+   2> miga-project.class.nwk
+
 # Finalize
 date "+%Y-%m-%d %H:%M:%S %z" > "miga-project.done"
 miga add_result -P "$PROJECT" -r clade_finding

data/scripts/distances.bash

@@ -1,5 +1,6 @@
 #!/bin/bash
 # Available variables: $PROJECT, $DATASET, $RUNTYPE, $MIGA, $CORES
+set -e
 echo "MiGA: $MIGA"
 echo "Project: $PROJECT"
 source "$MIGA/scripts/miga.bash" || exit 1

data/scripts/essential_genes.bash

@@ -1,5 +1,6 @@
 #!/bin/bash
 # Available variables: $PROJECT, $RUNTYPE, $MIGA, $CORES
+set -e
 echo "MiGA: $MIGA"
 echo "Project: $PROJECT"
 source "$MIGA/scripts/miga.bash" || exit 1