miga-base 0.2.0.6

data/lib/miga/remote_dataset.rb

@@ -0,0 +1,154 @@
+#
+# @package MiGA
+# @author Luis M. Rodriguez-R <lmrodriguezr at gmail dot com>
+# @license artistic license 2.0
+# @update Dec-07-2015
+#
+
+require "restclient"
+
+module MiGA
+   class RemoteDataset
+      # Class
+      @@UNIVERSE = {
+	 ebi:{
+	    dbs: { embl:{stage: :assembly, format: :fasta} },
+	    url: "http://www.ebi.ac.uk/Tools/dbfetch/dbfetch/%1$s/%2$s/%3$s",
+	    method: :rest
+	 },
+	 ncbi:{
+	    dbs: { nuccore:{stage: :assembly, format: :fasta} },
+	    url: "http://eutils.ncbi.nlm.nih.gov/entrez/eutils/" +
+	       "efetch.fcgi?db=%1$s&id=%2$s&rettype=%3$s&retmode=text",
+	    method: :rest
+	 },
+	 ncbi_map:{
+	    dbs: { assembly:{map_to: :nuccore, format: :text} },
+	    url: "http://eutils.ncbi.nlm.nih.gov/entrez/eutils/" +
+	       "elink.fcgi?dbfrom=%1$s&id=%2$s&db=%3$s - - - - -",
+	    method: :rest,
+	    map_to_universe: :ncbi
+	 }
+      }
+      def self.UNIVERSE ; @@UNIVERSE ; end
+      def self.download(universe, db, ids, format, file=nil)
+	 ids = [ids] unless ids.is_a? Array
+	 case @@UNIVERSE[universe][:method]
+	 when :rest
+	    map_to = @@UNIVERSE[universe][:dbs][db].nil? ? nil :
+	       @@UNIVERSE[universe][:dbs][db][:map_to]
+	    url = sprintf @@UNIVERSE[universe][:url],
+	       db, ids.join(","), format, map_to
+	    response = RestClient::Request.execute(:method=>:get,  :url=>url,
+	       :timeout=>600)
+	    raise "Unable to reach #{universe} client, error code "+
+	       "#{response.code}." unless response.code == 200
+	    doc = response.to_s
+	 else
+	    raise "Unexpected error: Unsupported download method for Universe "+
+	       "#{universe}."
+	 end
+	 unless file.nil?
+	    ofh = File.open(file, "w")
+	    ofh.print doc
+	    ofh.close
+	 end
+	 doc
+      end
+      # Instance
+      attr_reader :universe, :db, :ids
+      def initialize(ids, db, universe)
+	 ids = [ids] unless ids.is_a? Array
+	 @ids = (ids.is_a?(Array) ? ids : [ids])
+	 @db = db.to_sym
+	 @universe = universe.to_sym
+	 raise "Unknown Universe: #{@universe}. Try one of: "+
+	    "#{@@UNIVERSE.keys}" unless @@UNIVERSE.keys.include? @universe
+	 raise "Unknown Database: #{@db}. Try one of: "+
+	    "#{@@UNIVERSE[@universe][:dbs]}" unless
+	    @@UNIVERSE[@universe][:dbs].include? @db
+	 unless @@UNIVERSE[@universe][:dbs][@db][:map_to].nil?
+	    res = RemoteDataset.download
+	 end
+      end
+      def save_to(project, name=nil, is_ref=true, metadata={})
+	 name = ids.join("_").miga_name if name.nil?
+	 project = Project.new(project) if project.is_a? String
+	 raise "Dataset #{name} exists in the project, aborting..." if
+	    Dataset.exist?(project, name)
+	 metadata = get_metadata(metadata)
+	 case @@UNIVERSE[universe][:dbs][db][:stage]
+	 when :assembly
+	    base = project.path + "/data/" + Dataset.RESULT_DIRS[:assembly] +
+	       "/" + name
+	    ofh = File.open("#{base}.start", "w")
+	    ofh.puts Time.now.to_s
+	    ofh.close
+	    download("#{base}.LargeContigs.fna")
+	    File.symlink("#{base}.LargeContigs.fna", "#{base}.AllContigs.fna")
+	    ofh = File.open("#{base}.done", "w")
+	    ofh.puts Time.now.to_s
+	    ofh.close
+	 else
+	    raise "Unexpected error: Unsupported result for database #{db}."
+	 end
+	 dataset = Dataset.new(project, name, is_ref, metadata)
+	 project.add_dataset(dataset.name)
+	 result = dataset.add_result @@UNIVERSE[universe][:dbs][db][:stage]
+	 raise "Empty dataset created: seed result was not added due to "+
+	    "incomplete files." if result.nil?
+	 dataset
+      end
+      def get_metadata(metadata={})
+	 case universe
+	 when :ebi
+	    # Get taxonomy
+	    metadata[:tax] = get_ncbi_taxonomy
+	 when :ncbi
+	    # Get taxonomy
+	    metadata[:tax] = get_ncbi_taxonomy
+	 end
+	 metadata
+      end
+      def download(file)
+	 RemoteDataset.download(universe, db, ids,
+	    @@UNIVERSE[universe][:dbs][db][:format], file)
+      end
+      def get_ncbi_taxid
+	 case universe
+	 when :ebi
+	    doc = RemoteDataset.download(universe, db, ids, :annot).split(/\n/)
+	    ln = doc.grep(/^FT\s+\/db_xref="taxon:/).first
+	    ln = doc.grep(/^OX\s+NCBI_TaxID=/).first if ln.nil?
+	    return nil if ln.nil?
+	    ln.sub!(/.*(?:"taxon:|NCBI_TaxID=)(\d+)["; ].*/, "\\1")
+	    return nil unless ln =~ /^\d+$/
+	    ln
+	 when :ncbi
+	    doc = RemoteDataset.download(universe, db, ids, :gb).split(/\n/)
+	    ln = doc.grep(/^\s+\/db_xref="taxon:/).first
+	    return nil if ln.nil?
+	    ln.sub!(/.*(?:"taxon:)(\d+)["; ].*/, "\\1")
+	    return nil unless ln =~ /^\d+$/
+	    ln
+	 else
+	    raise "I don't know how to extract ncbi_taxids from #{universe}."
+	 end
+      end
+      def get_ncbi_taxonomy
+	 lineage = {}
+	 tax_id = get_ncbi_taxid
+	 loop do
+	    break if tax_id.nil? or %w{0 1}.include? tax_id
+	    doc = RemoteDataset.download(:ebi, :taxonomy, tax_id, "")
+	    name = (doc.scan(/SCIENTIFIC NAME\s+:\s+(.+)/).first||[]).first
+	    rank = (doc.scan(/RANK\s+:\s+(.+)/).first||[]).first
+	    rank = "dataset" if lineage.empty? and rank=="no rank"
+	    lineage[rank] = name unless rank.nil?
+	    tax_id = (doc.scan(/PARENT ID\s+:\s+(.+)/).first||[]).first
+	 end
+	 Taxonomy.new(lineage)
+      end
+   end
+end
+

data/lib/miga/result.rb

@@ -0,0 +1,102 @@
+#
+# @package MiGA
+# @author  Luis M. Rodriguez-R <lmrodriguezr at gmail dot com>
+# @license artistic license 2.0
+# @update  Dec-19-2015
+#
+
+module MiGA
+   class Result
+      # Class
+      def self.exist? path
+	 !!(File.size? path)
+      end
+      def self.load path
+	 return nil unless Result.exist? path
+	 Result.new path
+      end
+      # Instance
+      attr_reader :path, :data, :results
+      def initialize(path)
+	 @path = path
+	 if Result.exist? path
+	    self.load
+	 else
+	    self.create
+	 end
+      end
+      def dir
+	 File.dirname(path)
+      end
+      def file_path(k)
+	 k = k.to_sym
+	 return nil if self[:files].nil? or self[:files][k].nil?
+	 return File.expand_path(self[:files][k], dir) unless
+	    self[:files][k].is_a? Array
+         self[:files][k].map{ |f| File.expand_path(f, dir) }
+      end
+      def [](k) data[k.to_sym] ; end
+      def add_file(k, file)
+         k = k.to_sym
+	 self.data[:files] ||= {}
+	 self.data[:files][k] = file if
+	    File.exist? File.expand_path(file, dir)
+	 self.data[:files][k] = file + ".gz" if
+	    File.exist? File.expand_path(file + ".gz", dir)
+      end
+      def create
+	 @data = {:created=>Time.now.to_s, :results=>[], :stats=>{}, :files=>{}}
+	 self.save
+      end
+      def save
+	 self.data[:updated] = Time.now.to_s
+	 json = JSON.pretty_generate data
+	 ofh = File.open(path, "w")
+	 ofh.puts json
+	 ofh.close
+	 self.load
+      end
+      def load
+	 json = File.read path
+	 @data = JSON.parse(json, {:symbolize_names=>true})
+	 @results = self[:results].map{ |rs| Result.new rs }
+      end
+      def remove!
+	 each_file do |file|
+	    f = File.expand_path(file, dir)
+	    File.unlink_r(f) if File.exist? f
+	 end
+	 %w(.start .done).each do |ext|
+	    f = path.sub(/\.json$/, ext)
+	    File.unlink f if File.exist? f
+	 end
+	 File.unlink path
+      end
+      def each_file(&blk)
+	 self.data[:files] = {} if self.data[:files].nil?
+	 self.data[:files].each do |k,files|
+	    files = [files] unless files.kind_of? Array
+	    files.each do |file|
+	       if blk.arity==1
+		  blk.call file
+	       elsif blk.arity==2
+		  blk.call k, file
+	       else
+		  raise "Wrong number of arguments: #{blk.arity} for one or two"
+	       end
+	    end
+	 end
+      end
+      def add_result(result)
+         self.data[:results] << result.path
+	 self.save
+      end
+      def file_path(file)
+	 f = self.data[:files][file.to_sym]
+	 return nil if f.nil?
+	 return File.dirname(self.path) + "/" + f unless f.is_a?(Array)
+	 f.map{ |i| File.dirname(self.path) + "/" + i }
+      end
+   end
+end
+

data/lib/miga/tax_index.rb

@@ -0,0 +1,70 @@
+
+# @package MiGA
+# @author Luis M. Rodriguez-R <lmrodriguezr at gmail dot com>
+# @license artistic license 2.0
+# @update Jul-09-2015
+#
+
+require 'miga/taxonomy'
+
+module MiGA
+   class TaxIndex
+      # Instance
+      attr_reader :datasets, :root
+      def initialize()
+	 @root = TaxIndexTaxon.new :root, "biota"
+	 @datasets = []
+      end
+      def <<(dataset)
+         return nil if dataset.metadata[:tax].nil?
+	 taxon = @root
+	 Taxonomy.KNOWN_RANKS.each { |rank| taxon = taxon.add_child(rank, dataset.metadata[:tax][rank]) }
+	 taxon.add_dataset dataset
+	 @datasets << dataset
+      end
+      def to_json
+	 JSON.pretty_generate({ root:root.to_hash, datasets:datasets.map{|d| d.name} })
+      end
+      def to_tab(unknown=false) ; root.to_tab(unknown) ; end
+   end
+   class TaxIndexTaxon
+      # Instance
+      attr_reader :rank, :name,:children, :datasets
+      def initialize(rank, name)
+	 @rank = rank.to_sym
+	 @name = (name.nil? ? nil : name.miga_name)
+	 @children = []
+	 @datasets = []
+      end
+      def tax_str ; "#{rank}:#{name.nil? ? "?" : name}" ; end
+      def add_child(rank, name)
+	 rank = rank.to_sym
+	 name = name.miga_name unless name.nil?
+         child = children.find{ |it| it.rank==rank and it.name==name }
+	 if child.nil?
+	    child = TaxIndexTaxon.new(rank, name)
+	    @children << child
+	 end
+	 child
+      end
+      def add_dataset(dataset) @datasets << dataset ; end
+      def datasets_count
+	 datasets.size + children.map{ |it| it.datasets_count }.reduce(0, :+)
+      end
+      def to_json(*a)
+	 { str:tax_str, datasets:datasets.map{|d| d.name}, children:children }.to_json(a)
+      end
+      def to_hash
+         { str:tax_str, datasets:datasets.map{|d| d.name}, children:children.map{ |it| it.to_hash } }
+      end
+      def to_tab(unknown, indent=0)
+	 o = ""
+	 o = (" " * indent) + tax_str + ": " + datasets_count.to_s + "\n" if unknown or not datasets.empty? or not name.nil?
+	 indent += 2
+	 datasets.each{ |ds| o += (" " * indent) + "# " + ds.name + "\n" }
+	 children.each{ |it| o += it.to_tab(unknown, indent) }
+	 o
+      end
+   end
+end
+

data/lib/miga/taxonomy.rb

@@ -0,0 +1,107 @@
+#
+# @package MiGA
+# @author  Luis M. Rodriguez-R <lmrodriguezr at gmail dot com>
+# @license artistic license 2.0
+# @update  Oct-05-2015
+#
+
+module MiGA
+   class Taxonomy
+      # Class
+      # Cannonical ranks
+      @@KNOWN_RANKS = %w{ns d k p c o f g s ssp str ds}.map{|r| r.to_sym}
+      # Synonms for cannonical ranks
+      @@RANK_SYNONYMS = {
+	 "namespace"=>"ns",
+	 "domain"=>"d","superkingdom"=>"d",
+	 "kingdom"=>"k",
+	 "phylum"=>"p",
+	 "class"=>"c",
+	 "order"=>"o",
+	 "family"=>"f",
+	 "genus"=>"g",
+	 "species"=>"s","sp"=>"s",
+	 "subspecies"=>"ssp",
+	 "strain"=>"str","isolate"=>"str","culture"=>"str",
+	 "dataset"=>"ds","organism"=>"ds","genome"=>"ds","specimen"=>"ds"
+      }
+      def self.KNOWN_RANKS() @@KNOWN_RANKS ; end
+      def self.json_create(o) new(o["str"]) ; end
+      def self.normalize_rank(rank)
+	 rank = rank.to_s.downcase
+	 return nil if rank=="no rank"
+	 rank = @@RANK_SYNONYMS[rank] unless @@RANK_SYNONYMS[rank].nil?
+	 rank = rank.to_sym
+	 return nil unless @@KNOWN_RANKS.include? rank
+	 rank
+      end
+      # Instance
+      attr_reader :ranks
+      def initialize(str, ranks=nil)
+	 @ranks = {}
+	 if ranks.nil?
+	    if str.is_a? Array or str.is_a? Hash
+	       self << str
+	    else
+	       (str + " ").scan(/([A-Za-z]+):([^:]*)( )/) do |r,n,s|
+		  self << {r=>n}
+	       end
+	    end
+	 else
+	    ranks = ranks.split(/\s+/) unless ranks.is_a? Array
+	    str = str.split(/\s/) unless str.is_a? Array
+	    raise "Unequal number of ranks (#{ranks.size}) " +
+	       "and names (#{str.size}):#{ranks} => #{str}" unless
+	       ranks.size==str.size
+	    (0 .. str.size).each{ |i| self << "#{ranks[i]}:#{str[i]}" }
+	 end
+      end
+      
+      def <<(value)
+	 if value.is_a? Array
+	    value.each{ |v| self << v }
+	 elsif value.is_a? String
+	    (rank,name) = value.split /:/
+	    self << { rank => name }
+	 elsif value.is_a? Hash
+	    value.each_pair do |rank, name|
+	       next if name.nil? or name == ""
+	       @ranks[ Taxonomy.normalize_rank rank ] = name.gsub(/_/," ")
+	    end
+	 else
+	    raise "Unsupported class '#{value.class.name}'."
+	 end
+      end
+      
+      def [](rank) @ranks[ rank.to_sym ] ; end
+      
+      ### Evaluates if the loaded taxonomy includes `taxon`. It assumes that
+      ### `taxon` only has one informative rank. The evaluation is
+      ### case-insensitive.
+      def is_in? taxon
+	 r = taxon.ranks.keys.first
+	 return false if self[ r ].nil?
+	 self[ r ].downcase == taxon[ r ].downcase
+      end
+      
+      ### Sorted list of ranks, as two-entry arrays
+      def sorted_ranks
+	 @@KNOWN_RANKS.map do |r|
+	    ranks[r].nil? ? nil : [r, ranks[r]]
+	 end.compact
+      end
+
+      def highest; sorted_ranks.first ; end
+
+      def lowest; sorted_ranks.last ; end
+      
+      def to_s
+	 sorted_ranks.map{ |r| "#{r[0].to_s}:#{r[1].gsub(/\s/,"_")}" }.join(" ")
+      end
+      
+      def to_json(*a)
+	 { JSON.create_id => self.class.name, "str" => self.to_s }.to_json(*a)
+      end
+   end
+end
+

data/lib/miga.rb

@@ -0,0 +1,83 @@
+#
+# @package MiGA
+# @author  Luis M. Rodriguez-R <lmrodriguezr at gmail dot com>
+# @license artistic license 2.0
+#
+
+require "date"
+require "json"
+require "fileutils"
+require "miga/project"
+require "miga/taxonomy"
+
+module MiGA
+   VERSION = [0.2, 0, 6]
+   VERSION_NAME = "pochoir"
+   VERSION_DATE = Date.new(2015, 12, 07)
+   CITATION = "Rodriguez-R et al, in preparation. Microbial Genomes Atlas: " +
+      "Standardizing genomic and metagenomic analyses for Archaea and Bacteria."
+   class MiGA
+      @@DEBUG = false
+      @@DEBUG_TRACE = false
+      def self.DEBUG_ON() @@DEBUG=true end
+      def self.DEBUG_OFF() @@DEBUG=false end
+      def self.DEBUG_TRACE_ON
+	 @@DEBUG_TRACE=true
+	 self.DEBUG_ON
+      end
+      def self.DEBUG_TRACE_OFF
+	 @@DEBUG_TRACE=false
+	 self.DEBUG_OFF
+      end
+      def self.DEBUG *args
+	 $stderr.puts(*args) if @@DEBUG
+	 $stderr.puts caller.map{|v| v.gsub(/^/,"    ")}.join("\n") if
+	    @@DEBUG_TRACE
+      end
+      def self.VERSION ; VERSION[0] ; end
+      def self.FULL_VERSION ; VERSION.join(".") ; end
+      def self.LONG_VERSION
+	 "MiGA " + VERSION.join(".") + " - " + VERSION_NAME + " - " +
+	    VERSION_DATE.to_s
+      end
+      def self.VERSION_DATE ; VERSION_DATE ; end
+      def self.CITATION ; CITATION ; end
+   end
+end
+
+class File
+   def self.unlink_r(path)
+      if Dir.exists? path
+	 unless File.symlink? path
+	    Dir.entries(path).reject{|f| f =~ /^\.\.?$/}.each do |f|
+	       File.unlink_r path + "/" + f
+	    end
+	 end
+	 Dir.unlink path
+      elsif File.exists? path
+	 File.unlink path
+      else
+	 raise "Cannot find file: #{path}"
+      end
+   end
+   def self.generic_transfer(old_name, new_name, method)
+      return nil if exist? new_name
+      case method
+      when :symlink
+	 File.symlink(old_name, new_name)
+      when :hardlink
+	 File.link(old_name, new_name)
+      when :copy
+	 FileUtils.cp_r(old_name, new_name)
+      else
+	 raise "Unknown transfer method: #{method}."
+      end
+   end
+end
+
+class String
+   def miga_name ; gsub /[^A-Za-z0-9_]/, "_" ; end
+   def miga_name? ; not(self !~ /^[A-Za-z0-9_]+$/) ; end
+   def unmiga_name ; gsub /_/, " " ; end
+end
+

data/scripts/_distances_noref_nomulti.bash

@@ -0,0 +1,86 @@
+#!/bin/bash
+# Available variables: $PROJECT, $DATASET, $RUNTYPE, $MIGA, $CORES, $TMPDIR,
+# 			$NOMULTI, $REF
+
+# Deal with previous runs (if any)
+exists $DATASET.a[an]i.db && cp $DATASET.a[an]i.db $TMPDIR
+exists $DATASET.a[an]i.9[05] && rm $DATASET.a[an]i.9[05]
+N=0
+function checkpoint_n {
+   let N=$N+1
+   if [[ $N -ge 10 ]] ; then
+      for metric in aai ani ; do
+	 if [[ -s $TMPDIR/$DATASET.$metric.db ]] ; then
+	    echo "select count(*) from $metric;" \
+	       | sqlite3 $TMPDIR/$DATASET.$metric.db \
+	       || exit 1
+	    cp $TMPDIR/$DATASET.$metric.db .
+	 fi
+      done
+      N=0
+   fi
+}
+
+# Find 95%ANI clade(s) with AAI <= 90% / ANI <= 95%
+REFGENOMES=$(cat ../10.clades/01.find/miga-project.ani95-clades \
+   | tail -n +2 | cut -d , -f 1)
+for i in $REFGENOMES ; do
+   AAI=$(aai.rb -1 ../06.cds/$DATASET.faa \
+      -2 ../06.cds/$i.faa -t $CORES -a --lookup-first \
+      -S $TMPDIR/$DATASET.aai.db --name1 $DATASET --name2 $i || echo "")
+   checkpoint_n
+   if [[ $(perl -MPOSIX -e "print ceil $AAI") -ge 90 ]] ; then
+      echo $i >> $DATASET.aai90
+      [[ -e "../05.assembly/$DATASET.LargeContigs.fna" ]] || continue
+      [[ -e "../05.assembly/$i.LargeContigs.fna" ]] || continue
+      ANI=$(ani.rb -1 ../05.assembly/$DATASET.LargeContigs.fna \
+	 -2 ../05.assembly/$i.LargeContigs.fna -t $CORES -a \
+	 --no-save-regions --no-save-rbm --lookup-first \
+	 -S $TMPDIR/$DATASET.ani.db --name1 $DATASET --name2 $i || echo "")
+      checkpoint_n
+      if [[ $(perl -MPOSIX -e "print ceil $ANI") -ge 95 ]] ; then
+	 echo $i >> $DATASET.ani95
+      fi
+   fi
+done
+
+# Classify in-clade (if project type is clade)
+CLADES="../10.clades/02.ani"
+CLASSIF="."
+MAX_ANI=0
+ANI_MED=""
+[[ -e "$DATASET.medoids" ]] && rm "$DATASET.medoids"
+while [[ -e "$CLADES/$CLASSIF/miga-project.1.medoids" ]] ; do
+   for i in $(cat "$CLADES/$CLASSIF/miga-project.1.medoids") ; do
+      ANI=$(ani.rb -1 ../05.assembly/$DATASET.LargeContigs.fna \
+	 -2 ../05.assembly/$i.LargeContigs.fna -t $CORES -a \
+	 --no-save-regions --no-save-rbm --lookup-first \
+	 -S $TMPDIR/$DATASET.ani.db --name1 $DATASET --name2 $i || echo "")
+      checkpoint_n
+      if [[ $(perl -e "print 1 if $ANI > $MAX_ANI") == "1" ]] ; then
+         MAX_ANI=$ANI
+	 ANI_MED=$i
+      fi
+   done
+   echo $i >> "$DATASET.medoids"
+   CLASSIF="$CLASSIF/miga-project.1.subcl-$i"
+done
+echo $CLASSIF > "$DATASET.class"
+
+# Calculate all the ANIs against the lowest subclade (if classified in-clade)
+if [[ "$CLASSIF" != "." ]] ; then
+   if [[ -s "$CLADES/$CLASSIF/miga-project.all" ]] ; then
+      for i in $(cat "$CLADES/$CLASSIF/miga-project.all") ; do
+	 ANI=$(ani.rb -1 ../05.assembly/$DATASET.LargeContigs.fna \
+	    -2 ../05.assembly/$i.LargeContigs.fna -t $CORES -a \
+	    --no-save-regions --no-save-rbm --lookup-first \
+	    -S $TMPDIR/$DATASET.ani.db --name1 $DATASET --name2 $i || echo "")
+	 checkpoint_n
+      done
+   fi
+fi
+
+# Finalize
+mv $TMPDIR/$DATASET.aai.db 02.aai/$DATASET.db
+mv $TMPDIR/$DATASET.ani.db 03.ani/$DATASET.db
+