miga-base 0.2.0.6 → 0.2.0.7

data/lib/miga/remote_dataset.rb

@@ -1,154 +1,206 @@
-#
 # @package MiGA
-# @author Luis M. Rodriguez-R <lmrodriguezr at gmail dot com>
-# @license artistic license 2.0
-# @update Dec-07-2015
-#
+# @license Artistic-2.0
 
 require "restclient"
+require "open-uri"
 
-module MiGA
-   class RemoteDataset
-      # Class
-      @@UNIVERSE = {
-	 ebi:{
-	    dbs: { embl:{stage: :assembly, format: :fasta} },
-	    url: "http://www.ebi.ac.uk/Tools/dbfetch/dbfetch/%1$s/%2$s/%3$s",
-	    method: :rest
-	 },
-	 ncbi:{
-	    dbs: { nuccore:{stage: :assembly, format: :fasta} },
-	    url: "http://eutils.ncbi.nlm.nih.gov/entrez/eutils/" +
-	       "efetch.fcgi?db=%1$s&id=%2$s&rettype=%3$s&retmode=text",
-	    method: :rest
-	 },
-	 ncbi_map:{
-	    dbs: { assembly:{map_to: :nuccore, format: :text} },
-	    url: "http://eutils.ncbi.nlm.nih.gov/entrez/eutils/" +
-	       "elink.fcgi?dbfrom=%1$s&id=%2$s&db=%3$s - - - - -",
-	    method: :rest,
-	    map_to_universe: :ncbi
-	 }
-      }
-      def self.UNIVERSE ; @@UNIVERSE ; end
-      def self.download(universe, db, ids, format, file=nil)
-	 ids = [ids] unless ids.is_a? Array
-	 case @@UNIVERSE[universe][:method]
-	 when :rest
-	    map_to = @@UNIVERSE[universe][:dbs][db].nil? ? nil :
-	       @@UNIVERSE[universe][:dbs][db][:map_to]
-	    url = sprintf @@UNIVERSE[universe][:url],
-	       db, ids.join(","), format, map_to
-	    response = RestClient::Request.execute(:method=>:get,  :url=>url,
-	       :timeout=>600)
-	    raise "Unable to reach #{universe} client, error code "+
-	       "#{response.code}." unless response.code == 200
-	    doc = response.to_s
-	 else
-	    raise "Unexpected error: Unsupported download method for Universe "+
-	       "#{universe}."
-	 end
-	 unless file.nil?
-	    ofh = File.open(file, "w")
-	    ofh.print doc
-	    ofh.close
-	 end
-	 doc
-      end
-      # Instance
-      attr_reader :universe, :db, :ids
-      def initialize(ids, db, universe)
-	 ids = [ids] unless ids.is_a? Array
-	 @ids = (ids.is_a?(Array) ? ids : [ids])
-	 @db = db.to_sym
-	 @universe = universe.to_sym
-	 raise "Unknown Universe: #{@universe}. Try one of: "+
-	    "#{@@UNIVERSE.keys}" unless @@UNIVERSE.keys.include? @universe
-	 raise "Unknown Database: #{@db}. Try one of: "+
-	    "#{@@UNIVERSE[@universe][:dbs]}" unless
-	    @@UNIVERSE[@universe][:dbs].include? @db
-	 unless @@UNIVERSE[@universe][:dbs][@db][:map_to].nil?
-	    res = RemoteDataset.download
-	 end
-      end
-      def save_to(project, name=nil, is_ref=true, metadata={})
-	 name = ids.join("_").miga_name if name.nil?
-	 project = Project.new(project) if project.is_a? String
-	 raise "Dataset #{name} exists in the project, aborting..." if
-	    Dataset.exist?(project, name)
-	 metadata = get_metadata(metadata)
-	 case @@UNIVERSE[universe][:dbs][db][:stage]
-	 when :assembly
-	    base = project.path + "/data/" + Dataset.RESULT_DIRS[:assembly] +
-	       "/" + name
-	    ofh = File.open("#{base}.start", "w")
-	    ofh.puts Time.now.to_s
-	    ofh.close
-	    download("#{base}.LargeContigs.fna")
-	    File.symlink("#{base}.LargeContigs.fna", "#{base}.AllContigs.fna")
-	    ofh = File.open("#{base}.done", "w")
-	    ofh.puts Time.now.to_s
-	    ofh.close
-	 else
-	    raise "Unexpected error: Unsupported result for database #{db}."
-	 end
-	 dataset = Dataset.new(project, name, is_ref, metadata)
-	 project.add_dataset(dataset.name)
-	 result = dataset.add_result @@UNIVERSE[universe][:dbs][db][:stage]
-	 raise "Empty dataset created: seed result was not added due to "+
-	    "incomplete files." if result.nil?
-	 dataset
-      end
-      def get_metadata(metadata={})
-	 case universe
-	 when :ebi
-	    # Get taxonomy
-	    metadata[:tax] = get_ncbi_taxonomy
-	 when :ncbi
-	    # Get taxonomy
-	    metadata[:tax] = get_ncbi_taxonomy
-	 end
-	 metadata
-      end
-      def download(file)
-	 RemoteDataset.download(universe, db, ids,
-	    @@UNIVERSE[universe][:dbs][db][:format], file)
-      end
-      def get_ncbi_taxid
-	 case universe
-	 when :ebi
-	    doc = RemoteDataset.download(universe, db, ids, :annot).split(/\n/)
-	    ln = doc.grep(/^FT\s+\/db_xref="taxon:/).first
-	    ln = doc.grep(/^OX\s+NCBI_TaxID=/).first if ln.nil?
-	    return nil if ln.nil?
-	    ln.sub!(/.*(?:"taxon:|NCBI_TaxID=)(\d+)["; ].*/, "\\1")
-	    return nil unless ln =~ /^\d+$/
-	    ln
-	 when :ncbi
-	    doc = RemoteDataset.download(universe, db, ids, :gb).split(/\n/)
-	    ln = doc.grep(/^\s+\/db_xref="taxon:/).first
-	    return nil if ln.nil?
-	    ln.sub!(/.*(?:"taxon:)(\d+)["; ].*/, "\\1")
-	    return nil unless ln =~ /^\d+$/
-	    ln
-	 else
-	    raise "I don't know how to extract ncbi_taxids from #{universe}."
-	 end
-      end
-      def get_ncbi_taxonomy
-	 lineage = {}
-	 tax_id = get_ncbi_taxid
-	 loop do
-	    break if tax_id.nil? or %w{0 1}.include? tax_id
-	    doc = RemoteDataset.download(:ebi, :taxonomy, tax_id, "")
-	    name = (doc.scan(/SCIENTIFIC NAME\s+:\s+(.+)/).first||[]).first
-	    rank = (doc.scan(/RANK\s+:\s+(.+)/).first||[]).first
-	    rank = "dataset" if lineage.empty? and rank=="no rank"
-	    lineage[rank] = name unless rank.nil?
-	    tax_id = (doc.scan(/PARENT ID\s+:\s+(.+)/).first||[]).first
-	 end
-	 Taxonomy.new(lineage)
+##
+# MiGA representation of datasets with data in remote locations.
+class MiGA::RemoteDataset < MiGA::MiGA
+  # Class-level
+
+  ##
+  # Structure of the different database Universes or containers. The structure
+  # is a Hash with universe names as keys as Symbol and values being a Hash with
+  # supported keys as Symbol:
+  # - +:dbs+ => Hash with keys being the database name and the values a Hash of
+  #   properties such as +stage+, +format+, and +map_to+.
+  # - +url+ => Pattern of the URL where the data can be obtained, where +%1$s+
+  #   is the name of the database, +%2$s+ is the IDs, and +%3$s+ is format.
+  # - +method+ => Method used to query the URL. Only +:rest+ is currently
+  #   supported.
+  # - +map_to_universe+ => Universe where results map to. Currently unsupported.
+  def self.UNIVERSE ; @@UNIVERSE ; end
+  @@UNIVERSE = {
+    web:{
+      dbs: {
+        assembly:{stage: :assembly, format: :fasta},
+        assembly_gz:{stage: :assembly, format: :fasta_gz}
+      },
+      url: "%2$s",
+      method: :net
+    },
+    ebi:{
+      dbs: { embl:{stage: :assembly, format: :fasta} },
+      url: "http://www.ebi.ac.uk/Tools/dbfetch/dbfetch/%1$s/%2$s/%3$s",
+      method: :rest
+    },
+    ncbi:{
+      dbs: { nuccore:{stage: :assembly, format: :fasta} },
+      url: "http://eutils.ncbi.nlm.nih.gov/entrez/eutils/" +
+        "efetch.fcgi?db=%1$s&id=%2$s&rettype=%3$s&retmode=text",
+      method: :rest
+    },
+    ncbi_map:{
+      dbs: { assembly:{map_to: :nuccore, format: :text} },
+      url: "http://eutils.ncbi.nlm.nih.gov/entrez/eutils/" +
+        # FIXME ncbi_map is intended to do internal NCBI mapping between
+        # databases.
+        "elink.fcgi?dbfrom=%1$s&id=%2$s&db=%3$s - - - - -",
+      method: :rest,
+      map_to_universe: :ncbi
+    }
+  }
+
+  ##
+  # Download data from the +universe+ in the database +db+ with IDs +ids+ and
+  # in +format+. If passed, it saves the result in +file+. Returns String.
+  def self.download(universe, db, ids, format, file=nil)
+    ids = [ids] unless ids.is_a? Array
+    case @@UNIVERSE[universe][:method]
+    when :rest
+      map_to = @@UNIVERSE[universe][:dbs][db].nil? ? nil :
+        @@UNIVERSE[universe][:dbs][db][:map_to]
+      url = sprintf @@UNIVERSE[universe][:url],
+        db, ids.join(","), format, map_to
+      response = RestClient::Request.execute(:method=>:get,  :url=>url,
+        :timeout=>600)
+      raise "Unable to reach #{universe} client, error code "+
+        "#{response.code}." unless response.code == 200
+      doc = response.to_s
+    when :net
+      url = sprintf @@UNIVERSE[universe][:url],
+        db, ids.join(","), format, map_to
+      doc = ""
+      open(url) { |f| doc = f.read }
+    end
+    unless file.nil?
+      ofh = File.open(file, "w")
+      ofh.print doc
+      ofh.close
+    end
+    doc
+  end
+
+  # Instance-level
+
+  ##
+  # Universe of the dataset.
+  attr_reader :universe
+  # Database storing the dataset.
+  attr_reader :db
+  # IDs of the entries composing the dataset.
+  attr_reader :ids
+
+  ##
+  # Initialize MiGA::RemoteDataset with +ids+ in database +db+ from +universe+.
+  def initialize(ids, db, universe)
+    ids = [ids] unless ids.is_a? Array
+    @ids = (ids.is_a?(Array) ? ids : [ids])
+    @db = db.to_sym
+    @universe = universe.to_sym
+    raise "Unknown Universe: #{@universe}. Try one of: "+
+      "#{@@UNIVERSE.keys}" unless @@UNIVERSE.keys.include? @universe
+    raise "Unknown Database: #{@db}. Try one of: "+
+      "#{@@UNIVERSE[@universe][:dbs]}" unless
+      @@UNIVERSE[@universe][:dbs].include? @db
+    # FIXME Part of the +map_to+ support:
+    #unless @@UNIVERSE[@universe][:dbs][@db][:map_to].nil?
+    #  MiGA::RemoteDataset.download
+    #end
+  end
+
+  ##
+  # Save dataset to the MiGA::Project +project+ identified with +name+. +is_ref+
+  # indicates if it should be a reference dataset, and contains +metadata+.
+  def save_to(project, name=nil, is_ref=true, metadata={})
+    name = ids.join("_").miga_name if name.nil?
+    project = MiGA::Project.new(project) if project.is_a? String
+    raise "Dataset #{name} exists in the project, aborting..." if
+      MiGA::Dataset.exist?(project, name)
+    metadata = get_metadata(metadata)
+    case @@UNIVERSE[universe][:dbs][db][:stage]
+    when :assembly
+      base = project.path + "/data/" + MiGA::Dataset.RESULT_DIRS[:assembly] +
+        "/" + name
+      File.open("#{base}.start", "w") { |ofh| ofh.puts Time.now.to_s }
+      if @@UNIVERSE[universe][:dbs][db][:format] == :fasta_gz
+        download("#{base}.LargeContigs.fna.gz")
+        system("gzip -d #{base}.LargeContigs.fna.gz")
+      else
+        download("#{base}.LargeContigs.fna")
       end
-   end
-end
+      File.symlink("#{base}.LargeContigs.fna", "#{base}.AllContigs.fna")
+      File.open("#{base}.done", "w") { |ofh| ofh.puts Time.now.to_s }
+    else
+      raise "Unexpected error: Unsupported result for database #{db}."
+    end
+    dataset = MiGA::Dataset.new(project, name, is_ref, metadata)
+    project.add_dataset(dataset.name)
+    result = dataset.add_result @@UNIVERSE[universe][:dbs][db][:stage]
+    raise "Empty dataset created: seed result was not added due to "+
+      "incomplete files." if result.nil?
+    dataset
+  end
+
+  ##
+  # Get metadata from the remote location.
+  def get_metadata(metadata={})
+    case universe
+    when :ebi, :ncbi
+      # Get taxonomy
+      metadata[:tax] = get_ncbi_taxonomy
+    end
+    metadata
+  end
+
+  ##
+  # Download data into +file+.
+  def download(file)
+    MiGA::RemoteDataset.download(universe, db, ids,
+      @@UNIVERSE[universe][:dbs][db][:format], file)
+  end
 
+  ##
+  # Get NCBI Taxonomy ID.
+  def get_ncbi_taxid
+    send("get_ncbi_taxid_from_#{universe}")
+  end
+
+  ##
+  # Get NCBI taxonomy as MiGA::Taxonomy.
+  def get_ncbi_taxonomy
+    lineage = {}
+    tax_id = get_ncbi_taxid
+    while !(tax_id.nil? or %w{0 1}.include? tax_id)
+      doc = MiGA::RemoteDataset.download(:ebi, :taxonomy, tax_id, "")
+      name = doc.scan(/SCIENTIFIC NAME\s+:\s+(.+)/).first.to_a.first
+      rank = doc.scan(/RANK\s+:\s+(.+)/).first.to_a.first
+      rank = "dataset" if lineage.empty? and rank=="no rank"
+      lineage[rank] = name unless rank.nil?
+      tax_id = doc.scan(/PARENT ID\s+:\s+(.+)/).first.to_a.first
+    end
+    MiGA::Taxonomy.new(lineage)
+  end
+
+  private
+    
+    def get_ncbi_taxid_from_ncbi
+      doc = MiGA::RemoteDataset.download(universe, db, ids, :gb).split(/\n/)
+      ln = doc.grep(/^\s+\/db_xref="taxon:/).first
+      return nil if ln.nil?
+      ln.sub!(/.*(?:"taxon:)(\d+)["; ].*/, "\\1")
+      return nil unless ln =~ /^\d+$/
+      ln
+    end
+
+    def get_ncbi_taxid_from_ebi
+      doc = MiGA::RemoteDataset.download(universe, db, ids, :annot).split(/\n/)
+      ln = doc.grep(/^FT\s+\/db_xref="taxon:/).first
+      ln = doc.grep(/^OX\s+NCBI_TaxID=/).first if ln.nil?
+      return nil if ln.nil?
+      ln.sub!(/.*(?:"taxon:|NCBI_TaxID=)(\d+)["; ].*/, "\\1")
+      return nil unless ln =~ /^\d+$/
+      ln
+    end
+end

data/lib/miga/result.rb

@@ -1,102 +1,153 @@
-#
 # @package MiGA
-# @author  Luis M. Rodriguez-R <lmrodriguezr at gmail dot com>
-# @license artistic license 2.0
-# @update  Dec-19-2015
-#
-
-module MiGA
-   class Result
-      # Class
-      def self.exist? path
-	 !!(File.size? path)
-      end
-      def self.load path
-	 return nil unless Result.exist? path
-	 Result.new path
-      end
-      # Instance
-      attr_reader :path, :data, :results
-      def initialize(path)
-	 @path = path
-	 if Result.exist? path
-	    self.load
-	 else
-	    self.create
-	 end
-      end
-      def dir
-	 File.dirname(path)
-      end
-      def file_path(k)
-	 k = k.to_sym
-	 return nil if self[:files].nil? or self[:files][k].nil?
-	 return File.expand_path(self[:files][k], dir) unless
-	    self[:files][k].is_a? Array
-         self[:files][k].map{ |f| File.expand_path(f, dir) }
-      end
-      def [](k) data[k.to_sym] ; end
-      def add_file(k, file)
-         k = k.to_sym
-	 self.data[:files] ||= {}
-	 self.data[:files][k] = file if
-	    File.exist? File.expand_path(file, dir)
-	 self.data[:files][k] = file + ".gz" if
-	    File.exist? File.expand_path(file + ".gz", dir)
-      end
-      def create
-	 @data = {:created=>Time.now.to_s, :results=>[], :stats=>{}, :files=>{}}
-	 self.save
-      end
-      def save
-	 self.data[:updated] = Time.now.to_s
-	 json = JSON.pretty_generate data
-	 ofh = File.open(path, "w")
-	 ofh.puts json
-	 ofh.close
-	 self.load
-      end
-      def load
-	 json = File.read path
-	 @data = JSON.parse(json, {:symbolize_names=>true})
-	 @results = self[:results].map{ |rs| Result.new rs }
-      end
-      def remove!
-	 each_file do |file|
-	    f = File.expand_path(file, dir)
-	    File.unlink_r(f) if File.exist? f
-	 end
-	 %w(.start .done).each do |ext|
-	    f = path.sub(/\.json$/, ext)
-	    File.unlink f if File.exist? f
-	 end
-	 File.unlink path
-      end
-      def each_file(&blk)
-	 self.data[:files] = {} if self.data[:files].nil?
-	 self.data[:files].each do |k,files|
-	    files = [files] unless files.kind_of? Array
-	    files.each do |file|
-	       if blk.arity==1
-		  blk.call file
-	       elsif blk.arity==2
-		  blk.call k, file
-	       else
-		  raise "Wrong number of arguments: #{blk.arity} for one or two"
-	       end
-	    end
-	 end
-      end
-      def add_result(result)
-         self.data[:results] << result.path
-	 self.save
-      end
-      def file_path(file)
-	 f = self.data[:files][file.to_sym]
-	 return nil if f.nil?
-	 return File.dirname(self.path) + "/" + f unless f.is_a?(Array)
-	 f.map{ |i| File.dirname(self.path) + "/" + i }
+# @license Artistic-2.0
+
+##
+# The result from a task run. It can be project-wide or dataset-specific.
+class MiGA::Result < MiGA::MiGA
+  
+  # Class-level
+  
+  ##
+  # Check if the result described by the JSON in +path+ already exists.
+  def self.exist?(path)
+    !!(File.size? path)
+  end
+
+  ##
+  # Load the result described by the JSON in +path+. Returns MiGA::Result if it
+  # already exists, nil otherwise.
+  def self.load(path)
+    return nil unless MiGA::Result.exist? path
+    MiGA::Result.new(path)
+  end
+
+  # Instance-level
+
+  ##
+  # Path to the JSON file describing the result.
+  attr_reader :path
+  
+  ##
+  # Hash with the result metadata.
+  attr_reader :data
+
+  ##
+  # Array of MiGA::Result objects nested within the result (if any).
+  attr_reader :results
+  
+  ##
+  # Load or create the MiGA::Result described by the JSON file +path+.
+  def initialize(path)
+    @path = path
+    MiGA::Result.exist?(path) ? self.load : create
+  end
+
+  ##
+  # Directory containing the result.
+  def dir
+    File.dirname(path)
+  end
+
+  ##
+  # Absolute path to the file(s) defined by symbol +k+.
+  def file_path(k)
+    k = k.to_sym
+    f = self[:files].nil? ? nil : self[:files][k]
+    return nil if f.nil?
+    return File.expand_path(f, dir) unless f.is_a? Array
+    f.map{ |fi| File.expand_path(fi, dir) }
+  end
+
+  ##
+  # Entry with symbol +k+. 
+  def [](k) data[k.to_sym] ; end
+
+  ##
+  # Register +file+ (path relative to #dir) with the symbol +k+. If the file
+  # doesn't exist but the .gz extension does, the gzipped file is registered
+  # instead. If neither exists, nothing is registered.
+  def add_file(k, file)
+    k = k.to_sym
+    @data[:files] ||= {}
+    @data[:files][k] = file if File.exist? File.expand_path(file, dir)
+    @data[:files][k] = file + ".gz" if
+      File.exist? File.expand_path(file + ".gz", dir)
+  end
+
+  ##
+  # #add_file for each key-value pair in the +files+ Hash.
+  def add_files(files)
+    files.each { |k, v| add_file(k, v) }
+  end
+
+  ##
+  # Initialize and #save empty result.
+  def create
+    @data = {:created=>Time.now.to_s, :results=>[], :stats=>{}, :files=>{}}
+    save
+  end
+
+  ##
+  # Save the result persistently (in the JSON file #path).
+  def save
+    @data[:updated] = Time.now.to_s
+    json = JSON.pretty_generate data
+    ofh = File.open(path, "w")
+    ofh.puts json
+    ofh.close
+    self.load
+  end
+
+  ##
+  # Load (or reload) result data in the JSON file #path.
+  def load
+    json = File.read(path)
+    @data = JSON.parse(json, {:symbolize_names=>true})
+    @data[:files] ||= {}
+    @results = self[:results].map{ |rs| MiGA::Result.new rs }
+  end
+
+  ##
+  # Remove result, including all associated files.
+  def remove!
+    each_file do |file|
+      f = File.expand_path(file, dir)
+      FileUtils.rm_rf(f) if File.exist? f
+    end
+    %w(.start .done).each do |ext|
+      f = path.sub(/\.json$/, ext)
+      File.unlink f if File.exist? f
+    end
+    File.unlink path
+  end
+
+  ##
+  # Iterate +blk+ for each registered file. If +blk+ calls for one argument, the
+  # relative path to the file is passed. If it calls for two arguments, the
+  # symbol describing the file is passed first and the path second. Note that
+  # multiple files may have the same symbol, since arrays of files are
+  # supported.
+  def each_file(&blk)
+    @data[:files] ||= {}
+    self[:files].each do |k,files|
+      files = [files] unless files.kind_of? Array
+      files.each do |file|
+        if blk.arity==1
+          blk.call(file)
+        elsif blk.arity==2
+          blk.call(k, file)
+        else
+          raise "Wrong number of arguments: #{blk.arity} for one or two"
+        end
       end
-   end
-end
+    end
+  end
 
+  ##
+  # Add the MiGA::Result +result+ as part of the current result.
+  def add_result(result)
+    @data[:results] << result.path
+    save
+  end
+  
+end