miga-base 0.7.3.0 → 0.7.7.0

data/lib/miga/project/hooks.rb

@@ -1,4 +1,3 @@
-
 require 'miga/common/hooks'
 
 ##
@@ -18,7 +17,6 @@ require 'miga/common/hooks'
 # Internal hooks:
 # - _pull_result_hooks()
 module MiGA::Project::Hooks
-
   include MiGA::Common::Hooks
 
   def default_hooks
@@ -47,5 +45,4 @@ module MiGA::Project::Hooks
     pull_hook(:"on_result_ready_#{event_args.first}", *event_args)
     pull_hook(:on_processing_ready) if next_task(nil, false).nil?
   end
-
 end

data/lib/miga/project/result.rb

@@ -62,6 +62,7 @@ module MiGA::Project::Result
   # Add result of any type +:*_distances+ at +base+ (no +_opts+ supported).
   def add_result_distances(base, _opts)
     return nil unless result_files_exist?(base, %w[.Rdata .log .txt])
+
     r = MiGA::Result.new("#{base}.json")
     r.add_file(:rdata,  'miga-project.Rdata')
     r.add_file(:matrix, 'miga-project.txt')
@@ -79,8 +80,13 @@ module MiGA::Project::Result
       return r
     end
     return nil unless result_files_exist?(base, %w[.proposed-clades])
-    return nil unless is_clade? or result_files_exist?(base,
-        %w[.pdf .classif .medoids .class.tsv .class.nwk])
+    unless is_clade? ||
+           result_files_exist?(
+             base, %w[.pdf .classif .medoids .class.tsv .class.nwk]
+           )
+      return nil
+    end
+
     r = add_result_iter_clades(base)
     r.add_file(:aai_tree,      'miga-project.aai.nwk')
     r.add_file(:proposal,      'miga-project.proposed-clades')
@@ -99,8 +105,10 @@ module MiGA::Project::Result
       r.add_file(:empty, 'miga-project.empty')
       return r
     end
-    return nil unless result_files_exist?(base,
-      %w[.pdf .classif .medoids .class.tsv .class.nwk])
+    return nil unless result_files_exist?(
+      base, %w[.pdf .classif .medoids .class.tsv .class.nwk]
+    )
+
     r = add_result_iter_clades(base)
     r.add_file(:ani_tree, 'miga-project.ani.nwk')
     r
@@ -127,6 +135,7 @@ module MiGA::Project::Result
       return r
     end
     return nil unless result_files_exist?(base, %w[.ogs .stats])
+
     r = MiGA::Result.new("#{base}.json")
     r.add_file(:ogs,   'miga-project.ogs')
     r.add_file(:abc,   'miga-project.abc')
@@ -141,6 +150,7 @@ module MiGA::Project::Result
   def add_result_project_stats(base, _opts)
     return nil unless
       result_files_exist?(base, %w[.taxonomy.json .metadata.db])
+
     r = MiGA::Result.new("#{base}.json")
     r.add_file(:taxonomy_index, 'miga-project.taxonomy.json')
     r.add_file(:metadata_index, 'miga-project.metadata.db')
@@ -151,5 +161,4 @@ module MiGA::Project::Result
   alias add_result_aai_distances add_result_distances
   alias add_result_ani_distances add_result_distances
   alias add_result_ssu_distances add_result_distances
-
 end

data/lib/miga/remote_dataset.rb

@@ -14,9 +14,11 @@ class MiGA::RemoteDataset < MiGA::MiGA
   class << self
     def ncbi_asm_acc2id(acc)
       return acc if acc =~ /^\d+$/
+
       search_doc = MiGA::Json.parse(
         download(:ncbi_search, :assembly, acc, :json),
-        symbolize: false, contents: true)
+        symbolize: false, contents: true
+      )
       (search_doc['esearchresult']['idlist'] || []).first
     end
   end
@@ -90,14 +92,14 @@ class MiGA::RemoteDataset < MiGA::MiGA
   # and optionally the Hash +metadata+.
   def update_metadata(dataset, metadata = {})
     metadata = get_metadata(metadata)
-    metadata.each { |k,v| dataset.metadata[k] = v }
+    metadata.each { |k, v| dataset.metadata[k] = v }
     dataset.save
   end
 
   ##
   # Get metadata from the remote location.
   def get_metadata(metadata_def = {})
-    metadata_def.each { |k,v| @metadata[k] = v }
+    metadata_def.each { |k, v| @metadata[k] = v }
     case universe
     when :ebi, :ncbi, :web
       # Get taxonomy
@@ -131,7 +133,8 @@ class MiGA::RemoteDataset < MiGA::MiGA
   def get_ncbi_taxonomy
     tax_id = get_ncbi_taxid
     return nil if tax_id.nil?
-    lineage = {ns: 'ncbi'}
+
+    lineage = { ns: 'ncbi' }
     doc = MiGA::RemoteDataset.download(:ncbi, :taxonomy, tax_id, :xml)
     doc.scan(%r{<Taxon>(.*?)</Taxon>}m).map(&:first).each do |i|
       name = i.scan(%r{<ScientificName>(.*)</ScientificName>}).first.to_a.first
@@ -148,89 +151,99 @@ class MiGA::RemoteDataset < MiGA::MiGA
   # Get the JSON document describing an NCBI assembly entry.
   def ncbi_asm_json_doc
     return @_ncbi_asm_json_doc unless @_ncbi_asm_json_doc.nil?
+
     metadata[:ncbi_asm] ||= ids.first if universe == :ncbi and db == :assembly
     return nil unless metadata[:ncbi_asm]
+
     ncbi_asm_id = self.class.ncbi_asm_acc2id metadata[:ncbi_asm]
     doc = MiGA::Json.parse(
       self.class.download(:ncbi_summary, :assembly, ncbi_asm_id, :json),
-      symbolize: false, contents: true)
+      symbolize: false, contents: true
+    )
     @_ncbi_asm_json_doc = doc['result'][ doc['result']['uids'].first ]
   end
 
-
   private
 
-    def get_ncbi_taxid_from_web
-      return nil if ncbi_asm_json_doc.nil?
-      ncbi_asm_json_doc['taxid']
-    end
+  def get_ncbi_taxid_from_web
+    return nil if ncbi_asm_json_doc.nil?
 
-    def get_ncbi_taxid_from_ncbi
-      doc = self.class.download(universe, db, ids, :gb).split(/\n/)
-      ln = doc.grep(%r{^\s+/db_xref="taxon:}).first
-      return nil if ln.nil?
-      ln.sub!(/.*(?:"taxon:)(\d+)["; ].*/, '\\1')
-      return nil unless ln =~ /^\d+$/
-      ln
-    end
+    ncbi_asm_json_doc['taxid']
+  end
 
-    def get_ncbi_taxid_from_ebi
-      doc = self.class.download(universe, db, ids, :annot).split(/\n/)
-      ln = doc.grep(%r{^FT\s+/db_xref="taxon:}).first
-      ln = doc.grep(/^OX\s+NCBI_TaxID=/).first if ln.nil?
-      return nil if ln.nil?
-      ln.sub!(/.*(?:"taxon:|NCBI_TaxID=)(\d+)["; ].*/, '\\1')
-      return nil unless ln =~ /^\d+$/
-      ln
-    end
+  def get_ncbi_taxid_from_ncbi
+    doc = self.class.download(universe, db, ids, :gb).split(/\n/)
+    ln = doc.grep(%r{^\s+/db_xref="taxon:}).first
+    return nil if ln.nil?
 
-    def get_type_status_ncbi_nuccore(metadata)
-      return metadata if metadata[:ncbi_nuccore].nil?
-      biosample = self.class.ncbi_map(metadata[:ncbi_nuccore],
-        :nuccore, :biosample)
-      return metadata if biosample.nil?
-      asm = self.class.ncbi_map(biosample, :biosample, :assembly)
-      metadata[:ncbi_asm] = asm.to_s unless asm.nil?
-      get_type_status_ncbi_asm metadata
-    end
+    ln.sub!(/.*(?:"taxon:)(\d+)["; ].*/, '\\1')
+    return nil unless ln =~ /^\d+$/
 
-    def get_type_status_ncbi_asm(metadata)
-      return metadata if ncbi_asm_json_doc.nil?
-      from_type = ncbi_asm_json_doc['from_type']
-      from_type = ncbi_asm_json_doc['fromtype'] if from_type.nil?
-      case from_type
-      when nil
-        # Do nothing
-      when ''
-        metadata[:is_type] = false
-        metadata[:is_ref_type] = false
-      when 'assembly from reference material', 'assembly designated as reftype'
-        metadata[:is_type] = false
-        metadata[:is_ref_type] = true
-        metadata[:type_rel] = from_type
-      else
-        metadata[:is_type] = true
-        metadata[:type_rel] = from_type
-      end
-      metadata[:suspect] = (ncbi_asm_json_doc['exclfromrefseq'] || [])
-      metadata[:suspect] = nil if metadata[:suspect].empty?
-      MiGA.DEBUG "Got type: #{from_type}"
-      metadata
+    ln
+  end
+
+  def get_ncbi_taxid_from_ebi
+    doc = self.class.download(universe, db, ids, :annot).split(/\n/)
+    ln = doc.grep(%r{^FT\s+/db_xref="taxon:}).first
+    ln = doc.grep(/^OX\s+NCBI_TaxID=/).first if ln.nil?
+    return nil if ln.nil?
+
+    ln.sub!(/.*(?:"taxon:|NCBI_TaxID=)(\d+)["; ].*/, '\\1')
+    return nil unless ln =~ /^\d+$/
+
+    ln
+  end
+
+  def get_type_status_ncbi_nuccore(metadata)
+    return metadata if metadata[:ncbi_nuccore].nil?
+
+    biosample =
+      self.class.ncbi_map(metadata[:ncbi_nuccore], :nuccore, :biosample)
+    return metadata if biosample.nil?
+
+    asm = self.class.ncbi_map(biosample, :biosample, :assembly)
+    metadata[:ncbi_asm] = asm.to_s unless asm.nil?
+    get_type_status_ncbi_asm metadata
+  end
+
+  def get_type_status_ncbi_asm(metadata)
+    return metadata if ncbi_asm_json_doc.nil?
+
+    from_type = ncbi_asm_json_doc['from_type']
+    from_type = ncbi_asm_json_doc['fromtype'] if from_type.nil?
+    case from_type
+    when nil
+      # Do nothing
+    when ''
+      metadata[:is_type] = false
+      metadata[:is_ref_type] = false
+    when 'assembly from reference material', 'assembly designated as reftype'
+      metadata[:is_type] = false
+      metadata[:is_ref_type] = true
+      metadata[:type_rel] = from_type
+    else
+      metadata[:is_type] = true
+      metadata[:type_rel] = from_type
     end
+    metadata[:suspect] = (ncbi_asm_json_doc['exclfromrefseq'] || [])
+    metadata[:suspect] = nil if metadata[:suspect].empty?
+    MiGA.DEBUG "Got type: #{from_type}"
+    metadata
+  end
 
-    def save_assembly_to(project, name, udb)
-      dir = MiGA::Dataset.RESULT_DIRS[:assembly]
-      base = "#{project.path}/data/#{dir}/#{name}"
-      l_ctg = "#{base}.LargeContigs.fna"
-      a_ctg = "#{base}.AllContigs.fna"
-      File.open("#{base}.start", 'w') { |ofh| ofh.puts Time.now.to_s }
-      if udb[:format] == :fasta_gz
-        download "#{l_ctg}.gz"
-        system "gzip -d '#{l_ctg}.gz'"
-      else
-        download l_ctg
-      end
-      File.unlink(a_ctg) if File.exist? a_ctg
-      File.open("#{base}.done", 'w') { |ofh| ofh.puts Time.now.to_s }
+  def save_assembly_to(project, name, udb)
+    dir = MiGA::Dataset.RESULT_DIRS[:assembly]
+    base = "#{project.path}/data/#{dir}/#{name}"
+    l_ctg = "#{base}.LargeContigs.fna"
+    a_ctg = "#{base}.AllContigs.fna"
+    File.open("#{base}.start", 'w') { |ofh| ofh.puts Time.now.to_s }
+    if udb[:format] == :fasta_gz
+      download "#{l_ctg}.gz"
+      system "gzip -d '#{l_ctg}.gz'"
+    else
+      download l_ctg
     end
+    File.unlink(a_ctg) if File.exist? a_ctg
+    File.open("#{base}.done", 'w') { |ofh| ofh.puts Time.now.to_s }
+  end
 end

data/lib/miga/remote_dataset/base.rb

@@ -1,21 +1,20 @@
-
 require 'open-uri'
 require 'cgi'
 
 class MiGA::RemoteDataset < MiGA::MiGA
-
   # Class-level
   class << self
-    def UNIVERSE ; @@UNIVERSE ; end
+    def UNIVERSE
+      @@UNIVERSE
+    end
   end
-
 end
 
 module MiGA::RemoteDataset::Base
-
   @@_EUTILS = 'https://eutils.ncbi.nlm.nih.gov/entrez/eutils/'
   @@_NCBI_API_KEY = lambda { |url|
-    ENV['NCBI_API_KEY'].nil? ? url : "#{url}&api_key=#{ENV['NCBI_API_KEY']}" }
+    ENV['NCBI_API_KEY'].nil? ? url : "#{url}&api_key=#{ENV['NCBI_API_KEY']}"
+  }
 
   ##
   # Structure of the different database Universes or containers. The structure
@@ -43,13 +42,13 @@ module MiGA::RemoteDataset::Base
       method: :net
     },
     ebi: {
-      dbs: { embl: {stage: :assembly, format: :fasta} },
+      dbs: { embl: { stage: :assembly, format: :fasta } },
       url: 'https://www.ebi.ac.uk/Tools/dbfetch/dbfetch/%1$s/%2$s/%3$s',
       method: :rest
     },
     ncbi: {
       dbs: {
-        nuccore: { stage: :assembly, format: :fasta },
+        nuccore: { stage: :assembly, format: :fasta, getter: :ncbi_gb },
         assembly: { stage: :assembly, format: :fasta_gz, getter: :ncbi_asm },
         taxonomy: { stage: :metadata, format: :xml }
       },
@@ -59,9 +58,10 @@ module MiGA::RemoteDataset::Base
     },
     ncbi_map: {
       dbs: {
-        nuccore: { stage: :metadata, map_to: [:biosample, :assembly],
-          format: :json },
-        biosample: {stage: :metadata, map_to: [:assembly], format: :json}
+        nuccore: {
+          stage: :metadata, map_to: [:biosample, :assembly], format: :json
+        },
+        biosample: { stage: :metadata, map_to: [:assembly], format: :json }
       },
       url: "#{@@_EUTILS}elink.fcgi?dbfrom=%1$s&id=%2$s&db=%4$s&retmode=%3$s",
       method: :net,
@@ -81,6 +81,4 @@ module MiGA::RemoteDataset::Base
       api_key: @@_NCBI_API_KEY
     }
   }
-
 end
-

data/lib/miga/remote_dataset/download.rb

@@ -1,4 +1,3 @@
-
 require 'miga/remote_dataset/base'
 
 class MiGA::RemoteDataset
@@ -26,8 +25,8 @@ class MiGA::RemoteDataset
         obj: obj
       }
       doc = send("#{getter}_#{method}", opts)
-      unless file.nil?
-        ofh = File.open(file, 'w')
+      unless opts[:file].nil?
+        ofh = File.open(opts[:file], 'w')
         ofh.print doc.force_encoding('UTF-8')
         ofh.close
       end
@@ -45,8 +44,26 @@ class MiGA::RemoteDataset
     def ncbi_asm_rest(opts)
       url_dir = opts[:obj].ncbi_asm_json_doc['ftppath_genbank']
       url = "#{url_dir}/#{File.basename url_dir}_genomic.fna.gz"
-      download(:web, :assembly_gz, url,
-        opts[:format], opts[:file], opts[:extra], opts[:obj])
+      download(
+        :web, :assembly_gz, url,
+        opts[:format], opts[:file], opts[:extra], opts[:obj]
+      )
+    end
+
+    ##
+    # Download data from NCBI GenBank (nuccore) database using the REST method.
+    # Supported +opts+ (Hash) are the same as #download_rest and #ncbi_asm_rest.
+    def ncbi_gb_rest(opts)
+      o = download_rest(opts)
+      return o unless o.strip.empty?
+
+      MiGA::MiGA.DEBUG 'Empty sequence, attempting download from NCBI assembly'
+      opts[:format] = :fasta_gz
+      if opts[:file]
+        File.unlink(opts[:file]) if File.exist? opts[:file]
+        opts[:file] = "#{opts[:file]}.gz"
+      end
+      ncbi_asm_rest(opts)
     end
 
     ##
@@ -58,8 +75,9 @@ class MiGA::RemoteDataset
     # +extra+: Array
     def download_rest(opts)
       u = @@UNIVERSE[opts[:universe]]
-      url = sprintf(u[:url],
-        opts[:db], opts[:ids].join(','), opts[:format], *opts[:extra])
+      url = sprintf(
+        u[:url], opts[:db], opts[:ids].join(','), opts[:format], *opts[:extra]
+      )
       url = u[:api_key][url] unless u[:api_key].nil?
       download_url url
     end
@@ -80,17 +98,20 @@ class MiGA::RemoteDataset
       rescue => e
         @timeout_try += 1
         raise e if @timeout_try >= 3
+
+        sleep 5 # <- For: 429 Too Many Requests
         retry
       end
       doc
     end
-    
+
     ##
     # Looks for the entry +id+ in +dbfrom+, and returns the linked
     # identifier in +db+ (or nil).
     def ncbi_map(id, dbfrom, db)
       doc = download(:ncbi_map, dbfrom, id, :json, nil, [db])
       return if doc.empty?
+
       tree = MiGA::Json.parse(doc, contents: true)
       [:linksets, 0, :linksetdbs, 0, :links, 0].each do |i|
         tree = tree[i]
@@ -102,11 +123,12 @@ class MiGA::RemoteDataset
 end
 
 module MiGA::RemoteDataset::Download
-
   ##
-  # Download data into +file+.
+  # Download data into +file+
   def download(file)
-    self.class.download(universe, db, ids,
-          self.class.UNIVERSE[universe][:dbs][db][:format], file, [], self)
+    self.class.download(
+      universe, db, ids, self.class.UNIVERSE[universe][:dbs][db][:format],
+      file, [], self
+    )
   end
 end

data/lib/miga/result.rb

@@ -8,7 +8,6 @@ require 'miga/result/stats'
 ##
 # The result from a task run. It can be project-wide or dataset-specific.
 class MiGA::Result < MiGA::MiGA
-
   include MiGA::Result::Dates
   include MiGA::Result::Source
   include MiGA::Result::Stats
@@ -26,6 +25,7 @@ class MiGA::Result < MiGA::MiGA
     # Returns MiGA::Result if it already exists, nil otherwise.
     def load(path)
       return nil unless MiGA::Result.exist? path
+
       MiGA::Result.new(path)
     end
 
@@ -33,6 +33,7 @@ class MiGA::Result < MiGA::MiGA
       FileUtils.rm(path) if force && File.exist?(path)
       r_pre = self.load(path)
       return r_pre unless r_pre.nil?
+
       yield
       self.load(path)
     end
@@ -41,34 +42,38 @@ class MiGA::Result < MiGA::MiGA
   # Instance-level
 
   ##
-  # Hash with the result metadata.
+  # Hash with the result metadata
   attr_reader :data
 
   ##
-  # Array of MiGA::Result objects nested within the result (if any).
+  # Array of MiGA::Result objects nested within the result (if any)
   attr_reader :results
 
   ##
-  # Load or create the MiGA::Result described by the JSON file +path+.
+  # Load or create the MiGA::Result described by the JSON file +path+
   def initialize(path)
     @path = File.absolute_path(path)
     MiGA::Result.exist?(@path) ? self.load : create
   end
 
   ##
-  # Is the result clean? Returns Boolean.
-  def clean? ; !! self[:clean] ; end
+  # Is the result clean? Returns Boolean
+  def clean?
+    !!self[:clean]
+  end
 
   ##
-  # Register the result as cleaned.
-  def clean! ; self[:clean] = true ; end
+  # Register the result as cleaned
+  def clean!
+    self[:clean] = true
+  end
 
   ##
   # Path to the standard files of the result. +which+ must be one of:
   # - :json (default) : JSON file describing the result.
   # - :start : File with the date when the processing started.
   # - :done : File with the date when the processing ended.
-  def path(which=:json)
+  def path(which = :json)
     case which.to_sym
     when :json
       @path
@@ -80,28 +85,33 @@ class MiGA::Result < MiGA::MiGA
   end
 
   ##
-  # Directory containing the result.
+  # Directory containing the result
   def dir
     File.dirname(path)
   end
 
   ##
-  # Absolute path to the file(s) defined by symbol +k+.
+  # Absolute path to the file(s) defined by symbol +k+
   def file_path(k)
     k = k.to_sym
     f = self[:files].nil? ? nil : self[:files][k]
     return nil if f.nil?
     return File.expand_path(f, dir) unless f.is_a? Array
-    f.map{ |fi| File.expand_path(fi, dir) }
+
+    f.map { |fi| File.expand_path(fi, dir) }
   end
 
   ##
-  # Entry with symbol +k+. 
-  def [](k) data[k.to_sym] ; end
+  # Entry with symbol +k+
+  def [](k)
+    data[k.to_sym]
+  end
 
   ##
-  # Adds value +v+ to entry with symbol +k+.
-  def []=(k,v) data[k.to_sym]=v ; end
+  # Adds value +v+ to entry with symbol +k+
+  def []=(k, v)
+    data[k.to_sym] = v
+  end
 
   ##
   # Register +file+ (path relative to #dir) with the symbol +k+. If the file
@@ -116,20 +126,20 @@ class MiGA::Result < MiGA::MiGA
   end
 
   ##
-  # #add_file for each key-value pair in the +files+ Hash.
+  # #add_file for each key-value pair in the +files+ Hash
   def add_files(files)
     files.each { |k, v| add_file(k, v) }
   end
 
   ##
-  # Initialize and #save empty result.
+  # Initialize and #save empty result
   def create
     @data = { created: Time.now.to_s, results: [], stats: {}, files: {} }
     save
   end
 
   ##
-  # Save the result persistently (in the JSON file #path).
+  # Save the result persistently (in the JSON file #path)
   def save
     @data[:updated] = Time.now.to_s
     s = path(:start)
@@ -142,15 +152,15 @@ class MiGA::Result < MiGA::MiGA
   end
 
   ##
-  # Load (or reload) result data in the JSON file #path.
+  # Load (or reload) result data in the JSON file #path
   def load
     @data = MiGA::Json.parse(path)
     @data[:files] ||= {}
-    @results = (self[:results] || []).map{ |rs| MiGA::Result.new rs }
+    @results = (self[:results] || []).map { |rs| MiGA::Result.new rs }
   end
 
   ##
-  # Remove result, including all associated files.
+  # Remove result, including all associated files
   def remove!
     each_file do |file|
       f = File.expand_path(file, dir)
@@ -173,7 +183,7 @@ class MiGA::Result < MiGA::MiGA
   # arrays of files are supported.
   def each_file(&blk)
     @data[:files] ||= {}
-    self[:files].each do |k,files|
+    self[:files].each do |k, files|
       files = [files] unless files.kind_of? Array
       files.each do |file|
         case blk.arity
@@ -191,10 +201,9 @@ class MiGA::Result < MiGA::MiGA
   end
 
   ##
-  # Add the MiGA::Result +result+ as part of the current result.
+  # Add the MiGA::Result +result+ as part of the current result
   def add_result(result)
     @data[:results] << result.path
     save
   end
-
 end