miga-base 0.3.4.2 → 0.3.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0bb2e8453ba42c927af174b0212b8ab8b5af9281
-  data.tar.gz: 00750cc643ff8eb2a62f5fc83525984b6fe7b4ab
+  metadata.gz: c7f9c242373a48634effe2905d33ddd49c2c1bce
+  data.tar.gz: cbdf5d91e364987718b6f0d29b57a0bab372afdc
 SHA512:
-  metadata.gz: 2fd6479b87d09a021884598d6ad5bb43c849ba0809821de0b2a14eb2a6670bbf5331eb634fbee54c9def0c3f1192c4deef8b4576b0dc6721f193b90e32bf366c
-  data.tar.gz: 65cb9f2ac275ed29c34a18db17543e29a3f252374cef4ddc7aedbc9da8210700669b3412b70adb190b3693717149140e39d7d2a89444d5871d7ed9af02f20e74
+  metadata.gz: d790bb6056fc556ae86915093d6d16973222009e3249f1d33351ae9e35ab11614ea9163722a63592e2e1e95a0ea1345a537f98f8ffa675ef421ce6dd1f077642
+  data.tar.gz: d435fcda801d87f51f1dcc8d090812c7fbae12e74dfbbea5470ed573a993ad61edcf6855376f2ef3ea9acb9900407072b827852716c24ab64c4252e87a982d95

data/actions/get.rb

@@ -3,35 +3,39 @@
 # @package MiGA
 # @license Artistic-2.0
 
-require "miga/remote_dataset"
+require 'miga/remote_dataset'
 
-o = {q:true, query:false, universe: :ebi, db: :embl}
+o = {q: true, query: false, universe: :ebi, db: :embl}
 OptionParser.new do |opt|
   opt_banner(opt)
   opt_object(opt, o, [:project, :dataset, :dataset_type])
-  opt.on("-I", "--ids ID1,ID2,...",
-    "(Mandatory unless -F) IDs in the remote database separated by commas."
+  opt.on('-I', '--ids ID1,ID2,...',
+    '(Mandatory unless -F) IDs in the remote database separated by commas.'
     ){ |v| o[:ids]=v }
-  opt.on("-U", "--universe STRING",
+  opt.on('-U', '--universe STRING',
     "Universe where the remote database lives. By default: #{o[:universe]}."
     ){ |v| o[:universe]=v.to_sym }
-  opt.on("--db STRING",
+  opt.on('--db STRING',
     "Name of the remote database. By default: #{o[:db]}."
     ){ |v| o[:db]=v.to_sym }
-  opt.on("-F", "--file PATH",
-    "Tab-delimited file (with header) listing the datasets to download.",
-    "The long form of all the options are supported as header (without the --)",
-    "including dataset, ids, universe, and db. For query use true/false values."
+  opt.on('-F', '--file PATH',
+    'Tab-delimited file (with header) listing the datasets to download.',
+    'The long form of all the options are supported as header (without the --)',
+    'including dataset, ids, universe, and db. For query use true/false values.'
     ){ |v| o[:file] = v }
-  opt.on("-q", "--query",
-    "If set, the dataset is registered as a query, not a reference dataset."
+  opt.on('-q', '--query',
+    'If set, the dataset is registered as a query, not a reference dataset.'
     ){ |v| o[:query]=v }
-  opt.on("--ignore-dup",
-    "If set, ignores datasets that already exist."){ |v| o[:ignore_dup]=v }
-  opt.on("-d", "--description STRING",
-    "Description of the dataset."){ |v| o[:description]=v }
-  opt.on("-c", "--comments STRING",
-    "Comments on the dataset."){ |v| o[:comments]=v }
+  opt.on('--ignore-dup',
+    'If set, ignores datasets that already exist.'){ |v| o[:ignore_dup]=v }
+  opt.on('-d', '--description STRING',
+    'Description of the dataset.'){ |v| o[:description]=v }
+  opt.on('-c', '--comments STRING',
+    'Comments on the dataset.'){ |v| o[:comments]=v }
+  opt.on('-m', '--metadata STRING',
+    'Metadata as key-value pairs separated by = and delimited by comma.',
+    'Values are saved as strings except for booleans (true / false) or nil.'
+    ){ |v| o[:metadata]=v }
   opt_common(opt, o)
 end.parse!
 
@@ -40,7 +44,7 @@ end.parse!
 glob = [o]
 unless o[:file].nil?
   glob = []
-  fh = File.open(o[:file], "r")
+  fh = File.open(o[:file], 'r')
   h = nil
   fh.each do |ln|
     r = ln.chomp.split(/\t/)
@@ -49,7 +53,7 @@ unless o[:file].nil?
     else
        glob << o.dup
        h.each_index do |i|
-	 glob[glob.size-1][h[i].to_sym] = h[i]=="query" ? r[i]=="true" :
+	 glob[glob.size-1][h[i].to_sym] = h[i]=='query' ? r[i]=='true' :
 	   %w[type universe db].include?(h[i]) ? r[i].to_sym : r[i]
        end
     end
@@ -58,25 +62,24 @@ unless o[:file].nil?
 end
 
 glob.each do |o_i|
-  opt_require(o_i, project:"-P", dataset:"-D", ids:"-I")
+  opt_require(o_i, project: '-P', dataset: '-D', ids: '-I')
 
   $stderr.puts "Dataset: #{o_i[:dataset]}" unless o_i[:q]
-  $stderr.puts "Loading project." unless o_i[:q]
+  $stderr.puts 'Loading project.' unless o_i[:q]
   p = MiGA::Project.load(o_i[:project])
   raise "Impossible to load project: #{o_i[:project]}" if p.nil?
 
   next if o_i[:ignore_dup] and not p.dataset(o_i[:dataset]).nil?
 
-  $stderr.puts "Locating remote dataset." unless o_i[:q]
+  $stderr.puts 'Locating remote dataset.' unless o_i[:q]
   rd = MiGA::RemoteDataset.new(o_i[:ids], o_i[:db], o_i[:universe])
 
-  $stderr.puts "Creating dataset." unless o_i[:q]
-  md = {}
-  [:type, :description, :user, :comments].each do |k|
-    md[k]=o_i[k] unless o_i[k].nil?
-  end
+  $stderr.puts 'Creating dataset.' unless o_i[:q]
+  dummy_d = MiGA::Dataset.new(p, o_i[:dataset])
+  md = add_metadata(o_i, dummy_d).metadata.data
+  dummy_d.remove!
   rd.save_to(p, o_i[:dataset], !o_i[:query], md)
   p.add_dataset(o_i[:dataset])
 
-  $stderr.puts "Done." unless o_i[:q]
+  $stderr.puts 'Done.' unless o_i[:q]
 end

data/actions/init.rb

@@ -172,7 +172,7 @@ $stderr.puts ""
 
 # Check for Ruby gems
 $stderr.puts "Looking for Ruby gems:"
-%w(rest-client sqlite3 daemons json).each do |pkg|
+%w(sqlite3 daemons json).each do |pkg|
   $stderr.print "Testing #{pkg}... "
   `#{paths["ruby"].shellescape} -r "#{pkg}" -e "" 2>/dev/null`
   if $?.success?

data/actions/ncbi_get.rb

@@ -77,11 +77,7 @@ def get_list(taxon, status)
           status: status }
   end
   url = url_base + URI.encode_www_form(url_param)
-  response = RestClient::Request.execute(method: :get, url:url, timeout:600)
-  unless response.code == 200
-    raise "Unable to reach NCBI, error code #{response.code}."
-  end
-  response.to_s
+  MiGA::RemoteDataset.download_url url
 end
 
 # Download IDs with reference status
@@ -135,6 +131,7 @@ if o[:scaffold] or o[:contig]
           map{ |i| "#{i}/#{File.basename(i)}_genomic.fna.gz" }
     next if ids.empty?
     n = "#{r[0]}_#{asm}".miga_name
+    asm.gsub!(/\(.*\)/, '')
     ds[n] = {ids: ids, md: {type: :genome, ncbi_asm: asm},
           db: :assembly_gz, universe: :web}
   end

data/lib/miga/daemon.rb

@@ -19,9 +19,8 @@ class MiGA::Daemon < MiGA::MiGA
     DateTime.parse(File.read(f))
   end
 
-  # Shutdown all spawned daemons before exit.
+  # Array of all spawned daemons.
   $_MIGA_DAEMON_LAIR = []
-  END { $_MIGA_DAEMON_LAIR.each(&:terminate) }
 
   # MiGA::Project in which the daemon is running.
   attr_reader :project

data/lib/miga/project/result.rb

@@ -34,7 +34,9 @@ module MiGA::Project::Result
   def add_result(name, save=true, opts={})
     return nil if @@RESULT_DIRS[name].nil?
     base = "#{path}/data/#{@@RESULT_DIRS[name]}/miga-project"
-    unless opts[:force]
+    if opts[:force]
+      FileUtils.rm("#{base}.json") if File.exist?("#{base}.json")
+    else
       r_pre = MiGA::Result.load("#{base}.json")
       return r_pre if (r_pre.nil? and not save) or not r_pre.nil?
     end

data/lib/miga/remote_dataset.rb

@@ -103,15 +103,15 @@ class MiGA::RemoteDataset < MiGA::MiGA
   ##
   # Get NCBI taxonomy as MiGA::Taxonomy.
   def get_ncbi_taxonomy
-    lineage = {}
     tax_id = get_ncbi_taxid
-    until [nil, '0', '1'].include? tax_id
-      doc = MiGA::RemoteDataset.download(:ebi, :taxonomy, tax_id, '')
-      name = doc.scan(/SCIENTIFIC NAME\s+:\s+(.+)/).first.to_a.first
-      rank = doc.scan(/RANK\s+:\s+(.+)/).first.to_a.first
-      rank = 'dataset' if lineage.empty? and rank == 'no rank'
-      lineage[rank] = name unless rank.nil?
-      tax_id = doc.scan(/PARENT ID\s+:\s+(.+)/).first.to_a.first
+    lineage = {}
+    doc = MiGA::RemoteDataset.download(:ncbi, :taxonomy, tax_id, :xml)
+    doc.scan(%r{<Taxon>(.*?)</Taxon>}m).map(&:first).each do |i|
+      name = i.scan(%r{<ScientificName>(.*)</ScientificName>}).first.to_a.first
+      rank = i.scan(%r{<Rank>(.*)</Rank>}).first.to_a.first
+      rank = nil if rank == 'no rank' or rank.empty?
+      rank = 'dataset' if lineage.empty? and rank.nil?
+      lineage[rank] = name unless rank.nil? or rank.nil?
     end
     MiGA::Taxonomy.new(lineage)
   end

data/lib/miga/remote_dataset/base.rb

@@ -1,5 +1,4 @@
 
-require 'rest-client'
 require 'open-uri'
 require 'cgi'
 
@@ -56,7 +55,7 @@ module MiGA::RemoteDataset::Base
         biosample: {stage: :metadata, map_to: [:assembly], format: :json}
       },
       url: "#{@@_EUTILS}elink.fcgi?dbfrom=%1$s&id=%2$s&db=%4$s&retmode=%3$s",
-      method: :rest,
+      method: :net,
       map_to_universe: :ncbi
     }
   }

data/lib/miga/remote_dataset/download.rb

@@ -33,30 +33,28 @@ class MiGA::RemoteDataset
     # using +extra+. Returns the doc as String.
     def download_rest(universe, db, ids, format, extra = [])
       u = @@UNIVERSE[universe]
-      url ||= sprintf(u[:url], db, ids.join(","), format, *extra)
-      response = RestClient::Request.execute(method: :get, url:url, timeout:600)
-      unless response.code == 200
-        raise "Unable to reach #{universe} client, error code #{response.code}."
-      end
-      response.to_s
+      url = sprintf(u[:url], db, ids.join(","), format, *extra)
+      download_url url
     end
 
     ##
     # Download data using a GET request from the +universe+ in the database +db+
     # with IDs +ids+ and in +format+. Additional URL parameters can be passed
     # using +extra+. Returns the doc as String.
-    def download_net(universe, db, ids, format, extra = [])
-      u = @@UNIVERSE[universe]
-      url = sprintf(u[:url], db, ids.join(","), format, *extra)
-      doc = ""
+    alias download_net download_rest
+
+    ##
+    # Download the given +url+ and return the result regardless of response
+    # code. Attempts download up to three times before raising Net::ReadTimeout.
+    def download_url(url)
+      doc = ''
       @timeout_try = 0
       begin
-        open(url) { |f| doc = f.read }
+        open(url, open_timeout: 600, read_timeout: 600) { |f| doc = f.read }
       rescue Net::ReadTimeout
         @timeout_try += 1
-        if @timeout_try > 3 ; raise Net::ReadTimeout
-        else ; retry
-        end
+        raise Net::ReadTimeout if @timeout_try >= 3
+        retry
       end
       doc
     end

data/lib/miga/version.rb

@@ -10,7 +10,7 @@ module MiGA
   # - Float representing the major.minor version.
   # - Integer representing gem releases of the current version.
   # - Integer representing minor changes that require new version number.
-  VERSION = [0.3, 4, 2]
+  VERSION = [0.3, 5, 0]
   
   ##
   # Nickname for the current major.minor version.

data/scripts/init.bash

@@ -121,7 +121,7 @@ done
 # Check for ruby gems
 echo "
 Looking for Ruby gems:" >&2
-GEMS="rest-client sqlite3 daemons json"
+GEMS="sqlite3 daemons json"
 for gem in $GEMS ; do
   if ! check_gem "$gem" ; then
     echo "+ Installing $gem (user-only)" >&2

data/utils/representatives.rb

@@ -0,0 +1,36 @@
+#!/usr/bin/env ruby
+
+$:.push File.expand_path('../../lib', __FILE__)
+require 'miga'
+
+proj_path = ARGV.shift or raise "Usage: #{$0} path/to/project"
+
+# Load MiGA object
+p = MiGA::Project.load(proj_path) or raise "Cannot load project: #{proj_path}"
+pr = p.result(:clade_finding) or raise "Unavailable result: clade_finding"
+pf = pr.file_path(:clades_ani95) or raise "Unavailable result file: proposal"
+
+# Read ANIspp
+ani_spp = []
+File.open(pf, 'r') do |fh|
+  fh.each_line do |ln|
+    next if $.==1
+    ani_spp << ln.chomp.split(',')
+  end
+end
+
+# Find the best candidate
+ani_spp.each_with_index do |datasets, i|
+  best = nil
+  datasets.each do |ds_name|
+    d = p.dataset(ds_name) or next
+    dr = d.result(:essential_genes) or next
+    q = dr[:stats][:quality] or next
+    if best.nil? or q > best[:q]
+      best = {d: d, q: q}
+    end
+  end
+  raise "Unavailable statistics for any of:\n#{datasets}\n" if best.nil?
+  puts "ANIsp_#{i+1}\t#{best[:d].name}"
+end
+

metadata

@@ -1,29 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: miga-base
 version: !ruby/object:Gem::Version
-  version: 0.3.4.2
+  version: 0.3.5.0
 platform: ruby
 authors:
 - Luis M. Rodriguez-R
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-09-06 00:00:00.000000000 Z
+date: 2018-10-04 00:00:00.000000000 Z
 dependencies:
-- !ruby/object:Gem::Dependency
-  name: rest-client
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.7'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.7'
 - !ruby/object:Gem::Dependency
   name: daemons
   requirement: !ruby/object:Gem::Requirement
@@ -473,6 +459,7 @@ files:
 - utils/mytaxa_scan.rb
 - utils/plot-taxdist.R
 - utils/ref-tree.R
+- utils/representatives.rb
 - utils/requirements.txt
 - utils/subclade/base.rb
 - utils/subclade/pipeline.rb