miga-base 0.3.6.2 → 0.3.6.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 18d4ed2342a1b2d1e1b35b1bbb060a79bd573af5cdc9c286fe9dc9d7b7b9c6aa
-  data.tar.gz: 723c593505c7a54c26316862d0a5eef7072dbd2de174154517f7cb14a8bd3bae
+  metadata.gz: f989fccd5161208979728e293bfc89d18e24e1ca6941b18a804063749cce56e9
+  data.tar.gz: 7568cb56974c18279428a50f750a5302beae8063a190def53734eb985fe4b56b
 SHA512:
-  metadata.gz: d955e1011f1f02730adb0fdf689740cb494b15552fccc07ce86c4427170f42f599a87fc527512809af99d0d7365fb7a5af55c101e87e7b16df013e3e648781e7
-  data.tar.gz: 63fa2a296df01ba78e281be1bf4fed252bdad7959d355d5b112228b25f71a79e1a6af1094a623967eaaf69431fb440529b481a9b5b9a06ff90c6660a22d2c73f
+  metadata.gz: 8d026577fd38d399d643fbc9ef7c01ffa7969d8206d6f859655b9d04e41d7c3e75b1432581add654c5b3d58ce6b2bedf566783c2587165f811e12f6fbf87b695
+  data.tar.gz: 4fae078b18a74314a7654d5c7a83ca7277555977b299931e33ddfabc3356cfa82994b00c6950a646eb8c16289f4cdb34f686574fc9505aacfec9ae826242c742

data/README.md

@@ -7,10 +7,7 @@
 
 # MiGA: Microbial Genomes Atlas
 
-**Important**: MiGA is under active development, and we currently cannot ensure
-any stability on the different interfaces. We're currently running a Beta
-Testing program with dedicated support for a small number of laboratories. If 
-you're interested, please [contact us][contact].
+![Graphical abstract](manual/img/Graphical_abstract-01.png)
 
 For additional information on MiGA, visit:
 

data/actions/get.rb

@@ -39,10 +39,11 @@ OptionParser.new do |opt|
   opt.on('--get-metadata',
     'Only download and update metadata for existing datasets'
     ){ |v| o[:get_md] = v }
+  opt.on('--api-key STRING',
+    'API key for the given universe.'){ |v| o[:api_key] = v }
   opt_common(opt, o)
 end.parse!
 
-
 ##=> Main <=
 glob = [o]
 unless o[:file].nil?
@@ -66,6 +67,9 @@ end
 
 glob.each do |o_i|
   opt_require(o_i, project: '-P', dataset: '-D', ids: '-I')
+  unless o_i[:api_key].nil?
+    ENV["#{o_i[:universe].to_s.upcase}_API_KEY"] = o_i[:api_key]
+  end
 
   $stderr.puts "Dataset: #{o_i[:dataset]}" unless o_i[:q]
   $stderr.puts 'Loading project.' unless o_i[:q]

data/actions/ncbi_get.rb

@@ -50,6 +50,7 @@ OptionParser.new do |opt|
   opt.on('-R', '--remote-list PATH',
         'Path to an output file with the list of all datasets listed remotely.'
         ){ |v| o[:remote_list]=v }
+  opt.on('--api-key STRING', 'NCBI API key.'){ |v| ENV['NCBI_API_KEY'] = v }
   opt_common(opt, o)
 end.parse!
 

data/lib/miga/daemon.rb

@@ -95,15 +95,18 @@ class MiGA::Daemon < MiGA::MiGA
   def load_status
     f_path = File.expand_path('daemon/status.json', project.path)
     return unless File.size? f_path
+    say 'Loading previous status in daemon/status.json:'
     status = JSON.parse(File.read(f_path), symbolize_names: true)
     status.keys.each do |i|
       status[i].map! do |j|
         j.tap { |k| k[:ds] = project.dataset(k[:ds_name]) unless k[:ds].nil? }
       end
     end
-    @jobs_to_run = status[:jobs_to_run]
     @jobs_running = status[:jobs_running]
+    @jobs_to_run  = status[:jobs_to_run]
     purge!
+    say "- jobs running: #{@jobs_running.size}"
+    say "- jobs to run: #{@jobs_to_run.size}"
   end
 
   ##
@@ -111,7 +114,7 @@ class MiGA::Daemon < MiGA::MiGA
   def check_datasets
     project.each_dataset do |n, ds|
       if ds.nil?
-        say "Warning: Dataset #{n} listed but not loaded, reloading project."
+        say "Warning: Dataset #{n} listed but not loaded, reloading project"
         project.load
       else
         to_run = ds.next_preprocessing(true)

data/lib/miga/remote_dataset/base.rb

@@ -27,6 +27,8 @@ module MiGA::RemoteDataset::Base
   #   option.
   # - +method+ => Method used to query the URL. Only +:rest+ and +:net+ are
   #   currently supported.
+  # - +api_key+ => A lambda function that takes a URL as input and returns the
+  #   URL to be downloaded with an API Key (if available).
   # - +map_to_universe+ => Universe where results map to. Currently unsupported.
   @@UNIVERSE = {
     web: {
@@ -46,7 +48,10 @@ module MiGA::RemoteDataset::Base
     ncbi: {
       dbs: { nuccore: {stage: :assembly, format: :fasta} },
       url: "#{@@_EUTILS}efetch.fcgi?db=%1$s&id=%2$s&rettype=%3$s&retmode=text",
-      method: :rest
+      method: :rest,
+      api_key: lambda { |url|
+        ENV['NCBI_API_KEY'].nil? ?
+          url : "#{url}&api_key=#{ENV['NCBI_API_KEY']}" }
     },
     ncbi_map: {
       dbs: {
@@ -56,7 +61,10 @@ module MiGA::RemoteDataset::Base
       },
       url: "#{@@_EUTILS}elink.fcgi?dbfrom=%1$s&id=%2$s&db=%4$s&retmode=%3$s",
       method: :net,
-      map_to_universe: :ncbi
+      map_to_universe: :ncbi,
+      api_key: lambda { |url|
+        ENV['NCBI_API_KEY'].nil? ?
+          url : "#{url}&api_key=#{ENV['NCBI_API_KEY']}" }
     }
   }
 

data/lib/miga/remote_dataset/download.rb

@@ -34,6 +34,7 @@ class MiGA::RemoteDataset
     def download_rest(universe, db, ids, format, extra = [])
       u = @@UNIVERSE[universe]
       url = sprintf(u[:url], db, ids.join(","), format, *extra)
+      url = u[:api_key][url] unless u[:api_key].nil?
       download_url url
     end
 

data/lib/miga/version.rb

@@ -10,7 +10,7 @@ module MiGA
   # - Float representing the major.minor version.
   # - Integer representing gem releases of the current version.
   # - Integer representing minor changes that require new version number.
-  VERSION = [0.3, 6, 2]
+  VERSION = [0.3, 6, 3]
   
   ##
   # Nickname for the current major.minor version.

data/scripts/clade_finding.bash

@@ -11,9 +11,6 @@ cd "$PROJECT/data/10.clades/01.find"
 # Initialize
 miga date > "miga-project.start"
 
-# Cleanup databases
-ruby -I "$MIGA/lib" "$MIGA/utils/cleanup-databases.rb" "$PROJECT" "$CORES"
-
 # Run
 ruby -I "$MIGA/lib" "$MIGA/utils/subclades.rb" "$PROJECT" "$SCRIPT"
 

data/scripts/haai_distances.bash

@@ -11,6 +11,10 @@ cd "$PROJECT/data/09.distances/01.haai"
 # Initialize
 miga date > "miga-project.start"
 
+# Cleanup databases
+ruby -I "$MIGA/lib" "$MIGA/utils/cleanup-databases.rb" "$PROJECT" "$CORES"
+
+# Run hAAI
 echo -n "" > miga-project.log
 DS=$(miga ls -P "$PROJECT" --ref --no-multi --active)
 

data/utils/distance/commands.rb

@@ -53,11 +53,11 @@ module MiGA::DistanceRunner::Commands
   end
 
   ##
-  # Calculates and returns ANI against +target+ if AAI >= 85%. Returns
-  # +nil+ otherwise
-  def ani_after_aai(target)
+  # Calculates and returns ANI against +target+ if AAI >= +aai_limit+.
+  # Returns +nil+ otherwise
+  def ani_after_aai(target, aai_limit = 85.0)
     aai = aai(target)
-    ani(target) unless aai.nil? or aai < 85.0
+    ani(target) unless aai.nil? or aai < aai_limit
   end
 
   ##

data/utils/distance/runner.rb

@@ -94,19 +94,19 @@ class MiGA::DistanceRunner
         fh.each_line do |ln|
           r = ln.chomp.split("\t")
           next unless r[1].to_i == val_cls
-          ani = ani_after_aai(ref_project.dataset(r[0]))
+          ani = ani_after_aai(ref_project.dataset(r[0]), 80.0)
           closest = {ds: r[0], ani: ani} unless ani.nil? or ani < closest[:ani]
         end
       end
     end
 
-    # Calculate all the AAIs/ANIs against the closest ANI95-clade (if ANI > 95%)
-    cl_path = File.expand_path('miga-project.ani95-clades', home)
-    if File.size? cl_path and tsk[0] == :clade_finding and closest[:ani] >= 95.0
+    # Calculate all the AAIs/ANIs against the closest ANI95-clade (if AAI > 80%)
+    cl_path = res.file_path :clades_ani95
+    if File.size? cl_path and tsk[0] == :clade_finding
       File.foreach(cl_path).
         map  { |i| i.chomp.split(',') }.
-        find { |i| i.include? closest[:ds] }.
-        each { |i| ani_after_aai(ref_project.dataset(i)) }
+        find( lambda{[]} ){ |i| i.include? closest[:ds] }.
+        each { |i| ani_after_aai(ref_project.dataset(i), 80.0) }
     end
 
     # Finalize

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: miga-base
 version: !ruby/object:Gem::Version
-  version: 0.3.6.2
+  version: 0.3.6.3
 platform: ruby
 authors:
 - Luis M. Rodriguez-R
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-12-06 00:00:00.000000000 Z
+date: 2018-12-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: daemons