miga-base 0.3.5.0 → 0.3.5.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: c7f9c242373a48634effe2905d33ddd49c2c1bce
-  data.tar.gz: cbdf5d91e364987718b6f0d29b57a0bab372afdc
+SHA256:
+  metadata.gz: 25f6a339288dbdbeda1f84f5da6b5de697d3790c0b38d138a4416ff762cae936
+  data.tar.gz: 60187223750e983fafd6088935a912d016bbf4acafaf79b548e206c50076cc04
 SHA512:
-  metadata.gz: d790bb6056fc556ae86915093d6d16973222009e3249f1d33351ae9e35ab11614ea9163722a63592e2e1e95a0ea1345a537f98f8ffa675ef421ce6dd1f077642
-  data.tar.gz: d435fcda801d87f51f1dcc8d090812c7fbae12e74dfbbea5470ed573a993ad61edcf6855376f2ef3ea9acb9900407072b827852716c24ab64c4252e87a982d95
+  metadata.gz: 60b59ccc8fc3bf9f5a584f3221c268839ba4cd41eb3a6c16911f1808944c20100758f74908797f13ada9d2475e4fda382c1e204b257cf6c6adcfd17db694efcf
+  data.tar.gz: e62de30119c51dc4e92e64aec954c263f84a4820f1541f014c49d93dadd8de1efb5d331c35ecbda6ccfa79c402393894b7b6ffd52edf2362b8e117fe505ff230

data/actions/add.rb

@@ -60,7 +60,9 @@ def cp_result(o, d, p, sym, res_sym, ext)
   r_dir  = MiGA::Dataset.RESULT_DIRS[res_sym]
   r_path = File.expand_path("data/#{r_dir}/#{d.name}", p.path)
   ext.each_index do |i|
-    FileUtils.cp o[sym][i], "#{r_path}#{ext[i]}" unless o[sym][i].nil?
+    next if o[sym][i].nil?
+    gz = o[sym][i] =~ /\.gz/ ? '.gz' : ''
+    FileUtils.cp(o[sym][i], "#{r_path}#{ext[i]}#{gz}")
   end
   File.open("#{r_path}.done", "w") { |f| f.print Time.now.to_s }
 end

data/actions/get.rb

@@ -5,7 +5,7 @@
 
 require 'miga/remote_dataset'
 
-o = {q: true, query: false, universe: :ebi, db: :embl}
+o = {q: true, query: false, universe: :ebi, db: :embl, get_md: false}
 OptionParser.new do |opt|
   opt_banner(opt)
   opt_object(opt, o, [:project, :dataset, :dataset_type])
@@ -36,6 +36,9 @@ OptionParser.new do |opt|
     'Metadata as key-value pairs separated by = and delimited by comma.',
     'Values are saved as strings except for booleans (true / false) or nil.'
     ){ |v| o[:metadata]=v }
+  opt.on('--get-metadata',
+    'Only download and update metadata for existing datasets'
+    ){ |v| o[:get_md] = v }
   opt_common(opt, o)
 end.parse!
 
@@ -74,12 +77,20 @@ glob.each do |o_i|
   $stderr.puts 'Locating remote dataset.' unless o_i[:q]
   rd = MiGA::RemoteDataset.new(o_i[:ids], o_i[:db], o_i[:universe])
 
-  $stderr.puts 'Creating dataset.' unless o_i[:q]
-  dummy_d = MiGA::Dataset.new(p, o_i[:dataset])
-  md = add_metadata(o_i, dummy_d).metadata.data
-  dummy_d.remove!
-  rd.save_to(p, o_i[:dataset], !o_i[:query], md)
-  p.add_dataset(o_i[:dataset])
+  if o[:get_md]
+    $stderr.puts 'Updating dataset.' unless o_i[:q]
+    d = p.dataset(o_i[:dataset])
+    next if d.nil?
+    md = add_metadata(o_i, d).metadata.data
+    rd.update_metadata(d, md)
+  else
+    $stderr.puts 'Creating dataset.' unless o_i[:q]
+    dummy_d = MiGA::Dataset.new(p, o_i[:dataset])
+    md = add_metadata(o_i, dummy_d).metadata.data
+    dummy_d.remove!
+    rd.save_to(p, o_i[:dataset], !o_i[:query], md)
+    p.add_dataset(o_i[:dataset])
+  end
 
   $stderr.puts 'Done.' unless o_i[:q]
 end

data/bin/miga

@@ -19,7 +19,7 @@ $task_desc = {
   # Datasets
   add: "Creates an empty dataset in a pre-existing MiGA project.",
   get: "Downloads a dataset from public databases into a MiGA project.",
-  ncbi_get: "Downloads all genomes in a taxon or RefSeq from NCBI into a MiGA project.",
+  ncbi_get: "Downloads all genomes in a taxon from NCBI into a MiGA project.",
   rm: "Removes a dataset from an MiGA project.",
   find: "Finds unregistered datasets based on result files.",
   ln: "Link datasets (including results) from one project to another.",
@@ -137,13 +137,16 @@ def opt_common(opt, o)
 end
 
 # OptParse flags to filter lists of datasets.
-def opt_filter_datasets(opt, o, what=[:ref, :multi, :taxonomy])
+def opt_filter_datasets(opt, o, what=[:ref, :multi, :active, :taxonomy])
   opt.on("--[no-]ref",
     "If set, uses only reference (or only non-reference) datasets."
     ){ |v| o[:ref]=v } if what.include? :ref
   opt.on("--[no-]multi",
     "If set, uses only multi-species (or only single-species) datasets."
     ){ |v| o[:multi]=v } if what.include? :multi
+  opt.on("--[no-]active",
+    "If set, uses only active (or inactive) datasets."
+    ){ |v| o[:active]=v } if what.include? :active
   opt.on("-t", "--taxonomy RANK:TAXON", "Filter by taxonomy."
     ){ |v| o[:taxonomy]=MiGA::Taxonomy.new v } if what.include? :taxonomy
   opt.on("-k", "--key INTEGER",
@@ -163,7 +166,8 @@ end
 
 # Filters datasets by keys set in +opt_filter_datasets+.
 def filter_datasets!(ds, o)
-  ds.select!{|d| d.is_ref? == o[:ref] } unless o[:ref].nil?
+  ds.select! { |d| d.is_ref? == o[:ref] } unless o[:ref].nil?
+  ds.select! { |d| d.is_active? == o[:active] } unless o[:active].nil?
   ds.select! do |d|
     o[:multi] ? d.is_multi? : d.is_nonmulti?
   end unless o[:multi].nil?

data/lib/miga/dataset/result.rb

@@ -122,6 +122,7 @@ module MiGA::Dataset::Result
   # the project as reference datasets.
   def cleanup_distances!
     r = get_result(:distances)
+    ref = project.datasets.select(&:is_ref?).select(&:is_active?).map(&:name)
     return if r.nil?
     [:haai_db, :aai_db, :ani_db].each do |db_type|
       db = r.file_path(db_type)
@@ -130,7 +131,7 @@ module MiGA::Dataset::Result
       table = db_type[-6..-4]
       val = sqlite_db.execute "select seq2 from #{table}"
       next if val.empty?
-      (val.map{ |i| i.first } - project.dataset_names).each do |extra|
+      (val.map(&:first) - ref).each do |extra|
         sqlite_db.execute "delete from #{table} where seq2=?", extra
       end
     end

data/lib/miga/dataset.rb

@@ -17,7 +17,7 @@ class MiGA::Dataset < MiGA::MiGA
     ##
     # Does the +project+ already have a dataset with that +name+?
     def exist?(project, name)
-      project.dataset_names.include? name
+      not project.dataset_names_hash[name].nil?
     end
 
     ##

data/lib/miga/project/dataset.rb

@@ -5,7 +5,6 @@
 # Helper module including specific functions handle datasets.
 module MiGA::Project::Dataset
   
-
   ##
   # Returns Array of MiGA::Dataset.
   def datasets
@@ -17,6 +16,13 @@ module MiGA::Project::Dataset
   def dataset_names
     metadata[:datasets]
   end
+
+  ##
+  # Returns Hash of Strings => true. Similar to +dataset_names+ but as
+  # Hash for efficiency.
+  def dataset_names_hash
+    @dataset_names_hash ||= Hash[dataset_names.map{ |i| [i,true] }]
+  end
   
   ##
   # Returns MiGA::Dataset.

data/lib/miga/project.rb

@@ -63,6 +63,7 @@ class MiGA::Project < MiGA::MiGA
   # (Re-)load project data and metadata.
   def load
     @datasets = {}
+    @dataset_names_hash = nil
     @metadata = MiGA::Metadata.load "#{path}/miga.project.json"
     raise "Couldn't find project metadata at #{path}" if metadata.nil?
   end

data/lib/miga/remote_dataset/download.rb

@@ -50,10 +50,10 @@ class MiGA::RemoteDataset
       doc = ''
       @timeout_try = 0
       begin
-        open(url, open_timeout: 600, read_timeout: 600) { |f| doc = f.read }
-      rescue Net::ReadTimeout
+        open(url, read_timeout: 600) { |f| doc = f.read }
+      rescue => e
         @timeout_try += 1
-        raise Net::ReadTimeout if @timeout_try >= 3
+        raise e if @timeout_try >= 3
         retry
       end
       doc

data/lib/miga/remote_dataset.rb

@@ -17,6 +17,8 @@ class MiGA::RemoteDataset < MiGA::MiGA
   attr_reader :db
   # Array of IDs of the entries composing the dataset.
   attr_reader :ids
+  # Internal metadata hash
+  attr_reader :metadata
 
   ##
   # Initialize MiGA::RemoteDataset with +ids+ in database +db+ from +universe+.
@@ -25,6 +27,8 @@ class MiGA::RemoteDataset < MiGA::MiGA
     @ids = (ids.is_a?(Array) ? ids : [ids])
     @db = db.to_sym
     @universe = universe.to_sym
+    @metadata = {}
+    @metadata[:"#{universe}_#{db}"] = ids.join(",")
     @@UNIVERSE.keys.include?(@universe) or
       raise "Unknown Universe: #{@universe}. Try: #{@@UNIVERSE.keys}"
     @@UNIVERSE[@universe][:dbs].include?(@db) or
@@ -37,15 +41,15 @@ class MiGA::RemoteDataset < MiGA::MiGA
 
   ##
   # Save dataset to the MiGA::Project +project+ identified with +name+. +is_ref+
-  # indicates if it should be a reference dataset, and contains +metadata+.
-  def save_to(project, name = nil, is_ref = true, metadata = {})
+  # indicates if it should be a reference dataset, and contains +metadata_def+.
+  def save_to(project, name = nil, is_ref = true, metadata_def = {})
     name ||= ids.join('_').miga_name
     project = MiGA::Project.new(project) if project.is_a? String
     MiGA::Dataset.exist?(project, name) and
       raise "Dataset #{name} exists in the project, aborting..."
-    metadata = get_metadata(metadata)
+    @metadata = get_metadata(metadata_def)
     udb = @@UNIVERSE[universe][:dbs][db]
-    metadata["#{universe}_#{db}"] = ids.join(',')
+    @metadata["#{universe}_#{db}"] = ids.join(',')
     respond_to?("save_#{udb[:stage]}_to", true) or
       raise "Unexpected error: Unsupported stage #{udb[:stage]} for #{db}."
     send "save_#{udb[:stage]}_to", project, name, udb
@@ -70,15 +74,14 @@ class MiGA::RemoteDataset < MiGA::MiGA
 
   ##
   # Get metadata from the remote location.
-  def get_metadata(metadata = {})
+  def get_metadata(metadata_def = {})
+    metadata_def.each { |k,v| @metadata[k] = v }
     case universe
-    when :ebi, :ncbi
+    when :ebi, :ncbi, :web
       # Get taxonomy
-      metadata[:tax] = get_ncbi_taxonomy
+      @metadata[:tax] = get_ncbi_taxonomy
     end
-    metadata[:"#{universe}_#{db}"] = ids.join(",")
-    metadata = get_type_status(metadata)
-    metadata
+    @metadata = get_type_status(metadata)
   end
 
   ##
@@ -118,6 +121,15 @@ class MiGA::RemoteDataset < MiGA::MiGA
 
   private
 
+    def get_ncbi_taxid_from_web
+      return nil unless metadata[:ncbi_asm]
+      base_url = 'https://www.ncbi.nlm.nih.gov/assembly'
+      doc = self.class.download_url(
+        "#{base_url}/#{metadata[:ncbi_asm]}?report=xml&format=text")
+      taxid = doc.scan(%r{&lt;Taxid&gt;(\S+)&lt;/Taxid&gt;}).first
+      taxid.nil? ? taxid : taxid.first
+    end
+
     def get_ncbi_taxid_from_ncbi
       doc = self.class.download(universe, db, ids, :gb).split(/\n/)
       ln = doc.grep(%r{^\s+/db_xref="taxon:}).first

data/lib/miga/version.rb

@@ -10,7 +10,7 @@ module MiGA
   # - Float representing the major.minor version.
   # - Integer representing gem releases of the current version.
   # - Integer representing minor changes that require new version number.
-  VERSION = [0.3, 5, 0]
+  VERSION = [0.3, 5, 1]
   
   ##
   # Nickname for the current major.minor version.

data/scripts/aai_distances.bash

@@ -12,7 +12,7 @@ cd "$PROJECT/data/09.distances/02.aai"
 miga date > "miga-project.start"
 
 echo -n "" > miga-project.log
-DS=$(miga list_datasets -P "$PROJECT" --ref --no-multi)
+DS=$(miga ls -P "$PROJECT" --ref --no-multi --active)
 
 # Extract values
 echo "metric a b value sd n omega" | tr " " "\\t" >miga-project.txt

data/scripts/ani_distances.bash

@@ -12,7 +12,7 @@ cd "$PROJECT/data/09.distances/03.ani"
 miga date > "miga-project.start"
 
 echo -n "" > miga-project.log
-DS=$(miga list_datasets -P "$PROJECT" --ref --no-multi)
+DS=$(miga ls -P "$PROJECT" --ref --no-multi --active)
 
 # Extract values
 echo "metric a b value sd n omega" | tr " " "\\t" >miga-project.txt

data/scripts/haai_distances.bash

@@ -12,7 +12,7 @@ cd "$PROJECT/data/09.distances/01.haai"
 miga date > "miga-project.start"
 
 echo -n "" > miga-project.log
-DS=$(miga list_datasets -P "$PROJECT" --ref --no-multi)
+DS=$(miga ls -P "$PROJECT" --ref --no-multi --active)
 
 # Extract values
 echo "metric a b value sd n omega" | tr " " "\\t" >miga-project.txt

data/utils/enveomics/Pipelines/assembly.pbs/FastA.N50.pl

@@ -1 +1 @@
-../../Scripts/FastA.N50.pl
+utils/enveomics/Pipelines/assembly.pbs/../../Scripts/FastA.N50.pl

data/utils/enveomics/Pipelines/assembly.pbs/FastA.filterN.pl

@@ -1 +1 @@
-../../Scripts/FastA.filterN.pl
+utils/enveomics/Pipelines/assembly.pbs/../../Scripts/FastA.filterN.pl

data/utils/enveomics/Pipelines/assembly.pbs/FastA.length.pl

@@ -1 +1 @@
-../../Scripts/FastA.length.pl
+utils/enveomics/Pipelines/assembly.pbs/../../Scripts/FastA.length.pl

data/utils/enveomics/Pipelines/blast.pbs/FastA.split.pl

@@ -1 +1 @@
-../../Scripts/FastA.split.pl
+utils/enveomics/Pipelines/blast.pbs/../../Scripts/FastA.split.pl

data/utils/enveomics/Scripts/lib/enveomics.R

@@ -1 +1 @@
-../../enveomics.R
+utils/enveomics/Scripts/lib/../../enveomics.R

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: miga-base
 version: !ruby/object:Gem::Version
-  version: 0.3.5.0
+  version: 0.3.5.1
 platform: ruby
 authors:
 - Luis M. Rodriguez-R
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-10-04 00:00:00.000000000 Z
+date: 2018-11-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: daemons
@@ -495,7 +495,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.2.3
+rubygems_version: 2.7.7
 signing_key: 
 specification_version: 4
 summary: MiGA