miga-base 0.3.3.1 → 0.3.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b1b32b7800278dc330c5c8e01f4b94dfd1d97750
-  data.tar.gz: 2c3b6ef0e73568df8775fb98c65d454cdcf0f411
+  metadata.gz: b7140af2e2cb8525f5231ea065c79cb362348a9f
+  data.tar.gz: 257de78dbf14f7e01f4c239e7311883f98ccab21
 SHA512:
-  metadata.gz: 590a41c7bc94f5d36a53e0b9eb4f096211ccdae8724e63948480e0b57c8b7fa24a5779534868c7cb13405b3360f35be00d977728886ba7b7491ef5aeebb0bc0d
-  data.tar.gz: 64a273f14eea3aec6f9c8cfb388bdae4bdf2027d7ce95d89b1e8a27799e51420d2f58b42a58871037ee4a0e7f616f92e8ac485a12dbe226d7ab5cda99792f286
+  metadata.gz: 9a08c21c03a045d369078614dc41223db249b4105d20ece5198e4ef126dab468f7490278f50d7101652bba5433e4eee4653ea0fd67e85de55a78881d842d25ac
+  data.tar.gz: 62b07a2e62dbf801f5afbe87ac6bbb7c5a004ccc52153c640daf675e5e50e211fd568b7496d884a67759006d47e4ffd264dc280d28e299fb8351ba5ba14e5a6f

data/actions/about.rb

@@ -27,7 +27,7 @@ raise "Impossible to load project: #{o[:project]}" if p.nil?
 
 if not o[:datum].nil?
   v = p.metadata[ o[:datum] ]
-  puts v.nil? ? "?" : v
+  puts v.nil? ? '?' : v
 elsif o[:processing]
   keys = MiGA::Project.DISTANCE_TASKS + MiGA::Project.INCLADE_TASKS
   puts MiGA::MiGA.tabulate([:task, :status], keys.map do |k|

data/actions/ls.rb

@@ -13,7 +13,7 @@ OptionParser.new do |opt|
   opt.on("-p", "--processing",
     "Print information on processing advance."){ |v| o[:processing]=v }
   opt.on("-m", "--metadata STRING",
-    "Print name and metadata field only. If set, ignores -i."
+    "Print name and metadata field only. If set, ignores -i and assumes --tab."
     ){ |v| o[:datum]=v }
   opt.on("--tab",
     "Returns a tab-delimited table."){ |v| o[:tabular] = v }
@@ -43,7 +43,10 @@ ds = filter_datasets!(ds, o)
 exit(1) if o[:silent] and ds.empty?
 
 if not o[:datum].nil?
-  ds.each{|d| puts "#{d.name}\t#{d.metadata[ o[:datum] ] || "?"}"}
+  ds.each do |d|
+    v = d.metadata[ o[:datum] ]
+    puts "#{d.name}\t#{v.nil? ? '?' : v}"
+  end
 elsif o[:info]
   puts MiGA::MiGA.tabulate(
         MiGA::Dataset.INFO_FIELDS, ds.map{ |d| d.info }, o[:tabular])

data/actions/ncbi_get.rb

@@ -8,7 +8,8 @@ require 'miga/remote_dataset'
 o = {q:true, query:false, unlink:false,
       reference: false, ignore_plasmids: false,
       complete: false, chromosome: false,
-      scaffold: false, contig: false, add_version: true, dry: false}
+      scaffold: false, contig: false, add_version: true, dry: false,
+      get_md: false}
 OptionParser.new do |opt|
   opt_banner(opt)
   opt_object(opt, o, [:project])
@@ -37,6 +38,9 @@ OptionParser.new do |opt|
   opt.on('--blacklist PATH',
         'A file with dataset names to blacklist.'){ |v| o[:blacklist] = v }
   opt.on('--dry', 'Do not download or save the datasets.'){ |v| o[:dry] = v }
+  opt.on('--get-metadata',
+        'Only download and update metadata for existing datasets'
+        ){ |v| o[:get_md] = v }
   opt.on('-q', '--query',
         'Register the datasets as queries, not reference datasets.'
         ){ |v| o[:query]=v }
@@ -131,8 +135,7 @@ if o[:scaffold] or o[:contig]
           map{ |i| "#{i}/#{File.basename(i)}_genomic.fna.gz" }
     next if ids.empty?
     n = "#{r[0]}_#{asm}".miga_name
-    comm = "Assembly: #{asm}"
-    ds[n] = {ids: ids, md: {type: :genome, comments: comm},
+    ds[n] = {ids: ids, md: {type: :genome, ncbi_asm: asm},
           db: :assembly_gz, universe: :web}
   end
 end
@@ -144,23 +147,30 @@ unless o[:blacklist].nil?
 end
 
 # Download entries
-$stderr.puts "Downloading #{ds.size} #{ds.size>1 ? "entries" : "entry"}." unless o[:q]
+$stderr.puts "Downloading #{ds.size} " +
+  (ds.size > 1 ? "entries" : "entry") unless o[:q]
 ds.each do |name,body|
   d << name
   puts name
-  next unless p.dataset(name).nil?
+  next if p.dataset(name).nil? == o[:get_md]
   downloaded += 1
   next if o[:dry]
   $stderr.puts '  Locating remote dataset.' unless o[:q]
   rd = MiGA::RemoteDataset.new(body[:ids], body[:db], body[:universe])
-  $stderr.puts '  Creating dataset.' unless o[:q]
-  rd.save_to(p, name, !o[:query], body[:md])
-  p.add_dataset(name)
+  if o[:get_md]
+    $stderr.puts '  Updating dataset.' unless o[:q]
+    rd.update_metadata(p.dataset(name), body[:md])
+  else
+    $stderr.puts '  Creating dataset.' unless o[:q]
+    rd.save_to(p, name, !o[:query], body[:md])
+    p.add_dataset(name)
+  end
 end
 
 # Finalize
 $stderr.puts "Datasets listed: #{d.size}" unless o[:q]
-$stderr.puts "Datasets #{"to be " if o[:dry]}downloaded: #{downloaded}" unless o[:q]
+$stderr.puts "Datasets #{o[:dry] ? 'to download' : 'downloaded'}: " +
+  downloaded.to_s unless o[:q]
 unless o[:remote_list].nil?
   File.open(o[:remote_list], 'w') do |fh|
     d.each { |i| fh.puts i }

data/actions/stats.rb

@@ -45,14 +45,16 @@ if o[:compute]
   when :raw_reads
     if r[:files][:pair1].nil?
       s = MiGA::MiGA.seqs_length(r.file_path(:single), :fastq, gc: true)
-      stats = {reads: s[:n],
+      stats = {
+        reads: s[:n],
         length_average: [s[:avg], "bp"],
         length_standard_deviation: [s[:sd], "bp"],
         g_c_content: [s[:gc], "%"]}
     else
       s1 = MiGA::MiGA.seqs_length(r.file_path(:pair1), :fastq, gc: true)
       s2 = MiGA::MiGA.seqs_length(r.file_path(:pair2), :fastq, gc: true)
-      stats = {read_pairs: s1[:n],
+      stats = {
+        read_pairs: s1[:n],
         forward_length_average: [s1[:avg], "bp"],
         forward_length_standard_deviation: [s1[:sd], "bp"],
         forward_g_c_content: [s1[:gc], "%"],
@@ -63,22 +65,28 @@ if o[:compute]
   when :trimmed_fasta
     f = r[:files][:coupled].nil? ? r.file_path(:single) : r.file_path(:coupled)
     s = MiGA::MiGA.seqs_length(f, :fasta, gc: true)
-    stats = {reads: s[:n],
+    stats = {
+      reads: s[:n],
       length_average: [s[:avg], "bp"],
       length_standard_deviation: [s[:sd], "bp"],
       g_c_content: [s[:gc], "%"]}
   when :assembly
     s = MiGA::MiGA.seqs_length(r.file_path(:largecontigs), :fasta,
-      n50:true, gc:true)
-    stats = {contigs: s[:n], n50: [s[:n50], "bp"],
-      total_length: [s[:tot], "bp"], g_c_content: [s[:gc], "%"]}
+      n50: true, gc: true)
+    stats = {
+      contigs: s[:n],
+      n50: [s[:n50], "bp"],
+      total_length: [s[:tot], "bp"],
+      g_c_content: [s[:gc], "%"]}
   when :cds
     s = MiGA::MiGA.seqs_length(r.file_path(:proteins), :fasta)
-    stats = {predicted_proteins: s[:n], average_length: [s[:avg], "aa"]}
+    stats = {
+      predicted_proteins: s[:n],
+      average_length: [s[:avg], "aa"]}
     asm = d.add_result(:assembly, false)
     unless asm.nil? or asm[:stats][:total_length].nil?
       stats[:coding_density] =
-        [300.0*s[:tot]/asm[:stats][:total_length][0], "%"]
+        [300.0 * s[:tot] / asm[:stats][:total_length][0], "%"]
     end
   when :essential_genes
     if d.is_multi?
@@ -102,7 +110,7 @@ if o[:compute]
         r.add_file(:report, "#{d.name}.ess/log.archaea")
       end
       # Extract/compute quality values
-      stats = {completeness:[0.0,"%"], contamination:[0.0,"%"]}
+      stats = {completeness: [0.0,"%"], contamination: [0.0,"%"]}
       File.open(r.file_path(:report), "r") do |fh|
         fh.each_line do |ln|
           if /^! (Completeness|Contamination): (.*)%/.match(ln)
@@ -110,7 +118,7 @@ if o[:compute]
           end
         end
       end
-      stats[:quality] = stats[:completeness][0] - stats[:contamination][0]*5
+      stats[:quality] = stats[:completeness][0] - stats[:contamination][0] * 5
       d.metadata[:quality] = case stats[:quality]
         when 80..100 ; :excellent
         when 50..80  ; :high

data/lib/miga.rb

@@ -1,8 +1,8 @@
 # @package MiGA
 # @license Artistic-2.0
 
-require "json"
-require "fileutils"
-require "miga/common"
-require "miga/project"
-require "miga/taxonomy"
+require 'json'
+require 'fileutils'
+require 'miga/common'
+require 'miga/project'
+require 'miga/taxonomy'

data/lib/miga/common/format.rb

@@ -2,18 +2,20 @@
 require 'tempfile'
 require 'zlib'
 
+##
+# General formatting functions shared throughout MiGA.
 module MiGA::Common::Format
   ##
   # Tabulates an +values+, and Array of Arrays, all with the same number of
   # entries as +header+. Returns an Array of String, one per line.
-  def tabulate(header, values, tabular=false)
+  def tabulate(header, values, tabular = false)
     fields = [header.map(&:to_s)]
     fields << fields.first.map { |h| h.gsub(/\S/, '-') } unless tabular
     fields += values.map { |r| r.map { |cell| cell.nil? ? '?' : cell.to_s } }
     clen = tabular ? Array.new(header.size, 0) :
           fields.map { |r| r.map(&:length) }.transpose.map(&:max)
     fields.map do |r|
-      (0 .. clen.size - 1).map do |col_n|
+      (0..(clen.size - 1)).map do |col_n|
         col_n == 0 ? r[col_n].rjust(clen[col_n]) : r[col_n].ljust(clen[col_n])
       end.join(tabular ? "\t" : '  ')
     end
@@ -37,7 +39,7 @@ module MiGA::Common::Format
       fh.each_line do |ln|
         ln.chomp!
         if ln =~ /^>\s*(\S+)(.*)/
-          (id, df) = [$1, $2]
+          id, df = $1, $2
           tmp_fh.print buffer.wrap_width(80)
           buffer = ''
           tmp_fh.puts ">#{id.gsub(/[^A-Za-z0-9_\|\.]/, '_')}#{df}"
@@ -66,16 +68,17 @@ module MiGA::Common::Format
   # - +:n50+: If true, it also returns the N50 and the median (in bp).
   # - +gc+: If true, it also returns the G+C content (in %).
   def seqs_length(file, format, opts = {})
-    fh = (file =~ /\.gz/) ? Zlib::GzipReader.open(file) : File.open(file, 'r')
+    fh = file =~ /\.gz/ ? Zlib::GzipReader.open(file) : File.open(file, 'r')
     l = []
     gc = 0
-    i = 0 # <- Zlib::GzipReader doesn't set $.
+    i = 0 # <- Zlib::GzipReader doesn't set `$.`
     fh.each_line do |ln|
       i += 1
-      if (format == :fasta and ln =~ /^>/) or (format == :fastq and (i % 4)==1)
+      if (format == :fasta and ln =~ /^>/) or
+            (format == :fastq and (i % 4) == 1)
         l << 0
       elsif format == :fasta or (i % 4) == 2
-        l[l.size-1] += ln.chomp.size
+        l[l.size - 1] += ln.chomp.size
         gc += ln.scan(/[GCgc]/).count if opts[:gc]
       end
     end
@@ -131,4 +134,3 @@ class String
     gsub(/([^\n\r]{1,#{width}})/, "\\1\n")
   end
 end
-

data/lib/miga/dataset/result.rb

@@ -92,7 +92,7 @@ module MiGA::Dataset::Result
 
   ##
   # Are all the dataset-specific tasks done? Passes +save+ to #add_result.
-  def done_preprocessing?(save=false)
+  def done_preprocessing?(save = false)
     !first_preprocessing(save).nil? and next_preprocessing(save).nil?
   end
   
@@ -103,7 +103,7 @@ module MiGA::Dataset::Result
   # - 1 for a registered result (a completed task).
   # - 2 for a queued result (a task yet to be executed).
   # It passes +save+ to #add_result
-  def profile_advance(save=false)
+  def profile_advance(save = false)
     first_task = first_preprocessing(save)
     return Array.new(@@PREPROCESSING_TASKS.size, 0) if first_task.nil?
     adv = []
@@ -141,12 +141,12 @@ module MiGA::Dataset::Result
     ##
     # Add result type +:raw_reads+ at +base+ (no +_opts+ supported).
     def add_result_raw_reads(base, _opts)
-      return nil unless result_files_exist?(base, ".1.fastq")
+      return nil unless result_files_exist?(base, '.1.fastq')
       r = MiGA::Result.new("#{base}.json")
       add_files_to_ds_result(r, name,
-        ( result_files_exist?(base, ".2.fastq") ?
-          {pair1:".1.fastq", pair2:".2.fastq"} :
-          {single:".1.fastq"} ))
+        ( result_files_exist?(base, '.2.fastq') ?
+          {pair1: '.1.fastq', pair2: '.2.fastq'} :
+          {single: '.1.fastq'} ))
     end
 
     ##
@@ -156,13 +156,12 @@ module MiGA::Dataset::Result
       r = MiGA::Result.new("#{base}.json")
       if result_files_exist?(base, ".2.clipped.fastq")
         r = add_files_to_ds_result(r, name,
-          pair1:".1.clipped.fastq", pair2:".2.clipped.fastq",
-          single:".1.clipped.single.fastq")
+          pair1: ".1.clipped.fastq", pair2: ".2.clipped.fastq",
+          single: ".1.clipped.single.fastq")
       else
-        r = add_files_to_ds_result(r, name, single:".1.clipped.fastq")
+        r = add_files_to_ds_result(r, name, single: ".1.clipped.fastq")
       end
       r.add_file(:trimming_sumary, "#{name}.1.fastq.trimmed.summary.txt")
-      add_result(:raw_reads) #-> Post gunzip
       r
     end
 
@@ -171,10 +170,8 @@ module MiGA::Dataset::Result
     def add_result_read_quality(base, _opts)
       return nil unless result_files_exist?(base, %w[.solexaqa .fastqc])
       r = MiGA::Result.new("#{base}.json")
-      r = add_files_to_ds_result(r, name,
-        solexaqa:".solexaqa", fastqc:".fastqc")
-      add_result(:trimmed_reads) #-> Post cleaning
-      r
+      add_files_to_ds_result(r, name,
+        solexaqa: ".solexaqa", fastqc: ".fastqc")
     end
 
     ##
@@ -185,10 +182,8 @@ module MiGA::Dataset::Result
         result_files_exist?(base, ".SingleReads.fa") or
         result_files_exist?(base, %w[.1.fasta .2.fasta])
       r = MiGA::Result.new("#{base}.json")
-      r = add_files_to_ds_result(r, name, coupled:".CoupledReads.fa",
-        single:".SingleReads.fa", pair1:".1.fasta", pair2:".2.fasta")
-      add_result(:raw_reads) #-> Post gzip
-      r
+      add_files_to_ds_result(r, name, coupled: ".CoupledReads.fa",
+        single: ".SingleReads.fa", pair1: ".1.fasta", pair2: ".2.fasta")
     end
 
     ##
@@ -197,15 +192,14 @@ module MiGA::Dataset::Result
     def add_result_assembly(base, opts)
       return nil unless result_files_exist?(base, ".LargeContigs.fna")
       r = MiGA::Result.new("#{base}.json")
-      r = add_files_to_ds_result(r, name, largecontigs:".LargeContigs.fna",
-        allcontigs:".AllContigs.fna", assembly_data:"")
+      r = add_files_to_ds_result(r, name, largecontigs: ".LargeContigs.fna",
+        allcontigs: ".AllContigs.fna", assembly_data: '')
       opts[:is_clean] ||= false
       r.clean! if opts[:is_clean]
       unless r.clean?
         MiGA::MiGA.clean_fasta_file(r.file_path :largecontigs)
         r.clean!
       end
-      add_result(:trimmed_fasta) #-> Post interposing
       r
     end
 
@@ -214,8 +208,8 @@ module MiGA::Dataset::Result
     def add_result_cds(base, opts)
       return nil unless result_files_exist?(base, %w[.faa])
       r = MiGA::Result.new("#{base}.json")
-      r = add_files_to_ds_result(r, name, proteins:".faa", genes:".fna",
-        gff2:".gff2", gff3:".gff3", tab:".tab")
+      r = add_files_to_ds_result(r, name, proteins: ".faa", genes: ".fna",
+        gff2: ".gff2", gff3: ".gff3", tab: ".tab")
       opts[:is_clean] ||= false
       r.clean! if opts[:is_clean]
       unless r.clean?
@@ -231,8 +225,8 @@ module MiGA::Dataset::Result
     def add_result_essential_genes(base, _opts)
       return nil unless result_files_exist?(base, %w[.ess.faa .ess .ess/log])
       r = MiGA::Result.new("#{base}.json")
-      add_files_to_ds_result(r, name, ess_genes:".ess.faa",
-        collection:".ess", report:".ess/log")
+      add_files_to_ds_result(r, name, ess_genes: ".ess.faa",
+        collection: ".ess", report: ".ess/log")
     end
 
     ##
@@ -241,8 +235,8 @@ module MiGA::Dataset::Result
       return MiGA::Result.new("#{base}.json") if result(:assembly).nil?
       return nil unless result_files_exist?(base, ".ssu.fa")
       r = MiGA::Result.new("#{base}.json")
-      r = add_files_to_ds_result(r, name, longest_ssu_gene:".ssu.fa",
-        gff:".ssu.gff", all_ssu_genes:".ssu.all.fa")
+      r = add_files_to_ds_result(r, name, longest_ssu_gene: ".ssu.fa",
+        gff: ".ssu.gff", all_ssu_genes: ".ssu.all.fa")
       opts[:is_clean] ||= false
       r.clean! if opts[:is_clean]
       unless r.clean?
@@ -259,11 +253,11 @@ module MiGA::Dataset::Result
         return nil unless result_files_exist?(base, ".mytaxa") or
           result_files_exist?(base, ".nomytaxa.txt")
         r = MiGA::Result.new("#{base}.json")
-        add_files_to_ds_result(r, name, mytaxa:".mytaxa", blast:".blast",
-          mytaxain:".mytaxain", nomytaxa:".nomytaxa.txt",
-          species:".mytaxa.Species.txt", genus:".mytaxa.Genus.txt",
-          phylum:".mytaxa.Phylum.txt", innominate:".mytaxa.innominate",
-          kronain:".mytaxa.krona", krona:".html")
+        add_files_to_ds_result(r, name, mytaxa: ".mytaxa", blast: ".blast",
+          mytaxain: ".mytaxain", nomytaxa: ".nomytaxa.txt",
+          species: ".mytaxa.Species.txt", genus: ".mytaxa.Genus.txt",
+          phylum: ".mytaxa.Phylum.txt", innominate: ".mytaxa.innominate",
+          kronain: ".mytaxa.krona", krona: ".html")
       else
         MiGA::Result.new("#{base}.json")
       end
@@ -327,8 +321,8 @@ module MiGA::Dataset::Result
       return nil unless
         File.exist?("#{pref}/01.haai/#{name}.db")
       r = MiGA::Result.new("#{base}.json")
-      r.add_files(haai_db:"01.haai/#{name}.db", aai_db:"02.aai/#{name}.db",
-        ani_db:"03.ani/#{name}.db")
+      r.add_files(haai_db: "01.haai/#{name}.db", aai_db: "02.aai/#{name}.db",
+        ani_db: "03.ani/#{name}.db")
       r
     end
 
@@ -339,10 +333,10 @@ module MiGA::Dataset::Result
         result_files_exist?(base, %w[.aai-medoids.tsv .aai.db]) or
         result_files_exist?(base, %w[.ani-medoids.tsv .ani.db])
       r = MiGA::Result.new("#{base}.json")
-      add_files_to_ds_result(r, name, aai_medoids:".aai-medoids.tsv",
-        haai_db:".haai.db", aai_db:".aai.db", ani_medoids:".ani-medoids.tsv",
-        ani_db:".ani.db", ref_tree:".nwk", ref_tree_pdf:".nwk.pdf",
-        intax_test:".intax.txt")
+      add_files_to_ds_result(r, name, aai_medoids: ".aai-medoids.tsv",
+        haai_db: ".haai.db", aai_db: ".aai.db", ani_medoids: ".ani-medoids.tsv",
+        ani_db: ".ani.db", ref_tree: ".nwk", ref_tree_pdf: ".nwk.pdf",
+        intax_test: ".intax.txt")
     end
 
     ##

data/lib/miga/remote_dataset.rb

@@ -1,68 +1,12 @@
 # @package MiGA
 # @license Artistic-2.0
 
-require 'miga/remote_dataset/base'
+require 'miga/remote_dataset/download'
 
 ##
 # MiGA representation of datasets with data in remote locations.
 class MiGA::RemoteDataset < MiGA::MiGA
-
-  include MiGA::RemoteDataset::Base
-
-  # Class-level
-
-  ##
-  # Download data from the +universe+ in the database +db+ with IDs +ids+ and
-  # in +format+. If passed, it saves the result in +file+. Returns String.
-  def self.download(universe, db, ids, format, file=nil)
-    ids = [ids] unless ids.is_a? Array
-    case @@UNIVERSE[universe][:method]
-    when :rest
-      doc = download_rest(universe, db, ids, format)
-    when :net
-      doc = download_net(universe, db, ids, format)
-    end
-    unless file.nil?
-      ofh = File.open(file, "w")
-      ofh.print doc
-      ofh.close
-    end
-    doc
-  end
-
-  ##
-  # Download data using a REST method from the +universe+ in the database +db+
-  # with IDs +ids+ and in +format+. Returns the doc as String.
-  def self.download_rest(universe, db, ids, format)
-    u = @@UNIVERSE[universe]
-    map_to = u[:dbs][db].nil? ? nil : u[:dbs][db][:map_to]
-    url = sprintf(u[:url], db, ids.join(","), format, map_to)
-    response = RestClient::Request.execute(method: :get, url:url, timeout:600)
-    unless response.code == 200
-      raise "Unable to reach #{universe} client, error code #{response.code}."
-    end
-    response.to_s
-  end
-  
-  ##
-  # Download data using a GET request from the +universe+ in the database +db+
-  # with IDs +ids+ and in +format+. Returns the doc as String.
-  def self.download_net(universe, db, ids, format)
-    u = @@UNIVERSE[universe]
-    map_to = u[:dbs][db].nil? ? nil : u[:dbs][db][:map_to]
-    url = sprintf(u[:url], db, ids.join(","), format, map_to)
-    doc = ""
-    @timeout_try = 0
-    begin
-      open(url) { |f| doc = f.read }
-    rescue Net::ReadTimeout
-      @timeout_try += 1
-      if @timeout_try > 3 ; raise Net::ReadTimeout
-      else ; retry
-      end
-    end
-    doc
-  end
+  include MiGA::RemoteDataset::Download
 
   # Instance-level
 
@@ -71,7 +15,7 @@ class MiGA::RemoteDataset < MiGA::MiGA
   attr_reader :universe
   # Database storing the dataset.
   attr_reader :db
-  # IDs of the entries composing the dataset.
+  # Array of IDs of the entries composing the dataset.
   attr_reader :ids
 
   ##
@@ -81,92 +25,91 @@ class MiGA::RemoteDataset < MiGA::MiGA
     @ids = (ids.is_a?(Array) ? ids : [ids])
     @db = db.to_sym
     @universe = universe.to_sym
-    raise "Unknown Universe: #{@universe}. Try one of: "+
-      "#{@@UNIVERSE.keys}" unless @@UNIVERSE.keys.include? @universe
-    raise "Unknown Database: #{@db}. Try one of: "+
-      "#{@@UNIVERSE[@universe][:dbs]}" unless
-      @@UNIVERSE[@universe][:dbs].include? @db
-    # FIXME Part of the +map_to+ support:
-    #unless @@UNIVERSE[@universe][:dbs][@db][:map_to].nil?
-    #  MiGA::RemoteDataset.download
-    #end
+    @@UNIVERSE.keys.include?(@universe) or
+      raise "Unknown Universe: #{@universe}. Try: #{@@UNIVERSE.keys}"
+    @@UNIVERSE[@universe][:dbs].include?(@db) or
+      raise "Unknown Database: #{@db}. Try: #{@@UNIVERSE[@universe][:dbs]}"
+    # FIXME: Part of the +map_to+ support:
+    # unless @@UNIVERSE[@universe][:dbs][@db][:map_to].nil?
+    #   MiGA::RemoteDataset.download
+    # end
   end
 
   ##
   # Save dataset to the MiGA::Project +project+ identified with +name+. +is_ref+
   # indicates if it should be a reference dataset, and contains +metadata+.
-  def save_to(project, name=nil, is_ref=true, metadata={})
-    name ||= ids.join("_").miga_name
+  def save_to(project, name = nil, is_ref = true, metadata = {})
+    name ||= ids.join('_').miga_name
     project = MiGA::Project.new(project) if project.is_a? String
-    if MiGA::Dataset.exist?(project, name)
+    MiGA::Dataset.exist?(project, name) and
       raise "Dataset #{name} exists in the project, aborting..."
-    end
     metadata = get_metadata(metadata)
     udb = @@UNIVERSE[universe][:dbs][db]
-    metadata["#{universe}_#{db}"] = ids.join(",")
-    case udb[:stage]
-    when :assembly
-      dir = MiGA::Dataset.RESULT_DIRS[:assembly]
-      base = "#{project.path}/data/#{dir}/#{name}"
-      l_ctg = "#{base}.LargeContigs.fna"
-      a_ctg = "#{base}.AllContigs.fna"
-      File.open("#{base}.start", "w") { |ofh| ofh.puts Time.now.to_s }
-      if udb[:format] == :fasta_gz
-        download "#{l_ctg}.gz"
-        system "gzip -d '#{l_ctg}.gz'"
-      else
-        download l_ctg
-      end
-      File.unlink(a_ctg) if File.exist? a_ctg
-      File.symlink(File.basename(l_ctg), a_ctg)
-      File.open("#{base}.done", "w") { |ofh| ofh.puts Time.now.to_s }
-    else
-      raise "Unexpected error: Unsupported result for database #{db}."
-    end
+    metadata["#{universe}_#{db}"] = ids.join(',')
+    respond_to?("save_#{udb[:stage]}_to", true) or
+      raise "Unexpected error: Unsupported stage #{udb[:stage]} for #{db}."
+    send "save_#{udb[:stage]}_to", project, name, udb
     dataset = MiGA::Dataset.new(project, name, is_ref, metadata)
     project.add_dataset(dataset.name)
-    result = dataset.add_result(udb[:stage], true, is_clean:true)
-    raise "Empty dataset created: seed result was not added due to " +
-      "incomplete files." if result.nil?
+    result = dataset.add_result(udb[:stage], true, is_clean: true)
+    result.nil? and
+      raise 'Empty dataset: seed result not added due to incomplete files.'
     result.clean!
     result.save
     dataset
   end
 
+  ##
+  # Updates the MiGA::Dataset +dataset+ with the remotely available metadata,
+  # and optionally the Hash +metadata+.
+  def update_metadata(dataset, metadata = {})
+    metadata = get_metadata(metadata)
+    metadata.each { |k,v| dataset.metadata[k] = v }
+    dataset.save
+  end
+
   ##
   # Get metadata from the remote location.
-  def get_metadata(metadata={})
+  def get_metadata(metadata = {})
     case universe
     when :ebi, :ncbi
       # Get taxonomy
       metadata[:tax] = get_ncbi_taxonomy
     end
+    metadata[:"#{universe}_#{db}"] = ids.join(",")
+    metadata = get_type_status(metadata)
     metadata
   end
 
-  ##
-  # Download data into +file+.
-  def download(file)
-    MiGA::RemoteDataset.download(universe, db, ids,
-      @@UNIVERSE[universe][:dbs][db][:format], file)
-  end
-
   ##
   # Get NCBI Taxonomy ID.
   def get_ncbi_taxid
     send("get_ncbi_taxid_from_#{universe}")
   end
 
+  ##
+  # Get the type material status and return an (updated)
+  # +metadata+ hash.
+  def get_type_status(metadata)
+    if metadata[:ncbi_asm]
+      get_type_status_ncbi_asm metadata
+    elsif metadata[:ncbi_nuccore]
+      get_type_status_ncbi_nuccore metadata
+    else
+      metadata
+    end
+  end
+
   ##
   # Get NCBI taxonomy as MiGA::Taxonomy.
   def get_ncbi_taxonomy
     lineage = {}
     tax_id = get_ncbi_taxid
-    while !(tax_id.nil? or %w{0 1}.include? tax_id)
-      doc = MiGA::RemoteDataset.download(:ebi, :taxonomy, tax_id, "")
+    until [nil, '0', '1'].include? tax_id
+      doc = MiGA::RemoteDataset.download(:ebi, :taxonomy, tax_id, '')
       name = doc.scan(/SCIENTIFIC NAME\s+:\s+(.+)/).first.to_a.first
       rank = doc.scan(/RANK\s+:\s+(.+)/).first.to_a.first
-      rank = "dataset" if lineage.empty? and rank=="no rank"
+      rank = 'dataset' if lineage.empty? and rank == 'no rank'
       lineage[rank] = name unless rank.nil?
       tax_id = doc.scan(/PARENT ID\s+:\s+(.+)/).first.to_a.first
     end
@@ -174,24 +117,72 @@ class MiGA::RemoteDataset < MiGA::MiGA
   end
 
   private
-    
+
     def get_ncbi_taxid_from_ncbi
-      doc = MiGA::RemoteDataset.download(universe, db, ids, :gb).split(/\n/)
-      ln = doc.grep(/^\s+\/db_xref="taxon:/).first
+      doc = self.class.download(universe, db, ids, :gb).split(/\n/)
+      ln = doc.grep(%r{^\s+/db_xref="taxon:}).first
       return nil if ln.nil?
-      ln.sub!(/.*(?:"taxon:)(\d+)["; ].*/, "\\1")
+      ln.sub!(/.*(?:"taxon:)(\d+)["; ].*/, '\\1')
       return nil unless ln =~ /^\d+$/
       ln
     end
 
     def get_ncbi_taxid_from_ebi
-      doc = MiGA::RemoteDataset.download(universe, db, ids, :annot).split(/\n/)
-      ln = doc.grep(/^FT\s+\/db_xref="taxon:/).first
+      doc = self.class.download(universe, db, ids, :annot).split(/\n/)
+      ln = doc.grep(%r{^FT\s+/db_xref="taxon:}).first
       ln = doc.grep(/^OX\s+NCBI_TaxID=/).first if ln.nil?
       return nil if ln.nil?
-      ln.sub!(/.*(?:"taxon:|NCBI_TaxID=)(\d+)["; ].*/, "\\1")
+      ln.sub!(/.*(?:"taxon:|NCBI_TaxID=)(\d+)["; ].*/, '\\1')
       return nil unless ln =~ /^\d+$/
       ln
     end
 
+    def get_type_status_ncbi_nuccore(metadata)
+      return metadata if metadata[:ncbi_nuccore].nil?
+      biosample = self.class.ncbi_map(metadata[:ncbi_nuccore],
+        :nuccore, :biosample)
+      return metadata if biosample.nil?
+      asm = self.class.ncbi_map(biosample,
+        :biosample, :assembly)
+      metadata[:ncbi_asm] = asm.to_s unless asm.nil?
+      get_type_status_ncbi_asm metadata
+    end
+
+    def get_type_status_ncbi_asm(metadata)
+      return metadata if metadata[:ncbi_asm].nil?
+      doc = CGI.unescapeHTML(self.class.download(:web, :text,
+        "https://www.ncbi.nlm.nih.gov/assembly/" \
+          "#{metadata[:ncbi_asm]}?report=xml", :xml)).each_line
+      from_type = doc.grep(%r{<FromType/?>}).first or return metadata
+      if from_type =~ %r{<FromType/>}
+        metadata[:is_type] = false
+        metadata[:is_ref_type] = false
+      elsif from_type =~ %r{<FromType>(.*)</FromType>}
+        if $1 == 'assembly from reference material'
+          metadata[:is_type] = false
+          metadata[:is_ref_type] = true
+        else
+          metadata[:is_type] = true
+        end
+        metadata[:type_rel] = $1
+      end
+      metadata
+    end
+
+    def save_assembly_to(project, name, udb)
+      dir = MiGA::Dataset.RESULT_DIRS[:assembly]
+      base = "#{project.path}/data/#{dir}/#{name}"
+      l_ctg = "#{base}.LargeContigs.fna"
+      a_ctg = "#{base}.AllContigs.fna"
+      File.open("#{base}.start", 'w') { |ofh| ofh.puts Time.now.to_s }
+      if udb[:format] == :fasta_gz
+        download "#{l_ctg}.gz"
+        system "gzip -d '#{l_ctg}.gz'"
+      else
+        download l_ctg
+      end
+      File.unlink(a_ctg) if File.exist? a_ctg
+      File.symlink(File.basename(l_ctg), a_ctg)
+      File.open("#{base}.done", 'w') { |ofh| ofh.puts Time.now.to_s }
+    end
 end

data/lib/miga/remote_dataset/base.rb

@@ -1,6 +1,7 @@
 
-require 'restclient'
+require 'rest-client'
 require 'open-uri'
+require 'cgi'
 
 class MiGA::RemoteDataset < MiGA::MiGA
 
@@ -13,7 +14,7 @@ end
 
 module MiGA::RemoteDataset::Base
 
-  @@_EUTILS = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/"
+  @@_EUTILS = 'https://eutils.ncbi.nlm.nih.gov/entrez/eutils/'
 
   ##
   # Structure of the different database Universes or containers. The structure
@@ -23,33 +24,38 @@ module MiGA::RemoteDataset::Base
   #   properties such as +stage+, +format+, and +map_to+.
   # - +url+ => Pattern of the URL where the data can be obtained, where +%1$s+
   #   is the name of the database, +%2$s+ is the IDs, and +%3$s+ is format.
-  # - +method+ => Method used to query the URL. Only +:rest+ is currently
-  #   supported.
+  #   Additional parameters can be passed to certain functions using the +extra+
+  #   option.
+  # - +method+ => Method used to query the URL. Only +:rest+ and +:net+ are
+  #   currently supported.
   # - +map_to_universe+ => Universe where results map to. Currently unsupported.
   @@UNIVERSE = {
-    web:{
+    web: {
       dbs: {
-        assembly:{stage: :assembly, format: :fasta},
-        assembly_gz:{stage: :assembly, format: :fasta_gz}
+        assembly: {stage: :assembly, format: :fasta},
+        assembly_gz: {stage: :assembly, format: :fasta_gz},
+        text: {stage: :metadata, format: :text}
       },
       url: "%2$s",
       method: :net
     },
-    ebi:{
-      dbs: { embl:{stage: :assembly, format: :fasta} },
-      url: "http://www.ebi.ac.uk/Tools/dbfetch/dbfetch/%1$s/%2$s/%3$s",
+    ebi: {
+      dbs: { embl: {stage: :assembly, format: :fasta} },
+      url: "https://www.ebi.ac.uk/Tools/dbfetch/dbfetch/%1$s/%2$s/%3$s",
       method: :rest
     },
-    ncbi:{
-      dbs: { nuccore:{stage: :assembly, format: :fasta} },
+    ncbi: {
+      dbs: { nuccore: {stage: :assembly, format: :fasta} },
       url: "#{@@_EUTILS}efetch.fcgi?db=%1$s&id=%2$s&rettype=%3$s&retmode=text",
       method: :rest
     },
-    ncbi_map:{
-      dbs: { assembly:{map_to: :nuccore, format: :text} },
-        # FIXME ncbi_map is intended to do internal NCBI mapping between
-        # databases.
-      url: "#{@@_EUTILS}elink.fcgi?dbfrom=%1$s&id=%2$s&db=%3$s - - - - -",
+    ncbi_map: {
+      dbs: {
+        nuccore: {stage: :metadata, map_to: [:biosample, :assembly],
+          format: :json},
+        biosample: {stage: :metadata, map_to: [:assembly], format: :json}
+      },
+      url: "#{@@_EUTILS}elink.fcgi?dbfrom=%1$s&id=%2$s&db=%4$s&retmode=%3$s",
       method: :rest,
       map_to_universe: :ncbi
     }

data/lib/miga/remote_dataset/download.rb

@@ -0,0 +1,84 @@
+
+require 'miga/remote_dataset/base'
+
+class MiGA::RemoteDataset
+  include MiGA::RemoteDataset::Base
+
+  # Class-level
+  class << self
+    ##
+    # Download data from the +universe+ in the database +db+ with IDs +ids+ and
+    # in +format+. If passed, it saves the result in +file+. Additional
+    # parameters specific to the download method can be passed using +extra+.
+    # Returns String.
+    def download(universe, db, ids, format, file = nil, extra = [])
+      ids = [ids] unless ids.is_a? Array
+      case @@UNIVERSE[universe][:method]
+      when :rest
+        doc = download_rest(universe, db, ids, format, extra)
+      when :net
+        doc = download_net(universe, db, ids, format, extra)
+      end
+      unless file.nil?
+        ofh = File.open(file, 'w')
+        ofh.print doc
+        ofh.close
+      end
+      doc
+    end
+
+    ##
+    # Download data using a REST method from the +universe+ in the database +db+
+    # with IDs +ids+ and in +format+. Additional URL parameters can be passed
+    # using +extra+. Returns the doc as String.
+    def download_rest(universe, db, ids, format, extra = [])
+      u = @@UNIVERSE[universe]
+      url ||= sprintf(u[:url], db, ids.join(","), format, *extra)
+      response = RestClient::Request.execute(method: :get, url:url, timeout:600)
+      unless response.code == 200
+        raise "Unable to reach #{universe} client, error code #{response.code}."
+      end
+      response.to_s
+    end
+
+    ##
+    # Download data using a GET request from the +universe+ in the database +db+
+    # with IDs +ids+ and in +format+. Additional URL parameters can be passed
+    # using +extra+. Returns the doc as String.
+    def download_net(universe, db, ids, format, extra = [])
+      u = @@UNIVERSE[universe]
+      url = sprintf(u[:url], db, ids.join(","), format, *extra)
+      doc = ""
+      @timeout_try = 0
+      begin
+        open(url) { |f| doc = f.read }
+      rescue Net::ReadTimeout
+        @timeout_try += 1
+        if @timeout_try > 3 ; raise Net::ReadTimeout
+        else ; retry
+        end
+      end
+      doc
+    end
+
+    ##
+    # Looks for the entry +id+ in +dbfrom+, and returns the linked
+    # identifier in +db+ (or nil).
+    def ncbi_map(id, dbfrom, db)
+      doc = download(:ncbi_map, dbfrom, id, :json, nil, [db])
+      return if doc.empty?
+      tree = JSON.parse(doc, symbolize_names: true)
+      tree.dig(:linksets, 0, :linksetdbs, 0, :links, 0)
+    end
+  end
+end
+
+module MiGA::RemoteDataset::Download
+
+  ##
+  # Download data into +file+.
+  def download(file)
+    self.class.download(universe, db, ids,
+          self.class.UNIVERSE[universe][:dbs][db][:format], file)
+  end
+end

data/lib/miga/result/dates.rb

@@ -1,24 +1,25 @@
 
-require "miga/result/base"
+require 'miga/result/base'
 
 ##
 # Helper module including date-specific functions for results.
 module MiGA::Result::Dates
-
   include MiGA::Result::Base
-  
+
   ##
-  # Returns the start date of processing as DateTime or +nil+ if it doesn't exist.
+  # Returns the start date of processing as DateTime or +nil+ if it doesn't
+  # exist.
   def started_at
     date_at :start
   end
 
   ##
-  # Returns the end (done) date of processing as DateTime or +nil+ if it doesn't exist.
+  # Returns the end (done) date of processing as DateTime or +nil+ if it doesn't
+  # exist.
   def done_at
     date_at :done
   end
-  
+
   ##
   # Time it took for the result to complete as Float in minutes.
   def running_time
@@ -27,16 +28,17 @@ module MiGA::Result::Dates
     (b - a).to_f * 24 * 60
   end
 
-
   private
     
     ##
     # Internal function to detect start and end dates
     def date_at(event)
-      f = path event
-      return nil unless File.size? f
-      DateTime.parse File.read(f)
+      date = self[event]
+      if date.nil?
+        f = path event
+        date = File.read(f) if File.size? f
+      end
+      date.nil? ? nil : DateTime.parse(date)
     end
-
 end
 

data/lib/miga/version.rb

@@ -10,7 +10,7 @@ module MiGA
   # - Float representing the major.minor version.
   # - Integer representing gem releases of the current version.
   # - Integer representing minor changes that require new version number.
-  VERSION = [0.3, 3, 1]
+  VERSION = [0.3, 4, 1]
   
   ##
   # Nickname for the current major.minor version.

data/scripts/assembly.bash

@@ -19,7 +19,7 @@ if [[ -s "$TF/$DATASET.1.fasta" \
   FastA.interpose.pl "$TF/$DATASET.CoupledReads.fa" "$TF/$DATASET".[12].fasta
   gzip -9 -f "$TF/$DATASET.1.fasta"
   gzip -9 -f "$TF/$DATASET.2.fasta"
-  miga add_result -P "$PROJECT" -D "$DATASET" -r trimmed_fasta
+  miga add_result -P "$PROJECT" -D "$DATASET" -r trimmed_fasta -f
 fi
 
 # Assemble

data/scripts/read_quality.bash

@@ -28,6 +28,7 @@ rm -f "../02.trimmed_reads/$b".[12].fastq.trimmed.paired
 rm -f "../02.trimmed_reads/$b".[12].fastq.trimmed.single
 rm -f "../02.trimmed_reads/$b".[12].fastq.trimmed
 rm -f "../02.trimmed_reads/$b".[12].fastq
+miga add_result -P "$PROJECT" -D "$DATASET" -r trimmed_reads -f
 
 # Finalize
 miga date > "$DATASET.done"

data/scripts/trimmed_fasta.bash

@@ -19,6 +19,7 @@ for sis in 1 2 ; do
     && ! -e "../02.trimmed_reads/$b.$sis.clipped.fastq" ]] \
       && gunzip "../02.trimmed_reads/$b.$sis.clipped.fastq.gz"
 done
+miga add_result -P "$PROJECT" -D "$DATASET" -r trimmed_reads -f
 
 # FastQ -> FastA
 FQ2A="$MIGA/utils/enveomics/Scripts/FastQ.toFastA.awk"
@@ -44,6 +45,8 @@ for sis in 1 2 ; do
   [[ -e "../02.trimmed_reads/$b.$sis.clipped.single.fastq" ]] \
     && gzip -9 -f "../02.trimmed_reads/$b.$sis.clipped.single.fastq"
 done
+miga add_result -P "$PROJECT" -D "$DATASET" -r raw_reads -f
+miga add_result -P "$PROJECT" -D "$DATASET" -r trimmed_reads -f
 
 # Finalize
 miga date > "$DATASET.done"

data/scripts/trimmed_reads.bash

@@ -18,6 +18,7 @@ miga date > "$DATASET.start"
   && gunzip "../01.raw_reads/$b.1.fastq.gz"
 [[ -e "../01.raw_reads/$b.2.fastq.gz" && ! -e "../01.raw_reads/$b.2.fastq" ]] \
   && gunzip "../01.raw_reads/$b.2.fastq.gz"
+miga add_result -P "$PROJECT" -D "$DATASET" -r raw_reads -f
 
 # Clean existing files
 exists "$b".[12].* && rm "$b".[12].*

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: miga-base
 version: !ruby/object:Gem::Version
-  version: 0.3.3.1
+  version: 0.3.4.1
 platform: ruby
 authors:
 - Luis M. Rodriguez-R
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-08-29 00:00:00.000000000 Z
+date: 2018-09-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client
@@ -162,6 +162,7 @@ files:
 - lib/miga/project/result.rb
 - lib/miga/remote_dataset.rb
 - lib/miga/remote_dataset/base.rb
+- lib/miga/remote_dataset/download.rb
 - lib/miga/result.rb
 - lib/miga/result/base.rb
 - lib/miga/result/dates.rb
@@ -499,7 +500,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '1.9'
+      version: '2.3'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="