miga-base 1.3.9.1 → 1.3.9.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '09d0eb94fc3898aecede549b45d2a0478c944719c3e14b1efcefd83b8ba0f08d'
-  data.tar.gz: 667f5cca17495e22a1f80b396eb1d1d64b2f273c27dbab4628c6d758da51a4a6
+  metadata.gz: 32295b80e344eec3e534bfef0de472a19c14674c93a50ac6c066a3690be7499c
+  data.tar.gz: fa44c75572f39ae7dc60dabcdc2fcf11d2c17b5a4120dea0ea6fa66a4dc915ff
 SHA512:
-  metadata.gz: c73e6b163aa3be26e728744e29c8a1a6fcba73e3e27c9d2ff40604d0351a263fd6db6b9582bb4eef59a408f937af8ebac99d5360fc3cdc8c2ba1fd1c406c33c0
-  data.tar.gz: ffd3639052114019007db0600a0d84b272e9ef52c4d00f6e8f8ca5a172d36b05f51a365a038521415411f6eddb256132e9e98d1e22624713aed20e5149d0995c
+  metadata.gz: d3fd8c46e2daf0b0e6ee82435bc91b5d62784a36f2f2e0cff1b254d335dc6cecb9b5aacef2c982c743d026d87163049611a2bc73f99245f5253af8331be284b6
+  data.tar.gz: c3b3c69514dbb2cc035b78380f49789483ff96d510f7b044602e487512911bd8f6243451c38d473e27581a986302ae6f10e2ba4f464886c22273e6ef4ed066aa

data/lib/miga/cli/action/download/ncbi.rb

@@ -26,6 +26,9 @@ module MiGA::Cli::Action::Download::Ncbi
       cli[:scaffold] = true
       cli[:contig] = true
     end
+    opt.on('--ncbi-list-json STRING', '::HIDE::') do |v|
+      cli[:ncbi_list_json] = v
+    end
   end
 
   def cli_name_modifiers(opt)
@@ -35,7 +38,9 @@ module MiGA::Cli::Action::Download::Ncbi
       'Only affects --complete and --chromosome'
     ) { |v| cli[:add_version] = v }
     # For backwards compatibility
-    cli.opt_flag(opt, 'legacy-name', '::HIDE::', :legacy_name)
+    opt.on('--legacy-name', '::HIDE::') do
+      warn 'Deprecated flag --legacy-name ignored'
+    end
   end
 
   def sanitize_cli
@@ -49,6 +54,11 @@ module MiGA::Cli::Action::Download::Ncbi
   end
 
   def remote_list
+    if cli[:ncbi_list_json] && File.size?(cli[:ncbi_list_json])
+      cli.say "Reusing remote list: #{cli[:ncbi_list_json]}"
+      return MiGA::Json.parse(cli[:ncbi_list_json])
+    end
+
     list  = {}
     query = remote_list_query
     loop do
@@ -66,6 +76,12 @@ module MiGA::Cli::Action::Download::Ncbi
       break unless page[:next_page_token]
       query[:page_token] = page[:next_page_token]
     end
+
+    if cli[:ncbi_list_json]
+      cli.say "Saving remote list: #{cli[:ncbi_list_json]}"
+      MiGA::Json.generate_plain(list, cli[:ncbi_list_json])
+    end
+
     list
   end
   
@@ -80,7 +96,8 @@ module MiGA::Cli::Action::Download::Ncbi
       ds[n] = {
         ids: [asm], db: :assembly, universe: :ncbi,
         md: {
-          type: :genome, ncbi_asm: asm, strain: r.dig(:organism, :infraspecific_names, :strain)
+          type: :genome, ncbi_asm: asm,
+          strain: r.dig(:organism, :infraspecific_names, :strain)
         }
       }
       date = r.dig(:assembly_info, :release_date)

data/lib/miga/cli/action/get.rb

@@ -14,7 +14,7 @@ class MiGA::Cli::Action::Get < MiGA::Cli::Action
       opt.on(
         '-I', '--ids ID1,ID2,...', Array,
         '(Mandatory unless -F) IDs in the remote database separated by commas'
-      ) { |v| cli[:ids] = v }
+      ) { |v| cli[:ids] = v.map(&:strip) }
       opt.on(
         '-U', '--universe STRING',
         "Universe of the remote database. By default: #{cli[:universe]}",

data/lib/miga/json.rb

@@ -65,17 +65,26 @@ class MiGA::Json < MiGA::MiGA
     # Generates and returns prettyfied JSON to represent +obj+.
     # If +path+ is passed, it saves the JSON in that file.
     def generate(obj, path = nil)
-      y = JSON.pretty_generate(obj)
-      File.open(path, 'w') { |fh| fh.print y } unless path.nil?
-      y
+      generate_generic(:pretty_generate, obj, path)
     end
 
     ##
     # Generates and returns plain JSON to represent +obj+.
     # If +path+ is passed, it saves the JSON in that file.
     def generate_plain(obj, path = nil)
-      y = JSON.generate(obj)
-      File.open(path, 'w') { |fh| fh.print y } unless path.nil?
+      generate_generic(:generate, obj, path)
+    end
+
+    private
+
+    def generate_generic(method, obj, path)
+      y = JSON.send(method, obj)
+      return y unless path
+
+      io = StringIO.new(y)
+      File.open(path, 'w') do |fh|
+        fh.print(io.read(1024)) until io.eof?
+      end
       y
     end
   end

data/lib/miga/remote_dataset/base.rb

@@ -64,8 +64,8 @@ module MiGA::RemoteDataset::Base
     },
     gtdb: {
       dbs: {
-        # This is a dummy entry plugged directly to +ncbi_asm_rest+
-        assembly: { stage: :assembly, format: :fasta_gz, getter: :ncbi_asm },
+        # This is a dummy entry plugged directly to +ncbi_asm_get+
+        assembly: { stage: :assembly, format: :fasta, getter: :ncbi_asm },
         # The 'taxon' namespace actually returns a list of genomes (+format+)
         taxon: {
           stage: :metadata, format: :genomes, map_to: [:assembly],
@@ -84,8 +84,8 @@ module MiGA::RemoteDataset::Base
     },
     seqcode: {
       dbs: {
-        # These are dummy entries plugged directly to +ncbi_*_rest+
-        assembly: { stage: :assembly, format: :fasta_gz, getter: :ncbi_asm },
+        # These are dummy entries plugged directly to +ncbi_*_get+
+        assembly: { stage: :assembly, format: :fasta, getter: :ncbi_asm },
         nuccore:  { stage: :assembly, format: :fasta, getter: :ncbi_gb },
         # This is the list of type genomes
         :'type-genomes' => { stage: :metadata, format: :json }
@@ -100,7 +100,7 @@ module MiGA::RemoteDataset::Base
     ncbi: {
       dbs: {
         nuccore: { stage: :assembly, format: :fasta, getter: :ncbi_gb },
-        assembly: { stage: :assembly, format: :fasta_gz, getter: :ncbi_asm },
+        assembly: { stage: :assembly, format: :fasta, getter: :ncbi_asm },
         taxonomy: { stage: :metadata, format: :xml }
       },
       uri: lambda do |opts|

data/lib/miga/remote_dataset/download.rb

@@ -19,6 +19,10 @@ class MiGA::RemoteDataset
       getter = database_hash[:getter] || :download
       action = database_hash[:method] || universe_hash[:method]
 
+      # Clean IDs
+      ids = 
+
+      # Return options
       {
         universe: universe,  db:   db,    ids: ids.is_a?(Array) ? ids : [ids],
         format:   format,    file: file,  obj: obj,
@@ -50,22 +54,37 @@ class MiGA::RemoteDataset
     # Supported +opts+ (Hash) include:
     # +obj+ (mandatory): MiGA::RemoteDataset
     # +ids+ (mandatory): String or Array of String
-    # +file+: String, passed to download
+    # +file+ (mandatory): String, assembly saved here
     # +extra+: Hash, passed to download
-    # +format+: String, passed to download
+    # +format+: String, ignored
     def ncbi_asm_get(opts)
-      url_dir = opts[:obj].ncbi_asm_json_doc&.dig('ftppath_genbank')
-      if url_dir.nil? || url_dir.empty?
-        raise MiGA::RemoteDataMissingError.new(
-          "Missing ftppath_genbank in NCBI Assembly JSON"
-        )
-      end
+      require 'tempfile'
+      require 'zip'
 
-      url = '%s/%s_genomic.fna.gz' % [url_dir, File.basename(url_dir)]
-      download(
-        :web, :assembly_gz, url,
-        opts[:format], opts[:file], opts[:extra], opts[:obj]
+      zipped = download(
+        :ncbi_datasets_download, :genome, opts[:ids],
+        :zip, nil, opts[:extra], opts[:obj]
       )
+      zip_tmp = Tempfile.new('asm.zip')
+      zip_tmp.puts zipped
+      zip_tmp.close
+
+      o = ''
+      ofh = opts[:file] ? File.open(opts[:file], 'w') : nil
+      Zip::File.open(zip_tmp.path) do |zfh|
+        zfh.each do |entry|
+          if entry.file? && entry.name =~ /_genomic\.fna$/
+            DEBUG "Extracting: #{entry.name}"
+            entry.get_input_stream do |ifh|
+              cont = ifh.read
+              ofh&.puts cont
+              o += cont
+            end
+          end
+        end
+      end
+      ofh&.close
+      o
     end
 
     ##
@@ -77,11 +96,7 @@ class MiGA::RemoteDataset
       return o unless o.strip.empty?
 
       MiGA::MiGA.DEBUG 'Empty sequence, attempting download from NCBI assembly'
-      opts[:format] = :fasta_gz
-      if opts[:file]
-        File.unlink(opts[:file]) if File.exist? opts[:file]
-        opts[:file] = "#{opts[:file]}.gz"
-      end
+      opts[:format] = :fasta
       ncbi_asm_get(opts)
     end
 

data/lib/miga/result/stats.rb

@@ -29,33 +29,15 @@ module MiGA::Result::Stats
     seq_opts = { gc: true, x: true, skew: true }
     if self[:files][:pair1].nil?
       s = MiGA::MiGA.seqs_length(file_path(:single), :fastq, seq_opts)
-      stats = {
-        reads: s[:n],
-        length_average: [s[:avg], 'bp'],
-        length_standard_deviation: [s[:sd], 'bp'],
-        g_c_content: [s[:gc], '%'],
-        x_content: [s[:x], '%'],
-        g_c_skew: [s[:gc_skew], '%'],
-        a_t_skew: [s[:at_skew], '%']
-      }
+      stats = seqs_length_as_stats_hash(s)
     else
-      s1 = MiGA::MiGA.seqs_length(file_path(:pair1), :fastq, seq_opts)
-      s2 = MiGA::MiGA.seqs_length(file_path(:pair2), :fastq, seq_opts)
-      stats = {
-        read_pairs: s1[:n],
-        forward_length_average: [s1[:avg], 'bp'],
-        forward_length_standard_deviation: [s1[:sd], 'bp'],
-        forward_g_c_content: [s1[:gc], '%'],
-        forward_x_content: [s1[:x], '%'],
-        forward_g_c_skew: [s1[:gc_skew], '%'],
-        forward_a_t_skew: [s1[:at_skew], '%'],
-        reverse_length_average: [s2[:avg], 'bp'],
-        reverse_length_standard_deviation: [s2[:sd], 'bp'],
-        reverse_g_c_content: [s2[:gc], '%'],
-        reverse_x_content: [s2[:x], '%'],
-        reverse_g_c_skew: [s2[:gc_skew], '%'],
-        reverse_a_t_skew: [s2[:at_skew], '%']
-      }
+      stats = { read_pairs: nil }
+      { pair1: :forward, pair2: :reverse }.each do |pair, direction|
+        s = MiGA::MiGA.seqs_length(file_path(pair), :fastq, seq_opts)
+        seqs_length_as_stats_hash(s).each do |k, v|
+          stats[k == :reads ? :read_pairs : :"#{direction}_#{k}"] ||= v
+        end
+      end
     end
     stats
   end
@@ -63,15 +45,7 @@ module MiGA::Result::Stats
   def compute_stats_trimmed_fasta
     f = self[:files][:coupled].nil? ? file_path(:single) : file_path(:coupled)
     s = MiGA::MiGA.seqs_length(f, :fasta, gc: true, x: true, skew: true)
-    {
-      reads: s[:n],
-      length_average: [s[:avg], 'bp'],
-      length_standard_deviation: [s[:sd], 'bp'],
-      g_c_content: [s[:gc], '%'],
-      x_content: [s[:x], '%'],
-      g_c_skew: [s[:gc_skew], '%'],
-      a_t_skew: [s[:at_skew], '%']
-    }
+    seqs_length_as_stats_hash(s)
   end
 
   def compute_stats_assembly
@@ -79,16 +53,17 @@ module MiGA::Result::Stats
       file_path(:largecontigs), :fasta,
       n50: true, gc: true, x: true, skew: true
     )
+    h = seqs_length_as_stats_hash(s)
     {
       contigs: s[:n],
       n50: [s[:n50], 'bp'],
       total_length: [s[:tot], 'bp'],
-      longest_sequence: [s[:max], 'bp'],
-      g_c_content: [s[:gc], '%'],
-      x_content: [s[:x], '%'],
-      g_c_skew: [s[:gc_skew], '%'],
-      a_t_skew: [s[:at_skew], '%']
-    }
+      longest_sequence: [s[:max], 'bp']
+    }.tap do |stats|
+      %i[g_c_content x_content g_c_skew a_t_skew].each do |i|
+        stats[i] = h[i]
+      end
+    end
   end
 
   def compute_stats_cds
@@ -253,4 +228,16 @@ module MiGA::Result::Stats
     add_file(:raw_report, "#{source.name}.ess/log")
     add_file(:report, "#{source.name}.ess/log.domain")
   end
+
+  def seqs_length_as_stats_hash(s)
+    {
+      reads: s[:n],
+      length_average: [s[:avg], 'bp'],
+      length_standard_deviation: [s[:sd], 'bp'],
+      g_c_content: [s[:gc], '%'],
+      x_content: [s[:x], '%'],
+      g_c_skew: [s[:gc_skew], '%'],
+      a_t_skew: [s[:at_skew], '%']
+    }
+  end
 end

data/lib/miga/version.rb

@@ -12,7 +12,7 @@ module MiGA
   # - String indicating release status:
   #   - rc* release candidate, not released as gem
   #   - [0-9]+ stable release, released as gem
-  VERSION = [1.3, 9, 1].freeze
+  VERSION = [1.3, 9, 3].freeze
 
   ##
   # Nickname for the current major.minor version.

data/test/net_test.rb

@@ -47,4 +47,24 @@ class FormatTest < Test::Unit::TestCase
     ### m.download_file_ftp(:miga_db, '../api_test.txt', f)
     ### assert_equal('miga', File.read(f).chomp)
   end
+
+  def test_encoding
+    # Test original encoding
+    t1 = '()!@*#àøo'
+    t2 = "#{t1}"
+    assert_equal(t1, t2)
+    assert_equal(t1, MiGA::MiGA.normalize_encoding(t2))
+
+    # Test with a different encoding
+    t2 = t2.encode('windows-1252')
+    assert_equal('Windows-1252', t2.encoding.to_s)
+    assert_not_equal(t1, t2)
+    assert_equal(t1, MiGA::MiGA.normalize_encoding(t2))
+
+    # Test with a different encoding wrongly declared
+    t2.force_encoding('utf-8')
+    assert_equal('UTF-8', t2.encoding.to_s)
+    assert_not_equal(t1, t2)
+    assert_equal(t1, MiGA::MiGA.normalize_encoding(t2))
+  end
 end

data/test/remote_dataset_test.rb

@@ -142,7 +142,7 @@ class RemoteDatasetTest < Test::Unit::TestCase
 
   def test_missing_data
     declare_remote_access
-    rd = MiGA::RemoteDataset.new('GCA_000484975.1', :assembly, :ncbi)
+    rd = MiGA::RemoteDataset.new('XYZ_GCA_000484975.1', :assembly, :ncbi)
     assert_raise(MiGA::RemoteDataMissingError) { rd.save_to(project, 'bad') }
   end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: miga-base
 version: !ruby/object:Gem::Version
-  version: 1.3.9.1
+  version: 1.3.9.3
 platform: ruby
 authors:
 - Luis M. Rodriguez-R
@@ -80,6 +80,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: rubyzip
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement