miga-base 1.3.9.2 → 1.3.9.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2603688836b87b4be16c9c88bacbb206a93bd6b71b19a5a7febc219638f4e912
-  data.tar.gz: 1df2051e7d1d88facce4e0738828274ca9743807cdf90da959697addde25e424
+  metadata.gz: 436166fe9bbc76b69f014879610440e9646261f1a06dbc134418b79fa3da9b08
+  data.tar.gz: fb835c9bfa4829a3ac1c5ae8bdc1c5998cd8b9c4a55d3988538b9c24b2f4329f
 SHA512:
-  metadata.gz: a28d606569bdd37e5d4d33710564573639b103747c9622fae0a3a2d254f6293bf8462decb09bdf67d7656f84bca84fc7094e217c66ff081e75c5f5622bd9711c
-  data.tar.gz: 7e18ecf2487c31b5d99d3fb53b8716ce0dea2f2a8b2bab5015a871be9b703dd63c4384fb050380dad4dd28f5de3d1de5c9bcb1cb8544726870c7314fab626ca9
+  metadata.gz: 77e76a0dea664321aad58b4eee310d91cd9fa34c6ae0339451c83d2d169cbf9c160ed6c469401d0155280a28612ae8cde47a0c00be739371986f64621e44ef68
+  data.tar.gz: cdece24eb38d804b729c3c7f9018f7f3e835db258a59abe1ce560baf8bce29223a957f8498b96d6a647994361903d2d8179f2c6e6890451923dd44c5ae6cbcd8

data/lib/miga/cli/action/download/ncbi.rb

@@ -79,9 +79,7 @@ module MiGA::Cli::Action::Download::Ncbi
 
     if cli[:ncbi_list_json]
       cli.say "Saving remote list: #{cli[:ncbi_list_json]}"
-      File.open(cli[:ncbi_list_json], 'w') do |fh|
-        fh.puts MiGA::Json.generate_plain(list)
-      end
+      MiGA::Json.generate_plain(list, cli[:ncbi_list_json])
     end
 
     list

data/lib/miga/cli/action/get.rb

@@ -14,7 +14,7 @@ class MiGA::Cli::Action::Get < MiGA::Cli::Action
       opt.on(
         '-I', '--ids ID1,ID2,...', Array,
         '(Mandatory unless -F) IDs in the remote database separated by commas'
-      ) { |v| cli[:ids] = v }
+      ) { |v| cli[:ids] = v.map(&:strip) }
       opt.on(
         '-U', '--universe STRING',
         "Universe of the remote database. By default: #{cli[:universe]}",

data/lib/miga/json.rb

@@ -65,17 +65,26 @@ class MiGA::Json < MiGA::MiGA
     # Generates and returns prettyfied JSON to represent +obj+.
     # If +path+ is passed, it saves the JSON in that file.
     def generate(obj, path = nil)
-      y = JSON.pretty_generate(obj)
-      File.open(path, 'w') { |fh| fh.print y } unless path.nil?
-      y
+      generate_generic(:pretty_generate, obj, path)
     end
 
     ##
     # Generates and returns plain JSON to represent +obj+.
     # If +path+ is passed, it saves the JSON in that file.
     def generate_plain(obj, path = nil)
-      y = JSON.generate(obj)
-      File.open(path, 'w') { |fh| fh.print y } unless path.nil?
+      generate_generic(:generate, obj, path)
+    end
+
+    private
+
+    def generate_generic(method, obj, path)
+      y = JSON.send(method, obj)
+      return y unless path
+
+      io = StringIO.new(y)
+      File.open(path, 'w') do |fh|
+        fh.print(io.read(1024)) until io.eof?
+      end
       y
     end
   end

data/lib/miga/remote_dataset/base.rb

@@ -64,8 +64,8 @@ module MiGA::RemoteDataset::Base
     },
     gtdb: {
       dbs: {
-        # This is a dummy entry plugged directly to +ncbi_asm_rest+
-        assembly: { stage: :assembly, format: :fasta_gz, getter: :ncbi_asm },
+        # This is a dummy entry plugged directly to +ncbi_asm_get+
+        assembly: { stage: :assembly, format: :fasta, getter: :ncbi_asm },
         # The 'taxon' namespace actually returns a list of genomes (+format+)
         taxon: {
           stage: :metadata, format: :genomes, map_to: [:assembly],
@@ -84,8 +84,8 @@ module MiGA::RemoteDataset::Base
     },
     seqcode: {
       dbs: {
-        # These are dummy entries plugged directly to +ncbi_*_rest+
-        assembly: { stage: :assembly, format: :fasta_gz, getter: :ncbi_asm },
+        # These are dummy entries plugged directly to +ncbi_*_get+
+        assembly: { stage: :assembly, format: :fasta, getter: :ncbi_asm },
         nuccore:  { stage: :assembly, format: :fasta, getter: :ncbi_gb },
         # This is the list of type genomes
         :'type-genomes' => { stage: :metadata, format: :json }
@@ -100,7 +100,7 @@ module MiGA::RemoteDataset::Base
     ncbi: {
       dbs: {
         nuccore: { stage: :assembly, format: :fasta, getter: :ncbi_gb },
-        assembly: { stage: :assembly, format: :fasta_gz, getter: :ncbi_asm },
+        assembly: { stage: :assembly, format: :fasta, getter: :ncbi_asm },
         taxonomy: { stage: :metadata, format: :xml }
       },
       uri: lambda do |opts|

data/lib/miga/remote_dataset/download.rb

@@ -19,6 +19,10 @@ class MiGA::RemoteDataset
       getter = database_hash[:getter] || :download
       action = database_hash[:method] || universe_hash[:method]
 
+      # Clean IDs
+      ids = 
+
+      # Return options
       {
         universe: universe,  db:   db,    ids: ids.is_a?(Array) ? ids : [ids],
         format:   format,    file: file,  obj: obj,
@@ -50,22 +54,38 @@ class MiGA::RemoteDataset
     # Supported +opts+ (Hash) include:
     # +obj+ (mandatory): MiGA::RemoteDataset
     # +ids+ (mandatory): String or Array of String
-    # +file+: String, passed to download
+    # +file+ (mandatory): String, assembly saved here
     # +extra+: Hash, passed to download
-    # +format+: String, passed to download
+    # +format+: String, ignored
     def ncbi_asm_get(opts)
-      url_dir = opts[:obj].ncbi_asm_json_doc&.dig('ftppath_genbank')
-      if url_dir.nil? || url_dir.empty?
-        raise MiGA::RemoteDataMissingError.new(
-          "Missing ftppath_genbank in NCBI Assembly JSON"
-        )
-      end
+      require 'tempfile'
+      require 'zip'
 
-      url = '%s/%s_genomic.fna.gz' % [url_dir, File.basename(url_dir)]
-      download(
-        :web, :assembly_gz, url,
-        opts[:format], opts[:file], opts[:extra], opts[:obj]
+      zipped = download(
+        :ncbi_datasets_download, :genome, opts[:ids],
+        :zip, nil, opts[:extra], opts[:obj]
       )
+      zip_tmp = Tempfile.new('asm.zip')
+      zip_tmp.print(zipped)
+      zip_tmp.close
+
+      o = ''
+      ofh = opts[:file] ? File.open(opts[:file], 'w') : nil
+      Zip::File.open(zip_tmp.path) do |zfh|
+        zfh.each do |entry|
+          if entry.file? && entry.name =~ /_genomic\.fna$/
+            DEBUG "Extracting: #{entry.name}"
+            entry.get_input_stream do |ifh|
+              cont = MiGA::MiGA.normalize_encoding(ifh.read) + "\n"
+              ofh&.print(cont)
+              o += cont
+            end
+          end
+        end
+      end
+      ofh&.close
+      File.unlink(zip_tmp.path)
+      o
     end
 
     ##
@@ -77,11 +97,7 @@ class MiGA::RemoteDataset
       return o unless o.strip.empty?
 
       MiGA::MiGA.DEBUG 'Empty sequence, attempting download from NCBI assembly'
-      opts[:format] = :fasta_gz
-      if opts[:file]
-        File.unlink(opts[:file]) if File.exist? opts[:file]
-        opts[:file] = "#{opts[:file]}.gz"
-      end
+      opts[:format] = :fasta
       ncbi_asm_get(opts)
     end
 

data/lib/miga/version.rb

@@ -12,7 +12,7 @@ module MiGA
   # - String indicating release status:
   #   - rc* release candidate, not released as gem
   #   - [0-9]+ stable release, released as gem
-  VERSION = [1.3, 9, 2].freeze
+  VERSION = [1.3, 9, 4].freeze
 
   ##
   # Nickname for the current major.minor version.

data/test/net_test.rb

@@ -47,4 +47,24 @@ class FormatTest < Test::Unit::TestCase
     ### m.download_file_ftp(:miga_db, '../api_test.txt', f)
     ### assert_equal('miga', File.read(f).chomp)
   end
+
+  def test_encoding
+    # Test original encoding
+    t1 = '()!@*#àøo'
+    t2 = "#{t1}"
+    assert_equal(t1, t2)
+    assert_equal(t1, MiGA::MiGA.normalize_encoding(t2))
+
+    # Test with a different encoding
+    t2 = t2.encode('windows-1252')
+    assert_equal('Windows-1252', t2.encoding.to_s)
+    assert_not_equal(t1, t2)
+    assert_equal(t1, MiGA::MiGA.normalize_encoding(t2))
+
+    # Test with a different encoding wrongly declared
+    t2.force_encoding('utf-8')
+    assert_equal('UTF-8', t2.encoding.to_s)
+    assert_not_equal(t1, t2)
+    assert_equal(t1, MiGA::MiGA.normalize_encoding(t2))
+  end
 end

data/test/remote_dataset_test.rb

@@ -142,7 +142,7 @@ class RemoteDatasetTest < Test::Unit::TestCase
 
   def test_missing_data
     declare_remote_access
-    rd = MiGA::RemoteDataset.new('GCA_000484975.1', :assembly, :ncbi)
+    rd = MiGA::RemoteDataset.new('XYZ_GCA_000484975.1', :assembly, :ncbi)
     assert_raise(MiGA::RemoteDataMissingError) { rd.save_to(project, 'bad') }
   end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: miga-base
 version: !ruby/object:Gem::Version
-  version: 1.3.9.2
+  version: 1.3.9.4
 platform: ruby
 authors:
 - Luis M. Rodriguez-R
@@ -80,6 +80,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: rubyzip
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.3'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement