miga-base 1.2.14.1 → 1.2.15.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ae083dbb06efae44846887887e0706b0436340aeb7524846478a68867c602b67
-  data.tar.gz: 78bb2440e073e4d55fe6a439dfa1eddb25c2fa4aad7d06000384de20092ab710
+  metadata.gz: 9de962d747353644dddff05181f74c87a3f401df14e81d1810f125c9d1843351
+  data.tar.gz: 3dd90251f07976c36adfdabaa603e399bc046f68c14ff83680eb6a6de4219159
 SHA512:
-  metadata.gz: 185667da9d0e7dc3849dc851e2f2c6829901bfaee7975c7b249630b660cc41aa40988fcc9e27c3937a6ad533df3135e4d2f2115e7f16e8bd3f25ed9e0533d9f7
-  data.tar.gz: 77ec658cbfcecda4b32e7a0110b53953220f649f79f4a112c23c510511a4752869af9e3894452335d827311a9b8fedd0c5072860ecff035f1c018aa346c9894f
+  metadata.gz: 68f3c9616c1d2ad0fed1f03fc6b1054d428fa8b61999febcf6394d34c261550e203b108d2399a43077d95980b6c623e40799e0bf233cfcc130ff5ef97fdc4e47
+  data.tar.gz: e95d549756ad10f86f4abf1761fb763e8ee218ff81d1cf8f6b597c831227482b6e90871800eb38863a8a12ed622b3222385b5f730e3e4103de06aa24753e8a3a

data/lib/miga/cli/action/download/base.rb

@@ -9,14 +9,18 @@ end
 module MiGA::Cli::Action::Download::Base
   def cli_filters(opt)
     opt.on(
-      '--blacklist PATH',
-      'A file with dataset names to blacklist'
-    ) { |v| cli[:blacklist] = v }
+      '--exclude PATH',
+      'A file with dataset names to exclude'
+    ) { |v| cli[:exclude] = v }
     cli.opt_flag(opt, 'dry', 'Do not download or save the datasets')
     opt.on(
       '--ignore-until STRING',
       'Ignores all datasets until a name is found (useful for large reruns)'
     ) { |v| cli[:ignore_until] = v }
+    opt.on(
+      '--ignore-removed',
+      'Ignores entries removed from NCBI (by default fails on removed entries)'
+    ) { |v| cli[:ignore_removed] = v }
     cli.opt_flag(
       opt, 'get-metadata',
       'Only download and update metadata for existing datasets', :get_md
@@ -49,10 +53,10 @@ module MiGA::Cli::Action::Download::Base
     ) { |v| cli[:remote_list] = v }
   end
 
-  def discard_blacklisted(ds)
-    unless cli[:blacklist].nil?
-      cli.say "Discarding datasets in #{cli[:blacklist]}"
-      File.readlines(cli[:blacklist])
+  def discard_excluded(ds)
+    unless cli[:exclude].nil?
+      cli.say "Discarding datasets in #{cli[:exclude]}"
+      File.readlines(cli[:exclude])
           .select { |i| i !~ /^#/ }
           .map(&:chomp)
           .each { |i| ds.delete i }
@@ -84,7 +88,11 @@ module MiGA::Cli::Action::Download::Base
 
       downloaded += 1
       unless cli[:dry]
-        save_entry(name, body, p)
+        unless save_entry(name, body, p)
+          downloaded -= 1
+          d.pop
+          next
+        end
         p.save! if cli[:save_every] > 1 && (downloaded % cli[:save_every]).zero?
       end
     end
@@ -93,6 +101,9 @@ module MiGA::Cli::Action::Download::Base
     [d, downloaded]
   end
 
+  ##
+  # Saves the (generic remote) entry identified by +name+ with +body+ into the
+  # project +p+, and returns +true+ on success and +false+ otherwise
   def save_entry(name, body, p)
     cli.say '  Locating remote dataset'
     body[:md][:metadata_only] = true if cli[:only_md]
@@ -105,5 +116,9 @@ module MiGA::Cli::Action::Download::Base
       rd.save_to(p, name, !cli[:query], body[:md])
       cli.add_metadata(p.add_dataset(name))
     end
+    true
+  rescue MiGA::RemoteDataMissingError => e
+    raise(e) unless cli[:ignore_removed]
+    false
   end
 end

data/lib/miga/cli/action/gtdb_get.rb

@@ -42,7 +42,7 @@ class MiGA::Cli::Action::GtdbGet < MiGA::Cli::Action
     sanitize_cli
     p = cli.load_project
     ds = remote_list
-    ds = discard_blacklisted(ds)
+    ds = discard_excluded(ds)
     ds = impose_limit(ds)
     d, downloaded = download_entries(ds, p)
 

data/lib/miga/cli/action/ncbi_get.rb

@@ -44,7 +44,7 @@ class MiGA::Cli::Action::NcbiGet < MiGA::Cli::Action
     sanitize_cli
     p = cli.load_project
     ds = remote_list
-    ds = discard_blacklisted(ds)
+    ds = discard_excluded(ds)
     ds = impose_limit(ds)
     d, downloaded = download_entries(ds, p)
 

data/lib/miga/common/errors.rb

@@ -9,4 +9,14 @@ module MiGA
   # An error with a system call
   class SystemCallError < Error
   end
+
+  ##
+  # An error with remote data
+  class RemoteDataError < Error
+  end
+
+  ##
+  # An error caused by missing remote data
+  class RemoteDataMissingError < RemoteDataError
+  end
 end

data/lib/miga/remote_dataset/download.rb

@@ -43,6 +43,12 @@ class MiGA::RemoteDataset
     # +format+: String, passed to download
     def ncbi_asm_rest(opts)
       url_dir = opts[:obj].ncbi_asm_json_doc['ftppath_genbank']
+      if url_dir.nil? || url_dir.empty?
+        raise MiGA::RemoteDataMissingError.new(
+          "Missing ftppath_genbank in NCBI Assembly JSON"
+        )
+      end
+
       url = "#{url_dir}/#{File.basename url_dir}_genomic.fna.gz"
       download(
         :web, :assembly_gz, url,

data/lib/miga/version.rb

@@ -12,7 +12,7 @@ module MiGA
   # - String indicating release status:
   #   - rc* release candidate, not released as gem
   #   - [0-9]+ stable release, released as gem
-  VERSION = [1.2, 14, 1].freeze
+  VERSION = [1.2, 15, 0].freeze
 
   ##
   # Nickname for the current major.minor version.
@@ -20,7 +20,7 @@ module MiGA
 
   ##
   # Date of the current gem relese.
-  VERSION_DATE = Date.new(2023, 1, 16)
+  VERSION_DATE = Date.new(2023, 1, 17)
 
   ##
   # References of MiGA

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: miga-base
 version: !ruby/object:Gem::Version
-  version: 1.2.14.1
+  version: 1.2.15.0
 platform: ruby
 authors:
 - Luis M. Rodriguez-R
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-01-16 00:00:00.000000000 Z
+date: 2023-01-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: daemons