miga-base 1.3.10.1 → 1.3.10.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e3a7f2c88342fa48b9b5a1485c3b371bdff02e2e5bf19af7c24ab9b19b3fd373
-  data.tar.gz: cffaf705081cb22a9c3834e91a6ed46ccd86d138433d65a4e78269c6698f73be
+  metadata.gz: 601e480270ea7f04ff3f5057fa5f031146ac2e73247a7c191ea391a17e971bfc
+  data.tar.gz: 1577f19ec6b7798305a847da42556a16ff9e6a144c2e317e3349a8b49a0069fd
 SHA512:
-  metadata.gz: '093441822af4f362f210d0bbb0b4ade917757a5a5ba6938dec60038fdbdb73f75b0a479d2c930e223a54e81fce0ba9ab83b7f42ccc6a77944f33b66b53ad91a8'
-  data.tar.gz: 688e9925740ac96b258dd25521c5c5d63be05c2f0ebdf82a0b669bac0bd0d5c4def96537bcd4e1d411aeb03e6662b631447a5ed748c7de6c3631ea23f0d1021b
+  metadata.gz: '0293ffeea41a27ddef7a7b077d8cd93f86f5b9e95537576f8ce87120242cf3ced5f9554966cf648dcb817134977fbe6ee13a45b0815069cf2471014d780ce8c4'
+  data.tar.gz: b7ab71aa3a78d32861e1e9f6ff685424530093fd485185be722c6cc139a81d30692e00ccb6cbad4799bbcbd077ce32c7693db8336c98e8dfe0b18196748af174

data/lib/miga/cli/action/download/ncbi.rb

@@ -57,14 +57,14 @@ module MiGA::Cli::Action::Download::Ncbi
   def remote_list
     if cli[:ncbi_taxonomy_dump]
       cli.say "Reading NCBI Taxonomy dump: #{cli[:ncbi_taxonomy_dump]}"
-      MiGA::RemoteDataset.use_ncbi_taxonomy_dump(cli[:ncbi_taxonomy_dump])
+      MiGA::RemoteDataset.use_ncbi_taxonomy_dump(cli[:ncbi_taxonomy_dump], cli)
     end
 
     if cli[:ncbi_list_json] && File.size?(cli[:ncbi_list_json])
-      cli.say "Reusing remote list: #{cli[:ncbi_list_json]}"
-      return MiGA::Json.parse(cli[:ncbi_list_json])
+      return read_ncbi_list_json(cli[:ncbi_list_json])
     end
 
+    cli.say "Obtaining remote list of datasets"
     list  = {}
     query = remote_list_query
     loop do
@@ -79,18 +79,45 @@ module MiGA::Cli::Action::Download::Ncbi
       list.merge!(parse_reports_as_datasets(page[:reports]))
 
       # Next page
+      cli.advance('Datasets:', list.size, page[:total_count])
       break unless page[:next_page_token]
       query[:page_token] = page[:next_page_token]
     end
+    cli.say
 
-    if cli[:ncbi_list_json]
-      cli.say "Saving remote list: #{cli[:ncbi_list_json]}"
-      MiGA::Json.generate_fast(list, cli[:ncbi_list_json])
+    write_ncbi_list_json(cli[:ncbi_list_json], list) if cli[:ncbi_list_json]
+    list
+  end
+
+  def read_ncbi_list_json(file)
+    cli.say "Reusing remote list: #{file}"
+    list = {}
+    n_tot = nil
+    File.open(file, 'r') do |fh|
+      n_tot = fh.gets.chomp.sub(/^# /, '').to_i
+      fh.each_with_index do |ln, k|
+        row = ln.chomp.split("\t", 2)
+        list[row[0]] = MiGA::Json.parse(row[1], contents: true)
+        cli.advance('Lines:', k, n_tot)
+      end
+      cli.say
     end
+    return list
+  end
 
-    list
+  def write_ncbi_list_json(file, list)
+    cli.say "Saving remote list: #{file}"
+    File.open(file, 'w') do |fh|
+      fh.puts('# %i' % list.size)
+      kk = 0
+      list.each do |k, v|
+        fh.puts([k, MiGA::Json.generate_fast(v)].join("\t"))
+        cli.advance('Datasets:', kk += 1, list.size)
+      end
+      cli.say
+    end
   end
-  
+
   def parse_reports_as_datasets(reports)
     ds = {}
     reports.each do |r|

data/lib/miga/dataset.rb

@@ -3,6 +3,7 @@
 # @package MiGA
 # @license Artistic-2.0
 
+require'set'
 require 'miga/metadata'
 require 'miga/dataset/result'
 require 'miga/dataset/status'
@@ -27,7 +28,7 @@ class MiGA::Dataset < MiGA::MiGA
     ##
     # Does the +project+ already have a dataset with that +name+?
     def exist?(project, name)
-      !project.dataset_names_hash[name].nil?
+      project.dataset_names_set.include? name
     end
 
     ##

data/lib/miga/json.rb

@@ -15,6 +15,8 @@ class MiGA::Json < MiGA::MiGA
     # - +:symbolize+: If names should be symbolized. By default it's true if
     #   additions is false, or false otherwise. They can both be false, but an
     #   exception will be raised if both are true
+    # - +:large_file+: If passed, the file is treated as a file with very long
+    #   lines (possibly a single long line)
     def default_opts(opts = {})
       opts[:contents] ||= false
       opts[:additions] ||= false
@@ -36,11 +38,18 @@ class MiGA::Json < MiGA::MiGA
 
       # Read JSON
       cont = path
-      12.times do
-        cont = File.read(path)
-        break unless cont.empty?
-        sleep 1 # Wait up to 12 seconds for racing processes (iff empty file)
-      end unless opts[:contents]
+      if opts[:large_file]
+        cont = ''
+        File.open(path, 'r') do |fh|
+          cont += fh.read(2 ** 16) until fh.eof?
+        end
+      elsif !opts[:contents]
+        12.times do
+          cont = File.read(path)
+          break unless cont.empty?
+          sleep 1 # Wait up to 12 seconds for racing processes (iff empty file)
+        end
+      end
       raise "Empty descriptor: #{opts[:contents] ? "''" : path}" if cont.empty?
 
       # Parse JSON

data/lib/miga/metadata.rb

@@ -116,7 +116,7 @@ class MiGA::Metadata < MiGA::MiGA
   ##
   # Return the value of +k+ in #data
   def [](k)
-    if k.to_s =~ /(.+):(.+)/
+    if k.to_s =~ /^([^:]+):(.+)$/
       data[$1.to_sym]&.fetch($2)
     else
       data[k.to_sym]

data/lib/miga/project/dataset.rb

@@ -5,24 +5,32 @@
 # Helper module including specific functions handle datasets.
 module MiGA::Project::Dataset
   ##
-  # Returns Array of MiGA::Dataset.
+  # Returns Array of MiGA::Dataset
   def datasets
     metadata[:datasets].map { |name| dataset(name) }
   end
 
   ##
-  # Returns Array of String (without evaluating dataset objects).
+  # Returns Array of String (without evaluating dataset objects)
   def dataset_names
     metadata[:datasets]
   end
 
   ##
-  # Returns Hash of Strings => true. Similar to +dataset_names+ but as
-  # Hash for efficiency.
+  # Returns Hash of +{ String => true }+. Similar to +dataset_names+ but as
+  # Hash for efficiency
   def dataset_names_hash
+    warn 'The Project#dataset_names_hash method will be deprecated soon'
     @dataset_names_hash ||= Hash[dataset_names.map { |i| [i, true] }]
   end
 
+  ##
+  # Returns Set of Strings. Similar to +dataset_names+ but as Set for
+  # efficiency
+  def dataset_names_set
+    @dataset_names_set ||= Set.new(dataset_names)
+  end
+
   ##
   # Returns MiGA::Dataset
   def dataset(name)
@@ -50,7 +58,8 @@ module MiGA::Project::Dataset
     unless metadata[:datasets].include? name
       d = MiGA::Dataset.new(self, name)
       @metadata[:datasets] << name
-      @dataset_names_hash = nil # Ensure loading even if +do_not_save+ is true
+      @dataset_names_hash[name] = true if @dataset_names_hash
+      @dataset_names_set << name if @dataset_names_set
       save
       if d.ref? && d.active?
         recalculate_tasks("Reference dataset added: #{d.name}")

data/lib/miga/project.rb

@@ -77,6 +77,7 @@ class MiGA::Project < MiGA::MiGA
   def load
     @datasets = {}
     @dataset_names_hash = nil
+    @dataset_names_set = nil
     @metadata = MiGA::Metadata.load "#{path}/miga.project.json"
     raise "Couldn't find project metadata at #{path}" if metadata.nil?
 

data/lib/miga/remote_dataset.rb

@@ -16,7 +16,12 @@ class MiGA::RemoteDataset < MiGA::MiGA
     # Path to a directory with a recent NCBI Taxonomy dump to use instead of
     # making API calls to NCBI servers, which can be obtained at:
     # https://ftp.ncbi.nih.gov/pub/taxonomy/taxdump.tar.gz
-    def use_ncbi_taxonomy_dump(path)
+    #
+    # The +cli+ parameter, if passed, should be a MiGA::Cli object that will
+    # be used to report advance in the reading. Other objects can be passed,
+    # minimally supporting the MiGA::Cli#say and MiGA::Cli#advance method
+    # interfaces
+    def use_ncbi_taxonomy_dump(path, cli = nil)
       raise "Directory doesn't exist: #{path}" unless File.directory?(path)
 
       # Structure: { TaxID => ["name", "rank", parent TaxID] }
@@ -24,23 +29,31 @@ class MiGA::RemoteDataset < MiGA::MiGA
       @ncbi_taxonomy_names = {}
 
       # Read names.dmp
-      File.open(File.join(path, 'names.dmp')) do |fh|
+      File.open(file = File.join(path, 'names.dmp')) do |fh|
+        read = 0
+        size = File.size(file)
         fh.each do |ln|
+          cli&.advance('- names.dmp:', read += ln.size, size)
           row = ln.split(/\t\|\t?/)
           next unless row[3] == 'scientific name'
           @ncbi_taxonomy_names[row[0].to_i] = [row[1].strip]
         end
+        cli&.say
       end
 
       # Read nodes.dmp
-      File.open(File.join(path, 'nodes.dmp')) do |fh|
+      File.open(file = File.join(path, 'nodes.dmp')) do |fh|
+        read = 0
+        size = File.size(file)
         fh.each do |ln|
+          cli&.advance('- nodes.dmp:', read += ln.size, size)
           row = ln.split(/\t\|\t?/)
           child  = row[0].to_i
           parent = row[1].to_i
           @ncbi_taxonomy_names[child][1] = row[2]
           @ncbi_taxonomy_names[child][2] = parent unless parent == child
         end
+        cli&.say
       end
     end
 

data/lib/miga/taxonomy.rb

@@ -188,7 +188,7 @@ class MiGA::Taxonomy < MiGA::MiGA
     when Array, Hash
       self << str
     else
-      "#{str} ".scan(/([A-Za-z]+):([^:]*)( )/) { |r, n, _| self << { r => n } }
+      " #{str} ".scan(/(?<= )([A-Za-z]+):([^:]*) /) { |r, n| self << { r => n } }
     end
   end
 

data/lib/miga/version.rb

@@ -12,7 +12,7 @@ module MiGA
   # - String indicating release status:
   #   - rc* release candidate, not released as gem
   #   - [0-9]+ stable release, released as gem
-  VERSION = [1.3, 10, 1].freeze
+  VERSION = [1.3, 10, 2].freeze
 
   ##
   # Nickname for the current major.minor version.
@@ -20,7 +20,7 @@ module MiGA
 
   ##
   # Date of the current gem relese.
-  VERSION_DATE = Date.new(2024, 1, 31)
+  VERSION_DATE = Date.new(2024, 2, 6)
 
   ##
   # References of MiGA

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: miga-base
 version: !ruby/object:Gem::Version
-  version: 1.3.10.1
+  version: 1.3.10.2
 platform: ruby
 authors:
 - Luis M. Rodriguez-R
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2024-01-31 00:00:00.000000000 Z
+date: 2024-02-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: daemons