miga-base 1.3.10.0 → 1.3.10.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '0239e39a0588b73d042da7d970925d2d93a5334c858453e032d51b0af760fa27'
-  data.tar.gz: 81e6903e1feba6571d76fe5d113a60414bd0d3b1b3090d6e26367a93cf8d0da7
+  metadata.gz: 601e480270ea7f04ff3f5057fa5f031146ac2e73247a7c191ea391a17e971bfc
+  data.tar.gz: 1577f19ec6b7798305a847da42556a16ff9e6a144c2e317e3349a8b49a0069fd
 SHA512:
-  metadata.gz: d68e55d5335f3da03eb9cea737aad5fa21a7a272e3958db6130e7260387844c1bed92b0b2f655a5a5133772797212b717559da9701723c9885cc9ee7cffc962f
-  data.tar.gz: 8d27c2f580106c0d1f74e6daaf1cb81ffd7c6fbabf86e7569342027ff9e09b1810c245ad560eff0179c3d06a88b2d1a087344c56e34a2d54c799004e2f6370c0
+  metadata.gz: '0293ffeea41a27ddef7a7b077d8cd93f86f5b9e95537576f8ce87120242cf3ced5f9554966cf648dcb817134977fbe6ee13a45b0815069cf2471014d780ce8c4'
+  data.tar.gz: b7ab71aa3a78d32861e1e9f6ff685424530093fd485185be722c6cc139a81d30692e00ccb6cbad4799bbcbd077ce32c7693db8336c98e8dfe0b18196748af174

data/lib/miga/cli/action/download/ncbi.rb

@@ -29,7 +29,7 @@ module MiGA::Cli::Action::Download::Ncbi
     opt.on(
       '--ncbi-taxonomy-dump STRING',
       'Path to an NCBI Taxonomy dump directory to query instead of API calls'
-    ) { |v| MiGA::RemoteDataset.use_ncbi_taxonomy_dump(v) }
+    ) { |v| cli[:ncbi_taxonomy_dump] = v }
   end
 
   def cli_name_modifiers(opt)
@@ -55,11 +55,16 @@ module MiGA::Cli::Action::Download::Ncbi
   end
 
   def remote_list
+    if cli[:ncbi_taxonomy_dump]
+      cli.say "Reading NCBI Taxonomy dump: #{cli[:ncbi_taxonomy_dump]}"
+      MiGA::RemoteDataset.use_ncbi_taxonomy_dump(cli[:ncbi_taxonomy_dump], cli)
+    end
+
     if cli[:ncbi_list_json] && File.size?(cli[:ncbi_list_json])
-      cli.say "Reusing remote list: #{cli[:ncbi_list_json]}"
-      return MiGA::Json.parse(cli[:ncbi_list_json])
+      return read_ncbi_list_json(cli[:ncbi_list_json])
     end
 
+    cli.say "Obtaining remote list of datasets"
     list  = {}
     query = remote_list_query
     loop do
@@ -74,18 +79,45 @@ module MiGA::Cli::Action::Download::Ncbi
       list.merge!(parse_reports_as_datasets(page[:reports]))
 
       # Next page
+      cli.advance('Datasets:', list.size, page[:total_count])
       break unless page[:next_page_token]
       query[:page_token] = page[:next_page_token]
     end
+    cli.say
+
+    write_ncbi_list_json(cli[:ncbi_list_json], list) if cli[:ncbi_list_json]
+    list
+  end
 
-    if cli[:ncbi_list_json]
-      cli.say "Saving remote list: #{cli[:ncbi_list_json]}"
-      MiGA::Json.generate_fast(list, cli[:ncbi_list_json])
+  def read_ncbi_list_json(file)
+    cli.say "Reusing remote list: #{file}"
+    list = {}
+    n_tot = nil
+    File.open(file, 'r') do |fh|
+      n_tot = fh.gets.chomp.sub(/^# /, '').to_i
+      fh.each_with_index do |ln, k|
+        row = ln.chomp.split("\t", 2)
+        list[row[0]] = MiGA::Json.parse(row[1], contents: true)
+        cli.advance('Lines:', k, n_tot)
+      end
+      cli.say
     end
+    return list
+  end
 
-    list
+  def write_ncbi_list_json(file, list)
+    cli.say "Saving remote list: #{file}"
+    File.open(file, 'w') do |fh|
+      fh.puts('# %i' % list.size)
+      kk = 0
+      list.each do |k, v|
+        fh.puts([k, MiGA::Json.generate_fast(v)].join("\t"))
+        cli.advance('Datasets:', kk += 1, list.size)
+      end
+      cli.say
+    end
   end
-  
+
   def parse_reports_as_datasets(reports)
     ds = {}
     reports.each do |r|

data/lib/miga/dataset/status.rb

@@ -20,7 +20,10 @@ module MiGA::Dataset::Status
     old_status = metadata[:status]
     metadata[:status] =
       !active? ? 'inactive' : done_preprocessing? ? 'complete' : 'incomplete'
-    self.save if save && (old_status.nil? || old_status != metadata[:status])
+    if save && (old_status.nil? || old_status != metadata[:status])
+      self.save
+      MiGA::MiGA.DEBUG "Status changed: #{old_status} -> #{metadata[:status]}"
+    end
     metadata[:status].to_sym
   end
 end

data/lib/miga/dataset.rb

@@ -3,6 +3,7 @@
 # @package MiGA
 # @license Artistic-2.0
 
+require'set'
 require 'miga/metadata'
 require 'miga/dataset/result'
 require 'miga/dataset/status'
@@ -27,7 +28,7 @@ class MiGA::Dataset < MiGA::MiGA
     ##
     # Does the +project+ already have a dataset with that +name+?
     def exist?(project, name)
-      !project.dataset_names_hash[name].nil?
+      project.dataset_names_set.include? name
     end
 
     ##
@@ -61,6 +62,7 @@ class MiGA::Dataset < MiGA::MiGA
     @project, @name, @metadata = project, name, nil
     metadata[:ref] = is_ref
     metadata[:type] ||= :empty
+    metadata[:status] ||= 'incomplete'
     @metadata_future = [
       File.join(project.path, 'metadata', "#{name}.json"),
       metadata
@@ -84,15 +86,18 @@ class MiGA::Dataset < MiGA::MiGA
   ##
   # Save any changes you've made in the dataset
   def save
-    MiGA.DEBUG "Dataset.metadata: #{metadata.data}"
+    MiGA.DEBUG "Dataset.save: #{name}"
     metadata.save
     pull_hook :on_save
   end
 
   ##
-  # Currently +save!+ is simply an alias of +save+, for compatibility with the
-  # +Project+ interface
-  alias :save! :save
+  # Forces a save even if nothing has changed in the metadata
+  def save!
+    MiGA.DEBUG "Dataset.save!: #{name}"
+    metadata.save!
+    pull_hook :on_save
+  end
 
   ##
   # Delete the dataset with all it's contents (including results) and returns
@@ -148,7 +153,7 @@ class MiGA::Dataset < MiGA::MiGA
   ##
   # Is this dataset active?
   def active?
-    metadata[:inactive].nil? or !metadata[:inactive]
+    metadata[:inactive].nil? || !metadata[:inactive]
   end
 
   ##

data/lib/miga/json.rb

@@ -15,6 +15,8 @@ class MiGA::Json < MiGA::MiGA
     # - +:symbolize+: If names should be symbolized. By default it's true if
     #   additions is false, or false otherwise. They can both be false, but an
     #   exception will be raised if both are true
+    # - +:large_file+: If passed, the file is treated as a file with very long
+    #   lines (possibly a single long line)
     def default_opts(opts = {})
       opts[:contents] ||= false
       opts[:additions] ||= false
@@ -36,11 +38,18 @@ class MiGA::Json < MiGA::MiGA
 
       # Read JSON
       cont = path
-      12.times do
-        cont = File.read(path)
-        break unless cont.empty?
-        sleep 1 # Wait up to 12 seconds for racing processes (iff empty file)
-      end unless opts[:contents]
+      if opts[:large_file]
+        cont = ''
+        File.open(path, 'r') do |fh|
+          cont += fh.read(2 ** 16) until fh.eof?
+        end
+      elsif !opts[:contents]
+        12.times do
+          cont = File.read(path)
+          break unless cont.empty?
+          sleep 1 # Wait up to 12 seconds for racing processes (iff empty file)
+        end
+      end
       raise "Empty descriptor: #{opts[:contents] ? "''" : path}" if cont.empty?
 
       # Parse JSON

data/lib/miga/metadata.rb

@@ -26,12 +26,17 @@ class MiGA::Metadata < MiGA::MiGA
   # Path to the JSON file describing the metadata
   attr_reader :path
 
+  ##
+  # Hash (Integer) of the last saved data Hash (object)
+  attr_reader :saved_hash
+
   ##
   # Initiate a MiGA::Metadata object with description in +path+.
   # It will create it if it doesn't exist.
   def initialize(path, defaults = {})
     @data = nil
     @path = File.absolute_path(path)
+    @saved_hash = nil
     unless File.exist? path
       @data = {}
       defaults.each { |k, v| self[k] = v }
@@ -57,35 +62,41 @@ class MiGA::Metadata < MiGA::MiGA
   # Save the metadata into #path
   def save
     return if self[:never_save]
+    return if !saved_hash.nil? && saved_hash == data.hash
 
     MiGA::MiGA.DEBUG "Metadata.save #{path}"
+    path_tmp = "#{path}.tmp"
     self[:updated] = Time.now.to_s
+    @saved_hash = data.hash
     json = to_json
     wait_for_lock
     FileUtils.touch(lock_file)
-    ofh = File.open("#{path}.tmp", 'w')
-    ofh.puts json
-    ofh.close
+    File.open(path_tmp, 'w') { |ofh| ofh.puts json }
 
-    unless File.exist?("#{path}.tmp") && File.exist?(lock_file)
+    unless File.exist?(path_tmp) && File.exist?(lock_file)
       raise "Lock-racing detected for #{path}"
     end
 
-    File.rename("#{path}.tmp", path)
+    File.rename(path_tmp, path)
     File.unlink(lock_file)
   end
 
+  ##
+  # Force +save+ even if nothing has changed since the last save
+  # or load. However, it doesn't save if +:never_save+ is true.
+  def save!
+    @saved_hash = nil
+    save
+  end
+
   ##
   # (Re-)load metadata stored in #path
   def load
-    sleeper = 0.0
-    while File.exist? lock_file
-      sleeper += 0.1 if sleeper <= 10.0
-      sleep(sleeper.to_i)
-    end
+    wait_for_lock
     tmp = MiGA::Json.parse(path, additions: true)
     @data = {}
     tmp.each { |k, v| self[k] = v }
+    @saved_hash = data.hash
   end
 
   ##
@@ -105,7 +116,7 @@ class MiGA::Metadata < MiGA::MiGA
   ##
   # Return the value of +k+ in #data
   def [](k)
-    if k.to_s =~ /(.+):(.+)/
+    if k.to_s =~ /^([^:]+):(.+)$/
       data[$1.to_sym]&.fetch($2)
     else
       data[k.to_sym]

data/lib/miga/project/dataset.rb

@@ -5,24 +5,32 @@
 # Helper module including specific functions handle datasets.
 module MiGA::Project::Dataset
   ##
-  # Returns Array of MiGA::Dataset.
+  # Returns Array of MiGA::Dataset
   def datasets
     metadata[:datasets].map { |name| dataset(name) }
   end
 
   ##
-  # Returns Array of String (without evaluating dataset objects).
+  # Returns Array of String (without evaluating dataset objects)
   def dataset_names
     metadata[:datasets]
   end
 
   ##
-  # Returns Hash of Strings => true. Similar to +dataset_names+ but as
-  # Hash for efficiency.
+  # Returns Hash of +{ String => true }+. Similar to +dataset_names+ but as
+  # Hash for efficiency
   def dataset_names_hash
+    warn 'The Project#dataset_names_hash method will be deprecated soon'
     @dataset_names_hash ||= Hash[dataset_names.map { |i| [i, true] }]
   end
 
+  ##
+  # Returns Set of Strings. Similar to +dataset_names+ but as Set for
+  # efficiency
+  def dataset_names_set
+    @dataset_names_set ||= Set.new(dataset_names)
+  end
+
   ##
   # Returns MiGA::Dataset
   def dataset(name)
@@ -50,7 +58,8 @@ module MiGA::Project::Dataset
     unless metadata[:datasets].include? name
       d = MiGA::Dataset.new(self, name)
       @metadata[:datasets] << name
-      @dataset_names_hash = nil # Ensure loading even if +do_not_save+ is true
+      @dataset_names_hash[name] = true if @dataset_names_hash
+      @dataset_names_set << name if @dataset_names_set
       save
       if d.ref? && d.active?
         recalculate_tasks("Reference dataset added: #{d.name}")

data/lib/miga/project.rb

@@ -67,7 +67,7 @@ class MiGA::Project < MiGA::MiGA
   ##
   # Save any changes persistently, regardless of +do_not_save+
   def save!
-    metadata.save
+    metadata.save!
     pull_hook :on_save
     self.load
   end
@@ -77,6 +77,7 @@ class MiGA::Project < MiGA::MiGA
   def load
     @datasets = {}
     @dataset_names_hash = nil
+    @dataset_names_set = nil
     @metadata = MiGA::Metadata.load "#{path}/miga.project.json"
     raise "Couldn't find project metadata at #{path}" if metadata.nil?
 

data/lib/miga/remote_dataset.rb

@@ -16,30 +16,44 @@ class MiGA::RemoteDataset < MiGA::MiGA
     # Path to a directory with a recent NCBI Taxonomy dump to use instead of
     # making API calls to NCBI servers, which can be obtained at:
     # https://ftp.ncbi.nih.gov/pub/taxonomy/taxdump.tar.gz
-    def use_ncbi_taxonomy_dump(path)
+    #
+    # The +cli+ parameter, if passed, should be a MiGA::Cli object that will
+    # be used to report advance in the reading. Other objects can be passed,
+    # minimally supporting the MiGA::Cli#say and MiGA::Cli#advance method
+    # interfaces
+    def use_ncbi_taxonomy_dump(path, cli = nil)
       raise "Directory doesn't exist: #{path}" unless File.directory?(path)
 
       # Structure: { TaxID => ["name", "rank", parent TaxID] }
+      MiGA::MiGA.DEBUG "Loading NCBI Taxonomy dump: #{path}"
       @ncbi_taxonomy_names = {}
 
       # Read names.dmp
-      File.open(File.join(path, 'names.dmp')) do |fh|
+      File.open(file = File.join(path, 'names.dmp')) do |fh|
+        read = 0
+        size = File.size(file)
         fh.each do |ln|
+          cli&.advance('- names.dmp:', read += ln.size, size)
           row = ln.split(/\t\|\t?/)
           next unless row[3] == 'scientific name'
           @ncbi_taxonomy_names[row[0].to_i] = [row[1].strip]
         end
+        cli&.say
       end
 
       # Read nodes.dmp
-      File.open(File.join(path, 'nodes.dmp')) do |fh|
+      File.open(file = File.join(path, 'nodes.dmp')) do |fh|
+        read = 0
+        size = File.size(file)
         fh.each do |ln|
+          cli&.advance('- nodes.dmp:', read += ln.size, size)
           row = ln.split(/\t\|\t?/)
           child  = row[0].to_i
           parent = row[1].to_i
           @ncbi_taxonomy_names[child][1] = row[2]
           @ncbi_taxonomy_names[child][2] = parent unless parent == child
         end
+        cli&.say
       end
     end
 

data/lib/miga/taxonomy.rb

@@ -188,7 +188,7 @@ class MiGA::Taxonomy < MiGA::MiGA
     when Array, Hash
       self << str
     else
-      "#{str} ".scan(/([A-Za-z]+):([^:]*)( )/) { |r, n, _| self << { r => n } }
+      " #{str} ".scan(/(?<= )([A-Za-z]+):([^:]*) /) { |r, n| self << { r => n } }
     end
   end
 

data/lib/miga/version.rb

@@ -12,7 +12,7 @@ module MiGA
   # - String indicating release status:
   #   - rc* release candidate, not released as gem
   #   - [0-9]+ stable release, released as gem
-  VERSION = [1.3, 10, 0].freeze
+  VERSION = [1.3, 10, 2].freeze
 
   ##
   # Nickname for the current major.minor version.
@@ -20,7 +20,7 @@ module MiGA
 
   ##
   # Date of the current gem relese.
-  VERSION_DATE = Date.new(2024, 1, 31)
+  VERSION_DATE = Date.new(2024, 2, 6)
 
   ##
   # References of MiGA

data/test/metadata_test.rb

@@ -13,7 +13,7 @@ class MetadataTest < Test::Unit::TestCase
       File.unlink(md1.lock_file)
     end
     t1 = Time.new
-    md1.save
+    md1.save!
     t2 = Time.new
     assert_path_not_exist(md1.lock_file)
     assert_ge(t2 - t1, 1.0)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: miga-base
 version: !ruby/object:Gem::Version
-  version: 1.3.10.0
+  version: 1.3.10.2
 platform: ruby
 authors:
 - Luis M. Rodriguez-R
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2024-01-31 00:00:00.000000000 Z
+date: 2024-02-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: daemons