gn_crossmap 3.2.0 → 3.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: bfd0f87bbaf73d66c33e130c124e087104994879
-  data.tar.gz: 11020365c22465b4d804faaf2cf13c6ee168b051
+  metadata.gz: 24d63c7b0f6ce958c567008c74eb31108753e745
+  data.tar.gz: 988291015dc7712cbbf01e67b1da1272fff11db4
 SHA512:
-  metadata.gz: 5ca351156f9007cc6aaa879b3d86852b2a1aeb2584182351ccd3d228997cbccb47604bf97654bd583e58bc1a7280d0f0acb9f6903095db783e4bf0e614cebddf
-  data.tar.gz: 5580be6d8d50f4385e05b798b37a987338e54270d132560df1465f7d2a8cd101b22394a867481577861f03fea28efdd412542092f7f8b5928bf9d8654c434321
+  metadata.gz: 801270ddb8e7ed24a77958bb9bba8db9c8f1f355165fac2ecc119e90494d6cb2bf909e1b24762ad296586657c58c1edc12f86b25a828178b30f273666aa801bf
+  data.tar.gz: 43de612ec0c72ddf9c949c209495fb73d0a346632fc5a841185723858048bebcefcbeaefbbc3cb39fe31b291a4da49e1b4908a9ed709d0a6952e70feeeab097b

data/CHANGELOG.md

@@ -1,5 +1,11 @@
 # ``gn_crossmap`` CHANGELOG
 
+## 3.3.0
+
+* @dimus - Add option `opts.threads` for threads number for concurrent jobs.
+           Resolution now happens concurrently if the number of jobs is more
+           than 1. Max amount of jobs is 10.
+
 ## 3.2.0
 
 * @dimus - Add a column `matchSize` with number of matches for each name

data/exe/crossmap

@@ -8,6 +8,7 @@ puts "This program requires Ruby >= v. 2.4.1" if RUBY_VERSION < "2.4.1"
 
 CATALOGUE_OF_LIFE = 1
 OUTPUT = "output.csv"
+THREADS = 1
 opts = Trollop.options do
   banner "Compares a list of scientific names to scientific names from a " \
     "data source from Global Names Resolver\n\n " \
@@ -16,6 +17,7 @@ opts = Trollop.options do
 
   opt(:input, "Path to input file", type: :string)
   opt(:output, "Path to output file", default: OUTPUT)
+  opt(:threads, "Threads number for resolution", default: THREADS)
   opt(:data_source_id, "Data source id from GN Resolver",
       default: CATALOGUE_OF_LIFE)
   opt(:skip_original, "If given, only 'taxonID' is shown " \

data/gn_crossmap.gemspec

@@ -31,13 +31,14 @@ Gem::Specification.new do |gem|
   gem.require_paths = ["lib"]
 
   gem.add_dependency "biodiversity", "~> 3.1"
+  gem.add_dependency "concurrent-ruby", "~> 1.0"
   gem.add_dependency "gn_uuid", "~> 0.5"
   gem.add_dependency "logger-colors", "~> 1.0"
   gem.add_dependency "rest-client", "~> 2.0"
   gem.add_dependency "trollop", "~> 2.1"
 
   gem.add_development_dependency "bundler", "~> 1.7"
-  gem.add_development_dependency "byebug", "~> 9.0"
+  gem.add_development_dependency "byebug", "~> 9.1"
   gem.add_development_dependency "coveralls", "~> 0.8"
   gem.add_development_dependency "rake", "~> 12.0"
   gem.add_development_dependency "rspec", "~> 3.2"

data/lib/gn_crossmap.rb

@@ -7,9 +7,11 @@ require "tempfile"
 require "logger"
 require "logger/colors"
 require "biodiversity"
+require "concurrent"
 require "gn_uuid"
 require "gn_crossmap/errors"
 require "gn_crossmap/version"
+require "gn_crossmap/resolver_job"
 require "gn_crossmap/reader"
 require "gn_crossmap/writer"
 require "gn_crossmap/collector"
@@ -45,7 +47,7 @@ module GnCrossmap
       reader = create_reader(input_io, opts)
       data = block_given? ? reader.read(&Proc.new) : reader.read
       writer = create_writer(reader, output_io, opts)
-      resolver = create_resolver(writer, opts)
+      resolver = Resolver.new(writer, opts)
       block_given? ? resolver.resolve(data, &Proc.new) : resolver.resolve(data)
       resolver.stats
     end
@@ -68,13 +70,20 @@ module GnCrossmap
       end
     end
 
-    private
-
-    def create_resolver(writer, opts)
-      Resolver.new(writer, opts.data_source_id, opts.resolver_url,
-                   opts.stats, opts.with_classification)
+    def opts_struct(opts)
+      resolver_url = "http://resolver.globalnames.org/name_resolvers.json"
+      threads = opts[:threads].to_i
+      opts[:threads] = threads.between?(1, 10) ? threads : 2
+      with_classification = opts[:with_classification] ? true : false
+      opts[:with_classification] = with_classification
+      data_source_id = opts[:data_source_id].to_i
+      opts[:data_source_id] = data_source_id.zero? ? 1 : data_source_id
+      OpenStruct.new({ stats: Stats.new, alt_headers: [],
+                       resolver_url: resolver_url }.merge(opts))
     end
 
+    private
+
     def create_writer(reader, output_io, opts)
       Writer.new(output_io, reader.original_fields,
                  output_name(opts.output), opts.with_classification)
@@ -85,12 +94,6 @@ module GnCrossmap
                  opts.skip_original, opts.alt_headers, opts.stats)
     end
 
-    def opts_struct(opts)
-      resolver_url = "http://resolver.globalnames.org/name_resolvers.json"
-      OpenStruct.new({ stats: Stats.new, alt_headers: [],
-                       resolver_url: resolver_url }.merge(opts))
-    end
-
     def io(input, output)
       io_in = iogen(input, INPUT_MODE)
       io_out = iogen(output, OUTPUT_MODE)

data/lib/gn_crossmap/resolver.rb

@@ -1,108 +1,133 @@
 # frozen_string_literal: true
 
+# rubocop:disable Metrics/ClassLength
+
 module GnCrossmap
   # Sends data to GN Resolver and collects results
   class Resolver
     attr_reader :stats
 
-    def initialize(writer, data_source_id,
-                   resolver_url, stats, with_classification = false)
-      @stats = stats
-      @resolver_url = resolver_url
+    def initialize(writer, opts)
+      instance_vars_from_opts(opts)
       @processor = GnCrossmap::ResultProcessor.
-                   new(writer, @stats, with_classification)
-      @ds_id = data_source_id
+                   new(writer, @stats, @with_classification)
       @count = 0
-      @current_data = {}
+      @jobs = []
       @batch = 200
     end
 
     def resolve(data)
-      update_stats(data.size)
-      block_given? ? process(data, &Proc.new) : process(data)
+      resolution_stats(data.size)
+      @threads.times do
+        batch = data.shift(@batch)
+        add_job(batch)
+      end
+      block_given? ? traverse_jobs(data, &Proc.new) : traverse_jobs(data)
       wrap_up
       yield(@stats.stats) if block_given?
     end
 
     private
 
-    def process(data)
-      cmd = nil
-      data.each_slice(@batch) do |slice|
-        with_log do
-          remote_resolve(collect_names(slice))
-          cmd = yield(@stats.stats) if block_given?
-        end
-        break if cmd == "STOP"
-      end
-    end
-
     def wrap_up
       @stats.stats[:resolution_stop] = Time.now
       @stats.stats[:status] = :finish
       @processor.writer.close
     end
 
-    def update_stats(records_num)
+    def add_job(batch)
+      job = batch.empty? ? nil : create_job(batch)
+      @jobs << job
+    end
+
+    def traverse_jobs(data)
+      until data.empty? && @jobs.compact.empty?
+        process_results(data)
+        cmd = yield(@stats.stats) if block_given?
+        break if cmd == "STOP"
+        sleep(0.5)
+      end
+    end
+
+    def resolution_stats(records_num)
       @stats.stats[:total_records] = records_num
       @stats.stats[:resolution_start] = Time.now
       @stats.stats[:status] = :resolution
     end
 
-    def with_log
-      s = @count + 1
-      @count += @batch
-      e = [@count, @stats.stats[:total_records]].min
-      GnCrossmap.log("Resolve #{s}-#{e} out of " \
-                     "#{@stats.stats[:total_records]} records at " \
-                     "#{@resolver_url}")
-      yield
+    def process_results(data)
+      indices = []
+      @jobs.each_with_index do |job, i|
+        next if job.nil? || !job.complete?
+        with_log do
+          process_job(job)
+          indices << i
+        end
+      end
+      add_jobs(indices, data) unless indices.empty?
     end
 
-    def collect_names(slice)
-      @current_data = {}
-      slice.each_with_object([]) do |row, str|
-        id = row[:id].strip
-        @current_data[id] = row[:original]
-        @processor.input[id] = { rank: row[:rank] }
-        str << "#{id}|#{row[:name]}"
-      end.join("\n")
+    def add_jobs(indices, data)
+      indices.each do |i|
+        batch = data.shift(@batch)
+        @jobs[i] = batch.empty? ? nil : create_job(batch)
+      end
     end
 
-    def remote_resolve(names)
-      batch_start = Time.now
-      res = RestClient.post(@resolver_url, data: names, data_source_ids: @ds_id)
-      @processor.process(res, @current_data)
-    rescue RestClient::Exception
-      single_remote_resolve(names)
-    ensure
-      update_batch_times(batch_start)
+    def process_job(job)
+      if job.fulfilled?
+        results, current_data, stats = job.value
+        update_stats(stats)
+        @processor.process(results, current_data)
+      else
+        GnCrossmap.logger.error("Remote resolver server failed")
+      end
     end
 
-    def update_batch_times(batch_start)
+    # rubocop:disable Metrics/AbcSize
+    def update_stats(job_stats)
       s = @stats.stats
       s[:last_batches_time].shift if s[:last_batches_time].size > 2
-      s[:last_batches_time] << Time.now - batch_start
+      s[:last_batches_time] << job_stats.stats[:last_batches_time][0]
       s[:resolution_span] = Time.now - s[:resolution_start]
+      s[:resolved_records] += job_stats.stats[:resolved_records]
+      s[:matches][7] += job_stats.stats[:matches][7]
     end
+    # rubocop:enable all
 
-    def single_remote_resolve(names)
-      names.split("\n").each do |name|
-        begin
-          res = RestClient.post(@resolver_url, data: name,
-                                               data_source_ids: @ds_id)
-          @processor.process(res, @current_data)
-        rescue RestClient::Exception => e
-          process_resolver_error(e, name)
-          next
-        end
-      end
+    def create_job(batch)
+      names, batch_data = collect_names(batch)
+      rb = ResolverJob.new(names, batch_data, @resolver_url, @ds_id)
+      Concurrent::Future.execute { rb.run }
+    end
+
+    def instance_vars_from_opts(opts)
+      @stats = opts.stats
+      @with_classification = opts.with_classification.freeze
+      @ds_id = opts.data_source_id.freeze
+      @resolver_url = opts.resolver_url.freeze
+      @threads = opts.threads
+    end
+
+    def collect_names(batch)
+      batch_data = {}
+      names = batch.each_with_object([]) do |row, str|
+        id = row[:id].strip
+        batch_data[id] = row[:original]
+        @processor.input[id] = { rank: row[:rank] }
+        str << "#{id}|#{row[:name]}"
+      end.join("\n")
+      [names, batch_data]
     end
 
-    def process_resolver_error(err, name)
-      @stats.stats[:matches][7] += 1
-      @stats.stats[:resolved_records] += 1
-      GnCrossmap.logger.error("Resolver broke on '#{name}': #{err.message}")
+    def with_log
+      s = @count + 1
+      @count += @batch
+      e = [@count, @stats.stats[:total_records]].min
+      GnCrossmap.log("Resolve #{s}-#{e} out of " \
+                     "#{@stats.stats[:total_records]} records at " \
+                     "#{@resolver_url}")
+      yield
     end
   end
 end

data/lib/gn_crossmap/resolver_job.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module GnCrossmap
+  # Remote resolution for parallel jobs
+  class ResolverJob
+    def initialize(names, batch_data, resolver_url, ds_id)
+      @names = names
+      @batch_data = batch_data
+      @resolver_url = resolver_url
+      @ds_id = ds_id
+      @stats = Stats.new
+    end
+
+    def run
+      res = remote_resolve(@names)
+      [res, @batch_data, @stats]
+    end
+
+    private
+
+    def remote_resolve(names)
+      batch_start = Time.now
+      res = RestClient.post(@resolver_url, data: names,
+                                           data_source_ids: @ds_id)
+      [res.body]
+    rescue RestClient::Exception
+      single_remote_resolve(names)
+    ensure
+      stats_add_batch_time(batch_start)
+    end
+
+    def single_remote_resolve(names)
+      all_res = []
+      names.split("\n").each do |name|
+        res = single_post(name)
+        next unless res
+        all_res << res.body
+      end
+      all_res
+    end
+
+    def single_post
+      RestClient.post(@resolver_url, data: name,
+                                     data_source_ids: @ds_id)
+    rescue RestClient::Exception => e
+      process_resolver_error(e, name)
+      nil
+    end
+
+    def process_resolver_error(err, name)
+      @stats.stats[:matches][7] += 1
+      @stats.stats[:resolved_records] += 1
+      GnCrossmap.logger.error("Resolver broke on '#{name}': #{err.message}")
+    end
+
+    def stats_add_batch_time(batch_start)
+      @stats.stats[:last_batches_time] << Time.now - batch_start
+    end
+  end
+end

data/lib/gn_crossmap/result_processor.rb

@@ -13,11 +13,13 @@ module GnCrossmap
       @input = {}
     end
 
-    def process(result, original_data)
+    def process(results, original_data)
       @original_data = original_data
-      res = rubyfy(result)
-      res[:data].each do |d|
-        d[:results].nil? ? write_empty_result(d) : write_result(d)
+      results.each do |result|
+        res = rubyfy(result)
+        res[:data].each do |d|
+          d[:results].nil? ? write_empty_result(d) : write_result(d)
+        end
       end
     end
 

data/lib/gn_crossmap/version.rb

@@ -2,7 +2,7 @@
 
 # Namespace module for crossmapping checklists to GN sources
 module GnCrossmap
-  VERSION = "3.2.0"
+  VERSION = "3.3.0"
 
   def self.version
     VERSION

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gn_crossmap
 version: !ruby/object:Gem::Version
-  version: 3.2.0
+  version: 3.3.0
 platform: ruby
 authors:
 - Dmitry Mozzherin
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-08-23 00:00:00.000000000 Z
+date: 2017-09-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: biodiversity
@@ -24,6 +24,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.1'
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: gn_uuid
   requirement: !ruby/object:Gem::Requirement
@@ -100,14 +114,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '9.0'
+        version: '9.1'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '9.0'
+        version: '9.1'
 - !ruby/object:Gem::Dependency
   name: coveralls
   requirement: !ruby/object:Gem::Requirement
@@ -194,6 +208,7 @@ files:
 - lib/gn_crossmap/errors.rb
 - lib/gn_crossmap/reader.rb
 - lib/gn_crossmap/resolver.rb
+- lib/gn_crossmap/resolver_job.rb
 - lib/gn_crossmap/result_processor.rb
 - lib/gn_crossmap/sci_name_collector.rb
 - lib/gn_crossmap/stats.rb