gn_list_resolver 3.2.0.4 → 3.3.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f990bf592ce702c16e20f45e3c881e7f4f4df01c
-  data.tar.gz: e8b55f1f5e6b27e4b8eddb8e21c65ce900ede729
+  metadata.gz: cdcc523cf5153459744453bc7ac1a6ccd0dcc362
+  data.tar.gz: a5886cca5aef50a2d17a851152927f1653d850c8
 SHA512:
-  metadata.gz: c5586d503371d99d8e26cf26057e95805a5fc22ca6f2073eb4baabe31891a68ccd20b64958f1d8e6947a2ea6469c5e03981e3b514cc1696323809c03892c1f27
-  data.tar.gz: 44a2e55c363de3858449f0ea954e21ea4cdaf27d30c32a4d12c7d3064c9c69cb98b65fffcdab1261a86d25a01c02cffbc970a11e1909be20120c33749da66470
+  metadata.gz: c8f44334200d52e5d407793206a409727ad61c1ae045c2510ef4d8ca6cfd39163dda3ad38017041da5f4e7a9ef4ebd0eb1bb65d8ce8c3eb4c618e98e20037321
+  data.tar.gz: 62034abff0ab1a6a85499586ab259cf15138d49e0c0cb578a0e500e3354c38ef09253e579763698f9fa8ab849c09e11fe25411301d7aee36d0a1e551c615f714

data/CHANGELOG.md

@@ -17,6 +17,16 @@
 @dimus, @alexander-myltsev - sync with `gn_crossmap`'s 3.2.0, current name is
                              still a bit broken
 
+## 3.3.1
+
+* @dimus - Better error message in logger
+
+## 3.3.0
+
+* @dimus - Add option `opts.threads` for threads number for concurrent jobs.
+           Resolution now happens concurrently if the number of jobs is more
+           than 1. Max amount of jobs is 10.
+
 ## 3.2.0
 
 * @dimus - Add a column `matchSize` with number of matches for each name

data/exe/list_resolver

@@ -8,6 +8,7 @@ puts "This program requires Ruby >= v. 2.4.1" if RUBY_VERSION < "2.4.1"
 
 CATALOGUE_OF_LIFE = 1
 OUTPUT = "output.csv"
+THREADS = 1
 opts = Trollop.options do
   banner "Compares a list of scientific names to scientific names from a " \
     "data source from Global Names Resolver\n\n " \
@@ -16,6 +17,7 @@ opts = Trollop.options do
 
   opt(:input, "Path to input file", type: :string)
   opt(:output, "Path to output file", default: OUTPUT)
+  opt(:threads, "Threads number for resolution", default: THREADS)
   opt(:data_source_id, "Data source id from GN Resolver",
       default: CATALOGUE_OF_LIFE)
   opt(:skip_original, "If given, only 'taxonID' is shown " \

data/gn_list_resolver.gemspec

@@ -32,6 +32,7 @@ Gem::Specification.new do |gem|
   gem.require_paths = ["lib"]
 
   gem.add_dependency "biodiversity", "~> 3.1"
+  gem.add_dependency "concurrent-ruby", "~> 1.0"
   gem.add_dependency "gn_uuid", "~> 0.5"
   gem.add_dependency "graphql-client", "~> 0.11.3"
   gem.add_dependency "logger-colors", "~> 1.0"
@@ -39,7 +40,7 @@ Gem::Specification.new do |gem|
   gem.add_dependency "trollop", "~> 2.1"
 
   gem.add_development_dependency "bundler", "~> 1.7"
-  gem.add_development_dependency "byebug", "~> 9.0"
+  gem.add_development_dependency "byebug", "~> 9.1"
   gem.add_development_dependency "coveralls", "~> 0.8"
   gem.add_development_dependency "rake", "~> 12.0"
   gem.add_development_dependency "rspec", "~> 3.2"

data/lib/gn_list_resolver.rb

@@ -8,6 +8,7 @@ require "logger"
 require "logger/colors"
 require "pp"
 require "biodiversity"
+require "concurrent"
 require "gn_uuid"
 require "graphql/client"
 require "graphql/client/http"
@@ -20,6 +21,7 @@ require "gn_list_resolver/writer"
 require "gn_list_resolver/collector"
 require "gn_list_resolver/column_collector"
 require "gn_list_resolver/sci_name_collector"
+require "gn_list_resolver/resolver_job"
 require "gn_list_resolver/resolver"
 require "gn_list_resolver/result_processor"
 require "gn_list_resolver/stats"
@@ -37,7 +39,7 @@ module GnListResolver
       reader = create_reader(input_io, opts)
       data = block_given? ? reader.read(&Proc.new) : reader.read
       writer = create_writer(reader, output_io, opts)
-      resolver = create_resolver(writer, opts)
+      resolver = Resolver.new(writer, opts)
       block_given? ? resolver.resolve(data, &Proc.new) : resolver.resolve(data)
       logger.warn(resolver.stats.stats.pretty_inspect) if opts[:debug]
       resolver.stats
@@ -61,13 +63,18 @@ module GnListResolver
       end
     end
 
-    private
-
-    def create_resolver(writer, opts)
-      Resolver.new(writer, opts.data_source_id,
-                   opts.stats, opts.with_classification)
+    def opts_struct(opts)
+      threads = opts[:threads].to_i
+      opts[:threads] = threads.between?(1, 10) ? threads : 2
+      with_classification = opts[:with_classification] ? true : false
+      opts[:with_classification] = with_classification
+      data_source_id = opts[:data_source_id].to_i
+      opts[:data_source_id] = data_source_id.zero? ? 1 : data_source_id
+      OpenStruct.new({ stats: Stats.new, alt_headers: [] }.merge(opts))
     end
 
+    private
+
     def create_writer(reader, output_io, opts)
       Writer.new(output_io, reader.original_fields,
                  output_name(opts.output), opts.with_classification)
@@ -78,10 +85,6 @@ module GnListResolver
                  opts.skip_original, opts.alt_headers, opts.stats)
     end
 
-    def opts_struct(opts)
-      OpenStruct.new({ stats: Stats.new, alt_headers: [] }.merge(opts))
-    end
-
     def io(input, output)
       io_in = iogen(input, INPUT_MODE)
       io_out = iogen(output, OUTPUT_MODE)

data/lib/gn_list_resolver/resolver.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+# rubocop:disable Metrics/ClassLength
+
 module GnListResolver
   # Sends data to GN Resolver and collects results
   class Resolver
@@ -7,111 +9,122 @@ module GnListResolver
     QUERY = GRAPHQL.client.parse(GRAPHQL.query)
     attr_reader :stats
 
-    def initialize(writer, data_source_id, stats, with_classification = false)
-      @stats = stats
-      @processor = GnListResolver::ResultProcessor.new(writer, @stats,
-                                                       with_classification)
-      @ds_id = data_source_id
+    def initialize(writer, opts)
+      instance_vars_from_opts(opts)
+      @processor = GnListResolver::ResultProcessor.
+                   new(writer, @stats, @with_classification)
       @count = 0
-      @current_data = {}
+      @jobs = []
       @batch = 1000
     end
 
     def resolve(data)
-      update_stats(data.size)
-      block_given? ? process(data, &Proc.new) : process(data)
+      resolution_stats(data.size)
+      @threads.times do
+        batch = data.shift(@batch)
+        add_job(batch)
+      end
+      block_given? ? traverse_jobs(data, &Proc.new) : traverse_jobs(data)
       wrap_up
       block_given? ? yield(@stats.stats) : @stats.stats
     end
 
     private
 
-    def process(data)
-      cmd = nil
-      data.each_slice(@batch) do |slice|
-        with_log do
-          collect_names(slice)
-          remote_resolve(slice)
-          cmd = yield(@stats.stats) if block_given?
-        end
-        break if cmd == "STOP"
-      end
-    end
-
     def wrap_up
       @stats.stats[:resolution_stop] = Time.now
       @stats.stats[:status] = :finish
       @processor.writer.close
     end
 
-    def update_stats(records_num)
-      @stats.stats[:total_records] = records_num
-      @stats.stats[:resolution_start] = Time.now
-      @stats.stats[:status] = :resolution
+    def add_job(batch)
+      job = batch.empty? ? nil : create_job(batch)
+      @jobs << job
     end
 
-    def with_log
-      s = @count + 1
-      @count += @batch
-      e = [@count, @stats.stats[:total_records]].min
-      GnListResolver.log("Resolve #{s}-#{e} out of " \
-                         "#{@stats.stats[:total_records]} records at " \
-                         "#{RESOLVER_URL}")
-      yield
+    def traverse_jobs(data)
+      until data.empty? && @jobs.compact.empty?
+        process_results(data)
+        cmd = yield(@stats.stats) if block_given?
+        break if cmd == "STOP"
+        sleep(0.5)
+      end
     end
 
-    def collect_names(slice)
-      @current_data = {}
-      slice.each_with_object([]) do |row, str|
-        id = row[:id].strip
-        @current_data[id] = row[:original]
-        @processor.input[id] = { rank: row[:rank] }
-        str << "#{id}|#{row[:name]}"
-      end.join("\n")
+    def resolution_stats(records_num)
+      @stats.stats[:total_records] = records_num
+      @stats.stats[:resolution_start] = Time.now
+      @stats.stats[:status] = :resolution
     end
 
-    def variables(names)
-      { dataSourceIds: [@ds_id],
-        names: names.
-          map { |name| { value: name[:name], suppliedId: name[:id] } } }
+    def process_results(data)
+      indices = []
+      @jobs.each_with_index do |job, i|
+        next if job.nil? || !job.complete?
+        with_log do
+          process_job(job)
+          indices << i
+        end
+      end
+      add_jobs(indices, data) unless indices.empty?
     end
 
-    def remote_resolve(names)
-      batch_start = Time.now
-
-      res = GRAPHQL.client.query(QUERY, variables: variables(names))
-      if res.data
-        @processor.process(res.data.name_resolver.responses, @current_data)
-      else
-        single_remote_resolve(names)
+    def add_jobs(indices, data)
+      indices.each do |i|
+        batch = data.shift(@batch)
+        @jobs[i] = batch.empty? ? nil : create_job(batch)
       end
-      update_batch_times(batch_start)
     end
 
-    def single_remote_resolve(names)
-      names.each do |name|
-        res = GRAPHQL.client.query(QUERY, variables: variables([name]))
-        if res.data
-          @processor.process(res.data.name_resolver, @current_data)
-        else
-          process_resolver_error(res, name)
-        end
+    def process_job(job)
+      if job.fulfilled?
+        results, current_data, stats = job.value
+        update_stats(stats)
+        @processor.process(results, current_data)
+      else
+        GnResolver.logger.error(job.reason.message)
       end
     end
 
-    def update_batch_times(batch_start)
+    def update_stats(job_stats)
       s = @stats.stats
       s[:last_batches_time].shift if s[:last_batches_time].size > 2
-      s[:last_batches_time] << Time.now - batch_start
+      s[:last_batches_time] << job_stats.stats[:last_batches_time][0]
       s[:resolution_span] = Time.now - s[:resolution_start]
     end
 
-    def process_resolver_error(res, name)
-      @stats.stats[:matches][:ErrorInMatch] += 1
-      @stats.stats[:resolved_records] += 1
-      error =
-        "Resolver broke on '#{name}': #{res.errors.messages['data'].first}"
-      GnListResolver.logger.error(error)
+    def create_job(batch)
+      batch_data = collect_names(batch)
+      rb = ResolverJob.new(batch, batch_data, @ds_id)
+      Concurrent::Future.execute { rb.run }
+    end
+
+    def instance_vars_from_opts(opts)
+      @stats = opts.stats
+      @with_classification = opts.with_classification.freeze
+      @ds_id = opts.data_source_id.freeze
+      @threads = opts.threads
+    end
+
+    def collect_names(batch)
+      batch_data = {}
+      batch.each do |row|
+        id = row[:id].strip
+        batch_data[id] = row[:original]
+        @processor.input[id] = { rank: row[:rank] }
+      end
+      batch_data
+    end
+
+    def with_log
+      s = @count + 1
+      @count += @batch
+      e = [@count, @stats.stats[:total_records]].min
+      GnListResolver.log("Resolve #{s}-#{e} out of " \
+                     "#{@stats.stats[:total_records]} records at " \
+                     "#{RESOLVER_URL}")
+      yield
     end
   end
 end
+# rubocop:enable all

data/lib/gn_list_resolver/resolver.rb.orig

@@ -0,0 +1,134 @@
+# frozen_string_literal: true
+
+# rubocop:disable Metrics/ClassLength
+
+module GnListResolver
+  # Sends data to GN Resolver and collects results
+  class Resolver
+    GRAPHQL = GnGraphQL.new
+    QUERY = GRAPHQL.client.parse(GRAPHQL.query)
+    attr_reader :stats
+
+    def initialize(writer, opts)
+      instance_vars_from_opts(opts)
+      @processor = GnListResolver::ResultProcessor.
+                   new(writer, @stats, @with_classification)
+      @count = 0
+      @jobs = []
+      @batch = 1000
+    end
+
+    def resolve(data)
+      resolution_stats(data.size)
+      @threads.times do
+        batch = data.shift(@batch)
+        add_job(batch)
+      end
+      block_given? ? traverse_jobs(data, &Proc.new) : traverse_jobs(data)
+      wrap_up
+      block_given? ? yield(@stats.stats) : @stats.stats
+    end
+
+    private
+
+    def wrap_up
+      @stats.stats[:resolution_stop] = Time.now
+      @stats.stats[:status] = :finish
+      @processor.writer.close
+    end
+
+    def add_job(batch)
+      job = batch.empty? ? nil : create_job(batch)
+      @jobs << job
+    end
+
+    def traverse_jobs(data)
+      until data.empty? && @jobs.compact.empty?
+        process_results(data)
+        cmd = yield(@stats.stats) if block_given?
+        break if cmd == "STOP"
+        sleep(0.5)
+      end
+    end
+
+    def resolution_stats(records_num)
+      @stats.stats[:total_records] = records_num
+      @stats.stats[:resolution_start] = Time.now
+      @stats.stats[:status] = :resolution
+    end
+
+    def process_results(data)
+      indices = []
+      @jobs.each_with_index do |job, i|
+        next if job.nil? || !job.complete?
+        with_log do
+          process_job(job)
+          indices << i
+        end
+      end
+      add_jobs(indices, data) unless indices.empty?
+    end
+
+    def add_jobs(indices, data)
+      indices.each do |i|
+        batch = data.shift(@batch)
+        @jobs[i] = batch.empty? ? nil : create_job(batch)
+      end
+    end
+
+    def process_job(job)
+      if job.fulfilled?
+        results, current_data, stats = job.value
+        update_stats(stats)
+        @processor.process(results, current_data)
+      else
+<<<<<<< HEAD:lib/gn_list_resolver/resolver.rb
+        GnListResolver.logger.error("Remote resolver server failed")
+=======
+        GnCrossmap.logger.error(job.reason.message)
+>>>>>>> 36115cc... better error log:lib/gn_crossmap/resolver.rb
+      end
+    end
+
+    def update_stats(job_stats)
+      s = @stats.stats
+      s[:last_batches_time].shift if s[:last_batches_time].size > 2
+      s[:last_batches_time] << job_stats.stats[:last_batches_time][0]
+      s[:resolution_span] = Time.now - s[:resolution_start]
+    end
+
+    def create_job(batch)
+      batch_data = collect_names(batch)
+      rb = ResolverJob.new(batch, batch_data, @ds_id)
+      Concurrent::Future.execute { rb.run }
+    end
+
+    def instance_vars_from_opts(opts)
+      @stats = opts.stats
+      @with_classification = opts.with_classification.freeze
+      @ds_id = opts.data_source_id.freeze
+      @threads = opts.threads
+    end
+
+    def collect_names(batch)
+      batch_data = {}
+      batch.each do |row|
+        id = row[:id].strip
+        batch_data[id] = row[:original]
+        @processor.input[id] = { rank: row[:rank] }
+      end
+      batch_data
+    end
+
+    def with_log
+      s = @count + 1
+      @count += @batch
+      e = [@count, @stats.stats[:total_records]].min
+      GnListResolver.log("Resolve #{s}-#{e} out of " \
+                     "#{@stats.stats[:total_records]} records at " \
+                     "#{RESOLVER_URL}")
+      yield
+    end
+  end
+end
+# rubocop:enable all

data/lib/gn_list_resolver/resolver_job.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module GnListResolver
+  # Remote resolution for parallel jobs
+  class ResolverJob
+    GRAPHQL = GnGraphQL.new
+    QUERY = GRAPHQL.client.parse(GRAPHQL.query)
+    def initialize(names, batch_data, data_source_id)
+      @names = names
+      @batch_data = batch_data
+      @data_source_id = data_source_id
+      @stats = Stats.new
+    end
+
+    def run
+      res = remote_resolve(@names)
+      [res, @batch_data, @stats]
+    end
+
+    private
+
+    def variables(names)
+      { dataSourceIds: [@data_source_id],
+        names: names.
+          map { |name| { value: name[:name], suppliedId: name[:id] } } }
+    end
+
+    def remote_resolve(names)
+      batch_start = Time.now
+      res = GRAPHQL.client.query(QUERY, variables: variables(names))
+      stats_add_batch_time(batch_start)
+      res.data.name_resolver.responses
+    end
+
+    def stats_add_batch_time(batch_start)
+      @stats.stats[:last_batches_time] << Time.now - batch_start
+    end
+  end
+end

data/lib/gn_list_resolver/version.rb

@@ -2,7 +2,7 @@
 
 # Namespace module for crossmapping checklists to GN sources
 module GnListResolver
-  VERSION = "3.2.0.4"
+  VERSION = "3.3.1.0"
 
   def self.version
     VERSION

data/lib/gn_list_resolver/version.rb.orig

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+# Namespace module for crossmapping checklists to GN sources
+<<<<<<< HEAD:lib/gn_list_resolver/version.rb
+module GnListResolver
+  VERSION = "3.3.1.0"
+=======
+module GnCrossmap
+  VERSION = "3.3.1"
+>>>>>>> 36115cc... better error log:lib/gn_crossmap/version.rb
+
+  def self.version
+    VERSION
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gn_list_resolver
 version: !ruby/object:Gem::Version
-  version: 3.2.0.4
+  version: 3.3.1.0
 platform: ruby
 authors:
 - Dmitry Mozzherin
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-09-07 00:00:00.000000000 Z
+date: 2017-09-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: biodiversity
@@ -25,6 +25,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.1'
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: gn_uuid
   requirement: !ruby/object:Gem::Requirement
@@ -115,14 +129,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '9.0'
+        version: '9.1'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '9.0'
+        version: '9.1'
 - !ruby/object:Gem::Dependency
   name: coveralls
   requirement: !ruby/object:Gem::Requirement
@@ -212,10 +226,13 @@ files:
 - lib/gn_list_resolver/graphql.rb
 - lib/gn_list_resolver/reader.rb
 - lib/gn_list_resolver/resolver.rb
+- lib/gn_list_resolver/resolver.rb.orig
+- lib/gn_list_resolver/resolver_job.rb
 - lib/gn_list_resolver/result_processor.rb
 - lib/gn_list_resolver/sci_name_collector.rb
 - lib/gn_list_resolver/stats.rb
 - lib/gn_list_resolver/version.rb
+- lib/gn_list_resolver/version.rb.orig
 - lib/gn_list_resolver/writer.rb
 homepage: https://github.com/GlobalNamesArchitecture/gn_list_resolver
 licenses: