gn_list_resolver 4.0.1.0 → 4.0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 47abe84b5c8bd829c01d9c50978fe71b3f14aef6
-  data.tar.gz: 7852f24f3dc3ba40e3175c92275bcde26eda0463
+  metadata.gz: 0d3bfe28b67d56d76bed6c1a4a5be00e89bfad04
+  data.tar.gz: 5dd7d097a80376c9fcc574787980af55f956d766
 SHA512:
-  metadata.gz: e97f1949f12a97c814adb4e4a4b88223ac00bc5bc6095a553ed22461304f4986a2d32cc7071f8871b83a7957ba76c1fe102e6fd776cfea52a33278e9e1909584
-  data.tar.gz: f458552148b3981f6f9c5017ee1b8c9b90bed16e332ef06f5340f88815050905e51277e472be23089be94032c84032f770e3e338dcf7cf00b7dccb64e181e109
+  metadata.gz: 3bf460b69cc2fa211ec8a9d4099ef79c7c7ff25f5acd63c61bbada96609cab45d59967ddbcb454c75a067445f3acb41c0ec4b5ebb21d9155f9109c6c64ef3d55
+  data.tar.gz: 1496680694c8920362025e76327d4185638c1742108d628c1af4279c1c54228d40864b9106db3bb844591ba1592d478cf7799ee5ab59e39886e8051f18cc6168

data/CHANGELOG.md

@@ -25,6 +25,10 @@
 @dimus, @alexander-myltsev - sync with `gn_crossmap`'s 3.2.0, current name is
                              still a bit broken
 
+## 4.0.2
+
+* @dimus - Add resolution span_time to stats
+
 ## 4.0.1
 
 * @dimus - Clean up resolution stats

data/gn_list_resolver.gemspec

@@ -44,7 +44,7 @@ Gem::Specification.new do |gem|
   gem.add_development_dependency "coveralls", "~> 0.8"
   gem.add_development_dependency "rake", "~> 12.0"
   gem.add_development_dependency "rspec", "~> 3.2"
-  gem.add_development_dependency "rubocop", "~> 0.49"
+  gem.add_development_dependency "rubocop", "~> 0.50"
 end
 
 # rubocop:enable Metrics/BlockLength:

data/lib/gn_list_resolver/reader.rb

@@ -75,7 +75,7 @@ module GnListResolver
 
     def process_headers(row)
       @original_fields = headers(row)
-      row = produce_alt_headers(row) if @alt_headers && !@alt_headers.empty?
+      row = produce_alt_headers(row) unless @alt_headers&.empty?
       row
     end
 
@@ -104,7 +104,7 @@ module GnListResolver
 
     def taxon_id_header(hdrs)
       hdrs.each do |h|
-        return [h] if h && h.match?(/taxonid\s*$/i)
+        return [h] if h&.match?(/taxonid\s*$/i)
       end
       []
     end

data/lib/gn_list_resolver/resolver.rb

@@ -119,6 +119,7 @@ module GnListResolver
       s[:resolution][:completed_records] +=
         job_stats.stats[:resolution][:completed_records]
       @stats.update_eta(current_speed)
+      s[:resolution][:time_span] = Time.now - s[:resolution][:start_time]
     end
 
     def with_log

data/lib/gn_list_resolver/stats.rb

@@ -34,7 +34,7 @@ module GnListResolver
     end
 
     def eta_struct
-      { start_time: nil, completed_records: 0,
+      { start_time: nil, completed_records: 0, time_span: 0,
         speed: nil, eta: nil, stop_time: nil }
     end
 

data/lib/gn_list_resolver/version.rb

@@ -2,7 +2,7 @@
 
 # Namespace module for crossmapping checklists to GN sources
 module GnListResolver
-  VERSION = "4.0.1.0"
+  VERSION = "4.0.2.0"
 
   def self.version
     VERSION

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gn_list_resolver
 version: !ruby/object:Gem::Version
-  version: 4.0.1.0
+  version: 4.0.2.0
 platform: ruby
 authors:
 - Dmitry Mozzherin
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-09-17 00:00:00.000000000 Z
+date: 2017-09-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: biodiversity
@@ -185,14 +185,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.49'
+        version: '0.50'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.49'
+        version: '0.50'
 description: Gem uses a checklist in a comma-separated format as an input, and returns
   back a new comma-separated resolved list of scientific names to one of the data
   sources from http://resolver.globalnames.org
@@ -226,14 +226,11 @@ files:
 - lib/gn_list_resolver/graphql.rb
 - lib/gn_list_resolver/reader.rb
 - lib/gn_list_resolver/resolver.rb
-- lib/gn_list_resolver/resolver.rb.orig
 - lib/gn_list_resolver/resolver_job.rb
 - lib/gn_list_resolver/result_processor.rb
 - lib/gn_list_resolver/sci_name_collector.rb
 - lib/gn_list_resolver/stats.rb
-- lib/gn_list_resolver/stats.rb.orig
 - lib/gn_list_resolver/version.rb
-- lib/gn_list_resolver/version.rb.orig
 - lib/gn_list_resolver/writer.rb
 homepage: https://github.com/GlobalNamesArchitecture/gn_list_resolver
 licenses:

data/lib/gn_list_resolver/resolver.rb.orig

@@ -1,166 +0,0 @@
-# frozen_string_literal: true
-
-# rubocop:disable Metrics/ClassLength
-
-module GnListResolver
-  # Sends data to GN Resolver and collects results
-  class Resolver
-    GRAPHQL = GnGraphQL.new
-    QUERY = GRAPHQL.client.parse(GRAPHQL.query)
-    attr_reader :stats
-
-    def initialize(writer, opts)
-      instance_vars_from_opts(opts)
-      @processor = GnListResolver::ResultProcessor.
-                   new(writer, @stats, @with_classification)
-      @count = 0
-      @jobs = []
-<<<<<<< HEAD:lib/gn_list_resolver/resolver.rb
-      @batch = 1000
-=======
-      @batch = 200
-      @smoothing = 0.05
->>>>>>> 2a4afb8... Fix #42 - better speed estimation:lib/gn_crossmap/resolver.rb
-    end
-
-    def resolve(data)
-      resolution_stats(data.size)
-      @threads.times do
-        batch = data.shift(@batch)
-        add_job(batch)
-      end
-      block_given? ? traverse_jobs(data, &Proc.new) : traverse_jobs(data)
-      wrap_up
-      block_given? ? yield(@stats.stats) : @stats.stats
-    end
-
-    private
-
-    def wrap_up
-      @stats.stats[:resolution][:stop_time] = Time.now
-      @stats.stats[:status] = :finish
-      @processor.writer.close
-    end
-
-    def add_job(batch)
-      job = batch.empty? ? nil : create_job(batch)
-      @jobs << job
-    end
-
-    def traverse_jobs(data)
-      until data.empty? && @jobs.compact.empty?
-        process_results(data)
-        cmd = yield(@stats.stats) if block_given?
-        break if cmd == "STOP"
-        sleep(0.5)
-      end
-    end
-
-    def resolution_stats(records_num)
-      @stats.stats[:total_records] = records_num
-      @stats.stats[:resolution][:start_time] = Time.now
-      @stats.stats[:status] = :resolution
-    end
-
-    def process_results(data)
-      indices = []
-      @jobs.each_with_index do |job, i|
-        next if job.nil? || !job.complete?
-        with_log do
-          process_job(job)
-          indices << i
-        end
-      end
-      add_jobs(indices, data) unless indices.empty?
-    end
-
-    def add_jobs(indices, data)
-      indices.each do |i|
-        batch = data.shift(@batch)
-        @jobs[i] = batch.empty? ? nil : create_job(batch)
-      end
-    end
-
-    def process_job(job)
-      if job.fulfilled?
-        results, current_data, stats = job.value
-        update_stats(stats)
-        @processor.process(results, current_data)
-      else
-        GnResolver.logger.error(job.reason.message)
-      end
-    end
-
-<<<<<<< HEAD:lib/gn_list_resolver/resolver.rb
-    def update_stats(job_stats)
-      s = @stats.stats
-      s[:last_batches_time].shift if s[:last_batches_time].size > 2
-      s[:last_batches_time] << job_stats.stats[:last_batches_time][0]
-      s[:resolution_span] = Time.now - s[:resolution_start]
-    end
-
-=======
->>>>>>> 2a4afb8... Fix #42 - better speed estimation:lib/gn_crossmap/resolver.rb
-    def create_job(batch)
-      batch_data = collect_names(batch)
-      rb = ResolverJob.new(batch, batch_data, @ds_id)
-      Concurrent::Future.execute { rb.run }
-    end
-
-    def instance_vars_from_opts(opts)
-      @stats = opts.stats
-      @with_classification = opts.with_classification.freeze
-      @ds_id = opts.data_source_id.freeze
-      @threads = opts.threads
-    end
-
-    def collect_names(batch)
-      batch_data = {}
-      batch.each do |row|
-        id = row[:id].strip
-        batch_data[id] = row[:original]
-        @processor.input[id] = { rank: row[:rank] }
-<<<<<<< HEAD:lib/gn_list_resolver/resolver.rb
-      end
-      batch_data
-=======
-        str << "#{id}|#{row[:name]}"
-      end
-      [names, batch_data]
->>>>>>> 2a4afb8... Fix #42 - better speed estimation:lib/gn_crossmap/resolver.rb
-    end
-
-    # rubocop:disable Metrics/AbcSize
-    def update_stats(job_stats)
-      s = @stats.stats
-      current_speed = job_stats.stats[:current_speed] *
-                      @stats.penalty(@threads)
-
-      s[:resolution][:completed_records] +=
-        job_stats.stats[:resolution][:completed_records]
-      @stats.update_eta(current_speed)
-      s[:matches][7] += job_stats.stats[:matches][7]
-    end
-
-    def with_log
-      yield
-      s = @count + 1
-      @count += @batch
-      e = [@count, @stats.stats[:total_records]].min
-<<<<<<< HEAD:lib/gn_list_resolver/resolver.rb
-      GnListResolver.log("Resolve #{s}-#{e} out of " \
-                     "#{@stats.stats[:total_records]} records at " \
-                     "#{RESOLVER_URL}")
-      yield
-=======
-      eta = @stats.stats[:resolution][:eta].to_i + Time.now.to_i
-      msg = format("Resolve %s-%s/%s records %d rec/s; eta: %s", s, e,
-                   @stats.stats[:total_records],
-                   @stats.stats[:resolution][:speed].to_i,
-                   Time.at(eta))
-      GnCrossmap.log(msg)
->>>>>>> 2a4afb8... Fix #42 - better speed estimation:lib/gn_crossmap/resolver.rb
-    end
-  end
-end
-# rubocop:enable all

data/lib/gn_list_resolver/stats.rb.orig

@@ -1,62 +0,0 @@
-# frozen_string_literal: true
-
-module GnListResolver
-  # Collects statistics about list resolving process
-  class Stats
-    attr_accessor :stats
-
-    def initialize
-      @stats = { status: :init, total_records: 0, ingested_records: 0,
-<<<<<<< HEAD:lib/gn_list_resolver/stats.rb
-                 resolved_records: 0, ingestion_span: nil,
-                 resolution_span: nil, ingestion_start: nil,
-                 resolution_start: nil, resolution_stop: nil,
-                 last_batches_time: [], matches: init_matches,
-                 errors: [] }
-=======
-                 ingestion_span: nil, ingestion_start: nil,
-                 resolution: eta_struct,
-                 matches: match_types, errors: [] }
-      @smooth = 0.05
-    end
-
-    def penalty(threads)
-      pnlt = 0.7
-      penalty_adj(threads.to_i, 1, pnlt)
-    end
-
-    def update_eta(current_speed)
-      eta = @stats[:resolution]
-      eta[:speed] = current_speed if eta[:speed].nil?
-      eta[:speed] = eta[:speed] * (1 - @smooth) + current_speed * @smooth
-      eta[:eta] = (@stats[:total_records] -
-                   @stats[:resolution][:completed_records]) /
-                  eta[:speed]
->>>>>>> 2a4afb8... Fix #42 - better speed estimation:lib/gn_crossmap/stats.rb
-    end
-
-    private
-
-<<<<<<< HEAD:lib/gn_list_resolver/stats.rb
-    def init_matches
-      MATCH_TYPES.keys.each_with_object({}) { |t, h| h[t] = 0 }
-=======
-    def eta_struct
-      { start_time: nil, completed_records: 0,
-        speed: nil, eta: nil, stop_time: nil }
-    end
-
-    def match_types
-      matches = GnCrossmap::MATCH_TYPES.keys
-      matches.each_with_object({}) do |key, obj|
-        obj[key] = 0
-      end
->>>>>>> 2a4afb8... Fix #42 - better speed estimation:lib/gn_crossmap/stats.rb
-    end
-
-    def penalty_adj(threads, val, pnlt)
-      return val if threads < 2
-      val + penalty_adj(threads - 1, (val * pnlt), pnlt)
-    end
-  end
-end

data/lib/gn_list_resolver/version.rb.orig

@@ -1,15 +0,0 @@
-# frozen_string_literal: true
-
-# Namespace module for crossmapping checklists to GN sources
-<<<<<<< HEAD:lib/gn_list_resolver/version.rb
-module GnListResolver
-  VERSION = "3.3.1.1"
-=======
-module GnCrossmap
-  VERSION = "4.0.1"
->>>>>>> 2a4afb8... Fix #42 - better speed estimation:lib/gn_crossmap/version.rb
-
-  def self.version
-    VERSION
-  end
-end