gn_list_resolver 3.3.1.1 → 4.0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: bca6cb993b435d76acf6a14bb0c4031892a6a949
-  data.tar.gz: d7c2c9e5c5132cd93b4dc03857427a0e6e5ed8e2
+  metadata.gz: 47abe84b5c8bd829c01d9c50978fe71b3f14aef6
+  data.tar.gz: 7852f24f3dc3ba40e3175c92275bcde26eda0463
 SHA512:
-  metadata.gz: 64966a526648716221e6024c753fb9cb28b3af1450d928c1e9bc5f70cb614587ac7481674d93dbac1368a0ca90e621930d170ed5db5bf31d94a6478944919a58
-  data.tar.gz: aabced616396747ce23661c945b29b11b7aebc297ee57618ed5e2a659d581abf446f1af42aafd1032492e54b20b8cc49ccd550650a73222f15d6c4bb69e1d0b3
+  metadata.gz: e97f1949f12a97c814adb4e4a4b88223ac00bc5bc6095a553ed22461304f4986a2d32cc7071f8871b83a7957ba76c1fe102e6fd776cfea52a33278e9e1909584
+  data.tar.gz: f458552148b3981f6f9c5017ee1b8c9b90bed16e332ef06f5340f88815050905e51277e472be23089be94032c84032f770e3e338dcf7cf00b7dccb64e181e109

data/CHANGELOG.md

@@ -25,6 +25,14 @@
 @dimus, @alexander-myltsev - sync with `gn_crossmap`'s 3.2.0, current name is
                              still a bit broken
 
+## 4.0.1
+
+* @dimus - Clean up resolution stats
+
+## 4.0.0
+
+* @dimus - Better estimation for time left for resolver and speed
+
 ## 3.3.1
 
 * @dimus - Better error message in logger

data/README.md

@@ -166,7 +166,8 @@ end
 |resolution_stop   |time when resolution of names stopped                    |
 |resolution_span   |time of intermediate checkpoint of resolving names       |
 |resolved_records  |number of names already processed                        |
-|last_batches_time |time required to process the last batch of names         |
+|speed             |weighted speed average                                   |
+|current_speed     |speed of resolution for the last batch                   |
 |matches           |Distribution of processed data by match type (see below) |
 |errors            |First 0-10 errors found during the csv file processing   |
 

data/lib/gn_list_resolver/resolver.rb

@@ -16,6 +16,7 @@ module GnListResolver
       @count = 0
       @jobs = []
       @batch = 1000
+      @smoothing = 0.05
     end
 
     def resolve(data)
@@ -32,7 +33,7 @@ module GnListResolver
     private
 
     def wrap_up
-      @stats.stats[:resolution_stop] = Time.now
+      @stats.stats[:resolution][:stop_time] = Time.now
       @stats.stats[:status] = :finish
       @processor.writer.close
     end
@@ -53,7 +54,7 @@ module GnListResolver
 
     def resolution_stats(records_num)
       @stats.stats[:total_records] = records_num
-      @stats.stats[:resolution_start] = Time.now
+      @stats.stats[:resolution][:start_time] = Time.now
       @stats.stats[:status] = :resolution
     end
 
@@ -82,17 +83,10 @@ module GnListResolver
         update_stats(stats)
         @processor.process(results, current_data)
       else
-        GnResolver.logger.error(job.reason.message)
+        GnListResolver.logger.error(job.reason.message)
       end
     end
 
-    def update_stats(job_stats)
-      s = @stats.stats
-      s[:last_batches_time].shift if s[:last_batches_time].size > 2
-      s[:last_batches_time] << job_stats.stats[:last_batches_time][0]
-      s[:resolution_span] = Time.now - s[:resolution_start]
-    end
-
     def create_job(batch)
       batch_data = collect_names(batch)
       rb = ResolverJob.new(batch, batch_data, @ds_id)
@@ -116,14 +110,28 @@ module GnListResolver
       batch_data
     end
 
+    # rubocop:disable Metrics/AbcSize
+    def update_stats(job_stats)
+      s = @stats.stats
+      current_speed = job_stats.stats[:current_speed] *
+                      @stats.penalty(@threads)
+
+      s[:resolution][:completed_records] +=
+        job_stats.stats[:resolution][:completed_records]
+      @stats.update_eta(current_speed)
+    end
+
     def with_log
+      yield
       s = @count + 1
       @count += @batch
       e = [@count, @stats.stats[:total_records]].min
-      GnListResolver.log("Resolve #{s}-#{e} out of " \
-                     "#{@stats.stats[:total_records]} records at " \
-                     "#{RESOLVER_URL}")
-      yield
+      eta = @stats.stats[:resolution][:eta].to_i + Time.now.to_i
+      msg = format("Resolve %s-%s/%s records %d rec/s; eta: %s", s, e,
+                   @stats.stats[:total_records],
+                   @stats.stats[:resolution][:speed].to_i,
+                   Time.at(eta))
+      GnListResolver.log(msg)
     end
   end
 end

data/lib/gn_list_resolver/resolver.rb.orig

@@ -15,7 +15,12 @@ module GnListResolver
                    new(writer, @stats, @with_classification)
       @count = 0
       @jobs = []
+<<<<<<< HEAD:lib/gn_list_resolver/resolver.rb
       @batch = 1000
+=======
+      @batch = 200
+      @smoothing = 0.05
+>>>>>>> 2a4afb8... Fix #42 - better speed estimation:lib/gn_crossmap/resolver.rb
     end
 
     def resolve(data)
@@ -32,7 +37,7 @@ module GnListResolver
     private
 
     def wrap_up
-      @stats.stats[:resolution_stop] = Time.now
+      @stats.stats[:resolution][:stop_time] = Time.now
       @stats.stats[:status] = :finish
       @processor.writer.close
     end
@@ -53,7 +58,7 @@ module GnListResolver
 
     def resolution_stats(records_num)
       @stats.stats[:total_records] = records_num
-      @stats.stats[:resolution_start] = Time.now
+      @stats.stats[:resolution][:start_time] = Time.now
       @stats.stats[:status] = :resolution
     end
 
@@ -82,14 +87,11 @@ module GnListResolver
         update_stats(stats)
         @processor.process(results, current_data)
       else
-<<<<<<< HEAD:lib/gn_list_resolver/resolver.rb
-        GnListResolver.logger.error("Remote resolver server failed")
-=======
-        GnCrossmap.logger.error(job.reason.message)
->>>>>>> 36115cc... better error log:lib/gn_crossmap/resolver.rb
+        GnResolver.logger.error(job.reason.message)
       end
     end
 
+<<<<<<< HEAD:lib/gn_list_resolver/resolver.rb
     def update_stats(job_stats)
       s = @stats.stats
       s[:last_batches_time].shift if s[:last_batches_time].size > 2
@@ -97,6 +99,8 @@ module GnListResolver
       s[:resolution_span] = Time.now - s[:resolution_start]
     end
 
+=======
+>>>>>>> 2a4afb8... Fix #42 - better speed estimation:lib/gn_crossmap/resolver.rb
     def create_job(batch)
       batch_data = collect_names(batch)
       rb = ResolverJob.new(batch, batch_data, @ds_id)
@@ -116,18 +120,46 @@ module GnListResolver
         id = row[:id].strip
         batch_data[id] = row[:original]
         @processor.input[id] = { rank: row[:rank] }
+<<<<<<< HEAD:lib/gn_list_resolver/resolver.rb
       end
       batch_data
+=======
+        str << "#{id}|#{row[:name]}"
+      end
+      [names, batch_data]
+>>>>>>> 2a4afb8... Fix #42 - better speed estimation:lib/gn_crossmap/resolver.rb
+    end
+
+    # rubocop:disable Metrics/AbcSize
+    def update_stats(job_stats)
+      s = @stats.stats
+      current_speed = job_stats.stats[:current_speed] *
+                      @stats.penalty(@threads)
+
+      s[:resolution][:completed_records] +=
+        job_stats.stats[:resolution][:completed_records]
+      @stats.update_eta(current_speed)
+      s[:matches][7] += job_stats.stats[:matches][7]
     end
 
     def with_log
+      yield
       s = @count + 1
       @count += @batch
       e = [@count, @stats.stats[:total_records]].min
+<<<<<<< HEAD:lib/gn_list_resolver/resolver.rb
       GnListResolver.log("Resolve #{s}-#{e} out of " \
                      "#{@stats.stats[:total_records]} records at " \
                      "#{RESOLVER_URL}")
       yield
+=======
+      eta = @stats.stats[:resolution][:eta].to_i + Time.now.to_i
+      msg = format("Resolve %s-%s/%s records %d rec/s; eta: %s", s, e,
+                   @stats.stats[:total_records],
+                   @stats.stats[:resolution][:speed].to_i,
+                   Time.at(eta))
+      GnCrossmap.log(msg)
+>>>>>>> 2a4afb8... Fix #42 - better speed estimation:lib/gn_crossmap/resolver.rb
     end
   end
 end

data/lib/gn_list_resolver/resolver_job.rb

@@ -33,7 +33,8 @@ module GnListResolver
     end
 
     def stats_add_batch_time(batch_start)
-      @stats.stats[:last_batches_time] << Time.now - batch_start
+      @stats.stats[:current_speed] = @names.size / (Time.now - batch_start)
+      @stats.stats[:resolution][:completed_records] = @names.size
     end
   end
 end

data/lib/gn_list_resolver/result_processor.rb

@@ -24,7 +24,6 @@ module GnListResolver
 
     def write_empty_result(datum)
       @stats.stats[:matches][MATCH_TYPE_EMPTY] += 1
-      @stats.stats[:resolved_records] += 1
       res = compile_empty_result(datum)
       @writer.write(res)
     end
@@ -54,9 +53,7 @@ module GnListResolver
                          else
                            match_type_min.match_type.kind.to_sym
                          end
-      require "byebug"; byebug if @stats.stats[:matches][match_type_value].nil?
       @stats.stats[:matches][match_type_value] += 1
-      @stats.stats[:resolved_records] += 1
     end
 
     def compile_result(datum, result, match_size)

data/lib/gn_list_resolver/stats.rb

@@ -7,11 +7,24 @@ module GnListResolver
 
     def initialize
       @stats = { status: :init, total_records: 0, ingested_records: 0,
-                 resolved_records: 0, ingestion_span: nil,
-                 resolution_span: nil, ingestion_start: nil,
-                 resolution_start: nil, resolution_stop: nil,
-                 last_batches_time: [], matches: init_matches,
-                 errors: [] }
+                 ingestion_span: nil, ingestion_start: nil,
+                 resolution: eta_struct,
+                 matches: init_matches, errors: [] }
+      @smooth = 0.05
+    end
+
+    def penalty(threads)
+      pnlt = 0.7
+      penalty_adj(threads.to_i, 1, pnlt)
+    end
+
+    def update_eta(current_speed)
+      eta = @stats[:resolution]
+      eta[:speed] = current_speed if eta[:speed].nil?
+      eta[:speed] = eta[:speed] * (1 - @smooth) + current_speed * @smooth
+      eta[:eta] = (@stats[:total_records] -
+                   @stats[:resolution][:completed_records]) /
+                  eta[:speed]
     end
 
     private
@@ -19,5 +32,15 @@ module GnListResolver
     def init_matches
       MATCH_TYPES.keys.each_with_object({}) { |t, h| h[t] = 0 }
     end
+
+    def eta_struct
+      { start_time: nil, completed_records: 0,
+        speed: nil, eta: nil, stop_time: nil }
+    end
+
+    def penalty_adj(threads, val, pnlt)
+      return val if threads < 2
+      val + penalty_adj(threads - 1, (val * pnlt), pnlt)
+    end
   end
 end

data/lib/gn_list_resolver/stats.rb.orig

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+module GnListResolver
+  # Collects statistics about list resolving process
+  class Stats
+    attr_accessor :stats
+
+    def initialize
+      @stats = { status: :init, total_records: 0, ingested_records: 0,
+<<<<<<< HEAD:lib/gn_list_resolver/stats.rb
+                 resolved_records: 0, ingestion_span: nil,
+                 resolution_span: nil, ingestion_start: nil,
+                 resolution_start: nil, resolution_stop: nil,
+                 last_batches_time: [], matches: init_matches,
+                 errors: [] }
+=======
+                 ingestion_span: nil, ingestion_start: nil,
+                 resolution: eta_struct,
+                 matches: match_types, errors: [] }
+      @smooth = 0.05
+    end
+
+    def penalty(threads)
+      pnlt = 0.7
+      penalty_adj(threads.to_i, 1, pnlt)
+    end
+
+    def update_eta(current_speed)
+      eta = @stats[:resolution]
+      eta[:speed] = current_speed if eta[:speed].nil?
+      eta[:speed] = eta[:speed] * (1 - @smooth) + current_speed * @smooth
+      eta[:eta] = (@stats[:total_records] -
+                   @stats[:resolution][:completed_records]) /
+                  eta[:speed]
+>>>>>>> 2a4afb8... Fix #42 - better speed estimation:lib/gn_crossmap/stats.rb
+    end
+
+    private
+
+<<<<<<< HEAD:lib/gn_list_resolver/stats.rb
+    def init_matches
+      MATCH_TYPES.keys.each_with_object({}) { |t, h| h[t] = 0 }
+=======
+    def eta_struct
+      { start_time: nil, completed_records: 0,
+        speed: nil, eta: nil, stop_time: nil }
+    end
+
+    def match_types
+      matches = GnCrossmap::MATCH_TYPES.keys
+      matches.each_with_object({}) do |key, obj|
+        obj[key] = 0
+      end
+>>>>>>> 2a4afb8... Fix #42 - better speed estimation:lib/gn_crossmap/stats.rb
+    end
+
+    def penalty_adj(threads, val, pnlt)
+      return val if threads < 2
+      val + penalty_adj(threads - 1, (val * pnlt), pnlt)
+    end
+  end
+end

data/lib/gn_list_resolver/version.rb

@@ -2,7 +2,7 @@
 
 # Namespace module for crossmapping checklists to GN sources
 module GnListResolver
-  VERSION = "3.3.1.1"
+  VERSION = "4.0.1.0"
 
   def self.version
     VERSION

data/lib/gn_list_resolver/version.rb.orig

@@ -3,11 +3,11 @@
 # Namespace module for crossmapping checklists to GN sources
 <<<<<<< HEAD:lib/gn_list_resolver/version.rb
 module GnListResolver
-  VERSION = "3.3.1.0"
+  VERSION = "3.3.1.1"
 =======
 module GnCrossmap
-  VERSION = "3.3.1"
->>>>>>> 36115cc... better error log:lib/gn_crossmap/version.rb
+  VERSION = "4.0.1"
+>>>>>>> 2a4afb8... Fix #42 - better speed estimation:lib/gn_crossmap/version.rb
 
   def self.version
     VERSION

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gn_list_resolver
 version: !ruby/object:Gem::Version
-  version: 3.3.1.1
+  version: 4.0.1.0
 platform: ruby
 authors:
 - Dmitry Mozzherin
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-09-15 00:00:00.000000000 Z
+date: 2017-09-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: biodiversity
@@ -231,6 +231,7 @@ files:
 - lib/gn_list_resolver/result_processor.rb
 - lib/gn_list_resolver/sci_name_collector.rb
 - lib/gn_list_resolver/stats.rb
+- lib/gn_list_resolver/stats.rb.orig
 - lib/gn_list_resolver/version.rb
 - lib/gn_list_resolver/version.rb.orig
 - lib/gn_list_resolver/writer.rb