gn_crossmap 4.0.0 → 4.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 17c1750e71b66ba0d13b76a4acaa110f00b7cdd0
-  data.tar.gz: 96f3b4d6955c10724f78749c99d9c0414da21278
+  metadata.gz: 49f923dfdab693728fe296afcf09fa275ce870ad
+  data.tar.gz: da0ac647957f8c28c438af7954b1e35473bcfc28
 SHA512:
-  metadata.gz: 874afab0ba3e3c7e847ebcb8a86c439bcdcc56ab84ae86b0d5dc1912cd7dd8dff7fefa5bc0d031a082b9b854c11714879db44568edf79bcc698ba8337a633b9e
-  data.tar.gz: 4cff3ac93255cd203ef4c2e9083ff1da6babb94a6367afb244c38e743c23be620fb154aad8e7deed818c16ada4f7ad0b78643123b72dd2c54798cafa8f70728e
+  metadata.gz: 37e8d0183db3322a0bf627c336737a6dc42ae4471c4f52554ea761cfcdcc71210c6d0a793c948623a647cfbc97f7d782538049f4a62bdb97de6bded8ac2998ff
+  data.tar.gz: db2ccccf36fd9bfe04f77e3d75eca76286d82b4cc3f8e79d78fb54c443b6f991f2a235ba396a7bc4c41ec997a15aaec283eb676e577ac52358cb39017154d7d3

data/CHANGELOG.md

@@ -1,5 +1,9 @@
 # ``gn_crossmap`` CHANGELOG
 
+## 4.0.1
+
+* @dimus - Clean up resolution stats
+
 ## 4.0.0
 
 * @dimus - Better estimation for time left for resolver and speed

data/lib/gn_crossmap/resolver.rb

@@ -31,7 +31,7 @@ module GnCrossmap
     private
 
     def wrap_up
-      @stats.stats[:resolution_stop] = Time.now
+      @stats.stats[:resolution][:stop_time] = Time.now
       @stats.stats[:status] = :finish
       @processor.writer.close
     end
@@ -52,7 +52,7 @@ module GnCrossmap
 
     def resolution_stats(records_num)
       @stats.stats[:total_records] = records_num
-      @stats.stats[:resolution_start] = Time.now
+      @stats.stats[:resolution][:start_time] = Time.now
       @stats.stats[:status] = :resolution
     end
 
@@ -85,20 +85,6 @@ module GnCrossmap
       end
     end
 
-    # rubocop:disable Metrics/AbcSize
-    def update_stats(job_stats)
-      s = @stats.stats
-      s[:current_speed] = job_stats.stats[:current_speed]
-      s[:speed] = s[:current_speed] * @threads unless s[:speed]
-      s[:speed] = s[:speed] * (1 - @smoothing) +
-                  s[:current_speed] * @smoothing * @threads
-      s[:resolution_span] = Time.now - s[:resolution_start]
-      s[:resolved_records] += job_stats.stats[:resolved_records]
-      s[:eta] = (s[:total_records] - s[:resolved_records]) / s[:speed]
-      s[:matches][7] += job_stats.stats[:matches][7]
-    end
-    # rubocop:enable all
-
     def create_job(batch)
       names, batch_data = collect_names(batch)
       rb = ResolverJob.new(names, batch_data, @resolver_url, @ds_id)
@@ -120,19 +106,34 @@ module GnCrossmap
         batch_data[id] = row[:original]
         @processor.input[id] = { rank: row[:rank] }
         str << "#{id}|#{row[:name]}"
-      end.join("\n")
+      end
       [names, batch_data]
     end
 
+    # rubocop:disable Metrics/AbcSize
+    def update_stats(job_stats)
+      s = @stats.stats
+      current_speed = job_stats.stats[:current_speed] *
+                      @stats.penalty(@threads)
+
+      s[:resolution][:completed_records] +=
+        job_stats.stats[:resolution][:completed_records]
+      @stats.update_eta(current_speed)
+      s[:matches][7] += job_stats.stats[:matches][7]
+    end
+
     def with_log
       yield
       s = @count + 1
       @count += @batch
       e = [@count, @stats.stats[:total_records]].min
-      msg = format("Resolve %s-%s/%s records %d rec/s; eta: %d", s, e,
-                   @stats.stats[:total_records], @stats.stats[:speed],
-                   @stats.stats[:eta].to_i + Time.now.to_i)
+      eta = @stats.stats[:resolution][:eta].to_i + Time.now.to_i
+      msg = format("Resolve %s-%s/%s records %d rec/s; eta: %s", s, e,
+                   @stats.stats[:total_records],
+                   @stats.stats[:resolution][:speed].to_i,
+                   Time.at(eta))
       GnCrossmap.log(msg)
     end
   end
 end
+# rubocop:enable all

data/lib/gn_crossmap/resolver_job.rb

@@ -4,7 +4,8 @@ module GnCrossmap
   # Remote resolution for parallel jobs
   class ResolverJob
     def initialize(names, batch_data, resolver_url, ds_id)
-      @names = names
+      @names_size = names.size
+      @names = names.join("\n")
       @batch_data = batch_data
       @resolver_url = resolver_url
       @ds_id = ds_id
@@ -26,7 +27,7 @@ module GnCrossmap
     rescue RestClient::Exception
       single_remote_resolve(names)
     ensure
-      stats_add_batch_time(batch_start)
+      update_stats(batch_start)
     end
 
     def single_remote_resolve(names)
@@ -49,12 +50,12 @@ module GnCrossmap
 
     def process_resolver_error(err, name)
       @stats.stats[:matches][7] += 1
-      @stats.stats[:resolved_records] += 1
       GnCrossmap.logger.error("Resolver broke on '#{name}': #{err.message}")
     end
 
-    def stats_add_batch_time(batch_start)
+    def update_stats(batch_start)
       @stats.stats[:current_speed] = @batch_data.size / (Time.now - batch_start)
+      @stats.stats[:resolution][:completed_records] = @names_size
     end
   end
 end

data/lib/gn_crossmap/result_processor.rb

@@ -31,7 +31,6 @@ module GnCrossmap
 
     def write_empty_result(datum)
       @stats.stats[:matches][0] += 1
-      @stats.stats[:resolved_records] += 1
       res = compile_empty_result(datum)
       @writer.write(res)
     end
@@ -56,7 +55,6 @@ module GnCrossmap
     def collect_stats(datum)
       match_num = datum[:results].map { |d| d[:match_type] }.min
       @stats.stats[:matches][match_num] += 1
-      @stats.stats[:resolved_records] += 1
     end
 
     def compile_result(datum, result, match_size)

data/lib/gn_crossmap/stats.rb

@@ -7,20 +7,43 @@ module GnCrossmap
 
     def initialize
       @stats = { status: :init, total_records: 0, ingested_records: 0,
-                 resolved_records: 0, ingestion_span: nil,
-                 resolution_span: nil, ingestion_start: nil,
-                 resolution_start: nil, resolution_stop: nil,
-                 speed: nil, current_speed: nil, eta: nil,
+                 ingestion_span: nil, ingestion_start: nil,
+                 resolution: eta_struct,
                  matches: match_types, errors: [] }
+      @smooth = 0.05
+    end
+
+    def penalty(threads)
+      pnlt = 0.7
+      penalty_adj(threads.to_i, 1, pnlt)
+    end
+
+    def update_eta(current_speed)
+      eta = @stats[:resolution]
+      eta[:speed] = current_speed if eta[:speed].nil?
+      eta[:speed] = eta[:speed] * (1 - @smooth) + current_speed * @smooth
+      eta[:eta] = (@stats[:total_records] -
+                   @stats[:resolution][:completed_records]) /
+                  eta[:speed]
     end
 
     private
 
+    def eta_struct
+      { start_time: nil, completed_records: 0,
+        speed: nil, eta: nil, stop_time: nil }
+    end
+
     def match_types
       matches = GnCrossmap::MATCH_TYPES.keys
       matches.each_with_object({}) do |key, obj|
         obj[key] = 0
       end
     end
+
+    def penalty_adj(threads, val, pnlt)
+      return val if threads < 2
+      val + penalty_adj(threads - 1, (val * pnlt), pnlt)
+    end
   end
 end

data/lib/gn_crossmap/version.rb

@@ -2,7 +2,7 @@
 
 # Namespace module for crossmapping checklists to GN sources
 module GnCrossmap
-  VERSION = "4.0.0"
+  VERSION = "4.0.1"
 
   def self.version
     VERSION

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gn_crossmap
 version: !ruby/object:Gem::Version
-  version: 4.0.0
+  version: 4.0.1
 platform: ruby
 authors:
 - Dmitry Mozzherin
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-09-13 00:00:00.000000000 Z
+date: 2017-09-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: biodiversity