gn_crossmap 3.3.1 → 4.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 94a484698915810c34397b63d22a7c27febd2a65
-  data.tar.gz: 5fe783ff2bf35ad569d1be0393ffbf32fdf910a5
+  metadata.gz: 17c1750e71b66ba0d13b76a4acaa110f00b7cdd0
+  data.tar.gz: 96f3b4d6955c10724f78749c99d9c0414da21278
 SHA512:
-  metadata.gz: 216221d07d2fdb278c69f8c40ec36f6811f8adfbf8163ed86e8ee53b051d0d42d3b6dc309efee736da2adc5a038a5df9ec997ba845edf456c6e5c57a01cef1e4
-  data.tar.gz: 7adc10873cb2bebffc709541a0ef9238e9cce71f82e5c2980b5d14ded3332fc3533a2a96da7fa8be5db515dfae429578be8e0b7df9b9969f0c625e96c40e4ce9
+  metadata.gz: 874afab0ba3e3c7e847ebcb8a86c439bcdcc56ab84ae86b0d5dc1912cd7dd8dff7fefa5bc0d031a082b9b854c11714879db44568edf79bcc698ba8337a633b9e
+  data.tar.gz: 4cff3ac93255cd203ef4c2e9083ff1da6babb94a6367afb244c38e743c23be620fb154aad8e7deed818c16ada4f7ad0b78643123b72dd2c54798cafa8f70728e

data/CHANGELOG.md

@@ -1,5 +1,9 @@
 # ``gn_crossmap`` CHANGELOG
 
+## 4.0.0
+
+* @dimus - Better estimation for time left for resolver and speed
+
 ## 3.3.1
 
 * @dimus - Better error message in logger

data/README.md

@@ -163,7 +163,8 @@ end
 |resolution_stop   |time when resolution of names stopped                    |
 |resolution_span   |time of intermediate checkpoint of resolving names       |
 |resolved_records  |number of names already processed                        |
-|last_batches_time |time required to process the last batch of names         |
+|speed             |weighted speed average                                   |
+|current_speed     |speed of resolution for the last batch                   |
 |matches           |Distribution of processed data by match type (see below) |
 |errors            |First 0-10 errors found during the csv file processing   |
 

data/lib/gn_crossmap/resolver.rb

@@ -14,6 +14,7 @@ module GnCrossmap
       @count = 0
       @jobs = []
       @batch = 200
+      @smoothing = 0.05
     end
 
     def resolve(data)
@@ -87,10 +88,13 @@ module GnCrossmap
     # rubocop:disable Metrics/AbcSize
     def update_stats(job_stats)
       s = @stats.stats
-      s[:last_batches_time].shift if s[:last_batches_time].size > 2
-      s[:last_batches_time] << job_stats.stats[:last_batches_time][0]
+      s[:current_speed] = job_stats.stats[:current_speed]
+      s[:speed] = s[:current_speed] * @threads unless s[:speed]
+      s[:speed] = s[:speed] * (1 - @smoothing) +
+                  s[:current_speed] * @smoothing * @threads
       s[:resolution_span] = Time.now - s[:resolution_start]
       s[:resolved_records] += job_stats.stats[:resolved_records]
+      s[:eta] = (s[:total_records] - s[:resolved_records]) / s[:speed]
       s[:matches][7] += job_stats.stats[:matches][7]
     end
     # rubocop:enable all
@@ -121,13 +125,14 @@ module GnCrossmap
     end
 
     def with_log
+      yield
       s = @count + 1
       @count += @batch
       e = [@count, @stats.stats[:total_records]].min
-      GnCrossmap.log("Resolve #{s}-#{e} out of " \
-                     "#{@stats.stats[:total_records]} records at " \
-                     "#{@resolver_url}")
-      yield
+      msg = format("Resolve %s-%s/%s records %d rec/s; eta: %d", s, e,
+                   @stats.stats[:total_records], @stats.stats[:speed],
+                   @stats.stats[:eta].to_i + Time.now.to_i)
+      GnCrossmap.log(msg)
     end
   end
 end

data/lib/gn_crossmap/resolver_job.rb

@@ -54,7 +54,7 @@ module GnCrossmap
     end
 
     def stats_add_batch_time(batch_start)
-      @stats.stats[:last_batches_time] << Time.now - batch_start
+      @stats.stats[:current_speed] = @batch_data.size / (Time.now - batch_start)
     end
   end
 end

data/lib/gn_crossmap/stats.rb

@@ -10,8 +10,8 @@ module GnCrossmap
                  resolved_records: 0, ingestion_span: nil,
                  resolution_span: nil, ingestion_start: nil,
                  resolution_start: nil, resolution_stop: nil,
-                 last_batches_time: [], matches: match_types,
-                 errors: [] }
+                 speed: nil, current_speed: nil, eta: nil,
+                 matches: match_types, errors: [] }
     end
 
     private

data/lib/gn_crossmap/version.rb

@@ -2,7 +2,7 @@
 
 # Namespace module for crossmapping checklists to GN sources
 module GnCrossmap
-  VERSION = "3.3.1"
+  VERSION = "4.0.0"
 
   def self.version
     VERSION

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gn_crossmap
 version: !ruby/object:Gem::Version
-  version: 3.3.1
+  version: 4.0.0
 platform: ruby
 authors:
 - Dmitry Mozzherin
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-09-12 00:00:00.000000000 Z
+date: 2017-09-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: biodiversity