gn_crossmap 1.0.0 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 779a3a9193e896242a7e8717fd98bc779b417937
-  data.tar.gz: d8e9c4c7d72447a62d35f57fe26e33e9b5a69a29
+  metadata.gz: 215ff795d273c98f6c25eab1e93ac32d21b3e1be
+  data.tar.gz: 5fd8159546705e355fe662ddb6f0b136edc36ce4
 SHA512:
-  metadata.gz: 83f6f2f6be28c5891d93e5f6e5e09ac54481f4334640ef3b500822301bb5d1ea2cf0d5ca94f4ba0a2eb7b7e7afcd67fed8b37ad76c61ac184c2d9932d080ccf3
-  data.tar.gz: ae609d842d36b96de15ffe3013fe49e762da963191609486757d7ba9bbc5e85cdf674a19e24916fdcac7214d37adb1a9c089c95cd89ba9c4c7360256858cca11
+  metadata.gz: 9e467719f0be4a7c7a2293e9ec78dbdbf54b8dfc8198a4e1f12417a239cddf982da495434006c3b40d5386b58896b18bbedff68d358fced0b4f1d06d038ed9c4
+  data.tar.gz: c41b99ddafe9479773198e900e2dcd07f83d19ebe9929a3b9b228b0d10abad475b85429301f108fdc94ddb6529b91ebfc6921ac42215dbacfb03aacdbeb822fa

data/CHANGELOG.md

@@ -1,5 +1,10 @@
 # ``gn_crossmap`` CHANGELOG
 
+## 1.1.0
+
+* @dimus - #23 optional block for GnCrossmap.run gives access to intermediate
+           results
+
 ## 1.0.0
 
 * @dimus - #18 output file optionally removes original fields except `taxonID`

data/README.md

@@ -98,6 +98,44 @@ GnCrossmap.logger = MyCustomLogger.new
 GnCrossmap.run("path/to/input.csv", "path/to/output.csv", 5, true)
 ```
 
+If you want to get intermediate statistics for each resolution cycle use a
+block:
+
+```ruby
+GnCrossmap.run("path/to/input.csv", "path/to/output.csv", 5, true) do |stats|
+  puts stats
+  put "Matches:"
+  stats[:matches].each do |key, value|
+    puts "#{GnCrossmap::MATCH_TYPES[key]}: #{value}"
+  end
+end
+```
+
+#### Intermediate stats format
+
+|Field           | Description                                             |
+|----------------|---------------------------------------------------------|
+|total           | total number of names in original list                  |
+|current         | number of names already processed                       |
+|start_time      | Start of resolution                                     |
+|last_batch_time | time span of the last batch processing                  |
+|matches         | Distribution of processed data by match type (see below)|
+
+#### Match types
+
+Match types dictionary can be accessed with `GnCrossmap::MATCH_TYPES` constant
+
+| Match code | Match type                       |
+|------------|----------------------------------|
+|0           |No match                          |
+|1           |Exact string match                |
+|2           |Canonical form exact match        |
+|3           |Canonical form fuzzy match        |
+|4           |Partial canonical form match      |
+|5           |Partial canonical form fuzzy match|
+|6           |Genus part match                  |
+|7           |Error in matching                 |
+
 ### Input file format
 
 - Comma Separated File with names of fields in first row.

data/lib/gn_crossmap.rb

@@ -18,6 +18,16 @@ require "gn_crossmap/result_processor"
 module GnCrossmap
   INPUT_MODE = "r:utf-8".freeze
   OUTPUT_MODE = "w:utf-8".freeze
+  MATCH_TYPES = {
+    0 => "No match",
+    1 => "Exact string match",
+    2 => "Canonical form exact match",
+    3 => "Canonical form fuzzy match",
+    4 => "Partial canonical form match",
+    5 => "Partial canonical form fuzzy match",
+    6 => "Genus part match",
+    7 => "Error in matching"
+  }.freeze
 
   class << self
     attr_writer :logger
@@ -28,7 +38,8 @@ module GnCrossmap
       data = reader.read
       writer = Writer.new(output_io, reader.original_fields,
                           output_name(output))
-      Resolver.new(writer, data_source_id).resolve(data)
+      resolver = Resolver.new(writer, data_source_id)
+      block_given? ? resolver.resolve(data, &Proc.new) : resolver.resolve(data)
       output
     end
 

data/lib/gn_crossmap/resolver.rb

@@ -4,7 +4,9 @@ module GnCrossmap
     URL = "http://resolver.globalnames.org/name_resolvers.json".freeze
 
     def initialize(writer, data_source_id)
-      @processor = GnCrossmap::ResultProcessor.new(writer)
+      @stats = { total: 0, current: 0, start_time: nil, last_batch_time: nil,
+                 matches: match_types }
+      @processor = GnCrossmap::ResultProcessor.new(writer, @stats)
       @ds_id = data_source_id
       @count = 0
       @current_data = {}
@@ -12,11 +14,13 @@ module GnCrossmap
     end
 
     def resolve(data)
-      data_size = data.size
+      @stats[:total] = data.size
+      @stats[:start_time] = Time.now
       data.each_slice(@batch) do |slice|
-        with_log(data_size) do
+        with_log do
           names = collect_names(slice)
           remote_resolve(names)
+          yield(@stats) if block_given?
         end
       end
       @processor.writer.close
@@ -24,11 +28,18 @@ module GnCrossmap
 
     private
 
-    def with_log(size)
+    def match_types
+      matches = GnCrossmap::MATCH_TYPES.keys
+      matches.each_with_object({}) do |key, obj|
+        obj[key] = 0
+      end
+    end
+
+    def with_log
       s = @count + 1
       @count += @batch
-      e = [@count, size].min
-      GnCrossmap.log("Resolve #{s}-#{e} out of #{size} records")
+      e = [@count, @stats[:total]].min
+      GnCrossmap.log("Resolve #{s}-#{e} out of #{@stats[:total]} records")
       yield
     end
 
@@ -42,10 +53,13 @@ module GnCrossmap
     end
 
     def remote_resolve(names)
+      batch_start = Time.now
       res = RestClient.post(URL, data: names, data_source_ids: @ds_id)
       @processor.process(res, @current_data)
     rescue RestClient::Exception
       single_remote_resolve(names)
+    ensure
+      @stats[:last_batch_time] = Time.now - batch_start
     end
 
     def single_remote_resolve(names)
@@ -54,10 +68,16 @@ module GnCrossmap
           res = RestClient.post(URL, data: name, data_source_ids: @ds_id)
           @processor.process(res, @current_data)
         rescue RestClient::Exception => e
-          GnCrossmap.logger.error("Resolver broke on '#{name}': #{e.message}")
+          process_resolver_error(e, name)
           next
         end
       end
     end
+
+    def process_resolver_error(err, name)
+      @stats[:matches][7] += 1
+      @stats[:current] += 1
+      GnCrossmap.logger.error("Resolver broke on '#{name}': #{err.message}")
+    end
   end
 end

data/lib/gn_crossmap/result_processor.rb

@@ -1,19 +1,10 @@
 module GnCrossmap
   # Processes data received from the GN Resolver
   class ResultProcessor
-    MATCH_TYPES = {
-      0 => "No match",
-      1 => "Exact string match",
-      2 => "Canonical form exact match",
-      3 => "Canonical form fuzzy match",
-      4 => "Partial canonical form match",
-      5 => "Partial canonical form fuzzy match",
-      6 => "Genus part match"
-    }.freeze
-
     attr_reader :input, :writer
 
-    def initialize(writer)
+    def initialize(writer, stats)
+      @stats = stats
       @writer = writer
       @input = {}
     end
@@ -33,19 +24,28 @@ module GnCrossmap
     end
 
     def write_empty_result(datum)
+      @stats[:matches][0] += 1
+      @stats[:current] += 1
       res = @original_data[datum[:supplied_id]]
-      res += [MATCH_TYPES[0], datum[:supplied_name_string], nil,
+      res += [GnCrossmap::MATCH_TYPES[0], datum[:supplied_name_string], nil,
               nil, @input[datum[:supplied_id]][:rank], nil,
               nil, nil, nil]
       @writer.write(res)
     end
 
     def write_result(datum)
+      collect_stats(datum)
       datum[:results].each do |result|
         @writer.write(compile_result(datum, result))
       end
     end
 
+    def collect_stats(datum)
+      match_num = datum[:results].map { |d| d[:match_type] }.min
+      @stats[:matches][match_num] += 1
+      @stats[:current] += 1
+    end
+
     def compile_result(datum, result)
       @original_data[datum[:supplied_id]] + new_data(datum, result)
     end
@@ -64,7 +64,7 @@ module GnCrossmap
     end
 
     def matched_type(record)
-      MATCH_TYPES[record[:match_type]]
+      GnCrossmap::MATCH_TYPES[record[:match_type]]
     end
   end
 end

data/lib/gn_crossmap/version.rb

@@ -1,6 +1,6 @@
 # Namespace module for crossmapping checklists to GN sources
 module GnCrossmap
-  VERSION = "1.0.0".freeze
+  VERSION = "1.1.0".freeze
 
   def self.version
     VERSION

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gn_crossmap
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.1.0
 platform: ruby
 authors:
 - Dmitry Mozzherin
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-11-22 00:00:00.000000000 Z
+date: 2016-11-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: trollop