gn_crossmap 1.1.0 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 215ff795d273c98f6c25eab1e93ac32d21b3e1be
-  data.tar.gz: 5fd8159546705e355fe662ddb6f0b136edc36ce4
+  metadata.gz: f5a97cd143dd1e3bd3940b6457dd4b1f824236c7
+  data.tar.gz: 12a4ed179bdadaccacf34fe130300e77147d4c35
 SHA512:
-  metadata.gz: 9e467719f0be4a7c7a2293e9ec78dbdbf54b8dfc8198a4e1f12417a239cddf982da495434006c3b40d5386b58896b18bbedff68d358fced0b4f1d06d038ed9c4
-  data.tar.gz: c41b99ddafe9479773198e900e2dcd07f83d19ebe9929a3b9b228b0d10abad475b85429301f108fdc94ddb6529b91ebfc6921ac42215dbacfb03aacdbeb822fa
+  metadata.gz: 5cbcc463342999b5f9c9e7dcc85f031783f3e4d9e31c8a8a4152bd64dddae086f5f82e1bbcef44f14fdd8fd3721c819d46ae892c7b02cdc256c3728b8359c1f8
+  data.tar.gz: 4d5df19d77f565ab6ad268f9106e93a8aafe6d2de0b6baff7906d78bc469a50a583c29423a611b406612a6355f1aefe106ec16ad0e7653e31f0d5c9fe6ef6ed5

data/CHANGELOG.md

@@ -1,5 +1,10 @@
 # ``gn_crossmap`` CHANGELOG
 
+## 1.2.0
+
+* @dimus - #24 optional block for GnCrossmap.run now also gives access to
+           intermediate results of CSV file reading
+
 ## 1.1.0
 
 * @dimus - #23 optional block for GnCrossmap.run gives access to intermediate

data/README.md

@@ -104,7 +104,7 @@ block:
 ```ruby
 GnCrossmap.run("path/to/input.csv", "path/to/output.csv", 5, true) do |stats|
   puts stats
-  put "Matches:"
+  puts "Matches:"
   stats[:matches].each do |key, value|
     puts "#{GnCrossmap::MATCH_TYPES[key]}: #{value}"
   end
@@ -113,13 +113,18 @@ end
 
 #### Intermediate stats format
 
-|Field           | Description                                             |
-|----------------|---------------------------------------------------------|
-|total           | total number of names in original list                  |
-|current         | number of names already processed                       |
-|start_time      | Start of resolution                                     |
-|last_batch_time | time span of the last batch processing                  |
-|matches         | Distribution of processed data by match type (see below)|
+|Field             |Description                                              |
+|------------------|---------------------------------------------------------|
+|status            |current phase: (init, ingested                           |
+|total_records     |total number of names in original list                   |
+|ingestion_start   |time when the reading from csv started                   |
+|ingestion_span    |time of intermediate checkpoint of reading csv           |
+|ingested_records  |number of ingested records at an intermediate checkpoint |
+|resolution_start  |time when resolution of names started                    |
+|resolution_span   |time of intermediate checkpoint of resolving names       |
+|resolved_records  |number of names already processed                        |
+|last_batches_time |time required to process the last batch of names         |
+|matches           |Distribution of processed data by match type (see below) |
 
 #### Match types
 

data/lib/gn_crossmap.rb

@@ -13,6 +13,7 @@ require "gn_crossmap/column_collector"
 require "gn_crossmap/sci_name_collector"
 require "gn_crossmap/resolver"
 require "gn_crossmap/result_processor"
+require "gn_crossmap/stats"
 
 # Namespace module for crossmapping checklists wth GN sources
 module GnCrossmap
@@ -32,17 +33,22 @@ module GnCrossmap
   class << self
     attr_writer :logger
 
+    # rubocop:disable Metrics/AbcSize
+
     def run(input, output, data_source_id, skip_original)
+      stats = Stats.new
       input_io, output_io = io(input, output)
-      reader = Reader.new(input_io, input_name(input), skip_original)
-      data = reader.read
+      reader = Reader.new(input_io, input_name(input), skip_original, stats)
+      data = block_given? ? reader.read(&Proc.new) : reader.read
       writer = Writer.new(output_io, reader.original_fields,
                           output_name(output))
-      resolver = Resolver.new(writer, data_source_id)
+      resolver = Resolver.new(writer, data_source_id, stats)
       block_given? ? resolver.resolve(data, &Proc.new) : resolver.resolve(data)
       output
     end
 
+    # rubocop:enable all
+
     def logger
       @logger ||= Logger.new(STDERR)
     end

data/lib/gn_crossmap/reader.rb

@@ -4,7 +4,8 @@ module GnCrossmap
   class Reader
     attr_reader :original_fields
 
-    def initialize(csv_io, input_name, skip_original)
+    def initialize(csv_io, input_name, skip_original, stats)
+      @stats = stats
       @csv_io = csv_io
       @col_sep = col_sep
       @original_fields = nil
@@ -13,14 +14,17 @@ module GnCrossmap
     end
 
     def read
+      @stats.stats[:ingestion_start] = Time.now
+      @stats.stats[:status] = :ingestion
       GnCrossmap.log("Read input from #{@input_name}")
-      parse_input
+      block_given? ? parse_input(&Proc.new) : parse_input
     end
 
     private
 
     def col_sep
       line = @csv_io.first
+      @stats.stats[:total_records] = @csv_io.readlines.size
       @csv_io.rewind
       [";", ",", "\t"].map { |s| [line.count(s), s] }.sort.last.last
     end
@@ -30,13 +34,20 @@ module GnCrossmap
       csv = CSV.new(@csv_io, col_sep: col_sep)
       csv.each_with_index do |row, i|
         @original_fields = headers(row) if @original_fields.nil?
-        i += 1
-        GnCrossmap.log("Ingesting #{i}th csv row") if (i % 10_000).zero?
+        yield @stats.stats if log_progress(i) && block_given?
         dc.process_row(row)
       end && @csv_io.close
       dc.data
     end
 
+    def log_progress(count)
+      return false unless (count % 10_000).zero?
+      GnCrossmap.log("Ingesting csv row #{count + 1}")
+      @stats.stats[:ingested_records] = count + 1
+      @stats.stats[:ingestion_span] = Time.now - @stats.stats[:ingestion_start]
+      true
+    end
+
     def headers(row)
       hdrs = row.dup
       @skip_original ? taxon_id_header(hdrs) : hdrs

data/lib/gn_crossmap/resolver.rb

@@ -3,9 +3,8 @@ module GnCrossmap
   class Resolver
     URL = "http://resolver.globalnames.org/name_resolvers.json".freeze
 
-    def initialize(writer, data_source_id)
-      @stats = { total: 0, current: 0, start_time: nil, last_batch_time: nil,
-                 matches: match_types }
+    def initialize(writer, data_source_id, stats)
+      @stats = stats
       @processor = GnCrossmap::ResultProcessor.new(writer, @stats)
       @ds_id = data_source_id
       @count = 0
@@ -14,13 +13,12 @@ module GnCrossmap
     end
 
     def resolve(data)
-      @stats[:total] = data.size
-      @stats[:start_time] = Time.now
+      update_stats(data.size)
       data.each_slice(@batch) do |slice|
         with_log do
           names = collect_names(slice)
           remote_resolve(names)
-          yield(@stats) if block_given?
+          yield(@stats.stats) if block_given?
         end
       end
       @processor.writer.close
@@ -28,18 +26,18 @@ module GnCrossmap
 
     private
 
-    def match_types
-      matches = GnCrossmap::MATCH_TYPES.keys
-      matches.each_with_object({}) do |key, obj|
-        obj[key] = 0
-      end
+    def update_stats(records_num)
+      @stats.stats[:total_records] = records_num
+      @stats.stats[:resolution_start] = Time.now
+      @stats.stats[:status] = :resolution
     end
 
     def with_log
       s = @count + 1
       @count += @batch
-      e = [@count, @stats[:total]].min
-      GnCrossmap.log("Resolve #{s}-#{e} out of #{@stats[:total]} records")
+      e = [@count, @stats.stats[:total_records]].min
+      GnCrossmap.log("Resolve #{s}-#{e} out of " \
+                     "#{@stats.stats[:total_records]} records")
       yield
     end
 
@@ -59,7 +57,14 @@ module GnCrossmap
     rescue RestClient::Exception
       single_remote_resolve(names)
     ensure
-      @stats[:last_batch_time] = Time.now - batch_start
+      update_batch_times(batch_start)
+    end
+
+    def update_batch_times(batch_start)
+      s = @stats.stats
+      s[:last_batches_time].shift if s[:last_batches_time].size > 2
+      s[:last_batches_time] << Time.now - batch_start
+      s[:resolution_span] = Time.now - s[:resolution_start]
     end
 
     def single_remote_resolve(names)
@@ -75,8 +80,8 @@ module GnCrossmap
     end
 
     def process_resolver_error(err, name)
-      @stats[:matches][7] += 1
-      @stats[:current] += 1
+      @stats.stats[:matches][7] += 1
+      @stats.stats[:resolved_records] += 1
       GnCrossmap.logger.error("Resolver broke on '#{name}': #{err.message}")
     end
   end

data/lib/gn_crossmap/result_processor.rb

@@ -24,8 +24,8 @@ module GnCrossmap
     end
 
     def write_empty_result(datum)
-      @stats[:matches][0] += 1
-      @stats[:current] += 1
+      @stats.stats[:matches][0] += 1
+      @stats.stats[:resolved_records] += 1
       res = @original_data[datum[:supplied_id]]
       res += [GnCrossmap::MATCH_TYPES[0], datum[:supplied_name_string], nil,
               nil, @input[datum[:supplied_id]][:rank], nil,
@@ -42,8 +42,8 @@ module GnCrossmap
 
     def collect_stats(datum)
       match_num = datum[:results].map { |d| d[:match_type] }.min
-      @stats[:matches][match_num] += 1
-      @stats[:current] += 1
+      @stats.stats[:matches][match_num] += 1
+      @stats.stats[:resolved_records] += 1
     end
 
     def compile_result(datum, result)

data/lib/gn_crossmap/stats.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module GnCrossmap
+  # Collects statistics about crossmapping process
+  class Stats
+    attr_accessor :stats
+
+    def initialize
+      @stats = { status: :init, total_records: 0, ingested_records: 0,
+                 resolved_records: 0, ingestion_span: nil,
+                 resolution_span: nil, ingestion_start: nil,
+                 resolution_start: nil, last_batches_time: [],
+                 matches: match_types }
+    end
+
+    private
+
+    def match_types
+      matches = GnCrossmap::MATCH_TYPES.keys
+      matches.each_with_object({}) do |key, obj|
+        obj[key] = 0
+      end
+    end
+  end
+end

data/lib/gn_crossmap/version.rb

@@ -1,6 +1,6 @@
 # Namespace module for crossmapping checklists to GN sources
 module GnCrossmap
-  VERSION = "1.1.0".freeze
+  VERSION = "1.2.0".freeze
 
   def self.version
     VERSION

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gn_crossmap
 version: !ruby/object:Gem::Version
-  version: 1.1.0
+  version: 1.2.0
 platform: ruby
 authors:
 - Dmitry Mozzherin
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-11-28 00:00:00.000000000 Z
+date: 2016-11-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: trollop
@@ -181,6 +181,7 @@ files:
 - lib/gn_crossmap/resolver.rb
 - lib/gn_crossmap/result_processor.rb
 - lib/gn_crossmap/sci_name_collector.rb
+- lib/gn_crossmap/stats.rb
 - lib/gn_crossmap/version.rb
 - lib/gn_crossmap/writer.rb
 homepage: https://github.com/GlobalNamesArchitecture/gn_crossmap