gn_crossmap 2.1.3 → 2.2.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d9e9c42628d2d1a8a5d8aeea1b0b1614ffe5c7a9
-  data.tar.gz: ed427b1beb9cc204a1bdf622b1521645336af96e
+  metadata.gz: af88cbf0b3ca5ea98ebf8cbb37b41797ba696294
+  data.tar.gz: 75bc41909f7c00eee0311a254e1390aa6ae8307c
 SHA512:
-  metadata.gz: 5f61004fb4c0a2885f46fb1f16274694ee926d4d7c2ca3637f61e462e7ccf1b1c9eb7ac8ce77aad9cf46ad37760f72ecd46c61f52040c0807592de0b904d28e1
-  data.tar.gz: f6ae3e266a4d5738feb623d07176c6641e49edeacecb78378de746adf7addce82edd9d0d501e5efe962c37d99ae55194c31d60169a9a974e656fae09c055bd0f
+  metadata.gz: a784f2f6bab3b46fc4f4d7f3ef0aa69116d528589ff116ad85f050d28d430882672bdaef42e2c2fe21aa526bbaac5d27e55939ec9df4e0049b44f98311c9fc1f
+  data.tar.gz: 2b054ace46de4c2f11b15ab52a47a8371c126a4274c4be439115f1a6e37ce77e6f2148bfc38034f096a6f7d21807c58b7101764a4f0e169589140767a485e38c

data/CHANGELOG.md

@@ -1,5 +1,11 @@
 # ``gn_crossmap`` CHANGELOG
 
+## 2.2.3
+
+* @dimus - make csv processing more permissive, continue
+           it after meeting malformed csv rows. Add "errors"
+           field into status report.
+
 ## 2.1.3
 
 * @dimus - fix nil situation in headers' fields

data/README.md

@@ -154,6 +154,7 @@ end
 |resolved_records  |number of names already processed                        |
 |last_batches_time |time required to process the last batch of names         |
 |matches           |Distribution of processed data by match type (see below) |
+|errors            |First 0-10 errors found during the csv file processing   |
 
 #### Match types
 

data/lib/gn_crossmap/collector.rb

@@ -13,6 +13,7 @@ module GnCrossmap
     def process_row(row)
       @row = row
       @fields ? collect_data : init_fields_collector
+      true
     end
 
     private

data/lib/gn_crossmap/reader.rb

@@ -9,6 +9,7 @@ module GnCrossmap
       @alt_headers = alt_headers
       @csv_io = csv_io
       @col_sep = col_sep
+      @quote_char = quote_char(@col_sep)
       @original_fields = nil
       @input_name = input_name
       @skip_original = skip_original
@@ -30,9 +31,13 @@ module GnCrossmap
       [";", ",", "\t"].map { |s| [line.count(s), s] }.sort.last.last
     end
 
+    def quote_char(col_sep)
+      col_sep == "\t" ? "\x00" : '"'
+    end
+
     def parse_input
       dc = Collector.new(@skip_original)
-      csv = CSV.new(@csv_io, col_sep: col_sep)
+      csv = CSV.new(@csv_io, col_sep: @col_sep, quote_char: @quote_char)
       block_given? ? process(csv, dc, &Proc.new) : process(csv, dc)
       wrap_up
       yield @stats.stats if block_given?
@@ -40,13 +45,24 @@ module GnCrossmap
     end
 
     def process(csv, data_collector)
-      csv.each_with_index do |row, i|
-        row = process_headers(row) if @original_fields.nil?
-        yield @stats.stats if log_progress(i) && block_given?
-        data_collector.process_row(row)
+      counter = 0
+      loop do
+        yield @stats.stats if log_progress(counter) && block_given?
+        rl = read_line(csv, data_collector)
+        break unless rl
+        counter += 1
       end && @csv_io.close
     end
 
+    def read_line(csv, data_collector)
+      row = csv.readline
+      return false if row.nil?
+      row = process_headers(row) if @original_fields.nil?
+      data_collector.process_row(row)
+    rescue CSV::MalformedCSVError => e
+      @stats.stats[:errors] << e.message if @stats.stats[:errors].size < 10
+    end
+
     def wrap_up
       @stats.stats[:ingested_records] = @stats.stats[:total_records]
       @stats.stats[:ingestion_span] = Time.now - @stats.stats[:ingestion_start]

data/lib/gn_crossmap/stats.rb

@@ -10,7 +10,8 @@ module GnCrossmap
                  resolved_records: 0, ingestion_span: nil,
                  resolution_span: nil, ingestion_start: nil,
                  resolution_start: nil, resolution_stop: nil,
-                 last_batches_time: [], matches: match_types }
+                 last_batches_time: [], matches: match_types,
+                 errors: [] }
     end
 
     private

data/lib/gn_crossmap/version.rb

@@ -1,6 +1,6 @@
 # Namespace module for crossmapping checklists to GN sources
 module GnCrossmap
-  VERSION = "2.1.3".freeze
+  VERSION = "2.2.3".freeze
 
   def self.version
     VERSION

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gn_crossmap
 version: !ruby/object:Gem::Version
-  version: 2.1.3
+  version: 2.2.3
 platform: ruby
 authors:
 - Dmitry Mozzherin
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-02-03 00:00:00.000000000 Z
+date: 2017-02-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: trollop