gn_crossmap 0.1.4 → 0.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2860a0033f9ebfcf9d71d4ffb2cf9fa9d29ea7f3
-  data.tar.gz: b09ff0e2ab214279979e268adcaece8207e8c9ed
+  metadata.gz: 30c84b5910edf24b6b67fc45c1f80be36b29a183
+  data.tar.gz: d7481bc663e1b4edd225c83f711a84092fb9d602
 SHA512:
-  metadata.gz: 7c291b3a606eebf4da9ea8abe686bb046c63d30f34e6842346cd65a14a695d8b3f5f9af6c3f1f92a64ff6be69bc6bd8286d4a7a80386b8b239b2b65db1d8ccaa
-  data.tar.gz: 22bd5787bf73447fb8b4fc2795cec9d74ee68357e0163f90b1279d6fda5a9efa2e43c3a3c6656b158ad961923fdb9c27018f811a2babb647bb6eb7347d1b915a
+  metadata.gz: 5754ec3e9d65a69cf8fcf83aaeaa2c24fd4b565917ba37b860a0468e5f9e46ba1e6dcd37dbdb28935313469f704aa84b6faab79584a238b1e17d2e6284b23ac5
+  data.tar.gz: 11ba4423f9c31529f5f1d17ad25609ccfae8d7d6694efea0bcf30137e65e661334a89e30f9259dfd9934676f9bfa76ec59d4ffee07f0d79afffdb9a5bcc67abd

data/CHANGELOG.md

@@ -1,6 +1,21 @@
 gn_crossmap CHANGELOG
 =====================
 
+0.1.5
+-----
+
+* @dimus - #5 - All original fields are now preserved in the output file.
+
+* @dimus - #3 - If ingest has more than 10K rows -- user will see logging events
+
+* @dimus - #4 Bug - Error messages if headers are missing necessary fields
+
+* @dimus - #2 - Header fields are now allowed trailing spaces
+
+* @dimus - #7 Bug - Empty rank does not break crossmapping anymore
+
+* @dimus - #1 Bug - add missing rest-client gem
+
 0.1.4
 -----
 - [Dmitry Mozzherin][dimus] - Bug fixes

data/README.md

@@ -10,6 +10,8 @@ in [GN Resolver][resolver].
 
 Checklist has to be in a CSV format.
 
+[Issues on waffle.io][waffle]
+
 Compatibility
 -------------
 
@@ -35,6 +37,29 @@ Or install it yourself as:
 Usage
 -----
 
+### Usage from command line
+
+    # to see help
+    $ crossmap --help
+
+    # to compare with default source (Catalogue of Life)
+    $ crossmap -i my_list.csv -o my_list_col.csv
+
+    # to compare with other source (Index Fungorum in this example)
+    $ crossmap -i my_list.csv -o my_list_if.csv -d 5
+
+### Usage as Ruby Library
+
+```ruby
+require "gn_crossmap"
+
+# If you want to change logger -- default Logging is to standard output
+GnCrossmap.logger = MyCustomLogger.new
+
+GnCrossmap.run("path/to/input.csv", "path/to/output.csv", 5)
+```
+
+
 ### Input file format
 
 - Comma Separated File with names of fields in first row.
@@ -92,27 +117,33 @@ TaxonId|kingdom|subkingdom|phylum|subphylum|superclass|class|subclass|cohort|sup
 
 More examples can be found in [spec/files][files] directory
 
-### Usage from command line
-
-    # to see help
-    $ crossmap --help
-
-    # to compare with default source (Catalogue of Life)
-    $ crossmap -i my_list.csv -o my_list_col.csv
+### Output file format
 
-    # to compare with other source (Index Fungorum in this example)
-    $ crossmap -i my_list.csv -o my_list_if.csv -d 5
+[Output][output] includes following fields:
 
-### Usage as Ruby Library
+Field                | Description
+---------------------|-----------------------------------------------------------
+taxonID              | original ID attached to a name in the checklist
+scientificName       | name from the checklist
+matchedScientificName| name matched from the GN Reolver data source
+matchedCanonicalForm | canonical form of the matched name
+rank                 | rank from the source (if it was given/inferred)
+matchedRank          | corresponding rank from the data source
+matchType            | what kind of match it is
+editDistance         | for fuzzy-matching -- how many characters differ between checklist and data source name
+score                | heuristic score from 0 to 1 where 1 is a good match, 0.5 match requires further human investigation
 
-```ruby
-require "gn_crossmap"
+#### Types of Matches
 
-# If you want to change logger -- default Logging is to standard output
-GnCrossmap.logger = MyCustomLogger.new
+The output fomat returns 7 possible types of matches:
 
-GnCrossmap.run("path/to/input.csv", "path/to/output.csv", 5)
-```
+1. **Exact match** - The exact name was matched (but ignoring non-ascii characters)
+2. **Exact match by canonical form of a name** - The canonical form of the name (a version of a scientific name that contains complete versions of the latin words, but lacks insertions of subtaxa, annotations, or authority information) was matched
+3. **Fuzzy match by canonical form** - The canonical form gave a fuzzy (detecting lexical or spelling variations of a name using Tony Rees' algorithm TAXAMATCH) match
+4. **Partial exact match by species part of canonical form** - The canonical form returned a partial but exact match
+5. **Partial fuzzy match by species part of canonical form** - The canonical form returned a partial, fuzzy match
+6. **Exact match by genus part of a canonical form** - The genus part of the canonical form of the species name returned an exact match
+7. **[Blank]** - No match
 
 Development
 -----------
@@ -160,3 +191,5 @@ See [LICENSE][license] for details.
 [license]: https://github.com/GlobalNamesArchitecture/gn_crossmap/blob/master/LICENSE
 [terms]: http://rs.tdwg.org/dwc/terms
 [files]:  https://github.com/GlobalNamesArchitecture/gn_crossmap/tree/master/spec/files
+[output]: https://github.com/GlobalNamesArchitecture/gn_crossmap/tree/master/spec/files/output-example.csv
+[waffle]: https://waffle.io/GlobalNamesArchitecture/gn_crossmap

data/exe/crossmap

@@ -20,4 +20,8 @@ end
 Trollop.die :input, "must be set" if opts[:input].nil?
 Trollop.die :input, "file must exist" unless File.exist?(opts[:input])
 
-GnCrossmap.run(opts[:input], opts[:output], opts[:data_source_id])
+begin
+  GnCrossmap.run(opts[:input], opts[:output], opts[:data_source_id])
+rescue GnCrossmapError => e
+  GnCrossmap.logger.error(e.message)
+end

data/gn_crossmap.gemspec

@@ -27,6 +27,8 @@ Gem::Specification.new do |gem|
 
   gem.add_dependency "trollop", "~> 2.1"
   gem.add_dependency "biodiversity", "~> 3.1"
+  gem.add_dependency "rest-client", "~> 1.8"
+  gem.add_dependency "logger-colors", "~> 1.0"
 
   gem.add_development_dependency "bundler", "~> 1.7"
   gem.add_development_dependency "rake", "~> 10.0"

data/lib/gn_crossmap/collector.rb

@@ -17,14 +17,18 @@ module GnCrossmap
     private
 
     def init_fields_collector
-      @fields = @row.map { |f| f.downcase.to_sym }
+      @fields = @row.map { |f| f.to_s.strip.downcase.to_sym }
       @collector = collector_factory
+      err = "taxonID must be present in the csv header"
+      fail GnCrossmapError, err unless @fields.include?(:taxonid)
     end
 
     def collect_data
       @row = @fields.zip(@row).to_h
       data = @collector.id_name_rank(@row)
-      @data << data if data
+      return unless data
+      data[:original] = @row.values
+      @data << data
     end
 
     def collector_factory

data/lib/gn_crossmap/column_collector.rb

@@ -9,6 +9,9 @@ module GnCrossmap
 
     def initialize(fields)
       @fields = fields
+      err = "At least some of these fields must exist in " \
+        "the CSV header: '#{RANKS.join('\', \'')}'"
+      fail GnCrossmapError, err if (RANKS - @fields).size == RANKS.size
     end
 
     def id_name_rank(row)

data/lib/gn_crossmap/errors.rb

@@ -0,0 +1,2 @@
+# Error to raise in case of problems
+class GnCrossmapError < RuntimeError; end

data/lib/gn_crossmap/reader.rb

@@ -2,9 +2,12 @@ module GnCrossmap
   # Reads supplied csv file and creates ruby structure to compare
   # with a Global Names Resolver source
   class Reader
+    attr_reader :original_fields
+
     def initialize(csv_path)
       @csv_file = csv_path
       @col_sep = col_sep
+      @original_fields = nil
     end
 
     def read
@@ -21,7 +24,10 @@ module GnCrossmap
 
     def parse_input
       dc = Collector.new
-      CSV.open(@csv_file, col_sep: @col_sep).each do |row|
+      CSV.open(@csv_file, col_sep: @col_sep).each_with_index do |row, i|
+        @original_fields = row.dup if @original_fields.nil?
+        i += 1
+        GnCrossmap.log("Ingesting #{i}th csv row") if i % 10_000 == 0
         dc.process_row(row)
       end
       dc.data

data/lib/gn_crossmap/resolver.rb

@@ -7,6 +7,7 @@ module GnCrossmap
       @processor = GnCrossmap::ResultProcessor.new(writer)
       @ds_id = data_source_id
       @count = 0
+      @current_data = {}
       @batch = 200
     end
 
@@ -32,7 +33,9 @@ module GnCrossmap
     end
 
     def collect_names(slice)
+      @current_data = {}
       slice.each_with_object("") do |row, str|
+        @current_data[row[:id]] = row[:original]
         @processor.input[row[:id]] = { rank: row[:rank] }
         str << "#{row[:id]}|#{row[:name]}\n"
       end
@@ -40,7 +43,7 @@ module GnCrossmap
 
     def remote_resolve(names)
       res = RestClient.post(URL, data: names, data_source_ids: @ds_id)
-      @processor.process(res)
+      @processor.process(res, @current_data)
     rescue RestClient::Exception
       single_remote_resolve(names)
     end
@@ -51,7 +54,7 @@ module GnCrossmap
           res = RestClient.post(URL, data: name, data_source_ids: @ds_id)
           @processor.process(res)
         rescue RestClient::Exception => e
-          GnCrossmap.log("Resolver broke on '#{name}': #{e}")
+          GnCrossmap.logger.error("Resolver broke on '#{name}': #{e.message}")
           next
         end
       end

data/lib/gn_crossmap/result_processor.rb

@@ -17,7 +17,8 @@ module GnCrossmap
       @input = {}
     end
 
-    def process(result)
+    def process(result, original_data)
+      @original_data = original_data
       res = rubyfy(result)
       res[:data].each do |d|
         d[:results].nil? ? write_empty_result(d) : write_result(d)
@@ -31,22 +32,26 @@ module GnCrossmap
     end
 
     def write_empty_result(datum)
-      res = [datum[:supplied_id], datum[:supplied_name_string], nil, nil,
-             @input[datum[:supplied_id]][:rank], nil, nil, nil, nil]
+      res = @original_data[datum[:supplied_id]]
+      res += [datum[:supplied_name_string], nil, nil,
+              @input[datum[:supplied_id]][:rank], nil, nil, nil, nil]
       @writer.write(res)
     end
 
     def write_result(datum)
-      datum[:results].each do |r|
-        res = [datum[:supplied_id], datum[:supplied_name_string],
-               r[:name_string], r[:canonical_form],
-               @input[datum[:supplied_id]][:rank],
-               matched_rank(r), matched_type(r),
-               r[:edit_distance], r[:score]]
-        @writer.write(res)
+      datum[:results].each do |result|
+        @writer.write(compile_result(datum, result))
       end
     end
 
+    def compile_result(datum, result)
+      @original_data[datum[:supplied_id]] +
+        [datum[:supplied_name_string], result[:name_string],
+         result[:canonical_form], @input[datum[:supplied_id]][:rank],
+         matched_rank(result), matched_type(result),
+         result[:edit_distance], result[:score]]
+    end
+
     def matched_rank(record)
       record[:classification_path_ranks].split("|").last
     end

data/lib/gn_crossmap/sci_name_collector.rb

@@ -43,6 +43,8 @@ module GnCrossmap
       else
         normalize_rank(@parsed_name[:details][0][:infraspecies][-1][:rank])
       end
+    rescue NoMethodError
+      nil
     end
 
     def normalize_rank(rank)

data/lib/gn_crossmap/version.rb

@@ -1,6 +1,6 @@
 # Namespace module for crossmapping checklists to GN sources
 module GnCrossmap
-  VERSION = "0.1.4"
+  VERSION = "0.1.5"
 
   def self.version
     VERSION

data/lib/gn_crossmap/writer.rb

@@ -1,12 +1,11 @@
 module GnCrossmap
   # Saves output from GN Resolver to disk
   class Writer
-    def initialize(output_path)
+    def initialize(output_path, original_fields)
       @path = output_path
+      @output_fields = output_fields(original_fields)
       @output = CSV.open(@path, "w:utf-8")
-      @output << [:taxonID, :scientificName, :matchedScientificName,
-                  :matchedCanonicalForm, :rank, :matchedRank, :matchType,
-                  :editDistance, :score]
+      @output << @output_fields
       GnCrossmap.log("Open output file '#{@path}'")
     end
 
@@ -18,5 +17,13 @@ module GnCrossmap
       GnCrossmap.log("Close output file '#{@path}'")
       @output.close
     end
+
+    private
+
+    def output_fields(original_fields)
+      original_fields + [:inputName, :matchedName, :matchedCanonicalForm,
+                         :inputRank, :matchedRank, :matchedType,
+                         :matchedEditDistance, :marchedScore]
+    end
   end
 end

data/lib/gn_crossmap.rb

@@ -1,7 +1,9 @@
 require "csv"
 require "rest_client"
 require "logger"
+require "logger/colors"
 require "biodiversity"
+require "gn_crossmap/errors"
 require "gn_crossmap/version"
 require "gn_crossmap/reader"
 require "gn_crossmap/writer"
@@ -17,8 +19,9 @@ module GnCrossmap
     attr_writer :logger
 
     def run(input, output, data_source_id)
-      data = Reader.new(input).read
-      writer = Writer.new(output)
+      reader = Reader.new(input)
+      data = reader.read
+      writer = Writer.new(output, reader.original_fields)
       Resolver.new(writer, data_source_id).resolve(data)
       output
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gn_crossmap
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 0.1.5
 platform: ruby
 authors:
 - Dmitry Mozzherin
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2015-05-11 00:00:00.000000000 Z
+date: 2015-05-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: trollop
@@ -38,6 +38,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.1'
+- !ruby/object:Gem::Dependency
+  name: rest-client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+- !ruby/object:Gem::Dependency
+  name: logger-colors
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -148,6 +176,7 @@ files:
 - lib/gn_crossmap.rb
 - lib/gn_crossmap/collector.rb
 - lib/gn_crossmap/column_collector.rb
+- lib/gn_crossmap/errors.rb
 - lib/gn_crossmap/reader.rb
 - lib/gn_crossmap/resolver.rb
 - lib/gn_crossmap/result_processor.rb