gn_list_resolver 3.0.3.4 → 3.2.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6a630b9dd4ea1cc3bebb02baec56e9277a52399f
-  data.tar.gz: 35ac3f5944bb050604bb2c9f584ee9f47b3e9102
+  metadata.gz: 3b30d2905481948448c40c18922587551a228819
+  data.tar.gz: 3ff55484ce12de44275814909941c3ba9b63ce03
 SHA512:
-  metadata.gz: 383620a1423b7bc056ff08c75f28fefef9937f50bbe4322037f9f89685d6e7fb1bf5070c7e890d77c7b87aefa0a46384d84eba5abc36d01905778f4f3461c2c7
-  data.tar.gz: a734e466173e3e03ac960e700a851cb755af4cbf5dcb46a8a0e36c314adafe865ae5aac089937375bb62f0c1ff0d38e65e4ce70b55253dccbf472867e7d70f75
+  metadata.gz: a82ffc469b139df0cd3ca8444a74a9fc73a16c8334153e9cd48d28882c9509a2ffdb837a46f9d76a564df282d10760e2be937802d10655ef6c892c7d0431aa11
+  data.tar.gz: 78e74f0263368c2d339151162583e35c1fd3f9facdbfba4cb73cb5333de994a375951ca3d0a03d766d07a1feb9c1f39ae74f5d80dd9a369d060f067fb822a961

data/.gitignore

@@ -12,6 +12,7 @@ output.csv
 .vim.custom
 t
 t.*
+.~lock*
 gn_list_resolver-*.gem
 .byebug_history
 .nvimlog

data/.travis.yml

@@ -1,7 +1,6 @@
 language: ruby
 rvm:
-  - 2.1
-  - 2.2
+  - 2.4
 script:
   - bundle exec rake
 branches:

data/CHANGELOG.md

@@ -1,5 +1,42 @@
 # ``gn_list_resolver`` CHANGELOG
 
+## 3.2.0.1
+
+@dimus, @alexander-myltsev - sync with `gn_crossmap`'s 3.2.0, current name is
+                             still a bit broken
+
+## 3.2.0
+
+* @dimus - Add a column `matchSize` with number of matches for each name
+
+## 3.1.5
+
+* @dimus - Fix number of columns in the output when there is no taxon_id given
+           and skip_original flag is set in options
+
+## 3.1.4
+
+* @dimus - test typo
+
+## 3.1.3
+
+* @dimus - Fix formatting bug for empty results, remove BOM char from headers
+
+## 3.1.2
+
+* @dimus - Fixes #37 tab is now default if separator is not found
+
+## 3.1.1
+
+* @dimus - Fixes #36 move edit distance close to canonical forms in output
+
+## 3.1.0
+
+* @dimus - Fixes #34 add canonical form input
+
+* @dimus - Fixes #35 optionally returns classification path
+
+
 ## 3.0.3.4
 
 @alexander-myltsev - make compatible with API changes

data/README.md

@@ -59,6 +59,9 @@ cat my_list.csv | crossmap -i - -o - > output
 # to keep only taxonID (if given) from original input
 # no original fields will be kept without taxonID
 cat my_list.csv | crossmap -i my_list.csv -s
+
+# to show classification from the source
+cat my_list.csv | crossmap -i my_list.csv -w
 ```
 
 ### Usage as Ruby Library (API description)
@@ -102,6 +105,9 @@ instead of the headers supplied with the file
 
 #### `GnListResolver.logger=`
 
+``with_classification``
+: (boolean) if true, adds classification path to the output
+
 Allows to set logger to a custom logger (default is `STDERR`)
 
 #### Usage Example
@@ -248,15 +254,17 @@ More examples can be found in [spec/files][files] directory
 
 Field                | Description
 ---------------------|-----------------------------------------------------------
-taxonID              | original ID attached to a name in the checklist
-scientificName       | name from the checklist
-matchedScientificName| name matched from the GN Reolver data source
+classification       | classification path of the data source (if available)
+editDistance         | for fuzzy-matching -- how many characters differ between checklist and data source name
+matchSize            | number of returned matches for a name
+matchType            | what kind of match it is
 matchedCanonicalForm | canonical form of the matched name
-rank                 | rank from the source (if it was given/inferred)
 matchedRank          | corresponding rank from the data source
-matchType            | what kind of match it is
-editDistance         | for fuzzy-matching -- how many characters differ between checklist and data source name
+matchedScientificName| name matched from the GN Reolver data source
+rank                 | rank from the source (if it was given/inferred)
+scientificName       | name from the checklist
 score                | heuristic score from 0 to 1 where 1 is a good match, 0.5 match requires further human investigation
+taxonID              | original ID attached to a name in the checklist
 
 #### Types of Matches
 

data/Rakefile

@@ -11,3 +11,8 @@ end
 RuboCop::RakeTask.new
 
 task default: %i[rubocop rspec]
+
+desc "open an irb session preloaded with this gem"
+task :console do
+  sh "irb -r pp -r ./lib/gn_crossmap.rb"
+end

data/exe/list_resolver

@@ -4,7 +4,7 @@
 require "trollop"
 require "gn_list_resolver"
 
-puts "This program requires Ruby >= v. 2.1.0" if RUBY_VERSION < "2.1.0"
+puts "This program requires Ruby >= v. 2.4.1" if RUBY_VERSION < "2.4.1"
 
 CATALOGUE_OF_LIFE = 1
 OUTPUT = "output.csv"
@@ -20,6 +20,8 @@ opts = Trollop.options do
       default: CATALOGUE_OF_LIFE)
   opt(:skip_original, "If given, only 'taxonID' is shown " \
       "from the original input", type: :boolean)
+  opt(:with_classification, "If given, returns classification path of " \
+                            "matched names", type: :boolean)
   opt(:debug, "If given, prints statistics of name resolution to STDOUT",
       short: "D", type: :boolean)
 end

data/gn_list_resolver.gemspec

@@ -31,21 +31,19 @@ Gem::Specification.new do |gem|
   gem.executables   = gem.files.grep(%r{^exe/}) { |f| File.basename(f) }
   gem.require_paths = ["lib"]
 
-  gem.add_dependency "trollop", "~> 2.1"
   gem.add_dependency "biodiversity", "~> 3.1"
-  gem.add_dependency "rest-client", "~> 2.0"
-  gem.add_dependency "logger-colors", "~> 1.0"
-  gem.add_dependency "graphql-client", "~> 0.11.3"
   gem.add_dependency "gn_uuid", "~> 0.5"
+  gem.add_dependency "graphql-client", "~> 0.11.3"
+  gem.add_dependency "logger-colors", "~> 1.0"
+  gem.add_dependency "rest-client", "~> 2.0"
+  gem.add_dependency "trollop", "~> 2.1"
 
   gem.add_development_dependency "bundler", "~> 1.7"
-  gem.add_development_dependency "rake", "~> 11.0"
+  gem.add_development_dependency "byebug", "~> 9.0"
+  gem.add_development_dependency "coveralls", "~> 0.8"
+  gem.add_development_dependency "rake", "~> 12.0"
   gem.add_development_dependency "rspec", "~> 3.2"
   gem.add_development_dependency "rubocop", "~> 0.49"
-  gem.add_development_dependency "coveralls", "~> 0.8"
-  gem.add_development_dependency "byebug", "~> 9.0"
-  gem.add_development_dependency "guard", "~> 2.14"
-  gem.add_development_dependency "guard-rspec", "~> 4.7"
 end
 
 # rubocop:enable Metrics/BlockLength:

data/lib/gn_list_resolver.rb

@@ -54,17 +54,23 @@ module GnListResolver
     end
 
     def find_id(row, name)
-      row.key?(:taxonid) ? row[:taxonid].strip : GnUUID.uuid(name)
+      if row.key?(:taxonid) && row[:taxonid]
+        row[:taxonid].to_s.strip
+      else
+        GnUUID.uuid(name.to_s)
+      end
     end
 
     private
 
     def create_resolver(writer, opts)
-      Resolver.new(writer, opts.data_source_id, opts.stats)
+      Resolver.new(writer, opts.data_source_id,
+                   opts.stats, opts.with_classification)
     end
 
     def create_writer(reader, output_io, opts)
-      Writer.new(output_io, reader.original_fields, output_name(opts.output))
+      Writer.new(output_io, reader.original_fields,
+                 output_name(opts.output), opts.with_classification)
     end
 
     def create_reader(input_io, opts)

data/lib/gn_list_resolver/collector.rb

@@ -26,7 +26,7 @@ module GnListResolver
     end
 
     def prepare_field(field)
-      field = field.to_s.tr(":", "/")
+      field = field.to_s.tr(":", "/").delete("\u{feff}")
       return :none if field == ""
       field.split("/")[-1].strip.downcase.to_sym
     end
@@ -41,7 +41,7 @@ module GnListResolver
     end
 
     def prepare_original
-      @skip_original ? [@row_hash[:taxonid]] : @row.map(&:last)
+      @skip_original ? [@row_hash[:taxonid]].compact : @row.map(&:last)
     end
 
     def collector_factory

data/lib/gn_list_resolver/graphql.rb

@@ -17,9 +17,10 @@ module GnListResolver
               results {
                 name { value }
                 canonicalName { value }
+                acceptedName { name { value } }
                 synonym
                 matchType { kind score editDistance }
-                taxonId classification { pathRanks }
+                taxonId classification { path pathRanks }
                 score { value parsingQuality }
               }
             }

data/lib/gn_list_resolver/reader.rb

@@ -4,13 +4,13 @@ module GnListResolver
   # Reads supplied csv file and creates ruby structure to compare
   # with a Global Names Resolver source
   class Reader
-    attr_reader :original_fields
+    attr_reader :original_fields, :col_sep
 
     def initialize(csv_io, input_name, skip_original, alt_headers, stats)
       @stats = stats
       @alt_headers = alt_headers
       @csv_io = csv_io
-      @col_sep = col_sep
+      @col_sep = detect_col_sep
       @quote_char = quote_char(@col_sep)
       @original_fields = nil
       @input_name = input_name
@@ -26,11 +26,14 @@ module GnListResolver
 
     private
 
-    def col_sep
+    def detect_col_sep
       line = @csv_io.first
       @stats.stats[:total_records] = @csv_io.readlines.size
       @csv_io.rewind
-      [";", ",", "\t"].map { |s| [line.count(s), s] }.sort.last.last
+      data = [[";", 1], [",", 0], ["\t", 2]].map do |s, weight|
+        [line.count(s), weight, s]
+      end
+      data.sort.last.last
     end
 
     def quote_char(col_sep)

data/lib/gn_list_resolver/resolver.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-# GnListResolver::test
 module GnListResolver
   # Sends data to GN Resolver and collects results
   class Resolver
@@ -8,9 +7,10 @@ module GnListResolver
     QUERY = GRAPHQL.client.parse(GRAPHQL.query)
     attr_reader :stats
 
-    def initialize(writer, data_source_id, stats)
+    def initialize(writer, data_source_id, stats, with_classification = false)
       @stats = stats
-      @processor = GnListResolver::ResultProcessor.new(writer, @stats)
+      @processor = GnListResolver::ResultProcessor.new(writer, @stats,
+                                                       with_classification)
       @ds_id = data_source_id
       @count = 0
       @current_data = {}
@@ -21,12 +21,7 @@ module GnListResolver
       update_stats(data.size)
       block_given? ? process(data, &Proc.new) : process(data)
       wrap_up
-
-      if block_given?
-        yield(@stats.stats)
-      else
-        @stats.stats
-      end
+      block_given? ? yield(@stats.stats) : @stats.stats
     end
 
     private
@@ -67,11 +62,12 @@ module GnListResolver
 
     def collect_names(slice)
       @current_data = {}
-      slice.each do |row|
+      slice.each_with_object([]) do |row, str|
         id = row[:id].strip
         @current_data[id] = row[:original]
         @processor.input[id] = { rank: row[:rank] }
-      end
+        str << "#{id}|#{row[:name]}"
+      end.join("\n")
     end
 
     def variables(names)

data/lib/gn_list_resolver/result_processor.rb

@@ -5,7 +5,9 @@ module GnListResolver
   class ResultProcessor
     attr_reader :input, :writer
 
-    def initialize(writer, stats)
+    def initialize(writer, stats, with_classification = false)
+      @with_classification = with_classification
+      @parser = ScientificNameParser.new
       @stats = stats
       @writer = writer
       @input = {}
@@ -20,26 +22,28 @@ module GnListResolver
 
     private
 
-    def rubyfy(result)
-      JSON.parse(result, symbolize_names: true)
-    end
-
     def write_empty_result(datum)
       @stats.stats[:matches][MATCH_TYPE_EMPTY] += 1
       @stats.stats[:resolved_records] += 1
+      res = compile_empty_result(datum)
+      @writer.write(res)
+    end
+
+    def compile_empty_result(datum)
       res = @original_data[datum.supplied_id]
-      res += [MATCH_TYPES[MATCH_TYPE_EMPTY], datum.supplied_input, nil,
-              nil,
+      res += [MATCH_TYPES[MATCH_TYPE_EMPTY], 0, datum.supplied_input,
+              nil, nil, nil, nil,
               @input[datum.supplied_id][:rank],
-              nil,
-              nil, nil, nil]
-      @writer.write(res)
+              nil, nil, nil, nil, nil]
+      res <<  nil if @with_classification
+      res
     end
 
     def write_result(datum)
       collect_stats(datum)
+      match_size = datum.results.size
       datum.results.each do |result|
-        @writer.write(compile_result(datum, result))
+        @writer.write(compile_result(datum, result, match_size))
       end
     end
 
@@ -54,28 +58,56 @@ module GnListResolver
       @stats.stats[:resolved_records] += 1
     end
 
-    def compile_result(datum, result)
-      @original_data[datum.supplied_id] + prepare_data(datum, result)
+    def compile_result(datum, result, match_size)
+      @original_data[datum.supplied_id] + prepare_data(datum,
+                                                       result, match_size)
     end
 
     # rubocop:disable Metrics/AbcSize
 
-    def prepare_data(datum, result)
-      [MATCH_TYPES[result.match_type.kind.to_sym],
-       datum.supplied_input, result.name.value,
-       result.canonical_name.value, @input[datum.supplied_id][:rank],
-       matched_rank(result),
-       result.synonym,
-       result.name.value, # TODO: should be `current_name_string` field
-       result.match_type.edit_distance,
-       result.score.value ? result.score.value.round(3) : nil,
-       result.taxon_id]
+    def prepare_data(datum, result, match_size)
+      res = [MATCH_TYPES[result.match_type.kind.to_sym], match_size,
+             datum.supplied_input, result.name.value,
+             canonical(datum.supplied_input), result.canonical_name.value,
+             result.match_type.edit_distance, @input[datum.supplied_id][:rank],
+             matched_rank(result), result.synonym, current_name(result),
+             result.score.value ? result.score.value.round(3) : nil,
+             result.taxon_id]
+      res << classification(result) if @with_classification
+      res
     end
 
     # rubocop:enable all
 
+    def current_name(result)
+      result.accepted_name.name.value || result.name.value
+    end
+
+    def canonical(name_string)
+      parsed = @parser.parse(name_string)[:scientificName]
+      return nil if parsed[:canonical].nil? || parsed[:hybrid]
+      parsed[:canonical]
+    rescue StandardError
+      @parser = ScientificNameParser.new
+      nil
+    end
+
     def matched_rank(result)
       result.classification.path_ranks.split("|").last
     end
+
+    # rubocop:disable Metrics/AbcSize
+
+    def classification(result)
+      return nil if result.classification.path.to_s.strip == ""
+      path = result.classification.path.split("|")
+      ranks = result.classification.path_ranks.split("|")
+      if path.size == ranks.size
+        path = path.zip(ranks).map { |e| "#{e[0]}(#{e[1]})" }
+      end
+      path.join(", ")
+    end
+
+    # rubocop:enable all
   end
 end

data/lib/gn_list_resolver/version.rb

@@ -2,7 +2,7 @@
 
 # Namespace module for crossmapping checklists to GN sources
 module GnListResolver
-  VERSION = "3.0.3.4"
+  VERSION = "3.2.0.1"
 
   def self.version
     VERSION

data/lib/gn_list_resolver/writer.rb

@@ -3,9 +3,11 @@
 module GnListResolver
   # Saves output from GN Resolver to disk
   class Writer
-    def initialize(output_io, original_fields, output_name)
+    def initialize(output_io, original_fields, output_name,
+                   with_classification = false)
       @output_io = output_io
       @output_fields = output_fields(original_fields)
+      @output_fields << :classification if with_classification
       @output = CSV.new(@output_io, col_sep: "\t")
       @output << @output_fields
       @output_name = output_name
@@ -24,9 +26,10 @@ module GnListResolver
     private
 
     def output_fields(original_fields)
-      original_fields + %i[matchedType inputName matchedName
-                           matchedCanonicalForm inputRank matchedRank
-                           synonymStatus acceptedName matchedEditDistance
+      original_fields + %i[matchedType matchSize inputName matchedName
+                           inputCanonicalForm matchedCanonicalForm
+                           matchedEditDistance inputRank
+                           matchedRank synonymStatus acceptedName
                            matchedScore matchTaxonID]
     end
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gn_list_resolver
 version: !ruby/object:Gem::Version
-  version: 3.0.3.4
+  version: 3.2.0.1
 platform: ruby
 authors:
 - Dmitry Mozzherin
@@ -9,50 +9,50 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-08-25 00:00:00.000000000 Z
+date: 2017-08-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: trollop
+  name: biodiversity
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.1'
+        version: '3.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.1'
+        version: '3.1'
 - !ruby/object:Gem::Dependency
-  name: biodiversity
+  name: gn_uuid
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.1'
+        version: '0.5'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.1'
+        version: '0.5'
 - !ruby/object:Gem::Dependency
-  name: rest-client
+  name: graphql-client
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: 0.11.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: 0.11.3
 - !ruby/object:Gem::Dependency
   name: logger-colors
   requirement: !ruby/object:Gem::Requirement
@@ -68,33 +68,33 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.0'
 - !ruby/object:Gem::Dependency
-  name: graphql-client
+  name: rest-client
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.11.3
+        version: '2.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.11.3
+        version: '2.0'
 - !ruby/object:Gem::Dependency
-  name: gn_uuid
+  name: trollop
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.5'
+        version: '2.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.5'
+        version: '2.1'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -110,47 +110,19 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.7'
 - !ruby/object:Gem::Dependency
-  name: rake
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '11.0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '11.0'
-- !ruby/object:Gem::Dependency
-  name: rspec
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.2'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.2'
-- !ruby/object:Gem::Dependency
-  name: rubocop
+  name: byebug
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.49'
+        version: '9.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.49'
+        version: '9.0'
 - !ruby/object:Gem::Dependency
   name: coveralls
   requirement: !ruby/object:Gem::Requirement
@@ -166,47 +138,47 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0.8'
 - !ruby/object:Gem::Dependency
-  name: byebug
+  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '9.0'
+        version: '12.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '9.0'
+        version: '12.0'
 - !ruby/object:Gem::Dependency
-  name: guard
+  name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.14'
+        version: '3.2'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.14'
+        version: '3.2'
 - !ruby/object:Gem::Dependency
-  name: guard-rspec
+  name: rubocop
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.7'
+        version: '0.49'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.7'
+        version: '0.49'
 description: Gem uses a checklist in a comma-separated format as an input, and returns
   back a new comma-separated resolved list of scientific names to one of the data
   sources from http://resolver.globalnames.org