biodiversity 4.0.2 → 5.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8e1a5a67bcea7c0e234726d7702c4224b109a7852899a2d3b84d25b38ee9ad84
-  data.tar.gz: 69e7781d05858faa63ed40b4d964499e0718bec3b6fecc9e5620879bd72acf34
+  metadata.gz: b9df90a14b8fcc999def9b89b58ea277e37749eca303f858ebaee03159bb60d8
+  data.tar.gz: b6f8ed59091c85d12fb660a608444f94c2974c4f7720e7b085b5c02265cad296
 SHA512:
-  metadata.gz: be191bc9156447da5883c7a910b83012422869bbe7fdb6950833e2090b1ad33fb521dd5196376fba821d2991857a5c0adcc06b80f5752b7434c428a449d662b1
-  data.tar.gz: 4bc7244b2d4c24146aa19f191c78e9645779298cfb67a18904aaddae3c0baa866035ecfed82827bc2e78cde54e8ee4fd9dccdedd76f8a43d222f06212a07ce35
+  metadata.gz: b2c749612aed3f60f3fc3041a4685a7f45651ba438d7058687433a8b039443487b3e2e9ff90cf9a976cab5b183bac513ae611d238ceed2353e3ad491f963e81c
+  data.tar.gz: 2765731a3d3a9dfd5bb346246ce8afe741bf9a0b74e9c50bced34d3f82da9ce5e1a56c6d671373b6c0973e4afc63f913bc7badf2f7073f059a2c38aeebfc817c

data/.rubocop.yml

@@ -1,4 +1,7 @@
 AllCops:
+  TargetRubyVersion: 2.6
+  NewCops: disable
+  SuggestExtensions: false
   Exclude:
     - .bundle/**/*
     - bundle_bin/**/*
@@ -10,4 +13,10 @@ Metrics/MethodLength:
     - lib/**/*
 Metrics/BlockLength:
   Exclude:
-    - spec/**/*
+    - spec/**/*
+Style/HashEachMethods:
+  Enabled: true
+Style/HashTransformKeys:
+  Enabled: true
+Style/HashTransformValues:
+  Enabled: true

data/.ruby-version

@@ -1 +1 @@
-2.5.7
+3.0.0

data/.travis.yml

@@ -1,6 +1,7 @@
 rvm:
-  - 2.5
   - 2.6
+  - 2.7
+  - 3.0
 before_install: "gem update bundler"
 branches:
   only:

data/CHANGELOG

@@ -1,3 +1,7 @@
+5.0.0 -- changed GNparser to v 1.0.0
+
+4.0.3 -- fix memory leak in parse_ary method
+
 4.0.2 -- add MS Windows libraries
 
 4.0.1 -- fix for simple output

data/LICENSE

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2008-2018 Dmitry Mozzherin
+Copyright (c) 2008-2020 Dmitry Mozzherin
 
 Permission is hereby granted, free of charge, to any person obtaining
 a copy of this software and associated documentation files (the

data/README.md

@@ -1,7 +1,7 @@
 Biodiversity
 ============
 
-[![DOI](https://zenodo.org/badge/19435/GlobalNamesArchitecture/biodiversity.svg)](https://zenodo.org/badge/latestdoi/19435/GlobalNamesArchitecture/biodiversity)
+[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.3569596.svg)](https://doi.org/10.5281/zenodo.3569596)
 [![Gem Version][gem_svg]][gem_link]
 [![Continuous Integration Status][ci_svg]][ci_link]
 
@@ -17,6 +17,7 @@ For such features use https://gitlab.com/gogna/gnparser.
 
 - [Biodiversity](#biodiversity)
   - [Installation](#installation)
+  - [Benchmarks](#benchmarks)
   - [Example usage](#example-usage)
     - [What is "nameStringID" in the parsed results?](#what-is-%22namestringid%22-in-the-parsed-results)
   - [Copyright](#copyright)
@@ -27,6 +28,38 @@ For such features use https://gitlab.com/gogna/gnparser.
 
 The gem should work on Linux, Mac and Windows (64bit) machines
 
+## Benchmarks
+
+The fastest way to go through a massive amount of names is to use
+`Biodiversity::Parser.parse_ary([big array], simple = true)` function.
+
+For example parsing a large file with one name per line:
+
+```ruby
+#!/usr/bin/env ruby
+
+require 'biodiversity'
+
+P = Biodiversity::Parser
+count = 0
+File.open('all_names.txt').each_slice(50_000) do |sl|
+  count += 1
+  res = P.parse_ary(sl, true)
+  puts count * 50_000
+  puts res[0]
+end
+```
+
+Here are comparative results of running parsers against a file with 24
+million names on a 4CPU hyperthreaded laptop:
+
+| Program      | Version | Full/Simple | Names/min |
+| ------------ | ------- | ----------- | --------: |
+| gnparser     | 0.12.0  | Simple      | 3,000,000 |
+| biodiversity | 4.0.1   | Simple      | 2,000,000 |
+| biodiversity | 4.0.1   | Full JSON   |   800,000 |
+| biodiversity | 3.5.1   | n/a         |    40,000 |
+
 ## Example usage
 
 You can use it as a library in Ruby:
@@ -92,7 +125,11 @@ Copyright
 
 Authors: [Dmitry Mozzherin][dimus]
 
-Copyright (c) 2008-2019 Dmitry Mozzherin. See [LICENSE][license]
+Contributors: [Patrick Leary][pleary], [Hernán Lucas Pereira][hernan]
+
+
+
+Copyright (c) 2008-2020 Dmitry Mozzherin. See [LICENSE][license]
 for further details.
 
 [gem_svg]: https://badge.fury.io/rb/biodiversity.svg
@@ -100,6 +137,8 @@ for further details.
 [ci_svg]: https://secure.travis-ci.org/GlobalNamesArchitecture/biodiversity.svg
 [ci_link]: http://travis-ci.org/GlobalNamesArchitecture/biodiversity
 [dimus]: https://github.com/dimus
+[pleary]: https://github.com/pleary
+[hernan]: https://github.com/LocoDelAssembly
 [license]: https://github.com/GlobalNamesArchitecture/biodiversity/blob/master/LICENSE
 [uuid_examples]: https://github.com/GlobalNamesArchitecture/gn_uuid_examples
 [uuid_blog]: http://globalnamesarchitecture.github.io/gna/uuid/2015/05/31/gn-uuid-0-5-0.html

data/biodiversity.gemspec

@@ -5,6 +5,7 @@ $LOAD_PATH.push File.expand_path('lib', __dir__)
 require 'biodiversity/version'
 
 Gem::Specification.new do |gem|
+  gem.required_ruby_version = '>= 2.6'
   gem.name          = 'biodiversity'
   gem.version       = Biodiversity::VERSION
   gem.homepage      = 'https://github.com/GlobalNamesArchitecture/biodiversity'
@@ -17,11 +18,11 @@ Gem::Specification.new do |gem|
   gem.files         = `git ls-files`.split("\n")
   gem.require_paths = ['lib']
 
-  gem.add_runtime_dependency 'ffi', '~> 1.11'
+  gem.add_runtime_dependency 'ffi', '~> 1.14'
 
-  gem.add_development_dependency 'bundler', '~> 2.0'
-  gem.add_development_dependency 'byebug', '~> 11.0'
+  gem.add_development_dependency 'bundler', '~> 2.2'
+  gem.add_development_dependency 'byebug', '~> 11.1'
   gem.add_development_dependency 'rake', '~> 13.0'
-  gem.add_development_dependency 'rspec', '~> 3.9'
-  gem.add_development_dependency 'rubocop', '~> 0.76'
+  gem.add_development_dependency 'rspec', '~> 3.10'
+  gem.add_development_dependency 'rubocop', '~> 1.8'
 end

data/clib/linux/libgnparser.h

@@ -1,6 +1,6 @@
 /* Code generated by cmd/cgo; DO NOT EDIT. */
 
-/* package gitlab.com/gogna/gnparser/binding */
+/* package github.com/gnames/gnparser/binding */
 
 
 #line 1 "cgo-builtin-export-prolog"
@@ -74,19 +74,23 @@ extern "C" {
 #endif
 
 
-// ParseToString function takes a name-string, desired format, and parses
-// the name-string to either JSON, or pipe-separated values, depending on
-// the desired format. Format can take values of 'simple', 'compact', 'pretty'.
+// ParseToString function takes a name-string, desired format, a withDetails
+// flag as 0|1 integer. It parses the name-string to either JSON, or a CSV
+// string, depending on the desired format. Format argument can take values of
+// 'csv', 'compact', 'pretty'. If withDetails argument is 0, additional
+// parsed details are ommited, if it is 1 -- they are included.
+// true.
+extern char* ParseToString(char* name, char* f, int details);
 
-extern char* ParseToString(char* p0, char* p1);
+// FreeMemory takes a string pointer and frees its memory.
+extern void FreeMemory(char* p);
 
-// ParseAryToStrings function takes an array of names, parsing format and a
-// reference to an output: an empty array of strings to return the the data
-// back. It populates the output array with raw strings of either JSON or
-// pipe-separated parsed values (depending on a given format). Format can take
-// values of 'simple', 'compact', or 'pretty'.
-
-extern void ParseAryToStrings(char** p0, int p1, char* p2, char*** p3);
+// ParseAryToString function takes an array of names, parsing format, and a
+// withDetails flag as 0|1 integer.  Parsed outputs are sent as a string in
+// either CSV or JSON format.  Format argument can take values of 'csv',
+// 'compact', or 'pretty'. For withDetails argument 0 means false, 1 means
+// true.
+extern char* ParseAryToString(char** in, int length, char* f, int details);
 
 #ifdef __cplusplus
 }

data/clib/mac/libgnparser.h

@@ -1,6 +1,6 @@
 /* Code generated by cmd/cgo; DO NOT EDIT. */
 
-/* package gitlab.com/gogna/gnparser/binding */
+/* package github.com/gnames/gnparser/binding */
 
 
 #line 1 "cgo-builtin-export-prolog"
@@ -74,19 +74,26 @@ extern "C" {
 #endif
 
 
-// ParseToString function takes a name-string, desired format, and parses
-// the name-string to either JSON, or pipe-separated values, depending on
-// the desired format. Format can take values of 'simple', 'compact', 'pretty'.
+// ParseToString function takes a name-string, desired format, a withDetails
+// flag as 0|1 integer. It parses the name-string to either JSON, or a CSV
+// string, depending on the desired format. Format argument can take values of
+// 'csv', 'compact', 'pretty'. If withDetails argument is 0, additional
+// parsed details are ommited, if it is 1 -- they are included.
+// true.
 
-extern char* ParseToString(char* p0, char* p1);
+extern char* ParseToString(char* p0, char* p1, int p2);
 
-// ParseAryToStrings function takes an array of names, parsing format and a
-// reference to an output: an empty array of strings to return the the data
-// back. It populates the output array with raw strings of either JSON or
-// pipe-separated parsed values (depending on a given format). Format can take
-// values of 'simple', 'compact', or 'pretty'.
+// FreeMemory takes a string pointer and frees its memory.
 
-extern void ParseAryToStrings(char** p0, int p1, char* p2, char*** p3);
+extern void FreeMemory(char* p0);
+
+// ParseAryToString function takes an array of names, parsing format, and a
+// withDetails flag as 0|1 integer.  Parsed outputs are sent as a string in
+// either CSV or JSON format.  Format argument can take values of 'csv',
+// 'compact', or 'pretty'. For withDetails argument 0 means false, 1 means
+// true.
+
+extern char* ParseAryToString(char** p0, int p1, char* p2, int p3);
 
 #ifdef __cplusplus
 }

data/clib/win/libgnparser.h

@@ -1,6 +1,6 @@
 /* Code generated by cmd/cgo; DO NOT EDIT. */
 
-/* package gitlab.com/gogna/gnparser/binding */
+/* package github.com/gnames/gnparser/binding */
 
 
 #line 1 "cgo-builtin-export-prolog"
@@ -74,19 +74,23 @@ extern "C" {
 #endif
 
 
-// ParseToString function takes a name-string, desired format, and parses
-// the name-string to either JSON, or pipe-separated values, depending on
-// the desired format. Format can take values of 'simple', 'compact', 'pretty'.
+// ParseToString function takes a name-string, desired format, a withDetails
+// flag as 0|1 integer. It parses the name-string to either JSON, or a CSV
+// string, depending on the desired format. Format argument can take values of
+// 'csv', 'compact', 'pretty'. If withDetails argument is 0, additional
+// parsed details are ommited, if it is 1 -- they are included.
+// true.
+extern char* ParseToString(char* name, char* f, int details);
 
-extern char* ParseToString(char* p0, char* p1);
+// FreeMemory takes a string pointer and frees its memory.
+extern void FreeMemory(char* p);
 
-// ParseAryToStrings function takes an array of names, parsing format and a
-// reference to an output: an empty array of strings to return the the data
-// back. It populates the output array with raw strings of either JSON or
-// pipe-separated parsed values (depending on a given format). Format can take
-// values of 'simple', 'compact', or 'pretty'.
-
-extern void ParseAryToStrings(char** p0, int p1, char* p2, char*** p3);
+// ParseAryToString function takes an array of names, parsing format, and a
+// withDetails flag as 0|1 integer.  Parsed outputs are sent as a string in
+// either CSV or JSON format.  Format argument can take values of 'csv',
+// 'compact', or 'pretty'. For withDetails argument 0 means false, 1 means
+// true.
+extern char* ParseAryToString(char** in, int length, char* f, int details);
 
 #ifdef __cplusplus
 }

data/lib/biodiversity.rb

@@ -2,6 +2,7 @@
 
 require 'ffi'
 require 'json'
+require 'csv'
 require_relative 'biodiversity/version'
 require_relative 'biodiversity/parser'
 

data/lib/biodiversity/parser.rb

@@ -25,55 +25,67 @@ module Biodiversity
     ffi_lib File.join(__dir__, '..', '..', 'clib', platform, 'libgnparser.so')
     POINTER_SIZE = FFI.type_size(:pointer)
 
-    attach_function(:parse_go, :ParseToString, %i[string string], :string)
-    attach_function(:parse_ary_go, :ParseAryToStrings,
-                    %i[pointer int string pointer], :void)
+    callback(:parser_callback, %i[string], :void)
 
-    def self.parse(name, simple = false)
-      format = simple ? 'simple' : 'compact'
-      parsed = parse_go(name, format)
+    attach_function(:parse_go, :ParseToString,
+                    %i[string string int], :strptr)
+    attach_function(:parse_ary_go, :ParseAryToString,
+                    %i[pointer int string int], :strptr)
+    attach_function(:free_mem, :FreeMemory, %i[pointer], :void)
+
+    def self.parse(name, simple: false, details: false)
+      format = simple ? 'csv' : 'compact'
+      with_details = details ? 1 : 0
+
+      parsed, ptr = parse_go(name, format, with_details)
+      free_mem(ptr)
       output(parsed, simple)
     end
 
-    def self.parse_ary(ary, simple = false)
-      format = simple ? 'simple' : 'compact'
+    def self.parse_ary(ary, simple: false, details: false)
+      format = simple ? 'csv' : 'compact'
+      with_details = details ? 1 : 0
+
       in_ptr = FFI::MemoryPointer.new(:pointer, ary.length)
       in_ptr.write_array_of_pointer(
         ary.map { |s| FFI::MemoryPointer.from_string(s) }
       )
-      out_var = FFI::MemoryPointer.new(:pointer)
-      parse_ary_go(in_ptr, ary.length, format, out_var)
 
-      out_var.read_pointer
-             .get_array_of_string(0, ary.length)
-             .each_with_object([]) do |prsd, a|
-        a << output(prsd, simple)
-      end
-    ensure
-      out_var.read_pointer.get_array_of_pointer(0, ary.length).each do |p|
-        CLib.free(p)
+      parsed, ptr = parse_ary_go(in_ptr, ary.length, format, with_details)
+      free_mem(ptr)
+      if simple
+        CSV.new(parsed).map do |row|
+          csv_row(row)
+        end
+      else
+        JSON.parse(parsed, symbolize_names: true)
       end
-      CLib.free(out_var.read_pointer)
     end
 
     def self.output(parsed, simple)
       if simple
-        parsed = parsed.split('|')
-        {
-          id: parsed[0],
-          verbatim: parsed[1],
-          canonicalName: {
-            full: parsed[2],
-            simple: parsed[3],
-            stem: parsed[4]
-          },
-          authorship: parsed[5],
-          year: parsed[6],
-          quality: parsed[7]
-        }
+        csv = CSV.new(parsed)
+        row = csv.readlines[0]
+        csv_row(row)
       else
         JSON.parse(parsed, symbolize_names: true)
       end
     end
+
+    def self.csv_row(row)
+      {
+        id: row[0],
+        verbatim: row[1],
+        cardinality: row[2],
+        canonical: {
+          stem: row[3],
+          simple: row[4],
+          full: row[5]
+        },
+        authorship: row[6],
+        year: row[7],
+        quality: row[8]
+      }
+    end
   end
 end

data/lib/biodiversity/version.rb

@@ -2,7 +2,7 @@
 
 # Biodiversity module provides a namespace for scientific name parser.
 module Biodiversity
-  VERSION = '4.0.2'
+  VERSION = '5.1.0'
 
   def self.version
     VERSION

data/spec/lib/parser_spec.rb

@@ -5,20 +5,20 @@
 describe Biodiversity::Parser do
   describe('parse') do
     it 'parses name in simple format' do
-      parsed = subject.parse('Homo sapiens Linn.', true)
-      expect(parsed[:canonicalName][:simple]).to eq 'Homo sapiens'
+      parsed = subject.parse('Homo sapiens Linn.', simple: true)
+      expect(parsed[:canonical][:simple]).to eq 'Homo sapiens'
       expect(parsed[:normalized]).to be_nil
     end
 
     it 'parsed name in full format' do
       parsed = subject.parse('Homo sapiens  Linn.')
-      expect(parsed[:canonicalName][:simple]).to eq 'Homo sapiens'
+      expect(parsed[:canonical][:simple]).to eq 'Homo sapiens'
       expect(parsed[:normalized]).to eq 'Homo sapiens Linn.'
     end
 
     it 'gets quality and year correctly in simple form' do
-      parsed = subject.parse('Homo sapiens Linn. 1758', true)
-      expect(parsed[:canonicalName][:simple]).to eq 'Homo sapiens'
+      parsed = subject.parse('Homo sapiens Linn. 1758', simple: true)
+      expect(parsed[:canonical][:simple]).to eq 'Homo sapiens'
       expect(parsed[:year]).to eq '1758'
       expect(parsed[:quality]).to eq '1'
       expect(parsed[:normalized]).to be_nil
@@ -27,9 +27,12 @@ describe Biodiversity::Parser do
 
   describe('parse_ary') do
     it 'parses names in simple format' do
-      parsed = subject.parse_ary(['Homo sapiens Linn.', 'Pardosa moesta'], true)
-      expect(parsed[0][:canonicalName][:simple]).to eq 'Homo sapiens'
-      expect(parsed[1][:canonicalName][:simple]).to eq 'Pardosa moesta'
+      parsed = subject.parse_ary(
+        ['Homo sapiens Linn.', 'Pardosa moesta'],
+        simple: true
+      )
+      expect(parsed[0][:canonical][:simple]).to eq 'Homo sapiens'
+      expect(parsed[1][:canonical][:simple]).to eq 'Pardosa moesta'
       expect(parsed[0][:normalized]).to be_nil
     end
 
@@ -37,7 +40,7 @@ describe Biodiversity::Parser do
       parsed = subject.parse_ary(
         ['Homo sapiens  Linn.', 'Tobacco Mosaic Virus']
       )
-      expect(parsed[0][:canonicalName][:simple]).to eq 'Homo sapiens'
+      expect(parsed[0][:canonical][:simple]).to eq 'Homo sapiens'
       expect(parsed[0][:normalized]).to eq 'Homo sapiens Linn.'
       expect(parsed[1][:parsed]).to be false
       expect(parsed[1][:virus]).to be true

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: biodiversity
 version: !ruby/object:Gem::Version
-  version: 4.0.2
+  version: 5.1.0
 platform: ruby
 authors:
 - Dmitry Mozzherin
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-11-18 00:00:00.000000000 Z
+date: 2021-01-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ffi
@@ -16,42 +16,42 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.11'
+        version: '1.14'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.11'
+        version: '1.14'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: '2.2'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: '2.2'
 - !ruby/object:Gem::Dependency
   name: byebug
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '11.0'
+        version: '11.1'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '11.0'
+        version: '11.1'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -72,28 +72,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.9'
+        version: '3.10'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.9'
+        version: '3.10'
 - !ruby/object:Gem::Dependency
   name: rubocop
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.76'
+        version: '1.8'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.76'
+        version: '1.8'
 description: Parsing tool for biodiversity informatics
 email: dmozzherin@gmail.com
 executables: []
@@ -127,7 +127,7 @@ homepage: https://github.com/GlobalNamesArchitecture/biodiversity
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -135,16 +135,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: '2.6'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6.2
-signing_key: 
+rubygems_version: 3.2.3
+signing_key:
 specification_version: 4
 summary: Parser of scientific names
 test_files: []