biodiversity 4.0.2 → 4.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8e1a5a67bcea7c0e234726d7702c4224b109a7852899a2d3b84d25b38ee9ad84
-  data.tar.gz: 69e7781d05858faa63ed40b4d964499e0718bec3b6fecc9e5620879bd72acf34
+  metadata.gz: 8d1bb6dd3170d4a3d036cf5068a6d663606ec85f29003b24408e252e782ff4fd
+  data.tar.gz: 2ee9a9e168b107ecd851302c32c5b06cbafe25df83bae385fe00d620fbfd8cde
 SHA512:
-  metadata.gz: be191bc9156447da5883c7a910b83012422869bbe7fdb6950833e2090b1ad33fb521dd5196376fba821d2991857a5c0adcc06b80f5752b7434c428a449d662b1
-  data.tar.gz: 4bc7244b2d4c24146aa19f191c78e9645779298cfb67a18904aaddae3c0baa866035ecfed82827bc2e78cde54e8ee4fd9dccdedd76f8a43d222f06212a07ce35
+  metadata.gz: f2643c3916407220055a52581d357e09519a94be555e4703a9ecf4a8709f99f35a8be9232b3612913baa5a30e42812fcbf66d223f7824def39eee0bdef7e7da8
+  data.tar.gz: faaebcafb79ccc3273a28ace4c00f642e43b7813dd7477c4ec19c1ddf4252c8c871f7185746e40a9c94e37b04fc422cfd9c53aea53f97b532c74324b2fc66387

data/.rubocop.yml

@@ -10,4 +10,10 @@ Metrics/MethodLength:
     - lib/**/*
 Metrics/BlockLength:
   Exclude:
-    - spec/**/*
+    - spec/**/*
+Style/HashEachMethods:
+  Enabled: true
+Style/HashTransformKeys:
+  Enabled: true
+Style/HashTransformValues:
+  Enabled: true

data/.ruby-version

@@ -1 +1 @@
-2.5.7
+2.6.5

data/CHANGELOG

@@ -1,3 +1,5 @@
+4.0.3 -- fix memory leak in parse_ary method
+
 4.0.2 -- add MS Windows libraries
 
 4.0.1 -- fix for simple output

data/README.md

@@ -1,7 +1,7 @@
 Biodiversity
 ============
 
-[![DOI](https://zenodo.org/badge/19435/GlobalNamesArchitecture/biodiversity.svg)](https://zenodo.org/badge/latestdoi/19435/GlobalNamesArchitecture/biodiversity)
+[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.3569596.svg)](https://doi.org/10.5281/zenodo.3569596)
 [![Gem Version][gem_svg]][gem_link]
 [![Continuous Integration Status][ci_svg]][ci_link]
 
@@ -17,6 +17,7 @@ For such features use https://gitlab.com/gogna/gnparser.
 
 - [Biodiversity](#biodiversity)
   - [Installation](#installation)
+  - [Benchmarks](#benchmarks)
   - [Example usage](#example-usage)
     - [What is "nameStringID" in the parsed results?](#what-is-%22namestringid%22-in-the-parsed-results)
   - [Copyright](#copyright)
@@ -27,6 +28,38 @@ For such features use https://gitlab.com/gogna/gnparser.
 
 The gem should work on Linux, Mac and Windows (64bit) machines
 
+## Benchmarks
+
+The fastest way to go through a massive amount of names is to use
+`Biodiversity::Parser.parse_ary([big array], simple = true)` function.
+
+For example parsing a large file with one name per line:
+
+```ruby
+#!/usr/bin/env ruby
+
+require 'biodiversity'
+
+P = Biodiversity::Parser
+count = 0
+File.open('all_names.txt').each_slice(50_000) do |sl|
+  count += 1
+  res = P.parse_ary(sl, true)
+  puts count * 50_000
+  puts res[0]
+end
+```
+
+Here are comparative results of running parsers against a file with 24
+million names on a 4CPU hyperthreaded laptop:
+
+| Program      | Version | Full/Simple | Names/min |
+| ------------ | ------- | ----------- | --------: |
+| gnparser     | 0.12.0  | Simple      | 3,000,000 |
+| biodiversity | 4.0.1   | Simple      | 2,000,000 |
+| biodiversity | 4.0.1   | Full JSON   |   800,000 |
+| biodiversity | 3.5.1   | n/a         |    40,000 |
+
 ## Example usage
 
 You can use it as a library in Ruby:
@@ -92,6 +125,10 @@ Copyright
 
 Authors: [Dmitry Mozzherin][dimus]
 
+Contributors: [Patrick Leary][pleary], [Hernán Lucas Pereira][hernan]
+
+
+
 Copyright (c) 2008-2019 Dmitry Mozzherin. See [LICENSE][license]
 for further details.
 
@@ -100,6 +137,8 @@ for further details.
 [ci_svg]: https://secure.travis-ci.org/GlobalNamesArchitecture/biodiversity.svg
 [ci_link]: http://travis-ci.org/GlobalNamesArchitecture/biodiversity
 [dimus]: https://github.com/dimus
+[pleary]: https://github.com/pleary
+[hernan]: https://github.com/LocoDelAssembly
 [license]: https://github.com/GlobalNamesArchitecture/biodiversity/blob/master/LICENSE
 [uuid_examples]: https://github.com/GlobalNamesArchitecture/gn_uuid_examples
 [uuid_blog]: http://globalnamesarchitecture.github.io/gna/uuid/2015/05/31/gn-uuid-0-5-0.html

data/clib/linux/callback_bridge.h

@@ -0,0 +1,3 @@
+typedef void Callback(char *parsed);
+
+void callback_bridge(void *callback, char *parsed);

data/clib/linux/libgnparser.h

@@ -22,6 +22,7 @@ typedef struct { const char *p; ptrdiff_t n; } _GoString_;
 #line 3 "main.go"
 
 	#include "stdlib.h"
+	#include "callback_bridge.h"
 
 #line 1 "cgo-generated-wrapper"
 
@@ -77,8 +78,9 @@ extern "C" {
 // ParseToString function takes a name-string, desired format, and parses
 // the name-string to either JSON, or pipe-separated values, depending on
 // the desired format. Format can take values of 'simple', 'compact', 'pretty'.
+// NOTE: Read callback type as "void (*callback)(char *parsed)"
 
-extern char* ParseToString(char* p0, char* p1);
+extern void ParseToString(char* p0, char* p1, void* p2);
 
 // ParseAryToStrings function takes an array of names, parsing format and a
 // reference to an output: an empty array of strings to return the the data
@@ -86,7 +88,7 @@ extern char* ParseToString(char* p0, char* p1);
 // pipe-separated parsed values (depending on a given format). Format can take
 // values of 'simple', 'compact', or 'pretty'.
 
-extern void ParseAryToStrings(char** p0, int p1, char* p2, char*** p3);
+extern void ParseAryToStrings(char** p0, int p1, char* p2, void* p3);
 
 #ifdef __cplusplus
 }

data/clib/mac/callback_bridge.h

@@ -0,0 +1,3 @@
+typedef void Callback(char *parsed);
+
+void callback_bridge(void *callback, char *parsed);

data/clib/mac/libgnparser.h

@@ -22,6 +22,7 @@ typedef struct { const char *p; ptrdiff_t n; } _GoString_;
 #line 3 "main.go"
 
 	#include "stdlib.h"
+	#include "callback_bridge.h"
 
 #line 1 "cgo-generated-wrapper"
 
@@ -77,8 +78,9 @@ extern "C" {
 // ParseToString function takes a name-string, desired format, and parses
 // the name-string to either JSON, or pipe-separated values, depending on
 // the desired format. Format can take values of 'simple', 'compact', 'pretty'.
+// NOTE: Read callback type as "void (*callback)(char *parsed)"
 
-extern char* ParseToString(char* p0, char* p1);
+extern void ParseToString(char* p0, char* p1, void* p2);
 
 // ParseAryToStrings function takes an array of names, parsing format and a
 // reference to an output: an empty array of strings to return the the data
@@ -86,7 +88,7 @@ extern char* ParseToString(char* p0, char* p1);
 // pipe-separated parsed values (depending on a given format). Format can take
 // values of 'simple', 'compact', or 'pretty'.
 
-extern void ParseAryToStrings(char** p0, int p1, char* p2, char*** p3);
+extern void ParseAryToStrings(char** p0, int p1, char* p2, void* p3);
 
 #ifdef __cplusplus
 }

data/clib/win/callback_bridge.h

@@ -0,0 +1,3 @@
+typedef void Callback(char *parsed);
+
+void callback_bridge(void *callback, char *parsed);

data/clib/win/libgnparser.h

@@ -22,6 +22,7 @@ typedef struct { const char *p; ptrdiff_t n; } _GoString_;
 #line 3 "main.go"
 
 	#include "stdlib.h"
+	#include "callback_bridge.h"
 
 #line 1 "cgo-generated-wrapper"
 
@@ -77,8 +78,9 @@ extern "C" {
 // ParseToString function takes a name-string, desired format, and parses
 // the name-string to either JSON, or pipe-separated values, depending on
 // the desired format. Format can take values of 'simple', 'compact', 'pretty'.
+// NOTE: Read callback type as "void (*callback)(char *parsed)"
 
-extern char* ParseToString(char* p0, char* p1);
+extern void ParseToString(char* p0, char* p1, void* p2);
 
 // ParseAryToStrings function takes an array of names, parsing format and a
 // reference to an output: an empty array of strings to return the the data
@@ -86,7 +88,7 @@ extern char* ParseToString(char* p0, char* p1);
 // pipe-separated parsed values (depending on a given format). Format can take
 // values of 'simple', 'compact', or 'pretty'.
 
-extern void ParseAryToStrings(char** p0, int p1, char* p2, char*** p3);
+extern void ParseAryToStrings(char** p0, int p1, char* p2, void* p3);
 
 #ifdef __cplusplus
 }

data/lib/biodiversity.rb

@@ -2,6 +2,7 @@
 
 require 'ffi'
 require 'json'
+require 'csv'
 require_relative 'biodiversity/version'
 require_relative 'biodiversity/parser'
 

data/lib/biodiversity/parser.rb

@@ -25,40 +25,42 @@ module Biodiversity
     ffi_lib File.join(__dir__, '..', '..', 'clib', platform, 'libgnparser.so')
     POINTER_SIZE = FFI.type_size(:pointer)
 
-    attach_function(:parse_go, :ParseToString, %i[string string], :string)
+    callback(:parser_callback, %i[string], :void)
+
+    attach_function(:parse_go, :ParseToString,
+                    %i[string string parser_callback], :void)
     attach_function(:parse_ary_go, :ParseAryToStrings,
-                    %i[pointer int string pointer], :void)
+                    %i[pointer int string parser_callback], :void)
 
     def self.parse(name, simple = false)
       format = simple ? 'simple' : 'compact'
-      parsed = parse_go(name, format)
+
+      parsed = nil
+      callback = FFI::Function.new(:void, [:string]) { |str| parsed = str }
+      parse_go(name, format, callback)
       output(parsed, simple)
     end
 
     def self.parse_ary(ary, simple = false)
       format = simple ? 'simple' : 'compact'
       in_ptr = FFI::MemoryPointer.new(:pointer, ary.length)
+
       in_ptr.write_array_of_pointer(
         ary.map { |s| FFI::MemoryPointer.from_string(s) }
       )
-      out_var = FFI::MemoryPointer.new(:pointer)
-      parse_ary_go(in_ptr, ary.length, format, out_var)
 
-      out_var.read_pointer
-             .get_array_of_string(0, ary.length)
-             .each_with_object([]) do |prsd, a|
-        a << output(prsd, simple)
-      end
-    ensure
-      out_var.read_pointer.get_array_of_pointer(0, ary.length).each do |p|
-        CLib.free(p)
+      out_ary = []
+      callback = FFI::Function.new(:void, [:string]) do |str|
+        out_ary << output(str, simple)
       end
-      CLib.free(out_var.read_pointer)
+      parse_ary_go(in_ptr, ary.length, format, callback)
+      out_ary
     end
 
     def self.output(parsed, simple)
       if simple
-        parsed = parsed.split('|')
+        csv = CSV.new(parsed)
+        parsed = csv.read[0]
         {
           id: parsed[0],
           verbatim: parsed[1],

data/lib/biodiversity/version.rb

@@ -2,7 +2,7 @@
 
 # Biodiversity module provides a namespace for scientific name parser.
 module Biodiversity
-  VERSION = '4.0.2'
+  VERSION = '4.0.3'
 
   def self.version
     VERSION

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: biodiversity
 version: !ruby/object:Gem::Version
-  version: 4.0.2
+  version: 4.0.3
 platform: ruby
 authors:
 - Dmitry Mozzherin
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-11-18 00:00:00.000000000 Z
+date: 2020-03-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ffi
@@ -111,10 +111,13 @@ files:
 - README.md
 - Rakefile
 - biodiversity.gemspec
+- clib/linux/callback_bridge.h
 - clib/linux/libgnparser.h
 - clib/linux/libgnparser.so
+- clib/mac/callback_bridge.h
 - clib/mac/libgnparser.h
 - clib/mac/libgnparser.so
+- clib/win/callback_bridge.h
 - clib/win/libgnparser.h
 - clib/win/libgnparser.so
 - lib/biodiversity.rb
@@ -142,8 +145,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6.2
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: Parser of scientific names