biodiversity 1.0.5 → 1.0.8

data/.rvmrc

@@ -0,0 +1 @@
+rvm use ruby-1.9.2-p290@biodiversity --create

data/Gemfile

@@ -0,0 +1,13 @@
+source "http://rubygems.org"
+
+gem "treetop"
+gem "parallel"
+
+group :development do
+  gem "jeweler"
+end
+
+group :test do
+  gem "ruby-debug19", :require => "ruby-debug"
+  gem "rspec"
+end

data/Gemfile.lock

@@ -0,0 +1,47 @@
+GEM
+  remote: http://rubygems.org/
+  specs:
+    archive-tar-minitar (0.5.2)
+    columnize (0.3.4)
+    diff-lcs (1.1.3)
+    git (1.2.5)
+    jeweler (1.6.4)
+      bundler (~> 1.0)
+      git (>= 1.2.5)
+      rake
+    linecache19 (0.5.12)
+      ruby_core_source (>= 0.1.4)
+    parallel (0.5.9)
+    polyglot (0.3.3)
+    rake (0.9.2.2)
+    rspec (2.7.0)
+      rspec-core (~> 2.7.0)
+      rspec-expectations (~> 2.7.0)
+      rspec-mocks (~> 2.7.0)
+    rspec-core (2.7.1)
+    rspec-expectations (2.7.0)
+      diff-lcs (~> 1.1.2)
+    rspec-mocks (2.7.0)
+    ruby-debug-base19 (0.11.25)
+      columnize (>= 0.3.1)
+      linecache19 (>= 0.5.11)
+      ruby_core_source (>= 0.1.4)
+    ruby-debug19 (0.11.6)
+      columnize (>= 0.3.1)
+      linecache19 (>= 0.5.11)
+      ruby-debug-base19 (>= 0.11.19)
+    ruby_core_source (0.1.5)
+      archive-tar-minitar (>= 0.5.2)
+    treetop (1.4.10)
+      polyglot
+      polyglot (>= 0.3.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  jeweler
+  parallel
+  rspec
+  ruby-debug19
+  treetop

data/README.rdoc

@@ -85,6 +85,11 @@ You can use it as a library
   # to get detailed information about elements of the name
   parser.parse("Pseudocercospora dendrobii (H.C. Burnett 1883) U. Braun & Crous 2003")[:scientificName][:details]
 
+  # to parse using several CPUs (4 seem to be optimal)
+  parser = ParallelParser.new # ParallelParser.new(4) will try to run 4 processes if hardware allows
+  array_of_names = ["Betula alba", "Homo sapiens"....]
+  parser.parse(array_of_names) # -> {"Betula alba" => "{scientificName...}", "Homo sapiens" => "{scientificName...}", ...}
+
   # to resolve lsid and get back RDF file
   LsidResolver.resolve("urn:lsid:ubio.org:classificationbank:2232671")
 

data/Rakefile

@@ -30,6 +30,7 @@ begin
     gem.bindir = 'bin'
     gem.executables = ['nnparse', 'parserver']
     gem.add_dependency('treetop')
+    gem.add_dependency('parallel')
     gem.add_dependency('json') if ruby_version < 19
     gem.add_development_dependency "rspec"
     # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings

data/VERSION

@@ -1 +1 @@
-1.0.5
+1.0.8

data/lib/biodiversity/parser.rb

@@ -23,6 +23,34 @@ module PreProcessor
   end   
 end
 
+class ParallelParser
+
+  def initialize(processes_num = nil)
+    require 'parallel'
+    cpu_num
+    if processes_num.to_i > 0
+      @processes_num = [processes_num, cpu_num - 1].min
+    else
+      @processes_num = cpu_num > 3 ? cpu_num - 2 : 1
+    end
+  end
+
+  def parse(names_list)
+    parsed = Parallel.map(names_list.uniq, :in_processes => @processes_num) { |n| [n, parse_process(n)] }
+    parsed.inject({}) { |res, x| res[x[0]] = x[1]; res }
+  end
+
+  def cpu_num
+    @cpu_num ||= Parallel.processor_count
+  end
+
+  private
+  def parse_process(name)
+    p = ScientificNameParser.new
+    p.parse(name).to_json rescue {'scientificName' => {'parsed' => false, 'verbatim' => name,  'error' => 'Parser error'}}.to_json
+  end
+end
+
 # we can use these expressions when we are ready to parse virus names
 # class VirusParser
 #   def initialize

data/lib/biodiversity/parser/scientific_name_clean.treetop

@@ -403,7 +403,7 @@ grammar ScientificNameClean
   end
   
   rule rank
-    ("morph."/"f.sp."/"B"/"ssp."/"mut."/"nat"/"nothosubsp."/"pseudovar."/"sect."/"ser."/"var."/"subvar."/ "[var.]" /"var"/"subsp."/"subsp"/"subf."/"race"/"α" 
+    ("morph."/"f.sp."/"B"/"ssp."/"mut."/"nat"/"nothosubsp."/"convar."/"pseudovar."/"sect."/"ser."/"var."/"subvar."/ "[var.]" /"var"/"subsp."/"subsp"/"subf."/"race"/"α" 
     /"ββ"/"β"/"γ"/"δ"/"ε"/"φ"/"θ"/"μ"/"a."/"b."/"c."/"d."/"e."/"g."/"k."/"****"/"**"/"*")
     {
       def value

data/spec/parser/scientific_name.spec.rb

@@ -43,3 +43,31 @@ describe ScientificNameParser do
     parse('Nile virus')[:scientificName][:parser_version].should_not be_nil
   end
 end
+
+
+describe ParallelParser do
+  it "should find number of cpus" do
+    pparser = ParallelParser.new
+    pparser.cpu_num.should > 0
+  end
+
+  it "should parse several names in parallel" do
+    names = []
+    read_test_file { |n| names << (n[:name]) if n[:name] }
+    names.uniq!
+    pparser = ParallelParser.new
+    res = pparser.parse(names)
+    names.size.should > 100
+    res.keys.size.should == names.size
+  end
+
+  it "should parse several names in parallel with given num of processes" do
+    names = []
+    read_test_file { |n| names << (n[:name]) if n[:name] }
+    names.uniq!
+    pparser = ParallelParser.new(4)
+    res = pparser.parse(names)
+    names.size.should > 100
+    res.keys.size.should == names.size
+  end
+end

data/spec/parser/scientific_name_clean.spec.rb

@@ -254,6 +254,8 @@ describe ScientificNameClean do
     pos(sn).should == {0=>["genus", 12], 13=>["species", 19], 25=>["infraspecies", 34], 35=>["author_word", 42], 44=>["author_word", 48], 49=>["author_word", 52], 53=>["year", 57]}
     sn = "Cassytha peninsularis J. Z. Weber var. flindersii"
     canonical(sn).should == "Cassytha peninsularis flindersii"
+    sn = "Prunus armeniaca convar. budae (Pénzes) Soó"
+    canonical(sn).should == "Prunus armeniaca budae"
   end
 
   it 'should parse unknown original authors (auct.)/(hort.)/(?)' do

data/spec/parser/test_data.txt

@@ -109,6 +109,7 @@ Chlorocyperus glaber form. fasciculariforme (Lojac.) Soó|{"scientificName":{"pa
 Sphaerotheca    fuliginea    f.     dahliae    Movss.     1967|{"scientificName":{"parsed":true, "parser_version":"test_version", "parser_run":1,"verbatim":"Sphaerotheca    fuliginea    f.     dahliae    Movss.     1967","normalized":"Sphaerotheca fuliginea f. dahliae Movss. 1967","canonical":"Sphaerotheca fuliginea dahliae","hybrid":false,"details":[{"genus":{"string":"Sphaerotheca"},"species":{"string":"fuliginea"},"infraspecies":[{"string":"dahliae","rank":"f.","authorship":"Movss.     1967","basionymAuthorTeam":{"authorTeam":"Movss.","author":["Movss."],"year":"1967"}}]}],"positions":{"0":["genus",12],"16":["species",25],"36":["infraspecies",43],"47":["author_word",53],"58":["year",62]}}}
 Polypodium vulgare nothosubsp. mantoniae (Rothm.) Schidlay|{"scientificName":{"parsed":true, "parser_version":"test_version", "parser_run":1,"verbatim":"Polypodium vulgare nothosubsp. mantoniae (Rothm.) Schidlay","normalized":"Polypodium vulgare nothosubsp. mantoniae (Rothm.) Schidlay","canonical":"Polypodium vulgare mantoniae","hybrid":false,"details":[{"genus":{"string":"Polypodium"},"species":{"string":"vulgare"},"infraspecies":[{"string":"mantoniae","rank":"nothosubsp.","authorship":"(Rothm.) Schidlay","combinationAuthorTeam":{"authorTeam":"Schidlay","author":["Schidlay"]},"basionymAuthorTeam":{"authorTeam":"Rothm.","author":["Rothm."]}}]}],"positions":{"0":["genus",10],"11":["species",18],"31":["infraspecies",40],"42":["author_word",48],"50":["author_word",58]}}}
 Allophylus amazonicus var amazonicus|{"scientificName":{"parsed":true, "verbatim":"Allophylus amazonicus var amazonicus", "normalized":"Allophylus amazonicus var amazonicus", "canonical":"Allophylus amazonicus amazonicus", "hybrid":false, "details":[{"genus":{"string":"Allophylus"}, "species":{"string":"amazonicus"}, "infraspecies":[{"string":"amazonicus", "rank":"var"}]}],  "parser_version":"test_version", "parser_run":1, "positions":{"0":["genus", 10], "11":["species", 21], "26":["infraspecies", 36]}}}
+Prunus armeniaca convar. budae (Pénzes) Soó|{"scientificName":{"parsed":true, "parser_version":"test_version", "verbatim":"Prunus armeniaca convar. budae (Pénzes) Soó", "normalized":"Prunus armeniaca convar. budae (Pénzes) Soó", "canonical":"Prunus armeniaca budae", "hybrid":false, "details":[{"genus":{"string":"Prunus"}, "species":{"string":"armeniaca"}, "infraspecies":[{"string":"budae", "rank":"convar.", "authorship":"(Pénzes) Soó", "combinationAuthorTeam":{"authorTeam":"Soó", "author":["Soó"]}, "basionymAuthorTeam":{"authorTeam":"Pénzes", "author":["Pénzes"]}}]}], "parser_run":1, "positions":{"0":["genus", 6], "7":["species", 16], "25":["infraspecies", 30], "32":["author_word", 38], "40":["author_word", 43]}}}
 
 #infraspecies_multiple
 Hydnellum scrobiculatum var. zonatum f. parvum (Banker) D. Hall & D.E. Stuntz 1972|{"scientificName":{"parsed":true, "parser_version":"test_version", "parser_run":1,"verbatim":"Hydnellum scrobiculatum var. zonatum f. parvum (Banker) D. Hall & D.E. Stuntz 1972","normalized":"Hydnellum scrobiculatum var. zonatum f. parvum (Banker) D. Hall et D.E. Stuntz 1972","canonical":"Hydnellum scrobiculatum zonatum parvum","hybrid":false,"details":[{"genus":{"string":"Hydnellum"},"species":{"string":"scrobiculatum"},"infraspecies":[{"string":"zonatum","rank":"var."},{"string":"parvum","rank":"f.","authorship":"(Banker) D. Hall & D.E. Stuntz 1972","combinationAuthorTeam":{"authorTeam":"D. Hall & D.E. Stuntz","author":["D. Hall","D.E. Stuntz"],"year":"1972"},"basionymAuthorTeam":{"authorTeam":"Banker","author":["Banker"]}}]}],"positions":{"0":["genus",9],"10":["species",23],"29":["infraspecies",36],"40":["infraspecies",46],"48":["author_word",54],"56":["author_word",58],"59":["author_word",63],"66":["author_word",70],"71":["author_word",77],"78":["year",82]}}}

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: biodiversity
 version: !ruby/object:Gem::Version 
-  hash: 29
+  hash: 7
   prerelease: 
   segments: 
   - 1
   - 0
-  - 5
-  version: 1.0.5
+  - 8
+  version: 1.0.8
 platform: ruby
 authors: 
 - Dmitry Mozzherin
@@ -15,12 +15,24 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-08-09 00:00:00 Z
+date: 2011-11-18 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  version_requirements: *id001
   name: treetop
   prerelease: false
-  requirement: &id001 !ruby/object:Gem::Requirement 
+  type: :runtime
+- !ruby/object:Gem::Dependency 
+  requirement: &id002 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
@@ -29,12 +41,26 @@ dependencies:
         segments: 
         - 0
         version: "0"
+  version_requirements: *id002
+  name: parallel
+  prerelease: false
   type: :runtime
-  version_requirements: *id001
 - !ruby/object:Gem::Dependency 
-  name: json
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  version_requirements: *id003
+  name: jeweler
   prerelease: false
-  requirement: &id002 !ruby/object:Gem::Requirement 
+  type: :development
+- !ruby/object:Gem::Dependency 
+  requirement: &id004 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
@@ -43,12 +69,26 @@ dependencies:
         segments: 
         - 0
         version: "0"
+  version_requirements: *id004
+  name: treetop
+  prerelease: false
   type: :runtime
-  version_requirements: *id002
 - !ruby/object:Gem::Dependency 
-  name: rspec
+  requirement: &id005 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  version_requirements: *id005
+  name: parallel
   prerelease: false
-  requirement: &id003 !ruby/object:Gem::Requirement 
+  type: :runtime
+- !ruby/object:Gem::Dependency 
+  requirement: &id006 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
@@ -57,8 +97,24 @@ dependencies:
         segments: 
         - 0
         version: "0"
+  version_requirements: *id006
+  name: json
+  prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency 
+  requirement: &id007 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  version_requirements: *id007
+  name: rspec
+  prerelease: false
   type: :development
-  version_requirements: *id003
 description: Tools for biodiversity informatics
 email: dmozzherin@gmail.com
 executables: 
@@ -71,6 +127,9 @@ extra_rdoc_files:
 - README.rdoc
 files: 
 - .document
+- .rvmrc
+- Gemfile
+- Gemfile.lock
 - LICENSE
 - README.rdoc
 - Rakefile
@@ -125,7 +184,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.8.6
+rubygems_version: 1.8.10
 signing_key: 
 specification_version: 3
 summary: Parser of scientific names