biodiversity 0.5.14

data/.document

@@ -0,0 +1,5 @@
+README.rdoc
+lib/**/*.rb
+bin/*
+features/**/*.feature
+LICENSE

data/.gitignore

@@ -0,0 +1,13 @@
+*.sw?
+.DS_Store
+coverage
+rdoc
+pkg
+*.swp
+*.swo
+biodiversity*.gem
+*json
+*xml
+tmp
+.DS_Store
+spec/parser/test_data_new.txt

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2009 Dmitry Mozzherin
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,44 @@
+= Biodiversity
+
+Parses species scientific name and breaks it into elements.
+
+== Installation
+
+To install gem you need RubyGems >= 1.2.0
+
+  $ gem sources -a http://gems.github.com (you only have to do this once)
+  $ sudo gem install dimus-biodiversity
+
+== Example usage
+
+You can parse file with species names from command line. File should contain one scientific name per line
+
+  nnparser file_with_names
+  
+You can use it as a library
+
+  require 'biodiversity'
+  
+  parser = ScientificNameParser.new
+  
+  # to parse a scientific name into a ruby hash
+  parser.parse("Plantago major")
+  
+  #to get json representation
+  parser.parse("Plantago").to_json
+  #or
+  parser.parse("Plantago")
+  parser.all_json
+  
+  # to clean name up
+  parser.parse("             Plantago       major    ")[:scientificName][:normalized] 
+  
+  # to get only cleaned up latin part of the name
+  parser.parse("Pseudocercospora dendrobii (H.C. Burnett) U. Braun & Crous 2003")[:scientificName][:canonical]
+  
+  # to get detailed information about elements of the name
+  parser.parse("Pseudocercospora dendrobii (H.C. Burnett 1883) U. Braun & Crous 2003")[:scientificName][:details]
+  
+  # to resolve lsid and get back RDF file
+  LsidResolver.resolve("urn:lsid:ubio.org:classificationbank:2232671")
+

data/Rakefile

@@ -0,0 +1,43 @@
+dir = File.dirname(__FILE__)
+require 'rubygems'
+require 'rake'
+#$LOAD_PATH.unshift(File.join(dir, 'vendor', 'rspec', 'lib'))
+require 'spec/rake/spectask'
+
+#Gem::manage_gems
+#require 'rake/gempackagetask'
+
+task :default => :spec
+
+Spec::Rake::SpecTask.new do |t|
+  t.pattern = 'spec/**/*spec.rb'
+end
+
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gem|
+    gem.name = "biodiversity"
+    gem.summary = 'Parser of scientific names'
+    gem.description = 'Tools for biodiversity informatics'
+    gem.email = "dmozzherin@gmail.com"
+    gem.homepage = "http://github.com/dimus/biodiversity"
+    gem.authors = ["Dmitry Mozzherin"]
+    gem.has_rdoc = false
+    gem.bindir = 'bin'
+    gem.executables = ['nnparse']
+    gem.add_dependency('treetop')
+    gem.add_dependency('json') if RUBY_VERSION.split(".")[0..1].join('').to_i < 19
+    gem.add_development_dependency "rspec"
+    # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
+  end
+rescue LoadError
+  puts "Jeweler (or a dependency) not available. Install it with: sudo gem install jeweler"
+end
+
+task :tt do
+  system("tt #{dir}/lib/biodiversity/parser/scientific_name_clean.treetop")
+  system("tt #{dir}/lib/biodiversity/parser/scientific_name_dirty.treetop")
+  system("tt #{dir}/lib/biodiversity/parser/scientific_name_canonical.treetop")
+end
+

data/VERSION

@@ -0,0 +1 @@
+0.5.14

data/bin/nnparse

@@ -0,0 +1,43 @@
+#!/usr/bin/env ruby
+require 'rubygems'
+gem 'dimus-biodiversity' rescue gem 'biodiversity' rescue nil
+
+$LOAD_PATH.unshift(File.expand_path(File.dirname(__FILE__) + "/../lib"))
+require 'biodiversity'
+require 'json'
+
+
+if ARGV.empty?
+  puts "Usage:\n\nnnparse file_with_scientific_names [output_file]\n\ndefault output_file is parsed.json\n\n"
+  exit
+end
+
+input = ARGV[0]
+output = ARGV[1] || 'parsed.json'
+
+ruby_min_version = RUBY_VERSION.split(".")[0..1].join('').to_i
+
+p = ScientificNameParser.new
+o = open(output, 'w')
+count = 0
+puts 'Parsing...'
+IO.foreach(input) do |line|
+  count += 1
+  puts("%s lines parsed" % count) if count % 10000 == 0
+  name = line.gsub(/^[\d]*\s*/, '').strip
+  begin
+    if ruby_min_version < 19
+      old_kcode = $KCODE
+      $KCODE = 'NONE'
+    end
+    p.parse(name) 
+    parsed_data = p.parsed.all_json rescue {'parsed' => false, 'verbatim' => name,  'error' => 'Parser error'}.to_json
+    if ruby_min_version < 19
+      $KCODE = old_kcode
+    end
+  rescue 
+    parsed_data = {'parsed' => false, 'verbatim' => name,  'error' => 'Parser error'}.to_json
+  end
+  o.write parsed_data + "\n"
+end
+

data/bin/parserver

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+require 'rubygems'
+require 'socket'
+require 'biodiversity'          # Get sockets from stdlib
+parser = ScientificNameParser.new
+server = TCPServer.open(4334)  # Socket to listen on port 4334
+loop do                         # Servers run forever
+  client = server.accept       # Wait for a client to connect
+  while a = client.readline
+    client.close if ['end','exit','q', '.'].include? a.strip
+    client.puts parser.parse(a).to_json
+  end
+end
+

data/biodiversity.gemspec

@@ -0,0 +1,88 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{biodiversity}
+  s.version = "0.5.14"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Dmitry Mozzherin"]
+  s.date = %q{2010-03-19}
+  s.default_executable = %q{nnparse}
+  s.description = %q{Tools for biodiversity informatics}
+  s.email = %q{dmozzherin@gmail.com}
+  s.executables = ["nnparse"]
+  s.extra_rdoc_files = [
+    "LICENSE",
+     "README.rdoc"
+  ]
+  s.files = [
+    ".document",
+     ".gitignore",
+     "LICENSE",
+     "README.rdoc",
+     "Rakefile",
+     "VERSION",
+     "bin/nnparse",
+     "bin/parserver",
+     "biodiversity.gemspec",
+     "conf/environment.rb",
+     "lib/biodiversity.rb",
+     "lib/biodiversity/guid.rb",
+     "lib/biodiversity/guid/lsid.rb",
+     "lib/biodiversity/parser.rb",
+     "lib/biodiversity/parser/scientific_name_canonical.rb",
+     "lib/biodiversity/parser/scientific_name_canonical.treetop",
+     "lib/biodiversity/parser/scientific_name_clean.rb",
+     "lib/biodiversity/parser/scientific_name_clean.treetop",
+     "lib/biodiversity/parser/scientific_name_dirty.rb",
+     "lib/biodiversity/parser/scientific_name_dirty.treetop",
+     "pkg/.gitignore",
+     "spec/biodiversity_spec.rb",
+     "spec/guid/lsid.spec.rb",
+     "spec/parser/scientific_name.spec.rb",
+     "spec/parser/scientific_name_canonical.spec.rb",
+     "spec/parser/scientific_name_clean.spec.rb",
+     "spec/parser/scientific_name_dirty.spec.rb",
+     "spec/parser/spec_helper.rb",
+     "spec/parser/test_data.txt",
+     "spec/spec_helper.rb"
+  ]
+  s.homepage = %q{http://github.com/dimus/biodiversity}
+  s.rdoc_options = ["--charset=UTF-8"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = %q{1.3.6}
+  s.summary = %q{Parser of scientific names}
+  s.test_files = [
+    "spec/biodiversity_spec.rb",
+     "spec/guid/lsid.spec.rb",
+     "spec/parser/scientific_name.spec.rb",
+     "spec/parser/scientific_name_canonical.spec.rb",
+     "spec/parser/scientific_name_clean.spec.rb",
+     "spec/parser/scientific_name_dirty.spec.rb",
+     "spec/parser/spec_helper.rb",
+     "spec/spec_helper.rb"
+  ]
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<treetop>, [">= 0"])
+      s.add_runtime_dependency(%q<json>, [">= 0"])
+      s.add_development_dependency(%q<rspec>, [">= 0"])
+    else
+      s.add_dependency(%q<treetop>, [">= 0"])
+      s.add_dependency(%q<json>, [">= 0"])
+      s.add_dependency(%q<rspec>, [">= 0"])
+    end
+  else
+    s.add_dependency(%q<treetop>, [">= 0"])
+    s.add_dependency(%q<json>, [">= 0"])
+    s.add_dependency(%q<rspec>, [">= 0"])
+  end
+end
+

data/conf/environment.rb

@@ -0,0 +1,3 @@
+
+#Constants
+LSID_RESOLVER_URL = "http://lsid.tdwg.org/" 

data/lib/biodiversity.rb

@@ -0,0 +1,9 @@
+require 'rubygems'
+require 'treetop'
+
+dir = File.dirname(__FILE__)
+
+BIODIVERSITY_ROOT = File.join(dir, 'biodiversity')
+require File.join(dir, "/../conf/environment")
+require File.join(BIODIVERSITY_ROOT, "parser")
+require File.join(BIODIVERSITY_ROOT, "guid")

data/lib/biodiversity/guid.rb

@@ -0,0 +1,2 @@
+dir = File.dirname(__FILE__)
+require File.join(dir, *%w[guid lsid])

data/lib/biodiversity/guid/lsid.rb

@@ -0,0 +1,18 @@
+require 'open-uri'
+
+class LsidResolver
+  def self.resolve(lsid)
+    http_get_rdf(lsid)
+  end
+  
+protected
+  def self.http_get_rdf(lsid)
+    rdf = ''
+    open(LSID_RESOLVER_URL + lsid) do |f|
+      f.each do |line|
+        rdf += line if !line.strip.blank?
+      end
+    end
+    rdf
+  end
+end

data/lib/biodiversity/parser.rb

@@ -0,0 +1,57 @@
+# encoding: UTF-8
+dir = File.dirname(__FILE__)
+require File.join(dir, *%w[parser scientific_name_clean])
+require File.join(dir, *%w[parser scientific_name_dirty])
+require File.join(dir, *%w[parser scientific_name_canonical])
+require 'rubygems'
+require 'json'
+
+class ScientificNameParser
+  
+  def initialize
+    @verbatim = ''
+    @clean = ScientificNameCleanParser.new
+    @dirty = ScientificNameDirtyParser.new
+    @canonical = ScientificNameCanonicalParser.new
+    @parsed = nil
+  end
+  
+  def parsed
+    @parsed
+  end
+  
+  def parse(a_string)
+    @verbatim = a_string
+    @parsed = @clean.parse(a_string) || @dirty.parse(a_string) || @canonical.parse(a_string) || {:verbatim => a_string}
+    def @parsed.all
+      parsed = self.class != Hash
+      res = {:parsed => parsed}
+      if parsed
+        hybrid = self.hybrid rescue false
+        res.merge!({
+          :verbatim => self.text_value,
+          :normalized => self.value,
+          :canonical => self.canonical,
+          :hybrid => hybrid,
+          :details => self.details,
+          :positions => self.pos
+          })
+      else
+        res.merge!(self)
+      end
+      res = {:scientificName => res}
+      res
+    end
+    
+    def @parsed.pos_json
+      self.pos.to_json rescue ''
+    end
+    
+    def @parsed.all_json
+      self.all.to_json rescue ''
+    end
+    
+    @parsed.all
+  end
+end
+

data/lib/biodiversity/parser/scientific_name_canonical.rb

@@ -0,0 +1,462 @@
+# encoding: UTF-8
+module ScientificNameCanonical
+  include Treetop::Runtime
+
+  def root
+    @root || :root
+  end
+
+  include ScientificNameClean
+
+  include ScientificNameDirty
+
+  module Root0
+    def hybrid
+      false
+    end
+
+    def details
+      [super]
+    end
+  end
+
+  module Root1
+    def hybrid
+      false
+    end
+    
+    def details
+      [super]
+    end
+  end
+
+  def _nt_root
+    start_index = index
+    if node_cache[:root].has_key?(index)
+      cached = node_cache[:root][index]
+      @index = cached.interval.end if cached
+      return cached
+    end
+
+    i0 = index
+    r1 = _nt_multinomial_with_garbage
+    r1.extend(Root0)
+    if r1
+      r0 = r1
+    else
+      r2 = _nt_uninomial_with_garbage
+      r2.extend(Root1)
+      if r2
+        r0 = r2
+      else
+        @index = i0
+        r0 = nil
+      end
+    end
+
+    node_cache[:root][start_index] = r0
+
+    r0
+  end
+
+  module MultinomialWithGarbage0
+    def a
+      elements[0]
+    end
+
+    def space
+      elements[1]
+    end
+
+    def b
+      elements[2]
+    end
+
+    def space
+      elements[3]
+    end
+
+    def c
+      elements[4]
+    end
+
+    def garbage
+      elements[5]
+    end
+  end
+
+  module MultinomialWithGarbage1
+    def value
+      a.value + " " + b.value + " " + c.value
+    end
+    
+    def canonical
+      a.canonical + " " + b.canonical + " " + c.canonical
+    end
+    
+    def pos
+      a.pos.merge(b.pos).merge(c.pos)
+    end
+    
+    def details
+      a.details.merge(b.details).merge(c.details)
+    end
+  end
+
+  module MultinomialWithGarbage2
+    def a
+      elements[0]
+    end
+
+    def space
+      elements[1]
+    end
+
+    def b
+      elements[2]
+    end
+
+    def garbage
+      elements[3]
+    end
+  end
+
+  module MultinomialWithGarbage3
+    def value
+      a.value + " " + b.value
+    end
+    
+    def canonical
+      a.canonical + " " + b.canonical
+    end
+    
+    def pos
+      a.pos.merge(b.pos)
+    end
+    
+    def details
+      a.details.merge(b.details)
+    end
+  end
+
+  module MultinomialWithGarbage4
+    def a
+      elements[0]
+    end
+
+    def space
+      elements[1]
+    end
+
+    def b
+      elements[2]
+    end
+
+    def garbage
+      elements[3]
+    end
+  end
+
+  module MultinomialWithGarbage5
+    def value
+      a.value + " " + b.value
+    end
+    
+    def canonical
+      a.canonical + " " + b.canonical
+    end
+    
+    def pos
+      a.pos.merge(b.pos)
+    end
+    
+    def details
+      a.details.merge(b.details)
+    end
+  end
+
+  def _nt_multinomial_with_garbage
+    start_index = index
+    if node_cache[:multinomial_with_garbage].has_key?(index)
+      cached = node_cache[:multinomial_with_garbage][index]
+      @index = cached.interval.end if cached
+      return cached
+    end
+
+    i0 = index
+    i1, s1 = index, []
+    r2 = _nt_genus
+    s1 << r2
+    if r2
+      r3 = _nt_space
+      s1 << r3
+      if r3
+        r4 = _nt_subgenus
+        s1 << r4
+        if r4
+          r5 = _nt_space
+          s1 << r5
+          if r5
+            r6 = _nt_species
+            s1 << r6
+            if r6
+              r7 = _nt_garbage
+              s1 << r7
+            end
+          end
+        end
+      end
+    end
+    if s1.last
+      r1 = instantiate_node(SyntaxNode,input, i1...index, s1)
+      r1.extend(MultinomialWithGarbage0)
+      r1.extend(MultinomialWithGarbage1)
+    else
+      @index = i1
+      r1 = nil
+    end
+    if r1
+      r0 = r1
+    else
+      i8, s8 = index, []
+      r9 = _nt_genus
+      s8 << r9
+      if r9
+        r10 = _nt_space
+        s8 << r10
+        if r10
+          r11 = _nt_subgenus
+          s8 << r11
+          if r11
+            r12 = _nt_garbage
+            s8 << r12
+          end
+        end
+      end
+      if s8.last
+        r8 = instantiate_node(SyntaxNode,input, i8...index, s8)
+        r8.extend(MultinomialWithGarbage2)
+        r8.extend(MultinomialWithGarbage3)
+      else
+        @index = i8
+        r8 = nil
+      end
+      if r8
+        r0 = r8
+      else
+        i13, s13 = index, []
+        r14 = _nt_genus
+        s13 << r14
+        if r14
+          r15 = _nt_space
+          s13 << r15
+          if r15
+            r16 = _nt_species
+            s13 << r16
+            if r16
+              r17 = _nt_garbage
+              s13 << r17
+            end
+          end
+        end
+        if s13.last
+          r13 = instantiate_node(SyntaxNode,input, i13...index, s13)
+          r13.extend(MultinomialWithGarbage4)
+          r13.extend(MultinomialWithGarbage5)
+        else
+          @index = i13
+          r13 = nil
+        end
+        if r13
+          r0 = r13
+        else
+          @index = i0
+          r0 = nil
+        end
+      end
+    end
+
+    node_cache[:multinomial_with_garbage][start_index] = r0
+
+    r0
+  end
+
+  module UninomialWithGarbage0
+    def a
+      elements[0]
+    end
+
+    def b
+      elements[1]
+    end
+  end
+
+  module UninomialWithGarbage1
+    def value
+      a.value
+    end
+    
+    def canonical
+      a.canonical
+    end
+    
+    def pos
+      a.pos
+    end
+    
+    def details
+      {:uninomial => a.details[:uninomial]}
+    end
+  end
+
+  def _nt_uninomial_with_garbage
+    start_index = index
+    if node_cache[:uninomial_with_garbage].has_key?(index)
+      cached = node_cache[:uninomial_with_garbage][index]
+      @index = cached.interval.end if cached
+      return cached
+    end
+
+    i0, s0 = index, []
+    r1 = _nt_uninomial_epitheton
+    s0 << r1
+    if r1
+      r2 = _nt_garbage
+      s0 << r2
+    end
+    if s0.last
+      r0 = instantiate_node(SyntaxNode,input, i0...index, s0)
+      r0.extend(UninomialWithGarbage0)
+      r0.extend(UninomialWithGarbage1)
+    else
+      @index = i0
+      r0 = nil
+    end
+
+    node_cache[:uninomial_with_garbage][start_index] = r0
+
+    r0
+  end
+
+  module Garbage0
+    def space
+      elements[0]
+    end
+
+    def space
+      elements[2]
+    end
+
+  end
+
+  module Garbage1
+    def space_hard
+      elements[0]
+    end
+
+  end
+
+  def _nt_garbage
+    start_index = index
+    if node_cache[:garbage].has_key?(index)
+      cached = node_cache[:garbage][index]
+      @index = cached.interval.end if cached
+      return cached
+    end
+
+    i0 = index
+    i1, s1 = index, []
+    r2 = _nt_space
+    s1 << r2
+    if r2
+      if has_terminal?('\G["\',.]', true, index)
+        r3 = instantiate_node(SyntaxNode,input, index...(index + 1))
+        @index += 1
+      else
+        r3 = nil
+      end
+      s1 << r3
+      if r3
+        r4 = _nt_space
+        s1 << r4
+        if r4
+          s5, i5 = [], index
+          loop do
+            if has_terminal?('\G[^щ]', true, index)
+              r6 = instantiate_node(SyntaxNode,input, index...(index + 1))
+              @index += 1
+            else
+              r6 = nil
+            end
+            if r6
+              s5 << r6
+            else
+              break
+            end
+          end
+          r5 = instantiate_node(SyntaxNode,input, i5...index, s5)
+          s1 << r5
+        end
+      end
+    end
+    if s1.last
+      r1 = instantiate_node(SyntaxNode,input, i1...index, s1)
+      r1.extend(Garbage0)
+    else
+      @index = i1
+      r1 = nil
+    end
+    if r1
+      r0 = r1
+    else
+      i7, s7 = index, []
+      r8 = _nt_space_hard
+      s7 << r8
+      if r8
+        s9, i9 = [], index
+        loop do
+          if has_terminal?('\G[^ш]', true, index)
+            r10 = instantiate_node(SyntaxNode,input, index...(index + 1))
+            @index += 1
+          else
+            r10 = nil
+          end
+          if r10
+            s9 << r10
+          else
+            break
+          end
+        end
+        if s9.empty?
+          @index = i9
+          r9 = nil
+        else
+          r9 = instantiate_node(SyntaxNode,input, i9...index, s9)
+        end
+        s7 << r9
+      end
+      if s7.last
+        r7 = instantiate_node(SyntaxNode,input, i7...index, s7)
+        r7.extend(Garbage1)
+      else
+        @index = i7
+        r7 = nil
+      end
+      if r7
+        r0 = r7
+      else
+        @index = i0
+        r0 = nil
+      end
+    end
+
+    node_cache[:garbage][start_index] = r0
+
+    r0
+  end
+
+end
+
+class ScientificNameCanonicalParser < Treetop::Runtime::CompiledParser
+  include ScientificNameCanonical
+end