anystyle-parser 0.3.0 → 0.4.0

data/lib/anystyle/parser/version.rb

@@ -1,5 +1,5 @@
 module Anystyle
   module Parser
-    VERSION = '0.3.0'.freeze
+    VERSION = '0.4.0'.freeze
   end
 end

data/spec/anystyle/parser/features_spec.rb

@@ -1,24 +1,30 @@
 # -*- encoding: utf-8 -*-
 
 module Anystyle::Parser
-	describe "Features" do
-	  
-		describe "numbers" do
-			let(:f) { Parser.feature[:numbers] }
-			
-			%w{ (1992) 1992 2011 1776 }.each do |year|
-				it "returns :year for #{year.inspect}" do
-					f.match(year).should == :year
-				end
-			end
-			
-			%w{ (1) (12) (123) }.each do |year|
-				it "returns :year for #{year.inspect}" do
-					f.match(year).should == :numeric
-				end
-			end
-			
-		end
-						
-	end
-end
+  describe "Features" do
+
+    describe "numbers" do
+      let(:f) { Parser.feature[:numbers] }
+
+      %w{ (1992) 1992 2011 1776 }.each do |year|
+        it "returns :year for #{year.inspect}" do
+          f.match(year).should == :year
+        end
+      end
+
+      %w{ (1) (12) (123) }.each do |year|
+        it "returns :year for #{year.inspect}" do
+          f.match(year).should == :numeric
+        end
+      end
+
+      ['pp', 'pp.', '23-4', '6124--19', '48 - 9', '19–27'].each do |page|
+        it "returns :page for #{page.inspect}" do
+          f.match(page).should == :page
+        end
+      end
+
+    end
+
+  end
+end

data/spec/anystyle/parser/normalizer_spec.rb

@@ -1,63 +1,84 @@
 module Anystyle
-	module Parser
-
-		describe "Normalizer" do
-
-			describe "#tokenize_names" do
-
-				it "tokenizes 'A B'" do
-					Normalizer.instance.normalize_names('A B').should == 'B, A'
-				end
-
-				it "tokenizes 'A, B'" do
-					Normalizer.instance.normalize_names('A, B').should == 'A, B'
-				end
-
-				# it "tokenizes 'A, jr., B'" do
-				# 	Normalizer.instance.normalize_names('A, jr., B').should == 'A, jr., B'
-				# end
-				# 
-				# it "tokenizes 'A, B, jr.'" do
-				# 	Normalizer.instance.normalize_names('A, B, jr.').should == 'A, B, jr.'
-				# end
-
-				it "tokenizes 'A, B, C, D'" do
-					Normalizer.instance.normalize_names('A, B, C, D').should == 'A, B and C, D'
-				end
-
-				it "tokenizes 'A, B, C'" do
-					Normalizer.instance.normalize_names('A, B, C').should == 'A, B and C'
-				end
-
-				it "tokenizes 'Aa Bb, C.'" do
-					Normalizer.instance.normalize_names('Aa Bb, C.').should == 'Aa Bb, C.'
-				end
-				
-				it "tokenizes 'Aa Bb, Cc Dd, and E F G'" do
-					Normalizer.instance.normalize_names('Aa Bb, C D, and E F G').should == 'Bb, Aa and D, C and G, E F'
-				end
-				
-				[
-					['Poe, Edgar A.', 'Poe, Edgar A.'],
-					['Edgar A. Poe', 'Poe, Edgar A.'],
-					['Edgar A. Poe, Herman Melville', 'Poe, Edgar A. and Melville, Herman'],
-					['Poe, Edgar A., Melville, Herman', 'Poe, Edgar A. and Melville, Herman'],
-					['Aeschlimann Magnin, E.', 'Aeschlimann Magnin, E.']
-				].each do |name, normalized|
-					it "tokenizes #{name.inspect}" do
-						Normalizer.instance.normalize_names(name).should == normalized
-					end
-				end
-				
-			end
-			
-			describe 'date extraction' do
-				it 'should extract the month and year from a string like (July 2009)' do
-					Normalizer.instance.normalize_date(:date => '(July 2009).').should == { :year => 2009, :month => 7 }
-				end
-			end
-			
-		end
-
-	end
-end
+  module Parser
+
+    describe "Normalizer" do
+      let(:n) { Normalizer.instance }
+
+      describe "#tokenize_names" do
+
+        it "tokenizes 'A B'" do
+          Normalizer.instance.normalize_names('A B').should == 'B, A'
+        end
+
+        it "tokenizes 'A, B'" do
+          Normalizer.instance.normalize_names('A, B').should == 'A, B'
+        end
+
+        # it "tokenizes 'A, jr., B'" do
+        #   Normalizer.instance.normalize_names('A, jr., B').should == 'A, jr., B'
+        # end
+        #
+        # it "tokenizes 'A, B, jr.'" do
+        #   Normalizer.instance.normalize_names('A, B, jr.').should == 'A, B, jr.'
+        # end
+
+        it "tokenizes 'A, B, C, D'" do
+          Normalizer.instance.normalize_names('A, B, C, D').should == 'A, B and C, D'
+        end
+
+        it "tokenizes 'A, B, C'" do
+          Normalizer.instance.normalize_names('A, B, C').should == 'A, B and C'
+        end
+
+        it "tokenizes 'Aa Bb, C.'" do
+          Normalizer.instance.normalize_names('Aa Bb, C.').should == 'Aa Bb, C.'
+        end
+
+        it "tokenizes 'Aa Bb, Cc Dd, and E F G'" do
+          Normalizer.instance.normalize_names('Aa Bb, C D, and E F G').should == 'Bb, Aa and D, C and G, E F'
+        end
+
+        [
+          ['Poe, Edgar A.', 'Poe, Edgar A.'],
+          ['Edgar A. Poe', 'Poe, Edgar A.'],
+          ['Edgar A. Poe, Herman Melville', 'Poe, Edgar A. and Melville, Herman'],
+          ['Poe, Edgar A., Melville, Herman', 'Poe, Edgar A. and Melville, Herman'],
+          ['Aeschlimann Magnin, E.', 'Aeschlimann Magnin, E.']
+        ].each do |name, normalized|
+          it "tokenizes #{name.inspect}" do
+            Normalizer.instance.normalize_names(name).should == normalized
+          end
+        end
+
+      end
+
+      describe 'editors extraction' do
+        it 'recognizes editors in the author field' do
+          n.normalize_author(:author => 'D. Knuth (ed.)').should == { :editor => 'Knuth, D.' }
+        end
+      end
+
+      describe 'date extraction' do
+        it 'extracts month and year from a string like "(July 2009)"' do
+          h = Normalizer.instance.normalize_date(:date => '(July 2009)')
+          h[:year].should == 2009
+          h[:month].should == 7
+          h.should_not have_key(:date)
+        end
+
+        it 'extracts month and year from a string like "(1997 Sept.)"' do
+          h = Normalizer.instance.normalize_date(:date => '(1997 Sept.)')
+          h[:year].should == 1997
+          h[:month].should == 9
+          h.should_not have_key(:date)
+
+          h = Normalizer.instance.normalize_date(:date => '(1997 Okt.)')
+          h[:year].should == 1997
+          h[:month].should == 10
+        end
+      end
+
+    end
+
+  end
+end

data/spec/anystyle/parser/parser_spec.rb

@@ -1,3 +1,5 @@
+# -*- encoding: utf-8 -*-
+
 module Anystyle::Parser
   describe Parser do
 
@@ -82,18 +84,16 @@ module Anystyle::Parser
           subject.label('').should == []
         end
 
-        it 'returns an empty array for an empty line' do
+        it 'returns an empty array for empty lines' do
           subject.label("\n").should == []
-          subject.label("\n ").should == [[],[]]
-          subject.label(" \n ").should == [[],[]]
-          subject.label(" \n").should == [[]]
+          subject.label("\n ").should == []
+          subject.label(" \n ").should == []
+          subject.label(" \n").should == []
         end
 
         it 'does not fail for unrecognizable input' do
           lambda { subject.label("@misc{70213094902020,\n") }.should_not raise_error
           lambda { subject.label("doi = {DOI:10.1503/jpn.100140}\n}\n") }.should_not raise_error
-
-          pending
           lambda { subject.label("\n doi ") }.should_not raise_error
         end
       end
@@ -117,8 +117,51 @@ module Anystyle::Parser
       it 'returns the label/token arrays for format "raw"' do
         subject.parse(citation, :raw)[0][0].should == [:author, 'Perec,']
       end
+
+      it 'returns the token in original order for format "raw"' do
+        subject.parse(citation, :raw)[0].map(&:last).join(' ').should == citation
+
+        difference = 'Derrida, J. (1967). L’écriture et la différence (1 éd.). Paris: Éditions du Seuil.'
+        subject.parse(difference, :raw)[0].map(&:last).join(' ').should == difference
+      end
     end
 
+    describe "#train" do
+      let(:dps) { File.open(fixture_path('train_dps.txt'), 'r:UTF-8').read.split(/\n/) }
+
+      describe "a pristine model" do
+        before(:each) { subject.train '', true }
+
+        it 'recognizes trained references' do
+          subject.learn dps[0]
+          subject.parse(strip_tags(dps[0]), :tags)[0].should == dps[0]
+        end
+
+        it 'recognizes trained references when learnt in one go' do
+          subject.learn dps
+
+          dps.each do |d|
+            subject.parse(strip_tags(d), :tags)[0].should == d
+          end
+        end
+
+        it 'recognizes trained references when learnt separately' do
+          pending
+
+          dps.each do |d|
+            subject.learn d
+          end
+
+          dps.each do |d|
+            subject.parse(strip_tags(d), :tags)[0].should == d
+          end
+        end
+      end
+
+      describe "the default model" do
+
+      end
+    end
 
   end
 end

data/spec/fixtures/train_dps.txt

@@ -0,0 +1,12 @@
+<author>Williams, J.B., J.D. Shorthouse and R.E. Lee, Jr.</author> <date>2002.</date> <title>Extreme resistance to desiccation and microclimate-related differences in cold-hardiness of gall wasps (Hymenoptera; Cynipidae) overwintering on roses in southern Canada.</title> <journal>The Journal of Experimental Biology</journal> <volume>205:</volume> <pages>2115–2124.</pages>
+<author>MACDONALD, S., & FENNIAK T.</author> <date>(2007).</date> <title>Understory plant communities of boreal mixedwood forests in western Canada: Natural patterns and response to variable-retention harvesting.</title> <journal>Forest Ecology and Management.</journal> <volume>242(1):</volume> <pages>34–48.</pages>
+<author>Harris, P. and J.D. Shorthouse.</author> <date>1996.</date> <title>Effectiveness of gall inducers in weed biological control.</title> <journal>The Canadian Entomologist</journal> <volume>128:</volume> <pages>1021–1055.</pages>
+<author>Williams, J.B., J.D. Shorthouse and R.E. Lee, Jr.</author> <date>2003.</date> <title>Deleterious effects of mild simulated overwintering temperatures on survival and potential fecundity of rose-galling Diplolepis wasps (Hymenoptera: Cynipidae).</title> <journal>Journal of Experimental Zoology</journal> <volume>298A:</volume> <pages>23–31.</pages>
+<author>Shorthouse, J.D., H. Goulet and D.P. Shorthouse.</author> <date>2003.</date> <title>Notes on cynipid galls, ground beetles and ground-dwelling spiders collected at Fort Severn, Ontario.</title> <journal>Arctic</journal> <volume>56:</volume> <pages>159–167.</pages>
+<author>Epling, C., Lewis H., & Ball F. M.</author> <date>(1960).</date> <title>The Breeding Group and Seed Storage: A Study in Population Dynamics.</title> <journal>Evolution.</journal> <volume>14,</volume> <pages>238-255.</pages>
+<author>Bagatto, Giuseppe, Louise C. Paquette, and Joseph D. Shorthouse.</author> <date>1995.</date> <title>Influence of galls of Phanacis taraxaci on carbon partitioning within common dandelion, Taraxacum officinale.</title> <journal>Entomologia Experimentalis et Applicata</journal> <volume>79:</volume> <pages>111–117.</pages>
+<author>Shorthouse, J.D.</author> <date>1993.</date> <title>Adaptations of gall wasps of the genus Diplolepis (Hymenoptera: Cynipidae) and the role of gall anatomy in cynipid systematics.</title> <journal>Memoirs of the Entomological Society of Canada</journal> <volume>165:</volume> <pages>139–163.</pages>
+<author>Bagatto, G., and J.D. Shorthouse.</author> <date>2000.</date> <title>Evaluation of municipal solid waste (MSW) compost as a soil amendment for acidic, metalliferous mine tailings.</title> <journal>International Journal of Surface Mining, Reclamation and Environment</journal> <volume>14:</volume> <pages>205–214.</pages>
+<author>Shorthouse, J.D., D. Wool, and A. Raman</author> <date>2005.</date> <title>Gall-inducing insects - nature's most sophisticated herbivores.</title> <journal>Basic and Applied Ecology</journal> <volume>6:</volume> <pages>407–411.</pages>
+<author>Sopow, S.L., J.D. Shorthouse, W. Strong, and D.T. Quiring.</author> <date>2003.</date> <title>Evidence for long-distance, chemical gall induction by an insect.</title> <journal>Ecology Letters</journal> <volume>6:</volume> <pages>102–105.</pages>
+<author>Shorthouse, J.D. and J.J. Leggo.</author> <date>2002.</date> <title>Immature stages of the galler Diplolepis triforma (Hymenoptera: Cynipidae) with comments on the role of its prepupa.</title> <journal>The Canadian Entomologist</journal> <volume>134</volume> <pages>433–446.</pages>

data/spec/spec_helper.rb

@@ -22,10 +22,22 @@ $LOAD_PATH.unshift(File.dirname(__FILE__))
 require 'rspec'
 require 'anystyle/parser'
 
-# Requires supporting files with custom matchers and macros, etc,
-# in ./support/ and its subdirectories.
-Dir["#{File.dirname(__FILE__)}/support/**/*.rb"].each {|f| require f}
+module Fixtures
+	PATH = File.expand_path('../fixtures', __FILE__)
+
+	Dir[File.join(PATH, '*.rb')].each do |fixture|
+		require fixture
+	end
+
+  def fixture_path(path)
+    File.join(PATH, path)
+  end
+end
 
 RSpec.configure do |config|
+  config.include Fixtures
 
+  def strip_tags(string)
+    string.gsub(/<[^>]+>/, '')
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: anystyle-parser
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Sylvester Keil
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-02-07 00:00:00.000000000 Z
+date: 2014-02-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bibtex-ruby
@@ -30,14 +30,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.0'
+        version: '0.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.0'
+        version: '0.1'
 - !ruby/object:Gem::Dependency
   name: namae
   requirement: !ruby/object:Gem::Requirement
@@ -87,6 +87,7 @@ files:
 - spec/anystyle/parser/normalizer_spec.rb
 - spec/anystyle/parser/parser_spec.rb
 - spec/benchmark.rb
+- spec/fixtures/train_dps.txt
 - spec/profile.rb
 - spec/spec_helper.rb
 homepage: http://github.com/inukshuk/anystyle-parser
@@ -115,7 +116,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.1
+rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
 summary: Smart and fast academic bibliography parser.
@@ -127,6 +128,7 @@ test_files:
 - spec/anystyle/parser/normalizer_spec.rb
 - spec/anystyle/parser/parser_spec.rb
 - spec/benchmark.rb
+- spec/fixtures/train_dps.txt
 - spec/profile.rb
 - spec/spec_helper.rb
 has_rdoc: