anystyle-parser 0.0.1

data/lib/anystyle/parser/support/anystyle.pat

@@ -0,0 +1,72 @@
+# Feature numbers
+# 0       1 2 3  4   5    6 7  8   9    10      11      12   13 14      15        16      17       18    19           20         21   22 23     24
+# England a E En Eng Engl d nd and land england initial none 20 no-male no-female surname no-month place no-publisher no-journal none 0  others other
+
+u:%x[-3,0]
+u:%x[-2,0]
+u:%x[-1,0]
+u:%x[0,0]
+u:%x[1,0]
+u:%x[2,0]
+u:%x[3,0]
+u:%x[-1,0]/%x[0,0]
+u:%x[0,0]/%x[1,0]
+
+# last character type
+u:%x[0,1]
+u:%x[-1,1]
+
+# first 1-4 characters
+u:%x[0,2]
+u:%x[0,3]
+u:%x[0,4]
+u:%x[0,5]
+
+# last 1-4 characters
+u:%x[0,6]
+u:%x[0,7]
+u:%x[0,8]
+u:%x[0,9]
+
+# no punctuation lower-case
+u:%x[-2,10]
+u:%x[-1,10]
+u:%x[0,10]
+u:%x[1,10]
+u:%x[2,10]
+
+# capitalization
+u:%x[0,11]
+
+# numbers
+u:%x[-1,12]
+u:%x[0,12]
+u:%x[1,12]
+u:%x[-1,12]/%x[0,12]
+u:%x[0,12]/%x[1,12]
+
+# dictionary
+u:%x[0,13]
+u:%x[0,14]
+u:%x[0,15]
+u:%x[0,16]
+u:%x[0,17]
+u:%x[0,18]
+u:%x[0,19]
+u:%x[0,20]
+
+# possible editor
+u:%x[0,21]
+
+# position
+u:%x[0,22]
+
+# punctuation
+u:%x[0,23]
+
+# possible chapter
+u:%x[-1,23]/%x[0,24]/%x[0,21]
+u:%x[-1,23]/%x[0,24]/%x[1,11]
+
+# bigram
+b

data/lib/anystyle/parser/utility.rb

@@ -0,0 +1,19 @@
+module Anystyle
+	
+	def self.parse(*arguments)
+		Parser::Parser.instance.parse(*arguments)
+	end
+	
+	def self.parser
+		Parser::Parser.instance
+	end
+	
+	module Parser
+		
+		def self.instance
+			Parser.instance
+		end
+		
+	end
+	
+end

data/lib/anystyle/parser/version.rb

@@ -0,0 +1,5 @@
+module Anystyle
+  module Parser
+    VERSION = '0.0.1'.freeze
+  end
+end

data/lib/anystyle/parser.rb

@@ -0,0 +1,17 @@
+
+require 'singleton'
+
+require 'bibtex'
+require 'wapiti'
+
+# require 'ruby-debug'
+# Debugger.start
+
+require 'anystyle/parser/errors'
+
+require 'anystyle/parser/dictionary'
+require 'anystyle/parser/features'
+require 'anystyle/parser/parser'
+require 'anystyle/parser/normalizer'
+
+require 'anystyle/parser/utility'

data/spec/anystyle/parser/dictionary_spec.rb

@@ -0,0 +1,31 @@
+# -*- encoding: utf-8 -*-
+
+module Anystyle
+	module Parser
+
+		describe "Dictionary" do
+
+			let(:dict) { Dictionary.instance }
+
+			it { Dictionary.should_not respond_to(:new) }		
+			it { dict.should_not be nil }
+
+
+			describe "the dictionary" do
+
+				%w{ philippines italy }.each do |place|
+					it "#{place.inspect} should be a place name" do
+						dict[place].should == Dictionary.code[:place]
+					end
+				end
+
+				it "accepts unicode strins like 'çela' (surname)" do
+					(dict['çela'] & Dictionary.code[:surname]).should > 0
+				end
+
+			end
+
+		end
+
+	end
+end

data/spec/anystyle/parser/features_spec.rb

@@ -0,0 +1,24 @@
+# -*- encoding: utf-8 -*-
+
+module Anystyle::Parser
+	describe "Features" do
+	  
+		describe "numbers" do
+			let(:f) { Parser.feature[:numbers] }
+			
+			%w{ (1992) 1992 2011 1776 }.each do |year|
+				it "returns :year for #{year.inspect}" do
+					f.match(year).should == :year
+				end
+			end
+			
+			%w{ (1) (12) (123) }.each do |year|
+				it "returns :year for #{year.inspect}" do
+					f.match(year).should == :numeric
+				end
+			end
+			
+		end
+						
+	end
+end

data/spec/anystyle/parser/normalizer_spec.rb

@@ -0,0 +1,36 @@
+module Anystyle
+	module Parser
+
+		describe "Normalizer" do
+
+			describe "#tokenize_names" do
+
+				it "tokenizes 'A B'" do
+					Normalizer.instance.tokenize_names('A B').should == ['A B']
+				end
+
+				it "tokenizes 'A, B'" do
+					Normalizer.instance.tokenize_names('A, B').should == ['A, B']
+				end
+
+				it "tokenizes 'A, jr., B'" do
+					Normalizer.instance.tokenize_names('A, jr., B').should == ['A, jr., B']
+				end
+
+				it "tokenizes 'A, B, jr.'" do
+					Normalizer.instance.tokenize_names('A, B, jr.').should == ['A, B, jr.']
+				end
+
+				it "tokenizes 'A, B, C, D'" do
+					Normalizer.instance.tokenize_names('A, B, C, D').should == ['A, B', ' C, D']
+				end
+
+				it "tokenizes 'A, B, C'" do
+					Normalizer.instance.tokenize_names('A, B, C').should == ['A, B', ' C']
+				end
+
+			end
+		end
+
+	end
+end

data/spec/anystyle/parser/parser_spec.rb

@@ -0,0 +1,85 @@
+module Anystyle::Parser
+  describe Parser do
+    
+    it { should_not be nil }
+    
+		describe "#tokenize" do
+			it "returns [] when given an empty string" do
+				subject.tokenize('').should == []
+			end
+			
+		  it "takes a single line and returns an array of token sequences" do
+				subject.tokenize('hello, world!').should == [%w{ hello, world! }]
+			end
+			
+		  it "takes two lines and returns an array of token sequences" do
+				subject.tokenize("hello, world!\ngoodbye!").should == [%w{ hello, world! }, %w{ goodbye! }]
+			end
+			
+			context "when passing a string marked as tagged" do
+				it "returns [] when given an empty string" do
+					subject.tokenize('', true).should == []
+				end
+			
+				it "returns an array of :unknown token sequences when given an untagged single line" do
+					subject.tokenize('hello, world!', true).should == [[['hello,', :unknown], ['world!', :unknown]]]
+				end
+
+				it "returns an array of :unknown token sequences when given two untagged lines" do
+					subject.tokenize("hello,\nworld!", true).should == [[['hello,', :unknown]], [['world!', :unknown]]]
+				end
+
+				it "returns an array of token/tag pair for each line when given a single tagged string" do
+					subject.tokenize('<a>hello</a>', true).should == [[['hello', :a]]]
+				end
+			
+				it "returns an array of token/tag pair for each line when given a string with multiple tags" do
+					subject.tokenize('<a>hello world</a> <b> !</b>', true).should == [[['hello',:a], ['world', :a], ['!', :b]]]
+				end
+			
+				it "raises an argument error if the string contains mismatched tags" do
+					expect { subject.tokenize('<a> hello </b>', true) }.to raise_error(ArgumentError)
+					expect { subject.tokenize('<a> hello <b> world </a>', true) }.to raise_error(ArgumentError)
+				end
+			end
+			
+		end
+		
+		describe "#prepare" do
+			it 'returns an array of expanded token sequences' do
+				subject.prepare('hello, world!').should == [['hello, , h he hel hell , o, lo, llo, hello other none 0 no-male no-female no-surname no-month no-place no-publisher no-journal no-editors 0 internal other', 'world! ! w wo wor worl ! d! ld! rld! world other none 36 no-male no-female surname no-month no-place publisher no-journal no-editors 5 terminal other']]
+			end
+			
+			context 'when marking the input as being tagged' do
+				let(:input) { %{<author> A. Cau, R. Kuiper, and W.-P. de Roever. </author> <title> Formalising Dijkstra's development strategy within Stark's formalism. </title> <editor> In C. B. Jones, R. C. Shaw, and T. Denvir, editors, </editor> <booktitle> Proc. 5th. BCS-FACS Refinement Workshop, </booktitle> <date> 1992. </date>} }
+
+				it 'returns an array of expaned and labelled token sequences for a tagged string' do
+					subject.prepare(input, true)[0].map { |t| t[/\S+$/] }.should == %w{ author author author author author author author author title title title title title title title editor editor editor editor editor editor editor editor editor editor editor booktitle booktitle booktitle booktitle booktitle date }
+				end
+
+				it 'returns an array of expanded and labelled :unknown token sequences for an untagged input' do
+					subject.prepare('hello, world!', true)[0].map { |t| t[/\S+$/] }.should == %w{ unknown unknown }
+				end
+				
+			end
+		end
+
+		describe "#label" do
+			let(:citation) { 'Perec, Georges. A Void. London: The Harvill Press, 1995. p.108.' }
+			
+			it 'returns an array of labelled segments' do
+				subject.label(citation)[0].map(&:first).should == [:author, :title, :location, :publisher, :date, :pages]
+			end
+		end
+
+		describe "#parse" do
+			let(:citation) { 'Perec, Georges. A Void. London: The Harvill Press, 1995. p.108.' }
+			
+			it 'returns a hash of label/segment pairs by default' do
+				subject.parse(citation)[0].should == { :author => 'Perec, Georges', :title => 'A Void', :location => 'London', :publisher => 'The Harvill Press', :year => 1995, :pages => '108', :type => :book }
+			end
+		end
+
+		
+  end
+end

data/spec/benchmark.rb

@@ -0,0 +1,74 @@
+require 'anystyle/parser'
+
+require 'benchmark'
+include Benchmark
+
+data = <<-END_REFERENCES
+<author> A. Cau, R. Kuiper, and W.-P. de Roever. </author> <title> Formalising Dijkstra's development strategy within Stark's formalism. </title> <editor> In C. B. Jones, R. C. Shaw, and T. Denvir, editors, </editor> <booktitle> Proc. 5th. BCS-FACS Refinement Workshop, </booktitle> <date> 1992. </date>
+<author> M. Kitsuregawa, H. Tanaka, and T. Moto-oka. </author> <title> Application of hash to data base machine and its architecture. </title> <journal> New Generation Computing, </journal> <volume> 1(1), </volume> <date> 1983. </date>
+<author> Alexander Vrchoticky. </author> <title> Modula/R language definition. </title> <tech> Technical Report TU Wien rr-02-92, version 2.0, </tech> <institution> Dept. for Real-Time Systems, Technical University of Vienna, </institution> <date> May 1993. </date>
+<author> Marc Shapiro and Susan Horwitz. </author> <title> Fast and accurate flow-insensitive points-to analysis. </title> <booktitle> In Proceedings of the 24th Annual ACM Symposium on Principles of Programming Languages, </booktitle> <date> January 1997. </date>
+<author> W. Landi and B. G. Ryder. </author> <title> Aliasing with and without pointers: A problem taxonomy. </title> <institution> Center for Computer Aids for Industrial Productivity </institution> <tech> Technical Report CAIP-TR-125, </tech> <institution> Rutgers University, </institution> <date> September 1990. </date>
+<author> W. H. Enright. </author> <title> Improving the efficiency of matrix operations in the numerical solution of stiff ordinary differential equations. </title> <journal> ACM Trans. Math. Softw., </journal> <volume> 4(2), </volume> <pages> 127-136, </pages> <date> June 1978. </date>
+<author> Gmytrasiewicz, P. J., Durfee, E. H., & Wehe, D. K. </author> <date> (1991a). </date> <title> A decision theoretic approach to coordinating multiagent interaction. </title> <booktitle> In Proceedings of the Twelfth International Joint Conference on Artificial Intelligence, </booktitle> <pages> pp. 62-68 </pages> <location> Sydney, Australia. </location>
+<author> A. Bookstein and S. T. Klein, </author> <title> Detecting content-bearing words by serial clustering, </title> <booktitle> Proceedings of the Nineteenth Annual International ACM SIGIR Conference on Research and Development in Information Retrieval, </booktitle> <pages> pp. 319327, </pages> <date> 1995. </date>
+<author> U. Dayal, H. Garcia-Molina, M. Hsu, B. Kao, and M.- C. Shan. </author> <title> Third generation TP monitors: A database challenge. </title> <booktitle> In ACM SIGMOD Conference on Management of Data, </booktitle> <pages> pages 393-397, </pages> <location> Washington, D. C., </location> <date> May 1993. </date>
+<author> C. Qiao and R. Melhem, </author> <title> "Reducing Communication Latency with Path Multiplexing in Optically Interconnected Multiprocessor Systems", </title> <booktitle> Proc. of HPCA-1, </booktitle> <date> 1995. </date>
+END_REFERENCES
+
+data = data * 100
+
+data = data.split("\n")
+
+Anystyle::Parser::Feature.load_dictionary
+parser = Anystyle::Parser::Parser.instance
+
+n, k = 100, 5
+
+f = []
+g = []
+
+Benchmark.benchmark((" "*15) + CAPTION, 7, FMTSTR, '%14s:' % 'sum(f)', '%14s:' % 'sum(g)') do |b|
+	1.step(n,k) do |i|
+		
+		input = data[0,i]
+		f << b.report('%14s:' % "f(#{i})") do
+			input.each { |line| parser.prepare(line, true) }
+		end
+		
+		input = input.join("\n")
+		g << b.report('%14s:' % "g(#{i})") do
+			parser.prepare(input, true)
+		end
+		
+		[f.reduce(:+), g.reduce(:+)]
+	end
+	
+end
+
+require 'gnuplot'
+
+f = f.map(&:total)
+g = g.map(&:total)
+
+x = 1.step(n,k).to_a
+
+Gnuplot.open do |gp|
+	Gnuplot::Plot.new(gp) do |plot|
+		plot.title 'Anystyle Parser Benchmark'
+		plot.ylabel 't'
+		plot.xlabel 'n'
+		
+		plot.data << Gnuplot::DataSet.new([x,f]) do |ds|
+      ds.with = 'linespoints'
+      ds.title = 'f'
+    end
+
+    plot.data << Gnuplot::DataSet.new([x,g]) do |ds|
+      ds.with = 'linespoints'
+      ds.title = 'g'
+    end
+
+	end
+end
+

data/spec/profile.rb

@@ -0,0 +1,34 @@
+require 'anystyle/parser'
+
+require 'tempfile'
+require 'ruby-prof'
+
+data = <<-END_REFERENCES
+<author> A. Cau, R. Kuiper, and W.-P. de Roever. </author> <title> Formalising Dijkstra's development strategy within Stark's formalism. </title> <editor> In C. B. Jones, R. C. Shaw, and T. Denvir, editors, </editor> <booktitle> Proc. 5th. BCS-FACS Refinement Workshop, </booktitle> <date> 1992. </date>
+<author> M. Kitsuregawa, H. Tanaka, and T. Moto-oka. </author> <title> Application of hash to data base machine and its architecture. </title> <journal> New Generation Computing, </journal> <volume> 1(1), </volume> <date> 1983. </date>
+<author> Alexander Vrchoticky. </author> <title> Modula/R language definition. </title> <tech> Technical Report TU Wien rr-02-92, version 2.0, </tech> <institution> Dept. for Real-Time Systems, Technical University of Vienna, </institution> <date> May 1993. </date>
+<author> Marc Shapiro and Susan Horwitz. </author> <title> Fast and accurate flow-insensitive points-to analysis. </title> <booktitle> In Proceedings of the 24th Annual ACM Symposium on Principles of Programming Languages, </booktitle> <date> January 1997. </date>
+<author> W. Landi and B. G. Ryder. </author> <title> Aliasing with and without pointers: A problem taxonomy. </title> <institution> Center for Computer Aids for Industrial Productivity </institution> <tech> Technical Report CAIP-TR-125, </tech> <institution> Rutgers University, </institution> <date> September 1990. </date>
+<author> W. H. Enright. </author> <title> Improving the efficiency of matrix operations in the numerical solution of stiff ordinary differential equations. </title> <journal> ACM Trans. Math. Softw., </journal> <volume> 4(2), </volume> <pages> 127-136, </pages> <date> June 1978. </date>
+<author> Gmytrasiewicz, P. J., Durfee, E. H., & Wehe, D. K. </author> <date> (1991a). </date> <title> A decision theoretic approach to coordinating multiagent interaction. </title> <booktitle> In Proceedings of the Twelfth International Joint Conference on Artificial Intelligence, </booktitle> <pages> pp. 62-68 </pages> <location> Sydney, Australia. </location>
+<author> A. Bookstein and S. T. Klein, </author> <title> Detecting content-bearing words by serial clustering, </title> <booktitle> Proceedings of the Nineteenth Annual International ACM SIGIR Conference on Research and Development in Information Retrieval, </booktitle> <pages> pp. 319327, </pages> <date> 1995. </date>
+<author> U. Dayal, H. Garcia-Molina, M. Hsu, B. Kao, and M.- C. Shan. </author> <title> Third generation TP monitors: A database challenge. </title> <booktitle> In ACM SIGMOD Conference on Management of Data, </booktitle> <pages> pages 393-397, </pages> <location> Washington, D. C., </location> <date> May 1993. </date>
+<author> C. Qiao and R. Melhem, </author> <title> "Reducing Communication Latency with Path Multiplexing in Optically Interconnected Multiprocessor Systems", </title> <booktitle> Proc. of HPCA-1, </booktitle> <date> 1995. </date>
+END_REFERENCES
+
+
+Anystyle::Parser::Feature.load_dictionary
+parser = Anystyle::Parser::Parser.instance
+
+
+result = RubyProf.profile do
+	parser.prepare(data, true)
+end
+
+dot = Tempfile.new('dot')
+RubyProf::DotPrinter.new(result).print(dot, :min_percent => 5)
+dot.close
+
+system "dot -Tpng -oprofile.png #{dot.path}"
+
+# dot.unlink

data/spec/spec_helper.rb

@@ -0,0 +1 @@
+require 'anystyle/parser'

metadata

@@ -0,0 +1,169 @@
+--- !ruby/object:Gem::Specification
+name: anystyle-parser
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Sylvester Keil
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2011-09-05 00:00:00.000000000Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bibtex-ruby
+  requirement: &2152150220 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: *2152150220
+- !ruby/object:Gem::Dependency
+  name: wapiti
+  requirement: &2152110260 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *2152110260
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: &2152107240 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.9'
+  type: :development
+  prerelease: false
+  version_requirements: *2152107240
+- !ruby/object:Gem::Dependency
+  name: racc
+  requirement: &2152096280 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.4'
+  type: :development
+  prerelease: false
+  version_requirements: *2152096280
+- !ruby/object:Gem::Dependency
+  name: cucumber
+  requirement: &2152093880 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :development
+  prerelease: false
+  version_requirements: *2152093880
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: &2152091560 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.6'
+  type: :development
+  prerelease: false
+  version_requirements: *2152091560
+- !ruby/object:Gem::Dependency
+  name: ZenTest
+  requirement: &2152088380 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '4.6'
+  type: :development
+  prerelease: false
+  version_requirements: *2152088380
+description: A sophisticated parser for academic references based on conditional random
+  fields.
+email:
+- http://sylvester.keil.or.at
+executables: []
+extensions: []
+extra_rdoc_files:
+- README.md
+- LICENSE
+files:
+- .autotest
+- .gitignore
+- .rspec
+- Gemfile
+- HISTORY.md
+- LICENSE
+- README.md
+- anystyle-parser.gemspec
+- features/step_definitions/parser_steps.rb
+- features/support/env.rb
+- lib/anystyle/parser.rb
+- lib/anystyle/parser/dictionary.rb
+- lib/anystyle/parser/errors.rb
+- lib/anystyle/parser/features.rb
+- lib/anystyle/parser/normalizer.rb
+- lib/anystyle/parser/parser.rb
+- lib/anystyle/parser/support/anystyle.mod
+- lib/anystyle/parser/support/anystyle.pat
+- lib/anystyle/parser/support/dict.txt.gz
+- lib/anystyle/parser/utility.rb
+- lib/anystyle/parser/version.rb
+- spec/anystyle/parser/dictionary_spec.rb
+- spec/anystyle/parser/features_spec.rb
+- spec/anystyle/parser/normalizer_spec.rb
+- spec/anystyle/parser/parser_spec.rb
+- spec/benchmark.rb
+- spec/profile.rb
+- spec/spec_helper.rb
+homepage: http://inukshuk.github.com/anystyle-parser
+licenses:
+- FreeBSD
+post_install_message: 
+rdoc_options:
+- --line-numbers
+- --inline-source
+- --title
+- ! '"Anystyle Parser"'
+- --main
+- README.md
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.10
+signing_key: 
+specification_version: 3
+summary: Parser for academic references.
+test_files:
+- features/step_definitions/parser_steps.rb
+- features/support/env.rb
+- spec/anystyle/parser/dictionary_spec.rb
+- spec/anystyle/parser/features_spec.rb
+- spec/anystyle/parser/normalizer_spec.rb
+- spec/anystyle/parser/parser_spec.rb
+- spec/benchmark.rb
+- spec/profile.rb
+- spec/spec_helper.rb