anystyle-parser 0.1.1 → 0.2.0

data/Gemfile

@@ -1,14 +1,19 @@
 source :rubygems
 gemspec
 
-group :debug do
-	gem 'ruby-debug19', :require => 'ruby-debug', :platforms => [:mri_19]
-	gem 'ruby-debug', :platforms => [:mri_18, :jruby]
-	gem 'rbx-trepanning', :platforms => [:rbx]
+group :development do
+	gem 'debugger', :platforms => [:mri_19]	
+	gem 'simplecov'
+	gem 'yard'
 end
 
-group :osx_test do
-	gem 'autotest-fsevent', :require => false  
+group :test  do
+	gem 'rake'
+	gem 'racc', '~>1.4'
+
+	gem 'cucumber'
+	gem 'rspec'
+	gem 'ZenTest'
 end
 
 group :profile do
@@ -16,6 +21,7 @@ group :profile do
 	gem 'gnuplot'
 end
 
-group :kyotocabinet do
+group :extra do
 	gem 'kyotocabinet-ruby', :require => 'kyotocabinet'
+	gem 'autotest-fsevent', :require => false  
 end

data/HISTORY.md

@@ -1,3 +1,7 @@
+0.2.0 / 2012-10-29
+==================
+* Use Namae for name normalization
+
 0.1.1 / 2012-03-29
 ==================
 * Bugfix

data/README.md

@@ -5,7 +5,7 @@ Anystyle-Parser is a very fast and smart parser for academic references. It
 is inspired by [ParsCit](http://aye.comp.nus.edu.sg/parsCit/) and
 [FreeCite](http://freecite.library.brown.edu/); Anystyle-Parser uses machine
 learning algorithms and is designed
-for raw speed (it [wapiti](https://github.com/inukshuk/wapiti-ruby) based
+for raw speed (it uses [wapiti](https://github.com/inukshuk/wapiti-ruby) based
 conditional random fields and [Kyoto Cabinet](http://fallabs.com/kyotocabinet/)
 or [Redis](http://redis.io) as a key-value store), flexibility (it is easy to
 train the model with data that is relevant to your parsing needs), and

data/Rakefile

@@ -1,10 +1,19 @@
-lib = File.expand_path('../lib/', __FILE__)
-$:.unshift lib unless $:.include?(lib)
-
+require 'bundler'
+begin
+  Bundler.setup(:default, :development, :debug, :test, :extra, :profile)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'rake'
 require 'rake/clean'
 
+$:.unshift(File.join(File.dirname(__FILE__), './lib'))
+
 require 'anystyle/parser/version'
 
+task :default
 task :build => [:clean] do
   system 'gem build anystyle-parser.gemspec'
 end
@@ -14,5 +23,34 @@ task :release => [:build] do
   system "gem push anystyle-parser-#{Anystyle::Parser::VERSION}.gem"
 end
 
+require 'rspec/core'
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new(:spec) do |spec|
+  spec.pattern = FileList['spec/**/*_spec.rb']
+end
+
+require 'cucumber/rake/task'
+Cucumber::Rake::Task.new(:features)
+
+task :default => :spec
+
+begin
+  require 'yard'
+  YARD::Rake::YardocTask.new
+rescue LoadError
+  # ignore
+end
+
+desc 'Run an IRB session with CSL loaded'
+task :console, [:script] do |t, args|
+  ARGV.clear
+
+  require 'irb'
+  require 'anystyle/parser'
+  
+  IRB.conf[:SCRIPT] = args.script
+  IRB.start
+end
+
 CLEAN.include('*.gem')
 CLEAN.include('*.rbc')

data/anystyle-parser.gemspec

@@ -17,12 +17,7 @@ Gem::Specification.new do |s|
   
   s.add_runtime_dependency('bibtex-ruby', '~>2.0')
   s.add_runtime_dependency('wapiti', '~>0.0')
-  
-  s.add_development_dependency('rake', ['~>0.9'])
-  s.add_development_dependency('racc', ['~>1.4'])
-  s.add_development_dependency('cucumber', ['~>1.0'])
-  s.add_development_dependency('rspec', ['~>2.6'])
-  s.add_development_dependency('ZenTest', ['~>4.6'])
+  s.add_runtime_dependency('namae', '~>0.7')
 
   s.files        = `git ls-files`.split("\n") - Dir['resources/**/*']
   s.test_files   = `git ls-files -- {test,spec,features}/*`.split("\n")

data/cucumber.yml

@@ -0,0 +1 @@
+default: --format progress --require features --color

data/lib/anystyle/parser/normalizer.rb

@@ -129,47 +129,15 @@ module Anystyle
 				hash
 			end
 			
+			Namae::Parser.instance.options[:prefer_comma_as_separator] = true
+
 			def normalize_names(names)
-				names = tokenize_names(names).map do |name|
-					name.strip!
-					name.gsub!(/\b([[:upper:]]{2,3})\b/) { $1.split(//).join(' ') }
-					name.gsub!(/\b([[:upper:]])(\s|$)/) { [$1, $2 == ?. ? nil : ?., $2].compact.join }
-					name
-				end
-				names.join(' and ')
+				Namae.parse!(names).map(&:sort_order).join(' and ')
 			rescue => e
 				warn e.message
 				hash
 			end
-			
-			def tokenize_names(names)
-				s, n, ns, cc = StringScanner.new(names), '', [], 0
-				until s.eos?
-					case
-					when s.scan(/,?\s*(and\b|&|;)/)
-						ns << n
-						n, cc = '', 0
-					when s.scan(/\s+/)
-						n << ' '
-					when s.scan(/,?\s*(jr|sr|ph\.?d|m\.?d|esq)\.?/i)
-						n << s.matched
-					when s.scan(/,/)
-						if cc > 0 || (n =~ /\S{2,}\s+\S{2,}/ && s.rest !~ /^\s*[[:alpha:]]+(\.|,|$)/)
-							ns << n
-							n, cc = '', 0							
-						else
-							n << s.matched
-							cc += 1
-						end
-					when s.scan(/[[:alpha:]]+/)
-						n << s.matched
-					when  s.scan(/./)
-						n << s.matched
-					end
-				end
-				ns << n		
-			end
-			
+						
 			def normalize_title(hash)
 				title, container = hash[:title]
 				

data/lib/anystyle/parser/version.rb

@@ -1,5 +1,5 @@
 module Anystyle
   module Parser
-    VERSION = '0.1.1'.freeze
+    VERSION = '0.2.0'.freeze
   end
 end

data/lib/anystyle/parser.rb

@@ -3,9 +3,7 @@ require 'singleton'
 
 require 'bibtex'
 require 'wapiti'
-
-# require 'ruby-debug'
-# Debugger.start
+require 'namae'
 
 require 'anystyle/parser/errors'
 

data/spec/anystyle/parser/normalizer_spec.rb

@@ -6,46 +6,46 @@ module Anystyle
 			describe "#tokenize_names" do
 
 				it "tokenizes 'A B'" do
-					Normalizer.instance.tokenize_names('A B').should == ['A B']
+					Normalizer.instance.normalize_names('A B').should == 'B, A'
 				end
 
 				it "tokenizes 'A, B'" do
-					Normalizer.instance.tokenize_names('A, B').should == ['A, B']
+					Normalizer.instance.normalize_names('A, B').should == 'A, B'
 				end
 
-				it "tokenizes 'A, jr., B'" do
-					Normalizer.instance.tokenize_names('A, jr., B').should == ['A, jr., B']
-				end
-
-				it "tokenizes 'A, B, jr.'" do
-					Normalizer.instance.tokenize_names('A, B, jr.').should == ['A, B, jr.']
-				end
+				# it "tokenizes 'A, jr., B'" do
+				# 	Normalizer.instance.normalize_names('A, jr., B').should == 'A, jr., B'
+				# end
+				# 
+				# it "tokenizes 'A, B, jr.'" do
+				# 	Normalizer.instance.normalize_names('A, B, jr.').should == 'A, B, jr.'
+				# end
 
 				it "tokenizes 'A, B, C, D'" do
-					Normalizer.instance.tokenize_names('A, B, C, D').should == ['A, B', ' C, D']
+					Normalizer.instance.normalize_names('A, B, C, D').should == 'A, B and C, D'
 				end
 
 				it "tokenizes 'A, B, C'" do
-					Normalizer.instance.tokenize_names('A, B, C').should == ['A, B', ' C']
+					Normalizer.instance.normalize_names('A, B, C').should == 'A, B and C'
 				end
 
 				it "tokenizes 'Aa Bb, C.'" do
-					Normalizer.instance.tokenize_names('Aa Bb, C.').should == ['Aa Bb, C.']
+					Normalizer.instance.normalize_names('Aa Bb, C.').should == 'Aa Bb, C.'
 				end
 				
 				it "tokenizes 'Aa Bb, Cc Dd, and E F G'" do
-					Normalizer.instance.tokenize_names('Aa Bb, C D, and E F G').should == ['Aa Bb', ' C D', ' E F G']
+					Normalizer.instance.normalize_names('Aa Bb, C D, and E F G').should == 'Bb, Aa and D, C and G, E F'
 				end
 				
 				[
-					['Poe, Edgar A.', ['Poe, Edgar A.']],
-					['Edgar A. Poe', ['Edgar A. Poe']],
-					['Edgar A. Poe, Herman Melville', ['Edgar A. Poe', ' Herman Melville']],
-					['Poe, Edgar A., Melville, Herman', ['Poe, Edgar A.', ' Melville, Herman']],
-					['Aeschlimann Magnin, E.', ['Aeschlimann Magnin, E.']]
-				].each do |name, tokens|
+					['Poe, Edgar A.', 'Poe, Edgar A.'],
+					['Edgar A. Poe', 'Poe, Edgar A.'],
+					['Edgar A. Poe, Herman Melville', 'Poe, Edgar A. and Melville, Herman'],
+					['Poe, Edgar A., Melville, Herman', 'Poe, Edgar A. and Melville, Herman'],
+					['Aeschlimann Magnin, E.', 'Aeschlimann Magnin, E.']
+				].each do |name, normalized|
 					it "tokenizes #{name.inspect}" do
-						Normalizer.instance.tokenize_names(name).should == tokens
+						Normalizer.instance.normalize_names(name).should == normalized
 					end
 				end
 				

data/spec/spec_helper.rb

@@ -1 +1,24 @@
+begin
+  require 'simplecov'
+rescue LoadError
+  # ignore
+end
+
+begin
+  require 'debugger'
+rescue LoadError
+  # ignore
+end
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'rspec'
 require 'anystyle/parser'
+
+# Requires supporting files with custom matchers and macros, etc,
+# in ./support/ and its subdirectories.
+Dir["#{File.dirname(__FILE__)}/support/**/*.rb"].each {|f| require f}
+
+RSpec.configure do |config|
+  
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: anystyle-parser
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-03-29 00:00:00.000000000 Z
+date: 2012-10-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bibtex-ruby
-  requirement: &70180181117220 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,73 +21,44 @@ dependencies:
         version: '2.0'
   type: :runtime
   prerelease: false
-  version_requirements: *70180181117220
-- !ruby/object:Gem::Dependency
-  name: wapiti
-  requirement: &70180181145980 !ruby/object:Gem::Requirement
+  version_requirements: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '0.0'
-  type: :runtime
-  prerelease: false
-  version_requirements: *70180181145980
-- !ruby/object:Gem::Dependency
-  name: rake
-  requirement: &70180181145180 !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: '0.9'
-  type: :development
-  prerelease: false
-  version_requirements: *70180181145180
+        version: '2.0'
 - !ruby/object:Gem::Dependency
-  name: racc
-  requirement: &70180181144260 !ruby/object:Gem::Requirement
+  name: wapiti
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '1.4'
-  type: :development
+        version: '0.0'
+  type: :runtime
   prerelease: false
-  version_requirements: *70180181144260
-- !ruby/object:Gem::Dependency
-  name: cucumber
-  requirement: &70180181143520 !ruby/object:Gem::Requirement
+  version_requirements: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '1.0'
-  type: :development
-  prerelease: false
-  version_requirements: *70180181143520
+        version: '0.0'
 - !ruby/object:Gem::Dependency
-  name: rspec
-  requirement: &70180181142760 !ruby/object:Gem::Requirement
+  name: namae
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '2.6'
-  type: :development
+        version: '0.7'
+  type: :runtime
   prerelease: false
-  version_requirements: *70180181142760
-- !ruby/object:Gem::Dependency
-  name: ZenTest
-  requirement: &70180181142000 !ruby/object:Gem::Requirement
+  version_requirements: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '4.6'
-  type: :development
-  prerelease: false
-  version_requirements: *70180181142000
+        version: '0.7'
 description: A sophisticated parser for academic references based on machine learning
   algorithms using conditional random fields.
 email:
@@ -107,6 +78,7 @@ files:
 - README.md
 - Rakefile
 - anystyle-parser.gemspec
+- cucumber.yml
 - features/step_definitions/parser_steps.rb
 - features/support/env.rb
 - lib/anystyle/parser.rb
@@ -148,7 +120,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -3694608752673104144
+      hash: -4454145497019098220
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -157,10 +129,10 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -3694608752673104144
+      hash: -4454145497019098220
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.10
+rubygems_version: 1.8.24
 signing_key: 
 specification_version: 3
 summary: Parser for academic references.
@@ -174,3 +146,4 @@ test_files:
 - spec/benchmark.rb
 - spec/profile.rb
 - spec/spec_helper.rb
+has_rdoc: