wapiti 0.0.5 → 0.1.0

data/lib/wapiti/options.rb

@@ -1,125 +1,124 @@
 module Wapiti
-	
-	class Options
-		
-		include Comparable
-		
-		class << self
-				
-			# Returns a sorted list of available option attributes.
-			def attribute_names
-				@attribute_names ||= %w{ stop_window convergence_window posterior
-					max_iterations jobsize threads rho1 rho2 stop_epsilon score check
-					algorithm pattern development_data maxent compact sparse skip_tokens
-					compress }.sort.map(&:to_sym).freeze
-			end
-			
-			# Returns the default options.
-			def defaults
-				@defaults ||= new.attributes
-			end
-
-			# Returns the list of supported algorithm options.
-			def algorithms
-				@algorithms ||= %w{ l-bfgs sgd-l1 bcd rprop rprop+ rprop- auto }.freeze
-			end
-			
-		end
-		
-		attr_accessor :compress
-		
-		alias compress? compress
-		
-		# Returns the value of the attribute identified by +name+ or nil
-		# if there is no such attribute.
-		def [](name)
-			has_attribute?(name) ? send(name) : nil
-		end
-		
-		# Updates the value of the attribute identified by +name+ with the
-		# passed-in +value+.
-		def []=(name, value)
-			raise ArgumentError, "bad attribute name: #{name}" unless has_attribute?(name)
-			send("#{name}=", value)
-		end
-		
-		# Updates all the attributes from the passed-in hash.
-		def update(attributes = {})
-			attributes.each_pair do |k,v|
-				mid = "#{k}="
-				send(mid, v) if respond_to?(mid)
-			end	
-			self
-		end
-		
-		alias update_attributes update
-		
-		def lbfgs
-			{ :clip => clip, :histsz => histsz, :maxls => maxls }
-		end
-		
-		def sgdl1
-			{ :eta0 => eta0, :alpha => alpha }
-		end
-		
-		def bcd
-			{ :kappa => kappa }
-		end
-		
-		def rprop
-			{
-				:stpmin => stpmin, :stpmax => stpmax, :stpinc => stpinc,
-				:stpdec => stpdec, :cutoff => cutoff
-			}
-		end
-		
-		# Returns a hash of all the attributes with their names and values.
-		def attributes
-			Hash[*Options.attribute_names.map { |a| [a, send(a)] }.flatten]
-		end
-		
-		alias to_hash attributes
-		
-		def has_attribute?(attribute)
-			Options.attribute_names.include?(attribute)
-		end
-		
-		def valid_algorithm?
-			self.class.algorithms.include?(algorithm)
-		end
-		
-		def valid?
-			validate.empty?
-		end
-		
-		def validate
-			e = []
-
-			%w{ threads jobsize alpha histsz maxls eta0 alpha nbest }.each do |name|
-				e << "invalid value for #{name}: #{send(name)}" unless send(name) > 0
-			end
-
-			%w{ rho1 rho2 }.each do |name|
-				e << "invalid value for #{name}: #{send(name)}" unless send(name) >= 0.0
-			end
-			
-			e << "unknown algorithm: #{algorithm}" unless valid_algorithm?			
-			e << "BCD not supported for training maxent models" if maxent && algorithm == 'bcd'
-			e
-		end
-		
-		%w{ maxent compact sparse label check score posterior compress }.each do |m|
-			writer = "#{m}=".to_sym
-			define_method("#{m}!") do
-				send(writer, true)
-				self
-			end
-		end
-		
-		def <=>(other)
-			other.respond_to?(:attributes) ? attributes <=> other.attributes : nil
-		end
-				
-	end
-	
-end
+  class Options
+
+    include Comparable
+
+    class << self
+
+      # Returns a sorted list of available option attributes.
+      def attribute_names
+        @attribute_names ||= %w{ stop_window convergence_window posterior
+          max_iterations jobsize threads rho1 rho2 stop_epsilon score check
+          algorithm pattern development_data maxent compact sparse skip_tokens
+          compress }.sort.map(&:to_sym).freeze
+      end
+
+      # Returns the default options.
+      def defaults
+        @defaults ||= new.attributes
+      end
+
+      # Returns the list of supported algorithm options.
+      def algorithms
+        @algorithms ||= %w{ l-bfgs sgd-l1 bcd rprop rprop+ rprop- auto }.freeze
+      end
+
+    end
+
+    attr_accessor :compress
+
+    alias compress? compress
+
+    # Returns the value of the attribute identified by +name+ or nil
+    # if there is no such attribute.
+    def [](name)
+      has_attribute?(name) ? send(name) : nil
+    end
+
+    # Updates the value of the attribute identified by +name+ with the
+    # passed-in +value+.
+    def []=(name, value)
+      raise ArgumentError, "bad attribute name: #{name}" unless has_attribute?(name)
+      send("#{name}=", value)
+    end
+
+    # Updates all the attributes from the passed-in hash.
+    def update(attributes = {})
+      attributes.each_pair do |k,v|
+        mid = "#{k}="
+        send(mid, v) if respond_to?(mid)
+      end
+      self
+    end
+
+    alias update_attributes update
+
+    def lbfgs
+      { :clip => clip, :histsz => histsz, :maxls => maxls }
+    end
+
+    def sgdl1
+      { :eta0 => eta0, :alpha => alpha }
+    end
+
+    def bcd
+      { :kappa => kappa }
+    end
+
+    def rprop
+      {
+        :stpmin => stpmin, :stpmax => stpmax, :stpinc => stpinc,
+        :stpdec => stpdec, :cutoff => cutoff
+      }
+    end
+
+    # Returns a hash of all the attributes with their names and values.
+    def attributes
+      Hash[*Options.attribute_names.map { |a| [a, send(a)] }.flatten]
+    end
+
+    alias to_hash attributes
+
+    def has_attribute?(attribute)
+      Options.attribute_names.include?(attribute)
+    end
+
+    def valid_algorithm?
+      self.class.algorithms.include?(algorithm)
+    end
+
+    def valid?
+      validate.empty?
+    end
+
+    def validate
+      e = []
+
+      %w{ threads jobsize alpha histsz maxls eta0 alpha nbest }.each do |name|
+        e << "invalid value for #{name}: #{send(name)}" unless send(name) > 0
+      end
+
+      %w{ rho1 rho2 }.each do |name|
+        e << "invalid value for #{name}: #{send(name)}" unless send(name) >= 0.0
+      end
+
+      e << "unknown algorithm: #{algorithm}" unless valid_algorithm?
+      e << "BCD not supported for training maxent models" if maxent && algorithm == 'bcd'
+      e
+    end
+
+    %w{ maxent compact sparse label check score posterior compress }.each do |m|
+      writer = "#{m}=".to_sym
+      define_method("#{m}!") do
+        send(writer, true)
+        self
+      end
+    end
+
+    def <=>(other)
+      other.respond_to?(:attributes) ? attributes <=> other.attributes : nil
+    end
+
+  end
+
+end

data/lib/wapiti/utility.rb

@@ -1,15 +1,15 @@
 module Wapiti
-	
-	# Creates a Model based on the pattern in the passed-in options and training
-	# +data+ supplied as input.
-	def train(data, options, &block)
-		Model.train(data, options, &block)
-	end
-	
-	def load(model)
-		Model.load(model)
-	end
-	
-	module_function :train, :load
-	
-end
+
+  # Creates a Model based on the pattern in the passed-in options and training
+  # +data+ supplied as input.
+  def train(data, options, &block)
+    Model.train(data, options, &block)
+  end
+
+  def load(model)
+    Model.load(model)
+  end
+
+  module_function :train, :load
+
+end

data/lib/wapiti/version.rb

@@ -1,3 +1,3 @@
 module Wapiti
-  VERSION = '0.0.5'.freeze
-end
+  VERSION = '0.1.0'.freeze
+end

data/spec/spec_helper.rb

@@ -1,21 +1,41 @@
+begin
+  require 'simplecov'
+  require 'coveralls' if ENV['CI']
+rescue LoadError
+  # ignore
+end
+
+begin
+  if defined?(RUBY_ENGINE) && RUBY_ENGINE == 'rbx'
+    require 'rubinius/debugger'
+  else
+    require 'debugger'
+  end
+rescue LoadError
+  # ignore
+end
+
+$:.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$:.unshift(File.dirname(__FILE__))
+
 require 'wapiti'
 
 require 'fileutils'
 require 'tempfile'
 
 RSpec::Matchers.define :be_valid_model do
-	match do |model|
-		model.is_a?(Wapiti::Model) && model.nlbl > 0
-	end	
+  match do |model|
+    model.is_a?(Wapiti::Model) && model.nlbl > 0
+  end
 end
 
 RSpec::Matchers.define :be_valid_model_file do
-	match do |path|
-		File.exists?(path) && !File.open(path).read.empty?
-	end	
+  match do |path|
+    File.exists?(path) && !File.open(path).read.empty?
+  end
 end
 
 
-RSpec.configuration do |c|
-	c.include(FileUtils)
-end
+RSpec.configure do |c|
+  c.include(FileUtils)
+end

data/spec/wapiti/model_spec.rb

@@ -1,197 +1,233 @@
 # -*- encoding: utf-8 -*-
 
 module Wapiti
-	describe 'Model' do
-		
-		describe '.train' do
-			context 'given sufficient options' do
-				let(:pattern) { File.expand_path('../../fixtures/pattern.txt', __FILE__) }
-				let(:input) { File.expand_path('../../fixtures/train.txt', __FILE__) }
-				
-				it 'returns a valid model instance' do
-					Model.train(input, :pattern => pattern).labels.should == (1..6).map(&:to_s)
-				end
-				
-			end
-		end
-		
-		describe 'initialization' do
-		
-			context 'when passed no arguments' do
-				it 'creates a new model with default options' do
-					m = Model.new
-					m.should_not be nil
-					m.options.should be_instance_of(Options)
-					m.nlbl.should == 0
-					m.nobs.should == 0
-				end
-			end
-			
-			context 'when passed more than one argument' do
-				it 'should raise an error' do
-					expect { Model.new(1,2) }.to raise_error
-				end
-			end
-
-			context 'when passed a hash' do
-				let(:options) { { :threads => 42 } }
-				
-				it 'should create the options from the hash' do
-					Model.new(options).options[:threads].should == 42
-				end
-			end
-			
-			context 'when passed an options instance' do
-				let(:options) { Options.new(:threads => 42) }
-
-				it 'should create the options from the hash' do
-					Model.new(options).options[:threads].should == 42
-				end
-			end
-			
-			context 'when passed something other than a hash or an options instance' do
-				it 'should raise an error' do
-					expect { Model.new(1) }.to raise_error
-					expect { Model.new(nil) }.to raise_error
-					expect { Model.new(true) }.to raise_error
-					expect { Model.new('foo') }.to raise_error
-				end
-			end
-			
-			context 'when called with a block' do
-				it 'should pass the options instance to the block' do
-					Model.new(:threads => 42) { |o| o.threads = 23 }.options.threads.should == 23
-				end
-			end
-		end
-		
-		describe '#options' do
-			it 'returns the options for training' do
-				Model.new.options.should be_instance_of(Options)
-			end
-		end
-		
-		describe '#labels' do
-			it 'returns the number of labels (0 by default)' do
-				Model.new.nlbl.should == 0
-			end
-		end
-
-		describe '#observations' do
-			it 'returns the number of observations (0 by default)' do
-				Model.new.observations.should == 0
-			end
-		end
-
-		describe '#features' do
-			it 'returns the number of features (0 by default)' do
-				Model.new.features.should == 0
-			end
-		end
-
-		describe '#total' do
-			it 'returns the total training time (0.0 by default)' do
-				Model.new.total.should == 0.0
-			end
-		end
-
-		describe '#train' do
-			let(:model) { Model.new(:pattern => File.expand_path('../../fixtures/pattern.txt', __FILE__)) }
-			let(:data) { File.expand_path('../../fixtures/train.txt', __FILE__) }
-			
-			it 'accepts a filename as input' do
-				model.train(data).nlbl.should == 6
-			end
-			
-			it 'accepts a data array' do
-				# sequence = []
-				# File.open(data).each_line do |line|
-				# 	
-				# end
-				# 
-				# model.train([]).nlbl.should == 6
-			end
-			
-			context 'when called without a pattern' do
-				it 'fails because of wapiti' do
-					expect { Model.new.train(data).nlbl.should == 6 }.to raise_error(NativeError)
-				end
-			end
-			
-		end
-		
-		describe '#label' do
-			
-			context 'given an empty model' do
-				
-			end
-			
-			context 'given a trained model' do
-				let(:model) { Model.load(File.expand_path('../../fixtures/ch.mod', __FILE__)) }
-				
-				context 'when passed an array of arrays' do
-					let(:input) { [['Héllo NN B-VP', ', , O', 'world NN B-NP', '! ! O']] }
-					
-					it 'returns an array of token-label pairs' do
-						labels = model.label(input)
-						labels[0].map(&:first).should == input[0]
-						labels[0].map(&:last).should == %w{ B-NP O B-NP O }
-					end
-										
-					it 'yields each token/label pair to the supplied block' do
-						labels = model.label(input) do |token, label|
-				      [token.downcase, label.downcase]
-				    end
-						labels[0].map(&:last).should == %w{ b-np o b-np o }
-				  end
-				
-					context 'with the :score option set' do
-						before(:each) { model.options.score! }
-
-						it 'returns an array of token-label-score tuples' do
-							model.label(input)[0].map { |t,l,s| s.class }.uniq == [Float]
-						end
-					end
-
-					context 'with the :nbest option set to 2' do
-						before(:each) { model.options.nbest = 2 }
-
-						it 'returns an array of token-label-label tuples' do
-							model.label(input)[0][-1][1,2] == %w{ O O }
-						end
-					end
-				
-				end
-
-				
-				context 'when passed a filename' do
-					let(:input) { File.expand_path('../../fixtures/chtest.txt', __FILE__) }
-					
-					it 'returns an array of token-label pairs' do
-						labels = model.label(input)
-						labels.should have(77).elements
-						labels[0].take(5).map(&:last).should == %w{ B-NP B-PP B-NP I-NP B-VP }
-					end
-				end
-								
-			end
-			
-		end
-	
-		describe '#labels' do
-			it 'returns an empty list by default' do
-				Model.new.labels.should be_empty
-			end
-			
-			context 'given a trained model' do
-				let(:model) { Model.load(File.expand_path('../../fixtures/ch.mod', __FILE__)) }
-			
-				it 'returns a list of all known labels' do
-					model.labels.should have(model.nlbl).elements
-				end
-			end
-		end
-		
-		
-	end
-end
+  describe 'Model' do
+
+    describe '.train' do
+      context 'given sufficient options' do
+        let(:pattern) { File.expand_path('../../fixtures/pattern.txt', __FILE__) }
+        let(:input) { File.expand_path('../../fixtures/train.txt', __FILE__) }
+
+        it 'returns a valid model instance' do
+          Model.train(input, :pattern => pattern).labels.should == (1..6).map(&:to_s)
+        end
+
+        it 'is also exposed as Wapiti.train' do
+          Wapiti.train(input, :pattern => pattern).labels.should == (1..6).map(&:to_s)
+        end
+
+      end
+    end
+
+    describe 'initialization' do
+
+      context 'when passed no arguments' do
+        it 'creates a new model with default options' do
+          m = Model.new
+          m.should_not be nil
+          m.options.should be_instance_of(Options)
+          m.nlbl.should == 0
+          m.nobs.should == 0
+        end
+      end
+
+      context 'when passed more than one argument' do
+        it 'should raise an error' do
+          expect { Model.new(1,2) }.to raise_error
+        end
+      end
+
+      context 'when passed a hash' do
+        let(:options) { { :threads => 42 } }
+
+        it 'should create the options from the hash' do
+          Model.new(options).options[:threads].should == 42
+        end
+      end
+
+      context 'when passed an options instance' do
+        let(:options) { Options.new(:threads => 42) }
+
+        it 'should create the options from the hash' do
+          Model.new(options).options[:threads].should == 42
+        end
+      end
+
+      context 'when passed something other than a hash or an options instance' do
+        it 'should raise an error' do
+          expect { Model.new(1) }.to raise_error
+          expect { Model.new(nil) }.to raise_error
+          expect { Model.new(true) }.to raise_error
+          expect { Model.new('foo') }.to raise_error
+        end
+      end
+
+      context 'when called with a block' do
+        it 'should pass the options instance to the block' do
+          Model.new(:threads => 42) { |o| o.threads = 23 }.options.threads.should == 23
+        end
+      end
+    end
+
+    describe '#options' do
+      it 'returns the options for training' do
+        Model.new.options.should be_instance_of(Options)
+      end
+    end
+
+    describe '#labels' do
+      it 'returns the number of labels (0 by default)' do
+        Model.new.nlbl.should == 0
+      end
+    end
+
+    describe '#observations' do
+      it 'returns the number of observations (0 by default)' do
+        Model.new.observations.should == 0
+      end
+    end
+
+    describe '#features' do
+      it 'returns the number of features (0 by default)' do
+        Model.new.features.should == 0
+      end
+    end
+
+    describe '#total' do
+      it 'returns the total training time (0.0 by default)' do
+        Model.new.total.should == 0.0
+      end
+    end
+
+    describe '#train' do
+      let(:model) { Model.new(:pattern => File.expand_path('../../fixtures/pattern.txt', __FILE__)) }
+      let(:data) { File.expand_path('../../fixtures/train.txt', __FILE__) }
+
+      it 'accepts a filename as input' do
+        model.train(data).nlbl.should == 6
+      end
+
+      it 'accepts a data array' do
+        # sequence = []
+        # File.open(data).each_line do |line|
+        #
+        # end
+        #
+        # model.train([]).nlbl.should == 6
+      end
+
+      context 'when called without a pattern' do
+        it 'fails because of wapiti' do
+          expect { Model.new.train(data).nlbl.should == 6 }.to raise_error(NativeError)
+        end
+      end
+
+    end
+
+    describe '#statistics' do
+      context 'given an empty model' do
+        it 'returns zeroes' do
+          s = Model.new.statistics
+
+          s[:sequences][:total].should == 0
+          s[:sequences][:errors].should == 0
+          s[:sequences][:rate].should == 0
+          s[:tokens][:total].should == 0
+          s[:tokens][:errors].should == 0
+          s[:tokens][:rate].should == 0
+        end
+      end
+
+      context 'given a trained model' do
+        let(:model) { Wapiti.load(File.expand_path('../../fixtures/ch.mod', __FILE__)) }
+        let(:input) { [['Héllo NN B-VP', ', , O', 'world NN B-NP', '! ! O']] }
+
+        it 'returns token and sequcence counts and errors' do
+          model.statistics[:tokens][:total].should == 0
+          model.label input
+          model.statistics[:tokens][:total].should == 0
+
+          model.options.check = true
+          model.label input
+
+          model.statistics[:tokens][:total].should == input.map(&:length).reduce(&:+)
+          model.statistics[:sequences][:total].should == input.length
+        end
+      end
+    end
+
+    describe '#label' do
+
+      context 'given an empty model' do
+
+      end
+
+      context 'given a trained model' do
+        let(:model) { Wapiti.load(File.expand_path('../../fixtures/ch.mod', __FILE__)) }
+
+        context 'when passed an array of arrays' do
+          let(:input) { [['Héllo NN B-VP', ', , O', 'world NN B-NP', '! ! O']] }
+
+          it 'returns an array of token-label pairs' do
+            labels = model.label(input)
+            labels[0].map(&:first).should == input[0]
+            labels[0].map(&:last).should == %w{ B-NP O B-NP O }
+          end
+
+          it 'yields each token/label pair to the supplied block' do
+            labels = model.label(input) do |token, label|
+              [token.downcase, label.downcase]
+            end
+            labels[0].map(&:last).should == %w{ b-np o b-np o }
+          end
+
+          context 'with the :score option set' do
+            before(:each) { model.options.score! }
+
+            it 'returns an array of token-label-score tuples' do
+              model.label(input)[0].map { |t,l,s| s.class }.uniq == [Float]
+            end
+          end
+
+          context 'with the :nbest option set to 2' do
+            before(:each) { model.options.nbest = 2 }
+
+            it 'returns an array of token-label-label tuples' do
+              model.label(input)[0][-1][1,2] == %w{ O O }
+            end
+          end
+
+        end
+
+
+        context 'when passed a filename' do
+          let(:input) { File.expand_path('../../fixtures/chtest.txt', __FILE__) }
+
+          it 'returns an array of token-label pairs' do
+            labels = model.label(input)
+            labels.should have(77).elements
+            labels[0].take(5).map(&:last).should == %w{ B-NP B-PP B-NP I-NP B-VP }
+          end
+        end
+
+      end
+
+    end
+
+    describe '#labels' do
+      it 'returns an empty list by default' do
+        Model.new.labels.should be_empty
+      end
+
+      context 'given a trained model' do
+        let(:model) { Model.load(File.expand_path('../../fixtures/ch.mod', __FILE__)) }
+
+        it 'returns a list of all known labels' do
+          model.labels.should have(model.nlbl).elements
+        end
+      end
+    end
+
+
+  end
+end