crfpp 0.0.1 → 0.0.2

data/lib/crfpp.rb

@@ -1,4 +1,13 @@
 
+require 'forwardable'
+require 'tempfile'
+
 require 'crfpp/version'
+require 'crfpp/errors'
+require 'crfpp/filelike'
+require 'crfpp/macro'
+require 'crfpp/template'
+require 'crfpp/feature'
+require 'crfpp/model'
 require 'crfpp/native'
 require 'crfpp/utilities'

data/lib/crfpp/errors.rb

@@ -0,0 +1,16 @@
+module CRFPP
+
+  class Error < StandardError
+    
+    attr_accessor :original
+    
+    def initialize(message = nil, original = $!)
+      super(message)
+      @original = original
+    end
+    
+  end
+  
+  class NativeError < Error; end
+  
+end

data/lib/crfpp/feature.rb

@@ -0,0 +1,29 @@
+module CRFPP
+  
+  class Feature < Struct.new(:content, :type, :id)
+    
+    class << self
+      def parse(string)
+        if string =~ /^([UB])(\d*):(.+)\s*$/
+          Feature.new($3, $1.to_sym, $2)
+        else
+          string
+        end
+      end
+    end
+    
+    def initialize(content = Macro.new, type = :U, id = nil)
+      super
+    end
+    
+    def identifier(base = id)
+      base.is_a?(Numeric) ? ('%02d' % base) : base
+    end
+    
+    def to_s(number = id)
+      [type.to_s.upcase, identifier(number), ':', content].compact.join
+    end
+    
+  end
+  
+end

data/lib/crfpp/filelike.rb

@@ -0,0 +1,27 @@
+module CRFPP
+  
+  module Filelike
+    
+    attr_writer :path
+    
+    def path
+      @path ||= Tempfile.new('filelike').path
+    end
+
+    def write
+      File.open(path, 'w:UTF-8') do |f|
+        f.write(to_s)
+        f.close
+      end
+    end
+    
+    def read
+      f = File.open(path, 'r:UTF-8')
+      f.read
+    ensure
+      f.close
+    end
+
+  end
+  
+end

data/lib/crfpp/macro.rb

@@ -0,0 +1,17 @@
+module CRFPP
+  class Macro < Struct.new(:row, :column)
+
+    TEMPLATE = '%%x[%d,%d]'.freeze
+
+    alias col column
+    
+    def initialize(row = 0, column = 0)
+      super
+    end
+    
+    def to_s
+      TEMPLATE % values
+    end
+    
+  end
+end

data/lib/crfpp/model.rb

@@ -0,0 +1,23 @@
+module CRFPP
+  class Model
+    
+    include Filelike
+    
+    attr_reader :data
+    
+    def initialize(path = nil)
+      @data, @path = '', path
+    end
+    
+    def open
+      @data = read
+      self
+    end
+    
+    def save
+      write(@data)
+      self
+    end
+        
+  end
+end

data/lib/crfpp/template.rb

@@ -0,0 +1,32 @@
+module CRFPP
+  class Template
+    
+    extend Forwardable
+    
+    include Filelike
+    
+    attr_reader :features
+    
+    def_delegators :@features, :<<, :length
+    
+    def initialize(path = nil)
+      @path = path
+      open
+    end
+    
+    def open
+      @features = read.lines.map { |line| Feature.parse(line) }      
+      self
+    end
+    
+    def clear
+      @features = []
+    end
+    
+    def to_s
+      i = -1
+      features.map { |f| f.is_a?(Feature) ? f.to_s(i += 1) : f.chomp }.join("\n")
+    end
+    
+  end
+end

data/lib/crfpp/utilities.rb

@@ -1,12 +1,71 @@
 
 module CRFPP
   
-  def learn(template, data, model, options = {})
-    Native.learn(['--thread=2', template, data, model].join(' '))
+  # Creates a new Model based on a template and training data.
+  #
+  # The data parameter can either be an array of strings or a filename. The
+  # possible options are:
+  #
+  # :threads:   False or the number of threads to us (default is 2).
+  #
+  # :algorithm: L1 or L2 (default)
+  #
+  # :cost: With this option, you can change the hyper-parameter for the CRFs.
+  #        With larger C value, CRF tends to overfit to the give training
+  #        corpus. This parameter trades the balance between overfitting and
+  #        underfitting. The results will significantly be influenced by this
+  #        parameter. You can find an optimal value by using held-out data or
+  #        more general model selection method such as cross validation. 
+  #
+  # :frequency: This parameter sets the cut-off threshold for the features. CRF++
+  #         uses the features that occurs no less than NUM times in the given training
+  #         data. The default value is 1. When you apply CRF++ to large data, the
+  #         number of unique features would amount to several millions. This option is
+  #         useful in such cases. 
+  #
+  def learn(template, data, options = {})
+    options = { :threads => 2, :algorithm => :L2, :cost => 1.0, :frequency => 1}.merge(options)
+    
+    unless File.exists?(data)
+      data = save_data_to_tempfile([data].flatten)
+      temporary = true
+    end
+
+    template = Template.new(template) unless template.is_a?(Template)
+    model = Model.new
+    
+    arguments = []
+    
+    # TODO check algorithm names
+    # arguments << "--algorithm=#{options[:algorithm]}"
+    
+    arguments << "--cost=#{options[:cost]}"
+    arguments << "--thread=#{options[:threads]}"
+    arguments << "--freq=#{options[:frequency]}"
+    
+    arguments << template.path
+    arguments << data
+    arguments << model.path
+    
+    success = Native.learn(arguments.join(' '))
+    raise NativeError, 'crfpp learn failed' unless success
+    
+    model
+  ensure
+    data.unlink if temporary
   end
   
   alias train learn
   
   module_function :train, :learn
   
+  private
+  
+  # Saves data to temporary file and returns the path
+  def save_data_to_tempfile(data)
+    tmp = Tempfile.new(data)
+    tmp.write(data.join("\n"))
+    tmp.path
+  end
+  
 end

data/lib/crfpp/version.rb

@@ -1,3 +1,3 @@
 module CRFPP
-  VERSION = '0.0.1'.freeze
+  VERSION = '0.0.2'.freeze
 end

data/test/crfpp/test_feature.rb

@@ -0,0 +1,25 @@
+require 'helper'
+
+module CRFPP
+  
+  class TestFeature < Test::Unit::TestCase
+    
+    def test_assignment_of_constructor_arguments
+      assert_raise(ArgumentError) { Feature.new(1,2,3,4) }
+    end
+    
+    def test_to_s
+      assert_equal 'U:%x[0,0]', Feature.new.to_s
+      assert_equal 'U:%x[2,3]', Feature.new(Macro.new(2,3)).to_s
+      assert_equal 'U01:%x[2,3]', Feature.new(Macro.new(2,3)).to_s(1)
+      assert_equal 'U123:%x[2,3]', Feature.new(Macro.new(2,3)).to_s(123)
+    end
+    
+    def test_parse_feature_strings
+      assert_equal 'U:%x[42,23]/AB', Feature.parse('U:%x[42,23]/AB').to_s
+    end
+    
+    
+  end
+  
+end

data/test/crfpp/test_filelike.rb

@@ -0,0 +1,19 @@
+require 'helper'
+
+module CRFPP
+  class TestFilelike < Test::Unit::TestCase
+    def setup
+      @klass = Class.new
+      @klass.instance_eval { include Filelike }      
+    end
+    
+    def subject
+      @klass.new
+    end
+    
+    def test_has_a_path_by_default
+      assert subject.path
+    end
+        
+  end
+end

data/test/crfpp/test_macro.rb

@@ -0,0 +1,20 @@
+require 'helper'
+
+module CRFPP
+  
+  class TestMacro < Test::Unit::TestCase
+    
+    def test_assignment_of_constructor_arguments
+      assert_equal [0, 0], Macro.new.values
+      assert_equal [1, 1], Macro.new(1,1).values
+      assert_equal [1, 0], Macro.new(1).values
+      assert_raise(ArgumentError) { Macro.new(1,2,3) }
+    end
+    
+    def test_to_s
+      assert_equal '%x[0,0]', Macro.new.to_s
+    end
+    
+  end
+  
+end

data/test/crfpp/test_native.rb

@@ -3,12 +3,10 @@ require 'helper'
 module CRFPP
   class TestNative < Test::Unit::TestCase
     
-    FIXTURES_ROOT = File.expand_path('../../fixtures', __FILE__)
-    
     def test_create_a_new_model_file_through_training
       model = Tempfile.new('model')
-      assert Native.learn("#{FIXTURES_ROOT}/template #{FIXTURES_ROOT}/train.data #{model.path}")
-      assert model.length > 0
+      assert Native.learn("#{FixturesRoot}/template #{FixturesRoot}/train.data #{model.path}"), 'Native.learn returned with error exit status'
+      assert model.length > 0, 'nothing was written to model file'
     ensure
       model.close
       model.unlink

data/test/crfpp/test_template.rb

@@ -0,0 +1,23 @@
+require 'helper'
+
+module CRFPP
+  
+  class TestTemplate < Test::Unit::TestCase
+    
+    def test_load_template_from_empty_file
+      file = Tempfile.new('template')
+      assert Template.new(file.path).to_s.empty?
+    ensure
+      file.close
+      file.unlink
+    end
+
+    def test_load_template_from_file
+      path = "#{FixturesRoot}/template"
+      assert_equal Template.new(path).to_s.chomp, File.open(path).read.chomp
+    end
+    
+    
+  end
+  
+end

data/test/crfpp/test_utilities.rb

@@ -0,0 +1,12 @@
+require 'helper'
+
+module CRFPP
+  class TestUtilities < Test::Unit::TestCase
+    
+    def test_create_a_new_model_file_through_training
+      model = CRFPP.learn("#{FixturesRoot}/template", "#{FixturesRoot}/train.data")
+      assert !model.open.data.empty?
+    end
+        
+  end
+end

data/test/helper.rb

@@ -1,4 +1,6 @@
 require 'test/unit'
 require 'crfpp'
 
-require 'tempfile'
+module CRFPP
+  FixturesRoot = File.expand_path('../fixtures', __FILE__)
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: crfpp
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2011-08-16 00:00:00.000000000Z
+date: 2011-08-17 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &2157069020 !ruby/object:Gem::Requirement
+  requirement: &2156849560 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,10 +21,10 @@ dependencies:
         version: '0.9'
   type: :development
   prerelease: false
-  version_requirements: *2157069020
+  version_requirements: *2156849560
 - !ruby/object:Gem::Dependency
   name: rake-compiler
-  requirement: &2157068480 !ruby/object:Gem::Requirement
+  requirement: &2156849000 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -32,10 +32,10 @@ dependencies:
         version: '0.7'
   type: :development
   prerelease: false
-  version_requirements: *2157068480
+  version_requirements: *2156849000
 - !ruby/object:Gem::Dependency
   name: ZenTest
-  requirement: &2157067980 !ruby/object:Gem::Requirement
+  requirement: &2156848460 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -43,7 +43,7 @@ dependencies:
         version: '4.6'
   type: :development
   prerelease: false
-  version_requirements: *2157067980
+  version_requirements: *2156848460
 description: A Ruby extension to interface with CRF++, the Conditional Random Fields
   library written in C++. You need to install libcrfpp to use this gem.
 email:
@@ -67,10 +67,21 @@ files:
 - ext/crfpp/tagger.cpp
 - ext/crfpp/tagger.hpp
 - lib/crfpp.rb
+- lib/crfpp/errors.rb
+- lib/crfpp/feature.rb
+- lib/crfpp/filelike.rb
+- lib/crfpp/macro.rb
+- lib/crfpp/model.rb
+- lib/crfpp/template.rb
 - lib/crfpp/utilities.rb
 - lib/crfpp/version.rb
+- test/crfpp/test_feature.rb
+- test/crfpp/test_filelike.rb
+- test/crfpp/test_macro.rb
 - test/crfpp/test_native.rb
 - test/crfpp/test_tagger.rb
+- test/crfpp/test_template.rb
+- test/crfpp/test_utilities.rb
 - test/fixtures/template
 - test/fixtures/test.data
 - test/fixtures/train.data
@@ -108,8 +119,13 @@ signing_key:
 specification_version: 3
 summary: Conditional Random Fields for Ruby.
 test_files:
+- test/crfpp/test_feature.rb
+- test/crfpp/test_filelike.rb
+- test/crfpp/test_macro.rb
 - test/crfpp/test_native.rb
 - test/crfpp/test_tagger.rb
+- test/crfpp/test_template.rb
+- test/crfpp/test_utilities.rb
 - test/fixtures/template
 - test/fixtures/test.data
 - test/fixtures/train.data