natto 0.9.9 → 1.0.0

data/lib/natto/natto.rb

@@ -4,9 +4,9 @@ require 'natto/option_parse'
 require 'natto/struct'
 
 module Natto 
-  # `MeCab` is a wrapper class for the MeCab `Tagger`.
-  # Options to the MeCab `Tagger` are passed in as a string
-  # (MeCab command-line style) or as a Ruby-style hash at
+  # `MeCab` is a class providing an interface to the MeCab library.
+  # Options to the MeCab Model, Tagger and Lattice are passed in
+  # as a string (MeCab command-line style) or as a Ruby-style hash at
   # initialization.
   #
   # ## Usage
@@ -16,14 +16,16 @@ module Natto
   #     text = '凡人にしか見えねえ風景ってのがあるんだよ。'
   #
   #     nm = Natto::MeCab.new
-  #     => #<Natto::MeCab:0x28d3bdc8 \
-  #          @tagger=#<FFI::Pointer address=0x28afb980>, \
-  #          @libpath="/usr/local/lib/libmecab.so"       \
-  #          @options={},                                \
-  #          @dicts=[#<Natto::DictionaryInfo:0x289a1f14  \
+  #     => #<Natto::MeCab:0x0000080318d278                                  \
+  #          @model=#<FFI::Pointer address=0x000008039174c0>,               \
+  #          @tagger=#<FFI::Pointer address=0x0000080329ba60>,              \
+  #          @lattice=#<FFI::Pointer address=0x000008045bd140>,             \
+  #          @libpath="/usr/local/lib/libmecab.so"                          \
+  #          @options={},                                                   \
+  #          @dicts=[#<Natto::DictionaryInfo:0x0000080318ce90               \
   #                    @filepath="/usr/local/lib/mecab/dic/ipadic/sys.dic", \
-  #                    charset=utf8,                     \
-  #                    type=0>],                         \
+  #                    charset=utf8,                                        \
+  #                    type=0>],                                            \
   #          @version=0.996>
   #
   #     # print entire MeCab result to stdout
@@ -104,7 +106,7 @@ module Natto
   #
   #     # Boundary constraint parsing with output formatting.
   #     # %m   ... morpheme surface
-  #     # %F,  ... comma-delimited ChaSen feature values
+  #     # %f   ... tab-delimited ChaSen feature values
   #     #          part-of-speech (index 0) 
   #     # %2   ... MeCab node status value (1 unknown)
   #     #
@@ -148,19 +150,22 @@ module Natto
     MECAB_TOKEN_BOUNDARY = 1
     MECAB_INSIDE_TOKEN = 2
 
-    # @return [FFI:Pointer] pointer to MeCab tagger.
+    # @return [FFI:Pointer] pointer to MeCab Model.
+    attr_reader :model
+    # @return [FFI:Pointer] pointer to MeCab Tagger.
     attr_reader :tagger
+    # @return [FFI:Pointer] pointer to MeCab Lattice.
+    attr_reader :lattice
     # @return [String] absolute filepath to MeCab library.
     attr_reader :libpath
     # @return [Hash] MeCab options as key-value pairs.
     attr_reader :options
     # @return [Array] listing of all of dictionaries referenced.
     attr_reader :dicts
-    # @return [String] `MeCab` version.
+    # @return [String] MeCab version.
     attr_reader :version
 
-    # Initializes the wrapped `Tagger` instance with the
-    # given `options`.
+    # Initializes the wrapped Tagger instance with the given `options`.
     # 
     # Options supported are:
     #
@@ -186,19 +191,21 @@ module Natto
     # - :cost_factor --  cost factor (integer, default 700)
     # 
     # <p>MeCab command-line arguments (-F) or long (--node-format) may be used in 
-    # addition to Ruby-style `Hash`es</p>
+    # addition to Ruby-style hashs</p>
     # <i>Use single-quotes to preserve format options that contain escape chars.</i><br/>
     # e.g.<br/>
     #
     #     nm = Natto::MeCab.new(node_format: '%m¥t%f[7]¥n')
-    #     => #<Natto::MeCab:0x28d2ae10 
-    #          @tagger=#<FFI::Pointer address=0x28a97980>, \
-    #          @libpath="/usr/local/lib/libmecab.so",      \
-    #          @options={:node_format=>"%m¥t%f[7]¥n"},     \
-    #          @dicts=[#<Natto::DictionaryInfo:0x28d2a85c  \
+    #     => #<Natto::MeCab:0x00000803503ee8                                 \
+    #          @model=#<FFI::Pointer address=0x00000802b6d9c0>,              \
+    #          @tagger=#<FFI::Pointer address=0x00000802ad3ec0>,             \
+    #          @lattice=#<FFI::Pointer address=0x000008035f3980>,            \
+    #          @libpath="/usr/local/lib/libmecab.so",                        \
+    #          @options={:node_format=>"%m¥t%f[7]¥n"},                       \
+    #          @dicts=[#<Natto::DictionaryInfo:0x000008035038f8              \
     #                    @filepath="/usr/local/lib/mecab/dic/ipadic/sys.dic" \
-    #                    charset=utf8,                     \
-    #                    type=0>]                          \
+    #                    charset=utf8,                                       \
+    #                    type=0>]                                            \
     #          @version=0.996>
     # 
     #     puts nm.parse('才能とは求める人間に与えられるものではない。')
@@ -216,210 +223,216 @@ module Natto
     #     ない    ナイ
     #     。      。
     #     EOS
-    #
-    # @param options [Hash, String] the MeCab options for tagger
-    # @raise [MeCabError] if `mecab` cannot be initialized with the given `options`
+    # @param options [Hash, String] the MeCab options
+    # @raise [MeCabError] if MeCab cannot be initialized with the given `options`
     def initialize(options={})
       @options = self.class.parse_mecab_options(options) 
-      @dicts = []
-      # TODO invoke function for enhancing MeCabNode after this point
-
       opt_str  = self.class.build_options_str(@options)
-      @tagger  = self.class.mecab_new2(opt_str)
+
+      @model   = self.class.mecab_model_new2(opt_str)
+      if @model.address == 0x0
+        raise MeCabError.new("Could not initialize Model with options: '#{opt_str}'")
+      end
+
+      @tagger  = self.class.mecab_model_new_tagger(@model)
+      if @tagger.address == 0x0
+        raise MeCabError.new("Could not initialize Tagger with options: '#{opt_str}'")
+      end
+
+      @lattice = self.class.mecab_model_new_lattice(@model)
+      if @lattice.address == 0x0
+        raise MeCabError.new("Could not initialize Lattice with options: '#{opt_str}'")
+      end
+
       @libpath = self.class.find_library
-      raise MeCabError.new("Could not initialize MeCab with options: '#{opt_str}'") if @tagger.address == 0x0
-
-      self.mecab_set_theta(@tagger, @options[:theta]) if @options[:theta]
-      self.mecab_set_lattice_level(@tagger, @options[:lattice_level]) if @options[:lattice_level]
-      self.mecab_set_all_morphs(@tagger, 1) if @options[:all_morphs]
-      self.mecab_set_partial(@tagger, 1) if @options[:partial]
-       
-      # Define lambda for each major parsing type: _tostr, _tonode,
-      # boundary constraint _tostr, boundary constraint _node;
-      # and each parsing type will support both normal and N-best
-      # options
-      @parse_tostr = ->(text) {
-        if @options[:nbest] && @options[:nbest] > 1
-          #self.mecab_set_lattice_level(@tagger, (@options[:lattice_level] || 1))
-          retval = self.mecab_nbest_sparse_tostr(@tagger, @options[:nbest], text) || 
-                raise(MeCabError.new(self.mecab_strerror(@tagger))) 
-        else
-          retval = self.mecab_sparse_tostr(@tagger, text) || 
-                raise(MeCabError.new(self.mecab_strerror(@tagger))) 
-        end
 
-        retval.force_encoding(Encoding.default_external)
-      } 
+      if @options[:nbest] && @options[:nbest] > 1
+        self.mecab_lattice_set_request_type(@lattice, MECAB_LATTICE_NBEST)
+      else
+        self.mecab_lattice_set_request_type(@lattice, MECAB_LATTICE_ONE_BEST)
+      end
+      if @options[:partial]
+        self.mecab_lattice_add_request_type(@lattice, MECAB_LATTICE_PARTIAL)
+      end
+      if @options[:marginal]
+        self.mecab_lattice_add_request_type(@lattice,
+                                            MECAB_LATTICE_MARGINAL_PROB)
+      end
+      if @options[:all_morphs]
+        # required when node parsing
+        #self.mecab_lattice_add_request_type(@lattice, MECAB_LATTICE_NBEST)
+        self.mecab_lattice_add_request_type(@lattice,
+                                            MECAB_LATTICE_ALL_MORPHS)
+      end
+      if @options[:allocate_sentence]
+        self.mecab_lattice_add_request_type(@lattice, 
+                                            MECAB_LATTICE_ALLOCATE_SENTENCE)
+      end
 
-      @parse_tonodes = ->(text) { 
-        Enumerator.new do |y|
-          if @options[:nbest] && @options[:nbest] > 1
-            nlen = @options[:nbest]
-            #self.mecab_set_lattice_level(@tagger, (@options[:lattice_level] || 1))
-            self.mecab_nbest_init(@tagger, text) 
-            nptr = self.mecab_nbest_next_tonode(@tagger)
-          else
-            nlen = 1
-            nptr = self.mecab_sparse_tonode(@tagger, text) 
-          end
-          raise(MeCabError.new(self.mecab_strerror(@tagger))) if nptr.nil? || nptr.address==0x0
-
-          nlen.times do
-            s = text.bytes.to_a
-            while nptr && nptr.address != 0x0
-              mn = Natto::MeCabNode.new(nptr)
-              # ignore BOS nodes, since mecab does so
-              if !mn.is_bos?
-                s = s.drop_while {|e| (e==0xa || e==0x20)}
-                if !s.empty?
-                  sarr = []
-                  mn.length.times { sarr << s.shift }
-                  surf = sarr.pack('C*')
-                  mn.surface = surf.force_encoding(Encoding.default_external)
-                end
-                if @options[:output_format_type] || @options[:node_format]
-                  mn.feature = self.mecab_format_node(@tagger, nptr).force_encoding(Encoding.default_external)
-                end
-                y.yield mn
-              end
-              nptr = mn.next
-            end
-            if nlen > 1
-              nptr = self.mecab_nbest_next_tonode(@tagger)
-            end
-          end
-        end
-      }
-      
-      @bcparse_tostr = ->(text, boundary_constraints=/./) {
-        begin
-          lattice = self.mecab_lattice_new()
-          raise MeCabError.new("Could not create Lattice") if lattice.address == 0x0
+      if @options[:theta]
+        self.mecab_lattice_set_theta(@lattice, @options[:theta]) 
+      end
 
+      @parse_tostr = ->(text, constraints) {
+        begin
           if @options[:nbest] && @options[:nbest] > 1
             n = @options[:nbest]
-            self.mecab_lattice_set_request_type(lattice, MECAB_LATTICE_NBEST)
           else
             n = 1
-            self.mecab_lattice_set_request_type(lattice, MECAB_LATTICE_ONE_BEST)
-          end
-          if @options[:theta]
-            self.mecab_lattice_set_theta(lattice, @options[:theta])
           end
 
-          tokens = tokenize(text, boundary_constraints)
-          text = tokens.map {|t| t.first}.join
-          self.mecab_lattice_set_sentence(lattice, text)
-
-          bpos = 0
-          tokens.each do |token|
-            c = token.first.bytes.count
+          if constraints[:boundary_constraints]
+            tokens = tokenize_by_pattern(text,
+                                         constraints[:boundary_constraints])
+            text = tokens.map {|t| t.first}.join
+            self.mecab_lattice_set_sentence(@lattice, text)
 
-            self.mecab_lattice_set_boundary_constraint(lattice, bpos, MECAB_TOKEN_BOUNDARY)
-            bpos += 1
+            bpos = 0
+            tokens.each do |token|
+              c = token.first.bytes.count
 
-            mark = token.last ? MECAB_INSIDE_TOKEN : MECAB_ANY_BOUNDARY
-            (c-1).times do
-              self.mecab_lattice_set_boundary_constraint(lattice, bpos, mark)
+              self.mecab_lattice_set_boundary_constraint(@lattice,
+                                                         bpos,
+                                                         MECAB_TOKEN_BOUNDARY)
               bpos += 1
+
+              mark = token.last ? MECAB_INSIDE_TOKEN : MECAB_ANY_BOUNDARY
+              (c-1).times do
+                self.mecab_lattice_set_boundary_constraint(@lattice,
+                                                           bpos,
+                                                           mark)
+                bpos += 1
+              end
             end
+          elsif constraints[:feature_constraints]
+            features = constraints[:feature_constraints]
+            tokens = tokenize_by_features(text,
+                                          features.keys)
+            text = tokens.map {|t| t.first}.join
+            self.mecab_lattice_set_sentence(@lattice, text)
+
+            bpos = 0
+            tokens.each do |token|
+              chunk = token.first
+              c = chunk.bytes.count
+              if token.last
+                self.mecab_lattice_set_feature_constraint(@lattice,
+                                                          bpos,
+                                                          bpos+c,
+                                                          features[chunk])
+              end
+              bpos += c
+            end
+          else
+            self.mecab_lattice_set_sentence(@lattice, text)
           end
 
-          self.mecab_parse_lattice(@tagger, lattice)
+          self.mecab_parse_lattice(@tagger, @lattice)
           
           if n > 1
-            retval = self.mecab_lattice_nbest_tostr(lattice, n)
+            retval = self.mecab_lattice_nbest_tostr(@lattice, n)
           else
-            retval = self.mecab_lattice_tostr(lattice)
+            retval = self.mecab_lattice_tostr(@lattice)
           end
           retval.force_encoding(Encoding.default_external)
         rescue
-          raise(MeCabError.new(self.mecab_lattice_strerror(lattice))) 
-        ensure
-          if lattice.address != 0x0
-            self.mecab_lattice_destroy(lattice)
-          end
+          raise(MeCabError.new(self.mecab_lattice_strerror(@lattice))) 
         end
       }
         
-      @bcparse_tonodes = ->(text, boundary_constraints=/./) {
+      @parse_tonodes = ->(text, constraints) {
+        self.mecab_lattice_add_request_type(@lattice, MECAB_LATTICE_NBEST)
         Enumerator.new do |y|
           begin
-            lattice = self.mecab_lattice_new()
-            raise MeCabError.new("Could not create Lattice") if lattice.address == 0x0
-
             if @options[:nbest] && @options[:nbest] > 1
               n = @options[:nbest]
-              self.mecab_lattice_set_request_type(lattice, MECAB_LATTICE_NBEST)
             else
               n = 1
-              self.mecab_lattice_set_request_type(lattice, MECAB_LATTICE_ONE_BEST)
-            end
-            if @options[:theta]
-              self.mecab_lattice_set_theta(lattice, @options[:theta])
             end
 
-            tokens = tokenize(text, boundary_constraints)
-            text = tokens.map {|t| t.first}.join
-            self.mecab_lattice_set_sentence(lattice, text)
-
-            bpos = 0
-            tokens.each do |token|
-              c = token.first.bytes.count
+            if constraints[:boundary_constraints]
+              tokens = tokenize_by_pattern(text,
+                                           constraints[:boundary_constraints])
+              text = tokens.map {|t| t.first}.join
+              self.mecab_lattice_set_sentence(@lattice, text)
 
-              self.mecab_lattice_set_boundary_constraint(lattice, bpos, MECAB_TOKEN_BOUNDARY)
-              bpos += 1
+              bpos = 0
+              tokens.each do |token|
+                c = token.first.bytes.count
 
-              mark = token.last ? MECAB_INSIDE_TOKEN : MECAB_ANY_BOUNDARY
-              (c-1).times do
-                self.mecab_lattice_set_boundary_constraint(lattice, bpos, mark)
+                self.mecab_lattice_set_boundary_constraint(@lattice,
+                                                           bpos,
+                                                           MECAB_TOKEN_BOUNDARY)
                 bpos += 1
+
+                mark = token.last ? MECAB_INSIDE_TOKEN : MECAB_ANY_BOUNDARY
+                (c-1).times do
+                  self.mecab_lattice_set_boundary_constraint(@lattice, bpos, mark)
+                  bpos += 1
+                end
+              end
+            elsif constraints[:feature_constraints]
+              features = constraints[:feature_constraints]
+              tokens = tokenize_by_features(text,
+                                            features.keys)
+              text = tokens.map {|t| t.first}.join
+              self.mecab_lattice_set_sentence(@lattice, text)
+
+              bpos = 0
+              tokens.each do |token|
+                chunk = token.first
+                c = chunk.bytes.count
+                if token.last
+                  self.mecab_lattice_set_feature_constraint(@lattice,
+                                                            bpos,
+                                                            bpos+c,
+                                                            features[chunk])
+                end
+                bpos += c
               end
+            else
+              self.mecab_lattice_set_sentence(@lattice, text)
             end
 
-            self.mecab_parse_lattice(@tagger, lattice)
+            self.mecab_parse_lattice(@tagger, @lattice)
 
             n.times do
-              check = self.mecab_lattice_next(lattice)
+              check = self.mecab_lattice_next(@lattice)
               if check
-                nptr = self.mecab_lattice_get_bos_node(lattice)
+                nptr = self.mecab_lattice_get_bos_node(@lattice)
           
-                s = text.bytes.to_a
                 while nptr && nptr.address!=0x0
                   mn = Natto::MeCabNode.new(nptr)
-                  s = s.drop_while {|e| (e==0xa || e==0x20)}
-                  if !s.empty?
-                    sarr = []
-                    mn.length.times { sarr << s.shift }
-                    surf = sarr.pack('C*')
+                  if !mn.is_bos?
+                    surf = mn[:surface].bytes.to_a.slice(0,mn.length).pack('C*')
                     mn.surface = surf.force_encoding(Encoding.default_external)
+                    if @options[:output_format_type] || @options[:node_format]
+                      mn.feature = self.mecab_format_node(@tagger, nptr).force_encoding(Encoding.default_external)
+                    end
+                    y.yield mn
                   end
-                  if @options[:output_format_type] || @options[:node_format]
-                    mn.feature = self.mecab_format_node(@tagger, nptr).force_encoding(Encoding.default_external)
-                  end
-                  y.yield mn
-                  nptr = mn.next
+                  nptr = mn[:next]
                 end
               end
             end
+            nil
           rescue
-            raise(MeCabError.new(self.mecab_lattice_strerror(lattice))) 
-          ensure
-            if lattice.address != 0x0
-              self.mecab_lattice_destroy(lattice)
-            end
+            raise(MeCabError.new(self.mecab_lattice_strerror(@lattice))) 
           end
         end
       }
 
-      @dicts << Natto::DictionaryInfo.new(Natto::Binding.mecab_dictionary_info(@tagger))
+      @dicts = []
+      @dicts << Natto::DictionaryInfo.new(self.mecab_model_dictionary_info(@model))
       while @dicts.last.next.address != 0x0
         @dicts << Natto::DictionaryInfo.new(@dicts.last.next)
       end
 
       @version = self.mecab_version
 
-      ObjectSpace.define_finalizer(self, self.class.create_free_proc(@tagger))
+      ObjectSpace.define_finalizer(self, self.class.create_free_proc(@model,
+                                                                     @tagger,
+                                                                     @lattice))
     end
     
     # Parses the given `text`, returning the MeCab output as a single string. 
@@ -430,36 +443,45 @@ module Natto
     # `boundary_constraints` key in the `options` hash. Boundary constraints
     # parsing provides hints to MeCab on where the morpheme boundaries in the
     # given `text` are located. `boundary_constraints` value may be either a
-    # `Regexp` or `String`; please see
-    # [String#scan](http://ruby-doc.org/core-2.2.0/String.html#method-i-scan String#scan.
+    # `Regexp` or `String`; please see [String#scan](http://ruby-doc.org/core-2.2.1/String.html#method-i-scan)
     # The boundary constraint parsed output will be returned as a single
     # string, unless a block is passed to this method for node parsing.
     #
+    # Feature constraint parsing is available by passing in the 
+    # `feature_constraints` key in the `options` hash. Feature constraints
+    # parsing provides instructions to MeCab to use the feature indicated
+    # for any morpheme that is an exact match for the given key. 
+    # `feature_constraints` is a hash mapping a specific morpheme (String)
+    # to a corresponding feature value (String).
     # @param text [String] the Japanese text to parse
-    # @param options [Hash] only the `boundary_constraints` key is available
-    # @return [String] parsing result from `mecab`
-    # @raise [MeCabError] if the `mecab` tagger cannot parse the given `text`
+    # @param constraints [Hash] `boundary_constraints` or `feature_constraints`
+    # @return [String] parsing result from MeCab
+    # @raise [MeCabError] if the MeCab Tagger cannot parse the given `text`
     # @raise [ArgumentError] if the given string `text` argument is `nil`
     # @see MeCabNode
-    def parse(text, options={})
-      raise ArgumentError.new 'Text to parse cannot be nil' if text.nil?
-      if options[:boundary_constraints]
-        if block_given?
-          @bcparse_tonodes.call(text, options[:boundary_constraints]).each {|n| yield n }
-        else
-          @bcparse_tostr.call(text, options[:boundary_constraints])
+    def parse(text, constraints={})
+      if text.nil?
+        raise ArgumentError.new 'Text to parse cannot be nil'
+      elsif constraints[:boundary_constraints]
+        if !(constraints[:boundary_constraints].is_a?(Regexp) ||
+             constraints[:boundary_constraints].is_a?(String))
+          raise ArgumentError.new 'boundary constraints must be a Regexp or String'
         end
+      elsif constraints[:feature_constraints] && !constraints[:feature_constraints].is_a?(Hash)
+        raise ArgumentError.new 'feature constraints must be a Hash'
+      elsif @options[:partial] && !text.end_with?("\n")
+        raise ArgumentError.new 'partial parsing requires new-line char at end of text'
+      end
+
+      if block_given?
+        @parse_tonodes.call(text, constraints).each {|n| yield n }
       else
-        if block_given?
-          @parse_tonodes.call(text).each {|n| yield n }
-        else
-          @parse_tostr.call(text)
-        end
+        @parse_tostr.call(text, constraints)
       end
     end
 
     # Parses the given string `text`, returning an
-    # {http://www.ruby-doc.org/core-2.1.5/Enumerator.html Enumerator} that may be
+    # [Enumerator](http://www.ruby-doc.org/core-2.2.1/Enumerator.html) that may be
     # used to iterate over the resulting {MeCabNode} objects. This is more 
     # efficient than parsing to a simple string, since each node's
     # information will not be materialized all at once as it is with
@@ -469,45 +491,62 @@ module Natto
     # the morpheme. Node-formatting  may also be used to customize
     # the resulting node's `feature` attribute.
     #
-    # Boundary constraint parsing is available via passing in the
+    # Boundary constraint parsing is available by passing in the
     # `boundary_constraints` key in the `options` hash. Boundary constraints
     # parsing provides hints to MeCab on where the morpheme boundaries in the
     # given `text` are located. `boundary_constraints` value may be either a
     # `Regexp` or `String`; please see 
-    # [String#scan](http://ruby-doc.org/core-2.2.0/String.html#method-i-scan String#scan).
+    # [String#scan](http://ruby-doc.org/core-2.2.1/String.html#method-i-scan)
     #
+    # Feature constraint parsing is available by passing in the 
+    # `feature_constraints` key in the `options` hash. Feature constraints
+    # parsing provides instructions to MeCab to use the feature indicated
+    # for any morpheme that is an exact match for the given key. 
+    # `feature_constraints` is a hash mapping a specific morpheme (String)
+    # to a corresponding feature value (String).
     # @param text [String] the Japanese text to parse
-    # @param options [Hash] only the `boundary_constraints` key is available
+    # @param constraints [Hash] `boundary_constraints` or `feature_constraints`
     # @return [Enumerator] of MeCabNode instances
-    # @raise [MeCabError] if the `mecab` tagger cannot parse the given `text`
+    # @raise [MeCabError] if the MeCab Tagger cannot parse the given `text`
     # @raise [ArgumentError] if the given string `text` argument is `nil`
     # @see MeCabNode
     # @see http://ruby-doc.org/core-2.2.1/Enumerator.html
-    def enum_parse(text, options={})
-      raise ArgumentError.new 'Text to parse cannot be nil' if text.nil?
-      if options[:boundary_constraints]
-        @bcparse_tonodes.call(text, options[:boundary_constraints])
-      else
-        @parse_tonodes.call(text)
+    def enum_parse(text, constraints={})
+      if text.nil?
+        raise ArgumentError.new 'Text to parse cannot be nil'
+      elsif constraints[:boundary_constraints]
+        if !(constraints[:boundary_constraints].is_a?(Regexp) ||
+             constraints[:boundary_constraints].is_a?(String))
+          raise ArgumentError.new 'boundary constraints must be a Regexp or String'
+        end
+      elsif constraints[:feature_constraints] && !constraints[:feature_constraints].is_a?(Hash)
+        raise ArgumentError.new 'feature constraints must be a Hash'
+      elsif @options[:partial] && !text.end_with?("\n")
+        raise ArgumentError.new 'partial parsing requires new-line char at end of text'
       end
+
+      @parse_tonodes.call(text, constraints)
     end
 
-    # Returns human-readable details for the wrapped `mecab` tagger.
+    # Returns human-readable details for the wrapped MeCab library.
     # Overrides `Object#to_s`.
     #
     # - encoded object id
-    # - underlying FFI pointer to the `mecab` tagger
-    # - real file path to `mecab` library
+    # - underlying FFI pointer to the MeCab Model
+    # - underlying FFI pointer to the MeCab Tagger
+    # - underlying FFI pointer to the MeCab Lattice
+    # - real file path to MeCab library
     # - options hash
     # - list of dictionaries
     # - MeCab version
-    #
     # @return [String] encoded object id, underlying FFI pointer,
-    #   file path to `mecab` library, options hash,
+    #   file path to MeCab library, options hash,
     #   list of dictionaries and MeCab version
     def to_s
       [ super.chop,
+        "@model=#{@model},", 
         "@tagger=#{@tagger},", 
+        "@lattice=#{@lattice},", 
         "@libpath=\"#{@libpath}\",",
         "@options=#{@options.inspect},", 
         "@dicts=#{@dicts.to_s},", 
@@ -515,7 +554,6 @@ module Natto
     end
 
     # Overrides `Object#inspect`.
-    # 
     # @return [String] encoded object id, FFI pointer, options hash,
     #   list of dictionaries, and MeCab version
     # @see #to_s
@@ -524,26 +562,27 @@ module Natto
     end
 
     # Returns a `Proc` that will properly free resources
-    # when this `Tagger` instance is garbage collected.
-    # The `Proc` returned is registered to be invoked
-    # after the `Tagger` instance  owning `tptr` 
-    # has been destroyed.
-    #
-    # @param tptr [FFI::Pointer] pointer to `Tagger`
-    # @return [Proc] to release `mecab` resources properly
-    def self.create_free_proc(tptr)
+    # when this instance is garbage collected.
+    # @param mptr [FFI::Pointer] pointer to Model
+    # @param tptr [FFI::Pointer] pointer to Tagger
+    # @param lptr [FFI::Pointer] pointer to Lattice
+    # @return [Proc] to release MeCab resources properly
+    def self.create_free_proc(mptr, tptr, lptr)
       Proc.new do
+        self.mecab_lattice_destroy(lptr)
         self.mecab_destroy(tptr)
+        self.mecab_model_destory(mptr)
       end
     end
 
     private
 
     # @private
-    def tokenize(text, pattern)
+    # MeCab eats all leading and training whitespace char
+    def tokenize_by_pattern(text, pattern)
       matches = text.scan(pattern)
       
-      acc =[]
+      acc = []
       tmp = text
       matches.each_with_index do |m,i|
         bef, mat, aft = tmp.partition(m)
@@ -553,17 +592,34 @@ module Natto
         unless mat.empty?
           acc << [mat.strip, true]
         end
-        if i==matches.size-1 and !aft.empty?
+        if i==matches.size-1 && !aft.empty?
           acc << [aft.strip, false]
         end
         tmp = aft
       end
       acc
     end
+
+    def tokenize_by_features(text, features)
+      acc = []
+      acc << [text.strip, false]
+
+      features.each do |feature|
+        acc.each_with_index do |e,i|
+          if !e.last
+            tmp = tokenize_by_pattern(e.first, feature)
+            if !tmp.empty?
+              acc.delete_at(i)
+              acc.insert(i, *tmp)
+            end
+          end
+        end
+      end
+      acc
+    end
   end
 
-  # `MeCabError` is a general error class 
-  # for the `Natto` module.
+  # `MeCabError` is a general error class for the `Natto` module.
   class MeCabError < RuntimeError; end
 end