natto 0.4.1 → 0.5.0

data/CHANGELOG

@@ -1,5 +1,10 @@
 ## CHANGELOG
 
+- __2011/02/2?__: 0.5.0 release.
+    - Added support for node parsing using blocks
+    - Added support for mecab options nbest, all-morphs
+    - Pulling support for mecab option partial, since it is more of a command-line feature
+
 - __2011/01/27__: 0.4.1 release.
     - Tweaking the description in natto.gemspec a bit 
 

data/README.md

@@ -4,7 +4,7 @@ A Tasty Ruby Binding with MeCab
 ## What is natto?
 natto combines the [Ruby programming language](http://www.ruby-lang.org/) with [MeCab](http://mecab.sourceforge.net/), the part-of-speech and morphological analyzer for the Japanese language.
 
-natto is a gem bridging Ruby and MeCab using FFI (foreign function interface). No compilation is necessary, and natto will run on CRuby (mri/yarv) and JRuby (jvm) equally well, on any OS.
+natto is a gem bridging Ruby and MeCab using FFI (foreign function interface). No compilation is necessary, as natto is _not_ a C extension. natto will run on CRuby (mri/yarv) and JRuby (jvm) equally well. natto will also run on Windows, Unix/Linux, and Mac.
 
 You can learn more about [natto at Google Code Projects](http://code.google.com/p/natto/).
 
@@ -31,29 +31,36 @@ e.g., on Windows
     set MECAB_PATH=C:\Program Files\MeCab\bin\libmecab.dll
 e.g., for Cygwin
     export MECAB_PATH=cygmecab-1
+e.g., from within a Ruby program
+    ENV['MECAB_PATH']=/usr/local/lib/libmecab.so
 
 ## Usage
     require 'rubygems' if RUBY_VERSION.to_f < 1.9
     require 'natto'
 
-    mecab = Natto::MeCab.new
-    => #<Natto::MeCab:0x289b88e0 @ptr=#<FFI::Pointer address=0x288865c8>, \
-                                 @options={}, \
-                                 @version="0.98", \
-                                 @dicts=[/usr/local/lib/mecab/dic/ipadic/sys.dic]>
+    nm = Natto::MeCab.new
+    => #<Natto::MeCab:0x28d30748 
+         @ptr=#<FFI::Pointer address=0x28a97d50>, \
+         @options={}, \
+         @dicts=[#<Natto::DictionaryInfo:0x28d3061c 
+                 filename="/usr/local/lib/mecab/dic/ipadic/sys.dic", 
+                 charset="utf8">], 
+         @version="0.98">
 
-    puts mecab.version
-    => 0.98
+    puts nm.version
+    => "0.98" 
 
-    sysdic = mecab.dicts.first
+    sysdic = nm.dicts.first
 
     puts sysdic.filename
-    => /usr/local/lib/mecab/dic/ipadic/sys.dic
+    => "/usr/local/lib/mecab/dic/ipadic/sys.dic"
 
     puts sysdic.charset
-    => utf8
+    => "utf8" 
 
-    puts mecab.parse('暑い日にはもってこいの一品ですね。')
+    nm.parse('暑い日にはもってこいの一品ですね。') do |n|
+      puts "#{n.surface}\t#{n.feature}"
+    end
     暑い    形容詞,自立,*,*,形容詞・アウオ段,基本形,暑い,アツイ,アツイ
     日      名詞,非自立,副詞可能,*,*,*,日,ヒ,ヒ
     に      助詞,格助詞,一般,*,*,*,に,ニ,ニ
@@ -64,7 +71,7 @@ e.g., for Cygwin
     です    助動詞,*,*,*,特殊・デス,基本形,です,デス,デス
     ね      助詞,終助詞,*,*,*,*,ね,ネ,ネ
     。      終助詞記号,句点,*,*,*,*,。,。,。
-    EOS
+            BOS/EOS,*,*,*,*,*,*,*,*
     => nil
 
 ## Contributing to natto

data/lib/natto/binding.rb

@@ -13,7 +13,7 @@ module Natto
     # String name for the environment variable used by 
     # <tt>Natto</tt> to indicate the exact name / full path
     # to the <tt>mecab</tt> library.
-    MECAB_PATH = 'MECAB_PATH'
+    MECAB_PATH = 'MECAB_PATH'.freeze
     
     # @private
     def self.included(base)
@@ -31,11 +31,13 @@ module Natto
     # @raise [LoadError] if MECAB_PATH environment variable is not set in Windows
     # <br/>
     # e.g., for bash on UNIX/Linux
-    #     export MECAB_PATH=mecab.so
+    #     export MECAB_PATH=/usr/local/lib/libmecab.so
     # e.g., on Windows
     #     set MECAB_PATH=C:\Program Files\MeCab\bin\libmecab.dll
     # e.g., for Cygwin
     #     export MECAB_PATH=cygmecab-1
+    # e.g., from within a Ruby program
+    #     ENV['MECAB_PATH']=/usr/local/lib/libmecab.so
     def self.find_library
       host_os = RbConfig::CONFIG['host_os']
 
@@ -50,33 +52,72 @@ module Natto
 
     ffi_lib(ENV[MECAB_PATH] || find_library)
 
-    attach_function :mecab_version, [], :string
     attach_function :mecab_new2, [:string], :pointer
+    attach_function :mecab_version, [], :string
+    attach_function :mecab_strerror, [:pointer],:string
     attach_function :mecab_destroy, [:pointer], :void
+  
+    attach_function :mecab_set_theta, [:pointer, :float], :void
+    attach_function :mecab_set_lattice_level, [:pointer, :int], :void 
+    attach_function :mecab_set_all_morphs, [:pointer, :int], :void
+    
     attach_function :mecab_sparse_tostr, [:pointer, :string], :string
-    attach_function :mecab_strerror, [:pointer],:string
+    attach_function :mecab_sparse_tonode, [:pointer, :string], :pointer
+    
+    attach_function :mecab_nbest_init, [:pointer, :string], :int
+    attach_function :mecab_nbest_sparse_tostr, [:pointer, :int, :string], :string
+    attach_function :mecab_nbest_next_tonode, [:pointer], :pointer
+    
     attach_function :mecab_dictionary_info, [:pointer], :pointer
 
     # @private
     module ClassMethods
+      def mecab_new2(options_str)
+        Natto::Binding.mecab_new2(options_str)
+      end
+      
       def mecab_version
         Natto::Binding.mecab_version
       end
 
-      def mecab_new2(options_str)
-        Natto::Binding.mecab_new2(options_str)
+      def mecab_strerror(ptr)
+        Natto::Binding.mecab_strerror(ptr)
       end
 
       def mecab_destroy(ptr)
         Natto::Binding.mecab_destroy(ptr)
       end
 
+      def mecab_set_theta(ptr, t)
+        Natto::Binding.mecab_set_theta(ptr, t)
+      end
+
+      def mecab_set_lattice_level(ptr, ll)
+        Natto::Binding.mecab_set_lattice_level(ptr, ll)
+      end
+
+      def mecab_set_all_morphs(ptr, am)
+        Natto::Binding.mecab_set_all_morphs(ptr, am)
+      end
+
       def mecab_sparse_tostr(ptr, str)
         Natto::Binding.mecab_sparse_tostr(ptr, str)
       end
+      
+      def mecab_sparse_tonode(ptr, str)
+        Natto::Binding.mecab_sparse_tonode(ptr, str)
+      end
 
-      def mecab_strerror(ptr)
-        Natto::Binding.mecab_strerror(ptr)
+      def mecab_nbest_next_tonode(ptr)
+        Natto::Binding.mecab_nbest_next_tonode(ptr)
+      end
+
+      def mecab_nbest_init(ptr, str)
+        Natto::Binding.mecab_nbest_init(ptr, str)
+      end
+
+      def mecab_nbest_sparse_tostr(ptr, n, str)
+        Natto::Binding.mecab_nbest_sparse_tostr(ptr, n, str)
       end
 
       def mecab_dictionary_info(ptr)

data/lib/natto/version.rb

@@ -6,8 +6,15 @@
 # <tt>Natto::MeCab</tt> is a wrapper class for the <tt>mecab</tt> 
 # parser.
 #
-# <tt>Natto::DictionaryInfo</tt> is a wrapper for a <tt>Natto::MeCab</tt>
-# instance's related dictionary information.
+# <tt>Natto::MeCabStruct</tt> is a base class for a <tt>mecab</tt>
+# struct.
+#
+# <tt>Natto::MeCabNode</tt> is a wrapper for the struct representing
+# a <tt>mecab</tt>-parsed node.
+#
+# <tt>Natto::DictionaryInfo</tt> is a wrapper for the struct 
+# representing a <tt>Natto::MeCab</tt> instance's related 
+# dictionary information.
 #
 # <tt>Natto::MeCabError</tt> is a general error class for the 
 # <tt>Natto</tt> module.
@@ -16,5 +23,5 @@
 # which are made available via <tt>FFI</tt> bindings to <tt>mecab</tt>.
 module Natto
   # Version string for this Rubygem.
-  VERSION = "0.4.1"
+  VERSION = "0.5.0"
 end

data/lib/natto.rb

@@ -1,4 +1,5 @@
 # coding: utf-8
+
 require 'rubygems' if RUBY_VERSION.to_f < 1.9
 require 'natto/binding'
 
@@ -14,23 +15,25 @@ module Natto
   #     require 'rubygems' if RUBY_VERSION.to_f < 1.9
   #     require 'natto'
   #
-  #     mecab = Natto::MeCab.new(:output_format_type=>'wakati')
-  #     => #<Natto::MeCab:0x28dd471c @ptr=#<FFI::Pointer address=0x28a027d8>, \
-  #                                  @options={:output_format_type=>"wakati"}, \
-  #                                  @version="0.98", \
-  #                                  @dicts=[/usr/local/lib/mecab/dic/ipadic/sys.dic]>
-  #
-  #     output = mecab.parse('ネバネバの組み合わせ美味しいです。').split
+  #     nm = Natto::MeCab.new(:output_format_type=>'chasen2')
+  #     => #<Natto::MeCab:0x28d3bdc8 \
+  #          @ptr=#<FFI::Pointer address=0x28afb980>, \
+  #          @options={:output_format_type=>"chasen2"}, \
+  #          @dicts=[#<Natto::DictionaryInfo:0x289a1f14 \
+  #                    filename="/usr/local/lib/mecab/dic/ipadic/sys.dic", \
+  #                    charset="utf8">], \
+  #          @version="0.98">
   #
-  #     output.each do |token|
-  #       puts token
+  #     nm.parse('ネバネバの組み合わせ美味しいです。') do |n| 
+  #       puts "#{n.surface}\t#{n.feature}" 
   #     end
-  #     => ネバネバ
-  #        の
-  #        組み合わせ
-  #        美味しい
-  #        です
-  #        。
+  #
+  #     ネバネバ        名詞,サ変接続,*,*,*,*,ネバネバ,ネバネバ,ネバネバ
+  #     の              助詞,連体化,*,*,*,*,の,ノ,ノ
+  #     組み合わせ      名詞,一般,*,*,*,*,組み合わせ,クミアワセ,クミアワセ
+  #     美味しい        形容詞,自立,*,*,形容詞・イ段,基本形,美味しい,オイシイ,オイシイ
+  #     です            助動詞,*,*,*,特殊・デス,基本形,です,デス,デス
+  #     。              記号,句点,*,*,*,*,。,。,。
   #
   class MeCab
     include Natto::Binding
@@ -39,11 +42,11 @@ module Natto
 
     # Supported options to the <tt>mecab</tt> parser.
     # See the <tt>mecab</tt> help for more details. 
-    SUPPORTED_OPTS = [  :rcfile, :dicdir, :userdic, :lattice_level, :all_morphs,
-                        :output_format_type, :partial, :node_format, :unk_format, 
-                        :bos_format, :eos_format, :eon_format, :unk_feature, 
-                        :input_buffer_size, :allocate_sentence, :nbest, :theta, 
-                        :cost_factor ].freeze
+    SUPPORTED_OPTS = [ :rcfile, :dicdir, :userdic, :lattice_level, :all_morphs,
+                       :output_format_type, :node_format, :unk_format, 
+                       :bos_format, :eos_format, :eon_format, :unk_feature, 
+                       :input_buffer_size, :allocate_sentence, :nbest, :theta, 
+                       :cost_factor, :output ].freeze
 
     # Initializes the wrapped <tt>mecab</tt> instance with the
     # given <tt>options</tt> hash.
@@ -56,7 +59,6 @@ module Natto
     # - :lattice_level --  lattice information level (integer, default 0)
     # - :all_morphs --  output all morphs (default false)
     # - :output_format_type --  output format type (wakati, chasen, yomi, etc.)
-    # - :partial --  partial parsing mode
     # - :node_format --  user-defined node format
     # - :unk_format --  user-defined unknown node format
     # - :bos_format --  user-defined beginning-of-sentence format
@@ -65,20 +67,23 @@ module Natto
     # - :unk_feature --  feature for unknown word
     # - :input_buffer_size -- set input buffer size (default 8192) 
     # - :allocate_sentence -- allocate new memory for input sentence 
-    # - :nbest --  output N best results (integer, default 1)
+    # - :nbest --  output N best results (integer, default 1), requires lattice level >= 1
     # - :theta --  temperature parameter theta (float, default 0.75)
     # - :cost_factor --  cost factor (integer, default 700)
+    # - :output -- set the output file name
     # 
     # <i>Use single-quotes to preserve format options that contain escape chars.</i><br/>
     # e.g.<br/>
     #
-    #     mecab = Natto::MeCab.new(:node_format=>'%m\t%f[7]\n')
-    #     => #<Natto::MeCab:0x289b88e0 @ptr=#<FFI::Pointer address=0x288865c8>, \
-    #                                  @options={:node_format=>"%m\\t%f[7]\\n"}, \
-    #                                  @version="0.98", \
-    #                                  @dicts=[/usr/local/lib/mecab/dic/ipadic/sys.dic]>
+    #     nm = Natto::MeCab.new(:node_format=>'%m¥t%f[7]¥n')
+    #     => #<Natto::MeCab:0x28d2ae10 @ptr=#<FFI::Pointer address=0x28a97980>, \
+    #          @options={:node_format=>"%m¥t%f[7]¥n"}, \
+    #          @dicts=[#<Natto::DictionaryInfo:0x28d2a85c \
+    #                    filename="/usr/local/lib/mecab/dic/ipadic/sys.dic" \
+    #                    charset="utf8">], \
+    #          @version="0.98">
     #
-    #     puts mecab.parse('簡単で美味しくて良いですよね。')
+    #     puts nm.parse('簡単で美味しくて良いですよね。')
     #     簡単       カンタン
     #     で         デ
     #     美味しくて オイシクテ
@@ -101,23 +106,92 @@ module Natto
       @ptr = self.mecab_new2(opt_str)
       raise MeCabError.new("Could not initialize MeCab with options: '#{opt_str}'") if @ptr.address == 0x0
 
+      # set mecab parsing options
+      self.mecab_set_theta(@ptr, @options[:theta].to_f) if @options[:theta]
+      self.mecab_set_lattice_level(@ptr, @options[:lattice_level].to_i) if @options[:lattice_level]
+      self.mecab_set_all_morphs(@ptr, 1) if @options[:all_morphs]
+       
+      # set mecab parsing implementations 
+      if @options[:nbest] && @options[:nbest] > 1
+        # N-Best parsing implementations
+        self.mecab_nbest_init(@ptr, str) 
+        # nbest parsing require lattice level >= 1
+        self.mecab_set_lattice_level(@ptr, (@options[:lattice_level] || 1))
+        @parse_tostr = lambda { |str| 
+          return self.mecab_nbest_sparse_tostr(@ptr, @options[:nbest], str) || 
+                raise(MeCabError.new(self.mecab_strerror(@ptr))) 
+        } 
+        @parse_tonode = lambda { |str| return self.mecab_nbest_next_tonode(@ptr) }
+      else
+        # default parsing implementations
+        @parse_tostr = lambda { |str|
+          return self.mecab_sparse_tostr(@ptr, str) || raise(MeCabError.new(self.mecab_strerror(@ptr))) 
+        }
+        @parse_tonode = lambda { |str| return self.mecab_sparse_tonode(@ptr, str) }
+      end
+
+      # set ref to dictionaries
       @dicts << Natto::DictionaryInfo.new(Natto::Binding.mecab_dictionary_info(@ptr))
       while @dicts.last.next.address != 0x0
         @dicts << Natto::DictionaryInfo.new(@dicts.last.next)
       end
 
+      # set ref to mecab version string
       @version = self.mecab_version
 
+      # set Proc for freeing mecab pointer
       ObjectSpace.define_finalizer(self, self.class.create_free_proc(@ptr))
     end
-
-    # Parses the given string <tt>str</tt>.
+    
+    # Parses the given string <tt>str</tt>. If a block is passed to this method,
+    # then node parsing will be used and each node yielded to the given block.
     #
     # @param [String] str
     # @return parsing result from <tt>mecab</tt>
     # @raise [MeCabError] if the <tt>mecab</tt> parser cannot parse the given string <tt>str</tt>
+    # @see MeCabNode
     def parse(str)
-      self.mecab_sparse_tostr(@ptr, str) || raise(MeCabError.new(self.mecab_strerror(@ptr)))
+      if block_given?
+        m_node_ptr = @parse_tonode.call(str)
+        head = Natto::MeCabNode.new(m_node_ptr) 
+        if head && head[:next].address != 0x0
+          node = Natto::MeCabNode.new(head[:next])
+          while (node.nil? == false)
+            yield node
+            if node[:next].address != 0x0
+              node = Natto::MeCabNode.new(node[:next])
+            else
+              break
+            end
+          end
+        end
+      else
+        result = @parse_tostr.call(str)
+        result.force_encoding(Encoding.default_external) if result.respond_to?(:encoding) && result.encoding!=Encoding.default_external
+        result
+      end
+    end
+
+    # Returns human-readable details for the wrapped <tt>mecab</tt> parser.
+    # Overrides <tt>Object#to_s</tt>.
+    #
+    # - encoded object id
+    # - FFI pointer to <tt>mecab</tt> object
+    # - options hash
+    # - list of dictionaries
+    # - MeCab version
+    #
+    # @return [String] encoded object id, FFI pointer, options hash, list of dictionaries, and MeCab version
+    def to_s
+      %(#{super.chop} @ptr=#{@ptr.to_s}, @options=#{@options.to_s}, @dicts=#{@dicts.to_s}, @version="#{@version.to_s}">)
+    end
+
+    # Overrides <tt>Object#inspect</tt>.
+    # 
+    # @return [String] encoded object id, FFI pointer, options hash, list of dictionaries, and MeCab version
+    # @see #to_s
+    def inspect
+      self.to_s
     end
 
     # Returns a <tt>Proc</tt> that will properly free resources
@@ -138,14 +212,14 @@ module Natto
     # be passed in the construction of <tt>mecab</tt>.
     #
     # @param [Hash] options 
-    # @return string-representation of the options to the <tt>mecab</tt> parser
+    # @return [String] representation of the options to the <tt>mecab</tt> parser
     def self.build_options_str(options={})
       opt = []
       SUPPORTED_OPTS.each do |k|
         if options.has_key? k
           key = k.to_s.gsub('_', '-')  
-          # all-morphs, partial, and allocate-sentence are just flags
-          if %w( all-morphs partial allocate-sentence ).include? key
+          # all-morphs and allocate-sentence are just flags
+          if %w( all-morphs allocate-sentence ).include? key
             opt << "--#{key}" if options[k]==true
           else
             opt << "--#{key}=#{options[k]}"
@@ -160,6 +234,21 @@ module Natto
   # for the <tt>Natto</tt> module.
   class MeCabError < RuntimeError; end
 
+  # <tt>MeCabStruct</tt> is a general base class 
+  # for <tt>FFI::Struct</tt> objects in the <tt>Natto</tt> module.
+  class MeCabStruct < FFI::Struct
+    # Provides accessor methods for the members of the <tt>mecab</tt> struct.
+    #
+    # @param [String] attr_name
+    # @return member values for the <tt>mecab</tt> struct
+    # @raise [NoMethodError] if <tt>attr_name</tt> is not a member of this <tt>mecab</tt> struct 
+    def method_missing(attr_name)
+      member_sym = attr_name.id2name.to_sym
+      return self[member_sym] if self.members.include?(member_sym)
+      raise(NoMethodError.new("undefined method '#{attr_name}' for #{self}"))
+    end
+  end
+
   # <tt>DictionaryInfo</tt> is a wrapper for the structure holding
   # the <tt>MeCab</tt> instance's related dictionary information.
   # 
@@ -180,27 +269,16 @@ module Natto
   # <tt>mecab</tt> dictionary attributes can be obtained by
   # using their corresponding accessor.
   #
-  #     mecab = Natto::MeCab.new
+  #     nm = Natto::MeCab.new
   #
-  #     sysdic = m.dicts.first
+  #     sysdic = nm.dicts.first
   #
   #     puts sysdic.filename
-  #     => /usr/local/lib/mecab/dic/ipadic/sys.dic
+  #     => "/usr/local/lib/mecab/dic/ipadic/sys.dic"
   #
   #     puts sysdic.charset
-  #     => utf8
-  #
-  # It is also possible to use the <tt>Symbol</tt> for the
-  # <tt>mecab</tt> dictionary member to index into the 
-  # <tt>FFI::Struct</tt> layout associative array like so:
-  #     
-  #     puts sysdic[:filename]
-  #     => /usr/local/lib/mecab/dic/ipadic/sys.dic
-  #
-  #     puts sysdic[:charset]
-  #     => utf8
-  #
-  class DictionaryInfo < FFI::Struct
+  #     => "utf8"
+  class DictionaryInfo < MeCabStruct
 
     layout  :filename, :string,
             :charset,  :string,
@@ -211,7 +289,6 @@ module Natto
             :version,  :ushort,
             :next,     :pointer
    
-    # Hack to avoid that deprecation message Object#type thrown in Ruby 1.8.7.
     if RUBY_VERSION.to_f < 1.9
       alias_method :deprecated_type, :type
       # <tt>Object#type</tt> override defined when <tt>RUBY_VERSION</tt> is
@@ -226,22 +303,199 @@ module Natto
       end
     end
 
-    # Provides accessor methods for the members of the <tt>DictionaryInfo</tt> structure.
+    # Returns human-readable details for this <tt>mecab</tt> dictionary.
+    # Overrides <tt>Object#to_s</tt>.
     #
-    # @param [String] attr_name
-    # @return member values for the <tt>mecab</tt> dictionary
-    # @raise [NoMethodError] if <tt>attr_name</tt> is not a member of this <tt>mecab</tt> dictionary <tt>FFI::Struct</tt> 
-    def method_missing(attr_name)
-      member_sym = attr_name.id2name.to_sym
-      return self[member_sym] if self.members.include?(member_sym)
-      raise(NoMethodError.new("undefined method '#{attr_name}' for #{self}"))
+    # - encoded object id
+    # - full-path dictionary filename
+    # - dictionary charset
+    #
+    # @return [String] encoded object id, dictionary filename, and charset
+    def to_s
+      %(#{super.chop} filename="#{self.filename}", charset="#{self.charset}">)
+    end
+    
+    # Overrides <tt>Object#inspect</tt>.
+    #
+    # @return [String] encoded object id, dictionary filename, and charset
+    # @see #to_s
+    def inspect
+      self.to_s
     end
+  end
+
+  # <tt>MeCabNode</tt> is a wrapper for the structure holding
+  # the parsed <tt>node</tt>.
+  # 
+  # Values for the <tt>mecab</tt> node attributes may be 
+  # obtained by using the following <tt>Symbol</tt>s as keys 
+  # to the layout associative array of <tt>FFI::Struct</tt> members.
+  #
+  # - :prev
+  # - :next
+  # - :enext
+  # - :bnext
+  # - :rpath
+  # - :lpath
+  # - :begin_node_list
+  # - :end_node_list
+  # - :surface
+  # - :feature
+  # - :id
+  # - :length
+  # - :rlength
+  # - :rcAttr
+  # - :lcAttr
+  # - :posid
+  # - :char_type
+  # - :stat
+  # - :isbest
+  # - :sentence_length
+  # - :alpha
+  # - :beta
+  # - :beta
+  # - :prob
+  # - :wcost
+  # - :cost
+  # - :token
+  #
+  # <h2>Usage</h2>
+  # An instance of <tt>MeCabNode</tt> is yielded to a block
+  # used with <tt>MeCab#parse</tt>. Each resulting node is
+  # yielded to the block passed in, where the above-mentioned
+  # node attributes may be accessed.
+  #
+  #     nm = Natto::MeCab.new
+  #
+  #     nm.parse('めかぶの使い方がわからなくて困ってました。') do |n| 
+  #       puts "#{n.surface}¥t#{n.cost}" 
+  #     end
+  #
+  #     め      7961
+  #     かぶ    19303
+  #     の      25995
+  #     使い方  29182
+  #     が      28327
+  #     わから  33625
+  #     なく    34256
+  #     て      36454
+  #     困っ    43797
+  #     て      42178
+  #     まし    46708
+  #     た      46111
+  #     。      42677
+  #             41141
+  #     => nil
+  #
+  # It is also possible to use the <tt>Symbol</tt> for the
+  # <tt>mecab</tt> node member to index into the 
+  # <tt>FFI::Struct</tt> layout associative array like so:
+  #     
+  #     nm.parse('納豆に乗っけて頂きます！') {|n| puts n[:feature] }
+  #
+  #     名詞,一般,*,*,*,*,納豆,ナットウ,ナットー
+  #     助詞,格助詞,一般,*,*,*,に,ニ,ニ
+  #     動詞,自立,*,*,一段,連用形,乗っける,ノッケ,ノッケ
+  #     助詞,接続助詞,*,*,*,*,て,テ,テ
+  #     動詞,非自立,*,*,五段・カ行イ音便,連用形,頂く,イタダキ,イタダキ
+  #     助動詞,*,*,*,特殊・マス,基本形,ます,マス,マス
+  #     記号,一般,*,*,*,*,！,！,！
+  #     BOS/EOS,*,*,*,*,*,*,*,*
+  #     => nil
+  #
+  class MeCabNode < MeCabStruct
+
+    # Normal <tt>mecab</tt> node.
+    NOR_NODE = 0
+    # Unknown <tt>mecab</tt> node.
+    UNK_NODE = 1
+    # Beginning-of-string <tt>mecab</tt> node.
+    BOS_NODE = 2
+    # End-of-string <tt>mecab</tt> node.
+    EOS_NODE = 3
+    # End-of-NBest <tt>mecab</tt> node list.
+    EON_NODE = 4
 
-    # Returns the full-path file name for this dictionary. Overrides <tt>Object#to_s</tt>.
+    layout  :prev,            :pointer,
+            :next,            :pointer,
+            :enext,           :pointer,
+            :bnext,           :pointer,
+            :rpath,           :pointer,
+            :lpath,           :pointer,
+            :begin_node_list, :pointer,
+            :end_node_list,   :pointer,
+            :surface,         :string,
+            :feature,         :string,
+            :id,              :uint,
+            :length,          :ushort,
+            :rlength,         :ushort,
+            :rcAttr,          :ushort,
+            :lcAttr,          :ushort,
+            :posid,           :ushort,
+            :char_type,       :uchar,
+            :stat,            :uchar,
+            :isbest,          :uchar,
+            :sentence_length, :uint,
+            :alpha,           :float,
+            :beta,            :float,
+            :prob,            :float,
+            :wcost,           :short,
+            :cost,            :long,
+            :token,           :pointer
+   
+    if RUBY_VERSION.to_f < 1.9
+      alias_method :deprecated_id, :id
+      # <tt>Object#id</tt> override defined when <tt>RUBY_VERSION</tt> is
+      # older than 1.9. This is a hack to avoid the <tt>Object#id</tt>
+      # deprecation warning thrown up in Ruby 1.8.7.
+      #
+      # <i>This method override is not defined when the Ruby interpreter
+      # is 1.9 or greater.</i>
+      # @return [Fixnum] <tt>mecab</tt> node id
+      def id
+        self[:id]
+      end
+    end
+
+    # Returns the <tt>surface</tt> value for this node.
+    #
+    # @return [String] <tt>mecab</tt> node surface value
+    def surface
+      if self[:surface] && self[:length] > 0
+        @surface ||= self[:surface].bytes.to_a()[0,self[:length]].pack('C*')
+        @surface.force_encoding(Encoding.default_external) if @surface.respond_to?(:encoding) && @surface.encoding!=Encoding.default_external
+      end
+      @surface
+    end
+
+    # Returns the <tt>feature</tt> value for this node.
+    #
+    # @return [String] <tt>mecab</tt> node feature value
+    def feature
+      @feature ||= self[:feature]
+      @feature.force_encoding(Encoding.default_external) if @feature.respond_to?(:encoding) && @feature.encoding!=Encoding.default_external
+      @feature
+    end
+
+    # Returns human-readable details for the <tt>mecab</tt> node.
+    # Overrides <tt>Object#to_s</tt>.
+    #
+    # - encoded object id
+    # - stat 
+    # - surface 
+    # - feature
     #
-    # @return [String] full-path filename for this dictionary
+    # @return [String] encoded object id, stat, surface, and feature 
     def to_s
-      self[:filename]
+      %(#{super.chop} stat=#{self[:stat]}, surface="#{self.surface}", feature="#{self.feature}">)
+    end
+
+    # Overrides <tt>Object#inspect</tt>.
+    #
+    # @return [String] encoded object id, stat, surface, and feature 
+    # @see #to_s
+    def inspect
+      self.to_s
     end
   end
 end

data/test/natto/tc_binding.rb

@@ -21,11 +21,18 @@ class TestNattoBinding < Test::Unit::TestCase
   # Tests for the inclusion of mecab methods made available
   # to any classes including the Natto::Binding module.
   def test_functions_included
-    [  :mecab_version, 
-       :mecab_new2, 
+    [  :mecab_new2, 
+       :mecab_version, 
+       :mecab_strerror,
        :mecab_destroy, 
+       :mecab_set_theta,
+       :mecab_set_lattice_level,
+       :mecab_set_all_morphs,
        :mecab_sparse_tostr, 
-       :mecab_strerror,
+       :mecab_nbest_sparse_tostr, 
+       :mecab_nbest_init,
+       :mecab_nbest_sparse_tostr,
+       :mecab_nbest_next_tonode,
        :mecab_dictionary_info ].each do |f|
        assert(@klass.respond_to? f)
     end

data/test/natto/tc_dictionaryinfo.rb

@@ -4,11 +4,19 @@
 # behavior of Natto::DictionaryInfo
 class TestDictionaryInfo < Test::Unit::TestCase
   def setup
-    @m = Natto::MeCab.new
+    m = Natto::MeCab.new
+    @dicts = m.dicts
+
+    out = `mecab -D`.lines.to_a
+    out.each do |l|
+      tokens = l.split("\t")
+      @sysdic_filename = tokens[1].strip if tokens[0] =~ /filename:/i
+      @sysdic_charset = tokens[1].strip if tokens[0] =~ /charset:/i
+    end
   end
 
   def teardown
-    @m = nil
+    @dicts = nil
   end
 
   # Tests the dictionaries accessor method of Natto::MeCab.
@@ -17,28 +25,36 @@ class TestDictionaryInfo < Test::Unit::TestCase
   # b) system dictionary encoding is utf-8
   # c) only dealing w/ case of 1 dictionary being used
   def test_dictionaries_accessor
-    dicts = @m.dicts
-    assert dicts.empty? == false
-    sysdic = dicts.first
-    assert_equal('/usr/local/lib/mecab/dic/ipadic/sys.dic', sysdic[:filename])
-    assert_equal('utf8', sysdic[:charset])
+    assert @dicts.empty? == false
+    sysdic = @dicts.first
+    assert_equal(@sysdic_filename, sysdic[:filename])
+    assert_equal(@sysdic_charset, sysdic[:charset])
     assert_equal(0x0, sysdic[:next].address)
-    #assert_nil(sysdic.next)
   end
 
   # Tests the to_s method.
   def test_to_s
-    assert_equal('/usr/local/lib/mecab/dic/ipadic/sys.dic', @m.dicts.first.to_s)
+    #<Natto::DictionaryInfo:0x288879bc @filename=\"/usr/local/lib/mecab/dic/ipadic/sys.dic\", @charset=\"utf8\">
+    assert(@dicts.first.to_s.include?("filename=\"#{@sysdic_filename}\", charset=\"#{@sysdic_charset}\""))
   end
 
   # Tests the accessors of Natto::DictionaryInfo.
   # Note: Object#type is deprecated in 1.9.n, but comes with a warning
   #       in 1.8.n
   def test_dictionary_info_member_accessors
-    sysdic = @m.dicts.first
-    members = %w( filename charset type size lsize rsize version next )
+    sysdic = @dicts.first
+    members = [
+      :filename,
+      :charset,
+      :type,
+      :size,
+      :lsize,
+      :rsize,
+      :version,
+      :next 
+    ]
     members.each do |nomme|
-      assert_not_nil(sysdic.send nomme.to_sym ) 
+      assert_not_nil(sysdic.send nomme ) 
     end
     
     # NoMethodError will be raised for anything else!

data/test/natto/tc_mecab.rb

@@ -1,8 +1,19 @@
 # coding: utf-8
+require 'rbconfig'
+require 'nkf'
 
 # TestMeCab encapsulates tests for the basic
 # behavior of Natto::MeCab.
 class TestMeCab < Test::Unit::TestCase
+
+  host_os = RbConfig::CONFIG['host_os']
+  # we need to transfrom from UTF-8 ot SJIS if we are on Windows!
+  if host_os =~ /mswin|mingw/i
+    TEST_STR = NKF.nkf("-Ws", '試験ですよ、これが。')
+  else
+    TEST_STR = '試験ですよ、これが。'
+  end
+
   def setup
     @m = Natto::MeCab.new
   end
@@ -37,9 +48,6 @@ class TestMeCab < Test::Unit::TestCase
     res = Natto::MeCab.build_options_str(:output_format_type=>"natto")
     assert_equal('--output-format-type=natto', res)
     
-    res = Natto::MeCab.build_options_str(:partial=>true)
-    assert_equal('--partial', res)
-
     res = Natto::MeCab.build_options_str(:node_format=>'%m\t%f[7]\n')
     assert_equal('--node-format=%m\t%f[7]\n', res)
     
@@ -76,10 +84,8 @@ class TestMeCab < Test::Unit::TestCase
     res = Natto::MeCab.build_options_str(:output_format_type=>"natto", 
                                          :userdic=>"/some/file", 
                                          :dicdir=>"/some/other/file",
-                                         :partial=>true,
                                          :all_morphs=>true)
-    assert_equal('--dicdir=/some/other/file --userdic=/some/file --all-morphs --output-format-type=natto --partial', res)
-
+    assert_equal('--dicdir=/some/other/file --userdic=/some/file --all-morphs --output-format-type=natto', res)
   end
 
   # Tests the construction and initial state of a Natto::MeCab instance.
@@ -96,7 +102,13 @@ class TestMeCab < Test::Unit::TestCase
     end
     assert_equal(opts, m.options)
     
-    opts = {:all_morphs=>true, :partial=>true, :allocate_sentence=>true}
+    opts = {:all_morphs=>true, :allocate_sentence=>true}
+    assert_nothing_raised do
+      m = Natto::MeCab.new(opts)
+    end
+    assert_equal(opts, m.options)
+    
+    opts = {:lattice_level=>999}
     assert_nothing_raised do
       m = Natto::MeCab.new(opts)
     end
@@ -126,4 +138,41 @@ class TestMeCab < Test::Unit::TestCase
   def test_version_accessor
     assert_equal('0.98', @m.version)
   end
+
+  # Tests Natto::MeCab parsing using the --all-morphs option.
+  def test_all_morphs
+    m = Natto::MeCab.new(:all_morphs=>true)
+    expected = `echo #{TEST_STR} | mecab --all-morphs`.lines.to_a
+    expected.delete_if {|e| e =~ /^(EOS|BOS)/ }
+    
+    actual   = m.parse(TEST_STR).lines.to_a
+    actual.delete_if {|e| e =~ /^(EOS|BOS|\t)/ }
+    
+    assert_equal(expected, actual)
+  end
+
+  # Tests Natto::MeCab parsing (default parse_tostr).
+  def test_parse_tostr_default
+    expected = `echo #{TEST_STR} | mecab`.lines.to_a
+    expected.delete_if {|e| e =~ /^(EOS|BOS|\t)/ }
+
+    actual = @m.parse(TEST_STR).lines.to_a
+    actual.delete_if {|e| e =~ /^(EOS|BOS|\t)/ }
+
+    assert_equal(expected, actual)
+  end
+
+  # Tests Natto::MeCab parsing (default parse_tonode).
+  def test_parse_tonode_default
+    expected = `echo #{TEST_STR} | mecab`.lines.to_a
+    expected.delete_if {|e| e =~ /^(EOS|BOS|\t)/ }
+
+    actual = []
+    @m.parse(TEST_STR) do |node|
+      actual << "#{node.surface}\t#{node.feature}\n"
+    end
+    actual.delete_if {|e| e =~ /^(EOS|BOS|\t)/ }
+
+    assert_equal(expected, actual)
+  end
 end 

data/test/natto/tc_mecabnode.rb

@@ -0,0 +1,106 @@
+# coding: utf-8
+require 'rbconfig'
+require 'nkf'
+
+# TestMeCabNode encapsulates tests for the basic
+# behavior of Natto::MeCabNode
+class TestMeCabNode < Test::Unit::TestCase
+  
+  host_os = RbConfig::CONFIG['host_os']
+  # we need to transfrom from UTF-8 ot SJIS if we are on Windows!
+  if host_os =~ /mswin|mingw/i
+    TEST_STR = NKF.nkf("-Ws", '試験ですよ、これが。')
+  else
+    TEST_STR = '試験ですよ、これが。'
+  end
+
+  def setup
+    nm = Natto::MeCab.new
+    @nodes = []
+    nm.parse(TEST_STR) { |n| @nodes << n }
+  end
+
+  def teardown
+    @nodes = nil
+  end
+
+  # Tests the surface and feature accessors methods.
+  def test_surface_and_feature_accessors
+    raw = `echo #{TEST_STR} | mecab`.lines.to_a
+    raw.delete_if {|e| e =~ /^(EOS|BOS|\t)/ }
+    expected = {}
+    raw.each do |l|
+      tokens = l.split("\t")
+      expected[tokens[0]]=tokens[1].strip
+    end
+
+    actual = {}
+    @nodes.each do |n|
+      actual[n.surface]=n.feature if (n.stat==Natto::MeCabNode::NOR_NODE || 
+                                      n.stat==Natto::MeCabNode::UNK_NODE)
+    end
+    
+    assert_equal(expected, actual)
+  end
+
+  # Tests MeCabNode#surface to show that it is consistent
+  # no matter how many times it is invoked.
+  def test_manysurfaces
+    @nodes.each do |n|
+      expected = n.surface
+      5.times { assert_equal(expected, n.surface) }
+    end
+  end
+
+  # Tests MeCabNode#feature to show that it is consistent
+  # no matter how many times it is invoked.
+  def test_manyfeature
+    @nodes.each do |n|
+      expected = n.feature
+      5.times { assert_equal(expected, n.feature) }
+    end
+  end
+
+  # Tests that the accessors of Natto::MeCabNode exist.
+  # Note: Object#id is deprecated in 1.9.n, but comes with a warning
+  #       in 1.8.n
+  def test_mecabnode_accessors
+    node = @nodes[0]
+    members = [
+      :prev,
+      :next,
+      :enext,
+      :bnext,
+      :rpath,
+      :lpath,
+      :begin_node_list,
+      :end_node_list,
+      :surface,
+      :feature,
+      :id,
+      :length,
+      :rlength,
+      :rcAttr,
+      :lcAttr,
+      :posid,
+      :char_type,
+      :stat,
+      :isbest,
+      :sentence_length,
+      :alpha,
+      :beta,
+      :prob,
+      :wcost,
+      :cost,
+      :token
+    ]
+    members.each do |nomme|
+      assert_not_nil(node.respond_to? nomme ) 
+    end
+    
+    # NoMethodError will be raised for anything else!
+    assert_raise NoMethodError do
+      node.send :unknown_attr
+    end
+  end
+end

data/test/test_natto.rb

@@ -5,6 +5,7 @@ require 'test/unit'
 require 'natto'
 
 [ '/test/natto/tc_mecab.rb',
+  '/test/natto/tc_mecabnode.rb',
   '/test/natto/tc_dictionaryinfo.rb',
   '/test/natto/tc_binding.rb' ].each do |tc|
   require File.join(File.expand_path('.'), tc)

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: natto
 version: !ruby/object:Gem::Version 
-  hash: 13
+  hash: 11
   prerelease: 
   segments: 
   - 0
-  - 4
-  - 1
-  version: 0.4.1
+  - 5
+  - 0
+  version: 0.5.0
 platform: ruby
 authors: 
 - Brooke M. Fujita
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-01-27 00:00:00 +09:00
+date: 2011-02-26 00:00:00 +09:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -51,6 +51,7 @@ files:
 - test/natto/tc_binding.rb
 - test/natto/tc_dictionaryinfo.rb
 - test/natto/tc_mecab.rb
+- test/natto/tc_mecabnode.rb
 - README.md
 - LICENSE
 - CHANGELOG