bimyou_segmenter 1.1.1 → 1.2.0

data/bin/bimyou_segmenter

@@ -68,7 +68,9 @@ if (ARGV.size == 0)
         s = NKF.nkf(nkf_option, s)
       end
       begin
-        puts s
+        unless (s.empty?)
+          puts s
+        end
       rescue Errno::EPIPE
         exit 0
       end
@@ -84,7 +86,9 @@ else
         if (nkf_option)
           s = NKF.nkf(nkf_option, s)
         end
-        puts s
+        unless (s.empty?)
+          puts s
+        end
       end
     end
   end

data/lib/bimyou_segmenter/aozora_model.rb

@@ -16,6 +16,7 @@ module BimyouSegmenter
       end
       
       white_space = options[:white_space].nil? ? false : options[:white_space]
+      symbol = options[:symbol].nil? ? true : options[:symbol]
       
       wakachi = []
       text_chars = text.chars.to_a
@@ -155,17 +156,23 @@ module BimyouSegmenter
         end
       end
       wakachi << word
-      if (white_space)
-        wakachi
-      else
-        wakachi.reject{|v| v.match(WHITE_SPACE) }
+      unless (white_space)
+        wakachi = wakachi.reject{|v| v.match(WHITE_SPACE) }
       end
+      unless (symbol)
+        wakachi = wakachi.reject{|v| v.match(SYMBOL) }
+      end
+      wakachi
     end
     
     private
     KANJI = Regexp.new('[々〇ヵヶ' + [0x3400].pack('U') + '-' + [0x9FFF].pack('U') +
                        [0xF900].pack('U') + '-' + [0xFAFF].pack('U') +
                        [0x20000].pack('U') + '-' + [0x2FFFF].pack('U') + ']')
+    SYMBOL = Regexp.new('^[^々〇' + [0x3400].pack('U') + '-' + [0x9FFF].pack('U') +
+                        [0xF900].pack('U') + '-' + [0xFAFF].pack('U') +
+                        [0x20000].pack('U') + '-' + [0x2FFFF].pack('U') +
+                        '\s　ぁ-ゞァ-ヾa-zA-Zａ-ｚＡ-Ｚ0-9０-９]+$')
     WHITE_SPACE = /[\s　]/
     
     def self.char_types(chars)

data/lib/bimyou_segmenter/version.rb

@@ -1,4 +1,4 @@
 # -*- coding: utf-8 -*-
 module BimyouSegmenter
-  VERSION = "1.1.1"
+  VERSION = "1.2.0"
 end

data/test/bimyou_segmenter_test.rb

@@ -33,6 +33,18 @@ class BimyouSegmenterTest < Test::Unit::TestCase
     assert_equal tokens[2], "\r"
     assert_equal tokens[3], "\n"
     assert_equal tokens[4], "　"
+
+    tokens = BimyouSegmenter.segment("「私はトマトです。」", :symbol => false)
+    assert_equal tokens.size, 4
+    assert_equal tokens[0], "私"
+    assert_equal tokens[1], "は"
+    assert_equal tokens[2], "トマト"
+    assert_equal tokens[3], "です"
+    
+    tokens = BimyouSegmenter.segment("hello. world? = !", :symbol => false)
+    assert_equal tokens.size, 2
+    assert_equal tokens[0], "hello"
+    assert_equal tokens[1], "world"
     
     assert_equal BimyouSegmenter.segment("").size, 0
     assert_equal BimyouSegmenter.segment(nil).size, 0

metadata

@@ -1,44 +1,35 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: bimyou_segmenter
-version: !ruby/object:Gem::Version 
-  prerelease: false
-  segments: 
-  - 1
-  - 1
-  - 1
-  version: 1.1.1
+version: !ruby/object:Gem::Version
+  version: 1.2.0
+  prerelease: 
 platform: ruby
-authors: 
+authors:
 - nagadomi
 autorequire: 
 bindir: bin
 cert_chain: []
-
-date: 2012-05-28 00:00:00 +09:00
-default_executable: 
-dependencies: 
-- !ruby/object:Gem::Dependency 
+date: 2012-06-01 00:00:00.000000000Z
+dependencies:
+- !ruby/object:Gem::Dependency
   name: test-unit
-  prerelease: false
-  requirement: &id001 !ruby/object:Gem::Requirement 
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        segments: 
-        - 0
-        version: "0"
+  requirement: &5715420 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :development
-  version_requirements: *id001
+  prerelease: false
+  version_requirements: *5715420
 description: Japanese Word Segmenter
-email: 
+email:
 - nagadomi@nurs.or.jp
-executables: 
+executables:
 - bimyou_segmenter
 extensions: []
-
 extra_rdoc_files: []
-
-files: 
+files:
 - .gitignore
 - Changelog
 - Gemfile
@@ -52,36 +43,30 @@ files:
 - lib/bimyou_segmenter/version.rb
 - test/bimyou_segmenter_test.rb
 - test/test_helper.rb
-has_rdoc: true
 homepage: http://github.com/nagadomi/bimyou_segmenter
 licenses: []
-
 post_install_message: 
 rdoc_options: []
-
-require_paths: 
+require_paths:
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      segments: 
-      - 0
-      version: "0"
-required_rubygems_version: !ruby/object:Gem::Requirement 
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      segments: 
-      - 0
-      version: "0"
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
 requirements: []
-
 rubyforge_project: 
-rubygems_version: 1.3.6
+rubygems_version: 1.8.10
 signing_key: 
 specification_version: 3
 summary: Japanese Word Segmenter
-test_files: 
+test_files:
 - test/bimyou_segmenter_test.rb
 - test/test_helper.rb