bimyou_segmenter 1.0.1 → 1.1.1

data/Changelog

@@ -0,0 +1,5 @@
+2012-05-28 nagadomi <nagadomi@nurs.or.jp>
+	* bin/bimyou_segmenter: 文字コード変換に対応 (1.1.0)
+
+2012-05-27 nagadomi <nagadomi@nurs.or.jp>
+	* release

data/README.md

@@ -32,3 +32,11 @@ segmentは、分かち書きした結果を文字列の配列として返しま
 :white_space => trueを付けると空白文字や改行コードも含みます。この場合は、返した文字列配列を順に結合すると元の文字列と一致します。
 
 Ruby1.8.7の場合、$KCODE='u'にしないと動きません。
+
+    % bimyou_segmenter -d "|" -e ""
+    昼飯のスパゲティナポリタンを眺めながら、積年の疑問を考えていた。それは「なぜナポリタンは赤いのだろうか」という問いである。
+    昼飯|の|スパゲティナポリタン|を|眺め|ながら|、|積年|の|疑問|を|考え|て|い|た|。|それ|は|「|なぜ|ナポリタン|は|赤い|の|だろ|う|か|」|という|問い|で|ある|。
+
+    % bimyou_segmenter --help
+
+コマンドラインツールもあります。

data/bimyou_segmenter.gemspec

@@ -4,8 +4,8 @@ require File.expand_path('../lib/bimyou_segmenter/version', __FILE__)
 Gem::Specification.new do |gem|
   gem.authors       = ["nagadomi"]
   gem.email         = ["nagadomi@nurs.or.jp"]
-  gem.description   = %q{Japanese word segmenter.}
-  gem.summary       = %q{Japanese word segmenter.}
+  gem.description   = %q{Japanese Word Segmenter}
+  gem.summary       = %q{Japanese Word Segmenter}
   gem.homepage      = "http://github.com/nagadomi/bimyou_segmenter"
   
   gem.files         = `git ls-files`.split($\)

data/bin/bimyou_segmenter

@@ -3,7 +3,16 @@
 if (RUBY_VERSION < "1.9.0")
   require 'rubygems'
   $KCODE= 'u'
+  # jruby not work...
+  if (RUBY_PLATFORM.downcase =~ /mswin(?!ce)|mingw|cygwin|bccwin/)
+    nkf_option = '-s'
+  else
+    nkf_option = '-w'
+  end
+else
+  nkf_option = nil
 end
+require 'nkf'
 require 'bimyou_segmenter'
 require 'optparse'
 
@@ -21,6 +30,27 @@ opt = OptionParser.new do |opt|
   opt.on('-e STR', '--eos STR', 'EOS string') do |v|
     eos = v
   end
+  opt.on('-c ENCODING', '--output-encoding ENCODING', 'output encoding (euc-jp, sjis, utf-8)') do |v|
+    encoding = nil
+    option = nil
+    case v.downcase
+    when /(sjis)|(shift_jis)/
+      encoding = 'Shift_JIS'
+      option = '-s'
+    when /utf\-?8/
+      encoding = 'UTF-8'
+      option = '-w'
+    when /euc/
+      encoding = 'EUC-JP'
+      option = '-e'
+    end
+    if (encoding && $stdout.respond_to?(:set_encoding))
+      $stdout.set_encoding(encoding, 'UTF-8')
+      nkf_option = nil
+    else
+      nkf_option = option
+    end
+  end
 end
 opt.version = BimyouSegmenter::VERSION
 opt.banner = "Usage: bimyou_segmenter [options] [files]"
@@ -31,10 +61,14 @@ if (ARGV.size == 0)
   until $stdin.eof?
     line = $stdin.readline
     if (line)
-      line = line.chomp
+      line = NKF.nkf('-w', line).chomp
+      s = [BimyouSegmenter.segment(line, :white_space => white_space), eos
+          ].flatten(1).reject{|v| v.size == 0}.join(delimiter)
+      if (nkf_option)
+        s = NKF.nkf(nkf_option, s)
+      end
       begin
-        puts [BimyouSegmenter.segment(line, :white_space => white_space), eos
-             ].flatten(1).reject{|v| v.size == 0}.join(delimiter)
+        puts s
       rescue Errno::EPIPE
         exit 0
       end
@@ -44,9 +78,13 @@ else
   ARGV.each do |file|
     File.open(file) do |f|
       until f.eof?
-        line = f.readline
-        puts [BimyouSegmenter.segment(line, :white_space => white_space), eos
-             ].flatten(1).reject{|v| v.size == 0}.join(delimiter)
+        line = NKF.nkf('-w', f.readline)
+        s = [BimyouSegmenter.segment(line, :white_space => white_space), eos
+            ].flatten(1).reject{|v| v.size == 0}.join(delimiter)
+        if (nkf_option)
+          s = NKF.nkf(nkf_option, s)
+        end
+        puts s
       end
     end
   end

data/lib/bimyou_segmenter/version.rb

@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 module BimyouSegmenter
-  VERSION = "1.0.1"
+  VERSION = "1.1.1"
 end

data/lib/bimyou_segmenter.rb

@@ -1,3 +1,4 @@
+# coding: utf-8
 require 'bimyou_segmenter/version'
 require 'bimyou_segmenter/aozora_model'
 

metadata

@@ -1,36 +1,46 @@
---- !ruby/object:Gem::Specification
+--- !ruby/object:Gem::Specification 
 name: bimyou_segmenter
-version: !ruby/object:Gem::Version
-  version: 1.0.1
-  prerelease: 
+version: !ruby/object:Gem::Version 
+  prerelease: false
+  segments: 
+  - 1
+  - 1
+  - 1
+  version: 1.1.1
 platform: ruby
-authors:
+authors: 
 - nagadomi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-05-27 00:00:00.000000000Z
-dependencies:
-- !ruby/object:Gem::Dependency
+
+date: 2012-05-28 00:00:00 +09:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
   name: test-unit
-  requirement: &18169520 !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
   prerelease: false
-  version_requirements: *18169520
-description: Japanese word segmenter.
-email:
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id001
+description: Japanese Word Segmenter
+email: 
 - nagadomi@nurs.or.jp
-executables:
+executables: 
 - bimyou_segmenter
 extensions: []
+
 extra_rdoc_files: []
-files:
+
+files: 
 - .gitignore
+- Changelog
 - Gemfile
 - LICENSE
 - README.md
@@ -42,30 +52,36 @@ files:
 - lib/bimyou_segmenter/version.rb
 - test/bimyou_segmenter_test.rb
 - test/test_helper.rb
+has_rdoc: true
 homepage: http://github.com/nagadomi/bimyou_segmenter
 licenses: []
+
 post_install_message: 
 rdoc_options: []
-require_paths:
+
+require_paths: 
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
-  requirements:
-  - - ! '>='
-    - !ruby/object:Gem::Version
-      version: '0'
-required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
-  requirements:
-  - - ! '>='
-    - !ruby/object:Gem::Version
-      version: '0'
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
 requirements: []
+
 rubyforge_project: 
-rubygems_version: 1.8.10
+rubygems_version: 1.3.6
 signing_key: 
 specification_version: 3
-summary: Japanese word segmenter.
-test_files:
+summary: Japanese Word Segmenter
+test_files: 
 - test/bimyou_segmenter_test.rb
 - test/test_helper.rb