tiny_segmenter 0.0.2 → 0.0.4

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: ed996e5195f9e29609e110d175c3d002875e5496
+  data.tar.gz: 00405bd85ee522fc3d0b7271cf46fe06bca298a8
+SHA512:
+  metadata.gz: e5e0455da693e7eba988f3a921d05c13db13756d9658b0481643963d35367cc207ea02a61f45e02be288ba0a4315b202fc95e6c3bbf88b2fb97b74df6afcd8bd
+  data.tar.gz: b3d581557d7097c012bb565ee736265892b686f0b7b4fa286c07409e785da2664cb46f4e7971417d530a277ed8cb302df632c91fe7b85d106ef0d2b20b01a24c

data/.gitignore

@@ -1,2 +1,4 @@
 *.gem
 .DS_Store
+Gemfile.lock
+.rvmrc

data/.travis.yml

@@ -0,0 +1,7 @@
+language: ruby
+rvm:
+  - 1.9.2
+  - 1.9.3
+  - 2.0.0
+  - jruby-19mode # JRuby in 1.9 mode
+  - jruby-20mode

data/Gemfile

@@ -1,4 +1,3 @@
-source "http://rubygems.org"
+source "https://rubygems.org"
 
-# Specify your gem's dependencies in method_decorators.gemspec
 gemspec

data/README.md

@@ -1,4 +1,6 @@
-Ruby port of [TinySegmenter.js](http://chasen.org/~taku/software/TinySegmenter/) for tokenizing Japanese text.
+Ruby port of [TinySegmenter.js](http://chasen.org/~taku/software/TinySegmenter/) for tokenizing Japanese text. Ruby 1.9 or higher required.
+
+[![Build Status](https://secure.travis-ci.org/6/tiny_segmenter.png?branch=master)](http://travis-ci.org/6/tiny_segmenter)
 
 ### Install
 

data/Rakefile

@@ -0,0 +1,8 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"
+
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/lib/tiny_segmenter.rb

@@ -3,6 +3,9 @@ require "tiny_segmenter/version"
 require "tiny_segmenter/segmentation_model"
 
 class TinySegmenter
+  WhitespaceOnlyRegex = Regexp.compile("^[　 ]+$")
+  PunctuationRegex = Regexp.compile("^[-–—―.。・（）()［］｛｝{}【】⟨⟩、､,，،…‥〽「」『』〜~！!：:？?\"'|_＿“”‘’;/⁄／«»]+$")
+
   def initialize
     @chartype = []
     @model = SegmentationModel.new
@@ -20,7 +23,7 @@ class TinySegmenter
     end
   end
 
-  def segment(text)
+  def segment(text, options = {})
     return []  if text.nil? || text.strip.empty?
     text = text.strip
     result = []
@@ -28,7 +31,8 @@ class TinySegmenter
     ctypes = %w[O O O]
     text.split(//).each do |char|
       char.strip!
-      next  if char.empty?
+      next  if char.empty? || char.match(WhitespaceOnlyRegex)
+      next  if options[:ignore_punctuation] && char.match(PunctuationRegex)
       segments << char
       ctypes << ctype(char)
     end

data/lib/tiny_segmenter/version.rb

@@ -1,3 +1,3 @@
 class TinySegmenter
-  VERSION = "0.0.2"
+  VERSION = "0.0.4"
 end

data/spec/tiny_segmenter_spec.rb

@@ -4,20 +4,41 @@ require 'spec_helper'
 describe TinySegmenter do
   subject{ TinySegmenter.new }
 
-  it "tokenizes Japanese text fairly accurately" do
-    subject.segment("極めてコンパクトな日本語分かち書きソフトウェアです。").should == \
-      ["極めて", "コンパクト", "な", "日本", "語分", "かち", "書き", "ソフトウェア", "です", "。"]
-  end
+  describe "#segment" do
+    it "tokenizes Japanese text fairly accurately" do
+      subject.segment("極めてコンパクトな日本語分かち書きソフトウェアです。").should == \
+        ["極めて", "コンパクト", "な", "日本", "語分", "かち", "書き", "ソフトウェア", "です", "。"]
+    end
 
-  it "removes any whitespace-only or empty tokens" do
-    subject.segment("書かれた 極めて    コンパクト").should_not include("", " ")
-  end
+    it "removes any whitespace-only or empty tokens" do
+      subject.segment("書かれた 極めて    コンパクト").should_not include("", " ", nil)
+    end
+
+    it "removes full-width space (U+3000) tokens" do
+      sentence = "すてき！　男性が歌う「夢やぶれて」もいいね。"
+      full_width_space = "　"
+      sentence.should include(full_width_space)
+      subject.segment(sentence).should_not include (full_width_space)
+    end
+
+    it "tokenizes interspersed non-Japanese words correctly" do
+      subject.segment("TinySegmenterはRubyだけで").should == ["TinySegmenter", "は", "Ruby", "だけ", "で"]
+    end
+
+    context "with ignore_punctuation option not set" do
+      it "includes punctuation-only tokens" do
+        subject.segment("すてき！?　男性が、歌う「夢やぶれて」もいいね。...").should include("。", "！", "?", "、", "「", "」", "...")
+      end
+    end
 
-  it "tokenizes interspersed non-Japanese words correctly" do
-    subject.segment("TinySegmenterはRubyだけで").should == ["TinySegmenter", "は", "Ruby", "だけ", "で"]
+    context "with ignore_punctuation option set" do
+      it "removes all punctuation-only tokens" do
+        subject.segment("すてき！?　男性が、歌う「夢やぶれて」もいいね。...", ignore_punctuation: true).should_not include("。", "！", "?", "、", "「", "」", "...")
+      end
+    end
   end
 
   it "has a version" do
-    TinySegmenter::VERSION.should_not be_empty
+    TinySegmenter::VERSION.should be_kind_of(String)
   end
 end

data/tiny_segmenter.gemspec

@@ -5,7 +5,7 @@ require 'tiny_segmenter/version'
 Gem::Specification.new do |s|
   s.name = 'tiny_segmenter'
   s.version = TinySegmenter::VERSION
-  s.date = '2012-08-27'
+  s.date = '2013-03-30'
   s.summary = "Ruby port of TinySegmenter.js for tokenizing Japanese text."
   s.description = "Ruby port of TinySegmenter.js for tokenizing Japanese text."
   s.authors = ["Peter Graham"]

metadata

@@ -1,46 +1,41 @@
 --- !ruby/object:Gem::Specification
 name: tiny_segmenter
 version: !ruby/object:Gem::Version
-  version: 0.0.2
-  prerelease: 
+  version: 0.0.4
 platform: ruby
 authors:
 - Peter Graham
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-08-27 00:00:00.000000000 Z
+date: 2013-03-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 description: Ruby port of TinySegmenter.js for tokenizing Japanese text.
@@ -52,8 +47,10 @@ extra_rdoc_files: []
 files:
 - .gitignore
 - .rspec
+- .travis.yml
 - Gemfile
 - README.md
+- Rakefile
 - lib/tiny_segmenter.rb
 - lib/tiny_segmenter/segmentation_model.rb
 - lib/tiny_segmenter/version.rb
@@ -62,27 +59,26 @@ files:
 - tiny_segmenter.gemspec
 homepage: http://github.com/6/tiny_segmenter
 licenses: []
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.21
+rubygems_version: 2.0.0
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: Ruby port of TinySegmenter.js for tokenizing Japanese text.
 test_files:
 - spec/spec_helper.rb