text_detector 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 527080011ed97c18c7c900d0b9abdf52ff75f0f9
-  data.tar.gz: ef29428804f699210fac218d30dcb9ab8633820b
+  metadata.gz: a7d68797d530b617238813bcadb52cf8649d13f6
+  data.tar.gz: f615523b8676171aebb035c2d4432a3c596f2126
 SHA512:
-  metadata.gz: ddaaecd37594997719d38c5fdc6d96cac0efb7b9187ea9e5dedb16dda16bf8a4b5cad05988204d1c859186840feb9771eb5964a7713144764a0c1f43237c40e5
-  data.tar.gz: c6c3ebd4af514c0ae1f3d86a8625ba0dcdb4c25b69d3d20bf96e33157a03bfa8dff09f1607738534381694fc9feccb1285fe9e1384e5e3248409ab4fa5c2d6a8
+  metadata.gz: 1b0263082fe4138421f104844389afdf78aef50b864d6dbd7c2021176296da7f8ec86e41241bc4a80ff4d461de0fb38c8dcc5bfc3ff6ca8dc6eaae885c8bd274
+  data.tar.gz: 0c542dde4eff06e4a5a80be840acbde7e9ac3b061e03e84aea633a4b28fb032fa1c1e056d1b84d45f02ddaab79640408b820ddbfc532571de306f3f3923785cc

data/README.md

@@ -1,3 +1,4 @@
+[![Gem Version](https://badge.fury.io/rb/text_detector.svg)](http://badge.fury.io/rb/text_detector)
 [![Build Status](https://travis-ci.org/koshigoe/text_detector.svg)](https://travis-ci.org/koshigoe/text_detector)
 
 # TextDetector

data/bin/benchmark

@@ -51,39 +51,55 @@ regexp_detector = TextDetector.factory(:regexp, open('dictionary.txt'))
 mini_simple_detector = TextDetector.factory(:simple, open('mini-dictionary.txt'))
 mini_regexp_detector = TextDetector.factory(:regexp, open('mini-dictionary.txt'))
 
-results = { simple: [], regexp: [] }
-Benchmark.bm do |x|
-  x.report do
+Benchmark.bm(22) do |x|
+  x.report('simple detect') do
     testdata.each do |text|
-      results[:simple] << simple_detector.detect(text)
+      simple_detector.detect(text)
     end
   end
 
-  x.report do
+  x.report('regexp detect') do
     testdata.each do |text|
-      results[:regexp] << regexp_detector.detect(text)
+      regexp_detector.detect(text)
     end
   end
 
-  x.report do
+  x.report('mini simple detect') do
     testdata.each do |text|
-      results[:simple] << mini_simple_detector.detect(text)
+      mini_simple_detector.detect(text)
     end
   end
 
-  x.report do
+  x.report('mini regexp detect') do
     testdata.each do |text|
-      results[:regexp] << mini_regexp_detector.detect(text)
+      mini_regexp_detector.detect(text)
     end
   end
-end
 
-if results[:simple] == results[:regexp]
-  puts 'ok'
-else
-  puts 'ng'
-end
+  x.report('simple detect_all') do
+    testdata.each do |text|
+      simple_detector.detect_all(text)
+    end
+  end
+
+  x.report('regexp detect_all') do
+    testdata.each do |text|
+      regexp_detector.detect_all(text)
+    end
+  end
 
+  x.report('mini simple detect_all') do
+    testdata.each do |text|
+      mini_simple_detector.detect_all(text)
+    end
+  end
+
+  x.report('mini regexp detect_all') do
+    testdata.each do |text|
+      mini_regexp_detector.detect_all(text)
+    end
+  end
+end
 
 __END__
 　吾輩《わがはい》は猫である。名前はまだ無い。

data/lib/text_detector/detector/base.rb

@@ -12,6 +12,10 @@ module TextDetector
         raise NotImplementedError
       end
 
+      def detect_all(text)
+        raise NotImplementedError
+      end
+
       protected
 
       def setup

data/lib/text_detector/detector/regexp.rb

@@ -4,8 +4,20 @@ module TextDetector
   module Detector
     class Regexp < Base
       def detect(text)
-        detected = @re.match(text)
-        detected ? detected.to_s : nil
+        matched = @re.match(TextDetector.normalize(text))
+        if matched
+          offset = matched.offset(0)
+          text.slice(offset[0], offset[1] - offset[0])
+        else
+          nil
+        end
+      end
+
+      def detect_all(text)
+        TextDetector.normalize(text).to_enum(:scan, @re).map do
+          offset = ::Regexp.last_match.offset(0)
+          text.slice(offset[0], offset[1] - offset[0])
+        end
       end
 
       protected

data/lib/text_detector/detector/simple.rb

@@ -5,6 +5,20 @@ module TextDetector
     # BM法っぽく(トライ木を調べてる時に見かけた実装を参考に)
     class Simple < Base
       def detect(text)
+        detect_n(text, 1).first
+      end
+
+      def detect_all(text)
+        detect_n(text)
+      end
+
+      private
+
+      def detect_n(text, limit = nil)
+        results = []
+        original = text
+        text = TextDetector.normalize(original)
+
         # 0文字目から末尾の一つ前まで一文字ずつ始点を移動していく
         0.upto(text.size - 1) do |start|
           # 語の長さ配列から切り出し文字数を取り出していく
@@ -13,12 +27,15 @@ module TextDetector
             # 切り出した文字列の長さが、切り出し分より短ければ次のターン
             break if size > target.size
 
-            # 切り出した文字列が辞書に含まれていれば探索終了
-            return target if dictionary.lookup(target)
+            # 切り出した文字列が辞書に含まれていれば記録
+            results << original[start, size] if dictionary.lookup(target)
+
+            # 制限数までヒットしたら探索終了
+            return results if limit && results.size == limit
           end
         end
 
-        nil
+        results
       end
     end
   end

data/lib/text_detector/executor.rb

@@ -8,7 +8,11 @@ module TextDetector
     end
 
     def detect(text)
-      @detector.detect(TextDetector.normalize(text))
+      @detector.detect(TextDetector.shallow_normalize(text))
+    end
+
+    def detect_all(text)
+      @detector.detect_all(TextDetector.shallow_normalize(text))
     end
   end
 end

data/lib/text_detector/version.rb

@@ -1,3 +1,3 @@
 module TextDetector
-  VERSION = "0.1.0"
+  VERSION = "0.2.0"
 end

data/lib/text_detector.rb

@@ -7,7 +7,11 @@ module TextDetector
     Executor.new type, dictionary
   end
 
+  def self.shallow_normalize(text)
+    text.unicode_normalize(:nfc)
+  end
+
   def self.normalize(text)
-    NKF.nkf('--katakana -w', text).unicode_normalize(:nfc)
+    NKF.nkf('--katakana -w', shallow_normalize(text))
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: text_detector
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - koshigoe