furigana 0.0.3 → 0.0.4

data/lib/furigana/mecab.rb

@@ -11,7 +11,16 @@ module Furigana
       def tokenize(text)
         surface_form, reading = 0, 1
         stdout, stderr, status = Open3.capture3("mecab -Ochasen", :stdin_data => sanitize_text(text))
-        stdout.split("\n").inject([]) do |output, line|
+
+        # Avoid `ArgumentError - invalid byte sequence in UTF-8`
+        lines = if stdout.valid_encoding?
+                  stdout.split("\n")
+                else
+                  stdout.encode!("UTF-8", "UTF-8", :invalid => :replace, :undef => :replace, :replace => "�")
+                  stdout.split("\n")
+                end
+
+        lines.inject([]) do |output, line|
           columns = line.split("\t")
           output << {
             :surface_form => columns[surface_form],

data/lib/furigana/reader.rb

@@ -16,12 +16,8 @@ module Furigana
       NKF.nkf("-h1 -w", k)
     end
 
-    def sdiff(first, second)
-      Diff::LCS.sdiff(first, second)
-    end
-
     def diff_token_surface_form_and_reading(token)
-      sdiff(k2h(token[:surface_form]), k2h(token[:reading]))
+      Diff::LCS.sdiff(k2h(token[:surface_form]), k2h(token[:reading]))
     end
 
     def add_furigana(token)
@@ -29,18 +25,20 @@ module Furigana
       kanji, yomi = 0, 1
 
       list = []
-      on_kanji = false
-      diff_token_surface_form_and_reading(token).each do |part|
-        case part.action
-        when states[:kanji_and_yomi]
-          list.push ['',''] unless on_kanji
-          list.last[kanji] += part.old_element
-          list.last[yomi] += part.new_element
-          on_kanji = true
-        when states[:yomi]
-          list.last[yomi] += part.new_element
-        when states[:kana]
-          on_kanji = false
+      if /\p{Han}/.match(token[:surface_form])
+        on_kanji = false
+        diff_token_surface_form_and_reading(token).each do |part|
+          case part.action
+          when states[:kanji_and_yomi]
+            list.push ['',''] unless on_kanji
+            list.last[kanji] += part.old_element
+            list.last[yomi] += part.new_element
+            on_kanji = true
+          when states[:yomi]
+            list.last[yomi] += part.new_element
+          when states[:kana]
+            on_kanji = false
+          end
         end
       end
       list

data/lib/furigana/version.rb

@@ -1,3 +1,3 @@
 module Furigana
-  VERSION = '0.0.3'
+  VERSION = '0.0.4'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: furigana
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.4
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-01-19 00:00:00.000000000 Z
+date: 2013-01-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake