google_translate_diff 1.0.6 → 1.0.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0d3f09238c1e758c969c8e32d3c19997aee401c8
-  data.tar.gz: 3a9d0a82e79b6f0b7195972d38468d29883adb34
+  metadata.gz: f06bc3ed14da7bb671f3afc188375f72da6b1026
+  data.tar.gz: 62af00a1a43df3f8c1773346709c3f09e95cf186
 SHA512:
-  metadata.gz: 82046bb8370f241ed386c0f7c9e82438afd45f4e86a7aed6ebaf9090c090f7ff5c07fc5f90aa516856efcb1e1a9b0b29088971a0e279674fddf0a1349c3f4f89
-  data.tar.gz: ac93211c285e3b492e593b89ad28a04dd5af7997a1ed6f5a3daf07e38e0aea2d5d12b1b8383e31f7897d1702cd6087375a349023e7a6b67bda3345a8d8439b54
+  metadata.gz: f2662f618fc76e10877f78ca07dbce8bfda0b0f2c08da06643ea2d91491354631d15d44137f5fb76a33770fc59becb442471092fd030281e28c17d05722e7f1c
+  data.tar.gz: 39ddf3a505c96560016e24a9f3bd303e8e36556894e7aea7c612ce6c9078266ef1e4b7756f807e256507bf40105bcc226bab80f65273798b004b4756f8b0dd31

data/google_translate_diff.gemspec

@@ -44,7 +44,7 @@ between revisions of long texts.
   spec.add_development_dependency "simplecov"
 
   spec.add_dependency "google-cloud-translate"
-  spec.add_dependency "ox" #, "< 2.5"
+  spec.add_dependency "ox"
   spec.add_dependency "dry-initializer"
   spec.add_dependency "punkt-segmenter"
 end

data/lib/google_translate_diff/tokenizer.rb

@@ -1,40 +1,58 @@
 class GoogleTranslateDiff::Tokenizer < ::Ox::Sax
   def initialize(source)
     @pos = nil
-    @prev = 1
-    @skip = false
     @source = source
     @tokens = []
+    @context = []
+    @sequence = []
+    @indicies = []
   end
 
-  attr_reader :texts, :tokens, :prev, :pos
-
   def start_element(name)
-    @skip = true if SKIP.include?(name)
+    @context << name
+    @sequence << :markup
+    @indicies << @pos - 1
   end
 
   def end_element(name)
-    @skip = false if SKIP.include?(name)
+    @context.pop
+    @sequence << (nontranslate?(name) ? :notranslate : :markup)
+    @indicies << @pos - 1 unless @pos == @source.bytesize
   end
 
-  def text(value)
-    return if @skip
-    value = fix_utf(value)
-    return if value.strip.empty?
-
-    token.tap { |t| @tokens << [fix_utf(t), :markup] if t }
-    @tokens.concat(sentences(value))
+  def attr(name, value)
+    unless @context.last == :span && name == :class && value == "notranslate"
+      return
+    end
+    @sequence[-1] = :notranslate
+  end
 
-    @prev = @pos + value.bytesize
+  def text(_)
+    @sequence << (SKIP.include?(@context.last) ? :markup : :text)
+    @indicies << @pos - 1
   end
 
-  def token
-    return if @prev == @pos
-    fix_utf(@source.byteslice((@prev - 1)..(@pos - 2)))
+  # rubocop:disable Metrics/AbcSize
+  def tokens
+    raw_tokens.each_with_object([]) do |token, tokens|
+      if tokens.empty?
+        tokens << token
+      elsif tokens.last[1] == token[1]
+        tokens.last[0].concat(token[0])
+      else
+        tokens.concat(sentences(tokens.pop[0])) if tokens.last[1] == :text
+        tokens << token
+      end
+    end
   end
+  # rubocop:enable Metrics/AbcSize
 
-  # Splits text by sentences
+  private
+
+  # rubocop: disable Metrics/MethodLength
   def sentences(value)
+    return [] if value.empty?
+
     boundaries =
       Punkt::SentenceTokenizer
       .new(value)
@@ -49,10 +67,17 @@ class GoogleTranslateDiff::Tokenizer < ::Ox::Sax
       [value[left..right], :text]
     end
   end
+  # rubocop:enable Metrics/MethodLength
 
-  def cut_last_token
-    last_token = fix_utf(@source.byteslice((@prev - 1)..-1))
-    @tokens << [last_token, :markup] if last_token != ""
+  def raw_tokens
+    @indicies.map.with_index do |i, n|
+      first = i
+      last = (@indicies[n + 1] || 0) - 1
+      value = fix_utf(@source.byteslice(first..last))
+      type = @sequence[n]
+      type = :text if type == :notranslate
+      [value, type]
+    end
   end
 
   def fix_utf(value)
@@ -61,12 +86,15 @@ class GoogleTranslateDiff::Tokenizer < ::Ox::Sax
     )
   end
 
+  def nontranslate?(name)
+    @sequence[-2] == :notranslate && name == :span
+  end
+
   class << self
     def tokenize(value)
       return [] if value.nil?
       tokenizer = new(value).tap do |h|
         Ox.sax_parse(h, StringIO.new(value), HTML_OPTIONS)
-        h.cut_last_token
       end
       tokenizer.tokens
     end

data/lib/google_translate_diff/version.rb

@@ -1,3 +1,3 @@
 module GoogleTranslateDiff
-  VERSION = "1.0.6".freeze
+  VERSION = "1.0.7".freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: google_translate_diff
 version: !ruby/object:Gem::Version
-  version: 1.0.6
+  version: 1.0.7
 platform: ruby
 authors:
 - Victor Sokolov
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-08-30 00:00:00.000000000 Z
+date: 2017-08-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler