google_translate_diff 1.0.10 → 1.0.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: cbd316d964f6fdb9f119c66b2087e475bbed2b04
-  data.tar.gz: 02ae88623b5b4ddddebbf9b23a9bb02075516d6a
+  metadata.gz: 07c59774517f93999222ed67cd66eccb5ffad592
+  data.tar.gz: f75bd48f486fca6f1db63c1c188766b5112bc71c
 SHA512:
-  metadata.gz: 84d6f5f4287a586455900838a534760a1eec4755551ee920325336d6d513a0b59b0727acfadbcb84a82cc7a17b257bcce4c4ae94d4c8f1199948ce3a2b7be076
-  data.tar.gz: da22b6c0d1ddafeff3243f0e6b4aec4f4394451f22b307a8582c4a3581330f565f029d0db30907f1ef58f3c470a7a378e255527e2735998afae3af4f36039a56
+  metadata.gz: 7c7f741db39feb6880cb927a17ebe96b65f6e9d43a2bd0680e9aadece192c8786e086ef1e34f49581f981e86f504242a98c1b6cd3636b02cca3c94af564d176d
+  data.tar.gz: 96e64720cba873ba0a7513c7a390507ef78846281ae3b2dac4545cc65f8af437405f144260204e80654a9ed59bfe373622f858f2f3dc4a732c16728ac870c092

data/lib/google_translate_diff/tokenizer.rb

@@ -25,9 +25,10 @@ class GoogleTranslateDiff::Tokenizer < ::Ox::Sax
   end
 
   def attr(name, value)
-    unless @context.last == :span && name == :class && value == "notranslate"
-      return
-    end
+    return unless @context.last == :span
+    return unless name == :class && value == "notranslate"
+    return if notranslate?
+
     @sequence[-1] = :notranslate
   end
 
@@ -38,7 +39,8 @@ class GoogleTranslateDiff::Tokenizer < ::Ox::Sax
   end
 
   def tokens
-    @tokens ||= token_sequences_joined.tap { |tokens| make_sentences_from_last_token(tokens) }
+    @tokens ||= token_sequences_joined
+      .tap { |tokens| make_sentences_from_last_token(tokens) }
   end
 
   private
@@ -47,9 +49,11 @@ class GoogleTranslateDiff::Tokenizer < ::Ox::Sax
     raw_tokens.each_with_object([]) do |token, tokens|
       if tokens.empty? # Initial state
         tokens << token
-      elsif tokens.last[1] == token[1] # Join series of tokens of the same type into one
+      elsif tokens.last[1] == token[1]
+        # Join series of tokens of the same type into one
         tokens.last[0].concat(token[0])
-      else # If token before :markup is :text we need to split it into sentences
+      else
+        # If token before :markup is :text we need to split it into sentences
         make_sentences_from_last_token(tokens)
         tokens << token
       end
@@ -81,36 +85,47 @@ class GoogleTranslateDiff::Tokenizer < ::Ox::Sax
   end
   # rubocop:enable Metrics/MethodLength
 
+  # Whether the sequence is between `:notranslate` and `:end_notranslate`
+  def notranslate?
+    @sequence.select { |item| item[/notranslate/] }.last == :notranslate
+  end
+
+  # Returns the item for last opened span 
+  def end_span
+    return :markup unless notranslate?
+    opened_spans = @sequence
+      .reverse
+      .take_while { |item| item != :notranslate }
+      .map { |item| { span: 1, end_span: -1 }.fetch(item, 0) }
+      .reduce(0, :+)
+
+    opened_spans.positive? ? :end_span : :end_notranslate
+  end
+
   def raw_tokens
     @raw_tokens ||= @indicies.map.with_index do |i, n|
       first = i
       last = (@indicies[n + 1] || 0) - 1
       value = fix_utf(@source.byteslice(first..last))
       type = @sequence[n]
-      type = :text if type == :notranslate
+      type = :text if INNER_SPANS.include?(type)
       [value, type]
     end
   end
 
   def fix_utf(value)
-    value.encode(
-      "UTF-8", undef: :replace, invalid: :replace, replace: " "
-    )
-  end
-
-  def nontranslate?(name)
-    @sequence[-2] == :notranslate && name == :span
+    value.encode("UTF-8", undef: :replace, invalid: :replace, replace: " ")
   end
 
   def start_markup(name)
     @context << name
-    @sequence << :markup
+    @sequence << (notranslate? ? (name == :span ? :span : :text) : :markup)
     @indicies << @pos - 1
   end
 
   def end_markup(name)
     @context.pop
-    @sequence << (nontranslate?(name) ? :notranslate : :markup)
+    @sequence << (notranslate? ? (name == :span ? end_span : :text) : :markup)
     @indicies << @pos - 1 unless @pos == @source.bytesize
   end
 
@@ -125,5 +140,6 @@ class GoogleTranslateDiff::Tokenizer < ::Ox::Sax
   end
 
   SKIP = %i[script style].freeze
+  INNER_SPANS = %i[notranslate span end_span end_notranslate].freeze
   HTML_OPTIONS = { smart: true, skip: :skip_none }.freeze
 end

data/lib/google_translate_diff/version.rb

@@ -1,3 +1,3 @@
 module GoogleTranslateDiff
-  VERSION = "1.0.10".freeze
+  VERSION = "1.0.11".freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: google_translate_diff
 version: !ruby/object:Gem::Version
-  version: 1.0.10
+  version: 1.0.11
 platform: ruby
 authors:
 - Victor Sokolov
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-06-21 00:00:00.000000000 Z
+date: 2019-01-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler