google_translate_diff 1.0.9 → 1.0.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: bc126f78f09507a6cdfe8116479b3629d0eac062
-  data.tar.gz: a92719bc7447df9c530127ec0ce48e4f3e341329
+  metadata.gz: cbd316d964f6fdb9f119c66b2087e475bbed2b04
+  data.tar.gz: 02ae88623b5b4ddddebbf9b23a9bb02075516d6a
 SHA512:
-  metadata.gz: 0d120b67393f006ced3d955ece7afcc9913d7d6c43c4421ee8f87e7f5028743c0b7bfc6add578ccb2ea59bc814a76e7e51a9e04294676c1ae427ff59c433fe4a
-  data.tar.gz: d7230c98ea93cfc03de94a0496779e14ef3abb7fe70d4cca769a72a8708b08539222e4e369e0423b8ddcd870a995f0608dc83ccbd88b6cbaac95ea3e53944f0a
+  metadata.gz: 84d6f5f4287a586455900838a534760a1eec4755551ee920325336d6d513a0b59b0727acfadbcb84a82cc7a17b257bcce4c4ae94d4c8f1199948ce3a2b7be076
+  data.tar.gz: da22b6c0d1ddafeff3243f0e6b4aec4f4394451f22b307a8582c4a3581330f565f029d0db30907f1ef58f3c470a7a378e255527e2735998afae3af4f36039a56

data/.rubocop.yml

@@ -10,3 +10,6 @@ Style/ClassAndModuleChildren:
 Metrics/BlockLength:
   Exclude:
     - spec/**/*
+
+Metrics/LineLength:
+  Max: 120

data/lib/google_translate_diff/tokenizer.rb

@@ -8,16 +8,20 @@ class GoogleTranslateDiff::Tokenizer < ::Ox::Sax
     @indicies = []
   end
 
+  def instruct(target)
+    start_markup(target)
+  end
+
+  def end_instruct(target)
+    end_markup(target)
+  end
+
   def start_element(name)
-    @context << name
-    @sequence << :markup
-    @indicies << @pos - 1
+    start_markup(name)
   end
 
   def end_element(name)
-    @context.pop
-    @sequence << (nontranslate?(name) ? :notranslate : :markup)
-    @indicies << @pos - 1 unless @pos == @source.bytesize
+    end_markup(name)
   end
 
   def attr(name, value)
@@ -33,22 +37,29 @@ class GoogleTranslateDiff::Tokenizer < ::Ox::Sax
     @indicies << @pos - 1
   end
 
-  # rubocop:disable Metrics/AbcSize
   def tokens
-    @tokens ||= raw_tokens.each_with_object([]) do |token, tokens|
-      if tokens.empty?
+    @tokens ||= token_sequences_joined.tap { |tokens| make_sentences_from_last_token(tokens) }
+  end
+
+  private
+
+  def token_sequences_joined
+    raw_tokens.each_with_object([]) do |token, tokens|
+      if tokens.empty? # Initial state
         tokens << token
-      elsif tokens.last[1] == token[1]
+      elsif tokens.last[1] == token[1] # Join series of tokens of the same type into one
         tokens.last[0].concat(token[0])
-      else
-        tokens.concat(sentences(tokens.pop[0])) if tokens.last[1] == :text
+      else # If token before :markup is :text we need to split it into sentences
+        make_sentences_from_last_token(tokens)
         tokens << token
       end
     end
   end
-  # rubocop:enable Metrics/AbcSize
 
-  private
+  def make_sentences_from_last_token(tokens)
+    return if tokens.empty?
+    tokens.concat(sentences(tokens.pop[0])) if tokens.last[1] == :text
+  end
 
   # rubocop: disable Metrics/MethodLength
   def sentences(value)
@@ -91,13 +102,24 @@ class GoogleTranslateDiff::Tokenizer < ::Ox::Sax
     @sequence[-2] == :notranslate && name == :span
   end
 
+  def start_markup(name)
+    @context << name
+    @sequence << :markup
+    @indicies << @pos - 1
+  end
+
+  def end_markup(name)
+    @context.pop
+    @sequence << (nontranslate?(name) ? :notranslate : :markup)
+    @indicies << @pos - 1 unless @pos == @source.bytesize
+  end
+
   class << self
     def tokenize(value)
       return [] if value.nil?
       tokenizer = new(value).tap do |h|
         Ox.sax_parse(h, StringIO.new(value), HTML_OPTIONS)
       end
-      puts tokenizer.tokens.inspect
       tokenizer.tokens
     end
   end

data/lib/google_translate_diff/version.rb

@@ -1,3 +1,3 @@
 module GoogleTranslateDiff
-  VERSION = "1.0.9".freeze
+  VERSION = "1.0.10".freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: google_translate_diff
 version: !ruby/object:Gem::Version
-  version: 1.0.9
+  version: 1.0.10
 platform: ruby
 authors:
 - Victor Sokolov
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-10-17 00:00:00.000000000 Z
+date: 2018-06-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler