wovnrb 1.1.0 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e67d9a0a4d2f447222580c5d4e0a00995f8027fa
-  data.tar.gz: e8544f392fdf10c6c5101914cd7b721ed2e6c4e9
+  metadata.gz: 548c84031681d4273db42c24a7ccfd8faaad9b31
+  data.tar.gz: 6c82fab859281f1b946f4199a1737daaa5112842
 SHA512:
-  metadata.gz: f69d4ccb222bdfb97368d26c3585aaeb433c71d7b1a58b294d8b18cd098c8222149a1b41bb7912de82c88f7594e6d152600ca4e39167ed2f6987769761e2140c
-  data.tar.gz: 3e4ddf14d6c59a6eee023d878744c40a468874351ea27fe6138038506cef78529618950cf60c6af9670174dacd3ac98e12d70c2b306f54b927793cd7dec10bc7
+  metadata.gz: fd388862202899e8d3acbe2d5aa8fe8a55bc5c89684beec438df56da6d7c366bcd71bd72fcb049767586de4bd34f56cfdd275ba7cb1710f22e04dcf27f94c5b3
+  data.tar.gz: abee33fc5f353e531363d59001a258bed0b07ae4809b07560d10a0889371b4435e19b53ad00d3796157cbf96b87ae9859abe0baa33eafeaf48277b65d4310a1f

data/.circleci/config.yml

@@ -14,4 +14,5 @@ jobs:
           key: bundler-{{ checksum "Gemfile.lock" }}
           paths:
             - vendor/bundle
+      - run: bundle exec rubocop
       - run: bundle exec rake test

data/.gitignore

@@ -18,3 +18,5 @@ mkmf.log
 *.iml
 .ruby-version
 /*.gem
+wovn_error.log
+.byebug_history

data/.rubocop.yml

@@ -0,0 +1 @@
+inherit_from: .rubocop_todo.yml

data/.rubocop_todo.yml

@@ -0,0 +1,308 @@
+# This configuration was generated by
+# `rubocop --auto-gen-config`
+# on 2018-11-02 18:52:46 +0900 using RuboCop version 0.60.0.
+# The point is for the user to remove these configuration records
+# one by one as the offenses are removed from the code base.
+# Note that changes in the inspected code, or installation of new
+# versions of RuboCop, may require this file to be generated again.
+
+# Offense count: 1
+# Cop supports --auto-correct.
+Layout/EmptyLinesAroundAccessModifier:
+  Exclude:
+    - 'lib/wovnrb/text_caches/memory_cache.rb'
+
+# Offense count: 1
+# Cop supports --auto-correct.
+Layout/EmptyLinesAroundMethodBody:
+  Exclude:
+    - 'test/lib/text_caches/memory_cache_test.rb'
+
+# Offense count: 11
+# Cop supports --auto-correct.
+# Configuration parameters: AllowForAlignment, ForceEqualSignAlignment.
+Layout/ExtraSpacing:
+  Exclude:
+    - 'lib/wovnrb/lang.rb'
+
+# Offense count: 1
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedStyle, IndentationWidth.
+# SupportedStyles: special_inside_parentheses, consistent, align_braces
+Layout/IndentHash:
+  Exclude:
+    - 'lib/wovnrb/text_caches/cache_base.rb'
+
+# Offense count: 4
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedStyle.
+# SupportedStyles: auto_detection, squiggly, active_support, powerpack, unindent
+Layout/IndentHeredoc:
+  Exclude:
+    - 'test/lib/lang_test.rb'
+    - 'test/lib/wovnrb_test.rb'
+
+# Offense count: 1
+# Cop supports --auto-correct.
+# Configuration parameters: AllowForAlignment.
+Layout/SpaceAroundOperators:
+  Exclude:
+    - 'lib/wovnrb/text_caches/memory_cache.rb'
+
+# Offense count: 3
+# Configuration parameters: AllowSafeAssignment.
+Lint/AssignmentInCondition:
+  Exclude:
+    - 'lib/wovnrb/helpers/nokogumbo_helper.rb'
+
+# Offense count: 1
+Lint/DuplicateMethods:
+  Exclude:
+    - 'test/lib/headers_test.rb'
+
+# Offense count: 1
+Lint/ElseLayout:
+  Exclude:
+    - 'lib/wovnrb/headers.rb'
+
+# Offense count: 1
+Lint/ImplicitStringConcatenation:
+  Exclude:
+    - 'test/lib/lang_test.rb'
+
+# Offense count: 3
+# Cop supports --auto-correct.
+# Configuration parameters: AllowUnusedKeywordArguments, IgnoreEmptyMethods.
+Lint/UnusedMethodArgument:
+  Exclude:
+    - 'lib/wovnrb/text_caches/cache_base.rb'
+
+# Offense count: 5
+Lint/UselessAssignment:
+  Exclude:
+    - 'test/lib/store_test.rb'
+
+# Offense count: 22
+Metrics/AbcSize:
+  Max: 100
+
+# Offense count: 1
+# Configuration parameters: CountComments, ExcludedMethods.
+# ExcludedMethods: refine
+Metrics/BlockLength:
+  Max: 40
+
+# Offense count: 2
+# Configuration parameters: CountBlocks.
+Metrics/BlockNesting:
+  Max: 4
+
+# Offense count: 10
+# Configuration parameters: CountComments.
+Metrics/ClassLength:
+  Max: 4997
+
+# Offense count: 14
+Metrics/CyclomaticComplexity:
+  Max: 22
+
+# Offense count: 45
+# Configuration parameters: CountComments, ExcludedMethods.
+Metrics/MethodLength:
+  Max: 95
+
+# Offense count: 12
+Metrics/PerceivedComplexity:
+  Max: 22
+
+# Offense count: 3
+Naming/AccessorMethodName:
+  Exclude:
+    - 'lib/wovnrb/services/wovn_logger.rb'
+    - 'lib/wovnrb/text_caches/cache_base.rb'
+
+# Offense count: 120
+# Configuration parameters: .
+# SupportedStyles: snake_case, camelCase
+Naming/MethodName:
+  EnforcedStyle: snake_case
+  Exclude:
+    - 'test/lib/headers_test.rb'
+
+# Offense count: 5
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedStyle.
+# SupportedStyles: always, conditionals
+Style/AndOr:
+  Exclude:
+    - 'Rakefile'
+    - 'lib/wovnrb/helpers/nokogumbo_helper.rb'
+
+# Offense count: 1
+Style/CaseEquality:
+  Exclude:
+    - 'lib/wovnrb/lang.rb'
+
+# Offense count: 5
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedStyle.
+# SupportedStyles: is_a?, kind_of?
+Style/ClassCheck:
+  Exclude:
+    - 'lib/wovnrb/store.rb'
+
+# Offense count: 6
+Style/ClassVars:
+  Exclude:
+    - 'lib/wovnrb/text_caches/cache_base.rb'
+    - 'lib/wovnrb/text_caches/memory_cache.rb'
+
+# Offense count: 1
+# Cop supports --auto-correct.
+# Configuration parameters: Keywords.
+# Keywords: TODO, FIXME, OPTIMIZE, HACK, REVIEW
+Style/CommentAnnotation:
+  Exclude:
+    - 'lib/wovnrb/headers.rb'
+
+# Offense count: 13
+Style/Documentation:
+  Exclude:
+    - 'spec/**/*'
+    - 'test/**/*'
+    - 'lib/wovnrb.rb'
+    - 'lib/wovnrb/api_translator.rb'
+    - 'lib/wovnrb/headers.rb'
+    - 'lib/wovnrb/helpers/nokogumbo_helper.rb'
+    - 'lib/wovnrb/lang.rb'
+    - 'lib/wovnrb/railtie.rb'
+    - 'lib/wovnrb/services/html_converter.rb'
+    - 'lib/wovnrb/services/html_replace_marker.rb'
+    - 'lib/wovnrb/services/wovn_logger.rb'
+    - 'lib/wovnrb/settings.rb'
+    - 'lib/wovnrb/store.rb'
+    - 'lib/wovnrb/text_caches/cache_base.rb'
+    - 'lib/wovnrb/text_caches/memory_cache.rb'
+
+# Offense count: 2
+Style/DoubleNegation:
+  Exclude:
+    - 'lib/wovnrb.rb'
+    - 'lib/wovnrb/services/html_converter.rb'
+
+# Offense count: 3
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedStyle.
+# SupportedStyles: each, for
+Style/For:
+  Exclude:
+    - 'test/lib/headers_test.rb'
+
+# Offense count: 4
+# Configuration parameters: MinBodyLength.
+Style/GuardClause:
+  Exclude:
+    - 'lib/wovnrb/services/html_converter.rb'
+    - 'lib/wovnrb/store.rb'
+    - 'test/lib/api_translator_test.rb'
+
+# Offense count: 1
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedStyle, UseHashRocketsWithSymbolValues, PreferHashRocketsForNonAlnumEndingSymbols.
+# SupportedStyles: ruby19, hash_rockets, no_mixed_keys, ruby19_no_mixed_keys
+Style/HashSyntax:
+  Exclude:
+    - 'Rakefile'
+
+# Offense count: 13
+# Cop supports --auto-correct.
+Style/IfUnlessModifier:
+  Exclude:
+    - 'Gemfile'
+    - 'Rakefile'
+    - 'lib/wovnrb/helpers/nokogumbo_helper.rb'
+    - 'lib/wovnrb/store.rb'
+    - 'lib/wovnrb/text_caches/memory_cache.rb'
+
+# Offense count: 1
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedStyle.
+# SupportedStyles: both, prefix, postfix
+Style/NegatedIf:
+  Exclude:
+    - 'lib/wovnrb/store.rb'
+
+# Offense count: 2
+# Cop supports --auto-correct.
+# Configuration parameters: AutoCorrect, EnforcedStyle, IgnoredMethods.
+# SupportedStyles: predicate, comparison
+Style/NumericPredicate:
+  Exclude:
+    - 'spec/**/*'
+    - 'lib/wovnrb/store.rb'
+
+# Offense count: 9
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedStyle.
+# SupportedStyles: short, verbose
+Style/PreferredHashMethods:
+  Exclude:
+    - 'lib/wovnrb/store.rb'
+    - 'lib/wovnrb/text_caches/memory_cache.rb'
+
+# Offense count: 2
+# Cop supports --auto-correct.
+# Configuration parameters: .
+# SupportedStyles: compact, exploded
+Style/RaiseArgs:
+  EnforcedStyle: compact
+
+# Offense count: 22
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedStyle, AllowInnerSlashes.
+# SupportedStyles: slashes, percent_r, mixed
+Style/RegexpLiteral:
+  Exclude:
+    - 'lib/wovnrb/api_translator.rb'
+    - 'lib/wovnrb/headers.rb'
+    - 'lib/wovnrb/lang.rb'
+    - 'lib/wovnrb/services/html_converter.rb'
+
+# Offense count: 4
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedStyle.
+# SupportedStyles: implicit, explicit
+Style/RescueStandardError:
+  Exclude:
+    - 'lib/wovnrb/api_translator.rb'
+    - 'lib/wovnrb/lang.rb'
+    - 'lib/wovnrb/services/wovn_logger.rb'
+
+# Offense count: 1
+# Cop supports --auto-correct.
+Style/StderrPuts:
+  Exclude:
+    - 'lib/wovnrb/services/wovn_logger.rb'
+
+# Offense count: 4
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedStyle, ConsistentQuotesInMultiline.
+# SupportedStyles: single_quotes, double_quotes
+Style/StringLiterals:
+  Exclude:
+    - 'Rakefile'
+    - 'lib/wovnrb/store.rb'
+
+# Offense count: 7
+# Cop supports --auto-correct.
+Style/ZeroLengthPredicate:
+  Exclude:
+    - 'Rakefile'
+    - 'lib/wovnrb/helpers/nokogumbo_helper.rb'
+    - 'lib/wovnrb/store.rb'
+
+# Offense count: 1436
+# Configuration parameters: AllowHeredoc, AllowURI, URISchemes, IgnoreCopDirectives, IgnoredPatterns.
+# URISchemes: http, https
+Metrics/LineLength:
+  Max: 1424

data/Rakefile

@@ -1,5 +1,5 @@
-#require "bundler/gem_tasks"
-#require 'rake/extensiontask'
+# require "bundler/gem_tasks"
+# require 'rake/extensiontask'
 require 'rake/testtask'
 require 'pry'
 
@@ -24,18 +24,17 @@ end
 desc 'Run tests'
 task :default => :test
 
-#Rake::ExtensionTask.new 'dom' do |ext|
+# Rake::ExtensionTask.new 'dom' do |ext|
 #  ext.lib_dir = 'lib/wovnrb'
-#end
+# end
 
-#gemspec = Gem::Specification.load('wovnrb.gemspec')
-#Rake::ExtensionTask.new do |ext|
-#	ext.name = 'dom'
-#	ext.source_pattern = "*.{cpp,h}"
-#	ext.ext_dir = 'ext/wovnrb'
-#	ext.lib_dir = 'lib/wovnrb'
-#	ext.gem_spec = gemspec
-#end
+# gemspec = Gem::Specification.load('wovnrb.gemspec')
+# Rake::ExtensionTask.new do |ext|
+#  ext.name = 'dom'
+#  ext.source_pattern = "*.{cpp,h}"
+#  ext.ext_dir = 'ext/wovnrb'
+#  ext.lib_dir = 'lib/wovnrb'
+#  ext.gem_spec = gemspec
+# end
 #
-#task :default => [:compile]
-
+# task :default => [:compile]

data/lib/wovnrb.rb

@@ -1,135 +1,88 @@
 require 'rack'
-require 'wovnrb/store'
+require 'wovnrb/api_translator'
 require 'wovnrb/headers'
+require 'wovnrb/store'
 require 'wovnrb/lang'
+require 'wovnrb/services/html_converter'
+require 'wovnrb/services/html_replace_marker'
 require 'nokogumbo'
 require 'active_support'
-#require 'dom'
 require 'json'
 require 'wovnrb/helpers/nokogumbo_helper'
-require 'wovnrb/api_data'
 require 'wovnrb/text_caches/cache_base'
-require 'wovnrb/html_replacers/replacer_base'
-require 'wovnrb/html_replacers/link_replacer'
-require 'wovnrb/html_replacers/text_replacer'
-require 'wovnrb/html_replacers/meta_replacer'
-require 'wovnrb/html_replacers/input_replacer'
-require 'wovnrb/html_replacers/image_replacer'
-require 'wovnrb/html_replacers/script_replacer'
-require 'wovnrb/html_replacers/unified_values/text_replacer'
-require 'wovnrb/html_replacers/unified_values/text_scraper'
-require 'wovnrb/html_replacers/unified_values/values_stack'
-require 'wovnrb/html_replacers/unified_values/element_category'
-require 'wovnrb/html_replacers/unified_values/dst_swapping_targets_creator'
-require 'wovnrb/html_replacers/unified_values/node_swapping_targets_creator'
 require 'wovnrb/railtie' if defined?(Rails)
 require 'wovnrb/version'
 
 module Wovnrb
   class Interceptor
-    def initialize(app, opts={})
+    def initialize(app, opts = {})
       @app = app
       @store = Store.instance
-      opts = opts.each_with_object({}){|(k,v),memo| memo[k.to_s]=v}
+      opts = opts.each_with_object({}) { |(k, v), memo| memo[k.to_s] = v }
       @store.update_settings(opts)
       CacheBase.set_single(@store.settings)
     end
 
     def call(env)
       @store.settings.clear_dynamic_settings!
-      unless Store.instance.valid_settings?
-        return @app.call(env)
-      end
+      return @app.call(env) unless Store.instance.valid_settings?
+
       @env = env
       headers = Headers.new(env, @store.settings)
-      if @store.settings['test_mode'] && @store.settings['test_url'] != headers.url
-        return @app.call(env)
-      end
-      #redirect if the path is set to the default language (for SEO purposes)
-      if (headers.path_lang == @store.settings['default_lang'])
+      return @app.call(env) if @store.settings['test_mode'] && @store.settings['test_url'] != headers.url
+
+      # redirect if the path is set to the default language (for SEO purposes)
+      if headers.path_lang == @store.settings['default_lang']
         redirect_headers = headers.redirect(@store.settings['default_lang'])
         return [307, redirect_headers, ['']]
       end
-      lang = headers.lang_code
 
       # pass to application
       status, res_headers, body = @app.call(headers.request_out)
 
-      unless res_headers["Content-Type"] =~ /html/
-        return output(headers, status, res_headers, body)
-      end
+      return output(headers, status, res_headers, body) unless res_headers['Content-Type'] =~ /html/
 
       request = Rack::Request.new(env)
 
-      if request.params['wovn_disable'] == true
-        return output(headers, status, res_headers, body)
-      end
+      return output(headers, status, res_headers, body) if request.params['wovn_disable'] == true
 
       @store.settings.update_dynamic_settings!(request.params)
-      if @store.settings['ignore_globs'].any?{|g| g.match?(headers.pathname)}
-        return output(headers, status, res_headers, body)
-      end
+      return output(headers, status, res_headers, body) if @store.settings['ignore_globs'].any? { |g| g.match?(headers.pathname) }
 
-      # ApiData creates request for external server, but cannot use async.
-      # Because some server not allow multi thread. (env['async.callback'] is not supported at all Server).
-      api_data = ApiData.new(headers.redis_url, @store)
-      values = api_data.get_data
-
-      url = {
-        :protocol => headers.protocol,
-        :host => headers.host,
-        :pathname => headers.pathname
-      }
-      body = switch_lang(body, values, url, lang, headers) unless status.to_s =~ /^1|302/
+      body = switch_lang(headers, body) unless status.to_s =~ /^1|302/
 
       content_length = 0
       body.each { |b| content_length += b.respond_to?(:bytesize) ? b.bytesize : 0 }
-      res_headers["Content-Length"] = content_length.to_s
+      res_headers['Content-Length'] = content_length.to_s
 
       output(headers, status, res_headers, body)
     end
 
-    def switch_lang(body, values, url, lang=@store.settings['default_lang'], headers)
-      lang = Lang.new(lang)
-      ignore_all = false
-      new_body = []
-
-      # generate full_body to intercept
-      full_body = ''
-      body.each { |b| full_body += b }
-
-      [full_body].each do |b|
-        # temporarily remove noscripts
-        noscripts = []
-        b_without_noscripts = b
-        b.scan /<noscript.*?>.*?<\/noscript>/m do |match|
-          noscript_identifier = "<noscript wovn-id=\"#{noscripts.count}\"></noscript>"
-          noscripts << match
-          b_without_noscripts = b_without_noscripts.sub(match, noscript_identifier)
-        end
+    def switch_lang(headers, body)
+      translated_body = []
 
-        d = Helpers::NokogumboHelper::parse_html(b_without_noscripts)
+      # Must use `.each` for to support multiple-chunks in Sinatra
+      string_body = ''
+      body.each { |chunk| string_body += chunk }
+      html_body = Helpers::NokogumboHelper.parse_html(string_body)
 
-        # If this page has wovn-ignore in the html tag, don't do anything
-        if ignore_all || !d.xpath('//html[@wovn-ignore]').empty? || is_amp_page?(d)
-          ignore_all = true
-          output = d.to_html(save_with: 0).gsub(/href="([^"]*)"/) { |m| "href=\"#{URI.decode($1)}\"" }
-          put_back_noscripts!(output, noscripts)
-          new_body.push(output)
-          next
-        end
+      if !wovn_ignored?(html_body) && !amp_page?(html_body)
+        html_converter = HtmlConverter.new(html_body, @store, headers)
 
-        if must_translate?(d, values)
-          output = lang.switch_dom_lang(d, @store, values, url, headers)
+        if needs_api?(html_body, headers)
+          converted_html, marker = html_converter.build_api_compatible_html
+          translated_content = ApiTranslator.new(@store, headers).translate(converted_html)
+          translated_body.push(marker.revert(translated_content))
         else
-          ScriptReplacer.new(@store).replace(d, lang) if d.html?
-          output = d.to_html(save_with: 0)
+          string_body = html_converter.build if html_body.html?
+          translated_body.push(string_body)
         end
-        put_back_noscripts!(output, noscripts)
-        new_body.push(output)
+      else
+        translated_body.push(string_body)
       end
+
       body.close if body.respond_to?(:close)
-      new_body
+      translated_body
     end
 
     private
@@ -139,21 +92,13 @@ module Wovnrb
       [status, res_headers, body]
     end
 
-    def must_translate?(dom, values)
-      can_translate = dom.html? ? true : @store.settings['translate_fragment']
-
-      can_translate && have_data?(values)
-    end
-
-    def have_data?(values)
-      values.count > 0
+    def needs_api?(html_body, headers)
+      headers.lang_code != @store.settings['default_lang'] &&
+        (html_body.html? || @store.settings['translate_fragment'])
     end
 
-    def put_back_noscripts!(output, noscripts)
-      noscripts.each_with_index do |noscript, index|
-        noscript_identifier = "<noscript wovn-id=\"#{index}\"></noscript>"
-        output.sub!(noscript_identifier, noscript)
-      end
+    def wovn_ignored?(html_body)
+      !html_body.xpath('//html[@wovn-ignore]').empty?
     end
 
     # Checks if a given HTML body is an Accelerated Mobile Page (AMP).
@@ -163,10 +108,10 @@ module Wovnrb
     # @param {Nokogiri::HTML5::Document} body The HTML body to check.
     #
     # @returns {Boolean} True is the HTML body is an AMP, false otherwise.
-    def is_amp_page?(body)
-      html_attributes = body.xpath('//html')[0].try(:attributes) || {}
+    def amp_page?(html_body)
+      html_attributes = html_body.xpath('//html')[0].try(:attributes) || {}
 
-      html_attributes['amp'] || html_attributes["\u26A1"]
+      !!(html_attributes['amp'] || html_attributes["\u26A1"])
     end
   end
 end