wovnrb 3.5.0 → 3.6.0

data/lib/wovnrb/services/html_converter.rb

@@ -1,9 +1,10 @@
 module Wovnrb
   class HtmlConverter
-    def initialize(dom, store, headers)
+    def initialize(dom, store, headers, url_lang_switcher)
       @dom = dom
       @headers = headers
       @store = store
+      @url_lang_switcher = url_lang_switcher
     end
 
     def build
@@ -32,6 +33,7 @@ module Wovnrb
       replace_snippet
       replace_hreflangs
       inject_lang_html_tag
+      translate_canonical_tag if @store.settings['translate_canonical_tag']
     end
 
     def replace_snippet
@@ -48,6 +50,7 @@ module Wovnrb
       insert_snippet(adds_backend_error_mark: true)
       insert_hreflang_tags
       inject_lang_html_tag
+      translate_canonical_tag if @store.settings['translate_canonical_tag']
 
       html
     end
@@ -143,6 +146,19 @@ module Wovnrb
       end
     end
 
+    def translate_canonical_tag
+      canonical_node = @dom.at_css('link[rel="canonical"]')
+      return unless canonical_node
+
+      lang_code = @headers.lang_code
+      return if lang_code == @store.settings['default_lang'] && @store.settings['custom_lang_aliases'][lang_code].nil?
+
+      canonical_url = canonical_node['href']
+
+      translated_canonical_url = @url_lang_switcher.add_lang_code(canonical_url, lang_code, @headers)
+      canonical_node['href'] = translated_canonical_url
+    end
+
     # Remove wovn snippet code from dom
     def strip_snippet
       @dom.xpath('//script').each do |script_node|

data/lib/wovnrb/services/url.rb

@@ -0,0 +1,136 @@
+module Wovnrb
+  # URL utility ported from html-swapper
+  class URL
+    module FileExtension
+      IMG_FILES = 'jpe|jpe?g|bmp|gif|png|btif|tiff?|psd|djvu?|xif|wbmp|webp|p(n|b|g|p)m|rgb|tga|x(b|p)m|xwd|pic|ico|fh(c|4|5|7)?|xif|f(bs|px|st)'.freeze
+      AUDIO_FILES = 'mp(3|2)|m(p?2|3|p?4|pg)a|midi?|kar|rmi|web(m|a)|aif(f?|c)|w(ma|av|ax)|m(ka|3u)|sil|s3m|og(a|g)|uvv?a'.freeze
+      VIDEO_FILES = 'm(x|4)u|fl(i|v)|3g(p|2)|jp(gv|g?m)|mp(4v?|g4|(?!$)e?g?)|m(1|2)v|ogv|m(ov|ng)|qt|uvv?(h|m|p|s|v)|dvb|mk(v|3d|s)|f4v|as(x|f)|w(m(v|x)|vx)|xvid'.freeze
+      DOC_FILES = '(7|g)?zip|tar|rar|7z|gz|ez|aw|atom(cat|svc)?|(cc)?xa?ml|cdmi(a|c|d|o|q)?|epub|g(ml|px|xf)|jar|js|ser|class|json(ml)?|do(c|t)(m|x)?|xls(m|x)?|xps|pp(a|tx?|s)m?|potm?|sldm|mp(p|t)|bin|dms|lrf|mar|so|dist|distz|m?pkg|bpk|dump|rtf|tfi|pdf|pgp|apk|o(t|d)(b|c|ft?|g|h|i|p|s|t)'.freeze
+    end
+
+    # TODO: Maybe this should be applied to all get_attribute calls rather than just href
+    def self.normalize_url(href)
+      return nil unless href
+
+      href.delete("\u200b").strip
+    end
+
+    def self.absolute_url?(href)
+      href =~ %r{^(https?:)?//}i
+    end
+
+    def self.absolute_path?(href)
+      href.match?(%r{^/})
+    end
+
+    def self.relative_path?(href)
+      !absolute_url?(href) && !absolute_path?(href)
+    end
+
+    # @param parsed_uri [Addressable::URI]
+    def self.path_and_query(parsed_uri)
+      parsed_uri.path + (parsed_uri.query ? "?#{parsed_uri.query}" : '')
+    end
+
+    def self.path_and_query_and_hash(parsed_uri)
+      uri = parsed_uri.path
+      uri += "?#{parsed_uri.query}" if parsed_uri.query
+      uri += "##{parsed_uri.fragment}" if parsed_uri.fragment
+      uri
+    end
+
+    def self.host_with_port(parsed_uri)
+      if parsed_uri.port
+        "#{parsed_uri.host}:#{parsed_uri.port}"
+      else
+        parsed_uri.host.to_s
+      end
+    end
+
+    def self.resolve_absolute_uri(base_url, href)
+      # This resolves ./../ and also handles href already being absolute
+      Addressable::URI.join(base_url, href)
+    rescue Addressable::URI::InvalidURIError, ArgumentError => e
+      Rollbar.warning('Failed to resolve absolute URI', original_error: e, base_url: base_url, href: href)
+      raise
+    end
+
+    def self.resolve_absolute_path(base_url, href)
+      normalized_uri = resolve_absolute_uri(base_url, href)
+      path = normalized_uri.path
+      query = normalized_uri.query ? "?#{normalized_uri.query}" : ''
+      fragment = normalized_uri.fragment ? "##{normalized_uri.fragment}" : ''
+
+      path + query + fragment
+    end
+
+    # Set the path lang to
+    def self.prepend_path(url, dir)
+      url.sub(%r{(.+\.[^/]+)(/|$)}, "\\1/#{dir}\\2")
+    end
+
+    def self.trim_slashes(path)
+      path.gsub(%r{^/|/$}, '')
+    end
+
+    def self.prepend_path_slash(path)
+      path ||= ''
+      return path if path.starts_with?('/')
+
+      "/#{path}"
+    end
+
+    def self.join_paths(*paths)
+      paths.inject('') do |left, right|
+        case [left.end_with?('/'), right.start_with?('/')]
+        when [true, true]
+          left + right[1..-1]
+        when [false, false]
+          left + (right.blank? ? right : "/#{right}")
+        else
+          left + right
+        end
+      end
+    end
+
+    # @param uri [Addressable::URI]
+    # @param new_protocol [String | nil]
+    # @return copy of uri [Addressable::URI]
+    def self.change_protocol(uri, new_protocol)
+      result = uri.dup
+      result.scheme = new_protocol
+      result
+    end
+
+    def self.valid_protocol?(href)
+      scheme_matches = /^\s*(?<scheme>[a-zA-Z]+):/.match(href)
+      scheme = scheme_matches ? scheme_matches[:scheme] : nil
+
+      scheme.nil? || %w[http https].include?(scheme)
+    end
+
+    def self.file?(href_with_query_and_hash)
+      href = remove_query_and_hash(href_with_query_and_hash)
+      img_files = %r{^(https?://)?.*(\.(#{FileExtension::IMG_FILES}))((\?|#).*)?$}io
+      audio_files = %r{^(https?://)?.*(\.(#{FileExtension::AUDIO_FILES}))((\?|#).*)?$}io
+      video_files = %r{^(https?://)?.*(\.(#{FileExtension::VIDEO_FILES}))((\?|#).*)?$}io
+      doc_files = %r{^(https?://)?.*(\.(#{FileExtension::DOC_FILES}))((\?|#).*)?$}io
+      href.match?(img_files) || href.match?(audio_files) || href.match?(video_files) || href.match?(doc_files)
+    end
+
+    def self.remove_query_and_hash(href)
+      href.gsub(/[#?].*/, '')
+    end
+
+    # if original path does not end in slash, remove it from new path
+    # if original path ends in slash, add it to new path
+    def self.normalize_path_slash(original_path, new_path)
+      if !original_path.end_with?('/') && new_path.end_with?('/')
+        new_path = new_path.chomp('/')
+      elsif original_path.end_with?('/') && !new_path.end_with?('/')
+        new_path += '/'
+      end
+      new_path
+    end
+  end
+end

data/lib/wovnrb/store.rb

@@ -25,6 +25,7 @@ module Wovnrb
         'ignore_class' => [],
         'api_url' => 'https://wovn.global.ssl.fastly.net',
         'api_timeout_seconds' => 1.0,
+        'api_timeout_search_engine_bots' => 5.0,
         'default_lang' => 'ja',
         'supported_langs' => %w[en ja],
         'test_mode' => false,
@@ -36,7 +37,8 @@ module Wovnrb
         'translate_fragment' => true,
         'widget_url' => 'https://j.wovn.io/1',
         'wovn_dev_mode' => false,
-        'compress_api_requests' => true
+        'compress_api_requests' => true,
+        'translate_canonical_tag' => true
       )
     end
 
@@ -166,7 +168,7 @@ module Wovnrb
     end
 
     def custom_lang_aliases
-      @setttings['custom_lang_aliases'] || {}
+      @settings['custom_lang_aliases'] || {}
     end
 
     def default_lang
@@ -194,6 +196,10 @@ module Wovnrb
       @settings['wovn_dev_mode']
     end
 
+    def url_pattern
+      @settings['url_pattern']
+    end
+
     private
 
     def stringify_keys!(hash)

data/lib/wovnrb/url_language_switcher.rb

@@ -0,0 +1,124 @@
+require 'wovnrb/services/url'
+
+module Wovnrb
+  # URL Language switching helper, ported from html-swapper
+  class UrlLanguageSwitcher
+    def initialize(store)
+      @store = store
+    end
+
+    # Adds language code to URL in "href" variable by "pattern" variable and own lang_code.
+    #  When lang_code is 'ja', add_lang_code('https://wovn.io', 'path', url) returns 'https://wovn.io/ja/'.
+    # @param  [String] href original URL.
+    # @param  [String] to_lang_code language code.
+    def add_lang_code(href, to_lang_code, headers)
+      return nil if href.nil?
+      return href if href.match?(/^(#.*)?$/)
+
+      href_scheme = href[/^[A-Za-z][A-Za-z0-9+\-.]*(?=:)/]
+      return href if !href_scheme.nil? && href_scheme != 'http' && href_scheme != 'https'
+
+      code_to_add = @store.custom_lang_aliases[to_lang_code] || to_lang_code
+      if Wovnrb::URL.absolute_url?(href)
+        add_lang_code_absolute_url(href, code_to_add, headers)
+      else
+        add_lang_code_relative_url(href, code_to_add, headers)
+      end
+    end
+
+    private
+
+    delegate :default_lang_alias, to: :@store
+
+    def add_lang_code_absolute_url(href, code_to_add, headers)
+      # in the future, perhaps validate url rather than using begin rescue
+      # "#{url =~ /\// ? 'http:' : ''}#{url}" =~ URI::regexp
+      begin
+        href_uri = Addressable::URI.parse(href)
+      rescue Addressable::URI::InvalidURIError, ArgumentError => e
+        Rollbar.warning('Failed to parse URI', original_error: e, href: href)
+        return href
+      end
+
+      if internal_link?(href_uri, headers.host)
+        return case @store.settings['url_pattern']
+               when 'subdomain'
+                 sub_d = href.match(%r{//([^.]*)\.})[1]
+                 sub_code = Lang.get_code(sub_d)
+                 if sub_code&.casecmp(code_to_add.downcase)&.zero?
+                   href.sub(Regexp.new(code_to_add, 'i'), code_to_add.downcase)
+                 else
+                   href.sub(%r{(//)([^.]*)}, "\\1#{code_to_add.downcase}.\\2")
+                 end
+               when 'query'
+                 add_query_lang_code(href, code_to_add)
+               else # path
+                 href_uri.path = add_lang_code_for_path(href_uri.path, code_to_add, headers)
+                 href_uri.to_s
+               end
+      end
+
+      href
+    end
+
+    def internal_link?(absolute_uri, host_name)
+      absolute_uri.host == host_name.split(':')[0]
+    end
+
+    def add_lang_code_relative_url(href, code_to_add, headers)
+      begin
+        abs_path = normalize_absolute_path(headers.to_absolute_path(href), headers)
+      rescue Addressable::URI::InvalidURIError, ArgumentError
+        return href
+      end
+
+      case @store.url_pattern
+      when 'subdomain'
+        "#{headers.protocol}://#{code_to_add.downcase}.#{headers.host}#{abs_path}"
+      when 'query'
+        add_query_lang_code(href, code_to_add)
+      else # path
+        add_lang_code_for_path(href, code_to_add, headers)
+      end
+    end
+
+    def add_lang_code_for_path(href, code_to_add, headers)
+      new_href = href
+
+      if code_to_add != @store.default_lang || code_to_add == @store.default_lang_alias
+        new_href = headers.to_absolute_path(href)
+        lang_prefix_path = build_lang_path(code_to_add)
+        prefix_path = build_lang_path('')
+        suffix_path = new_href.sub(%r{^#{prefix_path}(/|$)}, '')
+                              .sub(%r{^#{@store.default_lang}(/|$)}, '')
+        new_href = URL.join_paths(lang_prefix_path, suffix_path)
+        new_href = URL.normalize_path_slash(href, new_href)
+        new_href
+      end
+
+      normalize_absolute_path(new_href, headers)
+    end
+
+    def normalize_absolute_path(input_path, headers)
+      URL.resolve_absolute_path(headers.url_with_scheme, input_path)
+    end
+
+    def sub_repeat!(string, pattern, replacement)
+      loop do
+        break unless string.sub!(pattern, replacement)
+      end
+    end
+
+    def add_query_lang_code(href, lang_code)
+      lang_param = @store.settings['lang_param_name']
+      return href if href.match?(/(&|&|\?)?#{lang_param}=[a-zA-Z_-]+/)
+
+      query_separator = href.include?('?') ? '&' : '?'
+      href.sub(/(#|$)/, "#{query_separator}#{lang_param}=#{lang_code}\\1")
+    end
+
+    def build_lang_path(lang_code)
+      lang_code.blank? ? '' : URL.prepend_path_slash(lang_code)
+    end
+  end
+end

data/lib/wovnrb/version.rb

@@ -1,3 +1,3 @@
 module Wovnrb
-  VERSION = '3.5.0'.freeze
+  VERSION = '3.6.0'.freeze
 end

data/lib/wovnrb.rb

@@ -5,6 +5,7 @@ require 'wovnrb/store'
 require 'wovnrb/lang'
 require 'wovnrb/services/html_converter'
 require 'wovnrb/services/html_replace_marker'
+require 'wovnrb/url_language_switcher'
 require 'nokogiri'
 require 'active_support'
 require 'json'
@@ -76,7 +77,8 @@ module Wovnrb
       html_body = Helpers::NokogumboHelper.parse_html(string_body)
 
       if !wovn_ignored?(html_body) && !amp_page?(html_body)
-        html_converter = HtmlConverter.new(html_body, @store, headers)
+        url_lang_switcher = Wovnrb::UrlLanguageSwitcher.new(@store)
+        html_converter = HtmlConverter.new(html_body, @store, headers, url_lang_switcher)
 
         if needs_api?(html_body, headers)
           converted_html, marker = html_converter.build_api_compatible_html

data/test/lib/api_translator_test.rb

@@ -54,6 +54,7 @@ module Wovnrb
                          'lang_code' => 'fr',
                          'url_pattern' => 'subdomain',
                          'lang_param_name' => 'lang',
+                         'translate_canonical_tag' => true,
                          'product' => 'WOVN.rb',
                          'version' => VERSION,
                          'body' => 'foo',
@@ -62,6 +63,46 @@ module Wovnrb
                        times: 1
     end
 
+    def test_api_timeout_is_search_engine_user_higher_default
+      settings = {
+        'project_token' => '123456',
+        'custom_lang_aliases' => { 'ja' => 'Japanese' },
+        'default_lang' => 'en',
+        'url_pattern' => 'subdomain',
+        'url_pattern_reg' => '^(?<lang>[^.]+).',
+        'lang_param_name' => 'lang'
+      }
+      store = Wovnrb::Store.instance
+      store.update_settings(settings)
+      headers = Wovnrb::Headers.new(
+        Wovnrb.get_env('url' => 'http://fr.wovn.io/test', 'HTTP_USER_AGENT' => 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)'),
+        Wovnrb.get_settings(settings)
+      )
+      api_translator = ApiTranslator.new(store, headers, REQUEST_UUID)
+      assert_equal(5.0, api_translator.send(:api_timeout))
+    end
+
+    def test_api_timeout_no_user_agent_use_normal_default
+      settings = {
+        'project_token' => '123456',
+        'custom_lang_aliases' => { 'ja' => 'Japanese' },
+        'default_lang' => 'en',
+        'url_pattern' => 'subdomain',
+        'url_pattern_reg' => '^(?<lang>[^.]+).',
+        'lang_param_name' => 'lang'
+      }
+      store = Wovnrb::Store.instance
+      store.update_settings(settings)
+      env = Wovnrb.get_env('url' => 'http://fr.wovn.io/test')
+      env.delete('HTTP_USER_AGENT')
+      headers = Wovnrb::Headers.new(
+        env,
+        Wovnrb.get_settings(settings)
+      )
+      api_translator = ApiTranslator.new(store, headers, REQUEST_UUID)
+      assert_equal(1.0, api_translator.send(:api_timeout))
+    end
+
     private
 
     def assert_translation(original_html_fixture, translated_html_fixture, success_expected, response: { encoding: 'gzip', status_code: 200 }, compress_data: true)
@@ -80,6 +121,8 @@ module Wovnrb
       api_translator, store, _headers = create_sut
       translation_request_stub = stub_translation_api_request(store, original_html, translated_html, response, compress_data: compress_data)
 
+      expected_api_timeout = store.settings['api_timeout_seconds']
+      assert_equal(expected_api_timeout, api_translator.send(:api_timeout))
       actual_translated_html = api_translator.translate(original_html)
       assert_requested(translation_request_stub, times: 1) if translation_request_stub
       actual_translated_html
@@ -152,6 +195,7 @@ module Wovnrb
         'lang_code' => 'fr',
         'url_pattern' => 'subdomain',
         'lang_param_name' => 'lang',
+        'translate_canonical_tag' => true,
         'product' => 'WOVN.rb',
         'version' => VERSION,
         'body' => original_html,