wovnrb 3.5.0 → 3.7.1

data/docker/scripts/jenkins/build.sh

@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+set -eux
+export AWS_PROFILE="wovn-code-staging-pipeline"
+export AWS_REGION="us-west-2"
+export ECR_HOST="257024234524.dkr.ecr.us-west-2.amazonaws.com"
+REPO_NAME_WOVNRB="wovnrb"
+REPO_NAME_NGINX="wovnrb-nginx"
+CLUSTER_NAME="wovn-library-testing"
+TASKDEF_FAMILY_NAME="wovnrb"
+ECS_SERVICE_NAME="wovnrb"
+PROJECT_DIR=$(dirname "$0")/../../..
+
+commit_hash=$(git rev-parse --short HEAD)
+image_tag="${commit_hash}"
+
+sh ${PROJECT_DIR}/build.sh "${REPO_NAME_WOVNRB}":"${image_tag}"
+sh ${PROJECT_DIR}/docker/nginx/build.sh "${REPO_NAME_NGINX}":"${image_tag}"
+
+source ${PROJECT_DIR}/docker/scripts/jenkins/tag_and_push_image.sh
+
+set +x
+$(aws ecr get-login --no-include-email --region "${AWS_REGION}" --profile "${AWS_PROFILE}")
+set -x
+
+tag_and_push_image "${AWS_REGION}" "${REPO_NAME_WOVNRB}" "${image_tag}" "staging"
+tag_and_push_image "${AWS_REGION}" "${REPO_NAME_NGINX}" "${image_tag}" "staging"
+
+sed -i "s#wovnrb:latest#"${REPO_NAME_WOVNRB}":"${image_tag}"#g" ${PROJECT_DIR}/docker/scripts/jenkins/taskdef.json
+sed -i "s#wovnrb-nginx:latest#"${REPO_NAME_NGINX}":"${image_tag}"#g" ${PROJECT_DIR}/docker/scripts/jenkins/taskdef.json
+
+cd ${PROJECT_DIR}/docker/scripts/jenkins/
+TASKDEF_REVISION=$(aws ecs register-task-definition \
+                         --profile "${AWS_PROFILE}" --region "${AWS_REGION}" \
+                         --cli-input-json file://$(pwd)/taskdef.json \
+                      | jq ."taskDefinition.revision")
+echo "${TASKDEF_REVISION}"
+
+echo "Start ECS Rolling deploy. Update ${ECS_SERVICE_NAME} by ${TASKDEF_FAMILY_NAME}:${TASKDEF_REVISION}"
+    aws ecs update-service \
+      --profile "${AWS_PROFILE}" --region "${AWS_REGION}" \
+      --cluster "${CLUSTER_NAME}" \
+      --service "${ECS_SERVICE_NAME}" \
+      --task-definition "${TASKDEF_FAMILY_NAME}:${TASKDEF_REVISION}"
+
+cd -

data/docker/scripts/jenkins/tag_and_push_image.sh

@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+
+# Tag to docker image and push it to specified ECR
+#
+# Required environment variables:
+#  - AWS_PROFILE (AWS profile authorized for ECR login and S3 release)
+#
+# Args:
+#  - region: ECR region (e.g. us-west-1 | us-west-2 )
+#  - image-name: docker image name (e.g. equalizer-nginx)
+#  - tag: docker image tag. we use git commit hash ordinary (git rev-parse --short HEAD)
+
+tag_and_push_image() {
+  local aws_region="${1}"
+  local image_name="${2}"
+  local tag="${3}"
+  local account="${4}"
+
+  if [ ${account} == "production" ]; then
+    local ecr_host="140249473629.dkr.ecr.${aws_region}.amazonaws.com"
+  elif [ ${account} == "staging" ]; then
+    local ecr_host="257024234524.dkr.ecr.${aws_region}.amazonaws.com"
+  else
+    echo "Passed account not recognized"
+    local ecr_host=""
+  fi
+
+  docker tag ${image_name}:${tag} ${ecr_host}/${image_name}:${tag}
+  docker push ${ecr_host}/${image_name}:${tag}
+}

data/docker/scripts/jenkins/taskdef.json

@@ -0,0 +1,104 @@
+{
+  "executionRoleArn": "arn:aws:iam::257024234524:role/ecsTaskExecutionRole",
+  "containerDefinitions": [
+    {
+      "name": "wovnrb-nginx",
+      "cpu": 0,
+      "image": "257024234524.dkr.ecr.us-west-2.amazonaws.com/wovnrb-nginx:latest",
+      "mountPoints": [],
+      "ulimits": [
+        {
+          "name": "nofile",
+          "softLimit": 65535,
+          "hardLimit": 65535
+        }
+      ],
+      "logConfiguration": {
+        "logDriver": "awslogs",
+        "options": {
+          "awslogs-group": "/ecs/wovnrb",
+          "awslogs-region": "us-west-2",
+          "awslogs-stream-prefix": "ecs"
+        }
+      },
+      "essential": true,
+      "portMappings": [
+        {
+          "hostPort": 80,
+          "protocol": "tcp",
+          "containerPort": 80
+        },
+        {
+          "hostPort": 40022,
+          "protocol": "tcp",
+          "containerPort": 40022
+        }
+      ],
+      "stopTimeout": 120,
+      "dependsOn": [
+        {
+          "containerName": "wovnrb",
+          "condition": "HEALTHY"
+        }
+      ]
+    },
+    {
+      "name": "wovnrb",
+      "cpu": 0,
+      "image": "257024234524.dkr.ecr.us-west-2.amazonaws.com/wovnrb:latest",
+      "mountPoints": [],
+      "workingDirectory": "/usr/src/app",
+      "linuxParameters": {
+        "initProcessEnabled": true
+      },
+      "environment": [],
+      "ulimits": [
+        {
+          "name": "nofile",
+          "softLimit": 65535,
+          "hardLimit": 65535
+        }
+      ],
+      "logConfiguration": {
+        "logDriver": "awslogs",
+        "options": {
+          "awslogs-group": "/ecs/wovnrb",
+          "awslogs-region": "us-west-2",
+          "awslogs-stream-prefix": "ecs"
+        }
+      },
+      "portMappings": [
+        {
+          "hostPort": 4000,
+          "protocol": "tcp",
+          "containerPort": 4000
+        },
+        {
+          "hostPort": 22,
+          "protocol": "tcp",
+          "containerPort": 22
+        }
+      ],
+      "volumesFrom": [],
+      "stopTimeout": 120,
+      "healthCheck": {
+        "retries": 3,
+        "command": [
+          "CMD-SHELL",
+          "curl -f http://localhost:4000/ || exit 1"
+        ],
+        "timeout": 20,
+        "interval": 60,
+        "startPeriod": 180
+      },
+      "essential": true
+    }
+  ],
+  "placementConstraints": [],
+  "cpu": "1024",
+  "memory": "2048",
+  "requiresCompatibilities": [ "FARGATE" ],
+  "family": "wovnrb",
+  "networkMode": "awsvpc",
+  "volumes": []
+}

data/docker/scripts/jenkins/taskdef.json.bak

@@ -0,0 +1,99 @@
+{
+  "executionRoleArn": "arn:aws:iam::257024234524:role/ecsTaskExecutionRole",
+  "containerDefinitions": [
+    {
+      "name": "wovnrb-nginx",
+      "cpu": 0,
+      "image": "257024234524.dkr.ecr.us-west-2.amazonaws.com/wovnrb-nginx:latest",
+      "mountPoints": [],
+      "ulimits": [
+        {
+          "name": "nofile",
+          "softLimit": 65535,
+          "hardLimit": 65535
+        }
+      ],
+      "logConfiguration": {
+        "logDriver": "awslogs",
+        "options": {
+          "awslogs-group": "/ecs/wovnrb",
+          "awslogs-region": "us-west-2",
+          "awslogs-stream-prefix": "ecs"
+        }
+      },
+      "essential": true,
+      "portMappings": [
+        {
+          "hostPort": 80,
+          "protocol": "tcp",
+          "containerPort": 80
+        },
+        {
+          "hostPort": 40022,
+          "protocol": "tcp",
+          "containerPort": 40022
+        }
+      ],
+      "stopTimeout": 120,
+      "dependsOn": [
+        {
+          "containerName": "wovnrb",
+          "condition": "HEALTHY"
+        }
+      ]
+    },
+    {
+      "name": "wovnrb",
+      "cpu": 0,
+      "image": "257024234524.dkr.ecr.us-west-2.amazonaws.com/wovnrb:36d9648",
+      "mountPoints": [],
+      "workingDirectory": "/usr/src/app",
+      "linuxParameters": {
+        "initProcessEnabled": true
+      },
+      "environment": [],
+      "ulimits": [
+        {
+          "name": "nofile",
+          "softLimit": 65535,
+          "hardLimit": 65535
+        }
+      ],
+      "logConfiguration": {
+        "logDriver": "awslogs",
+        "options": {
+          "awslogs-group": "/ecs/wovnrb",
+          "awslogs-region": "us-west-2",
+          "awslogs-stream-prefix": "ecs"
+        }
+      },
+      "portMappings": [
+        {
+          "hostPort": 4000,
+          "protocol": "tcp",
+          "containerPort": 4000
+        }
+      ],
+      "volumesFrom": [],
+      "stopTimeout": 120,
+      "healthCheck": {
+        "retries": 3,
+        "command": [
+          "CMD-SHELL",
+          "curl -f http://localhost:4000/ || exit 1"
+        ],
+        "timeout": 20,
+        "interval": 60,
+        "startPeriod": 180
+      },
+      "essential": true
+    }
+  ],
+  "placementConstraints": [],
+  "cpu": "1024",
+  "memory": "2048",
+  "requiresCompatibilities": [ "FARGATE" ],
+  "family": "wovnrb",
+  "networkMode": "awsvpc",
+  "volumes": []
+}

data/lib/wovnrb/api_translator.rb

@@ -111,6 +111,7 @@ module Wovnrb
         'lang_code' => lang_code,
         'url_pattern' => url_pattern,
         'lang_param_name' => lang_param_name,
+        'translate_canonical_tag' => translate_canonical_tag,
         'product' => 'WOVN.rb',
         'version' => VERSION,
         'body' => body
@@ -130,7 +131,7 @@ module Wovnrb
     end
 
     def api_timeout
-      @store.settings['api_timeout_seconds']
+      @headers.search_engine_bot? ? @store.settings['api_timeout_search_engine_bots'] : @store.settings['api_timeout_seconds']
     end
 
     def settings_hash
@@ -157,6 +158,10 @@ module Wovnrb
       @store.settings['custom_lang_aliases']
     end
 
+    def translate_canonical_tag
+      @store.settings['translate_canonical_tag']
+    end
+
     def page_url
       "#{@headers.protocol}://#{@headers.url}"
     end

data/lib/wovnrb/headers.rb

@@ -5,9 +5,9 @@ module Wovnrb
     # Generates new instance of Wovnrb::Headers.
     # Its parameters are set by parsing env variable.
 
-    def initialize(env, settings)
+    def initialize(env, settings, url_lang_switcher)
       request = Rack::Request.new(env)
-
+      @url_lang_switcher = url_lang_switcher
       @env = env
       @settings = settings
       @protocol = request.scheme
@@ -31,11 +31,11 @@ module Wovnrb
               else
                 @env['HTTP_HOST']
               end
-      @host = settings['url_pattern'] == 'subdomain' ? remove_lang(@host, lang_code) : @host
+      @host = settings['url_pattern'] == 'subdomain' ? @url_lang_switcher.remove_lang_from_uri_component(@host, lang_code) : @host
       @pathname, @query = @env['REQUEST_URI'].split('?')
-      @pathname = settings['url_pattern'] == 'path' ? remove_lang(@pathname, lang_code) : @pathname
+      @pathname = settings['url_pattern'] == 'path' ? @url_lang_switcher.remove_lang_from_uri_component(@pathname, lang_code) : @pathname
       @query ||= ''
-      @url = "#{@host}#{@pathname}#{(@query.empty? ? '' : '?') + remove_lang(@query, lang_code)}"
+      @url = "#{@host}#{@pathname}#{(@query.empty? ? '' : '?') + @url_lang_switcher.remove_lang_from_uri_component(@query, lang_code)}"
       if settings['query'].empty?
         @query = ''
       else
@@ -51,11 +51,15 @@ module Wovnrb
                    "?#{query_vals.sort.join('&')}"
                  end
       end
-      @query = remove_lang(@query, lang_code)
+      @query = @url_lang_switcher.remove_lang_from_uri_component(@query, lang_code)
       @pathname_with_trailing_slash_if_present = @pathname
       @pathname = @pathname.gsub(/\/$/, '')
     end
 
+    def url_with_scheme
+      "#{@protocol}://#{@url}"
+    end
+
     def unmasked_pathname_without_trailing_slash
       @unmasked_pathname.chomp('/')
     end
@@ -99,81 +103,38 @@ module Wovnrb
     def redirect_location(lang)
       if lang == @settings['default_lang']
         # IS THIS RIGHT??
-        "#{protocol}://#{url}"
-        # return remove_lang("#{@env['HTTP_HOST']}#{@env['REQUEST_URI']}", lang)
-      else
-        # TODO test
-        lang_code = Store.instance.settings['custom_lang_aliases'][lang] || lang
-        location = url
-        case @settings['url_pattern']
-        when 'query'
-          lang_param_name = @settings['lang_param_name']
-          location = if /\?/.match?(location)
-                       "#{location}&#{lang_param_name}=#{lang_code}"
-                     else
-                       "#{location}?#{lang_param_name}=#{lang_code}"
-                     end
-        when 'subdomain'
-          location = "#{lang_code.downcase}.#{location}"
-        # when 'path'
-        else
-          location = location.sub(/(\/|$)/, "/#{lang_code}/")
-        end
-        "#{protocol}://#{location}"
+        return url_with_scheme
       end
+
+      @url_lang_switcher.add_lang_code(url_with_scheme, lang, self)
     end
 
     def request_out(_def_lang = @settings['default_lang'])
       @env['wovnrb.target_lang'] = lang_code
       case @settings['url_pattern']
       when 'query'
-        @env['REQUEST_URI'] = remove_lang(@env['REQUEST_URI']) if @env.key?('REQUEST_URI')
-        @env['QUERY_STRING'] = remove_lang(@env['QUERY_STRING']) if @env.key?('QUERY_STRING')
-        @env['ORIGINAL_FULLPATH'] = remove_lang(@env['ORIGINAL_FULLPATH']) if @env.key?('ORIGINAL_FULLPATH')
+        @env['REQUEST_URI'] = @url_lang_switcher.remove_lang_from_uri_component(@env['REQUEST_URI'], lang_code) if @env.key?('REQUEST_URI')
+        @env['QUERY_STRING'] = @url_lang_switcher.remove_lang_from_uri_component(@env['QUERY_STRING'], lang_code) if @env.key?('QUERY_STRING')
+        @env['ORIGINAL_FULLPATH'] = @url_lang_switcher.remove_lang_from_uri_component(@env['ORIGINAL_FULLPATH'], lang_code) if @env.key?('ORIGINAL_FULLPATH')
       when 'subdomain'
         if @settings['use_proxy'] && @env.key?('HTTP_X_FORWARDED_HOST')
-          @env['HTTP_X_FORWARDED_HOST'] = remove_lang(@env['HTTP_X_FORWARDED_HOST'])
+          @env['HTTP_X_FORWARDED_HOST'] = @url_lang_switcher.remove_lang_from_uri_component(@env['HTTP_X_FORWARDED_HOST'], lang_code)
         else
-          @env['HTTP_HOST'] = remove_lang(@env['HTTP_HOST'])
-          @env['SERVER_NAME'] = remove_lang(@env['SERVER_NAME'])
+          @env['HTTP_HOST'] = @url_lang_switcher.remove_lang_from_uri_component(@env['HTTP_HOST'], lang_code)
+          @env['SERVER_NAME'] = @url_lang_switcher.remove_lang_from_uri_component(@env['SERVER_NAME'], lang_code)
         end
-        @env['HTTP_REFERER'] = remove_lang(@env['HTTP_REFERER']) if @env.key?('HTTP_REFERER')
+        @env['HTTP_REFERER'] = @url_lang_switcher.remove_lang_from_uri_component(@env['HTTP_REFERER'], lang_code) if @env.key?('HTTP_REFERER')
       # when 'path'
       else
-        @env['REQUEST_URI'] = remove_lang(@env['REQUEST_URI'])
-        @env['REQUEST_PATH'] = remove_lang(@env['REQUEST_PATH']) if @env.key?('REQUEST_PATH')
-        @env['PATH_INFO'] = remove_lang(@env['PATH_INFO'])
-        @env['ORIGINAL_FULLPATH'] = remove_lang(@env['ORIGINAL_FULLPATH']) if @env.key?('ORIGINAL_FULLPATH')
-        @env['HTTP_REFERER'] = remove_lang(@env['HTTP_REFERER']) if @env.key?('HTTP_REFERER')
+        @env['REQUEST_URI'] = @url_lang_switcher.remove_lang_from_uri_component(@env['REQUEST_URI'], lang_code)
+        @env['REQUEST_PATH'] = @url_lang_switcher.remove_lang_from_uri_component(@env['REQUEST_PATH'], lang_code) if @env.key?('REQUEST_PATH')
+        @env['PATH_INFO'] = @url_lang_switcher.remove_lang_from_uri_component(@env['PATH_INFO'], lang_code)
+        @env['ORIGINAL_FULLPATH'] = @url_lang_switcher.remove_lang_from_uri_component(@env['ORIGINAL_FULLPATH'], lang_code) if @env.key?('ORIGINAL_FULLPATH')
+        @env['HTTP_REFERER'] = @url_lang_switcher.remove_lang_from_uri_component(@env['HTTP_REFERER'], lang_code) if @env.key?('HTTP_REFERER')
       end
       @env
     end
 
-    # TODO: this should be in Lang for reusability
-    # Remove language code from the URI.
-    #
-    # @param uri  [String] original URI
-    # @param lang_code [String] language code
-    # @return     [String] removed URI
-    def remove_lang(uri, lang = path_lang)
-      lang_code = Store.instance.settings['custom_lang_aliases'][lang] || lang
-
-      # Do nothing if lang is empty.
-      return uri if lang_code.nil? || lang_code.empty?
-
-      case @settings['url_pattern']
-      when 'query'
-        lang_param_name = @settings['lang_param_name']
-        uri.sub(/(^|\?|&)#{lang_param_name}=#{lang_code}(&|$)/, '\1').gsub(/(\?|&)$/, '')
-      when 'subdomain'
-        rp = Regexp.new("(^|(//))#{lang_code}\\.", 'i')
-        uri.sub(rp, '\1')
-      # when 'path'
-      else
-        uri.sub(/\/#{lang_code}(\/|$)/, '/')
-      end
-    end
-
     def out(headers)
       r = Regexp.new("//#{@host}")
       lang_code = Store.instance.settings['custom_lang_aliases'][self.lang_code] || self.lang_code
@@ -197,11 +158,24 @@ module Wovnrb
     end
 
     def dirname
-      if pathname.include?('/')
-        pathname.end_with?('/') ? pathname : pathname[0, pathname.rindex('/') + 1]
+      if pathname_with_trailing_slash_if_present.include?('/')
+        pathname_with_trailing_slash_if_present.end_with?('/') ? pathname_with_trailing_slash_if_present : pathname_with_trailing_slash_if_present[0, pathname_with_trailing_slash_if_present.rindex('/') + 1]
       else
         '/'
       end
     end
+
+    def search_engine_bot?
+      return false unless @env.key?('HTTP_USER_AGENT')
+
+      bots = %w[Googlebot/ bingbot/ YandexBot/ YandexWebmaster/ DuckDuckBot-Https/ Baiduspider/ Slurp Yahoo]
+      bots.any? { |bot| @env['HTTP_USER_AGENT'].include?(bot) }
+    end
+
+    def to_absolute_path(path)
+      absolute_path = path.blank? ? '/' : path
+      absolute_path = absolute_path.starts_with?('/') ? absolute_path : URL.join_paths(dirname, absolute_path)
+      URL.normalize_path_slash(path, absolute_path)
+    end
   end
 end

data/lib/wovnrb/services/html_converter.rb

@@ -1,9 +1,10 @@
 module Wovnrb
   class HtmlConverter
-    def initialize(dom, store, headers)
+    def initialize(dom, store, headers, url_lang_switcher)
       @dom = dom
       @headers = headers
       @store = store
+      @url_lang_switcher = url_lang_switcher
     end
 
     def build
@@ -32,6 +33,7 @@ module Wovnrb
       replace_snippet
       replace_hreflangs
       inject_lang_html_tag
+      translate_canonical_tag if @store.settings['translate_canonical_tag']
     end
 
     def replace_snippet
@@ -48,6 +50,7 @@ module Wovnrb
       insert_snippet(adds_backend_error_mark: true)
       insert_hreflang_tags
       inject_lang_html_tag
+      translate_canonical_tag if @store.settings['translate_canonical_tag']
 
       html
     end
@@ -143,6 +146,19 @@ module Wovnrb
       end
     end
 
+    def translate_canonical_tag
+      canonical_node = @dom.at_css('link[rel="canonical"]')
+      return unless canonical_node
+
+      lang_code = @headers.lang_code
+      return if lang_code == @store.settings['default_lang'] && @store.settings['custom_lang_aliases'][lang_code].nil?
+
+      canonical_url = canonical_node['href']
+
+      translated_canonical_url = @url_lang_switcher.add_lang_code(canonical_url, lang_code, @headers)
+      canonical_node['href'] = translated_canonical_url
+    end
+
     # Remove wovn snippet code from dom
     def strip_snippet
       @dom.xpath('//script').each do |script_node|

data/lib/wovnrb/services/url.rb

@@ -0,0 +1,136 @@
+module Wovnrb
+  # URL utility ported from html-swapper
+  class URL
+    module FileExtension
+      IMG_FILES = 'jpe|jpe?g|bmp|gif|png|btif|tiff?|psd|djvu?|xif|wbmp|webp|p(n|b|g|p)m|rgb|tga|x(b|p)m|xwd|pic|ico|fh(c|4|5|7)?|xif|f(bs|px|st)'.freeze
+      AUDIO_FILES = 'mp(3|2)|m(p?2|3|p?4|pg)a|midi?|kar|rmi|web(m|a)|aif(f?|c)|w(ma|av|ax)|m(ka|3u)|sil|s3m|og(a|g)|uvv?a'.freeze
+      VIDEO_FILES = 'm(x|4)u|fl(i|v)|3g(p|2)|jp(gv|g?m)|mp(4v?|g4|(?!$)e?g?)|m(1|2)v|ogv|m(ov|ng)|qt|uvv?(h|m|p|s|v)|dvb|mk(v|3d|s)|f4v|as(x|f)|w(m(v|x)|vx)|xvid'.freeze
+      DOC_FILES = '(7|g)?zip|tar|rar|7z|gz|ez|aw|atom(cat|svc)?|(cc)?xa?ml|cdmi(a|c|d|o|q)?|epub|g(ml|px|xf)|jar|js|ser|class|json(ml)?|do(c|t)(m|x)?|xls(m|x)?|xps|pp(a|tx?|s)m?|potm?|sldm|mp(p|t)|bin|dms|lrf|mar|so|dist|distz|m?pkg|bpk|dump|rtf|tfi|pdf|pgp|apk|o(t|d)(b|c|ft?|g|h|i|p|s|t)'.freeze
+    end
+
+    # TODO: Maybe this should be applied to all get_attribute calls rather than just href
+    def self.normalize_url(href)
+      return nil unless href
+
+      href.delete("\u200b").strip
+    end
+
+    def self.absolute_url?(href)
+      href =~ %r{^(https?:)?//}i
+    end
+
+    def self.absolute_path?(href)
+      href.match?(%r{^/})
+    end
+
+    def self.relative_path?(href)
+      !absolute_url?(href) && !absolute_path?(href)
+    end
+
+    # @param parsed_uri [Addressable::URI]
+    def self.path_and_query(parsed_uri)
+      parsed_uri.path + (parsed_uri.query ? "?#{parsed_uri.query}" : '')
+    end
+
+    def self.path_and_query_and_hash(parsed_uri)
+      uri = parsed_uri.path
+      uri += "?#{parsed_uri.query}" if parsed_uri.query
+      uri += "##{parsed_uri.fragment}" if parsed_uri.fragment
+      uri
+    end
+
+    def self.host_with_port(parsed_uri)
+      if parsed_uri.port
+        "#{parsed_uri.host}:#{parsed_uri.port}"
+      else
+        parsed_uri.host.to_s
+      end
+    end
+
+    def self.resolve_absolute_uri(base_url, href)
+      # This resolves ./../ and also handles href already being absolute
+      Addressable::URI.join(base_url, href)
+    rescue Addressable::URI::InvalidURIError, ArgumentError => e
+      Rollbar.warning('Failed to resolve absolute URI', original_error: e, base_url: base_url, href: href)
+      raise
+    end
+
+    def self.resolve_absolute_path(base_url, href)
+      normalized_uri = resolve_absolute_uri(base_url, href)
+      path = normalized_uri.path
+      query = normalized_uri.query ? "?#{normalized_uri.query}" : ''
+      fragment = normalized_uri.fragment ? "##{normalized_uri.fragment}" : ''
+
+      path + query + fragment
+    end
+
+    # Set the path lang to
+    def self.prepend_path(url, dir)
+      url.sub(%r{(.+\.[^/]+)(/|$)}, "\\1/#{dir}\\2")
+    end
+
+    def self.trim_slashes(path)
+      path.gsub(%r{^/|/$}, '')
+    end
+
+    def self.prepend_path_slash(path)
+      path ||= ''
+      return path if path.starts_with?('/')
+
+      "/#{path}"
+    end
+
+    def self.join_paths(*paths)
+      paths.inject('') do |left, right|
+        case [left.end_with?('/'), right.start_with?('/')]
+        when [true, true]
+          left + right[1..-1]
+        when [false, false]
+          left + (right.blank? ? right : "/#{right}")
+        else
+          left + right
+        end
+      end
+    end
+
+    # @param uri [Addressable::URI]
+    # @param new_protocol [String | nil]
+    # @return copy of uri [Addressable::URI]
+    def self.change_protocol(uri, new_protocol)
+      result = uri.dup
+      result.scheme = new_protocol
+      result
+    end
+
+    def self.valid_protocol?(href)
+      scheme_matches = /^\s*(?<scheme>[a-zA-Z]+):/.match(href)
+      scheme = scheme_matches ? scheme_matches[:scheme] : nil
+
+      scheme.nil? || %w[http https].include?(scheme)
+    end
+
+    def self.file?(href_with_query_and_hash)
+      href = remove_query_and_hash(href_with_query_and_hash)
+      img_files = %r{^(https?://)?.*(\.(#{FileExtension::IMG_FILES}))((\?|#).*)?$}io
+      audio_files = %r{^(https?://)?.*(\.(#{FileExtension::AUDIO_FILES}))((\?|#).*)?$}io
+      video_files = %r{^(https?://)?.*(\.(#{FileExtension::VIDEO_FILES}))((\?|#).*)?$}io
+      doc_files = %r{^(https?://)?.*(\.(#{FileExtension::DOC_FILES}))((\?|#).*)?$}io
+      href.match?(img_files) || href.match?(audio_files) || href.match?(video_files) || href.match?(doc_files)
+    end
+
+    def self.remove_query_and_hash(href)
+      href.gsub(/[#?].*/, '')
+    end
+
+    # if original path does not end in slash, remove it from new path
+    # if original path ends in slash, add it to new path
+    def self.normalize_path_slash(original_path, new_path)
+      if !original_path.end_with?('/') && new_path.end_with?('/')
+        new_path = new_path.chomp('/')
+      elsif original_path.end_with?('/') && !new_path.end_with?('/')
+        new_path += '/'
+      end
+      new_path
+    end
+  end
+end