connectors_service 8.5.0.1

data/lib/utility/extension_mapping_util.rb

@@ -0,0 +1,123 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'active_support/core_ext/hash'
+
+module Utility
+  class ExtensionMappingUtil
+    @extension_to_mime = {
+      :doc => %w[
+        application/x-tika-msoffice
+        application/msword
+      ].freeze,
+      :docx => %w[
+        application/x-tika-ooxml
+        application/vnd.openxmlformats-officedocument.wordprocessingml.document
+        application/vnd.openxmlformats-officedocument.wordprocessingml.template
+        application/vnd.ms-word.template.macroenabled.12
+        application/vnd.ms-word.document.macroenabled.12
+      ].freeze,
+      :html => %w[
+        text/html
+        application/xhtml+xml
+      ].freeze,
+      :odt => %w[
+        application/x-vnd.oasis.opendocument.graphics-template
+        application/vnd.sun.xml.writer application/x-vnd.oasis.opendocument.text
+        application/x-vnd.oasis.opendocument.text-web
+        application/x-vnd.oasis.opendocument.spreadsheet-template
+        application/vnd.oasis.opendocument.formula-template
+        application/vnd.oasis.opendocument.presentation
+        application/vnd.oasis.opendocument.image-template
+        application/x-vnd.oasis.opendocument.graphics
+        application/vnd.oasis.opendocument.chart-template
+        application/vnd.oasis.opendocument.presentation-template
+        application/x-vnd.oasis.opendocument.image-template
+        application/vnd.oasis.opendocument.formula
+        application/x-vnd.oasis.opendocument.image
+        application/vnd.oasis.opendocument.spreadsheet-template
+        application/x-vnd.oasis.opendocument.chart-template
+        application/x-vnd.oasis.opendocument.formula
+        application/vnd.oasis.opendocument.spreadsheet
+        application/vnd.oasis.opendocument.text-web
+        application/vnd.oasis.opendocument.text-template
+        application/vnd.oasis.opendocument.text
+        application/x-vnd.oasis.opendocument.formula-template
+        application/x-vnd.oasis.opendocument.spreadsheet
+        application/x-vnd.oasis.opendocument.chart
+        application/vnd.oasis.opendocument.text-master
+        application/x-vnd.oasis.opendocument.text-master
+        application/x-vnd.oasis.opendocument.text-template
+        application/vnd.oasis.opendocument.graphics
+        application/vnd.oasis.opendocument.graphics-template
+        application/x-vnd.oasis.opendocument.presentation
+        application/vnd.oasis.opendocument.image
+        application/x-vnd.oasis.opendocument.presentation-template
+        application/vnd.oasis.opendocument.chart
+      ].freeze,
+      :one => %w[
+        application/onenote
+        application/msonenote
+      ].freeze,
+      :pdf => %w[
+        application/pdf
+      ].freeze,
+      :ppt => %w[
+        application/vnd.ms-powerpoint
+      ].freeze,
+      :pptx => %w[
+        application/vnd.openxmlformats-officedocument.presentationml.presentation
+        application/vnd.ms-powerpoint.presentation.macroenabled.12
+        application/vnd.openxmlformats-officedocument.presentationml.template
+        application/vnd.ms-powerpoint.slideshow.macroenabled.12
+        application/vnd.ms-powerpoint.addin.macroenabled.12
+        application/vnd.openxmlformats-officedocument.presentationml.slideshow
+      ].freeze,
+      :rtf => %w[
+        message/richtext
+        text/richtext
+        text/rtf
+        application/rtf
+      ].freeze,
+      :txt => %w[
+        text/plain
+      ].freeze,
+      :xls => %w[
+        application/x-tika-msoffice
+        application/vnd.ms-excel
+        application/vnd.ms-excel.sheet.3
+        application/vnd.ms-excel.sheet.2
+        application/vnd.ms-excel.workspace.3
+        application/vnd.ms-excel.workspace.4
+        application/vnd.ms-excel.sheet.4
+      ].freeze,
+      :xlsx => %w[
+        application/x-tika-ooxml
+        application/vnd.openxmlformats-officedocument.spreadsheetml.sheet
+        application/vnd.openxmlformats-officedocument.spreadsheetml.template
+        application/vnd.ms-excel.addin.macroenabled.12
+        application/vnd.ms-excel.template.macroenabled.12
+        application/vnd.ms-excel.sheet.macroenabled.12
+      ].freeze
+    }.with_indifferent_access.freeze
+
+    def self.mime_to_extension
+      @mime_to_extension ||= @extension_to_mime.each_with_object({}) do |(key, values), memo|
+        values.each { |value| memo[value] = key.to_s }
+      end.with_indifferent_access.freeze
+    end
+
+    def self.get_extension(mime_type)
+      mime_to_extension[mime_type.to_s.downcase]
+    end
+
+    def self.get_mime_types(extension)
+      @extension_to_mime[extension.to_s.downcase]
+    end
+  end
+end

data/lib/utility/logger.rb

@@ -0,0 +1,84 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+require 'logger'
+require 'active_support/core_ext/module'
+require 'active_support/core_ext/string/filters'
+require 'ecs_logging/logger'
+
+module Utility
+  class Logger
+    SUPPORTED_LOG_LEVELS = %i[fatal error warn info debug].freeze
+    MAX_SHORT_MESSAGE_LENGTH = 1000.freeze
+
+    class << self
+
+      delegate :formatter, :formatter=, :to => :logger
+
+      def level=(log_level)
+        logger.level = log_level
+      end
+
+      def logger
+        @logger ||= Settings[:ecs_logging] ? EcsLogging::Logger.new(STDOUT) : ::Logger.new(STDOUT)
+      end
+
+      SUPPORTED_LOG_LEVELS.each do |level|
+        define_method(level) do |message|
+          if logger.is_a?(EcsLogging::Logger)
+            logger.public_send(level, message, extra_ecs_fields)
+          else
+            logger.public_send(level, message)
+          end
+        end
+      end
+
+      def log_stacktrace(stacktrace)
+        if logger.is_a?(EcsLogging::Logger)
+          logger.error(nil, extra_ecs_fields.merge(:error => { :stack_trace => stacktrace }))
+        else
+          logger.error(stacktrace)
+        end
+      end
+
+      def error_with_backtrace(message: nil, exception: nil, prog_name: nil)
+        logger.error(prog_name) { message } if message
+        logger.error exception.message if exception
+        logger.error exception.backtrace.join("\n") if exception
+      end
+
+      def new_line
+        logger.info("\n")
+      end
+
+      def generate_trace_id
+        SecureRandom.uuid
+      end
+
+      def abbreviated_message(message)
+        message.gsub(/\s+/, ' ').strip.truncate(MAX_SHORT_MESSAGE_LENGTH)
+      end
+
+      private
+
+      def extra_ecs_fields
+        {
+          :labels => { :index_date => Time.now.strftime('%Y.%m.%d') },
+          :log => { :logger => logger.progname },
+          :service => {
+            :type => 'connectors-ruby',
+            :version => Settings.version
+          },
+          :process => {
+            :pid => Process.pid,
+            :name => $PROGRAM_NAME,
+            :thread => Thread.current.object_id
+          }
+        }
+      end
+    end
+  end
+end

data/lib/utility/middleware/basic_auth.rb

@@ -0,0 +1,27 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+module Utility
+  module Middleware
+    class BasicAuth
+      AUTHORIZATION = 'Authorization'
+
+      attr_reader :basic_auth_token
+
+      def initialize(app = nil, options = {})
+        @app = app
+        @basic_auth_token = options.fetch(:basic_auth_token)
+      end
+
+      def call(env)
+        env.request_headers[AUTHORIZATION] = "Basic #{basic_auth_token}"
+        @app.call(env)
+      end
+    end
+  end
+end

data/lib/utility/middleware/bearer_auth.rb

@@ -0,0 +1,27 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+module Utility
+  module Middleware
+    class BearerAuth
+      AUTHORIZATION = 'Authorization'
+
+      attr_reader :bearer_auth_token
+
+      def initialize(app = nil, options = {})
+        @app = app
+        @bearer_auth_token = options.fetch(:bearer_auth_token)
+      end
+
+      def call(env)
+        env.request_headers[AUTHORIZATION] = "Bearer #{bearer_auth_token}"
+        @app.call(env)
+      end
+    end
+  end
+end

data/lib/utility/middleware/restrict_hostnames.rb

@@ -0,0 +1,73 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'faraday/middleware'
+require 'resolv'
+
+require 'utility/errors'
+require 'utility/logger'
+
+module Utility
+  module Middleware
+    class RestrictHostnames < Faraday::Middleware
+      class AddressNotAllowed < Utility::ClientError; end
+      URL_PATTERN = /\Ahttp/
+
+      attr_reader :allowed_hosts, :allowed_ips
+
+      def initialize(app = nil, options = {})
+        super(app)
+        @allowed_hosts = options[:allowed_hosts]
+        @allowed_ips = ips_from_hosts(@allowed_hosts)
+      end
+
+      def call(env)
+        raise AddressNotAllowed.new("Address not allowed for #{env[:url]}") if denied?(env)
+        @app.call(env)
+      end
+
+      private
+
+      def ips_from_hosts(hosts)
+        hosts&.flat_map do |host|
+          if URL_PATTERN.match(host)
+            lookup_ips(Addressable::URI.parse(host).hostname)
+          elsif Resolv::IPv4::Regex.match(host) || Resolv::IPv6::Regex.match(host)
+            IPAddr.new(host)
+          else
+            lookup_ips(host)
+          end
+        end || []
+      end
+
+      def denied?(env)
+        requested_ips = lookup_ips(env[:url].hostname)
+        no_match = requested_ips.all? { |ip| !@allowed_ips.include?(ip) }
+        return false unless no_match
+        Utility::Logger.warn("Requested url #{env[:url]} with resolved ip addresses #{requested_ips} does not match " \
+                                  "allowed hosts #{@allowed_hosts} with resolved ip addresses #{@allowed_ips}. Retrying.")
+        @allowed_ips = ips_from_hosts(@allowed_hosts) # maybe the IP has changed for an allowed host. Re-do allowed_hosts DNS lookup
+        no_match = requested_ips.all? { |ip| !@allowed_ips.include?(ip) }
+        Utility::Logger.error("Requested url #{env[:url]} with resolved ip addresses #{requested_ips} does not match " \
+                                  "allowed hosts #{@allowed_hosts} with resolved ip addresses #{@allowed_ips}") if no_match
+        no_match
+      end
+
+      def lookup_ips(hostname)
+        addr_infos(hostname).map { |a| IPAddr.new(a.ip_address) }
+      end
+
+      def addr_infos(hostname)
+        Addrinfo.getaddrinfo(hostname, nil, :UNSPEC, :STREAM)
+      rescue SocketError
+        # In case of invalid hostname, return an empty list of addresses
+        []
+      end
+    end
+  end
+end

data/lib/utility.rb

@@ -0,0 +1,16 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+require 'utility/constants'
+require 'utility/cron'
+require 'utility/errors'
+require 'utility/es_client'
+require 'utility/environment'
+require 'utility/exception_tracking'
+require 'utility/extension_mapping_util'
+require 'utility/logger'
+require 'utility/elasticsearch/index/mappings'
+require 'utility/elasticsearch/index/text_analysis_settings'