connectors_service 8.5.0.1

data/lib/app/dispatcher.rb

@@ -0,0 +1,121 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'active_support'
+require 'concurrent'
+require 'connectors'
+require 'core'
+require 'utility'
+require 'app/config'
+
+module App
+  class Dispatcher
+    POLL_INTERVAL = (App::Config.poll_interval || 3).to_i
+    TERMINATION_TIMEOUT = (App::Config.termination_timeout || 60).to_i
+    HEARTBEAT_INTERVAL = (App::Config.heartbeat_interval || 60 * 30).to_i
+    MIN_THREADS = (App::Config.dig(:thread_pool, :min_threads) || 0).to_i
+    MAX_THREADS = (App::Config.dig(:thread_pool, :max_threads) || 5).to_i
+    MAX_QUEUE = (App::Config.dig(:thread_pool, :max_queue) || 100).to_i
+
+    @running = Concurrent::AtomicBoolean.new(false)
+
+    class << self
+      def start!
+        running!
+        Utility::Logger.info("Starting connector service in #{App::Config.native_mode ? 'native' : 'non-native'} mode...")
+        start_polling_jobs!
+      end
+
+      def shutdown!
+        Utility::Logger.info("Shutting down connector service with pool [#{pool.class}]...")
+        running.make_false
+        scheduler.shutdown
+        pool.shutdown
+        pool.wait_for_termination(TERMINATION_TIMEOUT)
+      end
+
+      private
+
+      attr_reader :running
+
+      def running!
+        raise 'connector service is already running!' unless running.make_true
+      end
+
+      def pool
+        @pool ||= Concurrent::ThreadPoolExecutor.new(
+          min_threads: MIN_THREADS,
+          max_threads: MAX_THREADS,
+          max_queue: MAX_QUEUE,
+          fallback_policy: :abort
+        )
+      end
+
+      def scheduler
+        @scheduler ||= if App::Config.native_mode
+                         Core::NativeScheduler.new(POLL_INTERVAL, HEARTBEAT_INTERVAL)
+                       else
+                         Core::SingleScheduler.new(App::Config.connector_id, POLL_INTERVAL, HEARTBEAT_INTERVAL)
+                       end
+      end
+
+      def start_polling_jobs!
+        scheduler.when_triggered do |connector_settings, task|
+          case task
+          when :sync
+            start_sync_task(connector_settings)
+          when :heartbeat
+            start_heartbeat_task(connector_settings)
+          when :configuration
+            start_configuration_task(connector_settings)
+          else
+            Utility::Logger.error("Unknown task type: #{task}. Skipping...")
+          end
+        end
+      rescue StandardError => e
+        Utility::ExceptionTracking.log_exception(e, 'The connector service failed due to unexpected error.')
+      end
+
+      def start_sync_task(connector_settings)
+        start_heartbeat_task(connector_settings)
+        pool.post do
+          Utility::Logger.info("Starting a sync job for #{connector_settings.formatted}...")
+          Core::ElasticConnectorActions.ensure_content_index_exists(connector_settings.index_name)
+          job_runner = Core::SyncJobRunner.new(connector_settings)
+          job_runner.execute
+        rescue StandardError => e
+          Utility::ExceptionTracking.log_exception(e, "Sync job for #{connector_settings.formatted} failed due to unexpected error.")
+        end
+      end
+
+      def start_heartbeat_task(connector_settings)
+        pool.post do
+          Utility::Logger.info("Sending heartbeat for #{connector_settings.formatted}...")
+          Core::Heartbeat.send(connector_settings)
+        rescue StandardError => e
+          Utility::ExceptionTracking.log_exception(e, "Heartbeat task for #{connector_settings.formatted} failed due to unexpected error.")
+        end
+      end
+
+      def start_configuration_task(connector_settings)
+        pool.post do
+          Utility::Logger.info("Updating configuration for #{connector_settings.formatted}...")
+          # when in non-native mode, populate the service type if it's not in connector settings
+          service_type = if !App::Config.native_mode && connector_settings.needs_service_type?
+                           App::Config.service_type
+                         else
+                           nil
+                         end
+          Core::Configuration.update(connector_settings, service_type)
+        rescue StandardError => e
+          Utility::ExceptionTracking.log_exception(e, "Configuration task for #{connector_settings.formatted} failed due to unexpected error.")
+        end
+      end
+    end
+  end
+end

data/lib/app/menu.rb

@@ -0,0 +1,104 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+# frozen_string_literal: true
+#
+require 'remedy'
+
+module App
+  class Menu
+    attr_reader :items
+    attr_reader :title
+    attr_reader :index
+
+    def initialize(title, items)
+      super()
+      @index = 0
+      @title = title
+      @items = items.map.with_index do |item, i|
+        item.is_a?(String) ? MenuItem.new(item, nil, i == 0) : MenuItem.new(item[:command], item[:hint], i == 0)
+      end
+    end
+
+    def select_item(index)
+      @index = index
+      @items.each_with_index { |item, i| item.selected = (i == index) }
+      display
+    end
+
+    def select_command
+      display
+      interaction = Remedy::Interaction.new
+      interaction.loop do |key|
+        if key.nil?
+          break
+        end
+        case key.to_s.to_sym
+        when :down
+          index = @index + 1
+          index = 0 if index >= @items.size
+          select_item(index)
+        when :up
+          index = @index - 1
+          index = 0 if index < 0
+          select_item(index)
+        when :control_m
+          return @items[@index].command
+        end
+      end
+    end
+
+    private
+
+    def display
+      clear_screen
+      puts(title)
+      @items.each do |item|
+        print(item.selected ? '--> ' : '    ')
+        puts item.hint.present? ? "#{item.hint} (#{item.command})" : item.command
+      end
+    end
+
+    def clear_screen
+      system('clear') || system('cls')
+    end
+
+    def read_char
+      STDIN.echo = false
+      STDIN.raw!
+
+      input = STDIN.getc
+      if input == "\e"
+        begin
+          input << STDIN.read_nonblock(3)
+        rescue StandardError
+          nil
+        end
+        begin
+          input << STDIN.read_nonblock(2)
+        rescue StandardError
+          nil
+        end
+      end
+      input
+    ensure
+      STDIN.echo = true
+      STDIN.cooked!
+    end
+  end
+
+  class MenuItem
+    attr_reader :command
+    attr_reader :hint
+    attr_accessor :selected
+
+    def initialize(command, hint = nil, selected = false)
+      super()
+      @command = command
+      @hint = hint
+      @selected = selected
+    end
+  end
+end

data/lib/app/preflight_check.rb

@@ -0,0 +1,134 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'app/version'
+require 'utility'
+require 'faraday'
+
+module App
+  class PreflightCheck
+    class CheckFailure < StandardError; end
+    class UnhealthyCluster < StandardError; end
+
+    STARTUP_RETRY_INTERVAL = 5
+    STARTUP_RETRY_TIMEOUT = 600
+
+    class << self
+      def run!
+        check_es_connection!
+        check_es_version!
+        check_system_indices!
+      end
+
+      private
+
+      #-------------------------------------------------------------------------------------------------
+      # Checks to make sure we can connect to Elasticsearch and make API requests to it
+      def check_es_connection!
+        check_es_connection_with_retries!(
+          :retry_interval => STARTUP_RETRY_INTERVAL,
+          :retry_timeout => STARTUP_RETRY_TIMEOUT
+        )
+      end
+
+      #-------------------------------------------------------------------------------------------------
+      # Ensures that the version of Elasticsearch is compatible with connector service
+      def check_es_version!
+        info = client.info
+        version = info.dig('version', 'number')
+        fail_check!("Cannot retrieve version from Elasticsearch response:\n#{info.to_json}") unless version
+
+        if match_es_version?(version)
+          Utility::Logger.info("Connector service version (#{App::VERSION}) matches Elasticsearch version (#{version}).")
+        else
+          fail_check!("Connector service (#{App::VERSION}) is required to run with the same major and minor version of Elasticsearch (#{version}).")
+        end
+      end
+
+      #-------------------------------------------------------------------------------------------------
+      # Ensures that the required system indices of connector service exist
+      def check_system_indices!
+        check_system_indices_with_retries!(
+          :retry_interval => STARTUP_RETRY_INTERVAL,
+          :retry_timeout => STARTUP_RETRY_TIMEOUT
+        )
+      end
+
+      def check_es_connection_with_retries!(retry_interval:, retry_timeout:)
+        started_at = Time.now
+
+        begin
+          response = client.cluster.health
+          Utility::Logger.info('Successfully connected to Elasticsearch')
+          case response['status']
+          when 'green'
+            Utility::Logger.info('Elasticsearch is running and healthy.')
+          when 'yellow'
+            Utility::Logger.warn('Elasticsearch is running but the status is yellow.')
+          when 'red'
+            raise UnhealthyCluster, 'Elasticsearch is running but unhealthy.'
+          else
+            raise UnhealthyCluster, "Unexpected cluster status: #{response['status']}"
+          end
+        rescue *App::RETRYABLE_CONNECTION_ERRORS => e
+          Utility::Logger.warn('Could not connect to Elasticsearch. Make sure it is running and healthy.')
+          Utility::Logger.debug("Error: #{e.full_message}")
+
+          sleep(retry_interval)
+          time_elapsed = Time.now - started_at
+          retry if time_elapsed < retry_timeout
+
+          # If we ran out of time, there is not much we can do but shut down
+          fail_check!("Could not connect to Elasticsearch after #{time_elapsed.to_i} seconds. Terminating...")
+        end
+      end
+
+      def match_es_version?(es_version)
+        parse_minor_version(App::VERSION) == parse_minor_version(es_version)
+      end
+
+      def parse_minor_version(version)
+        version.split('.').slice(0, 2).join('.')
+      end
+
+      def check_system_indices_with_retries!(retry_interval:, retry_timeout:)
+        started_at = Time.now
+        loop do
+          if client.indices.exists?(:index => Utility::Constants::CONNECTORS_INDEX) && client.indices.exists?(:index => Utility::Constants::JOB_INDEX)
+            Utility::Logger.info("Found system indices #{Utility::Constants::CONNECTORS_INDEX} and #{Utility::Constants::JOB_INDEX}.")
+            return
+          end
+          Utility::Logger.warn('Required system indices for connector service don\'t exist. Make sure to run Kibana first to create system indices.')
+          sleep(retry_interval)
+          time_elapsed = Time.now - started_at
+          if time_elapsed > retry_timeout
+            fail_check!("Could not find required system indices after #{time_elapsed.to_i} seconds. Terminating...")
+          end
+        end
+      end
+
+      def client
+        @client ||= Utility::EsClient.new(App::Config[:elasticsearch])
+      end
+
+      def fail_check!(message)
+        raise CheckFailure, message
+      end
+    end
+  end
+
+  RETRYABLE_CONNECTION_ERRORS = [
+      ::Faraday::ConnectionFailed,
+      ::Faraday::ClientError,
+      ::Errno::ECONNREFUSED,
+      ::SocketError,
+      ::Errno::ECONNRESET,
+      App::PreflightCheck::UnhealthyCluster,
+      ::HTTPClient::KeepAliveDisconnected
+  ]
+end

data/lib/app/version.rb

@@ -0,0 +1,10 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+require 'app/config'
+
+module App
+  VERSION = App::Config[:version]
+end

data/lib/connectors/base/adapter.rb

@@ -0,0 +1,119 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+require 'active_support/core_ext/array/wrap'
+require 'active_support/core_ext/numeric/time'
+require 'active_support/core_ext/object/deep_dup'
+require 'active_support/core_ext/object/json'
+require 'utility'
+require 'utility/extension_mapping_util'
+require 'date'
+require 'json'
+require 'mime-types'
+
+module Connectors
+  module Base
+    class Adapter
+      def self.fields_to_preserve
+        @fields_to_preserve ||= ['body']
+          .concat(Utility::Constants::THUMBNAIL_FIELDS)
+          .concat(Utility::Constants::SUBEXTRACTOR_RESERVED_FIELDS)
+          .map(&:freeze)
+          .freeze
+      end
+
+      def self.generate_id_helpers(method_prefix, id_prefix)
+        define_singleton_method("#{method_prefix}_id_to_es_id") do |id|
+          "#{id_prefix}_#{id}"
+        end
+
+        define_singleton_method("es_id_is_#{method_prefix}_id?") do |es_id|
+          regex_match = /#{id_prefix}_(.+)$/.match(es_id)
+          regex_match.present? && regex_match.size == 2
+        end
+
+        define_singleton_method("es_id_to_#{method_prefix}_id") do |es_id|
+          regex_match = /#{id_prefix}_(.+)$/.match(es_id)
+
+          raise ArgumentError, "Invalid id #{es_id} for source with method prefix #{method_prefix}." if regex_match.nil? || regex_match.length != 2
+          regex_match[1]
+        end
+      end
+
+      def self.mime_type_for_file(file_name)
+        ruby_detected_type = MIME::Types.type_for(file_name)
+        return ruby_detected_type.first.simplified if ruby_detected_type.present?
+        extension = extension_for_file(file_name)
+        Utility::ExtensionMappingUtil.get_mime_types(extension)&.first
+      end
+
+      def self.extension_for_file(file_name)
+        File.extname(file_name.downcase).delete_prefix!('.')
+      end
+
+      def self.strip_file_extension(file_name)
+        File.basename(file_name, File.extname(file_name))
+      end
+
+      def self.normalize_enum(enum)
+        enum&.to_s&.downcase
+      end
+
+      def self.normalize_date(date)
+        return nil if date.blank?
+
+        case date
+        when Date, Time, DateTime, ActiveSupport::TimeWithZone
+          date.to_datetime.rfc3339
+        else
+          begin
+            Time.zone.parse(date).to_datetime.rfc3339
+          rescue ArgumentError, TypeError => e
+            Utility::ExceptionTracking.capture_exception(e)
+            nil
+          end
+        end
+      end
+
+      def self.normalize_path(path)
+        return nil if path.blank?
+        return path if path.start_with?('/')
+        "/#{path}"
+      end
+
+      def self.url_to_path(url)
+        return nil if url.blank?
+        uri = URI(url)
+        return nil if uri.scheme.blank?
+        normalize_path(uri.path)
+      rescue URI::InvalidURIError, ArgumentError
+        nil
+      end
+
+      def self.es_document_from_configured_object_base(object_type:, object:, fields:)
+        object_as_json = object.as_json
+
+        adapted_object = {
+          :type => normalize_enum(object_type)
+        }
+
+        fields.each do |field_data|
+          remote_field_name = field_data.fetch(:remote)
+
+          value = object_as_json[remote_field_name]
+          value = object_as_json.dig(*remote_field_name.split('.')) if value.blank?
+          next if value.nil?
+
+          adapted_object[field_data.fetch(:target)] = value
+        end
+
+        adapted_object.symbolize_keys
+      end
+
+      delegate :normalize_enum, :normalize_date, :normalize_path, :to => :class
+    end
+  end
+end

data/lib/connectors/base/connector.rb

@@ -0,0 +1,57 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'bson'
+require 'core/output_sink'
+require 'utility/exception_tracking'
+require 'utility/errors'
+require 'app/config'
+
+module Connectors
+  module Base
+    class Connector
+      def self.display_name
+        raise 'Not implemented for this connector'
+      end
+
+      def self.configurable_fields
+        {}
+      end
+
+      def self.service_type
+        raise 'Not implemented for this connector'
+      end
+
+      def initialize(configuration: {})
+        @configuration = configuration.dup || {}
+      end
+
+      def yield_documents; end
+
+      def do_health_check
+        raise 'Not implemented for this connector'
+      end
+
+      def do_health_check!
+        do_health_check
+      rescue StandardError => e
+        Utility::ExceptionTracking.log_exception(e, "Connector for service #{self.class.service_type} failed the health check for 3rd-party service.")
+        raise Utility::HealthCheckFailedError.new, e.message
+      end
+
+      def is_healthy?
+        do_health_check
+
+        true
+      rescue StandardError => e
+        Utility::ExceptionTracking.log_exception(e, "Connector for service #{self.class.service_type} failed the health check for 3rd-party service.")
+        false
+      end
+    end
+  end
+end

data/lib/connectors/base/custom_client.rb

@@ -0,0 +1,111 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+require 'faraday'
+require 'httpclient'
+require 'active_support/core_ext/array/wrap'
+require 'active_support/core_ext/numeric/time'
+require 'active_support/core_ext/object/deep_dup'
+require 'utility'
+require 'date'
+
+module Connectors
+  module Base
+    class CustomClient
+      attr_reader :base_url, :middleware, :ensure_fresh_auth
+
+      MAX_RETRIES = 5
+
+      def initialize(base_url: nil, ensure_fresh_auth: nil)
+        @base_url = base_url
+        @ensure_fresh_auth = ensure_fresh_auth
+        middleware!
+      end
+
+      def middleware!
+        @middleware = Array.wrap(additional_middleware)
+        @middleware += Array.wrap(default_middleware)
+        @middleware.compact!
+      end
+
+      def additional_middleware
+        [] # define as needed in subclass
+      end
+
+      def default_middleware
+        [[Faraday::Request::Retry, retry_config]]
+      end
+
+      def retry_config
+        {
+          :retry_statuses => [429],
+          :backoff_factor => 2,
+          :max => MAX_RETRIES,
+          :interval => 0.05
+        }
+      end
+
+      [
+        :delete,
+        :get,
+        :head,
+        :options,
+        :patch,
+        :post,
+        :put,
+      ].each do |http_verb|
+        define_method http_verb do |*args, &block|
+          ensure_fresh_auth.call(self) if ensure_fresh_auth.present?
+          http_client.public_send(http_verb, *args, &block)
+        end
+      end
+
+      def http_client!
+        @http_client = nil
+        http_client
+      end
+
+      def http_client
+        @http_client ||= Faraday.new(base_url) do |faraday|
+          middleware.each do |middleware_config|
+            faraday.use(*middleware_config)
+          end
+
+          faraday.adapter :httpclient
+        end
+      end
+
+      private
+
+      # https://github.com/lostisland/faraday/blob/b09c6db31591dd1a58fffcc0979b0c7d96b5388b/lib/faraday/connection.rb#L171
+      METHODS_WITH_BODY = [:post, :put, :patch].freeze
+
+      def send_body?(method)
+        METHODS_WITH_BODY.include?(method.to_sym)
+      end
+
+      def request_with_throttling(method, url, options = {})
+        response =
+          if send_body?(method)
+            public_send(method, url, options[:body], options[:headers])
+          else
+            public_send(method, url, options[:params], options[:headers])
+          end
+
+        if response.status == 429
+          retry_after = response.headers['Retry-After']
+          multiplier = options.fetch(:retry_mulitplier, 1)
+          retry_after_secs = (retry_after.is_a?(Array) ? retry_after.first.to_i : retry_after.to_i) * multiplier
+          retry_after_secs = 60 if retry_after_secs <= 0
+          Utility::Logger.warn("Exceeded #{self.class} request limits. Going to sleep for #{retry_after_secs} seconds")
+          raise Utility::ThrottlingError.new(:suspend_until => DateTime.now + retry_after_secs.seconds, :cursors => options[:cursors])
+        else
+          response
+        end
+      end
+    end
+  end
+end

data/lib/connectors/connector_status.rb

@@ -0,0 +1,31 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+module Connectors
+  class ConnectorStatus
+    CREATED = 'created'
+    NEEDS_CONFIGURATION = 'needs_configuration'
+    CONFIGURED = 'configured'
+    CONNECTED = 'connected'
+    ERROR = 'error'
+
+    STATUSES = [
+      CREATED,
+      NEEDS_CONFIGURATION,
+      CONFIGURED,
+      CONNECTED,
+      ERROR
+    ]
+
+    STATUSES_ALLOWING_SYNC = [
+      CONFIGURED,
+      CONNECTED,
+      ERROR
+    ]
+  end
+end