connectors_service 8.5.0.1

data/lib/core/connector_settings.rb

@@ -0,0 +1,142 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'active_support/core_ext/hash/indifferent_access'
+require 'connectors/connector_status'
+require 'core/elastic_connector_actions'
+require 'utility'
+
+module Core
+  class ConnectorSettings
+
+    DEFAULT_REQUEST_PIPELINE = 'ent-search-generic-ingestion'
+    DEFAULT_EXTRACT_BINARY_CONTENT = true
+    DEFAULT_REDUCE_WHITESPACE = true
+    DEFAULT_RUN_ML_INFERENCE = true
+
+    DEFAULT_PAGE_SIZE = 100
+
+    # Error Classes
+    class ConnectorNotFoundError < StandardError; end
+
+    def self.fetch_by_id(connector_id)
+      es_response = ElasticConnectorActions.get_connector(connector_id)
+      connectors_meta = ElasticConnectorActions.connectors_meta
+
+      raise ConnectorNotFoundError.new("Connector with id=#{connector_id} was not found.") unless es_response[:found]
+      new(es_response, connectors_meta)
+    end
+
+    def initialize(es_response, connectors_meta)
+      @elasticsearch_response = es_response.with_indifferent_access
+      @connectors_meta = connectors_meta.with_indifferent_access
+    end
+
+    def self.fetch_native_connectors(page_size = DEFAULT_PAGE_SIZE)
+      query = { term: { is_native: true } }
+      fetch_connectors_by_query(query, page_size)
+    end
+
+    def self.fetch_crawler_connectors(page_size = DEFAULT_PAGE_SIZE)
+      query = { term: { service_type: Utility::Constants::CRAWLER_SERVICE_TYPE } }
+      fetch_connectors_by_query(query, page_size)
+    end
+
+    def id
+      @elasticsearch_response[:_id]
+    end
+
+    def [](property_name)
+      # TODO: handle not found
+      @elasticsearch_response[:_source][property_name]
+    end
+
+    def index_name
+      self[:index_name]
+    end
+
+    def connector_status
+      self[:status]
+    end
+
+    def connector_status_allows_sync?
+      Connectors::ConnectorStatus::STATUSES_ALLOWING_SYNC.include?(connector_status)
+    end
+
+    def service_type
+      self[:service_type]
+    end
+
+    def configuration
+      self[:configuration]
+    end
+
+    def scheduling_settings
+      self[:scheduling]
+    end
+
+    def request_pipeline
+      return_if_present(@elasticsearch_response.dig(:pipeline, :name), @connectors_meta.dig(:pipeline, :default_name), DEFAULT_REQUEST_PIPELINE)
+    end
+
+    def extract_binary_content?
+      return_if_present(@elasticsearch_response.dig(:pipeline, :extract_binary_content), @connectors_meta.dig(:pipeline, :default_extract_binary_content), DEFAULT_EXTRACT_BINARY_CONTENT)
+    end
+
+    def reduce_whitespace?
+      return_if_present(@elasticsearch_response.dig(:pipeline, :reduce_whitespace), @connectors_meta.dig(:pipeline, :default_reduce_whitespace), DEFAULT_REDUCE_WHITESPACE)
+    end
+
+    def run_ml_inference?
+      return_if_present(@elasticsearch_response.dig(:pipeline, :run_ml_inference), @connectors_meta.dig(:pipeline, :default_run_ml_inference), DEFAULT_RUN_ML_INFERENCE)
+    end
+
+    def formatted
+      properties = ["ID: #{id}"]
+      properties << "Service type: #{service_type}" if service_type
+      "connector (#{properties.join(', ')})"
+    end
+
+    def needs_service_type?
+      service_type.to_s.strip.empty?
+    end
+
+    def valid_index_name?
+      index_name&.start_with?(Utility::Constants::CONTENT_INDEX_PREFIX)
+    end
+
+    private
+
+    def self.fetch_connectors_by_query(query, page_size)
+      connectors_meta = ElasticConnectorActions.connectors_meta
+
+      results = []
+      offset = 0
+      loop do
+        response = ElasticConnectorActions.search_connectors(query, page_size, offset)
+
+        hits = response['hits']['hits']
+        total = response['hits']['total']['value']
+        results += hits.map do |hit|
+          Core::ConnectorSettings.new(hit, connectors_meta)
+        end
+        break if results.size >= total
+        offset += hits.size
+      end
+
+      results
+    end
+
+    def return_if_present(*args)
+      args.each do |arg|
+        return arg unless arg.nil?
+      end
+      nil
+    end
+  end
+end

data/lib/core/elastic_connector_actions.rb

@@ -0,0 +1,269 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+#
+require 'active_support/core_ext/hash'
+require 'connectors/connector_status'
+require 'connectors/sync_status'
+require 'utility'
+
+module Core
+  class ElasticConnectorActions
+    class << self
+
+      def force_sync(connector_id)
+        update_connector_fields(connector_id, :scheduling => { :enabled => true }, :sync_now => true)
+      end
+
+      def create_connector(index_name, service_type)
+        body = {
+          :scheduling => { :enabled => true },
+          :index_name => index_name,
+          :service_type => service_type
+        }
+        response = client.index(:index => Utility::Constants::CONNECTORS_INDEX, :body => body)
+        response['_id']
+      end
+
+      def get_connector(connector_id)
+        client.get(:index => Utility::Constants::CONNECTORS_INDEX, :id => connector_id, :ignore => 404).with_indifferent_access
+      end
+
+      def connectors_meta
+        alias_mappings = client.indices.get_mapping(:index => Utility::Constants::CONNECTORS_INDEX).with_indifferent_access
+        index = get_latest_index_in_alias(Utility::Constants::CONNECTORS_INDEX, alias_mappings.keys)
+        alias_mappings.dig(index, 'mappings', '_meta') || {}
+      end
+
+      def search_connectors(query, page_size, offset)
+        client.search(
+          :index => Utility::Constants::CONNECTORS_INDEX,
+          :ignore => 404,
+          :body => {
+            :size => page_size,
+            :from => offset,
+            :query => query,
+            :sort => ['name']
+          }
+        )
+      end
+
+      def update_connector_configuration(connector_id, configuration)
+        update_connector_fields(connector_id, :configuration => configuration)
+      end
+
+      def enable_connector_scheduling(connector_id, cron_expression)
+        payload = { :enabled => true, :interval => cron_expression }
+        update_connector_fields(connector_id, :scheduling => payload)
+      end
+
+      def disable_connector_scheduling(connector_id)
+        payload = { :enabled => false }
+        update_connector_fields(connector_id, :scheduling => payload)
+      end
+
+      def set_configurable_field(connector_id, field_name, label, value)
+        payload = { field_name => { :value => value, :label => label } }
+        update_connector_configuration(connector_id, payload)
+      end
+
+      def claim_job(connector_id)
+        update_connector_fields(connector_id,
+                                :sync_now => false,
+                                :last_sync_status => Connectors::SyncStatus::IN_PROGRESS,
+                                :last_synced => Time.now)
+
+        body = {
+          :connector_id => connector_id,
+          :status => Connectors::SyncStatus::IN_PROGRESS,
+          :worker_hostname => Socket.gethostname,
+          :created_at => Time.now
+        }
+        job = client.index(:index => Utility::Constants::JOB_INDEX, :body => body)
+
+        job['_id']
+      end
+
+      def update_connector_status(connector_id, status, error_message = nil)
+        if status == Connectors::ConnectorStatus::ERROR && error_message.nil?
+          raise ArgumentError, 'error_message is required when status is error'
+        end
+        body = {
+          :status => status,
+          :error => status == Connectors::ConnectorStatus::ERROR ? error_message : nil
+        }
+        update_connector_fields(connector_id, body)
+      end
+
+      def complete_sync(connector_id, job_id, status)
+        sync_status = status[:error] ? Connectors::SyncStatus::FAILED : Connectors::SyncStatus::COMPLETED
+
+        update_connector_fields(connector_id,
+                                :last_sync_status => sync_status,
+                                :last_sync_error => status[:error],
+                                :error => status[:error],
+                                :last_synced => Time.now,
+                                :last_indexed_document_count => status[:indexed_document_count],
+                                :last_deleted_document_count => status[:deleted_document_count])
+
+        body = {
+          :doc => {
+            :status => sync_status,
+            :completed_at => Time.now
+          }.merge(status)
+        }
+        client.update(:index => Utility::Constants::JOB_INDEX, :id => job_id, :body => body)
+      end
+
+      def fetch_document_ids(index_name)
+        page_size = 1000
+        result = []
+        begin
+          pit_id = client.open_point_in_time(:index => index_name, :keep_alive => '1m', :expand_wildcards => 'all')['id']
+          body = {
+            :query => { :match_all => {} },
+            :sort => [{ :id => { :order => :asc } }],
+            :pit => {
+              :id => pit_id,
+              :keep_alive => '1m'
+            },
+            :size => page_size,
+            :_source => false
+          }
+          loop do
+            response = client.search(:body => body)
+            hits = response['hits']['hits']
+
+            ids = hits.map { |h| h['_id'] }
+            result += ids
+            break if hits.size < page_size
+
+            body[:search_after] = hits.last['sort']
+            body[:pit][:id] = response['pit_id']
+          end
+        ensure
+          client.close_point_in_time(:index => index_name, :body => { :id => pit_id })
+        end
+
+        result
+      end
+
+      def ensure_content_index_exists(index_name, use_icu_locale = false, language_code = nil)
+        settings = Utility::Elasticsearch::Index::TextAnalysisSettings.new(:language_code => language_code, :analysis_icu => use_icu_locale).to_h
+        mappings = Utility::Elasticsearch::Index::Mappings.default_text_fields_mappings(:connectors_index => true)
+
+        body_payload = { settings: settings, mappings: mappings }
+        ensure_index_exists(index_name, body_payload)
+      end
+
+      def ensure_index_exists(index_name, body = {})
+        if client.indices.exists?(:index => index_name)
+          return unless body[:mappings]
+          Utility::Logger.debug("Index #{index_name} already exists. Checking mappings...")
+          Utility::Logger.debug("New mappings: #{body[:mappings]}")
+          response = client.indices.get_mapping(:index => index_name)
+          existing = response[index_name]['mappings']
+          if existing.empty?
+            Utility::Logger.debug("Index #{index_name} has no mappings. Adding mappings...")
+            client.indices.put_mapping(:index => index_name, :body => body[:mappings], :expand_wildcards => 'all')
+            Utility::Logger.debug("Index #{index_name} mappings added.")
+          else
+            Utility::Logger.debug("Index #{index_name} already has mappings: #{existing}. Skipping...")
+          end
+        else
+          client.indices.create(:index => index_name, :body => body)
+          Utility::Logger.debug("Created index #{index_name}")
+        end
+      end
+
+      def system_index_body(alias_name: nil, mappings: nil)
+        body = {
+          :settings => {
+            :index => {
+              :hidden => true,
+              :number_of_replicas => 0,
+              :auto_expand_replicas => '0-5'
+            }
+          }
+        }
+        body[:aliases] = { alias_name => { :is_write_index => true } } unless alias_name.nil? || alias_name.empty?
+        body[:mappings] = mappings unless mappings.nil?
+        body
+      end
+
+      # DO NOT USE this method
+      # Creation of connector index should be handled by Kibana, this method is only used by ftest.rb
+      def ensure_connectors_index_exists
+        mappings = {
+          :properties => {
+            :api_key_id => { :type => :keyword },
+            :configuration => { :type => :object },
+            :error => { :type => :text },
+            :index_name => { :type => :keyword },
+            :last_seen => { :type => :date },
+            :last_synced => { :type => :date },
+            :last_indexed_document_count => { :type => :integer },
+            :last_deleted_document_count => { :type => :integer },
+            :scheduling => {
+              :properties => {
+                :enabled => { :type => :boolean },
+                :interval => { :type => :text }
+              }
+            },
+            :service_type => { :type => :keyword },
+            :status => { :type => :keyword },
+            :sync_error => { :type => :text },
+            :sync_now => { :type => :boolean },
+            :sync_status => { :type => :keyword }
+          }
+        }
+        ensure_index_exists("#{Utility::Constants::CONNECTORS_INDEX}-v1", system_index_body(:alias_name => Utility::Constants::CONNECTORS_INDEX, :mappings => mappings))
+      end
+
+      # DO NOT USE this method
+      # Creation of job index should be handled by Kibana, this method is only used by ftest.rb
+      def ensure_job_index_exists
+        mappings = {
+          :properties => {
+            :connector_id => { :type => :keyword },
+            :status => { :type => :keyword },
+            :error => { :type => :text },
+            :worker_hostname => { :type => :keyword },
+            :indexed_document_count => { :type => :integer },
+            :deleted_document_count => { :type => :integer },
+            :created_at => { :type => :date },
+            :completed_at => { :type => :date }
+          }
+        }
+        ensure_index_exists("#{Utility::Constants::JOB_INDEX}-v1", system_index_body(:alias_name => Utility::Constants::JOB_INDEX, :mappings => mappings))
+      end
+
+      def update_connector_fields(connector_id, doc = {})
+        return if doc.empty?
+        client.update(
+          :index => Utility::Constants::CONNECTORS_INDEX,
+          :id => connector_id,
+          :body => { :doc => doc },
+          :refresh => true,
+          :retry_on_conflict => 3
+        )
+      end
+
+      private
+
+      def client
+        @client ||= Utility::EsClient.new(App::Config[:elasticsearch])
+      end
+
+      def get_latest_index_in_alias(alias_name, indicies)
+        index_versions = indicies.map { |index| index.gsub("#{alias_name}-v", '').to_i }
+        index_version = index_versions.max # gets the largest suffix number
+        "#{alias_name}-v#{index_version}"
+      end
+    end
+  end
+end

data/lib/core/heartbeat.rb

@@ -0,0 +1,32 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'connectors/connector_status'
+require 'connectors/registry'
+require 'core/connector_settings'
+require 'core/elastic_connector_actions'
+
+module Core
+  class Heartbeat
+    class << self
+      def send(connector_settings)
+        doc = {
+            :last_seen => Time.now
+        }
+        if connector_settings.connector_status_allows_sync?
+          connector_instance = Connectors::REGISTRY.connector(connector_settings.service_type, connector_settings.configuration)
+          doc[:status] = connector_instance.is_healthy? ? Connectors::ConnectorStatus::CONNECTED : Connectors::ConnectorStatus::ERROR
+          message = "Health check for 3d party service failed for connector [#{connector_settings.id}], service type [#{connector_settings.service_type}]. Check the application logs for more information."
+          doc[:error] = doc[:status] == Connectors::ConnectorStatus::ERROR ? message : nil
+        end
+
+        Core::ElasticConnectorActions.update_connector_fields(connector_settings.id, doc)
+      end
+    end
+  end
+end

data/lib/core/native_scheduler.rb

@@ -0,0 +1,24 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'core/scheduler'
+require 'core/connector_settings'
+require 'core/elastic_connector_actions'
+require 'utility/logger'
+require 'utility/exception_tracking'
+
+module Core
+  class NativeScheduler < Core::Scheduler
+    def connector_settings
+      Core::ConnectorSettings.fetch_native_connectors || []
+    rescue StandardError => e
+      Utility::ExceptionTracking.log_exception(e, 'Could not retrieve native connectors due to unexpected error.')
+      []
+    end
+  end
+end

data/lib/core/output_sink/base_sink.rb

@@ -0,0 +1,33 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+module Core
+  module OutputSink
+    class BaseSink
+      def ingest(_document)
+        raise 'not implemented'
+      end
+
+      def ingest_multiple(_documents)
+        raise 'not implemented'
+      end
+
+      def delete(_id)
+        raise 'not implemented'
+      end
+
+      def delete_multiple(_ids)
+        raise 'not implemented'
+      end
+
+      def flush(_size: nil)
+        raise 'not implemented'
+      end
+    end
+  end
+end

data/lib/core/output_sink/combined_sink.rb

@@ -0,0 +1,38 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'core/output_sink/base_sink'
+require 'utility/logger'
+
+module Core::OutputSink
+  class CombinedSink < Core::OutputSink::BaseSink
+    def initialize(sinks = [])
+      @sinks = sinks
+    end
+
+    def ingest(document)
+      @sinks.each { |sink| sink.ingest(document) }
+    end
+
+    def flush(size: nil)
+      @sinks.each { |sink| sink.flush(size: size) }
+    end
+
+    def ingest_multiple(documents)
+      @sinks.each { |sink| sink.ingest_multiple(documents) }
+    end
+
+    def delete(id)
+      @sinks.each { |sink| sink.delete(id) }
+    end
+
+    def delete_multiple(ids)
+      @sinks.each { |sink| sink.delete_multiple(ids) }
+    end
+  end
+end

data/lib/core/output_sink/console_sink.rb

@@ -0,0 +1,51 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'core/output_sink'
+require 'utility/logger'
+
+module Core::OutputSink
+  class ConsoleSink < Core::OutputSink::BaseSink
+    def ingest(document)
+      print_header 'Got a single document:'
+      puts document
+    end
+
+    def flush(size: nil)
+      print_header 'Flushing'
+      puts "Flush size: #{size}"
+    end
+
+    def ingest_multiple(documents)
+      print_header 'Got multiple documents:'
+      puts documents
+    end
+
+    def delete(id)
+      print_header "Deleting single id: #{id}"
+      puts id
+    end
+
+    def delete_multiple(ids)
+      print_header "Deleting several ids: #{ids}"
+      puts ids
+    end
+
+    private
+
+    def print_delim
+      puts '----------------------------------------------------'
+    end
+
+    def print_header(header)
+      print_delim
+      puts header
+      print_delim
+    end
+  end
+end

data/lib/core/output_sink/es_sink.rb

@@ -0,0 +1,74 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'active_support/core_ext/numeric/time'
+require 'app/config'
+require 'core/output_sink/base_sink'
+require 'utility/es_client'
+require 'utility/logger'
+
+module Core::OutputSink
+  class EsSink < Core::OutputSink::BaseSink
+    def initialize(index_name, request_pipeline, flush_threshold = 50)
+      super()
+      @client = Utility::EsClient.new(App::Config[:elasticsearch])
+      @index_name = index_name
+      @request_pipeline = request_pipeline
+      @operation_queue = []
+      @flush_threshold = flush_threshold
+    end
+
+    def ingest(document)
+      return if document.blank?
+
+      @operation_queue << { :index => { :_index => index_name, :_id => document[:id], :data => document } }
+      flush if ready_to_flush?
+    end
+
+    def delete(doc_id)
+      return if doc_id.nil?
+
+      @operation_queue << { :delete => { :_index => index_name, :_id => doc_id } }
+      flush if ready_to_flush?
+    end
+
+    def flush(size: nil)
+      flush_size = size || @flush_threshold
+
+      while @operation_queue.any?
+        data_to_flush = @operation_queue.pop(flush_size)
+        send_data(data_to_flush)
+      end
+    end
+
+    def ingest_multiple(documents)
+      Utility::Logger.debug "Enqueueing #{documents&.size} documents to the index #{index_name}."
+      documents.each { |doc| ingest(doc) }
+    end
+
+    def delete_multiple(ids)
+      Utility::Logger.debug "Enqueueing #{ids&.size} ids to delete from the index #{index_name}."
+      ids.each { |id| delete(id) }
+    end
+
+    private
+
+    attr_accessor :index_name
+
+    def send_data(ops)
+      return if ops.empty?
+
+      @client.bulk(:body => ops, :pipeline => @request_pipeline)
+      Utility::Logger.info "Applied #{ops.size} upsert/delete operations to the index #{index_name}."
+    end
+
+    def ready_to_flush?
+      @operation_queue.size >= @flush_threshold
+    end
+  end
+end

data/lib/core/output_sink.rb

@@ -0,0 +1,13 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'core/output_sink/es_sink'
+require 'core/output_sink/console_sink'
+require 'core/output_sink/combined_sink'
+
+module Core::OutputSink; end