connectors_utility 8.4.0.1 → 8.5.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e33f6b71650c6bf509ad4dec8065a97dae7b48e01feb442a163c63aa55acbeee
-  data.tar.gz: 6813830e1050d3c3361936976860ea08418c4d2ad7ae6764640096f83fb4572c
+  metadata.gz: 1aa92692e811978885951049ebd040709d0f286ce4498d93dce54189ff7b8135
+  data.tar.gz: deda3bf664bb03c0b41e75acfa68b4cfa0abce03fd9fd32909c2e0a130686d14
 SHA512:
-  metadata.gz: 73ac602ac9da3526e9104ee52c91b417a965c3e7ee1e8bd937695742ea0e5b8dd09fc72e56ae019c8b1ced48b1a72a26e2d1252e7fc2c99d35e495df6e5144b9
-  data.tar.gz: b97eef3df6c98684477ecd6d709504602e32add75f32e1df0c0c04f83d034631dde36458cd78d1c506d71309f4df0ab0eb7ac5411d16ae79800eb3652bd87d3e
+  metadata.gz: 648d92295a527ccb6541596e030322cd9d7f6d8858edb90981f635f00f83083667e1e29837649b3310226ba180d0dd2316dc6bcb3bd12d53d4ec7c27f0eeb287
+  data.tar.gz: ac6c2c9b5f1bdf0334f0e7379fe0cddcb7a9cf1d32ed6ad7843da9292cd1e5c025cdd31b10f7c45a7a0d5c23969879ddeee9d7938ecc8a13b4ed577696ba0e08

data/lib/connectors/connector_status.rb

@@ -0,0 +1,31 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+module Connectors
+  class ConnectorStatus
+    CREATED = 'created'
+    NEEDS_CONFIGURATION = 'needs_configuration'
+    CONFIGURED = 'configured'
+    CONNECTED = 'connected'
+    ERROR = 'error'
+
+    STATUSES = [
+      CREATED,
+      NEEDS_CONFIGURATION,
+      CONFIGURED,
+      CONNECTED,
+      ERROR
+    ]
+
+    STATUSES_ALLOWING_SYNC = [
+      CONFIGURED,
+      CONNECTED,
+      ERROR
+    ]
+  end
+end

data/lib/connectors/crawler/scheduler.rb

@@ -0,0 +1,26 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'core/scheduler'
+require 'core/connector_settings'
+require 'core/elastic_connector_actions'
+require 'utility/logger'
+require 'utility/exception_tracking'
+
+module Connectors
+  module Crawler
+    class Scheduler < Core::Scheduler
+      def connector_settings
+        Core::ConnectorSettings.fetch_crawler_connectors || []
+      rescue StandardError => e
+        Utility::ExceptionTracking.log_exception(e, 'Could not retrieve Crawler connectors due to unexpected error.')
+        []
+      end
+    end
+  end
+end

data/lib/connectors/sync_status.rb

@@ -0,0 +1,21 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+module Connectors
+  class SyncStatus
+    COMPLETED = 'completed'
+    IN_PROGRESS = 'in_progress'
+    FAILED = 'failed'
+
+    STATUSES = [
+      COMPLETED,
+      IN_PROGRESS,
+      FAILED
+    ]
+  end
+end

data/lib/connectors_utility.rb

@@ -6,5 +6,11 @@
 
 # frozen_string_literal: true
 
-require_relative 'utility/elasticsearch/index/text_analysis_settings'
-require_relative 'utility/elasticsearch/index/mappings'
+require_relative 'utility'
+
+require_relative 'connectors/connector_status'
+require_relative 'connectors/sync_status'
+require_relative 'core/scheduler'
+require_relative 'core/elastic_connector_actions'
+
+require_relative 'connectors/crawler/scheduler'

data/lib/core/connector_settings.rb

@@ -0,0 +1,142 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'active_support/core_ext/hash/indifferent_access'
+require 'connectors/connector_status'
+require 'core/elastic_connector_actions'
+require 'utility'
+
+module Core
+  class ConnectorSettings
+
+    DEFAULT_REQUEST_PIPELINE = 'ent-search-generic-ingestion'
+    DEFAULT_EXTRACT_BINARY_CONTENT = true
+    DEFAULT_REDUCE_WHITESPACE = true
+    DEFAULT_RUN_ML_INFERENCE = true
+
+    DEFAULT_PAGE_SIZE = 100
+
+    # Error Classes
+    class ConnectorNotFoundError < StandardError; end
+
+    def self.fetch_by_id(connector_id)
+      es_response = ElasticConnectorActions.get_connector(connector_id)
+      connectors_meta = ElasticConnectorActions.connectors_meta
+
+      raise ConnectorNotFoundError.new("Connector with id=#{connector_id} was not found.") unless es_response[:found]
+      new(es_response, connectors_meta)
+    end
+
+    def initialize(es_response, connectors_meta)
+      @elasticsearch_response = es_response.with_indifferent_access
+      @connectors_meta = connectors_meta.with_indifferent_access
+    end
+
+    def self.fetch_native_connectors(page_size = DEFAULT_PAGE_SIZE)
+      query = { term: { is_native: true } }
+      fetch_connectors_by_query(query, page_size)
+    end
+
+    def self.fetch_crawler_connectors(page_size = DEFAULT_PAGE_SIZE)
+      query = { term: { service_type: Utility::Constants::CRAWLER_SERVICE_TYPE } }
+      fetch_connectors_by_query(query, page_size)
+    end
+
+    def id
+      @elasticsearch_response[:_id]
+    end
+
+    def [](property_name)
+      # TODO: handle not found
+      @elasticsearch_response[:_source][property_name]
+    end
+
+    def index_name
+      self[:index_name]
+    end
+
+    def connector_status
+      self[:status]
+    end
+
+    def connector_status_allows_sync?
+      Connectors::ConnectorStatus::STATUSES_ALLOWING_SYNC.include?(connector_status)
+    end
+
+    def service_type
+      self[:service_type]
+    end
+
+    def configuration
+      self[:configuration]
+    end
+
+    def scheduling_settings
+      self[:scheduling]
+    end
+
+    def request_pipeline
+      return_if_present(@elasticsearch_response.dig(:pipeline, :name), @connectors_meta.dig(:pipeline, :default_name), DEFAULT_REQUEST_PIPELINE)
+    end
+
+    def extract_binary_content?
+      return_if_present(@elasticsearch_response.dig(:pipeline, :extract_binary_content), @connectors_meta.dig(:pipeline, :default_extract_binary_content), DEFAULT_EXTRACT_BINARY_CONTENT)
+    end
+
+    def reduce_whitespace?
+      return_if_present(@elasticsearch_response.dig(:pipeline, :reduce_whitespace), @connectors_meta.dig(:pipeline, :default_reduce_whitespace), DEFAULT_REDUCE_WHITESPACE)
+    end
+
+    def run_ml_inference?
+      return_if_present(@elasticsearch_response.dig(:pipeline, :run_ml_inference), @connectors_meta.dig(:pipeline, :default_run_ml_inference), DEFAULT_RUN_ML_INFERENCE)
+    end
+
+    def formatted
+      properties = ["ID: #{id}"]
+      properties << "Service type: #{service_type}" if service_type
+      "connector (#{properties.join(', ')})"
+    end
+
+    def needs_service_type?
+      service_type.to_s.strip.empty?
+    end
+
+    def valid_index_name?
+      index_name&.start_with?(Utility::Constants::CONTENT_INDEX_PREFIX)
+    end
+
+    private
+
+    def self.fetch_connectors_by_query(query, page_size)
+      connectors_meta = ElasticConnectorActions.connectors_meta
+
+      results = []
+      offset = 0
+      loop do
+        response = ElasticConnectorActions.search_connectors(query, page_size, offset)
+
+        hits = response['hits']['hits']
+        total = response['hits']['total']['value']
+        results += hits.map do |hit|
+          Core::ConnectorSettings.new(hit, connectors_meta)
+        end
+        break if results.size >= total
+        offset += hits.size
+      end
+
+      results
+    end
+
+    def return_if_present(*args)
+      args.each do |arg|
+        return arg unless arg.nil?
+      end
+      nil
+    end
+  end
+end

data/lib/core/elastic_connector_actions.rb

@@ -0,0 +1,269 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+#
+require 'active_support/core_ext/hash'
+require 'connectors/connector_status'
+require 'connectors/sync_status'
+require 'utility'
+
+module Core
+  class ElasticConnectorActions
+    class << self
+
+      def force_sync(connector_id)
+        update_connector_fields(connector_id, :scheduling => { :enabled => true }, :sync_now => true)
+      end
+
+      def create_connector(index_name, service_type)
+        body = {
+          :scheduling => { :enabled => true },
+          :index_name => index_name,
+          :service_type => service_type
+        }
+        response = client.index(:index => Utility::Constants::CONNECTORS_INDEX, :body => body)
+        response['_id']
+      end
+
+      def get_connector(connector_id)
+        client.get(:index => Utility::Constants::CONNECTORS_INDEX, :id => connector_id, :ignore => 404).with_indifferent_access
+      end
+
+      def connectors_meta
+        alias_mappings = client.indices.get_mapping(:index => Utility::Constants::CONNECTORS_INDEX).with_indifferent_access
+        index = get_latest_index_in_alias(Utility::Constants::CONNECTORS_INDEX, alias_mappings.keys)
+        alias_mappings.dig(index, 'mappings', '_meta') || {}
+      end
+
+      def search_connectors(query, page_size, offset)
+        client.search(
+          :index => Utility::Constants::CONNECTORS_INDEX,
+          :ignore => 404,
+          :body => {
+            :size => page_size,
+            :from => offset,
+            :query => query,
+            :sort => ['name']
+          }
+        )
+      end
+
+      def update_connector_configuration(connector_id, configuration)
+        update_connector_fields(connector_id, :configuration => configuration)
+      end
+
+      def enable_connector_scheduling(connector_id, cron_expression)
+        payload = { :enabled => true, :interval => cron_expression }
+        update_connector_fields(connector_id, :scheduling => payload)
+      end
+
+      def disable_connector_scheduling(connector_id)
+        payload = { :enabled => false }
+        update_connector_fields(connector_id, :scheduling => payload)
+      end
+
+      def set_configurable_field(connector_id, field_name, label, value)
+        payload = { field_name => { :value => value, :label => label } }
+        update_connector_configuration(connector_id, payload)
+      end
+
+      def claim_job(connector_id)
+        update_connector_fields(connector_id,
+                                :sync_now => false,
+                                :last_sync_status => Connectors::SyncStatus::IN_PROGRESS,
+                                :last_synced => Time.now)
+
+        body = {
+          :connector_id => connector_id,
+          :status => Connectors::SyncStatus::IN_PROGRESS,
+          :worker_hostname => Socket.gethostname,
+          :created_at => Time.now
+        }
+        job = client.index(:index => Utility::Constants::JOB_INDEX, :body => body)
+
+        job['_id']
+      end
+
+      def update_connector_status(connector_id, status, error_message = nil)
+        if status == Connectors::ConnectorStatus::ERROR && error_message.nil?
+          raise ArgumentError, 'error_message is required when status is error'
+        end
+        body = {
+          :status => status,
+          :error => status == Connectors::ConnectorStatus::ERROR ? error_message : nil
+        }
+        update_connector_fields(connector_id, body)
+      end
+
+      def complete_sync(connector_id, job_id, status)
+        sync_status = status[:error] ? Connectors::SyncStatus::FAILED : Connectors::SyncStatus::COMPLETED
+
+        update_connector_fields(connector_id,
+                                :last_sync_status => sync_status,
+                                :last_sync_error => status[:error],
+                                :error => status[:error],
+                                :last_synced => Time.now,
+                                :last_indexed_document_count => status[:indexed_document_count],
+                                :last_deleted_document_count => status[:deleted_document_count])
+
+        body = {
+          :doc => {
+            :status => sync_status,
+            :completed_at => Time.now
+          }.merge(status)
+        }
+        client.update(:index => Utility::Constants::JOB_INDEX, :id => job_id, :body => body)
+      end
+
+      def fetch_document_ids(index_name)
+        page_size = 1000
+        result = []
+        begin
+          pit_id = client.open_point_in_time(:index => index_name, :keep_alive => '1m', :expand_wildcards => 'all')['id']
+          body = {
+            :query => { :match_all => {} },
+            :sort => [{ :id => { :order => :asc } }],
+            :pit => {
+              :id => pit_id,
+              :keep_alive => '1m'
+            },
+            :size => page_size,
+            :_source => false
+          }
+          loop do
+            response = client.search(:body => body)
+            hits = response['hits']['hits']
+
+            ids = hits.map { |h| h['_id'] }
+            result += ids
+            break if hits.size < page_size
+
+            body[:search_after] = hits.last['sort']
+            body[:pit][:id] = response['pit_id']
+          end
+        ensure
+          client.close_point_in_time(:index => index_name, :body => { :id => pit_id })
+        end
+
+        result
+      end
+
+      def ensure_content_index_exists(index_name, use_icu_locale = false, language_code = nil)
+        settings = Utility::Elasticsearch::Index::TextAnalysisSettings.new(:language_code => language_code, :analysis_icu => use_icu_locale).to_h
+        mappings = Utility::Elasticsearch::Index::Mappings.default_text_fields_mappings(:connectors_index => true)
+
+        body_payload = { settings: settings, mappings: mappings }
+        ensure_index_exists(index_name, body_payload)
+      end
+
+      def ensure_index_exists(index_name, body = {})
+        if client.indices.exists?(:index => index_name)
+          return unless body[:mappings]
+          Utility::Logger.debug("Index #{index_name} already exists. Checking mappings...")
+          Utility::Logger.debug("New mappings: #{body[:mappings]}")
+          response = client.indices.get_mapping(:index => index_name)
+          existing = response[index_name]['mappings']
+          if existing.empty?
+            Utility::Logger.debug("Index #{index_name} has no mappings. Adding mappings...")
+            client.indices.put_mapping(:index => index_name, :body => body[:mappings], :expand_wildcards => 'all')
+            Utility::Logger.debug("Index #{index_name} mappings added.")
+          else
+            Utility::Logger.debug("Index #{index_name} already has mappings: #{existing}. Skipping...")
+          end
+        else
+          client.indices.create(:index => index_name, :body => body)
+          Utility::Logger.debug("Created index #{index_name}")
+        end
+      end
+
+      def system_index_body(alias_name: nil, mappings: nil)
+        body = {
+          :settings => {
+            :index => {
+              :hidden => true,
+              :number_of_replicas => 0,
+              :auto_expand_replicas => '0-5'
+            }
+          }
+        }
+        body[:aliases] = { alias_name => { :is_write_index => true } } unless alias_name.nil? || alias_name.empty?
+        body[:mappings] = mappings unless mappings.nil?
+        body
+      end
+
+      # DO NOT USE this method
+      # Creation of connector index should be handled by Kibana, this method is only used by ftest.rb
+      def ensure_connectors_index_exists
+        mappings = {
+          :properties => {
+            :api_key_id => { :type => :keyword },
+            :configuration => { :type => :object },
+            :error => { :type => :text },
+            :index_name => { :type => :keyword },
+            :last_seen => { :type => :date },
+            :last_synced => { :type => :date },
+            :last_indexed_document_count => { :type => :integer },
+            :last_deleted_document_count => { :type => :integer },
+            :scheduling => {
+              :properties => {
+                :enabled => { :type => :boolean },
+                :interval => { :type => :text }
+              }
+            },
+            :service_type => { :type => :keyword },
+            :status => { :type => :keyword },
+            :sync_error => { :type => :text },
+            :sync_now => { :type => :boolean },
+            :sync_status => { :type => :keyword }
+          }
+        }
+        ensure_index_exists("#{Utility::Constants::CONNECTORS_INDEX}-v1", system_index_body(:alias_name => Utility::Constants::CONNECTORS_INDEX, :mappings => mappings))
+      end
+
+      # DO NOT USE this method
+      # Creation of job index should be handled by Kibana, this method is only used by ftest.rb
+      def ensure_job_index_exists
+        mappings = {
+          :properties => {
+            :connector_id => { :type => :keyword },
+            :status => { :type => :keyword },
+            :error => { :type => :text },
+            :worker_hostname => { :type => :keyword },
+            :indexed_document_count => { :type => :integer },
+            :deleted_document_count => { :type => :integer },
+            :created_at => { :type => :date },
+            :completed_at => { :type => :date }
+          }
+        }
+        ensure_index_exists("#{Utility::Constants::JOB_INDEX}-v1", system_index_body(:alias_name => Utility::Constants::JOB_INDEX, :mappings => mappings))
+      end
+
+      def update_connector_fields(connector_id, doc = {})
+        return if doc.empty?
+        client.update(
+          :index => Utility::Constants::CONNECTORS_INDEX,
+          :id => connector_id,
+          :body => { :doc => doc },
+          :refresh => true,
+          :retry_on_conflict => 3
+        )
+      end
+
+      private
+
+      def client
+        @client ||= Utility::EsClient.new(App::Config[:elasticsearch])
+      end
+
+      def get_latest_index_in_alias(alias_name, indicies)
+        index_versions = indicies.map { |index| index.gsub("#{alias_name}-v", '').to_i }
+        index_version = index_versions.max # gets the largest suffix number
+        "#{alias_name}-v#{index_version}"
+      end
+    end
+  end
+end

data/lib/core/scheduler.rb

@@ -0,0 +1,138 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'time'
+require 'fugit'
+require 'core/connector_settings'
+require 'utility/cron'
+require 'utility/logger'
+require 'utility/exception_tracking'
+
+module Core
+  class Scheduler
+    def initialize(poll_interval, heartbeat_interval)
+      @poll_interval = poll_interval
+      @heartbeat_interval = heartbeat_interval
+      @is_shutting_down = false
+    end
+
+    def connector_settings
+      raise 'Not implemented'
+    end
+
+    def when_triggered
+      loop do
+        connector_settings.each do |cs|
+          if sync_triggered?(cs)
+            yield cs, :sync
+          end
+          if heartbeat_triggered?(cs)
+            yield cs, :heartbeat
+          end
+          if configuration_triggered?(cs)
+            yield cs, :configuration
+          end
+        end
+        if @is_shutting_down
+          break
+        end
+      rescue StandardError => e
+        Utility::ExceptionTracking.log_exception(e, 'Sync failed due to unexpected error.')
+      ensure
+        if @poll_interval > 0 && !@is_shutting_down
+          Utility::Logger.info("Sleeping for #{@poll_interval} seconds in #{self.class}.")
+          sleep(@poll_interval)
+        end
+      end
+    end
+
+    def shutdown
+      Utility::Logger.info("Shutting down scheduler #{self.class.name}.")
+      @is_shutting_down = true
+    end
+
+    private
+
+    def sync_triggered?(connector_settings)
+      unless connector_settings.connector_status_allows_sync?
+        Utility::Logger.info("Connector #{connector_settings.id} is in status \"#{connector_settings.connector_status}\" and won't sync yet. Connector needs to be in one of the following statuses: #{Connectors::ConnectorStatus::STATUSES_ALLOWING_SYNC} to run.")
+
+        return false
+      end
+
+      # Sync when sync_now flag is true for the connector
+      if connector_settings[:sync_now] == true
+        Utility::Logger.info("Connector #{connector_settings.id} is manually triggered to sync now.")
+        return true
+      end
+
+      # Don't sync if sync is explicitly disabled
+      scheduling_settings = connector_settings.scheduling_settings
+      unless scheduling_settings.present? && scheduling_settings[:enabled] == true
+        Utility::Logger.info("Connector #{connector_settings.id} scheduling is disabled.")
+        return false
+      end
+
+      # We want to sync when sync never actually happened
+      last_synced = connector_settings[:last_synced]
+      if last_synced.nil? || last_synced.empty?
+        Utility::Logger.info("Connector #{connector_settings.id} has never synced yet, running initial sync.")
+        return true
+      end
+
+      current_schedule = scheduling_settings[:interval]
+
+      # Don't sync if there is no actual scheduling interval
+      if current_schedule.nil? || current_schedule.empty?
+        Utility::Logger.warn("No sync schedule configured for connector #{connector_settings.id}.")
+        return false
+      end
+
+      current_schedule = begin
+        Utility::Cron.quartz_to_crontab(current_schedule)
+      rescue StandardError => e
+        Utility::ExceptionTracking.log_exception(e, "Unable to convert quartz (#{current_schedule}) to crontab.")
+        return false
+      end
+      cron_parser = Fugit::Cron.parse(current_schedule)
+
+      # Don't sync if the scheduling interval is non-parsable
+      unless cron_parser
+        Utility::Logger.error("Unable to parse sync schedule for connector #{connector_settings.id}: expression #{current_schedule} is not a valid Quartz Cron definition.")
+        return false
+      end
+
+      next_trigger_time = cron_parser.next_time(Time.parse(last_synced))
+
+      # Sync if next trigger for the connector is in past
+      if next_trigger_time < Time.now
+        Utility::Logger.info("Connector #{connector_settings.id} sync is triggered by cron schedule #{current_schedule}.")
+        return true
+      end
+
+      false
+    end
+
+    def heartbeat_triggered?(connector_settings)
+      last_seen = connector_settings[:last_seen]
+      return true if last_seen.nil? || last_seen.empty?
+      last_seen = begin
+        Time.parse(last_seen)
+      rescue StandardError
+        Utility::Logger.warn("Unable to parse last_seen #{last_seen}")
+        nil
+      end
+      return true unless last_seen
+      last_seen + @heartbeat_interval < Time.now
+    end
+
+    def configuration_triggered?(connector_settings)
+      connector_settings.connector_status == Connectors::ConnectorStatus::CREATED
+    end
+  end
+end

data/lib/utility/constants.rb

@@ -0,0 +1,20 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+module Utility
+  class Constants
+    THUMBNAIL_FIELDS = %w[_thumbnail_80x100 _thumbnail_310x430].freeze
+    SUBEXTRACTOR_RESERVED_FIELDS = %w[_subextracted_as_of _subextracted_version].freeze
+    ALLOW_FIELD = '_allow_permissions'
+    DENY_FIELD = '_deny_permissions'
+    CONNECTORS_INDEX = '.elastic-connectors'
+    JOB_INDEX = '.elastic-connectors-sync-jobs'
+    CONTENT_INDEX_PREFIX = 'search-'
+    CRAWLER_SERVICE_TYPE = 'elastic-crawler'
+  end
+end