connectors_service 8.6.0.3 → 8.6.0.4.pre.20221104T200814Z

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b550a78dba7e4cd4502b6eea4c187253e6bb77ab944815b59809d92c7ccc23bb
-  data.tar.gz: dbe8c32e1da94ed40777af57a84ecac883a15cab71aa164534157314bbcbfcff
+  metadata.gz: 23b78b5c5228f761b884af1bfd75b163ad912dc53331ce5a64cd3edb8f45ef86
+  data.tar.gz: 5c74e06f315ab9af4161e88fac626e84b6d2789c0b5b080c713e780a4560ef97
 SHA512:
-  metadata.gz: 92ef83e3bc94107b1cb11b4454760b17c217b61beb555b55d01ddb7b0758372c3f04ed1acc153e406f20c7abb2df5a3540c2b93733b48eafcc99bd752e7d6759
-  data.tar.gz: a7eade1996d683fcf47e072704d566479538249d9711e8c0d2019fbb7bd15c382c61cd35adc6ee52f989a75c1fa6d7a6fe1330a04801fb46952aa307f2e93ac5
+  metadata.gz: 75295f0d26061a977089e999190f4974f4929ca92c25c09e7ecbbd1f0117476f758b7f200b719461bdd9e7b9e6193df455d3e52bcec0634fc85a7b512721ab6a
+  data.tar.gz: fa12af986f72d10081245e782a9152a887512e62f48ad525af59e59a4ef72238727e7bfd521f516fd452a2e74a1101c5bd1965f1e9b97f7b236b8ae2ccf626ff

data/config/connectors.yml

@@ -1,11 +1,10 @@
 # general metadata
-version: 8.6.0.3
-repository: https://github.com/elastic/connectors-ruby.git
-revision: aa2faf8cc993a26980441adffe97d62fdaf5aa5c
+version: 8.6.0.4-20221104T200814Z
+repository: git@github.com:elastic/ent-search-connectors.git
+revision: 2051b3907639a1fe2ae68efdc33c06cf12d38383
 elasticsearch:
-  cloud_id: CHANGEME
   hosts: http://localhost:9200
-  api_key: CHANGEME
+  api_key: OW1FalJJUUI1clBtUVh5RVo1QmU6QVp5LV9pU3RRUXFYb2VVYnlCRWNZdw==
   retry_on_failure: 3
   request_timeout: 120
   disable_warnings: true
@@ -16,10 +15,10 @@ thread_pool:
   max_threads: 5
   max_queue: 100
 log_level: info
-ecs_logging: true
+ecs_logging: false
 poll_interval: 3
 termination_timeout: 60
 heartbeat_interval: 1800
-native_mode: true
-connector_id: CHANGEME
-service_type: CHANGEME
+native_mode: false
+connector_id: 9WEjRIQB5rPmQXyEWJB2
+service_type: example

data/lib/app/config.rb

@@ -35,6 +35,8 @@ puts "Parsing #{CONFIG_FILE} configuration file."
       optional(:disable_warnings).value(:bool?)
       optional(:trace).value(:bool?)
       optional(:log).value(:bool?)
+      optional(:ca_fingerprint).value(:string)
+      optional(:transport_options).value(:hash)
     end
 
     optional(:thread_pool).hash do

data/lib/app/dispatcher.rb

@@ -84,10 +84,14 @@ module App
       def start_sync_task(connector_settings)
         start_heartbeat_task(connector_settings)
         pool.post do
-          Utility::Logger.info("Starting a sync job for #{connector_settings.formatted}...")
+          Utility::Logger.info("Initiating a sync job for #{connector_settings.formatted}...")
           Core::ElasticConnectorActions.ensure_content_index_exists(connector_settings.index_name)
           job_runner = Core::SyncJobRunner.new(connector_settings)
           job_runner.execute
+        rescue Core::JobAlreadyRunningError
+          Utility::Logger.info("Sync job for #{connector_settings.formatted} is already running, skipping.")
+        rescue Core::ConnectorVersionChangedError => e
+          Utility::Logger.info("Could not start the job because #{connector_settings.formatted} has been updated externally. Message: #{e.message}")
         rescue StandardError => e
           Utility::ExceptionTracking.log_exception(e, "Sync job for #{connector_settings.formatted} failed due to unexpected error.")
         end

data/lib/app/preflight_check.rb

@@ -75,6 +75,10 @@ module App
           else
             raise UnhealthyCluster, "Unexpected cluster status: #{response['status']}"
           end
+        rescue *Utility::AUTHORIZATION_ERRORS => e
+          Utility::ExceptionTracking.log_exception(e)
+
+          fail_check!("Elasticsearch returned 'Unauthorized' response. Check your authentication details. Terminating...")
         rescue *App::RETRYABLE_CONNECTION_ERRORS => e
           Utility::Logger.warn('Could not connect to Elasticsearch. Make sure it is running and healthy.')
           Utility::Logger.debug("Error: #{e.full_message}")

data/lib/connectors/base/connector.rb

@@ -8,9 +8,9 @@
 
 require 'bson'
 require 'core/output_sink'
-require 'utility/exception_tracking'
-require 'utility/errors'
+require 'utility'
 require 'app/config'
+require 'active_support/core_ext/hash/indifferent_access'
 
 module Connectors
   module Base
@@ -19,6 +19,11 @@ module Connectors
         raise 'Not implemented for this connector'
       end
 
+      # Used as a framework util method, don't override
+      def self.configurable_fields_indifferent_access
+        configurable_fields.with_indifferent_access
+      end
+
       def self.configurable_fields
         {}
       end
@@ -27,8 +32,16 @@ module Connectors
         raise 'Not implemented for this connector'
       end
 
-      def initialize(configuration: {})
+      attr_reader :rules, :advanced_filter_config
+
+      def initialize(configuration: {}, job_description: {})
         @configuration = configuration.dup || {}
+        @job_description = job_description&.dup || {}
+
+        filter = get_filter(@job_description[:filtering])
+
+        @rules = Utility::Common.return_if_present(filter[:rules], [])
+        @advanced_filter_config = Utility::Common.return_if_present(filter[:advanced_config], {})
       end
 
       def yield_documents; end
@@ -52,6 +65,19 @@ module Connectors
         Utility::ExceptionTracking.log_exception(e, "Connector for service #{self.class.service_type} failed the health check for 3rd-party service.")
         false
       end
+
+      def filtering_present?
+        @advanced_filter_config.present? || @rules.present?
+      end
+
+      private
+
+      def get_filter(filtering)
+        # assume for now, that first object in filtering array or a filter object itself is the only filtering object
+        filter = filtering.is_a?(Array) ? filtering[0] : filtering
+
+        filter.present? ? filter : {}
+      end
     end
   end
 end

data/lib/connectors/connector_status.rb

@@ -8,11 +8,11 @@
 
 module Connectors
   class ConnectorStatus
-    CREATED = 'created'
+    CREATED             = 'created'
     NEEDS_CONFIGURATION = 'needs_configuration'
-    CONFIGURED = 'configured'
-    CONNECTED = 'connected'
-    ERROR = 'error'
+    CONFIGURED          = 'configured'
+    CONNECTED           = 'connected'
+    ERROR               = 'error'
 
     STATUSES = [
       CREATED,

data/lib/connectors/example/connector.rb

@@ -20,16 +20,21 @@ module Connectors
         'Example Connector'
       end
 
+      # Field 'Foo' won't have a default value. Field 'Bar' will have the default value 'Value'.
       def self.configurable_fields
         {
           'foo' => {
             'label' => 'Foo',
             'value' => nil
+          },
+          :bar => {
+            :label => 'Bar',
+            :value => 'Value'
           }
         }
       end
 
-      def initialize(configuration: {})
+      def initialize(configuration: {}, job_description: {})
         super
       end
 
@@ -42,16 +47,35 @@ module Connectors
 
       def yield_documents
         attachments = [
-          File.open('./lib/connectors/example/example_attachments/first_attachment.txt'),
-          File.open('./lib/connectors/example/example_attachments/second_attachment.txt'),
-          File.open('./lib/connectors/example/example_attachments/third_attachment.txt')
+          load_attachment('first_attachment.txt'),
+          load_attachment('second_attachment.txt'),
+          load_attachment('third_attachment.txt'),
         ]
 
         attachments.each_with_index do |att, index|
           data = { id: (index + 1).to_s, name: "example document #{index + 1}", _attachment: File.read(att) }
+
+          # Uncomment one of these two lines to simulate longer running sync jobs
+          #
+          # sleep(rand(10..60).seconds)
+          # sleep(rand(1..10).minutes)
+
           yield data
         end
       end
+
+      private
+
+      def load_attachment(path)
+        attachment_dir = "#{File.dirname(__FILE__)}/attachments"
+        attachment_path = "#{attachment_dir}/#{path}"
+
+        unless File.exist?(attachment_path)
+          raise "Attachment at location '#{attachment_path}' doesn't exist. Attachments should be located under #{attachment_dir}"
+        end
+
+        File.open(attachment_path)
+      end
     end
   end
 end

data/lib/connectors/gitlab/connector.rb

@@ -36,7 +36,7 @@ module Connectors
         }
       end
 
-      def initialize(configuration: {})
+      def initialize(configuration: {}, job_description: {})
         super
 
         @extractor = Connectors::GitLab::Extractor.new(

data/lib/connectors/mongodb/connector.rb

@@ -6,13 +6,18 @@
 
 # frozen_string_literal: true
 
-require 'active_support/core_ext/hash/indifferent_access'
 require 'connectors/base/connector'
 require 'mongo'
+require 'utility'
 
 module Connectors
   module MongoDB
     class Connector < Connectors::Base::Connector
+
+      ALLOWED_TOP_LEVEL_FILTER_KEYS = %w[find aggregate]
+
+      PAGE_SIZE = 100
+
       def self.service_type
         'mongodb'
       end
@@ -44,7 +49,7 @@ module Connectors
         }
       end
 
-      def initialize(configuration: {})
+      def initialize(configuration: {}, job_description: {})
         super
 
         @host = configuration.dig(:host, :value)
@@ -56,17 +61,108 @@ module Connectors
       end
 
       def yield_documents
+        check_filtering
+
         with_client do |client|
-          client[@collection].find.each do |document|
-            doc = document.with_indifferent_access
+          # We do paging using skip().limit() here to make Ruby recycle the memory for each page pulled from the server after it's not needed any more.
+          # This gives us more control on the usage of the memory (we can adjust PAGE_SIZE constant for that to decrease max memory consumption).
+          # It's done due to the fact that usage of .find.each leads to memory leaks or overuse of memory - the whole result set seems to stay in memory
+          # during the sync. Sometimes (not 100% sure) it even leads to a real leak, when the memory for these objects is never recycled.
+          cursor, options = create_db_cursor_on_collection(client[@collection])
+          skip = 0
+
+          found_overall = 0
+
+          # if no overall limit is specified by filtering use -1 to not break ingestion, when no overall limit is specified (found_overall is only increased,
+          # thus can never reach -1)
+          overall_limit = Float::INFINITY
+
+          if options.present?
+            # there could be a skip parameter defined for filtering
+            skip = options.fetch(:skip, skip)
+            # there could be a limit parameter defined for filtering -> used for an overall limit (not a page limit, which was introduced for memory optimization)
+            overall_limit = options.fetch(:limit, overall_limit)
+          end
+
+          overall_limit_reached = false
+
+          loop do
+            found_in_page = 0
+
+            view = cursor.skip(skip).limit(PAGE_SIZE)
+            view.each do |document|
+              yield serialize(document)
+
+              found_in_page += 1
+              found_overall += 1
+
+              overall_limit_reached = found_overall >= overall_limit && overall_limit != Float::INFINITY
+
+              break if overall_limit_reached
+            end
+
+            page_was_empty = found_in_page == 0
+
+            break if page_was_empty || overall_limit_reached
 
-            yield serialize(doc)
+            skip += PAGE_SIZE
           end
         end
       end
 
       private
 
+      def create_db_cursor_on_collection(collection)
+        return create_find_cursor(collection) if @advanced_filter_config[:find].present?
+
+        return create_aggregate_cursor(collection) if @advanced_filter_config[:aggregate].present?
+
+        collection.find
+      end
+
+      def check_filtering
+        return unless filtering_present?
+
+        check_find_and_aggregate
+      end
+
+      def check_find_and_aggregate
+        if @advanced_filter_config.keys.size != 1
+          invalid_keys_msg = "Only one of #{ALLOWED_TOP_LEVEL_FILTER_KEYS} is allowed in the filtering object. Keys present: '#{@advanced_filter_config.keys}'."
+          raise Utility::InvalidFilterConfigError.new(invalid_keys_msg)
+        end
+      end
+
+      def create_aggregate_cursor(collection)
+        aggregate = @advanced_filter_config[:aggregate]
+
+        pipeline = aggregate[:pipeline]
+        options = extract_options(aggregate)
+
+        if !pipeline.nil? && pipeline.empty? && !options.present?
+          Utility::Logger.warn('\'Aggregate\' was specified with an empty pipeline and empty options.')
+        end
+
+        [collection.aggregate(pipeline, options), options]
+      end
+
+      def create_find_cursor(collection)
+        find = @advanced_filter_config[:find]
+
+        filter = find[:filter]
+        options = extract_options(find)
+
+        if !filter.nil? && filter.empty? && !options.present?
+          Utility::Logger.warn('\'Find\' was specified with an empty filter and empty options.')
+        end
+
+        [collection.find(filter, options), options]
+      end
+
+      def extract_options(mongodb_function)
+        mongodb_function[:options].present? ? mongodb_function[:options] : {}
+      end
+
       def do_health_check
         with_client do |_client|
           Utility::Logger.debug("Mongo at #{@host}/#{@database} looks healthy.")
@@ -76,34 +172,43 @@ module Connectors
       def with_client
         raise "Invalid value for 'Direct connection' : #{@direct_connection}." unless %w[true false].include?(@direct_connection.to_s.strip.downcase)
 
-        client = if @user.present? || @password.present?
-                   Mongo::Client.new(
-                     @host,
-                     database: @database,
-                     direct_connection: to_boolean(@direct_connection),
-                     user: @user,
-                     password: @password
-                   )
-                 else
-                   Mongo::Client.new(
-                     @host,
-                     database: @database,
-                     direct_connection: to_boolean(@direct_connection)
-                   )
-                 end
-
-        begin
-          Utility::Logger.debug("Existing Databases #{client.database_names}")
-          Utility::Logger.debug('Existing Collections:')
-
-          client.collections.each { |coll| Utility::Logger.debug(coll.name) }
+        args = {
+                 database: @database,
+                 direct_connection: to_boolean(@direct_connection)
+               }
+
+        if @user.present? || @password.present?
+          args[:user] = @user
+          args[:password] = @password
+        end
+
+        Mongo::Client.new(@host, args) do |client|
+          databases = client.database_names
+
+          Utility::Logger.debug("Existing Databases: #{databases}")
+          check_database_exists!(databases, @database)
+
+          collections = client.database.collection_names
+
+          Utility::Logger.debug("Existing Collections: #{collections}")
+          check_collection_exists!(collections, @database, @collection)
 
           yield client
-        ensure
-          client.close
         end
       end
 
+      def check_database_exists!(databases, database)
+        return if databases.include?(database)
+
+        raise "Database (#{database}) does not exist. Existing databases: #{databases.join(', ')}"
+      end
+
+      def check_collection_exists!(collections, database, collection)
+        return if collections.include?(collection)
+
+        raise "Collection (#{collection}) does not exist within database '#{database}'. Existing collections: #{collections.join(', ')}"
+      end
+
       def serialize(mongodb_document)
         # This is some lazy serialization here.
         # Problem: MongoDB has its own format of things - e.g. ids are Bson::ObjectId, which when serialized to JSON
@@ -120,11 +225,10 @@ module Connectors
           mongodb_document.map { |v| serialize(v) }
         when Hash
           mongodb_document.map do |key, value|
-            remapped_key = key.to_sym == :_id ? :id : key.to_sym
-
+            key = 'id' if key == '_id'
             remapped_value = serialize(value)
-            [remapped_key, remapped_value]
-          end.to_h.with_indifferent_access
+            [key, remapped_value]
+          end.to_h
         else
           mongodb_document
         end

data/lib/connectors/registry.rb

@@ -24,10 +24,10 @@ module Connectors
       @connectors[name]
     end
 
-    def connector(name, configuration)
+    def connector(name, configuration, job_description: {})
       klass = connector_class(name)
       if klass.present?
-        return klass.new(configuration: configuration)
+        return klass.new(configuration: configuration, job_description: job_description)
       end
       raise "Connector #{name} is not yet registered. You need to register it before use"
     end

data/lib/connectors/sync_status.rb

@@ -8,14 +8,33 @@
 
 module Connectors
   class SyncStatus
-    COMPLETED = 'completed'
+    PENDING     = 'pending'
     IN_PROGRESS = 'in_progress'
-    FAILED = 'failed'
+    CANCELING   = 'canceling'
+    CANCELED    = 'canceled'
+    SUSPENDED   = 'suspended'
+    COMPLETED   = 'completed'
+    ERROR       = 'error'
 
     STATUSES = [
-      COMPLETED,
+      PENDING,
       IN_PROGRESS,
-      FAILED
+      CANCELING,
+      CANCELED,
+      SUSPENDED,
+      COMPLETED,
+      ERROR
+    ]
+
+    PENDING_STATUES = [
+      PENDING,
+      SUSPENDED
+    ]
+
+    TERMINAL_STATUSES = [
+      CANCELED,
+      COMPLETED,
+      ERROR
     ]
   end
 end

data/lib/core/configuration.rb

@@ -23,7 +23,7 @@ module Core
             Utility::Logger.error("Couldn't find connector for service type #{connector_settings.service_type || service_type}")
             return
           end
-          configuration = connector_class.configurable_fields
+          configuration = connector_class.configurable_fields_indifferent_access
           doc = {
             :configuration => configuration
           }

data/lib/core/connector_settings.rb

@@ -19,6 +19,8 @@ module Core
     DEFAULT_REDUCE_WHITESPACE = true
     DEFAULT_RUN_ML_INFERENCE = true
 
+    DEFAULT_FILTERING = {}
+
     DEFAULT_PAGE_SIZE = 100
 
     # Error Classes
@@ -80,20 +82,24 @@ module Core
       self[:scheduling]
     end
 
+    def filtering
+      Utility::Common.return_if_present(@elasticsearch_response[:filtering], DEFAULT_FILTERING)
+    end
+
     def request_pipeline
-      return_if_present(@elasticsearch_response.dig(:pipeline, :name), @connectors_meta.dig(:pipeline, :default_name), DEFAULT_REQUEST_PIPELINE)
+      Utility::Common.return_if_present(@elasticsearch_response.dig(:pipeline, :name), @connectors_meta.dig(:pipeline, :default_name), DEFAULT_REQUEST_PIPELINE)
     end
 
     def extract_binary_content?
-      return_if_present(@elasticsearch_response.dig(:pipeline, :extract_binary_content), @connectors_meta.dig(:pipeline, :default_extract_binary_content), DEFAULT_EXTRACT_BINARY_CONTENT)
+      Utility::Common.return_if_present(@elasticsearch_response.dig(:pipeline, :extract_binary_content), @connectors_meta.dig(:pipeline, :default_extract_binary_content), DEFAULT_EXTRACT_BINARY_CONTENT)
     end
 
     def reduce_whitespace?
-      return_if_present(@elasticsearch_response.dig(:pipeline, :reduce_whitespace), @connectors_meta.dig(:pipeline, :default_reduce_whitespace), DEFAULT_REDUCE_WHITESPACE)
+      Utility::Common.return_if_present(@elasticsearch_response.dig(:pipeline, :reduce_whitespace), @connectors_meta.dig(:pipeline, :default_reduce_whitespace), DEFAULT_REDUCE_WHITESPACE)
     end
 
     def run_ml_inference?
-      return_if_present(@elasticsearch_response.dig(:pipeline, :run_ml_inference), @connectors_meta.dig(:pipeline, :default_run_ml_inference), DEFAULT_RUN_ML_INFERENCE)
+      Utility::Common.return_if_present(@elasticsearch_response.dig(:pipeline, :run_ml_inference), @connectors_meta.dig(:pipeline, :default_run_ml_inference), DEFAULT_RUN_ML_INFERENCE)
     end
 
     def formatted
@@ -110,8 +116,6 @@ module Core
       index_name&.start_with?(Utility::Constants::CONTENT_INDEX_PREFIX)
     end
 
-    private
-
     def self.fetch_connectors_by_query(query, page_size)
       connectors_meta = ElasticConnectorActions.connectors_meta
 
@@ -120,8 +124,8 @@ module Core
       loop do
         response = ElasticConnectorActions.search_connectors(query, page_size, offset)
 
-        hits = response['hits']['hits']
-        total = response['hits']['total']['value']
+        hits = response.dig('hits', 'hits') || []
+        total = response.dig('hits', 'total', 'value') || 0
         results += hits.map do |hit|
           Core::ConnectorSettings.new(hit, connectors_meta)
         end
@@ -132,11 +136,5 @@ module Core
       results
     end
 
-    def return_if_present(*args)
-      args.each do |arg|
-        return arg unless arg.nil?
-      end
-      nil
-    end
   end
 end

data/lib/core/elastic_connector_actions.rb

@@ -10,8 +10,21 @@ require 'active_support/core_ext/hash'
 require 'connectors/connector_status'
 require 'connectors/sync_status'
 require 'utility'
+require 'elastic-transport'
 
 module Core
+  class JobAlreadyRunningError < StandardError
+    def initialize(connector_id)
+      super("Sync job for connector '#{connector_id}' is already running.")
+    end
+  end
+
+  class ConnectorVersionChangedError < StandardError
+    def initialize(connector_id, seq_no, primary_term)
+      super("Version conflict: seq_no [#{seq_no}] and primary_term [#{primary_term}] do not match for connector '#{connector_id}'.")
+    end
+  end
+
   class ElasticConnectorActions
     class << self
 
@@ -72,20 +85,53 @@ module Core
       end
 
       def claim_job(connector_id)
-        update_connector_fields(connector_id,
-                                :sync_now => false,
-                                :last_sync_status => Connectors::SyncStatus::IN_PROGRESS,
-                                :last_synced => Time.now)
+        seq_no = nil
+        primary_term = nil
+        sync_in_progress = false
+        connector_record = client.get(
+          :index => Utility::Constants::CONNECTORS_INDEX,
+          :id => connector_id,
+          :ignore => 404,
+          :refresh => true
+        ).tap do |response|
+          seq_no = response['_seq_no']
+          primary_term = response['_primary_term']
+          sync_in_progress = response.dig('_source', 'last_sync_status') == Connectors::SyncStatus::IN_PROGRESS
+        end
+        if sync_in_progress
+          raise JobAlreadyRunningError.new(connector_id)
+        end
+        update_connector_fields(
+          connector_id,
+          { :sync_now => false,
+            :last_sync_status => Connectors::SyncStatus::IN_PROGRESS,
+            :last_synced => Time.now },
+          seq_no,
+          primary_term
+        )
 
         body = {
           :connector_id => connector_id,
           :status => Connectors::SyncStatus::IN_PROGRESS,
           :worker_hostname => Socket.gethostname,
-          :created_at => Time.now
+          :created_at => Time.now,
+          :filtering => convert_connector_filtering_to_job_filtering(connector_record.dig('_source', 'filtering'))
         }
-        job = client.index(:index => Utility::Constants::JOB_INDEX, :body => body)
 
-        job['_id']
+        client.index(:index => Utility::Constants::JOB_INDEX, :body => body)
+      end
+
+      def convert_connector_filtering_to_job_filtering(connector_filtering)
+        return [] unless connector_filtering
+        connector_filtering = [connector_filtering] unless connector_filtering.is_a?(Array)
+        connector_filtering.each_with_object([]) do |filtering_domain, job_filtering|
+          job_filtering << {
+            'domain' => filtering_domain['domain'],
+            'rules' => filtering_domain.dig('active', 'rules'),
+            'advanced_snippet' => filtering_domain.dig('active', 'advanced_snippet'),
+            'warnings' => [] # TODO: in https://github.com/elastic/enterprise-search-team/issues/3174
+          }
+        end
       end
 
       def update_connector_status(connector_id, status, error_message = nil)
@@ -100,7 +146,7 @@ module Core
       end
 
       def complete_sync(connector_id, job_id, status)
-        sync_status = status[:error] ? Connectors::SyncStatus::FAILED : Connectors::SyncStatus::COMPLETED
+        sync_status = status[:error] ? Connectors::SyncStatus::ERROR : Connectors::SyncStatus::COMPLETED
 
         update_connector_fields(connector_id,
                                 :last_sync_status => sync_status,
@@ -136,7 +182,7 @@ module Core
           }
           loop do
             response = client.search(:body => body)
-            hits = response['hits']['hits']
+            hits = response.dig('hits', 'hits') || []
 
             ids = hits.map { |h| h['_id'] }
             result += ids
@@ -242,15 +288,29 @@ module Core
         ensure_index_exists("#{Utility::Constants::JOB_INDEX}-v1", system_index_body(:alias_name => Utility::Constants::JOB_INDEX, :mappings => mappings))
       end
 
-      def update_connector_fields(connector_id, doc = {})
+      def update_connector_fields(connector_id, doc = {}, seq_no = nil, primary_term = nil)
         return if doc.empty?
-        client.update(
+        update_args = {
           :index => Utility::Constants::CONNECTORS_INDEX,
           :id => connector_id,
           :body => { :doc => doc },
           :refresh => true,
           :retry_on_conflict => 3
-        )
+        }
+        # seq_no and primary_term are used for optimistic concurrency control
+        # see https://www.elastic.co/guide/en/elasticsearch/reference/current/optimistic-concurrency-control.html
+        if seq_no && primary_term
+          update_args[:if_seq_no] = seq_no
+          update_args[:if_primary_term] = primary_term
+          update_args.delete(:retry_on_conflict)
+        end
+        begin
+          client.update(update_args)
+        rescue Elastic::Transport::Transport::Errors::Conflict
+          # VersionConflictException
+          # see https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-index_.html#optimistic-concurrency-control-index
+          raise ConnectorVersionChangedError.new(connector_id, seq_no, primary_term)
+        end
       end
 
       private

data/lib/core/native_scheduler.rb

@@ -16,6 +16,9 @@ module Core
   class NativeScheduler < Core::Scheduler
     def connector_settings
       Core::ConnectorSettings.fetch_native_connectors || []
+    rescue *Utility::AUTHORIZATION_ERRORS => e
+      # should be handled by the general scheduler
+      raise e
     rescue StandardError => e
       Utility::ExceptionTracking.log_exception(e, 'Could not retrieve native connectors due to unexpected error.')
       []

data/lib/core/scheduler.rb

@@ -9,6 +9,7 @@
 require 'time'
 require 'fugit'
 require 'core/connector_settings'
+require 'core/elastic_connector_actions'
 require 'utility/cron'
 require 'utility/logger'
 require 'utility/exception_tracking'
@@ -41,6 +42,8 @@ module Core
         if @is_shutting_down
           break
         end
+      rescue *Utility::AUTHORIZATION_ERRORS => e
+        Utility::ExceptionTracking.log_exception(e, 'Could not retrieve connectors settings due to authorization error.')
       rescue StandardError => e
         Utility::ExceptionTracking.log_exception(e, 'Sync failed due to unexpected error.')
       ensure

data/lib/core/single_scheduler.rb

@@ -21,6 +21,9 @@ module Core
     def connector_settings
       connector_settings = Core::ConnectorSettings.fetch_by_id(@connector_id)
       [connector_settings]
+    rescue *Utility::AUTHORIZATION_ERRORS => e
+      # should be handled by the general scheduler
+      raise e
     rescue StandardError => e
       Utility::ExceptionTracking.log_exception(e, "Could not retrieve the connector by id #{@connector_id} due to unexpected error.")
       []

data/lib/core/sync_job_runner.rb

@@ -23,7 +23,7 @@ module Core
       @connector_settings = connector_settings
       @sink = Core::OutputSink::EsSink.new(connector_settings.index_name, @connector_settings.request_pipeline)
       @connector_class = Connectors::REGISTRY.connector_class(connector_settings.service_type)
-      @connector_instance = Connectors::REGISTRY.connector(connector_settings.service_type, connector_settings.configuration)
+      @sync_finished = false
       @status = {
         :indexed_document_count => 0,
         :deleted_document_count => 0,
@@ -39,9 +39,10 @@ module Core
     private
 
     def do_sync!
-      Utility::Logger.info("Starting sync for connector #{@connector_settings.id}.")
+      Utility::Logger.info("Claiming a sync job for connector #{@connector_settings.id}.")
 
-      job_id = ElasticConnectorActions.claim_job(@connector_settings.id)
+      job_description = ElasticConnectorActions.claim_job(@connector_settings.id)
+      job_id = job_description['_id']
 
       unless job_id.present?
         Utility::Logger.error("Failed to claim the job for #{@connector_settings.id}. Please check the logs for the cause of this error.")
@@ -51,17 +52,19 @@ module Core
       begin
         Utility::Logger.debug("Successfully claimed job for connector #{@connector_settings.id}.")
 
-        @connector_instance.do_health_check!
+        connector_instance = Connectors::REGISTRY.connector(@connector_settings.service_type, @connector_settings.configuration, job_description: job_description)
+
+        connector_instance.do_health_check!
 
         incoming_ids = []
         existing_ids = ElasticConnectorActions.fetch_document_ids(@connector_settings.index_name)
 
         Utility::Logger.debug("#{existing_ids.size} documents are present in index #{@connector_settings.index_name}.")
 
-        @connector_instance.yield_documents do |document|
+        connector_instance.yield_documents do |document|
           document = add_ingest_metadata(document)
           @sink.ingest(document)
-          incoming_ids << document[:id]
+          incoming_ids << document['id']
           @status[:indexed_document_count] += 1
         end
 
@@ -75,6 +78,10 @@ module Core
         end
 
         @sink.flush
+
+        # We use this mechanism for checking, whether an interrupt (or something else lead to the thread not finishing)
+        # occurred as most of the time the main execution thread is interrupted and we miss this Signal/Exception here
+        @sync_finished = true
       rescue StandardError => e
         @status[:error] = e.message
         Utility::ExceptionTracking.log_exception(e)
@@ -83,10 +90,15 @@ module Core
         Utility::Logger.info("Upserted #{@status[:indexed_document_count]} documents into #{@connector_settings.index_name}.")
         Utility::Logger.info("Deleted #{@status[:deleted_document_count]} documents into #{@connector_settings.index_name}.")
 
+        # Make sure to not override a previous error message
+        if !@sync_finished && @status[:error].nil?
+          @status[:error] = 'Sync thread didn\'t finish execution. Check connector logs for more details.'
+        end
+
         ElasticConnectorActions.complete_sync(@connector_settings.id, job_id, @status.dup)
 
         if @status[:error]
-          Utility::Logger.info("Failed to sync for connector #{@connector_settings.id} with error #{@status[:error]}.")
+          Utility::Logger.info("Failed to sync for connector #{@connector_settings.id} with error '#{@status[:error]}'.")
         else
           Utility::Logger.info("Successfully synced for connector #{@connector_settings.id}.")
         end

data/lib/{connectors_app/// → utility/common.rb}

@@ -4,10 +4,17 @@
 # you may not use this file except in compliance with the Elastic License.
 #
 
-module ConnectorsApp
-  module Errors
-    INVALID_API_KEY = 'INVALID_API_KEY'
-    UNSUPPORTED_AUTH_SCHEME = 'UNSUPPORTED_AUTH_SCHEME'
-    INTERNAL_SERVER_ERROR = 'INTERNAL_SERVER_ERROR'
+# frozen_string_literal: true
+
+module Utility
+  class Common
+    class << self
+      def return_if_present(*args)
+        args.each do |arg|
+          return arg unless arg.nil?
+        end
+        nil
+      end
+    end
   end
 end

data/lib/utility/errors.rb

@@ -5,6 +5,7 @@
 #
 
 require 'active_support/core_ext/string'
+require 'elasticsearch'
 
 module Utility
   class DocumentError
@@ -31,6 +32,8 @@ module Utility
   end
 
   class ClientError < StandardError; end
+
+  class InvalidFilterConfigError < StandardError; end
   class EvictionWithNoProgressError < StandardError; end
   class EvictionError < StandardError
     attr_accessor :cursors
@@ -89,6 +92,7 @@ module Utility
   class InvalidTokenError < StandardError; end
   class TokenRefreshFailedError < StandardError; end
   class ConnectorNotAvailableError < StandardError; end
+  class AuthorizationError < StandardError; end
 
   # For when we want to explicitly set a #cause but can't
   class ExplicitlyCausedError < StandardError
@@ -124,6 +128,7 @@ module Utility
     end
   end
 
+  AUTHORIZATION_ERRORS = [Elastic::Transport::Transport::Errors::Unauthorized]
   INTERNAL_SERVER_ERROR = Utility::Error.new(500, 'INTERNAL_SERVER_ERROR', 'Internal server error')
   INVALID_API_KEY = Utility::Error.new(401, 'INVALID_API_KEY', 'Invalid API key')
   UNSUPPORTED_AUTH_SCHEME = Utility::Error.new(401, 'UNSUPPORTED_AUTH_SCHEME', 'Unsupported authorization scheme')

data/lib/utility/es_client.rb

@@ -20,8 +20,8 @@ module Utility
       attr_reader :cause
     end
 
-    def initialize(es_config)
-      super(connection_configs(es_config))
+    def initialize(es_config, &block)
+      super(connection_configs(es_config), &block)
     end
 
     def connection_configs(es_config)
@@ -39,6 +39,10 @@ module Utility
       configs[:log] = es_config[:log] || false
       configs[:trace] = es_config[:trace] || false
 
+      # transport options
+      configs[:transport_options] = es_config[:transport_options] if es_config[:transport_options]
+      configs[:ca_fingerprint] = es_config[:ca_fingerprint] if es_config[:ca_fingerprint]
+
       # if log or trace is activated, we use the application logger
       configs[:logger] = if configs[:log] || configs[:trace]
                            Utility::Logger.logger

data/lib/utility.rb

@@ -6,6 +6,7 @@
 
 require 'utility/constants'
 require 'utility/cron'
+require 'utility/common'
 require 'utility/errors'
 require 'utility/es_client'
 require 'utility/environment'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: connectors_service
 version: !ruby/object:Gem::Version
-  version: 8.6.0.3
+  version: 8.6.0.4.pre.20221104T200814Z
 platform: ruby
 authors:
 - Elastic
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-10-03 00:00:00.000000000 Z
+date: 2022-11-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -198,14 +198,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 8.4.0
+        version: 8.5.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 8.4.0
+        version: 8.5.0
 - !ruby/object:Gem::Dependency
   name: faraday
   requirement: !ruby/object:Gem::Requirement
@@ -400,10 +400,10 @@ files:
 - lib/connectors/base/custom_client.rb
 - lib/connectors/connector_status.rb
 - lib/connectors/crawler/scheduler.rb
+- lib/connectors/example/attachments/first_attachment.txt
+- lib/connectors/example/attachments/second_attachment.txt
+- lib/connectors/example/attachments/third_attachment.txt
 - lib/connectors/example/connector.rb
-- lib/connectors/example/example_attachments/first_attachment.txt
-- lib/connectors/example/example_attachments/second_attachment.txt
-- lib/connectors/example/example_attachments/third_attachment.txt
 - lib/connectors/gitlab/adapter.rb
 - lib/connectors/gitlab/connector.rb
 - lib/connectors/gitlab/custom_client.rb
@@ -411,7 +411,6 @@ files:
 - lib/connectors/mongodb/connector.rb
 - lib/connectors/registry.rb
 - lib/connectors/sync_status.rb
-- lib/connectors_app/\
 - lib/connectors_service.rb
 - lib/connectors_utility.rb
 - lib/core.rb
@@ -433,6 +432,7 @@ files:
 - lib/stubs/connectors/stats.rb
 - lib/stubs/service_type.rb
 - lib/utility.rb
+- lib/utility/common.rb
 - lib/utility/constants.rb
 - lib/utility/cron.rb
 - lib/utility/elasticsearch/index/language_data.yml
@@ -451,7 +451,7 @@ homepage: https://github.com/elastic/connectors-ruby
 licenses:
 - Elastic-2.0
 metadata: {}
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -462,12 +462,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubygems_version: 3.0.3.1
-signing_key:
+signing_key: 
 specification_version: 4
 summary: Gem containing Elastic connectors service
 test_files: []