connectors_utility 8.6.0.4.pre.20221107T145613Z → 8.6.0.4.pre.20221114T235050Z

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c3dc548bffdaf2be6ac65e4c9231196f2b542d3cff6a092e3756272a62312d4b
-  data.tar.gz: fe6f7d03c9f2d1c56faeb501b7823c38abef582ce2edaade3a26d8c28dfad7ac
+  metadata.gz: b3b178cb45ca62a666074e0370d28f4b82477d2ee32e6e7a74b24c218f02c121
+  data.tar.gz: 0b99bd3126a5fdc7cad2ee2b9dc5862284992911043fc7c1931aaac9ea3844d8
 SHA512:
-  metadata.gz: 87400cec0be6b368834cd0171dd8ca38df5b5b7928b0e10da7c24a29a89a06a89af22c5a4317f8a683135cabfd2871412b26b9da61924ad2356612a4219be7c1
-  data.tar.gz: 249ed7dad12806e43c50b5c409085fcdb902f3e1e386bdfdf7db5d551dc0377d5a808cb8f4f49f9f01edebb548e7baebe1b8e9c249456a9efacb0496bf04d9c5
+  metadata.gz: 31b38cf34d989cb09c2ab61356f330a22224fe58f76b5a02db12da13ed3d3e6329244cc75b9eeb293312510df9ccb1e0ff615ec65c9dc0e832e8dbe2c5c25328
+  data.tar.gz: e9b2b753f2ac8135372303f750cb6630d364d0af802a7b5d980cb5ea470e57a3d23c949d2d60356e585da7c9d15383528982ecb80d1eaca2bb36f12854633950

data/lib/core/connector_settings.rb

@@ -8,6 +8,7 @@
 
 require 'active_support/core_ext/hash/indifferent_access'
 require 'connectors/connector_status'
+require 'connectors/registry'
 require 'core/elastic_connector_actions'
 require 'utility'
 
@@ -34,13 +35,15 @@ module Core
       new(es_response, connectors_meta)
     end
 
-    def initialize(es_response, connectors_meta)
-      @elasticsearch_response = es_response.with_indifferent_access
-      @connectors_meta = connectors_meta.with_indifferent_access
-    end
-
     def self.fetch_native_connectors(page_size = DEFAULT_PAGE_SIZE)
-      query = { term: { is_native: true } }
+      query = {
+        bool: {
+          filter: [
+            { term: { is_native: true } },
+            { terms: { service_type: Connectors::REGISTRY.registered_connectors } }
+          ]
+        }
+      }
       fetch_connectors_by_query(query, page_size)
     end
 
@@ -83,23 +86,26 @@ module Core
     end
 
     def filtering
-      Utility::Common.return_if_present(@elasticsearch_response[:filtering], DEFAULT_FILTERING)
+      # assume for now, that first object in filtering array or a filter object itself is the only filtering object
+      filtering = @elasticsearch_response.dig(:_source, :filtering)
+
+      Utility::Filtering.extract_filter(filtering)
     end
 
     def request_pipeline
-      Utility::Common.return_if_present(@elasticsearch_response.dig(:pipeline, :name), @connectors_meta.dig(:pipeline, :default_name), DEFAULT_REQUEST_PIPELINE)
+      Utility::Common.return_if_present(@elasticsearch_response.dig(:_source, :pipeline, :name), @connectors_meta.dig(:pipeline, :default_name), DEFAULT_REQUEST_PIPELINE)
     end
 
     def extract_binary_content?
-      Utility::Common.return_if_present(@elasticsearch_response.dig(:pipeline, :extract_binary_content), @connectors_meta.dig(:pipeline, :default_extract_binary_content), DEFAULT_EXTRACT_BINARY_CONTENT)
+      Utility::Common.return_if_present(@elasticsearch_response.dig(:_source, :pipeline, :extract_binary_content), @connectors_meta.dig(:pipeline, :default_extract_binary_content), DEFAULT_EXTRACT_BINARY_CONTENT)
     end
 
     def reduce_whitespace?
-      Utility::Common.return_if_present(@elasticsearch_response.dig(:pipeline, :reduce_whitespace), @connectors_meta.dig(:pipeline, :default_reduce_whitespace), DEFAULT_REDUCE_WHITESPACE)
+      Utility::Common.return_if_present(@elasticsearch_response.dig(:_source, :pipeline, :reduce_whitespace), @connectors_meta.dig(:pipeline, :default_reduce_whitespace), DEFAULT_REDUCE_WHITESPACE)
     end
 
     def run_ml_inference?
-      Utility::Common.return_if_present(@elasticsearch_response.dig(:pipeline, :run_ml_inference), @connectors_meta.dig(:pipeline, :default_run_ml_inference), DEFAULT_RUN_ML_INFERENCE)
+      Utility::Common.return_if_present(@elasticsearch_response.dig(:_source, :pipeline, :run_ml_inference), @connectors_meta.dig(:pipeline, :default_run_ml_inference), DEFAULT_RUN_ML_INFERENCE)
     end
 
     def formatted
@@ -116,6 +122,13 @@ module Core
       index_name&.start_with?(Utility::Constants::CONTENT_INDEX_PREFIX)
     end
 
+    private
+
+    def initialize(es_response, connectors_meta)
+      @elasticsearch_response = es_response.with_indifferent_access
+      @connectors_meta = connectors_meta.with_indifferent_access
+    end
+
     def self.fetch_connectors_by_query(query, page_size)
       connectors_meta = ElasticConnectorActions.connectors_meta
 

data/lib/core/elastic_connector_actions.rb

@@ -19,6 +19,12 @@ module Core
     end
   end
 
+  class JobNotCreatedError < StandardError
+    def initialize(connector_id, response)
+      super("Sync job for connector '#{connector_id}' could not be created. Response: #{response}")
+    end
+  end
+
   class ConnectorVersionChangedError < StandardError
     def initialize(connector_id, seq_no, primary_term)
       super("Version conflict: seq_no [#{seq_no}] and primary_term [#{primary_term}] do not match for connector '#{connector_id}'.")
@@ -43,10 +49,17 @@ module Core
       end
 
       def get_connector(connector_id)
+        # TODO: remove the usage of with_indifferent_access. Ideally this should return a hash or nil if not found
         client.get(:index => Utility::Constants::CONNECTORS_INDEX, :id => connector_id, :ignore => 404).with_indifferent_access
       end
 
+      def get_job(job_id)
+        # TODO: remove the usage of with_indifferent_access. Ideally this should return a hash or nil if not found
+        client.get(:index => Utility::Constants::JOB_INDEX, :id => job_id, :ignore => 404).with_indifferent_access
+      end
+
       def connectors_meta
+        # TODO: remove the usage of with_indifferent_access. Ideally this should return a hash or nil if not found
         alias_mappings = client.indices.get_mapping(:index => Utility::Constants::CONNECTORS_INDEX).with_indifferent_access
         index = get_latest_index_in_alias(Utility::Constants::CONNECTORS_INDEX, alias_mappings.keys)
         alias_mappings.dig(index, 'mappings', '_meta') || {}
@@ -65,6 +78,19 @@ module Core
         )
       end
 
+      def search_jobs(query, page_size, offset)
+        client.search(
+          :index => Utility::Constants::JOB_INDEX,
+          :ignore => 404,
+          :body => {
+              :size => page_size,
+              :from => offset,
+              :query => query,
+              :sort => ['created_at']
+          }
+        )
+      end
+
       def update_connector_configuration(connector_id, configuration)
         update_connector_fields(connector_id, :configuration => configuration)
       end
@@ -84,6 +110,28 @@ module Core
         update_connector_configuration(connector_id, payload)
       end
 
+      def update_filtering_validation(connector_id, filter_validation_results)
+        return if filter_validation_results.empty?
+
+        filtering = get_connector(connector_id).dig(:_source, :filtering)
+
+        case filtering
+        when Hash
+          update_filter_validation(filtering, filter_validation_results)
+        when Array
+          return unless should_update_validations?(filter_validation_results, filtering)
+
+          filtering.each do |filter|
+            update_filter_validation(filter, filter_validation_results)
+          end
+        else
+          Utility::Logger.warn("Elasticsearch returned invalid filtering format: #{filtering}. Skipping validation.")
+          return
+        end
+
+        update_connector_fields(connector_id, { :filtering => filtering })
+      end
+
       def claim_job(connector_id)
         seq_no = nil
         primary_term = nil
@@ -111,24 +159,38 @@ module Core
         )
 
         body = {
-          :connector_id => connector_id,
           :status => Connectors::SyncStatus::IN_PROGRESS,
           :worker_hostname => Socket.gethostname,
           :created_at => Time.now,
-          :filtering => convert_connector_filtering_to_job_filtering(connector_record.dig('_source', 'filtering'))
+          :started_at => Time.now,
+          :last_seen => Time.now,
+          :connector => {
+            :id => connector_id,
+            :filtering => convert_connector_filtering_to_job_filtering(connector_record.dig('_source', 'filtering'))
+          }
         }
 
-        client.index(:index => Utility::Constants::JOB_INDEX, :body => body)
+        index_response = client.index(:index => Utility::Constants::JOB_INDEX, :body => body, :refresh => true)
+        if index_response['result'] == 'created'
+          # TODO: remove the usage of with_indifferent_access. Ideally this should return a hash or nil if not found
+          return client.get(
+            :index => Utility::Constants::JOB_INDEX,
+            :id => index_response['_id'],
+            :ignore => 404
+          ).with_indifferent_access
+        end
+        raise JobNotCreatedError.new(connector_id, index_response)
       end
 
       def convert_connector_filtering_to_job_filtering(connector_filtering)
         return [] unless connector_filtering
         connector_filtering = [connector_filtering] unless connector_filtering.is_a?(Array)
         connector_filtering.each_with_object([]) do |filtering_domain, job_filtering|
+          snippet = filtering_domain.dig('active', 'advanced_snippet') || {}
           job_filtering << {
             'domain' => filtering_domain['domain'],
             'rules' => filtering_domain.dig('active', 'rules'),
-            'advanced_snippet' => filtering_domain.dig('active', 'advanced_snippet'),
+            'advanced_snippet' => snippet['value'] || snippet,
             'warnings' => [] # TODO: in https://github.com/elastic/enterprise-search-team/issues/3174
           }
         end
@@ -145,22 +207,33 @@ module Core
         update_connector_fields(connector_id, body)
       end
 
-      def complete_sync(connector_id, job_id, status)
-        sync_status = status[:error] ? Connectors::SyncStatus::ERROR : Connectors::SyncStatus::COMPLETED
+      def update_sync(job_id, metadata)
+        body = {
+          :doc => { :last_seen => Time.now }.merge(metadata)
+        }
+        client.update(:index => Utility::Constants::JOB_INDEX, :id => job_id, :body => body)
+      end
+
+      def complete_sync(connector_id, job_id, metadata, error)
+        sync_status = error ? Connectors::SyncStatus::ERROR : Connectors::SyncStatus::COMPLETED
+
+        metadata ||= {}
 
         update_connector_fields(connector_id,
                                 :last_sync_status => sync_status,
-                                :last_sync_error => status[:error],
-                                :error => status[:error],
+                                :last_sync_error => error,
+                                :error => error,
                                 :last_synced => Time.now,
-                                :last_indexed_document_count => status[:indexed_document_count],
-                                :last_deleted_document_count => status[:deleted_document_count])
+                                :last_indexed_document_count => metadata[:indexed_document_count],
+                                :last_deleted_document_count => metadata[:deleted_document_count])
 
         body = {
           :doc => {
             :status => sync_status,
-            :completed_at => Time.now
-          }.merge(status)
+            :completed_at => Time.now,
+            :last_seen => Time.now,
+            :error => error
+          }.merge(metadata)
         }
         client.update(:index => Utility::Constants::JOB_INDEX, :id => job_id, :body => body)
       end
@@ -248,12 +321,105 @@ module Core
           :properties => {
             :api_key_id => { :type => :keyword },
             :configuration => { :type => :object },
-            :error => { :type => :text },
+            :description => { :type => :text },
+            :error => { :type => :keyword },
+            :features => {
+              :properties => {
+                :filtering_advanced_config => { :type => :boolean },
+                :filtering_rules => { :type => :boolean }
+              }
+            },
+            :filtering => {
+              :properties => {
+                :domain => { :type => :keyword },
+                :active => {
+                  :properties => {
+                    :rules => {
+                      :properties => {
+                        :id => { :type => :keyword },
+                        :policy => { :type => :keyword },
+                        :field => { :type => :keyword },
+                        :rule => { :type => :keyword },
+                        :value => { :type => :keyword },
+                        :order => { :type => :short },
+                        :created_at => { :type => :date },
+                        :updated_at => { :type => :date }
+                      }
+                    },
+                    :advanced_snippet => {
+                      :properties => {
+                        :value => { :type => :object },
+                        :created_at => { :type => :date },
+                        :updated_at => { :type => :date }
+                       }
+                    },
+                    :validation => {
+                      :properties => {
+                        :state => { :type => :keyword },
+                        :errors => {
+                          :properties => {
+                            :ids => { :type => :keyword },
+                            :messages => { :type => :text }
+                          }
+                        }
+                      }
+                    }
+                  }
+                },
+                :draft => {
+                  :properties => {
+                    :rules => {
+                      :properties => {
+                        :id => { :type => :keyword },
+                        :policy => { :type => :keyword },
+                        :field => { :type => :keyword },
+                        :rule => { :type => :keyword },
+                        :value => { :type => :keyword },
+                        :order => { :type => :short },
+                        :created_at => { :type => :date },
+                        :updated_at => { :type => :date }
+                      }
+                    },
+                    :advanced_snippet => {
+                      :properties => {
+                        :value => { :type => :object },
+                        :created_at => { :type => :date },
+                        :updated_at => { :type => :date }
+                      }
+                    },
+                    :validation => {
+                      :properties => {
+                        :state => { :type => :keyword },
+                        :errors => {
+                          :properties => {
+                            :ids => { :type => :keyword },
+                            :messages => { :type => :text }
+                          }
+                        }
+                      }
+                    }
+                  }
+                }
+              }
+            },
             :index_name => { :type => :keyword },
+            :is_native => { :type => :boolean },
+            :language => { :type => :keyword },
             :last_seen => { :type => :date },
+            :last_sync_error => { :type => :keyword },
+            :last_sync_status => { :type => :keyword },
             :last_synced => { :type => :date },
-            :last_indexed_document_count => { :type => :integer },
-            :last_deleted_document_count => { :type => :integer },
+            :last_deleted_document_count => { :type => :long },
+            :last_indexed_document_count => { :type => :long },
+            :name => { :type => :keyword },
+            :pipeline => {
+              :properties => {
+                :extract_binary_content => { :type => :boolean },
+                :name => { :type => :keyword },
+                :reduce_whitespace => { :type => :boolean },
+                :run_ml_inference => { :type => :boolean }
+              }
+            },
             :scheduling => {
               :properties => {
                 :enabled => { :type => :boolean },
@@ -262,9 +428,7 @@ module Core
             },
             :service_type => { :type => :keyword },
             :status => { :type => :keyword },
-            :sync_error => { :type => :text },
-            :sync_now => { :type => :boolean },
-            :sync_status => { :type => :keyword }
+            :sync_now => { :type => :boolean }
           }
         }
         ensure_index_exists("#{Utility::Constants::CONNECTORS_INDEX}-v1", system_index_body(:alias_name => Utility::Constants::CONNECTORS_INDEX, :mappings => mappings))
@@ -275,14 +439,68 @@ module Core
       def ensure_job_index_exists
         mappings = {
           :properties => {
-            :connector_id => { :type => :keyword },
-            :status => { :type => :keyword },
+            :cancelation_requested_at => { :type => :date },
+            :canceled_at => { :type => :date },
+            :completed_at => { :type => :date },
+            :connector => {
+              :properties => {
+                :configuration => { :type => :object },
+                :filtering => {
+                  :properties => {
+                    :domain => { :type => :keyword },
+                    :rules => {
+                      :properties => {
+                        :id => { :type => :keyword },
+                        :policy => { :type => :keyword },
+                        :field => { :type => :keyword },
+                        :rule => { :type => :keyword },
+                        :value => { :type => :keyword },
+                        :order => { :type => :short },
+                        :created_at => { :type => :date },
+                        :updated_at => { :type => :date }
+                      }
+                    },
+                    :advanced_snippet => {
+                      :properties => {
+                        :value => { :type => :object },
+                        :created_at => { :type => :date },
+                        :updated_at => { :type => :date }
+                      }
+                    },
+                    :warnings => {
+                      :properties => {
+                        :ids => { :type => :keyword },
+                        :messages => { :type => :text }
+                      }
+                    }
+                  }
+                },
+                :id => { :type => :keyword },
+                :index_name => { :type => :keyword },
+                :language => { :type => :keyword },
+                :pipeline => {
+                  :properties => {
+                    :extract_binary_content => { :type => :boolean },
+                    :name => { :type => :keyword },
+                    :reduce_whitespace => { :type => :boolean },
+                    :run_ml_inference => { :type => :boolean }
+                  }
+                },
+                :service_type => { :type => :keyword }
+              }
+            },
+            :created_at => { :type => :date },
+            :deleted_document_count => { :type => :integer },
             :error => { :type => :text },
-            :worker_hostname => { :type => :keyword },
             :indexed_document_count => { :type => :integer },
-            :deleted_document_count => { :type => :integer },
-            :created_at => { :type => :date },
-            :completed_at => { :type => :date }
+            :indexed_document_volume => { :type => :integer },
+            :last_seen => { :type => :date },
+            :metadata => { :type => :object },
+            :started_at => { :type => :date },
+            :status => { :type => :keyword },
+            :total_document_count => { :type => :integer },
+            :trigger_method => { :type => :keyword },
+            :worker_hostname => { :type => :keyword }
           }
         }
         ensure_index_exists("#{Utility::Constants::JOB_INDEX}-v1", system_index_body(:alias_name => Utility::Constants::JOB_INDEX, :mappings => mappings))
@@ -313,8 +531,20 @@ module Core
         end
       end
 
+      def document_count(index_name)
+        client.count(:index => index_name)['count']
+      end
+
       private
 
+      def should_update_validations?(domain_validations, filtering)
+        domains_present = filtering.collect { |filter| filter[:domain] }
+        domains_to_update = domain_validations.keys
+
+        # non-empty intersection -> domains to update present
+        !(domains_present & domains_to_update).empty?
+      end
+
       def client
         @client ||= Utility::EsClient.new(App::Config[:elasticsearch])
       end
@@ -324,6 +554,15 @@ module Core
         index_version = index_versions.max # gets the largest suffix number
         "#{alias_name}-v#{index_version}"
       end
+
+      def update_filter_validation(filter, domain_validations)
+        domain = filter[:domain]
+
+        if domain_validations.key?(domain)
+          new_validation_state = { :draft => { :validation => domain_validations[domain] } }
+          filter.deep_merge!(new_validation_state)
+        end
+      end
     end
   end
 end

data/lib/core/scheduler.rb

@@ -10,6 +10,7 @@ require 'time'
 require 'fugit'
 require 'core/connector_settings'
 require 'core/elastic_connector_actions'
+require 'core/filtering/validation_status'
 require 'utility/cron'
 require 'utility/logger'
 require 'utility/exception_tracking'
@@ -38,15 +39,18 @@ module Core
           if configuration_triggered?(cs)
             yield cs, :configuration
           end
-        end
-        if @is_shutting_down
-          break
+          if filtering_validation_triggered?(cs)
+            yield cs, :filter_validation
+          end
         end
       rescue *Utility::AUTHORIZATION_ERRORS => e
         Utility::ExceptionTracking.log_exception(e, 'Could not retrieve connectors settings due to authorization error.')
       rescue StandardError => e
         Utility::ExceptionTracking.log_exception(e, 'Sync failed due to unexpected error.')
       ensure
+        if @is_shutting_down
+          break
+        end
         if @poll_interval > 0 && !@is_shutting_down
           Utility::Logger.debug("Sleeping for #{@poll_interval} seconds in #{self.class}.")
           sleep(@poll_interval)
@@ -62,8 +66,6 @@ module Core
     private
 
     def sync_triggered?(connector_settings)
-      return false unless connector_registered?(connector_settings.service_type)
-
       unless connector_settings.valid_index_name?
         Utility::Logger.warn("The index name of #{connector_settings.formatted} is invalid.")
         return false
@@ -129,8 +131,6 @@ module Core
     end
 
     def heartbeat_triggered?(connector_settings)
-      return false unless connector_registered?(connector_settings.service_type)
-
       last_seen = connector_settings[:last_seen]
       return true if last_seen.nil? || last_seen.empty?
       last_seen = begin
@@ -144,11 +144,41 @@ module Core
     end
 
     def configuration_triggered?(connector_settings)
-      if connector_settings.needs_service_type? || connector_registered?(connector_settings.service_type)
-        return connector_settings.connector_status == Connectors::ConnectorStatus::CREATED
+      connector_settings.needs_service_type? || connector_settings.connector_status == Connectors::ConnectorStatus::CREATED
+    end
+
+    def filtering_validation_triggered?(connector_settings)
+      filtering = connector_settings.filtering
+
+      unless filtering.present?
+        Utility::Logger.debug("#{connector_settings.formatted} does not contain filtering to be validated.")
+
+        return false
       end
 
-      false
+      draft_filters = filtering[:draft]
+
+      unless draft_filters.present?
+        Utility::Logger.debug("#{connector_settings.formatted} does not contain a draft filter to be validated.")
+
+        return false
+      end
+
+      validation = draft_filters[:validation]
+
+      unless validation.present?
+        Utility::Logger.warn("#{connector_settings.formatted} does not contain a validation object inside draft filtering. Check connectors index.")
+
+        return false
+      end
+
+      unless validation[:state] == Core::Filtering::ValidationStatus::EDITED
+        Utility::Logger.debug("#{connector_settings.formatted} filtering validation needs to be in state #{Core::Filtering::ValidationStatus::EDITED} to be able to validate it.")
+
+        return false
+      end
+
+      true
     end
 
     def connector_registered?(service_type)

data/lib/utility/bulk_queue.rb

@@ -0,0 +1,85 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+require 'json'
+
+module Utility
+  class BulkQueue
+    class QueueOverflowError < StandardError; end
+
+    # 500 items or 5MB
+    def initialize(operation_count_threshold = 500, size_threshold = 5 * 1024 * 1024)
+      @operation_count_threshold = operation_count_threshold.freeze
+      @size_threshold = size_threshold.freeze
+
+      @buffer = ''
+
+      @current_operation_count = 0
+
+      @current_buffer_size = 0
+      @current_data_size = 0
+    end
+
+    def pop_all
+      result = @buffer
+
+      reset
+
+      result
+    end
+
+    def add(operation, payload = nil)
+      raise QueueOverflowError unless will_fit?(operation, payload)
+
+      operation_size = get_size(operation)
+      payload_size = get_size(payload)
+
+      @current_operation_count += 1
+      @current_buffer_size += operation_size
+      @current_buffer_size += payload_size
+      @current_data_size += payload_size
+
+      @buffer << operation
+      @buffer << "\n"
+
+      if payload
+        @buffer << payload
+        @buffer << "\n"
+      end
+    end
+
+    def will_fit?(operation, payload = nil)
+      return false if @current_operation_count + 1 > @operation_count_threshold
+
+      operation_size = get_size(operation)
+      payload_size = get_size(payload)
+
+      @current_buffer_size + operation_size + payload_size < @size_threshold
+    end
+
+    def current_stats
+      {
+        :current_operation_count => @current_operation_count,
+        :current_buffer_size => @current_buffer_size
+      }
+    end
+
+    private
+
+    def get_size(str)
+      return 0 unless str
+      str.bytesize
+    end
+
+    def reset
+      @current_operation_count = 0
+      @current_buffer_size = 0
+      @current_data_size = 0
+
+      @buffer = ''
+    end
+  end
+end

data/lib/utility/constants.rb

@@ -16,5 +16,7 @@ module Utility
     JOB_INDEX = '.elastic-connectors-sync-jobs'
     CONTENT_INDEX_PREFIX = 'search-'
     CRAWLER_SERVICE_TYPE = 'elastic-crawler'
+    FILTERING_RULES_FEATURE = 'filtering_rules'
+    FILTERING_ADVANCED_FEATURE = 'filtering_advanced_config'
   end
 end

data/lib/utility/logger.rb

@@ -4,6 +4,7 @@
 # you may not use this file except in compliance with the Elastic License.
 #
 
+require 'config'
 require 'logger'
 require 'active_support/core_ext/module'
 require 'active_support/core_ext/string/filters'
@@ -23,7 +24,7 @@ module Utility
       end
 
       def logger
-        @logger ||= Settings[:ecs_logging] ? EcsLogging::Logger.new(STDOUT) : ::Logger.new(STDOUT)
+        @logger ||= defined?(::Settings) && ::Settings[:ecs_logging] ? EcsLogging::Logger.new(STDOUT) : ::Logger.new(STDOUT)
       end
 
       SUPPORTED_LOG_LEVELS.each do |level|

data/lib/utility.rb

@@ -4,14 +4,15 @@
 # you may not use this file except in compliance with the Elastic License.
 #
 
+require 'utility/bulk_queue'
+require 'utility/common'
 require 'utility/constants'
 require 'utility/cron'
-require 'utility/common'
+require 'utility/elasticsearch/index/mappings'
+require 'utility/elasticsearch/index/text_analysis_settings'
+require 'utility/environment'
 require 'utility/errors'
 require 'utility/es_client'
-require 'utility/environment'
 require 'utility/exception_tracking'
 require 'utility/extension_mapping_util'
 require 'utility/logger'
-require 'utility/elasticsearch/index/mappings'
-require 'utility/elasticsearch/index/text_analysis_settings'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: connectors_utility
 version: !ruby/object:Gem::Version
-  version: 8.6.0.4.pre.20221107T145613Z
+  version: 8.6.0.4.pre.20221114T235050Z
 platform: ruby
 authors:
 - Elastic
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-11-07 00:00:00.000000000 Z
+date: 2022-11-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -110,6 +110,7 @@ files:
 - lib/core/elastic_connector_actions.rb
 - lib/core/scheduler.rb
 - lib/utility.rb
+- lib/utility/bulk_queue.rb
 - lib/utility/common.rb
 - lib/utility/constants.rb
 - lib/utility/cron.rb
@@ -126,7 +127,7 @@ homepage: https://github.com/elastic/connectors-ruby
 licenses:
 - Elastic-2.0
 metadata:
-  revision: aa0e7e495d6f49de2bc05991b793b18b0c1ca78d
+  revision: f506d5e5ebedfb0c6058d347d8ce22adc42e2cc0
   repository: git@github.com:elastic/ent-search-connectors.git
 post_install_message: 
 rdoc_options: []