connectors_utility 8.7.0.0.pre.20221117T004939Z → 8.7.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5d2972f8e6974a79b6088ce6c03453c327132ce19ffb09dbf30f349eae4c2108
-  data.tar.gz: 4fd458de07be07923e0675dc0f341b8211ba3daeec8ac27bfb4f9eb9aff2334a
+  metadata.gz: 8bde5d9fcfd7af80dd1a20bc3fdffb3e509af46fc492607fa963858aacdb79bc
+  data.tar.gz: b3f26fba69d08e1add58b476a37a74f3fa855790d9bcea05c7a4f5ed3b1fd9bf
 SHA512:
-  metadata.gz: 9db02a3003d5645cbb5d57d4ca1bdb1acb65234f5a931afd9ccb06e2fbbe25be2394c65a72a9ae36038c8c127c35a0d937c83558ede0e4960fc688b073db052a
-  data.tar.gz: d02681e0d4009420b949ec649c9eac52bd533eee493c181ea1ffb15d939b561e81f234bac4885eefd0ea82c8564d7d330c0635c535b0616cfb9280e2e38512df
+  metadata.gz: 1eb4c63b6ae46d11b8b8e01224e1e1943a1971b7e12054978ebab97c99939fb1d0316b3ff06912d4908e8f34df482477e4e8d1f9b53c02fcebc809e27b597d2a
+  data.tar.gz: 2300a3a9c32ed95a1c25a54fba4737bb052caee10756887f2ac114948ec8a8ba0df066ee30420f5e3d64e0f5855df7e487f4782bc242e27ea66b592c6c60dbe6

data/lib/connectors/crawler/scheduler.rb

@@ -22,11 +22,47 @@ module Connectors
         []
       end
 
+      def when_triggered
+        loop do
+          connector_settings.each do |cs|
+            # crawler only supports :sync
+            if sync_triggered?(cs)
+              yield cs, :sync, nil
+              next
+            end
+
+            schedule_key = custom_schedule_triggered(cs)
+            yield cs, :sync, schedule_key if schedule_key
+          end
+        rescue *Utility::AUTHORIZATION_ERRORS => e
+          log_authorization_error(e)
+        rescue StandardError => e
+          log_standard_error(e)
+        ensure
+          if @is_shutting_down
+            break
+          end
+          sleep_for_poll_interval
+        end
+      end
+
       private
 
       def connector_registered?(service_type)
         service_type == 'elastic-crawler'
       end
+
+      # custom scheduling has no ordering, so the first-found schedule is returned
+      def custom_schedule_triggered(cs)
+        cs.custom_scheduling_settings.each do |key, custom_scheduling|
+          identifier = "#{cs.formatted} - #{custom_scheduling[:name]}"
+          if schedule_triggered?(custom_scheduling, identifier)
+            return key
+          end
+        end
+
+        nil
+      end
     end
   end
 end

data/lib/connectors/job_trigger_method.rb

@@ -0,0 +1,14 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+module Connectors
+  class JobTriggerMethod
+    ON_DEMAND = 'on_demand'
+    SCHEDULED = 'scheduled'
+  end
+end

data/lib/connectors_utility.rb

@@ -9,8 +9,11 @@
 require_relative 'utility'
 
 require_relative 'connectors/connector_status'
+require_relative 'connectors/crawler/scheduler'
+require_relative 'connectors/job_trigger_method'
 require_relative 'connectors/sync_status'
-require_relative 'core/scheduler'
+require_relative 'core/connector_job'
+require_relative 'core/connector_settings'
 require_relative 'core/elastic_connector_actions'
-
-require_relative 'connectors/crawler/scheduler'
+require_relative 'core/filtering/validation_status'
+require_relative 'core/scheduler'

data/lib/core/connector_job.rb

@@ -0,0 +1,251 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'active_support/core_ext/hash/indifferent_access'
+require 'connectors/sync_status'
+require 'core/connector_settings'
+require 'core/elastic_connector_actions'
+require 'utility'
+
+module Core
+  class ConnectorJob
+    DEFAULT_PAGE_SIZE = 100
+    IDLE_THRESHOLD = 60
+
+    def self.fetch_by_id(job_id)
+      es_response = ElasticConnectorActions.get_job(job_id)
+      return nil unless es_response[:found]
+
+      new(es_response)
+    end
+
+    def self.pending_jobs(connectors_ids: [], page_size: DEFAULT_PAGE_SIZE)
+      status_term = { status: Connectors::SyncStatus::PENDING_STATUSES }
+
+      query = { bool: { must: [{ terms: status_term }] } }
+
+      return fetch_jobs_by_query(query, page_size) if connectors_ids.empty?
+
+      query[:bool][:must] << { terms: { 'connector.id' => connectors_ids } }
+
+      fetch_jobs_by_query(query, page_size)
+    end
+
+    def self.orphaned_jobs(connector_ids = [], page_size = DEFAULT_PAGE_SIZE)
+      query = { bool: { must_not: { terms: { 'connector.id': connector_ids } } } }
+      fetch_jobs_by_query(query, page_size)
+    end
+
+    def self.delete_jobs(jobs)
+      query = { terms: { '_id': jobs.map(&:id) } }
+      ElasticConnectorActions.delete_jobs_by_query(query)
+    end
+
+    def self.idle_jobs(connector_id = nil, page_size = DEFAULT_PAGE_SIZE)
+      connector_ids = if connector_id
+                        [connector_id]
+                      else
+                        ConnectorSettings.fetch_native_connectors.map(&:id)
+                      end
+      query = {
+        bool: {
+          filter: [
+              { terms: { 'connector.id': connector_ids } },
+              { terms: { status: Connectors::SyncStatus::ACTIVE_STATUSES } },
+              { range: { last_seen: { lte: "now-#{IDLE_THRESHOLD}s" } } }
+          ]
+        }
+      }
+      fetch_jobs_by_query(query, page_size)
+    end
+
+    def self.enqueue(_connector_id)
+      nil
+    end
+
+    def id
+      @elasticsearch_response[:_id]
+    end
+
+    def [](property_name)
+      @elasticsearch_response[:_source][property_name]
+    end
+
+    def error
+      self[:error]
+    end
+
+    def status
+      self[:status]
+    end
+
+    def in_progress?
+      status == Connectors::SyncStatus::IN_PROGRESS
+    end
+
+    def canceling?
+      status == Connectors::SyncStatus::CANCELING
+    end
+
+    def suspended?
+      status == Connectors::SyncStatus::SUSPENDED
+    end
+
+    def canceled?
+      status == Connectors::SyncStatus::CANCELED
+    end
+
+    def pending?
+      Connectors::SyncStatus::PENDING_STATUSES.include?(status)
+    end
+
+    def active?
+      Connectors::SyncStatus::ACTIVE_STATUSES.include?(status)
+    end
+
+    def terminated?
+      Connectors::SyncStatus::TERMINAL_STATUSES.include?(status)
+    end
+
+    def connector_snapshot
+      self[:connector] || {}
+    end
+
+    def connector_id
+      connector_snapshot[:id]
+    end
+
+    def index_name
+      connector_snapshot[:index_name]
+    end
+
+    def language
+      connector_snapshot[:language]
+    end
+
+    def service_type
+      connector_snapshot[:service_type]
+    end
+
+    def configuration
+      connector_snapshot[:configuration]
+    end
+
+    def filtering
+      connector_snapshot[:filtering]
+    end
+
+    def pipeline
+      connector_snapshot[:pipeline] || {}
+    end
+
+    def extract_binary_content?
+      pipeline[:extract_binary_content]
+    end
+
+    def reduce_whitespace?
+      pipeline[:reduce_whitespace]
+    end
+
+    def run_ml_inference?
+      pipeline[:run_ml_inference]
+    end
+
+    def connector
+      @connector ||= ConnectorSettings.fetch_by_id(connector_id)
+    end
+
+    def update_metadata(ingestion_stats = {}, connector_metadata = {})
+      ingestion_stats ||= {}
+      doc = { :last_seen => Time.now }.merge(ingestion_stats)
+      doc[:metadata] = connector_metadata if connector_metadata&.any?
+      ElasticConnectorActions.update_job_fields(id, doc)
+    end
+
+    def done!(ingestion_stats = {}, connector_metadata = {})
+      terminate!(Connectors::SyncStatus::COMPLETED, nil, ingestion_stats, connector_metadata)
+    end
+
+    def error!(message, ingestion_stats = {}, connector_metadata = {})
+      terminate!(Connectors::SyncStatus::ERROR, message, ingestion_stats, connector_metadata)
+    end
+
+    def cancel!(ingestion_stats = {}, connector_metadata = {})
+      terminate!(Connectors::SyncStatus::CANCELED, nil, ingestion_stats, connector_metadata)
+    end
+
+    def with_concurrency_control
+      response = ElasticConnectorActions.get_job(id)
+
+      yield response, response['_seq_no'], response['_primary_term']
+    end
+
+    def make_running!
+      with_concurrency_control do |es_doc, seq_no, primary_term|
+        now = Time.now
+        doc = {
+          status: Connectors::SyncStatus::IN_PROGRESS,
+          started_at: now,
+          last_seen: now,
+          worker_hostname: Socket.gethostname
+        }
+
+        ElasticConnectorActions.update_job_fields(es_doc[:_id], doc, seq_no, primary_term)
+      end
+    end
+
+    def es_source
+      @elasticsearch_response[:_source]
+    end
+
+    private
+
+    def self.fetch_jobs_by_query(query, page_size)
+      results = []
+      offset = 0
+      loop do
+        response = ElasticConnectorActions.search_jobs(query, page_size, offset)
+
+        hits = response.dig('hits', 'hits') || []
+        total = response.dig('hits', 'total', 'value') || 0
+        results += hits.map { |hit| new(hit) }
+        break if results.size >= total
+        offset += hits.size
+      end
+
+      results
+    end
+
+    def initialize(es_response)
+      # TODO: remove the usage of with_indifferent_access. The initialize method should expect a hash argument
+      @elasticsearch_response = es_response.with_indifferent_access
+    end
+
+    def terminate!(status, error = nil, ingestion_stats = {}, connector_metadata = {})
+      ingestion_stats ||= {}
+      ingestion_stats[:total_document_count] = ElasticConnectorActions.document_count(index_name)
+      doc = {
+        :last_seen => Time.now,
+        :completed_at => Time.now,
+        :status => status,
+        :error => error
+      }.merge(ingestion_stats)
+      doc[:canceled_at] = Time.now if status == Connectors::SyncStatus::CANCELED
+      doc[:metadata] = connector_metadata if connector_metadata&.any?
+      ElasticConnectorActions.update_job_fields(id, doc)
+    end
+
+    def seq_no
+      @elasticsearch_response[:_seq_no]
+    end
+
+    def primary_term
+      @elasticsearch_response[:_primary_term]
+    end
+  end
+end

data/lib/core/connector_settings.rb

@@ -8,6 +8,7 @@
 
 require 'active_support/core_ext/hash/indifferent_access'
 require 'connectors/connector_status'
+require 'connectors/sync_status'
 require 'core/elastic_connector_actions'
 require 'utility'
 
@@ -49,6 +50,11 @@ module Core
       fetch_connectors_by_query(query, page_size)
     end
 
+    def self.fetch_all_connectors(page_size = DEFAULT_PAGE_SIZE)
+      query = { match_all: {} }
+      fetch_connectors_by_query(query, page_size)
+    end
+
     def id
       @elasticsearch_response[:_id]
     end
@@ -58,6 +64,24 @@ module Core
       @elasticsearch_response[:_source][property_name]
     end
 
+    def features
+      self[:features] || {}
+    end
+
+    # .dig version is the modern features way of doing things,
+    # Right-hand of OR operator is legacy features support
+    # When this is fixed with a migration, we can go ahead
+    def filtering_rule_feature_enabled?
+      !!features.dig(:sync_rules, :basic, :enabled) || !!features[:filtering_rules]
+    end
+    def filtering_advanced_config_feature_enabled?
+      !!features.dig(:sync_rules, :advanced, :enabled) || !!features[:filtering_advanced_config]
+    end
+
+    def any_filtering_feature_enabled?
+      filtering_rule_feature_enabled? || filtering_advanced_config_feature_enabled?
+    end
+
     def index_name
       self[:index_name]
     end
@@ -82,6 +106,18 @@ module Core
       self[:scheduling]
     end
 
+    def custom_scheduling_settings
+      self[:custom_scheduling]
+    end
+
+    def sync_now?
+      self[:sync_now] == true
+    end
+
+    def last_synced
+      self[:last_synced]
+    end
+
     def filtering
       # assume for now, that first object in filtering array or a filter object itself is the only filtering object
       filtering = @elasticsearch_response.dig(:_source, :filtering)
@@ -93,18 +129,6 @@ module Core
       Utility::Common.return_if_present(@elasticsearch_response.dig(:_source, :pipeline, :name), @connectors_meta.dig(:pipeline, :default_name), DEFAULT_REQUEST_PIPELINE)
     end
 
-    def extract_binary_content?
-      Utility::Common.return_if_present(@elasticsearch_response.dig(:_source, :pipeline, :extract_binary_content), @connectors_meta.dig(:pipeline, :default_extract_binary_content), DEFAULT_EXTRACT_BINARY_CONTENT)
-    end
-
-    def reduce_whitespace?
-      Utility::Common.return_if_present(@elasticsearch_response.dig(:_source, :pipeline, :reduce_whitespace), @connectors_meta.dig(:pipeline, :default_reduce_whitespace), DEFAULT_REDUCE_WHITESPACE)
-    end
-
-    def run_ml_inference?
-      Utility::Common.return_if_present(@elasticsearch_response.dig(:_source, :pipeline, :run_ml_inference), @connectors_meta.dig(:pipeline, :default_run_ml_inference), DEFAULT_RUN_ML_INFERENCE)
-    end
-
     def formatted
       properties = ["ID: #{id}"]
       properties << "Service type: #{service_type}" if service_type
@@ -130,19 +154,23 @@ module Core
     end
 
     def update_last_sync!(job)
+      # if job is nil, connector still needs to be updated, to avoid it stuck at in_progress
+      job_status = job&.status || Connectors::SyncStatus::ERROR
+      job_error = job.nil? ? 'Could\'t find the job' : job.error
+      job_error ||= 'unknown error' if job_status == Connectors::SyncStatus::ERROR
+      connector_status = (job_status == Connectors::SyncStatus::ERROR ? Connectors::ConnectorStatus::ERROR : Connectors::ConnectorStatus::CONNECTED)
       doc = {
-        :last_sync_status => job.status,
+        :last_sync_status => job_status,
         :last_synced => Time.now,
-        :last_sync_error => job.error,
-        :error => job.error
+        :last_sync_error => job_error,
+        :status => connector_status,
+        :error => job_error
       }
-
-      if job.terminated?
+      if job&.terminated?
         doc[:last_indexed_document_count] = job[:indexed_document_count]
         doc[:last_deleted_document_count] = job[:deleted_document_count]
       end
-
-      Core::ElasticConnectorActions.update_connector_fields(job.connector_id, doc)
+      Core::ElasticConnectorActions.update_connector_fields(id, doc)
     end
 
     private

data/lib/core/elastic_connector_actions.rb

@@ -8,6 +8,7 @@
 #
 require 'active_support/core_ext/hash'
 require 'connectors/connector_status'
+require 'connectors/job_trigger_method'
 require 'connectors/sync_status'
 require 'utility'
 require 'elastic-transport'
@@ -91,6 +92,17 @@ module Core
         )
       end
 
+      def delete_jobs_by_query(query)
+        client.delete_by_query(
+          :index => Utility::Constants::JOB_INDEX,
+          :body => { :query => query }
+        )
+      end
+
+      def delete_indices(indices)
+        client.indices.delete(:index => indices, :ignore_unavailable => true)
+      end
+
       def update_connector_configuration(connector_id, configuration)
         update_connector_fields(connector_id, :configuration => configuration)
       end
@@ -145,12 +157,37 @@ module Core
         )
       end
 
-      def update_connector_last_sync_status(connector_id, last_sync_status)
+      def update_connector_sync_start(connector_id)
+        doc = connector_with_concurrency_control(connector_id)
+
+        body = {
+            last_sync_status: Connectors::SyncStatus::IN_PROGRESS,
+            last_sync_error: nil,
+            status: Connectors::ConnectorStatus::CONNECTED
+        }
+
+        update_connector_fields(
+          connector_id,
+          body,
+          doc[:seq_no],
+          doc[:primary_term]
+        )
+      end
+
+      def update_connector_custom_scheduling_last_synced(connector_id, schedule_key)
         doc = connector_with_concurrency_control(connector_id)
 
+        body = {
+          :custom_scheduling => {
+            schedule_key => {
+              :last_synced => Time.now
+            }
+          }
+        }
+
         update_connector_fields(
           connector_id,
-          { last_sync_status: last_sync_status },
+          body,
           doc[:seq_no],
           doc[:primary_term]
         )
@@ -178,13 +215,15 @@ module Core
           status: Connectors::SyncStatus::PENDING,
           created_at: Time.now,
           last_seen: Time.now,
+          trigger_method: connector_settings.sync_now? ? Connectors::JobTriggerMethod::ON_DEMAND : Connectors::JobTriggerMethod::SCHEDULED,
           connector: {
             id: connector_settings.id,
             filtering: convert_connector_filtering_to_job_filtering(connector_settings.filtering),
             index_name: connector_settings.index_name,
             language: connector_settings[:language],
             pipeline: connector_settings[:pipeline],
-            service_type: connector_settings.service_type
+            service_type: connector_settings.service_type,
+            configuration: connector_settings.configuration
           }
         }
 
@@ -220,37 +259,6 @@ module Core
         update_connector_fields(connector_id, body)
       end
 
-      def update_sync(job_id, metadata)
-        body = {
-          :doc => { :last_seen => Time.now }.merge(metadata)
-        }
-        client.update(:index => Utility::Constants::JOB_INDEX, :id => job_id, :body => body)
-      end
-
-      def complete_sync(connector_id, job_id, metadata, error)
-        sync_status = error ? Connectors::SyncStatus::ERROR : Connectors::SyncStatus::COMPLETED
-
-        metadata ||= {}
-
-        update_connector_fields(connector_id,
-                                :last_sync_status => sync_status,
-                                :last_sync_error => error,
-                                :error => error,
-                                :last_synced => Time.now,
-                                :last_indexed_document_count => metadata[:indexed_document_count],
-                                :last_deleted_document_count => metadata[:deleted_document_count])
-
-        body = {
-          :doc => {
-            :status => sync_status,
-            :completed_at => Time.now,
-            :last_seen => Time.now,
-            :error => error
-          }.merge(metadata)
-        }
-        client.update(:index => Utility::Constants::JOB_INDEX, :id => job_id, :body => body)
-      end
-
       def fetch_document_ids(index_name)
         page_size = 1000
         result = []
@@ -331,9 +339,11 @@ module Core
       # Creation of connector index should be handled by Kibana, this method is only used by ftest.rb
       def ensure_connectors_index_exists
         mappings = {
+          :dynamic => false,
           :properties => {
             :api_key_id => { :type => :keyword },
             :configuration => { :type => :object },
+            :custom_schedule => { :type => :object },
             :description => { :type => :text },
             :error => { :type => :keyword },
             :features => {
@@ -451,6 +461,7 @@ module Core
       # Creation of job index should be handled by Kibana, this method is only used by ftest.rb
       def ensure_job_index_exists
         mappings = {
+          :dynamic => false,
           :properties => {
             :cancelation_requested_at => { :type => :date },
             :canceled_at => { :type => :date },
@@ -528,8 +539,8 @@ module Core
       end
 
       def document_count(index_name)
-        client.indices.refresh(:index => index_name)
-        client.count(:index => index_name)['count']
+        client.indices.refresh(:index => index_name, :ignore_unavailable => true)
+        client.count(:index => index_name, :ignore_unavailable => true)['count']
       end
 
       private

data/lib/core/scheduler.rb

@@ -44,17 +44,14 @@ module Core
           end
         end
       rescue *Utility::AUTHORIZATION_ERRORS => e
-        Utility::ExceptionTracking.log_exception(e, 'Could not retrieve connectors settings due to authorization error.')
+        log_authorization_error(e)
       rescue StandardError => e
-        Utility::ExceptionTracking.log_exception(e, 'Sync failed due to unexpected error.')
+        log_standard_error(e)
       ensure
         if @is_shutting_down
           break
         end
-        if @poll_interval > 0 && !@is_shutting_down
-          Utility::Logger.debug("Sleeping for #{@poll_interval} seconds in #{self.class}.")
-          sleep(@poll_interval)
-        end
+        sleep_for_poll_interval
       end
     end
 
@@ -78,56 +75,12 @@ module Core
       end
 
       # Sync when sync_now flag is true for the connector
-      if connector_settings[:sync_now] == true
+      if connector_settings.sync_now?
         Utility::Logger.info("#{connector_settings.formatted.capitalize} is manually triggered to sync now.")
         return true
       end
 
-      # Don't sync if sync is explicitly disabled
-      scheduling_settings = connector_settings.scheduling_settings
-      unless scheduling_settings.present? && scheduling_settings[:enabled] == true
-        Utility::Logger.debug("#{connector_settings.formatted.capitalize} scheduling is disabled.")
-        return false
-      end
-
-      # We want to sync when sync never actually happened
-      last_synced = connector_settings[:last_synced]
-      if last_synced.nil? || last_synced.empty?
-        Utility::Logger.info("#{connector_settings.formatted.capitalize} has never synced yet, running initial sync.")
-        return true
-      end
-
-      current_schedule = scheduling_settings[:interval]
-
-      # Don't sync if there is no actual scheduling interval
-      if current_schedule.nil? || current_schedule.empty?
-        Utility::Logger.warn("No sync schedule configured for #{connector_settings.formatted}.")
-        return false
-      end
-
-      current_schedule = begin
-        Utility::Cron.quartz_to_crontab(current_schedule)
-      rescue StandardError => e
-        Utility::ExceptionTracking.log_exception(e, "Unable to convert quartz (#{current_schedule}) to crontab.")
-        return false
-      end
-      cron_parser = Fugit::Cron.parse(current_schedule)
-
-      # Don't sync if the scheduling interval is non-parsable
-      unless cron_parser
-        Utility::Logger.error("Unable to parse sync schedule for #{connector_settings.formatted}: expression #{current_schedule} is not a valid Quartz Cron definition.")
-        return false
-      end
-
-      next_trigger_time = cron_parser.next_time(Time.parse(last_synced))
-
-      # Sync if next trigger for the connector is in past
-      if next_trigger_time < Time.now
-        Utility::Logger.info("#{connector_settings.formatted.capitalize} sync is triggered by cron schedule #{current_schedule}.")
-        return true
-      end
-
-      false
+      schedule_triggered?(connector_settings.scheduling_settings, connector_settings.formatted)
     end
 
     def heartbeat_triggered?(connector_settings)
@@ -148,6 +101,12 @@ module Core
     end
 
     def filtering_validation_triggered?(connector_settings)
+      unless connector_settings.any_filtering_feature_enabled?
+        Utility::Logger.debug("#{connector_settings.formatted} all filtering features are disabled. Skip filtering validation.")
+
+        return false
+      end
+
       filtering = connector_settings.filtering
 
       unless filtering.present?
@@ -189,5 +148,61 @@ module Core
         false
       end
     end
+
+    def schedule_triggered?(scheduling_settings, identifier)
+      # Don't sync if sync is explicitly disabled
+      unless scheduling_settings.present? && scheduling_settings[:enabled] == true
+        Utility::Logger.debug("#{identifier.capitalize} scheduling is disabled.")
+        return false
+      end
+
+      current_schedule = scheduling_settings[:interval]
+
+      # Don't sync if there is no actual scheduling interval
+      if current_schedule.nil? || current_schedule.empty?
+        Utility::Logger.warn("No sync schedule configured for #{identifier}.")
+        return false
+      end
+
+      current_schedule =
+        begin
+          Utility::Cron.quartz_to_crontab(current_schedule)
+        rescue StandardError => e
+          Utility::ExceptionTracking.log_exception(e, "Unable to convert quartz (#{current_schedule}) to crontab.")
+          return false
+        end
+      cron_parser = Fugit::Cron.parse(current_schedule)
+
+      # Don't sync if the scheduling interval is non-parsable
+      unless cron_parser
+        Utility::Logger.error("Unable to parse sync schedule for #{identifier}: expression #{current_schedule} is not a valid Quartz Cron definition.")
+        return false
+      end
+
+      next_trigger_time = cron_parser.next_time(Time.now)
+
+      # Sync if next trigger happens before the next poll
+      if next_trigger_time <= Time.now + @poll_interval
+        Utility::Logger.info("#{identifier.capitalize} sync is triggered by cron schedule #{current_schedule}.")
+        return true
+      end
+
+      false
+    end
+
+    def sleep_for_poll_interval
+      if @poll_interval > 0 && !@is_shutting_down
+        Utility::Logger.debug("Sleeping for #{@poll_interval} seconds in #{self.class}.")
+        sleep(@poll_interval)
+      end
+    end
+
+    def log_authorization_error(e)
+      Utility::ExceptionTracking.log_exception(e, 'Could not retrieve connectors settings due to authorization error.')
+    end
+
+    def log_standard_error(e)
+      Utility::ExceptionTracking.log_exception(e, 'Sync failed due to unexpected error.')
+    end
   end
 end

data/lib/utility/bulk_queue.rb

@@ -13,7 +13,7 @@ module Utility
     class QueueOverflowError < StandardError; end
 
     # 500 items or 5MB
-    def initialize(operation_count_threshold = Utility::Constants::DEFAULT_MAX_INGESTION_QUEUE_LENGTH, size_threshold = Utility::Constants::DEFAULT_MAX_INGESTION_QUEUE_BYTES)
+    def initialize(operation_count_threshold = Utility::Constants::DEFAULT_MAX_INGESTION_QUEUE_SIZE, size_threshold = Utility::Constants::DEFAULT_MAX_INGESTION_QUEUE_BYTES)
       @operation_count_threshold = operation_count_threshold.freeze
       @size_threshold = size_threshold.freeze
 

data/lib/utility/constants.rb

@@ -16,8 +16,6 @@ module Utility
     JOB_INDEX = '.elastic-connectors-sync-jobs'
     CONTENT_INDEX_PREFIX = 'search-'
     CRAWLER_SERVICE_TYPE = 'elastic-crawler'
-    FILTERING_RULES_FEATURE = 'filtering_rules'
-    FILTERING_ADVANCED_FEATURE = 'filtering_advanced_config'
 
     # Maximum number of operations in BULK Elasticsearch operation that will ingest the data
     DEFAULT_MAX_INGESTION_QUEUE_SIZE = 500

data/lib/utility/error_monitor.rb

@@ -51,7 +51,7 @@ module Utility
     def note_success
       @consecutive_error_count = 0
       @success_count += 1
-      increment_window_index
+      track_window_error(false)
     end
 
     def note_error(error, id: Time.now.to_i)
@@ -60,10 +60,9 @@ module Utility
       Utility::Logger.debug("Message id: #{id} - #{error_message}\n#{stack_trace}")
       @total_error_count += 1
       @consecutive_error_count += 1
-      @window_errors[@window_index] = true
       @error_queue << DocumentError.new(error.class.name, error_message, stack_trace, id)
       @error_queue = @error_queue.drop(1) if @error_queue.size > @error_queue_size
-      increment_window_index
+      track_window_error(true)
       @last_error = error
 
       raise_if_necessary
@@ -92,10 +91,32 @@ module Utility
     end
 
     def num_errors_in_window
-      @window_errors.count(&:itself).to_f
+      @window_errors.count(true).to_f
     end
 
-    def increment_window_index
+    def track_window_error(is_error)
+      # We keep the errors array of the size @window_size this way, imagine @window_size = 5
+      # Error array inits as falses:
+      # [ false, false, false, false, false ]
+      # Third document raises an error:
+      # [ false, false, true,  false, false ]
+      #                 ^^^^
+      #                 2 % 5 == 2
+      # Fifth document raises an error:
+      # [ false, false, true,  false, true  ]
+      #                               ^^^^
+      #                               4 % 5 == 4
+      # Sixth document raises an error:
+      # [ true, false, true,   false, true  ]
+      #   ^^^^
+      #   5 % 5 == 0
+      #
+      # Eigth document is successful:
+      # [ true, false, false,  false, true  ]
+      #                ^^^^^
+      #                7 % 5 == 2
+      # And so on.
+      @window_errors[@window_index] = is_error
       @window_index = (@window_index + 1) % @window_size
     end
 

data/lib/utility/es_client.rb

@@ -43,6 +43,10 @@ module Utility
       configs[:transport_options] = es_config[:transport_options] if es_config[:transport_options]
       configs[:ca_fingerprint] = es_config[:ca_fingerprint] if es_config[:ca_fingerprint]
 
+      # headers
+      # these are necessary for cloud-hosted native connectors
+      configs[:headers] = es_config[:headers].to_h if es_config[:headers]
+
       # if log or trace is activated, we use the application logger
       configs[:logger] = if configs[:log] || configs[:trace]
                            Utility::Logger.logger

data/lib/utility/filtering.rb

@@ -17,6 +17,10 @@ module Utility
 
         filter.present? ? filter : {}
       end
+
+      def rule_pre_processing_active?(filter)
+        !filter.dig('advanced_snippet', 'value')&.present?
+      end
     end
   end
 end

metadata

@@ -1,29 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: connectors_utility
 version: !ruby/object:Gem::Version
-  version: 8.7.0.0.pre.20221117T004939Z
+  version: 8.7.0.0
 platform: ruby
 authors:
 - Elastic
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-11-17 00:00:00.000000000 Z
+date: 2023-02-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 5.2.6
+        version: '5.2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 5.2.6
+        version: '5.2'
 - !ruby/object:Gem::Dependency
   name: ecs-logging
   requirement: !ruby/object:Gem::Requirement
@@ -104,8 +104,10 @@ files:
 - NOTICE.txt
 - lib/connectors/connector_status.rb
 - lib/connectors/crawler/scheduler.rb
+- lib/connectors/job_trigger_method.rb
 - lib/connectors/sync_status.rb
 - lib/connectors_utility.rb
+- lib/core/connector_job.rb
 - lib/core/connector_settings.rb
 - lib/core/elastic_connector_actions.rb
 - lib/core/filtering/validation_status.rb
@@ -130,9 +132,9 @@ homepage: https://github.com/elastic/connectors-ruby
 licenses:
 - Elastic-2.0
 metadata:
-  revision: 294214a26b0fe9a4347763b01de681c336e8daae
-  repository: https://github.com/elastic/connectors-ruby.git
-post_install_message:
+  revision: ae6292137eef9acac1259c5e7e71a3d0e149210b
+  repository: https://github.com/elastic/connectors-ruby
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -143,12 +145,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
 rubygems_version: 3.0.3.1
-signing_key:
+signing_key: 
 specification_version: 4
 summary: Gem containing shared Connector Services libraries
 test_files: []