connectors_utility 8.6.0.7 → 8.7.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d2406276300699ebf07cc2890f994c3a1a9fecb9127793bcad68fb69a4f49111
-  data.tar.gz: 0a35b249367837148c2f7fe2eac30f74de37c5fd5610f30859c703bfbcbb76be
+  metadata.gz: 8bde5d9fcfd7af80dd1a20bc3fdffb3e509af46fc492607fa963858aacdb79bc
+  data.tar.gz: b3f26fba69d08e1add58b476a37a74f3fa855790d9bcea05c7a4f5ed3b1fd9bf
 SHA512:
-  metadata.gz: e0cca6acc1faffd3312213d807e77425ed006f44313c3b49faf01c8f3b31817ef7a194249581d6058cc8309773fa1142f5daf8994a2a8e24533f1b3f08f73efd
-  data.tar.gz: '078ebf6f05133d0b3ffe252157e72bcfaee89fb98e65d8ffde1a400c69a95f1ea4edfd7dfd66a626955aa24a0e29217cb6f51ead3e30c1c1d2792d52c7ffbb33'
+  metadata.gz: 1eb4c63b6ae46d11b8b8e01224e1e1943a1971b7e12054978ebab97c99939fb1d0316b3ff06912d4908e8f34df482477e4e8d1f9b53c02fcebc809e27b597d2a
+  data.tar.gz: 2300a3a9c32ed95a1c25a54fba4737bb052caee10756887f2ac114948ec8a8ba0df066ee30420f5e3d64e0f5855df7e487f4782bc242e27ea66b592c6c60dbe6

data/lib/connectors/crawler/scheduler.rb

@@ -22,11 +22,47 @@ module Connectors
         []
       end
 
+      def when_triggered
+        loop do
+          connector_settings.each do |cs|
+            # crawler only supports :sync
+            if sync_triggered?(cs)
+              yield cs, :sync, nil
+              next
+            end
+
+            schedule_key = custom_schedule_triggered(cs)
+            yield cs, :sync, schedule_key if schedule_key
+          end
+        rescue *Utility::AUTHORIZATION_ERRORS => e
+          log_authorization_error(e)
+        rescue StandardError => e
+          log_standard_error(e)
+        ensure
+          if @is_shutting_down
+            break
+          end
+          sleep_for_poll_interval
+        end
+      end
+
       private
 
       def connector_registered?(service_type)
         service_type == 'elastic-crawler'
       end
+
+      # custom scheduling has no ordering, so the first-found schedule is returned
+      def custom_schedule_triggered(cs)
+        cs.custom_scheduling_settings.each do |key, custom_scheduling|
+          identifier = "#{cs.formatted} - #{custom_scheduling[:name]}"
+          if schedule_triggered?(custom_scheduling, identifier)
+            return key
+          end
+        end
+
+        nil
+      end
     end
   end
 end

data/lib/core/connector_job.rb

@@ -15,7 +15,7 @@ require 'utility'
 module Core
   class ConnectorJob
     DEFAULT_PAGE_SIZE = 100
-    STUCK_THRESHOLD = 60
+    IDLE_THRESHOLD = 60
 
     def self.fetch_by_id(job_id)
       es_response = ElasticConnectorActions.get_job(job_id)
@@ -36,8 +36,7 @@ module Core
       fetch_jobs_by_query(query, page_size)
     end
 
-    def self.orphaned_jobs(page_size = DEFAULT_PAGE_SIZE)
-      connector_ids = ConnectorSettings.fetch_all_connectors.map(&:id)
+    def self.orphaned_jobs(connector_ids = [], page_size = DEFAULT_PAGE_SIZE)
       query = { bool: { must_not: { terms: { 'connector.id': connector_ids } } } }
       fetch_jobs_by_query(query, page_size)
     end
@@ -47,7 +46,7 @@ module Core
       ElasticConnectorActions.delete_jobs_by_query(query)
     end
 
-    def self.stuck_jobs(connector_id = nil, page_size = DEFAULT_PAGE_SIZE)
+    def self.idle_jobs(connector_id = nil, page_size = DEFAULT_PAGE_SIZE)
       connector_ids = if connector_id
                         [connector_id]
                       else
@@ -58,7 +57,7 @@ module Core
           filter: [
               { terms: { 'connector.id': connector_ids } },
               { terms: { status: Connectors::SyncStatus::ACTIVE_STATUSES } },
-              { range: { last_seen: { lte: "now-#{STUCK_THRESHOLD}s" } } }
+              { range: { last_seen: { lte: "now-#{IDLE_THRESHOLD}s" } } }
           ]
         }
       }

data/lib/core/connector_settings.rb

@@ -64,6 +64,24 @@ module Core
       @elasticsearch_response[:_source][property_name]
     end
 
+    def features
+      self[:features] || {}
+    end
+
+    # .dig version is the modern features way of doing things,
+    # Right-hand of OR operator is legacy features support
+    # When this is fixed with a migration, we can go ahead
+    def filtering_rule_feature_enabled?
+      !!features.dig(:sync_rules, :basic, :enabled) || !!features[:filtering_rules]
+    end
+    def filtering_advanced_config_feature_enabled?
+      !!features.dig(:sync_rules, :advanced, :enabled) || !!features[:filtering_advanced_config]
+    end
+
+    def any_filtering_feature_enabled?
+      filtering_rule_feature_enabled? || filtering_advanced_config_feature_enabled?
+    end
+
     def index_name
       self[:index_name]
     end
@@ -88,10 +106,18 @@ module Core
       self[:scheduling]
     end
 
+    def custom_scheduling_settings
+      self[:custom_scheduling]
+    end
+
     def sync_now?
       self[:sync_now] == true
     end
 
+    def last_synced
+      self[:last_synced]
+    end
+
     def filtering
       # assume for now, that first object in filtering array or a filter object itself is the only filtering object
       filtering = @elasticsearch_response.dig(:_source, :filtering)

data/lib/core/elastic_connector_actions.rb

@@ -157,12 +157,37 @@ module Core
         )
       end
 
-      def update_connector_last_sync_status(connector_id, last_sync_status)
+      def update_connector_sync_start(connector_id)
         doc = connector_with_concurrency_control(connector_id)
 
+        body = {
+            last_sync_status: Connectors::SyncStatus::IN_PROGRESS,
+            last_sync_error: nil,
+            status: Connectors::ConnectorStatus::CONNECTED
+        }
+
         update_connector_fields(
           connector_id,
-          { last_sync_status: last_sync_status },
+          body,
+          doc[:seq_no],
+          doc[:primary_term]
+        )
+      end
+
+      def update_connector_custom_scheduling_last_synced(connector_id, schedule_key)
+        doc = connector_with_concurrency_control(connector_id)
+
+        body = {
+          :custom_scheduling => {
+            schedule_key => {
+              :last_synced => Time.now
+            }
+          }
+        }
+
+        update_connector_fields(
+          connector_id,
+          body,
           doc[:seq_no],
           doc[:primary_term]
         )
@@ -314,9 +339,11 @@ module Core
       # Creation of connector index should be handled by Kibana, this method is only used by ftest.rb
       def ensure_connectors_index_exists
         mappings = {
+          :dynamic => false,
           :properties => {
             :api_key_id => { :type => :keyword },
             :configuration => { :type => :object },
+            :custom_schedule => { :type => :object },
             :description => { :type => :text },
             :error => { :type => :keyword },
             :features => {
@@ -434,6 +461,7 @@ module Core
       # Creation of job index should be handled by Kibana, this method is only used by ftest.rb
       def ensure_job_index_exists
         mappings = {
+          :dynamic => false,
           :properties => {
             :cancelation_requested_at => { :type => :date },
             :canceled_at => { :type => :date },
@@ -511,8 +539,8 @@ module Core
       end
 
       def document_count(index_name)
-        client.indices.refresh(:index => index_name)
-        client.count(:index => index_name)['count']
+        client.indices.refresh(:index => index_name, :ignore_unavailable => true)
+        client.count(:index => index_name, :ignore_unavailable => true)['count']
       end
 
       private

data/lib/core/scheduler.rb

@@ -44,17 +44,14 @@ module Core
           end
         end
       rescue *Utility::AUTHORIZATION_ERRORS => e
-        Utility::ExceptionTracking.log_exception(e, 'Could not retrieve connectors settings due to authorization error.')
+        log_authorization_error(e)
       rescue StandardError => e
-        Utility::ExceptionTracking.log_exception(e, 'Sync failed due to unexpected error.')
+        log_standard_error(e)
       ensure
         if @is_shutting_down
           break
         end
-        if @poll_interval > 0 && !@is_shutting_down
-          Utility::Logger.debug("Sleeping for #{@poll_interval} seconds in #{self.class}.")
-          sleep(@poll_interval)
-        end
+        sleep_for_poll_interval
       end
     end
 
@@ -83,51 +80,7 @@ module Core
         return true
       end
 
-      # Don't sync if sync is explicitly disabled
-      scheduling_settings = connector_settings.scheduling_settings
-      unless scheduling_settings.present? && scheduling_settings[:enabled] == true
-        Utility::Logger.debug("#{connector_settings.formatted.capitalize} scheduling is disabled.")
-        return false
-      end
-
-      current_schedule = scheduling_settings[:interval]
-
-      # Don't sync if there is no actual scheduling interval
-      if current_schedule.nil? || current_schedule.empty?
-        Utility::Logger.warn("No sync schedule configured for #{connector_settings.formatted}.")
-        return false
-      end
-
-      current_schedule = begin
-        Utility::Cron.quartz_to_crontab(current_schedule)
-      rescue StandardError => e
-        Utility::ExceptionTracking.log_exception(e, "Unable to convert quartz (#{current_schedule}) to crontab.")
-        return false
-      end
-      cron_parser = Fugit::Cron.parse(current_schedule)
-
-      # Don't sync if the scheduling interval is non-parsable
-      unless cron_parser
-        Utility::Logger.error("Unable to parse sync schedule for #{connector_settings.formatted}: expression #{current_schedule} is not a valid Quartz Cron definition.")
-        return false
-      end
-
-      # We want to sync when sync never actually happened
-      last_synced = connector_settings[:last_synced]
-      if last_synced.nil? || last_synced.empty?
-        Utility::Logger.info("#{connector_settings.formatted.capitalize} has never synced yet, running initial sync.")
-        return true
-      end
-
-      next_trigger_time = cron_parser.next_time(Time.parse(last_synced))
-
-      # Sync if next trigger for the connector is in past
-      if next_trigger_time < Time.now
-        Utility::Logger.info("#{connector_settings.formatted.capitalize} sync is triggered by cron schedule #{current_schedule}.")
-        return true
-      end
-
-      false
+      schedule_triggered?(connector_settings.scheduling_settings, connector_settings.formatted)
     end
 
     def heartbeat_triggered?(connector_settings)
@@ -148,6 +101,12 @@ module Core
     end
 
     def filtering_validation_triggered?(connector_settings)
+      unless connector_settings.any_filtering_feature_enabled?
+        Utility::Logger.debug("#{connector_settings.formatted} all filtering features are disabled. Skip filtering validation.")
+
+        return false
+      end
+
       filtering = connector_settings.filtering
 
       unless filtering.present?
@@ -189,5 +148,61 @@ module Core
         false
       end
     end
+
+    def schedule_triggered?(scheduling_settings, identifier)
+      # Don't sync if sync is explicitly disabled
+      unless scheduling_settings.present? && scheduling_settings[:enabled] == true
+        Utility::Logger.debug("#{identifier.capitalize} scheduling is disabled.")
+        return false
+      end
+
+      current_schedule = scheduling_settings[:interval]
+
+      # Don't sync if there is no actual scheduling interval
+      if current_schedule.nil? || current_schedule.empty?
+        Utility::Logger.warn("No sync schedule configured for #{identifier}.")
+        return false
+      end
+
+      current_schedule =
+        begin
+          Utility::Cron.quartz_to_crontab(current_schedule)
+        rescue StandardError => e
+          Utility::ExceptionTracking.log_exception(e, "Unable to convert quartz (#{current_schedule}) to crontab.")
+          return false
+        end
+      cron_parser = Fugit::Cron.parse(current_schedule)
+
+      # Don't sync if the scheduling interval is non-parsable
+      unless cron_parser
+        Utility::Logger.error("Unable to parse sync schedule for #{identifier}: expression #{current_schedule} is not a valid Quartz Cron definition.")
+        return false
+      end
+
+      next_trigger_time = cron_parser.next_time(Time.now)
+
+      # Sync if next trigger happens before the next poll
+      if next_trigger_time <= Time.now + @poll_interval
+        Utility::Logger.info("#{identifier.capitalize} sync is triggered by cron schedule #{current_schedule}.")
+        return true
+      end
+
+      false
+    end
+
+    def sleep_for_poll_interval
+      if @poll_interval > 0 && !@is_shutting_down
+        Utility::Logger.debug("Sleeping for #{@poll_interval} seconds in #{self.class}.")
+        sleep(@poll_interval)
+      end
+    end
+
+    def log_authorization_error(e)
+      Utility::ExceptionTracking.log_exception(e, 'Could not retrieve connectors settings due to authorization error.')
+    end
+
+    def log_standard_error(e)
+      Utility::ExceptionTracking.log_exception(e, 'Sync failed due to unexpected error.')
+    end
   end
 end

data/lib/utility/constants.rb

@@ -16,8 +16,6 @@ module Utility
     JOB_INDEX = '.elastic-connectors-sync-jobs'
     CONTENT_INDEX_PREFIX = 'search-'
     CRAWLER_SERVICE_TYPE = 'elastic-crawler'
-    FILTERING_RULES_FEATURE = 'filtering_rules'
-    FILTERING_ADVANCED_FEATURE = 'filtering_advanced_config'
 
     # Maximum number of operations in BULK Elasticsearch operation that will ingest the data
     DEFAULT_MAX_INGESTION_QUEUE_SIZE = 500

data/lib/utility/es_client.rb

@@ -43,6 +43,10 @@ module Utility
       configs[:transport_options] = es_config[:transport_options] if es_config[:transport_options]
       configs[:ca_fingerprint] = es_config[:ca_fingerprint] if es_config[:ca_fingerprint]
 
+      # headers
+      # these are necessary for cloud-hosted native connectors
+      configs[:headers] = es_config[:headers].to_h if es_config[:headers]
+
       # if log or trace is activated, we use the application logger
       configs[:logger] = if configs[:log] || configs[:trace]
                            Utility::Logger.logger

metadata

@@ -1,29 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: connectors_utility
 version: !ruby/object:Gem::Version
-  version: 8.6.0.7
+  version: 8.7.0.0
 platform: ruby
 authors:
 - Elastic
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-11-29 00:00:00.000000000 Z
+date: 2023-02-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 5.2.6
+        version: '5.2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 5.2.6
+        version: '5.2'
 - !ruby/object:Gem::Dependency
   name: ecs-logging
   requirement: !ruby/object:Gem::Requirement
@@ -132,7 +132,7 @@ homepage: https://github.com/elastic/connectors-ruby
 licenses:
 - Elastic-2.0
 metadata:
-  revision: 86b6c162f41d3a837841d0df2430cbc26b57eb33
+  revision: ae6292137eef9acac1259c5e7e71a3d0e149210b
   repository: https://github.com/elastic/connectors-ruby
 post_install_message: 
 rdoc_options: []