connectors_service 8.7.0.0.pre.20221117T010623Z → 8.11.0.0

data/lib/core/job_cleanup.rb

@@ -0,0 +1,66 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'core'
+require 'utility/logger'
+
+module Core
+  class JobCleanUp
+    class << self
+      def execute(connector_id = nil)
+        process_orphaned_jobs
+        process_idle_jobs(connector_id)
+      end
+
+      private
+
+      def process_orphaned_jobs
+        Utility::Logger.debug('Start cleaning up orphaned jobs...')
+        all_connectors = ConnectorSettings.fetch_all_connectors
+        orphaned_jobs = ConnectorJob.orphaned_jobs(all_connectors.map(&:id))
+        if orphaned_jobs.empty?
+          Utility::Logger.debug('No orphaned jobs found. Skipping...')
+          return
+        end
+
+        # delete content indicies in case they are re-created by sync job
+        content_indices = (orphaned_jobs.map(&:index_name) - all_connectors.map(&:index_name)).compact.uniq
+        ElasticConnectorActions.delete_indices(content_indices) if content_indices.any?
+        result = ConnectorJob.delete_jobs(orphaned_jobs)
+        Utility::Logger.error("Error found when deleting jobs: #{result['failures']}") if result['failures']&.any?
+        Utility::Logger.info("Successfully deleted #{result['deleted']} out of #{result['total']} orphaned jobs.")
+      end
+
+      def process_idle_jobs(connector_id = nil)
+        Utility::Logger.debug("Start cleaning up idle jobs for #{connector_id ? "connector #{connector_id}" : 'native connectors'}...")
+        idle_jobs = ConnectorJob.idle_jobs(connector_id)
+        if idle_jobs.empty?
+          Utility::Logger.debug('No idle jobs found. Skipping...')
+          return
+        end
+
+        marked_count = 0
+        idle_jobs.each do |job|
+          job.error!('The job has not seen any update for some time.')
+          Utility::Logger.debug("Successfully marked job #{job.id} as error.")
+
+          job_id = job.id
+          job = ConnectorJob.fetch_by_id(job_id)
+          Utility::Logger.warn("Could not found job by id #{job_id}") if job.nil?
+          Utility::Logger.warn("Could not found connector by id #{job.connector_id}") if job && job.connector.nil?
+
+          job&.connector&.update_last_sync!(job)
+          marked_count += 1
+        rescue StandardError => e
+          Utility::ExceptionTracking.log_exception(e)
+        end
+        Utility::Logger.info("Successfully marked #{marked_count} out of #{idle_jobs.count} idle jobs as error.")
+      end
+    end
+  end
+end

data/lib/core/jobs/consumer.rb

@@ -6,7 +6,11 @@
 
 # frozen_string_literal: true
 
+require 'utility/logger'
 require 'utility/constants'
+require 'core/connector_job'
+require 'core/sync_job_runner'
+require 'concurrent'
 
 module Core
   module Jobs
@@ -30,90 +34,39 @@ module Core
 
         @max_ingestion_queue_size = max_ingestion_queue_size
         @max_ingestion_queue_bytes = max_ingestion_queue_bytes
-
-        @running = Concurrent::AtomicBoolean.new(false)
       end
 
       def subscribe!(index_name:)
-        @index_name = index_name
+        Utility::Logger.info("Starting a new consumer for #{index_name} index")
 
-        start_loop!
+        @index_name = index_name
+        start_timer_task!
+        start_thread_pool!
       end
 
       def running?
-        # @TODO check if a loop thread is alive
-        pool.running? && @running.true?
+        pool&.running? && timer_task&.running?
       end
 
       def shutdown!
         Utility::Logger.info("Shutting down consumer for #{@index_name} index")
-        @running.make_false
+
+        timer_task.shutdown
         pool.shutdown
         pool.wait_for_termination(@termination_timeout)
-        # reset pool
-        @pool = nil
+        reset_pool!
       end
 
       private
 
-      def start_loop!
-        Utility::Logger.info("Starting a new consumer for #{@index_name} index")
-
-        Thread.new do
-          # assign a name to the thread
-          # see @TODO in #self.running?
-          Thread.current[:name] = "consumer-group-#{@index_name}"
-
-          loop do
-            if @running.false?
-              Utility::Logger.info('Shutting down the loop')
-              break
-            end
-
-            sleep(@poll_interval)
-            Utility::Logger.debug('Getting registered connectors')
-
-            connectors = ready_for_sync_connectors
-            next unless connectors.any?
-
-            Utility::Logger.debug("Number of available connectors: #{connectors.size}")
-
-            # @TODO It is assumed that @index_name is used to retrive pending jobs.
-            # This will be discussed after 8.6 release
-            pending_jobs = Core::ConnectorJob.pending_jobs(connectors_ids: connectors.keys)
-            Utility::Logger.info("Number of pending jobs: #{pending_jobs.size}")
-
-            pending_jobs.each do |job|
-              connector_settings = connectors[job.connector_id]
-
-              pool.post do
-                Utility::Logger.info("Connector #{connector_settings.formatted} picked up the job #{job.id}")
-                Core::ElasticConnectorActions.ensure_content_index_exists(connector_settings.index_name)
-                job_runner = Core::SyncJobRunner.new(
-                  connector_settings,
-                  job,
-                  @max_ingestion_queue_size,
-                  @max_ingestion_queue_bytes
-                )
-                job_runner.execute
-              rescue Core::JobAlreadyRunningError
-                Utility::Logger.info("Sync job for #{connector_settings.formatted} is already running, skipping.")
-              rescue Core::ConnectorVersionChangedError => e
-                Utility::Logger.info("Could not start the job because #{connector_settings.formatted} has been updated externally. Message: #{e.message}")
-              rescue StandardError => e
-                Utility::ExceptionTracking.log_exception(e, "Sync job for #{connector_settings.formatted} failed due to unexpected error.")
-              end
-            end
-          rescue StandardError => e
-            Utility::ExceptionTracking.log_exception(e, 'The consumer group failed')
-          end
-        end
+      attr_reader :pool, :timer_task
 
-        @running.make_true
+      def start_timer_task!
+        @timer_task = Concurrent::TimerTask.execute(execution_interval: @poll_interval, run_now: true) { execute }
       end
 
-      def pool
-        @pool ||= Concurrent::ThreadPoolExecutor.new(
+      def start_thread_pool!
+        @pool = Concurrent::ThreadPoolExecutor.new(
           min_threads: @min_threads,
           max_threads: @max_threads,
           max_queue: @max_queue,
@@ -122,6 +75,51 @@ module Core
         )
       end
 
+      def reset_pool!
+        @pool = nil
+      end
+
+      def execute
+        Utility::Logger.debug('Getting registered connectors')
+
+        connectors = ready_for_sync_connectors
+        return unless connectors.any?
+
+        Utility::Logger.debug("Number of available connectors: #{connectors.size}")
+
+        # @TODO It is assumed that @index_name is used to retrive pending jobs.
+        # This will be discussed after 8.6 release
+        pending_jobs = Core::ConnectorJob.pending_jobs(connectors_ids: connectors.keys)
+        Utility::Logger.info("Number of pending jobs: #{pending_jobs.size}")
+
+        pending_jobs.each do |job|
+          connector_settings = connectors[job.connector_id]
+          execute_job(job, connector_settings)
+        end
+      rescue StandardError => e
+        Utility::ExceptionTracking.log_exception(e, 'The consumer group failed')
+      end
+
+      def execute_job(job, connector_settings)
+        pool.post do
+          Utility::Logger.info("Connector #{connector_settings.formatted} picked up the job #{job.id}")
+          Core::ElasticConnectorActions.ensure_content_index_exists(connector_settings.index_name)
+          job_runner = Core::SyncJobRunner.new(
+            connector_settings,
+            job,
+            @max_ingestion_queue_size,
+            @max_ingestion_queue_bytes
+          )
+          job_runner.execute
+        rescue Core::JobAlreadyRunningError
+          Utility::Logger.info("Sync job for #{connector_settings.formatted} is already running, skipping.")
+        rescue Core::ConnectorVersionChangedError => e
+          Utility::Logger.info("Could not start the job because #{connector_settings.formatted} has been updated externally. Message: #{e.message}")
+        rescue StandardError => e
+          Utility::ExceptionTracking.log_exception(e, "Sync job for #{connector_settings.formatted} failed due to unexpected error.")
+        end
+      end
+
       def ready_for_sync_connectors
         @scheduler.connector_settings
           .select(&:ready_for_sync?)

data/lib/core/jobs/producer.rb

@@ -6,6 +6,9 @@
 
 # frozen_string_literal: true
 
+require 'core/connector_settings'
+require 'core/elastic_connector_actions'
+
 module Core
   module Jobs
     class Producer

data/lib/core/scheduler.rb

@@ -44,17 +44,14 @@ module Core
           end
         end
       rescue *Utility::AUTHORIZATION_ERRORS => e
-        Utility::ExceptionTracking.log_exception(e, 'Could not retrieve connectors settings due to authorization error.')
+        log_authorization_error(e)
       rescue StandardError => e
-        Utility::ExceptionTracking.log_exception(e, 'Sync failed due to unexpected error.')
+        log_standard_error(e)
       ensure
         if @is_shutting_down
           break
         end
-        if @poll_interval > 0 && !@is_shutting_down
-          Utility::Logger.debug("Sleeping for #{@poll_interval} seconds in #{self.class}.")
-          sleep(@poll_interval)
-        end
+        sleep_for_poll_interval
       end
     end
 
@@ -78,56 +75,12 @@ module Core
       end
 
       # Sync when sync_now flag is true for the connector
-      if connector_settings[:sync_now] == true
+      if connector_settings.sync_now?
         Utility::Logger.info("#{connector_settings.formatted.capitalize} is manually triggered to sync now.")
         return true
       end
 
-      # Don't sync if sync is explicitly disabled
-      scheduling_settings = connector_settings.scheduling_settings
-      unless scheduling_settings.present? && scheduling_settings[:enabled] == true
-        Utility::Logger.debug("#{connector_settings.formatted.capitalize} scheduling is disabled.")
-        return false
-      end
-
-      # We want to sync when sync never actually happened
-      last_synced = connector_settings[:last_synced]
-      if last_synced.nil? || last_synced.empty?
-        Utility::Logger.info("#{connector_settings.formatted.capitalize} has never synced yet, running initial sync.")
-        return true
-      end
-
-      current_schedule = scheduling_settings[:interval]
-
-      # Don't sync if there is no actual scheduling interval
-      if current_schedule.nil? || current_schedule.empty?
-        Utility::Logger.warn("No sync schedule configured for #{connector_settings.formatted}.")
-        return false
-      end
-
-      current_schedule = begin
-        Utility::Cron.quartz_to_crontab(current_schedule)
-      rescue StandardError => e
-        Utility::ExceptionTracking.log_exception(e, "Unable to convert quartz (#{current_schedule}) to crontab.")
-        return false
-      end
-      cron_parser = Fugit::Cron.parse(current_schedule)
-
-      # Don't sync if the scheduling interval is non-parsable
-      unless cron_parser
-        Utility::Logger.error("Unable to parse sync schedule for #{connector_settings.formatted}: expression #{current_schedule} is not a valid Quartz Cron definition.")
-        return false
-      end
-
-      next_trigger_time = cron_parser.next_time(Time.parse(last_synced))
-
-      # Sync if next trigger for the connector is in past
-      if next_trigger_time < Time.now
-        Utility::Logger.info("#{connector_settings.formatted.capitalize} sync is triggered by cron schedule #{current_schedule}.")
-        return true
-      end
-
-      false
+      schedule_triggered?(connector_settings.full_sync_scheduling, connector_settings.formatted)
     end
 
     def heartbeat_triggered?(connector_settings)
@@ -148,6 +101,12 @@ module Core
     end
 
     def filtering_validation_triggered?(connector_settings)
+      unless connector_settings.any_filtering_feature_enabled?
+        Utility::Logger.debug("#{connector_settings.formatted} all filtering features are disabled. Skip filtering validation.")
+
+        return false
+      end
+
       filtering = connector_settings.filtering
 
       unless filtering.present?
@@ -189,5 +148,61 @@ module Core
         false
       end
     end
+
+    def schedule_triggered?(scheduling_settings, identifier)
+      # Don't sync if sync is explicitly disabled
+      unless scheduling_settings.present? && scheduling_settings[:enabled] == true
+        Utility::Logger.debug("#{identifier.capitalize} scheduling is disabled.")
+        return false
+      end
+
+      current_schedule = scheduling_settings[:interval]
+
+      # Don't sync if there is no actual scheduling interval
+      if current_schedule.nil? || current_schedule.empty?
+        Utility::Logger.warn("No sync schedule configured for #{identifier}.")
+        return false
+      end
+
+      current_schedule =
+        begin
+          Utility::Cron.quartz_to_crontab(current_schedule)
+        rescue StandardError => e
+          Utility::ExceptionTracking.log_exception(e, "Unable to convert quartz (#{current_schedule}) to crontab.")
+          return false
+        end
+      cron_parser = Fugit::Cron.parse(current_schedule)
+
+      # Don't sync if the scheduling interval is non-parsable
+      unless cron_parser
+        Utility::Logger.error("Unable to parse sync schedule for #{identifier}: expression #{current_schedule} is not a valid Quartz Cron definition.")
+        return false
+      end
+
+      next_trigger_time = cron_parser.next_time(Time.now)
+
+      # Sync if next trigger happens before the next poll
+      if next_trigger_time <= Time.now + @poll_interval
+        Utility::Logger.info("#{identifier.capitalize} sync is triggered by cron schedule #{current_schedule}.")
+        return true
+      end
+
+      false
+    end
+
+    def sleep_for_poll_interval
+      if @poll_interval > 0 && !@is_shutting_down
+        Utility::Logger.debug("Sleeping for #{@poll_interval} seconds in #{self.class}.")
+        sleep(@poll_interval)
+      end
+    end
+
+    def log_authorization_error(e)
+      Utility::ExceptionTracking.log_exception(e, 'Could not retrieve connectors settings due to authorization error.')
+    end
+
+    def log_standard_error(e)
+      Utility::ExceptionTracking.log_exception(e, 'Sync failed due to unexpected error.')
+    end
   end
 end