connectors_service 8.6.0.4.pre.20221116T024501Z → 8.6.0.4

data/lib/core/jobs/consumer.rb

@@ -1,114 +0,0 @@
-#
-# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
-# or more contributor license agreements. Licensed under the Elastic License;
-# you may not use this file except in compliance with the Elastic License.
-#
-
-# frozen_string_literal: true
-
-module Core
-  module Jobs
-    class Consumer
-      def initialize(scheduler:, poll_interval: 3, termination_timeout: 60, min_threads: 1, max_threads: 5, max_queue: 100, idle_time: 5)
-        @scheduler = scheduler
-        @poll_interval = poll_interval
-        @termination_timeout = termination_timeout
-        @min_threads = min_threads
-        @max_threads = max_threads
-        @max_queue = max_queue
-        @idle_time = idle_time
-
-        @running = Concurrent::AtomicBoolean.new(false)
-      end
-
-      def subscribe!(index_name:)
-        @index_name = index_name
-
-        start_loop!
-      end
-
-      def running?
-        # @TODO check if a loop thread is alive
-        pool.running? && @running.true?
-      end
-
-      def shutdown!
-        Utility::Logger.info("Shutting down consumer for #{@index_name} index")
-        @running.make_false
-        pool.shutdown
-        pool.wait_for_termination(@termination_timeout)
-        # reset pool
-        @pool = nil
-      end
-
-      private
-
-      def start_loop!
-        Utility::Logger.info("Starting a new consumer for #{@index_name} index")
-
-        Thread.new do
-          # assign a name to the thread
-          # see @TODO in #self.running?
-          Thread.current[:name] = "consumer-group-#{@index_name}"
-
-          loop do
-            if @running.false?
-              Utility::Logger.info('Shutting down the loop')
-              break
-            end
-
-            sleep(@poll_interval)
-            Utility::Logger.debug('Getting registered connectors')
-
-            connectors = ready_for_sync_connectors
-            next unless connectors.any?
-
-            Utility::Logger.debug("Number of available connectors: #{connectors.size}")
-
-            # @TODO It is assumed that @index_name is used to retrive pending jobs.
-            # This will be discussed after 8.6 release
-            pending_jobs = Core::ConnectorJob.pending_jobs(connectors_ids: connectors.keys)
-            Utility::Logger.info("Number of pending jobs: #{pending_jobs.size}")
-
-            pending_jobs.each do |job|
-              connector_settings = connectors[job.connector_id]
-
-              pool.post do
-                Utility::Logger.info("Connector #{connector_settings.formatted} picked up the job #{job.id}")
-                Core::ElasticConnectorActions.ensure_content_index_exists(connector_settings.index_name)
-                job_runner = Core::SyncJobRunner.new(connector_settings, job)
-                job_runner.execute
-              rescue Core::JobAlreadyRunningError
-                Utility::Logger.info("Sync job for #{connector_settings.formatted} is already running, skipping.")
-              rescue Core::ConnectorVersionChangedError => e
-                Utility::Logger.info("Could not start the job because #{connector_settings.formatted} has been updated externally. Message: #{e.message}")
-              rescue StandardError => e
-                Utility::ExceptionTracking.log_exception(e, "Sync job for #{connector_settings.formatted} failed due to unexpected error.")
-              end
-            end
-          rescue StandardError => e
-            Utility::ExceptionTracking.log_exception(e, 'The consumer group failed')
-          end
-        end
-
-        @running.make_true
-      end
-
-      def pool
-        @pool ||= Concurrent::ThreadPoolExecutor.new(
-          min_threads: @min_threads,
-          max_threads: @max_threads,
-          max_queue: @max_queue,
-          fallback_policy: :abort,
-          idletime: @idle_time
-        )
-      end
-
-      def ready_for_sync_connectors
-        @scheduler.connector_settings
-          .select(&:ready_for_sync?)
-          .inject({}) { |memo, cs| memo.merge(cs.id => cs) }
-      end
-    end
-  end
-end

data/lib/core/jobs/producer.rb

@@ -1,26 +0,0 @@
-#
-# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
-# or more contributor license agreements. Licensed under the Elastic License;
-# you may not use this file except in compliance with the Elastic License.
-#
-
-# frozen_string_literal: true
-
-module Core
-  module Jobs
-    class Producer
-      JOB_TYPES = %i(sync).freeze
-
-      class << self
-        def enqueue_job(job_type:, connector_settings:)
-          raise UnsupportedJobType unless JOB_TYPES.include?(job_type)
-          raise ArgumentError unless connector_settings.kind_of?(ConnectorSettings)
-
-          ElasticConnectorActions.create_job(connector_settings: connector_settings)
-        end
-      end
-    end
-
-    class UnsupportedJobType < StandardError; end
-  end
-end

data/lib/utility/bulk_queue.rb

@@ -1,85 +0,0 @@
-#
-# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
-# or more contributor license agreements. Licensed under the Elastic License;
-# you may not use this file except in compliance with the Elastic License.
-#
-
-require 'json'
-
-module Utility
-  class BulkQueue
-    class QueueOverflowError < StandardError; end
-
-    # 500 items or 5MB
-    def initialize(operation_count_threshold = 500, size_threshold = 5 * 1024 * 1024)
-      @operation_count_threshold = operation_count_threshold.freeze
-      @size_threshold = size_threshold.freeze
-
-      @buffer = ''
-
-      @current_operation_count = 0
-
-      @current_buffer_size = 0
-      @current_data_size = 0
-    end
-
-    def pop_all
-      result = @buffer
-
-      reset
-
-      result
-    end
-
-    def add(operation, payload = nil)
-      raise QueueOverflowError unless will_fit?(operation, payload)
-
-      operation_size = get_size(operation)
-      payload_size = get_size(payload)
-
-      @current_operation_count += 1
-      @current_buffer_size += operation_size
-      @current_buffer_size += payload_size
-      @current_data_size += payload_size
-
-      @buffer << operation
-      @buffer << "\n"
-
-      if payload
-        @buffer << payload
-        @buffer << "\n"
-      end
-    end
-
-    def will_fit?(operation, payload = nil)
-      return false if @current_operation_count + 1 > @operation_count_threshold
-
-      operation_size = get_size(operation)
-      payload_size = get_size(payload)
-
-      @current_buffer_size + operation_size + payload_size < @size_threshold
-    end
-
-    def current_stats
-      {
-        :current_operation_count => @current_operation_count,
-        :current_buffer_size => @current_buffer_size
-      }
-    end
-
-    private
-
-    def get_size(str)
-      return 0 unless str
-      str.bytesize
-    end
-
-    def reset
-      @current_operation_count = 0
-      @current_buffer_size = 0
-      @current_data_size = 0
-
-      @buffer = ''
-    end
-  end
-end

data/lib/utility/error_monitor.rb

@@ -1,108 +0,0 @@
-#
-# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
-# or more contributor license agreements. Licensed under the Elastic License;
-# you may not use this file except in compliance with the Elastic License.
-#
-
-# frozen_string_literal: true
-
-require 'time'
-require 'utility/errors'
-require 'utility/exception_tracking'
-
-module Utility
-  class ErrorMonitor
-    class MonitoringError < StandardError
-      attr_accessor :tripped_by
-
-      def initialize(message = nil, tripped_by: nil)
-        super("#{message}#{tripped_by.present? ? " Tripped by - #{tripped_by.class}: #{tripped_by.message}" : ''}")
-        @tripped_by = tripped_by
-      end
-    end
-
-    class MaxSuccessiveErrorsExceededError < MonitoringError; end
-    class MaxErrorsExceededError < MonitoringError; end
-    class MaxErrorsInWindowExceededError < MonitoringError; end
-
-    attr_reader :total_error_count, :success_count, :consecutive_error_count, :error_queue
-
-    def initialize(
-      max_errors: 1000,
-      max_consecutive_errors: 10,
-      max_error_ratio: 0.15,
-      window_size: 100,
-      error_queue_size: 20
-    )
-      @max_errors = max_errors
-      @max_consecutive_errors = max_consecutive_errors
-      @max_error_ratio = max_error_ratio
-      @window_size = window_size
-      @total_error_count = 0
-      @success_count = 0
-      @consecutive_error_count = 0
-      @window_errors = Array.new(window_size) { false }
-      @window_index = 0
-      @last_error = nil
-      @error_queue_size = error_queue_size
-      @error_queue = []
-    end
-
-    def note_success
-      @consecutive_error_count = 0
-      @success_count += 1
-      increment_window_index
-    end
-
-    def note_error(error, id: Time.now.to_i)
-      stack_trace = Utility::ExceptionTracking.generate_stack_trace(error)
-      error_message = Utility::ExceptionTracking.generate_error_message(error, nil, nil)
-      Utility::Logger.debug("Message id: #{id} - #{error_message}\n#{stack_trace}")
-      @total_error_count += 1
-      @consecutive_error_count += 1
-      @window_errors[@window_index] = true
-      @error_queue << DocumentError.new(error.class.name, error_message, stack_trace, id)
-      @error_queue = @error_queue.drop(1) if @error_queue.size > @error_queue_size
-      increment_window_index
-      @last_error = error
-
-      raise_if_necessary
-    end
-
-    def finalize
-      total_documents = @total_error_count + @success_count
-      if total_documents > 0 && @total_error_count.to_f / total_documents > @max_error_ratio
-        raise_with_last_cause(MaxErrorsInWindowExceededError.new("There were #{@total_error_count} errors out of #{total_documents} total documents", :tripped_by => @last_error))
-      end
-    end
-
-    private
-
-    def raise_if_necessary
-      error =
-        if @consecutive_error_count > @max_consecutive_errors
-          MaxSuccessiveErrorsExceededError.new("Exceeded maximum consecutive errors - saw #{@consecutive_error_count} errors in a row.", :tripped_by => @last_error)
-        elsif @total_error_count > @max_errors
-          MaxErrorsExceededError.new("Exceeded maximum number of errors - saw #{@total_error_count} errors in total.", :tripped_by => @last_error)
-        elsif @window_size > 0 && num_errors_in_window / @window_size > @max_error_ratio
-          MaxErrorsInWindowExceededError.new("Exceeded maximum error ratio of #{@max_error_ratio}. Of the last #{@window_size} documents, #{num_errors_in_window} had errors", :tripped_by => @last_error)
-        end
-
-      raise_with_last_cause(error) if error
-    end
-
-    def num_errors_in_window
-      @window_errors.count(&:itself).to_f
-    end
-
-    def increment_window_index
-      @window_index = (@window_index + 1) % @window_size
-    end
-
-    def raise_with_last_cause(error)
-      raise @last_error
-    rescue StandardError
-      raise error
-    end
-  end
-end

data/lib/utility/filtering.rb

@@ -1,22 +0,0 @@
-#
-# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
-# or more contributor license agreements. Licensed under the Elastic License;
-# you may not use this file except in compliance with the Elastic License.
-#
-
-# frozen_string_literal: true
-
-module Utility
-  class Filtering
-    class << self
-      def extract_filter(filtering)
-        return {} unless filtering.present?
-
-        # assume for now, that first object in filtering array or a filter object itself is the only filtering object
-        filter = filtering.is_a?(Array) ? filtering.first : filtering
-
-        filter.present? ? filter : {}
-      end
-    end
-  end
-end