connectors_service 8.6.0.4.pre.20221104T200814Z → 8.6.0.4.pre.20221116T024501Z

data/lib/core/sync_job_runner.rb

@@ -8,7 +8,9 @@
 
 require 'connectors/connector_status'
 require 'connectors/registry'
-require 'core/output_sink'
+require 'core/filtering/post_process_engine'
+require 'core/ingestion'
+require 'core/filtering/validation_status'
 require 'utility'
 
 module Core
@@ -19,16 +21,21 @@ module Core
   end
 
   class SyncJobRunner
-    def initialize(connector_settings)
+    JOB_REPORTING_INTERVAL = 10
+
+    def initialize(connector_settings, job)
       @connector_settings = connector_settings
-      @sink = Core::OutputSink::EsSink.new(connector_settings.index_name, @connector_settings.request_pipeline)
+      @sink = Core::Ingestion::EsSink.new(connector_settings.index_name, @connector_settings.request_pipeline)
       @connector_class = Connectors::REGISTRY.connector_class(connector_settings.service_type)
       @sync_finished = false
+      @sync_error = nil
       @status = {
         :indexed_document_count => 0,
         :deleted_document_count => 0,
+        :indexed_document_volume => 0,
         :error => nil
       }
+      @job = job
     end
 
     def execute
@@ -41,8 +48,17 @@ module Core
     def do_sync!
       Utility::Logger.info("Claiming a sync job for connector #{@connector_settings.id}.")
 
-      job_description = ElasticConnectorActions.claim_job(@connector_settings.id)
-      job_id = job_description['_id']
+      # connector service doesn't support multiple jobs running simultaneously
+      raise Core::JobAlreadyRunningError.new(@connector_settings.id) if @connector_settings.running?
+
+      Core::ElasticConnectorActions.update_connector_last_sync_status(@connector_settings.id, Connectors::SyncStatus::IN_PROGRESS)
+
+      # claim the job
+      @job.make_running!
+
+      job_description = @job.es_source
+      job_id = @job.id
+      job_description['_id'] = job_id
 
       unless job_id.present?
         Utility::Logger.error("Failed to claim the job for #{@connector_settings.id}. Please check the logs for the cause of this error.")
@@ -52,6 +68,10 @@ module Core
       begin
         Utility::Logger.debug("Successfully claimed job for connector #{@connector_settings.id}.")
 
+        Utility::Logger.info("Checking active filtering for sync job #{job_id} for connector #{@connector_settings.id}.")
+        validate_filtering(job_description.dig(:connector, :filtering))
+        Utility::Logger.debug("Active filtering for sync job #{job_id} for connector #{@connector_settings.id} is valid.")
+
         connector_instance = Connectors::REGISTRY.connector(@connector_settings.service_type, @connector_settings.configuration, job_description: job_description)
 
         connector_instance.do_health_check!
@@ -61,11 +81,21 @@ module Core
 
         Utility::Logger.debug("#{existing_ids.size} documents are present in index #{@connector_settings.index_name}.")
 
+        post_processing_engine = Core::Filtering::PostProcessEngine.new(job_description)
+        reporting_cycle_start = Time.now
+        Utility::Logger.info('Yielding documents...')
         connector_instance.yield_documents do |document|
           document = add_ingest_metadata(document)
-          @sink.ingest(document)
-          incoming_ids << document['id']
-          @status[:indexed_document_count] += 1
+          post_process_result = post_processing_engine.process(document)
+          if post_process_result.is_include?
+            @sink.ingest(document)
+            incoming_ids << document['id']
+          end
+
+          if Time.now - reporting_cycle_start >= JOB_REPORTING_INTERVAL
+            ElasticConnectorActions.update_sync(job_id, @sink.ingestion_stats.merge(:metadata => connector_instance.metadata))
+            reporting_cycle_start = Time.now
+          end
         end
 
         ids_to_delete = existing_ids - incoming_ids.uniq
@@ -74,7 +104,11 @@ module Core
 
         ids_to_delete.each do |id|
           @sink.delete(id)
-          @status[:deleted_document_count] += 1
+
+          if Time.now - reporting_cycle_start >= JOB_REPORTING_INTERVAL
+            ElasticConnectorActions.update_sync(job_id, @sink.ingestion_stats.merge(:metadata => connector_instance.metadata))
+            reporting_cycle_start = Time.now
+          end
         end
 
         @sink.flush
@@ -83,22 +117,34 @@ module Core
         # occurred as most of the time the main execution thread is interrupted and we miss this Signal/Exception here
         @sync_finished = true
       rescue StandardError => e
-        @status[:error] = e.message
+        @sync_error = e.message
         Utility::ExceptionTracking.log_exception(e)
-        ElasticConnectorActions.update_connector_status(@connector_settings.id, Connectors::ConnectorStatus::ERROR, Utility::Logger.abbreviated_message(e.message))
       ensure
+        stats = @sink.ingestion_stats
+
+        Utility::Logger.debug("Sync stats are: #{stats}")
+
+        @status[:indexed_document_count] = stats[:indexed_document_count]
+        @status[:deleted_document_count] = stats[:deleted_document_count]
+        @status[:indexed_document_volume] = stats[:indexed_document_volume]
+
         Utility::Logger.info("Upserted #{@status[:indexed_document_count]} documents into #{@connector_settings.index_name}.")
         Utility::Logger.info("Deleted #{@status[:deleted_document_count]} documents into #{@connector_settings.index_name}.")
 
         # Make sure to not override a previous error message
-        if !@sync_finished && @status[:error].nil?
-          @status[:error] = 'Sync thread didn\'t finish execution. Check connector logs for more details.'
+        if !@sync_finished && @sync_error.nil?
+          @sync_error = 'Sync thread didn\'t finish execution. Check connector logs for more details.'
+        end
+
+        unless connector_instance.nil?
+          metadata = @sink.ingestion_stats.merge(:metadata => connector_instance.metadata)
+          metadata[:total_document_count] = ElasticConnectorActions.document_count(@connector_settings.index_name)
         end
 
-        ElasticConnectorActions.complete_sync(@connector_settings.id, job_id, @status.dup)
+        ElasticConnectorActions.complete_sync(@connector_settings.id, job_id, metadata, @sync_error)
 
-        if @status[:error]
-          Utility::Logger.info("Failed to sync for connector #{@connector_settings.id} with error '#{@status[:error]}'.")
+        if @sync_error
+          Utility::Logger.info("Failed to sync for connector #{@connector_settings.id} with error '#{@sync_error}'.")
         else
           Utility::Logger.info("Successfully synced for connector #{@connector_settings.id}.")
         end
@@ -119,5 +165,15 @@ module Core
 
       raise IncompatibleConfigurableFieldsError.new(@connector_class.service_type, expected_fields, actual_fields) if expected_fields != actual_fields
     end
+
+    def validate_filtering(filtering)
+      validation_result = @connector_class.validate_filtering(filtering)
+
+      wrong_state_error = Utility::InvalidFilterConfigError.new("Active filtering is not in valid state (current state: #{validation_result[:state]}) for connector #{@connector_settings.id}. Please check active filtering in connectors index.")
+      raise wrong_state_error if validation_result[:state] != Core::Filtering::ValidationStatus::VALID
+
+      errors_present_error = Utility::InvalidFilterConfigError.new("Active filtering is in valid state, but errors were detected (errors: #{validation_result[:errors]}) for connector #{@connector_settings.id}. Please check active filtering in connectors index.")
+      raise errors_present_error if validation_result[:errors].present?
+    end
   end
 end

data/lib/core.rb

@@ -7,10 +7,14 @@
 # frozen_string_literal: true
 
 require 'core/configuration'
+require 'core/connector_job'
 require 'core/connector_settings'
 require 'core/elastic_connector_actions'
+require 'core/filtering'
 require 'core/heartbeat'
 require 'core/scheduler'
 require 'core/single_scheduler'
 require 'core/native_scheduler'
 require 'core/sync_job_runner'
+require 'core/jobs/producer'
+require 'core/jobs/consumer'

data/lib/utility/bulk_queue.rb

@@ -0,0 +1,85 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+require 'json'
+
+module Utility
+  class BulkQueue
+    class QueueOverflowError < StandardError; end
+
+    # 500 items or 5MB
+    def initialize(operation_count_threshold = 500, size_threshold = 5 * 1024 * 1024)
+      @operation_count_threshold = operation_count_threshold.freeze
+      @size_threshold = size_threshold.freeze
+
+      @buffer = ''
+
+      @current_operation_count = 0
+
+      @current_buffer_size = 0
+      @current_data_size = 0
+    end
+
+    def pop_all
+      result = @buffer
+
+      reset
+
+      result
+    end
+
+    def add(operation, payload = nil)
+      raise QueueOverflowError unless will_fit?(operation, payload)
+
+      operation_size = get_size(operation)
+      payload_size = get_size(payload)
+
+      @current_operation_count += 1
+      @current_buffer_size += operation_size
+      @current_buffer_size += payload_size
+      @current_data_size += payload_size
+
+      @buffer << operation
+      @buffer << "\n"
+
+      if payload
+        @buffer << payload
+        @buffer << "\n"
+      end
+    end
+
+    def will_fit?(operation, payload = nil)
+      return false if @current_operation_count + 1 > @operation_count_threshold
+
+      operation_size = get_size(operation)
+      payload_size = get_size(payload)
+
+      @current_buffer_size + operation_size + payload_size < @size_threshold
+    end
+
+    def current_stats
+      {
+        :current_operation_count => @current_operation_count,
+        :current_buffer_size => @current_buffer_size
+      }
+    end
+
+    private
+
+    def get_size(str)
+      return 0 unless str
+      str.bytesize
+    end
+
+    def reset
+      @current_operation_count = 0
+      @current_buffer_size = 0
+      @current_data_size = 0
+
+      @buffer = ''
+    end
+  end
+end

data/lib/utility/constants.rb

@@ -16,5 +16,7 @@ module Utility
     JOB_INDEX = '.elastic-connectors-sync-jobs'
     CONTENT_INDEX_PREFIX = 'search-'
     CRAWLER_SERVICE_TYPE = 'elastic-crawler'
+    FILTERING_RULES_FEATURE = 'filtering_rules'
+    FILTERING_ADVANCED_FEATURE = 'filtering_advanced_config'
   end
 end

data/lib/utility/error_monitor.rb

@@ -0,0 +1,108 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'time'
+require 'utility/errors'
+require 'utility/exception_tracking'
+
+module Utility
+  class ErrorMonitor
+    class MonitoringError < StandardError
+      attr_accessor :tripped_by
+
+      def initialize(message = nil, tripped_by: nil)
+        super("#{message}#{tripped_by.present? ? " Tripped by - #{tripped_by.class}: #{tripped_by.message}" : ''}")
+        @tripped_by = tripped_by
+      end
+    end
+
+    class MaxSuccessiveErrorsExceededError < MonitoringError; end
+    class MaxErrorsExceededError < MonitoringError; end
+    class MaxErrorsInWindowExceededError < MonitoringError; end
+
+    attr_reader :total_error_count, :success_count, :consecutive_error_count, :error_queue
+
+    def initialize(
+      max_errors: 1000,
+      max_consecutive_errors: 10,
+      max_error_ratio: 0.15,
+      window_size: 100,
+      error_queue_size: 20
+    )
+      @max_errors = max_errors
+      @max_consecutive_errors = max_consecutive_errors
+      @max_error_ratio = max_error_ratio
+      @window_size = window_size
+      @total_error_count = 0
+      @success_count = 0
+      @consecutive_error_count = 0
+      @window_errors = Array.new(window_size) { false }
+      @window_index = 0
+      @last_error = nil
+      @error_queue_size = error_queue_size
+      @error_queue = []
+    end
+
+    def note_success
+      @consecutive_error_count = 0
+      @success_count += 1
+      increment_window_index
+    end
+
+    def note_error(error, id: Time.now.to_i)
+      stack_trace = Utility::ExceptionTracking.generate_stack_trace(error)
+      error_message = Utility::ExceptionTracking.generate_error_message(error, nil, nil)
+      Utility::Logger.debug("Message id: #{id} - #{error_message}\n#{stack_trace}")
+      @total_error_count += 1
+      @consecutive_error_count += 1
+      @window_errors[@window_index] = true
+      @error_queue << DocumentError.new(error.class.name, error_message, stack_trace, id)
+      @error_queue = @error_queue.drop(1) if @error_queue.size > @error_queue_size
+      increment_window_index
+      @last_error = error
+
+      raise_if_necessary
+    end
+
+    def finalize
+      total_documents = @total_error_count + @success_count
+      if total_documents > 0 && @total_error_count.to_f / total_documents > @max_error_ratio
+        raise_with_last_cause(MaxErrorsInWindowExceededError.new("There were #{@total_error_count} errors out of #{total_documents} total documents", :tripped_by => @last_error))
+      end
+    end
+
+    private
+
+    def raise_if_necessary
+      error =
+        if @consecutive_error_count > @max_consecutive_errors
+          MaxSuccessiveErrorsExceededError.new("Exceeded maximum consecutive errors - saw #{@consecutive_error_count} errors in a row.", :tripped_by => @last_error)
+        elsif @total_error_count > @max_errors
+          MaxErrorsExceededError.new("Exceeded maximum number of errors - saw #{@total_error_count} errors in total.", :tripped_by => @last_error)
+        elsif @window_size > 0 && num_errors_in_window / @window_size > @max_error_ratio
+          MaxErrorsInWindowExceededError.new("Exceeded maximum error ratio of #{@max_error_ratio}. Of the last #{@window_size} documents, #{num_errors_in_window} had errors", :tripped_by => @last_error)
+        end
+
+      raise_with_last_cause(error) if error
+    end
+
+    def num_errors_in_window
+      @window_errors.count(&:itself).to_f
+    end
+
+    def increment_window_index
+      @window_index = (@window_index + 1) % @window_size
+    end
+
+    def raise_with_last_cause(error)
+      raise @last_error
+    rescue StandardError
+      raise error
+    end
+  end
+end

data/lib/utility/errors.rb

@@ -60,18 +60,6 @@ module Utility
   class JobDocumentLimitError < StandardError; end
   class JobClaimingError < StandardError; end
 
-  class MonitoringError < StandardError
-    attr_accessor :tripped_by
-
-    def initialize(message = nil, tripped_by: nil)
-      super("#{message}#{tripped_by.present? ? " Tripped by - #{tripped_by.class}: #{tripped_by.message}" : ''}")
-      @tripped_by = tripped_by
-    end
-  end
-  class MaxSuccessiveErrorsExceededError < MonitoringError; end
-  class MaxErrorsExceededError < MonitoringError; end
-  class MaxErrorsInWindowExceededError < MonitoringError; end
-
   class JobSyncNotPossibleYetError < StandardError
     attr_accessor :sync_will_be_possible_at
 

data/lib/utility/filtering.rb

@@ -0,0 +1,22 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+module Utility
+  class Filtering
+    class << self
+      def extract_filter(filtering)
+        return {} unless filtering.present?
+
+        # assume for now, that first object in filtering array or a filter object itself is the only filtering object
+        filter = filtering.is_a?(Array) ? filtering.first : filtering
+
+        filter.present? ? filter : {}
+      end
+    end
+  end
+end

data/lib/utility/logger.rb

@@ -23,7 +23,7 @@ module Utility
       end
 
       def logger
-        @logger ||= Settings[:ecs_logging] ? EcsLogging::Logger.new(STDOUT) : ::Logger.new(STDOUT)
+        @logger ||= defined?(::Settings) && ::Settings[:ecs_logging] ? EcsLogging::Logger.new(STDOUT) : ::Logger.new(STDOUT)
       end
 
       SUPPORTED_LOG_LEVELS.each do |level|

data/lib/utility.rb

@@ -4,14 +4,21 @@
 # you may not use this file except in compliance with the Elastic License.
 #
 
+# !!!!!!!!
+# IF YOU EDIT THIS FILE, YOU MUST EDIT THE `connectors_utility.gemspec`
+require 'utility/bulk_queue'
+require 'utility/common'
 require 'utility/constants'
 require 'utility/cron'
-require 'utility/common'
+require 'utility/elasticsearch/index/mappings'
+require 'utility/elasticsearch/index/text_analysis_settings'
+require 'utility/environment'
+require 'utility/error_monitor'
 require 'utility/errors'
+require 'utility/filtering'
 require 'utility/es_client'
-require 'utility/environment'
 require 'utility/exception_tracking'
 require 'utility/extension_mapping_util'
 require 'utility/logger'
-require 'utility/elasticsearch/index/mappings'
-require 'utility/elasticsearch/index/text_analysis_settings'
+# IF YOU EDIT THIS FILE, YOU MUST EDIT THE `connectors_utility.gemspec`
+# !!!!!!!!

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: connectors_service
 version: !ruby/object:Gem::Version
-  version: 8.6.0.4.pre.20221104T200814Z
+  version: 8.6.0.4.pre.20221116T024501Z
 platform: ruby
 authors:
 - Elastic
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-11-04 00:00:00.000000000 Z
+date: 2022-11-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -396,34 +396,49 @@ files:
 - lib/app/version.rb
 - lib/connectors.rb
 - lib/connectors/base/adapter.rb
+- lib/connectors/base/advanced_snippet_against_schema_validator.rb
+- lib/connectors/base/advanced_snippet_validator.rb
 - lib/connectors/base/connector.rb
 - lib/connectors/base/custom_client.rb
+- lib/connectors/base/simple_rules_parser.rb
 - lib/connectors/connector_status.rb
 - lib/connectors/crawler/scheduler.rb
 - lib/connectors/example/attachments/first_attachment.txt
 - lib/connectors/example/attachments/second_attachment.txt
 - lib/connectors/example/attachments/third_attachment.txt
 - lib/connectors/example/connector.rb
+- lib/connectors/example/example_advanced_snippet_validator.rb
 - lib/connectors/gitlab/adapter.rb
 - lib/connectors/gitlab/connector.rb
 - lib/connectors/gitlab/custom_client.rb
 - lib/connectors/gitlab/extractor.rb
+- lib/connectors/gitlab/gitlab_advanced_snippet_validator.rb
 - lib/connectors/mongodb/connector.rb
+- lib/connectors/mongodb/mongo_advanced_snippet_against_schema_validator.rb
+- lib/connectors/mongodb/mongo_advanced_snippet_schema.rb
+- lib/connectors/mongodb/mongo_rules_parser.rb
 - lib/connectors/registry.rb
 - lib/connectors/sync_status.rb
+- lib/connectors/tolerable_error_helper.rb
 - lib/connectors_service.rb
 - lib/connectors_utility.rb
 - lib/core.rb
 - lib/core/configuration.rb
+- lib/core/connector_job.rb
 - lib/core/connector_settings.rb
 - lib/core/elastic_connector_actions.rb
+- lib/core/filtering.rb
+- lib/core/filtering/post_process_engine.rb
+- lib/core/filtering/post_process_result.rb
+- lib/core/filtering/simple_rule.rb
+- lib/core/filtering/validation_job_runner.rb
+- lib/core/filtering/validation_status.rb
 - lib/core/heartbeat.rb
+- lib/core/ingestion.rb
+- lib/core/ingestion/es_sink.rb
+- lib/core/jobs/consumer.rb
+- lib/core/jobs/producer.rb
 - lib/core/native_scheduler.rb
-- lib/core/output_sink.rb
-- lib/core/output_sink/base_sink.rb
-- lib/core/output_sink/combined_sink.rb
-- lib/core/output_sink/console_sink.rb
-- lib/core/output_sink/es_sink.rb
 - lib/core/scheduler.rb
 - lib/core/single_scheduler.rb
 - lib/core/sync_job_runner.rb
@@ -432,6 +447,7 @@ files:
 - lib/stubs/connectors/stats.rb
 - lib/stubs/service_type.rb
 - lib/utility.rb
+- lib/utility/bulk_queue.rb
 - lib/utility/common.rb
 - lib/utility/constants.rb
 - lib/utility/cron.rb
@@ -439,10 +455,12 @@ files:
 - lib/utility/elasticsearch/index/mappings.rb
 - lib/utility/elasticsearch/index/text_analysis_settings.rb
 - lib/utility/environment.rb
+- lib/utility/error_monitor.rb
 - lib/utility/errors.rb
 - lib/utility/es_client.rb
 - lib/utility/exception_tracking.rb
 - lib/utility/extension_mapping_util.rb
+- lib/utility/filtering.rb
 - lib/utility/logger.rb
 - lib/utility/middleware/basic_auth.rb
 - lib/utility/middleware/bearer_auth.rb

data/lib/core/output_sink/base_sink.rb

@@ -1,33 +0,0 @@
-#
-# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
-# or more contributor license agreements. Licensed under the Elastic License;
-# you may not use this file except in compliance with the Elastic License.
-#
-
-# frozen_string_literal: true
-
-module Core
-  module OutputSink
-    class BaseSink
-      def ingest(_document)
-        raise 'not implemented'
-      end
-
-      def ingest_multiple(_documents)
-        raise 'not implemented'
-      end
-
-      def delete(_id)
-        raise 'not implemented'
-      end
-
-      def delete_multiple(_ids)
-        raise 'not implemented'
-      end
-
-      def flush(_size: nil)
-        raise 'not implemented'
-      end
-    end
-  end
-end

data/lib/core/output_sink/combined_sink.rb

@@ -1,38 +0,0 @@
-#
-# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
-# or more contributor license agreements. Licensed under the Elastic License;
-# you may not use this file except in compliance with the Elastic License.
-#
-
-# frozen_string_literal: true
-
-require 'core/output_sink/base_sink'
-require 'utility/logger'
-
-module Core::OutputSink
-  class CombinedSink < Core::OutputSink::BaseSink
-    def initialize(sinks = [])
-      @sinks = sinks
-    end
-
-    def ingest(document)
-      @sinks.each { |sink| sink.ingest(document) }
-    end
-
-    def flush(size: nil)
-      @sinks.each { |sink| sink.flush(size: size) }
-    end
-
-    def ingest_multiple(documents)
-      @sinks.each { |sink| sink.ingest_multiple(documents) }
-    end
-
-    def delete(id)
-      @sinks.each { |sink| sink.delete(id) }
-    end
-
-    def delete_multiple(ids)
-      @sinks.each { |sink| sink.delete_multiple(ids) }
-    end
-  end
-end

data/lib/core/output_sink/console_sink.rb

@@ -1,51 +0,0 @@
-#
-# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
-# or more contributor license agreements. Licensed under the Elastic License;
-# you may not use this file except in compliance with the Elastic License.
-#
-
-# frozen_string_literal: true
-
-require 'core/output_sink'
-require 'utility/logger'
-
-module Core::OutputSink
-  class ConsoleSink < Core::OutputSink::BaseSink
-    def ingest(document)
-      print_header 'Got a single document:'
-      puts document
-    end
-
-    def flush(size: nil)
-      print_header 'Flushing'
-      puts "Flush size: #{size}"
-    end
-
-    def ingest_multiple(documents)
-      print_header 'Got multiple documents:'
-      puts documents
-    end
-
-    def delete(id)
-      print_header "Deleting single id: #{id}"
-      puts id
-    end
-
-    def delete_multiple(ids)
-      print_header "Deleting several ids: #{ids}"
-      puts ids
-    end
-
-    private
-
-    def print_delim
-      puts '----------------------------------------------------'
-    end
-
-    def print_header(header)
-      print_delim
-      puts header
-      print_delim
-    end
-  end
-end