connectors_service 8.6.0.4.pre.20221104T200814Z → 8.6.0.4.pre.20221114T233727Z

data/lib/core/filtering/validation_job_runner.rb

@@ -0,0 +1,53 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'connectors/connector_status'
+require 'connectors/registry'
+
+module Core
+  module Filtering
+    DEFAULT_DOMAIN = 'DEFAULT'
+
+    class ValidationJobRunner
+      def initialize(connector_settings)
+        @connector_settings = connector_settings
+        @connector_class = Connectors::REGISTRY.connector_class(connector_settings.service_type)
+        @validation_finished = false
+        @status = { :error => nil }
+      end
+
+      def execute
+        Utility::Logger.info("Starting a validation job for connector #{@connector_settings.id}.")
+
+        validation_result = @connector_class.validate_filtering(@connector_settings.filtering)
+
+        # currently only used for connectors -> DEFAULT domain can be assumed (will be changed with the integration of crawler)
+        ElasticConnectorActions.update_filtering_validation(@connector_settings.id, { DEFAULT_DOMAIN => validation_result })
+
+        @validation_finished = true
+      rescue StandardError => e
+        Utility::ExceptionTracking.log_exception(e)
+        validation_failed_result = { :state => Core::Filtering::ValidationStatus::INVALID,
+                                     :errors => [
+                                       { :ids => [], :messages => ['Unknown problem occurred while validating, see logs for details.'] }
+                                     ] }
+        ElasticConnectorActions.update_filtering_validation(@connector_settings.id, { DEFAULT_DOMAIN => validation_failed_result })
+      ensure
+        if !@validation_finished && !@status[:error].present?
+          @status[:error] = 'Validation thread did not finish execution. Check connector logs for more details.'
+        end
+
+        if @status[:error]
+          Utility::Logger.warn("Failed to validate filtering for connector #{@connector_settings.id} with error '#{@status[:error]}'.")
+        else
+          Utility::Logger.info("Successfully validated filtering for connector #{@connector_settings.id}.")
+        end
+      end
+    end
+  end
+end

data/lib/core/filtering/validation_status.rb

@@ -0,0 +1,17 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+module Core
+  module Filtering
+    class ValidationStatus
+      INVALID = 'invalid'
+      VALID = 'valid'
+      EDITED = 'edited'
+    end
+  end
+end

data/lib/core/filtering.rb

@@ -0,0 +1,17 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'core/filtering/post_process_engine'
+require 'core/filtering/post_process_result'
+require 'core/filtering/simple_rule'
+require 'core/filtering/validation_job_runner'
+require 'core/filtering/validation_status'
+
+module Core::Filtering
+  DEFAULT_DOMAIN = 'DEFAULT'
+end

data/lib/core/ingestion/es_sink.rb

@@ -0,0 +1,59 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'app/config'
+require 'utility/bulk_queue'
+require 'utility/es_client'
+require 'utility/logger'
+require 'elasticsearch/api'
+
+module Core
+  module Ingestion
+    class EsSink
+      def initialize(index_name, request_pipeline, bulk_queue = Utility::BulkQueue.new)
+        @client = Utility::EsClient.new(App::Config[:elasticsearch])
+        @index_name = index_name
+        @request_pipeline = request_pipeline
+        @operation_queue = bulk_queue
+      end
+
+      def ingest(id, serialized_document)
+        index_op = serialize({ 'index' => { '_index' => index_name, '_id' => id } })
+
+        flush unless @operation_queue.will_fit?(index_op, serialized_document)
+
+        @operation_queue.add(
+          index_op,
+          serialized_document
+        )
+      end
+
+      def delete(doc_id)
+        delete_op = serialize({ 'delete' => { '_index' => index_name, '_id' => doc_id } })
+        flush unless @operation_queue.will_fit?(delete_op)
+
+        @operation_queue.add(delete_op)
+      end
+
+      def flush
+        data = @operation_queue.pop_all
+        return if data.empty?
+
+        @client.bulk(:body => data, :pipeline => @request_pipeline)
+      end
+
+      def serialize(obj)
+        Elasticsearch::API.serializer.dump(obj)
+      end
+
+      private
+
+      attr_accessor :index_name
+    end
+  end
+end

data/lib/core/ingestion/ingester.rb

@@ -0,0 +1,90 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'utility/logger'
+
+module Core
+  module Ingestion
+    class Ingester
+      def initialize(sink_strategy, max_allowed_document_size = 5 * 1024 * 1024)
+        @sink_strategy = sink_strategy
+        @max_allowed_document_size = max_allowed_document_size
+
+        @ingested_count = 0
+        @ingested_volume = 0
+        @deleted_count = 0
+      end
+
+      def ingest(document)
+        unless document&.any?
+          Utility::Logger.warn('Connector attempted to ingest an empty document, skipping')
+          return
+        end
+
+        serialized_document = @sink_strategy.serialize(document)
+        document_size = serialized_document.bytesize
+
+        if @max_allowed_document_size > 0 && document_size > @max_allowed_document_size
+          Utility::Logger.warn("Connector attempted to ingest too large document with id=#{document['id']} [#{document_size}/#{@max_allowed_document_size}], skipping the document.")
+          return
+        end
+
+        @sink_strategy.ingest(document['id'], serialized_document)
+
+        @ingested_count += 1
+        @ingested_volume += document_size
+      end
+
+      def ingest_multiple(documents)
+        documents.each { |doc| ingest(doc) }
+      end
+
+      def delete(id)
+        return if id.nil?
+
+        @sink_strategy.delete(id)
+
+        @deleted_count += 1
+      end
+
+      def delete_multiple(ids)
+        ids.each { |id| delete(id) }
+      end
+
+      def flush
+        @sink_strategy.flush
+      end
+
+      def ingestion_stats
+        {
+          :indexed_document_count => @ingested_count,
+          :indexed_document_volume => @ingested_volume,
+          :deleted_document_count => @deleted_count
+        }
+      end
+
+      private
+
+      def do_ingest(_id, _serialized_document)
+        raise NotImplementedError
+      end
+
+      def do_delete(_id)
+        raise NotImplementedError
+      end
+
+      def do_flush
+        raise NotImplementedError
+      end
+
+      def do_serialize(_document)
+        raise NotImplementedError
+      end
+    end
+  end
+end

data/lib/core/{output_sink.rb → ingestion.rb}

@@ -6,8 +6,5 @@
 
 # frozen_string_literal: true
 
-require 'core/output_sink/es_sink'
-require 'core/output_sink/console_sink'
-require 'core/output_sink/combined_sink'
-
-module Core::OutputSink; end
+require 'core/ingestion/ingester'
+require 'core/ingestion/es_sink'

data/lib/core/scheduler.rb

@@ -10,6 +10,7 @@ require 'time'
 require 'fugit'
 require 'core/connector_settings'
 require 'core/elastic_connector_actions'
+require 'core/filtering/validation_status'
 require 'utility/cron'
 require 'utility/logger'
 require 'utility/exception_tracking'
@@ -38,15 +39,18 @@ module Core
           if configuration_triggered?(cs)
             yield cs, :configuration
           end
-        end
-        if @is_shutting_down
-          break
+          if filtering_validation_triggered?(cs)
+            yield cs, :filter_validation
+          end
         end
       rescue *Utility::AUTHORIZATION_ERRORS => e
         Utility::ExceptionTracking.log_exception(e, 'Could not retrieve connectors settings due to authorization error.')
       rescue StandardError => e
         Utility::ExceptionTracking.log_exception(e, 'Sync failed due to unexpected error.')
       ensure
+        if @is_shutting_down
+          break
+        end
         if @poll_interval > 0 && !@is_shutting_down
           Utility::Logger.debug("Sleeping for #{@poll_interval} seconds in #{self.class}.")
           sleep(@poll_interval)
@@ -62,8 +66,6 @@ module Core
     private
 
     def sync_triggered?(connector_settings)
-      return false unless connector_registered?(connector_settings.service_type)
-
       unless connector_settings.valid_index_name?
         Utility::Logger.warn("The index name of #{connector_settings.formatted} is invalid.")
         return false
@@ -129,8 +131,6 @@ module Core
     end
 
     def heartbeat_triggered?(connector_settings)
-      return false unless connector_registered?(connector_settings.service_type)
-
       last_seen = connector_settings[:last_seen]
       return true if last_seen.nil? || last_seen.empty?
       last_seen = begin
@@ -144,11 +144,41 @@ module Core
     end
 
     def configuration_triggered?(connector_settings)
-      if connector_settings.needs_service_type? || connector_registered?(connector_settings.service_type)
-        return connector_settings.connector_status == Connectors::ConnectorStatus::CREATED
+      connector_settings.needs_service_type? || connector_settings.connector_status == Connectors::ConnectorStatus::CREATED
+    end
+
+    def filtering_validation_triggered?(connector_settings)
+      filtering = connector_settings.filtering
+
+      unless filtering.present?
+        Utility::Logger.debug("#{connector_settings.formatted} does not contain filtering to be validated.")
+
+        return false
       end
 
-      false
+      draft_filters = filtering[:draft]
+
+      unless draft_filters.present?
+        Utility::Logger.debug("#{connector_settings.formatted} does not contain a draft filter to be validated.")
+
+        return false
+      end
+
+      validation = draft_filters[:validation]
+
+      unless validation.present?
+        Utility::Logger.warn("#{connector_settings.formatted} does not contain a validation object inside draft filtering. Check connectors index.")
+
+        return false
+      end
+
+      unless validation[:state] == Core::Filtering::ValidationStatus::EDITED
+        Utility::Logger.debug("#{connector_settings.formatted} filtering validation needs to be in state #{Core::Filtering::ValidationStatus::EDITED} to be able to validate it.")
+
+        return false
+      end
+
+      true
     end
 
     def connector_registered?(service_type)

data/lib/core/sync_job_runner.rb

@@ -8,7 +8,9 @@
 
 require 'connectors/connector_status'
 require 'connectors/registry'
-require 'core/output_sink'
+require 'core/filtering/post_process_engine'
+require 'core/ingestion'
+require 'core/filtering/validation_status'
 require 'utility'
 
 module Core
@@ -19,14 +21,18 @@ module Core
   end
 
   class SyncJobRunner
+    JOB_REPORTING_INTERVAL = 10
+
     def initialize(connector_settings)
       @connector_settings = connector_settings
-      @sink = Core::OutputSink::EsSink.new(connector_settings.index_name, @connector_settings.request_pipeline)
+      @ingester = Core::Ingestion::Ingester.new(Core::Ingestion::EsSink.new(connector_settings.index_name, @connector_settings.request_pipeline))
       @connector_class = Connectors::REGISTRY.connector_class(connector_settings.service_type)
       @sync_finished = false
+      @sync_error = nil
       @status = {
         :indexed_document_count => 0,
         :deleted_document_count => 0,
+        :indexed_document_volume => 0,
         :error => nil
       }
     end
@@ -41,8 +47,10 @@ module Core
     def do_sync!
       Utility::Logger.info("Claiming a sync job for connector #{@connector_settings.id}.")
 
-      job_description = ElasticConnectorActions.claim_job(@connector_settings.id)
-      job_id = job_description['_id']
+      job_record = ElasticConnectorActions.claim_job(@connector_settings.id)
+      job_description = job_record['_source']
+      job_id = job_record['_id']
+      job_description['_id'] = job_id
 
       unless job_id.present?
         Utility::Logger.error("Failed to claim the job for #{@connector_settings.id}. Please check the logs for the cause of this error.")
@@ -52,6 +60,10 @@ module Core
       begin
         Utility::Logger.debug("Successfully claimed job for connector #{@connector_settings.id}.")
 
+        Utility::Logger.info("Checking active filtering for sync job #{job_id} for connector #{@connector_settings.id}.")
+        validate_filtering(job_description.dig(:connector, :filtering))
+        Utility::Logger.debug("Active filtering for sync job #{job_id} for connector #{@connector_settings.id} is valid.")
+
         connector_instance = Connectors::REGISTRY.connector(@connector_settings.service_type, @connector_settings.configuration, job_description: job_description)
 
         connector_instance.do_health_check!
@@ -61,11 +73,21 @@ module Core
 
         Utility::Logger.debug("#{existing_ids.size} documents are present in index #{@connector_settings.index_name}.")
 
+        post_processing_engine = Core::Filtering::PostProcessEngine.new(job_description)
+        reporting_cycle_start = Time.now
+        Utility::Logger.info('Yielding documents...')
         connector_instance.yield_documents do |document|
           document = add_ingest_metadata(document)
-          @sink.ingest(document)
-          incoming_ids << document['id']
-          @status[:indexed_document_count] += 1
+          post_process_result = post_processing_engine.process(document)
+          if post_process_result.is_include?
+            @ingester.ingest(document)
+            incoming_ids << document['id']
+          end
+
+          if Time.now - reporting_cycle_start >= JOB_REPORTING_INTERVAL
+            ElasticConnectorActions.update_sync(job_id, @ingester.ingestion_stats.merge(:metadata => connector_instance.metadata))
+            reporting_cycle_start = Time.now
+          end
         end
 
         ids_to_delete = existing_ids - incoming_ids.uniq
@@ -73,32 +95,48 @@ module Core
         Utility::Logger.info("Deleting #{ids_to_delete.size} documents from index #{@connector_settings.index_name}.")
 
         ids_to_delete.each do |id|
-          @sink.delete(id)
-          @status[:deleted_document_count] += 1
+          @ingester.delete(id)
+
+          if Time.now - reporting_cycle_start >= JOB_REPORTING_INTERVAL
+            ElasticConnectorActions.update_sync(job_id, @ingester.ingestion_stats.merge(:metadata => connector_instance.metadata))
+            reporting_cycle_start = Time.now
+          end
         end
 
-        @sink.flush
+        @ingester.flush
 
         # We use this mechanism for checking, whether an interrupt (or something else lead to the thread not finishing)
         # occurred as most of the time the main execution thread is interrupted and we miss this Signal/Exception here
         @sync_finished = true
       rescue StandardError => e
-        @status[:error] = e.message
+        @sync_error = e.message
         Utility::ExceptionTracking.log_exception(e)
-        ElasticConnectorActions.update_connector_status(@connector_settings.id, Connectors::ConnectorStatus::ERROR, Utility::Logger.abbreviated_message(e.message))
       ensure
+        stats = @ingester.ingestion_stats
+
+        Utility::Logger.debug("Sync stats are: #{stats}")
+
+        @status[:indexed_document_count] = stats[:indexed_document_count]
+        @status[:deleted_document_count] = stats[:deleted_document_count]
+        @status[:indexed_document_volume] = stats[:indexed_document_volume]
+
         Utility::Logger.info("Upserted #{@status[:indexed_document_count]} documents into #{@connector_settings.index_name}.")
         Utility::Logger.info("Deleted #{@status[:deleted_document_count]} documents into #{@connector_settings.index_name}.")
 
         # Make sure to not override a previous error message
-        if !@sync_finished && @status[:error].nil?
-          @status[:error] = 'Sync thread didn\'t finish execution. Check connector logs for more details.'
+        if !@sync_finished && @sync_error.nil?
+          @sync_error = 'Sync thread didn\'t finish execution. Check connector logs for more details.'
         end
 
-        ElasticConnectorActions.complete_sync(@connector_settings.id, job_id, @status.dup)
+        unless connector_instance.nil?
+          metadata = @ingester.ingestion_stats.merge(:metadata => connector_instance.metadata)
+          metadata[:total_document_count] = ElasticConnectorActions.document_count(@connector_settings.index_name)
+        end
+
+        ElasticConnectorActions.complete_sync(@connector_settings.id, job_id, metadata, @sync_error)
 
-        if @status[:error]
-          Utility::Logger.info("Failed to sync for connector #{@connector_settings.id} with error '#{@status[:error]}'.")
+        if @sync_error
+          Utility::Logger.info("Failed to sync for connector #{@connector_settings.id} with error '#{@sync_error}'.")
         else
           Utility::Logger.info("Successfully synced for connector #{@connector_settings.id}.")
         end
@@ -119,5 +157,15 @@ module Core
 
       raise IncompatibleConfigurableFieldsError.new(@connector_class.service_type, expected_fields, actual_fields) if expected_fields != actual_fields
     end
+
+    def validate_filtering(filtering)
+      validation_result = @connector_class.validate_filtering(filtering)
+
+      wrong_state_error = Utility::InvalidFilterConfigError.new("Active filtering is not in valid state (current state: #{validation_result[:state]}) for connector #{@connector_settings.id}. Please check active filtering in connectors index.")
+      raise wrong_state_error if validation_result[:state] != Core::Filtering::ValidationStatus::VALID
+
+      errors_present_error = Utility::InvalidFilterConfigError.new("Active filtering is in valid state, but errors were detected (errors: #{validation_result[:errors]}) for connector #{@connector_settings.id}. Please check active filtering in connectors index.")
+      raise errors_present_error if validation_result[:errors].present?
+    end
   end
 end

data/lib/core.rb

@@ -7,8 +7,10 @@
 # frozen_string_literal: true
 
 require 'core/configuration'
+require 'core/connector_job'
 require 'core/connector_settings'
 require 'core/elastic_connector_actions'
+require 'core/filtering'
 require 'core/heartbeat'
 require 'core/scheduler'
 require 'core/single_scheduler'

data/lib/utility/bulk_queue.rb

@@ -0,0 +1,85 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+require 'json'
+
+module Utility
+  class BulkQueue
+    class QueueOverflowError < StandardError; end
+
+    # 500 items or 5MB
+    def initialize(operation_count_threshold = 500, size_threshold = 5 * 1024 * 1024)
+      @operation_count_threshold = operation_count_threshold.freeze
+      @size_threshold = size_threshold.freeze
+
+      @buffer = ''
+
+      @current_operation_count = 0
+
+      @current_buffer_size = 0
+      @current_data_size = 0
+    end
+
+    def pop_all
+      result = @buffer
+
+      reset
+
+      result
+    end
+
+    def add(operation, payload = nil)
+      raise QueueOverflowError unless will_fit?(operation, payload)
+
+      operation_size = get_size(operation)
+      payload_size = get_size(payload)
+
+      @current_operation_count += 1
+      @current_buffer_size += operation_size
+      @current_buffer_size += payload_size
+      @current_data_size += payload_size
+
+      @buffer << operation
+      @buffer << "\n"
+
+      if payload
+        @buffer << payload
+        @buffer << "\n"
+      end
+    end
+
+    def will_fit?(operation, payload = nil)
+      return false if @current_operation_count + 1 > @operation_count_threshold
+
+      operation_size = get_size(operation)
+      payload_size = get_size(payload)
+
+      @current_buffer_size + operation_size + payload_size < @size_threshold
+    end
+
+    def current_stats
+      {
+        :current_operation_count => @current_operation_count,
+        :current_buffer_size => @current_buffer_size
+      }
+    end
+
+    private
+
+    def get_size(str)
+      return 0 unless str
+      str.bytesize
+    end
+
+    def reset
+      @current_operation_count = 0
+      @current_buffer_size = 0
+      @current_data_size = 0
+
+      @buffer = ''
+    end
+  end
+end

data/lib/utility/constants.rb

@@ -16,5 +16,7 @@ module Utility
     JOB_INDEX = '.elastic-connectors-sync-jobs'
     CONTENT_INDEX_PREFIX = 'search-'
     CRAWLER_SERVICE_TYPE = 'elastic-crawler'
+    FILTERING_RULES_FEATURE = 'filtering_rules'
+    FILTERING_ADVANCED_FEATURE = 'filtering_advanced_config'
   end
 end

data/lib/utility/filtering.rb

@@ -0,0 +1,22 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+module Utility
+  class Filtering
+    class << self
+      def extract_filter(filtering)
+        return {} unless filtering.present?
+
+        # assume for now, that first object in filtering array or a filter object itself is the only filtering object
+        filter = filtering.is_a?(Array) ? filtering.first : filtering
+
+        filter.present? ? filter : {}
+      end
+    end
+  end
+end

data/lib/utility/logger.rb

@@ -4,6 +4,7 @@
 # you may not use this file except in compliance with the Elastic License.
 #
 
+require 'config'
 require 'logger'
 require 'active_support/core_ext/module'
 require 'active_support/core_ext/string/filters'
@@ -23,7 +24,7 @@ module Utility
       end
 
       def logger
-        @logger ||= Settings[:ecs_logging] ? EcsLogging::Logger.new(STDOUT) : ::Logger.new(STDOUT)
+        @logger ||= defined?(::Settings) && ::Settings[:ecs_logging] ? EcsLogging::Logger.new(STDOUT) : ::Logger.new(STDOUT)
       end
 
       SUPPORTED_LOG_LEVELS.each do |level|

data/lib/utility.rb

@@ -4,14 +4,15 @@
 # you may not use this file except in compliance with the Elastic License.
 #
 
+require 'utility/bulk_queue'
+require 'utility/common'
 require 'utility/constants'
 require 'utility/cron'
-require 'utility/common'
+require 'utility/elasticsearch/index/mappings'
+require 'utility/elasticsearch/index/text_analysis_settings'
+require 'utility/environment'
 require 'utility/errors'
 require 'utility/es_client'
-require 'utility/environment'
 require 'utility/exception_tracking'
 require 'utility/extension_mapping_util'
 require 'utility/logger'
-require 'utility/elasticsearch/index/mappings'
-require 'utility/elasticsearch/index/text_analysis_settings'