connectors_service 8.6.0.4 → 8.7.0.0.pre.20221117T010623Z

data/lib/core/filtering/simple_rule.rb

@@ -0,0 +1,141 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'utility/logger'
+
+module Core
+  module Filtering
+    class SimpleRule
+      DEFAULT_RULE_ID = 'DEFAULT'
+
+      class Policy
+        INCLUDE = 'include'
+        EXCLUDE = 'exclude'
+      end
+
+      class Rule
+        REGEX = 'regex'
+        EQUALS = 'equals'
+        STARTS_WITH = 'starts_with'
+        ENDS_WITH = 'ends_with'
+        CONTAINS = 'contains'
+        LESS_THAN = '<'
+        GREATER_THAN = '>'
+      end
+
+      attr_reader :policy, :field, :rule, :value, :id
+
+      def initialize(rule_hash)
+        @policy = rule_hash.fetch('policy')
+        @field = rule_hash.fetch('field')
+        @rule = rule_hash.fetch('rule')
+        @value = rule_hash.fetch('value')
+        @id = rule_hash.fetch('id')
+        @rule_hash = rule_hash
+      rescue KeyError => e
+        raise "#{e.key} is required"
+      end
+
+      def self.from_args(id, policy, field, rule, value)
+        SimpleRule.new(
+          {
+            'id' => id,
+            'policy' => policy,
+            'field' => field,
+            'rule' => rule,
+            'value' => value
+          }
+        )
+      end
+
+      DEFAULT_RULE = SimpleRule.new(
+        'policy' => 'include',
+        'field' => '_',
+        'rule' => 'regex',
+        'value' => '.*',
+        'id' => SimpleRule::DEFAULT_RULE_ID
+      )
+
+      def match?(document)
+        return true if id == DEFAULT_RULE_ID
+        doc_value = document[field]
+        return false if doc_value.nil?
+        coerced_value = coerce(doc_value)
+        case rule
+        when Rule::EQUALS
+          case coerced_value
+          when Integer
+            doc_value == coerced_value
+          when DateTime, Time
+            doc_value.to_s == coerced_value.to_s
+          else
+            doc_value.to_s == coerced_value
+          end
+        when Rule::STARTS_WITH
+          doc_value.to_s.start_with?(value)
+        when Rule::ENDS_WITH
+          doc_value.to_s.end_with?(value)
+        when Rule::CONTAINS
+          doc_value.to_s.include?(value)
+        when Rule::REGEX
+          doc_value.to_s.match(/#{value}/)
+        when Rule::LESS_THAN
+          doc_value < coerced_value
+        when Rule::GREATER_THAN
+          doc_value > coerced_value
+        else
+          false
+        end
+      end
+
+      def coerce(doc_value)
+        case doc_value
+        when String
+          value.to_s
+        when Integer
+          value.to_i
+        when DateTime, Time
+          to_date(value)
+        when TrueClass, FalseClass # Ruby doesn't have a Boolean type, TIL
+          to_bool(value).to_s
+        else
+          value.to_s
+        end
+      rescue StandardError => e
+        Utility::Logger.debug("Failed to coerce value '#{value}' (#{value.class}) based on document value '#{doc_value}' (#{doc_value.class}) due to error: #{e.class}: #{e.message}")
+        value.to_s
+      end
+
+      def is_include?
+        policy == Policy::INCLUDE
+      end
+
+      def is_exclude?
+        policy == Policy::EXCLUDE
+      end
+
+      def to_h
+        @rule_hash
+      end
+
+      private
+
+      def to_bool(str)
+        return true if str == true || str =~ (/^(true|t|yes|y|on|1)$/i)
+        return false if str == false || str.blank? || str =~ (/^(false|f|no|n|off|0)$/i)
+        raise ArgumentError.new("invalid value for Boolean: \"#{str}\"")
+      end
+
+      def to_date(str)
+        DateTime.parse(str)
+      rescue ArgumentError
+        Time.at(str.to_i) # try with it as an int string of millis
+      end
+    end
+  end
+end

data/lib/core/filtering/validation_job_runner.rb

@@ -0,0 +1,53 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'connectors/connector_status'
+require 'connectors/registry'
+
+module Core
+  module Filtering
+    DEFAULT_DOMAIN = 'DEFAULT'
+
+    class ValidationJobRunner
+      def initialize(connector_settings)
+        @connector_settings = connector_settings
+        @connector_class = Connectors::REGISTRY.connector_class(connector_settings.service_type)
+        @validation_finished = false
+        @status = { :error => nil }
+      end
+
+      def execute
+        Utility::Logger.info("Starting a validation job for connector #{@connector_settings.id}.")
+
+        validation_result = @connector_class.validate_filtering(@connector_settings.filtering[:draft])
+
+        # currently only used for connectors -> DEFAULT domain can be assumed (will be changed with the integration of crawler)
+        ElasticConnectorActions.update_filtering_validation(@connector_settings.id, { DEFAULT_DOMAIN => validation_result })
+
+        @validation_finished = true
+      rescue StandardError => e
+        Utility::ExceptionTracking.log_exception(e)
+        validation_failed_result = { :state => Core::Filtering::ValidationStatus::INVALID,
+                                     :errors => [
+                                       { :ids => [], :messages => ['Unknown problem occurred while validating, see logs for details.'] }
+                                     ] }
+        ElasticConnectorActions.update_filtering_validation(@connector_settings.id, { DEFAULT_DOMAIN => validation_failed_result })
+      ensure
+        if !@validation_finished && !@status[:error].present?
+          @status[:error] = 'Validation thread did not finish execution. Check connector logs for more details.'
+        end
+
+        if @status[:error]
+          Utility::Logger.warn("Failed to validate filtering for connector #{@connector_settings.id} with error '#{@status[:error]}'.")
+        else
+          Utility::Logger.info("Successfully validated filtering for connector #{@connector_settings.id}.")
+        end
+      end
+    end
+  end
+end

data/lib/core/filtering/validation_status.rb

@@ -0,0 +1,17 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+module Core
+  module Filtering
+    class ValidationStatus
+      INVALID = 'invalid'
+      VALID = 'valid'
+      EDITED = 'edited'
+    end
+  end
+end

data/lib/core/filtering.rb

@@ -0,0 +1,17 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'core/filtering/post_process_engine'
+require 'core/filtering/post_process_result'
+require 'core/filtering/simple_rule'
+require 'core/filtering/validation_job_runner'
+require 'core/filtering/validation_status'
+
+module Core::Filtering
+  DEFAULT_DOMAIN = 'DEFAULT'
+end

data/lib/core/ingestion/es_sink.rb

@@ -0,0 +1,118 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'app/config'
+require 'utility/bulk_queue'
+require 'utility/es_client'
+require 'utility/logger'
+require 'elasticsearch/api'
+#
+# This class is responsible for sending the data to the data storage.
+# While we don't actually allow to output our data anywhere except
+# Elasticsearch, we still want to be able to do so sometime in future.
+#
+# This class should stay simple and any change to the class should be careful
+# with the thought of introducing other sinks in future.
+module Core
+  module Ingestion
+    class EsSink
+      def initialize(index_name, request_pipeline, bulk_queue = Utility::BulkQueue.new, max_allowed_document_size = Utility::Constants::DEFAULT_MAX_INGESTION_QUEUE_BYTES)
+        @client = Utility::EsClient.new(App::Config[:elasticsearch])
+        @index_name = index_name
+        @request_pipeline = request_pipeline
+        @operation_queue = bulk_queue
+
+        @max_allowed_document_size = max_allowed_document_size
+
+        @queued = {
+          :indexed_document_count => 0,
+          :deleted_document_count => 0,
+          :indexed_document_volume => 0
+        }
+
+        @completed = {
+          :indexed_document_count => 0,
+          :deleted_document_count => 0,
+          :indexed_document_volume => 0
+        }
+      end
+
+      def ingest(document)
+        if document.nil? || document.empty?
+          Utility::Logger.warn('Connector attempted to ingest an empty document, skipping')
+          return
+        end
+
+        id = document['id']
+        serialized_document = serialize(document)
+
+        document_size = serialized_document.bytesize
+
+        if @max_allowed_document_size > 0 && document_size > @max_allowed_document_size
+          Utility::Logger.warn("Connector attempted to ingest too large document with id=#{document['id']} [#{document_size}/#{@max_allowed_document_size}], skipping the document.")
+          return
+        end
+
+        index_op = serialize({ 'index' => { '_index' => @index_name, '_id' => id } })
+
+        flush unless @operation_queue.will_fit?(index_op, serialized_document)
+
+        @operation_queue.add(
+          index_op,
+          serialized_document
+        )
+
+        @queued[:indexed_document_count] += 1
+        @queued[:indexed_document_volume] += document_size
+      end
+
+      def ingest_multiple(documents)
+        documents.each { |doc| ingest(doc) }
+      end
+
+      def delete(id)
+        return if id.nil?
+
+        delete_op = serialize({ 'delete' => { '_index' => @index_name, '_id' => id } })
+        flush unless @operation_queue.will_fit?(delete_op)
+
+        @operation_queue.add(delete_op)
+        @queued[:deleted_document_count] += 1
+      end
+
+      def delete_multiple(ids)
+        ids.each { |id| delete(id) }
+      end
+
+      def flush
+        data = @operation_queue.pop_all
+        return if data.empty?
+
+        @client.bulk(:body => data, :pipeline => @request_pipeline)
+
+        @completed[:indexed_document_count] += @queued[:indexed_document_count]
+        @completed[:deleted_document_count] += @queued[:deleted_document_count]
+        @completed[:indexed_document_volume] += @queued[:indexed_document_volume]
+
+        @queued[:indexed_document_count] = 0
+        @queued[:deleted_document_count] = 0
+        @queued[:indexed_document_volume] = 0
+      end
+
+      def ingestion_stats
+        @completed.dup
+      end
+
+      private
+
+      def serialize(document)
+        Elasticsearch::API.serializer.dump(document)
+      end
+    end
+  end
+end

data/lib/core/{output_sink.rb → ingestion.rb}

@@ -6,8 +6,4 @@
 
 # frozen_string_literal: true
 
-require 'core/output_sink/es_sink'
-require 'core/output_sink/console_sink'
-require 'core/output_sink/combined_sink'
-
-module Core::OutputSink; end
+require 'core/ingestion/es_sink'

data/lib/core/jobs/consumer.rb

@@ -0,0 +1,132 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'utility/constants'
+
+module Core
+  module Jobs
+    class Consumer
+      def initialize(scheduler:,
+                     max_ingestion_queue_size:,
+                     max_ingestion_queue_bytes:,
+                     poll_interval: 3,
+                     termination_timeout: 60,
+                     min_threads: 1,
+                     max_threads: 5,
+                     max_queue: 100,
+                     idle_time: 5)
+        @scheduler = scheduler
+        @poll_interval = poll_interval
+        @termination_timeout = termination_timeout
+        @min_threads = min_threads
+        @max_threads = max_threads
+        @max_queue = max_queue
+        @idle_time = idle_time
+
+        @max_ingestion_queue_size = max_ingestion_queue_size
+        @max_ingestion_queue_bytes = max_ingestion_queue_bytes
+
+        @running = Concurrent::AtomicBoolean.new(false)
+      end
+
+      def subscribe!(index_name:)
+        @index_name = index_name
+
+        start_loop!
+      end
+
+      def running?
+        # @TODO check if a loop thread is alive
+        pool.running? && @running.true?
+      end
+
+      def shutdown!
+        Utility::Logger.info("Shutting down consumer for #{@index_name} index")
+        @running.make_false
+        pool.shutdown
+        pool.wait_for_termination(@termination_timeout)
+        # reset pool
+        @pool = nil
+      end
+
+      private
+
+      def start_loop!
+        Utility::Logger.info("Starting a new consumer for #{@index_name} index")
+
+        Thread.new do
+          # assign a name to the thread
+          # see @TODO in #self.running?
+          Thread.current[:name] = "consumer-group-#{@index_name}"
+
+          loop do
+            if @running.false?
+              Utility::Logger.info('Shutting down the loop')
+              break
+            end
+
+            sleep(@poll_interval)
+            Utility::Logger.debug('Getting registered connectors')
+
+            connectors = ready_for_sync_connectors
+            next unless connectors.any?
+
+            Utility::Logger.debug("Number of available connectors: #{connectors.size}")
+
+            # @TODO It is assumed that @index_name is used to retrive pending jobs.
+            # This will be discussed after 8.6 release
+            pending_jobs = Core::ConnectorJob.pending_jobs(connectors_ids: connectors.keys)
+            Utility::Logger.info("Number of pending jobs: #{pending_jobs.size}")
+
+            pending_jobs.each do |job|
+              connector_settings = connectors[job.connector_id]
+
+              pool.post do
+                Utility::Logger.info("Connector #{connector_settings.formatted} picked up the job #{job.id}")
+                Core::ElasticConnectorActions.ensure_content_index_exists(connector_settings.index_name)
+                job_runner = Core::SyncJobRunner.new(
+                  connector_settings,
+                  job,
+                  @max_ingestion_queue_size,
+                  @max_ingestion_queue_bytes
+                )
+                job_runner.execute
+              rescue Core::JobAlreadyRunningError
+                Utility::Logger.info("Sync job for #{connector_settings.formatted} is already running, skipping.")
+              rescue Core::ConnectorVersionChangedError => e
+                Utility::Logger.info("Could not start the job because #{connector_settings.formatted} has been updated externally. Message: #{e.message}")
+              rescue StandardError => e
+                Utility::ExceptionTracking.log_exception(e, "Sync job for #{connector_settings.formatted} failed due to unexpected error.")
+              end
+            end
+          rescue StandardError => e
+            Utility::ExceptionTracking.log_exception(e, 'The consumer group failed')
+          end
+        end
+
+        @running.make_true
+      end
+
+      def pool
+        @pool ||= Concurrent::ThreadPoolExecutor.new(
+          min_threads: @min_threads,
+          max_threads: @max_threads,
+          max_queue: @max_queue,
+          fallback_policy: :abort,
+          idletime: @idle_time
+        )
+      end
+
+      def ready_for_sync_connectors
+        @scheduler.connector_settings
+          .select(&:ready_for_sync?)
+          .inject({}) { |memo, cs| memo.merge(cs.id => cs) }
+      end
+    end
+  end
+end

data/lib/core/jobs/producer.rb

@@ -0,0 +1,26 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+module Core
+  module Jobs
+    class Producer
+      JOB_TYPES = %i(sync).freeze
+
+      class << self
+        def enqueue_job(job_type:, connector_settings:)
+          raise UnsupportedJobType unless JOB_TYPES.include?(job_type)
+          raise ArgumentError unless connector_settings.kind_of?(ConnectorSettings)
+
+          ElasticConnectorActions.create_job(connector_settings: connector_settings)
+        end
+      end
+    end
+
+    class UnsupportedJobType < StandardError; end
+  end
+end

data/lib/core/scheduler.rb

@@ -10,6 +10,7 @@ require 'time'
 require 'fugit'
 require 'core/connector_settings'
 require 'core/elastic_connector_actions'
+require 'core/filtering/validation_status'
 require 'utility/cron'
 require 'utility/logger'
 require 'utility/exception_tracking'
@@ -38,15 +39,18 @@ module Core
           if configuration_triggered?(cs)
             yield cs, :configuration
           end
-        end
-        if @is_shutting_down
-          break
+          if filtering_validation_triggered?(cs)
+            yield cs, :filter_validation
+          end
         end
       rescue *Utility::AUTHORIZATION_ERRORS => e
         Utility::ExceptionTracking.log_exception(e, 'Could not retrieve connectors settings due to authorization error.')
       rescue StandardError => e
         Utility::ExceptionTracking.log_exception(e, 'Sync failed due to unexpected error.')
       ensure
+        if @is_shutting_down
+          break
+        end
         if @poll_interval > 0 && !@is_shutting_down
           Utility::Logger.debug("Sleeping for #{@poll_interval} seconds in #{self.class}.")
           sleep(@poll_interval)
@@ -62,8 +66,6 @@ module Core
     private
 
     def sync_triggered?(connector_settings)
-      return false unless connector_registered?(connector_settings.service_type)
-
       unless connector_settings.valid_index_name?
         Utility::Logger.warn("The index name of #{connector_settings.formatted} is invalid.")
         return false
@@ -129,8 +131,6 @@ module Core
     end
 
     def heartbeat_triggered?(connector_settings)
-      return false unless connector_registered?(connector_settings.service_type)
-
       last_seen = connector_settings[:last_seen]
       return true if last_seen.nil? || last_seen.empty?
       last_seen = begin
@@ -144,11 +144,41 @@ module Core
     end
 
     def configuration_triggered?(connector_settings)
-      if connector_settings.needs_service_type? || connector_registered?(connector_settings.service_type)
-        return connector_settings.connector_status == Connectors::ConnectorStatus::CREATED
+      connector_settings.needs_service_type? || connector_settings.connector_status == Connectors::ConnectorStatus::CREATED
+    end
+
+    def filtering_validation_triggered?(connector_settings)
+      filtering = connector_settings.filtering
+
+      unless filtering.present?
+        Utility::Logger.debug("#{connector_settings.formatted} does not contain filtering to be validated.")
+
+        return false
       end
 
-      false
+      draft_filters = filtering[:draft]
+
+      unless draft_filters.present?
+        Utility::Logger.debug("#{connector_settings.formatted} does not contain a draft filter to be validated.")
+
+        return false
+      end
+
+      validation = draft_filters[:validation]
+
+      unless validation.present?
+        Utility::Logger.warn("#{connector_settings.formatted} does not contain a validation object inside draft filtering. Check connectors index.")
+
+        return false
+      end
+
+      unless validation[:state] == Core::Filtering::ValidationStatus::EDITED
+        Utility::Logger.debug("#{connector_settings.formatted} filtering validation needs to be in state #{Core::Filtering::ValidationStatus::EDITED} to be able to validate it.")
+
+        return false
+      end
+
+      true
     end
 
     def connector_registered?(service_type)

data/lib/core/single_scheduler.rb

@@ -20,7 +20,7 @@ module Core
 
     def connector_settings
       connector_settings = Core::ConnectorSettings.fetch_by_id(@connector_id)
-      [connector_settings]
+      [connector_settings].compact
     rescue *Utility::AUTHORIZATION_ERRORS => e
       # should be handled by the general scheduler
       raise e