connectors_service 8.6.0.4.pre.20221114T233727Z → 8.6.0.4.pre.20221116T024501Z

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8f69f05260d34b07ce34d569ce7c41fdd10349b33121823697ebbb6a4ebf9206
-  data.tar.gz: a2957118c80d0e2bc9ea6a8046307485c11e4f809efb01cabbb96e341dc947c2
+  metadata.gz: 9a610999d9c34543392e1dd9b7b9ab136557992cc1503fa5aa5973186d17af9e
+  data.tar.gz: '086282a4a15ce1b168e6a0add5f78646e1fe89c336b8a257677826cce3677635'
 SHA512:
-  metadata.gz: 63775eded9d9953b41950edd7ca86176200c4ae7510564f6f7995c336d6e78bbe40d494cb0a152a984b14b45055e7b7847a2779888d70905d4956b8c78d4bda1
-  data.tar.gz: 52b00d122ef43fc5afa0b4cb50bbe428111e7fe2cb7cee437dd2d2b6b32516cbd01c5803b0a11609a5021e85605a2e6bd2b30973807df3cae1420864e2fcb185
+  metadata.gz: bf0a0a22ea96074bcbb904000ec489a591fc8491bc2d91759bc1a097c60e31c2abad7b118bb97d16bbe80c5b00dbe08f7c65653710c2d6a77d3356607d87e045
+  data.tar.gz: 67ff773ed2ae3f869897b5fb3a2c0dc3fc55c5922c602e032ed5bafa800043bd4b7f4af90fdd055e575ea877f3cf3b9e308d91c58bad59cd6c63dd5eb8ab98f4

data/config/connectors.yml

@@ -1,10 +1,10 @@
 # general metadata
-version: 8.6.0.4-20221114T233727Z
+version: 8.6.0.4-20221116T024501Z
 repository: git@github.com:elastic/ent-search-connectors.git
-revision: f506d5e5ebedfb0c6058d347d8ce22adc42e2cc0
+revision: b3cc1332879a38930a272a63f8c6be1847578204
 elasticsearch:
   hosts: http://localhost:9200
-  api_key: WXNYeWQ0UUJ4Y3ZQV3ctbjVibnU6REx4eE8tbFhUMU94N2JoU2hIeVFMQQ==
+  api_key: MDJyaGZZUUJ1ZXdOLTB1ZnQ5aXQ6a2IwQURWSjVUc1NPcjVsOFBuVDBzZw==
   retry_on_failure: 3
   request_timeout: 120
   disable_warnings: true
@@ -20,5 +20,5 @@ poll_interval: 3
 termination_timeout: 60
 heartbeat_interval: 1800
 native_mode: false
-connector_id: YcXyd4QBxcvPWw-n2bkA
+connector_id: 0mrhfYQBuewN-0ufptgN
 service_type: mongodb

data/lib/app/app.rb

@@ -17,6 +17,10 @@ require 'utility/logger'
 module App
   Utility::Environment.set_execution_environment(App::Config) do
     App::PreflightCheck.run!
+
+    # set exit hook
+    Kernel.at_exit { App::Dispatcher.shutdown! }
+
     App::Dispatcher.start!
   rescue App::PreflightCheck::CheckFailure => e
     Utility::Logger.error("Preflight check failed: #{e.message}")

data/lib/app/dispatcher.rb

@@ -28,6 +28,10 @@ module App
       def start!
         running!
         Utility::Logger.info("Starting connector service in #{App::Config.native_mode ? 'native' : 'non-native'} mode...")
+
+        # start sync jobs consumer
+        start_consumer!
+
         start_polling_jobs!
       end
 
@@ -37,6 +41,8 @@ module App
         scheduler.shutdown
         pool.shutdown
         pool.wait_for_termination(TERMINATION_TIMEOUT)
+
+        stop_consumer!
       end
 
       private
@@ -68,7 +74,10 @@ module App
         scheduler.when_triggered do |connector_settings, task|
           case task
           when :sync
-            start_sync_task(connector_settings)
+            # update connector sync_now flag
+            Core::ElasticConnectorActions.update_connector_sync_now(connector_settings.id, false)
+
+            Core::Jobs::Producer.enqueue_job(job_type: :sync, connector_settings: connector_settings)
           when :heartbeat
             start_heartbeat_task(connector_settings)
           when :configuration
@@ -83,22 +92,6 @@ module App
         Utility::ExceptionTracking.log_exception(e, 'The connector service failed due to unexpected error.')
       end
 
-      def start_sync_task(connector_settings)
-        start_heartbeat_task(connector_settings)
-        pool.post do
-          Utility::Logger.info("Initiating a sync job for #{connector_settings.formatted}...")
-          Core::ElasticConnectorActions.ensure_content_index_exists(connector_settings.index_name)
-          job_runner = Core::SyncJobRunner.new(connector_settings)
-          job_runner.execute
-        rescue Core::JobAlreadyRunningError
-          Utility::Logger.info("Sync job for #{connector_settings.formatted} is already running, skipping.")
-        rescue Core::ConnectorVersionChangedError => e
-          Utility::Logger.info("Could not start the job because #{connector_settings.formatted} has been updated externally. Message: #{e.message}")
-        rescue StandardError => e
-          Utility::ExceptionTracking.log_exception(e, "Sync job for #{connector_settings.formatted} failed due to unexpected error.")
-        end
-      end
-
       def start_heartbeat_task(connector_settings)
         pool.post do
           Utility::Logger.info("Sending heartbeat for #{connector_settings.formatted}...")
@@ -132,6 +125,26 @@ module App
           Utility::ExceptionTracking.log_exception(e, "Filter validation task for #{connector_settings.formatted} failed due to unexpected error.")
         end
       end
+
+      def start_consumer!
+        @consumer = Core::Jobs::Consumer.new(
+          poll_interval: POLL_INTERVAL,
+          termination_timeout: TERMINATION_TIMEOUT,
+          min_threads: MIN_THREADS,
+          max_threads: MAX_THREADS,
+          max_queue: MAX_QUEUE,
+          scheduler: scheduler
+        )
+
+        @consumer.subscribe!(index_name: Utility::Constants::JOB_INDEX)
+      end
+
+      def stop_consumer!
+        return if @consumer.nil?
+        return unless @consumer.running?
+
+        @consumer.shutdown!
+      end
     end
   end
 end

data/lib/connectors/base/advanced_snippet_against_schema_validator.rb

@@ -0,0 +1,173 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+# frozen_string_literal: true
+
+require 'active_support/core_ext/hash'
+require 'utility/logger'
+require 'connectors/base/advanced_snippet_validator'
+require 'core/filtering/validation_status'
+
+module Connectors
+  module Base
+    class AdvancedSnippetAgainstSchemaValidator < Connectors::Base::AdvancedSnippetValidator
+
+      MAX_RECURSION_DEPTH = 50
+      ADVANCED_SNIPPET_ID = 'advanced_snippet'
+
+      def initialize(advanced_snippet, schema)
+        super(advanced_snippet)
+        @schema = schema
+      end
+
+      def is_snippet_valid?
+        validation_result = validate_against_schema(@schema, @advanced_snippet)
+        log_validation_result(validation_result)
+        validation_result
+      end
+
+      private
+
+      def validate_against_schema(config_schema, advanced_snippet, recursion_depth = 0)
+        # Prevent unintentional/intentional SystemStackErrors/crashes
+        return unexpected_error if exceeded_recursion_depth?(recursion_depth)
+
+        return valid_snippet if config_schema.nil? || config_schema.empty?
+
+        schema_fields = config_schema[:fields].is_a?(Hash) ? config_schema.dig(:fields, :values) : config_schema[:fields]
+        snippet_field_names = advanced_snippet&.keys&.map(&:to_s)
+        schema_field_names = schema_fields.map { |field| field[:name] }
+
+        return unexpected_field(schema_field_names, snippet_field_names) if unexpected_field_present?(snippet_field_names, schema_field_names)
+
+        return fields_constraint_violation(config_schema[:fields]) if fields_constraints_violated?(config_schema, advanced_snippet)
+
+        schema_fields.each do |field|
+          name = field[:name]
+          type = field[:type]
+          optional = field[:optional] || false
+
+          snippet_field_value = advanced_snippet.with_indifferent_access[name]
+
+          next if optional && (snippet_field_value.nil? || !snippet_field_value.present?)
+
+          return wrong_names(snippet_field_names, name) unless snippet_field_names.include?(name)
+
+          return wrong_type(name, type, snippet_field_value) if type_error_present?(type, snippet_field_value)
+
+          if field[:fields].present?
+            validation_result = validate_against_schema(field, snippet_field_value, recursion_depth + 1)
+
+            return validation_result unless validation_result[:is_valid]
+          end
+        end
+
+        valid_snippet
+      end
+
+      def fields_constraints_violated?(config_schema, advanced_snippet)
+        return false unless config_schema[:fields].is_a?(Hash)
+
+        constraints = config_schema.dig(:fields, :constraints)
+        constraints = constraints.is_a?(Array) ? constraints : [constraints]
+
+        constraints.each do |constraint|
+          return true unless constraint.call(advanced_snippet)
+        end
+
+        false
+      end
+
+      def type_error_present?(schema_type, snippet_value)
+        return !schema_type.call(snippet_value) if schema_type.is_a?(Proc)
+
+        !snippet_value.is_a?(schema_type)
+      end
+
+      def exceeded_recursion_depth?(recursion_depth)
+        if recursion_depth >= MAX_RECURSION_DEPTH
+          Utility::Logger.warn("Recursion depth for filtering validation exceeded. (Max recursion depth: #{MAX_RECURSION_DEPTH})")
+          return true
+        end
+
+        false
+      end
+
+      def unexpected_field_present?(actual_field_names, expected_field_names)
+        difference = actual_field_names - expected_field_names
+
+        # we have field names, which we didn't expect
+        !difference.empty?
+      end
+
+      def valid_snippet
+        {
+          :state => Core::Filtering::ValidationStatus::VALID,
+          :errors => []
+        }
+      end
+
+      def unexpected_field(expected_fields, actual_fields)
+        {
+          :state => Core::Filtering::ValidationStatus::INVALID,
+          :errors => [
+            {
+              :ids => [ADVANCED_SNIPPET_ID],
+              :messages => ["Encountered unexpected fields '#{actual_fields}'. Expected: '#{expected_fields}'."]
+            }
+          ]
+        }
+      end
+
+      def wrong_type(field_name, expected_type, actual_value)
+        {
+          :state => Core::Filtering::ValidationStatus::INVALID,
+          :errors => [
+            {
+              :ids => [ADVANCED_SNIPPET_ID],
+              :messages => ["Expected field type '#{expected_type.is_a?(Proc) ? 'custom matcher' : expected_type}' for field '#{field_name}', but got value '#{actual_value.inspect}' of type '#{actual_value.class}'."]
+            }
+          ]
+        }
+      end
+
+      def wrong_names(actual_field_names, expected_field_name)
+        {
+          :state => Core::Filtering::ValidationStatus::INVALID,
+          :errors => [
+            {
+              :ids => [ADVANCED_SNIPPET_ID],
+              :messages => ["Expected field name '#{expected_field_name}', but got #{actual_field_names}."]
+            }
+          ]
+        }
+      end
+
+      def fields_constraint_violation(fields)
+        {
+          :state => Core::Filtering::ValidationStatus::INVALID,
+          :errors => [
+            {
+              :ids => [ADVANCED_SNIPPET_ID],
+              :messages => ["A fields constraint was violated for fields: '#{fields[:values].map { |v| v[:name] }}'. Check advanced snippet field constraints."]
+            }
+          ]
+        }
+      end
+
+      def unexpected_error
+        {
+          :state => Core::Filtering::ValidationStatus::INVALID,
+          :errors => [
+            {
+              :ids => [ADVANCED_SNIPPET_ID],
+              :messages => ['Unexpected error. Check logs for details.']
+            }
+          ]
+        }
+      end
+    end
+  end
+end

data/lib/connectors/base/advanced_snippet_validator.rb

@@ -0,0 +1,34 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+# frozen_string_literal: true
+
+require 'utility/logger'
+
+module Connectors
+  module Base
+    class AdvancedSnippetValidator
+
+      def initialize(advanced_snippet)
+        @advanced_snippet = advanced_snippet || {}
+      end
+
+      def is_snippet_valid?
+        raise 'Advanced Snippet validation not implemented'
+      end
+
+      private
+
+      def log_validation_result(validation_result)
+        Utility::Logger.info("Filtering Advanced Configuration validation result: #{validation_result[:state]}")
+        if validation_result[:errors].present?
+          validation_result[:errors].each do |error|
+            Utility::Logger.warn("Validation error for: '#{error[:ids]}': '#{error[:messages]}'")
+          end
+        end
+      end
+    end
+  end
+end

data/lib/connectors/base/connector.rb

@@ -6,12 +6,15 @@
 
 # frozen_string_literal: true
 
+require 'active_support/core_ext/hash/indifferent_access'
+require 'app/config'
 require 'bson'
+require 'connectors/base/advanced_snippet_validator'
 require 'core/ingestion'
+require 'connectors/tolerable_error_helper'
+require 'core/filtering/validation_status'
 require 'utility'
 require 'utility/filtering'
-require 'app/config'
-require 'active_support/core_ext/hash/indifferent_access'
 
 module Connectors
   module Base
@@ -40,24 +43,43 @@ module Connectors
         ]
       end
 
-      def self.validate_filtering(_filtering = {})
-        raise 'Not implemented for this connector'
+      def self.advanced_snippet_validator
+        AdvancedSnippetValidator
+      end
+
+      def self.validate_filtering(filtering = {})
+        # nothing to validate
+        return { :state => Core::Filtering::ValidationStatus::VALID, :errors => [] } unless filtering.present?
+
+        filter = Utility::Filtering.extract_filter(filtering)
+        advanced_snippet = filter.dig(:advanced_snippet, :value)
+
+        snippet_validator_instance = advanced_snippet_validator.new(advanced_snippet)
+
+        snippet_validator_instance.is_snippet_valid?
       end
 
       attr_reader :rules, :advanced_filter_config
 
       def initialize(configuration: {}, job_description: {})
+        error_monitor = Utility::ErrorMonitor.new
+        @tolerable_error_helper = Connectors::TolerableErrorHelper.new(error_monitor)
+
         @configuration = configuration.dup || {}
         @job_description = job_description&.dup || {}
 
         filtering = Utility::Filtering.extract_filter(@job_description.dig(:connector, :filtering))
 
         @rules = filtering[:rules] || []
-        @advanced_filter_config = filtering[:advanced_snippet] || {}
+        @advanced_filter_config = filtering.dig(:advanced_snippet, :value) || {}
       end
 
       def yield_documents; end
 
+      def yield_with_handling_tolerable_errors(identifier: nil, &block)
+        @tolerable_error_helper.yield_single_document(identifier: identifier, &block)
+      end
+
       def do_health_check
         raise 'Not implemented for this connector'
       end

data/lib/connectors/example/connector.rb

@@ -7,6 +7,7 @@
 # frozen_string_literal: true
 
 require 'connectors/base/connector'
+require 'connectors/example/example_advanced_snippet_validator'
 require 'core/filtering/validation_status'
 require 'utility'
 
@@ -46,18 +47,8 @@ module Connectors
         # raise 'something went wrong'
       end
 
-      def self.validate_filtering(filtering = {})
-        # TODO: real filtering validation will follow later
-        errors = [
-          {
-            :ids => ['missing-implementation'],
-            :messages => ['Filtering is not implemented yet for the example connector']
-          }
-        ]
-
-        return { :state => Core::Filtering::ValidationStatus::INVALID, :errors => errors } if filtering.present?
-
-        { :state => Core::Filtering::ValidationStatus::VALID, :errors => [] }
+      def self.advanced_snippet_validator
+        ExampleAdvancedSnippetValidator
       end
 
       def yield_documents

data/lib/connectors/example/example_advanced_snippet_validator.rb

@@ -0,0 +1,35 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'connectors/base/advanced_snippet_validator'
+
+module Connectors
+  module Example
+    class ExampleAdvancedSnippetValidator < Connectors::Base::AdvancedSnippetValidator
+
+      def is_snippet_valid?
+        # TODO: real filtering validation will follow later
+        errors = [
+          {
+            :ids => ['missing-implementation'],
+            :messages => ['Filtering is not implemented yet for the example connector']
+          }
+        ]
+
+        validation_result = if @advanced_snippet.present? && !@advanced_snippet.empty?
+                              { :state => Core::Filtering::ValidationStatus::INVALID, :errors => errors }
+                            else
+                              { :state => Core::Filtering::ValidationStatus::VALID, :errors => [] }
+                            end
+        log_validation_result(validation_result)
+        validation_result
+      end
+
+    end
+  end
+end

data/lib/connectors/gitlab/connector.rb

@@ -11,6 +11,7 @@ require 'connectors/base/connector'
 require 'connectors/gitlab/extractor'
 require 'connectors/gitlab/custom_client'
 require 'connectors/gitlab/adapter'
+require 'connectors/gitlab/gitlab_advanced_snippet_validator'
 require 'core/ingestion'
 
 module Connectors
@@ -36,18 +37,8 @@ module Connectors
         }
       end
 
-      def self.validate_filtering(filtering = {})
-        # TODO: real filtering validation will follow later
-        errors = [
-          {
-            :ids => ['missing-implementation'],
-            :messages => ['Filtering is not implemented yet for the GitLab connector']
-          }
-        ]
-
-        return { :state => Core::Filtering::ValidationStatus::INVALID, :errors => errors } if filtering.present?
-
-        { :state => Core::Filtering::ValidationStatus::VALID, :errors => [] }
+      def self.advanced_snippet_validator
+        GitLabAdvancedSnippetValidator
       end
 
       def initialize(configuration: {}, job_description: {})

data/lib/connectors/gitlab/gitlab_advanced_snippet_validator.rb

@@ -0,0 +1,35 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'connectors/base/advanced_snippet_validator'
+
+module Connectors
+  module GitLab
+    class GitLabAdvancedSnippetValidator < Connectors::Base::AdvancedSnippetValidator
+
+      def is_snippet_valid?
+        # TODO: real filtering validation will follow later
+        errors = [
+          {
+            :ids => ['missing-implementation'],
+            :messages => ['Filtering is not implemented yet for the GitLab connector']
+          }
+        ]
+
+        validation_result = if @advanced_snippet.present? && !@advanced_snippet.empty?
+                              { :state => Core::Filtering::ValidationStatus::INVALID, :errors => errors }
+                            else
+                              { :state => Core::Filtering::ValidationStatus::VALID, :errors => [] }
+                            end
+        log_validation_result(validation_result)
+        validation_result
+      end
+
+    end
+  end
+end

data/lib/connectors/mongodb/connector.rb

@@ -9,6 +9,7 @@
 require 'connectors/base/connector'
 require 'core/filtering/validation_status'
 require 'connectors/mongodb/mongo_rules_parser'
+require 'connectors/mongodb/mongo_advanced_snippet_against_schema_validator'
 require 'mongo'
 require 'utility'
 
@@ -51,23 +52,8 @@ module Connectors
         }
       end
 
-      def self.validate_filtering(filtering = {})
-        valid_filtering = { :state => Core::Filtering::ValidationStatus::VALID, :errors => [] }
-
-        return valid_filtering unless filtering.present?
-
-        filter = Utility::Filtering.extract_filter(filtering)
-
-        advanced_filter_config = filter[:advanced_snippet] || {}
-        filter_keys = advanced_filter_config&.keys
-
-        if !filter_keys&.empty? && (filter_keys.size != 1 || !ALLOWED_TOP_LEVEL_FILTER_KEYS.include?(filter_keys[0]&.to_s))
-          return { :state => Core::Filtering::ValidationStatus::INVALID,
-                   :errors => [{ :ids => ['wrong-keys'],
-                                 :messages => ["Only one of #{ALLOWED_TOP_LEVEL_FILTER_KEYS} is allowed in the filtering object. Keys present: '#{filter_keys}'."] }] }
-        end
-
-        valid_filtering
+      def self.advanced_snippet_validator
+        MongoAdvancedSnippetAgainstSchemaValidator
       end
 
       def initialize(configuration: {}, job_description: {})
@@ -111,13 +97,12 @@ module Connectors
             Utility::Logger.info("Requesting #{PAGE_SIZE} documents from MongoDB (Starting at #{skip})")
             view = cursor.skip(skip).limit(PAGE_SIZE)
             view.each do |document|
-              yield serialize(document)
-
-              found_in_page += 1
-              found_overall += 1
-
-              overall_limit_reached = found_overall >= overall_limit && overall_limit != Float::INFINITY
-
+              yield_with_handling_tolerable_errors do
+                yield serialize(document)
+                found_in_page += 1
+                found_overall += 1
+                overall_limit_reached = found_overall >= overall_limit && overall_limit != Float::INFINITY
+              end
               break if overall_limit_reached
             end
 

data/lib/connectors/mongodb/mongo_advanced_snippet_against_schema_validator.rb

@@ -0,0 +1,22 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'connectors/base/advanced_snippet_against_schema_validator'
+require 'connectors/mongodb/mongo_advanced_snippet_schema'
+
+module Connectors
+  module MongoDB
+    class MongoAdvancedSnippetAgainstSchemaValidator < Connectors::Base::AdvancedSnippetAgainstSchemaValidator
+
+      def initialize(advanced_snippet, schema = Connectors::MongoDB::AdvancedSnippet::SCHEMA)
+        super
+      end
+
+    end
+  end
+end