connectors_service 8.6.0.4.pre.20221104T200814Z → 8.6.0.4.pre.20221116T024501Z

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 23b78b5c5228f761b884af1bfd75b163ad912dc53331ce5a64cd3edb8f45ef86
-  data.tar.gz: 5c74e06f315ab9af4161e88fac626e84b6d2789c0b5b080c713e780a4560ef97
+  metadata.gz: 9a610999d9c34543392e1dd9b7b9ab136557992cc1503fa5aa5973186d17af9e
+  data.tar.gz: '086282a4a15ce1b168e6a0add5f78646e1fe89c336b8a257677826cce3677635'
 SHA512:
-  metadata.gz: 75295f0d26061a977089e999190f4974f4929ca92c25c09e7ecbbd1f0117476f758b7f200b719461bdd9e7b9e6193df455d3e52bcec0634fc85a7b512721ab6a
-  data.tar.gz: fa12af986f72d10081245e782a9152a887512e62f48ad525af59e59a4ef72238727e7bfd521f516fd452a2e74a1101c5bd1965f1e9b97f7b236b8ae2ccf626ff
+  metadata.gz: bf0a0a22ea96074bcbb904000ec489a591fc8491bc2d91759bc1a097c60e31c2abad7b118bb97d16bbe80c5b00dbe08f7c65653710c2d6a77d3356607d87e045
+  data.tar.gz: 67ff773ed2ae3f869897b5fb3a2c0dc3fc55c5922c602e032ed5bafa800043bd4b7f4af90fdd055e575ea877f3cf3b9e308d91c58bad59cd6c63dd5eb8ab98f4

data/config/connectors.yml

@@ -1,10 +1,10 @@
 # general metadata
-version: 8.6.0.4-20221104T200814Z
+version: 8.6.0.4-20221116T024501Z
 repository: git@github.com:elastic/ent-search-connectors.git
-revision: 2051b3907639a1fe2ae68efdc33c06cf12d38383
+revision: b3cc1332879a38930a272a63f8c6be1847578204
 elasticsearch:
   hosts: http://localhost:9200
-  api_key: OW1FalJJUUI1clBtUVh5RVo1QmU6QVp5LV9pU3RRUXFYb2VVYnlCRWNZdw==
+  api_key: MDJyaGZZUUJ1ZXdOLTB1ZnQ5aXQ6a2IwQURWSjVUc1NPcjVsOFBuVDBzZw==
   retry_on_failure: 3
   request_timeout: 120
   disable_warnings: true
@@ -14,11 +14,11 @@ thread_pool:
   min_threads: 0
   max_threads: 5
   max_queue: 100
-log_level: info
+log_level: debug
 ecs_logging: false
 poll_interval: 3
 termination_timeout: 60
 heartbeat_interval: 1800
 native_mode: false
-connector_id: 9WEjRIQB5rPmQXyEWJB2
-service_type: example
+connector_id: 0mrhfYQBuewN-0ufptgN
+service_type: mongodb

data/lib/app/app.rb

@@ -17,6 +17,10 @@ require 'utility/logger'
 module App
   Utility::Environment.set_execution_environment(App::Config) do
     App::PreflightCheck.run!
+
+    # set exit hook
+    Kernel.at_exit { App::Dispatcher.shutdown! }
+
     App::Dispatcher.start!
   rescue App::PreflightCheck::CheckFailure => e
     Utility::Logger.error("Preflight check failed: #{e.message}")

data/lib/app/dispatcher.rb

@@ -28,6 +28,10 @@ module App
       def start!
         running!
         Utility::Logger.info("Starting connector service in #{App::Config.native_mode ? 'native' : 'non-native'} mode...")
+
+        # start sync jobs consumer
+        start_consumer!
+
         start_polling_jobs!
       end
 
@@ -37,6 +41,8 @@ module App
         scheduler.shutdown
         pool.shutdown
         pool.wait_for_termination(TERMINATION_TIMEOUT)
+
+        stop_consumer!
       end
 
       private
@@ -68,11 +74,16 @@ module App
         scheduler.when_triggered do |connector_settings, task|
           case task
           when :sync
-            start_sync_task(connector_settings)
+            # update connector sync_now flag
+            Core::ElasticConnectorActions.update_connector_sync_now(connector_settings.id, false)
+
+            Core::Jobs::Producer.enqueue_job(job_type: :sync, connector_settings: connector_settings)
           when :heartbeat
             start_heartbeat_task(connector_settings)
           when :configuration
             start_configuration_task(connector_settings)
+          when :filter_validation
+            start_filter_validation_task(connector_settings)
           else
             Utility::Logger.error("Unknown task type: #{task}. Skipping...")
           end
@@ -81,22 +92,6 @@ module App
         Utility::ExceptionTracking.log_exception(e, 'The connector service failed due to unexpected error.')
       end
 
-      def start_sync_task(connector_settings)
-        start_heartbeat_task(connector_settings)
-        pool.post do
-          Utility::Logger.info("Initiating a sync job for #{connector_settings.formatted}...")
-          Core::ElasticConnectorActions.ensure_content_index_exists(connector_settings.index_name)
-          job_runner = Core::SyncJobRunner.new(connector_settings)
-          job_runner.execute
-        rescue Core::JobAlreadyRunningError
-          Utility::Logger.info("Sync job for #{connector_settings.formatted} is already running, skipping.")
-        rescue Core::ConnectorVersionChangedError => e
-          Utility::Logger.info("Could not start the job because #{connector_settings.formatted} has been updated externally. Message: #{e.message}")
-        rescue StandardError => e
-          Utility::ExceptionTracking.log_exception(e, "Sync job for #{connector_settings.formatted} failed due to unexpected error.")
-        end
-      end
-
       def start_heartbeat_task(connector_settings)
         pool.post do
           Utility::Logger.info("Sending heartbeat for #{connector_settings.formatted}...")
@@ -120,6 +115,36 @@ module App
           Utility::ExceptionTracking.log_exception(e, "Configuration task for #{connector_settings.formatted} failed due to unexpected error.")
         end
       end
+
+      def start_filter_validation_task(connector_settings)
+        pool.post do
+          Utility::Logger.info("Validating filters for #{connector_settings.formatted}...")
+          validation_job_runner = Core::Filtering::ValidationJobRunner.new(connector_settings)
+          validation_job_runner.execute
+        rescue StandardError => e
+          Utility::ExceptionTracking.log_exception(e, "Filter validation task for #{connector_settings.formatted} failed due to unexpected error.")
+        end
+      end
+
+      def start_consumer!
+        @consumer = Core::Jobs::Consumer.new(
+          poll_interval: POLL_INTERVAL,
+          termination_timeout: TERMINATION_TIMEOUT,
+          min_threads: MIN_THREADS,
+          max_threads: MAX_THREADS,
+          max_queue: MAX_QUEUE,
+          scheduler: scheduler
+        )
+
+        @consumer.subscribe!(index_name: Utility::Constants::JOB_INDEX)
+      end
+
+      def stop_consumer!
+        return if @consumer.nil?
+        return unless @consumer.running?
+
+        @consumer.shutdown!
+      end
     end
   end
 end

data/lib/app/preflight_check.rb

@@ -23,6 +23,7 @@ module App
         check_es_connection!
         check_es_version!
         check_system_indices!
+        check_single_connector!
       end
 
       private
@@ -59,6 +60,16 @@ module App
         )
       end
 
+      #-------------------------------------------------------------------------------------------------
+      # Ensures the connector is supported when running in non-native mode
+      def check_single_connector!
+        if App::Config.native_mode
+          Utility::Logger.info('Skip single connector check for native mode.')
+        elsif !Connectors::REGISTRY.registered?(App::Config.service_type)
+          fail_check!("The service type #{App::Config.service_type} is not supported. Terminating...")
+        end
+      end
+
       def check_es_connection_with_retries!(retry_interval:, retry_timeout:)
         started_at = Time.now
 

data/lib/connectors/base/advanced_snippet_against_schema_validator.rb

@@ -0,0 +1,173 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+# frozen_string_literal: true
+
+require 'active_support/core_ext/hash'
+require 'utility/logger'
+require 'connectors/base/advanced_snippet_validator'
+require 'core/filtering/validation_status'
+
+module Connectors
+  module Base
+    class AdvancedSnippetAgainstSchemaValidator < Connectors::Base::AdvancedSnippetValidator
+
+      MAX_RECURSION_DEPTH = 50
+      ADVANCED_SNIPPET_ID = 'advanced_snippet'
+
+      def initialize(advanced_snippet, schema)
+        super(advanced_snippet)
+        @schema = schema
+      end
+
+      def is_snippet_valid?
+        validation_result = validate_against_schema(@schema, @advanced_snippet)
+        log_validation_result(validation_result)
+        validation_result
+      end
+
+      private
+
+      def validate_against_schema(config_schema, advanced_snippet, recursion_depth = 0)
+        # Prevent unintentional/intentional SystemStackErrors/crashes
+        return unexpected_error if exceeded_recursion_depth?(recursion_depth)
+
+        return valid_snippet if config_schema.nil? || config_schema.empty?
+
+        schema_fields = config_schema[:fields].is_a?(Hash) ? config_schema.dig(:fields, :values) : config_schema[:fields]
+        snippet_field_names = advanced_snippet&.keys&.map(&:to_s)
+        schema_field_names = schema_fields.map { |field| field[:name] }
+
+        return unexpected_field(schema_field_names, snippet_field_names) if unexpected_field_present?(snippet_field_names, schema_field_names)
+
+        return fields_constraint_violation(config_schema[:fields]) if fields_constraints_violated?(config_schema, advanced_snippet)
+
+        schema_fields.each do |field|
+          name = field[:name]
+          type = field[:type]
+          optional = field[:optional] || false
+
+          snippet_field_value = advanced_snippet.with_indifferent_access[name]
+
+          next if optional && (snippet_field_value.nil? || !snippet_field_value.present?)
+
+          return wrong_names(snippet_field_names, name) unless snippet_field_names.include?(name)
+
+          return wrong_type(name, type, snippet_field_value) if type_error_present?(type, snippet_field_value)
+
+          if field[:fields].present?
+            validation_result = validate_against_schema(field, snippet_field_value, recursion_depth + 1)
+
+            return validation_result unless validation_result[:is_valid]
+          end
+        end
+
+        valid_snippet
+      end
+
+      def fields_constraints_violated?(config_schema, advanced_snippet)
+        return false unless config_schema[:fields].is_a?(Hash)
+
+        constraints = config_schema.dig(:fields, :constraints)
+        constraints = constraints.is_a?(Array) ? constraints : [constraints]
+
+        constraints.each do |constraint|
+          return true unless constraint.call(advanced_snippet)
+        end
+
+        false
+      end
+
+      def type_error_present?(schema_type, snippet_value)
+        return !schema_type.call(snippet_value) if schema_type.is_a?(Proc)
+
+        !snippet_value.is_a?(schema_type)
+      end
+
+      def exceeded_recursion_depth?(recursion_depth)
+        if recursion_depth >= MAX_RECURSION_DEPTH
+          Utility::Logger.warn("Recursion depth for filtering validation exceeded. (Max recursion depth: #{MAX_RECURSION_DEPTH})")
+          return true
+        end
+
+        false
+      end
+
+      def unexpected_field_present?(actual_field_names, expected_field_names)
+        difference = actual_field_names - expected_field_names
+
+        # we have field names, which we didn't expect
+        !difference.empty?
+      end
+
+      def valid_snippet
+        {
+          :state => Core::Filtering::ValidationStatus::VALID,
+          :errors => []
+        }
+      end
+
+      def unexpected_field(expected_fields, actual_fields)
+        {
+          :state => Core::Filtering::ValidationStatus::INVALID,
+          :errors => [
+            {
+              :ids => [ADVANCED_SNIPPET_ID],
+              :messages => ["Encountered unexpected fields '#{actual_fields}'. Expected: '#{expected_fields}'."]
+            }
+          ]
+        }
+      end
+
+      def wrong_type(field_name, expected_type, actual_value)
+        {
+          :state => Core::Filtering::ValidationStatus::INVALID,
+          :errors => [
+            {
+              :ids => [ADVANCED_SNIPPET_ID],
+              :messages => ["Expected field type '#{expected_type.is_a?(Proc) ? 'custom matcher' : expected_type}' for field '#{field_name}', but got value '#{actual_value.inspect}' of type '#{actual_value.class}'."]
+            }
+          ]
+        }
+      end
+
+      def wrong_names(actual_field_names, expected_field_name)
+        {
+          :state => Core::Filtering::ValidationStatus::INVALID,
+          :errors => [
+            {
+              :ids => [ADVANCED_SNIPPET_ID],
+              :messages => ["Expected field name '#{expected_field_name}', but got #{actual_field_names}."]
+            }
+          ]
+        }
+      end
+
+      def fields_constraint_violation(fields)
+        {
+          :state => Core::Filtering::ValidationStatus::INVALID,
+          :errors => [
+            {
+              :ids => [ADVANCED_SNIPPET_ID],
+              :messages => ["A fields constraint was violated for fields: '#{fields[:values].map { |v| v[:name] }}'. Check advanced snippet field constraints."]
+            }
+          ]
+        }
+      end
+
+      def unexpected_error
+        {
+          :state => Core::Filtering::ValidationStatus::INVALID,
+          :errors => [
+            {
+              :ids => [ADVANCED_SNIPPET_ID],
+              :messages => ['Unexpected error. Check logs for details.']
+            }
+          ]
+        }
+      end
+    end
+  end
+end

data/lib/connectors/base/advanced_snippet_validator.rb

@@ -0,0 +1,34 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+# frozen_string_literal: true
+
+require 'utility/logger'
+
+module Connectors
+  module Base
+    class AdvancedSnippetValidator
+
+      def initialize(advanced_snippet)
+        @advanced_snippet = advanced_snippet || {}
+      end
+
+      def is_snippet_valid?
+        raise 'Advanced Snippet validation not implemented'
+      end
+
+      private
+
+      def log_validation_result(validation_result)
+        Utility::Logger.info("Filtering Advanced Configuration validation result: #{validation_result[:state]}")
+        if validation_result[:errors].present?
+          validation_result[:errors].each do |error|
+            Utility::Logger.warn("Validation error for: '#{error[:ids]}': '#{error[:messages]}'")
+          end
+        end
+      end
+    end
+  end
+end

data/lib/connectors/base/connector.rb

@@ -6,11 +6,15 @@
 
 # frozen_string_literal: true
 
+require 'active_support/core_ext/hash/indifferent_access'
+require 'app/config'
 require 'bson'
-require 'core/output_sink'
+require 'connectors/base/advanced_snippet_validator'
+require 'core/ingestion'
+require 'connectors/tolerable_error_helper'
+require 'core/filtering/validation_status'
 require 'utility'
-require 'app/config'
-require 'active_support/core_ext/hash/indifferent_access'
+require 'utility/filtering'
 
 module Connectors
   module Base
@@ -32,20 +36,50 @@ module Connectors
         raise 'Not implemented for this connector'
       end
 
+      def self.kibana_features
+        [
+          Utility::Constants::FILTERING_RULES_FEATURE,
+          Utility::Constants::FILTERING_ADVANCED_FEATURE
+        ]
+      end
+
+      def self.advanced_snippet_validator
+        AdvancedSnippetValidator
+      end
+
+      def self.validate_filtering(filtering = {})
+        # nothing to validate
+        return { :state => Core::Filtering::ValidationStatus::VALID, :errors => [] } unless filtering.present?
+
+        filter = Utility::Filtering.extract_filter(filtering)
+        advanced_snippet = filter.dig(:advanced_snippet, :value)
+
+        snippet_validator_instance = advanced_snippet_validator.new(advanced_snippet)
+
+        snippet_validator_instance.is_snippet_valid?
+      end
+
       attr_reader :rules, :advanced_filter_config
 
       def initialize(configuration: {}, job_description: {})
+        error_monitor = Utility::ErrorMonitor.new
+        @tolerable_error_helper = Connectors::TolerableErrorHelper.new(error_monitor)
+
         @configuration = configuration.dup || {}
         @job_description = job_description&.dup || {}
 
-        filter = get_filter(@job_description[:filtering])
+        filtering = Utility::Filtering.extract_filter(@job_description.dig(:connector, :filtering))
 
-        @rules = Utility::Common.return_if_present(filter[:rules], [])
-        @advanced_filter_config = Utility::Common.return_if_present(filter[:advanced_config], {})
+        @rules = filtering[:rules] || []
+        @advanced_filter_config = filtering.dig(:advanced_snippet, :value) || {}
       end
 
       def yield_documents; end
 
+      def yield_with_handling_tolerable_errors(identifier: nil, &block)
+        @tolerable_error_helper.yield_single_document(identifier: identifier, &block)
+      end
+
       def do_health_check
         raise 'Not implemented for this connector'
       end
@@ -67,16 +101,11 @@ module Connectors
       end
 
       def filtering_present?
-        @advanced_filter_config.present? || @rules.present?
+        @advanced_filter_config.present? && !@advanced_filter_config.empty? || @rules.present?
       end
 
-      private
-
-      def get_filter(filtering)
-        # assume for now, that first object in filtering array or a filter object itself is the only filtering object
-        filter = filtering.is_a?(Array) ? filtering[0] : filtering
-
-        filter.present? ? filter : {}
+      def metadata
+        {}
       end
     end
   end

data/lib/connectors/base/simple_rules_parser.rb

@@ -0,0 +1,42 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+# frozen_string_literal: true
+
+require 'active_support/core_ext/hash/indifferent_access'
+require 'active_support/core_ext/object/blank'
+require 'core/filtering/simple_rule'
+
+module Connectors
+  module Base
+    class SimpleRulesParser
+      def initialize(rules)
+        @rules = (rules || []).map(&:with_indifferent_access).filter { |r| r[:id] != 'DEFAULT' }.sort_by { |r| r[:order] }
+      end
+
+      def parse
+        merge_rules(@rules.map do |rule_hash|
+          rule = Core::Filtering::SimpleRule.new(rule_hash)
+          unless rule.is_include? || rule.is_exclude?
+            raise "Unknown policy: #{rule.policy}"
+          end
+          parse_rule(rule)
+        end)
+      end
+
+      private
+
+      # merge all rules into a filter object or array
+      # in a base case, does no transformations
+      def merge_rules(rules)
+        rules || []
+      end
+
+      def parse_rule(_rule)
+        raise 'Not implemented'
+      end
+    end
+  end
+end

data/lib/connectors/example/connector.rb

@@ -7,6 +7,8 @@
 # frozen_string_literal: true
 
 require 'connectors/base/connector'
+require 'connectors/example/example_advanced_snippet_validator'
+require 'core/filtering/validation_status'
 require 'utility'
 
 module Connectors
@@ -45,6 +47,10 @@ module Connectors
         # raise 'something went wrong'
       end
 
+      def self.advanced_snippet_validator
+        ExampleAdvancedSnippetValidator
+      end
+
       def yield_documents
         attachments = [
           load_attachment('first_attachment.txt'),

data/lib/connectors/example/example_advanced_snippet_validator.rb

@@ -0,0 +1,35 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'connectors/base/advanced_snippet_validator'
+
+module Connectors
+  module Example
+    class ExampleAdvancedSnippetValidator < Connectors::Base::AdvancedSnippetValidator
+
+      def is_snippet_valid?
+        # TODO: real filtering validation will follow later
+        errors = [
+          {
+            :ids => ['missing-implementation'],
+            :messages => ['Filtering is not implemented yet for the example connector']
+          }
+        ]
+
+        validation_result = if @advanced_snippet.present? && !@advanced_snippet.empty?
+                              { :state => Core::Filtering::ValidationStatus::INVALID, :errors => errors }
+                            else
+                              { :state => Core::Filtering::ValidationStatus::VALID, :errors => [] }
+                            end
+        log_validation_result(validation_result)
+        validation_result
+      end
+
+    end
+  end
+end

data/lib/connectors/gitlab/connector.rb

@@ -11,7 +11,8 @@ require 'connectors/base/connector'
 require 'connectors/gitlab/extractor'
 require 'connectors/gitlab/custom_client'
 require 'connectors/gitlab/adapter'
-require 'core/output_sink'
+require 'connectors/gitlab/gitlab_advanced_snippet_validator'
+require 'core/ingestion'
 
 module Connectors
   module GitLab
@@ -36,6 +37,10 @@ module Connectors
         }
       end
 
+      def self.advanced_snippet_validator
+        GitLabAdvancedSnippetValidator
+      end
+
       def initialize(configuration: {}, job_description: {})
         super
 

data/lib/connectors/gitlab/gitlab_advanced_snippet_validator.rb

@@ -0,0 +1,35 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'connectors/base/advanced_snippet_validator'
+
+module Connectors
+  module GitLab
+    class GitLabAdvancedSnippetValidator < Connectors::Base::AdvancedSnippetValidator
+
+      def is_snippet_valid?
+        # TODO: real filtering validation will follow later
+        errors = [
+          {
+            :ids => ['missing-implementation'],
+            :messages => ['Filtering is not implemented yet for the GitLab connector']
+          }
+        ]
+
+        validation_result = if @advanced_snippet.present? && !@advanced_snippet.empty?
+                              { :state => Core::Filtering::ValidationStatus::INVALID, :errors => errors }
+                            else
+                              { :state => Core::Filtering::ValidationStatus::VALID, :errors => [] }
+                            end
+        log_validation_result(validation_result)
+        validation_result
+      end
+
+    end
+  end
+end