connectors_service 8.6.0.4 → 8.7.0.0.pre.20221117T004928Z

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 43cb60284b3e1c74a6f34612c0f7509cb4be0375d5871c6b59d7338495c88236
-  data.tar.gz: 3f176c8770fc9b450a244b5f8cc7a5857668a452c17e19a07ddb412dcb8d2452
+  metadata.gz: 39e0bb6ae283bcb1afebd57e715342b7e30a7178c33653dfd05fee6f322061e6
+  data.tar.gz: e1fd33b8d28d7d906b4b8eea672a34c0be12e0a124977bbeb10c62c147690a8b
 SHA512:
-  metadata.gz: 59746d5ae600b0a77aaede033f5b371cd922e3a4e95f61afda146f5b6e0962026c02c987fc43c203c5274381539f5c76e5ef895fce593ef1883fa6cf620a1702
-  data.tar.gz: ee799a965b0f5b028c2a41003069d3b7b1e49cabe38453629af8a382ed7ac92b76fe835986c7e5157c1a06b1c2ebae84c20606319bbe7b0aad9a771b12a3f977
+  metadata.gz: bb6b96ac0c07e9ed4cbe87f16a17844771c12d7a7ccbb2e8c7aeb571cb146f67e5a0489b1e73ff907613dcda442912ba09c193e0e8e4e11fec553ebe9afdec12
+  data.tar.gz: c3c2fc12ac9de6ed37a85dbfb46725e2de37c08beba453c3e9d8a3f0c6c56a078a1c5987931251ca84f18e5a9388b6698ff43abe3ce547d291537b3040b15b0a

data/config/connectors.yml

@@ -1,10 +1,11 @@
 # general metadata
-version: 8.6.0.4-20221104T200420Z
-repository: git@github.com:elastic/ent-search-connectors.git
-revision: 2051b3907639a1fe2ae68efdc33c06cf12d38383
+version: 8.7.0.0-20221117T004928Z
+repository: https://github.com/elastic/connectors-ruby.git
+revision: 294214a26b0fe9a4347763b01de681c336e8daae
 elasticsearch:
+  cloud_id: CHANGEME
   hosts: http://localhost:9200
-  api_key: OW1FalJJUUI1clBtUVh5RVo1QmU6QVp5LV9pU3RRUXFYb2VVYnlCRWNZdw==
+  api_key: CHANGEME
   retry_on_failure: 3
   request_timeout: 120
   disable_warnings: true
@@ -15,10 +16,10 @@ thread_pool:
   max_threads: 5
   max_queue: 100
 log_level: info
-ecs_logging: false
+ecs_logging: true
 poll_interval: 3
 termination_timeout: 60
 heartbeat_interval: 1800
-native_mode: false
-connector_id: 9WEjRIQB5rPmQXyEWJB2
-service_type: example
+native_mode: true
+connector_id: CHANGEME
+service_type: CHANGEME

data/lib/app/app.rb

@@ -17,6 +17,10 @@ require 'utility/logger'
 module App
   Utility::Environment.set_execution_environment(App::Config) do
     App::PreflightCheck.run!
+
+    # set exit hook
+    Kernel.at_exit { App::Dispatcher.shutdown! }
+
     App::Dispatcher.start!
   rescue App::PreflightCheck::CheckFailure => e
     Utility::Logger.error("Preflight check failed: #{e.message}")

data/lib/app/config.rb

@@ -54,6 +54,9 @@ puts "Parsing #{CONFIG_FILE} configuration file."
     optional(:poll_interval).value(:integer)
     optional(:termination_timeout).value(:integer)
     optional(:heartbeat_interval).value(:integer)
+
+    optional(:max_ingestion_queue_size).value(:integer) # items
+    optional(:max_ingestion_queue_bytes).value(:integer) # bytes
   end
 end
 

data/lib/app/dispatcher.rb

@@ -28,6 +28,10 @@ module App
       def start!
         running!
         Utility::Logger.info("Starting connector service in #{App::Config.native_mode ? 'native' : 'non-native'} mode...")
+
+        # start sync jobs consumer
+        start_consumer!
+
         start_polling_jobs!
       end
 
@@ -37,6 +41,8 @@ module App
         scheduler.shutdown
         pool.shutdown
         pool.wait_for_termination(TERMINATION_TIMEOUT)
+
+        stop_consumer!
       end
 
       private
@@ -68,11 +74,16 @@ module App
         scheduler.when_triggered do |connector_settings, task|
           case task
           when :sync
-            start_sync_task(connector_settings)
+            # update connector sync_now flag
+            Core::ElasticConnectorActions.update_connector_sync_now(connector_settings.id, false)
+
+            Core::Jobs::Producer.enqueue_job(job_type: :sync, connector_settings: connector_settings)
           when :heartbeat
             start_heartbeat_task(connector_settings)
           when :configuration
             start_configuration_task(connector_settings)
+          when :filter_validation
+            start_filter_validation_task(connector_settings)
           else
             Utility::Logger.error("Unknown task type: #{task}. Skipping...")
           end
@@ -81,22 +92,6 @@ module App
         Utility::ExceptionTracking.log_exception(e, 'The connector service failed due to unexpected error.')
       end
 
-      def start_sync_task(connector_settings)
-        start_heartbeat_task(connector_settings)
-        pool.post do
-          Utility::Logger.info("Initiating a sync job for #{connector_settings.formatted}...")
-          Core::ElasticConnectorActions.ensure_content_index_exists(connector_settings.index_name)
-          job_runner = Core::SyncJobRunner.new(connector_settings)
-          job_runner.execute
-        rescue Core::JobAlreadyRunningError
-          Utility::Logger.info("Sync job for #{connector_settings.formatted} is already running, skipping.")
-        rescue Core::ConnectorVersionChangedError => e
-          Utility::Logger.info("Could not start the job because #{connector_settings.formatted} has been updated externally. Message: #{e.message}")
-        rescue StandardError => e
-          Utility::ExceptionTracking.log_exception(e, "Sync job for #{connector_settings.formatted} failed due to unexpected error.")
-        end
-      end
-
       def start_heartbeat_task(connector_settings)
         pool.post do
           Utility::Logger.info("Sending heartbeat for #{connector_settings.formatted}...")
@@ -120,6 +115,38 @@ module App
           Utility::ExceptionTracking.log_exception(e, "Configuration task for #{connector_settings.formatted} failed due to unexpected error.")
         end
       end
+
+      def start_filter_validation_task(connector_settings)
+        pool.post do
+          Utility::Logger.info("Validating filters for #{connector_settings.formatted}...")
+          validation_job_runner = Core::Filtering::ValidationJobRunner.new(connector_settings)
+          validation_job_runner.execute
+        rescue StandardError => e
+          Utility::ExceptionTracking.log_exception(e, "Filter validation task for #{connector_settings.formatted} failed due to unexpected error.")
+        end
+      end
+
+      def start_consumer!
+        @consumer = Core::Jobs::Consumer.new(
+          poll_interval: POLL_INTERVAL,
+          termination_timeout: TERMINATION_TIMEOUT,
+          min_threads: MIN_THREADS,
+          max_threads: MAX_THREADS,
+          max_queue: MAX_QUEUE,
+          max_ingestion_queue_size: (App::Config.max_ingestion_queue_size || Utility::Constants::DEFAULT_MAX_INGESTION_QUEUE_SIZE).to_i,
+          max_ingestion_queue_bytes: (App::Config.max_ingestion_queue_bytes || Utility::Constants::DEFAULT_MAX_INGESTION_QUEUE_BYTES).to_i,
+          scheduler: scheduler
+        )
+
+        @consumer.subscribe!(index_name: Utility::Constants::JOB_INDEX)
+      end
+
+      def stop_consumer!
+        return if @consumer.nil?
+        return unless @consumer.running?
+
+        @consumer.shutdown!
+      end
     end
   end
 end

data/lib/app/preflight_check.rb

@@ -23,6 +23,7 @@ module App
         check_es_connection!
         check_es_version!
         check_system_indices!
+        check_single_connector!
       end
 
       private
@@ -59,6 +60,16 @@ module App
         )
       end
 
+      #-------------------------------------------------------------------------------------------------
+      # Ensures the connector is supported when running in non-native mode
+      def check_single_connector!
+        if App::Config.native_mode
+          Utility::Logger.info('Skip single connector check for native mode.')
+        elsif !Connectors::REGISTRY.registered?(App::Config.service_type)
+          fail_check!("The service type #{App::Config.service_type} is not supported. Terminating...")
+        end
+      end
+
       def check_es_connection_with_retries!(retry_interval:, retry_timeout:)
         started_at = Time.now
 

data/lib/connectors/base/advanced_snippet_against_schema_validator.rb

@@ -0,0 +1,173 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+# frozen_string_literal: true
+
+require 'active_support/core_ext/hash'
+require 'utility/logger'
+require 'connectors/base/advanced_snippet_validator'
+require 'core/filtering/validation_status'
+
+module Connectors
+  module Base
+    class AdvancedSnippetAgainstSchemaValidator < Connectors::Base::AdvancedSnippetValidator
+
+      MAX_RECURSION_DEPTH = 50
+      ADVANCED_SNIPPET_ID = 'advanced_snippet'
+
+      def initialize(advanced_snippet, schema)
+        super(advanced_snippet)
+        @schema = schema
+      end
+
+      def is_snippet_valid?
+        validation_result = validate_against_schema(@schema, @advanced_snippet)
+        log_validation_result(validation_result)
+        validation_result
+      end
+
+      private
+
+      def validate_against_schema(config_schema, advanced_snippet, recursion_depth = 0)
+        # Prevent unintentional/intentional SystemStackErrors/crashes
+        return unexpected_error if exceeded_recursion_depth?(recursion_depth)
+
+        return valid_snippet if config_schema.nil? || config_schema.empty?
+
+        schema_fields = config_schema[:fields].is_a?(Hash) ? config_schema.dig(:fields, :values) : config_schema[:fields]
+        snippet_field_names = advanced_snippet&.keys&.map(&:to_s)
+        schema_field_names = schema_fields.map { |field| field[:name] }
+
+        return unexpected_field(schema_field_names, snippet_field_names) if unexpected_field_present?(snippet_field_names, schema_field_names)
+
+        return fields_constraint_violation(config_schema[:fields]) if fields_constraints_violated?(config_schema, advanced_snippet)
+
+        schema_fields.each do |field|
+          name = field[:name]
+          type = field[:type]
+          optional = field[:optional] || false
+
+          snippet_field_value = advanced_snippet.with_indifferent_access[name]
+
+          next if optional && (snippet_field_value.nil? || !snippet_field_value.present?)
+
+          return wrong_names(snippet_field_names, name) unless snippet_field_names.include?(name)
+
+          return wrong_type(name, type, snippet_field_value) if type_error_present?(type, snippet_field_value)
+
+          if field[:fields].present?
+            validation_result = validate_against_schema(field, snippet_field_value, recursion_depth + 1)
+
+            return validation_result unless validation_result[:is_valid]
+          end
+        end
+
+        valid_snippet
+      end
+
+      def fields_constraints_violated?(config_schema, advanced_snippet)
+        return false unless config_schema[:fields].is_a?(Hash)
+
+        constraints = config_schema.dig(:fields, :constraints)
+        constraints = constraints.is_a?(Array) ? constraints : [constraints]
+
+        constraints.each do |constraint|
+          return true unless constraint.call(advanced_snippet)
+        end
+
+        false
+      end
+
+      def type_error_present?(schema_type, snippet_value)
+        return !schema_type.call(snippet_value) if schema_type.is_a?(Proc)
+
+        !snippet_value.is_a?(schema_type)
+      end
+
+      def exceeded_recursion_depth?(recursion_depth)
+        if recursion_depth >= MAX_RECURSION_DEPTH
+          Utility::Logger.warn("Recursion depth for filtering validation exceeded. (Max recursion depth: #{MAX_RECURSION_DEPTH})")
+          return true
+        end
+
+        false
+      end
+
+      def unexpected_field_present?(actual_field_names, expected_field_names)
+        difference = actual_field_names - expected_field_names
+
+        # we have field names, which we didn't expect
+        !difference.empty?
+      end
+
+      def valid_snippet
+        {
+          :state => Core::Filtering::ValidationStatus::VALID,
+          :errors => []
+        }
+      end
+
+      def unexpected_field(expected_fields, actual_fields)
+        {
+          :state => Core::Filtering::ValidationStatus::INVALID,
+          :errors => [
+            {
+              :ids => [ADVANCED_SNIPPET_ID],
+              :messages => ["Encountered unexpected fields '#{actual_fields}'. Expected: '#{expected_fields}'."]
+            }
+          ]
+        }
+      end
+
+      def wrong_type(field_name, expected_type, actual_value)
+        {
+          :state => Core::Filtering::ValidationStatus::INVALID,
+          :errors => [
+            {
+              :ids => [ADVANCED_SNIPPET_ID],
+              :messages => ["Expected field type '#{expected_type.is_a?(Proc) ? 'custom matcher' : expected_type}' for field '#{field_name}', but got value '#{actual_value.inspect}' of type '#{actual_value.class}'."]
+            }
+          ]
+        }
+      end
+
+      def wrong_names(actual_field_names, expected_field_name)
+        {
+          :state => Core::Filtering::ValidationStatus::INVALID,
+          :errors => [
+            {
+              :ids => [ADVANCED_SNIPPET_ID],
+              :messages => ["Expected field name '#{expected_field_name}', but got #{actual_field_names}."]
+            }
+          ]
+        }
+      end
+
+      def fields_constraint_violation(fields)
+        {
+          :state => Core::Filtering::ValidationStatus::INVALID,
+          :errors => [
+            {
+              :ids => [ADVANCED_SNIPPET_ID],
+              :messages => ["A fields constraint was violated for fields: '#{fields[:values].map { |v| v[:name] }}'. Check advanced snippet field constraints."]
+            }
+          ]
+        }
+      end
+
+      def unexpected_error
+        {
+          :state => Core::Filtering::ValidationStatus::INVALID,
+          :errors => [
+            {
+              :ids => [ADVANCED_SNIPPET_ID],
+              :messages => ['Unexpected error. Check logs for details.']
+            }
+          ]
+        }
+      end
+    end
+  end
+end

data/lib/connectors/base/advanced_snippet_validator.rb

@@ -0,0 +1,34 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+# frozen_string_literal: true
+
+require 'utility/logger'
+
+module Connectors
+  module Base
+    class AdvancedSnippetValidator
+
+      def initialize(advanced_snippet)
+        @advanced_snippet = advanced_snippet || {}
+      end
+
+      def is_snippet_valid?
+        raise 'Advanced Snippet validation not implemented'
+      end
+
+      private
+
+      def log_validation_result(validation_result)
+        Utility::Logger.info("Filtering Advanced Configuration validation result: #{validation_result[:state]}")
+        if validation_result[:errors].present?
+          validation_result[:errors].each do |error|
+            Utility::Logger.warn("Validation error for: '#{error[:ids]}': '#{error[:messages]}'")
+          end
+        end
+      end
+    end
+  end
+end

data/lib/connectors/base/connector.rb

@@ -6,11 +6,15 @@
 
 # frozen_string_literal: true
 
+require 'active_support/core_ext/hash/indifferent_access'
+require 'app/config'
 require 'bson'
-require 'core/output_sink'
+require 'connectors/base/advanced_snippet_validator'
+require 'core/ingestion'
+require 'connectors/tolerable_error_helper'
+require 'core/filtering/validation_status'
 require 'utility'
-require 'app/config'
-require 'active_support/core_ext/hash/indifferent_access'
+require 'utility/filtering'
 
 module Connectors
   module Base
@@ -32,20 +36,50 @@ module Connectors
         raise 'Not implemented for this connector'
       end
 
+      def self.kibana_features
+        [
+          Utility::Constants::FILTERING_RULES_FEATURE,
+          Utility::Constants::FILTERING_ADVANCED_FEATURE
+        ]
+      end
+
+      def self.advanced_snippet_validator
+        AdvancedSnippetValidator
+      end
+
+      def self.validate_filtering(filtering = {})
+        # nothing to validate
+        return { :state => Core::Filtering::ValidationStatus::VALID, :errors => [] } unless filtering.present?
+
+        filter = Utility::Filtering.extract_filter(filtering)
+        advanced_snippet = filter.dig(:advanced_snippet, :value)
+
+        snippet_validator_instance = advanced_snippet_validator.new(advanced_snippet)
+
+        snippet_validator_instance.is_snippet_valid?
+      end
+
       attr_reader :rules, :advanced_filter_config
 
       def initialize(configuration: {}, job_description: {})
+        error_monitor = Utility::ErrorMonitor.new
+        @tolerable_error_helper = Connectors::TolerableErrorHelper.new(error_monitor)
+
         @configuration = configuration.dup || {}
         @job_description = job_description&.dup || {}
 
-        filter = get_filter(@job_description[:filtering])
+        filtering = Utility::Filtering.extract_filter(@job_description.dig(:connector, :filtering))
 
-        @rules = Utility::Common.return_if_present(filter[:rules], [])
-        @advanced_filter_config = Utility::Common.return_if_present(filter[:advanced_config], {})
+        @rules = filtering[:rules] || []
+        @advanced_filter_config = filtering.dig(:advanced_snippet, :value) || {}
       end
 
       def yield_documents; end
 
+      def yield_with_handling_tolerable_errors(identifier: nil, &block)
+        @tolerable_error_helper.yield_single_document(identifier: identifier, &block)
+      end
+
       def do_health_check
         raise 'Not implemented for this connector'
       end
@@ -67,16 +101,11 @@ module Connectors
       end
 
       def filtering_present?
-        @advanced_filter_config.present? || @rules.present?
+        @advanced_filter_config.present? && !@advanced_filter_config.empty? || @rules.present?
       end
 
-      private
-
-      def get_filter(filtering)
-        # assume for now, that first object in filtering array or a filter object itself is the only filtering object
-        filter = filtering.is_a?(Array) ? filtering[0] : filtering
-
-        filter.present? ? filter : {}
+      def metadata
+        {}
       end
     end
   end

data/lib/connectors/base/simple_rules_parser.rb

@@ -0,0 +1,42 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+# frozen_string_literal: true
+
+require 'active_support/core_ext/hash/indifferent_access'
+require 'active_support/core_ext/object/blank'
+require 'core/filtering/simple_rule'
+
+module Connectors
+  module Base
+    class SimpleRulesParser
+      def initialize(rules)
+        @rules = (rules || []).map(&:with_indifferent_access).filter { |r| r[:id] != 'DEFAULT' }.sort_by { |r| r[:order] }
+      end
+
+      def parse
+        merge_rules(@rules.map do |rule_hash|
+          rule = Core::Filtering::SimpleRule.new(rule_hash)
+          unless rule.is_include? || rule.is_exclude?
+            raise "Unknown policy: #{rule.policy}"
+          end
+          parse_rule(rule)
+        end)
+      end
+
+      private
+
+      # merge all rules into a filter object or array
+      # in a base case, does no transformations
+      def merge_rules(rules)
+        rules || []
+      end
+
+      def parse_rule(_rule)
+        raise 'Not implemented'
+      end
+    end
+  end
+end

data/lib/connectors/example/connector.rb

@@ -7,6 +7,8 @@
 # frozen_string_literal: true
 
 require 'connectors/base/connector'
+require 'connectors/example/example_advanced_snippet_validator'
+require 'core/filtering/validation_status'
 require 'utility'
 
 module Connectors
@@ -45,6 +47,10 @@ module Connectors
         # raise 'something went wrong'
       end
 
+      def self.advanced_snippet_validator
+        ExampleAdvancedSnippetValidator
+      end
+
       def yield_documents
         attachments = [
           load_attachment('first_attachment.txt'),

data/lib/connectors/example/example_advanced_snippet_validator.rb

@@ -0,0 +1,35 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'connectors/base/advanced_snippet_validator'
+
+module Connectors
+  module Example
+    class ExampleAdvancedSnippetValidator < Connectors::Base::AdvancedSnippetValidator
+
+      def is_snippet_valid?
+        # TODO: real filtering validation will follow later
+        errors = [
+          {
+            :ids => ['missing-implementation'],
+            :messages => ['Filtering is not implemented yet for the example connector']
+          }
+        ]
+
+        validation_result = if @advanced_snippet.present? && !@advanced_snippet.empty?
+                              { :state => Core::Filtering::ValidationStatus::INVALID, :errors => errors }
+                            else
+                              { :state => Core::Filtering::ValidationStatus::VALID, :errors => [] }
+                            end
+        log_validation_result(validation_result)
+        validation_result
+      end
+
+    end
+  end
+end

data/lib/connectors/gitlab/connector.rb

@@ -11,7 +11,8 @@ require 'connectors/base/connector'
 require 'connectors/gitlab/extractor'
 require 'connectors/gitlab/custom_client'
 require 'connectors/gitlab/adapter'
-require 'core/output_sink'
+require 'connectors/gitlab/gitlab_advanced_snippet_validator'
+require 'core/ingestion'
 
 module Connectors
   module GitLab
@@ -36,6 +37,10 @@ module Connectors
         }
       end
 
+      def self.advanced_snippet_validator
+        GitLabAdvancedSnippetValidator
+      end
+
       def initialize(configuration: {}, job_description: {})
         super
 

data/lib/connectors/gitlab/gitlab_advanced_snippet_validator.rb

@@ -0,0 +1,35 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'connectors/base/advanced_snippet_validator'
+
+module Connectors
+  module GitLab
+    class GitLabAdvancedSnippetValidator < Connectors::Base::AdvancedSnippetValidator
+
+      def is_snippet_valid?
+        # TODO: real filtering validation will follow later
+        errors = [
+          {
+            :ids => ['missing-implementation'],
+            :messages => ['Filtering is not implemented yet for the GitLab connector']
+          }
+        ]
+
+        validation_result = if @advanced_snippet.present? && !@advanced_snippet.empty?
+                              { :state => Core::Filtering::ValidationStatus::INVALID, :errors => errors }
+                            else
+                              { :state => Core::Filtering::ValidationStatus::VALID, :errors => [] }
+                            end
+        log_validation_result(validation_result)
+        validation_result
+      end
+
+    end
+  end
+end