connectors_service 8.7.0.0.pre.20221117T010623Z → 8.11.0.0

data/lib/core/sync_job_runner.rb

@@ -8,6 +8,7 @@
 
 require 'connectors/connector_status'
 require 'connectors/registry'
+require 'connectors/sync_status'
 require 'core/filtering/post_process_engine'
 require 'core/ingestion'
 require 'core/filtering/validation_status'
@@ -20,13 +21,42 @@ module Core
     end
   end
 
+  class ConnectorNotFoundError < StandardError
+    def initialize(connector_id)
+      super("Connector is not found for connector ID '#{connector_id}'.")
+    end
+  end
+
+  class ConnectorJobNotFoundError < StandardError
+    def initialize(job_id)
+      super("Connector job is not found for job ID '#{job_id}'.")
+    end
+  end
+
+  class ConnectorJobCanceledError < StandardError
+    def initialize(job_id)
+      super("Connector job (ID: '#{job_id}') is requested to be canceled.")
+    end
+  end
+
+  class ConnectorJobNotRunningError < StandardError
+    def initialize(job_id, status)
+      super("Connector job (ID: '#{job_id}') is not running but in status of '#{status}'.")
+    end
+  end
+
   class SyncJobRunner
     JOB_REPORTING_INTERVAL = 10
 
     def initialize(connector_settings, job, max_ingestion_queue_size, max_ingestion_queue_bytes)
       @connector_settings = connector_settings
+      @connector_id = connector_settings.id
+      @index_name = job.index_name
+      @service_type = job.service_type
+      @job = job
+      @job_id = job.id
       @sink = Core::Ingestion::EsSink.new(
-        connector_settings.index_name,
+        @index_name,
         @connector_settings.request_pipeline,
         Utility::BulkQueue.new(
           max_ingestion_queue_size,
@@ -34,16 +64,7 @@ module Core
         ),
         max_ingestion_queue_bytes
       )
-      @connector_class = Connectors::REGISTRY.connector_class(connector_settings.service_type)
-      @sync_finished = false
-      @sync_error = nil
-      @status = {
-        :indexed_document_count => 0,
-        :deleted_document_count => 0,
-        :indexed_document_volume => 0,
-        :error => nil
-      }
-      @job = job
+      @connector_class = Connectors::REGISTRY.connector_class(@service_type)
     end
 
     def execute
@@ -54,134 +75,200 @@ module Core
     private
 
     def do_sync!
-      Utility::Logger.info("Claiming a sync job for connector #{@connector_settings.id}.")
-
-      # connector service doesn't support multiple jobs running simultaneously
-      raise Core::JobAlreadyRunningError.new(@connector_settings.id) if @connector_settings.running?
-
-      Core::ElasticConnectorActions.update_connector_last_sync_status(@connector_settings.id, Connectors::SyncStatus::IN_PROGRESS)
-
-      # claim the job
-      @job.make_running!
-
-      job_description = @job.es_source
-      job_id = @job.id
-      job_description['_id'] = job_id
-
-      unless job_id.present?
-        Utility::Logger.error("Failed to claim the job for #{@connector_settings.id}. Please check the logs for the cause of this error.")
-        return
-      end
+      return unless claim_job!
 
       begin
-        Utility::Logger.debug("Successfully claimed job for connector #{@connector_settings.id}.")
-
-        Utility::Logger.info("Checking active filtering for sync job #{job_id} for connector #{@connector_settings.id}.")
-        validate_filtering(job_description.dig(:connector, :filtering))
-        Utility::Logger.debug("Active filtering for sync job #{job_id} for connector #{@connector_settings.id} is valid.")
+        # We want to validate advanced filtering rules even if basic rules are disabled
+        if @connector_settings.any_filtering_feature_enabled?
+          Utility::Logger.info("Checking active filtering for sync job #{@job_id} for connector #{@connector_id}.")
+          validate_filtering(@job.filtering)
+          Utility::Logger.debug("Active filtering for sync job #{@job_id} for connector #{@connector_id} is valid.")
+        end
 
-        connector_instance = Connectors::REGISTRY.connector(@connector_settings.service_type, @connector_settings.configuration, job_description: job_description)
+        @connector_instance = Connectors::REGISTRY.connector(@service_type, @connector_settings.configuration, job_description: @job)
+        @connector_instance.do_health_check!
 
-        connector_instance.do_health_check!
+        @sync_status = nil
+        @sync_error = nil
+        @reporting_cycle_start = Time.now
 
         incoming_ids = []
-        existing_ids = ElasticConnectorActions.fetch_document_ids(@connector_settings.index_name)
-
-        Utility::Logger.debug("#{existing_ids.size} documents are present in index #{@connector_settings.index_name}.")
-
-        post_processing_engine = Core::Filtering::PostProcessEngine.new(job_description)
-        reporting_cycle_start = Time.now
-        Utility::Logger.info('Yielding documents...')
-        connector_instance.yield_documents do |document|
-          document = add_ingest_metadata(document)
-          post_process_result = post_processing_engine.process(document)
-          if post_process_result.is_include?
-            @sink.ingest(document)
-            incoming_ids << document['id']
-          end
+        existing_ids = ElasticConnectorActions.fetch_document_ids(@index_name)
 
-          if Time.now - reporting_cycle_start >= JOB_REPORTING_INTERVAL
-            ElasticConnectorActions.update_sync(job_id, @sink.ingestion_stats.merge(:metadata => connector_instance.metadata))
-            reporting_cycle_start = Time.now
-          end
+        Utility::Logger.debug("#{existing_ids.size} documents are present in index #{@index_name}.")
+
+        post_processing_engine = @connector_settings.filtering_rule_feature_enabled? ? Core::Filtering::PostProcessEngine.new(@job.filtering) : nil
+
+        yield_docs do |document|
+          next if post_processing_engine && !post_processing_engine.process(document).is_include?
+          @sink.ingest(document)
+          incoming_ids << document['id']
         end
 
         ids_to_delete = existing_ids - incoming_ids.uniq
 
-        Utility::Logger.info("Deleting #{ids_to_delete.size} documents from index #{@connector_settings.index_name}.")
+        Utility::Logger.info("Deleting #{ids_to_delete.size} documents from index #{@index_name}.")
 
         ids_to_delete.each do |id|
           @sink.delete(id)
 
-          if Time.now - reporting_cycle_start >= JOB_REPORTING_INTERVAL
-            ElasticConnectorActions.update_sync(job_id, @sink.ingestion_stats.merge(:metadata => connector_instance.metadata))
-            reporting_cycle_start = Time.now
+          periodically do
+            check_job
+            @job.update_metadata(@sink.ingestion_stats, @connector_instance.metadata)
           end
         end
 
         @sink.flush
 
+        # force check at the end
+        check_job
+
         # We use this mechanism for checking, whether an interrupt (or something else lead to the thread not finishing)
         # occurred as most of the time the main execution thread is interrupted and we miss this Signal/Exception here
-        @sync_finished = true
+        @sync_status = Connectors::SyncStatus::COMPLETED
+        @sync_error = nil
+      rescue ConnectorNotFoundError, ConnectorJobNotFoundError, ConnectorJobNotRunningError => e
+        Utility::Logger.error(e.message)
+        @sync_status = Connectors::SyncStatus::ERROR
+        @sync_error = e.message
+      rescue ConnectorJobCanceledError => e
+        Utility::Logger.error(e.message)
+        @sync_status = Connectors::SyncStatus::CANCELED
+        # Cancelation is an expected action and we shouldn't log an error
+        @sync_error = nil
       rescue StandardError => e
+        @sync_status = Connectors::SyncStatus::ERROR
         @sync_error = e.message
         Utility::ExceptionTracking.log_exception(e)
       ensure
         stats = @sink.ingestion_stats
 
         Utility::Logger.debug("Sync stats are: #{stats}")
-
-        @status[:indexed_document_count] = stats[:indexed_document_count]
-        @status[:deleted_document_count] = stats[:deleted_document_count]
-        @status[:indexed_document_volume] = stats[:indexed_document_volume]
-
-        Utility::Logger.info("Upserted #{@status[:indexed_document_count]} documents into #{@connector_settings.index_name}.")
-        Utility::Logger.info("Deleted #{@status[:deleted_document_count]} documents into #{@connector_settings.index_name}.")
+        Utility::Logger.info("Upserted #{stats[:indexed_document_count]} documents into #{@index_name}.")
+        Utility::Logger.info("Deleted #{stats[:deleted_document_count]} documents into #{@index_name}.")
 
         # Make sure to not override a previous error message
-        if !@sync_finished && @sync_error.nil?
-          @sync_error = 'Sync thread didn\'t finish execution. Check connector logs for more details.'
+        @sync_status ||= Connectors::SyncStatus::ERROR
+        @sync_error = 'Sync thread didn\'t finish execution. Check connector logs for more details.' if @sync_status == Connectors::SyncStatus::ERROR && @sync_error.nil?
+
+        # update job if it's still present
+        if reload_job!
+          case @sync_status
+          when Connectors::SyncStatus::COMPLETED
+            @job.done!(stats, @connector_instance&.metadata)
+          when Connectors::SyncStatus::CANCELED
+            @job.cancel!(stats, @connector_instance&.metadata)
+          when Connectors::SyncStatus::ERROR
+            @job.error!(@sync_error, stats, @connector_instance&.metadata)
+          else
+            Utility::Logger.error("The job is supposed to be in one of the terminal statuses (#{Connectors::SyncStatus::TERMINAL_STATUSES.join(', ')}), but it's #{@sync_status}")
+            @sync_status = Connectors::SyncStatus::ERROR
+            @sync_error = 'The job is not ended as expected for unknown reason'
+            @job.error!(@sync_error, stats, @connector_instance&.metadata)
+          end
+          # need to reload the job to get the latest job status
+          reload_job!
         end
 
-        unless connector_instance.nil?
-          metadata = @sink.ingestion_stats.merge(:metadata => connector_instance.metadata)
-          metadata[:total_document_count] = ElasticConnectorActions.document_count(@connector_settings.index_name)
+        # update connector if it's still present
+        if reload_connector!
+          @connector_settings.update_last_sync!(@job)
         end
 
-        ElasticConnectorActions.complete_sync(@connector_settings.id, job_id, metadata, @sync_error)
+        Utility::Logger.info("Completed the job (ID: #{@job_id}) with status: #{@sync_status}#{@sync_error ? " and error: #{@sync_error}" : ''}")
+      end
+    end
+
+    def yield_docs
+      @connector_instance.yield_documents do |document|
+        document = add_ingest_metadata(document)
 
-        if @sync_error
-          Utility::Logger.info("Failed to sync for connector #{@connector_settings.id} with error '#{@sync_error}'.")
-        else
-          Utility::Logger.info("Successfully synced for connector #{@connector_settings.id}.")
+        yield(document) if block_given?
+
+        periodically do
+          check_job
+          @job.update_metadata(@sink.ingestion_stats, @connector_instance.metadata)
         end
       end
     end
 
+    def claim_job!
+      Utility::Logger.info("Claiming job (ID: #{@job_id}) for connector (ID: #{@connector_id}).")
+
+      # connector service doesn't support multiple jobs running simultaneously
+      if @connector_settings.running?
+        Utility::Logger.warn("Failed to claim job (ID: #{@job_id}) for connector (ID: #{@connector_id}), there are already jobs running.")
+        return false
+      end
+
+      begin
+        Core::ElasticConnectorActions.update_connector_sync_start(@connector_id)
+
+        @job.make_running!
+
+        Utility::Logger.info("Successfully claimed job (ID: #{@job_id}) for connector (ID: #{@connector_id}).")
+        true
+      rescue StandardError => e
+        Utility::ExceptionTracking.log_exception(e)
+        Utility::Logger.error("Failed to claim job (ID: #{@job_id}) for connector (ID: #{@connector_id}). Please check the logs for the cause of this error.")
+        false
+      end
+    end
+
     def add_ingest_metadata(document)
+      return document unless @job
       document.tap do |it|
-        it['_extract_binary_content'] = @connector_settings.extract_binary_content? if @connector_settings.extract_binary_content?
-        it['_reduce_whitespace'] = @connector_settings.reduce_whitespace? if @connector_settings.reduce_whitespace?
-        it['_run_ml_inference'] = @connector_settings.run_ml_inference? if @connector_settings.run_ml_inference?
+        it['_extract_binary_content'] = @job.extract_binary_content? if @job.extract_binary_content?
+        it['_reduce_whitespace'] = @job.reduce_whitespace? if @job.reduce_whitespace?
+        it['_run_ml_inference'] = @job.run_ml_inference? if @job.run_ml_inference?
       end
     end
 
     def validate_configuration!
       expected_fields = @connector_class.configurable_fields.keys.map(&:to_s).sort
-      actual_fields = @connector_settings.configuration.keys.map(&:to_s).sort
+      actual_fields = @job.configuration.keys.map(&:to_s).sort
 
-      raise IncompatibleConfigurableFieldsError.new(@connector_class.service_type, expected_fields, actual_fields) if expected_fields != actual_fields
+      raise IncompatibleConfigurableFieldsError.new(@service_type, expected_fields, actual_fields) if expected_fields != actual_fields
     end
 
     def validate_filtering(filtering)
       validation_result = @connector_class.validate_filtering(filtering)
 
-      wrong_state_error = Utility::InvalidFilterConfigError.new("Active filtering is not in valid state (current state: #{validation_result[:state]}) for connector #{@connector_settings.id}. Please check active filtering in connectors index.")
+      wrong_state_error = Utility::InvalidFilterConfigError.new("Active filtering is not in valid state (current state: #{validation_result[:state]}) for connector #{@connector_id}. Please check active filtering in connectors index.")
       raise wrong_state_error if validation_result[:state] != Core::Filtering::ValidationStatus::VALID
 
-      errors_present_error = Utility::InvalidFilterConfigError.new("Active filtering is in valid state, but errors were detected (errors: #{validation_result[:errors]}) for connector #{@connector_settings.id}. Please check active filtering in connectors index.")
+      errors_present_error = Utility::InvalidFilterConfigError.new("Active filtering is in valid state, but errors were detected (errors: #{validation_result[:errors]}) for connector #{@connector_id}. Please check active filtering in connectors index.")
       raise errors_present_error if validation_result[:errors].present?
     end
+
+    def periodically
+      return if Time.now - @reporting_cycle_start < JOB_REPORTING_INTERVAL
+
+      yield if block_given?
+
+      @reporting_cycle_start = Time.now
+    end
+
+    def check_job
+      # raise error if the connector is deleted
+      raise ConnectorNotFoundError.new(@connector_id) unless reload_connector!
+
+      # raise error if the job is deleted
+      raise ConnectorJobNotFoundError.new(@job_id) unless reload_job!
+
+      # raise error if the job is canceled
+      raise ConnectorJobCanceledError.new(@job_id) if @job.canceling?
+
+      # raise error if the job is not in the status in_progress
+      raise ConnectorJobNotRunningError.new(@job_id, @job.status) unless @job.in_progress?
+    end
+
+    def reload_job!
+      @job = ConnectorJob.fetch_by_id(@job_id)
+    end
+
+    def reload_connector!
+      @connector_settings = ConnectorSettings.fetch_by_id(@connector_id)
+    end
   end
 end

data/lib/core.rb

@@ -12,6 +12,7 @@ require 'core/connector_settings'
 require 'core/elastic_connector_actions'
 require 'core/filtering'
 require 'core/heartbeat'
+require 'core/job_cleanup'
 require 'core/scheduler'
 require 'core/single_scheduler'
 require 'core/native_scheduler'

data/lib/utility/bulk_queue.rb

@@ -13,7 +13,7 @@ module Utility
     class QueueOverflowError < StandardError; end
 
     # 500 items or 5MB
-    def initialize(operation_count_threshold = Utility::Constants::DEFAULT_MAX_INGESTION_QUEUE_LENGTH, size_threshold = Utility::Constants::DEFAULT_MAX_INGESTION_QUEUE_BYTES)
+    def initialize(operation_count_threshold = Utility::Constants::DEFAULT_MAX_INGESTION_QUEUE_SIZE, size_threshold = Utility::Constants::DEFAULT_MAX_INGESTION_QUEUE_BYTES)
       @operation_count_threshold = operation_count_threshold.freeze
       @size_threshold = size_threshold.freeze
 

data/lib/utility/constants.rb

@@ -16,8 +16,6 @@ module Utility
     JOB_INDEX = '.elastic-connectors-sync-jobs'
     CONTENT_INDEX_PREFIX = 'search-'
     CRAWLER_SERVICE_TYPE = 'elastic-crawler'
-    FILTERING_RULES_FEATURE = 'filtering_rules'
-    FILTERING_ADVANCED_FEATURE = 'filtering_advanced_config'
 
     # Maximum number of operations in BULK Elasticsearch operation that will ingest the data
     DEFAULT_MAX_INGESTION_QUEUE_SIZE = 500

data/lib/utility/error_monitor.rb

@@ -51,7 +51,7 @@ module Utility
     def note_success
       @consecutive_error_count = 0
       @success_count += 1
-      increment_window_index
+      track_window_error(false)
     end
 
     def note_error(error, id: Time.now.to_i)
@@ -60,10 +60,9 @@ module Utility
       Utility::Logger.debug("Message id: #{id} - #{error_message}\n#{stack_trace}")
       @total_error_count += 1
       @consecutive_error_count += 1
-      @window_errors[@window_index] = true
       @error_queue << DocumentError.new(error.class.name, error_message, stack_trace, id)
       @error_queue = @error_queue.drop(1) if @error_queue.size > @error_queue_size
-      increment_window_index
+      track_window_error(true)
       @last_error = error
 
       raise_if_necessary
@@ -92,10 +91,32 @@ module Utility
     end
 
     def num_errors_in_window
-      @window_errors.count(&:itself).to_f
+      @window_errors.count(true).to_f
     end
 
-    def increment_window_index
+    def track_window_error(is_error)
+      # We keep the errors array of the size @window_size this way, imagine @window_size = 5
+      # Error array inits as falses:
+      # [ false, false, false, false, false ]
+      # Third document raises an error:
+      # [ false, false, true,  false, false ]
+      #                 ^^^^
+      #                 2 % 5 == 2
+      # Fifth document raises an error:
+      # [ false, false, true,  false, true  ]
+      #                               ^^^^
+      #                               4 % 5 == 4
+      # Sixth document raises an error:
+      # [ true, false, true,   false, true  ]
+      #   ^^^^
+      #   5 % 5 == 0
+      #
+      # Eigth document is successful:
+      # [ true, false, false,  false, true  ]
+      #                ^^^^^
+      #                7 % 5 == 2
+      # And so on.
+      @window_errors[@window_index] = is_error
       @window_index = (@window_index + 1) % @window_size
     end
 

data/lib/utility/es_client.rb

@@ -43,6 +43,10 @@ module Utility
       configs[:transport_options] = es_config[:transport_options] if es_config[:transport_options]
       configs[:ca_fingerprint] = es_config[:ca_fingerprint] if es_config[:ca_fingerprint]
 
+      # headers
+      # these are necessary for cloud-hosted native connectors
+      configs[:headers] = es_config[:headers].to_h if es_config[:headers]
+
       # if log or trace is activated, we use the application logger
       configs[:logger] = if configs[:log] || configs[:trace]
                            Utility::Logger.logger

data/lib/utility/filtering.rb

@@ -17,6 +17,10 @@ module Utility
 
         filter.present? ? filter : {}
       end
+
+      def rule_pre_processing_active?(filter)
+        !filter.dig('advanced_snippet', 'value')&.present?
+      end
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: connectors_service
 version: !ruby/object:Gem::Version
-  version: 8.7.0.0.pre.20221117T010623Z
+  version: 8.11.0.0
 platform: ruby
 authors:
 - Elastic
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-11-17 00:00:00.000000000 Z
+date: 2023-11-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 5.2.6
+        version: 6.1.7.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 5.2.6
+        version: 6.1.7.3
 - !ruby/object:Gem::Dependency
   name: attr_extras
   requirement: !ruby/object:Gem::Requirement
@@ -198,14 +198,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 8.5.0
+        version: 8.8.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 8.5.0
+        version: 8.8.0
 - !ruby/object:Gem::Dependency
   name: faraday
   requirement: !ruby/object:Gem::Requirement
@@ -350,16 +350,16 @@ dependencies:
   name: tzinfo
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '2.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '2.0'
 - !ruby/object:Gem::Dependency
   name: tzinfo-data
   requirement: !ruby/object:Gem::Requirement
@@ -396,11 +396,8 @@ files:
 - lib/app/version.rb
 - lib/connectors.rb
 - lib/connectors/base/adapter.rb
-- lib/connectors/base/advanced_snippet_against_schema_validator.rb
-- lib/connectors/base/advanced_snippet_validator.rb
 - lib/connectors/base/connector.rb
 - lib/connectors/base/custom_client.rb
-- lib/connectors/base/simple_rules_parser.rb
 - lib/connectors/connector_status.rb
 - lib/connectors/crawler/scheduler.rb
 - lib/connectors/example/attachments/first_attachment.txt
@@ -413,14 +410,14 @@ files:
 - lib/connectors/gitlab/custom_client.rb
 - lib/connectors/gitlab/extractor.rb
 - lib/connectors/gitlab/gitlab_advanced_snippet_validator.rb
+- lib/connectors/job_trigger_method.rb
 - lib/connectors/mongodb/connector.rb
 - lib/connectors/mongodb/mongo_advanced_snippet_against_schema_validator.rb
 - lib/connectors/mongodb/mongo_advanced_snippet_schema.rb
-- lib/connectors/mongodb/mongo_rules_parser.rb
+- lib/connectors/mongodb/mongo_advanced_snippet_snake_case_transformer.rb
 - lib/connectors/registry.rb
 - lib/connectors/sync_status.rb
 - lib/connectors/tolerable_error_helper.rb
-- lib/connectors_app/\
 - lib/connectors_service.rb
 - lib/connectors_utility.rb
 - lib/core.rb
@@ -429,14 +426,28 @@ files:
 - lib/core/connector_settings.rb
 - lib/core/elastic_connector_actions.rb
 - lib/core/filtering.rb
+- lib/core/filtering/advanced_snippet/advanced_snippet_against_schema_validator.rb
+- lib/core/filtering/advanced_snippet/advanced_snippet_validator.rb
+- lib/core/filtering/filter_validator.rb
+- lib/core/filtering/hash_against_schema_validator.rb
 - lib/core/filtering/post_process_engine.rb
 - lib/core/filtering/post_process_result.rb
-- lib/core/filtering/simple_rule.rb
+- lib/core/filtering/processing_stage.rb
+- lib/core/filtering/simple_rules/simple_rule.rb
+- lib/core/filtering/simple_rules/simple_rules_parser.rb
+- lib/core/filtering/simple_rules/validation/no_conflicting_policies_rules_validator.rb
+- lib/core/filtering/simple_rules/validation/simple_rules_schema.rb
+- lib/core/filtering/simple_rules/validation/simple_rules_validator.rb
+- lib/core/filtering/simple_rules/validation/single_rule_against_schema_validator.rb
+- lib/core/filtering/transform/filter_transformer.rb
+- lib/core/filtering/transform/filter_transformer_facade.rb
+- lib/core/filtering/transform/transformation_target.rb
 - lib/core/filtering/validation_job_runner.rb
 - lib/core/filtering/validation_status.rb
 - lib/core/heartbeat.rb
 - lib/core/ingestion.rb
 - lib/core/ingestion/es_sink.rb
+- lib/core/job_cleanup.rb
 - lib/core/jobs/consumer.rb
 - lib/core/jobs/producer.rb
 - lib/core/native_scheduler.rb
@@ -470,7 +481,7 @@ homepage: https://github.com/elastic/connectors-ruby
 licenses:
 - Elastic-2.0
 metadata: {}
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -481,12 +492,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
 rubygems_version: 3.0.3.1
-signing_key:
+signing_key: 
 specification_version: 4
 summary: Gem containing Elastic connectors service
 test_files: []

data/lib/connectors/base/advanced_snippet_validator.rb

@@ -1,34 +0,0 @@
-#
-# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
-# or more contributor license agreements. Licensed under the Elastic License;
-# you may not use this file except in compliance with the Elastic License.
-#
-# frozen_string_literal: true
-
-require 'utility/logger'
-
-module Connectors
-  module Base
-    class AdvancedSnippetValidator
-
-      def initialize(advanced_snippet)
-        @advanced_snippet = advanced_snippet || {}
-      end
-
-      def is_snippet_valid?
-        raise 'Advanced Snippet validation not implemented'
-      end
-
-      private
-
-      def log_validation_result(validation_result)
-        Utility::Logger.info("Filtering Advanced Configuration validation result: #{validation_result[:state]}")
-        if validation_result[:errors].present?
-          validation_result[:errors].each do |error|
-            Utility::Logger.warn("Validation error for: '#{error[:ids]}': '#{error[:messages]}'")
-          end
-        end
-      end
-    end
-  end
-end