connectors_service 8.6.0.4.pre.20221116T024501Z → 8.6.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9a610999d9c34543392e1dd9b7b9ab136557992cc1503fa5aa5973186d17af9e
-  data.tar.gz: '086282a4a15ce1b168e6a0add5f78646e1fe89c336b8a257677826cce3677635'
+  metadata.gz: 43cb60284b3e1c74a6f34612c0f7509cb4be0375d5871c6b59d7338495c88236
+  data.tar.gz: 3f176c8770fc9b450a244b5f8cc7a5857668a452c17e19a07ddb412dcb8d2452
 SHA512:
-  metadata.gz: bf0a0a22ea96074bcbb904000ec489a591fc8491bc2d91759bc1a097c60e31c2abad7b118bb97d16bbe80c5b00dbe08f7c65653710c2d6a77d3356607d87e045
-  data.tar.gz: 67ff773ed2ae3f869897b5fb3a2c0dc3fc55c5922c602e032ed5bafa800043bd4b7f4af90fdd055e575ea877f3cf3b9e308d91c58bad59cd6c63dd5eb8ab98f4
+  metadata.gz: 59746d5ae600b0a77aaede033f5b371cd922e3a4e95f61afda146f5b6e0962026c02c987fc43c203c5274381539f5c76e5ef895fce593ef1883fa6cf620a1702
+  data.tar.gz: ee799a965b0f5b028c2a41003069d3b7b1e49cabe38453629af8a382ed7ac92b76fe835986c7e5157c1a06b1c2ebae84c20606319bbe7b0aad9a771b12a3f977

data/config/connectors.yml

@@ -1,10 +1,10 @@
 # general metadata
-version: 8.6.0.4-20221116T024501Z
+version: 8.6.0.4-20221104T200420Z
 repository: git@github.com:elastic/ent-search-connectors.git
-revision: b3cc1332879a38930a272a63f8c6be1847578204
+revision: 2051b3907639a1fe2ae68efdc33c06cf12d38383
 elasticsearch:
   hosts: http://localhost:9200
-  api_key: MDJyaGZZUUJ1ZXdOLTB1ZnQ5aXQ6a2IwQURWSjVUc1NPcjVsOFBuVDBzZw==
+  api_key: OW1FalJJUUI1clBtUVh5RVo1QmU6QVp5LV9pU3RRUXFYb2VVYnlCRWNZdw==
   retry_on_failure: 3
   request_timeout: 120
   disable_warnings: true
@@ -14,11 +14,11 @@ thread_pool:
   min_threads: 0
   max_threads: 5
   max_queue: 100
-log_level: debug
+log_level: info
 ecs_logging: false
 poll_interval: 3
 termination_timeout: 60
 heartbeat_interval: 1800
 native_mode: false
-connector_id: 0mrhfYQBuewN-0ufptgN
-service_type: mongodb
+connector_id: 9WEjRIQB5rPmQXyEWJB2
+service_type: example

data/lib/app/app.rb

@@ -17,10 +17,6 @@ require 'utility/logger'
 module App
   Utility::Environment.set_execution_environment(App::Config) do
     App::PreflightCheck.run!
-
-    # set exit hook
-    Kernel.at_exit { App::Dispatcher.shutdown! }
-
     App::Dispatcher.start!
   rescue App::PreflightCheck::CheckFailure => e
     Utility::Logger.error("Preflight check failed: #{e.message}")

data/lib/app/dispatcher.rb

@@ -28,10 +28,6 @@ module App
       def start!
         running!
         Utility::Logger.info("Starting connector service in #{App::Config.native_mode ? 'native' : 'non-native'} mode...")
-
-        # start sync jobs consumer
-        start_consumer!
-
         start_polling_jobs!
       end
 
@@ -41,8 +37,6 @@ module App
         scheduler.shutdown
         pool.shutdown
         pool.wait_for_termination(TERMINATION_TIMEOUT)
-
-        stop_consumer!
       end
 
       private
@@ -74,16 +68,11 @@ module App
         scheduler.when_triggered do |connector_settings, task|
           case task
           when :sync
-            # update connector sync_now flag
-            Core::ElasticConnectorActions.update_connector_sync_now(connector_settings.id, false)
-
-            Core::Jobs::Producer.enqueue_job(job_type: :sync, connector_settings: connector_settings)
+            start_sync_task(connector_settings)
           when :heartbeat
             start_heartbeat_task(connector_settings)
           when :configuration
             start_configuration_task(connector_settings)
-          when :filter_validation
-            start_filter_validation_task(connector_settings)
           else
             Utility::Logger.error("Unknown task type: #{task}. Skipping...")
           end
@@ -92,6 +81,22 @@ module App
         Utility::ExceptionTracking.log_exception(e, 'The connector service failed due to unexpected error.')
       end
 
+      def start_sync_task(connector_settings)
+        start_heartbeat_task(connector_settings)
+        pool.post do
+          Utility::Logger.info("Initiating a sync job for #{connector_settings.formatted}...")
+          Core::ElasticConnectorActions.ensure_content_index_exists(connector_settings.index_name)
+          job_runner = Core::SyncJobRunner.new(connector_settings)
+          job_runner.execute
+        rescue Core::JobAlreadyRunningError
+          Utility::Logger.info("Sync job for #{connector_settings.formatted} is already running, skipping.")
+        rescue Core::ConnectorVersionChangedError => e
+          Utility::Logger.info("Could not start the job because #{connector_settings.formatted} has been updated externally. Message: #{e.message}")
+        rescue StandardError => e
+          Utility::ExceptionTracking.log_exception(e, "Sync job for #{connector_settings.formatted} failed due to unexpected error.")
+        end
+      end
+
       def start_heartbeat_task(connector_settings)
         pool.post do
           Utility::Logger.info("Sending heartbeat for #{connector_settings.formatted}...")
@@ -115,36 +120,6 @@ module App
           Utility::ExceptionTracking.log_exception(e, "Configuration task for #{connector_settings.formatted} failed due to unexpected error.")
         end
       end
-
-      def start_filter_validation_task(connector_settings)
-        pool.post do
-          Utility::Logger.info("Validating filters for #{connector_settings.formatted}...")
-          validation_job_runner = Core::Filtering::ValidationJobRunner.new(connector_settings)
-          validation_job_runner.execute
-        rescue StandardError => e
-          Utility::ExceptionTracking.log_exception(e, "Filter validation task for #{connector_settings.formatted} failed due to unexpected error.")
-        end
-      end
-
-      def start_consumer!
-        @consumer = Core::Jobs::Consumer.new(
-          poll_interval: POLL_INTERVAL,
-          termination_timeout: TERMINATION_TIMEOUT,
-          min_threads: MIN_THREADS,
-          max_threads: MAX_THREADS,
-          max_queue: MAX_QUEUE,
-          scheduler: scheduler
-        )
-
-        @consumer.subscribe!(index_name: Utility::Constants::JOB_INDEX)
-      end
-
-      def stop_consumer!
-        return if @consumer.nil?
-        return unless @consumer.running?
-
-        @consumer.shutdown!
-      end
     end
   end
 end

data/lib/app/preflight_check.rb

@@ -23,7 +23,6 @@ module App
         check_es_connection!
         check_es_version!
         check_system_indices!
-        check_single_connector!
       end
 
       private
@@ -60,16 +59,6 @@ module App
         )
       end
 
-      #-------------------------------------------------------------------------------------------------
-      # Ensures the connector is supported when running in non-native mode
-      def check_single_connector!
-        if App::Config.native_mode
-          Utility::Logger.info('Skip single connector check for native mode.')
-        elsif !Connectors::REGISTRY.registered?(App::Config.service_type)
-          fail_check!("The service type #{App::Config.service_type} is not supported. Terminating...")
-        end
-      end
-
       def check_es_connection_with_retries!(retry_interval:, retry_timeout:)
         started_at = Time.now
 

data/lib/connectors/base/connector.rb

@@ -6,15 +6,11 @@
 
 # frozen_string_literal: true
 
-require 'active_support/core_ext/hash/indifferent_access'
-require 'app/config'
 require 'bson'
-require 'connectors/base/advanced_snippet_validator'
-require 'core/ingestion'
-require 'connectors/tolerable_error_helper'
-require 'core/filtering/validation_status'
+require 'core/output_sink'
 require 'utility'
-require 'utility/filtering'
+require 'app/config'
+require 'active_support/core_ext/hash/indifferent_access'
 
 module Connectors
   module Base
@@ -36,50 +32,20 @@ module Connectors
         raise 'Not implemented for this connector'
       end
 
-      def self.kibana_features
-        [
-          Utility::Constants::FILTERING_RULES_FEATURE,
-          Utility::Constants::FILTERING_ADVANCED_FEATURE
-        ]
-      end
-
-      def self.advanced_snippet_validator
-        AdvancedSnippetValidator
-      end
-
-      def self.validate_filtering(filtering = {})
-        # nothing to validate
-        return { :state => Core::Filtering::ValidationStatus::VALID, :errors => [] } unless filtering.present?
-
-        filter = Utility::Filtering.extract_filter(filtering)
-        advanced_snippet = filter.dig(:advanced_snippet, :value)
-
-        snippet_validator_instance = advanced_snippet_validator.new(advanced_snippet)
-
-        snippet_validator_instance.is_snippet_valid?
-      end
-
       attr_reader :rules, :advanced_filter_config
 
       def initialize(configuration: {}, job_description: {})
-        error_monitor = Utility::ErrorMonitor.new
-        @tolerable_error_helper = Connectors::TolerableErrorHelper.new(error_monitor)
-
         @configuration = configuration.dup || {}
         @job_description = job_description&.dup || {}
 
-        filtering = Utility::Filtering.extract_filter(@job_description.dig(:connector, :filtering))
+        filter = get_filter(@job_description[:filtering])
 
-        @rules = filtering[:rules] || []
-        @advanced_filter_config = filtering.dig(:advanced_snippet, :value) || {}
+        @rules = Utility::Common.return_if_present(filter[:rules], [])
+        @advanced_filter_config = Utility::Common.return_if_present(filter[:advanced_config], {})
       end
 
       def yield_documents; end
 
-      def yield_with_handling_tolerable_errors(identifier: nil, &block)
-        @tolerable_error_helper.yield_single_document(identifier: identifier, &block)
-      end
-
       def do_health_check
         raise 'Not implemented for this connector'
       end
@@ -101,11 +67,16 @@ module Connectors
       end
 
       def filtering_present?
-        @advanced_filter_config.present? && !@advanced_filter_config.empty? || @rules.present?
+        @advanced_filter_config.present? || @rules.present?
       end
 
-      def metadata
-        {}
+      private
+
+      def get_filter(filtering)
+        # assume for now, that first object in filtering array or a filter object itself is the only filtering object
+        filter = filtering.is_a?(Array) ? filtering[0] : filtering
+
+        filter.present? ? filter : {}
       end
     end
   end

data/lib/connectors/example/connector.rb

@@ -7,8 +7,6 @@
 # frozen_string_literal: true
 
 require 'connectors/base/connector'
-require 'connectors/example/example_advanced_snippet_validator'
-require 'core/filtering/validation_status'
 require 'utility'
 
 module Connectors
@@ -47,10 +45,6 @@ module Connectors
         # raise 'something went wrong'
       end
 
-      def self.advanced_snippet_validator
-        ExampleAdvancedSnippetValidator
-      end
-
       def yield_documents
         attachments = [
           load_attachment('first_attachment.txt'),

data/lib/connectors/gitlab/connector.rb

@@ -11,8 +11,7 @@ require 'connectors/base/connector'
 require 'connectors/gitlab/extractor'
 require 'connectors/gitlab/custom_client'
 require 'connectors/gitlab/adapter'
-require 'connectors/gitlab/gitlab_advanced_snippet_validator'
-require 'core/ingestion'
+require 'core/output_sink'
 
 module Connectors
   module GitLab
@@ -37,10 +36,6 @@ module Connectors
         }
       end
 
-      def self.advanced_snippet_validator
-        GitLabAdvancedSnippetValidator
-      end
-
       def initialize(configuration: {}, job_description: {})
         super
 

data/lib/connectors/mongodb/connector.rb

@@ -7,9 +7,6 @@
 # frozen_string_literal: true
 
 require 'connectors/base/connector'
-require 'core/filtering/validation_status'
-require 'connectors/mongodb/mongo_rules_parser'
-require 'connectors/mongodb/mongo_advanced_snippet_against_schema_validator'
 require 'mongo'
 require 'utility'
 
@@ -31,31 +28,27 @@ module Connectors
 
       def self.configurable_fields
         {
-          :host => {
-            :label => 'Server Hostname'
-          },
-          :user => {
-            :label => 'Username'
-          },
-          :password => {
-            :label => 'Password'
-          },
-          :database => {
-            :label => 'Database'
-          },
-          :collection => {
-            :label => 'Collection'
-          },
-          :direct_connection => {
-            :label => 'Direct connection? (true/false)'
-          }
+           :host => {
+             :label => 'Server Hostname'
+           },
+           :user => {
+             :label => 'Username'
+           },
+           :password => {
+             :label => 'Password'
+           },
+           :database => {
+             :label => 'Database'
+           },
+           :collection => {
+             :label => 'Collection'
+           },
+           :direct_connection => {
+             :label => 'Direct connection? (true/false)'
+           }
         }
       end
 
-      def self.advanced_snippet_validator
-        MongoAdvancedSnippetAgainstSchemaValidator
-      end
-
       def initialize(configuration: {}, job_description: {})
         super
 
@@ -68,6 +61,8 @@ module Connectors
       end
 
       def yield_documents
+        check_filtering
+
         with_client do |client|
           # We do paging using skip().limit() here to make Ruby recycle the memory for each page pulled from the server after it's not needed any more.
           # This gives us more control on the usage of the memory (we can adjust PAGE_SIZE constant for that to decrease max memory consumption).
@@ -94,15 +89,15 @@ module Connectors
           loop do
             found_in_page = 0
 
-            Utility::Logger.info("Requesting #{PAGE_SIZE} documents from MongoDB (Starting at #{skip})")
             view = cursor.skip(skip).limit(PAGE_SIZE)
             view.each do |document|
-              yield_with_handling_tolerable_errors do
-                yield serialize(document)
-                found_in_page += 1
-                found_overall += 1
-                overall_limit_reached = found_overall >= overall_limit && overall_limit != Float::INFINITY
-              end
+              yield serialize(document)
+
+              found_in_page += 1
+              found_overall += 1
+
+              overall_limit_reached = found_overall >= overall_limit && overall_limit != Float::INFINITY
+
               break if overall_limit_reached
             end
 
@@ -122,11 +117,22 @@ module Connectors
 
         return create_aggregate_cursor(collection) if @advanced_filter_config[:aggregate].present?
 
-        return create_simple_rules_cursor(collection) if @rules.present?
-
         collection.find
       end
 
+      def check_filtering
+        return unless filtering_present?
+
+        check_find_and_aggregate
+      end
+
+      def check_find_and_aggregate
+        if @advanced_filter_config.keys.size != 1
+          invalid_keys_msg = "Only one of #{ALLOWED_TOP_LEVEL_FILTER_KEYS} is allowed in the filtering object. Keys present: '#{@advanced_filter_config.keys}'."
+          raise Utility::InvalidFilterConfigError.new(invalid_keys_msg)
+        end
+      end
+
       def create_aggregate_cursor(collection)
         aggregate = @advanced_filter_config[:aggregate]
 
@@ -153,16 +159,6 @@ module Connectors
         [collection.find(filter, options), options]
       end
 
-      def create_simple_rules_cursor(collection)
-        filter = {}
-        if @rules.present?
-          parser = MongoRulesParser.new(@rules)
-          filter = parser.parse
-        end
-        Utility::Logger.info("Filtering with simple rules filter: #{filter}")
-        filter.present? ? collection.find(filter) : collection.find
-      end
-
       def extract_options(mongodb_function)
         mongodb_function[:options].present? ? mongodb_function[:options] : {}
       end
@@ -177,9 +173,9 @@ module Connectors
         raise "Invalid value for 'Direct connection' : #{@direct_connection}." unless %w[true false].include?(@direct_connection.to_s.strip.downcase)
 
         args = {
-          database: @database,
-          direct_connection: to_boolean(@direct_connection)
-        }
+                 database: @database,
+                 direct_connection: to_boolean(@direct_connection)
+               }
 
         if @user.present? || @password.present?
           args[:user] = @user

data/lib/connectors/sync_status.rb

@@ -26,16 +26,11 @@ module Connectors
       ERROR
     ]
 
-    PENDING_STATUSES = [
+    PENDING_STATUES = [
       PENDING,
       SUSPENDED
     ]
 
-    ACTIVE_STATUSES = [
-      IN_PROGRESS,
-      CANCELING
-    ]
-
     TERMINAL_STATUSES = [
       CANCELED,
       COMPLETED,

data/lib/core/configuration.rb

@@ -24,10 +24,8 @@ module Core
             return
           end
           configuration = connector_class.configurable_fields_indifferent_access
-          features = connector_class.kibana_features.each_with_object({}) { |feature, hsh| hsh[feature] = true }
           doc = {
-            :configuration => configuration,
-            :features => features
+            :configuration => configuration
           }
 
           doc[:service_type] = service_type if service_type && connector_settings.needs_service_type?

data/lib/core/connector_settings.rb

@@ -23,24 +23,24 @@ module Core
 
     DEFAULT_PAGE_SIZE = 100
 
+    # Error Classes
+    class ConnectorNotFoundError < StandardError; end
+
     def self.fetch_by_id(connector_id)
       es_response = ElasticConnectorActions.get_connector(connector_id)
-      return nil unless es_response[:found]
-
       connectors_meta = ElasticConnectorActions.connectors_meta
+
+      raise ConnectorNotFoundError.new("Connector with id=#{connector_id} was not found.") unless es_response[:found]
       new(es_response, connectors_meta)
     end
 
+    def initialize(es_response, connectors_meta)
+      @elasticsearch_response = es_response.with_indifferent_access
+      @connectors_meta = connectors_meta.with_indifferent_access
+    end
+
     def self.fetch_native_connectors(page_size = DEFAULT_PAGE_SIZE)
-      require 'connectors/registry' unless defined?(Connectors::REGISTRY)
-      query = {
-        bool: {
-          filter: [
-            { term: { is_native: true } },
-            { terms: { service_type: Connectors::REGISTRY.registered_connectors } }
-          ]
-        }
-      }
+      query = { term: { is_native: true } }
       fetch_connectors_by_query(query, page_size)
     end
 
@@ -83,26 +83,23 @@ module Core
     end
 
     def filtering
-      # assume for now, that first object in filtering array or a filter object itself is the only filtering object
-      filtering = @elasticsearch_response.dig(:_source, :filtering)
-
-      Utility::Filtering.extract_filter(filtering)
+      Utility::Common.return_if_present(@elasticsearch_response[:filtering], DEFAULT_FILTERING)
     end
 
     def request_pipeline
-      Utility::Common.return_if_present(@elasticsearch_response.dig(:_source, :pipeline, :name), @connectors_meta.dig(:pipeline, :default_name), DEFAULT_REQUEST_PIPELINE)
+      Utility::Common.return_if_present(@elasticsearch_response.dig(:pipeline, :name), @connectors_meta.dig(:pipeline, :default_name), DEFAULT_REQUEST_PIPELINE)
     end
 
     def extract_binary_content?
-      Utility::Common.return_if_present(@elasticsearch_response.dig(:_source, :pipeline, :extract_binary_content), @connectors_meta.dig(:pipeline, :default_extract_binary_content), DEFAULT_EXTRACT_BINARY_CONTENT)
+      Utility::Common.return_if_present(@elasticsearch_response.dig(:pipeline, :extract_binary_content), @connectors_meta.dig(:pipeline, :default_extract_binary_content), DEFAULT_EXTRACT_BINARY_CONTENT)
     end
 
     def reduce_whitespace?
-      Utility::Common.return_if_present(@elasticsearch_response.dig(:_source, :pipeline, :reduce_whitespace), @connectors_meta.dig(:pipeline, :default_reduce_whitespace), DEFAULT_REDUCE_WHITESPACE)
+      Utility::Common.return_if_present(@elasticsearch_response.dig(:pipeline, :reduce_whitespace), @connectors_meta.dig(:pipeline, :default_reduce_whitespace), DEFAULT_REDUCE_WHITESPACE)
     end
 
     def run_ml_inference?
-      Utility::Common.return_if_present(@elasticsearch_response.dig(:_source, :pipeline, :run_ml_inference), @connectors_meta.dig(:pipeline, :default_run_ml_inference), DEFAULT_RUN_ML_INFERENCE)
+      Utility::Common.return_if_present(@elasticsearch_response.dig(:pipeline, :run_ml_inference), @connectors_meta.dig(:pipeline, :default_run_ml_inference), DEFAULT_RUN_ML_INFERENCE)
     end
 
     def formatted
@@ -119,39 +116,6 @@ module Core
       index_name&.start_with?(Utility::Constants::CONTENT_INDEX_PREFIX)
     end
 
-    def ready_for_sync?
-      Connectors::REGISTRY.registered?(service_type) &&
-        valid_index_name? &&
-        connector_status_allows_sync?
-    end
-
-    def running?
-      @elasticsearch_response[:_source][:last_sync_status] == Connectors::SyncStatus::IN_PROGRESS
-    end
-
-    def update_last_sync!(job)
-      doc = {
-        :last_sync_status => job.status,
-        :last_synced => Time.now,
-        :last_sync_error => job.error,
-        :error => job.error
-      }
-
-      if job.terminated?
-        doc[:last_indexed_document_count] = job[:indexed_document_count]
-        doc[:last_deleted_document_count] = job[:deleted_document_count]
-      end
-
-      Core::ElasticConnectorActions.update_connector_fields(job.connector_id, doc)
-    end
-
-    private
-
-    def initialize(es_response, connectors_meta)
-      @elasticsearch_response = es_response.with_indifferent_access
-      @connectors_meta = connectors_meta.with_indifferent_access
-    end
-
     def self.fetch_connectors_by_query(query, page_size)
       connectors_meta = ElasticConnectorActions.connectors_meta