connectors_service 8.6.0.3 → 8.6.0.4.pre.20221114T233727Z

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b550a78dba7e4cd4502b6eea4c187253e6bb77ab944815b59809d92c7ccc23bb
-  data.tar.gz: dbe8c32e1da94ed40777af57a84ecac883a15cab71aa164534157314bbcbfcff
+  metadata.gz: 8f69f05260d34b07ce34d569ce7c41fdd10349b33121823697ebbb6a4ebf9206
+  data.tar.gz: a2957118c80d0e2bc9ea6a8046307485c11e4f809efb01cabbb96e341dc947c2
 SHA512:
-  metadata.gz: 92ef83e3bc94107b1cb11b4454760b17c217b61beb555b55d01ddb7b0758372c3f04ed1acc153e406f20c7abb2df5a3540c2b93733b48eafcc99bd752e7d6759
-  data.tar.gz: a7eade1996d683fcf47e072704d566479538249d9711e8c0d2019fbb7bd15c382c61cd35adc6ee52f989a75c1fa6d7a6fe1330a04801fb46952aa307f2e93ac5
+  metadata.gz: 63775eded9d9953b41950edd7ca86176200c4ae7510564f6f7995c336d6e78bbe40d494cb0a152a984b14b45055e7b7847a2779888d70905d4956b8c78d4bda1
+  data.tar.gz: 52b00d122ef43fc5afa0b4cb50bbe428111e7fe2cb7cee437dd2d2b6b32516cbd01c5803b0a11609a5021e85605a2e6bd2b30973807df3cae1420864e2fcb185

data/config/connectors.yml

@@ -1,11 +1,10 @@
 # general metadata
-version: 8.6.0.3
-repository: https://github.com/elastic/connectors-ruby.git
-revision: aa2faf8cc993a26980441adffe97d62fdaf5aa5c
+version: 8.6.0.4-20221114T233727Z
+repository: git@github.com:elastic/ent-search-connectors.git
+revision: f506d5e5ebedfb0c6058d347d8ce22adc42e2cc0
 elasticsearch:
-  cloud_id: CHANGEME
   hosts: http://localhost:9200
-  api_key: CHANGEME
+  api_key: WXNYeWQ0UUJ4Y3ZQV3ctbjVibnU6REx4eE8tbFhUMU94N2JoU2hIeVFMQQ==
   retry_on_failure: 3
   request_timeout: 120
   disable_warnings: true
@@ -15,11 +14,11 @@ thread_pool:
   min_threads: 0
   max_threads: 5
   max_queue: 100
-log_level: info
-ecs_logging: true
+log_level: debug
+ecs_logging: false
 poll_interval: 3
 termination_timeout: 60
 heartbeat_interval: 1800
-native_mode: true
-connector_id: CHANGEME
-service_type: CHANGEME
+native_mode: false
+connector_id: YcXyd4QBxcvPWw-n2bkA
+service_type: mongodb

data/lib/app/config.rb

@@ -35,6 +35,8 @@ puts "Parsing #{CONFIG_FILE} configuration file."
       optional(:disable_warnings).value(:bool?)
       optional(:trace).value(:bool?)
       optional(:log).value(:bool?)
+      optional(:ca_fingerprint).value(:string)
+      optional(:transport_options).value(:hash)
     end
 
     optional(:thread_pool).hash do

data/lib/app/dispatcher.rb

@@ -73,6 +73,8 @@ module App
             start_heartbeat_task(connector_settings)
           when :configuration
             start_configuration_task(connector_settings)
+          when :filter_validation
+            start_filter_validation_task(connector_settings)
           else
             Utility::Logger.error("Unknown task type: #{task}. Skipping...")
           end
@@ -84,10 +86,14 @@ module App
       def start_sync_task(connector_settings)
         start_heartbeat_task(connector_settings)
         pool.post do
-          Utility::Logger.info("Starting a sync job for #{connector_settings.formatted}...")
+          Utility::Logger.info("Initiating a sync job for #{connector_settings.formatted}...")
           Core::ElasticConnectorActions.ensure_content_index_exists(connector_settings.index_name)
           job_runner = Core::SyncJobRunner.new(connector_settings)
           job_runner.execute
+        rescue Core::JobAlreadyRunningError
+          Utility::Logger.info("Sync job for #{connector_settings.formatted} is already running, skipping.")
+        rescue Core::ConnectorVersionChangedError => e
+          Utility::Logger.info("Could not start the job because #{connector_settings.formatted} has been updated externally. Message: #{e.message}")
         rescue StandardError => e
           Utility::ExceptionTracking.log_exception(e, "Sync job for #{connector_settings.formatted} failed due to unexpected error.")
         end
@@ -116,6 +122,16 @@ module App
           Utility::ExceptionTracking.log_exception(e, "Configuration task for #{connector_settings.formatted} failed due to unexpected error.")
         end
       end
+
+      def start_filter_validation_task(connector_settings)
+        pool.post do
+          Utility::Logger.info("Validating filters for #{connector_settings.formatted}...")
+          validation_job_runner = Core::Filtering::ValidationJobRunner.new(connector_settings)
+          validation_job_runner.execute
+        rescue StandardError => e
+          Utility::ExceptionTracking.log_exception(e, "Filter validation task for #{connector_settings.formatted} failed due to unexpected error.")
+        end
+      end
     end
   end
 end

data/lib/app/preflight_check.rb

@@ -23,6 +23,7 @@ module App
         check_es_connection!
         check_es_version!
         check_system_indices!
+        check_single_connector!
       end
 
       private
@@ -59,6 +60,16 @@ module App
         )
       end
 
+      #-------------------------------------------------------------------------------------------------
+      # Ensures the connector is supported when running in non-native mode
+      def check_single_connector!
+        if App::Config.native_mode
+          Utility::Logger.info('Skip single connector check for native mode.')
+        elsif !Connectors::REGISTRY.registered?(App::Config.service_type)
+          fail_check!("The service type #{App::Config.service_type} is not supported. Terminating...")
+        end
+      end
+
       def check_es_connection_with_retries!(retry_interval:, retry_timeout:)
         started_at = Time.now
 
@@ -75,6 +86,10 @@ module App
           else
             raise UnhealthyCluster, "Unexpected cluster status: #{response['status']}"
           end
+        rescue *Utility::AUTHORIZATION_ERRORS => e
+          Utility::ExceptionTracking.log_exception(e)
+
+          fail_check!("Elasticsearch returned 'Unauthorized' response. Check your authentication details. Terminating...")
         rescue *App::RETRYABLE_CONNECTION_ERRORS => e
           Utility::Logger.warn('Could not connect to Elasticsearch. Make sure it is running and healthy.')
           Utility::Logger.debug("Error: #{e.full_message}")

data/lib/connectors/base/connector.rb

@@ -7,10 +7,11 @@
 # frozen_string_literal: true
 
 require 'bson'
-require 'core/output_sink'
-require 'utility/exception_tracking'
-require 'utility/errors'
+require 'core/ingestion'
+require 'utility'
+require 'utility/filtering'
 require 'app/config'
+require 'active_support/core_ext/hash/indifferent_access'
 
 module Connectors
   module Base
@@ -19,6 +20,11 @@ module Connectors
         raise 'Not implemented for this connector'
       end
 
+      # Used as a framework util method, don't override
+      def self.configurable_fields_indifferent_access
+        configurable_fields.with_indifferent_access
+      end
+
       def self.configurable_fields
         {}
       end
@@ -27,8 +33,27 @@ module Connectors
         raise 'Not implemented for this connector'
       end
 
-      def initialize(configuration: {})
+      def self.kibana_features
+        [
+          Utility::Constants::FILTERING_RULES_FEATURE,
+          Utility::Constants::FILTERING_ADVANCED_FEATURE
+        ]
+      end
+
+      def self.validate_filtering(_filtering = {})
+        raise 'Not implemented for this connector'
+      end
+
+      attr_reader :rules, :advanced_filter_config
+
+      def initialize(configuration: {}, job_description: {})
         @configuration = configuration.dup || {}
+        @job_description = job_description&.dup || {}
+
+        filtering = Utility::Filtering.extract_filter(@job_description.dig(:connector, :filtering))
+
+        @rules = filtering[:rules] || []
+        @advanced_filter_config = filtering[:advanced_snippet] || {}
       end
 
       def yield_documents; end
@@ -52,6 +77,14 @@ module Connectors
         Utility::ExceptionTracking.log_exception(e, "Connector for service #{self.class.service_type} failed the health check for 3rd-party service.")
         false
       end
+
+      def filtering_present?
+        @advanced_filter_config.present? && !@advanced_filter_config.empty? || @rules.present?
+      end
+
+      def metadata
+        {}
+      end
     end
   end
 end

data/lib/connectors/base/simple_rules_parser.rb

@@ -0,0 +1,42 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+# frozen_string_literal: true
+
+require 'active_support/core_ext/hash/indifferent_access'
+require 'active_support/core_ext/object/blank'
+require 'core/filtering/simple_rule'
+
+module Connectors
+  module Base
+    class SimpleRulesParser
+      def initialize(rules)
+        @rules = (rules || []).map(&:with_indifferent_access).filter { |r| r[:id] != 'DEFAULT' }.sort_by { |r| r[:order] }
+      end
+
+      def parse
+        merge_rules(@rules.map do |rule_hash|
+          rule = Core::Filtering::SimpleRule.new(rule_hash)
+          unless rule.is_include? || rule.is_exclude?
+            raise "Unknown policy: #{rule.policy}"
+          end
+          parse_rule(rule)
+        end)
+      end
+
+      private
+
+      # merge all rules into a filter object or array
+      # in a base case, does no transformations
+      def merge_rules(rules)
+        rules || []
+      end
+
+      def parse_rule(_rule)
+        raise 'Not implemented'
+      end
+    end
+  end
+end

data/lib/connectors/connector_status.rb

@@ -8,11 +8,11 @@
 
 module Connectors
   class ConnectorStatus
-    CREATED = 'created'
+    CREATED             = 'created'
     NEEDS_CONFIGURATION = 'needs_configuration'
-    CONFIGURED = 'configured'
-    CONNECTED = 'connected'
-    ERROR = 'error'
+    CONFIGURED          = 'configured'
+    CONNECTED           = 'connected'
+    ERROR               = 'error'
 
     STATUSES = [
       CREATED,

data/lib/connectors/example/connector.rb

@@ -7,6 +7,7 @@
 # frozen_string_literal: true
 
 require 'connectors/base/connector'
+require 'core/filtering/validation_status'
 require 'utility'
 
 module Connectors
@@ -20,16 +21,21 @@ module Connectors
         'Example Connector'
       end
 
+      # Field 'Foo' won't have a default value. Field 'Bar' will have the default value 'Value'.
       def self.configurable_fields
         {
           'foo' => {
             'label' => 'Foo',
             'value' => nil
+          },
+          :bar => {
+            :label => 'Bar',
+            :value => 'Value'
           }
         }
       end
 
-      def initialize(configuration: {})
+      def initialize(configuration: {}, job_description: {})
         super
       end
 
@@ -40,18 +46,51 @@ module Connectors
         # raise 'something went wrong'
       end
 
+      def self.validate_filtering(filtering = {})
+        # TODO: real filtering validation will follow later
+        errors = [
+          {
+            :ids => ['missing-implementation'],
+            :messages => ['Filtering is not implemented yet for the example connector']
+          }
+        ]
+
+        return { :state => Core::Filtering::ValidationStatus::INVALID, :errors => errors } if filtering.present?
+
+        { :state => Core::Filtering::ValidationStatus::VALID, :errors => [] }
+      end
+
       def yield_documents
         attachments = [
-          File.open('./lib/connectors/example/example_attachments/first_attachment.txt'),
-          File.open('./lib/connectors/example/example_attachments/second_attachment.txt'),
-          File.open('./lib/connectors/example/example_attachments/third_attachment.txt')
+          load_attachment('first_attachment.txt'),
+          load_attachment('second_attachment.txt'),
+          load_attachment('third_attachment.txt'),
         ]
 
         attachments.each_with_index do |att, index|
           data = { id: (index + 1).to_s, name: "example document #{index + 1}", _attachment: File.read(att) }
+
+          # Uncomment one of these two lines to simulate longer running sync jobs
+          #
+          # sleep(rand(10..60).seconds)
+          # sleep(rand(1..10).minutes)
+
           yield data
         end
       end
+
+      private
+
+      def load_attachment(path)
+        attachment_dir = "#{File.dirname(__FILE__)}/attachments"
+        attachment_path = "#{attachment_dir}/#{path}"
+
+        unless File.exist?(attachment_path)
+          raise "Attachment at location '#{attachment_path}' doesn't exist. Attachments should be located under #{attachment_dir}"
+        end
+
+        File.open(attachment_path)
+      end
     end
   end
 end

data/lib/connectors/gitlab/connector.rb

@@ -11,7 +11,7 @@ require 'connectors/base/connector'
 require 'connectors/gitlab/extractor'
 require 'connectors/gitlab/custom_client'
 require 'connectors/gitlab/adapter'
-require 'core/output_sink'
+require 'core/ingestion'
 
 module Connectors
   module GitLab
@@ -36,7 +36,21 @@ module Connectors
         }
       end
 
-      def initialize(configuration: {})
+      def self.validate_filtering(filtering = {})
+        # TODO: real filtering validation will follow later
+        errors = [
+          {
+            :ids => ['missing-implementation'],
+            :messages => ['Filtering is not implemented yet for the GitLab connector']
+          }
+        ]
+
+        return { :state => Core::Filtering::ValidationStatus::INVALID, :errors => errors } if filtering.present?
+
+        { :state => Core::Filtering::ValidationStatus::VALID, :errors => [] }
+      end
+
+      def initialize(configuration: {}, job_description: {})
         super
 
         @extractor = Connectors::GitLab::Extractor.new(

data/lib/connectors/mongodb/connector.rb

@@ -6,13 +6,20 @@
 
 # frozen_string_literal: true
 
-require 'active_support/core_ext/hash/indifferent_access'
 require 'connectors/base/connector'
+require 'core/filtering/validation_status'
+require 'connectors/mongodb/mongo_rules_parser'
 require 'mongo'
+require 'utility'
 
 module Connectors
   module MongoDB
     class Connector < Connectors::Base::Connector
+
+      ALLOWED_TOP_LEVEL_FILTER_KEYS = %w[find aggregate]
+
+      PAGE_SIZE = 100
+
       def self.service_type
         'mongodb'
       end
@@ -23,28 +30,47 @@ module Connectors
 
       def self.configurable_fields
         {
-           :host => {
-             :label => 'Server Hostname'
-           },
-           :user => {
-             :label => 'Username'
-           },
-           :password => {
-             :label => 'Password'
-           },
-           :database => {
-             :label => 'Database'
-           },
-           :collection => {
-             :label => 'Collection'
-           },
-           :direct_connection => {
-             :label => 'Direct connection? (true/false)'
-           }
+          :host => {
+            :label => 'Server Hostname'
+          },
+          :user => {
+            :label => 'Username'
+          },
+          :password => {
+            :label => 'Password'
+          },
+          :database => {
+            :label => 'Database'
+          },
+          :collection => {
+            :label => 'Collection'
+          },
+          :direct_connection => {
+            :label => 'Direct connection? (true/false)'
+          }
         }
       end
 
-      def initialize(configuration: {})
+      def self.validate_filtering(filtering = {})
+        valid_filtering = { :state => Core::Filtering::ValidationStatus::VALID, :errors => [] }
+
+        return valid_filtering unless filtering.present?
+
+        filter = Utility::Filtering.extract_filter(filtering)
+
+        advanced_filter_config = filter[:advanced_snippet] || {}
+        filter_keys = advanced_filter_config&.keys
+
+        if !filter_keys&.empty? && (filter_keys.size != 1 || !ALLOWED_TOP_LEVEL_FILTER_KEYS.include?(filter_keys[0]&.to_s))
+          return { :state => Core::Filtering::ValidationStatus::INVALID,
+                   :errors => [{ :ids => ['wrong-keys'],
+                                 :messages => ["Only one of #{ALLOWED_TOP_LEVEL_FILTER_KEYS} is allowed in the filtering object. Keys present: '#{filter_keys}'."] }] }
+        end
+
+        valid_filtering
+      end
+
+      def initialize(configuration: {}, job_description: {})
         super
 
         @host = configuration.dig(:host, :value)
@@ -57,16 +83,105 @@ module Connectors
 
       def yield_documents
         with_client do |client|
-          client[@collection].find.each do |document|
-            doc = document.with_indifferent_access
+          # We do paging using skip().limit() here to make Ruby recycle the memory for each page pulled from the server after it's not needed any more.
+          # This gives us more control on the usage of the memory (we can adjust PAGE_SIZE constant for that to decrease max memory consumption).
+          # It's done due to the fact that usage of .find.each leads to memory leaks or overuse of memory - the whole result set seems to stay in memory
+          # during the sync. Sometimes (not 100% sure) it even leads to a real leak, when the memory for these objects is never recycled.
+          cursor, options = create_db_cursor_on_collection(client[@collection])
+          skip = 0
+
+          found_overall = 0
+
+          # if no overall limit is specified by filtering use -1 to not break ingestion, when no overall limit is specified (found_overall is only increased,
+          # thus can never reach -1)
+          overall_limit = Float::INFINITY
+
+          if options.present?
+            # there could be a skip parameter defined for filtering
+            skip = options.fetch(:skip, skip)
+            # there could be a limit parameter defined for filtering -> used for an overall limit (not a page limit, which was introduced for memory optimization)
+            overall_limit = options.fetch(:limit, overall_limit)
+          end
 
-            yield serialize(doc)
+          overall_limit_reached = false
+
+          loop do
+            found_in_page = 0
+
+            Utility::Logger.info("Requesting #{PAGE_SIZE} documents from MongoDB (Starting at #{skip})")
+            view = cursor.skip(skip).limit(PAGE_SIZE)
+            view.each do |document|
+              yield serialize(document)
+
+              found_in_page += 1
+              found_overall += 1
+
+              overall_limit_reached = found_overall >= overall_limit && overall_limit != Float::INFINITY
+
+              break if overall_limit_reached
+            end
+
+            page_was_empty = found_in_page == 0
+
+            break if page_was_empty || overall_limit_reached
+
+            skip += PAGE_SIZE
           end
         end
       end
 
       private
 
+      def create_db_cursor_on_collection(collection)
+        return create_find_cursor(collection) if @advanced_filter_config[:find].present?
+
+        return create_aggregate_cursor(collection) if @advanced_filter_config[:aggregate].present?
+
+        return create_simple_rules_cursor(collection) if @rules.present?
+
+        collection.find
+      end
+
+      def create_aggregate_cursor(collection)
+        aggregate = @advanced_filter_config[:aggregate]
+
+        pipeline = aggregate[:pipeline]
+        options = extract_options(aggregate)
+
+        if !pipeline.nil? && pipeline.empty? && !options.present?
+          Utility::Logger.warn('\'Aggregate\' was specified with an empty pipeline and empty options.')
+        end
+
+        [collection.aggregate(pipeline, options), options]
+      end
+
+      def create_find_cursor(collection)
+        find = @advanced_filter_config[:find]
+
+        filter = find[:filter]
+        options = extract_options(find)
+
+        if !filter.nil? && filter.empty? && !options.present?
+          Utility::Logger.warn('\'Find\' was specified with an empty filter and empty options.')
+        end
+
+        [collection.find(filter, options), options]
+      end
+
+      def create_simple_rules_cursor(collection)
+        filter = {}
+        if @rules.present?
+          parser = MongoRulesParser.new(@rules)
+          filter = parser.parse
+        end
+        Utility::Logger.info("Filtering with simple rules filter: #{filter}")
+        filter.present? ? collection.find(filter) : collection.find
+      end
+
+      def extract_options(mongodb_function)
+        mongodb_function[:options].present? ? mongodb_function[:options] : {}
+      end
+
       def do_health_check
         with_client do |_client|
           Utility::Logger.debug("Mongo at #{@host}/#{@database} looks healthy.")
@@ -76,34 +191,43 @@ module Connectors
       def with_client
         raise "Invalid value for 'Direct connection' : #{@direct_connection}." unless %w[true false].include?(@direct_connection.to_s.strip.downcase)
 
-        client = if @user.present? || @password.present?
-                   Mongo::Client.new(
-                     @host,
-                     database: @database,
-                     direct_connection: to_boolean(@direct_connection),
-                     user: @user,
-                     password: @password
-                   )
-                 else
-                   Mongo::Client.new(
-                     @host,
-                     database: @database,
-                     direct_connection: to_boolean(@direct_connection)
-                   )
-                 end
-
-        begin
-          Utility::Logger.debug("Existing Databases #{client.database_names}")
-          Utility::Logger.debug('Existing Collections:')
-
-          client.collections.each { |coll| Utility::Logger.debug(coll.name) }
+        args = {
+          database: @database,
+          direct_connection: to_boolean(@direct_connection)
+        }
+
+        if @user.present? || @password.present?
+          args[:user] = @user
+          args[:password] = @password
+        end
+
+        Mongo::Client.new(@host, args) do |client|
+          databases = client.database_names
+
+          Utility::Logger.debug("Existing Databases: #{databases}")
+          check_database_exists!(databases, @database)
+
+          collections = client.database.collection_names
+
+          Utility::Logger.debug("Existing Collections: #{collections}")
+          check_collection_exists!(collections, @database, @collection)
 
           yield client
-        ensure
-          client.close
         end
       end
 
+      def check_database_exists!(databases, database)
+        return if databases.include?(database)
+
+        raise "Database (#{database}) does not exist. Existing databases: #{databases.join(', ')}"
+      end
+
+      def check_collection_exists!(collections, database, collection)
+        return if collections.include?(collection)
+
+        raise "Collection (#{collection}) does not exist within database '#{database}'. Existing collections: #{collections.join(', ')}"
+      end
+
       def serialize(mongodb_document)
         # This is some lazy serialization here.
         # Problem: MongoDB has its own format of things - e.g. ids are Bson::ObjectId, which when serialized to JSON
@@ -120,11 +244,10 @@ module Connectors
           mongodb_document.map { |v| serialize(v) }
         when Hash
           mongodb_document.map do |key, value|
-            remapped_key = key.to_sym == :_id ? :id : key.to_sym
-
+            key = 'id' if key == '_id'
             remapped_value = serialize(value)
-            [remapped_key, remapped_value]
-          end.to_h.with_indifferent_access
+            [key, remapped_value]
+          end.to_h
         else
           mongodb_document
         end