connectors_service 8.6.0.4.pre.20221104T200814Z → 8.6.0.4.pre.20221114T233727Z

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 23b78b5c5228f761b884af1bfd75b163ad912dc53331ce5a64cd3edb8f45ef86
-  data.tar.gz: 5c74e06f315ab9af4161e88fac626e84b6d2789c0b5b080c713e780a4560ef97
+  metadata.gz: 8f69f05260d34b07ce34d569ce7c41fdd10349b33121823697ebbb6a4ebf9206
+  data.tar.gz: a2957118c80d0e2bc9ea6a8046307485c11e4f809efb01cabbb96e341dc947c2
 SHA512:
-  metadata.gz: 75295f0d26061a977089e999190f4974f4929ca92c25c09e7ecbbd1f0117476f758b7f200b719461bdd9e7b9e6193df455d3e52bcec0634fc85a7b512721ab6a
-  data.tar.gz: fa12af986f72d10081245e782a9152a887512e62f48ad525af59e59a4ef72238727e7bfd521f516fd452a2e74a1101c5bd1965f1e9b97f7b236b8ae2ccf626ff
+  metadata.gz: 63775eded9d9953b41950edd7ca86176200c4ae7510564f6f7995c336d6e78bbe40d494cb0a152a984b14b45055e7b7847a2779888d70905d4956b8c78d4bda1
+  data.tar.gz: 52b00d122ef43fc5afa0b4cb50bbe428111e7fe2cb7cee437dd2d2b6b32516cbd01c5803b0a11609a5021e85605a2e6bd2b30973807df3cae1420864e2fcb185

data/config/connectors.yml

@@ -1,10 +1,10 @@
 # general metadata
-version: 8.6.0.4-20221104T200814Z
+version: 8.6.0.4-20221114T233727Z
 repository: git@github.com:elastic/ent-search-connectors.git
-revision: 2051b3907639a1fe2ae68efdc33c06cf12d38383
+revision: f506d5e5ebedfb0c6058d347d8ce22adc42e2cc0
 elasticsearch:
   hosts: http://localhost:9200
-  api_key: OW1FalJJUUI1clBtUVh5RVo1QmU6QVp5LV9pU3RRUXFYb2VVYnlCRWNZdw==
+  api_key: WXNYeWQ0UUJ4Y3ZQV3ctbjVibnU6REx4eE8tbFhUMU94N2JoU2hIeVFMQQ==
   retry_on_failure: 3
   request_timeout: 120
   disable_warnings: true
@@ -14,11 +14,11 @@ thread_pool:
   min_threads: 0
   max_threads: 5
   max_queue: 100
-log_level: info
+log_level: debug
 ecs_logging: false
 poll_interval: 3
 termination_timeout: 60
 heartbeat_interval: 1800
 native_mode: false
-connector_id: 9WEjRIQB5rPmQXyEWJB2
-service_type: example
+connector_id: YcXyd4QBxcvPWw-n2bkA
+service_type: mongodb

data/lib/app/dispatcher.rb

@@ -73,6 +73,8 @@ module App
             start_heartbeat_task(connector_settings)
           when :configuration
             start_configuration_task(connector_settings)
+          when :filter_validation
+            start_filter_validation_task(connector_settings)
           else
             Utility::Logger.error("Unknown task type: #{task}. Skipping...")
           end
@@ -120,6 +122,16 @@ module App
           Utility::ExceptionTracking.log_exception(e, "Configuration task for #{connector_settings.formatted} failed due to unexpected error.")
         end
       end
+
+      def start_filter_validation_task(connector_settings)
+        pool.post do
+          Utility::Logger.info("Validating filters for #{connector_settings.formatted}...")
+          validation_job_runner = Core::Filtering::ValidationJobRunner.new(connector_settings)
+          validation_job_runner.execute
+        rescue StandardError => e
+          Utility::ExceptionTracking.log_exception(e, "Filter validation task for #{connector_settings.formatted} failed due to unexpected error.")
+        end
+      end
     end
   end
 end

data/lib/app/preflight_check.rb

@@ -23,6 +23,7 @@ module App
         check_es_connection!
         check_es_version!
         check_system_indices!
+        check_single_connector!
       end
 
       private
@@ -59,6 +60,16 @@ module App
         )
       end
 
+      #-------------------------------------------------------------------------------------------------
+      # Ensures the connector is supported when running in non-native mode
+      def check_single_connector!
+        if App::Config.native_mode
+          Utility::Logger.info('Skip single connector check for native mode.')
+        elsif !Connectors::REGISTRY.registered?(App::Config.service_type)
+          fail_check!("The service type #{App::Config.service_type} is not supported. Terminating...")
+        end
+      end
+
       def check_es_connection_with_retries!(retry_interval:, retry_timeout:)
         started_at = Time.now
 

data/lib/connectors/base/connector.rb

@@ -7,8 +7,9 @@
 # frozen_string_literal: true
 
 require 'bson'
-require 'core/output_sink'
+require 'core/ingestion'
 require 'utility'
+require 'utility/filtering'
 require 'app/config'
 require 'active_support/core_ext/hash/indifferent_access'
 
@@ -32,16 +33,27 @@ module Connectors
         raise 'Not implemented for this connector'
       end
 
+      def self.kibana_features
+        [
+          Utility::Constants::FILTERING_RULES_FEATURE,
+          Utility::Constants::FILTERING_ADVANCED_FEATURE
+        ]
+      end
+
+      def self.validate_filtering(_filtering = {})
+        raise 'Not implemented for this connector'
+      end
+
       attr_reader :rules, :advanced_filter_config
 
       def initialize(configuration: {}, job_description: {})
         @configuration = configuration.dup || {}
         @job_description = job_description&.dup || {}
 
-        filter = get_filter(@job_description[:filtering])
+        filtering = Utility::Filtering.extract_filter(@job_description.dig(:connector, :filtering))
 
-        @rules = Utility::Common.return_if_present(filter[:rules], [])
-        @advanced_filter_config = Utility::Common.return_if_present(filter[:advanced_config], {})
+        @rules = filtering[:rules] || []
+        @advanced_filter_config = filtering[:advanced_snippet] || {}
       end
 
       def yield_documents; end
@@ -67,16 +79,11 @@ module Connectors
       end
 
       def filtering_present?
-        @advanced_filter_config.present? || @rules.present?
+        @advanced_filter_config.present? && !@advanced_filter_config.empty? || @rules.present?
       end
 
-      private
-
-      def get_filter(filtering)
-        # assume for now, that first object in filtering array or a filter object itself is the only filtering object
-        filter = filtering.is_a?(Array) ? filtering[0] : filtering
-
-        filter.present? ? filter : {}
+      def metadata
+        {}
       end
     end
   end

data/lib/connectors/base/simple_rules_parser.rb

@@ -0,0 +1,42 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+# frozen_string_literal: true
+
+require 'active_support/core_ext/hash/indifferent_access'
+require 'active_support/core_ext/object/blank'
+require 'core/filtering/simple_rule'
+
+module Connectors
+  module Base
+    class SimpleRulesParser
+      def initialize(rules)
+        @rules = (rules || []).map(&:with_indifferent_access).filter { |r| r[:id] != 'DEFAULT' }.sort_by { |r| r[:order] }
+      end
+
+      def parse
+        merge_rules(@rules.map do |rule_hash|
+          rule = Core::Filtering::SimpleRule.new(rule_hash)
+          unless rule.is_include? || rule.is_exclude?
+            raise "Unknown policy: #{rule.policy}"
+          end
+          parse_rule(rule)
+        end)
+      end
+
+      private
+
+      # merge all rules into a filter object or array
+      # in a base case, does no transformations
+      def merge_rules(rules)
+        rules || []
+      end
+
+      def parse_rule(_rule)
+        raise 'Not implemented'
+      end
+    end
+  end
+end

data/lib/connectors/example/connector.rb

@@ -7,6 +7,7 @@
 # frozen_string_literal: true
 
 require 'connectors/base/connector'
+require 'core/filtering/validation_status'
 require 'utility'
 
 module Connectors
@@ -45,6 +46,20 @@ module Connectors
         # raise 'something went wrong'
       end
 
+      def self.validate_filtering(filtering = {})
+        # TODO: real filtering validation will follow later
+        errors = [
+          {
+            :ids => ['missing-implementation'],
+            :messages => ['Filtering is not implemented yet for the example connector']
+          }
+        ]
+
+        return { :state => Core::Filtering::ValidationStatus::INVALID, :errors => errors } if filtering.present?
+
+        { :state => Core::Filtering::ValidationStatus::VALID, :errors => [] }
+      end
+
       def yield_documents
         attachments = [
           load_attachment('first_attachment.txt'),

data/lib/connectors/gitlab/connector.rb

@@ -11,7 +11,7 @@ require 'connectors/base/connector'
 require 'connectors/gitlab/extractor'
 require 'connectors/gitlab/custom_client'
 require 'connectors/gitlab/adapter'
-require 'core/output_sink'
+require 'core/ingestion'
 
 module Connectors
   module GitLab
@@ -36,6 +36,20 @@ module Connectors
         }
       end
 
+      def self.validate_filtering(filtering = {})
+        # TODO: real filtering validation will follow later
+        errors = [
+          {
+            :ids => ['missing-implementation'],
+            :messages => ['Filtering is not implemented yet for the GitLab connector']
+          }
+        ]
+
+        return { :state => Core::Filtering::ValidationStatus::INVALID, :errors => errors } if filtering.present?
+
+        { :state => Core::Filtering::ValidationStatus::VALID, :errors => [] }
+      end
+
       def initialize(configuration: {}, job_description: {})
         super
 

data/lib/connectors/mongodb/connector.rb

@@ -7,6 +7,8 @@
 # frozen_string_literal: true
 
 require 'connectors/base/connector'
+require 'core/filtering/validation_status'
+require 'connectors/mongodb/mongo_rules_parser'
 require 'mongo'
 require 'utility'
 
@@ -28,27 +30,46 @@ module Connectors
 
       def self.configurable_fields
         {
-           :host => {
-             :label => 'Server Hostname'
-           },
-           :user => {
-             :label => 'Username'
-           },
-           :password => {
-             :label => 'Password'
-           },
-           :database => {
-             :label => 'Database'
-           },
-           :collection => {
-             :label => 'Collection'
-           },
-           :direct_connection => {
-             :label => 'Direct connection? (true/false)'
-           }
+          :host => {
+            :label => 'Server Hostname'
+          },
+          :user => {
+            :label => 'Username'
+          },
+          :password => {
+            :label => 'Password'
+          },
+          :database => {
+            :label => 'Database'
+          },
+          :collection => {
+            :label => 'Collection'
+          },
+          :direct_connection => {
+            :label => 'Direct connection? (true/false)'
+          }
         }
       end
 
+      def self.validate_filtering(filtering = {})
+        valid_filtering = { :state => Core::Filtering::ValidationStatus::VALID, :errors => [] }
+
+        return valid_filtering unless filtering.present?
+
+        filter = Utility::Filtering.extract_filter(filtering)
+
+        advanced_filter_config = filter[:advanced_snippet] || {}
+        filter_keys = advanced_filter_config&.keys
+
+        if !filter_keys&.empty? && (filter_keys.size != 1 || !ALLOWED_TOP_LEVEL_FILTER_KEYS.include?(filter_keys[0]&.to_s))
+          return { :state => Core::Filtering::ValidationStatus::INVALID,
+                   :errors => [{ :ids => ['wrong-keys'],
+                                 :messages => ["Only one of #{ALLOWED_TOP_LEVEL_FILTER_KEYS} is allowed in the filtering object. Keys present: '#{filter_keys}'."] }] }
+        end
+
+        valid_filtering
+      end
+
       def initialize(configuration: {}, job_description: {})
         super
 
@@ -61,8 +82,6 @@ module Connectors
       end
 
       def yield_documents
-        check_filtering
-
         with_client do |client|
           # We do paging using skip().limit() here to make Ruby recycle the memory for each page pulled from the server after it's not needed any more.
           # This gives us more control on the usage of the memory (we can adjust PAGE_SIZE constant for that to decrease max memory consumption).
@@ -89,6 +108,7 @@ module Connectors
           loop do
             found_in_page = 0
 
+            Utility::Logger.info("Requesting #{PAGE_SIZE} documents from MongoDB (Starting at #{skip})")
             view = cursor.skip(skip).limit(PAGE_SIZE)
             view.each do |document|
               yield serialize(document)
@@ -117,20 +137,9 @@ module Connectors
 
         return create_aggregate_cursor(collection) if @advanced_filter_config[:aggregate].present?
 
-        collection.find
-      end
-
-      def check_filtering
-        return unless filtering_present?
+        return create_simple_rules_cursor(collection) if @rules.present?
 
-        check_find_and_aggregate
-      end
-
-      def check_find_and_aggregate
-        if @advanced_filter_config.keys.size != 1
-          invalid_keys_msg = "Only one of #{ALLOWED_TOP_LEVEL_FILTER_KEYS} is allowed in the filtering object. Keys present: '#{@advanced_filter_config.keys}'."
-          raise Utility::InvalidFilterConfigError.new(invalid_keys_msg)
-        end
+        collection.find
       end
 
       def create_aggregate_cursor(collection)
@@ -159,6 +168,16 @@ module Connectors
         [collection.find(filter, options), options]
       end
 
+      def create_simple_rules_cursor(collection)
+        filter = {}
+        if @rules.present?
+          parser = MongoRulesParser.new(@rules)
+          filter = parser.parse
+        end
+        Utility::Logger.info("Filtering with simple rules filter: #{filter}")
+        filter.present? ? collection.find(filter) : collection.find
+      end
+
       def extract_options(mongodb_function)
         mongodb_function[:options].present? ? mongodb_function[:options] : {}
       end
@@ -173,9 +192,9 @@ module Connectors
         raise "Invalid value for 'Direct connection' : #{@direct_connection}." unless %w[true false].include?(@direct_connection.to_s.strip.downcase)
 
         args = {
-                 database: @database,
-                 direct_connection: to_boolean(@direct_connection)
-               }
+          database: @database,
+          direct_connection: to_boolean(@direct_connection)
+        }
 
         if @user.present? || @password.present?
           args[:user] = @user

data/lib/connectors/mongodb/mongo_rules_parser.rb

@@ -0,0 +1,81 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'active_support/core_ext/object'
+require 'connectors/base/simple_rules_parser'
+require 'core/filtering/simple_rule'
+
+module Connectors
+  module MongoDB
+    class MongoRulesParser < Connectors::Base::SimpleRulesParser
+      def parse_rule(rule)
+        field = rule.field
+        value = rule.value
+        unless value.present?
+          raise "value is required for field: #{field}"
+        end
+        unless field.present?
+          raise "field is required for rule: #{rule}"
+        end
+        op = rule.rule
+        case op
+        when Core::Filtering::SimpleRule::Rule::EQUALS
+          parse_equals(rule)
+        when Core::Filtering::SimpleRule::Rule::GREATER_THAN
+          parse_greater_than(rule)
+        when Core::Filtering::SimpleRule::Rule::LESS_THAN
+          parse_less_than(rule)
+        when Core::Filtering::SimpleRule::Rule::REGEX
+          parse_regex(rule)
+        else
+          raise "Unknown operator: #{op}"
+        end
+      end
+
+      def merge_rules(rules)
+        return {} if rules.empty?
+        return rules[0] if rules.size == 1
+        { '$and' => rules }
+      end
+
+      private
+
+      def parse_equals(rule)
+        if rule.is_include?
+          { rule.field => rule.value }
+        else
+          { rule.field => { '$ne' => rule.value } }
+        end
+      end
+
+      def parse_greater_than(rule)
+        if rule.is_include?
+          { rule.field => { '$gt' => rule.value } }
+        else
+          { rule.field => { '$lte' => rule.value } }
+        end
+      end
+
+      def parse_less_than(rule)
+        if rule.is_include?
+          { rule.field => { '$lt' => rule.value } }
+        else
+          { rule.field => { '$gte' => rule.value } }
+        end
+      end
+
+      def parse_regex(rule)
+        if rule.is_include?
+          { rule.field => /#{rule.value}/ }
+        else
+          { rule.field => { '$not' => /#{rule.value}/ } }
+        end
+      end
+    end
+  end
+end

data/lib/core/configuration.rb

@@ -24,8 +24,10 @@ module Core
             return
           end
           configuration = connector_class.configurable_fields_indifferent_access
+          features = connector_class.kibana_features.each_with_object({}) { |feature, hsh| hsh[feature] = true }
           doc = {
-            :configuration => configuration
+            :configuration => configuration,
+            :features => features
           }
 
           doc[:service_type] = service_type if service_type && connector_settings.needs_service_type?

data/lib/core/connector_job.rb

@@ -0,0 +1,137 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'active_support/core_ext/hash/indifferent_access'
+require 'connectors/sync_status'
+require 'core/elastic_connector_actions'
+require 'utility'
+
+module Core
+  class ConnectorJob
+    DEFAULT_PAGE_SIZE = 100
+
+    # Error Classes
+    class ConnectorJobNotFoundError < StandardError; end
+
+    def self.fetch_by_id(job_id)
+      es_response = ElasticConnectorActions.get_job(job_id)
+
+      raise ConnectorJobNotFoundError.new("Connector job with id=#{job_id} was not found.") unless es_response[:found]
+      new(es_response)
+    end
+
+    def self.pending_jobs(page_size = DEFAULT_PAGE_SIZE)
+      query = { terms: { status: Connectors::SyncStatus::PENDING_STATUES } }
+      fetch_jobs_by_query(query, page_size)
+    end
+
+    def self.orphaned_jobs(_page_size = DEFAULT_PAGE_SIZE)
+      []
+    end
+
+    def self.stuck_jobs(_page_size = DEFAULT_PAGE_SIZE)
+      []
+    end
+
+    def self.enqueue(_connector_id)
+      nil
+    end
+
+    def id
+      @elasticsearch_response[:_id]
+    end
+
+    def [](property_name)
+      @elasticsearch_response[:_source][property_name]
+    end
+
+    def status
+      self[:status]
+    end
+
+    def in_progress?
+      status == Connectors::SyncStatus::IN_PROGRESS
+    end
+
+    def canceling?
+      status == Connectors::SyncStatus::CANCELING
+    end
+
+    def connector_snapshot
+      self[:connector]
+    end
+
+    def connector_id
+      connector_snapshot[:id]
+    end
+
+    def index_name
+      connector_snapshot[:configuration]
+    end
+
+    def language
+      connector_snapshot[:language]
+    end
+
+    def service_type
+      connector_snapshot[:service_type]
+    end
+
+    def configuration
+      connector_snapshot[:configuration]
+    end
+
+    def filtering
+      Utility::Filtering.extract_filter(connector_snapshot[:filtering])
+    end
+
+    def pipeline
+      connector_snapshot[:pipeline]
+    end
+
+    def connector
+      @connector ||= ConnectorSettings.fetch_by_id(connector_id)
+    end
+
+    def reload_connector!
+      @connector = nil
+      connector
+    end
+
+    def reload
+      es_response = ElasticConnectorActions.get_job(id)
+      raise ConnectorJobNotFoundError.new("Connector job with id=#{id} was not found.") unless es_response[:found]
+      # TODO: remove the usage of with_indifferent_access. get_id method is expected to return a hash
+      @elasticsearch_response = es_response.with_indifferent_access
+      @connector = nil
+    end
+
+    private
+
+    def initialize(es_response)
+      # TODO: remove the usage of with_indifferent_access. The initialize method should expect a hash argument
+      @elasticsearch_response = es_response.with_indifferent_access
+    end
+
+    def self.fetch_jobs_by_query(query, page_size)
+      results = []
+      offset = 0
+      loop do
+        response = ElasticConnectorActions.search_jobs(query, page_size, offset)
+
+        hits = response.dig('hits', 'hits') || []
+        total = response.dig('hits', 'total', 'value') || 0
+        results += hits.map { |hit| new(hit) }
+        break if results.size >= total
+        offset += hits.size
+      end
+
+      results
+    end
+  end
+end