connectors_service 8.7.0.0.pre.20221117T010623Z → 8.11.0.0

data/lib/connectors/mongodb/mongo_advanced_snippet_snake_case_transformer.rb

@@ -0,0 +1,49 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'core/filtering/transform/filter_transformer'
+require 'active_support'
+
+module Connectors
+  module MongoDB
+    class MongoAdvancedSnippetSnakeCaseTransformer < Core::Filtering::Transform::FilterTransformer
+
+      def initialize(advanced_snippet = {})
+        super
+
+        @advanced_snippet = advanced_snippet
+        @transformation = ->(snippet) { snake_case_filter(snippet) }
+      end
+
+      private
+
+      def snake_case_filter(advanced_snippet, transformed_filter = {})
+        advanced_snippet&.each do |key, value|
+          snake_case_key = key.to_s.underscore
+
+          value = value.is_a?(Hash) ? snake_case_filter(value, {}) : value
+
+          if value.is_a?(Array)
+            new_entries = []
+
+            value.each do |entry|
+              new_entry = entry.is_a?(Hash) ? snake_case_filter(entry, {}) : entry
+              new_entries.push(new_entry)
+            end
+
+            value = new_entries
+          end
+
+          transformed_filter[snake_case_key] = value
+        end
+
+        transformed_filter
+      end
+    end
+  end
+end

data/lib/connectors/registry.rb

@@ -24,7 +24,7 @@ module Connectors
       @connectors[name]
     end
 
-    def connector(name, configuration, job_description: {})
+    def connector(name, configuration, job_description: nil)
       klass = connector_class(name)
       if klass.present?
         return klass.new(configuration: configuration, job_description: job_description)

data/lib/connectors/tolerable_error_helper.rb

@@ -36,7 +36,11 @@ module Connectors
 
     def fatal_exception_classes
       [
-        Utility::ErrorMonitor::MonitoringError
+        Utility::ErrorMonitor::MonitoringError,
+        Core::ConnectorNotFoundError,
+        Core::ConnectorJobNotFoundError,
+        Core::ConnectorJobCanceledError,
+        Core::ConnectorJobNotRunningError
       ]
     end
   end

data/lib/connectors_utility.rb

@@ -9,8 +9,11 @@
 require_relative 'utility'
 
 require_relative 'connectors/connector_status'
+require_relative 'connectors/crawler/scheduler'
+require_relative 'connectors/job_trigger_method'
 require_relative 'connectors/sync_status'
-require_relative 'core/scheduler'
+require_relative 'core/connector_job'
+require_relative 'core/connector_settings'
 require_relative 'core/elastic_connector_actions'
-
-require_relative 'connectors/crawler/scheduler'
+require_relative 'core/filtering/validation_status'
+require_relative 'core/scheduler'

data/lib/core/configuration.rb

@@ -24,7 +24,19 @@ module Core
             return
           end
           configuration = connector_class.configurable_fields_indifferent_access
-          features = connector_class.kibana_features.each_with_object({}) { |feature, hsh| hsh[feature] = true }
+
+          features = {}
+
+          connector_class.kibana_features.each do |feature_definition, _hsh|
+            feature = feature_definition[:feature]
+            subfeature = feature_definition[:subfeature]
+            enabled = feature_definition[:enabled]
+
+            features[feature] = {} unless features.key?(feature)
+
+            features[feature][subfeature] = { :enabled => enabled }
+          end
+
           doc = {
             :configuration => configuration,
             :features => features

data/lib/core/connector_job.rb

@@ -8,12 +8,14 @@
 
 require 'active_support/core_ext/hash/indifferent_access'
 require 'connectors/sync_status'
+require 'core/connector_settings'
 require 'core/elastic_connector_actions'
 require 'utility'
 
 module Core
   class ConnectorJob
     DEFAULT_PAGE_SIZE = 100
+    IDLE_THRESHOLD = 60
 
     def self.fetch_by_id(job_id)
       es_response = ElasticConnectorActions.get_job(job_id)
@@ -34,12 +36,32 @@ module Core
       fetch_jobs_by_query(query, page_size)
     end
 
-    def self.orphaned_jobs(_page_size = DEFAULT_PAGE_SIZE)
-      []
+    def self.orphaned_jobs(connector_ids = [], page_size = DEFAULT_PAGE_SIZE)
+      query = { bool: { must_not: { terms: { 'connector.id': connector_ids } } } }
+      fetch_jobs_by_query(query, page_size)
     end
 
-    def self.stuck_jobs(_page_size = DEFAULT_PAGE_SIZE)
-      []
+    def self.delete_jobs(jobs)
+      query = { terms: { '_id': jobs.map(&:id) } }
+      ElasticConnectorActions.delete_jobs_by_query(query)
+    end
+
+    def self.idle_jobs(connector_id = nil, page_size = DEFAULT_PAGE_SIZE)
+      connector_ids = if connector_id
+                        [connector_id]
+                      else
+                        ConnectorSettings.fetch_native_connectors.map(&:id)
+                      end
+      query = {
+        bool: {
+          filter: [
+              { terms: { 'connector.id': connector_ids } },
+              { terms: { status: Connectors::SyncStatus::ACTIVE_STATUSES } },
+              { range: { last_seen: { lte: "now-#{IDLE_THRESHOLD}s" } } }
+          ]
+        }
+      }
+      fetch_jobs_by_query(query, page_size)
     end
 
     def self.enqueue(_connector_id)
@@ -95,7 +117,7 @@ module Core
     end
 
     def connector_id
-      @elasticsearch_response[:_source][:connector][:id]
+      connector_snapshot[:id]
     end
 
     def index_name
@@ -115,17 +137,36 @@ module Core
     end
 
     def filtering
-      Utility::Filtering.extract_filter(connector_snapshot[:filtering])
+      connector_snapshot[:filtering]
     end
 
     def pipeline
-      @elasticsearch_response[:_source][:pipeline]
+      connector_snapshot[:pipeline] || {}
+    end
+
+    def extract_binary_content?
+      pipeline[:extract_binary_content]
+    end
+
+    def reduce_whitespace?
+      pipeline[:reduce_whitespace]
+    end
+
+    def run_ml_inference?
+      pipeline[:run_ml_inference]
     end
 
     def connector
       @connector ||= ConnectorSettings.fetch_by_id(connector_id)
     end
 
+    def update_metadata(ingestion_stats = {}, connector_metadata = {})
+      ingestion_stats ||= {}
+      doc = { :last_seen => Time.now }.merge(ingestion_stats)
+      doc[:metadata] = connector_metadata if connector_metadata&.any?
+      ElasticConnectorActions.update_job_fields(id, doc)
+    end
+
     def done!(ingestion_stats = {}, connector_metadata = {})
       terminate!(Connectors::SyncStatus::COMPLETED, nil, ingestion_stats, connector_metadata)
     end

data/lib/core/connector_settings.rb

@@ -8,6 +8,7 @@
 
 require 'active_support/core_ext/hash/indifferent_access'
 require 'connectors/connector_status'
+require 'connectors/sync_status'
 require 'core/elastic_connector_actions'
 require 'utility'
 
@@ -49,6 +50,11 @@ module Core
       fetch_connectors_by_query(query, page_size)
     end
 
+    def self.fetch_all_connectors(page_size = DEFAULT_PAGE_SIZE)
+      query = { match_all: {} }
+      fetch_connectors_by_query(query, page_size)
+    end
+
     def id
       @elasticsearch_response[:_id]
     end
@@ -58,6 +64,24 @@ module Core
       @elasticsearch_response[:_source][property_name]
     end
 
+    def features
+      self[:features] || {}
+    end
+
+    # .dig version is the modern features way of doing things,
+    # Right-hand of OR operator is legacy features support
+    # When this is fixed with a migration, we can go ahead
+    def filtering_rule_feature_enabled?
+      !!features.dig(:sync_rules, :basic, :enabled) || !!features[:filtering_rules]
+    end
+    def filtering_advanced_config_feature_enabled?
+      !!features.dig(:sync_rules, :advanced, :enabled) || !!features[:filtering_advanced_config]
+    end
+
+    def any_filtering_feature_enabled?
+      filtering_rule_feature_enabled? || filtering_advanced_config_feature_enabled?
+    end
+
     def index_name
       self[:index_name]
     end
@@ -79,30 +103,34 @@ module Core
     end
 
     def scheduling_settings
-      self[:scheduling]
+      self[:scheduling] || {}
     end
 
-    def filtering
-      # assume for now, that first object in filtering array or a filter object itself is the only filtering object
-      filtering = @elasticsearch_response.dig(:_source, :filtering)
+    def full_sync_scheduling
+      scheduling_settings[:full]
+    end
 
-      Utility::Filtering.extract_filter(filtering)
+    def custom_scheduling_settings
+      self[:custom_scheduling]
     end
 
-    def request_pipeline
-      Utility::Common.return_if_present(@elasticsearch_response.dig(:_source, :pipeline, :name), @connectors_meta.dig(:pipeline, :default_name), DEFAULT_REQUEST_PIPELINE)
+    def sync_now?
+      self[:sync_now] == true
     end
 
-    def extract_binary_content?
-      Utility::Common.return_if_present(@elasticsearch_response.dig(:_source, :pipeline, :extract_binary_content), @connectors_meta.dig(:pipeline, :default_extract_binary_content), DEFAULT_EXTRACT_BINARY_CONTENT)
+    def last_synced
+      self[:last_synced]
     end
 
-    def reduce_whitespace?
-      Utility::Common.return_if_present(@elasticsearch_response.dig(:_source, :pipeline, :reduce_whitespace), @connectors_meta.dig(:pipeline, :default_reduce_whitespace), DEFAULT_REDUCE_WHITESPACE)
+    def filtering
+      # assume for now, that first object in filtering array or a filter object itself is the only filtering object
+      filtering = @elasticsearch_response.dig(:_source, :filtering)
+
+      Utility::Filtering.extract_filter(filtering)
     end
 
-    def run_ml_inference?
-      Utility::Common.return_if_present(@elasticsearch_response.dig(:_source, :pipeline, :run_ml_inference), @connectors_meta.dig(:pipeline, :default_run_ml_inference), DEFAULT_RUN_ML_INFERENCE)
+    def request_pipeline
+      Utility::Common.return_if_present(@elasticsearch_response.dig(:_source, :pipeline, :name), @connectors_meta.dig(:pipeline, :default_name), DEFAULT_REQUEST_PIPELINE)
     end
 
     def formatted
@@ -130,19 +158,23 @@ module Core
     end
 
     def update_last_sync!(job)
+      # if job is nil, connector still needs to be updated, to avoid it stuck at in_progress
+      job_status = job&.status || Connectors::SyncStatus::ERROR
+      job_error = job.nil? ? 'Could\'t find the job' : job.error
+      job_error ||= 'unknown error' if job_status == Connectors::SyncStatus::ERROR
+      connector_status = (job_status == Connectors::SyncStatus::ERROR ? Connectors::ConnectorStatus::ERROR : Connectors::ConnectorStatus::CONNECTED)
       doc = {
-        :last_sync_status => job.status,
+        :last_sync_status => job_status,
         :last_synced => Time.now,
-        :last_sync_error => job.error,
-        :error => job.error
+        :last_sync_error => job_error,
+        :status => connector_status,
+        :error => job_error
       }
-
-      if job.terminated?
+      if job&.terminated?
         doc[:last_indexed_document_count] = job[:indexed_document_count]
         doc[:last_deleted_document_count] = job[:deleted_document_count]
       end
-
-      Core::ElasticConnectorActions.update_connector_fields(job.connector_id, doc)
+      Core::ElasticConnectorActions.update_connector_fields(id, doc)
     end
 
     private

data/lib/core/elastic_connector_actions.rb

@@ -8,6 +8,7 @@
 #
 require 'active_support/core_ext/hash'
 require 'connectors/connector_status'
+require 'connectors/job_trigger_method'
 require 'connectors/sync_status'
 require 'utility'
 require 'elastic-transport'
@@ -60,9 +61,14 @@ module Core
 
       def connectors_meta
         # TODO: remove the usage of with_indifferent_access. Ideally this should return a hash or nil if not found
-        alias_mappings = client.indices.get_mapping(:index => Utility::Constants::CONNECTORS_INDEX).with_indifferent_access
+        alias_mappings = client.indices.get_mapping(:index => Utility::Constants::CONNECTORS_INDEX, :ignore => 404).with_indifferent_access
         index = get_latest_index_in_alias(Utility::Constants::CONNECTORS_INDEX, alias_mappings.keys)
-        alias_mappings.dig(index, 'mappings', '_meta') || {}
+        alias_mappings.dig(index, 'mappings', '_meta') || {
+          :extract_binary_content => true,
+          :name => 'ent-search-generic-ingestion',
+          :reduce_whitespace => true,
+          :run_ml_inference => false,
+        }
       end
 
       def search_connectors(query, page_size, offset)
@@ -91,6 +97,17 @@ module Core
         )
       end
 
+      def delete_jobs_by_query(query)
+        client.delete_by_query(
+          :index => Utility::Constants::JOB_INDEX,
+          :body => { :query => query }
+        )
+      end
+
+      def delete_indices(indices)
+        client.indices.delete(:index => indices, :ignore_unavailable => true)
+      end
+
       def update_connector_configuration(connector_id, configuration)
         update_connector_fields(connector_id, :configuration => configuration)
       end
@@ -145,12 +162,37 @@ module Core
         )
       end
 
-      def update_connector_last_sync_status(connector_id, last_sync_status)
+      def update_connector_sync_start(connector_id)
         doc = connector_with_concurrency_control(connector_id)
 
+        body = {
+            last_sync_status: Connectors::SyncStatus::IN_PROGRESS,
+            last_sync_error: nil,
+            status: Connectors::ConnectorStatus::CONNECTED
+        }
+
         update_connector_fields(
           connector_id,
-          { last_sync_status: last_sync_status },
+          body,
+          doc[:seq_no],
+          doc[:primary_term]
+        )
+      end
+
+      def update_connector_custom_scheduling_last_synced(connector_id, schedule_key)
+        doc = connector_with_concurrency_control(connector_id)
+
+        body = {
+          :custom_scheduling => {
+            schedule_key => {
+              :last_synced => Time.now
+            }
+          }
+        }
+
+        update_connector_fields(
+          connector_id,
+          body,
           doc[:seq_no],
           doc[:primary_term]
         )
@@ -178,13 +220,15 @@ module Core
           status: Connectors::SyncStatus::PENDING,
           created_at: Time.now,
           last_seen: Time.now,
+          trigger_method: connector_settings.sync_now? ? Connectors::JobTriggerMethod::ON_DEMAND : Connectors::JobTriggerMethod::SCHEDULED,
           connector: {
             id: connector_settings.id,
             filtering: convert_connector_filtering_to_job_filtering(connector_settings.filtering),
             index_name: connector_settings.index_name,
             language: connector_settings[:language],
             pipeline: connector_settings[:pipeline],
-            service_type: connector_settings.service_type
+            service_type: connector_settings.service_type,
+            configuration: connector_settings.configuration
           }
         }
 
@@ -220,37 +264,6 @@ module Core
         update_connector_fields(connector_id, body)
       end
 
-      def update_sync(job_id, metadata)
-        body = {
-          :doc => { :last_seen => Time.now }.merge(metadata)
-        }
-        client.update(:index => Utility::Constants::JOB_INDEX, :id => job_id, :body => body)
-      end
-
-      def complete_sync(connector_id, job_id, metadata, error)
-        sync_status = error ? Connectors::SyncStatus::ERROR : Connectors::SyncStatus::COMPLETED
-
-        metadata ||= {}
-
-        update_connector_fields(connector_id,
-                                :last_sync_status => sync_status,
-                                :last_sync_error => error,
-                                :error => error,
-                                :last_synced => Time.now,
-                                :last_indexed_document_count => metadata[:indexed_document_count],
-                                :last_deleted_document_count => metadata[:deleted_document_count])
-
-        body = {
-          :doc => {
-            :status => sync_status,
-            :completed_at => Time.now,
-            :last_seen => Time.now,
-            :error => error
-          }.merge(metadata)
-        }
-        client.update(:index => Utility::Constants::JOB_INDEX, :id => job_id, :body => body)
-      end
-
       def fetch_document_ids(index_name)
         page_size = 1000
         result = []
@@ -331,9 +344,11 @@ module Core
       # Creation of connector index should be handled by Kibana, this method is only used by ftest.rb
       def ensure_connectors_index_exists
         mappings = {
+          :dynamic => false,
           :properties => {
             :api_key_id => { :type => :keyword },
             :configuration => { :type => :object },
+            :custom_schedule => { :type => :object },
             :description => { :type => :text },
             :error => { :type => :keyword },
             :features => {
@@ -451,6 +466,7 @@ module Core
       # Creation of job index should be handled by Kibana, this method is only used by ftest.rb
       def ensure_job_index_exists
         mappings = {
+          :dynamic => false,
           :properties => {
             :cancelation_requested_at => { :type => :date },
             :canceled_at => { :type => :date },
@@ -528,8 +544,8 @@ module Core
       end
 
       def document_count(index_name)
-        client.indices.refresh(:index => index_name)
-        client.count(:index => index_name)['count']
+        client.indices.refresh(:index => index_name, :ignore_unavailable => true)
+        client.count(:index => index_name, :ignore_unavailable => true)['count']
       end
 
       private

data/lib/core/filtering/advanced_snippet/advanced_snippet_against_schema_validator.rb

@@ -0,0 +1,32 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+# frozen_string_literal: true
+
+require 'active_support/core_ext/hash'
+require 'utility/logger'
+require 'core/filtering/advanced_snippet/advanced_snippet_validator'
+require 'core/filtering/validation_status'
+require 'core/filtering/hash_against_schema_validator'
+
+module Core
+  module Filtering
+    module AdvancedSnippet
+      class AdvancedSnippetAgainstSchemaValidator < Core::Filtering::AdvancedSnippet::AdvancedSnippetValidator
+
+        def initialize(advanced_snippet, schema)
+          super(advanced_snippet)
+          @schema = schema
+          @schema_validator = Core::Filtering::SchemaValidator.new(schema: schema, payload: advanced_snippet, error_id: ADVANCED_SNIPPET_ID)
+        end
+
+        def is_snippet_valid
+          @schema_validator.validate_against_schema
+        end
+
+      end
+    end
+  end
+end

data/lib/core/filtering/advanced_snippet/advanced_snippet_validator.rb

@@ -0,0 +1,27 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+# frozen_string_literal: true
+
+require 'utility/logger'
+
+module Core
+  module Filtering
+    module AdvancedSnippet
+      class AdvancedSnippetValidator
+
+        ADVANCED_SNIPPET_ID = 'advanced_snippet'
+
+        def initialize(advanced_snippet)
+          @advanced_snippet = advanced_snippet || {}
+        end
+
+        def is_snippet_valid
+          raise 'Advanced Snippet validation not implemented'
+        end
+      end
+    end
+  end
+end

data/lib/core/filtering/filter_validator.rb

@@ -0,0 +1,103 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+# frozen_string_literal: true
+
+require 'core/filtering/validation_status'
+require 'core/filtering/processing_stage'
+require 'utility/logger'
+
+module Core
+  module Filtering
+    class FilterValidator
+
+      ADVANCED_SNIPPET = 'Advanced Snippet'
+      SIMPLE_RULES = 'Simple Rules'
+
+      def initialize(snippet_validator_classes: [], rules_validator_classes: [], rules_pre_processing_active: false)
+        @snippet_validators_classes = {
+            'classes' => extract_advanced_snippet_validators(snippet_validator_classes),
+            'type' => ADVANCED_SNIPPET
+          }
+
+        @rules_validator_classes = {
+            'classes' => extract_simple_rule_validators(rules_validator_classes, rules_pre_processing_active),
+            'type' => SIMPLE_RULES
+        }
+      end
+
+      def is_filter_valid(filter = {})
+        return { :state => Core::Filtering::ValidationStatus::VALID, :errors => [] } unless filter.present?
+
+        snippet_validation_result = execute_validation(@snippet_validators_classes, filter)
+        log_validation_result(snippet_validation_result, ADVANCED_SNIPPET)
+
+        rules_validation_result = execute_validation(@rules_validator_classes, filter)
+        log_validation_result(rules_validation_result, SIMPLE_RULES)
+
+        merge_validation_results(snippet_validation_result, rules_validation_result)
+      end
+
+      private
+
+      def log_validation_result(validation_result, validator_type)
+        Utility::Logger.info("Filtering #{validator_type} validation result: #{validation_result[:state]}")
+        if validation_result[:errors].present?
+          validation_result[:errors].each do |error|
+            Utility::Logger.warn("Filtering #{validator_type} validation error for: '#{error[:ids]}': '#{error[:messages]}'")
+          end
+        end
+      end
+
+      def merge_validation_results(*validation_results)
+        merged_result = { :state => Core::Filtering::ValidationStatus::VALID, :errors => [] }
+
+        validation_results.each do |validation_result|
+          merged_result[:state] = Core::Filtering::ValidationStatus::INVALID if validation_result[:state] == Core::Filtering::ValidationStatus::INVALID
+          merged_result[:errors].push(*validation_result[:errors]) if validation_result[:errors].present?
+        end
+
+        merged_result
+      end
+
+      def execute_validation(validators, filter)
+        validation_results = []
+
+        validator_type = validators['type']
+        advanced_snippet = filter.dig('advanced_snippet', 'value')
+
+        validators['classes'].each do |validator_class|
+          case validator_type
+          when ADVANCED_SNIPPET
+            validation_result = validator_class.new(advanced_snippet).is_snippet_valid
+          when SIMPLE_RULES
+            validation_result = validator_class.new(filter['rules']).are_rules_valid
+          else
+            raise "Unknown validator: #{validator_type}"
+          end
+
+          validation_results.push(validation_result) if validation_result[:state] == Core::Filtering::ValidationStatus::INVALID
+        end
+
+        merge_validation_results(*validation_results)
+      end
+
+      def extract_advanced_snippet_validators(snippet_validators)
+        snippet_validators.is_a?(Array) ? snippet_validators : [snippet_validators]
+      end
+
+      def extract_simple_rule_validators(rule_validators, pre_processing_active)
+        return rule_validators if rule_validators.is_a?(Array)
+
+        common_validators = rule_validators[Core::Filtering::ProcessingStage::ALL] || []
+        pre_validators = rule_validators[Core::Filtering::ProcessingStage::PRE] || []
+        post_validators = rule_validators[Core::Filtering::ProcessingStage::POST] || []
+
+        # post processing validation will always be executed
+        pre_processing_active ? common_validators + pre_validators + post_validators : common_validators + post_validators
+      end
+    end
+  end
+end