connectors_service 8.6.0.4.pre.20221104T200814Z → 8.6.0.4.pre.20221114T233727Z

data/lib/core/connector_settings.rb

@@ -8,6 +8,7 @@
 
 require 'active_support/core_ext/hash/indifferent_access'
 require 'connectors/connector_status'
+require 'connectors/registry'
 require 'core/elastic_connector_actions'
 require 'utility'
 
@@ -34,13 +35,15 @@ module Core
       new(es_response, connectors_meta)
     end
 
-    def initialize(es_response, connectors_meta)
-      @elasticsearch_response = es_response.with_indifferent_access
-      @connectors_meta = connectors_meta.with_indifferent_access
-    end
-
     def self.fetch_native_connectors(page_size = DEFAULT_PAGE_SIZE)
-      query = { term: { is_native: true } }
+      query = {
+        bool: {
+          filter: [
+            { term: { is_native: true } },
+            { terms: { service_type: Connectors::REGISTRY.registered_connectors } }
+          ]
+        }
+      }
       fetch_connectors_by_query(query, page_size)
     end
 
@@ -83,23 +86,26 @@ module Core
     end
 
     def filtering
-      Utility::Common.return_if_present(@elasticsearch_response[:filtering], DEFAULT_FILTERING)
+      # assume for now, that first object in filtering array or a filter object itself is the only filtering object
+      filtering = @elasticsearch_response.dig(:_source, :filtering)
+
+      Utility::Filtering.extract_filter(filtering)
     end
 
     def request_pipeline
-      Utility::Common.return_if_present(@elasticsearch_response.dig(:pipeline, :name), @connectors_meta.dig(:pipeline, :default_name), DEFAULT_REQUEST_PIPELINE)
+      Utility::Common.return_if_present(@elasticsearch_response.dig(:_source, :pipeline, :name), @connectors_meta.dig(:pipeline, :default_name), DEFAULT_REQUEST_PIPELINE)
     end
 
     def extract_binary_content?
-      Utility::Common.return_if_present(@elasticsearch_response.dig(:pipeline, :extract_binary_content), @connectors_meta.dig(:pipeline, :default_extract_binary_content), DEFAULT_EXTRACT_BINARY_CONTENT)
+      Utility::Common.return_if_present(@elasticsearch_response.dig(:_source, :pipeline, :extract_binary_content), @connectors_meta.dig(:pipeline, :default_extract_binary_content), DEFAULT_EXTRACT_BINARY_CONTENT)
     end
 
     def reduce_whitespace?
-      Utility::Common.return_if_present(@elasticsearch_response.dig(:pipeline, :reduce_whitespace), @connectors_meta.dig(:pipeline, :default_reduce_whitespace), DEFAULT_REDUCE_WHITESPACE)
+      Utility::Common.return_if_present(@elasticsearch_response.dig(:_source, :pipeline, :reduce_whitespace), @connectors_meta.dig(:pipeline, :default_reduce_whitespace), DEFAULT_REDUCE_WHITESPACE)
     end
 
     def run_ml_inference?
-      Utility::Common.return_if_present(@elasticsearch_response.dig(:pipeline, :run_ml_inference), @connectors_meta.dig(:pipeline, :default_run_ml_inference), DEFAULT_RUN_ML_INFERENCE)
+      Utility::Common.return_if_present(@elasticsearch_response.dig(:_source, :pipeline, :run_ml_inference), @connectors_meta.dig(:pipeline, :default_run_ml_inference), DEFAULT_RUN_ML_INFERENCE)
     end
 
     def formatted
@@ -116,6 +122,13 @@ module Core
       index_name&.start_with?(Utility::Constants::CONTENT_INDEX_PREFIX)
     end
 
+    private
+
+    def initialize(es_response, connectors_meta)
+      @elasticsearch_response = es_response.with_indifferent_access
+      @connectors_meta = connectors_meta.with_indifferent_access
+    end
+
     def self.fetch_connectors_by_query(query, page_size)
       connectors_meta = ElasticConnectorActions.connectors_meta
 

data/lib/core/elastic_connector_actions.rb

@@ -19,6 +19,12 @@ module Core
     end
   end
 
+  class JobNotCreatedError < StandardError
+    def initialize(connector_id, response)
+      super("Sync job for connector '#{connector_id}' could not be created. Response: #{response}")
+    end
+  end
+
   class ConnectorVersionChangedError < StandardError
     def initialize(connector_id, seq_no, primary_term)
       super("Version conflict: seq_no [#{seq_no}] and primary_term [#{primary_term}] do not match for connector '#{connector_id}'.")
@@ -43,10 +49,17 @@ module Core
       end
 
       def get_connector(connector_id)
+        # TODO: remove the usage of with_indifferent_access. Ideally this should return a hash or nil if not found
         client.get(:index => Utility::Constants::CONNECTORS_INDEX, :id => connector_id, :ignore => 404).with_indifferent_access
       end
 
+      def get_job(job_id)
+        # TODO: remove the usage of with_indifferent_access. Ideally this should return a hash or nil if not found
+        client.get(:index => Utility::Constants::JOB_INDEX, :id => job_id, :ignore => 404).with_indifferent_access
+      end
+
       def connectors_meta
+        # TODO: remove the usage of with_indifferent_access. Ideally this should return a hash or nil if not found
         alias_mappings = client.indices.get_mapping(:index => Utility::Constants::CONNECTORS_INDEX).with_indifferent_access
         index = get_latest_index_in_alias(Utility::Constants::CONNECTORS_INDEX, alias_mappings.keys)
         alias_mappings.dig(index, 'mappings', '_meta') || {}
@@ -65,6 +78,19 @@ module Core
         )
       end
 
+      def search_jobs(query, page_size, offset)
+        client.search(
+          :index => Utility::Constants::JOB_INDEX,
+          :ignore => 404,
+          :body => {
+              :size => page_size,
+              :from => offset,
+              :query => query,
+              :sort => ['created_at']
+          }
+        )
+      end
+
       def update_connector_configuration(connector_id, configuration)
         update_connector_fields(connector_id, :configuration => configuration)
       end
@@ -84,6 +110,28 @@ module Core
         update_connector_configuration(connector_id, payload)
       end
 
+      def update_filtering_validation(connector_id, filter_validation_results)
+        return if filter_validation_results.empty?
+
+        filtering = get_connector(connector_id).dig(:_source, :filtering)
+
+        case filtering
+        when Hash
+          update_filter_validation(filtering, filter_validation_results)
+        when Array
+          return unless should_update_validations?(filter_validation_results, filtering)
+
+          filtering.each do |filter|
+            update_filter_validation(filter, filter_validation_results)
+          end
+        else
+          Utility::Logger.warn("Elasticsearch returned invalid filtering format: #{filtering}. Skipping validation.")
+          return
+        end
+
+        update_connector_fields(connector_id, { :filtering => filtering })
+      end
+
       def claim_job(connector_id)
         seq_no = nil
         primary_term = nil
@@ -111,24 +159,38 @@ module Core
         )
 
         body = {
-          :connector_id => connector_id,
           :status => Connectors::SyncStatus::IN_PROGRESS,
           :worker_hostname => Socket.gethostname,
           :created_at => Time.now,
-          :filtering => convert_connector_filtering_to_job_filtering(connector_record.dig('_source', 'filtering'))
+          :started_at => Time.now,
+          :last_seen => Time.now,
+          :connector => {
+            :id => connector_id,
+            :filtering => convert_connector_filtering_to_job_filtering(connector_record.dig('_source', 'filtering'))
+          }
         }
 
-        client.index(:index => Utility::Constants::JOB_INDEX, :body => body)
+        index_response = client.index(:index => Utility::Constants::JOB_INDEX, :body => body, :refresh => true)
+        if index_response['result'] == 'created'
+          # TODO: remove the usage of with_indifferent_access. Ideally this should return a hash or nil if not found
+          return client.get(
+            :index => Utility::Constants::JOB_INDEX,
+            :id => index_response['_id'],
+            :ignore => 404
+          ).with_indifferent_access
+        end
+        raise JobNotCreatedError.new(connector_id, index_response)
       end
 
       def convert_connector_filtering_to_job_filtering(connector_filtering)
         return [] unless connector_filtering
         connector_filtering = [connector_filtering] unless connector_filtering.is_a?(Array)
         connector_filtering.each_with_object([]) do |filtering_domain, job_filtering|
+          snippet = filtering_domain.dig('active', 'advanced_snippet') || {}
           job_filtering << {
             'domain' => filtering_domain['domain'],
             'rules' => filtering_domain.dig('active', 'rules'),
-            'advanced_snippet' => filtering_domain.dig('active', 'advanced_snippet'),
+            'advanced_snippet' => snippet['value'] || snippet,
             'warnings' => [] # TODO: in https://github.com/elastic/enterprise-search-team/issues/3174
           }
         end
@@ -145,22 +207,33 @@ module Core
         update_connector_fields(connector_id, body)
       end
 
-      def complete_sync(connector_id, job_id, status)
-        sync_status = status[:error] ? Connectors::SyncStatus::ERROR : Connectors::SyncStatus::COMPLETED
+      def update_sync(job_id, metadata)
+        body = {
+          :doc => { :last_seen => Time.now }.merge(metadata)
+        }
+        client.update(:index => Utility::Constants::JOB_INDEX, :id => job_id, :body => body)
+      end
+
+      def complete_sync(connector_id, job_id, metadata, error)
+        sync_status = error ? Connectors::SyncStatus::ERROR : Connectors::SyncStatus::COMPLETED
+
+        metadata ||= {}
 
         update_connector_fields(connector_id,
                                 :last_sync_status => sync_status,
-                                :last_sync_error => status[:error],
-                                :error => status[:error],
+                                :last_sync_error => error,
+                                :error => error,
                                 :last_synced => Time.now,
-                                :last_indexed_document_count => status[:indexed_document_count],
-                                :last_deleted_document_count => status[:deleted_document_count])
+                                :last_indexed_document_count => metadata[:indexed_document_count],
+                                :last_deleted_document_count => metadata[:deleted_document_count])
 
         body = {
           :doc => {
             :status => sync_status,
-            :completed_at => Time.now
-          }.merge(status)
+            :completed_at => Time.now,
+            :last_seen => Time.now,
+            :error => error
+          }.merge(metadata)
         }
         client.update(:index => Utility::Constants::JOB_INDEX, :id => job_id, :body => body)
       end
@@ -248,12 +321,105 @@ module Core
           :properties => {
             :api_key_id => { :type => :keyword },
             :configuration => { :type => :object },
-            :error => { :type => :text },
+            :description => { :type => :text },
+            :error => { :type => :keyword },
+            :features => {
+              :properties => {
+                :filtering_advanced_config => { :type => :boolean },
+                :filtering_rules => { :type => :boolean }
+              }
+            },
+            :filtering => {
+              :properties => {
+                :domain => { :type => :keyword },
+                :active => {
+                  :properties => {
+                    :rules => {
+                      :properties => {
+                        :id => { :type => :keyword },
+                        :policy => { :type => :keyword },
+                        :field => { :type => :keyword },
+                        :rule => { :type => :keyword },
+                        :value => { :type => :keyword },
+                        :order => { :type => :short },
+                        :created_at => { :type => :date },
+                        :updated_at => { :type => :date }
+                      }
+                    },
+                    :advanced_snippet => {
+                      :properties => {
+                        :value => { :type => :object },
+                        :created_at => { :type => :date },
+                        :updated_at => { :type => :date }
+                       }
+                    },
+                    :validation => {
+                      :properties => {
+                        :state => { :type => :keyword },
+                        :errors => {
+                          :properties => {
+                            :ids => { :type => :keyword },
+                            :messages => { :type => :text }
+                          }
+                        }
+                      }
+                    }
+                  }
+                },
+                :draft => {
+                  :properties => {
+                    :rules => {
+                      :properties => {
+                        :id => { :type => :keyword },
+                        :policy => { :type => :keyword },
+                        :field => { :type => :keyword },
+                        :rule => { :type => :keyword },
+                        :value => { :type => :keyword },
+                        :order => { :type => :short },
+                        :created_at => { :type => :date },
+                        :updated_at => { :type => :date }
+                      }
+                    },
+                    :advanced_snippet => {
+                      :properties => {
+                        :value => { :type => :object },
+                        :created_at => { :type => :date },
+                        :updated_at => { :type => :date }
+                      }
+                    },
+                    :validation => {
+                      :properties => {
+                        :state => { :type => :keyword },
+                        :errors => {
+                          :properties => {
+                            :ids => { :type => :keyword },
+                            :messages => { :type => :text }
+                          }
+                        }
+                      }
+                    }
+                  }
+                }
+              }
+            },
             :index_name => { :type => :keyword },
+            :is_native => { :type => :boolean },
+            :language => { :type => :keyword },
             :last_seen => { :type => :date },
+            :last_sync_error => { :type => :keyword },
+            :last_sync_status => { :type => :keyword },
             :last_synced => { :type => :date },
-            :last_indexed_document_count => { :type => :integer },
-            :last_deleted_document_count => { :type => :integer },
+            :last_deleted_document_count => { :type => :long },
+            :last_indexed_document_count => { :type => :long },
+            :name => { :type => :keyword },
+            :pipeline => {
+              :properties => {
+                :extract_binary_content => { :type => :boolean },
+                :name => { :type => :keyword },
+                :reduce_whitespace => { :type => :boolean },
+                :run_ml_inference => { :type => :boolean }
+              }
+            },
             :scheduling => {
               :properties => {
                 :enabled => { :type => :boolean },
@@ -262,9 +428,7 @@ module Core
             },
             :service_type => { :type => :keyword },
             :status => { :type => :keyword },
-            :sync_error => { :type => :text },
-            :sync_now => { :type => :boolean },
-            :sync_status => { :type => :keyword }
+            :sync_now => { :type => :boolean }
           }
         }
         ensure_index_exists("#{Utility::Constants::CONNECTORS_INDEX}-v1", system_index_body(:alias_name => Utility::Constants::CONNECTORS_INDEX, :mappings => mappings))
@@ -275,14 +439,68 @@ module Core
       def ensure_job_index_exists
         mappings = {
           :properties => {
-            :connector_id => { :type => :keyword },
-            :status => { :type => :keyword },
+            :cancelation_requested_at => { :type => :date },
+            :canceled_at => { :type => :date },
+            :completed_at => { :type => :date },
+            :connector => {
+              :properties => {
+                :configuration => { :type => :object },
+                :filtering => {
+                  :properties => {
+                    :domain => { :type => :keyword },
+                    :rules => {
+                      :properties => {
+                        :id => { :type => :keyword },
+                        :policy => { :type => :keyword },
+                        :field => { :type => :keyword },
+                        :rule => { :type => :keyword },
+                        :value => { :type => :keyword },
+                        :order => { :type => :short },
+                        :created_at => { :type => :date },
+                        :updated_at => { :type => :date }
+                      }
+                    },
+                    :advanced_snippet => {
+                      :properties => {
+                        :value => { :type => :object },
+                        :created_at => { :type => :date },
+                        :updated_at => { :type => :date }
+                      }
+                    },
+                    :warnings => {
+                      :properties => {
+                        :ids => { :type => :keyword },
+                        :messages => { :type => :text }
+                      }
+                    }
+                  }
+                },
+                :id => { :type => :keyword },
+                :index_name => { :type => :keyword },
+                :language => { :type => :keyword },
+                :pipeline => {
+                  :properties => {
+                    :extract_binary_content => { :type => :boolean },
+                    :name => { :type => :keyword },
+                    :reduce_whitespace => { :type => :boolean },
+                    :run_ml_inference => { :type => :boolean }
+                  }
+                },
+                :service_type => { :type => :keyword }
+              }
+            },
+            :created_at => { :type => :date },
+            :deleted_document_count => { :type => :integer },
             :error => { :type => :text },
-            :worker_hostname => { :type => :keyword },
             :indexed_document_count => { :type => :integer },
-            :deleted_document_count => { :type => :integer },
-            :created_at => { :type => :date },
-            :completed_at => { :type => :date }
+            :indexed_document_volume => { :type => :integer },
+            :last_seen => { :type => :date },
+            :metadata => { :type => :object },
+            :started_at => { :type => :date },
+            :status => { :type => :keyword },
+            :total_document_count => { :type => :integer },
+            :trigger_method => { :type => :keyword },
+            :worker_hostname => { :type => :keyword }
           }
         }
         ensure_index_exists("#{Utility::Constants::JOB_INDEX}-v1", system_index_body(:alias_name => Utility::Constants::JOB_INDEX, :mappings => mappings))
@@ -313,8 +531,20 @@ module Core
         end
       end
 
+      def document_count(index_name)
+        client.count(:index => index_name)['count']
+      end
+
       private
 
+      def should_update_validations?(domain_validations, filtering)
+        domains_present = filtering.collect { |filter| filter[:domain] }
+        domains_to_update = domain_validations.keys
+
+        # non-empty intersection -> domains to update present
+        !(domains_present & domains_to_update).empty?
+      end
+
       def client
         @client ||= Utility::EsClient.new(App::Config[:elasticsearch])
       end
@@ -324,6 +554,15 @@ module Core
         index_version = index_versions.max # gets the largest suffix number
         "#{alias_name}-v#{index_version}"
       end
+
+      def update_filter_validation(filter, domain_validations)
+        domain = filter[:domain]
+
+        if domain_validations.key?(domain)
+          new_validation_state = { :draft => { :validation => domain_validations[domain] } }
+          filter.deep_merge!(new_validation_state)
+        end
+      end
     end
   end
 end

data/lib/core/filtering/post_process_engine.rb

@@ -0,0 +1,39 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'core/filtering'
+require 'utility/filtering'
+
+module Core
+  module Filtering
+    class PostProcessEngine
+      attr_reader :rules
+
+      def initialize(job_description)
+        @rules = ordered_rules(job_description.dig('connector', 'filtering'))
+      end
+
+      def process(document)
+        @rules.each do |rule|
+          if rule.match?(document.stringify_keys)
+            return PostProcessResult.new(document, rule)
+          end
+        end
+        PostProcessResult.new(document, SimpleRule::DEFAULT_RULE)
+      end
+
+      private
+
+      def ordered_rules(job_filtering)
+        job_rules = Utility::Filtering.extract_filter(job_filtering)['rules']
+        sorted_rules = job_rules.sort_by { |rule| rule['order'] }.reject { |rule| rule['id'] == Core::Filtering::SimpleRule::DEFAULT_RULE_ID }
+        sorted_rules.each_with_object([]) { |rule, output| output << SimpleRule.new(rule) }
+      end
+    end
+  end
+end

data/lib/core/filtering/post_process_result.rb

@@ -0,0 +1,27 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'utility/logger'
+
+module Core
+  module Filtering
+    class PostProcessResult
+      attr_reader :document, :matching_rule
+
+      def initialize(document, matching_rule)
+        @document = document
+        @matching_rule = matching_rule
+        Utility::Logger.debug("Document '#{document['id']}' matched filtering rule: #{matching_rule.id}. It will be #{matching_rule.policy}d")
+      end
+
+      def is_include?
+        matching_rule.is_include?
+      end
+    end
+  end
+end

data/lib/core/filtering/simple_rule.rb

@@ -0,0 +1,141 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'utility/logger'
+
+module Core
+  module Filtering
+    class SimpleRule
+      DEFAULT_RULE_ID = 'DEFAULT'
+
+      class Policy
+        INCLUDE = 'include'
+        EXCLUDE = 'exclude'
+      end
+
+      class Rule
+        REGEX = 'regex'
+        EQUALS = 'equals'
+        STARTS_WITH = 'starts_with'
+        ENDS_WITH = 'ends_with'
+        CONTAINS = 'contains'
+        LESS_THAN = '<'
+        GREATER_THAN = '>'
+      end
+
+      attr_reader :policy, :field, :rule, :value, :id
+
+      def initialize(rule_hash)
+        @policy = rule_hash.fetch('policy')
+        @field = rule_hash.fetch('field')
+        @rule = rule_hash.fetch('rule')
+        @value = rule_hash.fetch('value')
+        @id = rule_hash.fetch('id')
+        @rule_hash = rule_hash
+      rescue KeyError => e
+        raise "#{e.key} is required"
+      end
+
+      def self.from_args(id, policy, field, rule, value)
+        SimpleRule.new(
+          {
+            'id' => id,
+            'policy' => policy,
+            'field' => field,
+            'rule' => rule,
+            'value' => value
+          }
+        )
+      end
+
+      DEFAULT_RULE = SimpleRule.new(
+        'policy' => 'include',
+        'field' => '_',
+        'rule' => 'regex',
+        'value' => '.*',
+        'id' => SimpleRule::DEFAULT_RULE_ID
+      )
+
+      def match?(document)
+        return true if id == DEFAULT_RULE_ID
+        doc_value = document[field]
+        return false if doc_value.nil?
+        coerced_value = coerce(doc_value)
+        case rule
+        when Rule::EQUALS
+          case coerced_value
+          when Integer
+            doc_value == coerced_value
+          when DateTime, Time
+            doc_value.to_s == coerced_value.to_s
+          else
+            doc_value.to_s == coerced_value
+          end
+        when Rule::STARTS_WITH
+          doc_value.to_s.start_with?(value)
+        when Rule::ENDS_WITH
+          doc_value.to_s.end_with?(value)
+        when Rule::CONTAINS
+          doc_value.to_s.include?(value)
+        when Rule::REGEX
+          doc_value.to_s.match(/#{value}/)
+        when Rule::LESS_THAN
+          doc_value < coerced_value
+        when Rule::GREATER_THAN
+          doc_value > coerced_value
+        else
+          false
+        end
+      end
+
+      def coerce(doc_value)
+        case doc_value
+        when String
+          value.to_s
+        when Integer
+          value.to_i
+        when DateTime, Time
+          to_date(value)
+        when TrueClass, FalseClass # Ruby doesn't have a Boolean type, TIL
+          to_bool(value).to_s
+        else
+          value.to_s
+        end
+      rescue StandardError => e
+        Utility::Logger.debug("Failed to coerce value '#{value}' (#{value.class}) based on document value '#{doc_value}' (#{doc_value.class}) due to error: #{e.class}: #{e.message}")
+        value.to_s
+      end
+
+      def is_include?
+        policy == Policy::INCLUDE
+      end
+
+      def is_exclude?
+        policy == Policy::EXCLUDE
+      end
+
+      def to_h
+        @rule_hash
+      end
+
+      private
+
+      def to_bool(str)
+        return true if str == true || str =~ (/^(true|t|yes|y|on|1)$/i)
+        return false if str == false || str.blank? || str =~ (/^(false|f|no|n|off|0)$/i)
+        raise ArgumentError.new("invalid value for Boolean: \"#{str}\"")
+      end
+
+      def to_date(str)
+        DateTime.parse(str)
+      rescue ArgumentError
+        Time.at(str.to_i) # try with it as an int string of millis
+      end
+    end
+  end
+end