connectors_service 8.6.0.4.pre.20221104T200814Z → 8.6.0.4.pre.20221116T024501Z

data/lib/core/elastic_connector_actions.rb

@@ -19,6 +19,12 @@ module Core
     end
   end
 
+  class JobNotCreatedError < StandardError
+    def initialize(connector_id, response)
+      super("Sync job for connector '#{connector_id}' could not be created. Response: #{response}")
+    end
+  end
+
   class ConnectorVersionChangedError < StandardError
     def initialize(connector_id, seq_no, primary_term)
       super("Version conflict: seq_no [#{seq_no}] and primary_term [#{primary_term}] do not match for connector '#{connector_id}'.")
@@ -43,10 +49,17 @@ module Core
       end
 
       def get_connector(connector_id)
+        # TODO: remove the usage of with_indifferent_access. Ideally this should return a hash or nil if not found
         client.get(:index => Utility::Constants::CONNECTORS_INDEX, :id => connector_id, :ignore => 404).with_indifferent_access
       end
 
+      def get_job(job_id)
+        # TODO: remove the usage of with_indifferent_access. Ideally this should return a hash or nil if not found
+        client.get(:index => Utility::Constants::JOB_INDEX, :id => job_id, :ignore => 404).with_indifferent_access
+      end
+
       def connectors_meta
+        # TODO: remove the usage of with_indifferent_access. Ideally this should return a hash or nil if not found
         alias_mappings = client.indices.get_mapping(:index => Utility::Constants::CONNECTORS_INDEX).with_indifferent_access
         index = get_latest_index_in_alias(Utility::Constants::CONNECTORS_INDEX, alias_mappings.keys)
         alias_mappings.dig(index, 'mappings', '_meta') || {}
@@ -65,6 +78,19 @@ module Core
         )
       end
 
+      def search_jobs(query, page_size, offset)
+        client.search(
+          :index => Utility::Constants::JOB_INDEX,
+          :ignore => 404,
+          :body => {
+              :size => page_size,
+              :from => offset,
+              :query => query,
+              :sort => ['created_at']
+          }
+        )
+      end
+
       def update_connector_configuration(connector_id, configuration)
         update_connector_fields(connector_id, :configuration => configuration)
       end
@@ -84,11 +110,57 @@ module Core
         update_connector_configuration(connector_id, payload)
       end
 
-      def claim_job(connector_id)
+      def update_filtering_validation(connector_id, filter_validation_results)
+        return if filter_validation_results.empty?
+
+        filtering = get_connector(connector_id).dig(:_source, :filtering)
+
+        case filtering
+        when Hash
+          update_filter_validation(filtering, filter_validation_results)
+        when Array
+          return unless should_update_validations?(filter_validation_results, filtering)
+
+          filtering.each do |filter|
+            update_filter_validation(filter, filter_validation_results)
+          end
+        else
+          Utility::Logger.warn("Elasticsearch returned invalid filtering format: #{filtering}. Skipping validation.")
+          return
+        end
+
+        update_connector_fields(connector_id, { :filtering => filtering })
+      end
+
+      def update_connector_sync_now(connector_id, sync_now)
+        doc = connector_with_concurrency_control(connector_id)
+
+        body = { sync_now: sync_now, last_synced: Time.now }
+
+        update_connector_fields(
+          connector_id,
+          body,
+          doc[:seq_no],
+          doc[:primary_term]
+        )
+      end
+
+      def update_connector_last_sync_status(connector_id, last_sync_status)
+        doc = connector_with_concurrency_control(connector_id)
+
+        update_connector_fields(
+          connector_id,
+          { last_sync_status: last_sync_status },
+          doc[:seq_no],
+          doc[:primary_term]
+        )
+      end
+
+      def connector_with_concurrency_control(connector_id)
         seq_no = nil
         primary_term = nil
-        sync_in_progress = false
-        connector_record = client.get(
+
+        doc = client.get(
           :index => Utility::Constants::CONNECTORS_INDEX,
           :id => connector_id,
           :ignore => 404,
@@ -96,39 +168,42 @@ module Core
         ).tap do |response|
           seq_no = response['_seq_no']
           primary_term = response['_primary_term']
-          sync_in_progress = response.dig('_source', 'last_sync_status') == Connectors::SyncStatus::IN_PROGRESS
         end
-        if sync_in_progress
-          raise JobAlreadyRunningError.new(connector_id)
-        end
-        update_connector_fields(
-          connector_id,
-          { :sync_now => false,
-            :last_sync_status => Connectors::SyncStatus::IN_PROGRESS,
-            :last_synced => Time.now },
-          seq_no,
-          primary_term
-        )
 
+        { doc: doc, seq_no: seq_no, primary_term: primary_term }
+      end
+
+      def create_job(connector_settings:)
         body = {
-          :connector_id => connector_id,
-          :status => Connectors::SyncStatus::IN_PROGRESS,
-          :worker_hostname => Socket.gethostname,
-          :created_at => Time.now,
-          :filtering => convert_connector_filtering_to_job_filtering(connector_record.dig('_source', 'filtering'))
+          status: Connectors::SyncStatus::PENDING,
+          created_at: Time.now,
+          last_seen: Time.now,
+          connector: {
+            id: connector_settings.id,
+            filtering: convert_connector_filtering_to_job_filtering(connector_settings.filtering),
+            index_name: connector_settings.index_name,
+            language: connector_settings[:language],
+            pipeline: connector_settings[:pipeline],
+            service_type: connector_settings.service_type
+          }
         }
 
-        client.index(:index => Utility::Constants::JOB_INDEX, :body => body)
+        index_response = client.index(index: Utility::Constants::JOB_INDEX, body: body, refresh: true)
+
+        return index_response if index_response['result'] == 'created'
+
+        raise JobNotCreatedError.new(connector_settings.id, index_response)
       end
 
       def convert_connector_filtering_to_job_filtering(connector_filtering)
         return [] unless connector_filtering
         connector_filtering = [connector_filtering] unless connector_filtering.is_a?(Array)
         connector_filtering.each_with_object([]) do |filtering_domain, job_filtering|
+          snippet = filtering_domain.dig('active', 'advanced_snippet') || {}
           job_filtering << {
             'domain' => filtering_domain['domain'],
             'rules' => filtering_domain.dig('active', 'rules'),
-            'advanced_snippet' => filtering_domain.dig('active', 'advanced_snippet'),
+            'advanced_snippet' => snippet['value'] || snippet,
             'warnings' => [] # TODO: in https://github.com/elastic/enterprise-search-team/issues/3174
           }
         end
@@ -145,22 +220,33 @@ module Core
         update_connector_fields(connector_id, body)
       end
 
-      def complete_sync(connector_id, job_id, status)
-        sync_status = status[:error] ? Connectors::SyncStatus::ERROR : Connectors::SyncStatus::COMPLETED
+      def update_sync(job_id, metadata)
+        body = {
+          :doc => { :last_seen => Time.now }.merge(metadata)
+        }
+        client.update(:index => Utility::Constants::JOB_INDEX, :id => job_id, :body => body)
+      end
+
+      def complete_sync(connector_id, job_id, metadata, error)
+        sync_status = error ? Connectors::SyncStatus::ERROR : Connectors::SyncStatus::COMPLETED
+
+        metadata ||= {}
 
         update_connector_fields(connector_id,
                                 :last_sync_status => sync_status,
-                                :last_sync_error => status[:error],
-                                :error => status[:error],
+                                :last_sync_error => error,
+                                :error => error,
                                 :last_synced => Time.now,
-                                :last_indexed_document_count => status[:indexed_document_count],
-                                :last_deleted_document_count => status[:deleted_document_count])
+                                :last_indexed_document_count => metadata[:indexed_document_count],
+                                :last_deleted_document_count => metadata[:deleted_document_count])
 
         body = {
           :doc => {
             :status => sync_status,
-            :completed_at => Time.now
-          }.merge(status)
+            :completed_at => Time.now,
+            :last_seen => Time.now,
+            :error => error
+          }.merge(metadata)
         }
         client.update(:index => Utility::Constants::JOB_INDEX, :id => job_id, :body => body)
       end
@@ -248,12 +334,105 @@ module Core
           :properties => {
             :api_key_id => { :type => :keyword },
             :configuration => { :type => :object },
-            :error => { :type => :text },
+            :description => { :type => :text },
+            :error => { :type => :keyword },
+            :features => {
+              :properties => {
+                :filtering_advanced_config => { :type => :boolean },
+                :filtering_rules => { :type => :boolean }
+              }
+            },
+            :filtering => {
+              :properties => {
+                :domain => { :type => :keyword },
+                :active => {
+                  :properties => {
+                    :rules => {
+                      :properties => {
+                        :id => { :type => :keyword },
+                        :policy => { :type => :keyword },
+                        :field => { :type => :keyword },
+                        :rule => { :type => :keyword },
+                        :value => { :type => :keyword },
+                        :order => { :type => :short },
+                        :created_at => { :type => :date },
+                        :updated_at => { :type => :date }
+                      }
+                    },
+                    :advanced_snippet => {
+                      :properties => {
+                        :value => { :type => :object },
+                        :created_at => { :type => :date },
+                        :updated_at => { :type => :date }
+                       }
+                    },
+                    :validation => {
+                      :properties => {
+                        :state => { :type => :keyword },
+                        :errors => {
+                          :properties => {
+                            :ids => { :type => :keyword },
+                            :messages => { :type => :text }
+                          }
+                        }
+                      }
+                    }
+                  }
+                },
+                :draft => {
+                  :properties => {
+                    :rules => {
+                      :properties => {
+                        :id => { :type => :keyword },
+                        :policy => { :type => :keyword },
+                        :field => { :type => :keyword },
+                        :rule => { :type => :keyword },
+                        :value => { :type => :keyword },
+                        :order => { :type => :short },
+                        :created_at => { :type => :date },
+                        :updated_at => { :type => :date }
+                      }
+                    },
+                    :advanced_snippet => {
+                      :properties => {
+                        :value => { :type => :object },
+                        :created_at => { :type => :date },
+                        :updated_at => { :type => :date }
+                      }
+                    },
+                    :validation => {
+                      :properties => {
+                        :state => { :type => :keyword },
+                        :errors => {
+                          :properties => {
+                            :ids => { :type => :keyword },
+                            :messages => { :type => :text }
+                          }
+                        }
+                      }
+                    }
+                  }
+                }
+              }
+            },
             :index_name => { :type => :keyword },
+            :is_native => { :type => :boolean },
+            :language => { :type => :keyword },
             :last_seen => { :type => :date },
+            :last_sync_error => { :type => :keyword },
+            :last_sync_status => { :type => :keyword },
             :last_synced => { :type => :date },
-            :last_indexed_document_count => { :type => :integer },
-            :last_deleted_document_count => { :type => :integer },
+            :last_deleted_document_count => { :type => :long },
+            :last_indexed_document_count => { :type => :long },
+            :name => { :type => :keyword },
+            :pipeline => {
+              :properties => {
+                :extract_binary_content => { :type => :boolean },
+                :name => { :type => :keyword },
+                :reduce_whitespace => { :type => :boolean },
+                :run_ml_inference => { :type => :boolean }
+              }
+            },
             :scheduling => {
               :properties => {
                 :enabled => { :type => :boolean },
@@ -262,9 +441,7 @@ module Core
             },
             :service_type => { :type => :keyword },
             :status => { :type => :keyword },
-            :sync_error => { :type => :text },
-            :sync_now => { :type => :boolean },
-            :sync_status => { :type => :keyword }
+            :sync_now => { :type => :boolean }
           }
         }
         ensure_index_exists("#{Utility::Constants::CONNECTORS_INDEX}-v1", system_index_body(:alias_name => Utility::Constants::CONNECTORS_INDEX, :mappings => mappings))
@@ -275,55 +452,139 @@ module Core
       def ensure_job_index_exists
         mappings = {
           :properties => {
-            :connector_id => { :type => :keyword },
-            :status => { :type => :keyword },
+            :cancelation_requested_at => { :type => :date },
+            :canceled_at => { :type => :date },
+            :completed_at => { :type => :date },
+            :connector => {
+              :properties => {
+                :configuration => { :type => :object },
+                :filtering => {
+                  :properties => {
+                    :domain => { :type => :keyword },
+                    :rules => {
+                      :properties => {
+                        :id => { :type => :keyword },
+                        :policy => { :type => :keyword },
+                        :field => { :type => :keyword },
+                        :rule => { :type => :keyword },
+                        :value => { :type => :keyword },
+                        :order => { :type => :short },
+                        :created_at => { :type => :date },
+                        :updated_at => { :type => :date }
+                      }
+                    },
+                    :advanced_snippet => {
+                      :properties => {
+                        :value => { :type => :object },
+                        :created_at => { :type => :date },
+                        :updated_at => { :type => :date }
+                      }
+                    },
+                    :warnings => {
+                      :properties => {
+                        :ids => { :type => :keyword },
+                        :messages => { :type => :text }
+                      }
+                    }
+                  }
+                },
+                :id => { :type => :keyword },
+                :index_name => { :type => :keyword },
+                :language => { :type => :keyword },
+                :pipeline => {
+                  :properties => {
+                    :extract_binary_content => { :type => :boolean },
+                    :name => { :type => :keyword },
+                    :reduce_whitespace => { :type => :boolean },
+                    :run_ml_inference => { :type => :boolean }
+                  }
+                },
+                :service_type => { :type => :keyword }
+              }
+            },
+            :created_at => { :type => :date },
+            :deleted_document_count => { :type => :integer },
             :error => { :type => :text },
-            :worker_hostname => { :type => :keyword },
             :indexed_document_count => { :type => :integer },
-            :deleted_document_count => { :type => :integer },
-            :created_at => { :type => :date },
-            :completed_at => { :type => :date }
+            :indexed_document_volume => { :type => :integer },
+            :last_seen => { :type => :date },
+            :metadata => { :type => :object },
+            :started_at => { :type => :date },
+            :status => { :type => :keyword },
+            :total_document_count => { :type => :integer },
+            :trigger_method => { :type => :keyword },
+            :worker_hostname => { :type => :keyword }
           }
         }
         ensure_index_exists("#{Utility::Constants::JOB_INDEX}-v1", system_index_body(:alias_name => Utility::Constants::JOB_INDEX, :mappings => mappings))
       end
 
       def update_connector_fields(connector_id, doc = {}, seq_no = nil, primary_term = nil)
+        update_doc_fields(Utility::Constants::CONNECTORS_INDEX, connector_id, doc, seq_no, primary_term)
+      end
+
+      def update_job_fields(job_id, doc = {}, seq_no = nil, primary_term = nil)
+        update_doc_fields(Utility::Constants::JOB_INDEX, job_id, doc, seq_no, primary_term)
+      end
+
+      def document_count(index_name)
+        client.indices.refresh(:index => index_name)
+        client.count(:index => index_name)['count']
+      end
+
+      private
+
+      def should_update_validations?(domain_validations, filtering)
+        domains_present = filtering.collect { |filter| filter[:domain] }
+        domains_to_update = domain_validations.keys
+
+        # non-empty intersection -> domains to update present
+        !(domains_present & domains_to_update).empty?
+      end
+
+      def client
+        @client ||= Utility::EsClient.new(App::Config[:elasticsearch])
+      end
+
+      def get_latest_index_in_alias(alias_name, indicies)
+        index_versions = indicies.map { |index| index.gsub("#{alias_name}-v", '').to_i }
+        index_version = index_versions.max # gets the largest suffix number
+        "#{alias_name}-v#{index_version}"
+      end
+
+      def update_filter_validation(filter, domain_validations)
+        domain = filter[:domain]
+
+        if domain_validations.key?(domain)
+          new_validation_state = { :draft => { :validation => domain_validations[domain] } }
+          filter.deep_merge!(new_validation_state)
+        end
+      end
+
+      def update_doc_fields(index, id, doc = {}, seq_no = nil, primary_term = nil)
         return if doc.empty?
         update_args = {
-          :index => Utility::Constants::CONNECTORS_INDEX,
-          :id => connector_id,
+          :index => index,
+          :id => id,
           :body => { :doc => doc },
           :refresh => true,
           :retry_on_conflict => 3
         }
-        # seq_no and primary_term are used for optimistic concurrency control
-        # see https://www.elastic.co/guide/en/elasticsearch/reference/current/optimistic-concurrency-control.html
+
         if seq_no && primary_term
           update_args[:if_seq_no] = seq_no
           update_args[:if_primary_term] = primary_term
           update_args.delete(:retry_on_conflict)
         end
+
         begin
           client.update(update_args)
         rescue Elastic::Transport::Transport::Errors::Conflict
           # VersionConflictException
           # see https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-index_.html#optimistic-concurrency-control-index
-          raise ConnectorVersionChangedError.new(connector_id, seq_no, primary_term)
+          raise ConnectorVersionChangedError.new(id, seq_no, primary_term)
         end
       end
-
-      private
-
-      def client
-        @client ||= Utility::EsClient.new(App::Config[:elasticsearch])
-      end
-
-      def get_latest_index_in_alias(alias_name, indicies)
-        index_versions = indicies.map { |index| index.gsub("#{alias_name}-v", '').to_i }
-        index_version = index_versions.max # gets the largest suffix number
-        "#{alias_name}-v#{index_version}"
-      end
     end
   end
 end

data/lib/core/filtering/post_process_engine.rb

@@ -0,0 +1,39 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'core/filtering'
+require 'utility/filtering'
+
+module Core
+  module Filtering
+    class PostProcessEngine
+      attr_reader :rules
+
+      def initialize(job_description)
+        @rules = ordered_rules(job_description.dig('connector', 'filtering'))
+      end
+
+      def process(document)
+        @rules.each do |rule|
+          if rule.match?(document.stringify_keys)
+            return PostProcessResult.new(document, rule)
+          end
+        end
+        PostProcessResult.new(document, SimpleRule::DEFAULT_RULE)
+      end
+
+      private
+
+      def ordered_rules(job_filtering)
+        job_rules = Utility::Filtering.extract_filter(job_filtering)['rules']
+        sorted_rules = job_rules.sort_by { |rule| rule['order'] }.reject { |rule| rule['id'] == Core::Filtering::SimpleRule::DEFAULT_RULE_ID }
+        sorted_rules.each_with_object([]) { |rule, output| output << SimpleRule.new(rule) }
+      end
+    end
+  end
+end

data/lib/core/filtering/post_process_result.rb

@@ -0,0 +1,27 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'utility/logger'
+
+module Core
+  module Filtering
+    class PostProcessResult
+      attr_reader :document, :matching_rule
+
+      def initialize(document, matching_rule)
+        @document = document
+        @matching_rule = matching_rule
+        Utility::Logger.debug("Document '#{document['id']}' matched filtering rule: #{matching_rule.id}. It will be #{matching_rule.policy}d")
+      end
+
+      def is_include?
+        matching_rule.is_include?
+      end
+    end
+  end
+end