connectors_service 8.6.0.3 → 8.6.0.4.pre.20221114T233727Z

data/lib/core/elastic_connector_actions.rb

@@ -10,8 +10,27 @@ require 'active_support/core_ext/hash'
 require 'connectors/connector_status'
 require 'connectors/sync_status'
 require 'utility'
+require 'elastic-transport'
 
 module Core
+  class JobAlreadyRunningError < StandardError
+    def initialize(connector_id)
+      super("Sync job for connector '#{connector_id}' is already running.")
+    end
+  end
+
+  class JobNotCreatedError < StandardError
+    def initialize(connector_id, response)
+      super("Sync job for connector '#{connector_id}' could not be created. Response: #{response}")
+    end
+  end
+
+  class ConnectorVersionChangedError < StandardError
+    def initialize(connector_id, seq_no, primary_term)
+      super("Version conflict: seq_no [#{seq_no}] and primary_term [#{primary_term}] do not match for connector '#{connector_id}'.")
+    end
+  end
+
   class ElasticConnectorActions
     class << self
 
@@ -30,10 +49,17 @@ module Core
       end
 
       def get_connector(connector_id)
+        # TODO: remove the usage of with_indifferent_access. Ideally this should return a hash or nil if not found
         client.get(:index => Utility::Constants::CONNECTORS_INDEX, :id => connector_id, :ignore => 404).with_indifferent_access
       end
 
+      def get_job(job_id)
+        # TODO: remove the usage of with_indifferent_access. Ideally this should return a hash or nil if not found
+        client.get(:index => Utility::Constants::JOB_INDEX, :id => job_id, :ignore => 404).with_indifferent_access
+      end
+
       def connectors_meta
+        # TODO: remove the usage of with_indifferent_access. Ideally this should return a hash or nil if not found
         alias_mappings = client.indices.get_mapping(:index => Utility::Constants::CONNECTORS_INDEX).with_indifferent_access
         index = get_latest_index_in_alias(Utility::Constants::CONNECTORS_INDEX, alias_mappings.keys)
         alias_mappings.dig(index, 'mappings', '_meta') || {}
@@ -52,6 +78,19 @@ module Core
         )
       end
 
+      def search_jobs(query, page_size, offset)
+        client.search(
+          :index => Utility::Constants::JOB_INDEX,
+          :ignore => 404,
+          :body => {
+              :size => page_size,
+              :from => offset,
+              :query => query,
+              :sort => ['created_at']
+          }
+        )
+      end
+
       def update_connector_configuration(connector_id, configuration)
         update_connector_fields(connector_id, :configuration => configuration)
       end
@@ -71,21 +110,90 @@ module Core
         update_connector_configuration(connector_id, payload)
       end
 
+      def update_filtering_validation(connector_id, filter_validation_results)
+        return if filter_validation_results.empty?
+
+        filtering = get_connector(connector_id).dig(:_source, :filtering)
+
+        case filtering
+        when Hash
+          update_filter_validation(filtering, filter_validation_results)
+        when Array
+          return unless should_update_validations?(filter_validation_results, filtering)
+
+          filtering.each do |filter|
+            update_filter_validation(filter, filter_validation_results)
+          end
+        else
+          Utility::Logger.warn("Elasticsearch returned invalid filtering format: #{filtering}. Skipping validation.")
+          return
+        end
+
+        update_connector_fields(connector_id, { :filtering => filtering })
+      end
+
       def claim_job(connector_id)
-        update_connector_fields(connector_id,
-                                :sync_now => false,
-                                :last_sync_status => Connectors::SyncStatus::IN_PROGRESS,
-                                :last_synced => Time.now)
+        seq_no = nil
+        primary_term = nil
+        sync_in_progress = false
+        connector_record = client.get(
+          :index => Utility::Constants::CONNECTORS_INDEX,
+          :id => connector_id,
+          :ignore => 404,
+          :refresh => true
+        ).tap do |response|
+          seq_no = response['_seq_no']
+          primary_term = response['_primary_term']
+          sync_in_progress = response.dig('_source', 'last_sync_status') == Connectors::SyncStatus::IN_PROGRESS
+        end
+        if sync_in_progress
+          raise JobAlreadyRunningError.new(connector_id)
+        end
+        update_connector_fields(
+          connector_id,
+          { :sync_now => false,
+            :last_sync_status => Connectors::SyncStatus::IN_PROGRESS,
+            :last_synced => Time.now },
+          seq_no,
+          primary_term
+        )
 
         body = {
-          :connector_id => connector_id,
           :status => Connectors::SyncStatus::IN_PROGRESS,
           :worker_hostname => Socket.gethostname,
-          :created_at => Time.now
+          :created_at => Time.now,
+          :started_at => Time.now,
+          :last_seen => Time.now,
+          :connector => {
+            :id => connector_id,
+            :filtering => convert_connector_filtering_to_job_filtering(connector_record.dig('_source', 'filtering'))
+          }
         }
-        job = client.index(:index => Utility::Constants::JOB_INDEX, :body => body)
 
-        job['_id']
+        index_response = client.index(:index => Utility::Constants::JOB_INDEX, :body => body, :refresh => true)
+        if index_response['result'] == 'created'
+          # TODO: remove the usage of with_indifferent_access. Ideally this should return a hash or nil if not found
+          return client.get(
+            :index => Utility::Constants::JOB_INDEX,
+            :id => index_response['_id'],
+            :ignore => 404
+          ).with_indifferent_access
+        end
+        raise JobNotCreatedError.new(connector_id, index_response)
+      end
+
+      def convert_connector_filtering_to_job_filtering(connector_filtering)
+        return [] unless connector_filtering
+        connector_filtering = [connector_filtering] unless connector_filtering.is_a?(Array)
+        connector_filtering.each_with_object([]) do |filtering_domain, job_filtering|
+          snippet = filtering_domain.dig('active', 'advanced_snippet') || {}
+          job_filtering << {
+            'domain' => filtering_domain['domain'],
+            'rules' => filtering_domain.dig('active', 'rules'),
+            'advanced_snippet' => snippet['value'] || snippet,
+            'warnings' => [] # TODO: in https://github.com/elastic/enterprise-search-team/issues/3174
+          }
+        end
       end
 
       def update_connector_status(connector_id, status, error_message = nil)
@@ -99,22 +207,33 @@ module Core
         update_connector_fields(connector_id, body)
       end
 
-      def complete_sync(connector_id, job_id, status)
-        sync_status = status[:error] ? Connectors::SyncStatus::FAILED : Connectors::SyncStatus::COMPLETED
+      def update_sync(job_id, metadata)
+        body = {
+          :doc => { :last_seen => Time.now }.merge(metadata)
+        }
+        client.update(:index => Utility::Constants::JOB_INDEX, :id => job_id, :body => body)
+      end
+
+      def complete_sync(connector_id, job_id, metadata, error)
+        sync_status = error ? Connectors::SyncStatus::ERROR : Connectors::SyncStatus::COMPLETED
+
+        metadata ||= {}
 
         update_connector_fields(connector_id,
                                 :last_sync_status => sync_status,
-                                :last_sync_error => status[:error],
-                                :error => status[:error],
+                                :last_sync_error => error,
+                                :error => error,
                                 :last_synced => Time.now,
-                                :last_indexed_document_count => status[:indexed_document_count],
-                                :last_deleted_document_count => status[:deleted_document_count])
+                                :last_indexed_document_count => metadata[:indexed_document_count],
+                                :last_deleted_document_count => metadata[:deleted_document_count])
 
         body = {
           :doc => {
             :status => sync_status,
-            :completed_at => Time.now
-          }.merge(status)
+            :completed_at => Time.now,
+            :last_seen => Time.now,
+            :error => error
+          }.merge(metadata)
         }
         client.update(:index => Utility::Constants::JOB_INDEX, :id => job_id, :body => body)
       end
@@ -136,7 +255,7 @@ module Core
           }
           loop do
             response = client.search(:body => body)
-            hits = response['hits']['hits']
+            hits = response.dig('hits', 'hits') || []
 
             ids = hits.map { |h| h['_id'] }
             result += ids
@@ -202,12 +321,105 @@ module Core
           :properties => {
             :api_key_id => { :type => :keyword },
             :configuration => { :type => :object },
-            :error => { :type => :text },
+            :description => { :type => :text },
+            :error => { :type => :keyword },
+            :features => {
+              :properties => {
+                :filtering_advanced_config => { :type => :boolean },
+                :filtering_rules => { :type => :boolean }
+              }
+            },
+            :filtering => {
+              :properties => {
+                :domain => { :type => :keyword },
+                :active => {
+                  :properties => {
+                    :rules => {
+                      :properties => {
+                        :id => { :type => :keyword },
+                        :policy => { :type => :keyword },
+                        :field => { :type => :keyword },
+                        :rule => { :type => :keyword },
+                        :value => { :type => :keyword },
+                        :order => { :type => :short },
+                        :created_at => { :type => :date },
+                        :updated_at => { :type => :date }
+                      }
+                    },
+                    :advanced_snippet => {
+                      :properties => {
+                        :value => { :type => :object },
+                        :created_at => { :type => :date },
+                        :updated_at => { :type => :date }
+                       }
+                    },
+                    :validation => {
+                      :properties => {
+                        :state => { :type => :keyword },
+                        :errors => {
+                          :properties => {
+                            :ids => { :type => :keyword },
+                            :messages => { :type => :text }
+                          }
+                        }
+                      }
+                    }
+                  }
+                },
+                :draft => {
+                  :properties => {
+                    :rules => {
+                      :properties => {
+                        :id => { :type => :keyword },
+                        :policy => { :type => :keyword },
+                        :field => { :type => :keyword },
+                        :rule => { :type => :keyword },
+                        :value => { :type => :keyword },
+                        :order => { :type => :short },
+                        :created_at => { :type => :date },
+                        :updated_at => { :type => :date }
+                      }
+                    },
+                    :advanced_snippet => {
+                      :properties => {
+                        :value => { :type => :object },
+                        :created_at => { :type => :date },
+                        :updated_at => { :type => :date }
+                      }
+                    },
+                    :validation => {
+                      :properties => {
+                        :state => { :type => :keyword },
+                        :errors => {
+                          :properties => {
+                            :ids => { :type => :keyword },
+                            :messages => { :type => :text }
+                          }
+                        }
+                      }
+                    }
+                  }
+                }
+              }
+            },
             :index_name => { :type => :keyword },
+            :is_native => { :type => :boolean },
+            :language => { :type => :keyword },
             :last_seen => { :type => :date },
+            :last_sync_error => { :type => :keyword },
+            :last_sync_status => { :type => :keyword },
             :last_synced => { :type => :date },
-            :last_indexed_document_count => { :type => :integer },
-            :last_deleted_document_count => { :type => :integer },
+            :last_deleted_document_count => { :type => :long },
+            :last_indexed_document_count => { :type => :long },
+            :name => { :type => :keyword },
+            :pipeline => {
+              :properties => {
+                :extract_binary_content => { :type => :boolean },
+                :name => { :type => :keyword },
+                :reduce_whitespace => { :type => :boolean },
+                :run_ml_inference => { :type => :boolean }
+              }
+            },
             :scheduling => {
               :properties => {
                 :enabled => { :type => :boolean },
@@ -216,9 +428,7 @@ module Core
             },
             :service_type => { :type => :keyword },
             :status => { :type => :keyword },
-            :sync_error => { :type => :text },
-            :sync_now => { :type => :boolean },
-            :sync_status => { :type => :keyword }
+            :sync_now => { :type => :boolean }
           }
         }
         ensure_index_exists("#{Utility::Constants::CONNECTORS_INDEX}-v1", system_index_body(:alias_name => Utility::Constants::CONNECTORS_INDEX, :mappings => mappings))
@@ -229,32 +439,112 @@ module Core
       def ensure_job_index_exists
         mappings = {
           :properties => {
-            :connector_id => { :type => :keyword },
-            :status => { :type => :keyword },
+            :cancelation_requested_at => { :type => :date },
+            :canceled_at => { :type => :date },
+            :completed_at => { :type => :date },
+            :connector => {
+              :properties => {
+                :configuration => { :type => :object },
+                :filtering => {
+                  :properties => {
+                    :domain => { :type => :keyword },
+                    :rules => {
+                      :properties => {
+                        :id => { :type => :keyword },
+                        :policy => { :type => :keyword },
+                        :field => { :type => :keyword },
+                        :rule => { :type => :keyword },
+                        :value => { :type => :keyword },
+                        :order => { :type => :short },
+                        :created_at => { :type => :date },
+                        :updated_at => { :type => :date }
+                      }
+                    },
+                    :advanced_snippet => {
+                      :properties => {
+                        :value => { :type => :object },
+                        :created_at => { :type => :date },
+                        :updated_at => { :type => :date }
+                      }
+                    },
+                    :warnings => {
+                      :properties => {
+                        :ids => { :type => :keyword },
+                        :messages => { :type => :text }
+                      }
+                    }
+                  }
+                },
+                :id => { :type => :keyword },
+                :index_name => { :type => :keyword },
+                :language => { :type => :keyword },
+                :pipeline => {
+                  :properties => {
+                    :extract_binary_content => { :type => :boolean },
+                    :name => { :type => :keyword },
+                    :reduce_whitespace => { :type => :boolean },
+                    :run_ml_inference => { :type => :boolean }
+                  }
+                },
+                :service_type => { :type => :keyword }
+              }
+            },
+            :created_at => { :type => :date },
+            :deleted_document_count => { :type => :integer },
             :error => { :type => :text },
-            :worker_hostname => { :type => :keyword },
             :indexed_document_count => { :type => :integer },
-            :deleted_document_count => { :type => :integer },
-            :created_at => { :type => :date },
-            :completed_at => { :type => :date }
+            :indexed_document_volume => { :type => :integer },
+            :last_seen => { :type => :date },
+            :metadata => { :type => :object },
+            :started_at => { :type => :date },
+            :status => { :type => :keyword },
+            :total_document_count => { :type => :integer },
+            :trigger_method => { :type => :keyword },
+            :worker_hostname => { :type => :keyword }
           }
         }
         ensure_index_exists("#{Utility::Constants::JOB_INDEX}-v1", system_index_body(:alias_name => Utility::Constants::JOB_INDEX, :mappings => mappings))
       end
 
-      def update_connector_fields(connector_id, doc = {})
+      def update_connector_fields(connector_id, doc = {}, seq_no = nil, primary_term = nil)
         return if doc.empty?
-        client.update(
+        update_args = {
           :index => Utility::Constants::CONNECTORS_INDEX,
           :id => connector_id,
           :body => { :doc => doc },
           :refresh => true,
           :retry_on_conflict => 3
-        )
+        }
+        # seq_no and primary_term are used for optimistic concurrency control
+        # see https://www.elastic.co/guide/en/elasticsearch/reference/current/optimistic-concurrency-control.html
+        if seq_no && primary_term
+          update_args[:if_seq_no] = seq_no
+          update_args[:if_primary_term] = primary_term
+          update_args.delete(:retry_on_conflict)
+        end
+        begin
+          client.update(update_args)
+        rescue Elastic::Transport::Transport::Errors::Conflict
+          # VersionConflictException
+          # see https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-index_.html#optimistic-concurrency-control-index
+          raise ConnectorVersionChangedError.new(connector_id, seq_no, primary_term)
+        end
+      end
+
+      def document_count(index_name)
+        client.count(:index => index_name)['count']
       end
 
       private
 
+      def should_update_validations?(domain_validations, filtering)
+        domains_present = filtering.collect { |filter| filter[:domain] }
+        domains_to_update = domain_validations.keys
+
+        # non-empty intersection -> domains to update present
+        !(domains_present & domains_to_update).empty?
+      end
+
       def client
         @client ||= Utility::EsClient.new(App::Config[:elasticsearch])
       end
@@ -264,6 +554,15 @@ module Core
         index_version = index_versions.max # gets the largest suffix number
         "#{alias_name}-v#{index_version}"
       end
+
+      def update_filter_validation(filter, domain_validations)
+        domain = filter[:domain]
+
+        if domain_validations.key?(domain)
+          new_validation_state = { :draft => { :validation => domain_validations[domain] } }
+          filter.deep_merge!(new_validation_state)
+        end
+      end
     end
   end
 end

data/lib/core/filtering/post_process_engine.rb

@@ -0,0 +1,39 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'core/filtering'
+require 'utility/filtering'
+
+module Core
+  module Filtering
+    class PostProcessEngine
+      attr_reader :rules
+
+      def initialize(job_description)
+        @rules = ordered_rules(job_description.dig('connector', 'filtering'))
+      end
+
+      def process(document)
+        @rules.each do |rule|
+          if rule.match?(document.stringify_keys)
+            return PostProcessResult.new(document, rule)
+          end
+        end
+        PostProcessResult.new(document, SimpleRule::DEFAULT_RULE)
+      end
+
+      private
+
+      def ordered_rules(job_filtering)
+        job_rules = Utility::Filtering.extract_filter(job_filtering)['rules']
+        sorted_rules = job_rules.sort_by { |rule| rule['order'] }.reject { |rule| rule['id'] == Core::Filtering::SimpleRule::DEFAULT_RULE_ID }
+        sorted_rules.each_with_object([]) { |rule, output| output << SimpleRule.new(rule) }
+      end
+    end
+  end
+end

data/lib/core/filtering/post_process_result.rb

@@ -0,0 +1,27 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'utility/logger'
+
+module Core
+  module Filtering
+    class PostProcessResult
+      attr_reader :document, :matching_rule
+
+      def initialize(document, matching_rule)
+        @document = document
+        @matching_rule = matching_rule
+        Utility::Logger.debug("Document '#{document['id']}' matched filtering rule: #{matching_rule.id}. It will be #{matching_rule.policy}d")
+      end
+
+      def is_include?
+        matching_rule.is_include?
+      end
+    end
+  end
+end

data/lib/core/filtering/simple_rule.rb

@@ -0,0 +1,141 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'utility/logger'
+
+module Core
+  module Filtering
+    class SimpleRule
+      DEFAULT_RULE_ID = 'DEFAULT'
+
+      class Policy
+        INCLUDE = 'include'
+        EXCLUDE = 'exclude'
+      end
+
+      class Rule
+        REGEX = 'regex'
+        EQUALS = 'equals'
+        STARTS_WITH = 'starts_with'
+        ENDS_WITH = 'ends_with'
+        CONTAINS = 'contains'
+        LESS_THAN = '<'
+        GREATER_THAN = '>'
+      end
+
+      attr_reader :policy, :field, :rule, :value, :id
+
+      def initialize(rule_hash)
+        @policy = rule_hash.fetch('policy')
+        @field = rule_hash.fetch('field')
+        @rule = rule_hash.fetch('rule')
+        @value = rule_hash.fetch('value')
+        @id = rule_hash.fetch('id')
+        @rule_hash = rule_hash
+      rescue KeyError => e
+        raise "#{e.key} is required"
+      end
+
+      def self.from_args(id, policy, field, rule, value)
+        SimpleRule.new(
+          {
+            'id' => id,
+            'policy' => policy,
+            'field' => field,
+            'rule' => rule,
+            'value' => value
+          }
+        )
+      end
+
+      DEFAULT_RULE = SimpleRule.new(
+        'policy' => 'include',
+        'field' => '_',
+        'rule' => 'regex',
+        'value' => '.*',
+        'id' => SimpleRule::DEFAULT_RULE_ID
+      )
+
+      def match?(document)
+        return true if id == DEFAULT_RULE_ID
+        doc_value = document[field]
+        return false if doc_value.nil?
+        coerced_value = coerce(doc_value)
+        case rule
+        when Rule::EQUALS
+          case coerced_value
+          when Integer
+            doc_value == coerced_value
+          when DateTime, Time
+            doc_value.to_s == coerced_value.to_s
+          else
+            doc_value.to_s == coerced_value
+          end
+        when Rule::STARTS_WITH
+          doc_value.to_s.start_with?(value)
+        when Rule::ENDS_WITH
+          doc_value.to_s.end_with?(value)
+        when Rule::CONTAINS
+          doc_value.to_s.include?(value)
+        when Rule::REGEX
+          doc_value.to_s.match(/#{value}/)
+        when Rule::LESS_THAN
+          doc_value < coerced_value
+        when Rule::GREATER_THAN
+          doc_value > coerced_value
+        else
+          false
+        end
+      end
+
+      def coerce(doc_value)
+        case doc_value
+        when String
+          value.to_s
+        when Integer
+          value.to_i
+        when DateTime, Time
+          to_date(value)
+        when TrueClass, FalseClass # Ruby doesn't have a Boolean type, TIL
+          to_bool(value).to_s
+        else
+          value.to_s
+        end
+      rescue StandardError => e
+        Utility::Logger.debug("Failed to coerce value '#{value}' (#{value.class}) based on document value '#{doc_value}' (#{doc_value.class}) due to error: #{e.class}: #{e.message}")
+        value.to_s
+      end
+
+      def is_include?
+        policy == Policy::INCLUDE
+      end
+
+      def is_exclude?
+        policy == Policy::EXCLUDE
+      end
+
+      def to_h
+        @rule_hash
+      end
+
+      private
+
+      def to_bool(str)
+        return true if str == true || str =~ (/^(true|t|yes|y|on|1)$/i)
+        return false if str == false || str.blank? || str =~ (/^(false|f|no|n|off|0)$/i)
+        raise ArgumentError.new("invalid value for Boolean: \"#{str}\"")
+      end
+
+      def to_date(str)
+        DateTime.parse(str)
+      rescue ArgumentError
+        Time.at(str.to_i) # try with it as an int string of millis
+      end
+    end
+  end
+end