connectors_service 8.7.0.0.pre.20221117T004928Z → 8.11.0.0

data/lib/{connectors/base/advanced_snippet_against_schema_validator.rb → core/filtering/hash_against_schema_validator.rb}

@@ -7,83 +7,90 @@
 
 require 'active_support/core_ext/hash'
 require 'utility/logger'
-require 'connectors/base/advanced_snippet_validator'
-require 'core/filtering/validation_status'
 
-module Connectors
-  module Base
-    class AdvancedSnippetAgainstSchemaValidator < Connectors::Base::AdvancedSnippetValidator
+module Core
+  module Filtering
+    class SchemaValidator
 
       MAX_RECURSION_DEPTH = 50
-      ADVANCED_SNIPPET_ID = 'advanced_snippet'
 
-      def initialize(advanced_snippet, schema)
-        super(advanced_snippet)
+      def initialize(schema: {}, payload: {}, error_id: '')
         @schema = schema
+        @payload = payload
+        @error_id = error_id
       end
 
-      def is_snippet_valid?
-        validation_result = validate_against_schema(@schema, @advanced_snippet)
-        log_validation_result(validation_result)
-        validation_result
-      end
-
-      private
-
-      def validate_against_schema(config_schema, advanced_snippet, recursion_depth = 0)
+      def validate_against_schema(schema = @schema, payload = @payload, recursion_depth = 0)
         # Prevent unintentional/intentional SystemStackErrors/crashes
         return unexpected_error if exceeded_recursion_depth?(recursion_depth)
 
-        return valid_snippet if config_schema.nil? || config_schema.empty?
+        return valid_snippet unless schema.present?
 
-        schema_fields = config_schema[:fields].is_a?(Hash) ? config_schema.dig(:fields, :values) : config_schema[:fields]
-        snippet_field_names = advanced_snippet&.keys&.map(&:to_s)
+        schema_fields = schema[:fields].is_a?(Hash) ? schema.dig(:fields, :values) : schema[:fields]
+        snippet_field_names = payload&.keys&.map(&:to_s)
         schema_field_names = schema_fields.map { |field| field[:name] }
 
         return unexpected_field(schema_field_names, snippet_field_names) if unexpected_field_present?(snippet_field_names, schema_field_names)
 
-        return fields_constraint_violation(config_schema[:fields]) if fields_constraints_violated?(config_schema, advanced_snippet)
+        return fields_constraint_violation(schema[:fields]) if fields_constraints_violated?(schema[:fields], payload)
 
         schema_fields.each do |field|
           name = field[:name]
           type = field[:type]
           optional = field[:optional] || false
 
-          snippet_field_value = advanced_snippet.with_indifferent_access[name]
+          snippet_field_value = payload.nil? ? nil : payload.with_indifferent_access[name]
 
           next if optional && (snippet_field_value.nil? || !snippet_field_value.present?)
 
-          return wrong_names(snippet_field_names, name) unless snippet_field_names.include?(name)
+          return required_value_missing(name) if is_required_value_missing?(snippet_field_value)
 
-          return wrong_type(name, type, snippet_field_value) if type_error_present?(type, snippet_field_value)
+          type_error_present, error_message = type_error_present?(name, type, snippet_field_value)
+
+          return wrong_type(error_message) if type_error_present
 
           if field[:fields].present?
             validation_result = validate_against_schema(field, snippet_field_value, recursion_depth + 1)
 
-            return validation_result unless validation_result[:is_valid]
+            return validation_result unless validation_result[:state] == Core::Filtering::ValidationStatus::VALID
           end
         end
 
         valid_snippet
       end
 
-      def fields_constraints_violated?(config_schema, advanced_snippet)
-        return false unless config_schema[:fields].is_a?(Hash)
+      def fields_constraints_violated?(fields, payload)
+        return false if !fields.present? || !fields.is_a?(Hash)
 
-        constraints = config_schema.dig(:fields, :constraints)
+        constraints = fields[:constraints]
         constraints = constraints.is_a?(Array) ? constraints : [constraints]
 
         constraints.each do |constraint|
-          return true unless constraint.call(advanced_snippet)
+          return true unless constraint.call(payload)
         end
 
         false
       end
 
-      def type_error_present?(schema_type, snippet_value)
-        return !schema_type.call(snippet_value) if schema_type.is_a?(Proc)
+      def type_error_present?(field_name, schema_type, actual_value)
+        if schema_type.is_a?(Proc)
+          result = schema_type.call(actual_value)
+
+          # could already have a custom error message
+          if result.is_a?(Array)
+            is_valid, error_msg = result
+
+            return !is_valid, error_msg
+          end
+
+          # could only return a single boolean
+          return !result, 'Custom type matcher validation failed.'
+        end
+
+        error_msg = "Expected field type '#{schema_type}' for field '#{field_name}', but got value '#{actual_value.inspect}' of type '#{actual_value.class}'."
+        return true, error_msg unless actual_value.is_a?(schema_type)
 
-        !snippet_value.is_a?(schema_type)
+        false
       end
 
       def exceeded_recursion_depth?(recursion_depth)
@@ -96,12 +103,18 @@ module Connectors
       end
 
       def unexpected_field_present?(actual_field_names, expected_field_names)
+        return false unless actual_field_names.present?
+
         difference = actual_field_names - expected_field_names
 
         # we have field names, which we didn't expect
         !difference.empty?
       end
 
+      def is_required_value_missing?(snippet_field_value)
+        !snippet_field_value.present?
+      end
+
       def valid_snippet
         {
           :state => Core::Filtering::ValidationStatus::VALID,
@@ -109,37 +122,37 @@ module Connectors
         }
       end
 
-      def unexpected_field(expected_fields, actual_fields)
+      def required_value_missing(field)
         {
           :state => Core::Filtering::ValidationStatus::INVALID,
           :errors => [
             {
-              :ids => [ADVANCED_SNIPPET_ID],
-              :messages => ["Encountered unexpected fields '#{actual_fields}'. Expected: '#{expected_fields}'."]
+              :ids => [@error_id],
+              :messages => ["Required value missing for field '#{field}'."]
             }
           ]
         }
       end
 
-      def wrong_type(field_name, expected_type, actual_value)
+      def unexpected_field(expected_fields, actual_fields)
         {
           :state => Core::Filtering::ValidationStatus::INVALID,
           :errors => [
             {
-              :ids => [ADVANCED_SNIPPET_ID],
-              :messages => ["Expected field type '#{expected_type.is_a?(Proc) ? 'custom matcher' : expected_type}' for field '#{field_name}', but got value '#{actual_value.inspect}' of type '#{actual_value.class}'."]
+              :ids => [@error_id],
+              :messages => ["Encountered unexpected fields '#{actual_fields}'. Expected: '#{expected_fields}'."]
             }
           ]
         }
       end
 
-      def wrong_names(actual_field_names, expected_field_name)
+      def wrong_type(error_message)
         {
           :state => Core::Filtering::ValidationStatus::INVALID,
           :errors => [
             {
-              :ids => [ADVANCED_SNIPPET_ID],
-              :messages => ["Expected field name '#{expected_field_name}', but got #{actual_field_names}."]
+              :ids => [@error_id],
+              :messages => [error_message]
             }
           ]
         }
@@ -150,8 +163,8 @@ module Connectors
           :state => Core::Filtering::ValidationStatus::INVALID,
           :errors => [
             {
-              :ids => [ADVANCED_SNIPPET_ID],
-              :messages => ["A fields constraint was violated for fields: '#{fields[:values].map { |v| v[:name] }}'. Check advanced snippet field constraints."]
+              :ids => [@error_id],
+              :messages => ["A fields constraint was violated for fields: '#{fields[:values].map { |v| v[:name] }}'."]
             }
           ]
         }
@@ -162,12 +175,13 @@ module Connectors
           :state => Core::Filtering::ValidationStatus::INVALID,
           :errors => [
             {
-              :ids => [ADVANCED_SNIPPET_ID],
+              :ids => [@error_id],
               :messages => ['Unexpected error. Check logs for details.']
             }
           ]
         }
       end
+
     end
   end
 end

data/lib/core/filtering/post_process_engine.rb

@@ -14,8 +14,8 @@ module Core
     class PostProcessEngine
       attr_reader :rules
 
-      def initialize(job_description)
-        @rules = ordered_rules(job_description.dig('connector', 'filtering'))
+      def initialize(filtering)
+        @rules = ordered_rules(filtering)
       end
 
       def process(document)

data/lib/core/filtering/processing_stage.rb

@@ -0,0 +1,20 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+# frozen_string_literal: true
+
+module Core
+  module Filtering
+    class ProcessingStage
+      PRE = 'pre-processing'
+      POST = 'post-processing'
+
+      ALL = [
+        PRE,
+        POST
+      ]
+    end
+  end
+end

data/lib/core/filtering/{simple_rule.rb → simple_rules/simple_rule.rb}

@@ -16,6 +16,11 @@ module Core
       class Policy
         INCLUDE = 'include'
         EXCLUDE = 'exclude'
+
+        POLICIES = [
+          INCLUDE,
+          EXCLUDE
+        ]
       end
 
       class Rule
@@ -26,6 +31,16 @@ module Core
         CONTAINS = 'contains'
         LESS_THAN = '<'
         GREATER_THAN = '>'
+
+        RULES = [
+          REGEX,
+          EQUALS,
+          STARTS_WITH,
+          ENDS_WITH,
+          CONTAINS,
+          LESS_THAN,
+          GREATER_THAN
+        ]
       end
 
       attr_reader :policy, :field, :rule, :value, :id
@@ -123,6 +138,18 @@ module Core
         @rule_hash
       end
 
+      def try_coerce_value
+        coerced = to_float(value)
+        begin
+          coerced = to_date(value) if coerced.is_a?(String)
+        rescue ArgumentError
+          coerced = to_bool(value) if coerced.is_a?(String)
+        end
+        coerced
+      rescue StandardError
+        value
+      end
+
       private
 
       def to_bool(str)
@@ -134,7 +161,13 @@ module Core
       def to_date(str)
         DateTime.parse(str)
       rescue ArgumentError
-        Time.at(str.to_i) # try with it as an int string of millis
+        Time.at(Integer(value)) # try with it as an int string of millis
+      end
+
+      def to_float(value)
+        Float(value)
+      rescue StandardError
+        value
       end
     end
   end

data/lib/core/filtering/simple_rules/simple_rules_parser.rb

@@ -0,0 +1,44 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+# frozen_string_literal: true
+
+require 'active_support/core_ext/hash/indifferent_access'
+require 'active_support/core_ext/object/blank'
+require 'core/filtering/simple_rules/simple_rule'
+
+module Core
+  module Filtering
+    module SimpleRules
+      class SimpleRulesParser
+        def initialize(rules)
+          @rules = (rules || []).map(&:with_indifferent_access).filter { |r| r[:id] != 'DEFAULT' }.sort_by { |r| r[:order] }
+        end
+
+        def parse
+          merge_rules(@rules.map do |rule_hash|
+            rule = Core::Filtering::SimpleRule.new(rule_hash)
+            unless rule.is_include? || rule.is_exclude?
+              raise "Unknown policy: #{rule.policy}"
+            end
+            parse_rule(rule)
+          end)
+        end
+
+        private
+
+        # merge all rules into a filter object or array
+        # in a base case, does no transformations
+        def merge_rules(rules)
+          rules || []
+        end
+
+        def parse_rule(_rule)
+          raise 'Not implemented'
+        end
+      end
+    end
+  end
+end

data/lib/core/filtering/simple_rules/validation/no_conflicting_policies_rules_validator.rb

@@ -0,0 +1,47 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+# frozen_string_literal: true
+
+require 'core/filtering/simple_rules/validation/simple_rules_validator'
+require 'core/filtering/validation_status'
+
+module Core
+  module Filtering
+    module SimpleRules
+      module Validation
+        class NoConflictingPoliciesRulesValidator < Core::Filtering::SimpleRules::Validation::SimpleRulesValidator
+
+          def are_rules_valid
+            rule_field_value_to_policy = {}
+
+            @rules.each do |simple_rule|
+              rule_field_value = simple_rule.slice('rule', 'field', 'value')
+              policy = simple_rule['policy']
+
+              return conflicting_rules(rule_field_value) if rule_field_value_to_policy.key?(rule_field_value)
+
+              rule_field_value_to_policy[rule_field_value] = policy
+            end
+
+            { :state => Core::Filtering::ValidationStatus::VALID, :errors => [] }
+          end
+
+          private
+
+          def conflicting_rules(rule_type_value)
+            {
+              :state => Core::Filtering::ValidationStatus::INVALID,
+              :errors => [
+                :ids => [SIMPLE_RULES_ID],
+                :messages => ["Two simple rules with same rule (#{rule_type_value['rule']}), field (#{rule_type_value['field']}), value (#{rule_type_value['value']}) and conflicting policies detected."]
+              ]
+            }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/core/filtering/simple_rules/validation/simple_rules_schema.rb

@@ -0,0 +1,68 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+# frozen_string_literal: true
+
+require 'core/filtering/simple_rules/simple_rule'
+
+module Core
+  module Filtering
+    module SimpleRules
+      module Validation
+        DEFAULT_RULE_ID = 'DEFAULT'
+
+        ALLOWED_VALUE_TYPES = ->(rule_value) { rule_value.is_a?(String) || rule_value.is_a?(Integer) || rule_value.is_a?(TrueClass) || rule_value.is_a?(FalseClass) }
+        MATCH_ALL_REGEX_NOT_ALLOWED = ->(simple_rule) { simple_rule['id'] == DEFAULT_RULE_ID || !(simple_rule['rule'] == Core::Filtering::SimpleRule::Rule::REGEX && (simple_rule['value'] == '(.*)' || simple_rule['value'] == '.*')) }
+
+        SINGLE_RULE_SCHEMA = {
+          :fields => {
+            :constraints => [MATCH_ALL_REGEX_NOT_ALLOWED],
+            :values => [
+              {
+                :name => 'id',
+                :type => String,
+                :optional => false
+              },
+              {
+                :name => 'field',
+                :type => String,
+                :optional => false
+              },
+              {
+                :name => 'value',
+                :type => ALLOWED_VALUE_TYPES,
+                :optional => false
+              },
+              {
+                :name => 'policy',
+                :type => ->(policy) { Core::Filtering::SimpleRule::Policy::POLICIES.include?(policy) },
+                :optional => false
+              },
+              {
+                :name => 'rule',
+                :type => ->(rule) { Core::Filtering::SimpleRule::Rule::RULES.include?(rule) },
+                :optional => false
+              },
+              {
+                :name => 'order',
+                :type => ->(order) { order.is_a?(Integer) && order >= 0 },
+              },
+              {
+                :name => 'updated_at',
+                :type => String,
+                :optional => true
+              },
+              {
+                :name => 'created_at',
+                :type => String,
+                :optional => true
+              }
+            ]
+          }
+        }
+      end
+    end
+  end
+end

data/lib/core/filtering/simple_rules/validation/simple_rules_validator.rb

@@ -0,0 +1,25 @@
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+# frozen_string_literal: true
+
+module Core
+  module Filtering
+    module SimpleRules
+      module Validation
+        SIMPLE_RULES_ID = 'simple_rules'
+
+        class SimpleRulesValidator
+          def initialize(rules)
+            @rules = rules || []
+          end
+
+          def are_rules_valid
+            raise 'Simple rules validation not implemented'
+          end
+        end
+      end
+    end
+  end
+end

data/lib/core/filtering/simple_rules/validation/single_rule_against_schema_validator.rb

@@ -0,0 +1,37 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+# frozen_string_literal: true
+
+require 'core/filtering/simple_rules/validation/simple_rules_validator'
+require 'core/filtering/hash_against_schema_validator'
+require 'core/filtering/simple_rules/validation/simple_rules_schema'
+
+module Core
+  module Filtering
+    module SimpleRules
+      module Validation
+        class SingleRuleAgainstSchemaValidator < Core::Filtering::SimpleRules::Validation::SimpleRulesValidator
+
+          def initialize(rules, schema = Core::Filtering::SimpleRules::Validation::SINGLE_RULE_SCHEMA)
+            super(rules)
+            @schema = schema
+            @schema_validator = SchemaValidator.new(error_id: SIMPLE_RULES_ID)
+          end
+
+          def are_rules_valid
+            @rules.each do |rule|
+              validation_result = @schema_validator.validate_against_schema(@schema, rule)
+              return validation_result unless validation_result[:state] == Core::Filtering::ValidationStatus::VALID
+            end
+
+            { :state => Core::Filtering::ValidationStatus::VALID, :errors => [] }
+          end
+
+        end
+      end
+    end
+  end
+end

data/lib/core/filtering/transform/filter_transformer.rb

@@ -0,0 +1,26 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+module Core
+  module Filtering
+    module Transform
+      class FilterTransformer
+
+        def initialize(filter = {}, transformation = (->(_filter) { filter }))
+          @filter = filter
+          @transformation = transformation
+        end
+
+        def transform
+          @transformation.call(@filter)
+        end
+
+      end
+    end
+  end
+end

data/lib/core/filtering/transform/filter_transformer_facade.rb

@@ -0,0 +1,61 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'core/filtering/transform/transformation_target'
+require 'core/filtering/transform/filter_transformer'
+
+module Core
+  module Filtering
+    module Transform
+      class FilterTransformerFacade < Core::Filtering::Transform::FilterTransformer
+
+        def initialize(filter = {},
+                       filter_transformers = {
+                           Core::Filtering::Transform::TransformationTarget::ADVANCED_SNIPPET => [],
+                           Core::Filtering::Transform::TransformationTarget::RULES => [],
+                         })
+          super(filter)
+
+          rule_transformer_classes = filter_transformers[Core::Filtering::Transform::TransformationTarget::RULES]
+          snippet_transformer_classes = filter_transformers[Core::Filtering::Transform::TransformationTarget::ADVANCED_SNIPPET]
+
+          @rule_transformers = rule_transformer_classes.is_a?(Array) ? rule_transformer_classes : [rule_transformer_classes]
+          @snippet_transformers = snippet_transformer_classes.is_a?(Array) ? snippet_transformer_classes : [snippet_transformer_classes]
+
+          @facade = FilterTransformer.new(filter, execute_rule_and_snippet_transformations)
+        end
+
+        def transform
+          @facade.transform
+        end
+
+        private
+
+        def execute_rule_and_snippet_transformations
+          lambda do |filter|
+            rules = filter[:rules]
+            advanced_snippet = filter[:advanced_snippet]
+
+            {
+              :rules => call_transformers(@rule_transformers, rules),
+              :advanced_snippet => call_transformers(@snippet_transformers, advanced_snippet)
+            }
+          end
+        end
+
+        def call_transformers(transformer_classes, payload)
+          transformer_classes.each do |transformer_class|
+            payload = transformer_class.new(payload).transform if transformer_class.present?
+          end
+
+          payload
+        end
+      end
+    end
+  end
+end

data/lib/core/filtering/transform/transformation_target.rb

@@ -0,0 +1,10 @@
+module Core
+  module Filtering
+    module Transform
+      module TransformationTarget
+        RULES = 'rules'
+        ADVANCED_SNIPPET = 'advanced_snippet'
+      end
+    end
+  end
+end

data/lib/core/filtering/validation_job_runner.rb

@@ -11,8 +11,6 @@ require 'connectors/registry'
 
 module Core
   module Filtering
-    DEFAULT_DOMAIN = 'DEFAULT'
-
     class ValidationJobRunner
       def initialize(connector_settings)
         @connector_settings = connector_settings
@@ -27,7 +25,7 @@ module Core
         validation_result = @connector_class.validate_filtering(@connector_settings.filtering[:draft])
 
         # currently only used for connectors -> DEFAULT domain can be assumed (will be changed with the integration of crawler)
-        ElasticConnectorActions.update_filtering_validation(@connector_settings.id, { DEFAULT_DOMAIN => validation_result })
+        ElasticConnectorActions.update_filtering_validation(@connector_settings.id, { Core::Filtering::DEFAULT_DOMAIN => validation_result })
 
         @validation_finished = true
       rescue StandardError => e

data/lib/core/filtering.rb

@@ -8,10 +8,12 @@
 
 require 'core/filtering/post_process_engine'
 require 'core/filtering/post_process_result'
-require 'core/filtering/simple_rule'
+require 'core/filtering/simple_rules/simple_rule'
 require 'core/filtering/validation_job_runner'
 require 'core/filtering/validation_status'
 
-module Core::Filtering
-  DEFAULT_DOMAIN = 'DEFAULT'
+module Core
+  module Filtering
+    DEFAULT_DOMAIN = 'DEFAULT'
+  end
 end