connectors_service 8.6.0.4.pre.20221116T024501Z → 8.6.0.4

data/lib/connectors/base/simple_rules_parser.rb

@@ -1,42 +0,0 @@
-#
-# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
-# or more contributor license agreements. Licensed under the Elastic License;
-# you may not use this file except in compliance with the Elastic License.
-#
-# frozen_string_literal: true
-
-require 'active_support/core_ext/hash/indifferent_access'
-require 'active_support/core_ext/object/blank'
-require 'core/filtering/simple_rule'
-
-module Connectors
-  module Base
-    class SimpleRulesParser
-      def initialize(rules)
-        @rules = (rules || []).map(&:with_indifferent_access).filter { |r| r[:id] != 'DEFAULT' }.sort_by { |r| r[:order] }
-      end
-
-      def parse
-        merge_rules(@rules.map do |rule_hash|
-          rule = Core::Filtering::SimpleRule.new(rule_hash)
-          unless rule.is_include? || rule.is_exclude?
-            raise "Unknown policy: #{rule.policy}"
-          end
-          parse_rule(rule)
-        end)
-      end
-
-      private
-
-      # merge all rules into a filter object or array
-      # in a base case, does no transformations
-      def merge_rules(rules)
-        rules || []
-      end
-
-      def parse_rule(_rule)
-        raise 'Not implemented'
-      end
-    end
-  end
-end

data/lib/connectors/example/example_advanced_snippet_validator.rb

@@ -1,35 +0,0 @@
-#
-# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
-# or more contributor license agreements. Licensed under the Elastic License;
-# you may not use this file except in compliance with the Elastic License.
-#
-
-# frozen_string_literal: true
-
-require 'connectors/base/advanced_snippet_validator'
-
-module Connectors
-  module Example
-    class ExampleAdvancedSnippetValidator < Connectors::Base::AdvancedSnippetValidator
-
-      def is_snippet_valid?
-        # TODO: real filtering validation will follow later
-        errors = [
-          {
-            :ids => ['missing-implementation'],
-            :messages => ['Filtering is not implemented yet for the example connector']
-          }
-        ]
-
-        validation_result = if @advanced_snippet.present? && !@advanced_snippet.empty?
-                              { :state => Core::Filtering::ValidationStatus::INVALID, :errors => errors }
-                            else
-                              { :state => Core::Filtering::ValidationStatus::VALID, :errors => [] }
-                            end
-        log_validation_result(validation_result)
-        validation_result
-      end
-
-    end
-  end
-end

data/lib/connectors/gitlab/gitlab_advanced_snippet_validator.rb

@@ -1,35 +0,0 @@
-#
-# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
-# or more contributor license agreements. Licensed under the Elastic License;
-# you may not use this file except in compliance with the Elastic License.
-#
-
-# frozen_string_literal: true
-
-require 'connectors/base/advanced_snippet_validator'
-
-module Connectors
-  module GitLab
-    class GitLabAdvancedSnippetValidator < Connectors::Base::AdvancedSnippetValidator
-
-      def is_snippet_valid?
-        # TODO: real filtering validation will follow later
-        errors = [
-          {
-            :ids => ['missing-implementation'],
-            :messages => ['Filtering is not implemented yet for the GitLab connector']
-          }
-        ]
-
-        validation_result = if @advanced_snippet.present? && !@advanced_snippet.empty?
-                              { :state => Core::Filtering::ValidationStatus::INVALID, :errors => errors }
-                            else
-                              { :state => Core::Filtering::ValidationStatus::VALID, :errors => [] }
-                            end
-        log_validation_result(validation_result)
-        validation_result
-      end
-
-    end
-  end
-end

data/lib/connectors/mongodb/mongo_advanced_snippet_against_schema_validator.rb

@@ -1,22 +0,0 @@
-#
-# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
-# or more contributor license agreements. Licensed under the Elastic License;
-# you may not use this file except in compliance with the Elastic License.
-#
-
-# frozen_string_literal: true
-
-require 'connectors/base/advanced_snippet_against_schema_validator'
-require 'connectors/mongodb/mongo_advanced_snippet_schema'
-
-module Connectors
-  module MongoDB
-    class MongoAdvancedSnippetAgainstSchemaValidator < Connectors::Base::AdvancedSnippetAgainstSchemaValidator
-
-      def initialize(advanced_snippet, schema = Connectors::MongoDB::AdvancedSnippet::SCHEMA)
-        super
-      end
-
-    end
-  end
-end

data/lib/connectors/mongodb/mongo_advanced_snippet_schema.rb

@@ -1,292 +0,0 @@
-#
-# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
-# or more contributor license agreements. Licensed under the Elastic License;
-# you may not use this file except in compliance with the Elastic License.
-#
-
-# frozen_string_literal: true
-
-module Connectors
-  module MongoDB
-    module AdvancedSnippet
-      # Pipeline stages: https://www.mongodb.com/docs/manual/reference/operator/aggregation-pipeline/
-      ALLOWED_PIPELINE_STAGES = %w[
-        $addFields $bucket $bucketAuto $changeStream $collStats $count $densify
-        $documents $facet $fill $geoNear $graphLookup $group $indexStats $limit
-        $listSessions $lookup $match $merge $out $planCacheStats $project $redact
-        $replaceRoot $replaceWith $sample $search $searchMeta $set $setWindowFields
-        $skip $sort $sortByCount $unionWith $unset $unwind
-      ]
-
-      # All except the $out, $merge, $geoNear, and $changeStream stages can appear multiple times in a pipeline.
-      # Source: https://www.mongodb.com/docs/manual/reference/operator/aggregation-pipeline/
-      PIPELINE_STAGES_ALLOWED_ONCE = %w[$out $merge $geoNear $changeStream]
-
-      NON_NEGATIVE_INTEGER = ->(value) { value.is_a?(Integer) && value >= 0 }
-      READ_CONCERN_LEVEL = ->(level) { %w[local available majority linearizable].include?(level) }
-      STRING_OR_DOCUMENT = ->(value) { value.is_a?(Hash) || value.is_a?(String) }
-      MUTUAL_EXCLUSIVE_FILTER = ->(fields) { fields.size <= 1 }
-
-      AGGREGATION_PIPELINE = lambda { |pipeline|
-        return false unless pipeline.is_a?(Array)
-
-        allowed_once_appearances = Set.new
-
-        pipeline.flat_map(&:keys).each do |key|
-          return false unless ALLOWED_PIPELINE_STAGES.include?(key)
-
-          if PIPELINE_STAGES_ALLOWED_ONCE.include?(key)
-            return false if allowed_once_appearances.include?(key)
-
-            allowed_once_appearances.add(key)
-          end
-        end
-
-        true
-      }
-
-      # Ruby has no 'Boolean' class
-      BOOLEAN = ->(value) { value.is_a?(TrueClass) || value.is_a?(FalseClass) }
-
-      COLLATION = {
-        :name => 'collation',
-        :type => Hash,
-        :optional => true,
-        :fields => [
-          {
-            :name => 'locale',
-            :type => String,
-            :optional => true
-          },
-          {
-            :name => 'caseLevel',
-            :type => BOOLEAN,
-            :optional => true
-          },
-          {
-            :name => 'caseFirst',
-            :type => String,
-            :optional => true
-          },
-          {
-            :name => 'strength',
-            :type => Integer,
-            :optional => true
-          },
-          {
-            :name => 'numericOrdering',
-            :type => BOOLEAN,
-            :optional => true
-          },
-          {
-            :name => 'alternate',
-            :type => String,
-            :optional => true
-          },
-          {
-            :name => 'maxVariable',
-            :type => String,
-            :optional => true
-          },
-          {
-            :name => 'backwards',
-            :type => BOOLEAN,
-            :optional => true
-          },
-        ]
-      }
-
-      CURSOR_TYPE = ->(cursor) { [:tailable, :tailable_await].include?(cursor) }
-
-      # Aggregate options: https://www.mongodb.com/docs/manual/reference/method/db.collection.aggregate/
-      AGGREGATE_OPTIONS = {
-        :name => 'options',
-        :type => Hash,
-        :optional => true,
-        :fields => [
-          {
-            :name => 'explain',
-            :type => BOOLEAN,
-            :optional => true
-          },
-          {
-            :name => 'allowDiskUse',
-            :type => BOOLEAN,
-            :optional => true
-          },
-          {
-            :name => 'cursor',
-            :type => Hash,
-            :optional => true,
-            :fields => [
-              {
-                :name => 'batchSize',
-                :type => NON_NEGATIVE_INTEGER
-              }
-            ]
-          },
-          {
-            :name => 'maxTimeMS',
-            :type => NON_NEGATIVE_INTEGER,
-            :optional => true
-          },
-          {
-            :name => 'bypassDocumentValidation',
-            :type => BOOLEAN,
-            :optional => true
-          },
-          {
-            :name => 'readConcern',
-            :type => Hash,
-            :optional => true,
-            :fields => [
-              {
-                :name => 'level',
-                :type => READ_CONCERN_LEVEL
-              }
-            ]
-          },
-          COLLATION,
-          {
-            :name => 'hint',
-            :type => STRING_OR_DOCUMENT,
-            :optional => true
-          },
-          {
-            :name => 'comment',
-            :type => String,
-            :optional => true
-          },
-          {
-            :name => 'writeConcern',
-            :type => Hash,
-            :optional => true
-          },
-          {
-            :name => 'let',
-            :type => Hash,
-            :optional => true
-          }
-        ]
-      }
-
-      AGGREGATE_PIPELINE = {
-        :name => 'pipeline',
-        :type => AGGREGATION_PIPELINE,
-        :optional => true,
-      }
-
-      AGGREGATE = {
-        :name => 'aggregate',
-        :type => Hash,
-        :optional => true,
-        :fields => [
-          AGGREGATE_PIPELINE,
-          AGGREGATE_OPTIONS
-        ]
-      }
-
-      FIND_OPTIONS = {
-        :name => 'options',
-        :type => Hash,
-        :optional => true,
-        :fields => [
-          {
-            :name => 'allowDiskUse',
-            :type => BOOLEAN,
-            :optional => true
-          },
-          {
-            :name => 'allowPartialResults',
-            :type => BOOLEAN,
-            :optional => true
-          },
-          {
-            :name => 'batchSize',
-            :type => NON_NEGATIVE_INTEGER,
-            :optional => true
-          },
-          COLLATION,
-          {
-            :name => 'cursorType',
-            :type => CURSOR_TYPE,
-            :optional => true
-          },
-          {
-            :name => 'limit',
-            :type => NON_NEGATIVE_INTEGER,
-            :optional => true
-          },
-          {
-            :name => 'maxTimeMS',
-            :type => NON_NEGATIVE_INTEGER,
-            :optional => true
-          },
-          {
-            :name => 'modifiers',
-            :type => Hash,
-            :optional => true
-          },
-          {
-            :name => 'noCursorTimeout',
-            :type => BOOLEAN,
-            :optional => true
-          },
-          {
-            :name => 'oplogReplay',
-            :type => BOOLEAN,
-            :optional => true
-          },
-          {
-            :name => 'projection',
-            :type => Hash,
-            :optional => true
-          },
-          {
-            :name => 'skip',
-            :type => NON_NEGATIVE_INTEGER,
-            :optional => true
-          },
-          {
-            :name => 'sort',
-            :type => Hash,
-            :optional => true
-          },
-          {
-            :name => 'let',
-            :type => Hash,
-            :optional => true
-          }
-        ]
-      }
-
-      # TODO: return true for now. Will be more involved (basically needs full query parsing or "dummy" execution against a running instance)
-      FILTER = ->(_filter) { true }
-
-      FIND_FILTER = {
-        :name => 'filter',
-        :type => FILTER
-      }
-
-      FIND = {
-        :name => 'find',
-        :type => Hash,
-        :optional => true,
-        :fields => [
-          FIND_OPTIONS,
-          FIND_FILTER
-        ]
-      }
-
-      SCHEMA = {
-        :fields => {
-          :constraints => MUTUAL_EXCLUSIVE_FILTER,
-          :values => [
-            AGGREGATE,
-            FIND
-          ]
-        }
-      }
-    end
-  end
-end

data/lib/connectors/mongodb/mongo_rules_parser.rb

@@ -1,81 +0,0 @@
-#
-# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
-# or more contributor license agreements. Licensed under the Elastic License;
-# you may not use this file except in compliance with the Elastic License.
-#
-
-# frozen_string_literal: true
-
-require 'active_support/core_ext/object'
-require 'connectors/base/simple_rules_parser'
-require 'core/filtering/simple_rule'
-
-module Connectors
-  module MongoDB
-    class MongoRulesParser < Connectors::Base::SimpleRulesParser
-      def parse_rule(rule)
-        field = rule.field
-        value = rule.value
-        unless value.present?
-          raise "value is required for field: #{field}"
-        end
-        unless field.present?
-          raise "field is required for rule: #{rule}"
-        end
-        op = rule.rule
-        case op
-        when Core::Filtering::SimpleRule::Rule::EQUALS
-          parse_equals(rule)
-        when Core::Filtering::SimpleRule::Rule::GREATER_THAN
-          parse_greater_than(rule)
-        when Core::Filtering::SimpleRule::Rule::LESS_THAN
-          parse_less_than(rule)
-        when Core::Filtering::SimpleRule::Rule::REGEX
-          parse_regex(rule)
-        else
-          raise "Unknown operator: #{op}"
-        end
-      end
-
-      def merge_rules(rules)
-        return {} if rules.empty?
-        return rules[0] if rules.size == 1
-        { '$and' => rules }
-      end
-
-      private
-
-      def parse_equals(rule)
-        if rule.is_include?
-          { rule.field => rule.value }
-        else
-          { rule.field => { '$ne' => rule.value } }
-        end
-      end
-
-      def parse_greater_than(rule)
-        if rule.is_include?
-          { rule.field => { '$gt' => rule.value } }
-        else
-          { rule.field => { '$lte' => rule.value } }
-        end
-      end
-
-      def parse_less_than(rule)
-        if rule.is_include?
-          { rule.field => { '$lt' => rule.value } }
-        else
-          { rule.field => { '$gte' => rule.value } }
-        end
-      end
-
-      def parse_regex(rule)
-        if rule.is_include?
-          { rule.field => /#{rule.value}/ }
-        else
-          { rule.field => { '$not' => /#{rule.value}/ } }
-        end
-      end
-    end
-  end
-end

data/lib/connectors/tolerable_error_helper.rb

@@ -1,43 +0,0 @@
-#
-# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
-# or more contributor license agreements. Licensed under the Elastic License;
-# you may not use this file except in compliance with the Elastic License.
-#
-
-require 'utility/logger'
-require 'utility/exception_tracking'
-require 'utility/error_monitor'
-
-module Connectors
-  class TolerableErrorHelper
-    def initialize(error_monitor)
-      @error_monitor = error_monitor
-    end
-
-    def yield_single_document(identifier: nil)
-      Utility::Logger.debug("Extracting single document for #{identifier}") if identifier
-      yield
-      @error_monitor.note_success
-    rescue *fatal_exception_classes => e
-      Utility::ExceptionTracking.augment_exception(e)
-      Utility::Logger.error("Encountered a fall-through error during extraction#{identifying_error_message(identifier)}: #{e.class}: #{e.message} {:message_id => #{e.id}}")
-      raise
-    rescue StandardError => e
-      Utility::ExceptionTracking.augment_exception(e)
-      Utility::Logger.warn("Encountered error during extraction#{identifying_error_message(identifier)}: #{e.class}: #{e.message} {:message_id => #{e.id}}")
-      @error_monitor.note_error(e, :id => e.id)
-    end
-
-    private
-
-    def identifying_error_message(identifier)
-      identifier.present? ? " of '#{identifier}'" : ''
-    end
-
-    def fatal_exception_classes
-      [
-        Utility::ErrorMonitor::MonitoringError
-      ]
-    end
-  end
-end