connectors_service 8.6.0.4.pre.20221104T200814Z → 8.6.0.4.pre.20221116T024501Z

data/lib/connectors/mongodb/connector.rb

@@ -7,6 +7,9 @@
 # frozen_string_literal: true
 
 require 'connectors/base/connector'
+require 'core/filtering/validation_status'
+require 'connectors/mongodb/mongo_rules_parser'
+require 'connectors/mongodb/mongo_advanced_snippet_against_schema_validator'
 require 'mongo'
 require 'utility'
 
@@ -28,27 +31,31 @@ module Connectors
 
       def self.configurable_fields
         {
-           :host => {
-             :label => 'Server Hostname'
-           },
-           :user => {
-             :label => 'Username'
-           },
-           :password => {
-             :label => 'Password'
-           },
-           :database => {
-             :label => 'Database'
-           },
-           :collection => {
-             :label => 'Collection'
-           },
-           :direct_connection => {
-             :label => 'Direct connection? (true/false)'
-           }
+          :host => {
+            :label => 'Server Hostname'
+          },
+          :user => {
+            :label => 'Username'
+          },
+          :password => {
+            :label => 'Password'
+          },
+          :database => {
+            :label => 'Database'
+          },
+          :collection => {
+            :label => 'Collection'
+          },
+          :direct_connection => {
+            :label => 'Direct connection? (true/false)'
+          }
         }
       end
 
+      def self.advanced_snippet_validator
+        MongoAdvancedSnippetAgainstSchemaValidator
+      end
+
       def initialize(configuration: {}, job_description: {})
         super
 
@@ -61,8 +68,6 @@ module Connectors
       end
 
       def yield_documents
-        check_filtering
-
         with_client do |client|
           # We do paging using skip().limit() here to make Ruby recycle the memory for each page pulled from the server after it's not needed any more.
           # This gives us more control on the usage of the memory (we can adjust PAGE_SIZE constant for that to decrease max memory consumption).
@@ -89,15 +94,15 @@ module Connectors
           loop do
             found_in_page = 0
 
+            Utility::Logger.info("Requesting #{PAGE_SIZE} documents from MongoDB (Starting at #{skip})")
             view = cursor.skip(skip).limit(PAGE_SIZE)
             view.each do |document|
-              yield serialize(document)
-
-              found_in_page += 1
-              found_overall += 1
-
-              overall_limit_reached = found_overall >= overall_limit && overall_limit != Float::INFINITY
-
+              yield_with_handling_tolerable_errors do
+                yield serialize(document)
+                found_in_page += 1
+                found_overall += 1
+                overall_limit_reached = found_overall >= overall_limit && overall_limit != Float::INFINITY
+              end
               break if overall_limit_reached
             end
 
@@ -117,20 +122,9 @@ module Connectors
 
         return create_aggregate_cursor(collection) if @advanced_filter_config[:aggregate].present?
 
-        collection.find
-      end
-
-      def check_filtering
-        return unless filtering_present?
+        return create_simple_rules_cursor(collection) if @rules.present?
 
-        check_find_and_aggregate
-      end
-
-      def check_find_and_aggregate
-        if @advanced_filter_config.keys.size != 1
-          invalid_keys_msg = "Only one of #{ALLOWED_TOP_LEVEL_FILTER_KEYS} is allowed in the filtering object. Keys present: '#{@advanced_filter_config.keys}'."
-          raise Utility::InvalidFilterConfigError.new(invalid_keys_msg)
-        end
+        collection.find
       end
 
       def create_aggregate_cursor(collection)
@@ -159,6 +153,16 @@ module Connectors
         [collection.find(filter, options), options]
       end
 
+      def create_simple_rules_cursor(collection)
+        filter = {}
+        if @rules.present?
+          parser = MongoRulesParser.new(@rules)
+          filter = parser.parse
+        end
+        Utility::Logger.info("Filtering with simple rules filter: #{filter}")
+        filter.present? ? collection.find(filter) : collection.find
+      end
+
       def extract_options(mongodb_function)
         mongodb_function[:options].present? ? mongodb_function[:options] : {}
       end
@@ -173,9 +177,9 @@ module Connectors
         raise "Invalid value for 'Direct connection' : #{@direct_connection}." unless %w[true false].include?(@direct_connection.to_s.strip.downcase)
 
         args = {
-                 database: @database,
-                 direct_connection: to_boolean(@direct_connection)
-               }
+          database: @database,
+          direct_connection: to_boolean(@direct_connection)
+        }
 
         if @user.present? || @password.present?
           args[:user] = @user

data/lib/connectors/mongodb/mongo_advanced_snippet_against_schema_validator.rb

@@ -0,0 +1,22 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'connectors/base/advanced_snippet_against_schema_validator'
+require 'connectors/mongodb/mongo_advanced_snippet_schema'
+
+module Connectors
+  module MongoDB
+    class MongoAdvancedSnippetAgainstSchemaValidator < Connectors::Base::AdvancedSnippetAgainstSchemaValidator
+
+      def initialize(advanced_snippet, schema = Connectors::MongoDB::AdvancedSnippet::SCHEMA)
+        super
+      end
+
+    end
+  end
+end

data/lib/connectors/mongodb/mongo_advanced_snippet_schema.rb

@@ -0,0 +1,292 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+module Connectors
+  module MongoDB
+    module AdvancedSnippet
+      # Pipeline stages: https://www.mongodb.com/docs/manual/reference/operator/aggregation-pipeline/
+      ALLOWED_PIPELINE_STAGES = %w[
+        $addFields $bucket $bucketAuto $changeStream $collStats $count $densify
+        $documents $facet $fill $geoNear $graphLookup $group $indexStats $limit
+        $listSessions $lookup $match $merge $out $planCacheStats $project $redact
+        $replaceRoot $replaceWith $sample $search $searchMeta $set $setWindowFields
+        $skip $sort $sortByCount $unionWith $unset $unwind
+      ]
+
+      # All except the $out, $merge, $geoNear, and $changeStream stages can appear multiple times in a pipeline.
+      # Source: https://www.mongodb.com/docs/manual/reference/operator/aggregation-pipeline/
+      PIPELINE_STAGES_ALLOWED_ONCE = %w[$out $merge $geoNear $changeStream]
+
+      NON_NEGATIVE_INTEGER = ->(value) { value.is_a?(Integer) && value >= 0 }
+      READ_CONCERN_LEVEL = ->(level) { %w[local available majority linearizable].include?(level) }
+      STRING_OR_DOCUMENT = ->(value) { value.is_a?(Hash) || value.is_a?(String) }
+      MUTUAL_EXCLUSIVE_FILTER = ->(fields) { fields.size <= 1 }
+
+      AGGREGATION_PIPELINE = lambda { |pipeline|
+        return false unless pipeline.is_a?(Array)
+
+        allowed_once_appearances = Set.new
+
+        pipeline.flat_map(&:keys).each do |key|
+          return false unless ALLOWED_PIPELINE_STAGES.include?(key)
+
+          if PIPELINE_STAGES_ALLOWED_ONCE.include?(key)
+            return false if allowed_once_appearances.include?(key)
+
+            allowed_once_appearances.add(key)
+          end
+        end
+
+        true
+      }
+
+      # Ruby has no 'Boolean' class
+      BOOLEAN = ->(value) { value.is_a?(TrueClass) || value.is_a?(FalseClass) }
+
+      COLLATION = {
+        :name => 'collation',
+        :type => Hash,
+        :optional => true,
+        :fields => [
+          {
+            :name => 'locale',
+            :type => String,
+            :optional => true
+          },
+          {
+            :name => 'caseLevel',
+            :type => BOOLEAN,
+            :optional => true
+          },
+          {
+            :name => 'caseFirst',
+            :type => String,
+            :optional => true
+          },
+          {
+            :name => 'strength',
+            :type => Integer,
+            :optional => true
+          },
+          {
+            :name => 'numericOrdering',
+            :type => BOOLEAN,
+            :optional => true
+          },
+          {
+            :name => 'alternate',
+            :type => String,
+            :optional => true
+          },
+          {
+            :name => 'maxVariable',
+            :type => String,
+            :optional => true
+          },
+          {
+            :name => 'backwards',
+            :type => BOOLEAN,
+            :optional => true
+          },
+        ]
+      }
+
+      CURSOR_TYPE = ->(cursor) { [:tailable, :tailable_await].include?(cursor) }
+
+      # Aggregate options: https://www.mongodb.com/docs/manual/reference/method/db.collection.aggregate/
+      AGGREGATE_OPTIONS = {
+        :name => 'options',
+        :type => Hash,
+        :optional => true,
+        :fields => [
+          {
+            :name => 'explain',
+            :type => BOOLEAN,
+            :optional => true
+          },
+          {
+            :name => 'allowDiskUse',
+            :type => BOOLEAN,
+            :optional => true
+          },
+          {
+            :name => 'cursor',
+            :type => Hash,
+            :optional => true,
+            :fields => [
+              {
+                :name => 'batchSize',
+                :type => NON_NEGATIVE_INTEGER
+              }
+            ]
+          },
+          {
+            :name => 'maxTimeMS',
+            :type => NON_NEGATIVE_INTEGER,
+            :optional => true
+          },
+          {
+            :name => 'bypassDocumentValidation',
+            :type => BOOLEAN,
+            :optional => true
+          },
+          {
+            :name => 'readConcern',
+            :type => Hash,
+            :optional => true,
+            :fields => [
+              {
+                :name => 'level',
+                :type => READ_CONCERN_LEVEL
+              }
+            ]
+          },
+          COLLATION,
+          {
+            :name => 'hint',
+            :type => STRING_OR_DOCUMENT,
+            :optional => true
+          },
+          {
+            :name => 'comment',
+            :type => String,
+            :optional => true
+          },
+          {
+            :name => 'writeConcern',
+            :type => Hash,
+            :optional => true
+          },
+          {
+            :name => 'let',
+            :type => Hash,
+            :optional => true
+          }
+        ]
+      }
+
+      AGGREGATE_PIPELINE = {
+        :name => 'pipeline',
+        :type => AGGREGATION_PIPELINE,
+        :optional => true,
+      }
+
+      AGGREGATE = {
+        :name => 'aggregate',
+        :type => Hash,
+        :optional => true,
+        :fields => [
+          AGGREGATE_PIPELINE,
+          AGGREGATE_OPTIONS
+        ]
+      }
+
+      FIND_OPTIONS = {
+        :name => 'options',
+        :type => Hash,
+        :optional => true,
+        :fields => [
+          {
+            :name => 'allowDiskUse',
+            :type => BOOLEAN,
+            :optional => true
+          },
+          {
+            :name => 'allowPartialResults',
+            :type => BOOLEAN,
+            :optional => true
+          },
+          {
+            :name => 'batchSize',
+            :type => NON_NEGATIVE_INTEGER,
+            :optional => true
+          },
+          COLLATION,
+          {
+            :name => 'cursorType',
+            :type => CURSOR_TYPE,
+            :optional => true
+          },
+          {
+            :name => 'limit',
+            :type => NON_NEGATIVE_INTEGER,
+            :optional => true
+          },
+          {
+            :name => 'maxTimeMS',
+            :type => NON_NEGATIVE_INTEGER,
+            :optional => true
+          },
+          {
+            :name => 'modifiers',
+            :type => Hash,
+            :optional => true
+          },
+          {
+            :name => 'noCursorTimeout',
+            :type => BOOLEAN,
+            :optional => true
+          },
+          {
+            :name => 'oplogReplay',
+            :type => BOOLEAN,
+            :optional => true
+          },
+          {
+            :name => 'projection',
+            :type => Hash,
+            :optional => true
+          },
+          {
+            :name => 'skip',
+            :type => NON_NEGATIVE_INTEGER,
+            :optional => true
+          },
+          {
+            :name => 'sort',
+            :type => Hash,
+            :optional => true
+          },
+          {
+            :name => 'let',
+            :type => Hash,
+            :optional => true
+          }
+        ]
+      }
+
+      # TODO: return true for now. Will be more involved (basically needs full query parsing or "dummy" execution against a running instance)
+      FILTER = ->(_filter) { true }
+
+      FIND_FILTER = {
+        :name => 'filter',
+        :type => FILTER
+      }
+
+      FIND = {
+        :name => 'find',
+        :type => Hash,
+        :optional => true,
+        :fields => [
+          FIND_OPTIONS,
+          FIND_FILTER
+        ]
+      }
+
+      SCHEMA = {
+        :fields => {
+          :constraints => MUTUAL_EXCLUSIVE_FILTER,
+          :values => [
+            AGGREGATE,
+            FIND
+          ]
+        }
+      }
+    end
+  end
+end

data/lib/connectors/mongodb/mongo_rules_parser.rb

@@ -0,0 +1,81 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'active_support/core_ext/object'
+require 'connectors/base/simple_rules_parser'
+require 'core/filtering/simple_rule'
+
+module Connectors
+  module MongoDB
+    class MongoRulesParser < Connectors::Base::SimpleRulesParser
+      def parse_rule(rule)
+        field = rule.field
+        value = rule.value
+        unless value.present?
+          raise "value is required for field: #{field}"
+        end
+        unless field.present?
+          raise "field is required for rule: #{rule}"
+        end
+        op = rule.rule
+        case op
+        when Core::Filtering::SimpleRule::Rule::EQUALS
+          parse_equals(rule)
+        when Core::Filtering::SimpleRule::Rule::GREATER_THAN
+          parse_greater_than(rule)
+        when Core::Filtering::SimpleRule::Rule::LESS_THAN
+          parse_less_than(rule)
+        when Core::Filtering::SimpleRule::Rule::REGEX
+          parse_regex(rule)
+        else
+          raise "Unknown operator: #{op}"
+        end
+      end
+
+      def merge_rules(rules)
+        return {} if rules.empty?
+        return rules[0] if rules.size == 1
+        { '$and' => rules }
+      end
+
+      private
+
+      def parse_equals(rule)
+        if rule.is_include?
+          { rule.field => rule.value }
+        else
+          { rule.field => { '$ne' => rule.value } }
+        end
+      end
+
+      def parse_greater_than(rule)
+        if rule.is_include?
+          { rule.field => { '$gt' => rule.value } }
+        else
+          { rule.field => { '$lte' => rule.value } }
+        end
+      end
+
+      def parse_less_than(rule)
+        if rule.is_include?
+          { rule.field => { '$lt' => rule.value } }
+        else
+          { rule.field => { '$gte' => rule.value } }
+        end
+      end
+
+      def parse_regex(rule)
+        if rule.is_include?
+          { rule.field => /#{rule.value}/ }
+        else
+          { rule.field => { '$not' => /#{rule.value}/ } }
+        end
+      end
+    end
+  end
+end

data/lib/connectors/sync_status.rb

@@ -26,11 +26,16 @@ module Connectors
       ERROR
     ]
 
-    PENDING_STATUES = [
+    PENDING_STATUSES = [
       PENDING,
       SUSPENDED
     ]
 
+    ACTIVE_STATUSES = [
+      IN_PROGRESS,
+      CANCELING
+    ]
+
     TERMINAL_STATUSES = [
       CANCELED,
       COMPLETED,

data/lib/connectors/tolerable_error_helper.rb

@@ -0,0 +1,43 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+require 'utility/logger'
+require 'utility/exception_tracking'
+require 'utility/error_monitor'
+
+module Connectors
+  class TolerableErrorHelper
+    def initialize(error_monitor)
+      @error_monitor = error_monitor
+    end
+
+    def yield_single_document(identifier: nil)
+      Utility::Logger.debug("Extracting single document for #{identifier}") if identifier
+      yield
+      @error_monitor.note_success
+    rescue *fatal_exception_classes => e
+      Utility::ExceptionTracking.augment_exception(e)
+      Utility::Logger.error("Encountered a fall-through error during extraction#{identifying_error_message(identifier)}: #{e.class}: #{e.message} {:message_id => #{e.id}}")
+      raise
+    rescue StandardError => e
+      Utility::ExceptionTracking.augment_exception(e)
+      Utility::Logger.warn("Encountered error during extraction#{identifying_error_message(identifier)}: #{e.class}: #{e.message} {:message_id => #{e.id}}")
+      @error_monitor.note_error(e, :id => e.id)
+    end
+
+    private
+
+    def identifying_error_message(identifier)
+      identifier.present? ? " of '#{identifier}'" : ''
+    end
+
+    def fatal_exception_classes
+      [
+        Utility::ErrorMonitor::MonitoringError
+      ]
+    end
+  end
+end

data/lib/core/configuration.rb

@@ -24,8 +24,10 @@ module Core
             return
           end
           configuration = connector_class.configurable_fields_indifferent_access
+          features = connector_class.kibana_features.each_with_object({}) { |feature, hsh| hsh[feature] = true }
           doc = {
-            :configuration => configuration
+            :configuration => configuration,
+            :features => features
           }
 
           doc[:service_type] = service_type if service_type && connector_settings.needs_service_type?