connectors_service 8.7.0.0.pre.20221117T010623Z → 8.11.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: aaca309c6ef67d232a19c895ce21b1c9a0b75b1aed64bc90e4676ec5647c43ff
-  data.tar.gz: 1cc77648386b48bd863319ca309167bfce4a7fda5f50ceff18b659c134a88fa1
+  metadata.gz: 3c2c9d24a514189f2efb9fbfdcba52a2ee49d40dec97025b6524e1562087e750
+  data.tar.gz: f3ea6369ff7cbab5983178df08d5aa168f72cd66325c23c5dd6fe839688db1f8
 SHA512:
-  metadata.gz: 727ef952b3c32f38c56aaa05e6514ad9124e0086e0d898b02843932f1fa2889d870f099890e957c69e36d532bb65fb60f1996e5ec42233c09e8c19868767cc97
-  data.tar.gz: 66d47ed3e631562e080c01030eb1d5a0ea44f10f6d317d3181db5f89ec02f3d7ddbeeb486567a09e058c56b75cd225bb47dc31cf615762ce9602c56c69b27d24
+  metadata.gz: 2b80a16d7a1d60fe19d1ac95d9cd8a764629b256632fc205342513654fbbceae4b70d73f4f8ab8b2be808ed6d36f7d62773a32326a01e7579a15b6d4dbdc00f0
+  data.tar.gz: cfe21c6e49e6a95af8820a2a89b5dd6dc1b5b080cf174dd90c575bc5b390047334b27a4df7f01a3c26d19990162758b4f82cdb17ad01d048057bd62a3ded8863

data/config/connectors.yml

@@ -1,11 +1,10 @@
 # general metadata
-version: 8.7.0.0-20221117T010623Z
-repository: https://github.com/elastic/connectors-ruby.git
-revision: 294214a26b0fe9a4347763b01de681c336e8daae
+version: 8.11.0.0
+repository: git@github.com:elastic/connectors-ruby.git
+revision: b6c38305e5f7297064f4cf1cbda20f892483f267
 elasticsearch:
-  cloud_id: CHANGEME
   hosts: http://localhost:9200
-  api_key: CHANGEME
+  api_key: QWFHdV9JY0JDZDEzMU8tU3N1Y2Q6YXRyLTV5SlBTUENKYXcta1Yxc0RVZw==
   retry_on_failure: 3
   request_timeout: 120
   disable_warnings: true
@@ -16,10 +15,13 @@ thread_pool:
   max_threads: 5
   max_queue: 100
 log_level: info
-ecs_logging: true
+ecs_logging: false
 poll_interval: 3
 termination_timeout: 60
 heartbeat_interval: 1800
+job_cleanup_interval: 300
 native_mode: true
-connector_id: CHANGEME
-service_type: CHANGEME
+connector_id: _6Go_IcBCd131O-Ss-YH
+service_type: mongodb
+max_ingestion_queue_size: 500
+max_ingestion_queue_bytes: 5242880

data/lib/app/config.rb

@@ -37,6 +37,7 @@ puts "Parsing #{CONFIG_FILE} configuration file."
       optional(:log).value(:bool?)
       optional(:ca_fingerprint).value(:string)
       optional(:transport_options).value(:hash)
+      optional(:headers).value(:hash)
     end
 
     optional(:thread_pool).hash do
@@ -54,6 +55,7 @@ puts "Parsing #{CONFIG_FILE} configuration file."
     optional(:poll_interval).value(:integer)
     optional(:termination_timeout).value(:integer)
     optional(:heartbeat_interval).value(:integer)
+    optional(:job_cleanup_interval).value(:integer)
 
     optional(:max_ingestion_queue_size).value(:integer) # items
     optional(:max_ingestion_queue_bytes).value(:integer) # bytes
@@ -111,6 +113,8 @@ module App
       return nil
     end
 
+    headers = ent_search_config['elasticsearch.headers'] || ent_search_config.dig('elasticsearch', 'headers')
+
     {
       :hosts => [
         {
@@ -120,7 +124,8 @@ module App
           host: uri.host,
           port: uri.port
         }
-      ]
+      ],
+      :headers => headers
     }
   end
 

data/lib/app/console_app.rb

@@ -89,7 +89,7 @@ module App
     def enable_scheduling
       return unless connector_registered?
 
-      previous_schedule = Core::ConnectorSettings.fetch_by_id(connector_id)&.scheduling_settings&.fetch(:interval, nil)
+      previous_schedule = Core::ConnectorSettings.fetch_by_id(connector_id)&.full_sync_scheduling&.fetch(:interval, nil)
       if previous_schedule.present?
         puts "Please enter a valid crontab expression for scheduling. Previous schedule was: #{previous_schedule}."
       else

data/lib/app/dispatcher.rb

@@ -21,6 +21,7 @@ module App
     MIN_THREADS = (App::Config.dig(:thread_pool, :min_threads) || 0).to_i
     MAX_THREADS = (App::Config.dig(:thread_pool, :max_threads) || 5).to_i
     MAX_QUEUE = (App::Config.dig(:thread_pool, :max_queue) || 100).to_i
+    JOB_CLEANUP_INTERVAL = (App::Config.job_cleanup_interval || 60 * 5).to_i
 
     @running = Concurrent::AtomicBoolean.new(false)
 
@@ -28,16 +29,16 @@ module App
       def start!
         running!
         Utility::Logger.info("Starting connector service in #{App::Config.native_mode ? 'native' : 'non-native'} mode...")
-
+        start_job_cleanup_task!
         # start sync jobs consumer
         start_consumer!
-
         start_polling_jobs!
       end
 
       def shutdown!
         Utility::Logger.info("Shutting down connector service with pool [#{pool.class}]...")
         running.make_false
+        job_cleanup_timer.shutdown
         scheduler.shutdown
         pool.shutdown
         pool.wait_for_termination(TERMINATION_TIMEOUT)
@@ -70,11 +71,25 @@ module App
                        end
       end
 
+      def job_cleanup_timer
+        @job_cleanup_timer ||= Concurrent::TimerTask.new(:execution_interval => JOB_CLEANUP_INTERVAL, :run_now => true) do
+          connector_id = App::Config.native_mode ? nil : App::Config.connector_id
+          Core::JobCleanUp.execute(connector_id)
+        end
+      end
+
+      def start_job_cleanup_task!
+        job_cleanup_timer.execute
+      end
+
       def start_polling_jobs!
         scheduler.when_triggered do |connector_settings, task|
           case task
           when :sync
-            # update connector sync_now flag
+            # TODO: #update_connector_sync_now should be moved to Core::ConnectorSettings,
+            # there should not be any business logic related code in Core::ElasticConnectorActions.
+            # #update_connector_sync_now should not update `last_synced` after https://github.com/elastic/enterprise-search-team/issues/3366 is resolved,
+            # schedule should not based on `last_synced`
             Core::ElasticConnectorActions.update_connector_sync_now(connector_settings.id, false)
 
             Core::Jobs::Producer.enqueue_job(job_type: :sync, connector_settings: connector_settings)

data/lib/connectors/base/connector.rb

@@ -9,9 +9,15 @@
 require 'active_support/core_ext/hash/indifferent_access'
 require 'app/config'
 require 'bson'
-require 'connectors/base/advanced_snippet_validator'
 require 'core/ingestion'
 require 'connectors/tolerable_error_helper'
+require 'core/filtering/advanced_snippet/advanced_snippet_validator'
+require 'core/filtering/simple_rules/validation/no_conflicting_policies_rules_validator'
+require 'core/filtering/simple_rules/validation/single_rule_against_schema_validator'
+require 'core/filtering/transform/filter_transformer_facade'
+require 'core/filtering/transform/transformation_target'
+require 'core/filtering/filter_validator'
+require 'core/filtering/processing_stage'
 require 'core/filtering/validation_status'
 require 'utility'
 require 'utility/filtering'
@@ -38,40 +44,55 @@ module Connectors
 
       def self.kibana_features
         [
-          Utility::Constants::FILTERING_RULES_FEATURE,
-          Utility::Constants::FILTERING_ADVANCED_FEATURE
+          { :feature => :sync_rules, :subfeature => :basic, :enabled => true },
+          { :feature => :sync_rules, :subfeature => :advanced, :enabled => true }
         ]
       end
 
-      def self.advanced_snippet_validator
-        AdvancedSnippetValidator
+      def self.advanced_snippet_validators
+        Core::Filtering::AdvancedSnippet::AdvancedSnippetValidator
       end
 
-      def self.validate_filtering(filtering = {})
-        # nothing to validate
-        return { :state => Core::Filtering::ValidationStatus::VALID, :errors => [] } unless filtering.present?
+      def self.simple_rules_validators
+        {
+          Core::Filtering::ProcessingStage::ALL => [
+            Core::Filtering::SimpleRules::Validation::SingleRuleAgainstSchemaValidator,
+            Core::Filtering::SimpleRules::Validation::NoConflictingPoliciesRulesValidator
+          ]
+        }
+      end
 
-        filter = Utility::Filtering.extract_filter(filtering)
-        advanced_snippet = filter.dig(:advanced_snippet, :value)
+      def self.filter_transformers
+        {
+          Core::Filtering::Transform::TransformationTarget::ADVANCED_SNIPPET => [],
+          Core::Filtering::Transform::TransformationTarget::RULES => []
+        }
+      end
 
-        snippet_validator_instance = advanced_snippet_validator.new(advanced_snippet)
+      def self.validate_filtering(filtering = {})
+        filter = Utility::Filtering.extract_filter(filtering)
 
-        snippet_validator_instance.is_snippet_valid?
+        filter_validator = Core::Filtering::FilterValidator.new(snippet_validator_classes: advanced_snippet_validators,
+                                                                rules_validator_classes: simple_rules_validators,
+                                                                rules_pre_processing_active: Utility::Filtering.rule_pre_processing_active?(filter))
+        filter_validator.is_filter_valid(filter)
       end
 
       attr_reader :rules, :advanced_filter_config
 
-      def initialize(configuration: {}, job_description: {})
+      def initialize(configuration: {}, job_description: nil)
         error_monitor = Utility::ErrorMonitor.new
         @tolerable_error_helper = Connectors::TolerableErrorHelper.new(error_monitor)
 
-        @configuration = configuration.dup || {}
-        @job_description = job_description&.dup || {}
+        @configuration = job_description&.configuration&.dup || configuration&.dup || {}
+        @job_description = job_description&.dup
 
-        filtering = Utility::Filtering.extract_filter(@job_description.dig(:connector, :filtering))
+        filter = Utility::Filtering.extract_filter(@job_description&.filtering)
+        filter = Core::Filtering::Transform::FilterTransformerFacade.new(filter, self.class.filter_transformers).transform
 
-        @rules = filtering[:rules] || []
-        @advanced_filter_config = filtering.dig(:advanced_snippet, :value) || {}
+        @rules = filter[:rules] || []
+        # regression bug, we need to keep indifferent access here until we get rid of symbols in the connectors
+        @advanced_filter_config = filter[:advanced_snippet]&.with_indifferent_access || {}
       end
 
       def yield_documents; end
@@ -100,10 +121,6 @@ module Connectors
         false
       end
 
-      def filtering_present?
-        @advanced_filter_config.present? && !@advanced_filter_config.empty? || @rules.present?
-      end
-
       def metadata
         {}
       end

data/lib/connectors/crawler/scheduler.rb

@@ -22,11 +22,47 @@ module Connectors
         []
       end
 
+      def when_triggered
+        loop do
+          connector_settings.each do |cs|
+            # crawler only supports :sync
+            if sync_triggered?(cs)
+              yield cs, :sync, nil
+              next
+            end
+
+            schedule_key = custom_schedule_triggered(cs)
+            yield cs, :sync, schedule_key if schedule_key
+          end
+        rescue *Utility::AUTHORIZATION_ERRORS => e
+          log_authorization_error(e)
+        rescue StandardError => e
+          log_standard_error(e)
+        ensure
+          if @is_shutting_down
+            break
+          end
+          sleep_for_poll_interval
+        end
+      end
+
       private
 
       def connector_registered?(service_type)
         service_type == 'elastic-crawler'
       end
+
+      # custom scheduling has no ordering, so the first-found schedule is returned
+      def custom_schedule_triggered(cs)
+        cs.custom_scheduling_settings.each do |key, custom_scheduling|
+          identifier = "#{cs.formatted} - #{custom_scheduling[:name]}"
+          if schedule_triggered?(custom_scheduling, identifier)
+            return key
+          end
+        end
+
+        nil
+      end
     end
   end
 end

data/lib/connectors/example/connector.rb

@@ -36,7 +36,7 @@ module Connectors
         }
       end
 
-      def initialize(configuration: {}, job_description: {})
+      def initialize(configuration: {}, job_description: nil)
         super
       end
 
@@ -47,7 +47,7 @@ module Connectors
         # raise 'something went wrong'
       end
 
-      def self.advanced_snippet_validator
+      def self.advanced_snippet_validators
         ExampleAdvancedSnippetValidator
       end
 

data/lib/connectors/example/example_advanced_snippet_validator.rb

@@ -6,13 +6,14 @@
 
 # frozen_string_literal: true
 
-require 'connectors/base/advanced_snippet_validator'
+require 'core/filtering/advanced_snippet/advanced_snippet_validator'
+require 'core/filtering/validation_status'
 
 module Connectors
   module Example
-    class ExampleAdvancedSnippetValidator < Connectors::Base::AdvancedSnippetValidator
+    class ExampleAdvancedSnippetValidator < Core::Filtering::AdvancedSnippet::AdvancedSnippetValidator
 
-      def is_snippet_valid?
+      def is_snippet_valid
         # TODO: real filtering validation will follow later
         errors = [
           {

data/lib/connectors/gitlab/connector.rb

@@ -37,16 +37,16 @@ module Connectors
         }
       end
 
-      def self.advanced_snippet_validator
+      def self.advanced_snippet_validators
         GitLabAdvancedSnippetValidator
       end
 
-      def initialize(configuration: {}, job_description: {})
+      def initialize(configuration: {}, job_description: nil)
         super
 
         @extractor = Connectors::GitLab::Extractor.new(
-          :base_url => configuration.dig(:base_url, :value),
-          :api_token => configuration.dig(:api_token, :value)
+          :base_url => @configuration.dig(:base_url, :value),
+          :api_token => @configuration.dig(:api_token, :value)
         )
       end
 

data/lib/connectors/gitlab/gitlab_advanced_snippet_validator.rb

@@ -6,13 +6,13 @@
 
 # frozen_string_literal: true
 
-require 'connectors/base/advanced_snippet_validator'
+require 'core/filtering/advanced_snippet/advanced_snippet_validator'
 
 module Connectors
   module GitLab
-    class GitLabAdvancedSnippetValidator < Connectors::Base::AdvancedSnippetValidator
+    class GitLabAdvancedSnippetValidator < Core::Filtering::AdvancedSnippet::AdvancedSnippetValidator
 
-      def is_snippet_valid?
+      def is_snippet_valid
         # TODO: real filtering validation will follow later
         errors = [
           {
@@ -21,13 +21,11 @@ module Connectors
           }
         ]
 
-        validation_result = if @advanced_snippet.present? && !@advanced_snippet.empty?
-                              { :state => Core::Filtering::ValidationStatus::INVALID, :errors => errors }
-                            else
-                              { :state => Core::Filtering::ValidationStatus::VALID, :errors => [] }
-                            end
-        log_validation_result(validation_result)
-        validation_result
+        if @advanced_snippet.present? && !@advanced_snippet.empty?
+          { :state => Core::Filtering::ValidationStatus::INVALID, :errors => errors }
+        else
+          { :state => Core::Filtering::ValidationStatus::VALID, :errors => [] }
+        end
       end
 
     end

data/lib/{connectors_app/// → connectors/job_trigger_method.rb}

@@ -4,10 +4,11 @@
 # you may not use this file except in compliance with the Elastic License.
 #
 
-module ConnectorsApp
-  module Errors
-    INVALID_API_KEY = 'INVALID_API_KEY'
-    UNSUPPORTED_AUTH_SCHEME = 'UNSUPPORTED_AUTH_SCHEME'
-    INTERNAL_SERVER_ERROR = 'INTERNAL_SERVER_ERROR'
+# frozen_string_literal: true
+
+module Connectors
+  class JobTriggerMethod
+    ON_DEMAND = 'on_demand'
+    SCHEDULED = 'scheduled'
   end
 end

data/lib/connectors/mongodb/connector.rb

@@ -8,7 +8,8 @@
 
 require 'connectors/base/connector'
 require 'core/filtering/validation_status'
-require 'connectors/mongodb/mongo_rules_parser'
+require 'core/filtering/transform/transformation_target'
+require 'connectors/mongodb/mongo_advanced_snippet_snake_case_transformer'
 require 'connectors/mongodb/mongo_advanced_snippet_against_schema_validator'
 require 'mongo'
 require 'utility'
@@ -19,6 +20,9 @@ module Connectors
 
       ALLOWED_TOP_LEVEL_FILTER_KEYS = %w[find aggregate]
 
+      AGGREGATE = 'aggregate'
+      FIND = 'find'
+
       PAGE_SIZE = 100
 
       def self.service_type
@@ -52,19 +56,25 @@ module Connectors
         }
       end
 
-      def self.advanced_snippet_validator
+      def self.advanced_snippet_validators
         MongoAdvancedSnippetAgainstSchemaValidator
       end
 
-      def initialize(configuration: {}, job_description: {})
+      def self.filter_transformers
+        {
+          Core::Filtering::Transform::TransformationTarget::ADVANCED_SNIPPET => [MongoAdvancedSnippetSnakeCaseTransformer]
+        }
+      end
+
+      def initialize(configuration: {}, job_description: nil)
         super
 
-        @host = configuration.dig(:host, :value)
-        @database = configuration.dig(:database, :value)
-        @collection = configuration.dig(:collection, :value)
-        @user = configuration.dig(:user, :value)
-        @password = configuration.dig(:password, :value)
-        @direct_connection = configuration.dig(:direct_connection, :value)
+        @host = @configuration.dig(:host, :value)
+        @database = @configuration.dig(:database, :value)
+        @collection = @configuration.dig(:collection, :value)
+        @user = @configuration.dig(:user, :value)
+        @password = @configuration.dig(:password, :value)
+        @direct_connection = @configuration.dig(:direct_connection, :value)
       end
 
       def yield_documents
@@ -73,44 +83,56 @@ module Connectors
           # This gives us more control on the usage of the memory (we can adjust PAGE_SIZE constant for that to decrease max memory consumption).
           # It's done due to the fact that usage of .find.each leads to memory leaks or overuse of memory - the whole result set seems to stay in memory
           # during the sync. Sometimes (not 100% sure) it even leads to a real leak, when the memory for these objects is never recycled.
-          cursor, options = create_db_cursor_on_collection(client[@collection])
-          skip = 0
-
-          found_overall = 0
-
-          # if no overall limit is specified by filtering use -1 to not break ingestion, when no overall limit is specified (found_overall is only increased,
-          # thus can never reach -1)
-          overall_limit = Float::INFINITY
+          cursor_type, cursor_with_options = create_db_cursor_on_collection(client[@collection])
+          cursor, options = cursor_with_options
+
+          case cursor_type
+          when FIND
+            skip = 0
+            found_overall = 0
+
+            # if no overall limit is specified by filtering use -1 to not break ingestion, when no overall limit is specified (found_overall is only increased,
+            # thus can never reach -1)
+            overall_limit = Float::INFINITY
+
+            if options.present?
+              # there could be a skip parameter defined for filtering
+              skip = options.fetch(:skip, skip)
+              # there could be a limit parameter defined for filtering -> used for an overall limit (not a page limit, which was introduced for memory optimization)
+              overall_limit = options.fetch(:limit, overall_limit)
+            end
 
-          if options.present?
-            # there could be a skip parameter defined for filtering
-            skip = options.fetch(:skip, skip)
-            # there could be a limit parameter defined for filtering -> used for an overall limit (not a page limit, which was introduced for memory optimization)
-            overall_limit = options.fetch(:limit, overall_limit)
-          end
+            overall_limit_reached = false
+
+            loop do
+              found_in_page = 0
+
+              Utility::Logger.info("Requesting #{PAGE_SIZE} documents from MongoDB (Starting at #{skip})")
+              view = cursor.skip(skip).limit(PAGE_SIZE)
+              view.each do |document|
+                yield_with_handling_tolerable_errors do
+                  yield serialize(document)
+                  found_in_page += 1
+                  found_overall += 1
+                  overall_limit_reached = found_overall >= overall_limit && overall_limit != Float::INFINITY
+                end
+                break if overall_limit_reached
+              end
 
-          overall_limit_reached = false
+              page_was_empty = found_in_page == 0
 
-          loop do
-            found_in_page = 0
+              break if page_was_empty || overall_limit_reached
 
-            Utility::Logger.info("Requesting #{PAGE_SIZE} documents from MongoDB (Starting at #{skip})")
-            view = cursor.skip(skip).limit(PAGE_SIZE)
-            view.each do |document|
+              skip += PAGE_SIZE
+            end
+          when AGGREGATE
+            cursor.each do |document|
               yield_with_handling_tolerable_errors do
                 yield serialize(document)
-                found_in_page += 1
-                found_overall += 1
-                overall_limit_reached = found_overall >= overall_limit && overall_limit != Float::INFINITY
               end
-              break if overall_limit_reached
             end
-
-            page_was_empty = found_in_page == 0
-
-            break if page_was_empty || overall_limit_reached
-
-            skip += PAGE_SIZE
+          else
+            raise "Unknown retrieval function #{cursor_type} for MongoDB."
           end
         end
       end
@@ -118,22 +140,20 @@ module Connectors
       private
 
       def create_db_cursor_on_collection(collection)
-        return create_find_cursor(collection) if @advanced_filter_config[:find].present?
+        return [AGGREGATE, create_aggregate_cursor(collection)] if @advanced_filter_config[:aggregate].present?
 
-        return create_aggregate_cursor(collection) if @advanced_filter_config[:aggregate].present?
+        return [FIND, create_find_cursor(collection)] if @advanced_filter_config[:find].present?
 
-        return create_simple_rules_cursor(collection) if @rules.present?
-
-        collection.find
+        [FIND, collection.find]
       end
 
       def create_aggregate_cursor(collection)
         aggregate = @advanced_filter_config[:aggregate]
 
-        pipeline = aggregate[:pipeline]
+        pipeline = aggregate[:pipeline] || []
         options = extract_options(aggregate)
 
-        if !pipeline.nil? && pipeline.empty? && !options.present?
+        if pipeline.empty? && options.empty?
           Utility::Logger.warn('\'Aggregate\' was specified with an empty pipeline and empty options.')
         end
 
@@ -153,16 +173,6 @@ module Connectors
         [collection.find(filter, options), options]
       end
 
-      def create_simple_rules_cursor(collection)
-        filter = {}
-        if @rules.present?
-          parser = MongoRulesParser.new(@rules)
-          filter = parser.parse
-        end
-        Utility::Logger.info("Filtering with simple rules filter: #{filter}")
-        filter.present? ? collection.find(filter) : collection.find
-      end
-
       def extract_options(mongodb_function)
         mongodb_function[:options].present? ? mongodb_function[:options] : {}
       end

data/lib/connectors/mongodb/mongo_advanced_snippet_against_schema_validator.rb

@@ -6,12 +6,12 @@
 
 # frozen_string_literal: true
 
-require 'connectors/base/advanced_snippet_against_schema_validator'
+require 'core/filtering/advanced_snippet/advanced_snippet_against_schema_validator'
 require 'connectors/mongodb/mongo_advanced_snippet_schema'
 
 module Connectors
   module MongoDB
-    class MongoAdvancedSnippetAgainstSchemaValidator < Connectors::Base::AdvancedSnippetAgainstSchemaValidator
+    class MongoAdvancedSnippetAgainstSchemaValidator < Core::Filtering::AdvancedSnippet::AdvancedSnippetAgainstSchemaValidator
 
       def initialize(advanced_snippet, schema = Connectors::MongoDB::AdvancedSnippet::SCHEMA)
         super

data/lib/connectors/mongodb/mongo_advanced_snippet_schema.rb

@@ -25,7 +25,7 @@ module Connectors
       NON_NEGATIVE_INTEGER = ->(value) { value.is_a?(Integer) && value >= 0 }
       READ_CONCERN_LEVEL = ->(level) { %w[local available majority linearizable].include?(level) }
       STRING_OR_DOCUMENT = ->(value) { value.is_a?(Hash) || value.is_a?(String) }
-      MUTUAL_EXCLUSIVE_FILTER = ->(fields) { fields.size <= 1 }
+      MUTUAL_EXCLUSIVE_FILTER = ->(fields) { fields.nil? || fields.size <= 1 }
 
       AGGREGATION_PIPELINE = lambda { |pipeline|
         return false unless pipeline.is_a?(Array)
@@ -265,7 +265,8 @@ module Connectors
 
       FIND_FILTER = {
         :name => 'filter',
-        :type => FILTER
+        :type => FILTER,
+        :optional => true
       }
 
       FIND = {