connectors_service 8.6.0.3 → 8.6.0.4.pre.20221114T233727Z

data/lib/connectors/mongodb/mongo_rules_parser.rb

@@ -0,0 +1,81 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'active_support/core_ext/object'
+require 'connectors/base/simple_rules_parser'
+require 'core/filtering/simple_rule'
+
+module Connectors
+  module MongoDB
+    class MongoRulesParser < Connectors::Base::SimpleRulesParser
+      def parse_rule(rule)
+        field = rule.field
+        value = rule.value
+        unless value.present?
+          raise "value is required for field: #{field}"
+        end
+        unless field.present?
+          raise "field is required for rule: #{rule}"
+        end
+        op = rule.rule
+        case op
+        when Core::Filtering::SimpleRule::Rule::EQUALS
+          parse_equals(rule)
+        when Core::Filtering::SimpleRule::Rule::GREATER_THAN
+          parse_greater_than(rule)
+        when Core::Filtering::SimpleRule::Rule::LESS_THAN
+          parse_less_than(rule)
+        when Core::Filtering::SimpleRule::Rule::REGEX
+          parse_regex(rule)
+        else
+          raise "Unknown operator: #{op}"
+        end
+      end
+
+      def merge_rules(rules)
+        return {} if rules.empty?
+        return rules[0] if rules.size == 1
+        { '$and' => rules }
+      end
+
+      private
+
+      def parse_equals(rule)
+        if rule.is_include?
+          { rule.field => rule.value }
+        else
+          { rule.field => { '$ne' => rule.value } }
+        end
+      end
+
+      def parse_greater_than(rule)
+        if rule.is_include?
+          { rule.field => { '$gt' => rule.value } }
+        else
+          { rule.field => { '$lte' => rule.value } }
+        end
+      end
+
+      def parse_less_than(rule)
+        if rule.is_include?
+          { rule.field => { '$lt' => rule.value } }
+        else
+          { rule.field => { '$gte' => rule.value } }
+        end
+      end
+
+      def parse_regex(rule)
+        if rule.is_include?
+          { rule.field => /#{rule.value}/ }
+        else
+          { rule.field => { '$not' => /#{rule.value}/ } }
+        end
+      end
+    end
+  end
+end

data/lib/connectors/registry.rb

@@ -24,10 +24,10 @@ module Connectors
       @connectors[name]
     end
 
-    def connector(name, configuration)
+    def connector(name, configuration, job_description: {})
       klass = connector_class(name)
       if klass.present?
-        return klass.new(configuration: configuration)
+        return klass.new(configuration: configuration, job_description: job_description)
       end
       raise "Connector #{name} is not yet registered. You need to register it before use"
     end

data/lib/connectors/sync_status.rb

@@ -8,14 +8,33 @@
 
 module Connectors
   class SyncStatus
-    COMPLETED = 'completed'
+    PENDING     = 'pending'
     IN_PROGRESS = 'in_progress'
-    FAILED = 'failed'
+    CANCELING   = 'canceling'
+    CANCELED    = 'canceled'
+    SUSPENDED   = 'suspended'
+    COMPLETED   = 'completed'
+    ERROR       = 'error'
 
     STATUSES = [
-      COMPLETED,
+      PENDING,
       IN_PROGRESS,
-      FAILED
+      CANCELING,
+      CANCELED,
+      SUSPENDED,
+      COMPLETED,
+      ERROR
+    ]
+
+    PENDING_STATUES = [
+      PENDING,
+      SUSPENDED
+    ]
+
+    TERMINAL_STATUSES = [
+      CANCELED,
+      COMPLETED,
+      ERROR
     ]
   end
 end

data/lib/core/configuration.rb

@@ -23,9 +23,11 @@ module Core
             Utility::Logger.error("Couldn't find connector for service type #{connector_settings.service_type || service_type}")
             return
           end
-          configuration = connector_class.configurable_fields
+          configuration = connector_class.configurable_fields_indifferent_access
+          features = connector_class.kibana_features.each_with_object({}) { |feature, hsh| hsh[feature] = true }
           doc = {
-            :configuration => configuration
+            :configuration => configuration,
+            :features => features
           }
 
           doc[:service_type] = service_type if service_type && connector_settings.needs_service_type?

data/lib/core/connector_job.rb

@@ -0,0 +1,137 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'active_support/core_ext/hash/indifferent_access'
+require 'connectors/sync_status'
+require 'core/elastic_connector_actions'
+require 'utility'
+
+module Core
+  class ConnectorJob
+    DEFAULT_PAGE_SIZE = 100
+
+    # Error Classes
+    class ConnectorJobNotFoundError < StandardError; end
+
+    def self.fetch_by_id(job_id)
+      es_response = ElasticConnectorActions.get_job(job_id)
+
+      raise ConnectorJobNotFoundError.new("Connector job with id=#{job_id} was not found.") unless es_response[:found]
+      new(es_response)
+    end
+
+    def self.pending_jobs(page_size = DEFAULT_PAGE_SIZE)
+      query = { terms: { status: Connectors::SyncStatus::PENDING_STATUES } }
+      fetch_jobs_by_query(query, page_size)
+    end
+
+    def self.orphaned_jobs(_page_size = DEFAULT_PAGE_SIZE)
+      []
+    end
+
+    def self.stuck_jobs(_page_size = DEFAULT_PAGE_SIZE)
+      []
+    end
+
+    def self.enqueue(_connector_id)
+      nil
+    end
+
+    def id
+      @elasticsearch_response[:_id]
+    end
+
+    def [](property_name)
+      @elasticsearch_response[:_source][property_name]
+    end
+
+    def status
+      self[:status]
+    end
+
+    def in_progress?
+      status == Connectors::SyncStatus::IN_PROGRESS
+    end
+
+    def canceling?
+      status == Connectors::SyncStatus::CANCELING
+    end
+
+    def connector_snapshot
+      self[:connector]
+    end
+
+    def connector_id
+      connector_snapshot[:id]
+    end
+
+    def index_name
+      connector_snapshot[:configuration]
+    end
+
+    def language
+      connector_snapshot[:language]
+    end
+
+    def service_type
+      connector_snapshot[:service_type]
+    end
+
+    def configuration
+      connector_snapshot[:configuration]
+    end
+
+    def filtering
+      Utility::Filtering.extract_filter(connector_snapshot[:filtering])
+    end
+
+    def pipeline
+      connector_snapshot[:pipeline]
+    end
+
+    def connector
+      @connector ||= ConnectorSettings.fetch_by_id(connector_id)
+    end
+
+    def reload_connector!
+      @connector = nil
+      connector
+    end
+
+    def reload
+      es_response = ElasticConnectorActions.get_job(id)
+      raise ConnectorJobNotFoundError.new("Connector job with id=#{id} was not found.") unless es_response[:found]
+      # TODO: remove the usage of with_indifferent_access. get_id method is expected to return a hash
+      @elasticsearch_response = es_response.with_indifferent_access
+      @connector = nil
+    end
+
+    private
+
+    def initialize(es_response)
+      # TODO: remove the usage of with_indifferent_access. The initialize method should expect a hash argument
+      @elasticsearch_response = es_response.with_indifferent_access
+    end
+
+    def self.fetch_jobs_by_query(query, page_size)
+      results = []
+      offset = 0
+      loop do
+        response = ElasticConnectorActions.search_jobs(query, page_size, offset)
+
+        hits = response.dig('hits', 'hits') || []
+        total = response.dig('hits', 'total', 'value') || 0
+        results += hits.map { |hit| new(hit) }
+        break if results.size >= total
+        offset += hits.size
+      end
+
+      results
+    end
+  end
+end

data/lib/core/connector_settings.rb

@@ -8,6 +8,7 @@
 
 require 'active_support/core_ext/hash/indifferent_access'
 require 'connectors/connector_status'
+require 'connectors/registry'
 require 'core/elastic_connector_actions'
 require 'utility'
 
@@ -19,6 +20,8 @@ module Core
     DEFAULT_REDUCE_WHITESPACE = true
     DEFAULT_RUN_ML_INFERENCE = true
 
+    DEFAULT_FILTERING = {}
+
     DEFAULT_PAGE_SIZE = 100
 
     # Error Classes
@@ -32,13 +35,15 @@ module Core
       new(es_response, connectors_meta)
     end
 
-    def initialize(es_response, connectors_meta)
-      @elasticsearch_response = es_response.with_indifferent_access
-      @connectors_meta = connectors_meta.with_indifferent_access
-    end
-
     def self.fetch_native_connectors(page_size = DEFAULT_PAGE_SIZE)
-      query = { term: { is_native: true } }
+      query = {
+        bool: {
+          filter: [
+            { term: { is_native: true } },
+            { terms: { service_type: Connectors::REGISTRY.registered_connectors } }
+          ]
+        }
+      }
       fetch_connectors_by_query(query, page_size)
     end
 
@@ -80,20 +85,27 @@ module Core
       self[:scheduling]
     end
 
+    def filtering
+      # assume for now, that first object in filtering array or a filter object itself is the only filtering object
+      filtering = @elasticsearch_response.dig(:_source, :filtering)
+
+      Utility::Filtering.extract_filter(filtering)
+    end
+
     def request_pipeline
-      return_if_present(@elasticsearch_response.dig(:pipeline, :name), @connectors_meta.dig(:pipeline, :default_name), DEFAULT_REQUEST_PIPELINE)
+      Utility::Common.return_if_present(@elasticsearch_response.dig(:_source, :pipeline, :name), @connectors_meta.dig(:pipeline, :default_name), DEFAULT_REQUEST_PIPELINE)
     end
 
     def extract_binary_content?
-      return_if_present(@elasticsearch_response.dig(:pipeline, :extract_binary_content), @connectors_meta.dig(:pipeline, :default_extract_binary_content), DEFAULT_EXTRACT_BINARY_CONTENT)
+      Utility::Common.return_if_present(@elasticsearch_response.dig(:_source, :pipeline, :extract_binary_content), @connectors_meta.dig(:pipeline, :default_extract_binary_content), DEFAULT_EXTRACT_BINARY_CONTENT)
     end
 
     def reduce_whitespace?
-      return_if_present(@elasticsearch_response.dig(:pipeline, :reduce_whitespace), @connectors_meta.dig(:pipeline, :default_reduce_whitespace), DEFAULT_REDUCE_WHITESPACE)
+      Utility::Common.return_if_present(@elasticsearch_response.dig(:_source, :pipeline, :reduce_whitespace), @connectors_meta.dig(:pipeline, :default_reduce_whitespace), DEFAULT_REDUCE_WHITESPACE)
     end
 
     def run_ml_inference?
-      return_if_present(@elasticsearch_response.dig(:pipeline, :run_ml_inference), @connectors_meta.dig(:pipeline, :default_run_ml_inference), DEFAULT_RUN_ML_INFERENCE)
+      Utility::Common.return_if_present(@elasticsearch_response.dig(:_source, :pipeline, :run_ml_inference), @connectors_meta.dig(:pipeline, :default_run_ml_inference), DEFAULT_RUN_ML_INFERENCE)
     end
 
     def formatted
@@ -112,6 +124,11 @@ module Core
 
     private
 
+    def initialize(es_response, connectors_meta)
+      @elasticsearch_response = es_response.with_indifferent_access
+      @connectors_meta = connectors_meta.with_indifferent_access
+    end
+
     def self.fetch_connectors_by_query(query, page_size)
       connectors_meta = ElasticConnectorActions.connectors_meta
 
@@ -120,8 +137,8 @@ module Core
       loop do
         response = ElasticConnectorActions.search_connectors(query, page_size, offset)
 
-        hits = response['hits']['hits']
-        total = response['hits']['total']['value']
+        hits = response.dig('hits', 'hits') || []
+        total = response.dig('hits', 'total', 'value') || 0
         results += hits.map do |hit|
           Core::ConnectorSettings.new(hit, connectors_meta)
         end
@@ -132,11 +149,5 @@ module Core
       results
     end
 
-    def return_if_present(*args)
-      args.each do |arg|
-        return arg unless arg.nil?
-      end
-      nil
-    end
   end
 end