connectors_service 8.5.0.1

data/lib/connectors/crawler/scheduler.rb

@@ -0,0 +1,32 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'core/scheduler'
+require 'core/connector_settings'
+require 'core/elastic_connector_actions'
+require 'utility/logger'
+require 'utility/exception_tracking'
+
+module Connectors
+  module Crawler
+    class Scheduler < Core::Scheduler
+      def connector_settings
+        Core::ConnectorSettings.fetch_crawler_connectors || []
+      rescue StandardError => e
+        Utility::ExceptionTracking.log_exception(e, 'Could not retrieve Crawler connectors due to unexpected error.')
+        []
+      end
+
+      private
+
+      def connector_registered?(service_type)
+        service_type == 'elastic-crawler'
+      end
+    end
+  end
+end

data/lib/connectors/example/connector.rb

@@ -0,0 +1,57 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'connectors/base/connector'
+require 'utility'
+
+module Connectors
+  module Example
+    class Connector < Connectors::Base::Connector
+      def self.service_type
+        'example'
+      end
+
+      def self.display_name
+        'Example Connector'
+      end
+
+      def self.configurable_fields
+        {
+          'foo' => {
+            'label' => 'Foo',
+            'value' => nil
+          }
+        }
+      end
+
+      def initialize(configuration: {})
+        super
+      end
+
+      def do_health_check
+        # Do the health check by trying to access 3rd-party system just to verify that everything is set up properly.
+        #
+        # To emulate unhealthy 3rd-party system situation, uncomment the following line:
+        # raise 'something went wrong'
+      end
+
+      def yield_documents
+        attachments = [
+          File.open('./lib/connectors/example/example_attachments/first_attachment.txt'),
+          File.open('./lib/connectors/example/example_attachments/second_attachment.txt'),
+          File.open('./lib/connectors/example/example_attachments/third_attachment.txt')
+        ]
+
+        attachments.each_with_index do |att, index|
+          data = { id: (index + 1).to_s, name: "example document #{index + 1}", _attachment: File.read(att) }
+          yield data
+        end
+      end
+    end
+  end
+end

data/lib/connectors/example/example_attachments/first_attachment.txt

@@ -0,0 +1 @@
+e1xydGYxXGFuc2lcYW5zaWNwZzEyNTJcY29jb2FydGYyNjM5Clxjb2NvYXRleHRzY2FsaW5nMFxjb2NvYXBsYXRmb3JtMHtcZm9udHRibFxmMFxmc3dpc3NcZmNoYXJzZXQwIEhlbHZldGljYS1Cb2xkO1xmMVxmc3dpc3NcZmNoYXJzZXQwIEhlbHZldGljYTtcZjJcZnN3aXNzXGZjaGFyc2V0MCBIZWx2ZXRpY2EtT2JsaXF1ZTsKfQp7XGNvbG9ydGJsO1xyZWQyNTVcZ3JlZW4yNTVcYmx1ZTI1NTt9CntcKlxleHBhbmRlZGNvbG9ydGJsOzt9ClxwYXBlcncxMTkwMFxwYXBlcmgxNjg0MFxtYXJnbDE0NDBcbWFyZ3IxNDQwXHZpZXd3MTE1MjBcdmlld2g4NDAwXHZpZXdraW5kMApccGFyZFx0eDU2Nlx0eDExMzNcdHgxNzAwXHR4MjI2N1x0eDI4MzRcdHgzNDAxXHR4Mzk2OFx0eDQ1MzVcdHg1MTAyXHR4NTY2OVx0eDYyMzZcdHg2ODAzXHBhcmRpcm5hdHVyYWxccGFydGlnaHRlbmZhY3RvcjAKClxmMFxiXGZzMjQgXGNmMCBFeGFtcGxlIEF0dGFjaG1lbnQgMDFcClwKClxmMVxiMCBUaGlzIGlzIHRoZQpcZjJcaSAgZmlyc3QgClxmMVxpMCBvZiBcdWwgdGhyZWUgXHVsbm9uZSBleGFtcGxlIGF0dGFjaG1lbnRzfQ==

data/lib/connectors/example/example_attachments/second_attachment.txt

@@ -0,0 +1 @@
+e1xydGYxXGFuc2lcYW5zaWNwZzEyNTJcY29jb2FydGYyNjM5Clxjb2NvYXRleHRzY2FsaW5nMFxjb2NvYXBsYXRmb3JtMHtcZm9udHRibFxmMFxmc3dpc3NcZmNoYXJzZXQwIEhlbHZldGljYS1Cb2xkO1xmMVxmc3dpc3NcZmNoYXJzZXQwIEhlbHZldGljYTtcZjJcZnN3aXNzXGZjaGFyc2V0MCBIZWx2ZXRpY2EtT2JsaXF1ZTsKfQp7XGNvbG9ydGJsO1xyZWQyNTVcZ3JlZW4yNTVcYmx1ZTI1NTt9CntcKlxleHBhbmRlZGNvbG9ydGJsOzt9ClxwYXBlcncxMTkwMFxwYXBlcmgxNjg0MFxtYXJnbDE0NDBcbWFyZ3IxNDQwXHZpZXd3MTE1MjBcdmlld2g4NDAwXHZpZXdraW5kMApccGFyZFx0eDU2Nlx0eDExMzNcdHgxNzAwXHR4MjI2N1x0eDI4MzRcdHgzNDAxXHR4Mzk2OFx0eDQ1MzVcdHg1MTAyXHR4NTY2OVx0eDYyMzZcdHg2ODAzXHBhcmRpcm5hdHVyYWxccGFydGlnaHRlbmZhY3RvcjAKClxmMFxiXGZzMjQgXGNmMCBFeGFtcGxlIEF0dGFjaG1lbnQgMDJcClwKClxmMVxiMCBUaGlzIGlzIHRoZQpcZjJcaSAgc2Vjb25kIApcZjFcaTAgb2YgXHVsIHRocmVlIFx1bG5vbmUgZXhhbXBsZSBhdHRhY2htZW50c30=

data/lib/connectors/example/example_attachments/third_attachment.txt

@@ -0,0 +1 @@
+e1xydGYxXGFuc2lcYW5zaWNwZzEyNTJcY29jb2FydGYyNjM5Clxjb2NvYXRleHRzY2FsaW5nMFxjb2NvYXBsYXRmb3JtMHtcZm9udHRibFxmMFxmc3dpc3NcZmNoYXJzZXQwIEhlbHZldGljYS1Cb2xkO1xmMVxmc3dpc3NcZmNoYXJzZXQwIEhlbHZldGljYTtcZjJcZnN3aXNzXGZjaGFyc2V0MCBIZWx2ZXRpY2EtT2JsaXF1ZTsKfQp7XGNvbG9ydGJsO1xyZWQyNTVcZ3JlZW4yNTVcYmx1ZTI1NTt9CntcKlxleHBhbmRlZGNvbG9ydGJsOzt9ClxwYXBlcncxMTkwMFxwYXBlcmgxNjg0MFxtYXJnbDE0NDBcbWFyZ3IxNDQwXHZpZXd3MTE1MjBcdmlld2g4NDAwXHZpZXdraW5kMApccGFyZFx0eDU2Nlx0eDExMzNcdHgxNzAwXHR4MjI2N1x0eDI4MzRcdHgzNDAxXHR4Mzk2OFx0eDQ1MzVcdHg1MTAyXHR4NTY2OVx0eDYyMzZcdHg2ODAzXHBhcmRpcm5hdHVyYWxccGFydGlnaHRlbmZhY3RvcjAKClxmMFxiXGZzMjQgXGNmMCBFeGFtcGxlIEF0dGFjaG1lbnQgMDNcClwKClxmMVxiMCBUaGlzIGlzIHRoZQpcZjJcaSAgdGhpcmQgClxmMVxpMCBvZiBcdWwgdGhyZWUgXHVsbm9uZSBleGFtcGxlIGF0dGFjaG1lbnRzfQ==

data/lib/connectors/gitlab/adapter.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+require 'hashie/mash'
+require 'active_support/core_ext/hash/indifferent_access'
+require 'connectors/base/adapter'
+
+module Connectors
+  module GitLab
+    class Adapter < Connectors::Base::Adapter
+      # it's important to have this to generate ID converters between the GitLab ID and the
+      # Enterprise Search document ID. The Enterprise Search document ID will be prefixed with the service type,
+      # in our case - `gitlab`.
+      generate_id_helpers :gitlab, 'gitlab'
+
+      def self.to_es_document(type, source_doc)
+        source_doc = source_doc.with_indifferent_access
+        result = {}
+        case type.to_sym
+        when :project
+          result.merge!(
+            {
+              :url => source_doc[:web_url],
+              :body => source_doc[:description],
+              :title => source_doc[:name],
+              :created_at => source_doc[:created_at],
+              :last_modified_at => source_doc[:last_activity_at],
+              :visibility => source_doc[:visibility],
+              :namespace => if source_doc[:namespace].nil?
+                              nil
+                            else
+                              source_doc[:namespace][:name]
+                            end
+            }
+          )
+        else
+          # don't remap
+          result.merge!(source_doc)
+        end
+        result[:id] = gitlab_id_to_es_id(source_doc[:id])
+        result[:type] = type
+        result
+      end
+    end
+  end
+end

data/lib/connectors/gitlab/connector.rb

@@ -0,0 +1,67 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+require 'active_support/core_ext/hash/indifferent_access'
+
+require 'connectors/base/connector'
+require 'connectors/gitlab/extractor'
+require 'connectors/gitlab/custom_client'
+require 'connectors/gitlab/adapter'
+require 'core/output_sink'
+
+module Connectors
+  module GitLab
+    class Connector < Connectors::Base::Connector
+      def self.service_type
+        'gitlab'
+      end
+
+      def self.display_name
+        'GitLab Connector'
+      end
+
+      def self.configurable_fields
+        {
+          :base_url => {
+            :label => 'Base URL',
+            :value => Connectors::GitLab::DEFAULT_BASE_URL
+          },
+          :api_key => {
+             :label => 'API Key'
+          }
+        }
+      end
+
+      def initialize(configuration: {})
+        super
+
+        @extractor = Connectors::GitLab::Extractor.new(
+          :base_url => configuration.dig(:base_url, :value),
+          :api_token => configuration.dig(:api_token, :value)
+        )
+      end
+
+      def yield_documents
+        next_page_link = nil
+        loop do
+          next_page_link = @extractor.yield_projects_page(next_page_link) do |projects_chunk|
+            projects_chunk.each do |project|
+              yield Connectors::GitLab::Adapter.to_es_document(:project, project)
+            end
+          end
+          break unless next_page_link.present?
+        end
+      end
+
+      private
+
+      def do_health_check
+        @extractor.health_check
+      end
+    end
+  end
+end

data/lib/connectors/gitlab/custom_client.rb

@@ -0,0 +1,44 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+require 'faraday_middleware/response/follow_redirects'
+require 'connectors/base/custom_client'
+require 'utility/middleware/bearer_auth'
+require 'utility/middleware/basic_auth'
+require 'utility/middleware/restrict_hostnames'
+
+require 'app/config'
+
+module Connectors
+  module GitLab
+    DEFAULT_BASE_URL = 'https://gitlab.com/api/v4'
+
+    class CustomClient < Connectors::Base::CustomClient
+      attr_reader :api_token
+
+      class ClientError < StandardError
+        attr_reader :status_code, :endpoint, :api_token
+
+        def initialize(status_code, endpoint)
+          @status_code = status_code
+          @endpoint = endpoint
+        end
+      end
+
+      def initialize(base_url:, api_token:, ensure_fresh_auth: nil)
+        @api_token = api_token
+        super(:base_url => base_url || DEFAULT_BASE_URL, :ensure_fresh_auth => ensure_fresh_auth)
+      end
+
+      def additional_middleware
+        [
+          ::FaradayMiddleware::FollowRedirects,
+          [Utility::Middleware::RestrictHostnames, { :allowed_hosts => [base_url, DEFAULT_BASE_URL] }],
+          [Utility::Middleware::BearerAuth, { :bearer_auth_token => api_token }]
+        ]
+      end
+    end
+  end
+end

data/lib/connectors/gitlab/extractor.rb

@@ -0,0 +1,69 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'hashie'
+require 'json'
+require 'rack/utils'
+require 'active_support/core_ext/hash/indifferent_access'
+require 'connectors/gitlab/custom_client'
+
+module Connectors
+  module GitLab
+    class Extractor
+      PAGE_SIZE = 100 # max is 100
+
+      def initialize(base_url: nil, api_token: nil, owned_only: true)
+        super()
+        @base_url = base_url
+        @api_token = api_token
+        # only get projects that user owns
+        @owned_only = owned_only
+      end
+
+      def yield_projects_page(next_page_link = nil)
+        query_params = {
+          :pagination => :keyset,
+          :per_page => PAGE_SIZE,
+          :order_by => :id,
+          :sort => :desc,
+          :owned => @owned_only
+        }
+
+        if next_page_link.present?
+          if (matcher = /(https?:[^>]*)/.match(next_page_link))
+            clean_query = URI.parse(matcher.captures[0]).query
+            query_params = Rack::Utils.parse_query(clean_query)
+          else
+            raise "Next page link has unexpected format: #{next_page_link}"
+          end
+        end
+        response = client.get('projects', query_params)
+
+        projects_chunk = JSON.parse(response.body)
+        yield projects_chunk
+
+        # return next link
+        response.headers['Link'] || nil
+      end
+
+      def health_check
+        # let's do a simple call to get the current user
+        response = client.get('user')
+        unless response.present? && response.status == 200
+          raise "Health check failed with response status #{response.status} and body #{response.body}"
+        end
+      end
+
+      private
+
+      def client
+        @client ||= Connectors::GitLab::CustomClient.new(base_url: @base_url, api_token: @api_token)
+      end
+    end
+  end
+end

data/lib/connectors/mongodb/connector.rb

@@ -0,0 +1,138 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'active_support/core_ext/hash/indifferent_access'
+require 'connectors/base/connector'
+require 'mongo'
+
+module Connectors
+  module MongoDB
+    class Connector < Connectors::Base::Connector
+      def self.service_type
+        'mongodb'
+      end
+
+      def self.display_name
+        'MongoDB'
+      end
+
+      def self.configurable_fields
+        {
+           :host => {
+             :label => 'Server Hostname'
+           },
+           :user => {
+             :label => 'Username'
+           },
+           :password => {
+             :label => 'Password'
+           },
+           :database => {
+             :label => 'Database'
+           },
+           :collection => {
+             :label => 'Collection'
+           },
+           :direct_connection => {
+             :label => 'Direct connection? (true/false)'
+           }
+        }
+      end
+
+      def initialize(configuration: {})
+        super
+
+        @host = configuration.dig(:host, :value)
+        @database = configuration.dig(:database, :value)
+        @collection = configuration.dig(:collection, :value)
+        @user = configuration.dig(:user, :value)
+        @password = configuration.dig(:password, :value)
+        @direct_connection = configuration.dig(:direct_connection, :value)
+      end
+
+      def yield_documents
+        with_client do |client|
+          client[@collection].find.each do |document|
+            doc = document.with_indifferent_access
+
+            yield serialize(doc)
+          end
+        end
+      end
+
+      private
+
+      def do_health_check
+        with_client do |_client|
+          Utility::Logger.debug("Mongo at #{@host}/#{@database} looks healthy.")
+        end
+      end
+
+      def with_client
+        raise "Invalid value for 'Direct connection' : #{@direct_connection}." unless %w[true false].include?(@direct_connection.to_s.strip.downcase)
+
+        client = if @user.present? || @password.present?
+                   Mongo::Client.new(
+                     @host,
+                     database: @database,
+                     direct_connection: to_boolean(@direct_connection),
+                     user: @user,
+                     password: @password
+                   )
+                 else
+                   Mongo::Client.new(
+                     @host,
+                     database: @database,
+                     direct_connection: to_boolean(@direct_connection)
+                   )
+                 end
+
+        begin
+          Utility::Logger.debug("Existing Databases #{client.database_names}")
+          Utility::Logger.debug('Existing Collections:')
+
+          client.collections.each { |coll| Utility::Logger.debug(coll.name) }
+
+          yield client
+        ensure
+          client.close
+        end
+      end
+
+      def serialize(mongodb_document)
+        # This is some lazy serialization here.
+        # Problem: MongoDB has its own format of things - e.g. ids are Bson::ObjectId, which when serialized to JSON
+        # will produce something like: 'id': { '$oid': '536268a06d2d7019ba000000' }, which is not good for us
+        case mongodb_document
+        when BSON::ObjectId
+          mongodb_document.to_s
+        when BSON::Decimal128
+          mongodb_document.to_big_decimal # potential problems with NaNs but also will get treated as a string by Elasticsearch anyway
+        when String
+          # it's here cause Strings are Arrays too :/
+          mongodb_document.to_s
+        when Array
+          mongodb_document.map { |v| serialize(v) }
+        when Hash
+          mongodb_document.map do |key, value|
+            remapped_key = key == '_id' ? 'id' : key
+
+            remapped_value = serialize(value)
+            [remapped_key, remapped_value]
+          end.to_h
+        else
+          mongodb_document
+        end
+      end
+
+      def to_boolean(value)
+        value == true || value =~ (/(true|t|yes|y|1)$/i)
+      end
+    end
+  end
+end

data/lib/connectors/registry.rb

@@ -0,0 +1,52 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+module Connectors
+  class Factory
+    attr_reader :connectors
+
+    def initialize
+      @connectors = {}
+    end
+
+    def register(name, klass)
+      @connectors[name] = klass
+    end
+
+    def registered?(name)
+      @connectors.has_key?(name)
+    end
+
+    def connector_class(name)
+      @connectors[name]
+    end
+
+    def connector(name, configuration)
+      klass = connector_class(name)
+      if klass.present?
+        return klass.new(configuration: configuration)
+      end
+      raise "Connector #{name} is not yet registered. You need to register it before use"
+    end
+
+    def registered_connectors
+      @connectors.keys.sort
+    end
+  end
+
+  REGISTRY = Factory.new
+
+  require_relative './example/connector'
+  REGISTRY.register(Connectors::Example::Connector.service_type, Connectors::Example::Connector)
+
+  # loading plugins (might replace this with a directory scan and conventions on names)
+  require_relative './gitlab/connector'
+
+  REGISTRY.register(Connectors::GitLab::Connector.service_type, Connectors::GitLab::Connector)
+
+  require_relative 'mongodb/connector'
+  REGISTRY.register(Connectors::MongoDB::Connector.service_type, Connectors::MongoDB::Connector)
+end

data/lib/connectors/sync_status.rb

@@ -0,0 +1,21 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+module Connectors
+  class SyncStatus
+    COMPLETED = 'completed'
+    IN_PROGRESS = 'in_progress'
+    FAILED = 'failed'
+
+    STATUSES = [
+      COMPLETED,
+      IN_PROGRESS,
+      FAILED
+    ]
+  end
+end

data/lib/connectors.rb

@@ -0,0 +1,16 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+require 'utility'
+
+def required_path(absolute_path)
+  absolute_dir = File.dirname(absolute_path)
+  relative_dir = absolute_dir.sub(/.*lib\/connectors/, 'connectors')
+  name = File.basename(absolute_path, '.rb')
+  File.join(relative_dir, name)
+end
+
+Dir[File.join(__dir__, 'connectors/**/*.rb')].each { |f| require required_path(f) }

data/lib/connectors_app///

@@ -0,0 +1,13 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+module ConnectorsApp
+  module Errors
+    INVALID_API_KEY = 'INVALID_API_KEY'
+    UNSUPPORTED_AUTH_SCHEME = 'UNSUPPORTED_AUTH_SCHEME'
+    INTERNAL_SERVER_ERROR = 'INTERNAL_SERVER_ERROR'
+  end
+end

data/lib/connectors_service.rb

@@ -0,0 +1,24 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'app/config'
+require 'app/dispatcher'
+require 'app/preflight_check'
+require 'utility'
+
+class ConnectorsService
+  def self.run!
+    Utility::Environment.set_execution_environment(App::Config) do
+      App::PreflightCheck.run!
+      App::Dispatcher.start!
+    rescue App::PreflightCheck::CheckFailure => e
+      Utility::Logger.error("Preflight check failed: #{e.message}")
+      exit(-1)
+    end
+  end
+end

data/lib/connectors_utility.rb

@@ -0,0 +1,16 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require_relative 'utility'
+
+require_relative 'connectors/connector_status'
+require_relative 'connectors/sync_status'
+require_relative 'core/scheduler'
+require_relative 'core/elastic_connector_actions'
+
+require_relative 'connectors/crawler/scheduler'

data/lib/core/configuration.rb

@@ -0,0 +1,48 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'connectors/connector_status'
+require 'connectors/registry'
+require 'core/connector_settings'
+require 'core/elastic_connector_actions'
+require 'utility/logger'
+
+module Core
+  class Configuration
+    class << self
+
+      def update(connector_settings, service_type = nil)
+        if connector_settings.connector_status == Connectors::ConnectorStatus::CREATED
+          connector_class = Connectors::REGISTRY.connector_class(connector_settings.service_type || service_type)
+          unless connector_class
+            Utility::Logger.error("Couldn't find connector for service type #{connector_settings.service_type || service_type}")
+            return
+          end
+          configuration = connector_class.configurable_fields
+          doc = {
+            :configuration => configuration
+          }
+
+          doc[:service_type] = service_type if service_type && connector_settings.needs_service_type?
+
+          # We want to set connector to CONFIGURED status if all configurable fields have default values
+          new_connector_status = if configuration.values.all? { |setting| setting[:value].present? }
+                                   Utility::Logger.debug("All connector configurable fields provided default values for #{connector_settings.formatted}.")
+                                   Connectors::ConnectorStatus::CONFIGURED
+                                 else
+                                   Connectors::ConnectorStatus::NEEDS_CONFIGURATION
+                                 end
+
+          doc[:status] = new_connector_status
+          Utility::Logger.info("Changing connector status to #{new_connector_status} for #{connector_settings.formatted}.")
+          Core::ElasticConnectorActions.update_connector_fields(connector_settings.id, doc)
+        end
+      end
+    end
+  end
+end