connectors_sdk 8.3.0.0.pre.20220517T144653Z → 8.3.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b86ca5e489e3cef3b9f2c04a462baf71d6b43805731b0cb52ba2f56f5078d6d3
-  data.tar.gz: 044e860f11163e82c63f66276c3d5628b761c5dfcc5168fc8f80b63ca87d19f0
+  metadata.gz: 5e3f22d66affcaf3b17f9f7a42b6a782b43b612f7a0b8ee0ab5115f23305ff6d
+  data.tar.gz: 8d6a20fbaab9e04c0269f0afe520897be36c07053f0b6b56b257863ef66ad35d
 SHA512:
-  metadata.gz: 528fa5260cf80a3ebb918478e1be2e7cac1668588853ad72b1259095086090c15a0cbc028523b1de8b775b3b64ed7427c6fab7ffccc562d559c8164456b84c4b
-  data.tar.gz: a58d353e2b48ffda33aa287d0fa6bb1400c531aae2794995ff3b644b779426d10bb3efc8735d8e07c353fae64bc0b3b0af58427d989f499210a19244b23c1a35
+  metadata.gz: 9c87a941dcbb513f007bba44806a386ec1525cbb28863877fdeb9e57467e7049d05c971db6642cf29fa4e0910c007e12de97ae782f855e84d16d3748af5467a9
+  data.tar.gz: 46f89335d22031d52967ccdf5ad3e5b19021df9d689a5206820ae99a81cf99a6689deeafd8456d9072dcde9e597f14259d7cfc5c4f59bcc4075625e0fb33aa91

data/lib/connectors_sdk/atlassian/custom_client.rb

@@ -9,6 +9,7 @@
 require 'faraday_middleware'
 
 require 'connectors_shared/middleware/bearer_auth'
+require 'connectors_shared/middleware/basic_auth'
 require 'connectors_shared/middleware/restrict_hostnames'
 require 'connectors_sdk/base/custom_client'
 
@@ -30,10 +31,12 @@ module ConnectorsSdk
 
       MEDIA_API_BASE_URL = 'https://api.media.atlassian.com'
 
-      attr_reader :base_url, :access_token
+      attr_reader :base_url, :access_token, :basic_auth_token
 
-      def initialize(base_url:, access_token:, ensure_fresh_auth: nil)
+      def initialize(base_url:, access_token: nil, basic_auth_token: nil, ensure_fresh_auth: nil)
+        raise 'Either access_token or basic_auth_token must be provided' unless access_token.present? || basic_auth_token.present?
         @access_token = access_token
+        @basic_auth_token = basic_auth_token
         super(:base_url => base_url, :ensure_fresh_auth => ensure_fresh_auth)
       end
 
@@ -42,11 +45,17 @@ module ConnectorsSdk
       end
 
       def additional_middleware
-        [
-            ::FaradayMiddleware::FollowRedirects,
-            [ConnectorsShared::Middleware::RestrictHostnames, { :allowed_hosts => [base_url, MEDIA_API_BASE_URL] }],
-            [ConnectorsShared::Middleware::BearerAuth, { :bearer_auth_token => @access_token }]
+        result = [
+          FaradayMiddleware::FollowRedirects,
+          [ConnectorsShared::Middleware::RestrictHostnames, { :allowed_hosts => [base_url, MEDIA_API_BASE_URL] }]
         ]
+        if @access_token.present?
+          result.append([ConnectorsShared::Middleware::BearerAuth, { :bearer_auth_token => @access_token }])
+        elsif @basic_auth_token.present?
+          result.append([ConnectorsShared::Middleware::BasicAuth, { :basic_auth_token => @basic_auth_token }])
+        else
+          raise 'Either access token or basic auth must be provided'
+        end
       end
 
       def update_auth_data!(new_access_token)

data/lib/connectors_sdk/base/{http_call_wrapper.rb → connector.rb}

@@ -10,13 +10,25 @@ require 'bson'
 
 module ConnectorsSdk
   module Base
-    class HttpCallWrapper
+    class Connector
       def extractor(params)
+        content_source_id = params.fetch(:content_source_id)
+        secret_storage = params[:secret_storage]
+
         extractor_class.new(
-          content_source_id: params[:content_source_id] || "GENERATED-#{BSON::ObjectId.new}",
+          content_source_id: content_source_id || "GENERATED-#{BSON::ObjectId.new}",
           service_type: service_type,
-          authorization_data_proc: proc { { access_token: params[:access_token] } },
-          client_proc: proc { client(params) },
+          authorization_data_proc: proc do
+            secret = secret_storage.fetch_secret(content_source_id)
+            {
+              access_token: secret[:access_token]
+            }
+          end,
+          client_proc: proc {
+            secret = secret_storage.fetch_secret(content_source_id)
+            params[:access_token] = secret[:access_token]
+            client(params)
+          },
           config: config(params),
           features: params.fetch(:features, {}) || {}
         )
@@ -68,6 +80,10 @@ module ConnectorsSdk
         end
       end
 
+      def download(params)
+        extractor(params).download(params[:meta])
+      end
+
       def authorization_uri(params)
         authorization.authorization_uri(params)
       end
@@ -80,18 +96,18 @@ module ConnectorsSdk
         authorization.refresh(params)
       end
 
-      def download(params)
-        extractor(params).download(params[:meta])
-      end
-
       def source_status(params)
         health_check(params)
-        { :status => 'OK', :statusCode => 200, :message => "Connected to #{name}" }
+        { :status => 'OK', :statusCode => 200, :message => "Connected to #{display_name}" }
       rescue StandardError => e
         { :status => 'FAILURE', :statusCode => e.is_a?(custom_client_error) ? e.status_code : 500, :message => e.message }
       end
 
-      def name
+      def compare_secrets(*)
+        raise 'Not implemented for this connector'
+      end
+
+      def display_name
         raise 'Not implemented for this connector'
       end
 
@@ -99,6 +115,14 @@ module ConnectorsSdk
         self.class::SERVICE_TYPE
       end
 
+      def connection_requires_redirect
+        false
+      end
+
+      def configurable_fields
+        []
+      end
+
       private
 
       def convert_third_party_errors
@@ -130,6 +154,13 @@ module ConnectorsSdk
       def health_check(*)
         raise 'Not implemented for this connector'
       end
+
+      def missing_secrets?(params)
+        missing = %w[secret other_secret].select { |field| params[field.to_sym].nil? }
+        unless missing.blank?
+          raise ConnectorsShared::ClientError.new("Missing required fields: #{missing.join(', ')}")
+        end
+      end
     end
   end
 end

data/lib/connectors_sdk/base/extractor.rb

@@ -188,16 +188,19 @@ module ConnectorsSdk
       end
 
       def permissions(source_user_id, &block)
+        result = []
         Connectors::Stats.measure("extractor.#{Connectors::Stats.class_key(self.class)}.permissions") do
           with_auth_tokens_and_retry do
             Connectors::Stats.measure("extractor.#{Connectors::Stats.class_key(self.class)}.yield_permissions") do
               yield_permissions(source_user_id) do |permissions|
                 log_info("Extracted #{permissions.size} permissions for source user #{source_user_id}")
+                result = permissions
                 block.call(permissions) if block_given?
               end
             end
           end
         end
+        result.each
       end
 
       ConnectorsShared::Logger::SUPPORTED_LOG_LEVELS.each do |log_level|

data/lib/connectors_sdk/base/registry.rb

@@ -24,11 +24,16 @@ module ConnectorsSdk
 
     REGISTRY = Factory.new
 
+    require_relative '../stub_connector/connector'
+    REGISTRY.register(ConnectorsSdk::StubConnector::Connector::SERVICE_TYPE, ConnectorsSdk::StubConnector::Connector)
+
     # loading plugins (might replace this with a directory scan and conventions on names)
-    require_relative '../share_point/http_call_wrapper'
-    require_relative '../confluence_cloud//http_call_wrapper'
+    require_relative '../share_point/connector'
+    require_relative '../confluence_cloud/connector'
+    require_relative '../gitlab/connector'
 
-    REGISTRY.register(ConnectorsSdk::SharePoint::HttpCallWrapper::SERVICE_TYPE, ConnectorsSdk::SharePoint::HttpCallWrapper)
-    REGISTRY.register(ConnectorsSdk::ConfluenceCloud::HttpCallWrapper::SERVICE_TYPE, ConnectorsSdk::ConfluenceCloud::HttpCallWrapper)
+    REGISTRY.register(ConnectorsSdk::SharePoint::Connector::SERVICE_TYPE, ConnectorsSdk::SharePoint::Connector)
+    REGISTRY.register(ConnectorsSdk::ConfluenceCloud::Connector::SERVICE_TYPE, ConnectorsSdk::ConfluenceCloud::Connector)
+    REGISTRY.register(ConnectorsSdk::GitLab::Connector::SERVICE_TYPE, ConnectorsSdk::GitLab::Connector)
   end
 end

data/lib/connectors_sdk/confluence/extractor.rb

@@ -82,7 +82,7 @@ module ConnectorsSdk
       private
 
       def content_base_url
-        'https://workplace-search.atlassian.net/wiki'
+        config.base_url
       end
 
       def yield_spaces

data/lib/connectors_sdk/confluence_cloud/connector.rb

@@ -0,0 +1,110 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'base64'
+require 'connectors_sdk/atlassian/config'
+require 'connectors_sdk/confluence_cloud/extractor'
+require 'connectors_sdk/confluence_cloud/authorization'
+require 'connectors_sdk/confluence_cloud/custom_client'
+require 'connectors_sdk/base/connector'
+
+module ConnectorsSdk
+  module ConfluenceCloud
+    class Connector < ConnectorsSdk::Base::Connector
+      SERVICE_TYPE = 'confluence_cloud'
+
+      def compare_secrets(params)
+        missing_secrets?(params)
+
+        {
+          :equivalent => params[:secret] == params[:other_secret]
+        }
+      end
+
+      def display_name
+        'Confluence Cloud'
+      end
+
+      def configurable_fields
+        [
+          {
+            'key' => 'base_url',
+            'label' => 'Confluence Cloud Base URL'
+          },
+          {
+            'key' => 'confluence_user_email',
+            'label' => 'Confluence user email'
+          },
+          {
+            'key' => 'confluence_api_token',
+            'label' => 'Confluence user REST API Token'
+          },
+        ]
+      end
+
+      private
+
+      def extractor_class
+        ConnectorsSdk::ConfluenceCloud::Extractor
+      end
+
+      def authorization
+        ConnectorsSdk::ConfluenceCloud::Authorization
+      end
+
+      def client(params)
+        ConnectorsSdk::ConfluenceCloud::CustomClient.new(
+          :base_url => extract_base_url(params),
+          :basic_auth_token => extract_basic_auth_token(params)
+        )
+      end
+
+      def custom_client_error
+        ConnectorsSdk::Atlassian::CustomClient::ClientError
+      end
+
+      def config(params)
+        ConnectorsSdk::Atlassian::Config.new(
+          :base_url => extract_base_url(params),
+          :cursors => params.fetch(:cursors, {}) || {},
+          :index_permissions => params[:index_permissions] || false
+        )
+      end
+
+      def health_check(params)
+        client(params).me
+      end
+
+      def base_url(cloud_id)
+        "https://api.atlassian.com/ex/confluence/#{cloud_id}"
+      end
+
+      def is_basic_auth(params)
+        login = params.fetch('confluence_user_email', nil)
+        api_token = params.fetch('confluence_api_token', nil)
+        login.present? && api_token.present?
+      end
+
+      def extract_basic_auth_token(params)
+        login = params.fetch('confluence_user_email', nil)
+        api_token = params.fetch('confluence_api_token', nil)
+        nil unless login.present? && api_token.present?
+        Base64.strict_encode64("#{login}:#{api_token}")
+      end
+
+      def extract_base_url(params)
+        # From Confluence API documentation:
+        # Requests that use OAuth 2.0 (3LO) are made via api.atlassian.com (not https://your-domain.atlassian.net).
+        if is_basic_auth(params)
+          return params[:base_url].end_with?('/wiki') ? params[:base_url] : "#{params[:base_url]}/wiki"
+        end
+        base_url(params[:cloud_id])
+      end
+    end
+  end
+end

data/lib/connectors_sdk/confluence_cloud/extractor.rb

@@ -52,7 +52,8 @@ module ConnectorsSdk
       def download(item)
         content = item[:content]
         parent_id = content.dig('container', 'id')
-        client.download("#{client.base_url}/wiki/rest/api/content/#{parent_id}/child/attachment/#{content['id']}/download").body
+        base_url = client.base_url.end_with?('/wiki') ? client.base_url : "#{client.base_url}/wiki"
+        client.download("#{base_url}/rest/api/content/#{parent_id}/child/attachment/#{content['id']}/download").body
       end
     end
   end

data/lib/connectors_sdk/gitlab/adapter.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+require 'hashie/mash'
+require 'connectors_sdk/base/adapter'
+
+module ConnectorsSdk
+  module GitLab
+    class Adapter < ConnectorsSdk::Base::Adapter
+      # it's important to have this to generate ID converters between the GitLab ID and the
+      # Enterprise Search document ID. The Enterprise Search document ID will be prefixed with the service type,
+      # in our case - `gitlab`.
+      generate_id_helpers :gitlab, 'gitlab'
+
+      def self.to_es_document(type, source_doc)
+        result = {
+          :id => gitlab_id_to_es_id(source_doc[:id]),
+          :type => type,
+          :url => source_doc[:web_url],
+          :body => source_doc[:description],
+          :title => source_doc[:name],
+          :created_at => source_doc[:created_at],
+          :last_modified_at => source_doc[:last_activity_at],
+          :visibility => source_doc[:visibility],
+          :namespace => if source_doc[:namespace].nil?
+                          nil
+                        else
+                          source_doc[:namespace][:name]
+                        end
+        }
+        if source_doc[:_allow_permissions].present?
+          result[:_allow_permissions] = source_doc[:_allow_permissions]
+        end
+        result
+      end
+    end
+  end
+end

data/lib/connectors_sdk/gitlab/config.rb

@@ -0,0 +1,26 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'connectors_sdk/base/config'
+
+module ConnectorsSdk
+  module GitLab
+    class Config < ConnectorsSdk::Base::Config
+      attr_reader :index_permissions
+
+      def initialize(cursors:, index_permissions: false)
+        super(:cursors => cursors)
+        @index_permissions = index_permissions || false
+      end
+
+      def to_h
+        super.merge(:index_permissions => index_permissions)
+      end
+    end
+  end
+end

data/lib/connectors_sdk/gitlab/connector.rb

@@ -0,0 +1,71 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'connectors_sdk/base/connector'
+require 'connectors_sdk/gitlab/custom_client'
+require 'connectors_sdk/gitlab/adapter'
+require 'connectors_sdk/gitlab/config'
+require 'connectors_sdk/gitlab/extractor'
+require 'rack/utils'
+
+module ConnectorsSdk
+  module GitLab
+    class Connector < ConnectorsSdk::Base::Connector
+      SERVICE_TYPE = 'gitlab'
+
+      def display_name
+        'GitLab Connector'
+      end
+
+      def configurable_fields
+        [
+          {
+            'key' => 'api_token',
+            'label' => 'API Token'
+          },
+          {
+            'key' => 'base_url',
+            'label' => 'Base URL'
+          }
+        ]
+      end
+
+      private
+
+      def client(params)
+        ConnectorsSdk::GitLab::CustomClient.new(
+          :base_url => params[:base_url] || ConnectorsSdk::GitLab::API_BASE_URL,
+          :api_token => params[:api_token]
+        )
+      end
+
+      def config(params)
+        ConnectorsSdk::GitLab::Config.new(
+          :cursors => params.fetch(:cursors, {}) || {},
+          :index_permissions => params.fetch(:index_permissions, false)
+        )
+      end
+
+      def extractor_class
+        ConnectorsSdk::GitLab::Extractor
+      end
+
+      def custom_client_error
+        ConnectorsSdk::GitLab::CustomClient::ClientError
+      end
+
+      def health_check(params)
+        # let's do a simple call
+        response = client(params).get('user')
+        unless response.present? && response.status == 200
+          raise "Health check failed with response status #{response.status} and body #{response.body}"
+        end
+      end
+    end
+  end
+end

data/lib/connectors_sdk/gitlab/custom_client.rb

@@ -0,0 +1,40 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+require 'faraday_middleware/response/follow_redirects'
+require 'connectors_sdk/base/custom_client'
+require 'connectors_shared/middleware/bearer_auth'
+require 'connectors_shared/middleware/basic_auth'
+require 'connectors_shared/middleware/restrict_hostnames'
+
+module ConnectorsSdk
+  module GitLab
+    API_BASE_URL = 'https://gitlab.com/api/v4'
+
+    class CustomClient < ConnectorsSdk::Base::CustomClient
+      class ClientError < ConnectorsShared::ClientError
+        attr_reader :status_code, :endpoint
+
+        def initialize(status_code, endpoint)
+          @status_code = status_code
+          @endpoint = endpoint
+        end
+      end
+
+      def initialize(base_url:, api_token:, ensure_fresh_auth: nil)
+        @api_token = api_token
+        super(:base_url => base_url, :ensure_fresh_auth => ensure_fresh_auth)
+      end
+
+      def additional_middleware
+        [
+          ::FaradayMiddleware::FollowRedirects,
+          [ConnectorsShared::Middleware::RestrictHostnames, { :allowed_hosts => [base_url, API_BASE_URL] }],
+          [ConnectorsShared::Middleware::BearerAuth, { :bearer_auth_token => @api_token }]
+        ]
+      end
+    end
+  end
+end

data/lib/connectors_sdk/gitlab/extractor.rb

@@ -0,0 +1,123 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'connectors_sdk/base/extractor'
+require 'connectors_sdk/gitlab/custom_client'
+require 'connectors_sdk/gitlab/adapter'
+require 'connectors_sdk/gitlab/config'
+require 'rack/utils'
+
+module ConnectorsSdk
+  module GitLab
+    class Extractor < ConnectorsSdk::Base::Extractor
+      PAGE_SIZE = 100 # max is 100
+
+      def yield_document_changes(modified_since: nil)
+        query_params = {
+          :pagination => :keyset,
+          :per_page => PAGE_SIZE,
+          :order_by => :id,
+          :sort => :desc
+        }
+        # looks like it's an incremental sync
+        if modified_since.present?
+          date_since = modified_since.is_a?(Time) ? modified_since : Time.new(modified_since)
+          query_params[:last_activity_after] = date_since.iso8601
+        end
+
+        next_page_link = nil
+
+        loop do
+          if next_page_link.present?
+            if (matcher = /(https?:[^>]*)/.match(next_page_link))
+              clean_query = URI.parse(matcher.captures[0]).query
+              query_params = Rack::Utils.parse_query(clean_query)
+            else
+              raise "Next page link has unexpected format: #{next_page_link}"
+            end
+          end
+          response = client.get('projects', query_params)
+
+          JSON.parse(response.body).map do |doc|
+            doc = doc.with_indifferent_access
+            if config.index_permissions
+              doc = doc.merge(project_permissions(doc[:id], doc[:visibility]))
+            end
+            yield :create_or_update, ConnectorsSdk::GitLab::Adapter.to_es_document(:project, doc), nil
+          end
+
+          next_page_link = response.headers['Link'] || nil
+          break unless next_page_link.present?
+        end
+      end
+
+      def yield_deleted_ids(ids)
+        if ids.present?
+          ids.each do |id|
+            response = client.get("projects/#{id}")
+            if response.status == 404
+              # not found - assume deleted
+              yield id
+            else
+              unless response.success?
+                raise "Could not get a project by ID: #{id}, response code: #{response.status}, response: #{response.body}"
+              end
+            end
+          end
+        end
+      end
+
+      def yield_permissions(source_user_id)
+        result = []
+        if source_user_id.present?
+          result.push("user:#{source_user_id}")
+
+          user_response = client.get("users/#{source_user_id}")
+          if user_response.success?
+            username = JSON.parse(user_response.body).with_indifferent_access[:username]
+            query = { :external => true, :username => username }
+            external_response = client.get('users', query)
+            if external_response.success?
+              external_users = Hashie::Array.new(JSON.parse(external_response.body))
+              if external_users.empty?
+                # the user is not external
+                result.push('type:internal')
+              end
+            else
+              raise "Could not check external user status by ID: #{source_user_id}"
+            end
+          else
+            raise "User isn't found by ID: #{source_user_id}"
+          end
+        end
+        yield result
+      end
+
+      private
+
+      def project_permissions(id, visibility)
+        result = []
+        if visibility.to_sym == :public || !config.index_permissions
+          # visible-to-all
+          return {}
+        end
+        if visibility.to_sym == :internal
+          result.push('type:internal')
+        end
+        response = client.get("projects/#{id}/members/all")
+        if response.success?
+          members = Hashie::Array.new(JSON.parse(response.body))
+          result.concat(members.map { |user| "user:#{user[:id]}" })
+        else
+          raise "Could not get project members by project ID: #{id}, response code: #{response.status}, response: #{response.body}"
+        end
+        { :_allow_permissions => result }
+      end
+    end
+  end
+end

data/lib/connectors_sdk/share_point/{http_call_wrapper.rb → connector.rb}

@@ -6,22 +6,46 @@
 
 # frozen_string_literal: true
 
+require 'connectors_sdk/base/connector'
 require 'connectors_sdk/office365/config'
 require 'connectors_sdk/share_point/extractor'
 require 'connectors_sdk/share_point/authorization'
-require 'connectors_sdk/base/http_call_wrapper'
 
 module ConnectorsSdk
   module SharePoint
-    class HttpCallWrapper < ConnectorsSdk::Base::HttpCallWrapper
+    class Connector < ConnectorsSdk::Base::Connector
       SERVICE_TYPE = 'share_point'
 
-      def name
-        'SharePoint'
+      def compare_secrets(params)
+        missing_secrets?(params)
+
+        previous_user = client(:access_token => params[:other_secret][:access_token]).me
+        equivalent = previous_user.nil? ? false : previous_user.id == client(:access_token => params[:secret][:access_token]).me&.id
+
+        {
+          :equivalent => equivalent
+        }
+      end
+
+      def display_name
+        'SharePoint Online'
+      end
+
+      def connection_requires_redirect
+        true
       end
 
-      def service_type
-        SERVICE_TYPE
+      def configurable_fields
+        [
+          {
+            'key' => 'client_id',
+            'label' => 'Client ID'
+          },
+          {
+            'key' => 'client_secret',
+            'label' => 'Client Secret'
+          },
+        ]
       end
 
       private

data/lib/connectors_sdk/stub_connector/connector.rb

@@ -0,0 +1,62 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'connectors_sdk/base/connector'
+
+module ConnectorsSdk
+  module StubConnector
+    class Connector < ConnectorsSdk::Base::Connector
+      SERVICE_TYPE = 'stub_connector'
+
+      def display_name
+        'Stub Connector'
+      end
+
+      def configurable_fields
+        [
+          {
+            'key' => 'third_party_url',
+            'label' => 'Third Party URL'
+          },
+          {
+            'key' => 'third_party_api_key',
+            'label' => 'Third Party API Key'
+          }
+        ]
+      end
+
+      def health_check(_params)
+        true
+      end
+
+      def document_batch(_params)
+        results = 30.times.map do |i|
+          {
+            :action => :create_or_update,
+            :document => {
+              :id => "document_#{i}",
+              :type => 'document',
+              :body => "contents for document number: #{i}"
+            },
+            :download => nil
+          }
+        end
+
+        [results, {}, true]
+      end
+
+      def deleted(_params)
+        []
+      end
+
+      def permissions(_params)
+        []
+      end
+    end
+  end
+end

data/lib/connectors_shared/constants.rb

@@ -10,5 +10,17 @@ module ConnectorsShared
     SUBEXTRACTOR_RESERVED_FIELDS = %w[_subextracted_as_of _subextracted_version].freeze
     ALLOW_FIELD = '_allow_permissions'.freeze
     DENY_FIELD = '_deny_permissions'.freeze
+
+    # The following section reads as following:
+    # The job will extract documents until the job queue size will reach
+    # JOB_QUEUE_SIZE_IDLE_THRESHOLD items. After that, the job will attempt to sleep
+    # for IDLE_SLEEP_TIME seconds and check the queue size again. If the queue is still
+    # full, it will sleep for maximum MAX_IDDLE_ATTEMPTS times, and if the queue is still
+    # full, then job will be terminated.
+    JOB_QUEUE_SIZE_IDLE_THRESHOLD = 500 # How many documents the job queue stores until it sleeps
+    IDLE_SLEEP_TIME = 10 # For how long job queue will sleep before checking the queue size again
+    MAX_IDLE_ATTEMPTS = 30 # How many consecutive times job will try to sleep until it's destroyed
+
+    STALE_JOB_TIMEOUT = 60 * 30 # Time in seconds after which the job will be cleaned up if the job is considered stuck
   end
 end

data/lib/connectors_shared/middleware/restrict_hostnames.rb

@@ -36,7 +36,7 @@ module ConnectorsShared
       def ips_from_hosts(hosts)
         hosts&.flat_map do |host|
           if URL_PATTERN.match(host)
-            lookup_ips(URI.parse(host).host)
+            lookup_ips(Addressable::URI.parse(host).hostname)
           elsif Resolv::IPv4::Regex.match(host) || Resolv::IPv6::Regex.match(host)
             IPAddr.new(host)
           else

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: connectors_sdk
 version: !ruby/object:Gem::Version
-  version: 8.3.0.0.pre.20220517T144653Z
+  version: 8.3.3.0
 platform: ruby
 authors:
 - Elastic
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-05-17 00:00:00.000000000 Z
+date: 2022-07-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -52,6 +52,118 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: tzinfo-data
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: faraday
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: faraday_middleware
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: forwardable
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: hashie
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: httpclient
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: signet
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: ''
 email: ent-search-dev@elastic.co
 executables: []
@@ -66,17 +178,22 @@ files:
 - lib/connectors_sdk/base/adapter.rb
 - lib/connectors_sdk/base/authorization.rb
 - lib/connectors_sdk/base/config.rb
+- lib/connectors_sdk/base/connector.rb
 - lib/connectors_sdk/base/custom_client.rb
 - lib/connectors_sdk/base/extractor.rb
-- lib/connectors_sdk/base/http_call_wrapper.rb
 - lib/connectors_sdk/base/registry.rb
 - lib/connectors_sdk/confluence/adapter.rb
 - lib/connectors_sdk/confluence/custom_client.rb
 - lib/connectors_sdk/confluence/extractor.rb
 - lib/connectors_sdk/confluence_cloud/authorization.rb
+- lib/connectors_sdk/confluence_cloud/connector.rb
 - lib/connectors_sdk/confluence_cloud/custom_client.rb
 - lib/connectors_sdk/confluence_cloud/extractor.rb
-- lib/connectors_sdk/confluence_cloud/http_call_wrapper.rb
+- lib/connectors_sdk/gitlab/adapter.rb
+- lib/connectors_sdk/gitlab/config.rb
+- lib/connectors_sdk/gitlab/connector.rb
+- lib/connectors_sdk/gitlab/custom_client.rb
+- lib/connectors_sdk/gitlab/extractor.rb
 - lib/connectors_sdk/helpers/atlassian_time_formatter.rb
 - lib/connectors_sdk/office365/adapter.rb
 - lib/connectors_sdk/office365/config.rb
@@ -84,8 +201,9 @@ files:
 - lib/connectors_sdk/office365/extractor.rb
 - lib/connectors_sdk/share_point/adapter.rb
 - lib/connectors_sdk/share_point/authorization.rb
+- lib/connectors_sdk/share_point/connector.rb
 - lib/connectors_sdk/share_point/extractor.rb
-- lib/connectors_sdk/share_point/http_call_wrapper.rb
+- lib/connectors_sdk/stub_connector/connector.rb
 - lib/connectors_shared.rb
 - lib/connectors_shared/constants.rb
 - lib/connectors_shared/errors.rb
@@ -98,14 +216,13 @@ files:
 - lib/connectors_shared/middleware/bearer_auth.rb
 - lib/connectors_shared/middleware/restrict_hostnames.rb
 - lib/connectors_shared/monitor.rb
-- lib/stubs/enterprise_search/exception_tracking.rb
 homepage: https://github.com/elastic/connectors
 licenses:
 - Elastic-2.0
 metadata:
-  revision: 9f25f35e17ffb36dfda754d657794ed9b5d2d75a
-  repository: git@github.com:elastic/connectors.git
-post_install_message:
+  revision: e45e2d01a4902ee7888a1223c18d78e60a5bfcde
+  repository: git@github.com:elastic/ent-search-connectors.git
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -116,12 +233,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
 rubygems_version: 3.0.3.1
-signing_key:
+signing_key: 
 specification_version: 4
 summary: Gem containing apis used by Enterprise Search and implementations of Connectors
 test_files: []

data/lib/connectors_sdk/confluence_cloud/http_call_wrapper.rb

@@ -1,59 +0,0 @@
-#
-# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
-# or more contributor license agreements. Licensed under the Elastic License;
-# you may not use this file except in compliance with the Elastic License.
-#
-
-# frozen_string_literal: true
-
-require 'connectors_sdk/atlassian/config'
-require 'connectors_sdk/confluence_cloud/extractor'
-require 'connectors_sdk/confluence_cloud/authorization'
-require 'connectors_sdk/confluence_cloud/custom_client'
-require 'connectors_sdk/base/http_call_wrapper'
-
-module ConnectorsSdk
-  module ConfluenceCloud
-    class HttpCallWrapper < ConnectorsSdk::Base::HttpCallWrapper
-      SERVICE_TYPE = 'confluence_cloud'
-
-      def name
-        'Confluence Cloud'
-      end
-
-      def service_type
-        SERVICE_TYPE
-      end
-
-      private
-
-      def extractor_class
-        ConnectorsSdk::ConfluenceCloud::Extractor
-      end
-
-      def authorization
-        ConnectorsSdk::ConfluenceCloud::Authorization
-      end
-
-      def client(params)
-        ConnectorsSdk::ConfluenceCloud::CustomClient.new(:base_url => base_url(params[:cloud_id]), :access_token => params[:access_token])
-      end
-
-      def custom_client_error
-        ConnectorsSdk::Atlassian::CustomClient::ClientError
-      end
-
-      def config(params)
-        ConnectorsSdk::Atlassian::Config.new(:base_url => base_url(params[:cloud_id]), :cursors => params.fetch(:cursors, {}) || {})
-      end
-
-      def health_check(params)
-        client(params).me
-      end
-
-      def base_url(cloud_id)
-        "https://api.atlassian.com/ex/confluence/#{cloud_id}"
-      end
-    end
-  end
-end

data/lib/stubs/enterprise_search/exception_tracking.rb

@@ -1,43 +0,0 @@
-#
-# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
-# or more contributor license agreements. Licensed under the Elastic License;
-# you may not use this file except in compliance with the Elastic License.
-#
-
-module EnterpriseSearch
-  class ExceptionTracking
-    def self.capture_message(message, context = {})
-      AppConfig.connectors_logger.error { "Error: #{message}. Context: #{context.inspect}" }
-
-      # When the method is called from a rescue block, our return value may leak outside of its
-      # intended scope, so let's explicitly return nil here to be safe.
-      nil
-    end
-
-    def self.log_exception(exception, message = nil, context: nil, logger: AppConfig.connectors_logger)
-      logger.error { message } if message
-      logger.error { generate_stack_trace(exception) }
-      logger.error { "Context: #{context.inspect}" } if context
-    end
-
-    def self.generate_error_message(exception, message, context)
-      context = { :message_id => exception.id }.merge(context || {}) if exception.respond_to?(:id)
-      context_message = context && "Context: #{context.inspect}"
-      ['Exception', message, exception.class.to_s, exception.message, context_message]
-        .compact
-        .map { |part| part.to_s.dup.force_encoding('UTF-8') }
-        .join(': ')
-    end
-
-    def self.generate_stack_trace(exception)
-      full_message = exception.full_message
-
-      cause = exception
-      while cause.cause != cause && (cause = cause.cause)
-        full_message << "Cause:\n#{cause.full_message}"
-      end
-
-      full_message.dup.force_encoding('UTF-8')
-    end
-  end
-end