connectors_sdk 8.3.0.0.pre.20220510T144908Z → 8.3.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 94b75b9fa3a5f0c46a271a34d073f71977629db6095b6364c8710b0aab92374b
-  data.tar.gz: d133f34052f43e0b8b65ba10eb946c55a6ef421a6296d2d4438e7f4f92b45696
+  metadata.gz: 929c0ce8ad8eaf2ab34849ee4f0b476c37e5cfe12994bf73aa183be58c350213
+  data.tar.gz: 9e7189b26959b4bb74e097a6e0a2effb3ade9a0b730dbccee5c23f5d0cedebd1
 SHA512:
-  metadata.gz: 712b819efcfa755ce19e4cdb7060dbadefecf37321d07bfe4e7bc9b18254e0f11f5fc29967ff171b1314064d77427bde730ab400a4a2dcde89b31dc6344d4a34
-  data.tar.gz: fa1fedb0c7449b9b2b50b1a56a710178cd3343a108500938244987eb31c694ab0d0214f383df69b6efa6b384eb558697553813c9b93cb142f011b374e05f2c49
+  metadata.gz: 6ea84dda6c9c5980b2fa089e2ac049a8ce1d27ff98761f20bb5e760dbbf6851c452cdcb35e40d274cff0c692da37e9a77be029fd286bc38672ea3597ab013db7
+  data.tar.gz: 837884527017d23bb33a1370f6500db9b8d37afee351ffd6243d852309120f69d2dd1bcf18a70556785d2b460e011276b8d7b621d09c631519c7aa95603b32e9

data/lib/connectors_sdk/atlassian/custom_client.rb

@@ -9,6 +9,7 @@
 require 'faraday_middleware'
 
 require 'connectors_shared/middleware/bearer_auth'
+require 'connectors_shared/middleware/basic_auth'
 require 'connectors_shared/middleware/restrict_hostnames'
 require 'connectors_sdk/base/custom_client'
 
@@ -30,10 +31,12 @@ module ConnectorsSdk
 
       MEDIA_API_BASE_URL = 'https://api.media.atlassian.com'
 
-      attr_reader :base_url, :access_token
+      attr_reader :base_url, :access_token, :basic_auth_token
 
-      def initialize(base_url:, access_token:, ensure_fresh_auth: nil)
+      def initialize(base_url:, access_token: nil, basic_auth_token: nil, ensure_fresh_auth: nil)
+        raise 'Either access_token or basic_auth_token must be provided' unless access_token.present? || basic_auth_token.present?
         @access_token = access_token
+        @basic_auth_token = basic_auth_token
         super(:base_url => base_url, :ensure_fresh_auth => ensure_fresh_auth)
       end
 
@@ -42,11 +45,17 @@ module ConnectorsSdk
       end
 
       def additional_middleware
-        [
-            ::FaradayMiddleware::FollowRedirects,
-            [ConnectorsShared::Middleware::RestrictHostnames, { :allowed_hosts => [base_url, MEDIA_API_BASE_URL] }],
-            [ConnectorsShared::Middleware::BearerAuth, { :bearer_auth_token => @access_token }]
+        result = [
+          FaradayMiddleware::FollowRedirects,
+          [ConnectorsShared::Middleware::RestrictHostnames, { :allowed_hosts => [base_url, MEDIA_API_BASE_URL] }]
         ]
+        if @access_token.present?
+          result.append([ConnectorsShared::Middleware::BearerAuth, { :bearer_auth_token => @access_token }])
+        elsif @basic_auth_token.present?
+          result.append([ConnectorsShared::Middleware::BasicAuth, { :basic_auth_token => @basic_auth_token }])
+        else
+          raise 'Either access token or basic auth must be provided'
+        end
       end
 
       def update_auth_data!(new_access_token)

data/lib/connectors_sdk/base/{http_call_wrapper.rb → connector.rb}

@@ -10,43 +10,55 @@ require 'bson'
 
 module ConnectorsSdk
   module Base
-    class HttpCallWrapper
+    class Connector
       def extractor(params)
+        content_source_id = params.fetch(:content_source_id)
+        secret_storage = params[:secret_storage]
+
         extractor_class.new(
-          content_source_id: params[:content_source_id] || "GENERATED-#{BSON::ObjectId.new}",
+          content_source_id: content_source_id || "GENERATED-#{BSON::ObjectId.new}",
           service_type: service_type,
-          authorization_data_proc: proc { { access_token: params[:access_token] } },
-          client_proc: proc { client(params) },
+          authorization_data_proc: proc do
+            secret = secret_storage.fetch_secret(content_source_id)
+            {
+              access_token: secret[:access_token]
+            }
+          end,
+          client_proc: proc {
+            secret = secret_storage.fetch_secret(content_source_id)
+            params[:access_token] = secret[:access_token]
+            client(params)
+          },
           config: config(params),
           features: params.fetch(:features, {}) || {}
         )
       end
 
-      def document_batch(params)
+      def extract(params)
         convert_third_party_errors do
-          results = []
-
           extractor = extractor(params)
 
-          extractor.yield_document_changes(:break_after_page => true, :modified_since => extractor.config.cursors['modified_since']) do |action, doc, download_args_and_proc|
+          extractor.yield_document_changes(:modified_since => extractor.config.cursors[:modified_since]) do |action, doc, download_args_and_proc|
             download_obj = nil
             if download_args_and_proc
               download_obj = {
-                  id: download_args_and_proc[0],
-                  name: download_args_and_proc[1],
-                  size: download_args_and_proc[2],
-                  download_args: download_args_and_proc[3]
+                id: download_args_and_proc[0],
+                name: download_args_and_proc[1],
+                size: download_args_and_proc[2],
+                download_args: download_args_and_proc[3]
               }
             end
 
-            results << {
-                :action => action,
-                :document => doc,
-                :download => download_obj
+            doc = {
+              :action => action,
+              :document => doc,
+              :download => download_obj
             }
+
+            yield doc
           end
 
-          [results, extractor.config.cursors, extractor.completed]
+          extractor.config.to_h[:cursors]
         end
       end
 
@@ -68,6 +80,10 @@ module ConnectorsSdk
         end
       end
 
+      def download(params)
+        extractor(params).download(params[:meta])
+      end
+
       def authorization_uri(params)
         authorization.authorization_uri(params)
       end
@@ -80,18 +96,18 @@ module ConnectorsSdk
         authorization.refresh(params)
       end
 
-      def download(params)
-        extractor(params).download(params[:meta])
-      end
-
       def source_status(params)
         health_check(params)
-        { :status => 'OK', :statusCode => 200, :message => "Connected to #{name}" }
+        { :status => 'OK', :statusCode => 200, :message => "Connected to #{display_name}" }
       rescue StandardError => e
         { :status => 'FAILURE', :statusCode => e.is_a?(custom_client_error) ? e.status_code : 500, :message => e.message }
       end
 
-      def name
+      def compare_secrets(*)
+        raise 'Not implemented for this connector'
+      end
+
+      def display_name
         raise 'Not implemented for this connector'
       end
 
@@ -99,6 +115,14 @@ module ConnectorsSdk
         self.class::SERVICE_TYPE
       end
 
+      def connection_requires_redirect
+        false
+      end
+
+      def configurable_fields
+        []
+      end
+
       private
 
       def convert_third_party_errors
@@ -130,6 +154,13 @@ module ConnectorsSdk
       def health_check(*)
         raise 'Not implemented for this connector'
       end
+
+      def missing_secrets?(params)
+        missing = %w[secret other_secret].select { |field| params[field.to_sym].nil? }
+        unless missing.blank?
+          raise ConnectorsShared::ClientError.new("Missing required fields: #{missing.join(', ')}")
+        end
+      end
     end
   end
 end

data/lib/connectors_sdk/base/custom_client.rb

@@ -74,7 +74,7 @@ module ConnectorsSdk
             faraday.use(*middleware_config)
           end
 
-          faraday.adapter(:httpclient)
+          faraday.adapter :httpclient
         end
       end
 

data/lib/connectors_sdk/base/extractor.rb

@@ -188,16 +188,19 @@ module ConnectorsSdk
       end
 
       def permissions(source_user_id, &block)
+        result = []
         Connectors::Stats.measure("extractor.#{Connectors::Stats.class_key(self.class)}.permissions") do
           with_auth_tokens_and_retry do
             Connectors::Stats.measure("extractor.#{Connectors::Stats.class_key(self.class)}.yield_permissions") do
               yield_permissions(source_user_id) do |permissions|
                 log_info("Extracted #{permissions.size} permissions for source user #{source_user_id}")
+                result = permissions
                 block.call(permissions) if block_given?
               end
             end
           end
         end
+        result.each
       end
 
       ConnectorsShared::Logger::SUPPORTED_LOG_LEVELS.each do |log_level|

data/lib/connectors_sdk/base/registry.rb

@@ -24,11 +24,16 @@ module ConnectorsSdk
 
     REGISTRY = Factory.new
 
+    require_relative '../stub_connector/connector'
+    REGISTRY.register(ConnectorsSdk::StubConnector::Connector::SERVICE_TYPE, ConnectorsSdk::StubConnector::Connector)
+
     # loading plugins (might replace this with a directory scan and conventions on names)
-    require_relative '../share_point/http_call_wrapper'
-    require_relative '../confluence_cloud//http_call_wrapper'
+    require_relative '../share_point/connector'
+    require_relative '../confluence_cloud/connector'
+    require_relative '../gitlab/connector'
 
-    REGISTRY.register(ConnectorsSdk::SharePoint::HttpCallWrapper::SERVICE_TYPE, ConnectorsSdk::SharePoint::HttpCallWrapper)
-    REGISTRY.register(ConnectorsSdk::ConfluenceCloud::HttpCallWrapper::SERVICE_TYPE, ConnectorsSdk::ConfluenceCloud::HttpCallWrapper)
+    REGISTRY.register(ConnectorsSdk::SharePoint::Connector::SERVICE_TYPE, ConnectorsSdk::SharePoint::Connector)
+    REGISTRY.register(ConnectorsSdk::ConfluenceCloud::Connector::SERVICE_TYPE, ConnectorsSdk::ConfluenceCloud::Connector)
+    REGISTRY.register(ConnectorsSdk::GitLab::Connector::SERVICE_TYPE, ConnectorsSdk::GitLab::Connector)
   end
 end

data/lib/connectors_sdk/confluence/extractor.rb

@@ -20,7 +20,7 @@ module ConnectorsSdk
 
       ConnectorsSdk::Base::Extractor::TRANSIENT_SERVER_ERROR_CLASSES << Atlassian::CustomClient::ServiceUnavailableError
 
-      def yield_document_changes(modified_since: nil, break_after_page: false)
+      def yield_document_changes(modified_since: nil)
         @space_permissions_cache = {}
         @content_restriction_cache = {}
         yield_spaces do |space|
@@ -50,11 +50,6 @@ module ConnectorsSdk
               yield :create_or_update, Confluence::Adapter.es_document_from_confluence_content(content, content_base_url, restrictions)
             end
           end
-
-          if break_after_page
-            @completed = true
-            break
-          end
         end
       end
 
@@ -87,7 +82,7 @@ module ConnectorsSdk
       private
 
       def content_base_url
-        'https://workplace-search.atlassian.net/wiki'
+        config.base_url
       end
 
       def yield_spaces

data/lib/connectors_sdk/confluence_cloud/connector.rb

@@ -0,0 +1,110 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'base64'
+require 'connectors_sdk/atlassian/config'
+require 'connectors_sdk/confluence_cloud/extractor'
+require 'connectors_sdk/confluence_cloud/authorization'
+require 'connectors_sdk/confluence_cloud/custom_client'
+require 'connectors_sdk/base/connector'
+
+module ConnectorsSdk
+  module ConfluenceCloud
+    class Connector < ConnectorsSdk::Base::Connector
+      SERVICE_TYPE = 'confluence_cloud'
+
+      def compare_secrets(params)
+        missing_secrets?(params)
+
+        {
+          :equivalent => params[:secret] == params[:other_secret]
+        }
+      end
+
+      def display_name
+        'Confluence Cloud'
+      end
+
+      def configurable_fields
+        [
+          {
+            'key' => 'base_url',
+            'label' => 'Confluence Cloud Base URL'
+          },
+          {
+            'key' => 'confluence_user_email',
+            'label' => 'Confluence user email'
+          },
+          {
+            'key' => 'confluence_api_token',
+            'label' => 'Confluence user REST API Token'
+          },
+        ]
+      end
+
+      private
+
+      def extractor_class
+        ConnectorsSdk::ConfluenceCloud::Extractor
+      end
+
+      def authorization
+        ConnectorsSdk::ConfluenceCloud::Authorization
+      end
+
+      def client(params)
+        ConnectorsSdk::ConfluenceCloud::CustomClient.new(
+          :base_url => extract_base_url(params),
+          :basic_auth_token => extract_basic_auth_token(params)
+        )
+      end
+
+      def custom_client_error
+        ConnectorsSdk::Atlassian::CustomClient::ClientError
+      end
+
+      def config(params)
+        ConnectorsSdk::Atlassian::Config.new(
+          :base_url => extract_base_url(params),
+          :cursors => params.fetch(:cursors, {}) || {},
+          :index_permissions => params[:index_permissions] || false
+        )
+      end
+
+      def health_check(params)
+        client(params).me
+      end
+
+      def base_url(cloud_id)
+        "https://api.atlassian.com/ex/confluence/#{cloud_id}"
+      end
+
+      def is_basic_auth(params)
+        login = params.fetch('confluence_user_email', nil)
+        api_token = params.fetch('confluence_api_token', nil)
+        login.present? && api_token.present?
+      end
+
+      def extract_basic_auth_token(params)
+        login = params.fetch('confluence_user_email', nil)
+        api_token = params.fetch('confluence_api_token', nil)
+        nil unless login.present? && api_token.present?
+        Base64.strict_encode64("#{login}:#{api_token}")
+      end
+
+      def extract_base_url(params)
+        # From Confluence API documentation:
+        # Requests that use OAuth 2.0 (3LO) are made via api.atlassian.com (not https://your-domain.atlassian.net).
+        if is_basic_auth(params)
+          return params[:base_url].end_with?('/wiki') ? params[:base_url] : "#{params[:base_url]}/wiki"
+        end
+        base_url(params[:cloud_id])
+      end
+    end
+  end
+end

data/lib/connectors_sdk/confluence_cloud/extractor.rb

@@ -52,7 +52,8 @@ module ConnectorsSdk
       def download(item)
         content = item[:content]
         parent_id = content.dig('container', 'id')
-        client.download("#{client.base_url}/wiki/rest/api/content/#{parent_id}/child/attachment/#{content['id']}/download").body
+        base_url = client.base_url.end_with?('/wiki') ? client.base_url : "#{client.base_url}/wiki"
+        client.download("#{base_url}/rest/api/content/#{parent_id}/child/attachment/#{content['id']}/download").body
       end
     end
   end

data/lib/connectors_sdk/gitlab/adapter.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+require 'hashie/mash'
+require 'connectors_sdk/base/adapter'
+
+module ConnectorsSdk
+  module GitLab
+    class Adapter < ConnectorsSdk::Base::Adapter
+      # it's important to have this to generate ID converters between the GitLab ID and the
+      # Enterprise Search document ID. The Enterprise Search document ID will be prefixed with the service type,
+      # in our case - `gitlab`.
+      generate_id_helpers :gitlab, 'gitlab'
+
+      def self.to_es_document(type, source_doc)
+        result = {
+          :id => gitlab_id_to_es_id(source_doc[:id]),
+          :type => type,
+          :url => source_doc[:web_url],
+          :body => source_doc[:description],
+          :title => source_doc[:name],
+          :created_at => source_doc[:created_at],
+          :last_modified_at => source_doc[:last_activity_at],
+          :visibility => source_doc[:visibility],
+          :namespace => if source_doc[:namespace].nil?
+                          nil
+                        else
+                          source_doc[:namespace][:name]
+                        end
+        }
+        if source_doc[:_allow_permissions].present?
+          result[:_allow_permissions] = source_doc[:_allow_permissions]
+        end
+        result
+      end
+    end
+  end
+end

data/lib/connectors_sdk/gitlab/config.rb

@@ -0,0 +1,26 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'connectors_sdk/base/config'
+
+module ConnectorsSdk
+  module GitLab
+    class Config < ConnectorsSdk::Base::Config
+      attr_reader :index_permissions
+
+      def initialize(cursors:, index_permissions: false)
+        super(:cursors => cursors)
+        @index_permissions = index_permissions || false
+      end
+
+      def to_h
+        super.merge(:index_permissions => index_permissions)
+      end
+    end
+  end
+end

data/lib/connectors_sdk/gitlab/connector.rb

@@ -0,0 +1,71 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'connectors_sdk/base/connector'
+require 'connectors_sdk/gitlab/custom_client'
+require 'connectors_sdk/gitlab/adapter'
+require 'connectors_sdk/gitlab/config'
+require 'connectors_sdk/gitlab/extractor'
+require 'rack/utils'
+
+module ConnectorsSdk
+  module GitLab
+    class Connector < ConnectorsSdk::Base::Connector
+      SERVICE_TYPE = 'gitlab'
+
+      def display_name
+        'GitLab Connector'
+      end
+
+      def configurable_fields
+        [
+          {
+            'key' => 'api_token',
+            'label' => 'API Token'
+          },
+          {
+            'key' => 'base_url',
+            'label' => 'Base URL'
+          }
+        ]
+      end
+
+      private
+
+      def client(params)
+        ConnectorsSdk::GitLab::CustomClient.new(
+          :base_url => params[:base_url] || ConnectorsSdk::GitLab::API_BASE_URL,
+          :api_token => params[:api_token]
+        )
+      end
+
+      def config(params)
+        ConnectorsSdk::GitLab::Config.new(
+          :cursors => params.fetch(:cursors, {}) || {},
+          :index_permissions => params.fetch(:index_permissions, false)
+        )
+      end
+
+      def extractor_class
+        ConnectorsSdk::GitLab::Extractor
+      end
+
+      def custom_client_error
+        ConnectorsSdk::GitLab::CustomClient::ClientError
+      end
+
+      def health_check(params)
+        # let's do a simple call
+        response = client(params).get('user')
+        unless response.present? && response.status == 200
+          raise "Health check failed with response status #{response.status} and body #{response.body}"
+        end
+      end
+    end
+  end
+end

data/lib/connectors_sdk/gitlab/custom_client.rb

@@ -0,0 +1,40 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+require 'faraday_middleware/response/follow_redirects'
+require 'connectors_sdk/base/custom_client'
+require 'connectors_shared/middleware/bearer_auth'
+require 'connectors_shared/middleware/basic_auth'
+require 'connectors_shared/middleware/restrict_hostnames'
+
+module ConnectorsSdk
+  module GitLab
+    API_BASE_URL = 'https://gitlab.com/api/v4'
+
+    class CustomClient < ConnectorsSdk::Base::CustomClient
+      class ClientError < ConnectorsShared::ClientError
+        attr_reader :status_code, :endpoint
+
+        def initialize(status_code, endpoint)
+          @status_code = status_code
+          @endpoint = endpoint
+        end
+      end
+
+      def initialize(base_url:, api_token:, ensure_fresh_auth: nil)
+        @api_token = api_token
+        super(:base_url => base_url, :ensure_fresh_auth => ensure_fresh_auth)
+      end
+
+      def additional_middleware
+        [
+          ::FaradayMiddleware::FollowRedirects,
+          [ConnectorsShared::Middleware::RestrictHostnames, { :allowed_hosts => [base_url, API_BASE_URL] }],
+          [ConnectorsShared::Middleware::BearerAuth, { :bearer_auth_token => @api_token }]
+        ]
+      end
+    end
+  end
+end