connectors_sdk 8.3.0.0.pre.20220414T060419Z → 8.3.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a44a80ba5fe5032b6254fb8a2db25516542df5818044d627b6ba31f652a7812f
-  data.tar.gz: 52d1245cc24ab6e1a42c00483ec4a8ece48ede4a6465c9b46b3bf5e18c979678
+  metadata.gz: 21bd11a44ed7c45fa04b218346fda20d946b97ec8f7025c2cfb8c8f92321d6a3
+  data.tar.gz: b388f972974f32c59d719b809975229125e293045dd35559546fefe5485e1fc3
 SHA512:
-  metadata.gz: 7a4021e1d7b83ec7334913f1ef5e684e02296097f223e96610cb31eb5502c36f9b45b03eeaf80e876aac5bcc531f85d10445917459b86cedc30d6609b33f5351
-  data.tar.gz: ca04ab72f48ed6e76b3a0e0cc8c260f81779ea906325a782eb61456c9c5be231043d7a73ca9625df63459460de587c65aa352fb2e1150fc8c92a253536fcd634
+  metadata.gz: 88aa76506d6f2af049ea2a2ff54797270cef692319dd9ca647d33903a9fb3bb8bb274d9d99a3f94fe4d54cfce1722b31d63203cd157b627ccf3b2ca541b63523
+  data.tar.gz: 3faf8e16b18c0abd4f13fa988de8a6bb03e95bce5738389f132748fc17a230a9e516eb8b9f898d60aeb8a869f1ad1dadbb8606178e01be7d9ccc63ca98dcf681

data/lib/connectors_sdk/atlassian/config.rb

@@ -0,0 +1,27 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'connectors_sdk/base/config'
+
+module ConnectorsSdk
+  module Atlassian
+    class Config < ConnectorsSdk::Base::Config
+      attr_reader :base_url, :index_permissions
+
+      def initialize(cursors:, base_url:, index_permissions: false)
+        super(:cursors => cursors)
+        @base_url = base_url
+        @index_permissions = index_permissions
+      end
+
+      def to_h
+        super.merge(:base_url => base_url)
+      end
+    end
+  end
+end

data/lib/connectors_sdk/atlassian/custom_client.rb

@@ -0,0 +1,87 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'faraday_middleware'
+
+require 'connectors_shared/middleware/bearer_auth'
+require 'connectors_shared/middleware/restrict_hostnames'
+require 'connectors_sdk/base/custom_client'
+
+module ConnectorsSdk
+  module Atlassian
+    class CustomClient < ConnectorsSdk::Base::CustomClient
+      class ClientError < ConnectorsShared::ClientError
+        attr_reader :url, :status_code, :message
+
+        def initialize(url, status_code, message)
+          super("Failed to call #{url} because #{status_code}: #{message}")
+          @url = url
+          @status_code = status_code
+          @message = message
+        end
+      end
+      class ServiceUnavailableError < ClientError; end
+      class ContentConvertibleError < ClientError; end
+
+      MEDIA_API_BASE_URL = 'https://api.media.atlassian.com'
+
+      attr_reader :base_url, :access_token
+
+      def initialize(base_url:, access_token:, ensure_fresh_auth: nil)
+        @access_token = access_token
+        super(:base_url => base_url, :ensure_fresh_auth => ensure_fresh_auth)
+      end
+
+      def default_middleware
+        [] # Ignoring Base default for now, but we should probably revert to using Base default?
+      end
+
+      def additional_middleware
+        [
+            ::FaradayMiddleware::FollowRedirects,
+            [ConnectorsShared::Middleware::RestrictHostnames, { :allowed_hosts => [base_url, MEDIA_API_BASE_URL] }],
+            [ConnectorsShared::Middleware::BearerAuth, { :bearer_auth_token => @access_token }]
+        ]
+      end
+
+      def update_auth_data!(new_access_token)
+        @access_token = new_access_token
+        middleware!
+        http_client! # force a new client to pick up new middleware
+
+        self
+      end
+
+      def download(url)
+        response = get(url)
+        unless HTTP::Status.successful?(response.status)
+          raise ClientError.new(url, response.status, response.body)
+        end
+        response
+      end
+
+      private
+
+      def parse_and_raise_if_necessary!(response)
+        unless response.success?
+          status_code = response.status.to_i
+          atlassian_error_klass =
+            if status_code == 504
+              ServiceUnavailableError
+            elsif status_code == 400 && response.body.include?('is not ContentConvertible or API available')
+              ContentConvertibleError
+            else
+              ClientError
+            end
+          raise atlassian_error_klass.new(response.env.url.to_s, status_code, response.body)
+        end
+        JSON.parse(response.body)
+      end
+    end
+  end
+end

data/lib/connectors_sdk/base/adapter.rb

@@ -10,7 +10,6 @@ require 'active_support/core_ext/object/deep_dup'
 require 'connectors_shared'
 require 'connectors_shared/extension_mapping_util'
 require 'date'
-require 'active_support/all'
 require 'mime-types'
 
 module ConnectorsSdk
@@ -25,19 +24,19 @@ module ConnectorsSdk
       end
 
       def self.generate_id_helpers(method_prefix, id_prefix)
-        define_singleton_method("#{method_prefix}_id_to_fp_id") do |id|
+        define_singleton_method("#{method_prefix}_id_to_es_id") do |id|
           "#{id_prefix}_#{id}"
         end
 
-        define_singleton_method("fp_id_is_#{method_prefix}_id?") do |fp_id|
-          regex_match = /#{id_prefix}_(.+)$/.match(fp_id)
+        define_singleton_method("es_id_is_#{method_prefix}_id?") do |es_id|
+          regex_match = /#{id_prefix}_(.+)$/.match(es_id)
           regex_match.present? && regex_match.size == 2
         end
 
-        define_singleton_method("fp_id_to_#{method_prefix}_id") do |fp_id|
-          regex_match = /#{id_prefix}_(.+)$/.match(fp_id)
+        define_singleton_method("es_id_to_#{method_prefix}_id") do |es_id|
+          regex_match = /#{id_prefix}_(.+)$/.match(es_id)
 
-          raise ArgumentError, "Invalid id #{fp_id} for source with method prefix #{method_prefix}." if regex_match.nil? || regex_match.length != 2
+          raise ArgumentError, "Invalid id #{es_id} for source with method prefix #{method_prefix}." if regex_match.nil? || regex_match.length != 2
           regex_match[1]
         end
       end
@@ -92,7 +91,7 @@ module ConnectorsSdk
         nil
       end
 
-      def self.swiftype_document_from_configured_object_base(object_type:, object:, fields:)
+      def self.es_document_from_configured_object_base(object_type:, object:, fields:)
         object_as_json = object.as_json
 
         adapted_object = {

data/lib/connectors_sdk/base/authorization.rb

@@ -0,0 +1,89 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'connectors_shared'
+require 'signet'
+require 'signet/oauth_2'
+require 'signet/oauth_2/client'
+
+module ConnectorsSdk
+  module Base
+    class Authorization
+      class << self
+        def authorization_uri(params)
+          missing = missing_fields(params, %w[client_id])
+          unless missing.blank?
+            raise ConnectorsShared::ClientError.new("Missing required fields: #{missing.join(', ')}")
+          end
+
+          params[:response_type] = 'code'
+          params[:additional_parameters] = additional_parameters
+          client = oauth_client(params)
+          client.authorization_uri.to_s
+        end
+
+        def access_token(params)
+          missing = missing_fields(params, %w[client_id client_secret code redirect_uri])
+          unless missing.blank?
+            raise ConnectorsShared::ClientError.new("Missing required fields: #{missing.join(', ')}")
+          end
+
+          params[:grant_type] = 'authorization_code'
+          client = oauth_client(params)
+          client.fetch_access_token
+        end
+
+        def refresh(params)
+          missing = missing_fields(params, %w[client_id client_secret refresh_token])
+          unless missing.blank?
+            raise ConnectorsShared::ClientError.new("Missing required fields: #{missing.join(', ')}")
+          end
+
+          params[:grant_type] = 'refresh_token'
+          client = oauth_client(params)
+          client.refresh!
+        rescue StandardError => e
+          ConnectorsShared::ExceptionTracking.log_exception(e)
+          raise ConnectorsShared::TokenRefreshFailedError
+        end
+
+        def oauth_client(params)
+          options = params.merge(
+            :authorization_uri => authorization_url,
+            :token_credential_uri => token_credential_uri,
+            :scope => oauth_scope
+          )
+          options[:state] = JSON.dump(options[:state]) if options[:state]
+          Signet::OAuth2::Client.new(options)
+        end
+
+        def missing_fields(params, required = [])
+          Array.wrap(required).select { |field| params[field.to_sym].nil? }
+        end
+
+        def oauth_scope
+          raise 'Not implemented for this connector'
+        end
+
+        private
+
+        def authorization_url
+          raise 'Not implemented for this connector'
+        end
+
+        def token_credential_uri
+          raise 'Not implemented for this connector'
+        end
+
+        def additional_parameters
+          raise 'Not implemented for this connector'
+        end
+      end
+    end
+  end
+end

data/lib/connectors_sdk/base/connector.rb

@@ -0,0 +1,166 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'bson'
+
+module ConnectorsSdk
+  module Base
+    class Connector
+      def extractor(params)
+        content_source_id = params.fetch(:content_source_id)
+        secret_storage = params[:secret_storage]
+
+        extractor_class.new(
+          content_source_id: content_source_id || "GENERATED-#{BSON::ObjectId.new}",
+          service_type: service_type,
+          authorization_data_proc: proc do
+            secret = secret_storage.fetch_secret(content_source_id)
+            {
+              access_token: secret[:access_token]
+            }
+          end,
+          client_proc: proc {
+            secret = secret_storage.fetch_secret(content_source_id)
+            params[:access_token] = secret[:access_token]
+            client(params)
+          },
+          config: config(params),
+          features: params.fetch(:features, {}) || {}
+        )
+      end
+
+      def extract(params)
+        convert_third_party_errors do
+          extractor = extractor(params)
+
+          extractor.yield_document_changes(:modified_since => extractor.config.cursors[:modified_since]) do |action, doc, download_args_and_proc|
+            download_obj = nil
+            if download_args_and_proc
+              download_obj = {
+                id: download_args_and_proc[0],
+                name: download_args_and_proc[1],
+                size: download_args_and_proc[2],
+                download_args: download_args_and_proc[3]
+              }
+            end
+
+            doc = {
+              :action => action,
+              :document => doc,
+              :download => download_obj
+            }
+
+            yield doc
+          end
+
+          extractor.config.to_h[:cursors]
+        end
+      end
+
+      def deleted(params)
+        convert_third_party_errors do
+          results = []
+          extractor(params).yield_deleted_ids(params[:ids]) do |id|
+            results << id
+          end
+          results
+        end
+      end
+
+      def permissions(params)
+        convert_third_party_errors do
+          extractor(params).yield_permissions(params[:user_id]) do |permissions|
+            return permissions
+          end
+        end
+      end
+
+      def download(params)
+        extractor(params).download(params[:meta])
+      end
+
+      def authorization_uri(params)
+        authorization.authorization_uri(params)
+      end
+
+      def access_token(params)
+        authorization.access_token(params)
+      end
+
+      def refresh(params)
+        authorization.refresh(params)
+      end
+
+      def source_status(params)
+        health_check(params)
+        { :status => 'OK', :statusCode => 200, :message => "Connected to #{display_name}" }
+      rescue StandardError => e
+        { :status => 'FAILURE', :statusCode => e.is_a?(custom_client_error) ? e.status_code : 500, :message => e.message }
+      end
+
+      def compare_secrets(*)
+        raise 'Not implemented for this connector'
+      end
+
+      def display_name
+        raise 'Not implemented for this connector'
+      end
+
+      def service_type
+        self.class::SERVICE_TYPE
+      end
+
+      def connection_requires_redirect
+        false
+      end
+
+      def configurable_fields
+        []
+      end
+
+      private
+
+      def convert_third_party_errors
+        yield
+      rescue custom_client_error => e
+        raise e.status_code == 401 ? ConnectorsShared::InvalidTokenError : e
+      end
+
+      def extractor_class
+        raise 'Not implemented for this connector'
+      end
+
+      def authorization
+        raise 'Not implemented for this connector'
+      end
+
+      def client(*)
+        raise 'Not implemented for this connector'
+      end
+
+      def custom_client_error
+        raise 'Not implemented for this connector'
+      end
+
+      def config(*)
+        raise 'Not implemented for this connector'
+      end
+
+      def health_check(*)
+        raise 'Not implemented for this connector'
+      end
+
+      def missing_secrets?(params)
+        missing = %w[secret other_secret].select { |field| params[field.to_sym].nil? }
+        unless missing.blank?
+          raise ConnectorsShared::ClientError.new("Missing required fields: #{missing.join(', ')}")
+        end
+      end
+    end
+  end
+end

data/lib/connectors_sdk/base/custom_client.rb

@@ -11,7 +11,6 @@ require 'active_support/core_ext/numeric/time'
 require 'active_support/core_ext/object/deep_dup'
 require 'connectors_shared'
 require 'date'
-require 'active_support/all'
 
 module ConnectorsSdk
   module Base
@@ -75,7 +74,7 @@ module ConnectorsSdk
             faraday.use(*middleware_config)
           end
 
-          faraday.adapter(:httpclient)
+          faraday.adapter :httpclient
         end
       end
 

data/lib/connectors_sdk/base/extractor.rb

@@ -9,10 +9,11 @@ require 'httpclient'
 require 'active_support/core_ext/array/wrap'
 require 'active_support/core_ext/numeric/time'
 require 'active_support/core_ext/object/deep_dup'
-require 'connectors_shared'
 require 'date'
-require 'active_support/all'
+
+require 'connectors_shared'
 require 'stubs/connectors/stats' unless defined?(Rails)
+require 'stubs/connectors' unless defined?(Rails)
 
 module ConnectorsSdk
   module Base
@@ -187,16 +188,19 @@ module ConnectorsSdk
       end
 
       def permissions(source_user_id, &block)
+        result = []
         Connectors::Stats.measure("extractor.#{Connectors::Stats.class_key(self.class)}.permissions") do
           with_auth_tokens_and_retry do
             Connectors::Stats.measure("extractor.#{Connectors::Stats.class_key(self.class)}.yield_permissions") do
               yield_permissions(source_user_id) do |permissions|
                 log_info("Extracted #{permissions.size} permissions for source user #{source_user_id}")
+                result = permissions
                 block.call(permissions) if block_given?
               end
             end
           end
         end
+        result.each
       end
 
       ConnectorsShared::Logger::SUPPORTED_LOG_LEVELS.each do |log_level|

data/lib/connectors_sdk/base/registry.rb

@@ -17,16 +17,21 @@ module ConnectorsSdk
         @connectors[name] = klass
       end
 
-      def connector(name)
-        @connectors[name].new
+      def connector_class(name)
+        @connectors[name]
       end
     end
 
     REGISTRY = Factory.new
 
+    require_relative '../stub_connector/connector'
+    REGISTRY.register(ConnectorsSdk::StubConnector::Connector::SERVICE_TYPE, ConnectorsSdk::StubConnector::Connector)
+
     # loading plugins (might replace this with a directory scan and conventions on names)
-    require_relative '../share_point/http_call_wrapper'
+    require_relative '../share_point/connector'
+    require_relative '../confluence_cloud/connector'
 
-    REGISTRY.register(ConnectorsSdk::SharePoint::SERVICE_TYPE, ConnectorsSdk::SharePoint::HttpCallWrapper)
+    REGISTRY.register(ConnectorsSdk::SharePoint::Connector::SERVICE_TYPE, ConnectorsSdk::SharePoint::Connector)
+    REGISTRY.register(ConnectorsSdk::ConfluenceCloud::Connector::SERVICE_TYPE, ConnectorsSdk::ConfluenceCloud::Connector)
   end
 end