connectors_sdk 8.2.0.0 → 8.3.0.0.pre.20220517T144653Z

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c39fc76b8540ffda07563949daad4a4e1e5e57d225d64f479234c4ae14c15327
-  data.tar.gz: d864c7e38eac63a6d1092a1d4bec36d48913e5e894788f3704e48fb0f0e1ae4b
+  metadata.gz: b86ca5e489e3cef3b9f2c04a462baf71d6b43805731b0cb52ba2f56f5078d6d3
+  data.tar.gz: 044e860f11163e82c63f66276c3d5628b761c5dfcc5168fc8f80b63ca87d19f0
 SHA512:
-  metadata.gz: 598adcf8a50bd031374003792488a2942f88652429e94971be225702a498935cc3f0db2f687ee7089554bb3234f2c97bb46fec88f8cb4950962d4b06cb18ee7d
-  data.tar.gz: 53a5800aa40814cf98cbf176de29dd9ae2efb9a6b9ff12a3571bb87ec5b51f63f30bddfd8209bb952556279754568cd906af770b013f6044e5b78cd0d60122ae
+  metadata.gz: 528fa5260cf80a3ebb918478e1be2e7cac1668588853ad72b1259095086090c15a0cbc028523b1de8b775b3b64ed7427c6fab7ffccc562d559c8164456b84c4b
+  data.tar.gz: a58d353e2b48ffda33aa287d0fa6bb1400c531aae2794995ff3b644b779426d10bb3efc8735d8e07c353fae64bc0b3b0af58427d989f499210a19244b23c1a35

data/lib/connectors_sdk/atlassian/config.rb

@@ -0,0 +1,27 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'connectors_sdk/base/config'
+
+module ConnectorsSdk
+  module Atlassian
+    class Config < ConnectorsSdk::Base::Config
+      attr_reader :base_url, :index_permissions
+
+      def initialize(cursors:, base_url:, index_permissions: false)
+        super(:cursors => cursors)
+        @base_url = base_url
+        @index_permissions = index_permissions
+      end
+
+      def to_h
+        super.merge(:base_url => base_url)
+      end
+    end
+  end
+end

data/lib/connectors_sdk/atlassian/custom_client.rb

@@ -0,0 +1,87 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'faraday_middleware'
+
+require 'connectors_shared/middleware/bearer_auth'
+require 'connectors_shared/middleware/restrict_hostnames'
+require 'connectors_sdk/base/custom_client'
+
+module ConnectorsSdk
+  module Atlassian
+    class CustomClient < ConnectorsSdk::Base::CustomClient
+      class ClientError < ConnectorsShared::ClientError
+        attr_reader :url, :status_code, :message
+
+        def initialize(url, status_code, message)
+          super("Failed to call #{url} because #{status_code}: #{message}")
+          @url = url
+          @status_code = status_code
+          @message = message
+        end
+      end
+      class ServiceUnavailableError < ClientError; end
+      class ContentConvertibleError < ClientError; end
+
+      MEDIA_API_BASE_URL = 'https://api.media.atlassian.com'
+
+      attr_reader :base_url, :access_token
+
+      def initialize(base_url:, access_token:, ensure_fresh_auth: nil)
+        @access_token = access_token
+        super(:base_url => base_url, :ensure_fresh_auth => ensure_fresh_auth)
+      end
+
+      def default_middleware
+        [] # Ignoring Base default for now, but we should probably revert to using Base default?
+      end
+
+      def additional_middleware
+        [
+            ::FaradayMiddleware::FollowRedirects,
+            [ConnectorsShared::Middleware::RestrictHostnames, { :allowed_hosts => [base_url, MEDIA_API_BASE_URL] }],
+            [ConnectorsShared::Middleware::BearerAuth, { :bearer_auth_token => @access_token }]
+        ]
+      end
+
+      def update_auth_data!(new_access_token)
+        @access_token = new_access_token
+        middleware!
+        http_client! # force a new client to pick up new middleware
+
+        self
+      end
+
+      def download(url)
+        response = get(url)
+        unless HTTP::Status.successful?(response.status)
+          raise ClientError.new(url, response.status, response.body)
+        end
+        response
+      end
+
+      private
+
+      def parse_and_raise_if_necessary!(response)
+        unless response.success?
+          status_code = response.status.to_i
+          atlassian_error_klass =
+            if status_code == 504
+              ServiceUnavailableError
+            elsif status_code == 400 && response.body.include?('is not ContentConvertible or API available')
+              ContentConvertibleError
+            else
+              ClientError
+            end
+          raise atlassian_error_klass.new(response.env.url.to_s, status_code, response.body)
+        end
+        JSON.parse(response.body)
+      end
+    end
+  end
+end

data/lib/connectors_sdk/base/adapter.rb

@@ -10,7 +10,6 @@ require 'active_support/core_ext/object/deep_dup'
 require 'connectors_shared'
 require 'connectors_shared/extension_mapping_util'
 require 'date'
-require 'active_support/all'
 require 'mime-types'
 
 module ConnectorsSdk
@@ -25,19 +24,19 @@ module ConnectorsSdk
       end
 
       def self.generate_id_helpers(method_prefix, id_prefix)
-        define_singleton_method("#{method_prefix}_id_to_fp_id") do |id|
+        define_singleton_method("#{method_prefix}_id_to_es_id") do |id|
           "#{id_prefix}_#{id}"
         end
 
-        define_singleton_method("fp_id_is_#{method_prefix}_id?") do |fp_id|
-          regex_match = /#{id_prefix}_(.+)$/.match(fp_id)
+        define_singleton_method("es_id_is_#{method_prefix}_id?") do |es_id|
+          regex_match = /#{id_prefix}_(.+)$/.match(es_id)
           regex_match.present? && regex_match.size == 2
         end
 
-        define_singleton_method("fp_id_to_#{method_prefix}_id") do |fp_id|
-          regex_match = /#{id_prefix}_(.+)$/.match(fp_id)
+        define_singleton_method("es_id_to_#{method_prefix}_id") do |es_id|
+          regex_match = /#{id_prefix}_(.+)$/.match(es_id)
 
-          raise ArgumentError, "Invalid id #{fp_id} for source with method prefix #{method_prefix}." if regex_match.nil? || regex_match.length != 2
+          raise ArgumentError, "Invalid id #{es_id} for source with method prefix #{method_prefix}." if regex_match.nil? || regex_match.length != 2
           regex_match[1]
         end
       end
@@ -92,7 +91,7 @@ module ConnectorsSdk
         nil
       end
 
-      def self.swiftype_document_from_configured_object_base(object_type:, object:, fields:)
+      def self.es_document_from_configured_object_base(object_type:, object:, fields:)
         object_as_json = object.as_json
 
         adapted_object = {

data/lib/connectors_sdk/base/authorization.rb

@@ -0,0 +1,89 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'connectors_shared'
+require 'signet'
+require 'signet/oauth_2'
+require 'signet/oauth_2/client'
+
+module ConnectorsSdk
+  module Base
+    class Authorization
+      class << self
+        def authorization_uri(params)
+          missing = missing_fields(params, %w[client_id])
+          unless missing.blank?
+            raise ConnectorsShared::ClientError.new("Missing required fields: #{missing.join(', ')}")
+          end
+
+          params[:response_type] = 'code'
+          params[:additional_parameters] = additional_parameters
+          client = oauth_client(params)
+          client.authorization_uri.to_s
+        end
+
+        def access_token(params)
+          missing = missing_fields(params, %w[client_id client_secret code redirect_uri])
+          unless missing.blank?
+            raise ConnectorsShared::ClientError.new("Missing required fields: #{missing.join(', ')}")
+          end
+
+          params[:grant_type] = 'authorization_code'
+          client = oauth_client(params)
+          client.fetch_access_token
+        end
+
+        def refresh(params)
+          missing = missing_fields(params, %w[client_id client_secret refresh_token])
+          unless missing.blank?
+            raise ConnectorsShared::ClientError.new("Missing required fields: #{missing.join(', ')}")
+          end
+
+          params[:grant_type] = 'refresh_token'
+          client = oauth_client(params)
+          client.refresh!
+        rescue StandardError => e
+          ConnectorsShared::ExceptionTracking.log_exception(e)
+          raise ConnectorsShared::TokenRefreshFailedError
+        end
+
+        def oauth_client(params)
+          options = params.merge(
+            :authorization_uri => authorization_url,
+            :token_credential_uri => token_credential_uri,
+            :scope => oauth_scope
+          )
+          options[:state] = JSON.dump(options[:state]) if options[:state]
+          Signet::OAuth2::Client.new(options)
+        end
+
+        def missing_fields(params, required = [])
+          Array.wrap(required).select { |field| params[field.to_sym].nil? }
+        end
+
+        def oauth_scope
+          raise 'Not implemented for this connector'
+        end
+
+        private
+
+        def authorization_url
+          raise 'Not implemented for this connector'
+        end
+
+        def token_credential_uri
+          raise 'Not implemented for this connector'
+        end
+
+        def additional_parameters
+          raise 'Not implemented for this connector'
+        end
+      end
+    end
+  end
+end

data/lib/connectors_sdk/base/custom_client.rb

@@ -11,7 +11,6 @@ require 'active_support/core_ext/numeric/time'
 require 'active_support/core_ext/object/deep_dup'
 require 'connectors_shared'
 require 'date'
-require 'active_support/all'
 
 module ConnectorsSdk
   module Base
@@ -75,7 +74,7 @@ module ConnectorsSdk
             faraday.use(*middleware_config)
           end
 
-          faraday.adapter(:httpclient)
+          faraday.adapter :httpclient
         end
       end
 

data/lib/connectors_sdk/base/extractor.rb

@@ -9,10 +9,11 @@ require 'httpclient'
 require 'active_support/core_ext/array/wrap'
 require 'active_support/core_ext/numeric/time'
 require 'active_support/core_ext/object/deep_dup'
-require 'connectors_shared'
 require 'date'
-require 'active_support/all'
+
+require 'connectors_shared'
 require 'stubs/connectors/stats' unless defined?(Rails)
+require 'stubs/connectors' unless defined?(Rails)
 
 module ConnectorsSdk
   module Base

data/lib/connectors_sdk/base/http_call_wrapper.rb

@@ -0,0 +1,135 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'bson'
+
+module ConnectorsSdk
+  module Base
+    class HttpCallWrapper
+      def extractor(params)
+        extractor_class.new(
+          content_source_id: params[:content_source_id] || "GENERATED-#{BSON::ObjectId.new}",
+          service_type: service_type,
+          authorization_data_proc: proc { { access_token: params[:access_token] } },
+          client_proc: proc { client(params) },
+          config: config(params),
+          features: params.fetch(:features, {}) || {}
+        )
+      end
+
+      def extract(params)
+        convert_third_party_errors do
+          extractor = extractor(params)
+
+          extractor.yield_document_changes(:modified_since => extractor.config.cursors[:modified_since]) do |action, doc, download_args_and_proc|
+            download_obj = nil
+            if download_args_and_proc
+              download_obj = {
+                id: download_args_and_proc[0],
+                name: download_args_and_proc[1],
+                size: download_args_and_proc[2],
+                download_args: download_args_and_proc[3]
+              }
+            end
+
+            doc = {
+              :action => action,
+              :document => doc,
+              :download => download_obj
+            }
+
+            yield doc
+          end
+
+          extractor.config.to_h[:cursors]
+        end
+      end
+
+      def deleted(params)
+        convert_third_party_errors do
+          results = []
+          extractor(params).yield_deleted_ids(params[:ids]) do |id|
+            results << id
+          end
+          results
+        end
+      end
+
+      def permissions(params)
+        convert_third_party_errors do
+          extractor(params).yield_permissions(params[:user_id]) do |permissions|
+            return permissions
+          end
+        end
+      end
+
+      def authorization_uri(params)
+        authorization.authorization_uri(params)
+      end
+
+      def access_token(params)
+        authorization.access_token(params)
+      end
+
+      def refresh(params)
+        authorization.refresh(params)
+      end
+
+      def download(params)
+        extractor(params).download(params[:meta])
+      end
+
+      def source_status(params)
+        health_check(params)
+        { :status => 'OK', :statusCode => 200, :message => "Connected to #{name}" }
+      rescue StandardError => e
+        { :status => 'FAILURE', :statusCode => e.is_a?(custom_client_error) ? e.status_code : 500, :message => e.message }
+      end
+
+      def name
+        raise 'Not implemented for this connector'
+      end
+
+      def service_type
+        self.class::SERVICE_TYPE
+      end
+
+      private
+
+      def convert_third_party_errors
+        yield
+      rescue custom_client_error => e
+        raise e.status_code == 401 ? ConnectorsShared::InvalidTokenError : e
+      end
+
+      def extractor_class
+        raise 'Not implemented for this connector'
+      end
+
+      def authorization
+        raise 'Not implemented for this connector'
+      end
+
+      def client(*)
+        raise 'Not implemented for this connector'
+      end
+
+      def custom_client_error
+        raise 'Not implemented for this connector'
+      end
+
+      def config(*)
+        raise 'Not implemented for this connector'
+      end
+
+      def health_check(*)
+        raise 'Not implemented for this connector'
+      end
+    end
+  end
+end

data/lib/connectors_sdk/base/registry.rb

@@ -17,8 +17,8 @@ module ConnectorsSdk
         @connectors[name] = klass
       end
 
-      def connector(name)
-        @connectors[name].new
+      def connector_class(name)
+        @connectors[name]
       end
     end
 
@@ -26,7 +26,9 @@ module ConnectorsSdk
 
     # loading plugins (might replace this with a directory scan and conventions on names)
     require_relative '../share_point/http_call_wrapper'
+    require_relative '../confluence_cloud//http_call_wrapper'
 
-    REGISTRY.register(ConnectorsSdk::SharePoint::SERVICE_TYPE, ConnectorsSdk::SharePoint::HttpCallWrapper)
+    REGISTRY.register(ConnectorsSdk::SharePoint::HttpCallWrapper::SERVICE_TYPE, ConnectorsSdk::SharePoint::HttpCallWrapper)
+    REGISTRY.register(ConnectorsSdk::ConfluenceCloud::HttpCallWrapper::SERVICE_TYPE, ConnectorsSdk::ConfluenceCloud::HttpCallWrapper)
   end
 end

data/lib/connectors_sdk/confluence/adapter.rb

@@ -0,0 +1,216 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'connectors_sdk/base/adapter'
+require 'connectors_shared/extraction_utils'
+require 'nokogiri'
+
+module ConnectorsSdk
+  module Confluence
+    class Adapter < ConnectorsSdk::Base::Adapter
+
+      MAX_CONTENT_COMMENTS_TO_INDEX = 50
+      LEADING_SLASH_REGEXP = /\A\//
+
+      generate_id_helpers :confluence_space, 'confluence_space'
+      generate_id_helpers :confluence_content, 'confluence_content'
+      generate_id_helpers :confluence_attachment, 'confluence_attachment'
+
+      def self.es_document_from_confluence_space(space, base_url, permissions = [])
+        SpaceNode.new(:node => space, :base_url => base_url, :permissions => permissions).to_es_document
+      end
+
+      def self.es_document_from_confluence_content(content, base_url, restrictions = [])
+        ContentNode.new(:node => content, :base_url => base_url, :permissions => restrictions).to_es_document
+      end
+
+      def self.es_document_from_confluence_attachment(attachment, base_url, restrictions = [])
+        AttachmentNode.new(:node => attachment, :base_url => base_url, :permissions => restrictions).to_es_document
+      end
+
+      class Node
+        attr_reader :node, :base_url, :permissions
+
+        def initialize(node:, base_url:, permissions: [])
+          @node = node
+          @base_url = base_url
+          @base_url = "#{base_url}/" unless @base_url.ends_with?('/')
+          @permissions = permissions
+        end
+
+        def to_es_document
+          {
+            :id => id,
+            :title => title,
+            :url => url,
+            :type => ConnectorsSdk::Base::Adapter.normalize_enum(type),
+          }.merge(fields)
+        end
+
+        def id
+          raise NotImplementedError
+        end
+
+        def type
+          raise NotImplementedError
+        end
+
+        def title
+          raise NotImplementedError
+        end
+
+        def url
+          raise NotImplementedError
+        end
+
+        def fields
+          {}
+        end
+
+        protected
+
+        def permissions_hash
+          permissions.blank? ? {} : { ConnectorsShared::Constants::ALLOW_FIELD => permissions }
+        end
+      end
+
+      class SpaceNode < Node
+        def id
+          Confluence::Adapter.confluence_space_id_to_es_id(node.fetch('key'))
+        end
+
+        def type
+          'space'
+        end
+
+        def title
+          node.name
+        end
+
+        def url
+          Addressable::URI.join(base_url, (node._links.webui || node._links.self).gsub(LEADING_SLASH_REGEXP, '')).to_s
+        end
+
+        def path
+          title
+        end
+
+        def fields
+          permissions_hash
+        end
+      end
+
+      class ContentNode < Node
+        def id
+          Confluence::Adapter.confluence_content_id_to_es_id(node.id)
+        end
+
+        def type
+          case node.type
+          when 'page'
+            node.type
+          when 'blogpost'
+            'blog post'
+          else
+            ConnectorsShared::ExceptionTracking.capture_message("Unknown confluence type: #{node.type}")
+            nil
+          end
+        end
+
+        def title
+          node.title
+        end
+
+        def url
+          Addressable::URI.join(base_url, node._links.webui.gsub(LEADING_SLASH_REGEXP, '')).to_s
+        end
+
+        def body
+          text_from_html(node.body.export_view.value)
+        end
+
+        def comments
+          node.children&.comment&.results&.slice(0, MAX_CONTENT_COMMENTS_TO_INDEX)&.map do |comment|
+            text_from_html(comment.body.export_view.value)
+          end&.join("\n")
+        end
+
+        def description
+          [
+            node.space&.name,
+            node.ancestors&.map(&:title) # 'attachment' type nodes do not have `ancestors`, making this logic incomplete
+          ].flatten.select(&:present?).join('/')
+        end
+
+        def path
+          [
+            description,
+            title
+          ].select(&:present?).join('/')
+        end
+
+        def fields
+          {
+            :description => description,
+            :body => body,
+            :comments => comments,
+            :created_by => node.history&.createdBy&.displayName,
+            :project => node.space.try!(:[], :key),
+
+            :created_at => ConnectorsSdk::Base::Adapter.normalize_date(node.history&.createdDate),
+            :last_updated => ConnectorsSdk::Base::Adapter.normalize_date(node.history&.lastUpdated&.when)
+          }.merge(permissions_hash)
+        end
+
+        private
+
+        def text_from_html(raw_html)
+          ConnectorsShared::ExtractionUtils.node_descendant_text(Nokogiri::HTML(raw_html))
+        end
+      end
+
+      class AttachmentNode < ContentNode
+        def id
+          Confluence::Adapter.confluence_attachment_id_to_es_id(node.id)
+        end
+
+        def type
+          'attachment'
+        end
+
+        def fields
+          mime_type = [
+            node.extensions.mediaType,
+            ConnectorsSdk::Base::Adapter.mime_type_for_file(node.title)
+          ].detect(&:present?)
+          extension = ConnectorsSdk::Base::Adapter.extension_for_file(node.title)
+
+          {
+            :size => node.extensions.fileSize,
+            :container => node&.container&.title,
+
+            :description => description,
+            :comments => comments,
+            :created_by => node.history&.createdBy&.displayName,
+            :project => node.space.try!(:[], :key),
+
+            :created_at => ConnectorsSdk::Base::Adapter.normalize_date(node.history&.createdDate),
+            :last_updated => ConnectorsSdk::Base::Adapter.normalize_date(node.history&.lastUpdated&.when)
+          }.merge(permissions_hash).tap do |data|
+            data[:mime_type] = mime_type if mime_type.present?
+            data[:extension] = extension if extension.present?
+          end
+        end
+
+        def to_es_document
+          super.merge(:_fields_to_preserve => ConnectorsSdk::Confluence::Adapter.fields_to_preserve)
+        end
+      end
+    end
+  end
+end