connectors_sdk 8.3.0.0.pre.20220414T060419Z → 8.3.0.0

data/lib/connectors_sdk/confluence/adapter.rb

@@ -0,0 +1,216 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'connectors_sdk/base/adapter'
+require 'connectors_shared/extraction_utils'
+require 'nokogiri'
+
+module ConnectorsSdk
+  module Confluence
+    class Adapter < ConnectorsSdk::Base::Adapter
+
+      MAX_CONTENT_COMMENTS_TO_INDEX = 50
+      LEADING_SLASH_REGEXP = /\A\//
+
+      generate_id_helpers :confluence_space, 'confluence_space'
+      generate_id_helpers :confluence_content, 'confluence_content'
+      generate_id_helpers :confluence_attachment, 'confluence_attachment'
+
+      def self.es_document_from_confluence_space(space, base_url, permissions = [])
+        SpaceNode.new(:node => space, :base_url => base_url, :permissions => permissions).to_es_document
+      end
+
+      def self.es_document_from_confluence_content(content, base_url, restrictions = [])
+        ContentNode.new(:node => content, :base_url => base_url, :permissions => restrictions).to_es_document
+      end
+
+      def self.es_document_from_confluence_attachment(attachment, base_url, restrictions = [])
+        AttachmentNode.new(:node => attachment, :base_url => base_url, :permissions => restrictions).to_es_document
+      end
+
+      class Node
+        attr_reader :node, :base_url, :permissions
+
+        def initialize(node:, base_url:, permissions: [])
+          @node = node
+          @base_url = base_url
+          @base_url = "#{base_url}/" unless @base_url.ends_with?('/')
+          @permissions = permissions
+        end
+
+        def to_es_document
+          {
+            :id => id,
+            :title => title,
+            :url => url,
+            :type => ConnectorsSdk::Base::Adapter.normalize_enum(type),
+          }.merge(fields)
+        end
+
+        def id
+          raise NotImplementedError
+        end
+
+        def type
+          raise NotImplementedError
+        end
+
+        def title
+          raise NotImplementedError
+        end
+
+        def url
+          raise NotImplementedError
+        end
+
+        def fields
+          {}
+        end
+
+        protected
+
+        def permissions_hash
+          permissions.blank? ? {} : { ConnectorsShared::Constants::ALLOW_FIELD => permissions }
+        end
+      end
+
+      class SpaceNode < Node
+        def id
+          Confluence::Adapter.confluence_space_id_to_es_id(node.fetch('key'))
+        end
+
+        def type
+          'space'
+        end
+
+        def title
+          node.name
+        end
+
+        def url
+          Addressable::URI.join(base_url, (node._links.webui || node._links.self).gsub(LEADING_SLASH_REGEXP, '')).to_s
+        end
+
+        def path
+          title
+        end
+
+        def fields
+          permissions_hash
+        end
+      end
+
+      class ContentNode < Node
+        def id
+          Confluence::Adapter.confluence_content_id_to_es_id(node.id)
+        end
+
+        def type
+          case node.type
+          when 'page'
+            node.type
+          when 'blogpost'
+            'blog post'
+          else
+            ConnectorsShared::ExceptionTracking.capture_message("Unknown confluence type: #{node.type}")
+            nil
+          end
+        end
+
+        def title
+          node.title
+        end
+
+        def url
+          Addressable::URI.join(base_url, node._links.webui.gsub(LEADING_SLASH_REGEXP, '')).to_s
+        end
+
+        def body
+          text_from_html(node.body.export_view.value)
+        end
+
+        def comments
+          node.children&.comment&.results&.slice(0, MAX_CONTENT_COMMENTS_TO_INDEX)&.map do |comment|
+            text_from_html(comment.body.export_view.value)
+          end&.join("\n")
+        end
+
+        def description
+          [
+            node.space&.name,
+            node.ancestors&.map(&:title) # 'attachment' type nodes do not have `ancestors`, making this logic incomplete
+          ].flatten.select(&:present?).join('/')
+        end
+
+        def path
+          [
+            description,
+            title
+          ].select(&:present?).join('/')
+        end
+
+        def fields
+          {
+            :description => description,
+            :body => body,
+            :comments => comments,
+            :created_by => node.history&.createdBy&.displayName,
+            :project => node.space.try!(:[], :key),
+
+            :created_at => ConnectorsSdk::Base::Adapter.normalize_date(node.history&.createdDate),
+            :last_updated => ConnectorsSdk::Base::Adapter.normalize_date(node.history&.lastUpdated&.when)
+          }.merge(permissions_hash)
+        end
+
+        private
+
+        def text_from_html(raw_html)
+          ConnectorsShared::ExtractionUtils.node_descendant_text(Nokogiri::HTML(raw_html))
+        end
+      end
+
+      class AttachmentNode < ContentNode
+        def id
+          Confluence::Adapter.confluence_attachment_id_to_es_id(node.id)
+        end
+
+        def type
+          'attachment'
+        end
+
+        def fields
+          mime_type = [
+            node.extensions.mediaType,
+            ConnectorsSdk::Base::Adapter.mime_type_for_file(node.title)
+          ].detect(&:present?)
+          extension = ConnectorsSdk::Base::Adapter.extension_for_file(node.title)
+
+          {
+            :size => node.extensions.fileSize,
+            :container => node&.container&.title,
+
+            :description => description,
+            :comments => comments,
+            :created_by => node.history&.createdBy&.displayName,
+            :project => node.space.try!(:[], :key),
+
+            :created_at => ConnectorsSdk::Base::Adapter.normalize_date(node.history&.createdDate),
+            :last_updated => ConnectorsSdk::Base::Adapter.normalize_date(node.history&.lastUpdated&.when)
+          }.merge(permissions_hash).tap do |data|
+            data[:mime_type] = mime_type if mime_type.present?
+            data[:extension] = extension if extension.present?
+          end
+        end
+
+        def to_es_document
+          super.merge(:_fields_to_preserve => ConnectorsSdk::Confluence::Adapter.fields_to_preserve)
+        end
+      end
+    end
+  end
+end

data/lib/connectors_sdk/confluence/custom_client.rb

@@ -0,0 +1,143 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'active_support/json'
+require 'active_support/core_ext/array'
+require 'active_support/core_ext/object'
+require 'hashie/mash'
+
+require 'connectors_sdk/atlassian/custom_client'
+
+module ConnectorsSdk
+  module Confluence
+    class CustomClient < ConnectorsSdk::Atlassian::CustomClient
+      SEARCH_ENDPOINT = 'rest/api/search'
+      CONTENT_SEARCH_ENDPOINT = 'rest/api/content/search'
+      CONTENT_EXPAND_FIELDS = %w(body.export_view history.lastUpdated ancestors space children.comment.body.export_view container).freeze
+
+      def content(space:, types: [], start_at: 0, order_field_and_direction: 'created asc', updated_after: nil, next_value: nil)
+        search_helper(
+          :endpoint => CONTENT_SEARCH_ENDPOINT,
+          :space => space,
+          :types => types,
+          :start_at => start_at,
+          :order_field_and_direction => order_field_and_direction,
+          :updated_after => updated_after,
+          :next_value => next_value
+        )
+      end
+
+      def content_by_id(content_id, include_permissions: false, expand_fields: CONTENT_EXPAND_FIELDS)
+        endpoint = "rest/api/content/#{Faraday::Utils.escape(content_id)}"
+        if include_permissions
+          expand_fields = expand_fields.dup
+          expand_fields << 'restrictions.read.restrictions.user'
+          expand_fields << 'restrictions.read.restrictions.group'
+        end
+        response = begin
+          parse_and_raise_if_necessary!(get(endpoint, :status => 'any', :expand => expand_fields.join(',')))
+        rescue ContentConvertibleError
+          # Confluence has a bug when trying to expand `container` for certain items:
+          # https://jira.atlassian.com/browse/CONFSERVER-40475
+          Connectors::Stats.increment('custom_client.confluence.error.content_convertible')
+          parse_and_raise_if_necessary!(get(endpoint, :status => 'any', :expand => (expand_fields - ['container']).join(',')))
+        end
+        Hashie::Mash.new(response)
+      end
+
+      def spaces(start_at: 0, limit: 50, space_keys: nil, include_permissions: false)
+        params = {
+          :start => start_at,
+          :limit => limit
+        }
+        params[:spaceKey] = space_keys if space_keys.present?
+        params[:expand] = 'permissions' if include_permissions
+        response = get('rest/api/space', params)
+        Hashie::Mash.new(parse_and_raise_if_necessary!(response))
+      end
+
+      def search(
+        space: nil,
+        start_at: 0,
+        limit: 50,
+        types: [],
+        expand_fields: [],
+        order_field_and_direction: 'created asc',
+        updated_after: nil
+      )
+        search_helper(
+          :endpoint => SEARCH_ENDPOINT,
+          :space => space,
+          :start_at => start_at,
+          :limit => limit,
+          :types => types,
+          :expand_fields => expand_fields,
+          :order_field_and_direction => order_field_and_direction,
+          :updated_after => updated_after
+        )
+      end
+
+      def content_search(cql, expand: [], limit: 25)
+        response = get(CONTENT_SEARCH_ENDPOINT, :cql => cql, :expand => expand.join(','), :limit => limit)
+        Hashie::Mash.new(parse_and_raise_if_necessary!(response))
+      end
+
+      def me
+        response = get('rest/api/user/current')
+        Hashie::Mash.new(parse_and_raise_if_necessary!(response))
+      end
+
+      private
+
+      def search_helper(
+        endpoint:,
+        space: nil,
+        start_at: 0,
+        limit: 50,
+        types: [],
+        expand_fields: [],
+        order_field_and_direction: 'created asc',
+        updated_after: nil,
+        next_value: nil
+      )
+
+        response =
+          if next_value.present?
+            get(next_value.reverse.chomp('/').reverse)
+          else
+            params = {
+              :cql => generate_cql(:space => space, :types => types, :order_field_and_direction => order_field_and_direction, :updated_after => updated_after),
+              :start => start_at,
+              :expand => expand_fields.join(','),
+              :limit => limit
+            }
+
+            get(endpoint, params)
+          end
+
+        Hashie::Mash.new(parse_and_raise_if_necessary!(response))
+      end
+
+      def generate_cql(space: nil, types: nil, order_field_and_direction: nil, updated_after: nil)
+        query_conditions = []
+        query_conditions << "space=\"#{space}\"" if space
+        query_conditions << "type in (#{types.join(',')})" if types.any?
+        query_conditions << "lastmodified > \"#{format_date(updated_after)}\"" if updated_after.present?
+
+        query_parts = [query_conditions.join(' AND ')]
+        query_parts << "order by #{order_field_and_direction}" if order_field_and_direction
+
+        query_parts.join(' ').strip
+      end
+
+      def format_date(date)
+        DateTime.parse(date).strftime('%Y-%m-%d %H:%M')
+      end
+    end
+  end
+end

data/lib/connectors_sdk/confluence/extractor.rb

@@ -0,0 +1,265 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'connectors_sdk/atlassian/custom_client'
+require 'connectors_sdk/confluence/adapter'
+require 'connectors_sdk/base/extractor'
+
+module ConnectorsSdk
+  module Confluence
+    class Extractor < ConnectorsSdk::Base::Extractor
+      CONTENT_OFFSET_CURSOR_KEY = 'content'
+      CONTENT_NEXT_CURSOR_KEY = 'next'
+      CONTENT_MODIFIED_SINCE_NEXT_CURSOR_KEY = 'modified_since_next'
+      CONTENT_MODIFIED_SINCE_CURSOR_KEY = 'content_modified_at'
+
+      ConnectorsSdk::Base::Extractor::TRANSIENT_SERVER_ERROR_CLASSES << Atlassian::CustomClient::ServiceUnavailableError
+
+      def yield_document_changes(modified_since: nil)
+        @space_permissions_cache = {}
+        @content_restriction_cache = {}
+        yield_spaces do |space|
+          yield_single_document_change(:identifier => "Confluence Space: #{space&.fetch(:key)} (#{space&.webui})") do
+            permissions = config.index_permissions ? get_space_permissions(space) : []
+            yield :create_or_update, Confluence::Adapter.es_document_from_confluence_space(space, content_base_url, permissions)
+          end
+
+          yield_content_for_space(
+            :space => space[:key],
+            :types => %w(page blogpost attachment),
+            :modified_since => modified_since
+          ) do |content|
+            restrictions = config.index_permissions ? get_content_restrictions(content) : []
+            if content.type == 'attachment'
+              document = Confluence::Adapter.es_document_from_confluence_attachment(content, content_base_url, restrictions)
+              download_args = download_args_and_proc(
+                id: document.fetch(:id),
+                name: content.title,
+                size: content.extensions.fileSize,
+                download_args: { content: content }
+              ) do |args|
+                download(args)
+              end
+              yield :create_or_update, document, download_args
+            else
+              yield :create_or_update, Confluence::Adapter.es_document_from_confluence_content(content, content_base_url, restrictions)
+            end
+          end
+        end
+      end
+
+      def yield_deleted_ids(ids)
+        id_groups = ids.group_by do |id|
+          if Confluence::Adapter.es_id_is_confluence_space_id?(id)
+            :space
+          elsif Confluence::Adapter.es_id_is_confluence_content_id?(id)
+            :content
+          elsif Confluence::Adapter.es_id_is_confluence_attachment_id?(id)
+            :attachment
+          else
+            :unknown
+          end
+        end
+
+        %i(space content attachment).each do |group|
+          confluence_ids = Array(id_groups[group]).map { |id| Confluence::Adapter.public_send("es_id_to_confluence_#{group}_id", id) }
+          get_ids_for_deleted(confluence_ids, group).each do |deleted_id|
+            yield Confluence::Adapter.public_send("confluence_#{group}_id_to_es_id", deleted_id)
+          end
+        end
+      end
+
+      def download(item)
+        content = item[:content]
+        client.download("#{content._links.base}#{content._links.download}").body
+      end
+
+      private
+
+      def content_base_url
+        config.base_url
+      end
+
+      def yield_spaces
+        @space_cursor ||= 0
+        loop do
+          response = client.spaces(:start_at => @space_cursor, :include_permissions => config.index_permissions)
+          response.results.each do |space|
+            yield space
+            @space_cursor += 1
+          end
+          break unless should_continue_looping?(response)
+          log_info("Requesting more spaces with cursor: #{@space_cursor}")
+        end
+      end
+
+      def yield_content_for_space(space:, types:, modified_since:)
+        loop do
+          response = client.content(
+            :space => space,
+            :types => types,
+            :order_field_and_direction => modified_since ? 'lastmodified asc' : 'created asc',
+            cursoring_param(:modified_since => modified_since, :space => space) => cursoring_value(:modified_since => modified_since, :space => space)
+          )
+
+          response.results.each do |result|
+            yield_single_document_change(:identifier => "Confluence ID: #{result&.id} (#{result&.webui})") do
+              content = client.content_by_id(result.id, :include_permissions => config.index_permissions)
+              yield content if content.status == 'current'
+            end
+          end
+          update_content_cursors(space, response, modified_since)
+
+          break unless should_continue_looping?(response)
+          log_info("Requesting more content for space #{space} with cursor: #{cursoring_param(:modified_since => modified_since, :space => space)} #{cursoring_value(:modified_since => modified_since, :space => space)}")
+        end
+      end
+
+      def next_cursor_key(modified_since:)
+        modified_since ? CONTENT_MODIFIED_SINCE_NEXT_CURSOR_KEY : CONTENT_NEXT_CURSOR_KEY
+      end
+
+      def cursoring_param(modified_since:, space:)
+        return :next_value if config.cursors.dig(next_cursor_key(:modified_since => modified_since), space).present?
+
+        modified_since ? :updated_after : :start_at
+      end
+
+      def cursoring_value(modified_since:, space:)
+        next_value = config.cursors.dig(next_cursor_key(:modified_since => modified_since), space)
+        return next_value if next_value.present?
+
+        get_content_cursors(space, modified_since)[space]
+      end
+
+      def update_content_cursors(space, response, modified_since)
+        if response._links&.next.present?
+          config.cursors[next_cursor_key(:modified_since => modified_since)] ||= {}
+          config.cursors[next_cursor_key(:modified_since => modified_since)][space] = response._links.next
+        end
+
+        if response.results && modified_since
+          updated_cursor = response.results.last&.history&.lastUpdated&.when
+          config.cursors[CONTENT_MODIFIED_SINCE_CURSOR_KEY][space] = updated_cursor if updated_cursor
+        else
+          config.cursors[CONTENT_OFFSET_CURSOR_KEY][space] += response.results.size
+        end
+      end
+
+      def get_content_cursors(space, modified_since)
+        modified_since ? get_content_modified_since_cursors(space, modified_since) : get_content_offset_cursors(space)
+      end
+
+      def get_content_offset_cursors(space)
+        config.cursors[CONTENT_OFFSET_CURSOR_KEY] ||= {}
+        config.cursors[CONTENT_OFFSET_CURSOR_KEY][space] ||= 0
+        config.cursors[CONTENT_OFFSET_CURSOR_KEY]
+      end
+
+      def get_content_modified_since_cursors(space, modified_since)
+        config.cursors[CONTENT_MODIFIED_SINCE_CURSOR_KEY] ||= {}
+        config.cursors[CONTENT_MODIFIED_SINCE_CURSOR_KEY][space] ||= modified_since.to_s
+        config.cursors[CONTENT_MODIFIED_SINCE_CURSOR_KEY]
+      end
+
+      def should_continue_looping?(response)
+        response.results&.size.to_i > 0 && response._links.next.present?
+      end
+
+      def get_ids_for_deleted(search_ids, group)
+        return [] if search_ids.empty?
+
+        response, id_sym =
+          if group == :space
+            [client.spaces(:space_keys => search_ids, :limit => search_ids.size), :key]
+          else
+            [client.content_search("id in (#{search_ids.join(',')})", :limit => search_ids.size), :id]
+          end
+        found_ids = response.results.map { |result| result.fetch(id_sym) }.map(&:to_s)
+        search_ids - found_ids
+      end
+
+      def get_space_permissions(space)
+        space_permissions = space.permissions&.select { |permission| %w(read administer).include?(permission.operation.operation) }
+        if space_permissions.nil? || space_permissions.any?(&:anonymousAccess)
+          @space_permissions_cache[space.fetch('key')] = []
+          return []
+        end
+
+        subjects = space_permissions.flat_map(&:subjects)
+        permissions = [
+          subjects.select { |subject| subject.has_key?(:user) }.map { |subject| subject.user.results.map { |user| "user:#{user.accountId}" } },
+          subjects.select { |subject| subject.has_key?(:group) }.map { |subject| subject.group.results.map { |group| "group:#{group.name}" } }
+        ].flatten.uniq
+        @space_permissions_cache[space.fetch('key')] = permissions
+      end
+
+      def get_user_spaces(user_id)
+        (@space_permissions_cache || {}).select { |_, permissions| "user:#{user_id}".in?(permissions) }.keys.uniq
+      end
+
+      def get_group_spaces(group_name)
+        (@space_permissions_cache || {}).select { |_, permissions| "group:#{group_name}".in?(permissions) }.keys.uniq
+      end
+
+      # get_content_restrictions returns the final restriction of the content, taking into consideration inherited restrictions
+      # if the content is an attachment, the final restriction will will be either space permissions
+      # or (in case they aren't empty) the intersection of
+      #  1. its restriction
+      #  2. its container's restriction
+      #  3. its container's ancestors' restriction
+      # if the content is a page or blog post, the final restriction will be either space permissions
+      # or (in case they aren't empty) the intersection of
+      #  1. its restrictions
+      #  2. its ancestors' restrictions
+      def get_content_restrictions(content)
+        restrictions = []
+        restrictions << extract_restrictions(content.restrictions&.read&.restrictions)
+
+        # space permissions should always be available in cache
+        space_key = content.space&.fetch('key')
+        space_restrictions = (@space_permissions_cache[space_key] || [])
+
+        ancestors = content.ancestors || []
+
+        if content.type == 'attachment'
+          if content.container&.type == 'global'
+            log_info("Skipping ancestor restrictions as it is a space and should already be cached: [#{content.id}] in [#{content.container&.type}_#{content.container&.id}]")
+          else
+            container = client.content_by_id(content.container&.id, :include_permissions => true)
+            @content_restriction_cache[container.id] ||= extract_restrictions(container.restrictions&.read&.restrictions)
+            restrictions << @content_restriction_cache[container.id]
+            ancestors = container.ancestors
+          end
+        end
+
+        ancestors.each do |ancestor|
+          restrictions << get_restrictions_by_content_id(ancestor.id)
+        end
+
+        combined_restrictions = restrictions.select(&:any?).reduce(:&)
+        @content_restriction_cache[content.id] = combined_restrictions || []
+        (combined_restrictions || space_restrictions).map { |restriction| "#{space_key}/#{restriction}" }
+      end
+
+      def get_restrictions_by_content_id(content_id)
+        @content_restriction_cache[content_id] ||= begin
+          content = client.content_by_id(content_id, :include_permissions => true)
+          extract_restrictions(content.restrictions&.read&.restrictions)
+        end
+      end
+
+      def extract_restrictions(restrictions)
+        [
+          restrictions&.user&.results&.map { |user| "user:#{user.accountId}" },
+          restrictions&.group&.results&.map { |group| "group:#{group.name}" }
+        ].flatten.compact.uniq
+      end
+    end
+  end
+end

data/lib/connectors_sdk/confluence_cloud/authorization.rb

@@ -0,0 +1,64 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'connectors_sdk/base/authorization'
+
+module ConnectorsSdk
+  module ConfluenceCloud
+    class Authorization < ConnectorsSdk::Base::Authorization
+      class << self
+        def access_token(params)
+          tokens = super
+          tokens.merge(:cloud_id => fetch_cloud_id(tokens['access_token'], params[:external_connector_base_url]))
+        end
+
+        def oauth_scope
+          %w[
+            offline_access
+
+            read:confluence-content.all
+            read:confluence-content.summary
+            read:confluence-props
+            read:confluence-space.summary
+            read:confluence-user
+            readonly:content.attachment:confluence
+            search:confluence
+          ]
+        end
+
+        private
+
+        def authorization_url
+          'https://auth.atlassian.com/authorize'
+        end
+
+        def token_credential_uri
+          'https://auth.atlassian.com/oauth/token'
+        end
+
+        def additional_parameters
+          { :prompt => 'consent', :audience => 'api.atlassian.com' }
+        end
+
+        def fetch_cloud_id(access_token, base_url)
+          response = HTTPClient.new.get(
+            'https://api.atlassian.com/oauth/token/accessible-resources',
+            nil,
+            'Accept' => 'application/json',
+            'Authorization' => "Bearer #{access_token}"
+          )
+          raise 'unable to fetch cloud id' unless HTTP::Status.successful?(response.status)
+          json = JSON.parse(response.body)
+
+          site = json.find { |sites| sites['url'] == base_url } || {}
+          site.fetch('id') { raise 'unable to fetch cloud id' }
+        end
+      end
+    end
+  end
+end