connectors_sdk 8.3.0.0.pre.20220414T060419Z → 8.3.0.0.pre.20220510T144908Z

data/lib/connectors_sdk/confluence/custom_client.rb

@@ -0,0 +1,143 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'active_support/json'
+require 'active_support/core_ext/array'
+require 'active_support/core_ext/object'
+require 'hashie/mash'
+
+require 'connectors_sdk/atlassian/custom_client'
+
+module ConnectorsSdk
+  module Confluence
+    class CustomClient < ConnectorsSdk::Atlassian::CustomClient
+      SEARCH_ENDPOINT = 'rest/api/search'
+      CONTENT_SEARCH_ENDPOINT = 'rest/api/content/search'
+      CONTENT_EXPAND_FIELDS = %w(body.export_view history.lastUpdated ancestors space children.comment.body.export_view container).freeze
+
+      def content(space:, types: [], start_at: 0, order_field_and_direction: 'created asc', updated_after: nil, next_value: nil)
+        search_helper(
+          :endpoint => CONTENT_SEARCH_ENDPOINT,
+          :space => space,
+          :types => types,
+          :start_at => start_at,
+          :order_field_and_direction => order_field_and_direction,
+          :updated_after => updated_after,
+          :next_value => next_value
+        )
+      end
+
+      def content_by_id(content_id, include_permissions: false, expand_fields: CONTENT_EXPAND_FIELDS)
+        endpoint = "rest/api/content/#{Faraday::Utils.escape(content_id)}"
+        if include_permissions
+          expand_fields = expand_fields.dup
+          expand_fields << 'restrictions.read.restrictions.user'
+          expand_fields << 'restrictions.read.restrictions.group'
+        end
+        response = begin
+          parse_and_raise_if_necessary!(get(endpoint, :status => 'any', :expand => expand_fields.join(',')))
+        rescue ContentConvertibleError
+          # Confluence has a bug when trying to expand `container` for certain items:
+          # https://jira.atlassian.com/browse/CONFSERVER-40475
+          Connectors::Stats.increment('custom_client.confluence.error.content_convertible')
+          parse_and_raise_if_necessary!(get(endpoint, :status => 'any', :expand => (expand_fields - ['container']).join(',')))
+        end
+        Hashie::Mash.new(response)
+      end
+
+      def spaces(start_at: 0, limit: 50, space_keys: nil, include_permissions: false)
+        params = {
+          :start => start_at,
+          :limit => limit
+        }
+        params[:spaceKey] = space_keys if space_keys.present?
+        params[:expand] = 'permissions' if include_permissions
+        response = get('rest/api/space', params)
+        Hashie::Mash.new(parse_and_raise_if_necessary!(response))
+      end
+
+      def search(
+        space: nil,
+        start_at: 0,
+        limit: 50,
+        types: [],
+        expand_fields: [],
+        order_field_and_direction: 'created asc',
+        updated_after: nil
+      )
+        search_helper(
+          :endpoint => SEARCH_ENDPOINT,
+          :space => space,
+          :start_at => start_at,
+          :limit => limit,
+          :types => types,
+          :expand_fields => expand_fields,
+          :order_field_and_direction => order_field_and_direction,
+          :updated_after => updated_after
+        )
+      end
+
+      def content_search(cql, expand: [], limit: 25)
+        response = get(CONTENT_SEARCH_ENDPOINT, :cql => cql, :expand => expand.join(','), :limit => limit)
+        Hashie::Mash.new(parse_and_raise_if_necessary!(response))
+      end
+
+      def me
+        response = get('rest/api/user/current')
+        Hashie::Mash.new(parse_and_raise_if_necessary!(response))
+      end
+
+      private
+
+      def search_helper(
+        endpoint:,
+        space: nil,
+        start_at: 0,
+        limit: 50,
+        types: [],
+        expand_fields: [],
+        order_field_and_direction: 'created asc',
+        updated_after: nil,
+        next_value: nil
+      )
+
+        response =
+          if next_value.present?
+            get(next_value.reverse.chomp('/').reverse)
+          else
+            params = {
+              :cql => generate_cql(:space => space, :types => types, :order_field_and_direction => order_field_and_direction, :updated_after => updated_after),
+              :start => start_at,
+              :expand => expand_fields.join(','),
+              :limit => limit
+            }
+
+            get(endpoint, params)
+          end
+
+        Hashie::Mash.new(parse_and_raise_if_necessary!(response))
+      end
+
+      def generate_cql(space: nil, types: nil, order_field_and_direction: nil, updated_after: nil)
+        query_conditions = []
+        query_conditions << "space=\"#{space}\"" if space
+        query_conditions << "type in (#{types.join(',')})" if types.any?
+        query_conditions << "lastmodified > \"#{format_date(updated_after)}\"" if updated_after.present?
+
+        query_parts = [query_conditions.join(' AND ')]
+        query_parts << "order by #{order_field_and_direction}" if order_field_and_direction
+
+        query_parts.join(' ').strip
+      end
+
+      def format_date(date)
+        DateTime.parse(date).strftime('%Y-%m-%d %H:%M')
+      end
+    end
+  end
+end

data/lib/connectors_sdk/confluence/extractor.rb

@@ -0,0 +1,270 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'connectors_sdk/atlassian/custom_client'
+require 'connectors_sdk/confluence/adapter'
+require 'connectors_sdk/base/extractor'
+
+module ConnectorsSdk
+  module Confluence
+    class Extractor < ConnectorsSdk::Base::Extractor
+      CONTENT_OFFSET_CURSOR_KEY = 'content'
+      CONTENT_NEXT_CURSOR_KEY = 'next'
+      CONTENT_MODIFIED_SINCE_NEXT_CURSOR_KEY = 'modified_since_next'
+      CONTENT_MODIFIED_SINCE_CURSOR_KEY = 'content_modified_at'
+
+      ConnectorsSdk::Base::Extractor::TRANSIENT_SERVER_ERROR_CLASSES << Atlassian::CustomClient::ServiceUnavailableError
+
+      def yield_document_changes(modified_since: nil, break_after_page: false)
+        @space_permissions_cache = {}
+        @content_restriction_cache = {}
+        yield_spaces do |space|
+          yield_single_document_change(:identifier => "Confluence Space: #{space&.fetch(:key)} (#{space&.webui})") do
+            permissions = config.index_permissions ? get_space_permissions(space) : []
+            yield :create_or_update, Confluence::Adapter.es_document_from_confluence_space(space, content_base_url, permissions)
+          end
+
+          yield_content_for_space(
+            :space => space[:key],
+            :types => %w(page blogpost attachment),
+            :modified_since => modified_since
+          ) do |content|
+            restrictions = config.index_permissions ? get_content_restrictions(content) : []
+            if content.type == 'attachment'
+              document = Confluence::Adapter.es_document_from_confluence_attachment(content, content_base_url, restrictions)
+              download_args = download_args_and_proc(
+                id: document.fetch(:id),
+                name: content.title,
+                size: content.extensions.fileSize,
+                download_args: { content: content }
+              ) do |args|
+                download(args)
+              end
+              yield :create_or_update, document, download_args
+            else
+              yield :create_or_update, Confluence::Adapter.es_document_from_confluence_content(content, content_base_url, restrictions)
+            end
+          end
+
+          if break_after_page
+            @completed = true
+            break
+          end
+        end
+      end
+
+      def yield_deleted_ids(ids)
+        id_groups = ids.group_by do |id|
+          if Confluence::Adapter.es_id_is_confluence_space_id?(id)
+            :space
+          elsif Confluence::Adapter.es_id_is_confluence_content_id?(id)
+            :content
+          elsif Confluence::Adapter.es_id_is_confluence_attachment_id?(id)
+            :attachment
+          else
+            :unknown
+          end
+        end
+
+        %i(space content attachment).each do |group|
+          confluence_ids = Array(id_groups[group]).map { |id| Confluence::Adapter.public_send("es_id_to_confluence_#{group}_id", id) }
+          get_ids_for_deleted(confluence_ids, group).each do |deleted_id|
+            yield Confluence::Adapter.public_send("confluence_#{group}_id_to_es_id", deleted_id)
+          end
+        end
+      end
+
+      def download(item)
+        content = item[:content]
+        client.download("#{content._links.base}#{content._links.download}").body
+      end
+
+      private
+
+      def content_base_url
+        'https://workplace-search.atlassian.net/wiki'
+      end
+
+      def yield_spaces
+        @space_cursor ||= 0
+        loop do
+          response = client.spaces(:start_at => @space_cursor, :include_permissions => config.index_permissions)
+          response.results.each do |space|
+            yield space
+            @space_cursor += 1
+          end
+          break unless should_continue_looping?(response)
+          log_info("Requesting more spaces with cursor: #{@space_cursor}")
+        end
+      end
+
+      def yield_content_for_space(space:, types:, modified_since:)
+        loop do
+          response = client.content(
+            :space => space,
+            :types => types,
+            :order_field_and_direction => modified_since ? 'lastmodified asc' : 'created asc',
+            cursoring_param(:modified_since => modified_since, :space => space) => cursoring_value(:modified_since => modified_since, :space => space)
+          )
+
+          response.results.each do |result|
+            yield_single_document_change(:identifier => "Confluence ID: #{result&.id} (#{result&.webui})") do
+              content = client.content_by_id(result.id, :include_permissions => config.index_permissions)
+              yield content if content.status == 'current'
+            end
+          end
+          update_content_cursors(space, response, modified_since)
+
+          break unless should_continue_looping?(response)
+          log_info("Requesting more content for space #{space} with cursor: #{cursoring_param(:modified_since => modified_since, :space => space)} #{cursoring_value(:modified_since => modified_since, :space => space)}")
+        end
+      end
+
+      def next_cursor_key(modified_since:)
+        modified_since ? CONTENT_MODIFIED_SINCE_NEXT_CURSOR_KEY : CONTENT_NEXT_CURSOR_KEY
+      end
+
+      def cursoring_param(modified_since:, space:)
+        return :next_value if config.cursors.dig(next_cursor_key(:modified_since => modified_since), space).present?
+
+        modified_since ? :updated_after : :start_at
+      end
+
+      def cursoring_value(modified_since:, space:)
+        next_value = config.cursors.dig(next_cursor_key(:modified_since => modified_since), space)
+        return next_value if next_value.present?
+
+        get_content_cursors(space, modified_since)[space]
+      end
+
+      def update_content_cursors(space, response, modified_since)
+        if response._links&.next.present?
+          config.cursors[next_cursor_key(:modified_since => modified_since)] ||= {}
+          config.cursors[next_cursor_key(:modified_since => modified_since)][space] = response._links.next
+        end
+
+        if response.results && modified_since
+          updated_cursor = response.results.last&.history&.lastUpdated&.when
+          config.cursors[CONTENT_MODIFIED_SINCE_CURSOR_KEY][space] = updated_cursor if updated_cursor
+        else
+          config.cursors[CONTENT_OFFSET_CURSOR_KEY][space] += response.results.size
+        end
+      end
+
+      def get_content_cursors(space, modified_since)
+        modified_since ? get_content_modified_since_cursors(space, modified_since) : get_content_offset_cursors(space)
+      end
+
+      def get_content_offset_cursors(space)
+        config.cursors[CONTENT_OFFSET_CURSOR_KEY] ||= {}
+        config.cursors[CONTENT_OFFSET_CURSOR_KEY][space] ||= 0
+        config.cursors[CONTENT_OFFSET_CURSOR_KEY]
+      end
+
+      def get_content_modified_since_cursors(space, modified_since)
+        config.cursors[CONTENT_MODIFIED_SINCE_CURSOR_KEY] ||= {}
+        config.cursors[CONTENT_MODIFIED_SINCE_CURSOR_KEY][space] ||= modified_since.to_s
+        config.cursors[CONTENT_MODIFIED_SINCE_CURSOR_KEY]
+      end
+
+      def should_continue_looping?(response)
+        response.results&.size.to_i > 0 && response._links.next.present?
+      end
+
+      def get_ids_for_deleted(search_ids, group)
+        return [] if search_ids.empty?
+
+        response, id_sym =
+          if group == :space
+            [client.spaces(:space_keys => search_ids, :limit => search_ids.size), :key]
+          else
+            [client.content_search("id in (#{search_ids.join(',')})", :limit => search_ids.size), :id]
+          end
+        found_ids = response.results.map { |result| result.fetch(id_sym) }.map(&:to_s)
+        search_ids - found_ids
+      end
+
+      def get_space_permissions(space)
+        space_permissions = space.permissions&.select { |permission| %w(read administer).include?(permission.operation.operation) }
+        if space_permissions.nil? || space_permissions.any?(&:anonymousAccess)
+          @space_permissions_cache[space.fetch('key')] = []
+          return []
+        end
+
+        subjects = space_permissions.flat_map(&:subjects)
+        permissions = [
+          subjects.select { |subject| subject.has_key?(:user) }.map { |subject| subject.user.results.map { |user| "user:#{user.accountId}" } },
+          subjects.select { |subject| subject.has_key?(:group) }.map { |subject| subject.group.results.map { |group| "group:#{group.name}" } }
+        ].flatten.uniq
+        @space_permissions_cache[space.fetch('key')] = permissions
+      end
+
+      def get_user_spaces(user_id)
+        (@space_permissions_cache || {}).select { |_, permissions| "user:#{user_id}".in?(permissions) }.keys.uniq
+      end
+
+      def get_group_spaces(group_name)
+        (@space_permissions_cache || {}).select { |_, permissions| "group:#{group_name}".in?(permissions) }.keys.uniq
+      end
+
+      # get_content_restrictions returns the final restriction of the content, taking into consideration inherited restrictions
+      # if the content is an attachment, the final restriction will will be either space permissions
+      # or (in case they aren't empty) the intersection of
+      #  1. its restriction
+      #  2. its container's restriction
+      #  3. its container's ancestors' restriction
+      # if the content is a page or blog post, the final restriction will be either space permissions
+      # or (in case they aren't empty) the intersection of
+      #  1. its restrictions
+      #  2. its ancestors' restrictions
+      def get_content_restrictions(content)
+        restrictions = []
+        restrictions << extract_restrictions(content.restrictions&.read&.restrictions)
+
+        # space permissions should always be available in cache
+        space_key = content.space&.fetch('key')
+        space_restrictions = (@space_permissions_cache[space_key] || [])
+
+        ancestors = content.ancestors || []
+
+        if content.type == 'attachment'
+          if content.container&.type == 'global'
+            log_info("Skipping ancestor restrictions as it is a space and should already be cached: [#{content.id}] in [#{content.container&.type}_#{content.container&.id}]")
+          else
+            container = client.content_by_id(content.container&.id, :include_permissions => true)
+            @content_restriction_cache[container.id] ||= extract_restrictions(container.restrictions&.read&.restrictions)
+            restrictions << @content_restriction_cache[container.id]
+            ancestors = container.ancestors
+          end
+        end
+
+        ancestors.each do |ancestor|
+          restrictions << get_restrictions_by_content_id(ancestor.id)
+        end
+
+        combined_restrictions = restrictions.select(&:any?).reduce(:&)
+        @content_restriction_cache[content.id] = combined_restrictions || []
+        (combined_restrictions || space_restrictions).map { |restriction| "#{space_key}/#{restriction}" }
+      end
+
+      def get_restrictions_by_content_id(content_id)
+        @content_restriction_cache[content_id] ||= begin
+          content = client.content_by_id(content_id, :include_permissions => true)
+          extract_restrictions(content.restrictions&.read&.restrictions)
+        end
+      end
+
+      def extract_restrictions(restrictions)
+        [
+          restrictions&.user&.results&.map { |user| "user:#{user.accountId}" },
+          restrictions&.group&.results&.map { |group| "group:#{group.name}" }
+        ].flatten.compact.uniq
+      end
+    end
+  end
+end

data/lib/connectors_sdk/confluence_cloud/authorization.rb

@@ -0,0 +1,64 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'connectors_sdk/base/authorization'
+
+module ConnectorsSdk
+  module ConfluenceCloud
+    class Authorization < ConnectorsSdk::Base::Authorization
+      class << self
+        def access_token(params)
+          tokens = super
+          tokens.merge(:cloud_id => fetch_cloud_id(tokens['access_token'], params[:external_connector_base_url]))
+        end
+
+        def oauth_scope
+          %w[
+            offline_access
+
+            read:confluence-content.all
+            read:confluence-content.summary
+            read:confluence-props
+            read:confluence-space.summary
+            read:confluence-user
+            readonly:content.attachment:confluence
+            search:confluence
+          ]
+        end
+
+        private
+
+        def authorization_url
+          'https://auth.atlassian.com/authorize'
+        end
+
+        def token_credential_uri
+          'https://auth.atlassian.com/oauth/token'
+        end
+
+        def additional_parameters
+          { :prompt => 'consent', :audience => 'api.atlassian.com' }
+        end
+
+        def fetch_cloud_id(access_token, base_url)
+          response = HTTPClient.new.get(
+            'https://api.atlassian.com/oauth/token/accessible-resources',
+            nil,
+            'Accept' => 'application/json',
+            'Authorization' => "Bearer #{access_token}"
+          )
+          raise 'unable to fetch cloud id' unless HTTP::Status.successful?(response.status)
+          json = JSON.parse(response.body)
+
+          site = json.find { |sites| sites['url'] == base_url } || {}
+          site.fetch('id') { raise 'unable to fetch cloud id' }
+        end
+      end
+    end
+  end
+end

data/lib/connectors_sdk/confluence_cloud/custom_client.rb

@@ -0,0 +1,61 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'connectors_sdk/confluence/custom_client'
+
+module ConnectorsSdk
+  module ConfluenceCloud
+    class CustomClient < ConnectorsSdk::Confluence::CustomClient
+      def user_groups(account_id, limit: 200, start: 0)
+        size = Float::INFINITY
+
+        groups = []
+
+        while start + limit < size
+          params = {
+            :start => start,
+            :limit => limit,
+            :accountId => account_id
+          }
+          response = get('rest/api/user/memberof', params)
+          result = Hashie::Mash.new(parse_and_raise_if_necessary!(response))
+          size = result.size
+          start += limit
+          groups.concat(result.results)
+        end
+
+        groups
+      end
+
+      def user(account_id, expand: [])
+        params = {
+          :accountId => account_id
+        }
+        if expand.present?
+          params[:expand] = case expand
+                            when Array
+                              expand.join(',')
+                            when String
+                              expand
+                            else
+                              expand.to_s
+                            end
+        end
+        response = get('rest/api/user', params)
+        Hashie::Mash.new(parse_and_raise_if_necessary!(response))
+      rescue ConnectorsSdk::Atlassian::CustomClient::ClientError => e
+        if e.status_code == 404
+          ConnectorsShared::Logger.warn("Could not find a user with account id #{account_id}")
+          nil
+        else
+          raise
+        end
+      end
+    end
+  end
+end

data/lib/connectors_sdk/confluence_cloud/extractor.rb

@@ -0,0 +1,59 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'connectors_sdk/confluence/extractor'
+
+module ConnectorsSdk
+  module ConfluenceCloud
+    class Extractor < ConnectorsSdk::Confluence::Extractor
+
+      def yield_permissions(source_user_id)
+        # yield empty permissions if the user is suspended or deleted
+        user = client.user(source_user_id, :expand => 'operations')
+        if user.nil? || user.operations.blank?
+          yield [] and return
+        end
+
+        # refresh space permissions if not initialized
+        if @space_permissions_cache.nil?
+          @space_permissions_cache = {}
+          yield_spaces do |space|
+            if config.index_permissions
+              get_space_permissions(space)
+            end
+          end
+        end
+
+        direct_spaces = get_user_spaces(source_user_id)
+        indirect_spaces = []
+
+        group_permissions = []
+        client.user_groups(source_user_id).each do |group|
+          group_name = group.name
+          group_spaces = get_group_spaces(group_name)
+          indirect_spaces << group_spaces
+          group_permissions << "group:#{group_name}"
+        end
+
+        total_user_spaces = indirect_spaces.flatten.concat(direct_spaces).uniq
+        user_permissions = ["user:#{source_user_id}"]
+          .concat(group_permissions)
+          .product(total_user_spaces)
+          .collect { |permission, space| "#{space}/#{permission}" }
+
+        yield user_permissions.flatten.uniq.sort
+      end
+
+      def download(item)
+        content = item[:content]
+        parent_id = content.dig('container', 'id')
+        client.download("#{client.base_url}/wiki/rest/api/content/#{parent_id}/child/attachment/#{content['id']}/download").body
+      end
+    end
+  end
+end

data/lib/connectors_sdk/confluence_cloud/http_call_wrapper.rb

@@ -0,0 +1,59 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'connectors_sdk/atlassian/config'
+require 'connectors_sdk/confluence_cloud/extractor'
+require 'connectors_sdk/confluence_cloud/authorization'
+require 'connectors_sdk/confluence_cloud/custom_client'
+require 'connectors_sdk/base/http_call_wrapper'
+
+module ConnectorsSdk
+  module ConfluenceCloud
+    class HttpCallWrapper < ConnectorsSdk::Base::HttpCallWrapper
+      SERVICE_TYPE = 'confluence_cloud'
+
+      def name
+        'Confluence Cloud'
+      end
+
+      def service_type
+        SERVICE_TYPE
+      end
+
+      private
+
+      def extractor_class
+        ConnectorsSdk::ConfluenceCloud::Extractor
+      end
+
+      def authorization
+        ConnectorsSdk::ConfluenceCloud::Authorization
+      end
+
+      def client(params)
+        ConnectorsSdk::ConfluenceCloud::CustomClient.new(:base_url => base_url(params[:cloud_id]), :access_token => params[:access_token])
+      end
+
+      def custom_client_error
+        ConnectorsSdk::Atlassian::CustomClient::ClientError
+      end
+
+      def config(params)
+        ConnectorsSdk::Atlassian::Config.new(:base_url => base_url(params[:cloud_id]), :cursors => params.fetch(:cursors, {}) || {})
+      end
+
+      def health_check(params)
+        client(params).me
+      end
+
+      def base_url(cloud_id)
+        "https://api.atlassian.com/ex/confluence/#{cloud_id}"
+      end
+    end
+  end
+end

data/lib/connectors_sdk/helpers/atlassian_time_formatter.rb

@@ -0,0 +1,10 @@
+module ConnectorsSdk
+  module Helpers
+    module AtlassianTimeFormatter
+      def format_time(time)
+        time = Time.parse(time) if time.is_a?(String)
+        time.strftime('%Y-%m-%d %H:%M')
+      end
+    end
+  end
+end