connectors_sdk 8.3.0.0.pre.20220414T060419Z

data/lib/connectors_sdk/office365/extractor.rb

@@ -0,0 +1,230 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'connectors_sdk/office365/custom_client'
+require 'connectors_sdk/base/extractor'
+
+module ConnectorsSdk
+  module Office365
+    class Extractor < ConnectorsSdk::Base::Extractor
+      DRIVE_IDS_CURSOR_KEY = 'drive_ids'.freeze
+
+      def yield_document_changes(modified_since: nil, break_after_page: false, &block)
+        drives_to_index.each do |drive|
+          drive_id = drive.id
+
+          if break_after_page
+            current_drive_id = config.cursors['current_drive_id']
+            if current_drive_id.present? && current_drive_id > drive_id # they come alpha sorted
+              next
+            end
+            config.cursors['current_drive_id'] = drive_id
+          end
+
+          drive_owner_name = drive.dig(:owner, :user, :displayName)
+          drive_name = drive.name
+
+          drive_id_to_delta_link = config.cursors.fetch(DRIVE_IDS_CURSOR_KEY, {})
+          begin
+            if start_delta_link = drive_id_to_delta_link[drive_id]
+              log_debug("Starting an incremental crawl with cursor for #{service_type.classify} with drive_id: #{drive_id}")
+              begin
+                yield_changes(drive_id, :start_delta_link => start_delta_link, :drive_owner_name => drive_owner_name, :drive_name => drive_name, :break_after_page => break_after_page, &block)
+              rescue ConnectorsSdk::Office365::CustomClient::Office365InvalidCursorsError
+                log_warn("Error listing changes with start_delta_link: #{start_delta_link}, falling back to full crawl")
+                yield_drive_items(drive_id, :drive_owner_name => drive_owner_name, :drive_name => drive_name, :break_after_page => break_after_page, &block)
+              end
+            elsif modified_since.present?
+              log_debug("Starting an incremental crawl using last_modified (no cursor found) for #{service_type.classify} with drive_id: #{drive_id}")
+              yield_changes(drive_id, :last_modified => modified_since, :drive_owner_name => drive_owner_name, :drive_name => drive_name, :break_after_page => break_after_page, &block)
+            else
+              log_debug("Starting a full crawl #{service_type.classify} with drive_id: #{drive_id}")
+              yield_drive_items(drive_id, :drive_owner_name => drive_owner_name, :drive_name => drive_name, :break_after_page => break_after_page, &block)
+            end
+          rescue ConnectorsSdk::Office365::CustomClient::ClientError => e
+            log_warn("Error searching and listing drive #{drive_id}")
+            capture_exception(e)
+          end
+
+          if break_after_page && config.cursors['page_cursor'].present?
+            break
+          end
+        end
+
+        if break_after_page && config.cursors['page_cursor'].blank?
+          @completed = true
+          config.overwrite_cursors!(retrieve_latest_cursors)
+          log_debug("Completed #{modified_since.nil? ? 'full' : 'incremental'} extraction")
+        end
+
+        nil
+      end
+
+      def yield_deleted_ids(ids)
+        ids.each do |id|
+          yield id unless existing_drive_item_ids.include?(id)
+        end
+      end
+
+      def retrieve_latest_cursors
+        delta_links_for_drive_ids = drives_to_index.map(&:id).each_with_object({}) do |drive_id, h|
+          h[drive_id] = client.get_latest_delta_link(drive_id)
+        rescue ConnectorsSdk::Office365::CustomClient::ClientError => e
+          log_warn("Error getting delta link for #{drive_id}")
+          capture_exception(e)
+          raise e
+        end
+
+        {
+          DRIVE_IDS_CURSOR_KEY => delta_links_for_drive_ids
+        }
+      end
+
+      def yield_permissions(source_user_id)
+        permissions = [source_user_id]
+        client.user_groups(source_user_id, %w(id displayName)).each do |next_group|
+          # Adding "Members" suffix since that is how the item permissions endpoint return group permissions
+          permissions << "#{next_group.displayName} Members"
+        end
+
+        yield permissions.uniq
+      rescue ConnectorsSdk::Office365::CustomClient::ClientError => e
+        # if a user is deleted, client.user_groups will throw 404 Not Found error, saving another call to get user profile
+        if e.status_code == 404
+          log_warn("Could not find a user with id #{source_user_id}")
+          yield []
+        else
+          raise
+        end
+      end
+
+      def download(item)
+        download_url = item[:download_url]
+        client.download_item(download_url)
+      end
+
+      private
+
+      def drives
+        raise NotImplementedError
+      end
+
+      def drives_to_index
+        @drives_to_index ||= begin
+          value = if config.index_all_drives?
+                    drives
+                  else
+                    drives.select { |d| config.drive_ids.include?(d.id) }
+                  end
+
+          log_debug("Found drives to index with ids: #{value.map(&:id).join(', ')}")
+
+          value
+        end
+      end
+
+      def existing_drive_item_ids
+        @existing_drive_item_ids ||= Set.new.tap do |ids|
+          drives_to_index.each do |drive|
+            client.list_items(drive.id) do |item|
+              ids << convert_id_to_fp_id(item.id)
+            end
+          end
+        end
+      end
+
+      def adapter
+        raise NotImplementedError
+      end
+
+      def convert_id_to_fp_id(_id)
+        raise NotImplementedError
+      end
+
+      def capture_exception(office365_client_error)
+        options = {
+          :extra => {
+            :status_code => office365_client_error.status_code,
+            :endpoint => office365_client_error.endpoint
+          }
+        }
+        ConnectorsShared::ExceptionTracking.capture_exception(office365_client_error, options)
+      end
+
+      def yield_drive_items(drive_id, drive_owner_name:, drive_name:, break_after_page: false, &block)
+        client.list_items(drive_id, break_after_page: break_after_page) do |item|
+          yield_single_document_change(:identifier => "Office365 change: #{item&.id} (#{Office365::Adapter::GraphItem.get_path(item)})") do
+            item.drive_owner_name = drive_owner_name
+            item.drive_name = drive_name
+            yield_create_or_update(drive_id, item, &block)
+          end
+        end
+      end
+
+      def yield_correct_actions_and_converted_item(drive_id, item, &block)
+        if item.deleted.nil?
+          yield_create_or_update(drive_id, item, &block)
+        else
+          yield :delete, convert_id_to_fp_id(item.id)
+        end
+      end
+
+      def yield_changes(drive_id, drive_owner_name:, drive_name:, start_delta_link: nil, last_modified: nil, break_after_page: false, &block)
+        client.list_changes(:drive_id => drive_id, :start_delta_link => start_delta_link, :last_modified => last_modified, :break_after_page => break_after_page) do |item|
+          yield_single_document_change(:identifier => "Office365 change: #{item&.id} (#{Office365::Adapter::GraphItem.get_path(item)})") do
+            item.drive_owner_name = drive_owner_name
+            item.drive_name = drive_name
+            yield_correct_actions_and_converted_item(drive_id, item, &block)
+          end
+        end
+      end
+
+      def yield_create_or_update(drive_id, item)
+        item = with_permissions(drive_id, item)
+
+        document = generate_document(item)
+        download_args =
+          if downloadable?(item)
+            download_args_and_proc(
+              id: document.fetch(:id),
+              name: item.name,
+              size: item[:size],
+              download_args: { :download_url => item.fetch('@microsoft.graph.downloadUrl') }
+            ) do |args|
+              download(args)
+            end
+          else
+            []
+          end
+        yield :create_or_update, document, download_args
+      end
+
+      def downloadable?(item)
+        item.key?('@microsoft.graph.downloadUrl')
+      end
+
+      def generate_document(item)
+        if item.file
+          adapter.swiftype_document_from_file(item)
+        elsif item.folder
+          adapter.swiftype_document_from_folder(item)
+        elsif item.package
+          adapter.swiftype_document_from_package(item)
+        else
+          raise "Unexpected Office 365 item type for item #{item}"
+        end
+      end
+
+      def with_permissions(drive_id, item)
+        item = item.dup
+        item.permissions = client.item_permissions(drive_id, item.id) if config.index_permissions
+        item
+      end
+    end
+  end
+end

data/lib/connectors_sdk/share_point/adapter.rb

@@ -0,0 +1,47 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'connectors_sdk/office365/adapter'
+
+module ConnectorsSdk
+  module SharePoint
+    class Adapter < Office365::Adapter
+      generate_id_helpers :share_point, 'share_point'
+
+      def self.swiftype_document_from_file(file)
+        FileGraphItem.new(file).to_swiftype_document
+      end
+
+      def self.swiftype_document_from_folder(folder)
+        FolderGraphItem.new(folder).to_swiftype_document
+      end
+
+      def self.swiftype_document_from_package(package)
+        PackageGraphItem.new(package).to_swiftype_document
+      end
+
+      class FileGraphItem < Office365::Adapter::FileGraphItem
+        def self.convert_id_to_fp_id(id)
+          ConnectorsSdk::SharePoint::Adapter.share_point_id_to_fp_id(id)
+        end
+      end
+
+      class FolderGraphItem < Office365::Adapter::FolderGraphItem
+        def self.convert_id_to_fp_id(id)
+          ConnectorsSdk::SharePoint::Adapter.share_point_id_to_fp_id(id)
+        end
+      end
+
+      class PackageGraphItem < Office365::Adapter::PackageGraphItem
+        def self.convert_id_to_fp_id(id)
+          ConnectorsSdk::SharePoint::Adapter.share_point_id_to_fp_id(id)
+        end
+      end
+    end
+  end
+end

data/lib/connectors_sdk/share_point/authorization.rb

@@ -0,0 +1,91 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'connectors_shared'
+require 'signet'
+require 'signet/oauth_2'
+require 'signet/oauth_2/client'
+
+module ConnectorsSdk
+  module SharePoint
+    class Authorization
+      class << self
+        def authorization_url
+          'https://login.microsoftonline.com/common/oauth2/v2.0/authorize'
+        end
+
+        def token_credential_uri
+          'https://login.microsoftonline.com/common/oauth2/v2.0/token'
+        end
+
+        def authorization_uri(params)
+          missing = missing_fields(params, %w[client_id])
+          unless missing.blank?
+            raise ConnectorsShared::ClientError.new("Missing required fields: #{missing.join(', ')}")
+          end
+
+          params[:response_type] = 'code'
+          params[:additional_parameters] = { :prompt => 'consent' }
+          client = oauth_client(params)
+          client.authorization_uri.to_s
+        end
+
+        def access_token(params)
+          missing = missing_fields(params, %w[client_id client_secret code redirect_uri])
+          unless missing.blank?
+            raise ConnectorsShared::ClientError.new("Missing required fields: #{missing.join(', ')}")
+          end
+
+          params[:grant_type] = 'authorization_code'
+          client = oauth_client(params)
+          client.fetch_access_token
+        end
+
+        def refresh(params)
+          missing = missing_fields(params, %w[client_id client_secret refresh_token])
+          unless missing.blank?
+            raise ConnectorsShared::ClientError.new("Missing required fields: #{missing.join(', ')}")
+          end
+
+          params[:grant_type] = 'refresh_token'
+          client = oauth_client(params)
+          client.refresh!
+        rescue StandardError => e
+          ConnectorsShared::ExceptionTracking.log_exception(e)
+          raise ConnectorsShared::TokenRefreshFailedError
+        end
+
+        def oauth_client(params)
+          options = params.merge(
+            :authorization_uri => authorization_url,
+            :token_credential_uri => token_credential_uri,
+            :scope => oauth_scope
+          )
+          options[:state] = JSON.dump(options[:state]) if options[:state]
+          Signet::OAuth2::Client.new(options)
+        end
+
+        def oauth_scope
+          %w[
+            User.ReadBasic.All
+            Group.Read.All
+            Directory.AccessAsUser.All
+            Files.Read
+            Files.Read.All
+            Sites.Read.All
+            offline_access
+          ]
+        end
+
+        def missing_fields(params, required = [])
+          Array.wrap(required).select { |field| params[field.to_sym].nil? }
+        end
+      end
+    end
+  end
+end

data/lib/connectors_sdk/share_point/extractor.rb

@@ -0,0 +1,31 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'connectors_sdk/office365/extractor'
+require 'connectors_sdk/share_point/adapter'
+
+module ConnectorsSdk
+  module SharePoint
+    class Extractor < ConnectorsSdk::Office365::Extractor
+
+      private
+
+      def convert_id_to_fp_id(id)
+        ConnectorsSdk::SharePoint::Adapter.share_point_id_to_fp_id(id)
+      end
+
+      def adapter
+        ConnectorsSdk::SharePoint::Adapter
+      end
+
+      def drives
+        client.share_point_drives(:fields => %w(id owner name driveType)).sort_by(&:id)
+      end
+    end
+  end
+end

data/lib/connectors_sdk/share_point/http_call_wrapper.rb

@@ -0,0 +1,117 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'connectors_sdk/office365/config'
+require 'connectors_sdk/share_point/extractor'
+require 'connectors_sdk/share_point/authorization'
+require 'bson'
+
+module ConnectorsSdk
+  module SharePoint
+    SERVICE_TYPE = 'share_point'
+
+    class HttpCallWrapper
+      def extractor(params)
+        cursors = params.fetch(:cursors, {}) || {}
+        features = params.fetch(:features, {}) || {}
+
+        # XXX can we cache that class across calls?
+        ConnectorsSdk::SharePoint::Extractor.new(
+          content_source_id: BSON::ObjectId.new,
+          service_type: SERVICE_TYPE,
+          authorization_data_proc: proc { { access_token: params[:access_token] } },
+          client_proc: proc { ConnectorsSdk::Office365::CustomClient.new(:access_token => params[:access_token], :cursors => cursors) },
+          config: ConnectorsSdk::Office365::Config.new(:cursors => cursors, :drive_ids => 'all', :index_permissions => params[:index_permissions] || false),
+          features: features
+        )
+      end
+
+      def document_batch(params)
+        results = []
+
+        @extractor = extractor(params)
+
+        @extractor.yield_document_changes(:break_after_page => true, :modified_since => @extractor.config.cursors['modified_since']) do |action, doc, download_args_and_proc|
+          download_obj = nil
+          if download_args_and_proc
+            download_obj = {
+              id: download_args_and_proc[0],
+              name: download_args_and_proc[1],
+              size: download_args_and_proc[2],
+              download_args: download_args_and_proc[3]
+            }
+          end
+
+          results << {
+            :action => action,
+            :document => doc,
+            :download => download_obj
+          }
+        end
+
+        results
+      rescue ConnectorsSdk::Office365::CustomClient::ClientError => e
+        raise e.status_code == 401 ? ConnectorsShared::InvalidTokenError : e
+      end
+
+      def cursors
+        @extractor.config.cursors
+      end
+
+      def completed?
+        @extractor.completed
+      end
+
+      def deleted(params)
+        results = []
+        extractor(params).yield_deleted_ids(params[:ids]) do |id|
+          results << id
+        end
+        results
+      rescue ConnectorsSdk::Office365::CustomClient::ClientError => e
+        raise e.status_code == 401 ? ConnectorsShared::InvalidTokenError : e
+      end
+
+      def permissions(params)
+        extractor(params).yield_permissions(params[:user_id]) do |permissions|
+          return permissions
+        end
+      rescue ConnectorsSdk::Office365::CustomClient::ClientError => e
+        raise e.status_code == 401 ? ConnectorsShared::InvalidTokenError : e
+      end
+
+      def authorization_uri(body)
+        ConnectorsSdk::SharePoint::Authorization.authorization_uri(body)
+      end
+
+      def access_token(params)
+        ConnectorsSdk::SharePoint::Authorization.access_token(params)
+      end
+
+      def refresh(params)
+        ConnectorsSdk::SharePoint::Authorization.refresh(params)
+      end
+
+      def download(params)
+        extractor(params).download(params[:meta])
+      end
+
+      def name
+        'SharePoint'
+      end
+
+      def source_status(params)
+        client = ConnectorsSdk::Office365::CustomClient.new(:access_token => params[:access_token])
+        client.me
+        { :status => 'OK', :statusCode => 200, :message => 'Connected to SharePoint' }
+      rescue StandardError => e
+        { :status => 'FAILURE', :statusCode => e.is_a?(ConnectorsSdk::Office365::CustomClient::ClientError) ? e.status_code : 500, :message => e.message }
+      end
+    end
+  end
+end

data/lib/connectors_sdk.rb

@@ -0,0 +1,16 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+require 'connectors_shared'
+
+def required_path(absolute_path)
+  absolute_dir = File.dirname(absolute_path)
+  relative_dir = absolute_dir.sub(/.*lib\/connectors_sdk/, 'connectors_sdk')
+  name = File.basename(absolute_path, '.rb')
+  File.join(relative_dir, name)
+end
+
+Dir[File.join(__dir__, 'connectors_sdk/**/*.rb')].each { |f| require required_path(f) }

data/lib/connectors_shared/constants.rb

@@ -0,0 +1,14 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+module ConnectorsShared
+  class Constants
+    THUMBNAIL_FIELDS = %w[_thumbnail_80x100 _thumbnail_310x430].freeze
+    SUBEXTRACTOR_RESERVED_FIELDS = %w[_subextracted_as_of _subextracted_version].freeze
+    ALLOW_FIELD = '_allow_permissions'.freeze
+    DENY_FIELD = '_deny_permissions'.freeze
+  end
+end

data/lib/connectors_shared/errors.rb

@@ -0,0 +1,126 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+require 'active_support/core_ext/string'
+
+module ConnectorsShared
+  class DocumentError
+    attr_accessor :error_class, :error_message, :stack_trace, :error_id
+
+    def initialize(error_class, error_message, stack_trace, error_id)
+      @error_class = error_class
+      @error_message = error_message
+      @error_id = error_id
+
+      # keywords must be < 32kb, UTF-8 chars can be up to 3 bytes, thus 32k/3 ~= 10k
+      # See https://github.com/elastic/workplace-search-team/issues/1723
+      @stack_trace = stack_trace.truncate(10_000)
+    end
+
+    def to_h
+      {
+        'error_class' => error_class,
+        'error_message' => error_message,
+        'stack_trace' => stack_trace,
+        'error_id' => error_id
+      }
+    end
+  end
+
+  class ClientError < StandardError; end
+  class EvictionWithNoProgressError < StandardError; end
+  class EvictionError < StandardError
+    attr_accessor :cursors
+
+    def initialize(message = nil, cursors: nil)
+      super(message)
+      @cursors = cursors
+    end
+  end
+
+  class SuspendedJobError < StandardError
+    attr_accessor :suspend_until, :cursors
+
+    def initialize(message = nil, suspend_until:, cursors: nil)
+      super(message)
+      @suspend_until = suspend_until
+      @cursors = cursors
+    end
+  end
+  class ThrottlingError < SuspendedJobError; end
+  class TransientServerError < SuspendedJobError; end
+  class UnrecoverableServerError < StandardError; end
+  class TransientSubextractorError < StandardError; end
+  class JobDocumentLimitError < StandardError; end
+  class JobClaimingError < StandardError; end
+
+  class MonitoringError < StandardError
+    attr_accessor :tripped_by
+
+    def initialize(message = nil, tripped_by: nil)
+      super("#{message}#{tripped_by.present? ? " Tripped by - #{tripped_by.class}: #{tripped_by.message}" : ''}")
+      @tripped_by = tripped_by
+    end
+  end
+  class MaxSuccessiveErrorsExceededError < MonitoringError; end
+  class MaxErrorsExceededError < MonitoringError; end
+  class MaxErrorsInWindowExceededError < MonitoringError; end
+
+  class JobSyncNotPossibleYetError < StandardError
+    attr_accessor :sync_will_be_possible_at
+
+    def initialize(message = nil, sync_will_be_possible_at: nil)
+      human_readable_errors = []
+
+      human_readable_errors.push(message) unless message.nil?
+      human_readable_errors.push("Content source was created too recently to schedule jobs, next job scheduling is possible at #{sync_will_be_possible_at}.") unless sync_will_be_possible_at.nil?
+
+      super(human_readable_errors.join(' '))
+    end
+  end
+  class PlatinumLicenseRequiredError < StandardError; end
+  class JobInterruptedError < StandardError; end
+  class JobCannotBeUpdatedError < StandardError; end
+  class SecretInvalidError < StandardError; end
+  class InvalidIndexingConfigurationError < StandardError; end
+  class InvalidTokenError < StandardError; end
+  class TokenRefreshFailedError < StandardError; end
+  class ConnectorNotAvailableError < StandardError; end
+
+  # For when we want to explicitly set a #cause but can't
+  class ExplicitlyCausedError < StandardError
+    attr_reader :reason
+
+    def initialize(reason)
+      @reason = reason
+    end
+  end
+
+  class PublishingFailedError < ExplicitlyCausedError; end
+
+  class Error
+    attr_reader :status_code, :code, :message
+
+    def initialize(status_code, code, message)
+      @status_code = status_code
+      @code = code
+      @message = message
+    end
+
+    def to_h
+      {
+        'code' => @code,
+        'message' => @message
+      }
+    end
+  end
+
+  INTERNAL_SERVER_ERROR = ConnectorsShared::Error.new(500, 'INTERNAL_SERVER_ERROR', 'Internal server error')
+  INVALID_API_KEY = ConnectorsShared::Error.new(401, 'INVALID_API_KEY', 'Invalid API key')
+  UNSUPPORTED_AUTH_SCHEME = ConnectorsShared::Error.new(401, 'UNSUPPORTED_AUTH_SCHEME', 'Unsupported authorization scheme')
+  INVALID_ACCESS_TOKEN = ConnectorsShared::Error.new(401, 'INVALID_ACCESS_TOKEN', 'Invalid/expired access token, please refresh the token')
+  TOKEN_REFRESH_ERROR = ConnectorsShared::Error.new(401, 'TOKEN_REFRESH_ERROR', 'Failed to refresh token, please re-authenticate the application')
+end