connectors_sdk 8.3.0.0.pre.20220414T060419Z → 8.3.0.0.pre.20220510T144908Z

data/lib/connectors_sdk/office365/adapter.rb

@@ -11,11 +11,11 @@ require 'connectors_sdk/base/adapter'
 module ConnectorsSdk
   module Office365
     class Adapter < ConnectorsSdk::Base::Adapter
-      def self.swiftype_document_from_file(_file)
+      def self.es_document_from_file(_file)
         raise NotImplementedError
       end
 
-      def self.swiftype_document_from_folder(_folder)
+      def self.es_document_from_folder(_folder)
         raise NotImplementedError
       end
 
@@ -26,7 +26,7 @@ module ConnectorsSdk
           @item = item
         end
 
-        def self.convert_id_to_fp_id(_id)
+        def self.convert_id_to_es_id(_id)
           raise NotImplementedError
         end
 
@@ -41,10 +41,10 @@ module ConnectorsSdk
           ConnectorsSdk::Office365::Adapter.normalize_path("#{parent_folder_path}/#{item.name}")
         end
 
-        def to_swiftype_document
+        def to_es_document
           {
             :_fields_to_preserve => ConnectorsSdk::Office365::Adapter.fields_to_preserve,
-            :id => self.class.convert_id_to_fp_id(item.id),
+            :id => self.class.convert_id_to_es_id(item.id),
             :path => get_path(item),
             :title => item.name,
             :url => item.webUrl,
@@ -96,7 +96,7 @@ module ConnectorsSdk
       end
 
       class FileGraphItem < GraphItem
-        def self.convert_id_to_fp_id(_id)
+        def self.convert_id_to_es_id(_id)
           raise NotImplementedError
         end
 
@@ -132,7 +132,7 @@ module ConnectorsSdk
       end
 
       class PackageGraphItem < GraphItem
-        def self.convert_id_to_fp_id(id)
+        def self.convert_id_to_es_id(id)
           raise NotImplementedError
         end
 

data/lib/connectors_sdk/office365/config.rb

@@ -7,6 +7,7 @@
 # frozen_string_literal: true
 
 require 'connectors_sdk/base/config'
+require 'connectors_sdk/office365/extractor'
 
 module ConnectorsSdk
   module Office365

data/lib/connectors_sdk/office365/custom_client.rb

@@ -6,9 +6,11 @@
 
 # frozen_string_literal: true
 
+require 'hashie/mash'
+
 require 'connectors_sdk/base/custom_client'
+require 'connectors_sdk/office365/extractor'
 require 'connectors_shared'
-require 'hashie/mash'
 
 module ConnectorsSdk
   module Office365
@@ -125,7 +127,7 @@ module ConnectorsSdk
         yielded = 0
         while stack.any?
           folder_id = stack.pop
-          item_children(drive_id, folder_id, :fields => fields_with_id) do |item|
+          item_children(drive_id, folder_id, :fields => fields_with_id, :break_after_page => break_after_page) do |item|
             if item.folder
               stack << item.id
             end
@@ -134,9 +136,14 @@ module ConnectorsSdk
             yielded += 1
           end
 
-          if break_after_page && yielded >= 100 && stack.any?
-            cursors['page_cursor'] = stack.dup
-            break
+          if break_after_page && yielded >= 100
+            if cursors['item_children_next_link'].present?
+              stack << folder_id
+            end
+            if stack.any?
+              cursors['page_cursor'] = stack.dup
+              break
+            end
           end
         end
       end
@@ -256,15 +263,30 @@ module ConnectorsSdk
         request_endpoint(:endpoint => "drives/#{drive_id}/root", :query_params => query_params)
       end
 
-      def item_children(drive_id, item_id, fields: [], &block)
-        endpoint = "drives/#{drive_id}/items/#{item_id}/children"
-        query_params = transform_fields_to_request_query_params(fields)
-        response = request_endpoint(:endpoint => endpoint, :query_params => query_params)
+      def item_children(drive_id, item_id, fields: [], break_after_page: false, &block)
+        next_link = cursors.delete('item_children_next_link') if break_after_page
 
+        response = if next_link.present?
+                     request_json(:url => next_link)
+                   else
+                     endpoint = "drives/#{drive_id}/items/#{item_id}/children"
+                     query_params = transform_fields_to_request_query_params(fields)
+                     request_endpoint(:endpoint => endpoint, :query_params => query_params)
+                   end
+
+        yielded = 0
         loop do
           response.value.each(&block)
           next_link = response['@odata.nextLink']
+
           break if next_link.nil?
+
+          yielded += response.value.size
+          if break_after_page && yielded >= 100
+            cursors['item_children_next_link'] = next_link
+            break
+          end
+
           response = request_json(:url => next_link)
         end
       end

data/lib/connectors_sdk/office365/extractor.rb

@@ -51,12 +51,12 @@ module ConnectorsSdk
             capture_exception(e)
           end
 
-          if break_after_page && config.cursors['page_cursor'].present?
+          if break_after_page && (config.cursors['page_cursor'].present? || config.cursors['item_children_next_link'].present?)
             break
           end
         end
 
-        if break_after_page && config.cursors['page_cursor'].blank?
+        if break_after_page && config.cursors['page_cursor'].blank? && config.cursors['item_children_next_link'].blank?
           @completed = true
           config.overwrite_cursors!(retrieve_latest_cursors)
           log_debug("Completed #{modified_since.nil? ? 'full' : 'incremental'} extraction")
@@ -132,7 +132,7 @@ module ConnectorsSdk
         @existing_drive_item_ids ||= Set.new.tap do |ids|
           drives_to_index.each do |drive|
             client.list_items(drive.id) do |item|
-              ids << convert_id_to_fp_id(item.id)
+              ids << convert_id_to_es_id(item.id)
             end
           end
         end
@@ -142,7 +142,7 @@ module ConnectorsSdk
         raise NotImplementedError
       end
 
-      def convert_id_to_fp_id(_id)
+      def convert_id_to_es_id(_id)
         raise NotImplementedError
       end
 
@@ -170,7 +170,7 @@ module ConnectorsSdk
         if item.deleted.nil?
           yield_create_or_update(drive_id, item, &block)
         else
-          yield :delete, convert_id_to_fp_id(item.id)
+          yield :delete, convert_id_to_es_id(item.id)
         end
       end
 
@@ -210,11 +210,11 @@ module ConnectorsSdk
 
       def generate_document(item)
         if item.file
-          adapter.swiftype_document_from_file(item)
+          adapter.es_document_from_file(item)
         elsif item.folder
-          adapter.swiftype_document_from_folder(item)
+          adapter.es_document_from_folder(item)
         elsif item.package
-          adapter.swiftype_document_from_package(item)
+          adapter.es_document_from_package(item)
         else
           raise "Unexpected Office 365 item type for item #{item}"
         end

data/lib/connectors_sdk/share_point/adapter.rb

@@ -13,33 +13,33 @@ module ConnectorsSdk
     class Adapter < Office365::Adapter
       generate_id_helpers :share_point, 'share_point'
 
-      def self.swiftype_document_from_file(file)
-        FileGraphItem.new(file).to_swiftype_document
+      def self.es_document_from_file(file)
+        FileGraphItem.new(file).to_es_document
       end
 
-      def self.swiftype_document_from_folder(folder)
-        FolderGraphItem.new(folder).to_swiftype_document
+      def self.es_document_from_folder(folder)
+        FolderGraphItem.new(folder).to_es_document
       end
 
-      def self.swiftype_document_from_package(package)
-        PackageGraphItem.new(package).to_swiftype_document
+      def self.es_document_from_package(package)
+        PackageGraphItem.new(package).to_es_document
       end
 
       class FileGraphItem < Office365::Adapter::FileGraphItem
-        def self.convert_id_to_fp_id(id)
-          ConnectorsSdk::SharePoint::Adapter.share_point_id_to_fp_id(id)
+        def self.convert_id_to_es_id(id)
+          ConnectorsSdk::SharePoint::Adapter.share_point_id_to_es_id(id)
         end
       end
 
       class FolderGraphItem < Office365::Adapter::FolderGraphItem
-        def self.convert_id_to_fp_id(id)
-          ConnectorsSdk::SharePoint::Adapter.share_point_id_to_fp_id(id)
+        def self.convert_id_to_es_id(id)
+          ConnectorsSdk::SharePoint::Adapter.share_point_id_to_es_id(id)
         end
       end
 
       class PackageGraphItem < Office365::Adapter::PackageGraphItem
-        def self.convert_id_to_fp_id(id)
-          ConnectorsSdk::SharePoint::Adapter.share_point_id_to_fp_id(id)
+        def self.convert_id_to_es_id(id)
+          ConnectorsSdk::SharePoint::Adapter.share_point_id_to_es_id(id)
         end
       end
     end

data/lib/connectors_sdk/share_point/authorization.rb

@@ -6,70 +6,12 @@
 
 # frozen_string_literal: true
 
-require 'connectors_shared'
-require 'signet'
-require 'signet/oauth_2'
-require 'signet/oauth_2/client'
+require 'connectors_sdk/base/authorization'
 
 module ConnectorsSdk
   module SharePoint
-    class Authorization
+    class Authorization < ConnectorsSdk::Base::Authorization
       class << self
-        def authorization_url
-          'https://login.microsoftonline.com/common/oauth2/v2.0/authorize'
-        end
-
-        def token_credential_uri
-          'https://login.microsoftonline.com/common/oauth2/v2.0/token'
-        end
-
-        def authorization_uri(params)
-          missing = missing_fields(params, %w[client_id])
-          unless missing.blank?
-            raise ConnectorsShared::ClientError.new("Missing required fields: #{missing.join(', ')}")
-          end
-
-          params[:response_type] = 'code'
-          params[:additional_parameters] = { :prompt => 'consent' }
-          client = oauth_client(params)
-          client.authorization_uri.to_s
-        end
-
-        def access_token(params)
-          missing = missing_fields(params, %w[client_id client_secret code redirect_uri])
-          unless missing.blank?
-            raise ConnectorsShared::ClientError.new("Missing required fields: #{missing.join(', ')}")
-          end
-
-          params[:grant_type] = 'authorization_code'
-          client = oauth_client(params)
-          client.fetch_access_token
-        end
-
-        def refresh(params)
-          missing = missing_fields(params, %w[client_id client_secret refresh_token])
-          unless missing.blank?
-            raise ConnectorsShared::ClientError.new("Missing required fields: #{missing.join(', ')}")
-          end
-
-          params[:grant_type] = 'refresh_token'
-          client = oauth_client(params)
-          client.refresh!
-        rescue StandardError => e
-          ConnectorsShared::ExceptionTracking.log_exception(e)
-          raise ConnectorsShared::TokenRefreshFailedError
-        end
-
-        def oauth_client(params)
-          options = params.merge(
-            :authorization_uri => authorization_url,
-            :token_credential_uri => token_credential_uri,
-            :scope => oauth_scope
-          )
-          options[:state] = JSON.dump(options[:state]) if options[:state]
-          Signet::OAuth2::Client.new(options)
-        end
-
         def oauth_scope
           %w[
             User.ReadBasic.All
@@ -82,8 +24,18 @@ module ConnectorsSdk
           ]
         end
 
-        def missing_fields(params, required = [])
-          Array.wrap(required).select { |field| params[field.to_sym].nil? }
+        private
+
+        def authorization_url
+          'https://login.microsoftonline.com/common/oauth2/v2.0/authorize'
+        end
+
+        def token_credential_uri
+          'https://login.microsoftonline.com/common/oauth2/v2.0/token'
+        end
+
+        def additional_parameters
+          { :prompt => 'consent' }
         end
       end
     end

data/lib/connectors_sdk/share_point/extractor.rb

@@ -15,8 +15,8 @@ module ConnectorsSdk
 
       private
 
-      def convert_id_to_fp_id(id)
-        ConnectorsSdk::SharePoint::Adapter.share_point_id_to_fp_id(id)
+      def convert_id_to_es_id(id)
+        ConnectorsSdk::SharePoint::Adapter.share_point_id_to_es_id(id)
       end
 
       def adapter

data/lib/connectors_sdk/share_point/http_call_wrapper.rb

@@ -9,108 +9,49 @@
 require 'connectors_sdk/office365/config'
 require 'connectors_sdk/share_point/extractor'
 require 'connectors_sdk/share_point/authorization'
-require 'bson'
+require 'connectors_sdk/base/http_call_wrapper'
 
 module ConnectorsSdk
   module SharePoint
-    SERVICE_TYPE = 'share_point'
+    class HttpCallWrapper < ConnectorsSdk::Base::HttpCallWrapper
+      SERVICE_TYPE = 'share_point'
 
-    class HttpCallWrapper
-      def extractor(params)
-        cursors = params.fetch(:cursors, {}) || {}
-        features = params.fetch(:features, {}) || {}
-
-        # XXX can we cache that class across calls?
-        ConnectorsSdk::SharePoint::Extractor.new(
-          content_source_id: BSON::ObjectId.new,
-          service_type: SERVICE_TYPE,
-          authorization_data_proc: proc { { access_token: params[:access_token] } },
-          client_proc: proc { ConnectorsSdk::Office365::CustomClient.new(:access_token => params[:access_token], :cursors => cursors) },
-          config: ConnectorsSdk::Office365::Config.new(:cursors => cursors, :drive_ids => 'all', :index_permissions => params[:index_permissions] || false),
-          features: features
-        )
-      end
-
-      def document_batch(params)
-        results = []
-
-        @extractor = extractor(params)
-
-        @extractor.yield_document_changes(:break_after_page => true, :modified_since => @extractor.config.cursors['modified_since']) do |action, doc, download_args_and_proc|
-          download_obj = nil
-          if download_args_and_proc
-            download_obj = {
-              id: download_args_and_proc[0],
-              name: download_args_and_proc[1],
-              size: download_args_and_proc[2],
-              download_args: download_args_and_proc[3]
-            }
-          end
-
-          results << {
-            :action => action,
-            :document => doc,
-            :download => download_obj
-          }
-        end
-
-        results
-      rescue ConnectorsSdk::Office365::CustomClient::ClientError => e
-        raise e.status_code == 401 ? ConnectorsShared::InvalidTokenError : e
-      end
-
-      def cursors
-        @extractor.config.cursors
+      def name
+        'SharePoint'
       end
 
-      def completed?
-        @extractor.completed
+      def service_type
+        SERVICE_TYPE
       end
 
-      def deleted(params)
-        results = []
-        extractor(params).yield_deleted_ids(params[:ids]) do |id|
-          results << id
-        end
-        results
-      rescue ConnectorsSdk::Office365::CustomClient::ClientError => e
-        raise e.status_code == 401 ? ConnectorsShared::InvalidTokenError : e
-      end
+      private
 
-      def permissions(params)
-        extractor(params).yield_permissions(params[:user_id]) do |permissions|
-          return permissions
-        end
-      rescue ConnectorsSdk::Office365::CustomClient::ClientError => e
-        raise e.status_code == 401 ? ConnectorsShared::InvalidTokenError : e
+      def extractor_class
+        ConnectorsSdk::SharePoint::Extractor
       end
 
-      def authorization_uri(body)
-        ConnectorsSdk::SharePoint::Authorization.authorization_uri(body)
+      def authorization
+        ConnectorsSdk::SharePoint::Authorization
       end
 
-      def access_token(params)
-        ConnectorsSdk::SharePoint::Authorization.access_token(params)
+      def client(params)
+        ConnectorsSdk::Office365::CustomClient.new(:access_token => params[:access_token], :cursors => params.fetch(:cursors, {}) || {})
       end
 
-      def refresh(params)
-        ConnectorsSdk::SharePoint::Authorization.refresh(params)
+      def custom_client_error
+        ConnectorsSdk::Office365::CustomClient::ClientError
       end
 
-      def download(params)
-        extractor(params).download(params[:meta])
-      end
-
-      def name
-        'SharePoint'
+      def config(params)
+        ConnectorsSdk::Office365::Config.new(
+          :cursors => params.fetch(:cursors, {}) || {},
+          :drive_ids => 'all',
+          :index_permissions => params[:index_permissions] || false
+        )
       end
 
-      def source_status(params)
-        client = ConnectorsSdk::Office365::CustomClient.new(:access_token => params[:access_token])
-        client.me
-        { :status => 'OK', :statusCode => 200, :message => 'Connected to SharePoint' }
-      rescue StandardError => e
-        { :status => 'FAILURE', :statusCode => e.is_a?(ConnectorsSdk::Office365::CustomClient::ClientError) ? e.status_code : 500, :message => e.message }
+      def health_check(params)
+        client(params).me
       end
     end
   end

data/lib/connectors_shared/exception_tracking.rb

@@ -6,23 +6,23 @@
 
 # frozen_string_literal: true
 
-require 'stubs/swiftype/exception_tracking' unless defined?(Rails)
 require 'bson'
 require 'connectors_shared/logger'
+require 'stubs/enterprise_search/exception_tracking'
 
 module ConnectorsShared
   class ExceptionTracking
     class << self
       def capture_message(message, context = {})
-        Swiftype::ExceptionTracking.capture_message(message, context)
+        EnterpriseSearch::ExceptionTracking.capture_message(message, context)
       end
 
       def capture_exception(exception, context = {})
-        Swiftype::ExceptionTracking.log_exception(exception, :context => context)
+        EnterpriseSearch::ExceptionTracking.log_exception(exception, :context => context)
       end
 
       def log_exception(exception, message = nil)
-        Swiftype::ExceptionTracking.log_exception(exception, message, :logger => ConnectorsShared::Logger.logger)
+        EnterpriseSearch::ExceptionTracking.log_exception(exception, message, :logger => ConnectorsShared::Logger.logger)
       end
 
       def augment_exception(exception)

data/lib/connectors_shared/extraction_utils.rb

@@ -0,0 +1,109 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'set'
+
+module ConnectorsShared
+  module ExtractionUtils
+    # A list of tags tags we want to remove before extracting content
+    NON_CONTENT_TAGS = Set.new(%w[
+      comment
+      object
+      script
+      style
+      svg
+      video
+    ]).freeze
+
+    # Tags, that generate a word/line break when rendered
+    BREAK_ELEMENTS = Set.new(%w[
+      br
+      hr
+    ]).freeze
+
+    # The character used to signal that a string has been truncated
+    OMISSION = '…'
+
+    #-------------------------------------------------------------------------------------------------
+    # Expects a Nokogiri HTML node, returns textual content from the node and all of its children
+    def self.node_descendant_text(node)
+      return '' unless node&.present?
+
+      unless node.respond_to?(:children) && node.respond_to?(:name) && node.respond_to?(:text?)
+        raise ArgumentError, "Expecting something node-like but got a #{node.class}"
+      end
+
+      to_process_stack = [node]
+      text = []
+
+      loop do
+        # Get the next node to process
+        node = to_process_stack.pop
+        break unless node
+
+        # Base cases where we append content to the text buffer
+        if node.kind_of?(String)
+          text << node unless node == ' ' && text.last == ' '
+          next
+        end
+
+        # Remove tags that do not contain any text (and which sometimes are treated as CDATA, generating garbage text in jruby)
+        next if NON_CONTENT_TAGS.include?(node.name)
+
+        # Tags, that need to be replaced by spaces according to the standards
+        if replace_with_whitespace?(node)
+          text << ' ' unless text.last == ' '
+          next
+        end
+
+        # Extract the text from all text nodes
+        if node.text?
+          content = node.content
+          text << content.squish if content
+          next
+        end
+
+        # Add spaces before all tags
+        to_process_stack << ' '
+
+        # Recursion by adding the node's children to the stack and looping
+        node.children.reverse_each { |child| to_process_stack << child }
+
+        # Add spaces after all tags
+        to_process_stack << ' '
+      end
+
+      # Remove any duplicate spaces and return the content
+      text.join.squish!
+    end
+
+    #-------------------------------------------------------------------------------------------------
+    # Returns true, if the node should be replaced with a space when extracting text from a document
+    def self.replace_with_whitespace?(node)
+      BREAK_ELEMENTS.include?(node.name)
+    end
+
+    #-------------------------------------------------------------------------------------------------
+    # Limits the size of a given string value down to a given limit (in bytes)
+    # This is heavily inspired by https://github.com/rails/rails/pull/27319/files
+    def self.limit_bytesize(string, limit)
+      return string if string.nil? || string.bytesize <= limit
+      real_limit = limit - OMISSION.bytesize
+      (+'').tap do |cut|
+        string.scan(/\X/) do |grapheme|
+          if cut.bytesize + grapheme.bytesize <= real_limit
+            cut << grapheme
+          else
+            cut << OMISSION
+            break
+          end
+        end
+      end
+    end
+  end
+end

data/lib/connectors_shared/middleware/basic_auth.rb

@@ -0,0 +1,27 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+module ConnectorsShared
+  module Middleware
+    class BasicAuth
+      AUTHORIZATION = 'Authorization'
+
+      attr_reader :basic_auth_token
+
+      def initialize(app = nil, options = {})
+        @app = app
+        @basic_auth_token = options.fetch(:basic_auth_token)
+      end
+
+      def call(env)
+        env.request_headers[AUTHORIZATION] = "Basic #{basic_auth_token}"
+        @app.call(env)
+      end
+    end
+  end
+end

data/lib/connectors_shared/middleware/bearer_auth.rb

@@ -0,0 +1,27 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+module ConnectorsShared
+  module Middleware
+    class BearerAuth
+      AUTHORIZATION = 'Authorization'
+
+      attr_reader :bearer_auth_token
+
+      def initialize(app = nil, options = {})
+        @app = app
+        @bearer_auth_token = options.fetch(:bearer_auth_token)
+      end
+
+      def call(env)
+        env.request_headers[AUTHORIZATION] = "Bearer #{bearer_auth_token}"
+        @app.call(env)
+      end
+    end
+  end
+end