connectors_sdk 8.3.0.0.pre.20220414T060419Z

data/lib/connectors_shared/exception_tracking.rb

@@ -0,0 +1,39 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'stubs/swiftype/exception_tracking' unless defined?(Rails)
+require 'bson'
+require 'connectors_shared/logger'
+
+module ConnectorsShared
+  class ExceptionTracking
+    class << self
+      def capture_message(message, context = {})
+        Swiftype::ExceptionTracking.capture_message(message, context)
+      end
+
+      def capture_exception(exception, context = {})
+        Swiftype::ExceptionTracking.log_exception(exception, :context => context)
+      end
+
+      def log_exception(exception, message = nil)
+        Swiftype::ExceptionTracking.log_exception(exception, message, :logger => ConnectorsShared::Logger.logger)
+      end
+
+      def augment_exception(exception)
+        unless exception.respond_to?(:id)
+          exception.instance_eval do
+            def id
+              @error_id ||= BSON::ObjectId.new.to_s
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/connectors_shared/extension_mapping_util.rb

@@ -0,0 +1,123 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'active_support/core_ext/hash'
+
+module ConnectorsShared
+  class ExtensionMappingUtil
+    @extension_to_mime = {
+      :doc => %w[
+        application/x-tika-msoffice
+        application/msword
+      ].freeze,
+      :docx => %w[
+        application/x-tika-ooxml
+        application/vnd.openxmlformats-officedocument.wordprocessingml.document
+        application/vnd.openxmlformats-officedocument.wordprocessingml.template
+        application/vnd.ms-word.template.macroenabled.12
+        application/vnd.ms-word.document.macroenabled.12
+      ].freeze,
+      :html => %w[
+        text/html
+        application/xhtml+xml
+      ].freeze,
+      :odt => %w[
+        application/x-vnd.oasis.opendocument.graphics-template
+        application/vnd.sun.xml.writer application/x-vnd.oasis.opendocument.text
+        application/x-vnd.oasis.opendocument.text-web
+        application/x-vnd.oasis.opendocument.spreadsheet-template
+        application/vnd.oasis.opendocument.formula-template
+        application/vnd.oasis.opendocument.presentation
+        application/vnd.oasis.opendocument.image-template
+        application/x-vnd.oasis.opendocument.graphics
+        application/vnd.oasis.opendocument.chart-template
+        application/vnd.oasis.opendocument.presentation-template
+        application/x-vnd.oasis.opendocument.image-template
+        application/vnd.oasis.opendocument.formula
+        application/x-vnd.oasis.opendocument.image
+        application/vnd.oasis.opendocument.spreadsheet-template
+        application/x-vnd.oasis.opendocument.chart-template
+        application/x-vnd.oasis.opendocument.formula
+        application/vnd.oasis.opendocument.spreadsheet
+        application/vnd.oasis.opendocument.text-web
+        application/vnd.oasis.opendocument.text-template
+        application/vnd.oasis.opendocument.text
+        application/x-vnd.oasis.opendocument.formula-template
+        application/x-vnd.oasis.opendocument.spreadsheet
+        application/x-vnd.oasis.opendocument.chart
+        application/vnd.oasis.opendocument.text-master
+        application/x-vnd.oasis.opendocument.text-master
+        application/x-vnd.oasis.opendocument.text-template
+        application/vnd.oasis.opendocument.graphics
+        application/vnd.oasis.opendocument.graphics-template
+        application/x-vnd.oasis.opendocument.presentation
+        application/vnd.oasis.opendocument.image
+        application/x-vnd.oasis.opendocument.presentation-template
+        application/vnd.oasis.opendocument.chart
+      ].freeze,
+      :one => %w[
+        application/onenote
+        application/msonenote
+      ].freeze,
+      :pdf => %w[
+        application/pdf
+      ].freeze,
+      :ppt => %w[
+        application/vnd.ms-powerpoint
+      ].freeze,
+      :pptx => %w[
+        application/vnd.openxmlformats-officedocument.presentationml.presentation
+        application/vnd.ms-powerpoint.presentation.macroenabled.12
+        application/vnd.openxmlformats-officedocument.presentationml.template
+        application/vnd.ms-powerpoint.slideshow.macroenabled.12
+        application/vnd.ms-powerpoint.addin.macroenabled.12
+        application/vnd.openxmlformats-officedocument.presentationml.slideshow
+      ].freeze,
+      :rtf => %w[
+        message/richtext
+        text/richtext
+        text/rtf
+        application/rtf
+      ].freeze,
+      :txt => %w[
+        text/plain
+      ].freeze,
+      :xls => %w[
+        application/x-tika-msoffice
+        application/vnd.ms-excel
+        application/vnd.ms-excel.sheet.3
+        application/vnd.ms-excel.sheet.2
+        application/vnd.ms-excel.workspace.3
+        application/vnd.ms-excel.workspace.4
+        application/vnd.ms-excel.sheet.4
+      ].freeze,
+      :xlsx => %w[
+        application/x-tika-ooxml
+        application/vnd.openxmlformats-officedocument.spreadsheetml.sheet
+        application/vnd.openxmlformats-officedocument.spreadsheetml.template
+        application/vnd.ms-excel.addin.macroenabled.12
+        application/vnd.ms-excel.template.macroenabled.12
+        application/vnd.ms-excel.sheet.macroenabled.12
+      ].freeze
+    }.with_indifferent_access.freeze
+
+    def self.mime_to_extension
+      @mime_to_extension ||= @extension_to_mime.each_with_object({}) do |(key, values), memo|
+        values.each { |value| memo[value] = key.to_s }
+      end.with_indifferent_access.freeze
+    end
+
+    def self.get_extension(mime_type)
+      mime_to_extension[mime_type.to_s.downcase]
+    end
+
+    def self.get_mime_types(extension)
+      @extension_to_mime[extension.to_s.downcase]
+    end
+  end
+end

data/lib/connectors_shared/logger.rb

@@ -0,0 +1,33 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+require 'stubs/app_config' unless defined?(Rails)
+require 'active_support/core_ext/module'
+
+module ConnectorsShared
+  class Logger
+    SUPPORTED_LOG_LEVELS = %i[fatal error warn info debug].freeze
+
+    class << self
+
+      delegate :formatter, :formatter=, :to => :logger
+
+      def setup!(logger)
+        @logger = logger
+      end
+
+      def logger
+        @logger ||= AppConfig.connectors_logger
+      end
+
+      SUPPORTED_LOG_LEVELS.each do |level|
+        define_method(level) do |message|
+          logger.public_send(level, message)
+        end
+      end
+    end
+  end
+end

data/lib/connectors_shared/monitor.rb

@@ -0,0 +1,99 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'connectors_shared/errors'
+require 'stubs/app_config' unless defined?(Rails)
+require 'stubs/swiftype/exception_tracking' unless defined?(Rails)
+
+module ConnectorsShared
+  class Monitor
+    attr_reader :total_error_count, :success_count, :consecutive_error_count, :error_queue
+
+    def initialize(
+      connector:,
+      max_errors: AppConfig.content_source_sync_max_errors,
+      max_consecutive_errors: AppConfig.content_source_sync_max_consecutive_errors,
+      max_error_ratio: AppConfig.content_source_sync_max_error_ratio,
+      window_size: AppConfig.content_source_sync_error_ratio_window_size,
+      error_queue_size: 20
+    )
+      @connector = connector
+      @max_errors = max_errors
+      @max_consecutive_errors = max_consecutive_errors
+      @max_error_ratio = max_error_ratio
+      @window_size = window_size
+      @total_error_count = 0
+      @success_count = 0
+      @consecutive_error_count = 0
+      @window_errors = Array.new(window_size) { false }
+      @window_index = 0
+      @last_error = nil
+      @error_queue_size = error_queue_size
+      @error_queue = []
+    end
+
+    def note_success
+      @consecutive_error_count = 0
+      @success_count += 1
+      increment_window_index
+    end
+
+    def note_error(error, id: Time.now.to_i)
+      stack_trace = Swiftype::ExceptionTracking.generate_stack_trace(error)
+      error_message = Swiftype::ExceptionTracking.generate_error_message(error, nil, nil)
+      @connector.log_debug("Message id: #{id} - #{error_message}\n#{stack_trace}")
+      @total_error_count += 1
+      @consecutive_error_count += 1
+      @window_errors[@window_index] = true
+      @error_queue << DocumentError.new(error.class.name, error_message, stack_trace, id)
+      @error_queue = @error_queue.drop(1) if @error_queue.size > @error_queue_size
+      increment_window_index
+      @last_error = error
+
+      raise_if_necessary
+    end
+
+    def finalize
+      total_documents = @total_error_count + @success_count
+      if total_documents > 0 && @total_error_count.to_f / total_documents > @max_error_ratio
+        raise_with_last_cause(MaxErrorsInWindowExceededError.new("There were #{@total_error_count} errors out of #{total_documents} total documents", :tripped_by => @last_error))
+      end
+    end
+
+    private
+
+    def raise_if_necessary
+      error =
+        if @consecutive_error_count > @max_consecutive_errors
+          MaxSuccessiveErrorsExceededError.new("Exceeded maximum consecutive errors - saw #{@consecutive_error_count} errors in a row.", :tripped_by => @last_error)
+        elsif @total_error_count > @max_errors
+          MaxErrorsExceededError.new("Exceeded maximum number of errors - saw #{@total_error_count} errors in total.", :tripped_by => @last_error)
+        elsif @window_size > 0 && num_errors_in_window / @window_size > @max_error_ratio
+          MaxErrorsInWindowExceededError.new("Exceeded maximum error ratio of #{@max_error_ratio}. Of the last #{@window_size} documents, #{num_errors_in_window} had errors", :tripped_by => @last_error)
+        elsif @last_error.instance_of?(JobInterruptedError)
+          @last_error
+        end
+
+      raise_with_last_cause(error) if error
+    end
+
+    def num_errors_in_window
+      @window_errors.count(&:itself).to_f
+    end
+
+    def increment_window_index
+      @window_index = (@window_index + 1) % @window_size
+    end
+
+    def raise_with_last_cause(error)
+      raise @last_error
+    rescue StandardError
+      raise error
+    end
+  end
+end

data/lib/connectors_shared.rb

@@ -0,0 +1,12 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+require 'connectors_shared/constants'
+require 'connectors_shared/errors'
+require 'connectors_shared/exception_tracking'
+require 'connectors_shared/extension_mapping_util'
+require 'connectors_shared/logger'
+require 'connectors_shared/monitor'

metadata

@@ -0,0 +1,114 @@
+--- !ruby/object:Gem::Specification
+name: connectors_sdk
+version: !ruby/object:Gem::Version
+  version: 8.3.0.0.pre.20220414T060419Z
+platform: ruby
+authors:
+- Elastic
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2022-04-14 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bson
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: mime-types
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: ''
+email: ent-search-dev@elastic.co
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE
+- NOTICE.txt
+- lib/connectors_sdk.rb
+- lib/connectors_sdk/base/.config.rb.un~
+- lib/connectors_sdk/base/.connectors.rb.un~
+- lib/connectors_sdk/base/.registry.rb.un~
+- lib/connectors_sdk/base/adapter.rb
+- lib/connectors_sdk/base/config.rb
+- lib/connectors_sdk/base/custom_client.rb
+- lib/connectors_sdk/base/extractor.rb
+- lib/connectors_sdk/base/registry.rb
+- lib/connectors_sdk/office365/adapter.rb
+- lib/connectors_sdk/office365/config.rb
+- lib/connectors_sdk/office365/custom_client.rb
+- lib/connectors_sdk/office365/extractor.rb
+- lib/connectors_sdk/share_point/.http_call_wrapper.rb.un~
+- lib/connectors_sdk/share_point/adapter.rb
+- lib/connectors_sdk/share_point/authorization.rb
+- lib/connectors_sdk/share_point/extractor.rb
+- lib/connectors_sdk/share_point/http_call_wrapper.rb
+- lib/connectors_shared.rb
+- lib/connectors_shared/constants.rb
+- lib/connectors_shared/errors.rb
+- lib/connectors_shared/exception_tracking.rb
+- lib/connectors_shared/extension_mapping_util.rb
+- lib/connectors_shared/logger.rb
+- lib/connectors_shared/monitor.rb
+homepage: https://github.com/elastic/connectors
+licenses:
+- Elastic-2.0
+metadata:
+  revision: 427fad57fc69a8107c127ecbf99c6a5e803ea90c
+  repository: https://github.com/elastic/connectors.git
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">"
+    - !ruby/object:Gem::Version
+      version: 1.3.1
+requirements: []
+rubyforge_project:
+rubygems_version: 2.7.7
+signing_key:
+specification_version: 4
+summary: Gem containing apis used by Enterprise Search and implementations of Connectors
+test_files: []