connectors_service 8.6.0.3 → 8.6.0.4.pre.20221114T233727Z

data/lib/utility/errors.rb

@@ -5,6 +5,7 @@
 #
 
 require 'active_support/core_ext/string'
+require 'elasticsearch'
 
 module Utility
   class DocumentError
@@ -31,6 +32,8 @@ module Utility
   end
 
   class ClientError < StandardError; end
+
+  class InvalidFilterConfigError < StandardError; end
   class EvictionWithNoProgressError < StandardError; end
   class EvictionError < StandardError
     attr_accessor :cursors
@@ -89,6 +92,7 @@ module Utility
   class InvalidTokenError < StandardError; end
   class TokenRefreshFailedError < StandardError; end
   class ConnectorNotAvailableError < StandardError; end
+  class AuthorizationError < StandardError; end
 
   # For when we want to explicitly set a #cause but can't
   class ExplicitlyCausedError < StandardError
@@ -124,6 +128,7 @@ module Utility
     end
   end
 
+  AUTHORIZATION_ERRORS = [Elastic::Transport::Transport::Errors::Unauthorized]
   INTERNAL_SERVER_ERROR = Utility::Error.new(500, 'INTERNAL_SERVER_ERROR', 'Internal server error')
   INVALID_API_KEY = Utility::Error.new(401, 'INVALID_API_KEY', 'Invalid API key')
   UNSUPPORTED_AUTH_SCHEME = Utility::Error.new(401, 'UNSUPPORTED_AUTH_SCHEME', 'Unsupported authorization scheme')

data/lib/utility/es_client.rb

@@ -20,8 +20,8 @@ module Utility
       attr_reader :cause
     end
 
-    def initialize(es_config)
-      super(connection_configs(es_config))
+    def initialize(es_config, &block)
+      super(connection_configs(es_config), &block)
     end
 
     def connection_configs(es_config)
@@ -39,6 +39,10 @@ module Utility
       configs[:log] = es_config[:log] || false
       configs[:trace] = es_config[:trace] || false
 
+      # transport options
+      configs[:transport_options] = es_config[:transport_options] if es_config[:transport_options]
+      configs[:ca_fingerprint] = es_config[:ca_fingerprint] if es_config[:ca_fingerprint]
+
       # if log or trace is activated, we use the application logger
       configs[:logger] = if configs[:log] || configs[:trace]
                            Utility::Logger.logger

data/lib/utility/filtering.rb

@@ -0,0 +1,22 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+module Utility
+  class Filtering
+    class << self
+      def extract_filter(filtering)
+        return {} unless filtering.present?
+
+        # assume for now, that first object in filtering array or a filter object itself is the only filtering object
+        filter = filtering.is_a?(Array) ? filtering.first : filtering
+
+        filter.present? ? filter : {}
+      end
+    end
+  end
+end

data/lib/utility/logger.rb

@@ -4,6 +4,7 @@
 # you may not use this file except in compliance with the Elastic License.
 #
 
+require 'config'
 require 'logger'
 require 'active_support/core_ext/module'
 require 'active_support/core_ext/string/filters'
@@ -23,7 +24,7 @@ module Utility
       end
 
       def logger
-        @logger ||= Settings[:ecs_logging] ? EcsLogging::Logger.new(STDOUT) : ::Logger.new(STDOUT)
+        @logger ||= defined?(::Settings) && ::Settings[:ecs_logging] ? EcsLogging::Logger.new(STDOUT) : ::Logger.new(STDOUT)
       end
 
       SUPPORTED_LOG_LEVELS.each do |level|

data/lib/utility.rb

@@ -4,13 +4,15 @@
 # you may not use this file except in compliance with the Elastic License.
 #
 
+require 'utility/bulk_queue'
+require 'utility/common'
 require 'utility/constants'
 require 'utility/cron'
+require 'utility/elasticsearch/index/mappings'
+require 'utility/elasticsearch/index/text_analysis_settings'
+require 'utility/environment'
 require 'utility/errors'
 require 'utility/es_client'
-require 'utility/environment'
 require 'utility/exception_tracking'
 require 'utility/extension_mapping_util'
 require 'utility/logger'
-require 'utility/elasticsearch/index/mappings'
-require 'utility/elasticsearch/index/text_analysis_settings'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: connectors_service
 version: !ruby/object:Gem::Version
-  version: 8.6.0.3
+  version: 8.6.0.4.pre.20221114T233727Z
 platform: ruby
 authors:
 - Elastic
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-10-03 00:00:00.000000000 Z
+date: 2022-11-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -198,14 +198,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 8.4.0
+        version: 8.5.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 8.4.0
+        version: 8.5.0
 - !ruby/object:Gem::Dependency
   name: faraday
   requirement: !ruby/object:Gem::Requirement
@@ -398,33 +398,39 @@ files:
 - lib/connectors/base/adapter.rb
 - lib/connectors/base/connector.rb
 - lib/connectors/base/custom_client.rb
+- lib/connectors/base/simple_rules_parser.rb
 - lib/connectors/connector_status.rb
 - lib/connectors/crawler/scheduler.rb
+- lib/connectors/example/attachments/first_attachment.txt
+- lib/connectors/example/attachments/second_attachment.txt
+- lib/connectors/example/attachments/third_attachment.txt
 - lib/connectors/example/connector.rb
-- lib/connectors/example/example_attachments/first_attachment.txt
-- lib/connectors/example/example_attachments/second_attachment.txt
-- lib/connectors/example/example_attachments/third_attachment.txt
 - lib/connectors/gitlab/adapter.rb
 - lib/connectors/gitlab/connector.rb
 - lib/connectors/gitlab/custom_client.rb
 - lib/connectors/gitlab/extractor.rb
 - lib/connectors/mongodb/connector.rb
+- lib/connectors/mongodb/mongo_rules_parser.rb
 - lib/connectors/registry.rb
 - lib/connectors/sync_status.rb
-- lib/connectors_app/\
 - lib/connectors_service.rb
 - lib/connectors_utility.rb
 - lib/core.rb
 - lib/core/configuration.rb
+- lib/core/connector_job.rb
 - lib/core/connector_settings.rb
 - lib/core/elastic_connector_actions.rb
+- lib/core/filtering.rb
+- lib/core/filtering/post_process_engine.rb
+- lib/core/filtering/post_process_result.rb
+- lib/core/filtering/simple_rule.rb
+- lib/core/filtering/validation_job_runner.rb
+- lib/core/filtering/validation_status.rb
 - lib/core/heartbeat.rb
+- lib/core/ingestion.rb
+- lib/core/ingestion/es_sink.rb
+- lib/core/ingestion/ingester.rb
 - lib/core/native_scheduler.rb
-- lib/core/output_sink.rb
-- lib/core/output_sink/base_sink.rb
-- lib/core/output_sink/combined_sink.rb
-- lib/core/output_sink/console_sink.rb
-- lib/core/output_sink/es_sink.rb
 - lib/core/scheduler.rb
 - lib/core/single_scheduler.rb
 - lib/core/sync_job_runner.rb
@@ -433,6 +439,8 @@ files:
 - lib/stubs/connectors/stats.rb
 - lib/stubs/service_type.rb
 - lib/utility.rb
+- lib/utility/bulk_queue.rb
+- lib/utility/common.rb
 - lib/utility/constants.rb
 - lib/utility/cron.rb
 - lib/utility/elasticsearch/index/language_data.yml
@@ -443,6 +451,7 @@ files:
 - lib/utility/es_client.rb
 - lib/utility/exception_tracking.rb
 - lib/utility/extension_mapping_util.rb
+- lib/utility/filtering.rb
 - lib/utility/logger.rb
 - lib/utility/middleware/basic_auth.rb
 - lib/utility/middleware/bearer_auth.rb
@@ -451,7 +460,7 @@ homepage: https://github.com/elastic/connectors-ruby
 licenses:
 - Elastic-2.0
 metadata: {}
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -462,12 +471,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubygems_version: 3.0.3.1
-signing_key:
+signing_key: 
 specification_version: 4
 summary: Gem containing Elastic connectors service
 test_files: []

data/lib/core/output_sink/base_sink.rb

@@ -1,33 +0,0 @@
-#
-# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
-# or more contributor license agreements. Licensed under the Elastic License;
-# you may not use this file except in compliance with the Elastic License.
-#
-
-# frozen_string_literal: true
-
-module Core
-  module OutputSink
-    class BaseSink
-      def ingest(_document)
-        raise 'not implemented'
-      end
-
-      def ingest_multiple(_documents)
-        raise 'not implemented'
-      end
-
-      def delete(_id)
-        raise 'not implemented'
-      end
-
-      def delete_multiple(_ids)
-        raise 'not implemented'
-      end
-
-      def flush(_size: nil)
-        raise 'not implemented'
-      end
-    end
-  end
-end

data/lib/core/output_sink/combined_sink.rb

@@ -1,38 +0,0 @@
-#
-# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
-# or more contributor license agreements. Licensed under the Elastic License;
-# you may not use this file except in compliance with the Elastic License.
-#
-
-# frozen_string_literal: true
-
-require 'core/output_sink/base_sink'
-require 'utility/logger'
-
-module Core::OutputSink
-  class CombinedSink < Core::OutputSink::BaseSink
-    def initialize(sinks = [])
-      @sinks = sinks
-    end
-
-    def ingest(document)
-      @sinks.each { |sink| sink.ingest(document) }
-    end
-
-    def flush(size: nil)
-      @sinks.each { |sink| sink.flush(size: size) }
-    end
-
-    def ingest_multiple(documents)
-      @sinks.each { |sink| sink.ingest_multiple(documents) }
-    end
-
-    def delete(id)
-      @sinks.each { |sink| sink.delete(id) }
-    end
-
-    def delete_multiple(ids)
-      @sinks.each { |sink| sink.delete_multiple(ids) }
-    end
-  end
-end

data/lib/core/output_sink/console_sink.rb

@@ -1,51 +0,0 @@
-#
-# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
-# or more contributor license agreements. Licensed under the Elastic License;
-# you may not use this file except in compliance with the Elastic License.
-#
-
-# frozen_string_literal: true
-
-require 'core/output_sink'
-require 'utility/logger'
-
-module Core::OutputSink
-  class ConsoleSink < Core::OutputSink::BaseSink
-    def ingest(document)
-      print_header 'Got a single document:'
-      puts document
-    end
-
-    def flush(size: nil)
-      print_header 'Flushing'
-      puts "Flush size: #{size}"
-    end
-
-    def ingest_multiple(documents)
-      print_header 'Got multiple documents:'
-      puts documents
-    end
-
-    def delete(id)
-      print_header "Deleting single id: #{id}"
-      puts id
-    end
-
-    def delete_multiple(ids)
-      print_header "Deleting several ids: #{ids}"
-      puts ids
-    end
-
-    private
-
-    def print_delim
-      puts '----------------------------------------------------'
-    end
-
-    def print_header(header)
-      print_delim
-      puts header
-      print_delim
-    end
-  end
-end

data/lib/core/output_sink/es_sink.rb

@@ -1,74 +0,0 @@
-#
-# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
-# or more contributor license agreements. Licensed under the Elastic License;
-# you may not use this file except in compliance with the Elastic License.
-#
-
-# frozen_string_literal: true
-
-require 'active_support/core_ext/numeric/time'
-require 'app/config'
-require 'core/output_sink/base_sink'
-require 'utility/es_client'
-require 'utility/logger'
-
-module Core::OutputSink
-  class EsSink < Core::OutputSink::BaseSink
-    def initialize(index_name, request_pipeline, flush_threshold = 50)
-      super()
-      @client = Utility::EsClient.new(App::Config[:elasticsearch])
-      @index_name = index_name
-      @request_pipeline = request_pipeline
-      @operation_queue = []
-      @flush_threshold = flush_threshold
-    end
-
-    def ingest(document)
-      return if document.blank?
-
-      @operation_queue << { :index => { :_index => index_name, :_id => document[:id], :data => document } }
-      flush if ready_to_flush?
-    end
-
-    def delete(doc_id)
-      return if doc_id.nil?
-
-      @operation_queue << { :delete => { :_index => index_name, :_id => doc_id } }
-      flush if ready_to_flush?
-    end
-
-    def flush(size: nil)
-      flush_size = size || @flush_threshold
-
-      while @operation_queue.any?
-        data_to_flush = @operation_queue.pop(flush_size)
-        send_data(data_to_flush)
-      end
-    end
-
-    def ingest_multiple(documents)
-      Utility::Logger.debug "Enqueueing #{documents&.size} documents to the index #{index_name}."
-      documents.each { |doc| ingest(doc) }
-    end
-
-    def delete_multiple(ids)
-      Utility::Logger.debug "Enqueueing #{ids&.size} ids to delete from the index #{index_name}."
-      ids.each { |id| delete(id) }
-    end
-
-    private
-
-    attr_accessor :index_name
-
-    def send_data(ops)
-      return if ops.empty?
-
-      @client.bulk(:body => ops, :pipeline => @request_pipeline)
-      Utility::Logger.info "Applied #{ops.size} upsert/delete operations to the index #{index_name}."
-    end
-
-    def ready_to_flush?
-      @operation_queue.size >= @flush_threshold
-    end
-  end
-end