connectors_service 8.6.0.4.pre.20221104T200814Z → 8.6.0.4.pre.20221114T233727Z

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: connectors_service
 version: !ruby/object:Gem::Version
-  version: 8.6.0.4.pre.20221104T200814Z
+  version: 8.6.0.4.pre.20221114T233727Z
 platform: ruby
 authors:
 - Elastic
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-11-04 00:00:00.000000000 Z
+date: 2022-11-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -398,6 +398,7 @@ files:
 - lib/connectors/base/adapter.rb
 - lib/connectors/base/connector.rb
 - lib/connectors/base/custom_client.rb
+- lib/connectors/base/simple_rules_parser.rb
 - lib/connectors/connector_status.rb
 - lib/connectors/crawler/scheduler.rb
 - lib/connectors/example/attachments/first_attachment.txt
@@ -409,21 +410,27 @@ files:
 - lib/connectors/gitlab/custom_client.rb
 - lib/connectors/gitlab/extractor.rb
 - lib/connectors/mongodb/connector.rb
+- lib/connectors/mongodb/mongo_rules_parser.rb
 - lib/connectors/registry.rb
 - lib/connectors/sync_status.rb
 - lib/connectors_service.rb
 - lib/connectors_utility.rb
 - lib/core.rb
 - lib/core/configuration.rb
+- lib/core/connector_job.rb
 - lib/core/connector_settings.rb
 - lib/core/elastic_connector_actions.rb
+- lib/core/filtering.rb
+- lib/core/filtering/post_process_engine.rb
+- lib/core/filtering/post_process_result.rb
+- lib/core/filtering/simple_rule.rb
+- lib/core/filtering/validation_job_runner.rb
+- lib/core/filtering/validation_status.rb
 - lib/core/heartbeat.rb
+- lib/core/ingestion.rb
+- lib/core/ingestion/es_sink.rb
+- lib/core/ingestion/ingester.rb
 - lib/core/native_scheduler.rb
-- lib/core/output_sink.rb
-- lib/core/output_sink/base_sink.rb
-- lib/core/output_sink/combined_sink.rb
-- lib/core/output_sink/console_sink.rb
-- lib/core/output_sink/es_sink.rb
 - lib/core/scheduler.rb
 - lib/core/single_scheduler.rb
 - lib/core/sync_job_runner.rb
@@ -432,6 +439,7 @@ files:
 - lib/stubs/connectors/stats.rb
 - lib/stubs/service_type.rb
 - lib/utility.rb
+- lib/utility/bulk_queue.rb
 - lib/utility/common.rb
 - lib/utility/constants.rb
 - lib/utility/cron.rb
@@ -443,6 +451,7 @@ files:
 - lib/utility/es_client.rb
 - lib/utility/exception_tracking.rb
 - lib/utility/extension_mapping_util.rb
+- lib/utility/filtering.rb
 - lib/utility/logger.rb
 - lib/utility/middleware/basic_auth.rb
 - lib/utility/middleware/bearer_auth.rb

data/lib/core/output_sink/base_sink.rb

@@ -1,33 +0,0 @@
-#
-# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
-# or more contributor license agreements. Licensed under the Elastic License;
-# you may not use this file except in compliance with the Elastic License.
-#
-
-# frozen_string_literal: true
-
-module Core
-  module OutputSink
-    class BaseSink
-      def ingest(_document)
-        raise 'not implemented'
-      end
-
-      def ingest_multiple(_documents)
-        raise 'not implemented'
-      end
-
-      def delete(_id)
-        raise 'not implemented'
-      end
-
-      def delete_multiple(_ids)
-        raise 'not implemented'
-      end
-
-      def flush(_size: nil)
-        raise 'not implemented'
-      end
-    end
-  end
-end

data/lib/core/output_sink/combined_sink.rb

@@ -1,38 +0,0 @@
-#
-# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
-# or more contributor license agreements. Licensed under the Elastic License;
-# you may not use this file except in compliance with the Elastic License.
-#
-
-# frozen_string_literal: true
-
-require 'core/output_sink/base_sink'
-require 'utility/logger'
-
-module Core::OutputSink
-  class CombinedSink < Core::OutputSink::BaseSink
-    def initialize(sinks = [])
-      @sinks = sinks
-    end
-
-    def ingest(document)
-      @sinks.each { |sink| sink.ingest(document) }
-    end
-
-    def flush(size: nil)
-      @sinks.each { |sink| sink.flush(size: size) }
-    end
-
-    def ingest_multiple(documents)
-      @sinks.each { |sink| sink.ingest_multiple(documents) }
-    end
-
-    def delete(id)
-      @sinks.each { |sink| sink.delete(id) }
-    end
-
-    def delete_multiple(ids)
-      @sinks.each { |sink| sink.delete_multiple(ids) }
-    end
-  end
-end

data/lib/core/output_sink/console_sink.rb

@@ -1,51 +0,0 @@
-#
-# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
-# or more contributor license agreements. Licensed under the Elastic License;
-# you may not use this file except in compliance with the Elastic License.
-#
-
-# frozen_string_literal: true
-
-require 'core/output_sink'
-require 'utility/logger'
-
-module Core::OutputSink
-  class ConsoleSink < Core::OutputSink::BaseSink
-    def ingest(document)
-      print_header 'Got a single document:'
-      puts document
-    end
-
-    def flush(size: nil)
-      print_header 'Flushing'
-      puts "Flush size: #{size}"
-    end
-
-    def ingest_multiple(documents)
-      print_header 'Got multiple documents:'
-      puts documents
-    end
-
-    def delete(id)
-      print_header "Deleting single id: #{id}"
-      puts id
-    end
-
-    def delete_multiple(ids)
-      print_header "Deleting several ids: #{ids}"
-      puts ids
-    end
-
-    private
-
-    def print_delim
-      puts '----------------------------------------------------'
-    end
-
-    def print_header(header)
-      print_delim
-      puts header
-      print_delim
-    end
-  end
-end

data/lib/core/output_sink/es_sink.rb

@@ -1,74 +0,0 @@
-#
-# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
-# or more contributor license agreements. Licensed under the Elastic License;
-# you may not use this file except in compliance with the Elastic License.
-#
-
-# frozen_string_literal: true
-
-require 'active_support/core_ext/numeric/time'
-require 'app/config'
-require 'core/output_sink/base_sink'
-require 'utility/es_client'
-require 'utility/logger'
-
-module Core::OutputSink
-  class EsSink < Core::OutputSink::BaseSink
-    def initialize(index_name, request_pipeline, flush_threshold = 50)
-      super()
-      @client = Utility::EsClient.new(App::Config[:elasticsearch])
-      @index_name = index_name
-      @request_pipeline = request_pipeline
-      @operation_queue = []
-      @flush_threshold = flush_threshold
-    end
-
-    def ingest(document)
-      return if document.blank?
-
-      @operation_queue << { :index => { :_index => index_name, :_id => document[:id], :data => document } }
-      flush if ready_to_flush?
-    end
-
-    def delete(doc_id)
-      return if doc_id.nil?
-
-      @operation_queue << { :delete => { :_index => index_name, :_id => doc_id } }
-      flush if ready_to_flush?
-    end
-
-    def flush(size: nil)
-      flush_size = size || @flush_threshold
-
-      while @operation_queue.any?
-        data_to_flush = @operation_queue.pop(flush_size)
-        send_data(data_to_flush)
-      end
-    end
-
-    def ingest_multiple(documents)
-      Utility::Logger.debug "Enqueueing #{documents&.size} documents to the index #{index_name}."
-      documents.each { |doc| ingest(doc) }
-    end
-
-    def delete_multiple(ids)
-      Utility::Logger.debug "Enqueueing #{ids&.size} ids to delete from the index #{index_name}."
-      ids.each { |id| delete(id) }
-    end
-
-    private
-
-    attr_accessor :index_name
-
-    def send_data(ops)
-      return if ops.empty?
-
-      @client.bulk(:body => ops, :pipeline => @request_pipeline)
-      Utility::Logger.info "Applied #{ops.size} upsert/delete operations to the index #{index_name}."
-    end
-
-    def ready_to_flush?
-      @operation_queue.size >= @flush_threshold
-    end
-  end
-end