connectors_service 8.6.0.4.pre.20221114T233727Z → 8.6.0.4.pre.20221116T024501Z

data/lib/utility/logger.rb

@@ -4,7 +4,6 @@
 # you may not use this file except in compliance with the Elastic License.
 #
 
-require 'config'
 require 'logger'
 require 'active_support/core_ext/module'
 require 'active_support/core_ext/string/filters'

data/lib/utility.rb

@@ -4,6 +4,8 @@
 # you may not use this file except in compliance with the Elastic License.
 #
 
+# !!!!!!!!
+# IF YOU EDIT THIS FILE, YOU MUST EDIT THE `connectors_utility.gemspec`
 require 'utility/bulk_queue'
 require 'utility/common'
 require 'utility/constants'
@@ -11,8 +13,12 @@ require 'utility/cron'
 require 'utility/elasticsearch/index/mappings'
 require 'utility/elasticsearch/index/text_analysis_settings'
 require 'utility/environment'
+require 'utility/error_monitor'
 require 'utility/errors'
+require 'utility/filtering'
 require 'utility/es_client'
 require 'utility/exception_tracking'
 require 'utility/extension_mapping_util'
 require 'utility/logger'
+# IF YOU EDIT THIS FILE, YOU MUST EDIT THE `connectors_utility.gemspec`
+# !!!!!!!!

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: connectors_service
 version: !ruby/object:Gem::Version
-  version: 8.6.0.4.pre.20221114T233727Z
+  version: 8.6.0.4.pre.20221116T024501Z
 platform: ruby
 authors:
 - Elastic
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-11-14 00:00:00.000000000 Z
+date: 2022-11-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -396,6 +396,8 @@ files:
 - lib/app/version.rb
 - lib/connectors.rb
 - lib/connectors/base/adapter.rb
+- lib/connectors/base/advanced_snippet_against_schema_validator.rb
+- lib/connectors/base/advanced_snippet_validator.rb
 - lib/connectors/base/connector.rb
 - lib/connectors/base/custom_client.rb
 - lib/connectors/base/simple_rules_parser.rb
@@ -405,14 +407,19 @@ files:
 - lib/connectors/example/attachments/second_attachment.txt
 - lib/connectors/example/attachments/third_attachment.txt
 - lib/connectors/example/connector.rb
+- lib/connectors/example/example_advanced_snippet_validator.rb
 - lib/connectors/gitlab/adapter.rb
 - lib/connectors/gitlab/connector.rb
 - lib/connectors/gitlab/custom_client.rb
 - lib/connectors/gitlab/extractor.rb
+- lib/connectors/gitlab/gitlab_advanced_snippet_validator.rb
 - lib/connectors/mongodb/connector.rb
+- lib/connectors/mongodb/mongo_advanced_snippet_against_schema_validator.rb
+- lib/connectors/mongodb/mongo_advanced_snippet_schema.rb
 - lib/connectors/mongodb/mongo_rules_parser.rb
 - lib/connectors/registry.rb
 - lib/connectors/sync_status.rb
+- lib/connectors/tolerable_error_helper.rb
 - lib/connectors_service.rb
 - lib/connectors_utility.rb
 - lib/core.rb
@@ -429,7 +436,8 @@ files:
 - lib/core/heartbeat.rb
 - lib/core/ingestion.rb
 - lib/core/ingestion/es_sink.rb
-- lib/core/ingestion/ingester.rb
+- lib/core/jobs/consumer.rb
+- lib/core/jobs/producer.rb
 - lib/core/native_scheduler.rb
 - lib/core/scheduler.rb
 - lib/core/single_scheduler.rb
@@ -447,6 +455,7 @@ files:
 - lib/utility/elasticsearch/index/mappings.rb
 - lib/utility/elasticsearch/index/text_analysis_settings.rb
 - lib/utility/environment.rb
+- lib/utility/error_monitor.rb
 - lib/utility/errors.rb
 - lib/utility/es_client.rb
 - lib/utility/exception_tracking.rb

data/lib/core/ingestion/ingester.rb

@@ -1,90 +0,0 @@
-#
-# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
-# or more contributor license agreements. Licensed under the Elastic License;
-# you may not use this file except in compliance with the Elastic License.
-#
-
-# frozen_string_literal: true
-
-require 'utility/logger'
-
-module Core
-  module Ingestion
-    class Ingester
-      def initialize(sink_strategy, max_allowed_document_size = 5 * 1024 * 1024)
-        @sink_strategy = sink_strategy
-        @max_allowed_document_size = max_allowed_document_size
-
-        @ingested_count = 0
-        @ingested_volume = 0
-        @deleted_count = 0
-      end
-
-      def ingest(document)
-        unless document&.any?
-          Utility::Logger.warn('Connector attempted to ingest an empty document, skipping')
-          return
-        end
-
-        serialized_document = @sink_strategy.serialize(document)
-        document_size = serialized_document.bytesize
-
-        if @max_allowed_document_size > 0 && document_size > @max_allowed_document_size
-          Utility::Logger.warn("Connector attempted to ingest too large document with id=#{document['id']} [#{document_size}/#{@max_allowed_document_size}], skipping the document.")
-          return
-        end
-
-        @sink_strategy.ingest(document['id'], serialized_document)
-
-        @ingested_count += 1
-        @ingested_volume += document_size
-      end
-
-      def ingest_multiple(documents)
-        documents.each { |doc| ingest(doc) }
-      end
-
-      def delete(id)
-        return if id.nil?
-
-        @sink_strategy.delete(id)
-
-        @deleted_count += 1
-      end
-
-      def delete_multiple(ids)
-        ids.each { |id| delete(id) }
-      end
-
-      def flush
-        @sink_strategy.flush
-      end
-
-      def ingestion_stats
-        {
-          :indexed_document_count => @ingested_count,
-          :indexed_document_volume => @ingested_volume,
-          :deleted_document_count => @deleted_count
-        }
-      end
-
-      private
-
-      def do_ingest(_id, _serialized_document)
-        raise NotImplementedError
-      end
-
-      def do_delete(_id)
-        raise NotImplementedError
-      end
-
-      def do_flush
-        raise NotImplementedError
-      end
-
-      def do_serialize(_document)
-        raise NotImplementedError
-      end
-    end
-  end
-end