connectors_service 8.6.0.4.pre.20221116T024501Z → 8.7.0.0.pre.20221117T004928Z

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9a610999d9c34543392e1dd9b7b9ab136557992cc1503fa5aa5973186d17af9e
-  data.tar.gz: '086282a4a15ce1b168e6a0add5f78646e1fe89c336b8a257677826cce3677635'
+  metadata.gz: 39e0bb6ae283bcb1afebd57e715342b7e30a7178c33653dfd05fee6f322061e6
+  data.tar.gz: e1fd33b8d28d7d906b4b8eea672a34c0be12e0a124977bbeb10c62c147690a8b
 SHA512:
-  metadata.gz: bf0a0a22ea96074bcbb904000ec489a591fc8491bc2d91759bc1a097c60e31c2abad7b118bb97d16bbe80c5b00dbe08f7c65653710c2d6a77d3356607d87e045
-  data.tar.gz: 67ff773ed2ae3f869897b5fb3a2c0dc3fc55c5922c602e032ed5bafa800043bd4b7f4af90fdd055e575ea877f3cf3b9e308d91c58bad59cd6c63dd5eb8ab98f4
+  metadata.gz: bb6b96ac0c07e9ed4cbe87f16a17844771c12d7a7ccbb2e8c7aeb571cb146f67e5a0489b1e73ff907613dcda442912ba09c193e0e8e4e11fec553ebe9afdec12
+  data.tar.gz: c3c2fc12ac9de6ed37a85dbfb46725e2de37c08beba453c3e9d8a3f0c6c56a078a1c5987931251ca84f18e5a9388b6698ff43abe3ce547d291537b3040b15b0a

data/config/connectors.yml

@@ -1,10 +1,11 @@
 # general metadata
-version: 8.6.0.4-20221116T024501Z
-repository: git@github.com:elastic/ent-search-connectors.git
-revision: b3cc1332879a38930a272a63f8c6be1847578204
+version: 8.7.0.0-20221117T004928Z
+repository: https://github.com/elastic/connectors-ruby.git
+revision: 294214a26b0fe9a4347763b01de681c336e8daae
 elasticsearch:
+  cloud_id: CHANGEME
   hosts: http://localhost:9200
-  api_key: MDJyaGZZUUJ1ZXdOLTB1ZnQ5aXQ6a2IwQURWSjVUc1NPcjVsOFBuVDBzZw==
+  api_key: CHANGEME
   retry_on_failure: 3
   request_timeout: 120
   disable_warnings: true
@@ -14,11 +15,11 @@ thread_pool:
   min_threads: 0
   max_threads: 5
   max_queue: 100
-log_level: debug
-ecs_logging: false
+log_level: info
+ecs_logging: true
 poll_interval: 3
 termination_timeout: 60
 heartbeat_interval: 1800
-native_mode: false
-connector_id: 0mrhfYQBuewN-0ufptgN
-service_type: mongodb
+native_mode: true
+connector_id: CHANGEME
+service_type: CHANGEME

data/lib/app/config.rb

@@ -54,6 +54,9 @@ puts "Parsing #{CONFIG_FILE} configuration file."
     optional(:poll_interval).value(:integer)
     optional(:termination_timeout).value(:integer)
     optional(:heartbeat_interval).value(:integer)
+
+    optional(:max_ingestion_queue_size).value(:integer) # items
+    optional(:max_ingestion_queue_bytes).value(:integer) # bytes
   end
 end
 

data/lib/app/dispatcher.rb

@@ -133,6 +133,8 @@ module App
           min_threads: MIN_THREADS,
           max_threads: MAX_THREADS,
           max_queue: MAX_QUEUE,
+          max_ingestion_queue_size: (App::Config.max_ingestion_queue_size || Utility::Constants::DEFAULT_MAX_INGESTION_QUEUE_SIZE).to_i,
+          max_ingestion_queue_bytes: (App::Config.max_ingestion_queue_bytes || Utility::Constants::DEFAULT_MAX_INGESTION_QUEUE_BYTES).to_i,
           scheduler: scheduler
         )
 

data/lib/connectors_app///

@@ -0,0 +1,13 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+module ConnectorsApp
+  module Errors
+    INVALID_API_KEY = 'INVALID_API_KEY'
+    UNSUPPORTED_AUTH_SCHEME = 'UNSUPPORTED_AUTH_SCHEME'
+    INTERNAL_SERVER_ERROR = 'INTERNAL_SERVER_ERROR'
+  end
+end

data/lib/core/ingestion/es_sink.rb

@@ -21,7 +21,7 @@ require 'elasticsearch/api'
 module Core
   module Ingestion
     class EsSink
-      def initialize(index_name, request_pipeline, bulk_queue = Utility::BulkQueue.new, max_allowed_document_size = 5 * 1024 * 1024)
+      def initialize(index_name, request_pipeline, bulk_queue = Utility::BulkQueue.new, max_allowed_document_size = Utility::Constants::DEFAULT_MAX_INGESTION_QUEUE_BYTES)
         @client = Utility::EsClient.new(App::Config[:elasticsearch])
         @index_name = index_name
         @request_pipeline = request_pipeline

data/lib/core/jobs/consumer.rb

@@ -6,10 +6,20 @@
 
 # frozen_string_literal: true
 
+require 'utility/constants'
+
 module Core
   module Jobs
     class Consumer
-      def initialize(scheduler:, poll_interval: 3, termination_timeout: 60, min_threads: 1, max_threads: 5, max_queue: 100, idle_time: 5)
+      def initialize(scheduler:,
+                     max_ingestion_queue_size:,
+                     max_ingestion_queue_bytes:,
+                     poll_interval: 3,
+                     termination_timeout: 60,
+                     min_threads: 1,
+                     max_threads: 5,
+                     max_queue: 100,
+                     idle_time: 5)
         @scheduler = scheduler
         @poll_interval = poll_interval
         @termination_timeout = termination_timeout
@@ -18,6 +28,9 @@ module Core
         @max_queue = max_queue
         @idle_time = idle_time
 
+        @max_ingestion_queue_size = max_ingestion_queue_size
+        @max_ingestion_queue_bytes = max_ingestion_queue_bytes
+
         @running = Concurrent::AtomicBoolean.new(false)
       end
 
@@ -76,7 +89,12 @@ module Core
               pool.post do
                 Utility::Logger.info("Connector #{connector_settings.formatted} picked up the job #{job.id}")
                 Core::ElasticConnectorActions.ensure_content_index_exists(connector_settings.index_name)
-                job_runner = Core::SyncJobRunner.new(connector_settings, job)
+                job_runner = Core::SyncJobRunner.new(
+                  connector_settings,
+                  job,
+                  @max_ingestion_queue_size,
+                  @max_ingestion_queue_bytes
+                )
                 job_runner.execute
               rescue Core::JobAlreadyRunningError
                 Utility::Logger.info("Sync job for #{connector_settings.formatted} is already running, skipping.")

data/lib/core/sync_job_runner.rb

@@ -23,9 +23,17 @@ module Core
   class SyncJobRunner
     JOB_REPORTING_INTERVAL = 10
 
-    def initialize(connector_settings, job)
+    def initialize(connector_settings, job, max_ingestion_queue_size, max_ingestion_queue_bytes)
       @connector_settings = connector_settings
-      @sink = Core::Ingestion::EsSink.new(connector_settings.index_name, @connector_settings.request_pipeline)
+      @sink = Core::Ingestion::EsSink.new(
+        connector_settings.index_name,
+        @connector_settings.request_pipeline,
+        Utility::BulkQueue.new(
+          max_ingestion_queue_size,
+          max_ingestion_queue_bytes
+        ),
+        max_ingestion_queue_bytes
+      )
       @connector_class = Connectors::REGISTRY.connector_class(connector_settings.service_type)
       @sync_finished = false
       @sync_error = nil

data/lib/utility/bulk_queue.rb

@@ -6,12 +6,14 @@
 
 require 'json'
 
+require 'utility/constants'
+
 module Utility
   class BulkQueue
     class QueueOverflowError < StandardError; end
 
     # 500 items or 5MB
-    def initialize(operation_count_threshold = 500, size_threshold = 5 * 1024 * 1024)
+    def initialize(operation_count_threshold = Utility::Constants::DEFAULT_MAX_INGESTION_QUEUE_LENGTH, size_threshold = Utility::Constants::DEFAULT_MAX_INGESTION_QUEUE_BYTES)
       @operation_count_threshold = operation_count_threshold.freeze
       @size_threshold = size_threshold.freeze
 

data/lib/utility/constants.rb

@@ -18,5 +18,10 @@ module Utility
     CRAWLER_SERVICE_TYPE = 'elastic-crawler'
     FILTERING_RULES_FEATURE = 'filtering_rules'
     FILTERING_ADVANCED_FEATURE = 'filtering_advanced_config'
+
+    # Maximum number of operations in BULK Elasticsearch operation that will ingest the data
+    DEFAULT_MAX_INGESTION_QUEUE_SIZE = 500
+    # Maximum size of either whole BULK Elasticsearch operation or one document in it
+    DEFAULT_MAX_INGESTION_QUEUE_BYTES = 5 * 1024 * 1024
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: connectors_service
 version: !ruby/object:Gem::Version
-  version: 8.6.0.4.pre.20221116T024501Z
+  version: 8.7.0.0.pre.20221117T004928Z
 platform: ruby
 authors:
 - Elastic
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-11-16 00:00:00.000000000 Z
+date: 2022-11-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -420,6 +420,7 @@ files:
 - lib/connectors/registry.rb
 - lib/connectors/sync_status.rb
 - lib/connectors/tolerable_error_helper.rb
+- lib/connectors_app/\
 - lib/connectors_service.rb
 - lib/connectors_utility.rb
 - lib/core.rb
@@ -469,7 +470,7 @@ homepage: https://github.com/elastic/connectors-ruby
 licenses:
 - Elastic-2.0
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -485,7 +486,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: 1.3.1
 requirements: []
 rubygems_version: 3.0.3.1
-signing_key: 
+signing_key:
 specification_version: 4
 summary: Gem containing Elastic connectors service
 test_files: []