connectors_utility 8.6.0.4.pre.20221114T233920Z → 8.6.0.4.pre.20221114T235434Z

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 37c7557fa83583533bb2259248ab405166619dce6ba8c89667c4377ff0e9f4a2
-  data.tar.gz: 83e9e53681e78993213d11d4f100bbc93a788c85dc559423df108cc26a8807bc
+  metadata.gz: 5925861ca9c2d09d5a4b27cbb1262e7dee74420815b0df1988d97b5f1c4cbb3b
+  data.tar.gz: 06bc7686285b134ac9198bb8ad3579b759fda60d2e4962e1fe4fc730828a8428
 SHA512:
-  metadata.gz: 1dcde38d997b4bf23fe9bbcc6b90ac0d2617b8bd70692db21178e8ae7ceaf38406242c0a782a78a64d9822e3fa3e01a41f217bd1c39e6e2be0b6c10629775bac
-  data.tar.gz: ad902aa4782a28c8630192376c8873cb8e2fd9df2b10e72d2e6598b8e544a896f7b0e00b9e5fa4e14a20d718e45b50c8c8213f8d1f0bac5b02fc7d2d3a149c8c
+  metadata.gz: ffb17fd4a35bbecacddc59b018b20b3d24ef1581e5ca6b182dc73aa42100da4976fbb0d2e2e2504767769a0ea259e3091c877e5ac8ad58683cd1a45022215a0b
+  data.tar.gz: ce59bc3313006d3324d62c572cf9c743028cab4d052ea985a7ca183bf3bd9f55ad5edd6bf93fabd4eb54f2614226c1d8ed195a236d6916cb16682575c902b099

data/lib/utility/bulk_queue.rb

@@ -0,0 +1,85 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+require 'json'
+
+module Utility
+  class BulkQueue
+    class QueueOverflowError < StandardError; end
+
+    # 500 items or 5MB
+    def initialize(operation_count_threshold = 500, size_threshold = 5 * 1024 * 1024)
+      @operation_count_threshold = operation_count_threshold.freeze
+      @size_threshold = size_threshold.freeze
+
+      @buffer = ''
+
+      @current_operation_count = 0
+
+      @current_buffer_size = 0
+      @current_data_size = 0
+    end
+
+    def pop_all
+      result = @buffer
+
+      reset
+
+      result
+    end
+
+    def add(operation, payload = nil)
+      raise QueueOverflowError unless will_fit?(operation, payload)
+
+      operation_size = get_size(operation)
+      payload_size = get_size(payload)
+
+      @current_operation_count += 1
+      @current_buffer_size += operation_size
+      @current_buffer_size += payload_size
+      @current_data_size += payload_size
+
+      @buffer << operation
+      @buffer << "\n"
+
+      if payload
+        @buffer << payload
+        @buffer << "\n"
+      end
+    end
+
+    def will_fit?(operation, payload = nil)
+      return false if @current_operation_count + 1 > @operation_count_threshold
+
+      operation_size = get_size(operation)
+      payload_size = get_size(payload)
+
+      @current_buffer_size + operation_size + payload_size < @size_threshold
+    end
+
+    def current_stats
+      {
+        :current_operation_count => @current_operation_count,
+        :current_buffer_size => @current_buffer_size
+      }
+    end
+
+    private
+
+    def get_size(str)
+      return 0 unless str
+      str.bytesize
+    end
+
+    def reset
+      @current_operation_count = 0
+      @current_buffer_size = 0
+      @current_data_size = 0
+
+      @buffer = ''
+    end
+  end
+end

data/lib/utility/logger.rb

@@ -4,7 +4,6 @@
 # you may not use this file except in compliance with the Elastic License.
 #
 
-require 'config'
 require 'logger'
 require 'active_support/core_ext/module'
 require 'active_support/core_ext/string/filters'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: connectors_utility
 version: !ruby/object:Gem::Version
-  version: 8.6.0.4.pre.20221114T233920Z
+  version: 8.6.0.4.pre.20221114T235434Z
 platform: ruby
 authors:
 - Elastic
@@ -110,6 +110,7 @@ files:
 - lib/core/elastic_connector_actions.rb
 - lib/core/scheduler.rb
 - lib/utility.rb
+- lib/utility/bulk_queue.rb
 - lib/utility/common.rb
 - lib/utility/constants.rb
 - lib/utility/cron.rb