connectors_utility 8.6.0.4.pre.20221114T233920Z → 8.6.0.4.pre.20221114T235050Z

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 37c7557fa83583533bb2259248ab405166619dce6ba8c89667c4377ff0e9f4a2
-  data.tar.gz: 83e9e53681e78993213d11d4f100bbc93a788c85dc559423df108cc26a8807bc
+  metadata.gz: b3b178cb45ca62a666074e0370d28f4b82477d2ee32e6e7a74b24c218f02c121
+  data.tar.gz: 0b99bd3126a5fdc7cad2ee2b9dc5862284992911043fc7c1931aaac9ea3844d8
 SHA512:
-  metadata.gz: 1dcde38d997b4bf23fe9bbcc6b90ac0d2617b8bd70692db21178e8ae7ceaf38406242c0a782a78a64d9822e3fa3e01a41f217bd1c39e6e2be0b6c10629775bac
-  data.tar.gz: ad902aa4782a28c8630192376c8873cb8e2fd9df2b10e72d2e6598b8e544a896f7b0e00b9e5fa4e14a20d718e45b50c8c8213f8d1f0bac5b02fc7d2d3a149c8c
+  metadata.gz: 31b38cf34d989cb09c2ab61356f330a22224fe58f76b5a02db12da13ed3d3e6329244cc75b9eeb293312510df9ccb1e0ff615ec65c9dc0e832e8dbe2c5c25328
+  data.tar.gz: e9b2b753f2ac8135372303f750cb6630d364d0af802a7b5d980cb5ea470e57a3d23c949d2d60356e585da7c9d15383528982ecb80d1eaca2bb36f12854633950

data/lib/utility/bulk_queue.rb

@@ -0,0 +1,85 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+require 'json'
+
+module Utility
+  class BulkQueue
+    class QueueOverflowError < StandardError; end
+
+    # 500 items or 5MB
+    def initialize(operation_count_threshold = 500, size_threshold = 5 * 1024 * 1024)
+      @operation_count_threshold = operation_count_threshold.freeze
+      @size_threshold = size_threshold.freeze
+
+      @buffer = ''
+
+      @current_operation_count = 0
+
+      @current_buffer_size = 0
+      @current_data_size = 0
+    end
+
+    def pop_all
+      result = @buffer
+
+      reset
+
+      result
+    end
+
+    def add(operation, payload = nil)
+      raise QueueOverflowError unless will_fit?(operation, payload)
+
+      operation_size = get_size(operation)
+      payload_size = get_size(payload)
+
+      @current_operation_count += 1
+      @current_buffer_size += operation_size
+      @current_buffer_size += payload_size
+      @current_data_size += payload_size
+
+      @buffer << operation
+      @buffer << "\n"
+
+      if payload
+        @buffer << payload
+        @buffer << "\n"
+      end
+    end
+
+    def will_fit?(operation, payload = nil)
+      return false if @current_operation_count + 1 > @operation_count_threshold
+
+      operation_size = get_size(operation)
+      payload_size = get_size(payload)
+
+      @current_buffer_size + operation_size + payload_size < @size_threshold
+    end
+
+    def current_stats
+      {
+        :current_operation_count => @current_operation_count,
+        :current_buffer_size => @current_buffer_size
+      }
+    end
+
+    private
+
+    def get_size(str)
+      return 0 unless str
+      str.bytesize
+    end
+
+    def reset
+      @current_operation_count = 0
+      @current_buffer_size = 0
+      @current_data_size = 0
+
+      @buffer = ''
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: connectors_utility
 version: !ruby/object:Gem::Version
-  version: 8.6.0.4.pre.20221114T233920Z
+  version: 8.6.0.4.pre.20221114T235050Z
 platform: ruby
 authors:
 - Elastic
@@ -110,6 +110,7 @@ files:
 - lib/core/elastic_connector_actions.rb
 - lib/core/scheduler.rb
 - lib/utility.rb
+- lib/utility/bulk_queue.rb
 - lib/utility/common.rb
 - lib/utility/constants.rb
 - lib/utility/cron.rb