logstash-input-azure_event_hubs 1.1.3 → 1.2.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0ce1293dd83e610effb52c31c5aed6e6b565de16a373631956777e339f76343c
-  data.tar.gz: a1456715ca44e1b9eb749d2abc93f392d792f9246538d74fd469ea6821f63c91
+  metadata.gz: 504e5a3185917fb8bad235bc1c9198919be6186d2ce5c40b4a26679fb4cd2a6a
+  data.tar.gz: 3b50e6b5485c8af058624264881a9c0ba43f916861774582d1e021e98339305d
 SHA512:
-  metadata.gz: 9ad2df11598dc8b07fd01ffe414ea67d8a3b779c396077a2c1bb4aa5019065ee589f5f8015a763df8b2813181c4790d3a73410acf080429e4a9351168bb3570d
-  data.tar.gz: 94598d9b66bd83e232c2b808c2945e0bb234a118cad2ec7d824c6abd44daa1c97a2d740e20ac02eb0fbd97253adf71d9cf94a5a0bc9cbea246a142b53700c62a
+  metadata.gz: 7c9307f7a58a4cb971f1a3417740e47aec017de26c4554df0f5ddeb18d1cd44e3f1220f00e2dd3dfb1a228402cf06c810d8840998bdef9f985ac17fa4d69004c
+  data.tar.gz: 4bad8dbe16d7169093fd93d644957ce10dfbde9ff3898cb4b408ef82f6befc494a77c9aa7bf9ba53904acb77dbdf80b4d3465288fb9d40653f3a330df107428d

data/CHANGELOG.md

@@ -1,3 +1,18 @@
+## 1.2.3
+ - Fixed missing configuration of `prefetch_count` and `receive_timeout` [#61](https://github.com/logstash-plugins/logstash-input-azure_event_hubs/pull/61)
+
+## 1.2.2
+ - Refactor: scope and review global java_imports [#57](https://github.com/logstash-plugins/logstash-input-azure_event_hubs/pull/57)
+
+## 1.2.1
+ - [DOC] Changed documentation to update the default number of threads [#55](https://github.com/logstash-plugins/logstash-input-azure_event_hubs/pull/55)
+
+## 1.2.0
+ - Changed the default number of threads from `4` to `16` to match the default number from the Azure-Sdk EventProcessorHost [#54](https://github.com/logstash-plugins/logstash-input-azure_event_hubs/pull/54)
+
+## 1.1.4
+ - Fixed missing configuration of the `max_batch_size`setting [#52](https://github.com/logstash-plugins/logstash-input-azure_event_hubs/pull/52)
+
 ## 1.1.3
  - [DOC] Added clarification for threads parameter [#50](https://github.com/logstash-plugins/logstash-input-azure_event_hubs/pull/50)
 
@@ -25,6 +40,6 @@
 ## 1.0.1
  - Fixed logging for exception handler
 
-## 1.0.0 
+## 1.0.0
  - Initial release of `logstash-input-azure_event_hubs` supersedes `logstash-input-azureeventhub`
  - Re-implementation that uses Event Processor Host API and new configuration options.

data/LICENSE

@@ -1,3 +1,4 @@
+
                                  Apache License
                            Version 2.0, January 2004
                         http://www.apache.org/licenses/
@@ -186,7 +187,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright [yyyy] [name of copyright owner]
+   Copyright 2020 Elastic and contributors
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

data/VERSION

@@ -1 +1 @@
-1.1.3
+1.2.3

data/lib/logstash/inputs/azure_event_hubs.rb

@@ -8,18 +8,18 @@ require "logstash/inputs/error_notification_handler"
 require "logstash/inputs/named_thread_factory"
 require "logstash/inputs/look_back_position_provider"
 
+class LogStash::Inputs::AzureEventHubs < LogStash::Inputs::Base
 
-java_import com.microsoft.azure.eventprocessorhost.EventProcessorHost
-java_import com.microsoft.azure.eventprocessorhost.EventProcessorOptions
-java_import com.microsoft.azure.eventprocessorhost.InMemoryCheckpointManager
-java_import com.microsoft.azure.eventprocessorhost.InMemoryLeaseManager
-java_import com.microsoft.azure.eventprocessorhost.HostContext
-java_import com.microsoft.azure.eventhubs.ConnectionStringBuilder
-java_import java.util.concurrent.Executors
-java_import java.util.concurrent.TimeUnit
-
+  java_import com.microsoft.azure.eventprocessorhost.EventProcessorHost
+  java_import com.microsoft.azure.eventprocessorhost.EventProcessorOptions
+  java_import com.microsoft.azure.eventprocessorhost.InMemoryCheckpointManager
+  java_import com.microsoft.azure.eventprocessorhost.InMemoryLeaseManager
+  java_import com.microsoft.azure.eventprocessorhost.HostContext
+  java_import com.microsoft.azure.eventhubs.ConnectionStringBuilder
+  java_import java.util.concurrent.Executors
+  java_import java.util.concurrent.TimeUnit
+  java_import java.time.Duration
 
-class LogStash::Inputs::AzureEventHubs < LogStash::Inputs::Base
   config_name "azure_event_hubs"
 
   # This plugin supports two styles of configuration
@@ -111,9 +111,9 @@ class LogStash::Inputs::AzureEventHubs < LogStash::Inputs::Base
 
   # Total threads used process events. Requires at minimum 2 threads. This option can not be set per Event Hub.
   # azure_event_hubs {
-  #    threads => 4
+  #    threads => 16
   # }
-  config :threads, :validate => :number, :default => 4
+  config :threads, :validate => :number, :default => 16
 
   # Consumer group used to read the Event Hub(s). It is recommended to change from the $Default to a consumer group specifically for Logstash, and ensure that all instances of Logstash use that consumer group.
   # basic Example:
@@ -291,7 +291,7 @@ class LogStash::Inputs::AzureEventHubs < LogStash::Inputs::Base
   # }
   config :decorate_events, :validate => :boolean, :default => false
 
-  attr_reader :count, :pre_count
+  attr_reader :count
 
   def initialize(params)
 
@@ -409,6 +409,10 @@ class LogStash::Inputs::AzureEventHubs < LogStash::Inputs::Base
             checkpoint_manager.java_send :initialize, [HostContext], event_processor_host.getHostContext
           end
           options = EventProcessorOptions.new
+          options.setMaxBatchSize(max_batch_size)
+          options.setPrefetchCount(prefetch_count)
+          options.setReceiveTimeOut(Duration.ofSeconds(receive_timeout))
+          
           options.setExceptionNotification(LogStash::Inputs::Azure::ErrorNotificationHandler.new)
           case @initial_position
           when 'beginning'
@@ -475,5 +479,3 @@ class LogStash::Inputs::AzureEventHubs < LogStash::Inputs::Base
     end
   end
 end
-
-

data/lib/logstash/inputs/error_notification_handler.rb

@@ -1,12 +1,11 @@
 # encoding: utf-8
 require "logstash/util/loggable"
-java_import java.util.function.Consumer
 
 module LogStash
   module Inputs
     module Azure
       class ErrorNotificationHandler
-        include Consumer
+        include java.util.function.Consumer
         include LogStash::Util::Loggable
 
         def initialize
@@ -14,9 +13,9 @@ module LogStash
         end
 
         def accept(exception_received_event_args)
-          @logger.error("Error with Event Processor Host. ", 
+          @logger.error("Error with Event Processor Host. ",
             :host_name => exception_received_event_args.getHostname(),
-            :action => exception_received_event_args.getAction(), 
+            :action => exception_received_event_args.getAction(),
             :exception => exception_received_event_args.getException().toString())
         end
       end

data/lib/logstash/inputs/look_back_position_provider.rb

@@ -1,14 +1,15 @@
 # encoding: utf-8
 require "logstash/util/loggable"
-java_import java.util.function.Function
-java_import com.microsoft.azure.eventhubs.EventPosition
-java_import java.time.Instant
 
 module LogStash
   module Inputs
     module Azure
       class LookBackPositionProvider
-        include Function
+
+        java_import com.microsoft.azure.eventhubs.EventPosition
+        java_import java.time.Instant
+
+        include java.util.function.Function
         include LogStash::Util::Loggable
 
         def initialize(look_back_seconds)

data/lib/logstash/inputs/processor.rb

@@ -26,7 +26,7 @@ module LogStash
         end
 
         def onEvents(context, batch)
-          @logger.debug("Event Hub: #{context.getEventHubPath.to_s}, Partition: #{context.getPartitionId.to_s} is processing a batch.") if @logger.debug?
+          @logger.debug("Event Hub: #{context.getEventHubPath.to_s}, Partition: #{context.getPartitionId.to_s} is processing a batch of size #{batch.size}.") if @logger.debug?
           last_payload = nil
           batch_size = 0
           batch.each do |payload|

data/lib/logstash/inputs/processor_factory.rb

@@ -1,5 +1,6 @@
 # encoding: utf-8
 require "logstash/inputs/processor"
+
 module LogStash
   module Inputs
     module Azure
@@ -22,6 +23,3 @@ module LogStash
     end
   end
 end
-
-
-

data/logstash-input-azure_event_hubs.gemspec

@@ -24,5 +24,5 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'logstash-codec-plain'
   s.add_runtime_dependency 'logstash-codec-json'
   s.add_runtime_dependency 'stud', '>= 0.0.22'
-  s.add_development_dependency 'logstash-devutils', '>= 0.0.16'
+  s.add_development_dependency 'logstash-devutils', '>= 1.0.0'
 end

data/spec/inputs/azure_event_hub_spec.rb

@@ -2,16 +2,9 @@
 require "logstash/devutils/rspec/spec_helper"
 require "logstash/inputs/azure_event_hubs"
 
-
 java_import com.microsoft.azure.eventprocessorhost.EventProcessorHost
-java_import com.microsoft.azure.eventprocessorhost.EventProcessorOptions
 java_import com.microsoft.azure.eventprocessorhost.InMemoryCheckpointManager
 java_import com.microsoft.azure.eventprocessorhost.InMemoryLeaseManager
-java_import java.util.concurrent.ScheduledThreadPoolExecutor
-java_import java.util.concurrent.CompletableFuture
-java_import java.util.concurrent.TimeUnit
-java_import java.util.concurrent.atomic.AtomicInteger
-
 
 describe LogStash::Inputs::AzureEventHubs do
 
@@ -28,7 +21,7 @@ describe LogStash::Inputs::AzureEventHubs do
           expect(exploded_config[i]['event_hub_connections'][0].value).to start_with('Endpoint=sb://...')
           expect(exploded_config[i]['storage_connection'].value).to eql('DefaultEndpointsProtocol=https;AccountName=...')
           expect(exploded_config[i]['threads']).to be == 9
-          expect(exploded_config[i]['codec']).to be_a_kind_of(LogStash::Codecs::Plain)
+          expect(exploded_config[i]['codec'].class.to_s).to eq("LogStash::Codecs::Plain")
           expect(exploded_config[i]['consumer_group']).to eql('cg')
           expect(exploded_config[i]['max_batch_size']).to be == 20
           expect(exploded_config[i]['prefetch_count']).to be == 30
@@ -67,7 +60,7 @@ describe LogStash::Inputs::AzureEventHubs do
         mock_queue = double("queue")
         mock_host = double("event_processor_host")
         mock_host_context = double("host_context")
-        completable_future = CompletableFuture.new
+        completable_future = java.util.concurrent.CompletableFuture.new
         #simulate work being done before completing the future
         Thread.new do
           sleep 2
@@ -75,10 +68,10 @@ describe LogStash::Inputs::AzureEventHubs do
         end
 
         # rspec has issues with counters and concurrent code, so use threadsafe counters instead
-        host_counter = AtomicInteger.new
-        register_counter = AtomicInteger.new
-        unregister_counter = AtomicInteger.new
-        assertion_count = AtomicInteger.new
+        host_counter = java.util.concurrent.atomic.AtomicInteger.new
+        register_counter = java.util.concurrent.atomic.AtomicInteger.new
+        unregister_counter = java.util.concurrent.atomic.AtomicInteger.new
+        assertion_count = java.util.concurrent.atomic.AtomicInteger.new
 
         allow(mock_host).to receive(:getHostContext) {mock_host_context}
         allow(mock_host_context).to receive(:getEventHubPath) {"foo"}
@@ -186,7 +179,7 @@ describe LogStash::Inputs::AzureEventHubs do
         expect(exploded_config[1]['event_hub_connections'][0].value).to eql('1Endpoint=sb://...')
         expect(exploded_config[1]['storage_connection'].value).to eql('1DefaultEndpointsProtocol=https;AccountName=...')
         expect(exploded_config[1]['threads']).to be == 9
-        expect(exploded_config[1]['codec']).to be_a_kind_of(LogStash::Codecs::JSON) # different between configs
+        expect(exploded_config[1]['codec'].class.to_s).to eq("LogStash::Codecs::JSON") # different between configs
         expect(exploded_config[1]['consumer_group']).to eql('cg1') # override global
         expect(exploded_config[1]['max_batch_size']).to be == 21 # filled from global
         expect(exploded_config[1]['prefetch_count']).to be == 300 # default
@@ -202,7 +195,7 @@ describe LogStash::Inputs::AzureEventHubs do
         mock_queue = double("queue")
         mock_host = double("event_processor_host")
         mock_host_context = double("host_context")
-        completable_future = CompletableFuture.new
+        completable_future = java.util.concurrent.CompletableFuture.new
         #simulate work being done before completing the future
         Thread.new do
           sleep 2
@@ -210,10 +203,10 @@ describe LogStash::Inputs::AzureEventHubs do
         end
 
         # rspec has issues with counters and concurrent code, so use threadsafe counters instead
-        host_counter = AtomicInteger.new
-        register_counter = AtomicInteger.new
-        unregister_counter = AtomicInteger.new
-        assertion_count = AtomicInteger.new
+        host_counter = java.util.concurrent.atomic.AtomicInteger.new
+        register_counter = java.util.concurrent.atomic.AtomicInteger.new
+        unregister_counter = java.util.concurrent.atomic.AtomicInteger.new
+        assertion_count = java.util.concurrent.atomic.AtomicInteger.new
         allow_any_instance_of(InMemoryLeaseManager).to receive(:java_send)
         allow_any_instance_of(InMemoryCheckpointManager).to receive(:java_send)
 
@@ -319,4 +312,3 @@ describe LogStash::Inputs::AzureEventHubs do
     end
   end
 end
-

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-azure_event_hubs
 version: !ruby/object:Gem::Version
-  version: 1.1.3
+  version: 1.2.3
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-02-28 00:00:00.000000000 Z
+date: 2020-11-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -71,7 +71,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.0.16
+        version: 1.0.0
   name: logstash-devutils
   prerelease: false
   type: :development
@@ -79,7 +79,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.0.16
+        version: 1.0.0
 description: This gem is a Logstash plugin required to be installed on top of the
   Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This
   gem is not a stand-alone program