logstash-input-azure_event_hubs 1.1.4 → 1.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4dcc045714f46695cef8987d6c568d0222f603d795e14dd29d4d4b473bbd4f41
-  data.tar.gz: e69a6f42084ea5e6c4422f63aa94ae50c8c5aa292f216c3de30e066d78fa2cae
+  metadata.gz: 7d4912fed67f958fd8ae63335bea88a35cea37b9dd42acfbd57a053544ccfa14
+  data.tar.gz: 4dc5b535f4498f09ccd8c5487637b5f480d6ab41bdf6c8ce48f1f1b3f7d87cff
 SHA512:
-  metadata.gz: c00d6e820334a00c64d9b1b9ed358deee11df1e2ff81ffc2d8a47741fb81b7e6ddb9e6fdbe60a9e82e094334ca854869275470f1ab52c79066e69b6aed159594
-  data.tar.gz: 723964b7b0dfadad14756fad6f9f236b040c0e0993a59e5ef9f798b1faa36ca9fcd6adccfc2587958f3ab8e02fa04c6f30b63cb311f2c062f50d150d061075ad
+  metadata.gz: 7867920f3d3a00106d69a1ad3c94b8577b441addd6f2cddcf65c36f73835e4ee82d0db4cefd2962598c9d75cbfbfbab726dec437c2b7ffc82537a8e95ea1f2f6
+  data.tar.gz: cd2590603352dfd1207ab02ee4f5b4413de332d66ef58a3c3efcdde8d89046c3e8d9b923f04b13ce4e9b6fd12b39e52c25c32374ce40e7dbc27017414f887f22

data/CHANGELOG.md

@@ -1,3 +1,18 @@
+## 1.3.0
+ - Add EventHub `user properties` in `@metadata` object [#66](https://github.com/logstash-plugins/logstash-input-azure_event_hubs/pull/66)
+
+## 1.2.3
+ - Fixed missing configuration of `prefetch_count` and `receive_timeout` [#61](https://github.com/logstash-plugins/logstash-input-azure_event_hubs/pull/61)
+
+## 1.2.2
+ - Refactor: scope and review global java_imports [#57](https://github.com/logstash-plugins/logstash-input-azure_event_hubs/pull/57)
+
+## 1.2.1
+ - [DOC] Changed documentation to update the default number of threads [#55](https://github.com/logstash-plugins/logstash-input-azure_event_hubs/pull/55)
+
+## 1.2.0
+ - Changed the default number of threads from `4` to `16` to match the default number from the Azure-Sdk EventProcessorHost [#54](https://github.com/logstash-plugins/logstash-input-azure_event_hubs/pull/54)
+
 ## 1.1.4
  - Fixed missing configuration of the `max_batch_size`setting [#52](https://github.com/logstash-plugins/logstash-input-azure_event_hubs/pull/52)
 
@@ -28,6 +43,6 @@
 ## 1.0.1
  - Fixed logging for exception handler
 
-## 1.0.0 
+## 1.0.0
  - Initial release of `logstash-input-azure_event_hubs` supersedes `logstash-input-azureeventhub`
  - Re-implementation that uses Event Processor Host API and new configuration options.

data/LICENSE

@@ -1,3 +1,4 @@
+
                                  Apache License
                            Version 2.0, January 2004
                         http://www.apache.org/licenses/
@@ -186,7 +187,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright [yyyy] [name of copyright owner]
+   Copyright 2020 Elastic and contributors
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

data/VERSION

@@ -1 +1 @@
-1.1.4
+1.3.0

data/lib/logstash/inputs/azure_event_hubs.rb

@@ -8,18 +8,18 @@ require "logstash/inputs/error_notification_handler"
 require "logstash/inputs/named_thread_factory"
 require "logstash/inputs/look_back_position_provider"
 
+class LogStash::Inputs::AzureEventHubs < LogStash::Inputs::Base
 
-java_import com.microsoft.azure.eventprocessorhost.EventProcessorHost
-java_import com.microsoft.azure.eventprocessorhost.EventProcessorOptions
-java_import com.microsoft.azure.eventprocessorhost.InMemoryCheckpointManager
-java_import com.microsoft.azure.eventprocessorhost.InMemoryLeaseManager
-java_import com.microsoft.azure.eventprocessorhost.HostContext
-java_import com.microsoft.azure.eventhubs.ConnectionStringBuilder
-java_import java.util.concurrent.Executors
-java_import java.util.concurrent.TimeUnit
-
+  java_import com.microsoft.azure.eventprocessorhost.EventProcessorHost
+  java_import com.microsoft.azure.eventprocessorhost.EventProcessorOptions
+  java_import com.microsoft.azure.eventprocessorhost.InMemoryCheckpointManager
+  java_import com.microsoft.azure.eventprocessorhost.InMemoryLeaseManager
+  java_import com.microsoft.azure.eventprocessorhost.HostContext
+  java_import com.microsoft.azure.eventhubs.ConnectionStringBuilder
+  java_import java.util.concurrent.Executors
+  java_import java.util.concurrent.TimeUnit
+  java_import java.time.Duration
 
-class LogStash::Inputs::AzureEventHubs < LogStash::Inputs::Base
   config_name "azure_event_hubs"
 
   # This plugin supports two styles of configuration
@@ -111,9 +111,9 @@ class LogStash::Inputs::AzureEventHubs < LogStash::Inputs::Base
 
   # Total threads used process events. Requires at minimum 2 threads. This option can not be set per Event Hub.
   # azure_event_hubs {
-  #    threads => 4
+  #    threads => 16
   # }
-  config :threads, :validate => :number, :default => 4
+  config :threads, :validate => :number, :default => 16
 
   # Consumer group used to read the Event Hub(s). It is recommended to change from the $Default to a consumer group specifically for Logstash, and ensure that all instances of Logstash use that consumer group.
   # basic Example:
@@ -291,7 +291,7 @@ class LogStash::Inputs::AzureEventHubs < LogStash::Inputs::Base
   # }
   config :decorate_events, :validate => :boolean, :default => false
 
-  attr_reader :count, :pre_count
+  attr_reader :count
 
   def initialize(params)
 
@@ -410,6 +410,9 @@ class LogStash::Inputs::AzureEventHubs < LogStash::Inputs::Base
           end
           options = EventProcessorOptions.new
           options.setMaxBatchSize(max_batch_size)
+          options.setPrefetchCount(prefetch_count)
+          options.setReceiveTimeOut(Duration.ofSeconds(receive_timeout))
+          
           options.setExceptionNotification(LogStash::Inputs::Azure::ErrorNotificationHandler.new)
           case @initial_position
           when 'beginning'
@@ -476,5 +479,3 @@ class LogStash::Inputs::AzureEventHubs < LogStash::Inputs::Base
     end
   end
 end
-
-

data/lib/logstash/inputs/error_notification_handler.rb

@@ -1,12 +1,11 @@
 # encoding: utf-8
 require "logstash/util/loggable"
-java_import java.util.function.Consumer
 
 module LogStash
   module Inputs
     module Azure
       class ErrorNotificationHandler
-        include Consumer
+        include java.util.function.Consumer
         include LogStash::Util::Loggable
 
         def initialize
@@ -14,9 +13,9 @@ module LogStash
         end
 
         def accept(exception_received_event_args)
-          @logger.error("Error with Event Processor Host. ", 
+          @logger.error("Error with Event Processor Host. ",
             :host_name => exception_received_event_args.getHostname(),
-            :action => exception_received_event_args.getAction(), 
+            :action => exception_received_event_args.getAction(),
             :exception => exception_received_event_args.getException().toString())
         end
       end

data/lib/logstash/inputs/look_back_position_provider.rb

@@ -1,14 +1,15 @@
 # encoding: utf-8
 require "logstash/util/loggable"
-java_import java.util.function.Function
-java_import com.microsoft.azure.eventhubs.EventPosition
-java_import java.time.Instant
 
 module LogStash
   module Inputs
     module Azure
       class LookBackPositionProvider
-        include Function
+
+        java_import com.microsoft.azure.eventhubs.EventPosition
+        java_import java.time.Instant
+
+        include java.util.function.Function
         include LogStash::Util::Loggable
 
         def initialize(look_back_seconds)

data/lib/logstash/inputs/processor.rb

@@ -48,6 +48,7 @@ module LogStash
                 event.set("[@metadata][azure_event_hubs][sequence]", payload.getSystemProperties.getSequenceNumber)
                 event.set("[@metadata][azure_event_hubs][timestamp]",payload.getSystemProperties.getEnqueuedTime.getEpochSecond)
                 event.set("[@metadata][azure_event_hubs][event_size]", bytes.size)
+                event.set("[@metadata][azure_event_hubs][user_properties]", payload.getProperties)
               end
               @queue << event
               if @checkpoint_interval > 0

data/lib/logstash/inputs/processor_factory.rb

@@ -1,5 +1,6 @@
 # encoding: utf-8
 require "logstash/inputs/processor"
+
 module LogStash
   module Inputs
     module Azure
@@ -22,6 +23,3 @@ module LogStash
     end
   end
 end
-
-
-

data/logstash-input-azure_event_hubs.gemspec

@@ -24,5 +24,5 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'logstash-codec-plain'
   s.add_runtime_dependency 'logstash-codec-json'
   s.add_runtime_dependency 'stud', '>= 0.0.22'
-  s.add_development_dependency 'logstash-devutils', '>= 0.0.16'
+  s.add_development_dependency 'logstash-devutils', '>= 1.0.0'
 end

data/spec/inputs/azure_event_hub_spec.rb

@@ -2,16 +2,9 @@
 require "logstash/devutils/rspec/spec_helper"
 require "logstash/inputs/azure_event_hubs"
 
-
 java_import com.microsoft.azure.eventprocessorhost.EventProcessorHost
-java_import com.microsoft.azure.eventprocessorhost.EventProcessorOptions
 java_import com.microsoft.azure.eventprocessorhost.InMemoryCheckpointManager
 java_import com.microsoft.azure.eventprocessorhost.InMemoryLeaseManager
-java_import java.util.concurrent.ScheduledThreadPoolExecutor
-java_import java.util.concurrent.CompletableFuture
-java_import java.util.concurrent.TimeUnit
-java_import java.util.concurrent.atomic.AtomicInteger
-
 
 describe LogStash::Inputs::AzureEventHubs do
 
@@ -28,7 +21,7 @@ describe LogStash::Inputs::AzureEventHubs do
           expect(exploded_config[i]['event_hub_connections'][0].value).to start_with('Endpoint=sb://...')
           expect(exploded_config[i]['storage_connection'].value).to eql('DefaultEndpointsProtocol=https;AccountName=...')
           expect(exploded_config[i]['threads']).to be == 9
-          expect(exploded_config[i]['codec']).to be_a_kind_of(LogStash::Codecs::Plain)
+          expect(exploded_config[i]['codec'].class.to_s).to eq("LogStash::Codecs::Plain")
           expect(exploded_config[i]['consumer_group']).to eql('cg')
           expect(exploded_config[i]['max_batch_size']).to be == 20
           expect(exploded_config[i]['prefetch_count']).to be == 30
@@ -67,7 +60,7 @@ describe LogStash::Inputs::AzureEventHubs do
         mock_queue = double("queue")
         mock_host = double("event_processor_host")
         mock_host_context = double("host_context")
-        completable_future = CompletableFuture.new
+        completable_future = java.util.concurrent.CompletableFuture.new
         #simulate work being done before completing the future
         Thread.new do
           sleep 2
@@ -75,10 +68,10 @@ describe LogStash::Inputs::AzureEventHubs do
         end
 
         # rspec has issues with counters and concurrent code, so use threadsafe counters instead
-        host_counter = AtomicInteger.new
-        register_counter = AtomicInteger.new
-        unregister_counter = AtomicInteger.new
-        assertion_count = AtomicInteger.new
+        host_counter = java.util.concurrent.atomic.AtomicInteger.new
+        register_counter = java.util.concurrent.atomic.AtomicInteger.new
+        unregister_counter = java.util.concurrent.atomic.AtomicInteger.new
+        assertion_count = java.util.concurrent.atomic.AtomicInteger.new
 
         allow(mock_host).to receive(:getHostContext) {mock_host_context}
         allow(mock_host_context).to receive(:getEventHubPath) {"foo"}
@@ -186,7 +179,7 @@ describe LogStash::Inputs::AzureEventHubs do
         expect(exploded_config[1]['event_hub_connections'][0].value).to eql('1Endpoint=sb://...')
         expect(exploded_config[1]['storage_connection'].value).to eql('1DefaultEndpointsProtocol=https;AccountName=...')
         expect(exploded_config[1]['threads']).to be == 9
-        expect(exploded_config[1]['codec']).to be_a_kind_of(LogStash::Codecs::JSON) # different between configs
+        expect(exploded_config[1]['codec'].class.to_s).to eq("LogStash::Codecs::JSON") # different between configs
         expect(exploded_config[1]['consumer_group']).to eql('cg1') # override global
         expect(exploded_config[1]['max_batch_size']).to be == 21 # filled from global
         expect(exploded_config[1]['prefetch_count']).to be == 300 # default
@@ -202,7 +195,7 @@ describe LogStash::Inputs::AzureEventHubs do
         mock_queue = double("queue")
         mock_host = double("event_processor_host")
         mock_host_context = double("host_context")
-        completable_future = CompletableFuture.new
+        completable_future = java.util.concurrent.CompletableFuture.new
         #simulate work being done before completing the future
         Thread.new do
           sleep 2
@@ -210,10 +203,10 @@ describe LogStash::Inputs::AzureEventHubs do
         end
 
         # rspec has issues with counters and concurrent code, so use threadsafe counters instead
-        host_counter = AtomicInteger.new
-        register_counter = AtomicInteger.new
-        unregister_counter = AtomicInteger.new
-        assertion_count = AtomicInteger.new
+        host_counter = java.util.concurrent.atomic.AtomicInteger.new
+        register_counter = java.util.concurrent.atomic.AtomicInteger.new
+        unregister_counter = java.util.concurrent.atomic.AtomicInteger.new
+        assertion_count = java.util.concurrent.atomic.AtomicInteger.new
         allow_any_instance_of(InMemoryLeaseManager).to receive(:java_send)
         allow_any_instance_of(InMemoryCheckpointManager).to receive(:java_send)
 
@@ -319,4 +312,3 @@ describe LogStash::Inputs::AzureEventHubs do
     end
   end
 end
-

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-azure_event_hubs
 version: !ruby/object:Gem::Version
-  version: 1.1.4
+  version: 1.3.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-02-29 00:00:00.000000000 Z
+date: 2021-06-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -71,7 +71,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.0.16
+        version: 1.0.0
   name: logstash-devutils
   prerelease: false
   type: :development
@@ -79,7 +79,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.0.16
+        version: 1.0.0
 description: This gem is a Logstash plugin required to be installed on top of the
   Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This
   gem is not a stand-alone program