logstash-input-azure_event_hubs 1.2.1 → 1.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 636123e36c33570aaf89bdcc73ea5278e12d31c82744b8438d3e547a2e26226d
-  data.tar.gz: 04d9bcdd2cf90d9cbd9618c029e296d6d67ba4925da1d47a2ed6178955b3cf26
+  metadata.gz: f90b79dfae3e67cbaf4d5fd164760332bac2b60467abc04ea35050b938646ea7
+  data.tar.gz: fc5dde1bc64cc82d28522b79958a5cda08c7208f3fd69167b5a3639bbdb0536f
 SHA512:
-  metadata.gz: ff260a97dbed5ba989b14752858893eeec2a94ed1774e07c509987c709a5e91459128e5ca8680f4f7a42be2918694cb176aa74d83707101ff35e09f8557d743a
-  data.tar.gz: 22a1e7019abc970d1fbba51162f7dea7cce8f0070f82a31ecfe24e95cad8380c8949b0ee51352e608f5d01cf50fd5f9578c26da3ba654356e8da69a8471164d8
+  metadata.gz: 2e74ccf4e7293b76a2b5913c558a59692ccb3d284448b78059eb49104cc85a3604f0548a3b2b567bf841ddac6659d9105f03979ff46b2603937182ac5009d0e4
+  data.tar.gz: 91e7c1e01871badb346c6381304047aae7d507e431919de040b258e0678c5f3f5d559e1aec07e065e25901330fccec1a82643288f4e4f0f2644f1ceb4dde73aa

data/CHANGELOG.md

@@ -1,3 +1,15 @@
+## 1.4.0
+ - Updated the minor version of Azure SDK and other dependencies to ensure users of this plugin get upstream fixes and improvements [#67](https://github.com/logstash-plugins/logstash-input-azure_event_hubs/pull/67)
+
+## 1.3.0
+ - Add EventHub `user properties` in `@metadata` object [#66](https://github.com/logstash-plugins/logstash-input-azure_event_hubs/pull/66)
+
+## 1.2.3
+ - Fixed missing configuration of `prefetch_count` and `receive_timeout` [#61](https://github.com/logstash-plugins/logstash-input-azure_event_hubs/pull/61)
+
+## 1.2.2
+ - Refactor: scope and review global java_imports [#57](https://github.com/logstash-plugins/logstash-input-azure_event_hubs/pull/57)
+
 ## 1.2.1
  - [DOC] Changed documentation to update the default number of threads [#55](https://github.com/logstash-plugins/logstash-input-azure_event_hubs/pull/55)
 
@@ -34,6 +46,6 @@
 ## 1.0.1
  - Fixed logging for exception handler
 
-## 1.0.0 
+## 1.0.0
  - Initial release of `logstash-input-azure_event_hubs` supersedes `logstash-input-azureeventhub`
  - Re-implementation that uses Event Processor Host API and new configuration options.

data/LICENSE

@@ -1,3 +1,4 @@
+
                                  Apache License
                            Version 2.0, January 2004
                         http://www.apache.org/licenses/
@@ -186,7 +187,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright [yyyy] [name of copyright owner]
+   Copyright 2020 Elastic and contributors
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

data/VERSION

@@ -1 +1 @@
-1.2.1
+1.4.0

data/lib/logstash/inputs/azure_event_hubs.rb

@@ -8,18 +8,18 @@ require "logstash/inputs/error_notification_handler"
 require "logstash/inputs/named_thread_factory"
 require "logstash/inputs/look_back_position_provider"
 
+class LogStash::Inputs::AzureEventHubs < LogStash::Inputs::Base
 
-java_import com.microsoft.azure.eventprocessorhost.EventProcessorHost
-java_import com.microsoft.azure.eventprocessorhost.EventProcessorOptions
-java_import com.microsoft.azure.eventprocessorhost.InMemoryCheckpointManager
-java_import com.microsoft.azure.eventprocessorhost.InMemoryLeaseManager
-java_import com.microsoft.azure.eventprocessorhost.HostContext
-java_import com.microsoft.azure.eventhubs.ConnectionStringBuilder
-java_import java.util.concurrent.Executors
-java_import java.util.concurrent.TimeUnit
-
+  java_import com.microsoft.azure.eventprocessorhost.EventProcessorHost
+  java_import com.microsoft.azure.eventprocessorhost.EventProcessorOptions
+  java_import com.microsoft.azure.eventprocessorhost.InMemoryCheckpointManager
+  java_import com.microsoft.azure.eventprocessorhost.InMemoryLeaseManager
+  java_import com.microsoft.azure.eventprocessorhost.HostContext
+  java_import com.microsoft.azure.eventhubs.ConnectionStringBuilder
+  java_import java.util.concurrent.Executors
+  java_import java.util.concurrent.TimeUnit
+  java_import java.time.Duration
 
-class LogStash::Inputs::AzureEventHubs < LogStash::Inputs::Base
   config_name "azure_event_hubs"
 
   # This plugin supports two styles of configuration
@@ -291,7 +291,7 @@ class LogStash::Inputs::AzureEventHubs < LogStash::Inputs::Base
   # }
   config :decorate_events, :validate => :boolean, :default => false
 
-  attr_reader :count, :pre_count
+  attr_reader :count
 
   def initialize(params)
 
@@ -410,6 +410,9 @@ class LogStash::Inputs::AzureEventHubs < LogStash::Inputs::Base
           end
           options = EventProcessorOptions.new
           options.setMaxBatchSize(max_batch_size)
+          options.setPrefetchCount(prefetch_count)
+          options.setReceiveTimeOut(Duration.ofSeconds(receive_timeout))
+          
           options.setExceptionNotification(LogStash::Inputs::Azure::ErrorNotificationHandler.new)
           case @initial_position
           when 'beginning'
@@ -476,5 +479,3 @@ class LogStash::Inputs::AzureEventHubs < LogStash::Inputs::Base
     end
   end
 end
-
-

data/lib/logstash/inputs/error_notification_handler.rb

@@ -1,12 +1,11 @@
 # encoding: utf-8
 require "logstash/util/loggable"
-java_import java.util.function.Consumer
 
 module LogStash
   module Inputs
     module Azure
       class ErrorNotificationHandler
-        include Consumer
+        include java.util.function.Consumer
         include LogStash::Util::Loggable
 
         def initialize
@@ -14,9 +13,9 @@ module LogStash
         end
 
         def accept(exception_received_event_args)
-          @logger.error("Error with Event Processor Host. ", 
+          @logger.error("Error with Event Processor Host. ",
             :host_name => exception_received_event_args.getHostname(),
-            :action => exception_received_event_args.getAction(), 
+            :action => exception_received_event_args.getAction(),
             :exception => exception_received_event_args.getException().toString())
         end
       end

data/lib/logstash/inputs/look_back_position_provider.rb

@@ -1,14 +1,15 @@
 # encoding: utf-8
 require "logstash/util/loggable"
-java_import java.util.function.Function
-java_import com.microsoft.azure.eventhubs.EventPosition
-java_import java.time.Instant
 
 module LogStash
   module Inputs
     module Azure
       class LookBackPositionProvider
-        include Function
+
+        java_import com.microsoft.azure.eventhubs.EventPosition
+        java_import java.time.Instant
+
+        include java.util.function.Function
         include LogStash::Util::Loggable
 
         def initialize(look_back_seconds)

data/lib/logstash/inputs/processor.rb

@@ -48,6 +48,7 @@ module LogStash
                 event.set("[@metadata][azure_event_hubs][sequence]", payload.getSystemProperties.getSequenceNumber)
                 event.set("[@metadata][azure_event_hubs][timestamp]",payload.getSystemProperties.getEnqueuedTime.getEpochSecond)
                 event.set("[@metadata][azure_event_hubs][event_size]", bytes.size)
+                event.set("[@metadata][azure_event_hubs][user_properties]", payload.getProperties)
               end
               @queue << event
               if @checkpoint_interval > 0

data/lib/logstash/inputs/processor_factory.rb

@@ -1,5 +1,6 @@
 # encoding: utf-8
 require "logstash/inputs/processor"
+
 module LogStash
   module Inputs
     module Azure
@@ -22,6 +23,3 @@ module LogStash
     end
   end
 end
-
-
-

data/lib/logstash-input-azure_event_hubs.rb

@@ -2,10 +2,10 @@
 
 require 'jar_dependencies'
 require_jar('com.google.code.gson', 'gson', '2.8.5')
-require_jar('org.apache.qpid', 'proton-j', '0.33.3')
+require_jar('org.apache.qpid', 'proton-j', '0.33.9')
 require_jar('org.apache.logging.log4j', 'log4j-slf4j-impl', '2.9.1')
-require_jar('com.microsoft.azure', 'azure-eventhubs', '2.2.0')
-require_jar('com.microsoft.azure', 'qpid-proton-j-extensions', '1.1.0')
-require_jar('com.microsoft.azure', 'azure-eventhubs-eph', '2.4.0')
-require_jar('com.microsoft.azure', 'azure-storage', '8.0.0')
+require_jar('com.microsoft.azure', 'azure-eventhubs', '2.3.2')
+require_jar('com.microsoft.azure', 'qpid-proton-j-extensions', '1.2.4')
+require_jar('com.microsoft.azure', 'azure-eventhubs-eph', '2.5.2')
+require_jar('com.microsoft.azure', 'azure-storage', '8.6.6')
 require_jar('org.apache.logging.log4j', 'log4j-api', '2.9.1')

data/spec/inputs/azure_event_hub_spec.rb

@@ -2,16 +2,9 @@
 require "logstash/devutils/rspec/spec_helper"
 require "logstash/inputs/azure_event_hubs"
 
-
 java_import com.microsoft.azure.eventprocessorhost.EventProcessorHost
-java_import com.microsoft.azure.eventprocessorhost.EventProcessorOptions
 java_import com.microsoft.azure.eventprocessorhost.InMemoryCheckpointManager
 java_import com.microsoft.azure.eventprocessorhost.InMemoryLeaseManager
-java_import java.util.concurrent.ScheduledThreadPoolExecutor
-java_import java.util.concurrent.CompletableFuture
-java_import java.util.concurrent.TimeUnit
-java_import java.util.concurrent.atomic.AtomicInteger
-
 
 describe LogStash::Inputs::AzureEventHubs do
 
@@ -28,7 +21,7 @@ describe LogStash::Inputs::AzureEventHubs do
           expect(exploded_config[i]['event_hub_connections'][0].value).to start_with('Endpoint=sb://...')
           expect(exploded_config[i]['storage_connection'].value).to eql('DefaultEndpointsProtocol=https;AccountName=...')
           expect(exploded_config[i]['threads']).to be == 9
-          expect(exploded_config[i]['codec']).to be_a_kind_of(LogStash::Codecs::Plain)
+          expect(exploded_config[i]['codec'].class.to_s).to eq("LogStash::Codecs::Plain")
           expect(exploded_config[i]['consumer_group']).to eql('cg')
           expect(exploded_config[i]['max_batch_size']).to be == 20
           expect(exploded_config[i]['prefetch_count']).to be == 30
@@ -67,7 +60,7 @@ describe LogStash::Inputs::AzureEventHubs do
         mock_queue = double("queue")
         mock_host = double("event_processor_host")
         mock_host_context = double("host_context")
-        completable_future = CompletableFuture.new
+        completable_future = java.util.concurrent.CompletableFuture.new
         #simulate work being done before completing the future
         Thread.new do
           sleep 2
@@ -75,10 +68,10 @@ describe LogStash::Inputs::AzureEventHubs do
         end
 
         # rspec has issues with counters and concurrent code, so use threadsafe counters instead
-        host_counter = AtomicInteger.new
-        register_counter = AtomicInteger.new
-        unregister_counter = AtomicInteger.new
-        assertion_count = AtomicInteger.new
+        host_counter = java.util.concurrent.atomic.AtomicInteger.new
+        register_counter = java.util.concurrent.atomic.AtomicInteger.new
+        unregister_counter = java.util.concurrent.atomic.AtomicInteger.new
+        assertion_count = java.util.concurrent.atomic.AtomicInteger.new
 
         allow(mock_host).to receive(:getHostContext) {mock_host_context}
         allow(mock_host_context).to receive(:getEventHubPath) {"foo"}
@@ -186,7 +179,7 @@ describe LogStash::Inputs::AzureEventHubs do
         expect(exploded_config[1]['event_hub_connections'][0].value).to eql('1Endpoint=sb://...')
         expect(exploded_config[1]['storage_connection'].value).to eql('1DefaultEndpointsProtocol=https;AccountName=...')
         expect(exploded_config[1]['threads']).to be == 9
-        expect(exploded_config[1]['codec']).to be_a_kind_of(LogStash::Codecs::JSON) # different between configs
+        expect(exploded_config[1]['codec'].class.to_s).to eq("LogStash::Codecs::JSON") # different between configs
         expect(exploded_config[1]['consumer_group']).to eql('cg1') # override global
         expect(exploded_config[1]['max_batch_size']).to be == 21 # filled from global
         expect(exploded_config[1]['prefetch_count']).to be == 300 # default
@@ -202,7 +195,7 @@ describe LogStash::Inputs::AzureEventHubs do
         mock_queue = double("queue")
         mock_host = double("event_processor_host")
         mock_host_context = double("host_context")
-        completable_future = CompletableFuture.new
+        completable_future = java.util.concurrent.CompletableFuture.new
         #simulate work being done before completing the future
         Thread.new do
           sleep 2
@@ -210,10 +203,10 @@ describe LogStash::Inputs::AzureEventHubs do
         end
 
         # rspec has issues with counters and concurrent code, so use threadsafe counters instead
-        host_counter = AtomicInteger.new
-        register_counter = AtomicInteger.new
-        unregister_counter = AtomicInteger.new
-        assertion_count = AtomicInteger.new
+        host_counter = java.util.concurrent.atomic.AtomicInteger.new
+        register_counter = java.util.concurrent.atomic.AtomicInteger.new
+        unregister_counter = java.util.concurrent.atomic.AtomicInteger.new
+        assertion_count = java.util.concurrent.atomic.AtomicInteger.new
         allow_any_instance_of(InMemoryLeaseManager).to receive(:java_send)
         allow_any_instance_of(InMemoryCheckpointManager).to receive(:java_send)
 
@@ -319,4 +312,3 @@ describe LogStash::Inputs::AzureEventHubs do
     end
   end
 end
-

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-azure_event_hubs
 version: !ruby/object:Gem::Version
-  version: 1.2.1
+  version: 1.4.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-03-12 00:00:00.000000000 Z
+date: 2021-09-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -104,13 +104,13 @@ files:
 - logstash-input-azure_event_hubs.gemspec
 - spec/inputs/azure_event_hub_spec.rb
 - vendor/jar-dependencies/com/google/code/gson/gson/2.8.5/gson-2.8.5.jar
-- vendor/jar-dependencies/com/microsoft/azure/azure-eventhubs-eph/2.4.0/azure-eventhubs-eph-2.4.0.jar
-- vendor/jar-dependencies/com/microsoft/azure/azure-eventhubs/2.2.0/azure-eventhubs-2.2.0.jar
-- vendor/jar-dependencies/com/microsoft/azure/azure-storage/8.0.0/azure-storage-8.0.0.jar
-- vendor/jar-dependencies/com/microsoft/azure/qpid-proton-j-extensions/1.1.0/qpid-proton-j-extensions-1.1.0.jar
+- vendor/jar-dependencies/com/microsoft/azure/azure-eventhubs-eph/2.5.2/azure-eventhubs-eph-2.5.2.jar
+- vendor/jar-dependencies/com/microsoft/azure/azure-eventhubs/2.3.2/azure-eventhubs-2.3.2.jar
+- vendor/jar-dependencies/com/microsoft/azure/azure-storage/8.6.6/azure-storage-8.6.6.jar
+- vendor/jar-dependencies/com/microsoft/azure/qpid-proton-j-extensions/1.2.4/qpid-proton-j-extensions-1.2.4.jar
 - vendor/jar-dependencies/org/apache/logging/log4j/log4j-api/2.9.1/log4j-api-2.9.1.jar
 - vendor/jar-dependencies/org/apache/logging/log4j/log4j-slf4j-impl/2.9.1/log4j-slf4j-impl-2.9.1.jar
-- vendor/jar-dependencies/org/apache/qpid/proton-j/0.33.3/proton-j-0.33.3.jar
+- vendor/jar-dependencies/org/apache/qpid/proton-j/0.33.9/proton-j-0.33.9.jar
 homepage: http://www.elastic.co/guide/en/logstash/current/index.html
 licenses:
 - Apache-2.0
@@ -133,8 +133,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project:
-rubygems_version: 2.6.13
+rubygems_version: 3.1.6
 signing_key:
 specification_version: 4
 summary: Consumes events from Azure Event Hubs for use with Logstash