logstash-input-azure_event_hubs 1.4.8 → 1.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 534090e6c5ff8c6f47ce9b358069e9ee456b182c12a5bd29e91c7b4afd61bbbf
-  data.tar.gz: f2b13ba670593366c8d07fd22175752c954fd26cd5570fc759b9791df83a2e90
+  metadata.gz: 639751f75915a94cd8b1b35a7736317a23d471505bd399ad63abb7b8994e5348
+  data.tar.gz: c8d788acbff10038bacae242fdcfca4560b21413995a29da74346c05331ef41e
 SHA512:
-  metadata.gz: 06ca7c423993f970ab2b0e7bdabb6664a2c118189747382beac9dae5a8a9191bdc3d477bc02ea81fc16fd8d8edf95ab5c26ebe40ee0fc004ce9734ab5cd28621
-  data.tar.gz: 8b7106c47551404b8d70c499537eee339ffd6fe364c1db124fafe68c2bd0c495b7c1cc4bb07ec7590bfe85452c120c69f8dbafff5acac3826318d9cfc8c21f88
+  metadata.gz: ba716f7ce51e9230423a894fa99be446c4bf74612288bfd15734d92d0c283b8bf954fcbc9ec6914133b20f85542a3e1adc680dadbbd48387de9a40c9a8f120cd
+  data.tar.gz: 79d7e217c3f3de4f7074fb56c3de6dcfd60b5f7a468c5ca7e63ce7d58d70a4181d204f05d1977de8d5fcf3f9d8586c7ce2e1a68f9b1c938becf474538ae8c61f

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 1.5.0
+  - Updated Azure Event Hub client library to version 3.3.0 [#96](https://github.com/logstash-plugins/logstash-input-azure_event_hubs/pull/96)
+
+## 1.4.9
+  - Fixed issue with `getHostContext` method accessibility, causing plugin not to be able to run  [#93](https://github.com/logstash-plugins/logstash-input-azure_event_hubs/pull/93)
+
 ## 1.4.8
   - Fixed connection placeholder replacements errors with Logstash `8.15.1` and `8.15.2` [#92](https://github.com/logstash-plugins/logstash-input-azure_event_hubs/pull/92)
 

data/VERSION

@@ -1 +1 @@
-1.4.8
+1.5.0

data/lib/logstash/inputs/azure_event_hubs.rb

@@ -401,30 +401,20 @@ class LogStash::Inputs::AzureEventHubs < LogStash::Inputs::Base
         @logger.info("Event Hub #{event_hub_name} is initializing... ")
         begin
           if event_hub['storage_connection']
-            event_processor_host = EventProcessorHost.new(
-                EventProcessorHost.createHostName('logstash'),
-                event_hub_name,
-                event_hub['consumer_group'],
-                event_hub['event_hub_connections'].first.value, #there will only be one in this array by the time it gets here
-                event_hub['storage_connection'].value,
-                event_hub.fetch('storage_container', event_hub_name),
-                scheduled_executor_service)
+            event_processor_host = EventProcessorHost::EventProcessorHostBuilder.newBuilder(EventProcessorHost.createHostName('logstash'), event_hub['consumer_group'])
+                                                         .useAzureStorageCheckpointLeaseManager(
+                                                           event_hub['storage_connection'].value,
+                                                           event_hub.fetch('storage_container', event_hub_name),
+                                                           nil
+                                                         )
+                                                         .useEventHubConnectionString(
+                                                           event_hub['event_hub_connections'].first.value, #there will only be one in this array by the time it gets here
+                                                         )
+                                                     .setExecutor(scheduled_executor_service)
+                                                     .build
           else
             @logger.warn("You have NOT specified a `storage_connection_string` for #{event_hub_name}. This configuration is only supported for a single Logstash instance.")
-            checkpoint_manager = InMemoryCheckpointManager.new
-            lease_manager = InMemoryLeaseManager.new
-            event_processor_host = EventProcessorHost.new(
-                EventProcessorHost.createHostName('logstash'),
-                event_hub_name,
-                event_hub['consumer_group'],
-                event_hub['event_hub_connections'].first.value, #there will only be one in this array by the time it gets here
-                checkpoint_manager,
-                lease_manager,
-                scheduled_executor_service,
-                nil)
-            #using java_send to avoid naming conflicts with 'initialize' method
-            lease_manager.java_send :initialize, [HostContext], event_processor_host.getHostContext
-            checkpoint_manager.java_send :initialize, [HostContext], event_processor_host.getHostContext
+            event_processor_host = create_in_memory_event_processor_host(event_hub, event_hub_name, scheduled_executor_service)
           end
           options = EventProcessorOptions.new
           options.setMaxBatchSize(max_batch_size)
@@ -496,4 +486,34 @@ class LogStash::Inputs::AzureEventHubs < LogStash::Inputs::Base
       @logger.debug("interrupted while waiting to close executor service, this can generally be ignored", :exception => e, :backtrace => e.backtrace) if e
     end
   end
+
+  def create_in_memory_event_processor_host(event_hub, event_hub_name, scheduled_executor_service)
+    checkpoint_manager = InMemoryCheckpointManager.new
+    lease_manager = InMemoryLeaseManager.new
+    event_processor_host = EventProcessorHost::EventProcessorHostBuilder.newBuilder(EventProcessorHost.createHostName('logstash'), event_hub['consumer_group'])
+                                             .useUserCheckpointAndLeaseManagers(checkpoint_manager, lease_manager)
+                                             .useEventHubConnectionString(event_hub['event_hub_connections'].first.value) #there will only be one in this array by the time it gets here
+                                             .setExecutor(scheduled_executor_service)
+                                             .build
+    host_context = get_host_context(event_processor_host)
+    #using java_send to avoid naming conflicts with 'initialize' method
+    lease_manager.java_send :initialize, [HostContext], host_context
+    checkpoint_manager.java_send :initialize, [HostContext], host_context
+    event_processor_host
+  end
+
+  private
+
+  # This method is used to get around the fact that recent versions of jruby do not
+  # allow access to the package private protected method `getHostContext`
+  def get_host_context(event_processor_host)
+    call_private(event_processor_host, 'getHostContext')
+  end
+
+  def call_private(clazz, method)
+    method = clazz.java_class.declared_method(method)
+    method.accessible = true
+    method.invoke(clazz)
+  end
+
 end

data/lib/logstash-input-azure_event_hubs.rb

@@ -1,9 +1,38 @@
 # AUTOGENERATED BY THE GRADLE SCRIPT. DO NOT EDIT.
 
 require 'jar_dependencies'
-require_jar('com.microsoft.azure', 'azure-eventhubs', '2.3.2')
+require_jar('com.microsoft.azure', 'azure-eventhubs', '3.3.0')
 require_jar('com.microsoft.azure', 'qpid-proton-j-extensions', '1.2.4')
-require_jar('com.microsoft.azure', 'azure-eventhubs-eph', '2.5.2')
+require_jar('com.microsoft.azure', 'azure-eventhubs-eph', '3.3.0')
 require_jar('com.microsoft.azure', 'azure-storage', '8.6.6')
-require_jar('com.google.code.gson', 'gson', '2.10.1')
-require_jar('org.apache.qpid', 'proton-j', '0.33.9')
+require_jar('com.google.code.gson', 'gson', '2.8.9')
+require_jar('org.apache.qpid', 'proton-j', '0.33.8')
+require_jar('com.microsoft.azure', 'azure-keyvault-core', '1.2.4')
+require_jar('com.microsoft.azure', 'adal4j', '1.6.4')
+require_jar('com.microsoft.azure', 'azure-annotations', '1.10.0')
+require_jar('com.microsoft.azure', 'azure-client-authentication', '1.7.3')
+require_jar('com.microsoft.azure', 'azure-client-runtime', '1.7.3')
+require_jar('com.microsoft.rest', 'client-runtime', '1.7.3')
+require_jar('com.fasterxml.jackson.core', 'jackson-annotations', '2.12.7')
+require_jar('com.fasterxml.jackson.core', 'jackson-core', '2.12.7')
+require_jar('com.fasterxml.jackson.core', 'jackson-databind', '2.12.7.1')
+require_jar('com.fasterxml.jackson.datatype', 'jackson-datatype-joda', '2.12.7')
+require_jar('com.github.stephenc.jcip', 'jcip-annotations', '1.0-1')
+require_jar('com.google.guava', 'guava', '32.0.1-jre')
+require_jar('com.nimbusds', 'lang-tag', '1.7')
+require_jar('com.nimbusds', 'nimbus-jose-jwt', '9.37.2')
+require_jar('com.nimbusds', 'oauth2-oidc-sdk', '6.5')
+require_jar('com.squareup.okhttp3', 'logging-interceptor', '3.12.2')
+require_jar('com.squareup.okhttp3', 'okhttp-urlconnection', '3.12.2')
+require_jar('com.squareup.okhttp3', 'okhttp', '3.14.7')
+require_jar('com.squareup.okio', 'okio', '1.17.6')
+require_jar('com.squareup.retrofit2', 'adapter-rxjava', '2.7.2')
+require_jar('com.squareup.retrofit2', 'converter-jackson', '2.7.2')
+require_jar('com.squareup.retrofit2', 'retrofit', '2.7.2')
+require_jar('com.sun.mail', 'javax.mail', '1.6.1')
+require_jar('commons-codec', 'commons-codec', '1.11')
+require_jar('io.reactivex', 'rxjava', '1.3.8')
+require_jar('javax.activation', 'activation', '1.1')
+require_jar('net.minidev', 'json-smart', '2.4.9')
+require_jar('org.checkerframework', 'checker-compat-qual', '2.0.0')
+require_jar('org.codehaus.mojo', 'animal-sniffer-annotations', '1.14')

data/spec/inputs/azure_event_hub_spec.rb

@@ -84,25 +84,39 @@ describe LogStash::Inputs::AzureEventHubs do
           unregister_counter.incrementAndGet
           completable_future
         }
-        expect(EventProcessorHost).to receive(:new).at_most(2).times {|host_name, event_hub_name, consumer_group, event_hub_connection, storage_connection, container, executor|
-          case event_hub_name
-          when 'event_hub_name0'
 
-            assertion_count.incrementAndGet
+        build_step_mock = double("final build step")
+        expect(build_step_mock).to receive(:build).at_most(2).times.and_return(mock_host)
+
+        executor_step = double("executor step")
+        expect(executor_step).to receive(:setExecutor).at_most(2).times.and_return(build_step_mock)
+
+        mock_connection_string = double("connection string")
+        expect(mock_connection_string).to receive(:useEventHubConnectionString).at_most(2).times {|event_hub_connection|
+          case event_hub_connection
+          when /.*event_hub_name0$/
             expect(event_hub_connection).to eql(config['event_hub_connections'][0])
-            expect(container).to eql('event_hub_name0') # default
+          when /.*event_hub_name1$/
+            expect(event_hub_connection).to eql(config['event_hub_connections'][1])
+          end
+          executor_step
+        }
 
+        mock_builder = double("storage and lease managers")
+        expect(mock_builder).to receive(:useAzureStorageCheckpointLeaseManager).at_most(2).times {|storage_connection_str, storage_container, storage_blob_prefix|
+          case storage_container
+          when 'event_hub_name0'
+            assertion_count.incrementAndGet
           when 'event_hub_name1'
             assertion_count.incrementAndGet
-            expect(host_name).to start_with('logstash')
-            expect(event_hub_connection).to eql(config['event_hub_connections'][1])
-            expect(container).to eql('event_hub_name1') # default
           end
-          expect(host_name).to start_with('logstash')
-          expect(storage_connection).to eql(config['storage_connection'])
+          mock_connection_string
+        }
 
+        expect(EventProcessorHost::EventProcessorHostBuilder).to receive(:newBuilder).at_most(2).times {|host_name, consumer_group|
+          expect(host_name).to start_with('logstash')
           host_counter.incrementAndGet
-          mock_host
+          mock_builder
         }
         # signal the stop first since the run method blocks until stop is called.
         input.do_stop
@@ -126,8 +140,6 @@ describe LogStash::Inputs::AzureEventHubs do
           expect(exploded_config[0]['event_hub_connections'][0].value).to eql('Endpoint=sb://logstash/;SharedAccessKeyName=activity-log-readonly;SharedAccessKey=something;EntityPath=event_hub1')
         end
       end
-
-
     end
 
     describe "Advanced Config" do
@@ -207,10 +219,10 @@ describe LogStash::Inputs::AzureEventHubs do
         register_counter = java.util.concurrent.atomic.AtomicInteger.new
         unregister_counter = java.util.concurrent.atomic.AtomicInteger.new
         assertion_count = java.util.concurrent.atomic.AtomicInteger.new
+        allow(input).to receive(:get_host_context) {mock_host_context}
         allow_any_instance_of(InMemoryLeaseManager).to receive(:java_send)
         allow_any_instance_of(InMemoryCheckpointManager).to receive(:java_send)
 
-        allow(mock_host).to receive(:getHostContext) {mock_host_context}
         allow(mock_host_context).to receive(:getEventHubPath) {"foo"}
 
         expect(mock_host).to receive(:registerEventProcessorFactory).at_most(3).times {
@@ -221,32 +233,45 @@ describe LogStash::Inputs::AzureEventHubs do
           unregister_counter.incrementAndGet
           completable_future
         }
-        expect(EventProcessorHost).to receive(:new).at_most(3).times {|host_name, event_hub_name, consumer_group, event_hub_connection, storage_connection, container, executor|
-          case event_hub_name
+
+        build_step_mock = double("final build step")
+        expect(build_step_mock).to receive(:build).at_most(3).times.and_return(mock_host)
+
+        executor_step = double("executor step")
+        expect(executor_step).to receive(:setExecutor).at_most(3).times.and_return(build_step_mock)
+
+        mock_connection_string = double("connection string")
+        expect(mock_connection_string).to receive(:useEventHubConnectionString).at_most(3).times {|event_hub_connection|
+          case event_hub_connection
+          when /.*event_hub_name0$/
+            expect(event_hub_connection).to eql(config['event_hubs'][0]['event_hub_name0']['event_hub_connections'][0].value)
+          when /.*event_hub_name1$/
+            expect(event_hub_connection).to eql(config['event_hubs'][1]['event_hub_name1']['event_hub_connections'][0].value)
+          end
+          executor_step
+        }
+
+        managers_mock = double("checkpoint and lease managers")
+        expect(managers_mock).to receive(:useUserCheckpointAndLeaseManagers).at_most(3).times {|checkpoint_mngr, lease_mngr|
+          assertion_count.incrementAndGet
+          mock_connection_string
+        }
+        expect(managers_mock).to receive(:useAzureStorageCheckpointLeaseManager).at_most(3).times {|storage_connection_str, storage_container, storage_blob_prefix|
+          case storage_container
           when 'event_hub_name0'
-            if consumer_group.eql?('cg')
-              assertion_count.incrementAndGet
-              expect(host_name).to start_with('logstash')
-              expect(event_hub_connection).to eql(config['event_hubs'][0]['event_hub_name0']['event_hub_connections'][0].value)
-              expect(storage_connection).to eql(config['event_hubs'][0]['event_hub_name0']['storage_connection'].value)
-              expect(container).to eql('event_hub_name0') # default
-            elsif consumer_group.eql?('ls')
-              assertion_count.incrementAndGet
-              expect(event_hub_connection).to eql(config['event_hubs'][2]['event_hub_name0']['event_hub_connections'][0].value)
-              # in this mode, storage connection and container are replaced with in memory offset management
-              expect(storage_connection).to be_kind_of(InMemoryCheckpointManager)
-              expect(container).to be_kind_of(InMemoryLeaseManager)
-            end
-          when 'event_hub_name1'
             assertion_count.incrementAndGet
-            expect(host_name).to start_with('logstash')
-            expect(event_hub_connection).to eql(config['event_hubs'][1]['event_hub_name1']['event_hub_connections'][0].value)
-            expect(storage_connection).to eql(config['event_hubs'][1]['event_hub_name1']['storage_connection'].value)
-            expect(container).to eql(config['event_hubs'][1]['event_hub_name1']['storage_container'])
+          when 'alt_container'
+            assertion_count.incrementAndGet
           end
+          mock_connection_string
+        }
+
+        expect(EventProcessorHost::EventProcessorHostBuilder).to receive(:newBuilder).at_most(3).times {|host_name, consumer_group|
+          expect(host_name).to start_with('logstash')
           host_counter.incrementAndGet
-          mock_host
+          managers_mock
         }
+
         # signal the stop first since the run method blocks until stop is called.
         input.do_stop
         input.run(mock_queue)
@@ -256,10 +281,17 @@ describe LogStash::Inputs::AzureEventHubs do
         expect(assertion_count.get).to be == 3
       end
 
+      it "can create an in memory EPH" do
+        #event_hub, event_hub_name, scheduled_executor_service
+        exploded_config = input.event_hubs_exploded
+        # During build step Azure libraries does a syntax check of EventHub connection string, so needs to be a pseudo real
+        exploded_config[0]['event_hub_connections'] = [::LogStash::Util::Password.new("Endpoint=sb://logstash.windows.net/;SharedAccessKeyName=activity-log-read-only;SharedAccessKey=blabla;EntityPath=ops-logs")]
+        input.create_in_memory_event_processor_host(exploded_config[0], exploded_config[0]['event_hubs'].first, nil)
+      end
     end
 
     describe "Bad Basic Config" do
-      describe "Offset overwritting" do
+      describe "Offset overwriting" do
         let(:config) do
           {
               'event_hub_connections' => ['Endpoint=sb://...;EntityPath=event_hub_name0', 'Endpoint=sb://...;EntityPath=event_hub_name0'],

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-azure_event_hubs
 version: !ruby/object:Gem::Version
-  version: 1.4.8
+  version: 1.5.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-10-02 00:00:00.000000000 Z
+date: 2024-10-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -103,12 +103,41 @@ files:
 - lib/logstash/inputs/processor_factory.rb
 - logstash-input-azure_event_hubs.gemspec
 - spec/inputs/azure_event_hub_spec.rb
-- vendor/jar-dependencies/com/google/code/gson/gson/2.10.1/gson-2.10.1.jar
-- vendor/jar-dependencies/com/microsoft/azure/azure-eventhubs-eph/2.5.2/azure-eventhubs-eph-2.5.2.jar
-- vendor/jar-dependencies/com/microsoft/azure/azure-eventhubs/2.3.2/azure-eventhubs-2.3.2.jar
+- vendor/jar-dependencies/com/fasterxml/jackson/core/jackson-annotations/2.12.7/jackson-annotations-2.12.7.jar
+- vendor/jar-dependencies/com/fasterxml/jackson/core/jackson-core/2.12.7/jackson-core-2.12.7.jar
+- vendor/jar-dependencies/com/fasterxml/jackson/core/jackson-databind/2.12.7.1/jackson-databind-2.12.7.1.jar
+- vendor/jar-dependencies/com/fasterxml/jackson/datatype/jackson-datatype-joda/2.12.7/jackson-datatype-joda-2.12.7.jar
+- vendor/jar-dependencies/com/github/stephenc/jcip/jcip-annotations/1.0-1/jcip-annotations-1.0-1.jar
+- vendor/jar-dependencies/com/google/code/gson/gson/2.8.9/gson-2.8.9.jar
+- vendor/jar-dependencies/com/google/guava/guava/32.0.1-jre/guava-32.0.1-jre.jar
+- vendor/jar-dependencies/com/microsoft/azure/adal4j/1.6.4/adal4j-1.6.4.jar
+- vendor/jar-dependencies/com/microsoft/azure/azure-annotations/1.10.0/azure-annotations-1.10.0.jar
+- vendor/jar-dependencies/com/microsoft/azure/azure-client-authentication/1.7.3/azure-client-authentication-1.7.3.jar
+- vendor/jar-dependencies/com/microsoft/azure/azure-client-runtime/1.7.3/azure-client-runtime-1.7.3.jar
+- vendor/jar-dependencies/com/microsoft/azure/azure-eventhubs-eph/3.3.0/azure-eventhubs-eph-3.3.0.jar
+- vendor/jar-dependencies/com/microsoft/azure/azure-eventhubs/3.3.0/azure-eventhubs-3.3.0.jar
+- vendor/jar-dependencies/com/microsoft/azure/azure-keyvault-core/1.2.4/azure-keyvault-core-1.2.4.jar
 - vendor/jar-dependencies/com/microsoft/azure/azure-storage/8.6.6/azure-storage-8.6.6.jar
 - vendor/jar-dependencies/com/microsoft/azure/qpid-proton-j-extensions/1.2.4/qpid-proton-j-extensions-1.2.4.jar
-- vendor/jar-dependencies/org/apache/qpid/proton-j/0.33.9/proton-j-0.33.9.jar
+- vendor/jar-dependencies/com/microsoft/rest/client-runtime/1.7.3/client-runtime-1.7.3.jar
+- vendor/jar-dependencies/com/nimbusds/lang-tag/1.7/lang-tag-1.7.jar
+- vendor/jar-dependencies/com/nimbusds/nimbus-jose-jwt/9.37.2/nimbus-jose-jwt-9.37.2.jar
+- vendor/jar-dependencies/com/nimbusds/oauth2-oidc-sdk/6.5/oauth2-oidc-sdk-6.5.jar
+- vendor/jar-dependencies/com/squareup/okhttp3/logging-interceptor/3.12.2/logging-interceptor-3.12.2.jar
+- vendor/jar-dependencies/com/squareup/okhttp3/okhttp-urlconnection/3.12.2/okhttp-urlconnection-3.12.2.jar
+- vendor/jar-dependencies/com/squareup/okhttp3/okhttp/3.14.7/okhttp-3.14.7.jar
+- vendor/jar-dependencies/com/squareup/okio/okio/1.17.6/okio-1.17.6.jar
+- vendor/jar-dependencies/com/squareup/retrofit2/adapter-rxjava/2.7.2/adapter-rxjava-2.7.2.jar
+- vendor/jar-dependencies/com/squareup/retrofit2/converter-jackson/2.7.2/converter-jackson-2.7.2.jar
+- vendor/jar-dependencies/com/squareup/retrofit2/retrofit/2.7.2/retrofit-2.7.2.jar
+- vendor/jar-dependencies/com/sun/mail/javax.mail/1.6.1/javax.mail-1.6.1.jar
+- vendor/jar-dependencies/commons-codec/commons-codec/1.11/commons-codec-1.11.jar
+- vendor/jar-dependencies/io/reactivex/rxjava/1.3.8/rxjava-1.3.8.jar
+- vendor/jar-dependencies/javax/activation/activation/1.1/activation-1.1.jar
+- vendor/jar-dependencies/net/minidev/json-smart/2.4.9/json-smart-2.4.9.jar
+- vendor/jar-dependencies/org/apache/qpid/proton-j/0.33.8/proton-j-0.33.8.jar
+- vendor/jar-dependencies/org/checkerframework/checker-compat-qual/2.0.0/checker-compat-qual-2.0.0.jar
+- vendor/jar-dependencies/org/codehaus/mojo/animal-sniffer-annotations/1.14/animal-sniffer-annotations-1.14.jar
 homepage: http://www.elastic.co/guide/en/logstash/current/index.html
 licenses:
 - Apache-2.0