logstash-input-azure_event_hubs 1.0.0

data/lib/logstash/inputs/error_notification_handler.rb

@@ -0,0 +1,22 @@
+# encoding: utf-8
+require "logstash/util/loggable"
+java_import java.util.function.Consumer
+
+module LogStash
+  module Inputs
+    module Azure
+      class ErrorNotificationHandler
+        include Consumer
+        include LogStash::Util::Loggable
+
+        def initialize
+          @logger = self.logger
+        end
+
+        def accept(exception_received_event_args)
+          @logger.error("Error with Event Processor Host. ", :exception_received_event_args => exception_received_event_args.to_s)
+        end
+      end
+    end
+  end
+end

data/lib/logstash/inputs/look_back_position_provider.rb

@@ -0,0 +1,26 @@
+# encoding: utf-8
+require "logstash/util/loggable"
+java_import java.util.function.Function
+java_import com.microsoft.azure.eventhubs.EventPosition
+java_import java.time.Instant
+
+module LogStash
+  module Inputs
+    module Azure
+      class LookBackPositionProvider
+        include Function
+        include LogStash::Util::Loggable
+
+        def initialize(look_back_seconds)
+          @logger = self.logger
+          @look_back = Instant.ofEpochSecond(Instant.now.getEpochSecond - look_back_seconds.to_i)
+          @logger.debug("Look back date/time: #{@look_back}")
+        end
+
+        def apply(t)
+          EventPosition.fromEnqueuedTime(@look_back);
+        end
+      end
+    end
+  end
+end

data/lib/logstash/inputs/named_thread_factory.rb

@@ -0,0 +1,20 @@
+module LogStash
+  module Inputs
+    module Azure
+      class NamedThreadFactory
+        include java.util.concurrent.ThreadFactory
+        java_import java.util.concurrent.atomic.AtomicInteger
+
+        def initialize(name, id)
+          @name = name
+          @id = id
+          @counter = AtomicInteger.new(-1)
+        end
+
+        def newThread(runnable)
+          java.lang.Thread.new(runnable, @name + "-" + @counter.increment_and_get.to_s + "-" + @id.to_s)
+        end
+      end
+    end
+  end
+end

data/lib/logstash/inputs/processor.rb

@@ -0,0 +1,82 @@
+# encoding: utf-8
+require "logstash/util/loggable"
+module LogStash
+  module Inputs
+    module Azure
+      class Processor
+        include LogStash::Util::Loggable
+        include com.microsoft.azure.eventprocessorhost.IEventProcessor
+
+        def initialize(queue, codec, checkpoint_interval, decorator, meta_data)
+          @queue = queue
+          @codec = codec
+          @checkpoint_interval = checkpoint_interval
+          @decorator = decorator
+          @meta_data = meta_data
+          @logger = self.logger
+
+        end
+
+        def onOpen(context)
+          @logger.info("Event Hub: #{context.getEventHubPath.to_s}, Partition: #{context.getPartitionId.to_s} is opening.")
+        end
+
+        def onClose(context, reason)
+          @logger.info("Event Hub: #{context.getEventHubPath.to_s}, Partition: #{context.getPartitionId.to_s} is closing. (reason=#{reason.to_s})")
+        end
+
+        def onEvents(context, batch)
+          @logger.debug("Event Hub: #{context.getEventHubPath.to_s}, Partition: #{context.getPartitionId.to_s} is processing a batch.") if @logger.debug?
+          last_payload = nil
+          batch_size = 0
+          batch.each do |payload|
+            last_checkpoint = Time.now.to_i
+            bytes = payload.getBytes
+            batch_size += bytes.size
+            @logger.trace("Event Hub: #{context.getEventHubPath.to_s}, Partition: #{context.getPartitionId.to_s}, Offset: #{payload.getSystemProperties.getOffset.to_s},"+
+                              " Sequence: #{payload.getSystemProperties.getSequenceNumber.to_s}, Size: #{bytes.size}") if @logger.trace?
+
+            @codec.decode(bytes.to_a.pack('C*')) do |event|
+
+              @decorator.call(event)
+              if @meta_data
+                event.set("[@metadata][azure_event_hubs][name]", context.getEventHubPath)
+                event.set("[@metadata][azure_event_hubs][consumer_group]", context.getConsumerGroupName)
+                event.set("[@metadata][azure_event_hubs][processor_host]", context.getOwner)
+                event.set("[@metadata][azure_event_hubs][partition]", context.getPartitionId)
+                event.set("[@metadata][azure_event_hubs][offset]", payload.getSystemProperties.getOffset)
+                event.set("[@metadata][azure_event_hubs][sequence]", payload.getSystemProperties.getSequenceNumber)
+                event.set("[@metadata][azure_event_hubs][timestamp]",payload.getSystemProperties.getEnqueuedTime.getEpochSecond)
+                event.set("[@metadata][azure_event_hubs][event_size]", bytes.size)
+              end
+              @queue << event
+              if @checkpoint_interval > 0
+                now = Time.now.to_i
+                since_last_check_point = now - last_checkpoint
+                if since_last_check_point >= @checkpoint_interval
+                  context.checkpoint(payload).get
+                  last_checkpoint = now
+                end
+              end
+            end
+            last_payload = payload
+          end
+
+          @codec.flush
+          #always create checkpoint at end of onEvents in case of sparse events
+          context.checkpoint(last_payload).get if last_payload
+          @logger.debug("Event Hub: #{context.getEventHubPath.to_s}, Partition: #{context.getPartitionId.to_s} finished processing a batch of #{batch_size} bytes.") if @logger.debug?
+        end
+
+        def onError(context, error)
+          @logger.error("Event Hub: #{context.getEventHubPath.to_s}, Partition: #{context.getPartitionId.to_s} experienced an error #{error.to_s})")
+        end
+      end
+    end
+  end
+end
+
+
+
+
+

data/lib/logstash/inputs/processor_factory.rb

@@ -0,0 +1,27 @@
+# encoding: utf-8
+require "logstash/inputs/processor"
+module LogStash
+  module Inputs
+    module Azure
+      class ProcessorFactory
+        include com.microsoft.azure.eventprocessorhost.IEventProcessorFactory
+
+        def initialize(queue, codec, checkpoint_interval, decorator, meta_data)
+          @queue = queue
+          @codec = codec
+          @checkpoint_interval = checkpoint_interval
+          @decorator = decorator
+          @meta_data = meta_data
+        end
+
+        def createEventProcessor(context)
+          Processor.new(@queue, @codec.clone, @checkpoint_interval, @decorator, @meta_data)
+        end
+
+      end
+    end
+  end
+end
+
+
+

data/logstash-input-azure_event_hubs.gemspec

@@ -0,0 +1,28 @@
+GEM_VERSION = File.read(File.expand_path(File.join(File.dirname(__FILE__), "VERSION"))).strip unless defined?(GEM_VERSION)
+
+Gem::Specification.new do |s|
+  s.name          = 'logstash-input-azure_event_hubs'
+  s.version       = GEM_VERSION
+  s.licenses      = ['Apache-2.0']
+  s.summary       = 'Consumes events from Azure Event Hubs for use with Logstash'
+  s.description   = 'This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program'
+  s.homepage      = 'http://www.elastic.co/guide/en/logstash/current/index.html'
+  s.authors       = ['Elastic']
+  s.email         = 'info@elastic.co'
+  s.require_paths = ['lib', 'vendor/jar-dependencies']
+
+  # Files
+  s.files = Dir['lib/**/*','spec/**/*','vendor/**/*', 'vendor/jar-dependencies/**/*.jar','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT', 'VERSION']
+   # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "input" }
+
+  # Gem dependencies
+  s.add_runtime_dependency "logstash-core-plugin-api", "~> 2.0"
+  s.add_runtime_dependency 'logstash-codec-plain'
+  s.add_runtime_dependency 'logstash-codec-json'
+  s.add_runtime_dependency 'stud', '>= 0.0.22'
+  s.add_development_dependency 'logstash-devutils', '>= 0.0.16'
+end

data/spec/inputs/azure_event_hub_spec.rb

@@ -0,0 +1,322 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/inputs/azure_event_hubs"
+
+
+java_import com.microsoft.azure.eventprocessorhost.EventProcessorHost
+java_import com.microsoft.azure.eventprocessorhost.EventProcessorOptions
+java_import com.microsoft.azure.eventprocessorhost.InMemoryCheckpointManager
+java_import com.microsoft.azure.eventprocessorhost.InMemoryLeaseManager
+java_import java.util.concurrent.ScheduledThreadPoolExecutor
+java_import java.util.concurrent.CompletableFuture
+java_import java.util.concurrent.TimeUnit
+java_import java.util.concurrent.atomic.AtomicInteger
+
+
+describe LogStash::Inputs::AzureEventHubs do
+
+
+  subject(:input) {LogStash::Plugin.lookup("input", "azure_event_hubs").new(config)}
+
+  describe "Event Hubs Configuration -> " do
+    shared_examples "an exploded Event Hub config" do |x|
+      it "it explodes #{x} event hub(s) correctly" do
+        exploded_config = input.event_hubs_exploded
+        x.times do |i|
+          expect(exploded_config[i]['event_hubs'].size).to be == 1 #always 1 in the exploded form
+          expect(exploded_config[i]['event_hubs'][0]).to eql('event_hub_name' + i.to_s)
+          expect(exploded_config[i]['event_hub_connections'][0].value).to start_with('Endpoint=sb://...')
+          expect(exploded_config[i]['storage_connection'].value).to eql('DefaultEndpointsProtocol=https;AccountName=...')
+          expect(exploded_config[i]['threads']).to be == 9
+          expect(exploded_config[i]['codec']).to be_a_kind_of(LogStash::Codecs::Plain)
+          expect(exploded_config[i]['consumer_group']).to eql('cg')
+          expect(exploded_config[i]['max_batch_size']).to be == 20
+          expect(exploded_config[i]['prefetch_count']).to be == 30
+          expect(exploded_config[i]['receive_timeout']).to be == 40
+          expect(exploded_config[i]['initial_position']).to eql('look_back')
+          expect(exploded_config[i]['initial_position_look_back']).to be == 50
+          expect(exploded_config[i]['checkpoint_interval']).to be == 60
+          expect(exploded_config[i]['decorate_events']).to be_truthy
+        end
+      end
+    end
+
+    describe "Basic Config" do
+      before do
+        input.register
+      end
+      let(:config) do
+        {
+            'event_hub_connections' => ['Endpoint=sb://...;EntityPath=event_hub_name0', 'Endpoint=sb://...;EntityPath=event_hub_name1'],
+            'storage_connection' => 'DefaultEndpointsProtocol=https;AccountName=...',
+            'threads' => 9,
+            'codec' => 'plain',
+            'consumer_group' => 'cg',
+            'max_batch_size' => 20,
+            'prefetch_count' => 30,
+            'receive_timeout' => 40,
+            'initial_position' => 'look_back',
+            'initial_position_look_back' => 50,
+            'checkpoint_interval' => 60,
+            'decorate_events' => true
+        }
+      end
+      it_behaves_like "an exploded Event Hub config", 2
+
+      it "it runs the Event Processor Host" do
+        mock_queue = double("queue")
+        mock_host = double("event_processor_host")
+        mock_host_context = double("host_context")
+        completable_future = CompletableFuture.new
+        #simulate work being done before completing the future
+        Thread.new do
+          sleep 2
+          completable_future.complete("")
+        end
+
+        # rspec has issues with counters and concurrent code, so use threadsafe counters instead
+        host_counter = AtomicInteger.new
+        register_counter = AtomicInteger.new
+        unregister_counter = AtomicInteger.new
+        assertion_count = AtomicInteger.new
+
+        allow(mock_host).to receive(:getHostContext) {mock_host_context}
+        allow(mock_host_context).to receive(:getEventHubPath) {"foo"}
+
+        expect(mock_host).to receive(:registerEventProcessorFactory).at_most(2).times {
+          register_counter.incrementAndGet
+          completable_future
+        }
+        expect(mock_host).to receive(:unregisterEventProcessor).at_most(2).times {
+          unregister_counter.incrementAndGet
+          completable_future
+        }
+        expect(EventProcessorHost).to receive(:new).at_most(2).times {|host_name, event_hub_name, consumer_group, event_hub_connection, storage_connection, container, executor|
+          case event_hub_name
+          when 'event_hub_name0'
+
+            assertion_count.incrementAndGet
+            expect(event_hub_connection).to eql(config['event_hub_connections'][0])
+            expect(container).to eql('event_hub_name0') # default
+
+          when 'event_hub_name1'
+            assertion_count.incrementAndGet
+            expect(host_name).to start_with('logstash')
+            expect(event_hub_connection).to eql(config['event_hub_connections'][1])
+            expect(container).to eql('event_hub_name1') # default
+          end
+          expect(host_name).to start_with('logstash')
+          expect(storage_connection).to eql(config['storage_connection'])
+
+          host_counter.incrementAndGet
+          mock_host
+        }
+        # signal the stop first since the run method blocks until stop is called.
+        input.do_stop
+        input.run(mock_queue)
+        expect(host_counter.get).to be == 2
+        expect(register_counter.get).to be == 2
+        expect(unregister_counter.get).to be == 2
+        expect(assertion_count.get).to be == 2
+      end
+
+      describe "single connection, no array syntax" do
+        let(:config) do
+          {
+              'event_hub_connections' => 'Endpoint=sb://logstash/;SharedAccessKeyName=activity-log-readonly;SharedAccessKey=something;EntityPath=event_hub1'
+          }
+        end
+        it "it can handle a single connection without the array notation" do
+          expect {input}.to_not raise_error
+          exploded_config = input.event_hubs_exploded
+          expect(exploded_config.size).to be == 1
+          expect(exploded_config[0]['event_hub_connections'][0].value).to eql('Endpoint=sb://logstash/;SharedAccessKeyName=activity-log-readonly;SharedAccessKey=something;EntityPath=event_hub1')
+        end
+      end
+
+
+    end
+
+    describe "Advanced Config" do
+      before do
+        input.register
+      end
+      let(:config) do
+        {
+            'config_mode' => 'advanced',
+            'event_hubs' => [
+                {'event_hub_name0' => {
+                    'event_hub_connection' => 'Endpoint=sb://...',
+                    'storage_connection' => 'DefaultEndpointsProtocol=https;AccountName=...',
+                    'codec' => 'plain',
+                    'consumer_group' => 'cg',
+                    'max_batch_size' => 20,
+                    'prefetch_count' => 30,
+                    'receive_timeout' => 40,
+                    'initial_position' => 'look_back',
+                    'initial_position_look_back' => 50,
+                    'checkpoint_interval' => 60,
+                    'decorate_events' => true}},
+                {'event_hub_name1' => {
+                    'event_hub_connection' => '1Endpoint=sb://...',
+                    'storage_connection' => '1DefaultEndpointsProtocol=https;AccountName=...',
+                    'codec' => 'json',
+                    'consumer_group' => 'cg1',
+                    'receive_timeout' => 41,
+                    'initial_position' => 'end',
+                    'checkpoint_interval' => 61,
+                    'decorate_events' => false,
+                    'storage_container' => 'alt_container'}},
+                # same named event hub with different configuration is allowed
+                {'event_hub_name0' => {
+                    'event_hub_connection' => 'Endpoint=sb://...',
+                    'consumer_group' => 'ls'}}
+            ],
+            'codec' => 'plain',
+            'consumer_group' => 'default_consumer_group',
+            'max_batch_size' => 21,
+            'threads' => 9
+        }
+      end
+      it_behaves_like "an exploded Event Hub config", 1
+      it "it explodes the 2cnd advanced config event hub correctly" do
+        exploded_config = input.event_hubs_exploded
+        expect(exploded_config[1]['event_hubs'].size).to be == 1 #always 1 in the exploded form
+        expect(exploded_config[1]['event_hubs'][0]).to eql('event_hub_name1')
+        expect(exploded_config[1]['event_hub_connections'][0].value).to eql('1Endpoint=sb://...')
+        expect(exploded_config[1]['storage_connection'].value).to eql('1DefaultEndpointsProtocol=https;AccountName=...')
+        expect(exploded_config[1]['threads']).to be == 9
+        expect(exploded_config[1]['codec']).to be_a_kind_of(LogStash::Codecs::JSON) # different between configs
+        expect(exploded_config[1]['consumer_group']).to eql('cg1') # override global
+        expect(exploded_config[1]['max_batch_size']).to be == 21 # filled from global
+        expect(exploded_config[1]['prefetch_count']).to be == 300 # default
+        expect(exploded_config[1]['receive_timeout']).to be == 41
+        expect(exploded_config[1]['initial_position']).to eql('end')
+        expect(exploded_config[1]['initial_position_look_back']).to be == 86400 # default
+        expect(exploded_config[1]['checkpoint_interval']).to be == 61
+        expect(exploded_config[1]['decorate_events']).to be_falsy
+        expect(exploded_config[1]['storage_container']).to eq('alt_container')
+      end
+
+      it "it runs the Event Processor Host" do
+        mock_queue = double("queue")
+        mock_host = double("event_processor_host")
+        mock_host_context = double("host_context")
+        completable_future = CompletableFuture.new
+        #simulate work being done before completing the future
+        Thread.new do
+          sleep 2
+          completable_future.complete("")
+        end
+
+        # rspec has issues with counters and concurrent code, so use threadsafe counters instead
+        host_counter = AtomicInteger.new
+        register_counter = AtomicInteger.new
+        unregister_counter = AtomicInteger.new
+        assertion_count = AtomicInteger.new
+        allow_any_instance_of(InMemoryLeaseManager).to receive(:java_send)
+        allow_any_instance_of(InMemoryCheckpointManager).to receive(:java_send)
+
+        allow(mock_host).to receive(:getHostContext) {mock_host_context}
+        allow(mock_host_context).to receive(:getEventHubPath) {"foo"}
+
+        expect(mock_host).to receive(:registerEventProcessorFactory).at_most(3).times {
+          register_counter.incrementAndGet
+          completable_future
+        }
+        expect(mock_host).to receive(:unregisterEventProcessor).at_most(3).times {
+          unregister_counter.incrementAndGet
+          completable_future
+        }
+        expect(EventProcessorHost).to receive(:new).at_most(3).times {|host_name, event_hub_name, consumer_group, event_hub_connection, storage_connection, container, executor|
+          case event_hub_name
+          when 'event_hub_name0'
+            if consumer_group.eql?('cg')
+              assertion_count.incrementAndGet
+              expect(host_name).to start_with('logstash')
+              expect(event_hub_connection).to eql(config['event_hubs'][0]['event_hub_name0']['event_hub_connections'][0].value)
+              expect(storage_connection).to eql(config['event_hubs'][0]['event_hub_name0']['storage_connection'].value)
+              expect(container).to eql('event_hub_name0') # default
+            elsif consumer_group.eql?('ls')
+              assertion_count.incrementAndGet
+              expect(event_hub_connection).to eql(config['event_hubs'][2]['event_hub_name0']['event_hub_connections'][0].value)
+              # in this mode, storage connection and container are replaced with in memory offset management
+              expect(storage_connection).to be_kind_of(InMemoryCheckpointManager)
+              expect(container).to be_kind_of(InMemoryLeaseManager)
+            end
+          when 'event_hub_name1'
+            assertion_count.incrementAndGet
+            expect(host_name).to start_with('logstash')
+            expect(event_hub_connection).to eql(config['event_hubs'][1]['event_hub_name1']['event_hub_connections'][0].value)
+            expect(storage_connection).to eql(config['event_hubs'][1]['event_hub_name1']['storage_connection'].value)
+            expect(container).to eql(config['event_hubs'][1]['event_hub_name1']['storage_container'])
+          end
+          host_counter.incrementAndGet
+          mock_host
+        }
+        # signal the stop first since the run method blocks until stop is called.
+        input.do_stop
+        input.run(mock_queue)
+        expect(host_counter.get).to be == 3
+        expect(register_counter.get).to be == 3
+        expect(unregister_counter.get).to be == 3
+        expect(assertion_count.get).to be == 3
+      end
+
+    end
+
+    describe "Bad Basic Config" do
+      describe "Offset overwritting" do
+        let(:config) do
+          {
+              'event_hub_connections' => ['Endpoint=sb://...;EntityPath=event_hub_name0', 'Endpoint=sb://...;EntityPath=event_hub_name0'],
+              'storage_connection' => 'DefaultEndpointsProtocol=https;AccountName=...'
+          }
+        end
+        it "it errors when using same consumer group and storage container" do
+          expect {input}.to raise_error(/overwriting offsets/)
+        end
+      end
+
+      describe "Invalid Event Hub name" do
+        let(:config) do
+          {
+              'event_hub_connections' => ['Endpoint=sb://logstash/;SharedAccessKeyName=activity-log-readonly;SharedAccessKey=thisshouldnotbepartofthelogmessage'],
+              'storage_connection' => 'DefaultEndpointsProtocol=https;AccountName=...'
+          }
+        end
+        it "it errors when using same consumer group and storage container" do
+          expect {input}.to raise_error(/that the connection string contains the EntityPath/)
+          expect {input}.to raise_error(/redacted/)
+          expect {input}.to raise_error(/^((?!thisshouldnotbepartofthelogmessage).)*$/)
+        end
+      end
+    end
+
+    describe "Bad Advanced Config" do
+      describe "Offset overwritting" do
+        let(:config) do
+          {
+              'config_mode' => 'advanced',
+              'event_hubs' => [
+                  {'event_hub_name0' => {
+                      'event_hub_connection' => 'Endpoint=sb://...',
+                  }},
+                  {'event_hub_name1' => {
+                      'event_hub_connection' => '1Endpoint=sb://...',
+                  }}
+              ],
+
+              'storage_connection' => 'DefaultEndpointsProtocol=https;AccountName=...',
+              'consumer_group' => 'default_consumer_group',
+              'storage_container' => 'logstash'
+          }
+        end
+        it "it errors when using same consumer group and storage container" do
+          expect {input}.to raise_error(/overwriting offsets/)
+        end
+      end
+    end
+  end
+end
+