logstash-input-azureeventhub 0.9.7-java → 0.9.8-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 73a61719253d1cf4d9a302b34e5085f4b4a296ef
-  data.tar.gz: 6333763f8de1ff33ad6dfde0e80c421684a7e3ed
+SHA256:
+  metadata.gz: 924c45dec9708f85316b26facc34c9308f73f0773bbc056703fb4816ae1d72a1
+  data.tar.gz: 6a0eed913702d19b82d1f1fc8deb7036202f1f941bc30d3d0bff66ad5e722426
 SHA512:
-  metadata.gz: 612c3610534f016259ad133f42806783b80b2f446e4f2ac61ffaf0ed4f2ed7eb9496c93ecd5fe8cbebebd5f17afc5f8601823d8ca8d97bd640a1240222675cc1
-  data.tar.gz: 556505e4589f34e2dc9a3f8ca2d53ddd590d7afee93bfe1e1e75a31895ee242f21ae9130a11d35e3c74f170823dcee6ae5acb807cddb6216be0ff640edba0325
+  metadata.gz: b3462e2f63501a9183916ba6f366205c73bf2dd671686747b0b77f0a37d6b086706123c40c6bfb6638e35fa435312fd847cfba25833e3a445b3f6e72ae81fa39
+  data.tar.gz: 35bdb5244ba4abaacba931d8960d056198a6f37770d8d84c449fdaa9b77257b7f3e2c433b1d42ac699db7abe53edf3e22f85b5e5a35c21ee088d6230ec5251bd

data/README.md

@@ -43,7 +43,7 @@ Port of the target Event Hub. Default value is 5671.
 
 __*receive_credits*__
 
-The credit number to limit the number of messages to receive in a processing cycle. Default value is 1000.
+The credit number to limit the number of messages to receive in a processing cycle. Value must be between 10 and 999. Default is 999.
 
 __*consumer_group*__
 
@@ -59,6 +59,10 @@ __*thread_wait_sec*__
 
 Specifies the time (in seconds) to wait before another try if no message was received.
 
+__*partition_receiver_epochs*__
+
+A map from partition (string) to epoch (integer). By default each partition doesn't have an epoch defined. For more information read https://blogs.msdn.microsoft.com/gyan/2014/09/02/event-hubs-receiver-epoch/ .
+
 ### Examples
 ```
 input
@@ -70,6 +74,7 @@ input
         namespace => "mysbns"
         eventhub => "myeventhub"
         partitions => 4
+        partition_receiver_epochs => { '2' => 42 '0' => 15 }
     }
 }
 ```

data/lib/logstash-input-azureeventhub_jars.rb

@@ -0,0 +1,14 @@
+# this is a generated file, to avoid over-writing it just delete this comment
+begin
+  require 'jar_dependencies'
+rescue LoadError
+  require 'org/slf4j/slf4j-api/1.7.25/slf4j-api-1.7.25.jar'
+  require 'com/microsoft/azure/azure-eventhubs/0.15.0/azure-eventhubs-0.15.0.jar'
+  require 'org/apache/qpid/proton-j/0.22.0/proton-j-0.22.0.jar'
+end
+
+if defined? Jars
+  require_jar( 'org.slf4j', 'slf4j-api', '1.7.25' )
+  require_jar( 'com.microsoft.azure', 'azure-eventhubs', '0.15.0' )
+  require_jar( 'org.apache.qpid', 'proton-j', '0.22.0' )
+end

data/lib/logstash/inputs/azureeventhub.rb

@@ -16,19 +16,21 @@ class LogStash::Inputs::Azureeventhub < LogStash::Inputs::Base
 
   default :codec, "json"
   
-  config :key, :validate => :string
-  config :username, :validate => :string
-  config :namespace, :validate => :string
+  config :key, :validate => :string, :required => true
+  config :username, :validate => :string, :required => true
+  config :namespace, :validate => :string, :required => true
   config :domain, :validate => :string, :default => "servicebus.windows.net"
   config :port, :validate => :number, :default => 5671
-  config :receive_credits, :validate => :number, :default => 1000
+  config :receive_credits, :validate => :number, :default => 999
   
-  config :eventhub, :validate => :string
-  config :partitions, :validate => :number
+  config :eventhub, :validate => :string, :required => true
+  config :partitions, :validate => :number, :required => true
   config :consumer_group, :validate => :string, :default => "$default"
   
   config :time_since_epoch_millis, :validate => :number, :default => Time.now.utc.to_i * 1000
   config :thread_wait_sec, :validate => :number, :default => 5
+
+  config :partition_receiver_epochs, :validate => :hash, :default => {}
   
   
   def initialize(*args)
@@ -37,72 +39,94 @@ class LogStash::Inputs::Azureeventhub < LogStash::Inputs::Base
 
   public
   def register
+    user_agent = "logstash-input-azureeventhub"
+    user_agent << "/" << Gem.latest_spec_for("logstash-input-azureeventhub").version.to_s
+    com::microsoft::azure::eventhubs::EventHubClient.userAgent = user_agent
   end # def register
 
-  def get_pay_load(message)
-    return nil if not message
-    message.getPayload().each do |section|
-      if section.java_kind_of? org::apache::qpid::amqp_1_0::type::messaging::Data or section.java_kind_of? org::apache::qpid::amqp_1_0::type::messaging::AmqpValue
-        return section.getValue().to_s.gsub("\\x5c", "\\")
-      end
-    end
-    return nil
-  end
-  
-  def process(output_queue, receiver, partition)
+  def process(output_queue, receiver, partition, last_event_offset)
     while !stop?
       begin
-        msg = receiver.receive(10)
-        if msg
-          codec.decode(get_pay_load(msg)) do |event|
-            decorate(event)
-            output_queue << event
-          end
-          receiver.acknowledge(msg)
+        events = receiver.receiveSync(10)
+        if events
+          events.each{ |msg|
+            body = msg.getBytes().to_s.gsub("\\x5c", "\\")
+            props = msg.getSystemProperties()
+
+            last_event_offset = props.getOffset()
+            
+            @logger.debug("[#{partition.to_s.rjust(2,"0")}] Event: #{body[0..50] unless body.nil?}... " <<
+              "Offset: #{props.getOffset()} " <<
+              "Time: #{props.getEnqueuedTime()} " <<
+              "Sequence: #{props.getSequenceNumber()}")
+
+            codec.decode(body) do |event|
+              decorate(event)
+              output_queue << event
+            end
+          }
         else
-          @logger.debug("  " + partition.to_s.rjust(2,"0") + " --- " + "No message")
-          sleep(@thread_wait_sec)
+          @logger.debug("[#{partition.to_s.rjust(2,"0")}] No message")
         end
-      rescue LogStash::ShutdownSignal => e
-        raise e
-      rescue org::apache::qpid::amqp_1_0::client::ConnectionErrorException => e
-        raise e
-      rescue => e
-        @logger.error("  " + partition.to_s.rjust(2,"0") + " --- " + "Oh My, An error occurred.", :exception => e)
       end
-    end # process
+    end
+  rescue LogStash::ShutdownSignal => e
+    @logger.debug("[#{partition.to_s.rjust(2,"0")}] ShutdownSignal received")
+    raise e
+  rescue => e
+    @logger.error("[#{partition.to_s.rjust(2,"0")}] Oh My, An error occurred. Error:#{e}: Trace: #{e.backtrace}", :exception => e)
+    raise e
+  ensure
+    return last_event_offset
   end # process
   
-  def process_partition(output_queue, partition)
+  def process_partition(output_queue, partition, epoch) 
+    last_event_offset = nil
     while !stop?
       begin
-        filter = SelectorFilter.new "amqp.annotation.x-opt-enqueuedtimeutc > '" + @time_since_epoch_millis.to_s + "'"
-        filters = { org::apache::qpid::amqp_1_0::type::Symbol.valueOf("apache.org:selector-filter:string") => filter }
+        host = java::net::URI.new("amqps://" << @namespace << "." << @domain)
+        connStr = com::microsoft::azure::eventhubs::ConnectionStringBuilder.new(host, @eventhub, @username, @key).toString()
+        ehClient = com::microsoft::azure::eventhubs::EventHubClient.createFromConnectionStringSync(connStr)
+
+        if !epoch.nil?
+          if !last_event_offset.nil?
+            @logger.debug("[#{partition.to_s.rjust(2,"0")}] Create receiver with epoch=#{epoch} & offset > #{last_event_offset}")
+            receiver = ehClient.createEpochReceiverSync(@consumer_group, partition.to_s, last_event_offset, false, epoch)
+          else
+            @logger.debug("[#{partition.to_s.rjust(2,"0")}] Create receiver with epoch=#{epoch} & timestamp > #{@time_since_epoch_millis}")
+            receiver = ehClient.createEpochReceiverSync(@consumer_group, partition.to_s, java::time::Instant::ofEpochMilli(@time_since_epoch_millis), epoch)
+          end
+        else
+          if !last_event_offset.nil?
+            @logger.debug("[#{partition.to_s.rjust(2,"0")}] Create receiver with offset > #{last_event_offset}")
+            receiver = ehClient.createReceiverSync(@consumer_group, partition.to_s, last_event_offset, false)
+          else
+            @logger.debug("[#{partition.to_s.rjust(2,"0")}] Create receiver with timestamp > #{@time_since_epoch_millis}")
+            receiver = ehClient.createReceiverSync(@consumer_group, partition.to_s, java::time::Instant::ofEpochMilli(@time_since_epoch_millis))
+          end
+        end
+        receiver.setReceiveTimeout(java::time::Duration::ofSeconds(@thread_wait_sec));
+        receiver.setPrefetchCount(@receive_credits)
 
-        host = @namespace + "." + @domain
-        connection = org::apache::qpid::amqp_1_0::client::Connection.new(host, @port, @username, @key, host, true)
-        connection.getEndpoint().getDescribedTypeRegistry().register(filter.java_class, WriterFactory.new)
-        receiveSession = connection.createSession()
-        receiver = receiveSession.createReceiver(@eventhub + "/ConsumerGroups/" + @consumer_group + "/Partitions/" + partition.to_s, org::apache::qpid::amqp_1_0::client::AcknowledgeMode::ALO, "eventhubs-receiver-link-" + partition.to_s, false, filters, nil)
-        receiver.setCredit(org::apache::qpid::amqp_1_0::type::UnsignedInteger.valueOf(@receive_credits), true)
-        process(output_queue,receiver,partition)
-      rescue org::apache::qpid::amqp_1_0::client::ConnectionErrorException => e
-        @logger.debug("  " + partition.to_s.rjust(2,"0") + " --- " + "resetting connection")
-        @time_since_epoch_millis = Time.now.utc.to_i * 1000
+        last_event_offset = process(output_queue, receiver, partition, last_event_offset)
+      rescue com::microsoft::azure::eventhubs::EventHubException => e
+        sleep(@thread_wait_sec)
+        @logger.debug("[#{partition.to_s.rjust(2,"0")}] resetting connection. Error:#{e}: Trace: #{e.backtrace}", :exception => e)
       end
     end
   rescue LogStash::ShutdownSignal => e
     receiver.close()
     raise e
   rescue => e
-    @logger.error("  " + partition.to_s.rjust(2,"0") + " --- Oh My, An error occurred.", :exception => e)
-  end # process
+    @logger.error("[#{partition.to_s.rjust(2,"0")}] Oh My, An error occurred. Error:#{e}: Trace: #{e.backtrace}", :exception => e)
+  end # process_partition
 
   public
   def run(output_queue)
     threads = []
     (0..(@partitions-1)).each do |p_id|
-      threads << Thread.new { process_partition(output_queue, p_id) }
+      epoch = partition_receiver_epochs[p_id.to_s] if @partition_receiver_epochs.key?(p_id.to_s)
+      threads << Thread.new { process_partition(output_queue, p_id, epoch) }
     end
     threads.each { |thr| thr.join }
   end # def run
@@ -110,51 +134,4 @@ class LogStash::Inputs::Azureeventhub < LogStash::Inputs::Base
   public
   def teardown
   end # def teardown
-end # class LogStash::Inputs::Azureeventhub
-
-
-class SelectorFilter
-  include org::apache::qpid::amqp_1_0::type::messaging::Filter
-
-  def initialize(value)
-    @value = value
-  end
-
-  def getValue
-    return @value
-  end
-
-  def toString
-    return @value
-  end
-end
-
-class SelectorFilterWriter < org::apache::qpid::amqp_1_0::codec::AbstractDescribedTypeWriter
-  def initialize(registry)
-    super(registry)
-  end
-  
-  def onSetValue(value)
-    @value = value
-  end
-  
-  def clear
-    @value = nil
-  end
-  
-  def getDescriptor
-    return org::apache::qpid::amqp_1_0::type::UnsignedLong.valueOf(83483426826);
-  end
-  
-  def createDescribedWriter
-    return getRegistry().getValueWriter(@value.getValue());
-  end
-end
-
-class WriterFactory
-  include org::apache::qpid::amqp_1_0::codec::ValueWriter::Factory
-
-  def newInstance(registry)
-    return SelectorFilterWriter.new registry
-  end
-end
+end # class LogStash::Inputs::Azureeventhub

data/logstash-input-azureeventhub.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name          = 'logstash-input-azureeventhub'
-  s.version       = '0.9.7'
+  s.version       = '0.9.8'
   s.platform      = "java"
   s.licenses      = ['Apache License (2.0)']
   s.summary       = "This plugin collects data from Azure Event Hubs."
@@ -24,9 +24,6 @@ Gem::Specification.new do |s|
     s.add_development_dependency 'logstash-devutils'
   
   #Jar dependencies
-  s.requirements << "jar 'org.apache.qpid:qpid-amqp-1-0-common', '0.32'"
-  s.requirements << "jar 'org.apache.qpid:qpid-amqp-1-0-client-jms', '0.32'"
-  s.requirements << "jar 'org.apache.qpid:qpid-client', '0.32'"
-  s.requirements << "jar 'org.apache.geronimo.specs:geronimo-jms_1.1_spec', '1.1.1'"
+  s.requirements << "jar 'com.microsoft.azure:azure-eventhubs', '0.15.0'"
   s.add_runtime_dependency 'jar-dependencies', '~> 0.3.2'
 end

metadata

@@ -1,18 +1,17 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-azureeventhub
 version: !ruby/object:Gem::Version
-  version: 0.9.7
+  version: 0.9.8
 platform: java
 authors:
 - Microsoft Corporation
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-01-10 00:00:00.000000000 Z
+date: 2017-10-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: logstash-core-plugin-api
-  version_requirements: !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
@@ -20,7 +19,10 @@ dependencies:
     - - "<="
       - !ruby/object:Gem::Version
         version: '2.99'
-  requirement: !ruby/object:Gem::Requirement
+  name: logstash-core-plugin-api
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
@@ -28,50 +30,48 @@ dependencies:
     - - "<="
       - !ruby/object:Gem::Version
         version: '2.99'
-  prerelease: false
-  type: :runtime
 - !ruby/object:Gem::Dependency
-  name: azure
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 0.7.1
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.7.1
+  name: azure
   prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: logstash-devutils
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.7.1
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+  name: logstash-devutils
   prerelease: false
   type: :development
-- !ruby/object:Gem::Dependency
-  name: jar-dependencies
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 0.3.2
+        version: '0'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.3.2
+  name: jar-dependencies
   prerelease: false
   type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.3.2
 description: This gem is a Logstash plugin. It reads data from Azure Event Hubs.
 email: azdiag@microsoft.com
 executables: []
@@ -82,7 +82,11 @@ files:
 - Gemfile
 - LICENSE
 - README.md
+- lib/com/microsoft/azure/azure-eventhubs/0.15.0/azure-eventhubs-0.15.0.jar
+- lib/logstash-input-azureeventhub_jars.rb
 - lib/logstash/inputs/azureeventhub.rb
+- lib/org/apache/qpid/proton-j/0.22.0/proton-j-0.22.0.jar
+- lib/org/slf4j/slf4j-api/1.7.25/slf4j-api-1.7.25.jar
 - logstash-input-azureeventhub.gemspec
 - spec/inputs/azureeventhub_spec.rb
 homepage: https://github.com/Azure/azure-diagnostics-tools
@@ -106,12 +110,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements:
-- jar 'org.apache.qpid:qpid-amqp-1-0-common', '0.32'
-- jar 'org.apache.qpid:qpid-amqp-1-0-client-jms', '0.32'
-- jar 'org.apache.qpid:qpid-client', '0.32'
-- jar 'org.apache.geronimo.specs:geronimo-jms_1.1_spec', '1.1.1'
+- jar 'com.microsoft.azure:azure-eventhubs', '0.15.0'
 rubyforge_project:
-rubygems_version: 2.4.8
+rubygems_version: 2.6.13
 signing_key:
 specification_version: 4
 summary: This plugin collects data from Azure Event Hubs.