logstash-input-azureeventhub 0.9.5-java

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 28e1da54f720c2d11303e4a4c28dcec2332814bc
+  data.tar.gz: 3e43bdce6632c8b822b556b166f7558bbabaaccc
+SHA512:
+  metadata.gz: 27c5b6bd3740836cabeecac6e9ba9eaf240f83ca17e11ac2fda95b65839a48737a71b4ce6e4a37eef854066ccfad6ba59310255ec63e23fc9db27b15427548f8
+  data.tar.gz: 9d187d4ba4cdd1579cd80fafac81cf6df45b7f6295276946702652ade9619dc2db07ebdc24438a3b50309c615497b723b12cd1c67dad0853c75c8bdba6466b07

data/CHANGELOG.md

@@ -0,0 +1,6 @@
+## 2016.05.03
+* Fixed the jar dependency problem. Now installing the plugin by using the "plugin" or "logstash-plugin" (for newer versions of Logstash) should automatically download and jar files needed.
+* Fixed the default value of *time_since_epoch_millis* to use UTC time to match the SelectorFilter.
+* Made the plugin to respect Logstash shutdown signal.
+* Updated the *logstash-core* runtime dependency requirement to '~> 2.0'.
+* Updated the *logstash-devutils* development dependency requirement to '>= 0.0.16'

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org'
+gemspec

data/LICENSE

@@ -0,0 +1,17 @@
+
+Copyright (c) Microsoft.  All rights reserved.
+Microsoft would like to thank its contributors, a list
+of whom are at http://aka.ms/entlib-contributors
+
+Licensed under the Apache License, Version 2.0 (the "License"); you
+may not use this file except in compliance with the License. You may
+obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+implied. See the License for the specific language governing permissions
+and limitations under the License.
+

data/README.md

@@ -0,0 +1,80 @@
+# Logstash input plugin for data from Event Hubs 
+
+## Summary
+This plugin reads data from specified Azure Event Hubs.
+
+## Installation
+You can install this plugin using the Logstash "plugin" or "logstash-plugin" (for newer versions of Logstash) command:
+```sh
+logstash-plugin install logstash-input-azureeventhub
+```
+For more information, see Logstash reference [Working with plugins](https://www.elastic.co/guide/en/logstash/current/working-with-plugins.html).
+
+## Configuration
+### Required Parameters
+__*key*__
+
+The shared access key to the target event hub.
+
+__*username*__
+
+The name of the shared access policy.
+
+__*namespace*__
+
+Event Hub namespace.
+
+__*eventhub*__
+
+Event Hub name.
+
+__*partitions*__
+
+Partition count of the target event hub.
+
+### Optional Parameters
+__*domain*__
+
+Domain of the target Event Hub. Default value is "servicebus.windows.net".
+
+__*port*__
+
+Port of the target Event Hub. Default value is 5671.
+
+__*receive_credits*__
+
+The credit number to limit the number of messages to receive in a processing cycle. Default value is 1000.
+
+__*consumer_group*__
+
+Name of the consumer group. Default value is "$default".
+
+__*time_since_epoch_millis*__
+
+Specifies the point of time after which the messages are received. Default value is the time when this plugin is initialized:
+```ruby
+Time.now.utc.to_i * 1000
+```
+__*thread_wait_sec*__
+
+Specifies the time (in seconds) to wait before another try if no message was received.
+
+### Examples
+```
+input
+{
+    azurewadeventhub
+    {
+        key => "VGhpcyBpcyBhIGZha2Uga2V5Lg=="
+        username => "receivepolicy"
+        namespace => "mysbns"
+        eventhub => "myeventhub"
+        partitions => 4
+    }
+}
+```
+
+## More information
+The source code of this plugin is hosted in GitHub repo [Microsoft Azure Diagnostics with ELK](https://github.com/Azure/azure-diagnostics-tools). We welcome you to provide feedback and/or contribute to the project.
+
+Please also see [Analyze Diagnostics Data with ELK template](https://github.com/Azure/azure-quickstart-templates/tree/master/diagnostics-with-elk) for quick deployment of ELK to Azure.   

data/lib/logstash/inputs/azureeventhub.rb

@@ -0,0 +1,159 @@
+# encoding: utf-8
+require "logstash/inputs/base"
+require "logstash/namespace"
+
+require "securerandom"
+require "open-uri"
+require "thread"
+
+require Dir[ File.dirname(__FILE__) + "/../../*_jars.rb" ].first
+
+# Reads events from Azure event-hub
+class LogStash::Inputs::Azureeventhub < LogStash::Inputs::Base
+
+  config_name "azureeventhub"
+  milestone 1
+
+  default :codec, "json"
+  
+  config :key, :validate => :string
+  config :username, :validate => :string
+  config :namespace, :validate => :string
+  config :domain, :validate => :string, :default => "servicebus.windows.net"
+  config :port, :validate => :number, :default => 5671
+  config :receive_credits, :validate => :number, :default => 1000
+  
+  config :eventhub, :validate => :string
+  config :partitions, :validate => :number
+  config :consumer_group, :validate => :string, :default => "$default"
+  
+  config :time_since_epoch_millis, :validate => :number, :default => Time.now.utc.to_i * 1000
+  config :thread_wait_sec, :validate => :number, :default => 5
+  
+  
+  def initialize(*args)
+    super(*args)
+  end # def initialize
+
+  public
+  def register
+  end # def register
+
+  def get_pay_load(message)
+    return nil if not message
+    message.getPayload().each do |section|
+      if section.java_kind_of? org::apache::qpid::amqp_1_0::type::messaging::Data or section.java_kind_of? org::apache::qpid::amqp_1_0::type::messaging::AmqpValue
+        return section.getValue().to_s
+      end
+    end
+    return nil
+  end
+  
+  def process(output_queue, receiver, partition)
+    while !stop?
+      begin
+        msg = receiver.receive(10)
+        if msg
+          codec.decode(get_pay_load(msg)) do |event|
+            output_queue << event
+          end
+          receiver.acknowledge(msg)
+        else
+          @logger.debug("  " + partition.to_s.rjust(2,"0") + " --- " + "No message")
+          sleep(@thread_wait_sec)
+        end
+      rescue LogStash::ShutdownSignal => e
+        raise e
+      rescue org::apache::qpid::amqp_1_0::client::ConnectionErrorException => e
+        raise e
+      rescue => e
+        @logger.error("  " + partition.to_s.rjust(2,"0") + " --- " + "Oh My, An error occurred.", :exception => e)
+      end
+    end # process
+  end # process
+  
+  def process_partition(output_queue, partition)
+    while !stop?
+      begin
+        filter = SelectorFilter.new "amqp.annotation.x-opt-enqueuedtimeutc > '" + @time_since_epoch_millis.to_s + "'"
+        filters = { org::apache::qpid::amqp_1_0::type::Symbol.valueOf("apache.org:selector-filter:string") => filter }
+
+        host = @namespace + "." + @domain
+        connection = org::apache::qpid::amqp_1_0::client::Connection.new(host, @port, @username, @key, host, true)
+        connection.getEndpoint().getDescribedTypeRegistry().register(filter.java_class, WriterFactory.new)
+        receiveSession = connection.createSession()
+        receiver = receiveSession.createReceiver(@eventhub + "/ConsumerGroups/" + @consumer_group + "/Partitions/" + partition.to_s, org::apache::qpid::amqp_1_0::client::AcknowledgeMode::ALO, "eventhubs-receiver-link-" + partition.to_s, false, filters, nil)
+        receiver.setCredit(org::apache::qpid::amqp_1_0::type::UnsignedInteger.valueOf(@receive_credits), true)
+        process(output_queue,receiver,partition)
+      rescue org::apache::qpid::amqp_1_0::client::ConnectionErrorException => e
+        @logger.debug("  " + partition.to_s.rjust(2,"0") + " --- " + "resetting connection")
+        @time_since_epoch_millis = Time.now.utc.to_i * 1000
+      end
+    end
+  rescue LogStash::ShutdownSignal => e
+    receiver.close()
+    raise e
+  rescue => e
+    @logger.error("  " + partition.to_s.rjust(2,"0") + " --- Oh My, An error occurred.", :exception => e)
+  end # process
+
+  public
+  def run(output_queue)
+    threads = []
+    (0..(@partitions-1)).each do |p_id|
+      threads << Thread.new { process_partition(output_queue, p_id) }
+    end
+    threads.each { |thr| thr.join }
+  end # def run
+
+  public
+  def teardown
+  end # def teardown
+end # class LogStash::Inputs::Azureeventhub
+
+
+class SelectorFilter
+  include org::apache::qpid::amqp_1_0::type::messaging::Filter
+
+  def initialize(value)
+    @value = value
+  end
+
+  def getValue
+    return @value
+  end
+
+  def toString
+    return @value
+  end
+end
+
+class SelectorFilterWriter < org::apache::qpid::amqp_1_0::codec::AbstractDescribedTypeWriter
+  def initialize(registry)
+    super(registry)
+  end
+  
+  def onSetValue(value)
+    @value = value
+  end
+  
+  def clear
+    @value = nil
+  end
+  
+  def getDescriptor
+    return org::apache::qpid::amqp_1_0::type::UnsignedLong.valueOf(83483426826);
+  end
+  
+  def createDescribedWriter
+    return getRegistry().getValueWriter(@value.getValue());
+  end
+end
+
+class WriterFactory
+  include org::apache::qpid::amqp_1_0::codec::ValueWriter::Factory
+
+  def newInstance(registry)
+    return SelectorFilterWriter.new registry
+  end
+end

data/logstash-input-azureeventhub.gemspec

@@ -0,0 +1,32 @@
+Gem::Specification.new do |s|
+  s.name          = 'logstash-input-azureeventhub'
+  s.version       = '0.9.5'
+  s.platform      = "java"
+  s.licenses      = ['Apache License (2.0)']
+  s.summary       = "This plugin collects data from Azure Event Hubs."
+  s.description   = "This gem is a Logstash plugin. It reads data from Azure Event Hubs."
+  s.authors       = ["Microsoft Corporation"]
+  s.email         = 'azdiag@microsoft.com'
+  s.homepage      = "https://github.com/Azure/azure-diagnostics-tools"
+  s.require_paths = ["lib"]
+
+  # Files
+  s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','Gemfile','LICENSE']
+  # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "input" }
+
+  # Gem dependencies
+  s.add_runtime_dependency 'logstash-core', '~> 2.0'
+  s.add_runtime_dependency 'azure', '~> 0.7.1'
+  s.add_development_dependency 'logstash-devutils', '>= 0.0.16'
+  
+  #Jar dependencies
+  s.requirements << "jar 'org.apache.qpid:qpid-amqp-1-0-common', '0.32'"
+  s.requirements << "jar 'org.apache.qpid:qpid-amqp-1-0-client-jms', '0.32'"
+  s.requirements << "jar 'org.apache.qpid:qpid-client', '0.32'"
+  s.requirements << "jar 'org.apache.geronimo.specs:geronimo-jms_1.1_spec', '1.1.1'"
+  s.add_runtime_dependency 'jar-dependencies', '~> 0.3.2'
+end

data/spec/inputs/azureeventhub_spec.rb

@@ -0,0 +1 @@
+require "logstash/devutils/rspec/spec_helper"

metadata

@@ -0,0 +1,113 @@
+--- !ruby/object:Gem::Specification
+name: logstash-input-azureeventhub
+version: !ruby/object:Gem::Version
+  version: 0.9.5
+platform: java
+authors:
+- Microsoft Corporation
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2016-05-03 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: logstash-core
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: azure
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.7.1
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.7.1
+  prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: logstash-devutils
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.0.16
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.0.16
+  prerelease: false
+  type: :development
+- !ruby/object:Gem::Dependency
+  name: jar-dependencies
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.3.2
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.3.2
+  prerelease: false
+  type: :runtime
+description: This gem is a Logstash plugin. It reads data from Azure Event Hubs.
+email: azdiag@microsoft.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- Gemfile
+- LICENSE
+- README.md
+- lib/logstash/inputs/azureeventhub.rb
+- logstash-input-azureeventhub.gemspec
+- spec/inputs/azureeventhub_spec.rb
+homepage: https://github.com/Azure/azure-diagnostics-tools
+licenses:
+- Apache License (2.0)
+metadata:
+  logstash_plugin: 'true'
+  logstash_group: input
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements:
+- jar 'org.apache.qpid:qpid-amqp-1-0-common', '0.32'
+- jar 'org.apache.qpid:qpid-amqp-1-0-client-jms', '0.32'
+- jar 'org.apache.qpid:qpid-client', '0.32'
+- jar 'org.apache.geronimo.specs:geronimo-jms_1.1_spec', '1.1.1'
+rubyforge_project:
+rubygems_version: 2.4.8
+signing_key:
+specification_version: 4
+summary: This plugin collects data from Azure Event Hubs.
+test_files:
+- spec/inputs/azureeventhub_spec.rb