logstash-input-event-hub 0.9.8-java

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: d91c45e960c5cbc93915934fad7f29af488d5fef
+  data.tar.gz: 82fd3dfc287c37cc6f89bbe3ab6a49ac6e6d17ee
+SHA512:
+  metadata.gz: 81344c87d6c3bd82106a6dcab73d18b6bf77829b76ad706566e6f979ebb791ce681b70038292083598b83aa6973629ff6267ccf239a74d37253adc3713eb789e
+  data.tar.gz: 5ee58b5c8d391b881dcda450ff842981e99d28384c7d229421f033ee18dd190fced1fa2778c29371a7d1a64d592ab82eb8df19cf3508fd5226c03665689d069c

data/CHANGELOG.md

@@ -0,0 +1,6 @@
+## 2016.05.03
+* Fixed the jar dependency problem. Now installing the plugin by using the "plugin" or "logstash-plugin" (for newer versions of Logstash) should automatically download and jar files needed.
+* Fixed the default value of *time_since_epoch_millis* to use UTC time to match the SelectorFilter.
+* Made the plugin to respect Logstash shutdown signal.
+* Updated the *logstash-core* runtime dependency requirement to '~> 2.0'.
+* Updated the *logstash-devutils* development dependency requirement to '>= 0.0.16'

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org'
+gemspec

data/LICENSE

@@ -0,0 +1,17 @@
+
+Copyright (c) Microsoft.  All rights reserved.
+Microsoft would like to thank its contributors, a list
+of whom are at http://aka.ms/entlib-contributors
+
+Licensed under the Apache License, Version 2.0 (the "License"); you
+may not use this file except in compliance with the License. You may
+obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+implied. See the License for the specific language governing permissions
+and limitations under the License.
+

data/README.md

@@ -0,0 +1,110 @@
+# Logstash input plugin for data from Event Hubs 
+
+## Summary
+This plugin reads data from specified Azure Event Hubs.
+
+## Installation
+You can install this plugin using the Logstash "plugin" or "logstash-plugin" (for newer versions of Logstash) command:
+```sh
+logstash-plugin install logstash-input-azureeventhub
+```
+For more information, see Logstash reference [Working with plugins](https://www.elastic.co/guide/en/logstash/current/working-with-plugins.html).
+
+## Configuration
+### Required Parameters
+__*key*__
+
+The shared access key to the target event hub.
+
+__*username*__
+
+The name of the shared access policy.
+
+__*namespace*__
+
+Event Hub namespace.
+
+__*eventhub*__
+
+Event Hub name.
+
+__*partitions*__
+
+Partition count of the target event hub.
+
+### Optional Parameters
+__*domain*__
+
+Domain of the target Event Hub. Default value is "servicebus.windows.net".
+
+__*port*__
+
+Port of the target Event Hub. Default value is 5671.
+
+__*receive_credits*__
+
+The credit number to limit the number of messages to receive in a processing cycle. Value must be between 10 and 999. Default is 999.
+
+__*consumer_group*__
+
+Name of the consumer group. Default value is "$default".
+
+__*time_since_epoch_millis*__
+
+Specifies the point of time after which the messages are received. Default value is the time when this plugin is initialized:
+```ruby
+Time.now.utc.to_i * 1000
+```
+__*thread_wait_sec*__
+
+Specifies the time (in seconds) to wait before another try if no message was received.
+
+__*partition_receiver_epochs*__
+
+A map from partition (string) to epoch (integer). By default each partition doesn't have an epoch defined. For more information read https://blogs.msdn.microsoft.com/gyan/2014/09/02/event-hubs-receiver-epoch/ .
+
+### Examples
+* Bare-bone settings
+```
+input
+{
+    azureeventhub
+    {
+        key => "VGhpcyBpcyBhIGZha2Uga2V5Lg=="
+        username => "receivepolicy"
+        namespace => "mysbns"
+        eventhub => "myeventhub"
+        partitions => 4
+        partition_receiver_epochs => { '2' => 42 '0' => 15 }
+    }
+}
+```
+
+* Example for WAD (Azure Diagnostics)
+```
+input
+{
+    azureeventhub
+    {
+        key => "VGhpcyBpcyBhIGZha2Uga2V5Lg=="
+        username => "receivepolicy"
+        namespace => "mysbns"
+        eventhub => "myeventhub"
+        partitions => 4
+        partition_receiver_epochs => { '2' => 42 '0' => 15 }
+    }
+}
+filter {
+    split {field => 'records'} #split the records array in individual events
+}
+output {
+    stdout { 
+        codec => rubydebug
+    }
+}
+```
+
+## More information
+The source code of this plugin is hosted in GitHub repo [Microsoft Azure Diagnostics with ELK](https://github.com/Azure/azure-diagnostics-tools). We welcome you to provide feedback and/or contribute to the project.
+
+Please also see [Analyze Diagnostics Data with ELK template](https://github.com/Azure/azure-quickstart-templates/tree/master/diagnostics-with-elk) for quick deployment of ELK to Azure.   

data/lib/logstash/inputs/azureeventhub.rb

@@ -0,0 +1,137 @@
+# encoding: utf-8
+require "logstash/inputs/base"
+require "logstash/namespace"
+
+require "securerandom"
+require "open-uri"
+require "thread"
+
+require Dir[ File.dirname(__FILE__) + "/../../*_jars.rb" ].first
+
+# Reads events from Azure event-hub
+class LogStash::Inputs::Azureeventhub < LogStash::Inputs::Base
+
+  config_name "azureeventhub"
+  milestone 1
+
+  default :codec, "json"
+  
+  config :key, :validate => :string, :required => true
+  config :username, :validate => :string, :required => true
+  config :namespace, :validate => :string, :required => true
+  config :domain, :validate => :string, :default => "servicebus.windows.net"
+  config :port, :validate => :number, :default => 5671
+  config :receive_credits, :validate => :number, :default => 999
+  
+  config :eventhub, :validate => :string, :required => true
+  config :partitions, :validate => :number, :required => true
+  config :consumer_group, :validate => :string, :default => "$default"
+  
+  config :time_since_epoch_millis, :validate => :number, :default => Time.now.utc.to_i * 1000
+  config :thread_wait_sec, :validate => :number, :default => 5
+
+  config :partition_receiver_epochs, :validate => :hash, :default => {}
+  
+  
+  def initialize(*args)
+    super(*args)
+  end # def initialize
+
+  public
+  def register
+    user_agent = "logstash-input-azureeventhub"
+    user_agent << "/" << Gem.latest_spec_for("logstash-input-azureeventhub").version.to_s
+    com::microsoft::azure::eventhubs::EventHubClient.userAgent = user_agent
+  end # def register
+
+  def process(output_queue, receiver, partition, last_event_offset)
+    while !stop?
+      begin
+        events = receiver.receiveSync(10)
+        if events
+          events.each{ |msg|
+            body = msg.getBytes().to_s.gsub("\\x5c", "\\")
+            props = msg.getSystemProperties()
+
+            last_event_offset = props.getOffset()
+            
+            @logger.debug("[#{partition.to_s.rjust(2,"0")}] Event: #{body[0..50] unless body.nil?}... " <<
+              "Offset: #{props.getOffset()} " <<
+              "Time: #{props.getEnqueuedTime()} " <<
+              "Sequence: #{props.getSequenceNumber()}")
+
+            codec.decode(body) do |event|
+              decorate(event)
+              output_queue << event
+            end
+          }
+        else
+          @logger.debug("[#{partition.to_s.rjust(2,"0")}] No message")
+        end
+      end
+    end
+  rescue LogStash::ShutdownSignal => e
+    @logger.debug("[#{partition.to_s.rjust(2,"0")}] ShutdownSignal received")
+    raise e
+  rescue => e
+    @logger.error("[#{partition.to_s.rjust(2,"0")}] Oh My, An error occurred. Error:#{e}: Trace: #{e.backtrace}", :exception => e)
+    raise e
+  ensure
+    return last_event_offset
+  end # process
+  
+  def process_partition(output_queue, partition, epoch) 
+    last_event_offset = nil
+    while !stop?
+      begin
+        host = java::net::URI.new("amqps://" << @namespace << "." << @domain)
+        connStr = com::microsoft::azure::eventhubs::ConnectionStringBuilder.new(host, @eventhub, @username, @key).toString()
+        ehClient = com::microsoft::azure::eventhubs::EventHubClient.createFromConnectionStringSync(connStr)
+
+        if !epoch.nil?
+          if !last_event_offset.nil?
+            @logger.debug("[#{partition.to_s.rjust(2,"0")}] Create receiver with epoch=#{epoch} & offset > #{last_event_offset}")
+            receiver = ehClient.createEpochReceiverSync(@consumer_group, partition.to_s, last_event_offset, false, epoch)
+          else
+            @logger.debug("[#{partition.to_s.rjust(2,"0")}] Create receiver with epoch=#{epoch} & timestamp > #{@time_since_epoch_millis}")
+            receiver = ehClient.createEpochReceiverSync(@consumer_group, partition.to_s, java::time::Instant::ofEpochMilli(@time_since_epoch_millis), epoch)
+          end
+        else
+          if !last_event_offset.nil?
+            @logger.debug("[#{partition.to_s.rjust(2,"0")}] Create receiver with offset > #{last_event_offset}")
+            receiver = ehClient.createReceiverSync(@consumer_group, partition.to_s, last_event_offset, false)
+          else
+            @logger.debug("[#{partition.to_s.rjust(2,"0")}] Create receiver with timestamp > #{@time_since_epoch_millis}")
+            receiver = ehClient.createReceiverSync(@consumer_group, partition.to_s, java::time::Instant::ofEpochMilli(@time_since_epoch_millis))
+          end
+        end
+        receiver.setReceiveTimeout(java::time::Duration::ofSeconds(@thread_wait_sec));
+        receiver.setPrefetchCount(@receive_credits)
+
+        last_event_offset = process(output_queue, receiver, partition, last_event_offset)
+      rescue com::microsoft::azure::eventhubs::EventHubException => e
+        sleep(@thread_wait_sec)
+        @logger.debug("[#{partition.to_s.rjust(2,"0")}] resetting connection. Error:#{e}: Trace: #{e.backtrace}", :exception => e)
+      end
+    end
+  rescue LogStash::ShutdownSignal => e
+    receiver.close()
+    raise e
+  rescue => e
+    @logger.error("[#{partition.to_s.rjust(2,"0")}] Oh My, An error occurred. Error:#{e}: Trace: #{e.backtrace}", :exception => e)
+  end # process_partition
+
+  public
+  def run(output_queue)
+    threads = []
+    (0..(@partitions-1)).each do |p_id|
+      epoch = partition_receiver_epochs[p_id.to_s] if @partition_receiver_epochs.key?(p_id.to_s)
+      threads << Thread.new { process_partition(output_queue, p_id, epoch) }
+    end
+    threads.each { |thr| thr.join }
+  end # def run
+
+  public
+  def teardown
+  end # def teardown
+end # class LogStash::Inputs::Azureeventhub

data/logstash-input-azureeventhub.gemspec

@@ -0,0 +1,29 @@
+Gem::Specification.new do |s|
+  s.name          = 'logstash-input-event-hub'
+  s.version       = '0.9.8'
+  s.platform      = "java"
+  s.licenses      = ['Apache License (2.0)']
+  s.summary       = "This plugin collects data from Azure Event Hubs."
+  s.description   = "This gem is a Logstash plugin. It reads data from Azure Event Hubs."
+  s.authors       = ["Shailesh Srivastava"]
+  s.email         = 'shailesh17mar@gmail.com'
+  s.homepage      = "https://github.com/Azure/azure-diagnostics-tools"
+  s.require_paths = ["lib"]
+
+  # Files
+  s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','Gemfile','LICENSE']
+  # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "input" }
+
+  # Gem dependencies
+  s.add_runtime_dependency 'logstash-core-plugin-api', '~> 2.1', '>= 2.1.28'
+  s.add_runtime_dependency 'azure', '~> 0.7.1'
+    s.add_development_dependency 'logstash-devutils'
+  
+  #Jar dependencies
+  s.requirements << "jar 'com.microsoft.azure:azure-eventhubs', '0.15.0'"
+  s.add_runtime_dependency 'jar-dependencies', '~> 0.3.2'
+end

data/spec/inputs/azureeventhub_spec.rb

@@ -0,0 +1 @@
+require "logstash/devutils/rspec/spec_helper"

metadata

@@ -0,0 +1,116 @@
+--- !ruby/object:Gem::Specification
+name: logstash-input-event-hub
+version: !ruby/object:Gem::Version
+  version: 0.9.8
+platform: java
+authors:
+- Shailesh Srivastava
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-12-27 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: logstash-core-plugin-api
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.1'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.1.28
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.1'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.1.28
+- !ruby/object:Gem::Dependency
+  name: azure
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.7.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.7.1
+- !ruby/object:Gem::Dependency
+  name: logstash-devutils
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: jar-dependencies
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.3.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.3.2
+description: This gem is a Logstash plugin. It reads data from Azure Event Hubs.
+email: shailesh17mar@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- Gemfile
+- LICENSE
+- README.md
+- lib/logstash/inputs/azureeventhub.rb
+- logstash-input-azureeventhub.gemspec
+- spec/inputs/azureeventhub_spec.rb
+homepage: https://github.com/Azure/azure-diagnostics-tools
+licenses:
+- Apache License (2.0)
+metadata:
+  logstash_plugin: 'true'
+  logstash_group: input
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements:
+- jar 'com.microsoft.azure:azure-eventhubs', '0.15.0'
+rubyforge_project: 
+rubygems_version: 2.4.5.1
+signing_key: 
+specification_version: 4
+summary: This plugin collects data from Azure Event Hubs.
+test_files:
+- spec/inputs/azureeventhub_spec.rb