logstash-input-rabbitmq 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +15 -0
- data/.gitignore +4 -0
- data/Gemfile +4 -0
- data/LICENSE +13 -0
- data/Rakefile +6 -0
- data/lib/logstash/inputs/rabbitmq.rb +128 -0
- data/lib/logstash/inputs/rabbitmq/bunny.rb +118 -0
- data/lib/logstash/inputs/rabbitmq/hot_bunnies.rb +1 -0
- data/lib/logstash/inputs/rabbitmq/march_hare.rb +129 -0
- data/logstash-input-rabbitmq.gemspec +34 -0
- data/rakelib/publish.rake +9 -0
- data/rakelib/vendor.rake +169 -0
- data/spec/inputs/rabbitmq_spec.rb +1 -0
- metadata +106 -0
checksums.yaml
ADDED
@@ -0,0 +1,15 @@
|
|
1
|
+
---
|
2
|
+
!binary "U0hBMQ==":
|
3
|
+
metadata.gz: !binary |-
|
4
|
+
ODdmYmIwZDg5ZGFiZmMxZGExNGE3MmM2M2RmODRhODg3MTJkNDhkMQ==
|
5
|
+
data.tar.gz: !binary |-
|
6
|
+
ZDlmNDIzZGY3Y2NmOWFlMTlhMzQ5ZDk5MjQxMTM0NTJmMDA2MDBlYw==
|
7
|
+
SHA512:
|
8
|
+
metadata.gz: !binary |-
|
9
|
+
OTU1NGViZTA0Y2NhNmVkNGZhZTE3ZDVkZmVkN2RlZmVhYzMyMjBkMjNlMzUy
|
10
|
+
OWVkZTAwMWMxZDdiNjAxZjJmMGNhNGMzN2E3MzQyZTdhNzhkYjhkMDFkZTlj
|
11
|
+
ZWM3MTQyOTU5NWZjYjE2M2JkZGIyMTBjNDBjYTM5Y2M5NGViOTY=
|
12
|
+
data.tar.gz: !binary |-
|
13
|
+
YzllMjI4NzNlZDJmMmI4MTk1OTM1N2UxNzVmNjhjYjNiNzQwNWE4ZmMwMDZk
|
14
|
+
YmM4NjUxN2MyMmExYWUwOWY1MDY3Yjk2MDllZWE3MjA3MzE5OGZhMTlmN2E2
|
15
|
+
NjRhNTdmYmYwOWJhYzZjMDM1MTkyNzVhMDQwNjliMTRjYWU3MWU=
|
data/.gitignore
ADDED
data/Gemfile
ADDED
data/LICENSE
ADDED
@@ -0,0 +1,13 @@
|
|
1
|
+
Copyright (c) 2012-2014 Elasticsearch <http://www.elasticsearch.org>
|
2
|
+
|
3
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
you may not use this file except in compliance with the License.
|
5
|
+
You may obtain a copy of the License at
|
6
|
+
|
7
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
|
9
|
+
Unless required by applicable law or agreed to in writing, software
|
10
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
See the License for the specific language governing permissions and
|
13
|
+
limitations under the License.
|
data/Rakefile
ADDED
@@ -0,0 +1,128 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require "logstash/inputs/threadable"
|
3
|
+
require "logstash/namespace"
|
4
|
+
|
5
|
+
# Pull events from a RabbitMQ exchange.
|
6
|
+
#
|
7
|
+
# The default settings will create an entirely transient queue and listen for all messages by default.
|
8
|
+
# If you need durability or any other advanced settings, please set the appropriate options
|
9
|
+
#
|
10
|
+
# This has been tested with Bunny 0.9.x, which supports RabbitMQ 2.x and 3.x. You can
|
11
|
+
# find links to both here:
|
12
|
+
#
|
13
|
+
# * RabbitMQ - <http://www.rabbitmq.com/>
|
14
|
+
# * March Hare: <http://rubymarchhare.info>
|
15
|
+
# * Bunny - <https://github.com/ruby-amqp/bunny>
|
16
|
+
class LogStash::Inputs::RabbitMQ < LogStash::Inputs::Threadable
|
17
|
+
|
18
|
+
config_name "rabbitmq"
|
19
|
+
milestone 1
|
20
|
+
|
21
|
+
#
|
22
|
+
# Connection
|
23
|
+
#
|
24
|
+
|
25
|
+
# RabbitMQ server address
|
26
|
+
config :host, :validate => :string, :required => true
|
27
|
+
|
28
|
+
# RabbitMQ port to connect on
|
29
|
+
config :port, :validate => :number, :default => 5672
|
30
|
+
|
31
|
+
# RabbitMQ username
|
32
|
+
config :user, :validate => :string, :default => "guest"
|
33
|
+
|
34
|
+
# RabbitMQ password
|
35
|
+
config :password, :validate => :password, :default => "guest"
|
36
|
+
|
37
|
+
# The vhost to use. If you don't know what this is, leave the default.
|
38
|
+
config :vhost, :validate => :string, :default => "/"
|
39
|
+
|
40
|
+
# Enable or disable SSL
|
41
|
+
config :ssl, :validate => :boolean, :default => false
|
42
|
+
|
43
|
+
# Validate SSL certificate
|
44
|
+
config :verify_ssl, :validate => :boolean, :default => false
|
45
|
+
|
46
|
+
# Enable or disable logging
|
47
|
+
config :debug, :validate => :boolean, :default => false, :deprecated => "Use the logstash --debug flag for this instead."
|
48
|
+
|
49
|
+
|
50
|
+
|
51
|
+
#
|
52
|
+
# Queue & Consumer
|
53
|
+
#
|
54
|
+
|
55
|
+
# The name of the queue Logstash will consume events from.
|
56
|
+
config :queue, :validate => :string, :default => ""
|
57
|
+
|
58
|
+
# Is this queue durable? (aka; Should it survive a broker restart?)
|
59
|
+
config :durable, :validate => :boolean, :default => false
|
60
|
+
|
61
|
+
# Should the queue be deleted on the broker when the last consumer
|
62
|
+
# disconnects? Set this option to 'false' if you want the queue to remain
|
63
|
+
# on the broker, queueing up messages until a consumer comes along to
|
64
|
+
# consume them.
|
65
|
+
config :auto_delete, :validate => :boolean, :default => false
|
66
|
+
|
67
|
+
# Is the queue exclusive? Exclusive queues can only be used by the connection
|
68
|
+
# that declared them and will be deleted when it is closed (e.g. due to a Logstash
|
69
|
+
# restart).
|
70
|
+
config :exclusive, :validate => :boolean, :default => false
|
71
|
+
|
72
|
+
# Extra queue arguments as an array.
|
73
|
+
# To make a RabbitMQ queue mirrored, use: {"x-ha-policy" => "all"}
|
74
|
+
config :arguments, :validate => :array, :default => {}
|
75
|
+
|
76
|
+
# Prefetch count. Number of messages to prefetch
|
77
|
+
config :prefetch_count, :validate => :number, :default => 256
|
78
|
+
|
79
|
+
# Enable message acknowledgement
|
80
|
+
config :ack, :validate => :boolean, :default => true
|
81
|
+
|
82
|
+
# Passive queue creation? Useful for checking queue existance without modifying server state
|
83
|
+
config :passive, :validate => :boolean, :default => false
|
84
|
+
|
85
|
+
|
86
|
+
|
87
|
+
#
|
88
|
+
# (Optional) Exchange binding
|
89
|
+
#
|
90
|
+
|
91
|
+
# Optional.
|
92
|
+
#
|
93
|
+
# The name of the exchange to bind the queue to.
|
94
|
+
config :exchange, :validate => :string
|
95
|
+
|
96
|
+
# Optional.
|
97
|
+
#
|
98
|
+
# The routing key to use when binding a queue to the exchange.
|
99
|
+
# This is only relevant for direct or topic exchanges.
|
100
|
+
#
|
101
|
+
# * Routing keys are ignored on fanout exchanges.
|
102
|
+
# * Wildcards are not valid on direct exchanges.
|
103
|
+
config :key, :validate => :string, :default => "logstash"
|
104
|
+
|
105
|
+
|
106
|
+
def initialize(params)
|
107
|
+
params["codec"] = "json" if !params["codec"]
|
108
|
+
|
109
|
+
super
|
110
|
+
end
|
111
|
+
|
112
|
+
# Use March Hare on JRuby to avoid IO#select CPU spikes
|
113
|
+
# (see github.com/ruby-amqp/bunny/issues/95).
|
114
|
+
#
|
115
|
+
# On MRI, use Bunny.
|
116
|
+
#
|
117
|
+
# See http://rubybunny.info and http://rubymarchhare.info
|
118
|
+
# for the docs.
|
119
|
+
if RUBY_ENGINE == "jruby"
|
120
|
+
require "logstash/inputs/rabbitmq/march_hare"
|
121
|
+
|
122
|
+
include MarchHareImpl
|
123
|
+
else
|
124
|
+
require "logstash/inputs/rabbitmq/bunny"
|
125
|
+
|
126
|
+
include BunnyImpl
|
127
|
+
end
|
128
|
+
end # class LogStash::Inputs::RabbitMQ
|
@@ -0,0 +1,118 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
class LogStash::Inputs::RabbitMQ
|
3
|
+
module BunnyImpl
|
4
|
+
def register
|
5
|
+
require "bunny"
|
6
|
+
|
7
|
+
@vhost ||= Bunny::DEFAULT_HOST
|
8
|
+
# 5672. Will be switched to 5671 by Bunny if TLS is enabled.
|
9
|
+
@port ||= AMQ::Protocol::DEFAULT_PORT
|
10
|
+
@routing_key ||= "#"
|
11
|
+
|
12
|
+
@settings = {
|
13
|
+
:vhost => @vhost,
|
14
|
+
:host => @host,
|
15
|
+
:port => @port,
|
16
|
+
:automatically_recover => false
|
17
|
+
}
|
18
|
+
@settings[:user] = @user || Bunny::DEFAULT_USER
|
19
|
+
@settings[:pass] = if @password
|
20
|
+
@password.value
|
21
|
+
else
|
22
|
+
Bunny::DEFAULT_PASSWORD
|
23
|
+
end
|
24
|
+
|
25
|
+
@settings[:log_level] = if @debug || @logger.debug?
|
26
|
+
:debug
|
27
|
+
else
|
28
|
+
:error
|
29
|
+
end
|
30
|
+
|
31
|
+
@settings[:tls] = @ssl if @ssl
|
32
|
+
@settings[:verify_ssl] = @verify_ssl if @verify_ssl
|
33
|
+
|
34
|
+
proto = if @ssl
|
35
|
+
"amqps"
|
36
|
+
else
|
37
|
+
"amqp"
|
38
|
+
end
|
39
|
+
@connection_url = "#{proto}://#{@user}@#{@host}:#{@port}#{vhost}/#{@queue}"
|
40
|
+
|
41
|
+
@logger.info("Registering input #{@connection_url}")
|
42
|
+
end
|
43
|
+
|
44
|
+
def run(output_queue)
|
45
|
+
@output_queue = output_queue
|
46
|
+
|
47
|
+
begin
|
48
|
+
setup
|
49
|
+
consume
|
50
|
+
rescue Bunny::NetworkFailure, Bunny::ConnectionClosedError, Bunny::ConnectionLevelException, Bunny::TCPConnectionFailed => e
|
51
|
+
n = Bunny::Session::DEFAULT_NETWORK_RECOVERY_INTERVAL * 2
|
52
|
+
|
53
|
+
# Because we manually reconnect instead of letting Bunny
|
54
|
+
# handle failures,
|
55
|
+
# make sure we don't leave any consumer work pool
|
56
|
+
# threads behind. MK.
|
57
|
+
@ch.maybe_kill_consumer_work_pool!
|
58
|
+
@logger.error("RabbitMQ connection error: #{e.message}. Will attempt to reconnect in #{n} seconds...")
|
59
|
+
|
60
|
+
sleep n
|
61
|
+
retry
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
65
|
+
def teardown
|
66
|
+
@consumer.cancel
|
67
|
+
|
68
|
+
@ch.close if @ch && @ch.open?
|
69
|
+
@conn.close if @conn && @conn.open?
|
70
|
+
|
71
|
+
finished
|
72
|
+
end
|
73
|
+
|
74
|
+
def setup
|
75
|
+
@conn = Bunny.new(@settings)
|
76
|
+
|
77
|
+
@logger.debug("Connecting to RabbitMQ. Settings: #{@settings.inspect}, queue: #{@queue.inspect}")
|
78
|
+
return if terminating?
|
79
|
+
@conn.start
|
80
|
+
|
81
|
+
@ch = @conn.create_channel.tap do |ch|
|
82
|
+
ch.prefetch(@prefetch_count)
|
83
|
+
end
|
84
|
+
@logger.info("Connected to RabbitMQ at #{@settings[:host]}")
|
85
|
+
|
86
|
+
@arguments_hash = Hash[*@arguments]
|
87
|
+
|
88
|
+
@q = @ch.queue(@queue,
|
89
|
+
:durable => @durable,
|
90
|
+
:auto_delete => @auto_delete,
|
91
|
+
:exclusive => @exclusive,
|
92
|
+
:passive => @passive,
|
93
|
+
:arguments => @arguments)
|
94
|
+
|
95
|
+
# exchange binding is optional for the input
|
96
|
+
if @exchange
|
97
|
+
@q.bind(@exchange, :routing_key => @key)
|
98
|
+
end
|
99
|
+
end
|
100
|
+
|
101
|
+
def consume
|
102
|
+
@logger.info("Will consume events from queue #{@q.name}")
|
103
|
+
|
104
|
+
# we both need to block the caller in Bunny::Queue#subscribe and have
|
105
|
+
# a reference to the consumer so that we can cancel it, so
|
106
|
+
# a consumer manually. MK.
|
107
|
+
@consumer = Bunny::Consumer.new(@ch, @q)
|
108
|
+
@q.subscribe(:manual_ack => @ack, :block => true) do |delivery_info, properties, data|
|
109
|
+
@codec.decode(data) do |event|
|
110
|
+
decorate(event)
|
111
|
+
@output_queue << event
|
112
|
+
end
|
113
|
+
|
114
|
+
@ch.acknowledge(delivery_info.delivery_tag) if @ack
|
115
|
+
end
|
116
|
+
end
|
117
|
+
end # BunnyImpl
|
118
|
+
end
|
@@ -0,0 +1 @@
|
|
1
|
+
require "logstash/inputs/rabbitmq/march_hare"
|
@@ -0,0 +1,129 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
class LogStash::Inputs::RabbitMQ
|
3
|
+
# MarchHare-based implementation for JRuby
|
4
|
+
module MarchHareImpl
|
5
|
+
def register
|
6
|
+
require "hot_bunnies"
|
7
|
+
require "java"
|
8
|
+
|
9
|
+
@vhost ||= "127.0.0.1"
|
10
|
+
# 5672. Will be switched to 5671 by Bunny if TLS is enabled.
|
11
|
+
@port ||= 5672
|
12
|
+
@key ||= "#"
|
13
|
+
|
14
|
+
@settings = {
|
15
|
+
:vhost => @vhost,
|
16
|
+
:host => @host,
|
17
|
+
:port => @port,
|
18
|
+
:user => @user,
|
19
|
+
:automatic_recovery => false
|
20
|
+
}
|
21
|
+
@settings[:pass] = @password.value if @password
|
22
|
+
@settings[:tls] = @ssl if @ssl
|
23
|
+
|
24
|
+
proto = if @ssl
|
25
|
+
"amqps"
|
26
|
+
else
|
27
|
+
"amqp"
|
28
|
+
end
|
29
|
+
@connection_url = "#{proto}://#{@user}@#{@host}:#{@port}#{vhost}/#{@queue}"
|
30
|
+
|
31
|
+
@logger.info("Registering input #{@connection_url}")
|
32
|
+
end
|
33
|
+
|
34
|
+
def run(output_queue)
|
35
|
+
@output_queue = output_queue
|
36
|
+
@break_out_of_the_loop = java.util.concurrent.atomic.AtomicBoolean.new(false)
|
37
|
+
|
38
|
+
# MarchHare does not raise exceptions when connection goes down with a blocking
|
39
|
+
# consumer running (it uses callbacks, as the RabbitMQ Java client does).
|
40
|
+
#
|
41
|
+
# However, MarchHare::Channel will make sure to unblock all blocking consumers
|
42
|
+
# on any internal shutdown, so #consume will return and another loop iteration
|
43
|
+
# will run.
|
44
|
+
#
|
45
|
+
# This is very similar to how the Bunny implementation works and is sufficient
|
46
|
+
# for our needs: it recovers successfully after RabbitMQ is kill -9ed, the
|
47
|
+
# network device is shut down, etc. MK.
|
48
|
+
until @break_out_of_the_loop.get do
|
49
|
+
begin
|
50
|
+
setup
|
51
|
+
consume
|
52
|
+
rescue MarchHare::Exception, java.lang.Throwable, com.rabbitmq.client.AlreadyClosedException => e
|
53
|
+
n = 10
|
54
|
+
@logger.error("RabbitMQ connection error: #{e}. Will reconnect in #{n} seconds...")
|
55
|
+
|
56
|
+
sleep n
|
57
|
+
retry
|
58
|
+
rescue LogStash::ShutdownSignal => ss
|
59
|
+
shutdown_consumer
|
60
|
+
end
|
61
|
+
|
62
|
+
n = 10
|
63
|
+
@logger.error("RabbitMQ connection error: #{e}. Will reconnect in #{n} seconds...")
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
def teardown
|
68
|
+
shutdown_consumer
|
69
|
+
|
70
|
+
@ch.close if @ch && @ch.open?
|
71
|
+
@connection.close if @connection && @connection.open?
|
72
|
+
|
73
|
+
finished
|
74
|
+
end
|
75
|
+
|
76
|
+
#
|
77
|
+
# Implementation
|
78
|
+
#
|
79
|
+
|
80
|
+
protected
|
81
|
+
|
82
|
+
def setup
|
83
|
+
return if terminating?
|
84
|
+
|
85
|
+
@conn = MarchHare.connect(@settings)
|
86
|
+
@logger.info("Connected to RabbitMQ #{@connection_url}")
|
87
|
+
|
88
|
+
@ch = @conn.create_channel.tap do |ch|
|
89
|
+
ch.prefetch = @prefetch_count
|
90
|
+
end
|
91
|
+
|
92
|
+
@arguments_hash = Hash[*@arguments]
|
93
|
+
|
94
|
+
@q = @ch.queue(@queue,
|
95
|
+
:durable => @durable,
|
96
|
+
:auto_delete => @auto_delete,
|
97
|
+
:exclusive => @exclusive,
|
98
|
+
:passive => @passive,
|
99
|
+
:arguments => @arguments)
|
100
|
+
|
101
|
+
# exchange binding is optional for the input
|
102
|
+
if @exchange
|
103
|
+
@q.bind(@exchange, :routing_key => @key)
|
104
|
+
end
|
105
|
+
end
|
106
|
+
|
107
|
+
def consume
|
108
|
+
return if terminating?
|
109
|
+
|
110
|
+
# we manually build a consumer here to be able to keep a reference to it
|
111
|
+
# in an @ivar even though we use a blocking version of HB::Queue#subscribe
|
112
|
+
@consumer = @q.build_consumer(:block => true) do |metadata, data|
|
113
|
+
@codec.decode(data) do |event|
|
114
|
+
decorate(event)
|
115
|
+
@output_queue << event if event
|
116
|
+
end
|
117
|
+
@ch.ack(metadata.delivery_tag) if @ack
|
118
|
+
end
|
119
|
+
@q.subscribe_with(@consumer, :manual_ack => @ack, :block => true)
|
120
|
+
end
|
121
|
+
|
122
|
+
def shutdown_consumer
|
123
|
+
@break_out_of_the_loop.set(true)
|
124
|
+
|
125
|
+
@consumer.cancel
|
126
|
+
@consumer.gracefully_shut_down
|
127
|
+
end
|
128
|
+
end # MarchHareImpl
|
129
|
+
end
|
@@ -0,0 +1,34 @@
|
|
1
|
+
Gem::Specification.new do |s|
|
2
|
+
|
3
|
+
s.name = 'logstash-input-rabbitmq'
|
4
|
+
s.version = '0.1.0'
|
5
|
+
s.licenses = ['Apache License (2.0)']
|
6
|
+
s.summary = "Pull events from a RabbitMQ exchange."
|
7
|
+
s.description = "Pull events from a RabbitMQ exchange."
|
8
|
+
s.authors = ["Elasticsearch"]
|
9
|
+
s.email = 'richard.pijnenburg@elasticsearch.com'
|
10
|
+
s.homepage = "http://logstash.net/"
|
11
|
+
s.require_paths = ["lib"]
|
12
|
+
|
13
|
+
# Files
|
14
|
+
s.files = `git ls-files`.split($\)+::Dir.glob('vendor/*')
|
15
|
+
|
16
|
+
# Tests
|
17
|
+
s.test_files = s.files.grep(%r{^(test|spec|features)/})
|
18
|
+
|
19
|
+
# Special flag to let us know this is actually a logstash plugin
|
20
|
+
s.metadata = { "logstash_plugin" => "true", "group" => "input" }
|
21
|
+
|
22
|
+
# Gem dependencies
|
23
|
+
s.add_runtime_dependency 'logstash', '>= 1.4.0', '< 2.0.0'
|
24
|
+
|
25
|
+
s.add_runtime_dependency 'logstash-codec-json'
|
26
|
+
|
27
|
+
if RUBY_PLATFORM == 'java'
|
28
|
+
s.add_runtime_dependency 'march_hare', ['~> 2.5.1']
|
29
|
+
else
|
30
|
+
s.add_runtime_dependency 'bunny', ['>= 1.6.0']
|
31
|
+
end
|
32
|
+
|
33
|
+
end
|
34
|
+
|
@@ -0,0 +1,9 @@
|
|
1
|
+
require "gem_publisher"
|
2
|
+
|
3
|
+
desc "Publish gem to RubyGems.org"
|
4
|
+
task :publish_gem do |t|
|
5
|
+
gem_file = Dir.glob(File.expand_path('../*.gemspec',File.dirname(__FILE__))).first
|
6
|
+
gem = GemPublisher.publish_if_updated(gem_file, :rubygems)
|
7
|
+
puts "Published #{gem}" if gem
|
8
|
+
end
|
9
|
+
|
data/rakelib/vendor.rake
ADDED
@@ -0,0 +1,169 @@
|
|
1
|
+
require "net/http"
|
2
|
+
require "uri"
|
3
|
+
require "digest/sha1"
|
4
|
+
|
5
|
+
def vendor(*args)
|
6
|
+
return File.join("vendor", *args)
|
7
|
+
end
|
8
|
+
|
9
|
+
directory "vendor/" => ["vendor"] do |task, args|
|
10
|
+
mkdir task.name
|
11
|
+
end
|
12
|
+
|
13
|
+
def fetch(url, sha1, output)
|
14
|
+
|
15
|
+
puts "Downloading #{url}"
|
16
|
+
actual_sha1 = download(url, output)
|
17
|
+
|
18
|
+
if actual_sha1 != sha1
|
19
|
+
fail "SHA1 does not match (expected '#{sha1}' but got '#{actual_sha1}')"
|
20
|
+
end
|
21
|
+
end # def fetch
|
22
|
+
|
23
|
+
def file_fetch(url, sha1)
|
24
|
+
filename = File.basename( URI(url).path )
|
25
|
+
output = "vendor/#{filename}"
|
26
|
+
task output => [ "vendor/" ] do
|
27
|
+
begin
|
28
|
+
actual_sha1 = file_sha1(output)
|
29
|
+
if actual_sha1 != sha1
|
30
|
+
fetch(url, sha1, output)
|
31
|
+
end
|
32
|
+
rescue Errno::ENOENT
|
33
|
+
fetch(url, sha1, output)
|
34
|
+
end
|
35
|
+
end.invoke
|
36
|
+
|
37
|
+
return output
|
38
|
+
end
|
39
|
+
|
40
|
+
def file_sha1(path)
|
41
|
+
digest = Digest::SHA1.new
|
42
|
+
fd = File.new(path, "r")
|
43
|
+
while true
|
44
|
+
begin
|
45
|
+
digest << fd.sysread(16384)
|
46
|
+
rescue EOFError
|
47
|
+
break
|
48
|
+
end
|
49
|
+
end
|
50
|
+
return digest.hexdigest
|
51
|
+
ensure
|
52
|
+
fd.close if fd
|
53
|
+
end
|
54
|
+
|
55
|
+
def download(url, output)
|
56
|
+
uri = URI(url)
|
57
|
+
digest = Digest::SHA1.new
|
58
|
+
tmp = "#{output}.tmp"
|
59
|
+
Net::HTTP.start(uri.host, uri.port, :use_ssl => (uri.scheme == "https")) do |http|
|
60
|
+
request = Net::HTTP::Get.new(uri.path)
|
61
|
+
http.request(request) do |response|
|
62
|
+
fail "HTTP fetch failed for #{url}. #{response}" if [200, 301].include?(response.code)
|
63
|
+
size = (response["content-length"].to_i || -1).to_f
|
64
|
+
count = 0
|
65
|
+
File.open(tmp, "w") do |fd|
|
66
|
+
response.read_body do |chunk|
|
67
|
+
fd.write(chunk)
|
68
|
+
digest << chunk
|
69
|
+
if size > 0 && $stdout.tty?
|
70
|
+
count += chunk.bytesize
|
71
|
+
$stdout.write(sprintf("\r%0.2f%%", count/size * 100))
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|
75
|
+
$stdout.write("\r \r") if $stdout.tty?
|
76
|
+
end
|
77
|
+
end
|
78
|
+
|
79
|
+
File.rename(tmp, output)
|
80
|
+
|
81
|
+
return digest.hexdigest
|
82
|
+
rescue SocketError => e
|
83
|
+
puts "Failure while downloading #{url}: #{e}"
|
84
|
+
raise
|
85
|
+
ensure
|
86
|
+
File.unlink(tmp) if File.exist?(tmp)
|
87
|
+
end # def download
|
88
|
+
|
89
|
+
def untar(tarball, &block)
|
90
|
+
require "archive/tar/minitar"
|
91
|
+
tgz = Zlib::GzipReader.new(File.open(tarball))
|
92
|
+
# Pull out typesdb
|
93
|
+
tar = Archive::Tar::Minitar::Input.open(tgz)
|
94
|
+
tar.each do |entry|
|
95
|
+
path = block.call(entry)
|
96
|
+
next if path.nil?
|
97
|
+
parent = File.dirname(path)
|
98
|
+
|
99
|
+
mkdir_p parent unless File.directory?(parent)
|
100
|
+
|
101
|
+
# Skip this file if the output file is the same size
|
102
|
+
if entry.directory?
|
103
|
+
mkdir path unless File.directory?(path)
|
104
|
+
else
|
105
|
+
entry_mode = entry.instance_eval { @mode } & 0777
|
106
|
+
if File.exists?(path)
|
107
|
+
stat = File.stat(path)
|
108
|
+
# TODO(sissel): Submit a patch to archive-tar-minitar upstream to
|
109
|
+
# expose headers in the entry.
|
110
|
+
entry_size = entry.instance_eval { @size }
|
111
|
+
# If file sizes are same, skip writing.
|
112
|
+
next if stat.size == entry_size && (stat.mode & 0777) == entry_mode
|
113
|
+
end
|
114
|
+
puts "Extracting #{entry.full_name} from #{tarball} #{entry_mode.to_s(8)}"
|
115
|
+
File.open(path, "w") do |fd|
|
116
|
+
# eof? check lets us skip empty files. Necessary because the API provided by
|
117
|
+
# Archive::Tar::Minitar::Reader::EntryStream only mostly acts like an
|
118
|
+
# IO object. Something about empty files in this EntryStream causes
|
119
|
+
# IO.copy_stream to throw "can't convert nil into String" on JRuby
|
120
|
+
# TODO(sissel): File a bug about this.
|
121
|
+
while !entry.eof?
|
122
|
+
chunk = entry.read(16384)
|
123
|
+
fd.write(chunk)
|
124
|
+
end
|
125
|
+
#IO.copy_stream(entry, fd)
|
126
|
+
end
|
127
|
+
File.chmod(entry_mode, path)
|
128
|
+
end
|
129
|
+
end
|
130
|
+
tar.close
|
131
|
+
File.unlink(tarball) if File.file?(tarball)
|
132
|
+
end # def untar
|
133
|
+
|
134
|
+
def ungz(file)
|
135
|
+
|
136
|
+
outpath = file.gsub('.gz', '')
|
137
|
+
tgz = Zlib::GzipReader.new(File.open(file))
|
138
|
+
begin
|
139
|
+
File.open(outpath, "w") do |out|
|
140
|
+
IO::copy_stream(tgz, out)
|
141
|
+
end
|
142
|
+
File.unlink(file)
|
143
|
+
rescue
|
144
|
+
File.unlink(outpath) if File.file?(outpath)
|
145
|
+
raise
|
146
|
+
end
|
147
|
+
tgz.close
|
148
|
+
end
|
149
|
+
|
150
|
+
desc "Process any vendor files required for this plugin"
|
151
|
+
task "vendor" do |task, args|
|
152
|
+
|
153
|
+
@files.each do |file|
|
154
|
+
download = file_fetch(file['url'], file['sha1'])
|
155
|
+
if download =~ /.tar.gz/
|
156
|
+
prefix = download.gsub('.tar.gz', '').gsub('vendor/', '')
|
157
|
+
untar(download) do |entry|
|
158
|
+
if !file['files'].nil?
|
159
|
+
next unless file['files'].include?(entry.full_name.gsub(prefix, ''))
|
160
|
+
out = entry.full_name.split("/").last
|
161
|
+
end
|
162
|
+
File.join('vendor', out)
|
163
|
+
end
|
164
|
+
elsif download =~ /.gz/
|
165
|
+
ungz(download)
|
166
|
+
end
|
167
|
+
end
|
168
|
+
|
169
|
+
end
|
@@ -0,0 +1 @@
|
|
1
|
+
require 'spec_helper'
|
metadata
ADDED
@@ -0,0 +1,106 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: logstash-input-rabbitmq
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Elasticsearch
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2014-11-05 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: logstash
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ! '>='
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: 1.4.0
|
20
|
+
- - <
|
21
|
+
- !ruby/object:Gem::Version
|
22
|
+
version: 2.0.0
|
23
|
+
type: :runtime
|
24
|
+
prerelease: false
|
25
|
+
version_requirements: !ruby/object:Gem::Requirement
|
26
|
+
requirements:
|
27
|
+
- - ! '>='
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
version: 1.4.0
|
30
|
+
- - <
|
31
|
+
- !ruby/object:Gem::Version
|
32
|
+
version: 2.0.0
|
33
|
+
- !ruby/object:Gem::Dependency
|
34
|
+
name: logstash-codec-json
|
35
|
+
requirement: !ruby/object:Gem::Requirement
|
36
|
+
requirements:
|
37
|
+
- - ! '>='
|
38
|
+
- !ruby/object:Gem::Version
|
39
|
+
version: '0'
|
40
|
+
type: :runtime
|
41
|
+
prerelease: false
|
42
|
+
version_requirements: !ruby/object:Gem::Requirement
|
43
|
+
requirements:
|
44
|
+
- - ! '>='
|
45
|
+
- !ruby/object:Gem::Version
|
46
|
+
version: '0'
|
47
|
+
- !ruby/object:Gem::Dependency
|
48
|
+
name: bunny
|
49
|
+
requirement: !ruby/object:Gem::Requirement
|
50
|
+
requirements:
|
51
|
+
- - ! '>='
|
52
|
+
- !ruby/object:Gem::Version
|
53
|
+
version: 1.6.0
|
54
|
+
type: :runtime
|
55
|
+
prerelease: false
|
56
|
+
version_requirements: !ruby/object:Gem::Requirement
|
57
|
+
requirements:
|
58
|
+
- - ! '>='
|
59
|
+
- !ruby/object:Gem::Version
|
60
|
+
version: 1.6.0
|
61
|
+
description: Pull events from a RabbitMQ exchange.
|
62
|
+
email: richard.pijnenburg@elasticsearch.com
|
63
|
+
executables: []
|
64
|
+
extensions: []
|
65
|
+
extra_rdoc_files: []
|
66
|
+
files:
|
67
|
+
- .gitignore
|
68
|
+
- Gemfile
|
69
|
+
- LICENSE
|
70
|
+
- Rakefile
|
71
|
+
- lib/logstash/inputs/rabbitmq.rb
|
72
|
+
- lib/logstash/inputs/rabbitmq/bunny.rb
|
73
|
+
- lib/logstash/inputs/rabbitmq/hot_bunnies.rb
|
74
|
+
- lib/logstash/inputs/rabbitmq/march_hare.rb
|
75
|
+
- logstash-input-rabbitmq.gemspec
|
76
|
+
- rakelib/publish.rake
|
77
|
+
- rakelib/vendor.rake
|
78
|
+
- spec/inputs/rabbitmq_spec.rb
|
79
|
+
homepage: http://logstash.net/
|
80
|
+
licenses:
|
81
|
+
- Apache License (2.0)
|
82
|
+
metadata:
|
83
|
+
logstash_plugin: 'true'
|
84
|
+
group: input
|
85
|
+
post_install_message:
|
86
|
+
rdoc_options: []
|
87
|
+
require_paths:
|
88
|
+
- lib
|
89
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
90
|
+
requirements:
|
91
|
+
- - ! '>='
|
92
|
+
- !ruby/object:Gem::Version
|
93
|
+
version: '0'
|
94
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
95
|
+
requirements:
|
96
|
+
- - ! '>='
|
97
|
+
- !ruby/object:Gem::Version
|
98
|
+
version: '0'
|
99
|
+
requirements: []
|
100
|
+
rubyforge_project:
|
101
|
+
rubygems_version: 2.4.1
|
102
|
+
signing_key:
|
103
|
+
specification_version: 4
|
104
|
+
summary: Pull events from a RabbitMQ exchange.
|
105
|
+
test_files:
|
106
|
+
- spec/inputs/rabbitmq_spec.rb
|