logstash-input-sqs 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml ADDED
@@ -0,0 +1,15 @@
1
+ ---
2
+ !binary "U0hBMQ==":
3
+ metadata.gz: !binary |-
4
+ ZjkxZDNkOTIxNDBjMjllOTEyYzljNjM5NzUxMmJkOTBiZDQ0YTRmMg==
5
+ data.tar.gz: !binary |-
6
+ N2FlMjg0OWJlMzJlOTdjY2E1ODI2OTFiOWVlYWY1ZmU2OGI2MTAwZA==
7
+ SHA512:
8
+ metadata.gz: !binary |-
9
+ OWZlYmY4YjFhODk2YzNmY2NmODYyNzkwZmZlNjNkYjhlYmIwZjk3NzJjY2Uz
10
+ ZmUyZTdlNDc1NWI0MGJmOTczZGRmMjEyNTY0NzQ3MGFjYWZlYTQ5ZDFjZTM2
11
+ YjQwYWMyOTM3MDVjZDY0NTY3ZDEwNTk0MzI1ZTk4NzBjNDRlODc=
12
+ data.tar.gz: !binary |-
13
+ NDg4ZDcwNGRjNGMwNTA5MjlhNWJlOTNmNmU5YjU4MmJjYTgwNTcxZmU5YzBi
14
+ MmU4NTAxZjk5MTg4OTY3YTBkZWYyNzc0Y2EzNDZmMDJlMmMxZTdmMTkyMTY3
15
+ OTkwNmUwZGMyNTZjMmQ4OGEzOGQwODc5NGM4YTg3ZjRkYmU2MGE=
data/.gitignore ADDED
@@ -0,0 +1,4 @@
1
+ *.gem
2
+ Gemfile.lock
3
+ .bundle
4
+ vendor
data/Gemfile ADDED
@@ -0,0 +1,4 @@
1
+ source 'http://rubygems.org'
2
+ gem 'rake'
3
+ gem 'gem_publisher'
4
+ gem 'archive-tar-minitar'
data/LICENSE ADDED
@@ -0,0 +1,13 @@
1
+ Copyright (c) 2012-2014 Elasticsearch <http://www.elasticsearch.org>
2
+
3
+ Licensed under the Apache License, Version 2.0 (the "License");
4
+ you may not use this file except in compliance with the License.
5
+ You may obtain a copy of the License at
6
+
7
+ http://www.apache.org/licenses/LICENSE-2.0
8
+
9
+ Unless required by applicable law or agreed to in writing, software
10
+ distributed under the License is distributed on an "AS IS" BASIS,
11
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ See the License for the specific language governing permissions and
13
+ limitations under the License.
data/Rakefile ADDED
@@ -0,0 +1,6 @@
1
+ @files=[]
2
+
3
+ task :default do
4
+ system("rake -T")
5
+ end
6
+
@@ -0,0 +1,173 @@
1
+ # encoding: utf-8
2
+ require "logstash/inputs/threadable"
3
+ require "logstash/namespace"
4
+ require "logstash/timestamp"
5
+ require "logstash/plugin_mixins/aws_config"
6
+ require "digest/sha2"
7
+
8
+ # Pull events from an Amazon Web Services Simple Queue Service (SQS) queue.
9
+ #
10
+ # SQS is a simple, scalable queue system that is part of the
11
+ # Amazon Web Services suite of tools.
12
+ #
13
+ # Although SQS is similar to other queuing systems like AMQP, it
14
+ # uses a custom API and requires that you have an AWS account.
15
+ # See http://aws.amazon.com/sqs/ for more details on how SQS works,
16
+ # what the pricing schedule looks like and how to setup a queue.
17
+ #
18
+ # To use this plugin, you *must*:
19
+ #
20
+ # * Have an AWS account
21
+ # * Setup an SQS queue
22
+ # * Create an identify that has access to consume messages from the queue.
23
+ #
24
+ # The "consumer" identity must have the following permissions on the queue:
25
+ #
26
+ # * sqs:ChangeMessageVisibility
27
+ # * sqs:ChangeMessageVisibilityBatch
28
+ # * sqs:DeleteMessage
29
+ # * sqs:DeleteMessageBatch
30
+ # * sqs:GetQueueAttributes
31
+ # * sqs:GetQueueUrl
32
+ # * sqs:ListQueues
33
+ # * sqs:ReceiveMessage
34
+ #
35
+ # Typically, you should setup an IAM policy, create a user and apply the IAM policy to the user.
36
+ # A sample policy is as follows:
37
+ #
38
+ # {
39
+ # "Statement": [
40
+ # {
41
+ # "Action": [
42
+ # "sqs:ChangeMessageVisibility",
43
+ # "sqs:ChangeMessageVisibilityBatch",
44
+ # "sqs:GetQueueAttributes",
45
+ # "sqs:GetQueueUrl",
46
+ # "sqs:ListQueues",
47
+ # "sqs:SendMessage",
48
+ # "sqs:SendMessageBatch"
49
+ # ],
50
+ # "Effect": "Allow",
51
+ # "Resource": [
52
+ # "arn:aws:sqs:us-east-1:123456789012:Logstash"
53
+ # ]
54
+ # }
55
+ # ]
56
+ # }
57
+ #
58
+ # See http://aws.amazon.com/iam/ for more details on setting up AWS identities.
59
+ #
60
+ class LogStash::Inputs::SQS < LogStash::Inputs::Threadable
61
+ include LogStash::PluginMixins::AwsConfig
62
+
63
+ config_name "sqs"
64
+ milestone 1
65
+
66
+ default :codec, "json"
67
+
68
+ # Name of the SQS Queue name to pull messages from. Note that this is just the name of the queue, not the URL or ARN.
69
+ config :queue, :validate => :string, :required => true
70
+
71
+ # Name of the event field in which to store the SQS message ID
72
+ config :id_field, :validate => :string
73
+
74
+ # Name of the event field in which to store the SQS message MD5 checksum
75
+ config :md5_field, :validate => :string
76
+
77
+ # Name of the event field in which to store the SQS message Sent Timestamp
78
+ config :sent_timestamp_field, :validate => :string
79
+
80
+ public
81
+ def aws_service_endpoint(region)
82
+ return {
83
+ :sqs_endpoint => "sqs.#{region}.amazonaws.com"
84
+ }
85
+ end
86
+
87
+ public
88
+ def register
89
+ @logger.info("Registering SQS input", :queue => @queue)
90
+ require "aws-sdk"
91
+
92
+ @sqs = AWS::SQS.new(aws_options_hash)
93
+
94
+ begin
95
+ @logger.debug("Connecting to AWS SQS queue", :queue => @queue)
96
+ @sqs_queue = @sqs.queues.named(@queue)
97
+ @logger.info("Connected to AWS SQS queue successfully.", :queue => @queue)
98
+ rescue Exception => e
99
+ @logger.error("Unable to access SQS queue.", :error => e.to_s, :queue => @queue)
100
+ throw e
101
+ end # begin/rescue
102
+ end # def register
103
+
104
+ public
105
+ def run(output_queue)
106
+ @logger.debug("Polling SQS queue", :queue => @queue)
107
+
108
+ receive_opts = {
109
+ :limit => 10,
110
+ :visibility_timeout => 30,
111
+ :attributes => [:sent_at]
112
+ }
113
+
114
+ continue_polling = true
115
+ while running? && continue_polling
116
+ continue_polling = run_with_backoff(60, 1) do
117
+ @sqs_queue.receive_message(receive_opts) do |message|
118
+ if message
119
+ @codec.decode(message.body) do |event|
120
+ decorate(event)
121
+ if @id_field
122
+ event[@id_field] = message.id
123
+ end
124
+ if @md5_field
125
+ event[@md5_field] = message.md5
126
+ end
127
+ if @sent_timestamp_field
128
+ event[@sent_timestamp_field] = LogStash::Timestamp.new(message.sent_timestamp).utc
129
+ end
130
+ @logger.debug? && @logger.debug("Processed SQS message", :message_id => message.id, :message_md5 => message.md5, :sent_timestamp => message.sent_timestamp, :queue => @queue)
131
+ output_queue << event
132
+ message.delete
133
+ end # codec.decode
134
+ end # valid SQS message
135
+ end # receive_message
136
+ end # run_with_backoff
137
+ end # polling loop
138
+ end # def run
139
+
140
+ def teardown
141
+ @sqs_queue = nil
142
+ finished
143
+ end # def teardown
144
+
145
+ private
146
+ # Runs an AWS request inside a Ruby block with an exponential backoff in case
147
+ # we exceed the allowed AWS RequestLimit.
148
+ #
149
+ # @param [Integer] max_time maximum amount of time to sleep before giving up.
150
+ # @param [Integer] sleep_time the initial amount of time to sleep before retrying.
151
+ # @param [Block] block Ruby code block to execute.
152
+ def run_with_backoff(max_time, sleep_time, &block)
153
+ if sleep_time > max_time
154
+ @logger.error("AWS::EC2::Errors::RequestLimitExceeded ... failed.", :queue => @queue)
155
+ return false
156
+ end # retry limit exceeded
157
+
158
+ begin
159
+ block.call
160
+ rescue AWS::EC2::Errors::RequestLimitExceeded
161
+ @logger.info("AWS::EC2::Errors::RequestLimitExceeded ... retrying SQS request", :queue => @queue, :sleep_time => sleep_time)
162
+ sleep sleep_time
163
+ run_with_backoff(max_time, sleep_time * 2, &block)
164
+ rescue AWS::EC2::Errors::InstanceLimitExceeded
165
+ @logger.warn("AWS::EC2::Errors::InstanceLimitExceeded ... aborting SQS message retreival.")
166
+ return false
167
+ rescue Exception => bang
168
+ @logger.error("Error reading SQS queue.", :error => bang, :queue => @queue)
169
+ return false
170
+ end # begin/rescue
171
+ return true
172
+ end # def run_with_backoff
173
+ end # class LogStash::Inputs::SQS
@@ -0,0 +1,29 @@
1
+ Gem::Specification.new do |s|
2
+
3
+ s.name = 'logstash-input-sqs'
4
+ s.version = '0.1.0'
5
+ s.licenses = ['Apache License (2.0)']
6
+ s.summary = "Pull events from an Amazon Web Services Simple Queue Service (SQS) queue."
7
+ s.description = "Pull events from an Amazon Web Services Simple Queue Service (SQS) queue."
8
+ s.authors = ["Elasticsearch"]
9
+ s.email = 'richard.pijnenburg@elasticsearch.com'
10
+ s.homepage = "http://logstash.net/"
11
+ s.require_paths = ["lib"]
12
+
13
+ # Files
14
+ s.files = `git ls-files`.split($\)+::Dir.glob('vendor/*')
15
+
16
+ # Tests
17
+ s.test_files = s.files.grep(%r{^(test|spec|features)/})
18
+
19
+ # Special flag to let us know this is actually a logstash plugin
20
+ s.metadata = { "logstash_plugin" => "true", "group" => "input" }
21
+
22
+ # Gem dependencies
23
+ s.add_runtime_dependency 'logstash', '>= 1.4.0', '< 2.0.0'
24
+
25
+ s.add_runtime_dependency 'logstash-codec-json'
26
+ s.add_runtime_dependency 'aws-sdk'
27
+
28
+ end
29
+
@@ -0,0 +1,9 @@
1
+ require "gem_publisher"
2
+
3
+ desc "Publish gem to RubyGems.org"
4
+ task :publish_gem do |t|
5
+ gem_file = Dir.glob(File.expand_path('../*.gemspec',File.dirname(__FILE__))).first
6
+ gem = GemPublisher.publish_if_updated(gem_file, :rubygems)
7
+ puts "Published #{gem}" if gem
8
+ end
9
+
@@ -0,0 +1,169 @@
1
+ require "net/http"
2
+ require "uri"
3
+ require "digest/sha1"
4
+
5
+ def vendor(*args)
6
+ return File.join("vendor", *args)
7
+ end
8
+
9
+ directory "vendor/" => ["vendor"] do |task, args|
10
+ mkdir task.name
11
+ end
12
+
13
+ def fetch(url, sha1, output)
14
+
15
+ puts "Downloading #{url}"
16
+ actual_sha1 = download(url, output)
17
+
18
+ if actual_sha1 != sha1
19
+ fail "SHA1 does not match (expected '#{sha1}' but got '#{actual_sha1}')"
20
+ end
21
+ end # def fetch
22
+
23
+ def file_fetch(url, sha1)
24
+ filename = File.basename( URI(url).path )
25
+ output = "vendor/#{filename}"
26
+ task output => [ "vendor/" ] do
27
+ begin
28
+ actual_sha1 = file_sha1(output)
29
+ if actual_sha1 != sha1
30
+ fetch(url, sha1, output)
31
+ end
32
+ rescue Errno::ENOENT
33
+ fetch(url, sha1, output)
34
+ end
35
+ end.invoke
36
+
37
+ return output
38
+ end
39
+
40
+ def file_sha1(path)
41
+ digest = Digest::SHA1.new
42
+ fd = File.new(path, "r")
43
+ while true
44
+ begin
45
+ digest << fd.sysread(16384)
46
+ rescue EOFError
47
+ break
48
+ end
49
+ end
50
+ return digest.hexdigest
51
+ ensure
52
+ fd.close if fd
53
+ end
54
+
55
+ def download(url, output)
56
+ uri = URI(url)
57
+ digest = Digest::SHA1.new
58
+ tmp = "#{output}.tmp"
59
+ Net::HTTP.start(uri.host, uri.port, :use_ssl => (uri.scheme == "https")) do |http|
60
+ request = Net::HTTP::Get.new(uri.path)
61
+ http.request(request) do |response|
62
+ fail "HTTP fetch failed for #{url}. #{response}" if [200, 301].include?(response.code)
63
+ size = (response["content-length"].to_i || -1).to_f
64
+ count = 0
65
+ File.open(tmp, "w") do |fd|
66
+ response.read_body do |chunk|
67
+ fd.write(chunk)
68
+ digest << chunk
69
+ if size > 0 && $stdout.tty?
70
+ count += chunk.bytesize
71
+ $stdout.write(sprintf("\r%0.2f%%", count/size * 100))
72
+ end
73
+ end
74
+ end
75
+ $stdout.write("\r \r") if $stdout.tty?
76
+ end
77
+ end
78
+
79
+ File.rename(tmp, output)
80
+
81
+ return digest.hexdigest
82
+ rescue SocketError => e
83
+ puts "Failure while downloading #{url}: #{e}"
84
+ raise
85
+ ensure
86
+ File.unlink(tmp) if File.exist?(tmp)
87
+ end # def download
88
+
89
+ def untar(tarball, &block)
90
+ require "archive/tar/minitar"
91
+ tgz = Zlib::GzipReader.new(File.open(tarball))
92
+ # Pull out typesdb
93
+ tar = Archive::Tar::Minitar::Input.open(tgz)
94
+ tar.each do |entry|
95
+ path = block.call(entry)
96
+ next if path.nil?
97
+ parent = File.dirname(path)
98
+
99
+ mkdir_p parent unless File.directory?(parent)
100
+
101
+ # Skip this file if the output file is the same size
102
+ if entry.directory?
103
+ mkdir path unless File.directory?(path)
104
+ else
105
+ entry_mode = entry.instance_eval { @mode } & 0777
106
+ if File.exists?(path)
107
+ stat = File.stat(path)
108
+ # TODO(sissel): Submit a patch to archive-tar-minitar upstream to
109
+ # expose headers in the entry.
110
+ entry_size = entry.instance_eval { @size }
111
+ # If file sizes are same, skip writing.
112
+ next if stat.size == entry_size && (stat.mode & 0777) == entry_mode
113
+ end
114
+ puts "Extracting #{entry.full_name} from #{tarball} #{entry_mode.to_s(8)}"
115
+ File.open(path, "w") do |fd|
116
+ # eof? check lets us skip empty files. Necessary because the API provided by
117
+ # Archive::Tar::Minitar::Reader::EntryStream only mostly acts like an
118
+ # IO object. Something about empty files in this EntryStream causes
119
+ # IO.copy_stream to throw "can't convert nil into String" on JRuby
120
+ # TODO(sissel): File a bug about this.
121
+ while !entry.eof?
122
+ chunk = entry.read(16384)
123
+ fd.write(chunk)
124
+ end
125
+ #IO.copy_stream(entry, fd)
126
+ end
127
+ File.chmod(entry_mode, path)
128
+ end
129
+ end
130
+ tar.close
131
+ File.unlink(tarball) if File.file?(tarball)
132
+ end # def untar
133
+
134
+ def ungz(file)
135
+
136
+ outpath = file.gsub('.gz', '')
137
+ tgz = Zlib::GzipReader.new(File.open(file))
138
+ begin
139
+ File.open(outpath, "w") do |out|
140
+ IO::copy_stream(tgz, out)
141
+ end
142
+ File.unlink(file)
143
+ rescue
144
+ File.unlink(outpath) if File.file?(outpath)
145
+ raise
146
+ end
147
+ tgz.close
148
+ end
149
+
150
+ desc "Process any vendor files required for this plugin"
151
+ task "vendor" do |task, args|
152
+
153
+ @files.each do |file|
154
+ download = file_fetch(file['url'], file['sha1'])
155
+ if download =~ /.tar.gz/
156
+ prefix = download.gsub('.tar.gz', '').gsub('vendor/', '')
157
+ untar(download) do |entry|
158
+ if !file['files'].nil?
159
+ next unless file['files'].include?(entry.full_name.gsub(prefix, ''))
160
+ out = entry.full_name.split("/").last
161
+ end
162
+ File.join('vendor', out)
163
+ end
164
+ elsif download =~ /.gz/
165
+ ungz(download)
166
+ end
167
+ end
168
+
169
+ end
@@ -0,0 +1 @@
1
+ require 'spec_helper'
metadata ADDED
@@ -0,0 +1,103 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: logstash-input-sqs
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.0
5
+ platform: ruby
6
+ authors:
7
+ - Elasticsearch
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2014-11-05 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: logstash
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - ! '>='
18
+ - !ruby/object:Gem::Version
19
+ version: 1.4.0
20
+ - - <
21
+ - !ruby/object:Gem::Version
22
+ version: 2.0.0
23
+ type: :runtime
24
+ prerelease: false
25
+ version_requirements: !ruby/object:Gem::Requirement
26
+ requirements:
27
+ - - ! '>='
28
+ - !ruby/object:Gem::Version
29
+ version: 1.4.0
30
+ - - <
31
+ - !ruby/object:Gem::Version
32
+ version: 2.0.0
33
+ - !ruby/object:Gem::Dependency
34
+ name: logstash-codec-json
35
+ requirement: !ruby/object:Gem::Requirement
36
+ requirements:
37
+ - - ! '>='
38
+ - !ruby/object:Gem::Version
39
+ version: '0'
40
+ type: :runtime
41
+ prerelease: false
42
+ version_requirements: !ruby/object:Gem::Requirement
43
+ requirements:
44
+ - - ! '>='
45
+ - !ruby/object:Gem::Version
46
+ version: '0'
47
+ - !ruby/object:Gem::Dependency
48
+ name: aws-sdk
49
+ requirement: !ruby/object:Gem::Requirement
50
+ requirements:
51
+ - - ! '>='
52
+ - !ruby/object:Gem::Version
53
+ version: '0'
54
+ type: :runtime
55
+ prerelease: false
56
+ version_requirements: !ruby/object:Gem::Requirement
57
+ requirements:
58
+ - - ! '>='
59
+ - !ruby/object:Gem::Version
60
+ version: '0'
61
+ description: Pull events from an Amazon Web Services Simple Queue Service (SQS) queue.
62
+ email: richard.pijnenburg@elasticsearch.com
63
+ executables: []
64
+ extensions: []
65
+ extra_rdoc_files: []
66
+ files:
67
+ - .gitignore
68
+ - Gemfile
69
+ - LICENSE
70
+ - Rakefile
71
+ - lib/logstash/inputs/sqs.rb
72
+ - logstash-input-sqs.gemspec
73
+ - rakelib/publish.rake
74
+ - rakelib/vendor.rake
75
+ - spec/inputs/sqs_spec.rb
76
+ homepage: http://logstash.net/
77
+ licenses:
78
+ - Apache License (2.0)
79
+ metadata:
80
+ logstash_plugin: 'true'
81
+ group: input
82
+ post_install_message:
83
+ rdoc_options: []
84
+ require_paths:
85
+ - lib
86
+ required_ruby_version: !ruby/object:Gem::Requirement
87
+ requirements:
88
+ - - ! '>='
89
+ - !ruby/object:Gem::Version
90
+ version: '0'
91
+ required_rubygems_version: !ruby/object:Gem::Requirement
92
+ requirements:
93
+ - - ! '>='
94
+ - !ruby/object:Gem::Version
95
+ version: '0'
96
+ requirements: []
97
+ rubyforge_project:
98
+ rubygems_version: 2.4.1
99
+ signing_key:
100
+ specification_version: 4
101
+ summary: Pull events from an Amazon Web Services Simple Queue Service (SQS) queue.
102
+ test_files:
103
+ - spec/inputs/sqs_spec.rb