logstash-input-s3sqs 1.0.2 → 1.0.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +10 -0
- data/CONTRIBUTORS +2 -1
- data/lib/logstash/inputs/s3sqs.rb +12 -8
- data/logstash-input-s3sqs.gemspec +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 0bc3f016a0e987f5299a013d717f89f92b36cc66
|
4
|
+
data.tar.gz: 41ff3344f0114535e7bf699ba99f2770889e9e4d
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: cd6ceebadbfe378d7e83c55c1d8912ee5dbf294e9ea1dce5f16be8455797a6aed5d35e1cf78ea8cc2371ce16d3392e5ee2bdacdb05db0600878fe87204ee2acf
|
7
|
+
data.tar.gz: 97ca5b6b89bca7631e74b1377c9cf52526c3bff12aaddd827eda3cf773257dcbccaade6f60fabc394d88c9265bf0a63a7b1ad6da0540cbed5d172086460da9b6
|
data/CHANGELOG.md
CHANGED
@@ -1,6 +1,16 @@
|
|
1
1
|
# 1.0.0
|
2
|
+
|
2
3
|
- Initial Release
|
4
|
+
|
3
5
|
# 1.0.1
|
6
|
+
|
4
7
|
- same (because of screwed up rubygems.org release)
|
8
|
+
|
5
9
|
# 1.0.2
|
10
|
+
|
6
11
|
- fix for broken UTF-8 (so we won't lose a whole s3 log file because of a single invalid line, ruby's split will die on those)
|
12
|
+
|
13
|
+
# 1.0.3
|
14
|
+
|
15
|
+
- added some metadata to the event (bucket and object name as commited by joshuaspence)
|
16
|
+
- also try to unzip files ending with ".gz" (ALB logs are zipped but not marked with proper Content-Encoding)
|
data/CONTRIBUTORS
CHANGED
@@ -2,7 +2,8 @@ The following is a list of people who have contributed ideas, code, bug
|
|
2
2
|
reports, or in general have helped logstash along its way.
|
3
3
|
|
4
4
|
Contributors:
|
5
|
-
*
|
5
|
+
* joshuaspence (event metadata)
|
6
|
+
* Heiko-san (initial contributor)
|
6
7
|
* logstash-input-sqs plugin as code base
|
7
8
|
|
8
9
|
Note: If you've sent us patches, bug reports, or otherwise contributed to
|
@@ -26,7 +26,7 @@ require "logstash/errors"
|
|
26
26
|
# This plugin is meant for high availability setups, in contrast to logstash-input-s3 you can safely
|
27
27
|
# use multiple logstash nodes, since the usage of sqs will ensure that each logfile is processed
|
28
28
|
# only once and no file will get lost on node failure or downscaling for auto-scaling groups.
|
29
|
-
# (You should use a "Message Retention Period" >= 4 days for your sqs to ensure you can survive
|
29
|
+
# (You should use a "Message Retention Period" >= 4 days for your sqs to ensure you can survive
|
30
30
|
# a weekend of faulty log file processing)
|
31
31
|
# The plugin will not delete objects from s3 buckets, so make sure to have a reasonable "Lifecycle"
|
32
32
|
# configured for your buckets, which should keep the files at least "Message Retention Period" days.
|
@@ -37,7 +37,7 @@ require "logstash/errors"
|
|
37
37
|
# (The plugin supports gzipped content if it is marked with "contend-encoding: gzip" as it is the
|
38
38
|
# case for cloudtrail logs)
|
39
39
|
#
|
40
|
-
# The logstash node therefore must have sqs permissions + the permissions to download objects
|
40
|
+
# The logstash node therefore must have sqs permissions + the permissions to download objects
|
41
41
|
# from the s3 buckets that send events to the queue.
|
42
42
|
# (If logstash nodes are running on EC2 you should use a ServerRole to provide permissions)
|
43
43
|
# [source,json]
|
@@ -107,20 +107,20 @@ class LogStash::Inputs::S3SQS < LogStash::Inputs::Threadable
|
|
107
107
|
end
|
108
108
|
|
109
109
|
def polling_options
|
110
|
-
{
|
110
|
+
{
|
111
111
|
# we will query 1 message at a time, so we can ensure correct error handling if we can't download a single file correctly
|
112
112
|
# (we will throw :skip_delete if download size isn't correct to process the event again later
|
113
113
|
# -> set a reasonable "Default Visibility Timeout" for your queue, so that there's enough time to process the log files)
|
114
|
-
:max_number_of_messages => 1,
|
114
|
+
:max_number_of_messages => 1,
|
115
115
|
# we will use the queue's setting, a good value is 10 seconds
|
116
116
|
# (to ensure fast logstash shutdown on the one hand and few api calls on the other hand)
|
117
|
-
:wait_time_seconds => nil,
|
117
|
+
:wait_time_seconds => nil,
|
118
118
|
}
|
119
119
|
end
|
120
120
|
|
121
121
|
def handle_message(message, queue)
|
122
122
|
hash = JSON.parse message.body
|
123
|
-
# there may be test events sent from the s3 bucket which won't contain a Records array,
|
123
|
+
# there may be test events sent from the s3 bucket which won't contain a Records array,
|
124
124
|
# we will skip those events and remove them from queue
|
125
125
|
if hash['Records'] then
|
126
126
|
# typically there will be only 1 record per event, but since it is an array we will
|
@@ -143,7 +143,7 @@ class LogStash::Inputs::S3SQS < LogStash::Inputs::Threadable
|
|
143
143
|
if response.content_length == record['s3']['object']['size'] then
|
144
144
|
body = response.body
|
145
145
|
# if necessary unzip
|
146
|
-
if response.content_encoding == "gzip" then
|
146
|
+
if response.content_encoding == "gzip" or record['s3']['object']['key'].end_with?(".gz") then
|
147
147
|
begin
|
148
148
|
temp = Zlib::GzipReader.new(body)
|
149
149
|
rescue => e
|
@@ -158,6 +158,10 @@ class LogStash::Inputs::S3SQS < LogStash::Inputs::Threadable
|
|
158
158
|
lines.each do |line|
|
159
159
|
@codec.decode(line) do |event|
|
160
160
|
decorate(event)
|
161
|
+
|
162
|
+
event['[@metadata][s3_bucket_name]'] = record['s3']['bucket']['name']
|
163
|
+
event['[@metadata][s3_object_key]'] = record['s3']['object']['key']
|
164
|
+
|
161
165
|
queue << event
|
162
166
|
end
|
163
167
|
end
|
@@ -208,7 +212,7 @@ class LogStash::Inputs::S3SQS < LogStash::Inputs::Threadable
|
|
208
212
|
rescue Aws::SQS::Errors::ServiceError => e
|
209
213
|
@logger.warn("Aws::SQS::Errors::ServiceError ... retrying SQS request with exponential backoff", :queue => @queue, :sleep_time => sleep_time, :error => e)
|
210
214
|
sleep(next_sleep)
|
211
|
-
next_sleep = next_sleep > max_time ? sleep_time : sleep_time * BACKOFF_FACTOR
|
215
|
+
next_sleep = next_sleep > max_time ? sleep_time : sleep_time * BACKOFF_FACTOR
|
212
216
|
retry
|
213
217
|
end
|
214
218
|
end
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
s.name = 'logstash-input-s3sqs'
|
3
|
-
s.version = '1.0.
|
3
|
+
s.version = '1.0.3'
|
4
4
|
s.licenses = ['Apache License (2.0)']
|
5
5
|
s.summary = "Get logs from AWS s3 buckets as issued by an object-created event via sqs."
|
6
6
|
s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-input-s3sqs
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0.
|
4
|
+
version: 1.0.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Heiko Finzel
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-
|
11
|
+
date: 2016-11-29 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|