logstash-input-sqs_s3 1.1.11 → 1.1.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +3 -0
- data/lib/logstash/inputs/sqs_s3.rb +11 -3
- data/logstash-input-sqs_s3.gemspec +1 -1
- metadata +1 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: d9a94aef8c03a9df8145522e48b48942a6fb355675cbb244e54d07b4bfc78fb7
|
|
4
|
+
data.tar.gz: ed70e0cb422c557900aa59b507527c5fdc59e34e4d06c45ac379906d9dd3e1e3
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: c5029542e78e3671d4a29f152f719fb651f184a774cd2608933e65310c03fd5a4977e549bfeb68920b68b6cb1de32b952743e1e76196912d8da516bcd389dd01
|
|
7
|
+
data.tar.gz: 1f5deb0bc501959d6c6fa7220ac075c1dcb8a71ee89e938df54d50baea2bd5511f6679291fb99a175eed052621d0569820dd5c6e1e47a5f14a3b60c507f2d39c
|
data/CHANGELOG.md
CHANGED
|
@@ -127,6 +127,8 @@ class LogStash::Inputs::SQSS3 < LogStash::Inputs::Threadable
|
|
|
127
127
|
@logger.info("Registering SQS input", :queue => @queue)
|
|
128
128
|
@logger.info("Skip Delete", :skip_delete => @skip_delete)
|
|
129
129
|
@current_load = 0.0
|
|
130
|
+
@jsonCodec = LogStash::Codecs::JSON.new
|
|
131
|
+
@plainCodec = LogStash::Codecs::Plain.new
|
|
130
132
|
setup_queue
|
|
131
133
|
end
|
|
132
134
|
|
|
@@ -191,6 +193,13 @@ class LogStash::Inputs::SQSS3 < LogStash::Inputs::Threadable
|
|
|
191
193
|
# assess currently running load (in MB)
|
|
192
194
|
@current_load += (record['s3']['object']['size'].to_f / 1000000)
|
|
193
195
|
|
|
196
|
+
# Set the codec to json if required, otherwise the default is plain text
|
|
197
|
+
if response.content_type == "application/json" then
|
|
198
|
+
@codec = @jsonCodec
|
|
199
|
+
else
|
|
200
|
+
@codec = @plainCodec
|
|
201
|
+
end
|
|
202
|
+
|
|
194
203
|
lines = body.read.encode('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: "\u2370").split(/\n/)
|
|
195
204
|
lines.each do |line|
|
|
196
205
|
@codec.decode(line) do |event|
|
|
@@ -240,9 +249,8 @@ class LogStash::Inputs::SQSS3 < LogStash::Inputs::Threadable
|
|
|
240
249
|
|
|
241
250
|
# Throttle requests is overloaded by big files
|
|
242
251
|
if @current_load > @max_load_before_throttling/1000000 then
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
throttle_seconds_sleep = @seconds_to_throttle * (@current_load / (@max_load_before_throttling.to_f/1000000)).floor
|
|
252
|
+
throttle_seconds_sleep = @seconds_to_throttle * (@current_load / (@max_load_before_throttling.to_f/1000000)).floor
|
|
253
|
+
@logger.warn("**********Current load has exceeded " + (@max_load_before_throttling.to_f/1000000).to_s + " MB. Load is currently: " + @current_load.to_s + ". Throttling back by " + throttle_seconds_sleep
|
|
246
254
|
|
|
247
255
|
if(throttle_seconds_sleep != 0) then
|
|
248
256
|
sleep(throttle_seconds_sleep)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Gem::Specification.new do |s|
|
|
2
2
|
s.name = 'logstash-input-sqs_s3'
|
|
3
|
-
s.version = '1.1.
|
|
3
|
+
s.version = '1.1.12'
|
|
4
4
|
s.licenses = ['Apache License (2.0)']
|
|
5
5
|
s.summary = "Get logs from AWS s3 buckets as issued by an object-created event via sqs."
|
|
6
6
|
s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program. Full credit goes to Heiko Finzel. Republishing this gem to support Logstash 5."
|