logstash-input-s3-sns-sqs 1.1.8 → 1.1.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +2 -0
- data/lib/logstash/inputs/s3snssqs.rb +11 -1
- data/logstash-input-s3-sns-sqs.gemspec +1 -1
- metadata +1 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 4847b8327b6f423cd1c9c547651cd90494a95ea9
|
|
4
|
+
data.tar.gz: cdba3fa9f2923e14d92a8eab686a522f98797198
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 5635694cc34a510b8e2b4ea6f600839f227d2029b415a6e0d1d240c6a27c4e8d4b35bff8d88a78a764d87afa03c9d02e290e9ef1090d27974d38778ec5ed7a9f
|
|
7
|
+
data.tar.gz: 2bd71a99eeeb872c4b485c1ee1ee08dcb7470b859b5880f895595b78ff9496f1c6ebef75cf3158abd12e642f2bf29cdfce5e7116036d748f1a116e29a5f2f779
|
data/CHANGELOG.md
CHANGED
|
@@ -90,6 +90,7 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
|
|
|
90
90
|
|
|
91
91
|
# Name of the SQS Queue to pull messages from. Note that this is just the name of the queue, not the URL or ARN.
|
|
92
92
|
config :queue, :validate => :string, :required => true
|
|
93
|
+
config :s3_key_prefix, :validate => :string, :default => ''
|
|
93
94
|
config :queue_owner_aws_account_id, :validate => :string, :required => false
|
|
94
95
|
# Whether to delete files from S3 after processing.
|
|
95
96
|
config :delete_on_success, :validate => :boolean, :default => false
|
|
@@ -149,6 +150,10 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
|
|
|
149
150
|
bucket = CGI.unescape(record['s3']['bucket']['name'])
|
|
150
151
|
key = CGI.unescape(record['s3']['object']['key'])
|
|
151
152
|
|
|
153
|
+
if match=/\/#{s3_key_prefix}\/(?<type_folder>.*?)\/.*/.match(key)
|
|
154
|
+
event.set(s3_event_type, match['type_folder'])
|
|
155
|
+
end
|
|
156
|
+
|
|
152
157
|
# try download and :skip_delete if it fails
|
|
153
158
|
begin
|
|
154
159
|
response = @s3.get_object(
|
|
@@ -174,6 +179,12 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
|
|
|
174
179
|
end
|
|
175
180
|
body = temp
|
|
176
181
|
end
|
|
182
|
+
if @codec == "auto" and (response.content_type == "application/json" or record['s3']['object']['key'].end_with?(".json")) then
|
|
183
|
+
@logger.debug("This is an json files you should switch codec to json")
|
|
184
|
+
@codec = "json"
|
|
185
|
+
else
|
|
186
|
+
@codec = "plain"
|
|
187
|
+
end
|
|
177
188
|
# process the plain text content
|
|
178
189
|
begin
|
|
179
190
|
lines = body.read.encode('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: "\u2370").split(/\n/)
|
|
@@ -208,7 +219,6 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
|
|
|
208
219
|
end
|
|
209
220
|
end
|
|
210
221
|
end
|
|
211
|
-
end
|
|
212
222
|
end
|
|
213
223
|
|
|
214
224
|
def run(queue)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Gem::Specification.new do |s|
|
|
2
2
|
s.name = 'logstash-input-s3-sns-sqs'
|
|
3
|
-
s.version = '1.1.
|
|
3
|
+
s.version = '1.1.9'
|
|
4
4
|
s.licenses = ['Apache License (2.0)']
|
|
5
5
|
s.summary = "Get logs from AWS s3 buckets as issued by an object-created event via sns -> sqs."
|
|
6
6
|
s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
|