logstash-input-s3-sns-sqs 1.4.2 → 1.4.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +5 -0
- data/lib/logstash/inputs/s3snssqs.rb +58 -35
- data/logstash-input-s3-sns-sqs.gemspec +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: cb572034a924e029ae569b3748f0ccebeb640c44
|
4
|
+
data.tar.gz: 8108bfac0091dbd9e9c213dc43743f2e626301ba
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 1631222559bfb47c5b85a48b4ad6f227ea3e7b0d77bb9f8dbb78f3d5fb185edf2cc8b205848d9a3978fc408ea0d3b427d03fccb537e5ff148914581b4a87530e
|
7
|
+
data.tar.gz: 21b616763c272dbbb005c89a81a1b48316aba0c02b65aadab38c81ea8f27c51838fdb5b475be51fecbb48ccd496ec40e35d00523b8c8dc770ab500b4620fb12c
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,8 @@
|
|
1
|
+
## 1.4.3
|
2
|
+
- Fix: skip_delete on S3::Errors::AccessDenied
|
3
|
+
- Feature: codec per s3 folder
|
4
|
+
- Feature: Alpha phase: different credentials for s3 / default credentials for sqs
|
5
|
+
- Feature: Find files folder.
|
1
6
|
## 1.4.2
|
2
7
|
- Fix: Thread shutdown method should kill in case of wakeup fails
|
3
8
|
## 1.4.1
|
@@ -104,8 +104,13 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
|
|
104
104
|
# Name of the SQS Queue to pull messages from. Note that this is just the name of the queue, not the URL or ARN.
|
105
105
|
config :queue, :validate => :string, :required => true
|
106
106
|
config :s3_key_prefix, :validate => :string, :default => ''
|
107
|
+
#Sometimes you need another key for s3. This is a first test...
|
108
|
+
config :s3_access_key_id, :validate => :string
|
109
|
+
config :s3_secret_access_key, :validate => :string
|
107
110
|
config :queue_owner_aws_account_id, :validate => :string, :required => false
|
108
|
-
#
|
111
|
+
#If you have different file-types in you s3 bucket, you could define codec by folder
|
112
|
+
#set_codec_by_folder => {"My-ELB-logs" => "plain"}
|
113
|
+
config :set_codec_by_folder, :validate => :hash, :default => {}
|
109
114
|
config :delete_on_success, :validate => :boolean, :default => false
|
110
115
|
# Whether the event is processed though an SNS to SQS. (S3>SNS>SQS = true |S3>SQS=false)
|
111
116
|
config :from_sns, :validate => :boolean, :default => true
|
@@ -117,18 +122,10 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
|
|
117
122
|
attr_reader :poller
|
118
123
|
attr_reader :s3
|
119
124
|
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
require "logstash/codecs/json_lines"
|
125
|
-
if content_type == "application/json_lines" then
|
126
|
-
@logger.info("Automatically switching from #{@codec.class.config_name} to json_lines codec", :plugin => self.class.config_name)
|
127
|
-
@codec = LogStash::Codecs::JSONLines.new("charset" => @codec.charset)
|
128
|
-
elsif content_type == "application/json" or key.end_with?(".json") then
|
129
|
-
@logger.info("Automatically switching from #{@codec.class.config_name} to json codec", :plugin => self.class.config_name)
|
130
|
-
@codec = LogStash::Codecs::JSON.new("charset" => @codec.charset)
|
131
|
-
end
|
125
|
+
|
126
|
+
def set_codec (folder)
|
127
|
+
@logger.debug("Automatically switching from #{@codec.class.config_name} to #{set_codec_by_folder[folder]} codec", :plugin => self.class.config_name)
|
128
|
+
LogStash::Plugin.lookup("codec", "#{set_codec_by_folder[folder]}").new("charset" => @codec.charset)
|
132
129
|
end
|
133
130
|
|
134
131
|
public
|
@@ -145,14 +142,22 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
|
|
145
142
|
end
|
146
143
|
|
147
144
|
def setup_queue
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
|
145
|
+
begin
|
146
|
+
aws_sqs_client = Aws::SQS::Client.new(aws_options_hash)
|
147
|
+
queue_url = aws_sqs_client.get_queue_url({ queue_name: @queue, queue_owner_aws_account_id: @queue_owner_aws_account_id})[:queue_url]
|
148
|
+
@poller = Aws::SQS::QueuePoller.new(queue_url, :client => aws_sqs_client)
|
149
|
+
if s3_access_key_id and s3_secret_access_key
|
150
|
+
@logger.debug("Using S3 Credentials from config", :ID => aws_options_hash.merge(:access_key_id => s3_access_key_id, :secret_access_key => s3_secret_access_key) )
|
151
|
+
@s3_client = Aws::S3::Client.new(aws_options_hash.merge(:access_key_id => s3_access_key_id, :secret_access_key => s3_secret_access_key))
|
152
|
+
else
|
153
|
+
@s3_client = Aws::S3::Client.new(aws_options_hash)
|
154
|
+
end
|
155
|
+
|
156
|
+
@s3_resource = get_s3object
|
157
|
+
rescue Aws::SQS::Errors::ServiceError => e
|
158
|
+
@logger.error("Cannot establish connection to Amazon SQS", :error => e)
|
159
|
+
raise LogStash::ConfigurationError, "Verify the SQS queue name and your credentials"
|
160
|
+
end
|
156
161
|
end
|
157
162
|
|
158
163
|
def polling_options
|
@@ -187,10 +192,16 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
|
|
187
192
|
@logger.debug("It is a valid record")
|
188
193
|
bucket = CGI.unescape(record['s3']['bucket']['name'])
|
189
194
|
key = CGI.unescape(record['s3']['object']['key'])
|
190
|
-
|
195
|
+
type_folder = get_object_folder(key)
|
196
|
+
# Set input codec by :set_codec_by_folder
|
197
|
+
begin
|
198
|
+
instance_codec = set_codec(type_folder) unless set_codec_by_folder["#{type_folder}"].nil?
|
199
|
+
rescue
|
200
|
+
@logger.error("Failed to set_codec with error", :error => e)
|
201
|
+
end
|
191
202
|
# try download and :skip_delete if it fails
|
192
203
|
#if record['s3']['object']['size'] < 10000000 then
|
193
|
-
process_log(bucket, key, instance_codec, queue)
|
204
|
+
process_log(bucket, key, type_folder, instance_codec, queue)
|
194
205
|
#else
|
195
206
|
# @logger.info("Your file is too big")
|
196
207
|
#end
|
@@ -200,13 +211,13 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
|
|
200
211
|
end
|
201
212
|
|
202
213
|
private
|
203
|
-
def process_log(bucket , key, instance_codec, queue)
|
214
|
+
def process_log(bucket , key, folder, instance_codec, queue)
|
204
215
|
s3bucket = @s3_resource.bucket(bucket)
|
205
216
|
@logger.debug("Lets go reading file", :bucket => bucket, :key => key)
|
206
217
|
object = s3bucket.object(key)
|
207
218
|
filename = File.join(temporary_directory, File.basename(key))
|
208
219
|
if download_remote_file(object, filename)
|
209
|
-
if process_local_log( filename, key, instance_codec, queue, bucket)
|
220
|
+
if process_local_log( filename, key, folder, instance_codec, queue, bucket)
|
210
221
|
delete_file_from_bucket(object)
|
211
222
|
FileUtils.remove_entry_secure(filename, true)
|
212
223
|
end
|
@@ -226,7 +237,12 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
|
|
226
237
|
@logger.debug("S3 input: Download remote file", :remote_key => remote_object.key, :local_filename => local_filename)
|
227
238
|
File.open(local_filename, 'wb') do |s3file|
|
228
239
|
return completed if stop?
|
229
|
-
|
240
|
+
begin
|
241
|
+
remote_object.get(:response_target => s3file)
|
242
|
+
rescue Aws::S3::Errors::AccessDenied => e
|
243
|
+
@logger.debug("Unable to download file. We´ll requeue the message", :file => remote_object.inspect)
|
244
|
+
throw :skip_delete
|
245
|
+
end
|
230
246
|
end
|
231
247
|
completed = true
|
232
248
|
|
@@ -240,7 +256,7 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
|
|
240
256
|
# @param [Queue] Where to push the event
|
241
257
|
# @param [String] Which file to read from
|
242
258
|
# @return [Boolean] True if the file was completely read, false otherwise.
|
243
|
-
def process_local_log(filename, key, instance_codec, queue, bucket)
|
259
|
+
def process_local_log(filename, key, folder, instance_codec, queue, bucket)
|
244
260
|
@logger.debug('Processing file', :filename => filename)
|
245
261
|
metadata = {}
|
246
262
|
# Currently codecs operates on bytes instead of stream.
|
@@ -264,7 +280,7 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
|
|
264
280
|
# The line need to go through the codecs to replace
|
265
281
|
# unknown bytes in the log stream before doing a regexp match or
|
266
282
|
# you will get a `Error: invalid byte sequence in UTF-8'
|
267
|
-
local_decorate(event, key, metadata, bucket)
|
283
|
+
local_decorate(event, key, folder, metadata, bucket)
|
268
284
|
queue << event
|
269
285
|
end
|
270
286
|
end
|
@@ -272,7 +288,7 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
|
|
272
288
|
#@logger.info("event pre flush", :event => event)
|
273
289
|
# #ensure any stateful codecs (such as multi-line ) are flushed to the queue
|
274
290
|
instance_codec.flush do |event|
|
275
|
-
local_decorate(event, key, metadata, bucket)
|
291
|
+
local_decorate(event, key, folder, metadata, bucket)
|
276
292
|
@logger.debug("We´e to flush an incomplete event...", :event => event)
|
277
293
|
queue << event
|
278
294
|
end
|
@@ -281,7 +297,7 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
|
|
281
297
|
end # def process_local_log
|
282
298
|
|
283
299
|
private
|
284
|
-
def local_decorate(event, key, metadata, bucket)
|
300
|
+
def local_decorate(event, key, folder, metadata, bucket)
|
285
301
|
if event_is_metadata?(event)
|
286
302
|
@logger.debug('Event is metadata, updating the current cloudfront metadata', :event => event)
|
287
303
|
update_metadata(metadata, event)
|
@@ -292,14 +308,21 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
|
|
292
308
|
event.set("cloudfront_version", metadata[:cloudfront_version]) unless metadata[:cloudfront_version].nil?
|
293
309
|
event.set("cloudfront_fields", metadata[:cloudfront_fields]) unless metadata[:cloudfront_fields].nil?
|
294
310
|
|
295
|
-
event.set("[@metadata][s3]", { "object_key"
|
296
|
-
event.set("[@metadata][s3]", { "bucket_name"
|
297
|
-
|
298
|
-
event.set('[@metadata][s3]', {"object_folder" => match['type_folder']})
|
299
|
-
end
|
311
|
+
event.set("[@metadata][s3]", { "object_key" => key })
|
312
|
+
event.set("[@metadata][s3]", { "bucket_name" => bucket })
|
313
|
+
event.set("[@metadata][s3]", { "object_folder" => folder})
|
300
314
|
end
|
301
315
|
end
|
302
316
|
|
317
|
+
|
318
|
+
private
|
319
|
+
def get_object_folder(key)
|
320
|
+
if match=/#{s3_key_prefix}\/?(?<type_folder>.*?)\/.*/.match(key)
|
321
|
+
return match['type_folder']
|
322
|
+
else
|
323
|
+
return ""
|
324
|
+
end
|
325
|
+
end
|
303
326
|
private
|
304
327
|
def read_file(filename, &block)
|
305
328
|
if gzip?(filename)
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
s.name = 'logstash-input-s3-sns-sqs'
|
3
|
-
s.version = '1.4.
|
3
|
+
s.version = '1.4.3'
|
4
4
|
s.licenses = ['Apache License (2.0)']
|
5
5
|
s.summary = "Get logs from AWS s3 buckets as issued by an object-created event via sns -> sqs."
|
6
6
|
s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-input-s3-sns-sqs
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.4.
|
4
|
+
version: 1.4.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Christian Herweg
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-03-
|
11
|
+
date: 2018-03-21 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|