logstash-input-s3-sns-sqs 1.4.2 → 1.4.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: ed1788afadf83807b126ee90abb0bc0d07852bdf
4
- data.tar.gz: 64c1c4a0d088339c66e234b68607191e4a733115
3
+ metadata.gz: cb572034a924e029ae569b3748f0ccebeb640c44
4
+ data.tar.gz: 8108bfac0091dbd9e9c213dc43743f2e626301ba
5
5
  SHA512:
6
- metadata.gz: 086ea149de024ca358c9ca48b1a1d1e557dea44f550c428d42ff05c4ee6c5d263e9ff9c534f00a90e9e58738cb4dd6c02f8a65d831d46d7e980aba5b11df85bd
7
- data.tar.gz: f6d21b1564c83997d20ef60a05be2f28efa9e90c29a63acea5d93073afef24ed732457300dd59dbf5843460ef3976155fd68648bf5630472256470240c6be175
6
+ metadata.gz: 1631222559bfb47c5b85a48b4ad6f227ea3e7b0d77bb9f8dbb78f3d5fb185edf2cc8b205848d9a3978fc408ea0d3b427d03fccb537e5ff148914581b4a87530e
7
+ data.tar.gz: 21b616763c272dbbb005c89a81a1b48316aba0c02b65aadab38c81ea8f27c51838fdb5b475be51fecbb48ccd496ec40e35d00523b8c8dc770ab500b4620fb12c
data/CHANGELOG.md CHANGED
@@ -1,3 +1,8 @@
1
+ ## 1.4.3
2
+ - Fix: skip_delete on S3::Errors::AccessDenied
3
+ - Feature: codec per s3 folder
4
+ - Feature: Alpha phase: different credentials for s3 / default credentials for sqs
5
+ - Feature: Find files folder.
1
6
  ## 1.4.2
2
7
  - Fix: Thread shutdown method should kill in case of wakeup fails
3
8
  ## 1.4.1
@@ -104,8 +104,13 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
104
104
  # Name of the SQS Queue to pull messages from. Note that this is just the name of the queue, not the URL or ARN.
105
105
  config :queue, :validate => :string, :required => true
106
106
  config :s3_key_prefix, :validate => :string, :default => ''
107
+ #Sometimes you need another key for s3. This is a first test...
108
+ config :s3_access_key_id, :validate => :string
109
+ config :s3_secret_access_key, :validate => :string
107
110
  config :queue_owner_aws_account_id, :validate => :string, :required => false
108
- # Whether to delete files from S3 after processing.
111
+ #If you have different file-types in you s3 bucket, you could define codec by folder
112
+ #set_codec_by_folder => {"My-ELB-logs" => "plain"}
113
+ config :set_codec_by_folder, :validate => :hash, :default => {}
109
114
  config :delete_on_success, :validate => :boolean, :default => false
110
115
  # Whether the event is processed though an SNS to SQS. (S3>SNS>SQS = true |S3>SQS=false)
111
116
  config :from_sns, :validate => :boolean, :default => true
@@ -117,18 +122,10 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
117
122
  attr_reader :poller
118
123
  attr_reader :s3
119
124
 
120
- def suggest_codec(content_type, key)
121
- require "logstash/codecs/plain"
122
- require "logstash/codecs/line"
123
- require "logstash/codecs/json"
124
- require "logstash/codecs/json_lines"
125
- if content_type == "application/json_lines" then
126
- @logger.info("Automatically switching from #{@codec.class.config_name} to json_lines codec", :plugin => self.class.config_name)
127
- @codec = LogStash::Codecs::JSONLines.new("charset" => @codec.charset)
128
- elsif content_type == "application/json" or key.end_with?(".json") then
129
- @logger.info("Automatically switching from #{@codec.class.config_name} to json codec", :plugin => self.class.config_name)
130
- @codec = LogStash::Codecs::JSON.new("charset" => @codec.charset)
131
- end
125
+
126
+ def set_codec (folder)
127
+ @logger.debug("Automatically switching from #{@codec.class.config_name} to #{set_codec_by_folder[folder]} codec", :plugin => self.class.config_name)
128
+ LogStash::Plugin.lookup("codec", "#{set_codec_by_folder[folder]}").new("charset" => @codec.charset)
132
129
  end
133
130
 
134
131
  public
@@ -145,14 +142,22 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
145
142
  end
146
143
 
147
144
  def setup_queue
148
- aws_sqs_client = Aws::SQS::Client.new(aws_options_hash)
149
- queue_url = aws_sqs_client.get_queue_url({ queue_name: @queue, queue_owner_aws_account_id: @queue_owner_aws_account_id})[:queue_url]
150
- @poller = Aws::SQS::QueuePoller.new(queue_url, :client => aws_sqs_client)
151
- @s3_client = Aws::S3::Client.new(aws_options_hash)
152
- @s3_resource = get_s3object
153
- rescue Aws::SQS::Errors::ServiceError => e
154
- @logger.error("Cannot establish connection to Amazon SQS", :error => e)
155
- raise LogStash::ConfigurationError, "Verify the SQS queue name and your credentials"
145
+ begin
146
+ aws_sqs_client = Aws::SQS::Client.new(aws_options_hash)
147
+ queue_url = aws_sqs_client.get_queue_url({ queue_name: @queue, queue_owner_aws_account_id: @queue_owner_aws_account_id})[:queue_url]
148
+ @poller = Aws::SQS::QueuePoller.new(queue_url, :client => aws_sqs_client)
149
+ if s3_access_key_id and s3_secret_access_key
150
+ @logger.debug("Using S3 Credentials from config", :ID => aws_options_hash.merge(:access_key_id => s3_access_key_id, :secret_access_key => s3_secret_access_key) )
151
+ @s3_client = Aws::S3::Client.new(aws_options_hash.merge(:access_key_id => s3_access_key_id, :secret_access_key => s3_secret_access_key))
152
+ else
153
+ @s3_client = Aws::S3::Client.new(aws_options_hash)
154
+ end
155
+
156
+ @s3_resource = get_s3object
157
+ rescue Aws::SQS::Errors::ServiceError => e
158
+ @logger.error("Cannot establish connection to Amazon SQS", :error => e)
159
+ raise LogStash::ConfigurationError, "Verify the SQS queue name and your credentials"
160
+ end
156
161
  end
157
162
 
158
163
  def polling_options
@@ -187,10 +192,16 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
187
192
  @logger.debug("It is a valid record")
188
193
  bucket = CGI.unescape(record['s3']['bucket']['name'])
189
194
  key = CGI.unescape(record['s3']['object']['key'])
190
-
195
+ type_folder = get_object_folder(key)
196
+ # Set input codec by :set_codec_by_folder
197
+ begin
198
+ instance_codec = set_codec(type_folder) unless set_codec_by_folder["#{type_folder}"].nil?
199
+ rescue
200
+ @logger.error("Failed to set_codec with error", :error => e)
201
+ end
191
202
  # try download and :skip_delete if it fails
192
203
  #if record['s3']['object']['size'] < 10000000 then
193
- process_log(bucket, key, instance_codec, queue)
204
+ process_log(bucket, key, type_folder, instance_codec, queue)
194
205
  #else
195
206
  # @logger.info("Your file is too big")
196
207
  #end
@@ -200,13 +211,13 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
200
211
  end
201
212
 
202
213
  private
203
- def process_log(bucket , key, instance_codec, queue)
214
+ def process_log(bucket , key, folder, instance_codec, queue)
204
215
  s3bucket = @s3_resource.bucket(bucket)
205
216
  @logger.debug("Lets go reading file", :bucket => bucket, :key => key)
206
217
  object = s3bucket.object(key)
207
218
  filename = File.join(temporary_directory, File.basename(key))
208
219
  if download_remote_file(object, filename)
209
- if process_local_log( filename, key, instance_codec, queue, bucket)
220
+ if process_local_log( filename, key, folder, instance_codec, queue, bucket)
210
221
  delete_file_from_bucket(object)
211
222
  FileUtils.remove_entry_secure(filename, true)
212
223
  end
@@ -226,7 +237,12 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
226
237
  @logger.debug("S3 input: Download remote file", :remote_key => remote_object.key, :local_filename => local_filename)
227
238
  File.open(local_filename, 'wb') do |s3file|
228
239
  return completed if stop?
229
- remote_object.get(:response_target => s3file)
240
+ begin
241
+ remote_object.get(:response_target => s3file)
242
+ rescue Aws::S3::Errors::AccessDenied => e
243
+ @logger.debug("Unable to download file. We´ll requeue the message", :file => remote_object.inspect)
244
+ throw :skip_delete
245
+ end
230
246
  end
231
247
  completed = true
232
248
 
@@ -240,7 +256,7 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
240
256
  # @param [Queue] Where to push the event
241
257
  # @param [String] Which file to read from
242
258
  # @return [Boolean] True if the file was completely read, false otherwise.
243
- def process_local_log(filename, key, instance_codec, queue, bucket)
259
+ def process_local_log(filename, key, folder, instance_codec, queue, bucket)
244
260
  @logger.debug('Processing file', :filename => filename)
245
261
  metadata = {}
246
262
  # Currently codecs operates on bytes instead of stream.
@@ -264,7 +280,7 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
264
280
  # The line need to go through the codecs to replace
265
281
  # unknown bytes in the log stream before doing a regexp match or
266
282
  # you will get a `Error: invalid byte sequence in UTF-8'
267
- local_decorate(event, key, metadata, bucket)
283
+ local_decorate(event, key, folder, metadata, bucket)
268
284
  queue << event
269
285
  end
270
286
  end
@@ -272,7 +288,7 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
272
288
  #@logger.info("event pre flush", :event => event)
273
289
  # #ensure any stateful codecs (such as multi-line ) are flushed to the queue
274
290
  instance_codec.flush do |event|
275
- local_decorate(event, key, metadata, bucket)
291
+ local_decorate(event, key, folder, metadata, bucket)
276
292
  @logger.debug("We´e to flush an incomplete event...", :event => event)
277
293
  queue << event
278
294
  end
@@ -281,7 +297,7 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
281
297
  end # def process_local_log
282
298
 
283
299
  private
284
- def local_decorate(event, key, metadata, bucket)
300
+ def local_decorate(event, key, folder, metadata, bucket)
285
301
  if event_is_metadata?(event)
286
302
  @logger.debug('Event is metadata, updating the current cloudfront metadata', :event => event)
287
303
  update_metadata(metadata, event)
@@ -292,14 +308,21 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
292
308
  event.set("cloudfront_version", metadata[:cloudfront_version]) unless metadata[:cloudfront_version].nil?
293
309
  event.set("cloudfront_fields", metadata[:cloudfront_fields]) unless metadata[:cloudfront_fields].nil?
294
310
 
295
- event.set("[@metadata][s3]", { "object_key" => key })
296
- event.set("[@metadata][s3]", { "bucket_name" => bucket })
297
- if match=/#{s3_key_prefix}\/?(?<type_folder>.*?)\/.*/.match(key)
298
- event.set('[@metadata][s3]', {"object_folder" => match['type_folder']})
299
- end
311
+ event.set("[@metadata][s3]", { "object_key" => key })
312
+ event.set("[@metadata][s3]", { "bucket_name" => bucket })
313
+ event.set("[@metadata][s3]", { "object_folder" => folder})
300
314
  end
301
315
  end
302
316
 
317
+
318
+ private
319
+ def get_object_folder(key)
320
+ if match=/#{s3_key_prefix}\/?(?<type_folder>.*?)\/.*/.match(key)
321
+ return match['type_folder']
322
+ else
323
+ return ""
324
+ end
325
+ end
303
326
  private
304
327
  def read_file(filename, &block)
305
328
  if gzip?(filename)
@@ -1,6 +1,6 @@
1
1
  Gem::Specification.new do |s|
2
2
  s.name = 'logstash-input-s3-sns-sqs'
3
- s.version = '1.4.2'
3
+ s.version = '1.4.3'
4
4
  s.licenses = ['Apache License (2.0)']
5
5
  s.summary = "Get logs from AWS s3 buckets as issued by an object-created event via sns -> sqs."
6
6
  s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-input-s3-sns-sqs
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.4.2
4
+ version: 1.4.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Christian Herweg
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2018-03-20 00:00:00.000000000 Z
11
+ date: 2018-03-21 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement