logstash-input-s3-sns-sqs 1.4.2 → 1.4.3

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: ed1788afadf83807b126ee90abb0bc0d07852bdf
4
- data.tar.gz: 64c1c4a0d088339c66e234b68607191e4a733115
3
+ metadata.gz: cb572034a924e029ae569b3748f0ccebeb640c44
4
+ data.tar.gz: 8108bfac0091dbd9e9c213dc43743f2e626301ba
5
5
  SHA512:
6
- metadata.gz: 086ea149de024ca358c9ca48b1a1d1e557dea44f550c428d42ff05c4ee6c5d263e9ff9c534f00a90e9e58738cb4dd6c02f8a65d831d46d7e980aba5b11df85bd
7
- data.tar.gz: f6d21b1564c83997d20ef60a05be2f28efa9e90c29a63acea5d93073afef24ed732457300dd59dbf5843460ef3976155fd68648bf5630472256470240c6be175
6
+ metadata.gz: 1631222559bfb47c5b85a48b4ad6f227ea3e7b0d77bb9f8dbb78f3d5fb185edf2cc8b205848d9a3978fc408ea0d3b427d03fccb537e5ff148914581b4a87530e
7
+ data.tar.gz: 21b616763c272dbbb005c89a81a1b48316aba0c02b65aadab38c81ea8f27c51838fdb5b475be51fecbb48ccd496ec40e35d00523b8c8dc770ab500b4620fb12c
data/CHANGELOG.md CHANGED
@@ -1,3 +1,8 @@
1
+ ## 1.4.3
2
+ - Fix: skip_delete on S3::Errors::AccessDenied
3
+ - Feature: codec per s3 folder
4
+ - Feature: Alpha phase: different credentials for s3 / default credentials for sqs
5
+ - Feature: Find files folder.
1
6
  ## 1.4.2
2
7
  - Fix: Thread shutdown method should kill in case of wakeup fails
3
8
  ## 1.4.1
@@ -104,8 +104,13 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
104
104
  # Name of the SQS Queue to pull messages from. Note that this is just the name of the queue, not the URL or ARN.
105
105
  config :queue, :validate => :string, :required => true
106
106
  config :s3_key_prefix, :validate => :string, :default => ''
107
+ #Sometimes you need another key for s3. This is a first test...
108
+ config :s3_access_key_id, :validate => :string
109
+ config :s3_secret_access_key, :validate => :string
107
110
  config :queue_owner_aws_account_id, :validate => :string, :required => false
108
- # Whether to delete files from S3 after processing.
111
+ #If you have different file-types in you s3 bucket, you could define codec by folder
112
+ #set_codec_by_folder => {"My-ELB-logs" => "plain"}
113
+ config :set_codec_by_folder, :validate => :hash, :default => {}
109
114
  config :delete_on_success, :validate => :boolean, :default => false
110
115
  # Whether the event is processed though an SNS to SQS. (S3>SNS>SQS = true |S3>SQS=false)
111
116
  config :from_sns, :validate => :boolean, :default => true
@@ -117,18 +122,10 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
117
122
  attr_reader :poller
118
123
  attr_reader :s3
119
124
 
120
- def suggest_codec(content_type, key)
121
- require "logstash/codecs/plain"
122
- require "logstash/codecs/line"
123
- require "logstash/codecs/json"
124
- require "logstash/codecs/json_lines"
125
- if content_type == "application/json_lines" then
126
- @logger.info("Automatically switching from #{@codec.class.config_name} to json_lines codec", :plugin => self.class.config_name)
127
- @codec = LogStash::Codecs::JSONLines.new("charset" => @codec.charset)
128
- elsif content_type == "application/json" or key.end_with?(".json") then
129
- @logger.info("Automatically switching from #{@codec.class.config_name} to json codec", :plugin => self.class.config_name)
130
- @codec = LogStash::Codecs::JSON.new("charset" => @codec.charset)
131
- end
125
+
126
+ def set_codec (folder)
127
+ @logger.debug("Automatically switching from #{@codec.class.config_name} to #{set_codec_by_folder[folder]} codec", :plugin => self.class.config_name)
128
+ LogStash::Plugin.lookup("codec", "#{set_codec_by_folder[folder]}").new("charset" => @codec.charset)
132
129
  end
133
130
 
134
131
  public
@@ -145,14 +142,22 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
145
142
  end
146
143
 
147
144
  def setup_queue
148
- aws_sqs_client = Aws::SQS::Client.new(aws_options_hash)
149
- queue_url = aws_sqs_client.get_queue_url({ queue_name: @queue, queue_owner_aws_account_id: @queue_owner_aws_account_id})[:queue_url]
150
- @poller = Aws::SQS::QueuePoller.new(queue_url, :client => aws_sqs_client)
151
- @s3_client = Aws::S3::Client.new(aws_options_hash)
152
- @s3_resource = get_s3object
153
- rescue Aws::SQS::Errors::ServiceError => e
154
- @logger.error("Cannot establish connection to Amazon SQS", :error => e)
155
- raise LogStash::ConfigurationError, "Verify the SQS queue name and your credentials"
145
+ begin
146
+ aws_sqs_client = Aws::SQS::Client.new(aws_options_hash)
147
+ queue_url = aws_sqs_client.get_queue_url({ queue_name: @queue, queue_owner_aws_account_id: @queue_owner_aws_account_id})[:queue_url]
148
+ @poller = Aws::SQS::QueuePoller.new(queue_url, :client => aws_sqs_client)
149
+ if s3_access_key_id and s3_secret_access_key
150
+ @logger.debug("Using S3 Credentials from config", :ID => aws_options_hash.merge(:access_key_id => s3_access_key_id, :secret_access_key => s3_secret_access_key) )
151
+ @s3_client = Aws::S3::Client.new(aws_options_hash.merge(:access_key_id => s3_access_key_id, :secret_access_key => s3_secret_access_key))
152
+ else
153
+ @s3_client = Aws::S3::Client.new(aws_options_hash)
154
+ end
155
+
156
+ @s3_resource = get_s3object
157
+ rescue Aws::SQS::Errors::ServiceError => e
158
+ @logger.error("Cannot establish connection to Amazon SQS", :error => e)
159
+ raise LogStash::ConfigurationError, "Verify the SQS queue name and your credentials"
160
+ end
156
161
  end
157
162
 
158
163
  def polling_options
@@ -187,10 +192,16 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
187
192
  @logger.debug("It is a valid record")
188
193
  bucket = CGI.unescape(record['s3']['bucket']['name'])
189
194
  key = CGI.unescape(record['s3']['object']['key'])
190
-
195
+ type_folder = get_object_folder(key)
196
+ # Set input codec by :set_codec_by_folder
197
+ begin
198
+ instance_codec = set_codec(type_folder) unless set_codec_by_folder["#{type_folder}"].nil?
199
+ rescue
200
+ @logger.error("Failed to set_codec with error", :error => e)
201
+ end
191
202
  # try download and :skip_delete if it fails
192
203
  #if record['s3']['object']['size'] < 10000000 then
193
- process_log(bucket, key, instance_codec, queue)
204
+ process_log(bucket, key, type_folder, instance_codec, queue)
194
205
  #else
195
206
  # @logger.info("Your file is too big")
196
207
  #end
@@ -200,13 +211,13 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
200
211
  end
201
212
 
202
213
  private
203
- def process_log(bucket , key, instance_codec, queue)
214
+ def process_log(bucket , key, folder, instance_codec, queue)
204
215
  s3bucket = @s3_resource.bucket(bucket)
205
216
  @logger.debug("Lets go reading file", :bucket => bucket, :key => key)
206
217
  object = s3bucket.object(key)
207
218
  filename = File.join(temporary_directory, File.basename(key))
208
219
  if download_remote_file(object, filename)
209
- if process_local_log( filename, key, instance_codec, queue, bucket)
220
+ if process_local_log( filename, key, folder, instance_codec, queue, bucket)
210
221
  delete_file_from_bucket(object)
211
222
  FileUtils.remove_entry_secure(filename, true)
212
223
  end
@@ -226,7 +237,12 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
226
237
  @logger.debug("S3 input: Download remote file", :remote_key => remote_object.key, :local_filename => local_filename)
227
238
  File.open(local_filename, 'wb') do |s3file|
228
239
  return completed if stop?
229
- remote_object.get(:response_target => s3file)
240
+ begin
241
+ remote_object.get(:response_target => s3file)
242
+ rescue Aws::S3::Errors::AccessDenied => e
243
+ @logger.debug("Unable to download file. We´ll requeue the message", :file => remote_object.inspect)
244
+ throw :skip_delete
245
+ end
230
246
  end
231
247
  completed = true
232
248
 
@@ -240,7 +256,7 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
240
256
  # @param [Queue] Where to push the event
241
257
  # @param [String] Which file to read from
242
258
  # @return [Boolean] True if the file was completely read, false otherwise.
243
- def process_local_log(filename, key, instance_codec, queue, bucket)
259
+ def process_local_log(filename, key, folder, instance_codec, queue, bucket)
244
260
  @logger.debug('Processing file', :filename => filename)
245
261
  metadata = {}
246
262
  # Currently codecs operates on bytes instead of stream.
@@ -264,7 +280,7 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
264
280
  # The line need to go through the codecs to replace
265
281
  # unknown bytes in the log stream before doing a regexp match or
266
282
  # you will get a `Error: invalid byte sequence in UTF-8'
267
- local_decorate(event, key, metadata, bucket)
283
+ local_decorate(event, key, folder, metadata, bucket)
268
284
  queue << event
269
285
  end
270
286
  end
@@ -272,7 +288,7 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
272
288
  #@logger.info("event pre flush", :event => event)
273
289
  # #ensure any stateful codecs (such as multi-line ) are flushed to the queue
274
290
  instance_codec.flush do |event|
275
- local_decorate(event, key, metadata, bucket)
291
+ local_decorate(event, key, folder, metadata, bucket)
276
292
  @logger.debug("We´e to flush an incomplete event...", :event => event)
277
293
  queue << event
278
294
  end
@@ -281,7 +297,7 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
281
297
  end # def process_local_log
282
298
 
283
299
  private
284
- def local_decorate(event, key, metadata, bucket)
300
+ def local_decorate(event, key, folder, metadata, bucket)
285
301
  if event_is_metadata?(event)
286
302
  @logger.debug('Event is metadata, updating the current cloudfront metadata', :event => event)
287
303
  update_metadata(metadata, event)
@@ -292,14 +308,21 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
292
308
  event.set("cloudfront_version", metadata[:cloudfront_version]) unless metadata[:cloudfront_version].nil?
293
309
  event.set("cloudfront_fields", metadata[:cloudfront_fields]) unless metadata[:cloudfront_fields].nil?
294
310
 
295
- event.set("[@metadata][s3]", { "object_key" => key })
296
- event.set("[@metadata][s3]", { "bucket_name" => bucket })
297
- if match=/#{s3_key_prefix}\/?(?<type_folder>.*?)\/.*/.match(key)
298
- event.set('[@metadata][s3]', {"object_folder" => match['type_folder']})
299
- end
311
+ event.set("[@metadata][s3]", { "object_key" => key })
312
+ event.set("[@metadata][s3]", { "bucket_name" => bucket })
313
+ event.set("[@metadata][s3]", { "object_folder" => folder})
300
314
  end
301
315
  end
302
316
 
317
+
318
+ private
319
+ def get_object_folder(key)
320
+ if match=/#{s3_key_prefix}\/?(?<type_folder>.*?)\/.*/.match(key)
321
+ return match['type_folder']
322
+ else
323
+ return ""
324
+ end
325
+ end
303
326
  private
304
327
  def read_file(filename, &block)
305
328
  if gzip?(filename)
@@ -1,6 +1,6 @@
1
1
  Gem::Specification.new do |s|
2
2
  s.name = 'logstash-input-s3-sns-sqs'
3
- s.version = '1.4.2'
3
+ s.version = '1.4.3'
4
4
  s.licenses = ['Apache License (2.0)']
5
5
  s.summary = "Get logs from AWS s3 buckets as issued by an object-created event via sns -> sqs."
6
6
  s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-input-s3-sns-sqs
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.4.2
4
+ version: 1.4.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Christian Herweg
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2018-03-20 00:00:00.000000000 Z
11
+ date: 2018-03-21 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement