logstash-input-s3-sns-sqs 1.4.8 → 1.4.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +2 -0
- data/lib/logstash/inputs/mime/mimemagic/overlay.rb +7 -0
- data/lib/logstash/inputs/mime/mimemagic/tables.rb +1580 -0
- data/lib/logstash/inputs/mime/mimemagic/version.rb +5 -0
- data/lib/logstash/inputs/mime/mimemagic.rb +139 -0
- data/lib/logstash/inputs/s3snssqs.rb +13 -7
- data/logstash-input-s3-sns-sqs.gemspec +1 -2
- metadata +6 -2
@@ -0,0 +1,139 @@
|
|
1
|
+
require 'logstash/inputs/mime/mimemagic/tables'
|
2
|
+
require 'logstash/inputs/mime/mimemagic/version'
|
3
|
+
|
4
|
+
require 'stringio'
|
5
|
+
|
6
|
+
# Mime type detection
|
7
|
+
class MimeMagic
|
8
|
+
attr_reader :type, :mediatype, :subtype
|
9
|
+
|
10
|
+
# Mime type by type string
|
11
|
+
def initialize(type)
|
12
|
+
@type = type
|
13
|
+
@mediatype, @subtype = type.split('/', 2)
|
14
|
+
end
|
15
|
+
|
16
|
+
# Add custom mime type. Arguments:
|
17
|
+
# * <i>type</i>: Mime type
|
18
|
+
# * <i>options</i>: Options hash
|
19
|
+
#
|
20
|
+
# Option keys:
|
21
|
+
# * <i>:extensions</i>: String list or single string of file extensions
|
22
|
+
# * <i>:parents</i>: String list or single string of parent mime types
|
23
|
+
# * <i>:magic</i>: Mime magic specification
|
24
|
+
# * <i>:comment</i>: Comment string
|
25
|
+
def self.add(type, options)
|
26
|
+
extensions = [options[:extensions]].flatten.compact
|
27
|
+
TYPES[type] = [extensions,
|
28
|
+
[options[:parents]].flatten.compact,
|
29
|
+
options[:comment]]
|
30
|
+
extensions.each {|ext| EXTENSIONS[ext] = type }
|
31
|
+
MAGIC.unshift [type, options[:magic]] if options[:magic]
|
32
|
+
end
|
33
|
+
|
34
|
+
# Removes a mime type from the dictionary. You might want to do this if
|
35
|
+
# you're seeing impossible conflicts (for instance, application/x-gmc-link).
|
36
|
+
# * <i>type</i>: The mime type to remove. All associated extensions and magic are removed too.
|
37
|
+
def self.remove(type)
|
38
|
+
EXTENSIONS.delete_if {|ext, t| t == type }
|
39
|
+
MAGIC.delete_if {|t, m| t == type }
|
40
|
+
TYPES.delete(type)
|
41
|
+
end
|
42
|
+
|
43
|
+
# Returns true if type is a text format
|
44
|
+
def text?; mediatype == 'text' || child_of?('text/plain'); end
|
45
|
+
|
46
|
+
# Mediatype shortcuts
|
47
|
+
def image?; mediatype == 'image'; end
|
48
|
+
def audio?; mediatype == 'audio'; end
|
49
|
+
def video?; mediatype == 'video'; end
|
50
|
+
|
51
|
+
# Returns true if type is child of parent type
|
52
|
+
def child_of?(parent)
|
53
|
+
MimeMagic.child?(type, parent)
|
54
|
+
end
|
55
|
+
|
56
|
+
# Get string list of file extensions
|
57
|
+
def extensions
|
58
|
+
TYPES.key?(type) ? TYPES[type][0] : []
|
59
|
+
end
|
60
|
+
|
61
|
+
# Get mime comment
|
62
|
+
def comment
|
63
|
+
(TYPES.key?(type) ? TYPES[type][2] : nil).to_s
|
64
|
+
end
|
65
|
+
|
66
|
+
# Lookup mime type by file extension
|
67
|
+
def self.by_extension(ext)
|
68
|
+
ext = ext.to_s.downcase
|
69
|
+
mime = ext[0..0] == '.' ? EXTENSIONS[ext[1..-1]] : EXTENSIONS[ext]
|
70
|
+
mime && new(mime)
|
71
|
+
end
|
72
|
+
|
73
|
+
# Lookup mime type by filename
|
74
|
+
def self.by_path(path)
|
75
|
+
by_extension(File.extname(path))
|
76
|
+
end
|
77
|
+
|
78
|
+
# Lookup mime type by magic content analysis.
|
79
|
+
# This is a slow operation.
|
80
|
+
def self.by_magic(io)
|
81
|
+
mime = magic_match(io, :find)
|
82
|
+
mime && new(mime[0])
|
83
|
+
end
|
84
|
+
|
85
|
+
# Lookup all mime types by magic content analysis.
|
86
|
+
# This is a slower operation.
|
87
|
+
def self.all_by_magic(io)
|
88
|
+
magic_match(io, :select).map { |mime| new(mime[0]) }
|
89
|
+
end
|
90
|
+
|
91
|
+
# Return type as string
|
92
|
+
def to_s
|
93
|
+
type
|
94
|
+
end
|
95
|
+
|
96
|
+
# Allow comparison with string
|
97
|
+
def eql?(other)
|
98
|
+
type == other.to_s
|
99
|
+
end
|
100
|
+
|
101
|
+
def hash
|
102
|
+
type.hash
|
103
|
+
end
|
104
|
+
|
105
|
+
alias == eql?
|
106
|
+
|
107
|
+
def self.child?(child, parent)
|
108
|
+
child == parent || TYPES.key?(child) && TYPES[child][1].any? {|p| child?(p, parent) }
|
109
|
+
end
|
110
|
+
|
111
|
+
def self.magic_match(io, method)
|
112
|
+
return magic_match(StringIO.new(io.to_s), method) unless io.respond_to?(:read)
|
113
|
+
|
114
|
+
io.binmode if io.respond_to?(:binmode)
|
115
|
+
io.set_encoding(Encoding::BINARY) if io.respond_to?(:set_encoding)
|
116
|
+
buffer = "".force_encoding(Encoding::BINARY)
|
117
|
+
|
118
|
+
MAGIC.send(method) { |type, matches| magic_match_io(io, matches, buffer) }
|
119
|
+
end
|
120
|
+
|
121
|
+
def self.magic_match_io(io, matches, buffer)
|
122
|
+
matches.any? do |offset, value, children|
|
123
|
+
match =
|
124
|
+
if Range === offset
|
125
|
+
io.read(offset.begin, buffer)
|
126
|
+
x = io.read(offset.end - offset.begin + value.bytesize, buffer)
|
127
|
+
x && x.include?(value)
|
128
|
+
else
|
129
|
+
io.read(offset, buffer)
|
130
|
+
io.read(value.bytesize, buffer) == value
|
131
|
+
end
|
132
|
+
io.rewind
|
133
|
+
match && (!children || magic_match_io(io, children, buffer))
|
134
|
+
end
|
135
|
+
end
|
136
|
+
|
137
|
+
private_class_method :magic_match, :magic_match_io
|
138
|
+
end
|
139
|
+
|
@@ -8,6 +8,7 @@ require "logstash/errors"
|
|
8
8
|
require 'logstash/inputs/s3sqs/patch'
|
9
9
|
require "aws-sdk"
|
10
10
|
require 'cgi'
|
11
|
+
require 'logstash/inputs/mime/mimemagic'
|
11
12
|
|
12
13
|
require 'java'
|
13
14
|
java_import java.io.InputStream
|
@@ -237,7 +238,7 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
|
|
237
238
|
begin
|
238
239
|
remote_object.get(:response_target => s3file)
|
239
240
|
rescue Aws::S3::Errors::AccessDenied => e
|
240
|
-
@logger.
|
241
|
+
@logger.error("Unable to download file. We´ll requeue the message", :file => remote_object.inspect)
|
241
242
|
throw :skip_delete
|
242
243
|
end
|
243
244
|
end
|
@@ -264,10 +265,10 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
|
|
264
265
|
@logger.warn("Logstash S3 input, stop reading in the middle of the file, we will read it again when logstash is started")
|
265
266
|
return false
|
266
267
|
end
|
267
|
-
#@logger.info("read line #{i}", :line => line)
|
268
268
|
#line = line.encode('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: "\u2370")
|
269
|
+
#@logger.debug("read line", :line => line)
|
269
270
|
instance_codec.decode(line) do |event|
|
270
|
-
|
271
|
+
@logger.debug("decorate event")
|
271
272
|
# We are making an assumption concerning cloudfront
|
272
273
|
# log format, the user will use the plain or the line codec
|
273
274
|
# and the message key will represent the actual line content.
|
@@ -293,6 +294,7 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
|
|
293
294
|
|
294
295
|
private
|
295
296
|
def local_decorate_and_queue(event, queue, key, folder, metadata, bucket)
|
297
|
+
@logger.debug('decorating event', :event => event.to_s)
|
296
298
|
if event_is_metadata?(event)
|
297
299
|
@logger.debug('Event is metadata, updating the current cloudfront metadata', :event => event)
|
298
300
|
update_metadata(metadata, event)
|
@@ -306,6 +308,7 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
|
|
306
308
|
event.set("[@metadata][s3]", { "object_key" => key })
|
307
309
|
event.set("[@metadata][s3]", { "bucket_name" => bucket })
|
308
310
|
event.set("[@metadata][s3]", { "object_folder" => folder})
|
311
|
+
#@logger.debug('queuing event', :event => event.to_s)
|
309
312
|
queue << event
|
310
313
|
end
|
311
314
|
end
|
@@ -319,6 +322,7 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
|
|
319
322
|
return ""
|
320
323
|
end
|
321
324
|
end
|
325
|
+
|
322
326
|
private
|
323
327
|
def read_file(filename, &block)
|
324
328
|
if gzip?(filename)
|
@@ -355,10 +359,12 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
|
|
355
359
|
|
356
360
|
private
|
357
361
|
def gzip?(filename)
|
358
|
-
filename.end_with?('.gz','.gzip')
|
362
|
+
return true if filename.end_with?('.gz','.gzip')
|
363
|
+
return true if MimeMagic.by_magic(File.open(filename)).to_s == 'application/gzip'
|
364
|
+
rescue Exception => e
|
365
|
+
@logger.debug("Problem while gzip detection", :error => e)
|
359
366
|
end
|
360
367
|
|
361
|
-
|
362
368
|
private
|
363
369
|
def delete_file_from_bucket(object)
|
364
370
|
if @delete_on_success
|
@@ -370,7 +376,7 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
|
|
370
376
|
private
|
371
377
|
def get_s3client
|
372
378
|
if s3_access_key_id and s3_secret_access_key
|
373
|
-
@logger.debug("Using S3 Credentials from config", :ID => aws_options_hash.merge(:access_key_id => s3_access_key_id) )
|
379
|
+
@logger.debug("Using S3 Credentials from config", :ID => aws_options_hash.merge(:access_key_id => s3_access_key_id, :secret_access_key => s3_secret_access_key) )
|
374
380
|
@s3_client = Aws::S3::Client.new(aws_options_hash.merge(:access_key_id => s3_access_key_id, :secret_access_key => s3_secret_access_key))
|
375
381
|
elsif @s3_role_arn
|
376
382
|
@s3_client = Aws::S3::Client.new(aws_options_hash.merge!({ :credentials => s3_assume_role }))
|
@@ -499,4 +505,4 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
|
|
499
505
|
result
|
500
506
|
end
|
501
507
|
end
|
502
|
-
end # class
|
508
|
+
end # class
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
s.name = 'logstash-input-s3-sns-sqs'
|
3
|
-
s.version = '1.4.
|
3
|
+
s.version = '1.4.9'
|
4
4
|
s.licenses = ['Apache License (2.0)']
|
5
5
|
s.summary = "Get logs from AWS s3 buckets as issued by an object-created event via sns -> sqs."
|
6
6
|
s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
|
@@ -23,7 +23,6 @@ Gem::Specification.new do |s|
|
|
23
23
|
|
24
24
|
s.add_runtime_dependency 'logstash-codec-json'
|
25
25
|
s.add_runtime_dependency "logstash-mixin-aws"
|
26
|
-
|
27
26
|
s.add_development_dependency 'logstash-devutils'
|
28
27
|
|
29
28
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-input-s3-sns-sqs
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.4.
|
4
|
+
version: 1.4.9
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Christian Herweg
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-03-
|
11
|
+
date: 2018-03-30 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -84,6 +84,10 @@ files:
|
|
84
84
|
- LICENSE
|
85
85
|
- NOTICE.TXT
|
86
86
|
- README.md
|
87
|
+
- lib/logstash/inputs/mime/mimemagic.rb
|
88
|
+
- lib/logstash/inputs/mime/mimemagic/overlay.rb
|
89
|
+
- lib/logstash/inputs/mime/mimemagic/tables.rb
|
90
|
+
- lib/logstash/inputs/mime/mimemagic/version.rb
|
87
91
|
- lib/logstash/inputs/s3snssqs.rb
|
88
92
|
- lib/logstash/inputs/s3sqs/patch.rb
|
89
93
|
- logstash-input-s3-sns-sqs.gemspec
|