logstash-input-s3-sns-sqs 2.2.0.pre → 2.2.1.pre
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/logstash/inputs/s3/downloader.rb +13 -1
- data/lib/logstash/inputs/s3snssqs/log_processor.rb +1 -2
- data/lib/logstash/inputs/s3snssqs.rb +4 -1
- data/lib/logstash/plugin_mixins/aws_config/generic.rb +61 -0
- data/lib/logstash/plugin_mixins/aws_config/v2.rb +100 -0
- data/lib/logstash/plugin_mixins/aws_config.rb +8 -0
- data/logstash-input-s3-sns-sqs.gemspec +4 -4
- metadata +26 -29
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 045e651e9ce6ed3e99a4f06b52be476e30ef7501b5e53a5bf97bcf0118c463d6
|
4
|
+
data.tar.gz: 25b9a355da317dec8b235fd84ce0cb04aec7fec9843edbbd418aa7f0cb570237
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 51a683ac50b21cd5e27c7dbcf73a494adeb3c5f04c7f590b38e3cb3730c308d331a8ab475a06e0df92204172841c62563ea29f92a1304932e15e3fe828c6ea0b
|
7
|
+
data.tar.gz: a8957a59f15f3d134f8c1fb7e60f1bbfa0461ceca58ec4569d58419634309bef27e780f6b43a084533690ccde3c040ec7549286aef49ecfc9bdee71da83c4bfa
|
@@ -9,6 +9,7 @@ class S3Downloader
|
|
9
9
|
@stopped = stop_semaphore
|
10
10
|
@factory = options[:s3_client_factory]
|
11
11
|
@delete_on_success = options[:delete_on_success]
|
12
|
+
@move_to_bucket = options[:move_to_bucket]
|
12
13
|
@include_object_properties = options[:include_object_properties]
|
13
14
|
end
|
14
15
|
|
@@ -40,7 +41,7 @@ class S3Downloader
|
|
40
41
|
end
|
41
42
|
|
42
43
|
def cleanup_s3object(record)
|
43
|
-
return unless @delete_on_success
|
44
|
+
return unless @delete_on_success || @move_to_bucket
|
44
45
|
begin
|
45
46
|
@factory.get_s3_client(record[:bucket]) do |s3|
|
46
47
|
s3.delete_object(bucket: record[:bucket], key: record[:key])
|
@@ -50,6 +51,17 @@ class S3Downloader
|
|
50
51
|
end
|
51
52
|
end
|
52
53
|
|
54
|
+
def move_s3object(record)
|
55
|
+
return unless @move_to_bucket
|
56
|
+
begin
|
57
|
+
@factory.get_s3_client(@move_to_bucket) do |s3|
|
58
|
+
s3.copy_object(bucket: @move_to_bucket, copy_source: "/#{record[:bucket]}/#{record[:key]}", key: record[:key])
|
59
|
+
end
|
60
|
+
rescue Exception => e
|
61
|
+
@logger.warn("Failed to move s3 object", :record => record, :error => e)
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
53
65
|
def stop?
|
54
66
|
@stopped.value
|
55
67
|
end
|
@@ -77,8 +77,7 @@ module LogProcessor
|
|
77
77
|
end
|
78
78
|
|
79
79
|
def gzip?(filename)
|
80
|
-
|
81
|
-
MagicGzipValidator.new(File.new(filename, 'rb')).valid?
|
80
|
+
filename.end_with?('.gz','.gzip') && MagicGzipValidator.new(File.new(filename, 'rb')).valid?
|
82
81
|
rescue Exception => e
|
83
82
|
@logger.warn("Problem while gzip detection", :error => e)
|
84
83
|
end
|
@@ -26,7 +26,7 @@ java_import java.io.FileInputStream
|
|
26
26
|
java_import java.io.BufferedReader
|
27
27
|
java_import java.util.zip.GZIPInputStream
|
28
28
|
java_import java.util.zip.ZipException
|
29
|
-
|
29
|
+
java_import java.lang.StringBuilder
|
30
30
|
|
31
31
|
# our helper classes
|
32
32
|
# these may go into this file for brevity...
|
@@ -160,6 +160,7 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
|
|
160
160
|
# Session name to use when assuming an IAM role
|
161
161
|
config :s3_role_session_name, :validate => :string, :default => "logstash"
|
162
162
|
config :delete_on_success, :validate => :boolean, :default => false
|
163
|
+
config :move_to_bucket, :validate => :string, :default => nil
|
163
164
|
# Whether or not to include the S3 object's properties (last_modified, content_type, metadata)
|
164
165
|
# into each Event at [@metadata][s3]. Regardless of this setting, [@metdata][s3][key] will always
|
165
166
|
# be present.
|
@@ -272,6 +273,7 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
|
|
272
273
|
@s3_downloader = S3Downloader.new(@logger, @received_stop, {
|
273
274
|
s3_client_factory: @s3_client_factory,
|
274
275
|
delete_on_success: @delete_on_success,
|
276
|
+
move_to_bucket: @move_to_bucket,
|
275
277
|
include_object_properties: @include_object_properties
|
276
278
|
})
|
277
279
|
@codec_factory = CodecFactory.new(@logger, {
|
@@ -343,6 +345,7 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
|
|
343
345
|
@s3_downloader.cleanup_local_object(record)
|
344
346
|
# re-throw if necessary:
|
345
347
|
throw :skip_delete unless completed
|
348
|
+
@s3_downloader.move_s3object(record)
|
346
349
|
@s3_downloader.cleanup_s3object(record)
|
347
350
|
end
|
348
351
|
end
|
@@ -0,0 +1,61 @@
|
|
1
|
+
module LogStash::PluginMixins::AwsConfig::Generic
|
2
|
+
def self.included(base)
|
3
|
+
base.extend(self)
|
4
|
+
base.generic_aws_config
|
5
|
+
end
|
6
|
+
|
7
|
+
def generic_aws_config
|
8
|
+
# The AWS Region
|
9
|
+
config :region, :validate => :string, :default => LogStash::PluginMixins::AwsConfig::US_EAST_1
|
10
|
+
|
11
|
+
# This plugin uses the AWS SDK and supports several ways to get credentials, which will be tried in this order:
|
12
|
+
#
|
13
|
+
# 1. Static configuration, using `access_key_id` and `secret_access_key` params in the logstash plugin config
|
14
|
+
# 2. External credentials file specified by `aws_credentials_file`
|
15
|
+
# 3. Environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`
|
16
|
+
# 4. Environment variables `AMAZON_ACCESS_KEY_ID` and `AMAZON_SECRET_ACCESS_KEY`
|
17
|
+
# 5. IAM Instance Profile (available when running inside EC2)
|
18
|
+
config :access_key_id, :validate => :string
|
19
|
+
|
20
|
+
# The AWS Secret Access Key
|
21
|
+
config :secret_access_key, :validate => :password
|
22
|
+
|
23
|
+
# The AWS Session token for temporary credential
|
24
|
+
config :session_token, :validate => :password
|
25
|
+
|
26
|
+
# URI to proxy server if required
|
27
|
+
config :proxy_uri, :validate => :string
|
28
|
+
|
29
|
+
# Custom endpoint to connect to s3
|
30
|
+
config :endpoint, :validate => :string
|
31
|
+
|
32
|
+
# The AWS IAM Role to assume, if any.
|
33
|
+
# This is used to generate temporary credentials typically for cross-account access.
|
34
|
+
# See https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html for more information.
|
35
|
+
# When `role_arn` is set, AWS (`access_key_id`/`secret_access_key`) credentials still get used if they're configured.
|
36
|
+
config :role_arn, :validate => :string
|
37
|
+
|
38
|
+
# Session name to use when assuming an IAM role
|
39
|
+
config :role_session_name, :validate => :string, :default => "logstash"
|
40
|
+
|
41
|
+
# Path to YAML file containing a hash of AWS credentials.
|
42
|
+
# This file will only be loaded if `access_key_id` and
|
43
|
+
# `secret_access_key` aren't set. The contents of the
|
44
|
+
# file should look like this:
|
45
|
+
#
|
46
|
+
# [source,ruby]
|
47
|
+
# ----------------------------------
|
48
|
+
# :access_key_id: "12345"
|
49
|
+
# :secret_access_key: "54321"
|
50
|
+
# ----------------------------------
|
51
|
+
#
|
52
|
+
config :aws_credentials_file, :validate => :string
|
53
|
+
|
54
|
+
# By default, this plugin uses cert available to OpenSSL provided by OS
|
55
|
+
# when verifying SSL peer certificates.
|
56
|
+
# For cases where the default cert is unavailable, e.g. Windows,
|
57
|
+
# you can use the bundled ca certificate provided by AWS SDK
|
58
|
+
# by setting `use_aws_bundled_ca` to true
|
59
|
+
config :use_aws_bundled_ca, :validate => :boolean, :default => false
|
60
|
+
end
|
61
|
+
end
|
@@ -0,0 +1,100 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require "logstash/plugin_mixins/aws_config/generic"
|
3
|
+
|
4
|
+
module LogStash::PluginMixins::AwsConfig::V2
|
5
|
+
def self.included(base)
|
6
|
+
base.extend(self)
|
7
|
+
base.send(:include, LogStash::PluginMixins::AwsConfig::Generic)
|
8
|
+
end
|
9
|
+
|
10
|
+
public
|
11
|
+
def aws_options_hash
|
12
|
+
opts = {}
|
13
|
+
|
14
|
+
opts[:http_proxy] = @proxy_uri if @proxy_uri
|
15
|
+
|
16
|
+
if @role_arn
|
17
|
+
credentials = assume_role(opts.dup)
|
18
|
+
opts[:credentials] = credentials
|
19
|
+
else
|
20
|
+
credentials = aws_credentials
|
21
|
+
opts[:credentials] = credentials if credentials
|
22
|
+
end
|
23
|
+
|
24
|
+
if self.respond_to?(:aws_service_endpoint)
|
25
|
+
# used by CloudWatch to basically do the same as bellow (returns { region: region })
|
26
|
+
opts.merge!(self.aws_service_endpoint(@region))
|
27
|
+
else
|
28
|
+
# NOTE: setting :region works with the aws sdk (resolves correct endpoint)
|
29
|
+
opts[:region] = @region
|
30
|
+
end
|
31
|
+
|
32
|
+
opts[:endpoint] = @endpoint unless @endpoint.nil?
|
33
|
+
|
34
|
+
if respond_to?(:additional_settings)
|
35
|
+
opts = symbolize_keys_and_cast_true_false(additional_settings).merge(opts)
|
36
|
+
end
|
37
|
+
|
38
|
+
if @use_aws_bundled_ca
|
39
|
+
aws_core_library = Gem.loaded_specs['aws-sdk-core']&.full_gem_path or fail("AWS Core library not available")
|
40
|
+
opts[:ssl_ca_bundle] = File.expand_path('ca-bundle.crt', aws_core_library).tap do |aws_core_ca_bundle|
|
41
|
+
fail("AWS Core CA bundle not found") unless File.exists?(aws_core_ca_bundle)
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
return opts
|
46
|
+
end
|
47
|
+
|
48
|
+
private
|
49
|
+
|
50
|
+
def aws_credentials
|
51
|
+
if @access_key_id && @secret_access_key
|
52
|
+
Aws::Credentials.new(@access_key_id, @secret_access_key.value, @session_token ? @session_token.value : nil)
|
53
|
+
elsif @access_key_id.nil? ^ @secret_access_key.nil?
|
54
|
+
@logger.warn("Likely config error: Only one of access_key_id or secret_access_key was provided but not both.")
|
55
|
+
secret_access_key = @secret_access_key ? @secret_access_key.value : nil
|
56
|
+
Aws::Credentials.new(@access_key_id, secret_access_key, @session_token ? @session_token.value : nil)
|
57
|
+
elsif @aws_credentials_file
|
58
|
+
credentials_opts = YAML.load_file(@aws_credentials_file)
|
59
|
+
credentials_opts.default_proc = lambda { |hash, key| hash.fetch(key.to_s, nil) }
|
60
|
+
Aws::Credentials.new(credentials_opts[:access_key_id],
|
61
|
+
credentials_opts[:secret_access_key],
|
62
|
+
credentials_opts[:session_token])
|
63
|
+
else
|
64
|
+
nil # AWS client will read ENV or ~/.aws/credentials
|
65
|
+
end
|
66
|
+
end
|
67
|
+
alias credentials aws_credentials
|
68
|
+
|
69
|
+
def assume_role(opts = {})
|
70
|
+
unless opts.key?(:credentials)
|
71
|
+
credentials = aws_credentials
|
72
|
+
opts[:credentials] = credentials if credentials
|
73
|
+
end
|
74
|
+
|
75
|
+
# for a regional endpoint :region is always required by AWS
|
76
|
+
opts[:region] = @region
|
77
|
+
|
78
|
+
Aws::AssumeRoleCredentials.new(
|
79
|
+
:client => Aws::STS::Client.new(opts),
|
80
|
+
:role_arn => @role_arn,
|
81
|
+
:role_session_name => @role_session_name
|
82
|
+
)
|
83
|
+
end
|
84
|
+
|
85
|
+
def symbolize_keys_and_cast_true_false(hash)
|
86
|
+
case hash
|
87
|
+
when Hash
|
88
|
+
symbolized = {}
|
89
|
+
hash.each { |key, value| symbolized[key.to_sym] = symbolize_keys_and_cast_true_false(value) }
|
90
|
+
symbolized
|
91
|
+
when 'true'
|
92
|
+
true
|
93
|
+
when 'false'
|
94
|
+
false
|
95
|
+
else
|
96
|
+
hash
|
97
|
+
end
|
98
|
+
end
|
99
|
+
|
100
|
+
end
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
s.name = 'logstash-input-s3-sns-sqs'
|
3
|
-
s.version = '2.2.
|
3
|
+
s.version = '2.2.1.pre'
|
4
4
|
s.licenses = ['Apache-2.0']
|
5
5
|
s.summary = "Get logs from AWS s3 buckets as issued by an object-created event via sns -> sqs."
|
6
6
|
s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
|
@@ -19,8 +19,9 @@ Gem::Specification.new do |s|
|
|
19
19
|
s.metadata = { "logstash_plugin" => "true", "logstash_group" => "input" }
|
20
20
|
|
21
21
|
# Gem dependencies
|
22
|
-
s.add_runtime_dependency "logstash-core-plugin-api", "
|
22
|
+
s.add_runtime_dependency "logstash-core-plugin-api", "~> 2.0"
|
23
23
|
s.add_runtime_dependency "concurrent-ruby"
|
24
|
+
s.add_runtime_dependency 'stud', '>= 0.0.22'
|
24
25
|
s.add_runtime_dependency "logstash-codec-json"
|
25
26
|
s.add_runtime_dependency "logstash-codec-plain"
|
26
27
|
s.add_runtime_dependency "aws-sdk-core", "~> 3"
|
@@ -31,9 +32,8 @@ Gem::Specification.new do |s|
|
|
31
32
|
|
32
33
|
s.add_development_dependency "logstash-codec-json_lines"
|
33
34
|
s.add_development_dependency "logstash-codec-multiline"
|
34
|
-
s.add_development_dependency "logstash-codec-json"
|
35
35
|
s.add_development_dependency "logstash-codec-line"
|
36
|
-
s.add_development_dependency
|
36
|
+
s.add_development_dependency 'logstash-devutils', '>= 0.0.16'
|
37
37
|
s.add_development_dependency "logstash-input-generator"
|
38
38
|
s.add_development_dependency "timecop"
|
39
39
|
|
metadata
CHANGED
@@ -1,35 +1,29 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-input-s3-sns-sqs
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.2.
|
4
|
+
version: 2.2.1.pre
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Christian Herweg
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2023-11-13 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
15
15
|
requirements:
|
16
|
-
- - "
|
17
|
-
- !ruby/object:Gem::Version
|
18
|
-
version: 2.1.12
|
19
|
-
- - "<="
|
16
|
+
- - "~>"
|
20
17
|
- !ruby/object:Gem::Version
|
21
|
-
version: '2.
|
18
|
+
version: '2.0'
|
22
19
|
name: logstash-core-plugin-api
|
23
20
|
prerelease: false
|
24
21
|
type: :runtime
|
25
22
|
version_requirements: !ruby/object:Gem::Requirement
|
26
23
|
requirements:
|
27
|
-
- - "
|
28
|
-
- !ruby/object:Gem::Version
|
29
|
-
version: 2.1.12
|
30
|
-
- - "<="
|
24
|
+
- - "~>"
|
31
25
|
- !ruby/object:Gem::Version
|
32
|
-
version: '2.
|
26
|
+
version: '2.0'
|
33
27
|
- !ruby/object:Gem::Dependency
|
34
28
|
requirement: !ruby/object:Gem::Requirement
|
35
29
|
requirements:
|
@@ -44,6 +38,20 @@ dependencies:
|
|
44
38
|
- - ">="
|
45
39
|
- !ruby/object:Gem::Version
|
46
40
|
version: '0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
requirement: !ruby/object:Gem::Requirement
|
43
|
+
requirements:
|
44
|
+
- - ">="
|
45
|
+
- !ruby/object:Gem::Version
|
46
|
+
version: 0.0.22
|
47
|
+
name: stud
|
48
|
+
prerelease: false
|
49
|
+
type: :runtime
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ">="
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: 0.0.22
|
47
55
|
- !ruby/object:Gem::Dependency
|
48
56
|
requirement: !ruby/object:Gem::Requirement
|
49
57
|
requirements:
|
@@ -170,20 +178,6 @@ dependencies:
|
|
170
178
|
- - ">="
|
171
179
|
- !ruby/object:Gem::Version
|
172
180
|
version: '0'
|
173
|
-
- !ruby/object:Gem::Dependency
|
174
|
-
requirement: !ruby/object:Gem::Requirement
|
175
|
-
requirements:
|
176
|
-
- - ">="
|
177
|
-
- !ruby/object:Gem::Version
|
178
|
-
version: '0'
|
179
|
-
name: logstash-codec-json
|
180
|
-
prerelease: false
|
181
|
-
type: :development
|
182
|
-
version_requirements: !ruby/object:Gem::Requirement
|
183
|
-
requirements:
|
184
|
-
- - ">="
|
185
|
-
- !ruby/object:Gem::Version
|
186
|
-
version: '0'
|
187
181
|
- !ruby/object:Gem::Dependency
|
188
182
|
requirement: !ruby/object:Gem::Requirement
|
189
183
|
requirements:
|
@@ -203,7 +197,7 @@ dependencies:
|
|
203
197
|
requirements:
|
204
198
|
- - ">="
|
205
199
|
- !ruby/object:Gem::Version
|
206
|
-
version:
|
200
|
+
version: 0.0.16
|
207
201
|
name: logstash-devutils
|
208
202
|
prerelease: false
|
209
203
|
type: :development
|
@@ -211,7 +205,7 @@ dependencies:
|
|
211
205
|
requirements:
|
212
206
|
- - ">="
|
213
207
|
- !ruby/object:Gem::Version
|
214
|
-
version:
|
208
|
+
version: 0.0.16
|
215
209
|
- !ruby/object:Gem::Dependency
|
216
210
|
requirement: !ruby/object:Gem::Requirement
|
217
211
|
requirements:
|
@@ -261,6 +255,9 @@ files:
|
|
261
255
|
- lib/logstash/inputs/s3snssqs.rb
|
262
256
|
- lib/logstash/inputs/s3snssqs/log_processor.rb
|
263
257
|
- lib/logstash/inputs/sqs/poller.rb
|
258
|
+
- lib/logstash/plugin_mixins/aws_config.rb
|
259
|
+
- lib/logstash/plugin_mixins/aws_config/generic.rb
|
260
|
+
- lib/logstash/plugin_mixins/aws_config/v2.rb
|
264
261
|
- logstash-input-s3-sns-sqs.gemspec
|
265
262
|
- spec/inputs/s3sqs_spec.rb
|
266
263
|
homepage: https://github.com/cherweg/logstash-input-s3-sns-sqs
|
@@ -284,7 +281,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
284
281
|
- !ruby/object:Gem::Version
|
285
282
|
version: 1.3.1
|
286
283
|
requirements: []
|
287
|
-
rubygems_version: 3.
|
284
|
+
rubygems_version: 3.2.29
|
288
285
|
signing_key:
|
289
286
|
specification_version: 4
|
290
287
|
summary: Get logs from AWS s3 buckets as issued by an object-created event via sns
|