logstash-input-s3-sns-sqs 2.2.0.pre → 2.2.1.pre

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 9d5ea48e146f2c80413f41fa2be04ab09396ce2fd67f0275032d34641e0c6fd2
4
- data.tar.gz: c4301ebe9b7abb940891bd06843de67b66b400cc6470bfa69b4a208320ef6387
3
+ metadata.gz: 045e651e9ce6ed3e99a4f06b52be476e30ef7501b5e53a5bf97bcf0118c463d6
4
+ data.tar.gz: 25b9a355da317dec8b235fd84ce0cb04aec7fec9843edbbd418aa7f0cb570237
5
5
  SHA512:
6
- metadata.gz: 7c478aa632537f4db570327ff25df1102faab0f9480515bbb4991e52540e086cfb881935b29725e34b4bd5de65a06de77ef65d682ee6b7cc909fce72b5359080
7
- data.tar.gz: 52cbe75d68326c5c29fdd5e2b13b3724e0c829664726667fb031c22c99318dfe04d950eb06524ee6d4d5cff76f3df13074db813c663b044f7f126907c8106a25
6
+ metadata.gz: 51a683ac50b21cd5e27c7dbcf73a494adeb3c5f04c7f590b38e3cb3730c308d331a8ab475a06e0df92204172841c62563ea29f92a1304932e15e3fe828c6ea0b
7
+ data.tar.gz: a8957a59f15f3d134f8c1fb7e60f1bbfa0461ceca58ec4569d58419634309bef27e780f6b43a084533690ccde3c040ec7549286aef49ecfc9bdee71da83c4bfa
@@ -9,6 +9,7 @@ class S3Downloader
9
9
  @stopped = stop_semaphore
10
10
  @factory = options[:s3_client_factory]
11
11
  @delete_on_success = options[:delete_on_success]
12
+ @move_to_bucket = options[:move_to_bucket]
12
13
  @include_object_properties = options[:include_object_properties]
13
14
  end
14
15
 
@@ -40,7 +41,7 @@ class S3Downloader
40
41
  end
41
42
 
42
43
  def cleanup_s3object(record)
43
- return unless @delete_on_success
44
+ return unless @delete_on_success || @move_to_bucket
44
45
  begin
45
46
  @factory.get_s3_client(record[:bucket]) do |s3|
46
47
  s3.delete_object(bucket: record[:bucket], key: record[:key])
@@ -50,6 +51,17 @@ class S3Downloader
50
51
  end
51
52
  end
52
53
 
54
+ def move_s3object(record)
55
+ return unless @move_to_bucket
56
+ begin
57
+ @factory.get_s3_client(@move_to_bucket) do |s3|
58
+ s3.copy_object(bucket: @move_to_bucket, copy_source: "/#{record[:bucket]}/#{record[:key]}", key: record[:key])
59
+ end
60
+ rescue Exception => e
61
+ @logger.warn("Failed to move s3 object", :record => record, :error => e)
62
+ end
63
+ end
64
+
53
65
  def stop?
54
66
  @stopped.value
55
67
  end
@@ -77,8 +77,7 @@ module LogProcessor
77
77
  end
78
78
 
79
79
  def gzip?(filename)
80
- return true if filename.end_with?('.gz','.gzip')
81
- MagicGzipValidator.new(File.new(filename, 'rb')).valid?
80
+ filename.end_with?('.gz','.gzip') && MagicGzipValidator.new(File.new(filename, 'rb')).valid?
82
81
  rescue Exception => e
83
82
  @logger.warn("Problem while gzip detection", :error => e)
84
83
  end
@@ -26,7 +26,7 @@ java_import java.io.FileInputStream
26
26
  java_import java.io.BufferedReader
27
27
  java_import java.util.zip.GZIPInputStream
28
28
  java_import java.util.zip.ZipException
29
- import java.lang.StringBuilder
29
+ java_import java.lang.StringBuilder
30
30
 
31
31
  # our helper classes
32
32
  # these may go into this file for brevity...
@@ -160,6 +160,7 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
160
160
  # Session name to use when assuming an IAM role
161
161
  config :s3_role_session_name, :validate => :string, :default => "logstash"
162
162
  config :delete_on_success, :validate => :boolean, :default => false
163
+ config :move_to_bucket, :validate => :string, :default => nil
163
164
  # Whether or not to include the S3 object's properties (last_modified, content_type, metadata)
164
165
  # into each Event at [@metadata][s3]. Regardless of this setting, [@metdata][s3][key] will always
165
166
  # be present.
@@ -272,6 +273,7 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
272
273
  @s3_downloader = S3Downloader.new(@logger, @received_stop, {
273
274
  s3_client_factory: @s3_client_factory,
274
275
  delete_on_success: @delete_on_success,
276
+ move_to_bucket: @move_to_bucket,
275
277
  include_object_properties: @include_object_properties
276
278
  })
277
279
  @codec_factory = CodecFactory.new(@logger, {
@@ -343,6 +345,7 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
343
345
  @s3_downloader.cleanup_local_object(record)
344
346
  # re-throw if necessary:
345
347
  throw :skip_delete unless completed
348
+ @s3_downloader.move_s3object(record)
346
349
  @s3_downloader.cleanup_s3object(record)
347
350
  end
348
351
  end
@@ -0,0 +1,61 @@
1
+ module LogStash::PluginMixins::AwsConfig::Generic
2
+ def self.included(base)
3
+ base.extend(self)
4
+ base.generic_aws_config
5
+ end
6
+
7
+ def generic_aws_config
8
+ # The AWS Region
9
+ config :region, :validate => :string, :default => LogStash::PluginMixins::AwsConfig::US_EAST_1
10
+
11
+ # This plugin uses the AWS SDK and supports several ways to get credentials, which will be tried in this order:
12
+ #
13
+ # 1. Static configuration, using `access_key_id` and `secret_access_key` params in the logstash plugin config
14
+ # 2. External credentials file specified by `aws_credentials_file`
15
+ # 3. Environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`
16
+ # 4. Environment variables `AMAZON_ACCESS_KEY_ID` and `AMAZON_SECRET_ACCESS_KEY`
17
+ # 5. IAM Instance Profile (available when running inside EC2)
18
+ config :access_key_id, :validate => :string
19
+
20
+ # The AWS Secret Access Key
21
+ config :secret_access_key, :validate => :password
22
+
23
+ # The AWS Session token for temporary credential
24
+ config :session_token, :validate => :password
25
+
26
+ # URI to proxy server if required
27
+ config :proxy_uri, :validate => :string
28
+
29
+ # Custom endpoint to connect to s3
30
+ config :endpoint, :validate => :string
31
+
32
+ # The AWS IAM Role to assume, if any.
33
+ # This is used to generate temporary credentials typically for cross-account access.
34
+ # See https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html for more information.
35
+ # When `role_arn` is set, AWS (`access_key_id`/`secret_access_key`) credentials still get used if they're configured.
36
+ config :role_arn, :validate => :string
37
+
38
+ # Session name to use when assuming an IAM role
39
+ config :role_session_name, :validate => :string, :default => "logstash"
40
+
41
+ # Path to YAML file containing a hash of AWS credentials.
42
+ # This file will only be loaded if `access_key_id` and
43
+ # `secret_access_key` aren't set. The contents of the
44
+ # file should look like this:
45
+ #
46
+ # [source,ruby]
47
+ # ----------------------------------
48
+ # :access_key_id: "12345"
49
+ # :secret_access_key: "54321"
50
+ # ----------------------------------
51
+ #
52
+ config :aws_credentials_file, :validate => :string
53
+
54
+ # By default, this plugin uses cert available to OpenSSL provided by OS
55
+ # when verifying SSL peer certificates.
56
+ # For cases where the default cert is unavailable, e.g. Windows,
57
+ # you can use the bundled ca certificate provided by AWS SDK
58
+ # by setting `use_aws_bundled_ca` to true
59
+ config :use_aws_bundled_ca, :validate => :boolean, :default => false
60
+ end
61
+ end
@@ -0,0 +1,100 @@
1
+ # encoding: utf-8
2
+ require "logstash/plugin_mixins/aws_config/generic"
3
+
4
+ module LogStash::PluginMixins::AwsConfig::V2
5
+ def self.included(base)
6
+ base.extend(self)
7
+ base.send(:include, LogStash::PluginMixins::AwsConfig::Generic)
8
+ end
9
+
10
+ public
11
+ def aws_options_hash
12
+ opts = {}
13
+
14
+ opts[:http_proxy] = @proxy_uri if @proxy_uri
15
+
16
+ if @role_arn
17
+ credentials = assume_role(opts.dup)
18
+ opts[:credentials] = credentials
19
+ else
20
+ credentials = aws_credentials
21
+ opts[:credentials] = credentials if credentials
22
+ end
23
+
24
+ if self.respond_to?(:aws_service_endpoint)
25
+ # used by CloudWatch to basically do the same as bellow (returns { region: region })
26
+ opts.merge!(self.aws_service_endpoint(@region))
27
+ else
28
+ # NOTE: setting :region works with the aws sdk (resolves correct endpoint)
29
+ opts[:region] = @region
30
+ end
31
+
32
+ opts[:endpoint] = @endpoint unless @endpoint.nil?
33
+
34
+ if respond_to?(:additional_settings)
35
+ opts = symbolize_keys_and_cast_true_false(additional_settings).merge(opts)
36
+ end
37
+
38
+ if @use_aws_bundled_ca
39
+ aws_core_library = Gem.loaded_specs['aws-sdk-core']&.full_gem_path or fail("AWS Core library not available")
40
+ opts[:ssl_ca_bundle] = File.expand_path('ca-bundle.crt', aws_core_library).tap do |aws_core_ca_bundle|
41
+ fail("AWS Core CA bundle not found") unless File.exists?(aws_core_ca_bundle)
42
+ end
43
+ end
44
+
45
+ return opts
46
+ end
47
+
48
+ private
49
+
50
+ def aws_credentials
51
+ if @access_key_id && @secret_access_key
52
+ Aws::Credentials.new(@access_key_id, @secret_access_key.value, @session_token ? @session_token.value : nil)
53
+ elsif @access_key_id.nil? ^ @secret_access_key.nil?
54
+ @logger.warn("Likely config error: Only one of access_key_id or secret_access_key was provided but not both.")
55
+ secret_access_key = @secret_access_key ? @secret_access_key.value : nil
56
+ Aws::Credentials.new(@access_key_id, secret_access_key, @session_token ? @session_token.value : nil)
57
+ elsif @aws_credentials_file
58
+ credentials_opts = YAML.load_file(@aws_credentials_file)
59
+ credentials_opts.default_proc = lambda { |hash, key| hash.fetch(key.to_s, nil) }
60
+ Aws::Credentials.new(credentials_opts[:access_key_id],
61
+ credentials_opts[:secret_access_key],
62
+ credentials_opts[:session_token])
63
+ else
64
+ nil # AWS client will read ENV or ~/.aws/credentials
65
+ end
66
+ end
67
+ alias credentials aws_credentials
68
+
69
+ def assume_role(opts = {})
70
+ unless opts.key?(:credentials)
71
+ credentials = aws_credentials
72
+ opts[:credentials] = credentials if credentials
73
+ end
74
+
75
+ # for a regional endpoint :region is always required by AWS
76
+ opts[:region] = @region
77
+
78
+ Aws::AssumeRoleCredentials.new(
79
+ :client => Aws::STS::Client.new(opts),
80
+ :role_arn => @role_arn,
81
+ :role_session_name => @role_session_name
82
+ )
83
+ end
84
+
85
+ def symbolize_keys_and_cast_true_false(hash)
86
+ case hash
87
+ when Hash
88
+ symbolized = {}
89
+ hash.each { |key, value| symbolized[key.to_sym] = symbolize_keys_and_cast_true_false(value) }
90
+ symbolized
91
+ when 'true'
92
+ true
93
+ when 'false'
94
+ false
95
+ else
96
+ hash
97
+ end
98
+ end
99
+
100
+ end
@@ -0,0 +1,8 @@
1
+ # encoding: utf-8
2
+ require "logstash/config/mixin"
3
+
4
+ module LogStash::PluginMixins::AwsConfig
5
+ require "logstash/plugin_mixins/aws_config/v2"
6
+
7
+ US_EAST_1 = "us-east-1"
8
+ end
@@ -1,6 +1,6 @@
1
1
  Gem::Specification.new do |s|
2
2
  s.name = 'logstash-input-s3-sns-sqs'
3
- s.version = '2.2.0.pre'
3
+ s.version = '2.2.1.pre'
4
4
  s.licenses = ['Apache-2.0']
5
5
  s.summary = "Get logs from AWS s3 buckets as issued by an object-created event via sns -> sqs."
6
6
  s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
@@ -19,8 +19,9 @@ Gem::Specification.new do |s|
19
19
  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "input" }
20
20
 
21
21
  # Gem dependencies
22
- s.add_runtime_dependency "logstash-core-plugin-api", ">= 2.1.12", "<= 2.99"
22
+ s.add_runtime_dependency "logstash-core-plugin-api", "~> 2.0"
23
23
  s.add_runtime_dependency "concurrent-ruby"
24
+ s.add_runtime_dependency 'stud', '>= 0.0.22'
24
25
  s.add_runtime_dependency "logstash-codec-json"
25
26
  s.add_runtime_dependency "logstash-codec-plain"
26
27
  s.add_runtime_dependency "aws-sdk-core", "~> 3"
@@ -31,9 +32,8 @@ Gem::Specification.new do |s|
31
32
 
32
33
  s.add_development_dependency "logstash-codec-json_lines"
33
34
  s.add_development_dependency "logstash-codec-multiline"
34
- s.add_development_dependency "logstash-codec-json"
35
35
  s.add_development_dependency "logstash-codec-line"
36
- s.add_development_dependency "logstash-devutils"
36
+ s.add_development_dependency 'logstash-devutils', '>= 0.0.16'
37
37
  s.add_development_dependency "logstash-input-generator"
38
38
  s.add_development_dependency "timecop"
39
39
 
metadata CHANGED
@@ -1,35 +1,29 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-input-s3-sns-sqs
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.2.0.pre
4
+ version: 2.2.1.pre
5
5
  platform: ruby
6
6
  authors:
7
7
  - Christian Herweg
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2022-06-13 00:00:00.000000000 Z
11
+ date: 2023-11-13 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
15
15
  requirements:
16
- - - ">="
17
- - !ruby/object:Gem::Version
18
- version: 2.1.12
19
- - - "<="
16
+ - - "~>"
20
17
  - !ruby/object:Gem::Version
21
- version: '2.99'
18
+ version: '2.0'
22
19
  name: logstash-core-plugin-api
23
20
  prerelease: false
24
21
  type: :runtime
25
22
  version_requirements: !ruby/object:Gem::Requirement
26
23
  requirements:
27
- - - ">="
28
- - !ruby/object:Gem::Version
29
- version: 2.1.12
30
- - - "<="
24
+ - - "~>"
31
25
  - !ruby/object:Gem::Version
32
- version: '2.99'
26
+ version: '2.0'
33
27
  - !ruby/object:Gem::Dependency
34
28
  requirement: !ruby/object:Gem::Requirement
35
29
  requirements:
@@ -44,6 +38,20 @@ dependencies:
44
38
  - - ">="
45
39
  - !ruby/object:Gem::Version
46
40
  version: '0'
41
+ - !ruby/object:Gem::Dependency
42
+ requirement: !ruby/object:Gem::Requirement
43
+ requirements:
44
+ - - ">="
45
+ - !ruby/object:Gem::Version
46
+ version: 0.0.22
47
+ name: stud
48
+ prerelease: false
49
+ type: :runtime
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - ">="
53
+ - !ruby/object:Gem::Version
54
+ version: 0.0.22
47
55
  - !ruby/object:Gem::Dependency
48
56
  requirement: !ruby/object:Gem::Requirement
49
57
  requirements:
@@ -170,20 +178,6 @@ dependencies:
170
178
  - - ">="
171
179
  - !ruby/object:Gem::Version
172
180
  version: '0'
173
- - !ruby/object:Gem::Dependency
174
- requirement: !ruby/object:Gem::Requirement
175
- requirements:
176
- - - ">="
177
- - !ruby/object:Gem::Version
178
- version: '0'
179
- name: logstash-codec-json
180
- prerelease: false
181
- type: :development
182
- version_requirements: !ruby/object:Gem::Requirement
183
- requirements:
184
- - - ">="
185
- - !ruby/object:Gem::Version
186
- version: '0'
187
181
  - !ruby/object:Gem::Dependency
188
182
  requirement: !ruby/object:Gem::Requirement
189
183
  requirements:
@@ -203,7 +197,7 @@ dependencies:
203
197
  requirements:
204
198
  - - ">="
205
199
  - !ruby/object:Gem::Version
206
- version: '0'
200
+ version: 0.0.16
207
201
  name: logstash-devutils
208
202
  prerelease: false
209
203
  type: :development
@@ -211,7 +205,7 @@ dependencies:
211
205
  requirements:
212
206
  - - ">="
213
207
  - !ruby/object:Gem::Version
214
- version: '0'
208
+ version: 0.0.16
215
209
  - !ruby/object:Gem::Dependency
216
210
  requirement: !ruby/object:Gem::Requirement
217
211
  requirements:
@@ -261,6 +255,9 @@ files:
261
255
  - lib/logstash/inputs/s3snssqs.rb
262
256
  - lib/logstash/inputs/s3snssqs/log_processor.rb
263
257
  - lib/logstash/inputs/sqs/poller.rb
258
+ - lib/logstash/plugin_mixins/aws_config.rb
259
+ - lib/logstash/plugin_mixins/aws_config/generic.rb
260
+ - lib/logstash/plugin_mixins/aws_config/v2.rb
264
261
  - logstash-input-s3-sns-sqs.gemspec
265
262
  - spec/inputs/s3sqs_spec.rb
266
263
  homepage: https://github.com/cherweg/logstash-input-s3-sns-sqs
@@ -284,7 +281,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
284
281
  - !ruby/object:Gem::Version
285
282
  version: 1.3.1
286
283
  requirements: []
287
- rubygems_version: 3.1.6
284
+ rubygems_version: 3.2.29
288
285
  signing_key:
289
286
  specification_version: 4
290
287
  summary: Get logs from AWS s3 buckets as issued by an object-created event via sns