logstash-output-s3 4.3.7 → 4.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +9 -0
- data/README.md +1 -1
- data/VERSION +1 -0
- data/lib/logstash/outputs/s3/file_repository.rb +67 -11
- data/lib/logstash/outputs/s3/size_rotation_policy.rb +1 -1
- data/lib/logstash/outputs/s3/temporary_file.rb +48 -5
- data/lib/logstash/outputs/s3/temporary_file_factory.rb +1 -4
- data/lib/logstash/outputs/s3/uploader.rb +2 -0
- data/lib/logstash/outputs/s3.rb +61 -28
- data/lib/logstash-output-s3_jars.rb +4 -0
- data/lib/tasks/build.rake +15 -0
- data/logstash-output-s3.gemspec +2 -2
- data/spec/integration/restore_from_crash_spec.rb +69 -4
- data/spec/outputs/s3/size_rotation_policy_spec.rb +2 -2
- data/spec/supports/helpers.rb +3 -1
- data/vendor/jar-dependencies/org/logstash/plugins/outputs/s3/logstash-output-s3/4.4.0/logstash-output-s3-4.4.0.jar +0 -0
- metadata +7 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 1915e4499e1950b269e287e9bbec3d88efca2f274a390f99949c27af5e2da105
|
4
|
+
data.tar.gz: 6a464adee35655b6f06f5fdf8dea9461af550b51912c4edbe46fb0d7c656774c
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 31ff8fa96f60dffc824f4881ec162c2db15a707ddc43dee4ff81a043ad23dd9296dd5bafaedcef26a27e053f6db8156c918da4c68ec95b4dd496783218595cbe
|
7
|
+
data.tar.gz: 67e830c249b6a5042b442b89b0b0e0b5b7d88dd6064119d0b9816a680aff1d1ae975addcc02b6f5cbf8a36fc5faf2d5e9b6af164e7a471f08f8f8ecd010188a8
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,12 @@
|
|
1
|
+
## 4.4.1
|
2
|
+
- Fixes several closely-related race conditions that could cause plugin crashes or data-loss [#252](https://github.com/logstash-plugins/logstash-output-s3/pull/252)
|
3
|
+
- race condition in initializing a prefix could cause one or more local temp files to be abandoned and only recovered after next pipeline start
|
4
|
+
- race condition in stale watcher could cause the plugin to crash when working with a stale (empty) file that had been deleted
|
5
|
+
- race condition in stale watcher could cause a non-empty file to be deleted if bytes were written to it after it was detected as stale
|
6
|
+
|
7
|
+
## 4.4.0
|
8
|
+
- Logstash recovers corrupted gzip and uploads to S3 [#249](https://github.com/logstash-plugins/logstash-output-s3/pull/249)
|
9
|
+
|
1
10
|
## 4.3.7
|
2
11
|
- Refactor: avoid usage of CHM (JRuby 9.3.4 work-around) [#248](https://github.com/logstash-plugins/logstash-output-s3/pull/248)
|
3
12
|
|
data/README.md
CHANGED
@@ -19,7 +19,7 @@ Need help? Try #logstash on freenode IRC or the https://discuss.elastic.co/c/log
|
|
19
19
|
|
20
20
|
## Developing
|
21
21
|
|
22
|
-
### 1. Plugin
|
22
|
+
### 1. Plugin Development and Testing
|
23
23
|
|
24
24
|
#### Code
|
25
25
|
- To get started, you'll need JRuby with the Bundler gem installed.
|
data/VERSION
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
4.4.0
|
@@ -15,8 +15,9 @@ module LogStash
|
|
15
15
|
class PrefixedValue
|
16
16
|
def initialize(file_factory, stale_time)
|
17
17
|
@file_factory = file_factory
|
18
|
-
@lock =
|
18
|
+
@lock = Monitor.new
|
19
19
|
@stale_time = stale_time
|
20
|
+
@is_deleted = false
|
20
21
|
end
|
21
22
|
|
22
23
|
def with_lock
|
@@ -34,7 +35,14 @@ module LogStash
|
|
34
35
|
end
|
35
36
|
|
36
37
|
def delete!
|
37
|
-
with_lock
|
38
|
+
with_lock do |factory|
|
39
|
+
factory.current.delete!
|
40
|
+
@is_deleted = true
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
def deleted?
|
45
|
+
with_lock { |_| @is_deleted }
|
38
46
|
end
|
39
47
|
end
|
40
48
|
|
@@ -72,17 +80,52 @@ module LogStash
|
|
72
80
|
end
|
73
81
|
|
74
82
|
def each_files
|
75
|
-
|
76
|
-
|
83
|
+
each_factory(keys) do |factory|
|
84
|
+
yield factory.current
|
77
85
|
end
|
78
86
|
end
|
79
87
|
|
80
|
-
|
88
|
+
##
|
89
|
+
# Yields the file factory while the current thread has exclusive access to it, creating a new
|
90
|
+
# one if one does not exist or if the current one is being reaped by the stale watcher.
|
91
|
+
# @param prefix_key [String]: the prefix key
|
92
|
+
# @yieldparam factory [TemporaryFileFactory]: a temporary file factory that this thread has exclusive access to
|
93
|
+
# @yieldreturn [Object]: a value to return; should NOT be the factory, which should be contained by the exclusive access scope.
|
94
|
+
# @return [Object]: the value returned by the provided block
|
81
95
|
def get_factory(prefix_key)
|
82
|
-
|
96
|
+
|
97
|
+
# fast-path: if factory exists and is not deleted, yield it with exclusive access and return
|
98
|
+
prefix_val = @prefixed_factories.get(prefix_key)
|
99
|
+
prefix_val&.with_lock do |factory|
|
100
|
+
# intentional local-jump to ensure deletion detection
|
101
|
+
# is done inside the exclusive access.
|
102
|
+
return yield(factory) unless prefix_val.deleted?
|
103
|
+
end
|
104
|
+
|
105
|
+
# slow-path:
|
106
|
+
# the Concurrent::Map#get operation is lock-free, but may have returned an entry that was being deleted by
|
107
|
+
# another thread (such as via stale detection). If we failed to retrieve a value, or retrieved one that had
|
108
|
+
# been marked deleted, use the atomic Concurrent::Map#compute to retrieve a non-deleted entry.
|
109
|
+
prefix_val = @prefixed_factories.compute(prefix_key) do |existing|
|
110
|
+
existing && !existing.deleted? ? existing : @factory_initializer.create_value(prefix_key)
|
111
|
+
end
|
83
112
|
prefix_val.with_lock { |factory| yield factory }
|
84
113
|
end
|
85
114
|
|
115
|
+
##
|
116
|
+
# Yields each non-deleted file factory while the current thread has exclusive access to it.
|
117
|
+
# @param prefixes [Array<String>]: the prefix keys
|
118
|
+
# @yieldparam factory [TemporaryFileFactory]
|
119
|
+
# @return [void]
|
120
|
+
def each_factory(prefixes)
|
121
|
+
prefixes.each do |prefix_key|
|
122
|
+
prefix_val = @prefixed_factories.get(prefix_key)
|
123
|
+
prefix_val&.with_lock do |factory|
|
124
|
+
yield factory unless prefix_val.deleted?
|
125
|
+
end
|
126
|
+
end
|
127
|
+
end
|
128
|
+
|
86
129
|
def get_file(prefix_key)
|
87
130
|
get_factory(prefix_key) { |factory| yield factory.current }
|
88
131
|
end
|
@@ -95,10 +138,21 @@ module LogStash
|
|
95
138
|
@prefixed_factories.size
|
96
139
|
end
|
97
140
|
|
98
|
-
def
|
99
|
-
|
100
|
-
|
101
|
-
|
141
|
+
def remove_if_stale(prefix_key)
|
142
|
+
# we use the ATOMIC `Concurrent::Map#compute_if_present` to atomically
|
143
|
+
# detect the staleness, mark a stale prefixed factory as deleted, and delete from the map.
|
144
|
+
@prefixed_factories.compute_if_present(prefix_key) do |prefixed_factory|
|
145
|
+
# once we have retrieved an instance, we acquire exclusive access to it
|
146
|
+
# for stale detection, marking it as deleted before releasing the lock
|
147
|
+
# and causing it to become deleted from the map.
|
148
|
+
prefixed_factory.with_lock do |_|
|
149
|
+
if prefixed_factory.stale?
|
150
|
+
prefixed_factory.delete! # mark deleted to prevent reuse
|
151
|
+
nil # cause deletion
|
152
|
+
else
|
153
|
+
prefixed_factory # keep existing
|
154
|
+
end
|
155
|
+
end
|
102
156
|
end
|
103
157
|
end
|
104
158
|
|
@@ -106,7 +160,9 @@ module LogStash
|
|
106
160
|
@stale_sweeper = Concurrent::TimerTask.new(:execution_interval => @sweeper_interval) do
|
107
161
|
LogStash::Util.set_thread_name("S3, Stale factory sweeper")
|
108
162
|
|
109
|
-
@prefixed_factories.each
|
163
|
+
@prefixed_factories.keys.each do |prefix|
|
164
|
+
remove_if_stale(prefix)
|
165
|
+
end
|
110
166
|
end
|
111
167
|
|
112
168
|
@stale_sweeper.execute
|
@@ -2,15 +2,23 @@
|
|
2
2
|
require "thread"
|
3
3
|
require "forwardable"
|
4
4
|
require "fileutils"
|
5
|
+
require "logstash-output-s3_jars"
|
5
6
|
|
6
7
|
module LogStash
|
7
8
|
module Outputs
|
8
9
|
class S3
|
9
|
-
|
10
|
-
|
10
|
+
|
11
|
+
java_import 'org.logstash.plugins.outputs.s3.GzipUtil'
|
12
|
+
|
13
|
+
# Wrap the actual file descriptor into an utility class
|
14
|
+
# Make it more OOP and easier to reason with the paths.
|
11
15
|
class TemporaryFile
|
12
16
|
extend Forwardable
|
13
17
|
|
18
|
+
GZIP_EXTENSION = "txt.gz"
|
19
|
+
TXT_EXTENSION = "txt"
|
20
|
+
RECOVERED_FILE_NAME_TAG = "-recovered"
|
21
|
+
|
14
22
|
def_delegators :@fd, :path, :write, :close, :fsync
|
15
23
|
|
16
24
|
attr_reader :fd
|
@@ -33,8 +41,10 @@ module LogStash
|
|
33
41
|
def size
|
34
42
|
# Use the fd size to get the accurate result,
|
35
43
|
# so we dont have to deal with fsync
|
36
|
-
# if the file is close we
|
44
|
+
# if the file is close, fd.size raises an IO exception so we use the File::size
|
37
45
|
begin
|
46
|
+
# fd is nil when LS tries to recover gzip file but fails
|
47
|
+
return 0 unless @fd != nil
|
38
48
|
@fd.size
|
39
49
|
rescue IOError
|
40
50
|
::File.size(path)
|
@@ -45,7 +55,7 @@ module LogStash
|
|
45
55
|
@key.gsub(/^\//, "")
|
46
56
|
end
|
47
57
|
|
48
|
-
# Each temporary file is
|
58
|
+
# Each temporary file is created inside a directory named with an UUID,
|
49
59
|
# instead of deleting the file directly and having the risk of deleting other files
|
50
60
|
# we delete the root of the UUID, using a UUID also remove the risk of deleting unwanted file, it acts as
|
51
61
|
# a sandbox.
|
@@ -58,13 +68,46 @@ module LogStash
|
|
58
68
|
size == 0
|
59
69
|
end
|
60
70
|
|
71
|
+
# only to cover the case where LS cannot restore corrupted file, file is not exist
|
72
|
+
def recoverable?
|
73
|
+
!@fd.nil?
|
74
|
+
end
|
75
|
+
|
61
76
|
def self.create_from_existing_file(file_path, temporary_folder)
|
62
77
|
key_parts = Pathname.new(file_path).relative_path_from(temporary_folder).to_s.split(::File::SEPARATOR)
|
63
78
|
|
79
|
+
# recover gzip file and compress back before uploading to S3
|
80
|
+
if file_path.end_with?("." + GZIP_EXTENSION)
|
81
|
+
file_path = self.recover(file_path)
|
82
|
+
end
|
64
83
|
TemporaryFile.new(key_parts.slice(1, key_parts.size).join("/"),
|
65
|
-
::File.open(file_path, "r"),
|
84
|
+
::File.exist?(file_path) ? ::File.open(file_path, "r") : nil, # for the nil case, file size will be 0 and upload will be ignored.
|
66
85
|
::File.join(temporary_folder, key_parts.slice(0, 1)))
|
67
86
|
end
|
87
|
+
|
88
|
+
def self.gzip_extension
|
89
|
+
GZIP_EXTENSION
|
90
|
+
end
|
91
|
+
|
92
|
+
def self.text_extension
|
93
|
+
TXT_EXTENSION
|
94
|
+
end
|
95
|
+
|
96
|
+
def self.recovery_file_name_tag
|
97
|
+
RECOVERED_FILE_NAME_TAG
|
98
|
+
end
|
99
|
+
|
100
|
+
private
|
101
|
+
def self.recover(file_path)
|
102
|
+
full_gzip_extension = "." + GZIP_EXTENSION
|
103
|
+
recovered_txt_file_path = file_path.gsub(full_gzip_extension, RECOVERED_FILE_NAME_TAG + "." + TXT_EXTENSION)
|
104
|
+
recovered_gzip_file_path = file_path.gsub(full_gzip_extension, RECOVERED_FILE_NAME_TAG + full_gzip_extension)
|
105
|
+
GzipUtil.recover(file_path, recovered_txt_file_path)
|
106
|
+
if ::File.exist?(recovered_txt_file_path) && !::File.zero?(recovered_txt_file_path)
|
107
|
+
GzipUtil.compress(recovered_txt_file_path, recovered_gzip_file_path)
|
108
|
+
end
|
109
|
+
recovered_gzip_file_path
|
110
|
+
end
|
68
111
|
end
|
69
112
|
end
|
70
113
|
end
|
@@ -19,9 +19,6 @@ module LogStash
|
|
19
19
|
# I do not have to mess around to check if the other directory have file in it before destroying them.
|
20
20
|
class TemporaryFileFactory
|
21
21
|
FILE_MODE = "a"
|
22
|
-
GZIP_ENCODING = "gzip"
|
23
|
-
GZIP_EXTENSION = "txt.gz"
|
24
|
-
TXT_EXTENSION = "txt"
|
25
22
|
STRFTIME = "%Y-%m-%dT%H.%M"
|
26
23
|
|
27
24
|
attr_accessor :counter, :tags, :prefix, :encoding, :temporary_directory, :current
|
@@ -48,7 +45,7 @@ module LogStash
|
|
48
45
|
|
49
46
|
private
|
50
47
|
def extension
|
51
|
-
gzip? ?
|
48
|
+
gzip? ? TemporaryFile.gzip_extension : TemporaryFile.text_extension
|
52
49
|
end
|
53
50
|
|
54
51
|
def gzip?
|
@@ -31,6 +31,7 @@ module LogStash
|
|
31
31
|
end
|
32
32
|
end
|
33
33
|
|
34
|
+
# uploads a TemporaryFile to S3
|
34
35
|
def upload(file, options = {})
|
35
36
|
upload_options = options.fetch(:upload_options, {})
|
36
37
|
|
@@ -68,6 +69,7 @@ module LogStash
|
|
68
69
|
@workers_pool.shutdown
|
69
70
|
@workers_pool.wait_for_termination(nil) # block until its done
|
70
71
|
end
|
72
|
+
|
71
73
|
end
|
72
74
|
end
|
73
75
|
end
|
data/lib/logstash/outputs/s3.rb
CHANGED
@@ -97,6 +97,7 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
|
|
97
97
|
:fallback_policy => :caller_runs
|
98
98
|
})
|
99
99
|
|
100
|
+
GZIP_ENCODING = "gzip"
|
100
101
|
|
101
102
|
config_name "s3"
|
102
103
|
default :codec, "line"
|
@@ -181,7 +182,7 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
|
|
181
182
|
config :tags, :validate => :array, :default => []
|
182
183
|
|
183
184
|
# Specify the content encoding. Supports ("gzip"). Defaults to "none"
|
184
|
-
config :encoding, :validate => ["none",
|
185
|
+
config :encoding, :validate => ["none", GZIP_ENCODING], :default => "none"
|
185
186
|
|
186
187
|
# Define the strategy to use to decide when we need to rotate the file and push it to S3,
|
187
188
|
# The default strategy is to check for both size and time, the first one to match will rotate the file.
|
@@ -265,6 +266,8 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
|
|
265
266
|
|
266
267
|
@logger.debug("Uploading current workspace")
|
267
268
|
|
269
|
+
@file_repository.shutdown # stop stale sweeps
|
270
|
+
|
268
271
|
# The plugin has stopped receiving new events, but we still have
|
269
272
|
# data on disk, lets make sure it get to S3.
|
270
273
|
# If Logstash get interrupted, the `restore_from_crash` (when set to true) method will pickup
|
@@ -274,8 +277,6 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
|
|
274
277
|
upload_file(file)
|
275
278
|
end
|
276
279
|
|
277
|
-
@file_repository.shutdown
|
278
|
-
|
279
280
|
@uploader.stop # wait until all the current upload are complete
|
280
281
|
@crash_uploader.stop if @restore # we might have still work to do for recovery so wait until we are done
|
281
282
|
end
|
@@ -315,7 +316,7 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
|
|
315
316
|
:server_side_encryption => @server_side_encryption ? @server_side_encryption_algorithm : nil,
|
316
317
|
:ssekms_key_id => @server_side_encryption_algorithm == "aws:kms" ? @ssekms_key_id : nil,
|
317
318
|
:storage_class => @storage_class,
|
318
|
-
:content_encoding => @encoding ==
|
319
|
+
:content_encoding => @encoding == GZIP_ENCODING ? GZIP_ENCODING : nil,
|
319
320
|
:multipart_threshold => @upload_multipart_threshold
|
320
321
|
}
|
321
322
|
end
|
@@ -343,22 +344,22 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
|
|
343
344
|
end
|
344
345
|
|
345
346
|
def rotate_if_needed(prefixes)
|
346
|
-
|
347
|
-
|
348
|
-
|
349
|
-
|
350
|
-
|
351
|
-
|
352
|
-
|
353
|
-
|
354
|
-
|
355
|
-
|
356
|
-
|
357
|
-
|
358
|
-
|
359
|
-
|
360
|
-
|
361
|
-
|
347
|
+
# Each file access is thread safe,
|
348
|
+
# until the rotation is done then only
|
349
|
+
# one thread has access to the resource.
|
350
|
+
@file_repository.each_factory(prefixes) do |factory|
|
351
|
+
# we have exclusive access to the one-and-only
|
352
|
+
# prefix WRAPPER for this factory.
|
353
|
+
temp_file = factory.current
|
354
|
+
|
355
|
+
if @rotation.rotate?(temp_file)
|
356
|
+
@logger.debug? && @logger.debug("Rotate file",
|
357
|
+
:key => temp_file.key,
|
358
|
+
:path => temp_file.path,
|
359
|
+
:strategy => @rotation.class.name)
|
360
|
+
|
361
|
+
upload_file(temp_file) # may be async or blocking
|
362
|
+
factory.rotate!
|
362
363
|
end
|
363
364
|
end
|
364
365
|
end
|
@@ -397,16 +398,48 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
|
|
397
398
|
@crash_uploader = Uploader.new(bucket_resource, @logger, CRASH_RECOVERY_THREADPOOL)
|
398
399
|
|
399
400
|
temp_folder_path = Pathname.new(@temporary_directory)
|
400
|
-
Dir.glob(::File.join(@temporary_directory, "**/*"))
|
401
|
-
|
402
|
-
|
403
|
-
|
404
|
-
|
405
|
-
|
406
|
-
|
401
|
+
files = Dir.glob(::File.join(@temporary_directory, "**/*"))
|
402
|
+
.select { |file_path| ::File.file?(file_path) }
|
403
|
+
under_recovery_files = get_under_recovery_files(files)
|
404
|
+
|
405
|
+
files.each do |file_path|
|
406
|
+
# when encoding is GZIP, if file is already recovering or recovered and uploading to S3, log and skip
|
407
|
+
if under_recovery_files.include?(file_path)
|
408
|
+
unless file_path.include?(TemporaryFile.gzip_extension)
|
409
|
+
@logger.warn("The #{file_path} file either under recover process or failed to recover before.")
|
410
|
+
end
|
407
411
|
else
|
408
|
-
|
412
|
+
temp_file = TemporaryFile.create_from_existing_file(file_path, temp_folder_path)
|
413
|
+
# do not remove or upload if Logstash tries to recover file but fails
|
414
|
+
if temp_file.recoverable?
|
415
|
+
if temp_file.size > 0
|
416
|
+
@logger.debug? && @logger.debug("Recovering from crash and uploading", :path => temp_file.path)
|
417
|
+
@crash_uploader.upload_async(temp_file,
|
418
|
+
:on_complete => method(:clean_temporary_file),
|
419
|
+
:upload_options => upload_options)
|
420
|
+
else
|
421
|
+
clean_temporary_file(temp_file)
|
422
|
+
end
|
423
|
+
end
|
424
|
+
end
|
425
|
+
end
|
426
|
+
end
|
427
|
+
|
428
|
+
# figures out the recovering files and
|
429
|
+
# creates a skip list to ignore for the rest of processes
|
430
|
+
def get_under_recovery_files(files)
|
431
|
+
skip_files = Set.new
|
432
|
+
return skip_files unless @encoding == GZIP_ENCODING
|
433
|
+
|
434
|
+
files.each do |file_path|
|
435
|
+
if file_path.include?(TemporaryFile.recovery_file_name_tag)
|
436
|
+
skip_files << file_path
|
437
|
+
if file_path.include?(TemporaryFile.gzip_extension)
|
438
|
+
# also include the original corrupted gzip file
|
439
|
+
skip_files << file_path.gsub(TemporaryFile.recovery_file_name_tag, "")
|
440
|
+
end
|
409
441
|
end
|
410
442
|
end
|
443
|
+
skip_files
|
411
444
|
end
|
412
445
|
end
|
@@ -0,0 +1,15 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require "jars/installer"
|
3
|
+
require "fileutils"
|
4
|
+
|
5
|
+
task :vendor do
|
6
|
+
exit(1) unless system './gradlew vendor'
|
7
|
+
version = File.read("VERSION").strip
|
8
|
+
end
|
9
|
+
|
10
|
+
desc "clean"
|
11
|
+
task :clean do
|
12
|
+
["build", "vendor/jar-dependencies", "Gemfile.lock"].each do |p|
|
13
|
+
FileUtils.rm_rf(p)
|
14
|
+
end
|
15
|
+
end
|
data/logstash-output-s3.gemspec
CHANGED
@@ -1,13 +1,13 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
s.name = 'logstash-output-s3'
|
3
|
-
s.version = '4.
|
3
|
+
s.version = '4.4.1'
|
4
4
|
s.licenses = ['Apache-2.0']
|
5
5
|
s.summary = "Sends Logstash events to the Amazon Simple Storage Service"
|
6
6
|
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
7
7
|
s.authors = ["Elastic"]
|
8
8
|
s.email = 'info@elastic.co'
|
9
9
|
s.homepage = "http://www.elastic.co/guide/en/logstash/current/index.html"
|
10
|
-
s.require_paths = ["lib"]
|
10
|
+
s.require_paths = ["lib", "vendor/jar-dependencies"]
|
11
11
|
|
12
12
|
# Files
|
13
13
|
s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "docs/**/*"]
|
@@ -7,18 +7,17 @@ require "stud/temporary"
|
|
7
7
|
describe "Restore from crash", :integration => true do
|
8
8
|
include_context "setup plugin"
|
9
9
|
|
10
|
-
let(:options) { main_options.merge({ "restore" => true, "canned_acl" => "public-read-write" }) }
|
11
|
-
|
12
10
|
let(:number_of_files) { 5 }
|
13
11
|
let(:dummy_content) { "foobar\n" * 100 }
|
14
|
-
let(:factory) { LogStash::Outputs::S3::TemporaryFileFactory.new(prefix, tags, "none", temporary_directory)}
|
15
12
|
|
16
13
|
before do
|
17
14
|
clean_remote_files(prefix)
|
18
15
|
end
|
19
16
|
|
20
|
-
|
21
17
|
context 'with a non-empty tempfile' do
|
18
|
+
let(:options) { main_options.merge({ "restore" => true, "canned_acl" => "public-read-write" }) }
|
19
|
+
let(:factory) { LogStash::Outputs::S3::TemporaryFileFactory.new(prefix, tags, "none", temporary_directory)}
|
20
|
+
|
22
21
|
before do
|
23
22
|
# Creating a factory always create a file
|
24
23
|
factory.current.write(dummy_content)
|
@@ -41,6 +40,9 @@ describe "Restore from crash", :integration => true do
|
|
41
40
|
end
|
42
41
|
|
43
42
|
context 'with an empty tempfile' do
|
43
|
+
let(:options) { main_options.merge({ "restore" => true, "canned_acl" => "public-read-write" }) }
|
44
|
+
let(:factory) { LogStash::Outputs::S3::TemporaryFileFactory.new(prefix, tags, "none", temporary_directory)}
|
45
|
+
|
44
46
|
before do
|
45
47
|
factory.current
|
46
48
|
factory.rotate!
|
@@ -63,5 +65,68 @@ describe "Restore from crash", :integration => true do
|
|
63
65
|
expect(bucket_resource.objects(:prefix => prefix).count).to eq(0)
|
64
66
|
end
|
65
67
|
end
|
68
|
+
|
69
|
+
context "#gzip encoding" do
|
70
|
+
let(:options) { main_options.merge({ "restore" => true, "canned_acl" => "public-read-write", "encoding" => "gzip" }) }
|
71
|
+
let(:factory) { LogStash::Outputs::S3::TemporaryFileFactory.new(prefix, tags, "gzip", temporary_directory)}
|
72
|
+
describe "with empty recovered file" do
|
73
|
+
before do
|
74
|
+
# Creating a factory always create a file
|
75
|
+
factory.current.write('')
|
76
|
+
factory.current.fsync
|
77
|
+
factory.current.close
|
78
|
+
end
|
79
|
+
|
80
|
+
it 'should not upload and not remove temp file' do
|
81
|
+
subject.register
|
82
|
+
try(20) do
|
83
|
+
expect(bucket_resource.objects(:prefix => prefix).count).to eq(0)
|
84
|
+
expect(Dir.glob(File.join(temporary_directory, "*")).size).to eq(1)
|
85
|
+
end
|
86
|
+
end
|
87
|
+
end
|
88
|
+
|
89
|
+
describe "with healthy recovered, size is greater than zero file" do
|
90
|
+
before do
|
91
|
+
# Creating a factory always create a file
|
92
|
+
factory.current.write(dummy_content)
|
93
|
+
factory.current.fsync
|
94
|
+
factory.current.close
|
95
|
+
|
96
|
+
(number_of_files - 1).times do
|
97
|
+
factory.rotate!
|
98
|
+
factory.current.write(dummy_content)
|
99
|
+
factory.current.fsync
|
100
|
+
factory.current.close
|
101
|
+
end
|
102
|
+
end
|
103
|
+
|
104
|
+
it 'should recover, upload to S3 and remove temp file' do
|
105
|
+
subject.register
|
106
|
+
try(20) do
|
107
|
+
expect(bucket_resource.objects(:prefix => prefix).count).to eq(number_of_files)
|
108
|
+
expect(Dir.glob(File.join(temporary_directory, "*")).size).to eq(0)
|
109
|
+
expect(bucket_resource.objects(:prefix => prefix).first.acl.grants.collect(&:permission)).to include("READ", "WRITE")
|
110
|
+
end
|
111
|
+
end
|
112
|
+
end
|
113
|
+
|
114
|
+
describe "with failure when recovering" do
|
115
|
+
before do
|
116
|
+
# Creating a factory always create a file
|
117
|
+
factory.current.write(dummy_content)
|
118
|
+
factory.current.fsync
|
119
|
+
end
|
120
|
+
|
121
|
+
it 'should not upload to S3 and not remove temp file' do
|
122
|
+
subject.register
|
123
|
+
try(20) do
|
124
|
+
expect(bucket_resource.objects(:prefix => prefix).count).to eq(0)
|
125
|
+
expect(Dir.glob(File.join(temporary_directory, "*")).size).to eq(1)
|
126
|
+
end
|
127
|
+
end
|
128
|
+
end
|
129
|
+
end
|
130
|
+
|
66
131
|
end
|
67
132
|
|
@@ -25,11 +25,11 @@ describe LogStash::Outputs::S3::SizeRotationPolicy do
|
|
25
25
|
end
|
26
26
|
|
27
27
|
it "raises an exception if the `size_file` is 0" do
|
28
|
-
expect { described_class.new(0) }.to raise_error(LogStash::ConfigurationError, /need to be
|
28
|
+
expect { described_class.new(0) }.to raise_error(LogStash::ConfigurationError, /need to be greater than 0/)
|
29
29
|
end
|
30
30
|
|
31
31
|
it "raises an exception if the `size_file` is < 0" do
|
32
|
-
expect { described_class.new(-100) }.to raise_error(LogStash::ConfigurationError, /need to be
|
32
|
+
expect { described_class.new(-100) }.to raise_error(LogStash::ConfigurationError, /need to be greater than 0/)
|
33
33
|
end
|
34
34
|
|
35
35
|
context "#needs_periodic?" do
|
data/spec/supports/helpers.rb
CHANGED
@@ -5,6 +5,7 @@ shared_context "setup plugin" do
|
|
5
5
|
let(:bucket) { ENV["AWS_LOGSTASH_TEST_BUCKET"] }
|
6
6
|
let(:access_key_id) { ENV["AWS_ACCESS_KEY_ID"] }
|
7
7
|
let(:secret_access_key) { ENV["AWS_SECRET_ACCESS_KEY"] }
|
8
|
+
let(:session_token) { ENV["AWS_SESSION_TOKEN"] }
|
8
9
|
let(:size_file) { 100 }
|
9
10
|
let(:time_file) { 100 }
|
10
11
|
let(:tags) { [] }
|
@@ -18,6 +19,7 @@ shared_context "setup plugin" do
|
|
18
19
|
"temporary_directory" => temporary_directory,
|
19
20
|
"access_key_id" => access_key_id,
|
20
21
|
"secret_access_key" => secret_access_key,
|
22
|
+
"session_token" => session_token,
|
21
23
|
"size_file" => size_file,
|
22
24
|
"time_file" => time_file,
|
23
25
|
"region" => region,
|
@@ -25,7 +27,7 @@ shared_context "setup plugin" do
|
|
25
27
|
}
|
26
28
|
end
|
27
29
|
|
28
|
-
let(:client_credentials) { Aws::Credentials.new(access_key_id, secret_access_key) }
|
30
|
+
let(:client_credentials) { Aws::Credentials.new(access_key_id, secret_access_key, session_token) }
|
29
31
|
let(:bucket_resource) { Aws::S3::Bucket.new(bucket, { :credentials => client_credentials, :region => region }) }
|
30
32
|
|
31
33
|
subject { LogStash::Outputs::S3.new(options) }
|
Binary file
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-output-s3
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 4.
|
4
|
+
version: 4.4.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2022-
|
11
|
+
date: 2022-12-22 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -129,7 +129,9 @@ files:
|
|
129
129
|
- LICENSE
|
130
130
|
- NOTICE.TXT
|
131
131
|
- README.md
|
132
|
+
- VERSION
|
132
133
|
- docs/index.asciidoc
|
134
|
+
- lib/logstash-output-s3_jars.rb
|
133
135
|
- lib/logstash/outputs/s3.rb
|
134
136
|
- lib/logstash/outputs/s3/file_repository.rb
|
135
137
|
- lib/logstash/outputs/s3/patch.rb
|
@@ -142,6 +144,7 @@ files:
|
|
142
144
|
- lib/logstash/outputs/s3/uploader.rb
|
143
145
|
- lib/logstash/outputs/s3/writable_directory_validator.rb
|
144
146
|
- lib/logstash/outputs/s3/write_bucket_permission_validator.rb
|
147
|
+
- lib/tasks/build.rake
|
145
148
|
- logstash-output-s3.gemspec
|
146
149
|
- spec/integration/dynamic_prefix_spec.rb
|
147
150
|
- spec/integration/gzip_file_spec.rb
|
@@ -164,6 +167,7 @@ files:
|
|
164
167
|
- spec/outputs/s3_spec.rb
|
165
168
|
- spec/spec_helper.rb
|
166
169
|
- spec/supports/helpers.rb
|
170
|
+
- vendor/jar-dependencies/org/logstash/plugins/outputs/s3/logstash-output-s3/4.4.0/logstash-output-s3-4.4.0.jar
|
167
171
|
homepage: http://www.elastic.co/guide/en/logstash/current/index.html
|
168
172
|
licenses:
|
169
173
|
- Apache-2.0
|
@@ -174,6 +178,7 @@ post_install_message:
|
|
174
178
|
rdoc_options: []
|
175
179
|
require_paths:
|
176
180
|
- lib
|
181
|
+
- vendor/jar-dependencies
|
177
182
|
required_ruby_version: !ruby/object:Gem::Requirement
|
178
183
|
requirements:
|
179
184
|
- - ">="
|