logstash-output-s3 4.0.2 → 4.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +21 -18
- data/lib/logstash/outputs/s3.rb +8 -8
- data/logstash-output-s3.gemspec +1 -1
- data/spec/outputs/s3_spec.rb +18 -0
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: fa4450798ae34f2e3d086e2468afaae71f2d26a0
|
4
|
+
data.tar.gz: 024ad272f37c82f91dd1c25e4636e6837dbc97b7
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: e9b0b212a751b10ff9de9bc8d16695777ea53a4dfe5de72b91b3aa31c54f5ad7ce4a44da2e089d5d5b70d6f8c531e9dfdbf8696fcf81349f7b3f6ab40ca57dcf
|
7
|
+
data.tar.gz: 1e3bb81b19480cae9370046f728d81feed5afc54eed8a58f81e865eb760a2090618f300052573c5bd66821f2c6c5790d7b81e490fcce5a68f7ead633658aeb87
|
data/CHANGELOG.md
CHANGED
@@ -1,27 +1,30 @@
|
|
1
|
+
## 4.0.3
|
2
|
+
- When configuring the `canned_acl` options of the plugins the role was not applied correctly to the created object: #7
|
3
|
+
|
1
4
|
## 4.0.2
|
2
|
-
- Fixed AWS authentication when using instance profile credentials.
|
5
|
+
- Fixed AWS authentication when using instance profile credentials.
|
3
6
|
|
4
7
|
## 4.0.1
|
5
8
|
- Improved Error logging for S3 validation. Now specific S3 perms errors are logged
|
6
9
|
|
7
10
|
## 4.0.0
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
11
|
+
- This version is a complete rewrite over version 3.0.0 See #103
|
12
|
+
- This Plugin now uses the V2 version of the SDK, this make sure we receive the latest updates and changes.
|
13
|
+
- We now uses S3's `upload_file` instead of reading chunks, this method is more efficient and will uses the multipart with threads if the files is too big.
|
14
|
+
- You can now use the `fieldref` syntax in the prefix to dynamically changes the target with the events it receives.
|
15
|
+
- The Upload queue is now a bounded list, this options is necessary to allow back pressure to be communicated back to the pipeline but its configurable by the user.
|
16
|
+
- If the queue is full the plugin will start the upload in the current thread.
|
17
|
+
- The plugin now threadsafe and support the concurrency model `shared`
|
18
|
+
- The rotation strategy can be selected, the recommended is `size_and_time` that will check for both the configured limits (`size` and `time` are also available)
|
19
|
+
- The `restore` option will now use a separate threadpool with an unbounded queue
|
20
|
+
- The `restore` option will not block the launch of logstash and will uses less resources than the real time path
|
21
|
+
- The plugin now uses `multi_receive_encode`, this will optimize the writes to the files
|
22
|
+
- rotate operation are now batched to reduce the number of IO calls.
|
23
|
+
- Empty file will not be uploaded by any rotation rotation strategy
|
24
|
+
- We now use Concurrent-Ruby for the implementation of the java executor
|
25
|
+
- If you have finer grain permission on prefixes or want faster boot, you can disable the credentials check with `validate_credentials_on_root_bucket`
|
26
|
+
- The credentials check will no longer fails if we can't delete the file
|
27
|
+
- We now have a full suite of integration test for all the defined rotation
|
25
28
|
|
26
29
|
Fixes: #4 #81 #44 #59 #50
|
27
30
|
|
data/lib/logstash/outputs/s3.rb
CHANGED
@@ -262,6 +262,14 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
|
|
262
262
|
prefix_key.gsub(PathValidator.matches_re, PREFIX_KEY_NORMALIZE_CHARACTER)
|
263
263
|
end
|
264
264
|
|
265
|
+
def upload_options
|
266
|
+
{
|
267
|
+
:acl => @canned_acl,
|
268
|
+
:server_side_encryption => @server_side_encryption ? :aes256 : nil,
|
269
|
+
:content_encoding => @encoding == "gzip" ? "gzip" : nil
|
270
|
+
}
|
271
|
+
end
|
272
|
+
|
265
273
|
private
|
266
274
|
# We start a task in the background for check for stale files and make sure we rotate them to S3 if needed.
|
267
275
|
def start_periodic_check
|
@@ -288,14 +296,6 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
|
|
288
296
|
{ :s3_endpoint => region == 'us-east-1' ? 's3.amazonaws.com' : "s3-#{region}.amazonaws.com"}
|
289
297
|
end
|
290
298
|
|
291
|
-
def upload_options
|
292
|
-
{
|
293
|
-
:acl => @cannel_acl,
|
294
|
-
:server_side_encryption => @server_side_encryption ? :aes256 : nil,
|
295
|
-
:content_encoding => @encoding == "gzip" ? "gzip" : nil
|
296
|
-
}
|
297
|
-
end
|
298
|
-
|
299
299
|
def rotate_if_needed(prefixes)
|
300
300
|
prefixes.each do |prefix|
|
301
301
|
# Each file access is thread safe,
|
data/logstash-output-s3.gemspec
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
s.name = 'logstash-output-s3'
|
3
|
-
s.version = '4.0.
|
3
|
+
s.version = '4.0.3'
|
4
4
|
s.licenses = ['Apache-2.0']
|
5
5
|
s.summary = "This plugin was created for store the logstash's events into Amazon Simple Storage Service (Amazon S3)"
|
6
6
|
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
data/spec/outputs/s3_spec.rb
CHANGED
@@ -43,6 +43,24 @@ describe LogStash::Outputs::S3 do
|
|
43
43
|
end
|
44
44
|
end
|
45
45
|
|
46
|
+
describe "Access control list" do
|
47
|
+
context "when configured" do
|
48
|
+
["private", "public_read", "public_read_write", "authenticated_read"].each do |permission|
|
49
|
+
it "should return the configured ACL permissions: #{permission}" do
|
50
|
+
s3 = described_class.new(options.merge({ "canned_acl" => permission }))
|
51
|
+
expect(s3.upload_options).to include(:acl => permission)
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
context "when not configured" do
|
57
|
+
it "uses private as the default" do
|
58
|
+
s3 = described_class.new(options)
|
59
|
+
expect(s3.upload_options).to include(:acl => "private")
|
60
|
+
end
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
46
64
|
describe "temporary directory" do
|
47
65
|
let(:temporary_directory) { Stud::Temporary.pathname }
|
48
66
|
let(:options) { super.merge({ "temporary_directory" => temporary_directory }) }
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-output-s3
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 4.0.
|
4
|
+
version: 4.0.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-01-
|
11
|
+
date: 2017-01-05 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|