logstash-output-s3 3.2.0 → 4.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +21 -0
  3. data/lib/logstash/outputs/s3.rb +188 -308
  4. data/lib/logstash/outputs/s3/file_repository.rb +120 -0
  5. data/lib/logstash/outputs/s3/patch.rb +22 -0
  6. data/lib/logstash/outputs/s3/path_validator.rb +18 -0
  7. data/lib/logstash/outputs/s3/size_and_time_rotation_policy.rb +24 -0
  8. data/lib/logstash/outputs/s3/size_rotation_policy.rb +26 -0
  9. data/lib/logstash/outputs/s3/temporary_file.rb +71 -0
  10. data/lib/logstash/outputs/s3/temporary_file_factory.rb +123 -0
  11. data/lib/logstash/outputs/s3/time_rotation_policy.rb +26 -0
  12. data/lib/logstash/outputs/s3/uploader.rb +59 -0
  13. data/lib/logstash/outputs/s3/writable_directory_validator.rb +17 -0
  14. data/lib/logstash/outputs/s3/write_bucket_permission_validator.rb +49 -0
  15. data/logstash-output-s3.gemspec +2 -2
  16. data/spec/integration/dynamic_prefix_spec.rb +92 -0
  17. data/spec/integration/gzip_file_spec.rb +62 -0
  18. data/spec/integration/gzip_size_rotation_spec.rb +63 -0
  19. data/spec/integration/restore_from_crash_spec.rb +39 -0
  20. data/spec/integration/size_rotation_spec.rb +59 -0
  21. data/spec/integration/stress_test_spec.rb +60 -0
  22. data/spec/integration/time_based_rotation_with_constant_write_spec.rb +60 -0
  23. data/spec/integration/time_based_rotation_with_stale_write_spec.rb +60 -0
  24. data/spec/integration/upload_current_file_on_shutdown_spec.rb +51 -0
  25. data/spec/outputs/s3/file_repository_spec.rb +146 -0
  26. data/spec/outputs/s3/size_and_time_rotation_policy_spec.rb +77 -0
  27. data/spec/outputs/s3/size_rotation_policy_spec.rb +41 -0
  28. data/spec/outputs/s3/temporary_file_factory_spec.rb +85 -0
  29. data/spec/outputs/s3/temporary_file_spec.rb +40 -0
  30. data/spec/outputs/s3/time_rotation_policy_spec.rb +60 -0
  31. data/spec/outputs/s3/uploader_spec.rb +57 -0
  32. data/spec/outputs/s3/writable_directory_validator_spec.rb +40 -0
  33. data/spec/outputs/s3/write_bucket_permission_validator_spec.rb +38 -0
  34. data/spec/outputs/s3_spec.rb +52 -335
  35. data/spec/spec_helper.rb +6 -0
  36. data/spec/supports/helpers.rb +33 -9
  37. metadata +65 -4
  38. data/spec/integration/s3_spec.rb +0 -97
@@ -0,0 +1,6 @@
1
+ # encoding: utf-8
2
+ require "logstash/devutils/rspec/spec_helper"
3
+ require_relative "supports/helpers"
4
+ require "logstash/logging/logger"
5
+
6
+ LogStash::Logging::Logger::configure_logging("debug") if ENV["DEBUG"]
@@ -1,14 +1,38 @@
1
- def delete_matching_keys_on_bucket(prefix)
2
- s3_object.buckets[minimal_settings["bucket"]].objects.with_prefix(prefix).each do |obj|
3
- obj.delete
1
+ # encoding: utf-8
2
+ shared_context "setup plugin" do
3
+ let(:temporary_directory) { Stud::Temporary.pathname }
4
+
5
+ let(:bucket) { ENV["AWS_LOGSTASH_TEST_BUCKET"] }
6
+ let(:access_key_id) { ENV["AWS_ACCESS_KEY_ID"] }
7
+ let(:secret_access_key) { ENV["AWS_SECRET_ACCESS_KEY"] }
8
+ let(:size_file) { 100 }
9
+ let(:time_file) { 100 }
10
+ let(:tags) { [] }
11
+ let(:prefix) { "home" }
12
+ let(:region) { "us-east-1" }
13
+
14
+ let(:main_options) do
15
+ {
16
+ "bucket" => bucket,
17
+ "prefix" => prefix,
18
+ "temporary_directory" => temporary_directory,
19
+ "access_key_id" => access_key_id,
20
+ "secret_access_key" => secret_access_key,
21
+ "size_file" => size_file,
22
+ "time_file" => time_file,
23
+ "region" => region,
24
+ "tags" => []
25
+ }
4
26
  end
5
- end
6
27
 
7
- def key_exists_on_bucket?(key)
8
- s3_object.buckets[minimal_settings["bucket"]].objects[key].exists?
9
- end
28
+ let(:client_credentials) { Aws::Credentials.new(access_key_id, secret_access_key) }
29
+ let(:bucket_resource) { Aws::S3::Bucket.new(bucket, { :credentials => client_credentials, :region => region }) }
10
30
 
11
- def events_in_files(files)
12
- files.collect { |file| File.foreach(file).count }.inject(&:+)
31
+ subject { LogStash::Outputs::S3.new(options) }
13
32
  end
14
33
 
34
+ def clean_remote_files(prefix = "")
35
+ bucket_resource.objects(:prefix => prefix).each do |object|
36
+ object.delete
37
+ end
38
+ end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-output-s3
3
3
  version: !ruby/object:Gem::Version
4
- version: 3.2.0
4
+ version: 4.0.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Elastic
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-09-12 00:00:00.000000000 Z
11
+ date: 2016-12-15 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
@@ -44,6 +44,20 @@ dependencies:
44
44
  - - ">="
45
45
  - !ruby/object:Gem::Version
46
46
  version: '0'
47
+ - !ruby/object:Gem::Dependency
48
+ requirement: !ruby/object:Gem::Requirement
49
+ requirements:
50
+ - - ">="
51
+ - !ruby/object:Gem::Version
52
+ version: '0'
53
+ name: concurrent-ruby
54
+ prerelease: false
55
+ type: :runtime
56
+ version_requirements: !ruby/object:Gem::Requirement
57
+ requirements:
58
+ - - ">="
59
+ - !ruby/object:Gem::Version
60
+ version: '0'
47
61
  - !ruby/object:Gem::Dependency
48
62
  requirement: !ruby/object:Gem::Requirement
49
63
  requirements:
@@ -114,9 +128,38 @@ files:
114
128
  - NOTICE.TXT
115
129
  - README.md
116
130
  - lib/logstash/outputs/s3.rb
131
+ - lib/logstash/outputs/s3/file_repository.rb
132
+ - lib/logstash/outputs/s3/patch.rb
133
+ - lib/logstash/outputs/s3/path_validator.rb
134
+ - lib/logstash/outputs/s3/size_and_time_rotation_policy.rb
135
+ - lib/logstash/outputs/s3/size_rotation_policy.rb
136
+ - lib/logstash/outputs/s3/temporary_file.rb
137
+ - lib/logstash/outputs/s3/temporary_file_factory.rb
138
+ - lib/logstash/outputs/s3/time_rotation_policy.rb
139
+ - lib/logstash/outputs/s3/uploader.rb
140
+ - lib/logstash/outputs/s3/writable_directory_validator.rb
141
+ - lib/logstash/outputs/s3/write_bucket_permission_validator.rb
117
142
  - logstash-output-s3.gemspec
118
- - spec/integration/s3_spec.rb
143
+ - spec/integration/dynamic_prefix_spec.rb
144
+ - spec/integration/gzip_file_spec.rb
145
+ - spec/integration/gzip_size_rotation_spec.rb
146
+ - spec/integration/restore_from_crash_spec.rb
147
+ - spec/integration/size_rotation_spec.rb
148
+ - spec/integration/stress_test_spec.rb
149
+ - spec/integration/time_based_rotation_with_constant_write_spec.rb
150
+ - spec/integration/time_based_rotation_with_stale_write_spec.rb
151
+ - spec/integration/upload_current_file_on_shutdown_spec.rb
152
+ - spec/outputs/s3/file_repository_spec.rb
153
+ - spec/outputs/s3/size_and_time_rotation_policy_spec.rb
154
+ - spec/outputs/s3/size_rotation_policy_spec.rb
155
+ - spec/outputs/s3/temporary_file_factory_spec.rb
156
+ - spec/outputs/s3/temporary_file_spec.rb
157
+ - spec/outputs/s3/time_rotation_policy_spec.rb
158
+ - spec/outputs/s3/uploader_spec.rb
159
+ - spec/outputs/s3/writable_directory_validator_spec.rb
160
+ - spec/outputs/s3/write_bucket_permission_validator_spec.rb
119
161
  - spec/outputs/s3_spec.rb
162
+ - spec/spec_helper.rb
120
163
  - spec/supports/helpers.rb
121
164
  homepage: http://www.elastic.co/guide/en/logstash/current/index.html
122
165
  licenses:
@@ -145,6 +188,24 @@ signing_key:
145
188
  specification_version: 4
146
189
  summary: This plugin was created for store the logstash's events into Amazon Simple Storage Service (Amazon S3)
147
190
  test_files:
148
- - spec/integration/s3_spec.rb
191
+ - spec/integration/dynamic_prefix_spec.rb
192
+ - spec/integration/gzip_file_spec.rb
193
+ - spec/integration/gzip_size_rotation_spec.rb
194
+ - spec/integration/restore_from_crash_spec.rb
195
+ - spec/integration/size_rotation_spec.rb
196
+ - spec/integration/stress_test_spec.rb
197
+ - spec/integration/time_based_rotation_with_constant_write_spec.rb
198
+ - spec/integration/time_based_rotation_with_stale_write_spec.rb
199
+ - spec/integration/upload_current_file_on_shutdown_spec.rb
200
+ - spec/outputs/s3/file_repository_spec.rb
201
+ - spec/outputs/s3/size_and_time_rotation_policy_spec.rb
202
+ - spec/outputs/s3/size_rotation_policy_spec.rb
203
+ - spec/outputs/s3/temporary_file_factory_spec.rb
204
+ - spec/outputs/s3/temporary_file_spec.rb
205
+ - spec/outputs/s3/time_rotation_policy_spec.rb
206
+ - spec/outputs/s3/uploader_spec.rb
207
+ - spec/outputs/s3/writable_directory_validator_spec.rb
208
+ - spec/outputs/s3/write_bucket_permission_validator_spec.rb
149
209
  - spec/outputs/s3_spec.rb
210
+ - spec/spec_helper.rb
150
211
  - spec/supports/helpers.rb
@@ -1,97 +0,0 @@
1
- # encoding: utf-8
2
- require "logstash/devutils/rspec/spec_helper"
3
- require "logstash/outputs/s3"
4
- require 'socket'
5
- require "aws-sdk"
6
- require "fileutils"
7
- require "stud/temporary"
8
- require_relative "../supports/helpers"
9
-
10
- describe LogStash::Outputs::S3, :integration => true, :s3 => true do
11
- before do
12
- Thread.abort_on_exception = true
13
- end
14
-
15
- let!(:minimal_settings) { { "access_key_id" => ENV['AWS_ACCESS_KEY_ID'],
16
- "secret_access_key" => ENV['AWS_SECRET_ACCESS_KEY'],
17
- "bucket" => ENV['AWS_LOGSTASH_TEST_BUCKET'],
18
- "region" => ENV["AWS_REGION"] || "us-east-1",
19
- "temporary_directory" => Stud::Temporary.pathname('temporary_directory') }}
20
-
21
- let!(:s3_object) do
22
- s3output = LogStash::Outputs::S3.new(minimal_settings)
23
- s3output.register
24
- s3output.s3
25
- end
26
-
27
- after(:each) do
28
- delete_matching_keys_on_bucket('studtmp')
29
- delete_matching_keys_on_bucket('my-prefix')
30
- end
31
-
32
- describe "#register" do
33
- it "write a file on the bucket to check permissions" do
34
- s3 = LogStash::Outputs::S3.new(minimal_settings)
35
- expect(s3.register).not_to raise_error
36
- end
37
- end
38
-
39
- describe "#write_on_bucket" do
40
- after(:each) do
41
- File.unlink(fake_data.path)
42
- end
43
-
44
- let!(:fake_data) { Stud::Temporary.file }
45
-
46
- it "should prefix the file on the bucket if a prefix is specified" do
47
- prefix = "my-prefix"
48
-
49
- config = minimal_settings.merge({
50
- "prefix" => prefix,
51
- })
52
-
53
- s3 = LogStash::Outputs::S3.new(config)
54
- s3.register
55
- s3.write_on_bucket(fake_data)
56
-
57
- expect(key_exists_on_bucket?("#{prefix}#{File.basename(fake_data.path)}")).to eq(true)
58
- end
59
-
60
- it 'should use the same local filename if no prefix is specified' do
61
- s3 = LogStash::Outputs::S3.new(minimal_settings)
62
- s3.register
63
- s3.write_on_bucket(fake_data)
64
-
65
- expect(key_exists_on_bucket?(File.basename(fake_data.path))).to eq(true)
66
- end
67
- end
68
-
69
- describe "#move_file_to_bucket" do
70
- let!(:s3) { LogStash::Outputs::S3.new(minimal_settings) }
71
-
72
- before do
73
- s3.register
74
- end
75
-
76
- it "should upload the file if the size > 0" do
77
- tmp = Stud::Temporary.file
78
- allow(File).to receive(:zero?).and_return(false)
79
- s3.move_file_to_bucket(tmp)
80
- expect(key_exists_on_bucket?(File.basename(tmp.path))).to eq(true)
81
- end
82
- end
83
-
84
- describe "#restore_from_crashes" do
85
- it "read the temp directory and upload the matching file to s3" do
86
- Stud::Temporary.pathname do |temp_path|
87
- tempfile = File.open(File.join(temp_path, 'A'), 'w+') { |f| f.write('test')}
88
-
89
- s3 = LogStash::Outputs::S3.new(minimal_settings.merge({ "temporary_directory" => temp_path }))
90
- s3.restore_from_crashes
91
-
92
- expect(File.exist?(tempfile.path)).to eq(false)
93
- expect(key_exists_on_bucket?(File.basename(tempfile.path))).to eq(true)
94
- end
95
- end
96
- end
97
- end