logstash-input-s3-local 3.3.6 → 3.3.7
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/logstash-input-s3.gemspec +1 -1
- data/spec/inputs/s3_spec.rb +17 -17
- metadata +1 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 345f078e7ee7276bf4ce930b94a2f284e90c814aaa8d6aa257f6dc8b3f2dd530
|
4
|
+
data.tar.gz: ff1f443a7103873b9d75ff7815359ecd136fc1b0e48caff4c82c7b07ba9e82e8
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: '086073ce10e0870f4228e3222e8227492675ebd8a0ad94902560b4f5f39789fa97c04d90ec70076c70d5052e72398990d5d7e63f915e0fa6a93cfbb66d31c780'
|
7
|
+
data.tar.gz: feb5db98fa944605e15591c13eeebf0cc8e2b6db00e3c99b826c172e1b298848a3e2fa9d72c0396450c0f5746b74d013439b882e1afd02ef026da9ce5615f480
|
data/logstash-input-s3.gemspec
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
|
3
3
|
s.name = 'logstash-input-s3-local'
|
4
|
-
s.version = '3.3.
|
4
|
+
s.version = '3.3.7'
|
5
5
|
s.licenses = ['Apache-2.0']
|
6
6
|
s.summary = "Streams events from files in a S3 bucket"
|
7
7
|
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
data/spec/inputs/s3_spec.rb
CHANGED
@@ -9,7 +9,7 @@ require "stud/temporary"
|
|
9
9
|
require "aws-sdk"
|
10
10
|
require "fileutils"
|
11
11
|
|
12
|
-
describe LogStash::Inputs::
|
12
|
+
describe LogStash::Inputs::S3L do
|
13
13
|
let(:temporary_directory) { Stud::Temporary.pathname }
|
14
14
|
let(:sincedb_path) { Stud::Temporary.pathname }
|
15
15
|
let(:day) { 3600 * 24 }
|
@@ -35,14 +35,14 @@ describe LogStash::Inputs::S3 do
|
|
35
35
|
let(:config) { super.merge({ "interval" => 5 }) }
|
36
36
|
|
37
37
|
before do
|
38
|
-
expect_any_instance_of(LogStash::Inputs::
|
38
|
+
expect_any_instance_of(LogStash::Inputs::S3L).to receive(:list_new_files).and_return(TestInfiniteS3Object.new)
|
39
39
|
end
|
40
40
|
|
41
41
|
it_behaves_like "an interruptible input plugin"
|
42
42
|
end
|
43
43
|
|
44
44
|
describe "#register" do
|
45
|
-
subject { LogStash::Inputs::
|
45
|
+
subject { LogStash::Inputs::S3L.new(config) }
|
46
46
|
|
47
47
|
context "with temporary directory" do
|
48
48
|
let(:temporary_directory) { Stud::Temporary.pathname }
|
@@ -54,7 +54,7 @@ describe LogStash::Inputs::S3 do
|
|
54
54
|
end
|
55
55
|
|
56
56
|
describe '#get_s3object' do
|
57
|
-
subject { LogStash::Inputs::
|
57
|
+
subject { LogStash::Inputs::S3L.new(settings) }
|
58
58
|
|
59
59
|
context 'with modern access key options' do
|
60
60
|
let(:settings) {
|
@@ -124,13 +124,13 @@ describe LogStash::Inputs::S3 do
|
|
124
124
|
}
|
125
125
|
|
126
126
|
it 'should allow user to exclude files from the s3 bucket' do
|
127
|
-
plugin = LogStash::Inputs::
|
127
|
+
plugin = LogStash::Inputs::S3L.new(config.merge({ "exclude_pattern" => "^exclude" }))
|
128
128
|
plugin.register
|
129
129
|
expect(plugin.list_new_files).to eq([present_object.key])
|
130
130
|
end
|
131
131
|
|
132
132
|
it 'should support not providing a exclude pattern' do
|
133
|
-
plugin = LogStash::Inputs::
|
133
|
+
plugin = LogStash::Inputs::S3L.new(config)
|
134
134
|
plugin.register
|
135
135
|
expect(plugin.list_new_files).to eq(objects_list.map(&:key))
|
136
136
|
end
|
@@ -144,7 +144,7 @@ describe LogStash::Inputs::S3 do
|
|
144
144
|
}
|
145
145
|
|
146
146
|
it 'should not log that no files were found in the bucket' do
|
147
|
-
plugin = LogStash::Inputs::
|
147
|
+
plugin = LogStash::Inputs::S3L.new(config.merge({ "exclude_pattern" => "^exclude" }))
|
148
148
|
plugin.register
|
149
149
|
allow(plugin.logger).to receive(:debug).with(anything, anything)
|
150
150
|
|
@@ -158,7 +158,7 @@ describe LogStash::Inputs::S3 do
|
|
158
158
|
let(:objects_list) { [] }
|
159
159
|
|
160
160
|
it 'should log that no files were found in the bucket' do
|
161
|
-
plugin = LogStash::Inputs::
|
161
|
+
plugin = LogStash::Inputs::S3L.new(config)
|
162
162
|
plugin.register
|
163
163
|
expect(plugin.logger).to receive(:info).with(/No files found/, anything)
|
164
164
|
expect(plugin.list_new_files).to be_empty
|
@@ -174,7 +174,7 @@ describe LogStash::Inputs::S3 do
|
|
174
174
|
|
175
175
|
allow_any_instance_of(Aws::S3::Bucket).to receive(:objects) { objects_list }
|
176
176
|
|
177
|
-
plugin = LogStash::Inputs::
|
177
|
+
plugin = LogStash::Inputs::S3L.new(config.merge({ 'backup_add_prefix' => 'mybackup',
|
178
178
|
'backup_to_bucket' => config['bucket']}))
|
179
179
|
plugin.register
|
180
180
|
expect(plugin.list_new_files).to eq([present_object.key])
|
@@ -182,9 +182,9 @@ describe LogStash::Inputs::S3 do
|
|
182
182
|
end
|
183
183
|
|
184
184
|
it 'should ignore files older than X' do
|
185
|
-
plugin = LogStash::Inputs::
|
185
|
+
plugin = LogStash::Inputs::S3L.new(config.merge({ 'backup_add_prefix' => 'exclude-this-file'}))
|
186
186
|
|
187
|
-
expect_any_instance_of(LogStash::Inputs::
|
187
|
+
expect_any_instance_of(LogStash::Inputs::S3L::SinceDB::File).to receive(:read).exactly(objects_list.size) { Time.now - day }
|
188
188
|
plugin.register
|
189
189
|
|
190
190
|
expect(plugin.list_new_files).to eq([present_object.key])
|
@@ -200,7 +200,7 @@ describe LogStash::Inputs::S3 do
|
|
200
200
|
|
201
201
|
allow_any_instance_of(Aws::S3::Bucket).to receive(:objects).with(:prefix => prefix) { objects_list }
|
202
202
|
|
203
|
-
plugin = LogStash::Inputs::
|
203
|
+
plugin = LogStash::Inputs::S3L.new(config.merge({ 'prefix' => prefix }))
|
204
204
|
plugin.register
|
205
205
|
expect(plugin.list_new_files).to eq([present_object.key])
|
206
206
|
end
|
@@ -215,14 +215,14 @@ describe LogStash::Inputs::S3 do
|
|
215
215
|
allow_any_instance_of(Aws::S3::Bucket).to receive(:objects) { objects }
|
216
216
|
|
217
217
|
|
218
|
-
plugin = LogStash::Inputs::
|
218
|
+
plugin = LogStash::Inputs::S3L.new(config)
|
219
219
|
plugin.register
|
220
220
|
expect(plugin.list_new_files).to eq(['TWO_DAYS_AGO', 'YESTERDAY', 'TODAY'])
|
221
221
|
end
|
222
222
|
|
223
223
|
describe "when doing backup on the s3" do
|
224
224
|
it 'should copy to another s3 bucket when keeping the original file' do
|
225
|
-
plugin = LogStash::Inputs::
|
225
|
+
plugin = LogStash::Inputs::S3L.new(config.merge({ "backup_to_bucket" => "mybackup"}))
|
226
226
|
plugin.register
|
227
227
|
|
228
228
|
s3object = Aws::S3::Object.new('mybucket', 'testkey')
|
@@ -233,7 +233,7 @@ describe LogStash::Inputs::S3 do
|
|
233
233
|
end
|
234
234
|
|
235
235
|
it 'should copy to another s3 bucket when deleting the original file' do
|
236
|
-
plugin = LogStash::Inputs::
|
236
|
+
plugin = LogStash::Inputs::S3L.new(config.merge({ "backup_to_bucket" => "mybackup", "delete" => true }))
|
237
237
|
plugin.register
|
238
238
|
|
239
239
|
s3object = Aws::S3::Object.new('mybucket', 'testkey')
|
@@ -244,7 +244,7 @@ describe LogStash::Inputs::S3 do
|
|
244
244
|
end
|
245
245
|
|
246
246
|
it 'should add the specified prefix to the backup file' do
|
247
|
-
plugin = LogStash::Inputs::
|
247
|
+
plugin = LogStash::Inputs::S3L.new(config.merge({ "backup_to_bucket" => "mybackup",
|
248
248
|
"backup_add_prefix" => 'backup-' }))
|
249
249
|
plugin.register
|
250
250
|
|
@@ -261,7 +261,7 @@ describe LogStash::Inputs::S3 do
|
|
261
261
|
Stud::Temporary.file do |source_file|
|
262
262
|
backup_file = File.join(backup_dir.to_s, Pathname.new(source_file.path).basename.to_s)
|
263
263
|
|
264
|
-
plugin = LogStash::Inputs::
|
264
|
+
plugin = LogStash::Inputs::S3L.new(config.merge({ "backup_to_dir" => backup_dir }))
|
265
265
|
|
266
266
|
plugin.backup_to_dir(source_file)
|
267
267
|
|