logstash-input-s3-local 3.3.6 → 3.3.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/logstash-input-s3.gemspec +1 -1
- data/spec/inputs/s3_spec.rb +17 -17
- metadata +1 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 345f078e7ee7276bf4ce930b94a2f284e90c814aaa8d6aa257f6dc8b3f2dd530
|
4
|
+
data.tar.gz: ff1f443a7103873b9d75ff7815359ecd136fc1b0e48caff4c82c7b07ba9e82e8
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: '086073ce10e0870f4228e3222e8227492675ebd8a0ad94902560b4f5f39789fa97c04d90ec70076c70d5052e72398990d5d7e63f915e0fa6a93cfbb66d31c780'
|
7
|
+
data.tar.gz: feb5db98fa944605e15591c13eeebf0cc8e2b6db00e3c99b826c172e1b298848a3e2fa9d72c0396450c0f5746b74d013439b882e1afd02ef026da9ce5615f480
|
data/logstash-input-s3.gemspec
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
|
3
3
|
s.name = 'logstash-input-s3-local'
|
4
|
-
s.version = '3.3.
|
4
|
+
s.version = '3.3.7'
|
5
5
|
s.licenses = ['Apache-2.0']
|
6
6
|
s.summary = "Streams events from files in a S3 bucket"
|
7
7
|
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
data/spec/inputs/s3_spec.rb
CHANGED
@@ -9,7 +9,7 @@ require "stud/temporary"
|
|
9
9
|
require "aws-sdk"
|
10
10
|
require "fileutils"
|
11
11
|
|
12
|
-
describe LogStash::Inputs::
|
12
|
+
describe LogStash::Inputs::S3L do
|
13
13
|
let(:temporary_directory) { Stud::Temporary.pathname }
|
14
14
|
let(:sincedb_path) { Stud::Temporary.pathname }
|
15
15
|
let(:day) { 3600 * 24 }
|
@@ -35,14 +35,14 @@ describe LogStash::Inputs::S3 do
|
|
35
35
|
let(:config) { super.merge({ "interval" => 5 }) }
|
36
36
|
|
37
37
|
before do
|
38
|
-
expect_any_instance_of(LogStash::Inputs::
|
38
|
+
expect_any_instance_of(LogStash::Inputs::S3L).to receive(:list_new_files).and_return(TestInfiniteS3Object.new)
|
39
39
|
end
|
40
40
|
|
41
41
|
it_behaves_like "an interruptible input plugin"
|
42
42
|
end
|
43
43
|
|
44
44
|
describe "#register" do
|
45
|
-
subject { LogStash::Inputs::
|
45
|
+
subject { LogStash::Inputs::S3L.new(config) }
|
46
46
|
|
47
47
|
context "with temporary directory" do
|
48
48
|
let(:temporary_directory) { Stud::Temporary.pathname }
|
@@ -54,7 +54,7 @@ describe LogStash::Inputs::S3 do
|
|
54
54
|
end
|
55
55
|
|
56
56
|
describe '#get_s3object' do
|
57
|
-
subject { LogStash::Inputs::
|
57
|
+
subject { LogStash::Inputs::S3L.new(settings) }
|
58
58
|
|
59
59
|
context 'with modern access key options' do
|
60
60
|
let(:settings) {
|
@@ -124,13 +124,13 @@ describe LogStash::Inputs::S3 do
|
|
124
124
|
}
|
125
125
|
|
126
126
|
it 'should allow user to exclude files from the s3 bucket' do
|
127
|
-
plugin = LogStash::Inputs::
|
127
|
+
plugin = LogStash::Inputs::S3L.new(config.merge({ "exclude_pattern" => "^exclude" }))
|
128
128
|
plugin.register
|
129
129
|
expect(plugin.list_new_files).to eq([present_object.key])
|
130
130
|
end
|
131
131
|
|
132
132
|
it 'should support not providing a exclude pattern' do
|
133
|
-
plugin = LogStash::Inputs::
|
133
|
+
plugin = LogStash::Inputs::S3L.new(config)
|
134
134
|
plugin.register
|
135
135
|
expect(plugin.list_new_files).to eq(objects_list.map(&:key))
|
136
136
|
end
|
@@ -144,7 +144,7 @@ describe LogStash::Inputs::S3 do
|
|
144
144
|
}
|
145
145
|
|
146
146
|
it 'should not log that no files were found in the bucket' do
|
147
|
-
plugin = LogStash::Inputs::
|
147
|
+
plugin = LogStash::Inputs::S3L.new(config.merge({ "exclude_pattern" => "^exclude" }))
|
148
148
|
plugin.register
|
149
149
|
allow(plugin.logger).to receive(:debug).with(anything, anything)
|
150
150
|
|
@@ -158,7 +158,7 @@ describe LogStash::Inputs::S3 do
|
|
158
158
|
let(:objects_list) { [] }
|
159
159
|
|
160
160
|
it 'should log that no files were found in the bucket' do
|
161
|
-
plugin = LogStash::Inputs::
|
161
|
+
plugin = LogStash::Inputs::S3L.new(config)
|
162
162
|
plugin.register
|
163
163
|
expect(plugin.logger).to receive(:info).with(/No files found/, anything)
|
164
164
|
expect(plugin.list_new_files).to be_empty
|
@@ -174,7 +174,7 @@ describe LogStash::Inputs::S3 do
|
|
174
174
|
|
175
175
|
allow_any_instance_of(Aws::S3::Bucket).to receive(:objects) { objects_list }
|
176
176
|
|
177
|
-
plugin = LogStash::Inputs::
|
177
|
+
plugin = LogStash::Inputs::S3L.new(config.merge({ 'backup_add_prefix' => 'mybackup',
|
178
178
|
'backup_to_bucket' => config['bucket']}))
|
179
179
|
plugin.register
|
180
180
|
expect(plugin.list_new_files).to eq([present_object.key])
|
@@ -182,9 +182,9 @@ describe LogStash::Inputs::S3 do
|
|
182
182
|
end
|
183
183
|
|
184
184
|
it 'should ignore files older than X' do
|
185
|
-
plugin = LogStash::Inputs::
|
185
|
+
plugin = LogStash::Inputs::S3L.new(config.merge({ 'backup_add_prefix' => 'exclude-this-file'}))
|
186
186
|
|
187
|
-
expect_any_instance_of(LogStash::Inputs::
|
187
|
+
expect_any_instance_of(LogStash::Inputs::S3L::SinceDB::File).to receive(:read).exactly(objects_list.size) { Time.now - day }
|
188
188
|
plugin.register
|
189
189
|
|
190
190
|
expect(plugin.list_new_files).to eq([present_object.key])
|
@@ -200,7 +200,7 @@ describe LogStash::Inputs::S3 do
|
|
200
200
|
|
201
201
|
allow_any_instance_of(Aws::S3::Bucket).to receive(:objects).with(:prefix => prefix) { objects_list }
|
202
202
|
|
203
|
-
plugin = LogStash::Inputs::
|
203
|
+
plugin = LogStash::Inputs::S3L.new(config.merge({ 'prefix' => prefix }))
|
204
204
|
plugin.register
|
205
205
|
expect(plugin.list_new_files).to eq([present_object.key])
|
206
206
|
end
|
@@ -215,14 +215,14 @@ describe LogStash::Inputs::S3 do
|
|
215
215
|
allow_any_instance_of(Aws::S3::Bucket).to receive(:objects) { objects }
|
216
216
|
|
217
217
|
|
218
|
-
plugin = LogStash::Inputs::
|
218
|
+
plugin = LogStash::Inputs::S3L.new(config)
|
219
219
|
plugin.register
|
220
220
|
expect(plugin.list_new_files).to eq(['TWO_DAYS_AGO', 'YESTERDAY', 'TODAY'])
|
221
221
|
end
|
222
222
|
|
223
223
|
describe "when doing backup on the s3" do
|
224
224
|
it 'should copy to another s3 bucket when keeping the original file' do
|
225
|
-
plugin = LogStash::Inputs::
|
225
|
+
plugin = LogStash::Inputs::S3L.new(config.merge({ "backup_to_bucket" => "mybackup"}))
|
226
226
|
plugin.register
|
227
227
|
|
228
228
|
s3object = Aws::S3::Object.new('mybucket', 'testkey')
|
@@ -233,7 +233,7 @@ describe LogStash::Inputs::S3 do
|
|
233
233
|
end
|
234
234
|
|
235
235
|
it 'should copy to another s3 bucket when deleting the original file' do
|
236
|
-
plugin = LogStash::Inputs::
|
236
|
+
plugin = LogStash::Inputs::S3L.new(config.merge({ "backup_to_bucket" => "mybackup", "delete" => true }))
|
237
237
|
plugin.register
|
238
238
|
|
239
239
|
s3object = Aws::S3::Object.new('mybucket', 'testkey')
|
@@ -244,7 +244,7 @@ describe LogStash::Inputs::S3 do
|
|
244
244
|
end
|
245
245
|
|
246
246
|
it 'should add the specified prefix to the backup file' do
|
247
|
-
plugin = LogStash::Inputs::
|
247
|
+
plugin = LogStash::Inputs::S3L.new(config.merge({ "backup_to_bucket" => "mybackup",
|
248
248
|
"backup_add_prefix" => 'backup-' }))
|
249
249
|
plugin.register
|
250
250
|
|
@@ -261,7 +261,7 @@ describe LogStash::Inputs::S3 do
|
|
261
261
|
Stud::Temporary.file do |source_file|
|
262
262
|
backup_file = File.join(backup_dir.to_s, Pathname.new(source_file.path).basename.to_s)
|
263
263
|
|
264
|
-
plugin = LogStash::Inputs::
|
264
|
+
plugin = LogStash::Inputs::S3L.new(config.merge({ "backup_to_dir" => backup_dir }))
|
265
265
|
|
266
266
|
plugin.backup_to_dir(source_file)
|
267
267
|
|