logstash-input-s3 3.3.3 → 3.3.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +3 -0
- data/lib/logstash/inputs/s3.rb +6 -2
- data/logstash-input-s3.gemspec +1 -1
- data/spec/inputs/s3_spec.rb +30 -0
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: c14073f2906b9c1958697f2fac9d9c94f63451194494cb6c2b7152d7884fbf5e
|
4
|
+
data.tar.gz: fd650fb97f1624cb0586c47be4c12d5cfd6742135499474854d2f3f05526ee84
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 2dbb14ca04cebda808918a9cd2fec7a1fda0c5d41cb39ba30d86e83e3c62e9eb9ed77dd6e24ff77c5e8cdb2338061bf503ea1bad9dddeaffc65694258548d0eb
|
7
|
+
data.tar.gz: b0f938a69ba7968a7dacfba808eb8d4e983c6c2f01990a324e170579574789b3504db2cc670cfe2c290f3417eea3abbc03331bf5b071f697c1efdf3c4db86eb7
|
data/CHANGELOG.md
CHANGED
data/lib/logstash/inputs/s3.rb
CHANGED
@@ -109,18 +109,22 @@ class LogStash::Inputs::S3 < LogStash::Inputs::Base
|
|
109
109
|
public
|
110
110
|
def list_new_files
|
111
111
|
objects = {}
|
112
|
-
|
112
|
+
found = false
|
113
113
|
begin
|
114
114
|
@s3bucket.objects(:prefix => @prefix).each do |log|
|
115
|
+
found = true
|
115
116
|
@logger.debug("S3 input: Found key", :key => log.key)
|
116
|
-
|
117
|
+
if !ignore_filename?(log.key)
|
117
118
|
if sincedb.newer?(log.last_modified) && log.content_length > 0
|
118
119
|
objects[log.key] = log.last_modified
|
119
120
|
@logger.debug("S3 input: Adding to objects[]", :key => log.key)
|
120
121
|
@logger.debug("objects[] length is: ", :length => objects.length)
|
121
122
|
end
|
123
|
+
else
|
124
|
+
@logger.debug('S3 input: Ignoring', :key => log.key)
|
122
125
|
end
|
123
126
|
end
|
127
|
+
@logger.info('S3 input: No files found in bucket', :prefix => prefix) unless found
|
124
128
|
rescue Aws::Errors::ServiceError => e
|
125
129
|
@logger.error("S3 input: Unable to list objects in bucket", :prefix => prefix, :message => e.message)
|
126
130
|
end
|
data/logstash-input-s3.gemspec
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
|
3
3
|
s.name = 'logstash-input-s3'
|
4
|
-
s.version = '3.3.
|
4
|
+
s.version = '3.3.4'
|
5
5
|
s.licenses = ['Apache License (2.0)']
|
6
6
|
s.summary = "Streams events from files in a S3 bucket"
|
7
7
|
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
data/spec/inputs/s3_spec.rb
CHANGED
@@ -135,6 +135,36 @@ describe LogStash::Inputs::S3 do
|
|
135
135
|
expect(plugin.list_new_files).to eq(objects_list.map(&:key))
|
136
136
|
end
|
137
137
|
|
138
|
+
context 'when all files are excluded from a bucket' do
|
139
|
+
let(:objects_list) {
|
140
|
+
[
|
141
|
+
double(:key => 'exclude-this-file-1', :last_modified => Time.now - 2 * day, :content_length => 100),
|
142
|
+
double(:key => 'exclude/logstash', :last_modified => Time.now - 2 * day, :content_length => 50),
|
143
|
+
]
|
144
|
+
}
|
145
|
+
|
146
|
+
it 'should not log that no files were found in the bucket' do
|
147
|
+
plugin = LogStash::Inputs::S3.new(config.merge({ "exclude_pattern" => "^exclude" }))
|
148
|
+
plugin.register
|
149
|
+
allow(plugin.logger).to receive(:debug).with(anything, anything)
|
150
|
+
|
151
|
+
expect(plugin.logger).not_to receive(:info).with(/No files found/, anything)
|
152
|
+
expect(plugin.logger).to receive(:debug).with(/Ignoring/, anything)
|
153
|
+
expect(plugin.list_new_files).to be_empty
|
154
|
+
end
|
155
|
+
end
|
156
|
+
|
157
|
+
context 'with an empty bucket' do
|
158
|
+
let(:objects_list) { [] }
|
159
|
+
|
160
|
+
it 'should log that no files were found in the bucket' do
|
161
|
+
plugin = LogStash::Inputs::S3.new(config)
|
162
|
+
plugin.register
|
163
|
+
expect(plugin.logger).to receive(:info).with(/No files found/, anything)
|
164
|
+
expect(plugin.list_new_files).to be_empty
|
165
|
+
end
|
166
|
+
end
|
167
|
+
|
138
168
|
context "If the bucket is the same as the backup bucket" do
|
139
169
|
it 'should ignore files from the bucket if they match the backup prefix' do
|
140
170
|
objects_list = [
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-input-s3
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 3.3.
|
4
|
+
version: 3.3.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-05-
|
11
|
+
date: 2018-05-08 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|