logstash-input-s3 3.1.1 → 3.1.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +5 -0
- data/README.md +2 -1
- data/lib/logstash/inputs/s3.rb +15 -6
- data/lib/logstash/inputs/s3/patch.rb +20 -0
- data/logstash-input-s3.gemspec +1 -4
- data/spec/fixtures/multiple_compressed_streams.gz +0 -0
- data/spec/inputs/s3_spec.rb +12 -1
- metadata +6 -31
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 44b70069c72227302a7d1c75314f1c3bcc40c67d
|
4
|
+
data.tar.gz: 3a42b5a34ae5df3f77732dddf7929a91d7494685
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: be6f6e9873262bd8236ed76797c05c547c718e30a283bd5ada48b7566a1eb9da314a3e92b1859c7bd128f3f761fcb1337a6a0af6386066fb89677f4b3eec2b39
|
7
|
+
data.tar.gz: 6ddb29cde0e89777bbf924620f49658ca0400c6c852322a54ec38753a627e40d7252da5d55419b1d7cf9118c2813d12cf9d4b5c11f767dbbad9e63d4189c6fe7
|
data/CHANGELOG.md
CHANGED
data/README.md
CHANGED
@@ -17,6 +17,7 @@ permissions applied to the AWS IAM Policy being used:
|
|
17
17
|
You might also need `s3:DeleteObject` when setting S3 input to delete on read.
|
18
18
|
And the `s3:CreateBucket` permission to create a backup bucket unless already
|
19
19
|
exists.
|
20
|
+
In addition, when `backup_to_bucket` is used, the `s3:PutObject` action is also required.
|
20
21
|
|
21
22
|
For buckets that have versioning enabled, you might need to add additional
|
22
23
|
permissions.
|
@@ -113,4 +114,4 @@ Programming is not a required skill. Whatever you've seen about open source and
|
|
113
114
|
|
114
115
|
It is more important to the community that you are able to contribute.
|
115
116
|
|
116
|
-
For more information about contributing, see the [CONTRIBUTING](https://github.com/elastic/logstash/blob/master/CONTRIBUTING.md) file.
|
117
|
+
For more information about contributing, see the [CONTRIBUTING](https://github.com/elastic/logstash/blob/master/CONTRIBUTING.md) file.
|
data/lib/logstash/inputs/s3.rb
CHANGED
@@ -6,7 +6,10 @@ require "time"
|
|
6
6
|
require "tmpdir"
|
7
7
|
require "stud/interval"
|
8
8
|
require "stud/temporary"
|
9
|
+
require "aws-sdk"
|
10
|
+
require "logstash/inputs/s3/patch"
|
9
11
|
|
12
|
+
Aws.eager_autoload!
|
10
13
|
# Stream events from files from a S3 bucket.
|
11
14
|
#
|
12
15
|
# Each line from each file generates an event.
|
@@ -242,14 +245,20 @@ class LogStash::Inputs::S3 < LogStash::Inputs::Base
|
|
242
245
|
|
243
246
|
private
|
244
247
|
def read_gzip_file(filename, block)
|
245
|
-
|
246
|
-
|
247
|
-
|
248
|
+
# Details about multiple streams and the usage of unused from: http://code.activestate.com/lists/ruby-talk/11168/
|
249
|
+
File.open(filename) do |zio|
|
250
|
+
while true do
|
251
|
+
io = Zlib::GzipReader.new(zio)
|
252
|
+
io.each_line { |line| block.call(line) }
|
253
|
+
unused = io.unused
|
254
|
+
io.finish
|
255
|
+
break if unused.nil?
|
256
|
+
zio.pos -= unused.length # reset the position to the other block in the stream
|
248
257
|
end
|
249
|
-
rescue Zlib::Error, Zlib::GzipFile::Error => e
|
250
|
-
@logger.error("Gzip codec: We cannot uncompress the gzip file", :filename => filename)
|
251
|
-
raise e
|
252
258
|
end
|
259
|
+
rescue Zlib::Error, Zlib::GzipFile::Error => e
|
260
|
+
@logger.error("Gzip codec: We cannot uncompress the gzip file", :filename => filename)
|
261
|
+
raise e
|
253
262
|
end
|
254
263
|
|
255
264
|
private
|
@@ -0,0 +1,20 @@
|
|
1
|
+
# This is patch related to the autoloading and ruby
|
2
|
+
#
|
3
|
+
# The fix exist in jruby 9k but not in the current jruby, not sure when or it will be backported
|
4
|
+
# https://github.com/jruby/jruby/issues/3645
|
5
|
+
#
|
6
|
+
# AWS is doing tricky name discovery in the module to generate the correct error class and
|
7
|
+
# this strategy is bogus in jruby and `eager_autoload` don't fix this issue.
|
8
|
+
#
|
9
|
+
# This will be a short lived patch since AWS is removing the need.
|
10
|
+
# see: https://github.com/aws/aws-sdk-ruby/issues/1301#issuecomment-261115960
|
11
|
+
old_stderr = $stderr
|
12
|
+
|
13
|
+
$stderr = StringIO.new
|
14
|
+
begin
|
15
|
+
module Aws
|
16
|
+
const_set(:S3, Aws::S3)
|
17
|
+
end
|
18
|
+
ensure
|
19
|
+
$stderr = old_stderr
|
20
|
+
end
|
data/logstash-input-s3.gemspec
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
|
3
3
|
s.name = 'logstash-input-s3'
|
4
|
-
s.version = '3.1.
|
4
|
+
s.version = '3.1.2'
|
5
5
|
s.licenses = ['Apache License (2.0)']
|
6
6
|
s.summary = "Stream events from files from a S3 bucket."
|
7
7
|
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
@@ -24,10 +24,7 @@ Gem::Specification.new do |s|
|
|
24
24
|
s.add_runtime_dependency 'logstash-mixin-aws'
|
25
25
|
s.add_runtime_dependency 'stud', '~> 0.0.18'
|
26
26
|
# s.add_runtime_dependency 'aws-sdk-resources', '>= 2.0.33'
|
27
|
-
|
28
27
|
s.add_development_dependency 'logstash-devutils'
|
29
|
-
s.add_development_dependency 'simplecov'
|
30
|
-
s.add_development_dependency 'coveralls'
|
31
28
|
s.add_development_dependency "logstash-codec-json"
|
32
29
|
end
|
33
30
|
|
Binary file
|
data/spec/inputs/s3_spec.rb
CHANGED
@@ -208,9 +208,11 @@ describe LogStash::Inputs::S3 do
|
|
208
208
|
end
|
209
209
|
|
210
210
|
shared_examples "generated events" do
|
211
|
+
let(:events_to_process) { 2 }
|
212
|
+
|
211
213
|
it 'should process events' do
|
212
214
|
events = fetch_events(config)
|
213
|
-
expect(events.size).to eq(
|
215
|
+
expect(events.size).to eq(events_to_process)
|
214
216
|
end
|
215
217
|
|
216
218
|
it "deletes the temporary file" do
|
@@ -251,6 +253,15 @@ describe LogStash::Inputs::S3 do
|
|
251
253
|
include_examples "generated events"
|
252
254
|
end
|
253
255
|
|
256
|
+
context "multiple compressed streams" do
|
257
|
+
let(:log) { double(:key => 'log.gz', :last_modified => Time.now - 2 * day) }
|
258
|
+
let(:log_file) { File.join(File.dirname(__FILE__), '..', 'fixtures', 'multiple_compressed_streams.gz') }
|
259
|
+
|
260
|
+
include_examples "generated events" do
|
261
|
+
let(:events_to_process) { 16 }
|
262
|
+
end
|
263
|
+
end
|
264
|
+
|
254
265
|
context 'compressed' do
|
255
266
|
let(:log) { double(:key => 'log.gz', :last_modified => Time.now - 2 * day) }
|
256
267
|
let(:log_file) { File.join(File.dirname(__FILE__), '..', 'fixtures', 'compressed.log.gz') }
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-input-s3
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 3.1.
|
4
|
+
version: 3.1.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2017-01-24 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -72,34 +72,6 @@ dependencies:
|
|
72
72
|
- - ">="
|
73
73
|
- !ruby/object:Gem::Version
|
74
74
|
version: '0'
|
75
|
-
- !ruby/object:Gem::Dependency
|
76
|
-
requirement: !ruby/object:Gem::Requirement
|
77
|
-
requirements:
|
78
|
-
- - ">="
|
79
|
-
- !ruby/object:Gem::Version
|
80
|
-
version: '0'
|
81
|
-
name: simplecov
|
82
|
-
prerelease: false
|
83
|
-
type: :development
|
84
|
-
version_requirements: !ruby/object:Gem::Requirement
|
85
|
-
requirements:
|
86
|
-
- - ">="
|
87
|
-
- !ruby/object:Gem::Version
|
88
|
-
version: '0'
|
89
|
-
- !ruby/object:Gem::Dependency
|
90
|
-
requirement: !ruby/object:Gem::Requirement
|
91
|
-
requirements:
|
92
|
-
- - ">="
|
93
|
-
- !ruby/object:Gem::Version
|
94
|
-
version: '0'
|
95
|
-
name: coveralls
|
96
|
-
prerelease: false
|
97
|
-
type: :development
|
98
|
-
version_requirements: !ruby/object:Gem::Requirement
|
99
|
-
requirements:
|
100
|
-
- - ">="
|
101
|
-
- !ruby/object:Gem::Version
|
102
|
-
version: '0'
|
103
75
|
- !ruby/object:Gem::Dependency
|
104
76
|
requirement: !ruby/object:Gem::Requirement
|
105
77
|
requirements:
|
@@ -127,11 +99,13 @@ files:
|
|
127
99
|
- NOTICE.TXT
|
128
100
|
- README.md
|
129
101
|
- lib/logstash/inputs/s3.rb
|
102
|
+
- lib/logstash/inputs/s3/patch.rb
|
130
103
|
- logstash-input-s3.gemspec
|
131
104
|
- spec/fixtures/cloudfront.log
|
132
105
|
- spec/fixtures/compressed.log.gz
|
133
106
|
- spec/fixtures/invalid_utf8.log
|
134
107
|
- spec/fixtures/json.log
|
108
|
+
- spec/fixtures/multiple_compressed_streams.gz
|
135
109
|
- spec/fixtures/uncompressed.log
|
136
110
|
- spec/inputs/s3_spec.rb
|
137
111
|
- spec/inputs/sincedb_spec.rb
|
@@ -159,7 +133,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
159
133
|
version: '0'
|
160
134
|
requirements: []
|
161
135
|
rubyforge_project:
|
162
|
-
rubygems_version: 2.
|
136
|
+
rubygems_version: 2.4.8
|
163
137
|
signing_key:
|
164
138
|
specification_version: 4
|
165
139
|
summary: Stream events from files from a S3 bucket.
|
@@ -168,6 +142,7 @@ test_files:
|
|
168
142
|
- spec/fixtures/compressed.log.gz
|
169
143
|
- spec/fixtures/invalid_utf8.log
|
170
144
|
- spec/fixtures/json.log
|
145
|
+
- spec/fixtures/multiple_compressed_streams.gz
|
171
146
|
- spec/fixtures/uncompressed.log
|
172
147
|
- spec/inputs/s3_spec.rb
|
173
148
|
- spec/inputs/sincedb_spec.rb
|