logstash-input-s3 3.1.9 → 3.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +4 -0
- data/LICENSE +1 -1
- data/lib/logstash/inputs/s3.rb +22 -12
- data/logstash-input-s3.gemspec +1 -1
- data/spec/fixtures/compressed.log.gzip +0 -0
- data/spec/fixtures/{invalid_utf8.log → invalid_utf8.gbk.log} +1 -1
- data/spec/inputs/s3_spec.rb +8 -1
- metadata +6 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: e1328e9b8c79b50ccbd386fa6f304bafffffbb4a2dfe6921311d9cae022d47c1
|
4
|
+
data.tar.gz: 261262cb55fed216fbcc1799925ccbaf1d75799289b9fa3b298cdb21fd8364a8
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 5a68a70f262ec1d005e122ba4498119a6649aefb430b8d970ba48c3b358691cd76d301f52004e326455edfcc0e2503419264c87b48b2fcdb8e25601370f8fadc
|
7
|
+
data.tar.gz: d9ade73b5c471199fbe37dcbbf92bfff6f86839294d7641bc07666822c994c16c51229bebed7e6221f45d9bb45e33eae6dde580c990bce3054e841959516002f
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,7 @@
|
|
1
|
+
## 3.2.0
|
2
|
+
- Add support for auto-detecting gzip files with `.gzip` extension, in addition to existing support for `*.gz`
|
3
|
+
- Improve performance of gzip decoding by 10x by using Java's Zlib
|
4
|
+
|
1
5
|
## 3.1.9
|
2
6
|
- Change default sincedb path to live in `{path.data}/plugins/inputs/s3` instead of $HOME.
|
3
7
|
Prior Logstash installations (using $HOME default) are automatically migrated.
|
data/LICENSE
CHANGED
data/lib/logstash/inputs/s3.rb
CHANGED
@@ -9,6 +9,14 @@ require "stud/temporary"
|
|
9
9
|
require "aws-sdk"
|
10
10
|
require "logstash/inputs/s3/patch"
|
11
11
|
|
12
|
+
require 'java'
|
13
|
+
java_import java.io.InputStream
|
14
|
+
java_import java.io.InputStreamReader
|
15
|
+
java_import java.io.FileInputStream
|
16
|
+
java_import java.io.BufferedReader
|
17
|
+
java_import java.util.zip.GZIPInputStream
|
18
|
+
java_import java.util.zip.ZipException
|
19
|
+
|
12
20
|
Aws.eager_autoload!
|
13
21
|
# Stream events from files from a S3 bucket.
|
14
22
|
#
|
@@ -253,25 +261,27 @@ class LogStash::Inputs::S3 < LogStash::Inputs::Base
|
|
253
261
|
|
254
262
|
private
|
255
263
|
def read_gzip_file(filename, block)
|
256
|
-
|
257
|
-
|
258
|
-
|
259
|
-
|
260
|
-
|
261
|
-
|
262
|
-
|
263
|
-
break if unused.nil?
|
264
|
-
zio.pos -= unused.length # reset the position to the other block in the stream
|
265
|
-
end
|
264
|
+
file_stream = FileInputStream.new(filename)
|
265
|
+
gzip_stream = GZIPInputStream.new(file_stream)
|
266
|
+
decoder = InputStreamReader.new(gzip_stream, "UTF-8")
|
267
|
+
buffered = BufferedReader.new(decoder)
|
268
|
+
|
269
|
+
while (line = buffered.readLine())
|
270
|
+
block.call(line)
|
266
271
|
end
|
267
|
-
rescue
|
272
|
+
rescue ZipException => e
|
268
273
|
@logger.error("Gzip codec: We cannot uncompress the gzip file", :filename => filename)
|
269
274
|
raise e
|
275
|
+
ensure
|
276
|
+
buffered.close unless buffered.nil?
|
277
|
+
decoder.close unless decoder.nil?
|
278
|
+
gzip_stream.close unless gzip_stream.nil?
|
279
|
+
file_stream.close unless file_stream.nil?
|
270
280
|
end
|
271
281
|
|
272
282
|
private
|
273
283
|
def gzip?(filename)
|
274
|
-
filename.end_with?('.gz')
|
284
|
+
filename.end_with?('.gz','.gzip')
|
275
285
|
end
|
276
286
|
|
277
287
|
private
|
data/logstash-input-s3.gemspec
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
|
3
3
|
s.name = 'logstash-input-s3'
|
4
|
-
s.version = '3.
|
4
|
+
s.version = '3.2.0'
|
5
5
|
s.licenses = ['Apache License (2.0)']
|
6
6
|
s.summary = "Streams events from files in a S3 bucket"
|
7
7
|
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
Binary file
|
@@ -1,2 +1,2 @@
|
|
1
|
-
2015-01-01T02:52:45.866722Z no "GET http://www.logstash.com:80/utfmadness
|
1
|
+
2015-01-01T02:52:45.866722Z no "GET http://www.logstash.com:80/utfmadness/��4od HTTP/1.1"
|
2
2
|
|
data/spec/inputs/s3_spec.rb
CHANGED
@@ -285,6 +285,13 @@ describe LogStash::Inputs::S3 do
|
|
285
285
|
include_examples "generated events"
|
286
286
|
end
|
287
287
|
|
288
|
+
context 'compressed with gzip extension' do
|
289
|
+
let(:log) { double(:key => 'log.gz', :last_modified => Time.now - 2 * day, :content_length => 5) }
|
290
|
+
let(:log_file) { File.join(File.dirname(__FILE__), '..', 'fixtures', 'compressed.log.gzip') }
|
291
|
+
|
292
|
+
include_examples "generated events"
|
293
|
+
end
|
294
|
+
|
288
295
|
context 'plain text' do
|
289
296
|
let(:log_file) { File.join(File.dirname(__FILE__), '..', 'fixtures', 'uncompressed.log') }
|
290
297
|
|
@@ -306,7 +313,7 @@ describe LogStash::Inputs::S3 do
|
|
306
313
|
end
|
307
314
|
|
308
315
|
context 'encoded' do
|
309
|
-
let(:log_file) { File.join(File.dirname(__FILE__), '..', 'fixtures', 'invalid_utf8.log') }
|
316
|
+
let(:log_file) { File.join(File.dirname(__FILE__), '..', 'fixtures', 'invalid_utf8.gbk.log') }
|
310
317
|
|
311
318
|
include_examples "generated events"
|
312
319
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-input-s3
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 3.
|
4
|
+
version: 3.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2018-01-09 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -120,7 +120,8 @@ files:
|
|
120
120
|
- logstash-input-s3.gemspec
|
121
121
|
- spec/fixtures/cloudfront.log
|
122
122
|
- spec/fixtures/compressed.log.gz
|
123
|
-
- spec/fixtures/
|
123
|
+
- spec/fixtures/compressed.log.gzip
|
124
|
+
- spec/fixtures/invalid_utf8.gbk.log
|
124
125
|
- spec/fixtures/json.log
|
125
126
|
- spec/fixtures/json_with_message.log
|
126
127
|
- spec/fixtures/multiline.log
|
@@ -159,7 +160,8 @@ summary: Streams events from files in a S3 bucket
|
|
159
160
|
test_files:
|
160
161
|
- spec/fixtures/cloudfront.log
|
161
162
|
- spec/fixtures/compressed.log.gz
|
162
|
-
- spec/fixtures/
|
163
|
+
- spec/fixtures/compressed.log.gzip
|
164
|
+
- spec/fixtures/invalid_utf8.gbk.log
|
163
165
|
- spec/fixtures/json.log
|
164
166
|
- spec/fixtures/json_with_message.log
|
165
167
|
- spec/fixtures/multiline.log
|