logstash-codec-gzip_lines 0.1.1

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: acfa995154ea52cce3e6a9029e7cb788006b6cc1
4
+ data.tar.gz: 5eb6871e7906aa9a41254315a6cfa24a176f0acd
5
+ SHA512:
6
+ metadata.gz: f20668db3ce6196a1010b86b85b0fc497377f156fe8ea1ddd08e8629c6cdb70992121d3f005c97954db47da1e8304baa6bef3f14d7d2fb0e68b67a1eb8bbaec6
7
+ data.tar.gz: 3892666771ef048e4270c095f213830474d1efea04b1002baae5cb6d25453e8a0379b5dd7e402d5164c5b60961160a799c8309387ae4cfe28b3157944c097daf
data/.gitignore ADDED
@@ -0,0 +1,3 @@
1
+ *.gem
2
+ Gemfile.lock
3
+ .bundle
data/Gemfile ADDED
@@ -0,0 +1,3 @@
1
+ source 'https://rubygems.org'
2
+ gemspec
3
+ gem "logstash", :github => "elasticsearch/logstash", :branch => "1.5"
data/Rakefile ADDED
@@ -0,0 +1,7 @@
1
+ @files=[]
2
+
3
+ task :default do
4
+ system("rake -T")
5
+ end
6
+
7
+ require "logstash/devutils/rake"
data/gzip_lines.rb ADDED
@@ -0,0 +1,32 @@
1
+ # encoding: utf-8
2
+ require "logstash/devutils/rspec/spec_helper"
3
+ require "logstash/codecs/gzip_lines"
4
+ require "logstash/errors"
5
+ require "stringio"
6
+
7
+
8
+ describe LogStash::Codecs::GzipLines do
9
+ let!(:uncompressed_log) do
10
+ # Using format from
11
+ # http://docs.aws.amazon.com/AmazonCloudFront/latest/DeveloperGuide/AccessLogs.html
12
+ str = StringIO.new
13
+
14
+ str << "2010-03-12 23:51:20 SEA4 192.0.2.147 connect 2014 OK bfd8a98bee0840d9b871b7f6ade9908f rtmp://shqshne4jdp4b6.cloudfront.net/cfx/st​ key=value http://player.longtailvideo.com/player.swf http://www.longtailvideo.com/support/jw-player-setup-wizard?example=204 LNX%2010,0,32,18 - - - -\n"
15
+ str << "2010-03-12 23:51:21 SEA4 192.0.2.222 play 3914 OK bfd8a98bee0840d9b871b7f6ade9908f rtmp://shqshne4jdp4b6.cloudfront.net/cfx/st​ key=value http://player.longtailvideo.com/player.swf http://www.longtailvideo.com/support/jw-player-setup-wizard?example=204 LNX%2010,0,32,18 myvideo p=2&q=4 flv 1\n"
16
+
17
+ str.rewind
18
+ str
19
+ end
20
+
21
+ describe "#decode" do
22
+ it "should create events from a gzip file" do
23
+ events = []
24
+
25
+ subject.decode(compress_with_gzip(uncompressed_log)) do |event|
26
+ events << event
27
+ end
28
+
29
+ expect(events.size).to eq(2)
30
+ end
31
+ end
32
+ end
@@ -0,0 +1,46 @@
1
+ # encoding: utf-8
2
+ require "logstash/codecs/base"
3
+ require "logstash/codecs/plain"
4
+ require "logstash/json"
5
+ require "zlib"
6
+
7
+ # This codec will read gzip encoded content
8
+ class LogStash::Codecs::GzipLines < LogStash::Codecs::Base
9
+ config_name "gzip_lines"
10
+
11
+ milestone 3
12
+
13
+ # The character encoding used in this codec. Examples include "UTF-8" and
14
+ # "CP1252"
15
+ #
16
+ # JSON requires valid UTF-8 strings, but in some cases, software that
17
+ # emits JSON does so in another encoding (nxlog, for example). In
18
+ # weird cases like this, you can set the charset setting to the
19
+ # actual encoding of the text and logstash will convert it for you.
20
+ #
21
+ # For nxlog users, you'll want to set this to "CP1252"
22
+ config :charset, :validate => ::Encoding.name_list, :default => "UTF-8"
23
+
24
+ public
25
+ def initialize(params={})
26
+ super(params)
27
+ @converter = LogStash::Util::Charset.new(@charset)
28
+ @converter.logger = @logger
29
+ end
30
+
31
+ public
32
+ def decode(data)
33
+ @decoder = Zlib::GzipReader.new(data)
34
+
35
+ begin
36
+ @decoder.each_line do |line|
37
+ yield LogStash::Event.new("message" => @converter.convert(line))
38
+ end
39
+ rescue Zlib::Error, Zlib::GzipFile::Error=> e
40
+ file = data.is_a?(String) ? data : data.class
41
+
42
+ @logger.error("Gzip codec: We cannot uncompress the gzip file", :filename => file)
43
+ raise e
44
+ end
45
+ end # def decode
46
+ end # class LogStash::Codecs::GzipLines
@@ -0,0 +1,28 @@
1
+ Gem::Specification.new do |s|
2
+
3
+ s.name = 'logstash-codec-gzip_lines'
4
+ s.version = '0.1.1'
5
+ s.licenses = ['Apache License (2.0)']
6
+ s.summary = 'This codec may be used to decode (via inputs) gzip encoded file'
7
+ s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
8
+ s.authors = ["Elasticsearch"]
9
+ s.email = 'richard.pijnenburg@elasticsearch.com'
10
+ s.homepage = "http://www.elasticsearch.org/guide/en/logstash/current/index.html"
11
+ s.require_paths = ['lib']
12
+
13
+ # Files
14
+ s.files = `git ls-files`.split($\)
15
+
16
+ # Tests
17
+ s.test_files = s.files.grep(%r{^(test|spec|features)/})
18
+
19
+ # Special flag to let us know this is actually a logstash plugin
20
+ s.metadata = { 'logstash_plugin' => 'true', 'group' => 'codec' }
21
+
22
+ # Gem dependencies
23
+ s.add_runtime_dependency 'logstash', '>= 1.4.0', '< 2.0.0'
24
+ s.add_runtime_dependency 'logstash-codec-plain'
25
+
26
+ s.add_development_dependency 'logstash-devutils'
27
+ end
28
+
@@ -0,0 +1,43 @@
1
+ # encoding: utf-8
2
+ require "logstash/devutils/rspec/spec_helper"
3
+ require "logstash/codecs/gzip_lines"
4
+ require "logstash/errors"
5
+ require "stringio"
6
+
7
+ def compress_with_gzip(io)
8
+ compressed = StringIO.new('', 'r+b')
9
+
10
+ gzip = Zlib::GzipWriter.new(compressed)
11
+ gzip.write(io.read)
12
+ gzip.finish
13
+
14
+ compressed.rewind
15
+
16
+ compressed
17
+ end
18
+
19
+ describe LogStash::Codecs::GzipLines do
20
+ let!(:uncompressed_log) do
21
+ # Using format from
22
+ # http://docs.aws.amazon.com/AmazonCloudFront/latest/DeveloperGuide/AccessLogs.html
23
+ str = StringIO.new
24
+
25
+ str << "2010-03-12 23:51:20 SEA4 192.0.2.147 connect 2014 OK bfd8a98bee0840d9b871b7f6ade9908f rtmp://shqshne4jdp4b6.cloudfront.net/cfx/st​ key=value http://player.longtailvideo.com/player.swf http://www.longtailvideo.com/support/jw-player-setup-wizard?example=204 LNX%2010,0,32,18 - - - -\n"
26
+ str << "2010-03-12 23:51:21 SEA4 192.0.2.222 play 3914 OK bfd8a98bee0840d9b871b7f6ade9908f rtmp://shqshne4jdp4b6.cloudfront.net/cfx/st​ key=value http://player.longtailvideo.com/player.swf http://www.longtailvideo.com/support/jw-player-setup-wizard?example=204 LNX%2010,0,32,18 myvideo p=2&q=4 flv 1\n"
27
+
28
+ str.rewind
29
+ str
30
+ end
31
+
32
+ describe "#decode" do
33
+ it "should create events from a gzip file" do
34
+ events = []
35
+
36
+ subject.decode(compress_with_gzip(uncompressed_log)) do |event|
37
+ events << event
38
+ end
39
+
40
+ expect(events.size).to eq(2)
41
+ end
42
+ end
43
+ end
metadata ADDED
@@ -0,0 +1,101 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: logstash-codec-gzip_lines
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.1
5
+ platform: ruby
6
+ authors:
7
+ - Elasticsearch
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2014-11-19 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: logstash
15
+ version_requirements: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - '>='
18
+ - !ruby/object:Gem::Version
19
+ version: 1.4.0
20
+ - - <
21
+ - !ruby/object:Gem::Version
22
+ version: 2.0.0
23
+ requirement: !ruby/object:Gem::Requirement
24
+ requirements:
25
+ - - '>='
26
+ - !ruby/object:Gem::Version
27
+ version: 1.4.0
28
+ - - <
29
+ - !ruby/object:Gem::Version
30
+ version: 2.0.0
31
+ prerelease: false
32
+ type: :runtime
33
+ - !ruby/object:Gem::Dependency
34
+ name: logstash-codec-plain
35
+ version_requirements: !ruby/object:Gem::Requirement
36
+ requirements:
37
+ - - '>='
38
+ - !ruby/object:Gem::Version
39
+ version: '0'
40
+ requirement: !ruby/object:Gem::Requirement
41
+ requirements:
42
+ - - '>='
43
+ - !ruby/object:Gem::Version
44
+ version: '0'
45
+ prerelease: false
46
+ type: :runtime
47
+ - !ruby/object:Gem::Dependency
48
+ name: logstash-devutils
49
+ version_requirements: !ruby/object:Gem::Requirement
50
+ requirements:
51
+ - - '>='
52
+ - !ruby/object:Gem::Version
53
+ version: '0'
54
+ requirement: !ruby/object:Gem::Requirement
55
+ requirements:
56
+ - - '>='
57
+ - !ruby/object:Gem::Version
58
+ version: '0'
59
+ prerelease: false
60
+ type: :development
61
+ description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
62
+ email: richard.pijnenburg@elasticsearch.com
63
+ executables: []
64
+ extensions: []
65
+ extra_rdoc_files: []
66
+ files:
67
+ - .gitignore
68
+ - Gemfile
69
+ - Rakefile
70
+ - gzip_lines.rb
71
+ - lib/logstash/codecs/gzip_lines.rb
72
+ - logstash-codec-gzip_lines.gemspec
73
+ - spec/codecs/gzip_lines_spec.rb
74
+ homepage: http://www.elasticsearch.org/guide/en/logstash/current/index.html
75
+ licenses:
76
+ - Apache License (2.0)
77
+ metadata:
78
+ logstash_plugin: 'true'
79
+ group: codec
80
+ post_install_message:
81
+ rdoc_options: []
82
+ require_paths:
83
+ - lib
84
+ required_ruby_version: !ruby/object:Gem::Requirement
85
+ requirements:
86
+ - - '>='
87
+ - !ruby/object:Gem::Version
88
+ version: '0'
89
+ required_rubygems_version: !ruby/object:Gem::Requirement
90
+ requirements:
91
+ - - '>='
92
+ - !ruby/object:Gem::Version
93
+ version: '0'
94
+ requirements: []
95
+ rubyforge_project:
96
+ rubygems_version: 2.4.4
97
+ signing_key:
98
+ specification_version: 4
99
+ summary: This codec may be used to decode (via inputs) gzip encoded file
100
+ test_files:
101
+ - spec/codecs/gzip_lines_spec.rb