logstash-input-lumberjack 0.1.5 → 0.1.7
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/logstash/inputs/lumberjack.rb +27 -7
- data/logstash-input-lumberjack.gemspec +2 -2
- data/spec/inputs/lumberjack_spec.rb +71 -24
- data/spec/support/logstash_test.rb +23 -0
- metadata +35 -19
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 040459673c2faeacdaf421d43d5eae5fa69960d0
|
4
|
+
data.tar.gz: 23b1f847f4bd65ec54890c9869ec9c8c751119d4
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 3971428610a78bf39614ff760100272a8c820814eaff288d773e748f8401213722e73b2cbd32d8a09cdad50c0f2b967af281a673d89188dce2334e19c6873daa
|
7
|
+
data.tar.gz: a79c6bda1d57f5a7aff678bcd63ba67049910706728b8e695b32d4b96e8dc3adc4d4a37bb93eeabeb3309881a5824ab49b2cad67ab35ba7fb64caf0283c06bd1
|
@@ -29,19 +29,35 @@ class LogStash::Inputs::Lumberjack < LogStash::Inputs::Base
|
|
29
29
|
# SSL key passphrase to use.
|
30
30
|
config :ssl_key_passphrase, :validate => :password
|
31
31
|
|
32
|
+
# Number of maximum clients that the lumberjack input will accept, this allow you
|
33
|
+
# to control the back pressure up to the client and stop logstash to go OOM with
|
34
|
+
# connection. This settings is a temporary solution and will be deprecated really soon.
|
35
|
+
config :max_clients, :validate => :number, :default => 1000, :deprecated => true
|
36
|
+
|
32
37
|
# TODO(sissel): Add CA to authenticate clients with.
|
33
38
|
|
34
39
|
public
|
35
40
|
def register
|
36
41
|
require "lumberjack/server"
|
42
|
+
require "concurrent"
|
43
|
+
require "concurrent/executors"
|
37
44
|
|
38
45
|
@logger.info("Starting lumberjack input listener", :address => "#{@host}:#{@port}")
|
39
46
|
@lumberjack = Lumberjack::Server.new(:address => @host, :port => @port,
|
40
47
|
:ssl_certificate => @ssl_certificate, :ssl_key => @ssl_key,
|
41
48
|
:ssl_key_passphrase => @ssl_key_passphrase)
|
49
|
+
|
50
|
+
# Limit the number of thread that can be created by the
|
51
|
+
# lumberjack output, if the queue is full the input will
|
52
|
+
# start rejecting new connection and raise an exception
|
53
|
+
@threadpool = Concurrent::ThreadPoolExecutor.new(
|
54
|
+
:min_threads => 1,
|
55
|
+
:max_threads => @max_clients,
|
56
|
+
:max_queue => 1, # in concurrent-ruby, bounded queue need to be at least 1.
|
57
|
+
fallback_policy: :abort
|
58
|
+
)
|
42
59
|
end # def register
|
43
60
|
|
44
|
-
public
|
45
61
|
def run(output_queue)
|
46
62
|
while true do
|
47
63
|
accept do |connection, codec|
|
@@ -62,16 +78,20 @@ class LogStash::Inputs::Lumberjack < LogStash::Inputs::Base
|
|
62
78
|
private
|
63
79
|
def accept(&block)
|
64
80
|
connection = @lumberjack.accept # Blocking call that creates a new connection
|
65
|
-
|
81
|
+
|
82
|
+
if @threadpool.length < @threadpool.max_length
|
83
|
+
block.call(connection, @codec.clone)
|
84
|
+
else
|
85
|
+
@logger.warn("Lumberjack input, maximum connection exceeded, new connection are rejected.", :max_clients => @max_clients_queue)
|
86
|
+
connection.close
|
87
|
+
end
|
66
88
|
end
|
67
89
|
|
68
|
-
private
|
69
90
|
def invoke(connection, codec, &block)
|
70
|
-
|
71
|
-
|
72
|
-
block.call(
|
91
|
+
@threadpool.post do
|
92
|
+
connection.run do |fields|
|
93
|
+
block.call(codec, fields.delete("line"), fields)
|
73
94
|
end
|
74
95
|
end
|
75
96
|
end
|
76
|
-
|
77
97
|
end # class LogStash::Inputs::Lumberjack
|
@@ -1,7 +1,7 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
|
3
3
|
s.name = 'logstash-input-lumberjack'
|
4
|
-
s.version = '0.1.
|
4
|
+
s.version = '0.1.7'
|
5
5
|
s.licenses = ['Apache License (2.0)']
|
6
6
|
s.summary = "Receive events using the lumberjack protocol."
|
7
7
|
s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
|
@@ -24,10 +24,10 @@ Gem::Specification.new do |s|
|
|
24
24
|
|
25
25
|
s.add_runtime_dependency 'logstash-codec-plain'
|
26
26
|
s.add_runtime_dependency 'jls-lumberjack', ['>=0.0.20']
|
27
|
+
s.add_runtime_dependency "concurrent-ruby"
|
27
28
|
|
28
29
|
s.add_development_dependency 'logstash-devutils'
|
29
30
|
s.add_development_dependency 'stud'
|
30
31
|
s.add_development_dependency 'logstash-codec-multiline'
|
31
|
-
|
32
32
|
end
|
33
33
|
|
@@ -4,54 +4,101 @@ require "stud/temporary"
|
|
4
4
|
require 'logstash/inputs/lumberjack'
|
5
5
|
require "logstash/codecs/plain"
|
6
6
|
require "logstash/codecs/multiline"
|
7
|
+
require "logstash/event"
|
8
|
+
require "lumberjack/client"
|
9
|
+
require_relative "../support/logstash_test"
|
7
10
|
|
8
11
|
describe LogStash::Inputs::Lumberjack do
|
9
|
-
|
10
|
-
let(:
|
11
|
-
let(:
|
12
|
-
|
12
|
+
let(:connection) { double("connection") }
|
13
|
+
let(:certificate) { LogStashTest.certificate }
|
14
|
+
let(:port) { LogStashTest.random_port }
|
13
15
|
let(:queue) { Queue.new }
|
14
|
-
let(:config) { { "port" => 0, "ssl_certificate" => ssl_cert, "ssl_key" => ssl_key, "type" => "example" } }
|
16
|
+
let(:config) { { "port" => 0, "ssl_certificate" => certificate.ssl_cert, "ssl_key" => certificate.ssl_key, "type" => "example", "tags" => "lumberjack" } }
|
15
17
|
|
16
|
-
|
17
|
-
before do
|
18
|
-
system("openssl req -x509 -batch -nodes -newkey rsa:2048 -keyout #{ssl_key} -out #{ssl_cert} -subj /CN=localhost > /dev/null 2>&1")
|
19
|
-
end
|
18
|
+
subject(:lumberjack) { LogStash::Inputs::Lumberjack.new(config) }
|
20
19
|
|
21
20
|
context "#register" do
|
22
|
-
|
23
21
|
it "raise no exception" do
|
24
22
|
plugin = LogStash::Inputs::Lumberjack.new(config)
|
25
|
-
expect { plugin.register}.not_to raise_error
|
23
|
+
expect { plugin.register }.not_to raise_error
|
26
24
|
end
|
27
25
|
end
|
28
26
|
|
29
27
|
describe "#processing of events" do
|
28
|
+
let(:lines) { {"line" => "one\ntwo\n two.2\nthree\n", "tags" => ["syslog"]} }
|
30
29
|
|
31
|
-
|
30
|
+
before do
|
31
|
+
allow(connection).to receive(:run).and_yield(lines)
|
32
|
+
lumberjack.register
|
33
|
+
expect_any_instance_of(Lumberjack::Server).to receive(:accept).and_return(connection)
|
34
|
+
end
|
32
35
|
|
36
|
+
context "#codecs" do
|
33
37
|
let(:config) do
|
34
|
-
{ "port" =>
|
38
|
+
{ "port" => port, "ssl_certificate" => certificate.ssl_cert, "ssl_key" => certificate.ssl_key,
|
35
39
|
"type" => "example", "codec" => codec }
|
36
40
|
end
|
37
41
|
|
38
42
|
let(:codec) { LogStash::Codecs::Multiline.new("pattern" => '\n', "what" => "previous") }
|
39
|
-
subject!(:lumberjack) { LogStash::Inputs::Lumberjack.new(config) }
|
40
|
-
let(:connection) { double("connection") }
|
41
|
-
let(:lines) { {"line" => "one\ntwo\n two.2\nthree\n"} }
|
42
|
-
|
43
|
-
before(:each) do
|
44
|
-
allow(connection).to receive(:run).and_yield(lines)
|
45
|
-
lumberjack.register
|
46
|
-
end
|
47
|
-
|
48
43
|
it "clone the codec per connection" do
|
49
|
-
expect_any_instance_of(Lumberjack::Server).to receive(:accept).and_return(connection)
|
50
44
|
expect(lumberjack.codec).to receive(:clone).once
|
51
45
|
expect(lumberjack).to receive(:invoke).and_throw(:msg)
|
52
46
|
lumberjack.run(queue)
|
53
47
|
end
|
54
|
-
|
55
48
|
end
|
56
49
|
end
|
50
|
+
|
51
|
+
context "when we have the maximum clients connected" do
|
52
|
+
let(:max_clients) { 1 }
|
53
|
+
let(:window_size) { 1 }
|
54
|
+
let(:config) do
|
55
|
+
{
|
56
|
+
"port" => port,
|
57
|
+
"ssl_certificate" => certificate.ssl_cert,
|
58
|
+
"ssl_key" => certificate.ssl_key,
|
59
|
+
"type" => "testing",
|
60
|
+
"max_clients" => max_clients
|
61
|
+
}
|
62
|
+
end
|
63
|
+
|
64
|
+
let(:client_options) do
|
65
|
+
{
|
66
|
+
:port => port,
|
67
|
+
:address => "127.0.0.1",
|
68
|
+
:ssl_certificate => certificate.ssl_cert,
|
69
|
+
:window_size => window_size
|
70
|
+
}
|
71
|
+
end
|
72
|
+
|
73
|
+
before do
|
74
|
+
lumberjack.register
|
75
|
+
|
76
|
+
@server = Thread.new do
|
77
|
+
lumberjack.run(queue)
|
78
|
+
end
|
79
|
+
|
80
|
+
sleep(0.1) # wait for the server to correctly accept messages
|
81
|
+
end
|
82
|
+
|
83
|
+
after do
|
84
|
+
@server.raise(LogStash::ShutdownSignal)
|
85
|
+
@server.join
|
86
|
+
end
|
87
|
+
|
88
|
+
it "stops accepting new connection" do
|
89
|
+
client1 = Lumberjack::Socket.new(client_options)
|
90
|
+
|
91
|
+
# Since the connection is stopped on the other side and OS X and
|
92
|
+
# linux doesn't behave the same. The client could raise a IOError
|
93
|
+
# or an SSLError. On OSX I had to try to send some data to trip
|
94
|
+
# the error.
|
95
|
+
expect {
|
96
|
+
client2 = Lumberjack::Socket.new(client_options)
|
97
|
+
|
98
|
+
(window_size + 1).times do
|
99
|
+
client2.write_hash({"line" => "message"})
|
100
|
+
end
|
101
|
+
}.to raise_error
|
102
|
+
end
|
103
|
+
end
|
57
104
|
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
require "stud/temporary"
|
2
|
+
module LogStashTest
|
3
|
+
class Certicate
|
4
|
+
attr_reader :ssl_key, :ssl_cert
|
5
|
+
|
6
|
+
def initialize
|
7
|
+
@ssl_cert = Stud::Temporary.pathname("ssl_certificate")
|
8
|
+
@ssl_key = Stud::Temporary.pathname("ssl_key")
|
9
|
+
|
10
|
+
system("openssl req -x509 -batch -nodes -newkey rsa:2048 -keyout #{ssl_key} -out #{ssl_cert} -subj /CN=localhost > /dev/null 2>&1")
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
class << self
|
15
|
+
def certificate
|
16
|
+
Certicate.new
|
17
|
+
end
|
18
|
+
|
19
|
+
def random_port
|
20
|
+
rand(2000..10000)
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
metadata
CHANGED
@@ -1,17 +1,18 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-input-lumberjack
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.7
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-
|
11
|
+
date: 2015-05-05 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
|
-
|
14
|
+
name: logstash-core
|
15
|
+
version_requirements: !ruby/object:Gem::Requirement
|
15
16
|
requirements:
|
16
17
|
- - '>='
|
17
18
|
- !ruby/object:Gem::Version
|
@@ -19,10 +20,7 @@ dependencies:
|
|
19
20
|
- - <
|
20
21
|
- !ruby/object:Gem::Version
|
21
22
|
version: 2.0.0
|
22
|
-
|
23
|
-
prerelease: false
|
24
|
-
type: :runtime
|
25
|
-
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirement: !ruby/object:Gem::Requirement
|
26
24
|
requirements:
|
27
25
|
- - '>='
|
28
26
|
- !ruby/object:Gem::Version
|
@@ -30,76 +28,92 @@ dependencies:
|
|
30
28
|
- - <
|
31
29
|
- !ruby/object:Gem::Version
|
32
30
|
version: 2.0.0
|
31
|
+
prerelease: false
|
32
|
+
type: :runtime
|
33
33
|
- !ruby/object:Gem::Dependency
|
34
|
+
name: logstash-codec-plain
|
35
|
+
version_requirements: !ruby/object:Gem::Requirement
|
36
|
+
requirements:
|
37
|
+
- - '>='
|
38
|
+
- !ruby/object:Gem::Version
|
39
|
+
version: '0'
|
34
40
|
requirement: !ruby/object:Gem::Requirement
|
35
41
|
requirements:
|
36
42
|
- - '>='
|
37
43
|
- !ruby/object:Gem::Version
|
38
44
|
version: '0'
|
39
|
-
name: logstash-codec-plain
|
40
45
|
prerelease: false
|
41
46
|
type: :runtime
|
47
|
+
- !ruby/object:Gem::Dependency
|
48
|
+
name: jls-lumberjack
|
42
49
|
version_requirements: !ruby/object:Gem::Requirement
|
43
50
|
requirements:
|
44
51
|
- - '>='
|
45
52
|
- !ruby/object:Gem::Version
|
46
|
-
version:
|
47
|
-
- !ruby/object:Gem::Dependency
|
53
|
+
version: 0.0.20
|
48
54
|
requirement: !ruby/object:Gem::Requirement
|
49
55
|
requirements:
|
50
56
|
- - '>='
|
51
57
|
- !ruby/object:Gem::Version
|
52
58
|
version: 0.0.20
|
53
|
-
name: jls-lumberjack
|
54
59
|
prerelease: false
|
55
60
|
type: :runtime
|
61
|
+
- !ruby/object:Gem::Dependency
|
62
|
+
name: concurrent-ruby
|
56
63
|
version_requirements: !ruby/object:Gem::Requirement
|
57
64
|
requirements:
|
58
65
|
- - '>='
|
59
66
|
- !ruby/object:Gem::Version
|
60
|
-
version: 0
|
61
|
-
- !ruby/object:Gem::Dependency
|
67
|
+
version: '0'
|
62
68
|
requirement: !ruby/object:Gem::Requirement
|
63
69
|
requirements:
|
64
70
|
- - '>='
|
65
71
|
- !ruby/object:Gem::Version
|
66
72
|
version: '0'
|
67
|
-
name: logstash-devutils
|
68
73
|
prerelease: false
|
69
|
-
type: :
|
74
|
+
type: :runtime
|
75
|
+
- !ruby/object:Gem::Dependency
|
76
|
+
name: logstash-devutils
|
70
77
|
version_requirements: !ruby/object:Gem::Requirement
|
71
78
|
requirements:
|
72
79
|
- - '>='
|
73
80
|
- !ruby/object:Gem::Version
|
74
81
|
version: '0'
|
75
|
-
- !ruby/object:Gem::Dependency
|
76
82
|
requirement: !ruby/object:Gem::Requirement
|
77
83
|
requirements:
|
78
84
|
- - '>='
|
79
85
|
- !ruby/object:Gem::Version
|
80
86
|
version: '0'
|
81
|
-
name: stud
|
82
87
|
prerelease: false
|
83
88
|
type: :development
|
89
|
+
- !ruby/object:Gem::Dependency
|
90
|
+
name: stud
|
84
91
|
version_requirements: !ruby/object:Gem::Requirement
|
85
92
|
requirements:
|
86
93
|
- - '>='
|
87
94
|
- !ruby/object:Gem::Version
|
88
95
|
version: '0'
|
89
|
-
- !ruby/object:Gem::Dependency
|
90
96
|
requirement: !ruby/object:Gem::Requirement
|
91
97
|
requirements:
|
92
98
|
- - '>='
|
93
99
|
- !ruby/object:Gem::Version
|
94
100
|
version: '0'
|
95
|
-
name: logstash-codec-multiline
|
96
101
|
prerelease: false
|
97
102
|
type: :development
|
103
|
+
- !ruby/object:Gem::Dependency
|
104
|
+
name: logstash-codec-multiline
|
98
105
|
version_requirements: !ruby/object:Gem::Requirement
|
99
106
|
requirements:
|
100
107
|
- - '>='
|
101
108
|
- !ruby/object:Gem::Version
|
102
109
|
version: '0'
|
110
|
+
requirement: !ruby/object:Gem::Requirement
|
111
|
+
requirements:
|
112
|
+
- - '>='
|
113
|
+
- !ruby/object:Gem::Version
|
114
|
+
version: '0'
|
115
|
+
prerelease: false
|
116
|
+
type: :development
|
103
117
|
description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
|
104
118
|
email: info@elastic.co
|
105
119
|
executables: []
|
@@ -116,6 +130,7 @@ files:
|
|
116
130
|
- logstash-input-lumberjack.gemspec
|
117
131
|
- spec/inputs/lumberjack_spec.rb
|
118
132
|
- spec/spec_helper.rb
|
133
|
+
- spec/support/logstash_test.rb
|
119
134
|
homepage: http://www.elastic.co/guide/en/logstash/current/index.html
|
120
135
|
licenses:
|
121
136
|
- Apache License (2.0)
|
@@ -145,3 +160,4 @@ summary: Receive events using the lumberjack protocol.
|
|
145
160
|
test_files:
|
146
161
|
- spec/inputs/lumberjack_spec.rb
|
147
162
|
- spec/spec_helper.rb
|
163
|
+
- spec/support/logstash_test.rb
|