logstash-core 1.5.0.beta2-java → 1.5.0-java
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of logstash-core might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/lib/logstash-core.rb +2 -0
- data/lib/logstash/agent.rb +0 -41
- data/lib/logstash/config/config_ast.rb +62 -29
- data/lib/logstash/config/mixin.rb +3 -3
- data/lib/logstash/environment.rb +37 -100
- data/lib/logstash/event.rb +32 -20
- data/lib/logstash/filters/base.rb +20 -0
- data/lib/logstash/java_integration.rb +72 -18
- data/lib/logstash/namespace.rb +0 -3
- data/lib/logstash/outputs/base.rb +1 -1
- data/lib/logstash/patches/bundler.rb +20 -0
- data/lib/logstash/patches/rubygems.rb +37 -0
- data/lib/logstash/pipeline.rb +59 -39
- data/lib/logstash/runner.rb +4 -50
- data/lib/logstash/util.rb +0 -1
- data/lib/logstash/util/accessors.rb +6 -0
- data/lib/logstash/version.rb +1 -1
- data/locales/en.yml +5 -0
- data/logstash-core.gemspec +51 -0
- data/spec/core/conditionals_spec.rb +428 -0
- data/spec/core/config_mixin_spec.rb +99 -0
- data/spec/core/config_spec.rb +108 -0
- data/spec/core/environment_spec.rb +44 -0
- data/spec/core/event_spec.rb +473 -0
- data/spec/core/pipeline_spec.rb +198 -0
- data/spec/core/plugin_spec.rb +106 -0
- data/spec/core/runner_spec.rb +39 -0
- data/spec/core/timestamp_spec.rb +83 -0
- data/spec/filters/base_spec.rb +318 -0
- data/spec/inputs/base_spec.rb +13 -0
- data/spec/lib/logstash/bundler_spec.rb +120 -0
- data/spec/lib/logstash/java_integration_spec.rb +257 -0
- data/spec/logstash/agent_spec.rb +37 -0
- data/spec/outputs/base_spec.rb +47 -0
- data/spec/spec_helper.rb +1 -0
- data/spec/util/accessors_spec.rb +215 -0
- data/spec/util/charset_spec.rb +74 -0
- data/spec/util/fieldeval_spec.rb +96 -0
- data/spec/util/gemfile_spec.rb +212 -0
- data/spec/util/json_spec.rb +97 -0
- data/spec/util/plugin_version_spec.rb +48 -0
- data/spec/util/retryable_spec.rb +145 -0
- data/spec/util_spec.rb +34 -0
- metadata +96 -253
- data/lib/logstash-event.rb +0 -2
- data/lib/logstash.rb +0 -4
- data/lib/logstash/JRUBY-PR1448.rb +0 -32
- data/lib/logstash/bundler.rb +0 -124
- data/lib/logstash/gemfile.rb +0 -175
- data/lib/logstash/pluginmanager.rb +0 -17
- data/lib/logstash/pluginmanager/install.rb +0 -112
- data/lib/logstash/pluginmanager/list.rb +0 -38
- data/lib/logstash/pluginmanager/main.rb +0 -22
- data/lib/logstash/pluginmanager/maven_tools_patch.rb +0 -12
- data/lib/logstash/pluginmanager/uninstall.rb +0 -49
- data/lib/logstash/pluginmanager/update.rb +0 -50
- data/lib/logstash/pluginmanager/util.rb +0 -88
data/lib/logstash/event.rb
CHANGED
@@ -16,6 +16,13 @@ require "logstash/json"
|
|
16
16
|
class LogStash::ShutdownEvent; end
|
17
17
|
class LogStash::FlushEvent; end
|
18
18
|
|
19
|
+
module LogStash
|
20
|
+
FLUSH = LogStash::FlushEvent.new
|
21
|
+
|
22
|
+
# LogStash::SHUTDOWN is used by plugins
|
23
|
+
SHUTDOWN = LogStash::ShutdownEvent.new
|
24
|
+
end
|
25
|
+
|
19
26
|
# the logstash event object.
|
20
27
|
#
|
21
28
|
# An event is simply a tuple of (timestamp, data).
|
@@ -48,25 +55,26 @@ class LogStash::Event
|
|
48
55
|
TIMESTAMP_FAILURE_TAG = "_timestampparsefailure"
|
49
56
|
TIMESTAMP_FAILURE_FIELD = "_@timestamp"
|
50
57
|
|
58
|
+
METADATA = "@metadata".freeze
|
59
|
+
METADATA_BRACKETS = "[#{METADATA}]".freeze
|
60
|
+
|
51
61
|
# Floats outside of these upper and lower bounds are forcibly converted
|
52
62
|
# to scientific notation by Float#to_s
|
53
63
|
MIN_FLOAT_BEFORE_SCI_NOT = 0.0001
|
54
64
|
MAX_FLOAT_BEFORE_SCI_NOT = 1000000000000000.0
|
55
65
|
|
66
|
+
LOGGER = Cabin::Channel.get(LogStash)
|
67
|
+
|
56
68
|
public
|
57
69
|
def initialize(data = {})
|
58
|
-
@logger = Cabin::Channel.get(LogStash)
|
59
70
|
@cancelled = false
|
60
71
|
@data = data
|
61
72
|
@accessors = LogStash::Util::Accessors.new(data)
|
62
73
|
@data[VERSION] ||= VERSION_ONE
|
63
|
-
|
74
|
+
ts = @data[TIMESTAMP]
|
75
|
+
@data[TIMESTAMP] = ts ? init_timestamp(ts) : LogStash::Timestamp.now
|
64
76
|
|
65
|
-
@metadata =
|
66
|
-
@data.delete("@metadata")
|
67
|
-
else
|
68
|
-
{}
|
69
|
-
end
|
77
|
+
@metadata = @data.delete(METADATA) || {}
|
70
78
|
@metadata_accessors = LogStash::Util::Accessors.new(@metadata)
|
71
79
|
end # def initialize
|
72
80
|
|
@@ -113,9 +121,6 @@ class LogStash::Event
|
|
113
121
|
raise DeprecatedMethod
|
114
122
|
end # def unix_timestamp
|
115
123
|
|
116
|
-
# field-related access
|
117
|
-
METADATA = "@metadata".freeze
|
118
|
-
METADATA_BRACKETS = "[#{METADATA}]".freeze
|
119
124
|
public
|
120
125
|
def [](fieldref)
|
121
126
|
if fieldref.start_with?(METADATA_BRACKETS)
|
@@ -136,6 +141,7 @@ class LogStash::Event
|
|
136
141
|
@metadata_accessors.set(fieldref[METADATA_BRACKETS.length .. -1], value)
|
137
142
|
elsif fieldref == METADATA
|
138
143
|
@metadata = value
|
144
|
+
@metadata_accessors = LogStash::Util::Accessors.new(@metadata)
|
139
145
|
else
|
140
146
|
@accessors.set(fieldref, value)
|
141
147
|
end
|
@@ -171,8 +177,14 @@ class LogStash::Event
|
|
171
177
|
end
|
172
178
|
|
173
179
|
public
|
174
|
-
def include?(
|
175
|
-
|
180
|
+
def include?(fieldref)
|
181
|
+
if fieldref.start_with?(METADATA_BRACKETS)
|
182
|
+
@metadata_accessors.include?(fieldref[METADATA_BRACKETS.length .. -1])
|
183
|
+
elsif fieldref == METADATA
|
184
|
+
true
|
185
|
+
else
|
186
|
+
@accessors.include?(fieldref)
|
187
|
+
end
|
176
188
|
end # def include?
|
177
189
|
|
178
190
|
# Append an event to this one.
|
@@ -224,6 +236,10 @@ class LogStash::Event
|
|
224
236
|
# Take the inside of the %{ ... }
|
225
237
|
key = tok[2 ... -1]
|
226
238
|
|
239
|
+
if key[0] == "+" && !@data.has_key?(TIMESTAMP)
|
240
|
+
raise LogStash::Error, "Unable to format \"#{key}\" in string \"#{format}\", #{TIMESTAMP} field not found"
|
241
|
+
end
|
242
|
+
|
227
243
|
if key == "+%s"
|
228
244
|
# Got %{+%s}, support for unix epoch time
|
229
245
|
next @data[TIMESTAMP].to_i
|
@@ -263,12 +279,12 @@ class LogStash::Event
|
|
263
279
|
|
264
280
|
def init_timestamp(o)
|
265
281
|
begin
|
266
|
-
timestamp =
|
282
|
+
timestamp = LogStash::Timestamp.coerce(o)
|
267
283
|
return timestamp if timestamp
|
268
284
|
|
269
|
-
|
285
|
+
LOGGER.warn("Unrecognized #{TIMESTAMP} value, setting current time to #{TIMESTAMP}, original in #{TIMESTAMP_FAILURE_FIELD}field", :value => o.inspect)
|
270
286
|
rescue LogStash::TimestampParserError => e
|
271
|
-
|
287
|
+
LOGGER.warn("Error parsing #{TIMESTAMP} string, setting current time to #{TIMESTAMP}, original in #{TIMESTAMP_FAILURE_FIELD} field", :value => o.inspect, :exception => e.message)
|
272
288
|
end
|
273
289
|
|
274
290
|
@data["tags"] ||= []
|
@@ -280,11 +296,7 @@ class LogStash::Event
|
|
280
296
|
|
281
297
|
public
|
282
298
|
def to_hash_with_metadata
|
283
|
-
|
284
|
-
to_hash
|
285
|
-
else
|
286
|
-
to_hash.merge("@metadata" => @metadata)
|
287
|
-
end
|
299
|
+
@metadata.empty? ? to_hash : to_hash.merge(METADATA => @metadata)
|
288
300
|
end
|
289
301
|
|
290
302
|
public
|
@@ -1,5 +1,6 @@
|
|
1
1
|
# encoding: utf-8
|
2
2
|
require "logstash/namespace"
|
3
|
+
require "logstash/event"
|
3
4
|
require "logstash/logging"
|
4
5
|
require "logstash/plugin"
|
5
6
|
require "logstash/config/mixin"
|
@@ -145,6 +146,25 @@ class LogStash::Filters::Base < LogStash::Plugin
|
|
145
146
|
raise "#{self.class}#filter must be overidden"
|
146
147
|
end # def filter
|
147
148
|
|
149
|
+
# in 1.5.0 multi_filter is meant to be used in the generated filter function in LogStash::Config::AST::Plugin only
|
150
|
+
# and is temporary until we refactor the filter method interface to accept events list and return events list,
|
151
|
+
# just list in multi_filter see https://github.com/elastic/logstash/issues/2872.
|
152
|
+
# refactoring the filter method will mean updating all plugins which we want to avoid doing for 1.5.0.
|
153
|
+
#
|
154
|
+
# @param events [Array<LogStash::Event] list of events to filter
|
155
|
+
# @return [Array<LogStash::Event] filtered events and any new events generated by the filter
|
156
|
+
public
|
157
|
+
def multi_filter(events)
|
158
|
+
result = []
|
159
|
+
events.each do |event|
|
160
|
+
unless event.cancelled?
|
161
|
+
result << event
|
162
|
+
filter(event){|new_event| result << new_event}
|
163
|
+
end
|
164
|
+
end
|
165
|
+
result
|
166
|
+
end
|
167
|
+
|
148
168
|
public
|
149
169
|
def execute(event, &block)
|
150
170
|
filter(event, &block)
|
@@ -6,27 +6,11 @@ require "java"
|
|
6
6
|
# not test for is_a?(Array) or is_a?(Hash) and we do not want to include tests for
|
7
7
|
# both classes everywhere. see LogStash::JSon.
|
8
8
|
|
9
|
-
class Java::JavaUtil::ArrayList
|
10
|
-
# have ArrayList objects report is_a?(Array) == true
|
11
|
-
def is_a?(clazz)
|
12
|
-
return true if clazz == Array
|
13
|
-
super
|
14
|
-
end
|
15
|
-
end
|
16
|
-
|
17
|
-
class Java::JavaUtil::LinkedHashMap
|
18
|
-
# have LinkedHashMap objects report is_a?(Array) == true
|
19
|
-
def is_a?(clazz)
|
20
|
-
return true if clazz == Hash
|
21
|
-
super
|
22
|
-
end
|
23
|
-
end
|
24
|
-
|
25
9
|
class Array
|
26
10
|
# enable class equivalence between Array and ArrayList
|
27
11
|
# so that ArrayList will work with case o when Array ...
|
28
12
|
def self.===(other)
|
29
|
-
return true if other.is_a?(Java::JavaUtil::
|
13
|
+
return true if other.is_a?(Java::JavaUtil::Collection)
|
30
14
|
super
|
31
15
|
end
|
32
16
|
end
|
@@ -35,7 +19,77 @@ class Hash
|
|
35
19
|
# enable class equivalence between Hash and LinkedHashMap
|
36
20
|
# so that LinkedHashMap will work with case o when Hash ...
|
37
21
|
def self.===(other)
|
38
|
-
return true if other.is_a?(Java::JavaUtil::
|
22
|
+
return true if other.is_a?(Java::JavaUtil::Map)
|
39
23
|
super
|
40
24
|
end
|
41
25
|
end
|
26
|
+
|
27
|
+
# map_mixin to patch LinkedHashMap and HashMap. it must be done directly on the classes,
|
28
|
+
# using a module mixin does not work, and injecting in the Map interface does not work either
|
29
|
+
# but injecting in the class works.
|
30
|
+
|
31
|
+
map_mixin = lambda do
|
32
|
+
# this is a temporary fix to solve a bug in JRuby where classes implementing the Map interface, like LinkedHashMap
|
33
|
+
# have a bug in the has_key? method that is implemented in the Enumerable module that is somehow mixed in the Map interface.
|
34
|
+
# this bug makes has_key? (and all its aliases) return false for a key that has a nil value.
|
35
|
+
# Only LinkedHashMap is patched here because patching the Map interface is not working.
|
36
|
+
# TODO find proper fix, and submit upstream
|
37
|
+
# releavant JRuby files:
|
38
|
+
# https://github.com/jruby/jruby/blob/master/core/src/main/ruby/jruby/java/java_ext/java.util.rb
|
39
|
+
# https://github.com/jruby/jruby/blob/master/core/src/main/java/org/jruby/java/proxies/MapJavaProxy.java
|
40
|
+
def has_key?(key)
|
41
|
+
self.containsKey(key)
|
42
|
+
end
|
43
|
+
alias_method :include?, :has_key?
|
44
|
+
alias_method :member?, :has_key?
|
45
|
+
alias_method :key?, :has_key?
|
46
|
+
|
47
|
+
# Java 8 Map implements a merge method with a different signature from
|
48
|
+
# the Ruby Hash#merge. see https://github.com/jruby/jruby/issues/1249
|
49
|
+
# this can be removed when fixed upstream
|
50
|
+
if ENV_JAVA['java.specification.version'] >= '1.8'
|
51
|
+
def merge(other)
|
52
|
+
dup.merge!(other)
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
Java::JavaUtil::LinkedHashMap.module_exec(&map_mixin)
|
58
|
+
Java::JavaUtil::HashMap.module_exec(&map_mixin)
|
59
|
+
|
60
|
+
module java::util::Map
|
61
|
+
# have Map objects like LinkedHashMap objects report is_a?(Array) == true
|
62
|
+
def is_a?(clazz)
|
63
|
+
return true if clazz == Hash
|
64
|
+
super
|
65
|
+
end
|
66
|
+
end
|
67
|
+
|
68
|
+
module java::util::Collection
|
69
|
+
# have Collections objects like ArrayList report is_a?(Array) == true
|
70
|
+
def is_a?(clazz)
|
71
|
+
return true if clazz == Array
|
72
|
+
super
|
73
|
+
end
|
74
|
+
|
75
|
+
# support the Ruby Array delete method on a Java Collection
|
76
|
+
def delete(o)
|
77
|
+
self.removeAll([o]) ? o : block_given? ? yield : nil
|
78
|
+
end
|
79
|
+
|
80
|
+
# support the Ruby intersection method on Java Collection
|
81
|
+
def &(other)
|
82
|
+
# transform self into a LinkedHashSet to remove duplicates and preserve order as defined by the Ruby Array intersection contract
|
83
|
+
duped = Java::JavaUtil::LinkedHashSet.new(self)
|
84
|
+
duped.retainAll(other)
|
85
|
+
duped
|
86
|
+
end
|
87
|
+
|
88
|
+
# support the Ruby union method on Java Collection
|
89
|
+
def |(other)
|
90
|
+
# transform self into a LinkedHashSet to remove duplicates and preserve order as defined by the Ruby Array union contract
|
91
|
+
duped = Java::JavaUtil::LinkedHashSet.new(self)
|
92
|
+
duped.addAll(other)
|
93
|
+
duped
|
94
|
+
end
|
95
|
+
end
|
data/lib/logstash/namespace.rb
CHANGED
@@ -1,5 +1,4 @@
|
|
1
1
|
# encoding: utf-8
|
2
|
-
#$: << File.join(File.dirname(__FILE__), "..", "..", "vendor", "bundle")
|
3
2
|
|
4
3
|
module LogStash
|
5
4
|
module Inputs; end
|
@@ -12,6 +11,4 @@ module LogStash
|
|
12
11
|
module Util; end
|
13
12
|
module PluginMixins; end
|
14
13
|
module PluginManager; end
|
15
|
-
|
16
|
-
SHUTDOWN = :shutdown
|
17
14
|
end # module LogStash
|
@@ -69,7 +69,7 @@ class LogStash::Outputs::Base < LogStash::Plugin
|
|
69
69
|
else
|
70
70
|
define_singleton_method(:handle, method(:handle_worker))
|
71
71
|
@worker_queue = SizedQueue.new(20)
|
72
|
-
@worker_plugins = @workers.times.map { self.class.new(
|
72
|
+
@worker_plugins = @workers.times.map { self.class.new(@original_params.merge("workers" => 1)) }
|
73
73
|
@worker_plugins.map.with_index do |plugin, i|
|
74
74
|
Thread.new(original_params, @worker_queue) do |params, queue|
|
75
75
|
LogStash::Util::set_thread_name(">#{self.class.config_name}.#{i}")
|
@@ -0,0 +1,20 @@
|
|
1
|
+
# Bundler monkey patches
|
2
|
+
module ::Bundler
|
3
|
+
# Patch bundler to write a .lock file specific to the version of ruby.
|
4
|
+
# This keeps MRI/JRuby/RBX from conflicting over the Gemfile.lock updates
|
5
|
+
module SharedHelpers
|
6
|
+
def default_lockfile
|
7
|
+
ruby = "#{LogStash::Environment.ruby_engine}-#{LogStash::Environment.ruby_abi_version}"
|
8
|
+
Pathname.new("#{default_gemfile}.#{ruby}.lock")
|
9
|
+
end
|
10
|
+
end
|
11
|
+
|
12
|
+
# Add the Bundler.reset! method which has been added in master but is not in 1.7.9.
|
13
|
+
class << self
|
14
|
+
unless self.method_defined?("reset!")
|
15
|
+
def reset!
|
16
|
+
@definition = nil
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
@@ -0,0 +1,37 @@
|
|
1
|
+
# monkey patch RubyGems to silence ffi warnings:
|
2
|
+
#
|
3
|
+
# WARN: Unresolved specs during Gem::Specification.reset:
|
4
|
+
# ffi (>= 0)
|
5
|
+
# WARN: Clearing out unresolved specs.
|
6
|
+
# Please report a bug if this causes problems.
|
7
|
+
#
|
8
|
+
# see https://github.com/elasticsearch/logstash/issues/2556 and https://github.com/rubygems/rubygems/issues/1070
|
9
|
+
#
|
10
|
+
# this code is from Rubygems v2.1.9 in JRuby 1.7.17. Per tickets this issue should be solved at JRuby >= 1.7.20.
|
11
|
+
#
|
12
|
+
# this method implementation works for Rubygems version 2.1.0 and up, verified up to 2.4.6
|
13
|
+
if ::Gem::Version.new(::Gem::VERSION) >= ::Gem::Version.new("2.1.0") && ::Gem::Version.new(::Gem::VERSION) < ::Gem::Version.new("2.5.0")
|
14
|
+
class ::Gem::Specification
|
15
|
+
def self.reset
|
16
|
+
@@dirs = nil
|
17
|
+
::Gem.pre_reset_hooks.each { |hook| hook.call }
|
18
|
+
@@all = nil
|
19
|
+
@@stubs = nil
|
20
|
+
_clear_load_cache
|
21
|
+
unresolved = unresolved_deps
|
22
|
+
unless unresolved.empty?
|
23
|
+
unless (unresolved.size == 1 && unresolved["ffi"])
|
24
|
+
w = "W" + "ARN"
|
25
|
+
warn "#{w}: Unresolved specs during Gem::Specification.reset:"
|
26
|
+
unresolved.values.each do |dep|
|
27
|
+
warn " #{dep}"
|
28
|
+
end
|
29
|
+
warn "#{w}: Clearing out unresolved specs."
|
30
|
+
warn "Please report a bug if this causes problems."
|
31
|
+
end
|
32
|
+
unresolved.clear
|
33
|
+
end
|
34
|
+
::Gem.post_reset_hooks.each { |hook| hook.call }
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
data/lib/logstash/pipeline.rb
CHANGED
@@ -11,8 +11,6 @@ require "logstash/outputs/base"
|
|
11
11
|
|
12
12
|
class LogStash::Pipeline
|
13
13
|
|
14
|
-
FLUSH_EVENT = LogStash::FlushEvent.new
|
15
|
-
|
16
14
|
def initialize(configstr)
|
17
15
|
@logger = Cabin::Channel.get(LogStash)
|
18
16
|
grammar = LogStashConfigParser.new
|
@@ -45,14 +43,19 @@ class LogStash::Pipeline
|
|
45
43
|
@settings = {
|
46
44
|
"filter-workers" => 1,
|
47
45
|
}
|
46
|
+
|
47
|
+
@run_mutex = Mutex.new
|
48
|
+
@ready = false
|
49
|
+
@started = false
|
50
|
+
@input_threads = []
|
48
51
|
end # def initialize
|
49
52
|
|
50
53
|
def ready?
|
51
|
-
|
54
|
+
@run_mutex.synchronize{@ready}
|
52
55
|
end
|
53
56
|
|
54
57
|
def started?
|
55
|
-
|
58
|
+
@run_mutex.synchronize{@started}
|
56
59
|
end
|
57
60
|
|
58
61
|
def configure(setting, value)
|
@@ -71,14 +74,14 @@ class LogStash::Pipeline
|
|
71
74
|
end
|
72
75
|
|
73
76
|
def run
|
74
|
-
@started = true
|
75
|
-
@input_threads = []
|
77
|
+
@run_mutex.synchronize{@started = true}
|
76
78
|
|
77
|
-
|
79
|
+
# synchronize @input_threads between run and shutdown
|
80
|
+
@run_mutex.synchronize{start_inputs}
|
78
81
|
start_filters if filters?
|
79
82
|
start_outputs
|
80
83
|
|
81
|
-
@ready = true
|
84
|
+
@run_mutex.synchronize{@ready = true}
|
82
85
|
|
83
86
|
@logger.info("Pipeline started")
|
84
87
|
@logger.terminal("Logstash startup completed")
|
@@ -113,7 +116,7 @@ class LogStash::Pipeline
|
|
113
116
|
|
114
117
|
def shutdown_filters
|
115
118
|
@flusher_lock.synchronize { @flusher_thread.kill }
|
116
|
-
@input_to_filter.push(LogStash::
|
119
|
+
@input_to_filter.push(LogStash::SHUTDOWN)
|
117
120
|
end
|
118
121
|
|
119
122
|
def wait_filters
|
@@ -122,7 +125,7 @@ class LogStash::Pipeline
|
|
122
125
|
|
123
126
|
def shutdown_outputs
|
124
127
|
# nothing, filters will do this
|
125
|
-
@filter_to_output.push(LogStash::
|
128
|
+
@filter_to_output.push(LogStash::SHUTDOWN)
|
126
129
|
end
|
127
130
|
|
128
131
|
def wait_outputs
|
@@ -134,7 +137,7 @@ class LogStash::Pipeline
|
|
134
137
|
moreinputs = []
|
135
138
|
@inputs.each do |input|
|
136
139
|
if input.threadable && input.threads > 1
|
137
|
-
(input.threads-1).times do |i|
|
140
|
+
(input.threads - 1).times do |i|
|
138
141
|
moreinputs << input.clone
|
139
142
|
end
|
140
143
|
end
|
@@ -154,7 +157,7 @@ class LogStash::Pipeline
|
|
154
157
|
end
|
155
158
|
|
156
159
|
@flusher_lock = Mutex.new
|
157
|
-
@flusher_thread = Thread.new { Stud.interval(5) { @flusher_lock.synchronize { @input_to_filter.push(
|
160
|
+
@flusher_thread = Thread.new { Stud.interval(5) { @flusher_lock.synchronize { @input_to_filter.push(LogStash::FLUSH) } } }
|
158
161
|
end
|
159
162
|
|
160
163
|
def start_outputs
|
@@ -173,7 +176,7 @@ class LogStash::Pipeline
|
|
173
176
|
begin
|
174
177
|
plugin.run(@input_to_filter)
|
175
178
|
rescue LogStash::ShutdownSignal
|
176
|
-
|
179
|
+
# ignore and quit
|
177
180
|
rescue => e
|
178
181
|
if @logger.debug?
|
179
182
|
@logger.error(I18n.t("logstash.pipeline.worker-error-debug",
|
@@ -185,14 +188,23 @@ class LogStash::Pipeline
|
|
185
188
|
:plugin => plugin.inspect, :error => e))
|
186
189
|
end
|
187
190
|
puts e.backtrace if @logger.debug?
|
188
|
-
|
191
|
+
# input teardown must be synchronized since is can be called concurrently by
|
192
|
+
# the input worker thread and from the pipeline thread shutdown method.
|
193
|
+
# this means that input teardown methods must support multiple calls.
|
194
|
+
@run_mutex.synchronize{plugin.teardown}
|
189
195
|
sleep 1
|
190
196
|
retry
|
191
197
|
end
|
192
|
-
rescue LogStash::ShutdownSignal
|
193
|
-
# nothing
|
194
198
|
ensure
|
195
|
-
|
199
|
+
begin
|
200
|
+
# input teardown must be synchronized since is can be called concurrently by
|
201
|
+
# the input worker thread and from the pipeline thread shutdown method.
|
202
|
+
# this means that input teardown methods must support multiple calls.
|
203
|
+
@run_mutex.synchronize{plugin.teardown}
|
204
|
+
rescue LogStash::ShutdownSignal
|
205
|
+
# teardown could receive the ShutdownSignal, retry it
|
206
|
+
retry
|
207
|
+
end
|
196
208
|
end # def inputworker
|
197
209
|
|
198
210
|
def filterworker
|
@@ -203,11 +215,8 @@ class LogStash::Pipeline
|
|
203
215
|
|
204
216
|
case event
|
205
217
|
when LogStash::Event
|
206
|
-
#
|
207
|
-
|
208
|
-
events = []
|
209
|
-
filter(event) { |newevent| events << newevent }
|
210
|
-
events.each { |event| @filter_to_output.push(event) }
|
218
|
+
# filter_func returns all filtered events, including cancelled ones
|
219
|
+
filter_func(event).each { |e| @filter_to_output.push(e) unless e.cancelled? }
|
211
220
|
when LogStash::FlushEvent
|
212
221
|
# handle filter flushing here so that non threadsafe filters (thus only running one filterworker)
|
213
222
|
# don't have to deal with thread safety implementing the flush method
|
@@ -231,8 +240,8 @@ class LogStash::Pipeline
|
|
231
240
|
|
232
241
|
while true
|
233
242
|
event = @filter_to_output.pop
|
234
|
-
break if event
|
235
|
-
|
243
|
+
break if event == LogStash::SHUTDOWN
|
244
|
+
output_func(event)
|
236
245
|
end # while true
|
237
246
|
|
238
247
|
@outputs.each do |output|
|
@@ -246,18 +255,30 @@ class LogStash::Pipeline
|
|
246
255
|
def shutdown
|
247
256
|
@input_threads.each do |thread|
|
248
257
|
# Interrupt all inputs
|
249
|
-
@logger.info("Sending shutdown signal to input thread",
|
250
|
-
|
251
|
-
|
252
|
-
|
253
|
-
|
254
|
-
|
258
|
+
@logger.info("Sending shutdown signal to input thread", :thread => thread)
|
259
|
+
|
260
|
+
# synchronize both ShutdownSignal and teardown below. by synchronizing both
|
261
|
+
# we will avoid potentially sending a shutdown signal when the inputworker is
|
262
|
+
# executing the teardown method.
|
263
|
+
@run_mutex.synchronize do
|
264
|
+
thread.raise(LogStash::ShutdownSignal)
|
265
|
+
begin
|
266
|
+
thread.wakeup # in case it's in blocked IO or sleeping
|
267
|
+
rescue ThreadError
|
268
|
+
end
|
255
269
|
end
|
270
|
+
end
|
256
271
|
|
257
|
-
|
258
|
-
|
259
|
-
|
260
|
-
input
|
272
|
+
# sometimes an input is stuck in a blocking I/O so we need to tell it to teardown directly
|
273
|
+
@inputs.each do |input|
|
274
|
+
begin
|
275
|
+
# input teardown must be synchronized since is can be called concurrently by
|
276
|
+
# the input worker thread and from the pipeline thread shutdown method.
|
277
|
+
# this means that input teardown methods must support multiple calls.
|
278
|
+
@run_mutex.synchronize{input.teardown}
|
279
|
+
rescue LogStash::ShutdownSignal
|
280
|
+
# teardown could receive the ShutdownSignal, retry it
|
281
|
+
retry
|
261
282
|
end
|
262
283
|
end
|
263
284
|
|
@@ -271,12 +292,11 @@ class LogStash::Pipeline
|
|
271
292
|
return klass.new(*args)
|
272
293
|
end
|
273
294
|
|
295
|
+
# for backward compatibility in devutils for the rspec helpers, this method is not used
|
296
|
+
# in the pipeline anymore.
|
274
297
|
def filter(event, &block)
|
275
|
-
|
276
|
-
|
277
|
-
|
278
|
-
def output(event)
|
279
|
-
@output_func.call(event)
|
298
|
+
# filter_func returns all filtered events, including cancelled ones
|
299
|
+
filter_func(event).each { |e| block.call(e) }
|
280
300
|
end
|
281
301
|
|
282
302
|
# perform filters flush and yeild flushed event to the passed block
|