logstash-core 6.8.23-java → 7.0.0.alpha1-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/logstash/agent.rb +69 -85
- data/lib/logstash/api/modules/stats.rb +1 -1
- data/lib/logstash/compiler/lscl.rb +7 -7
- data/lib/logstash/config/config_ast.rb +1 -1
- data/lib/logstash/config/mixin.rb +1 -1
- data/lib/logstash/config/modules_common.rb +3 -3
- data/lib/logstash/dependency_report.rb +1 -2
- data/lib/logstash/environment.rb +4 -9
- data/lib/logstash/event.rb +1 -24
- data/lib/logstash/filter_delegator.rb +69 -2
- data/lib/logstash/filters/base.rb +2 -0
- data/lib/logstash/instrument/metric_store.rb +1 -1
- data/lib/logstash/instrument/periodic_poller/dlq.rb +7 -5
- data/lib/logstash/instrument/periodic_poller/jvm.rb +3 -3
- data/lib/logstash/instrument/periodic_poller/pq.rb +8 -6
- data/lib/logstash/instrument/periodic_pollers.rb +3 -3
- data/lib/logstash/java_pipeline.rb +11 -38
- data/lib/logstash/modules/kibana_config.rb +1 -1
- data/lib/logstash/modules/logstash_config.rb +1 -1
- data/lib/logstash/patches/resolv.rb +32 -17
- data/lib/logstash/pipeline.rb +11 -28
- data/lib/logstash/pipeline_action/base.rb +1 -1
- data/lib/logstash/pipeline_action/create.rb +13 -7
- data/lib/logstash/pipeline_action/reload.rb +12 -35
- data/lib/logstash/pipeline_action/stop.rb +6 -4
- data/lib/logstash/pipeline_settings.rb +1 -2
- data/lib/logstash/plugins/registry.rb +2 -5
- data/lib/logstash/runner.rb +0 -24
- data/lib/logstash/settings.rb +5 -5
- data/lib/logstash/state_resolver.rb +5 -5
- data/lib/logstash/util/duration_formatter.rb +1 -1
- data/lib/logstash/util/safe_uri.rb +0 -1
- data/lib/logstash/util/substitution_variables.rb +1 -22
- data/lib/logstash/util/thread_dump.rb +1 -1
- data/lib/logstash/util.rb +1 -11
- data/lib/logstash-core/version.rb +3 -1
- data/locales/en.yml +7 -16
- data/logstash-core.gemspec +11 -2
- data/spec/logstash/acked_queue_concurrent_stress_spec.rb +2 -2
- data/spec/logstash/agent/converge_spec.rb +31 -25
- data/spec/logstash/agent/metrics_spec.rb +1 -1
- data/spec/logstash/agent_spec.rb +7 -6
- data/spec/logstash/compiler/compiler_spec.rb +0 -28
- data/spec/logstash/config/config_ast_spec.rb +0 -15
- data/spec/logstash/config/mixin_spec.rb +2 -3
- data/spec/logstash/converge_result_spec.rb +1 -1
- data/spec/logstash/environment_spec.rb +4 -4
- data/spec/logstash/event_spec.rb +2 -10
- data/spec/logstash/filter_delegator_spec.rb +12 -2
- data/spec/logstash/filters/base_spec.rb +9 -45
- data/spec/logstash/instrument/periodic_poller/cgroup_spec.rb +2 -0
- data/spec/logstash/instrument/wrapped_write_client_spec.rb +1 -1
- data/spec/logstash/java_filter_delegator_spec.rb +11 -1
- data/spec/logstash/legacy_ruby_event_spec.rb +5 -6
- data/spec/logstash/patches_spec.rb +3 -1
- data/spec/logstash/pipeline_action/create_spec.rb +8 -14
- data/spec/logstash/pipeline_action/reload_spec.rb +9 -16
- data/spec/logstash/pipeline_action/stop_spec.rb +3 -4
- data/spec/logstash/queue_factory_spec.rb +1 -2
- data/spec/logstash/runner_spec.rb +0 -2
- data/spec/logstash/settings/array_coercible_spec.rb +1 -1
- data/spec/logstash/settings/bytes_spec.rb +2 -2
- data/spec/logstash/settings/port_range_spec.rb +1 -1
- data/spec/logstash/settings_spec.rb +0 -10
- data/spec/logstash/state_resolver_spec.rb +22 -26
- data/spec/logstash/util/safe_uri_spec.rb +0 -40
- data/spec/logstash/util/secretstore_spec.rb +1 -1
- data/spec/logstash/util/time_value_spec.rb +1 -1
- data/spec/logstash/util/wrapped_acked_queue_spec.rb +1 -1
- data/spec/logstash/webserver_spec.rb +5 -9
- data/spec/support/matchers.rb +19 -25
- data/spec/support/shared_contexts.rb +3 -3
- data/versions-gem-copy.yml +9 -9
- metadata +31 -44
- data/lib/logstash/patches/resolv_9270.rb +0 -2903
- data/lib/logstash/pipelines_registry.rb +0 -166
- data/lib/logstash/util/lazy_singleton.rb +0 -33
- data/spec/logstash/jruby_version_spec.rb +0 -15
- data/spec/logstash/pipelines_registry_spec.rb +0 -220
@@ -11,12 +11,14 @@ module LogStash module Instrument module PeriodicPoller
|
|
11
11
|
end
|
12
12
|
|
13
13
|
def collect
|
14
|
-
pipelines = @agent.
|
15
|
-
pipelines.
|
16
|
-
|
17
|
-
pipeline.
|
18
|
-
|
14
|
+
pipelines = @agent.with_running_user_defined_pipelines {|pipelines| pipelines}
|
15
|
+
unless pipelines.nil?
|
16
|
+
pipelines.each {|_, pipeline|
|
17
|
+
unless pipeline.nil?
|
18
|
+
pipeline.collect_stats
|
19
|
+
end
|
20
|
+
}
|
19
21
|
end
|
20
22
|
end
|
21
23
|
end
|
22
|
-
end end end
|
24
|
+
end; end; end
|
@@ -11,12 +11,12 @@ module LogStash module Instrument
|
|
11
11
|
class PeriodicPollers
|
12
12
|
attr_reader :metric
|
13
13
|
|
14
|
-
def initialize(metric, queue_type,
|
14
|
+
def initialize(metric, queue_type, pipelines)
|
15
15
|
@metric = metric
|
16
16
|
@periodic_pollers = [PeriodicPoller::Os.new(metric),
|
17
17
|
PeriodicPoller::JVM.new(metric),
|
18
|
-
PeriodicPoller::PersistentQueue.new(metric, queue_type,
|
19
|
-
PeriodicPoller::DeadLetterQueue.new(metric,
|
18
|
+
PeriodicPoller::PersistentQueue.new(metric, queue_type, pipelines),
|
19
|
+
PeriodicPoller::DeadLetterQueue.new(metric, pipelines)]
|
20
20
|
end
|
21
21
|
|
22
22
|
def start
|
@@ -1,7 +1,6 @@
|
|
1
1
|
# encoding: utf-8
|
2
2
|
require "thread"
|
3
3
|
require "concurrent"
|
4
|
-
require "logstash/event"
|
5
4
|
require "logstash/filters/base"
|
6
5
|
require "logstash/inputs/base"
|
7
6
|
require "logstash/outputs/base"
|
@@ -39,23 +38,9 @@ module LogStash; class JavaPipeline < JavaBasePipeline
|
|
39
38
|
@flushRequested = java.util.concurrent.atomic.AtomicBoolean.new(false)
|
40
39
|
@shutdownRequested = java.util.concurrent.atomic.AtomicBoolean.new(false)
|
41
40
|
@outputs_registered = Concurrent::AtomicBoolean.new(false)
|
42
|
-
|
43
|
-
# @finished_execution signals that the pipeline thread has finished its execution
|
44
|
-
# regardless of any exceptions; it will always be true when the thread completes
|
45
41
|
@finished_execution = Concurrent::AtomicBoolean.new(false)
|
46
|
-
|
47
|
-
# @finished_run signals that the run methods called in the pipeline thread was completed
|
48
|
-
# without errors and it will NOT be set if the run method exits from an exception; this
|
49
|
-
# is by design and necessary for the wait_until_started semantic
|
50
|
-
@finished_run = Concurrent::AtomicBoolean.new(false)
|
51
|
-
|
52
|
-
@thread = nil
|
53
42
|
end # def initialize
|
54
43
|
|
55
|
-
def finished_execution?
|
56
|
-
@finished_execution.true?
|
57
|
-
end
|
58
|
-
|
59
44
|
def ready?
|
60
45
|
@ready.value
|
61
46
|
end
|
@@ -97,18 +82,15 @@ module LogStash; class JavaPipeline < JavaBasePipeline
|
|
97
82
|
@logger.debug("Starting pipeline", default_logging_keys)
|
98
83
|
|
99
84
|
@finished_execution.make_false
|
100
|
-
@finished_run.make_false
|
101
85
|
|
102
86
|
@thread = Thread.new do
|
103
87
|
begin
|
104
88
|
LogStash::Util.set_thread_name("pipeline.#{pipeline_id}")
|
105
89
|
run
|
106
|
-
@
|
90
|
+
@finished_execution.make_true
|
107
91
|
rescue => e
|
108
92
|
close
|
109
93
|
logger.error("Pipeline aborted due to error", default_logging_keys(:exception => e, :backtrace => e.backtrace))
|
110
|
-
ensure
|
111
|
-
@finished_execution.make_true
|
112
94
|
end
|
113
95
|
end
|
114
96
|
|
@@ -123,14 +105,15 @@ module LogStash; class JavaPipeline < JavaBasePipeline
|
|
123
105
|
|
124
106
|
def wait_until_started
|
125
107
|
while true do
|
126
|
-
|
127
|
-
|
108
|
+
# This should be changed with an appropriate FSM
|
109
|
+
# It's an edge case, if we have a pipeline with
|
110
|
+
# a generator { count => 1 } its possible that `Thread#alive?` doesn't return true
|
111
|
+
# because the execution of the thread was successful and complete
|
112
|
+
if @finished_execution.true?
|
128
113
|
return true
|
129
114
|
elsif thread.nil? || !thread.alive?
|
130
|
-
# some exception occurred and the thread is dead
|
131
115
|
return false
|
132
116
|
elsif running?
|
133
|
-
# fully initialized and running
|
134
117
|
return true
|
135
118
|
else
|
136
119
|
sleep 0.01
|
@@ -232,11 +215,11 @@ module LogStash; class JavaPipeline < JavaBasePipeline
|
|
232
215
|
|
233
216
|
pipeline_workers.times do |t|
|
234
217
|
thread = Thread.new do
|
235
|
-
Util.set_thread_name("[#{pipeline_id}]>worker#{t}")
|
236
218
|
org.logstash.execution.WorkerLoop.new(
|
237
219
|
lir_execution, filter_queue_client, @events_filtered, @events_consumed,
|
238
220
|
@flushRequested, @flushing, @shutdownRequested, @drain_queue).run
|
239
221
|
end
|
222
|
+
Util.set_thread_name("[#{pipeline_id}]>worker#{t}")
|
240
223
|
@worker_threads << thread
|
241
224
|
end
|
242
225
|
|
@@ -257,13 +240,7 @@ module LogStash; class JavaPipeline < JavaBasePipeline
|
|
257
240
|
end
|
258
241
|
|
259
242
|
def wait_inputs
|
260
|
-
@input_threads.each
|
261
|
-
if thread.class == Java::JavaObject
|
262
|
-
thread.to_java.join
|
263
|
-
else
|
264
|
-
thread.join
|
265
|
-
end
|
266
|
-
end
|
243
|
+
@input_threads.each(&:join)
|
267
244
|
end
|
268
245
|
|
269
246
|
def start_inputs
|
@@ -285,11 +262,7 @@ module LogStash; class JavaPipeline < JavaBasePipeline
|
|
285
262
|
end
|
286
263
|
|
287
264
|
def start_input(plugin)
|
288
|
-
|
289
|
-
@input_threads << plugin.start
|
290
|
-
else
|
291
|
-
@input_threads << Thread.new { inputworker(plugin) }
|
292
|
-
end
|
265
|
+
@input_threads << Thread.new { inputworker(plugin) }
|
293
266
|
end
|
294
267
|
|
295
268
|
def inputworker(plugin)
|
@@ -335,7 +308,7 @@ module LogStash; class JavaPipeline < JavaBasePipeline
|
|
335
308
|
|
336
309
|
stop_inputs
|
337
310
|
|
338
|
-
# We make this call blocking, so we know for sure when the method return the
|
311
|
+
# We make this call blocking, so we know for sure when the method return the shutdown is
|
339
312
|
# stopped
|
340
313
|
wait_for_workers
|
341
314
|
clear_pipeline_metrics
|
@@ -399,7 +372,7 @@ module LogStash; class JavaPipeline < JavaBasePipeline
|
|
399
372
|
end
|
400
373
|
|
401
374
|
def plugin_threads_info
|
402
|
-
input_threads = @input_threads.select {|t| t.
|
375
|
+
input_threads = @input_threads.select {|t| t.alive? }
|
403
376
|
worker_threads = @worker_threads.select {|t| t.alive? }
|
404
377
|
(input_threads + worker_threads).map {|t| Util.thread_info(t) }
|
405
378
|
end
|
@@ -8,7 +8,7 @@ module LogStash module Modules class KibanaConfig
|
|
8
8
|
include LogStash::Util::Loggable
|
9
9
|
|
10
10
|
ALLOWED_DIRECTORIES = ["search", "visualization"]
|
11
|
-
attr_reader :index_name # not used when importing via kibana but for BWC with
|
11
|
+
attr_reader :index_name # not used when importing via kibana but for BWC with ElasticsearchConfig
|
12
12
|
|
13
13
|
# We name it `modul` here because `module` has meaning in Ruby.
|
14
14
|
def initialize(modul, settings)
|
@@ -39,7 +39,7 @@ module LogStash module Modules class LogStashConfig
|
|
39
39
|
def get_setting(setting_class)
|
40
40
|
raw_value = @settings[setting_class.name]
|
41
41
|
# If we dont check for NIL, the Settings class will try to coerce the value
|
42
|
-
# and most of the it will fails when a NIL value is
|
42
|
+
# and most of the it will fails when a NIL value is explicitly set.
|
43
43
|
# This will be fixed once we wrap the plugins settings into a Settings class
|
44
44
|
setting_class.set(raw_value) unless raw_value.nil?
|
45
45
|
setting_class.value
|
@@ -1,25 +1,40 @@
|
|
1
1
|
require "resolv"
|
2
2
|
|
3
|
-
# ref:
|
4
|
-
# https://github.com/logstash-plugins/logstash-filter-dns/issues/51
|
5
|
-
# https://github.com/jruby/jruby/pull/5722
|
3
|
+
# ref: https://github.com/logstash-plugins/logstash-filter-dns/issues/40
|
6
4
|
#
|
7
|
-
# JRuby versions
|
8
|
-
#
|
5
|
+
# JRuby 9k versions prior to 9.1.16.0 have a bug which crashes IP address
|
6
|
+
# resolution after 64k unique IP addresses resolutions.
|
9
7
|
#
|
10
|
-
#
|
11
|
-
#
|
12
|
-
# this will be fixed and we want to avoid potential conflicting monkey patches.
|
13
|
-
# A spec which will break on JRuby upgrade will redirect here
|
14
|
-
# to make a manual verification and eventually remove that patch here once the fix is
|
15
|
-
# made in the JRuby version of resolv.rb.
|
8
|
+
# Note that the oldest JRuby version in LS 6 is 9.1.13.0 and
|
9
|
+
# JRuby 1.7.25 and 1.7.27 (the 2 versions used across LS 5) are not affected by this bug.
|
16
10
|
|
17
|
-
if
|
18
|
-
|
19
|
-
|
20
|
-
$VERBOSE = nil
|
11
|
+
# make sure we abort if a known correct JRuby version is installed
|
12
|
+
# to avoid having an unnecessary legacy patch being applied in the future.
|
13
|
+
raise("Unnecessary patch on resolv.rb for JRuby version 9.1.16+") if Gem::Version.new(JRUBY_VERSION) >= Gem::Version.new("9.1.16.0")
|
21
14
|
|
22
|
-
|
15
|
+
# The code below is copied from JRuby 9.1.16.0 resolv.rb:
|
16
|
+
# https://github.com/jruby/jruby/blob/9.1.16.0/lib/ruby/stdlib/resolv.rb#L775-L784
|
17
|
+
#
|
18
|
+
# JRuby is Copyright (c) 2007-2017 The JRuby project, and is released
|
19
|
+
# under a tri EPL/GPL/LGPL license.
|
20
|
+
# Full license available at https://github.com/jruby/jruby/blob/9.1.16.0/COPYING
|
23
21
|
|
24
|
-
|
22
|
+
class Resolv
|
23
|
+
class DNS
|
24
|
+
class Requester
|
25
|
+
class UnconnectedUDP
|
26
|
+
def sender(msg, data, host, port=Port)
|
27
|
+
sock = @socks_hash[host.index(':') ? "::" : "0.0.0.0"]
|
28
|
+
return nil if !sock
|
29
|
+
service = [IPAddr.new(host), port]
|
30
|
+
id = DNS.allocate_request_id(service[0], service[1])
|
31
|
+
request = msg.encode
|
32
|
+
request[0,2] = [id].pack('n')
|
33
|
+
return @senders[[service, id]] =
|
34
|
+
Sender.new(request, data, sock, host, port)
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
25
39
|
end
|
40
|
+
|
data/lib/logstash/pipeline.rb
CHANGED
@@ -107,23 +107,8 @@ module LogStash; class Pipeline < BasePipeline
|
|
107
107
|
@flushing = Concurrent::AtomicReference.new(false)
|
108
108
|
@outputs_registered = Concurrent::AtomicBoolean.new(false)
|
109
109
|
@worker_shutdown = java.util.concurrent.atomic.AtomicBoolean.new(false)
|
110
|
-
|
111
|
-
# @finished_execution signals that the pipeline thread has finished its execution
|
112
|
-
# regardless of any exceptions; it will always be true when the thread completes
|
113
|
-
@finished_execution = Concurrent::AtomicBoolean.new(false)
|
114
|
-
|
115
|
-
# @finished_run signals that the run methods called in the pipeline thread was completed
|
116
|
-
# without errors and it will NOT be set if the run method exits from an exception; this
|
117
|
-
# is by design and necessary for the wait_until_started semantic
|
118
|
-
@finished_run = Concurrent::AtomicBoolean.new(false)
|
119
|
-
|
120
|
-
@thread = nil
|
121
110
|
end # def initialize
|
122
111
|
|
123
|
-
def finished_execution?
|
124
|
-
@finished_execution.true?
|
125
|
-
end
|
126
|
-
|
127
112
|
def ready?
|
128
113
|
@ready.value
|
129
114
|
end
|
@@ -167,19 +152,16 @@ module LogStash; class Pipeline < BasePipeline
|
|
167
152
|
"pipeline.batch.size" => settings.get("pipeline.batch.size"),
|
168
153
|
"pipeline.batch.delay" => settings.get("pipeline.batch.delay")))
|
169
154
|
|
170
|
-
@finished_execution.
|
171
|
-
@finished_run.make_false
|
155
|
+
@finished_execution = Concurrent::AtomicBoolean.new(false)
|
172
156
|
|
173
157
|
@thread = Thread.new do
|
174
158
|
begin
|
175
159
|
LogStash::Util.set_thread_name("pipeline.#{pipeline_id}")
|
176
160
|
run
|
177
|
-
@
|
161
|
+
@finished_execution.make_true
|
178
162
|
rescue => e
|
179
163
|
close
|
180
164
|
@logger.error("Pipeline aborted due to error", default_logging_keys(:exception => e, :backtrace => e.backtrace))
|
181
|
-
ensure
|
182
|
-
@finished_execution.make_true
|
183
165
|
end
|
184
166
|
end
|
185
167
|
|
@@ -194,14 +176,15 @@ module LogStash; class Pipeline < BasePipeline
|
|
194
176
|
|
195
177
|
def wait_until_started
|
196
178
|
while true do
|
197
|
-
|
198
|
-
|
179
|
+
# This should be changed with an appropriate FSM
|
180
|
+
# It's an edge case, if we have a pipeline with
|
181
|
+
# a generator { count => 1 } its possible that `Thread#alive?` doesn't return true
|
182
|
+
# because the execution of the thread was successful and complete
|
183
|
+
if @finished_execution.true?
|
199
184
|
return true
|
200
|
-
elsif
|
201
|
-
# some exception occured and the thread is dead
|
185
|
+
elsif !thread.alive?
|
202
186
|
return false
|
203
187
|
elsif running?
|
204
|
-
# fully initialized and running
|
205
188
|
return true
|
206
189
|
else
|
207
190
|
sleep 0.01
|
@@ -300,9 +283,9 @@ module LogStash; class Pipeline < BasePipeline
|
|
300
283
|
|
301
284
|
pipeline_workers.times do |t|
|
302
285
|
thread = Thread.new(batch_size, batch_delay, self) do |_b_size, _b_delay, _pipeline|
|
303
|
-
Util.set_thread_name("[#{pipeline_id}]>worker#{t}")
|
304
286
|
_pipeline.worker_loop(_b_size, _b_delay)
|
305
287
|
end
|
288
|
+
Util.set_thread_name("[#{pipeline_id}]>worker#{t}")
|
306
289
|
@worker_threads << thread
|
307
290
|
end
|
308
291
|
|
@@ -463,7 +446,7 @@ module LogStash; class Pipeline < BasePipeline
|
|
463
446
|
|
464
447
|
stop_inputs
|
465
448
|
|
466
|
-
# We make this call blocking, so we know for sure when the method return the
|
449
|
+
# We make this call blocking, so we know for sure when the method return the shutdown is
|
467
450
|
# stopped
|
468
451
|
wait_for_workers
|
469
452
|
clear_pipeline_metrics
|
@@ -542,7 +525,7 @@ module LogStash; class Pipeline < BasePipeline
|
|
542
525
|
|
543
526
|
# Calculate the uptime in milliseconds
|
544
527
|
#
|
545
|
-
# @return [
|
528
|
+
# @return [Fixnum] Uptime in milliseconds, 0 if the pipeline is not started
|
546
529
|
def uptime
|
547
530
|
return 0 if started_at.nil?
|
548
531
|
((Time.now.to_f - started_at.to_f) * 1000.0).to_i
|
@@ -3,7 +3,6 @@ require "logstash/pipeline_action/base"
|
|
3
3
|
require "logstash/pipeline"
|
4
4
|
require "logstash/java_pipeline"
|
5
5
|
|
6
|
-
|
7
6
|
module LogStash module PipelineAction
|
8
7
|
class Create < Base
|
9
8
|
include LogStash::Util::Loggable
|
@@ -31,26 +30,33 @@ module LogStash module PipelineAction
|
|
31
30
|
|
32
31
|
# The execute assume that the thread safety access of the pipeline
|
33
32
|
# is managed by the caller.
|
34
|
-
def execute(agent,
|
35
|
-
|
33
|
+
def execute(agent, pipelines)
|
34
|
+
pipeline =
|
36
35
|
if @pipeline_config.settings.get_value("pipeline.java_execution")
|
37
36
|
LogStash::JavaPipeline.new(@pipeline_config, @metric, agent)
|
38
37
|
else
|
39
38
|
agent.exclusive do
|
40
39
|
# The Ruby pipeline initialization is not thread safe because of the module level
|
41
40
|
# shared state in LogsStash::Config::AST. When using multiple pipelines this gets
|
42
|
-
# executed simultaneously in different threads and we need to
|
41
|
+
# executed simultaneously in different threads and we need to synchonize this initialization.
|
43
42
|
LogStash::Pipeline.new(@pipeline_config, @metric, agent)
|
44
43
|
end
|
45
44
|
end
|
46
45
|
|
47
|
-
|
48
|
-
|
46
|
+
status = nil
|
47
|
+
pipelines.compute(pipeline_id) do |id,value|
|
48
|
+
if value
|
49
|
+
LogStash::ConvergeResult::ActionResult.create(self, true)
|
50
|
+
end
|
51
|
+
status = pipeline.start # block until the pipeline is correctly started or crashed
|
52
|
+
pipeline # The pipeline is successfully started we can add it to the map
|
49
53
|
end
|
50
54
|
|
51
|
-
|
55
|
+
|
56
|
+
LogStash::ConvergeResult::ActionResult.create(self, status)
|
52
57
|
end
|
53
58
|
|
59
|
+
|
54
60
|
def to_s
|
55
61
|
"PipelineAction::Create<#{pipeline_id}>"
|
56
62
|
end
|
@@ -20,12 +20,8 @@ module LogStash module PipelineAction
|
|
20
20
|
"PipelineAction::Reload<#{pipeline_id}>"
|
21
21
|
end
|
22
22
|
|
23
|
-
def execute(agent,
|
24
|
-
old_pipeline =
|
25
|
-
|
26
|
-
if old_pipeline.nil?
|
27
|
-
return LogStash::ConvergeResult::FailedAction.new("Cannot reload pipeline, because the pipeline does not exist")
|
28
|
-
end
|
23
|
+
def execute(agent, pipelines)
|
24
|
+
old_pipeline = pipelines[pipeline_id]
|
29
25
|
|
30
26
|
if !old_pipeline.reloadable?
|
31
27
|
return LogStash::ConvergeResult::FailedAction.new("Cannot reload pipeline, because the existing pipeline is not reloadable")
|
@@ -38,8 +34,8 @@ module LogStash module PipelineAction
|
|
38
34
|
else
|
39
35
|
agent.exclusive do
|
40
36
|
# The Ruby pipeline initialization is not thread safe because of the module level
|
41
|
-
# shared state in LogsStash::Config::AST. When using multiple pipelines this gets
|
42
|
-
# executed simultaneously in different threads and we need to
|
37
|
+
# shared state in LogsStash::Config::AST. When using multiple pipelines this can gets
|
38
|
+
# executed simultaneously in different threads and we need to synchonize this initialization.
|
43
39
|
LogStash::BasePipeline.new(@pipeline_config)
|
44
40
|
end
|
45
41
|
end
|
@@ -53,35 +49,16 @@ module LogStash module PipelineAction
|
|
53
49
|
|
54
50
|
logger.info("Reloading pipeline", "pipeline.id" => pipeline_id)
|
55
51
|
|
56
|
-
|
57
|
-
|
58
|
-
# the block must emit a success boolean value
|
52
|
+
pipelines.compute(pipeline_id) do |_,pipeline|
|
53
|
+
status = Stop.new(pipeline_id).execute(agent, pipelines)
|
59
54
|
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
if @pipeline_config.settings.get_value("pipeline.java_execution")
|
67
|
-
LogStash::JavaPipeline.new(@pipeline_config, @metric, agent)
|
68
|
-
else
|
69
|
-
agent.exclusive do
|
70
|
-
# The Ruby pipeline initialization is not thread safe because of the module level
|
71
|
-
# shared state in LogsStash::Config::AST. When using multiple pipelines this gets
|
72
|
-
# executed simultaneously in different threads and we need to synchronize this initialization.
|
73
|
-
LogStash::Pipeline.new(@pipeline_config, @metric, agent)
|
74
|
-
end
|
75
|
-
end
|
76
|
-
|
77
|
-
success = new_pipeline.start # block until the pipeline is correctly started or crashed
|
78
|
-
|
79
|
-
# return success and new_pipeline to registry reload_pipeline
|
80
|
-
[success, new_pipeline]
|
55
|
+
if status
|
56
|
+
return Create.new(@pipeline_config, @metric).execute(agent, pipelines)
|
57
|
+
else
|
58
|
+
return status
|
59
|
+
end
|
60
|
+
pipeline
|
81
61
|
end
|
82
|
-
|
83
|
-
LogStash::ConvergeResult::ActionResult.create(self, success)
|
84
62
|
end
|
85
|
-
|
86
63
|
end
|
87
64
|
end end
|
@@ -9,13 +9,15 @@ module LogStash module PipelineAction
|
|
9
9
|
@pipeline_id = pipeline_id
|
10
10
|
end
|
11
11
|
|
12
|
-
def execute(agent,
|
13
|
-
|
12
|
+
def execute(agent, pipelines)
|
13
|
+
pipelines.compute(pipeline_id) do |_,pipeline|
|
14
14
|
pipeline.shutdown { LogStash::ShutdownWatcher.start(pipeline) }
|
15
15
|
pipeline.thread.join
|
16
|
+
nil # delete the pipeline
|
16
17
|
end
|
17
|
-
|
18
|
-
|
18
|
+
# If we reach this part of the code we have succeeded because
|
19
|
+
# the shutdown call will block.
|
20
|
+
return LogStash::ConvergeResult::SuccessfulAction.new
|
19
21
|
end
|
20
22
|
|
21
23
|
def to_s
|
@@ -28,7 +28,6 @@ module LogStash
|
|
28
28
|
"queue.checkpoint.acks",
|
29
29
|
"queue.checkpoint.interval",
|
30
30
|
"queue.checkpoint.writes",
|
31
|
-
"queue.checkpoint.retry",
|
32
31
|
"queue.drain",
|
33
32
|
"queue.max_bytes",
|
34
33
|
"queue.max_events",
|
@@ -47,7 +46,7 @@ module LogStash
|
|
47
46
|
|
48
47
|
def register(setting)
|
49
48
|
unless SETTINGS_WHITE_LIST.include?(setting.name)
|
50
|
-
raise ArgumentError.new("Only pipeline related settings can be
|
49
|
+
raise ArgumentError.new("Only pipeline related settings can be registered in a PipelineSettings object. Received \"#{setting.name}\". Allowed settings: #{SETTINGS_WHITE_LIST}")
|
51
50
|
end
|
52
51
|
super(setting)
|
53
52
|
end
|
@@ -262,14 +262,11 @@ module LogStash module Plugins
|
|
262
262
|
# @param name [String] plugin name
|
263
263
|
# @return [Boolean] true if klass is a valid plugin for name
|
264
264
|
def is_a_plugin?(klass, name)
|
265
|
-
|
266
|
-
(klass.ancestors.include?(LogStash::Plugin) && klass.respond_to?(:config_name) && klass.config_name == name)
|
265
|
+
klass.ancestors.include?(LogStash::Plugin) && klass.respond_to?(:config_name) && klass.config_name == name
|
267
266
|
end
|
268
267
|
|
269
268
|
def add_plugin(type, name, klass)
|
270
|
-
if
|
271
|
-
@registry[key_for(type, name)] = PluginSpecification.new(type, name, klass.javaClass)
|
272
|
-
elsif !exists?(type, name)
|
269
|
+
if !exists?(type, name)
|
273
270
|
specification_klass = type == :universal ? UniversalPluginSpecification : PluginSpecification
|
274
271
|
@registry[key_for(type, name)] = specification_klass.new(type, name, klass)
|
275
272
|
else
|
data/lib/logstash/runner.rb
CHANGED
@@ -276,17 +276,6 @@ class LogStash::Runner < Clamp::StrictCommand
|
|
276
276
|
LogStash::PLUGIN_REGISTRY.hooks.register_emitter(self.class, @dispatcher)
|
277
277
|
|
278
278
|
@settings.validate_all
|
279
|
-
|
280
|
-
if @settings.registered?("xpack.monitoring.enabled")
|
281
|
-
verify_deprecated_setting("xpack.monitoring.elasticsearch.url", "xpack.monitoring.elasticsearch.hosts")
|
282
|
-
verify_deprecated_setting("xpack.monitoring.elasticsearch.ssl.ca", "xpack.monitoring.elasticsearch.ssl.certificate_authority")
|
283
|
-
end
|
284
|
-
|
285
|
-
if @settings.registered?("xpack.management.enabled")
|
286
|
-
verify_deprecated_setting("xpack.management.elasticsearch.url", "xpack.management.elasticsearch.hosts")
|
287
|
-
verify_deprecated_setting("xpack.management.elasticsearch.ssl.ca", "xpack.management.elasticsearch.ssl.certificate_authority")
|
288
|
-
end
|
289
|
-
|
290
279
|
@dispatcher.fire(:before_bootstrap_checks)
|
291
280
|
|
292
281
|
return start_shell(setting("interactive"), binding) if setting("interactive")
|
@@ -380,8 +369,6 @@ class LogStash::Runner < Clamp::StrictCommand
|
|
380
369
|
|
381
370
|
@agent.shutdown
|
382
371
|
|
383
|
-
logger.info("Logstash shut down.")
|
384
|
-
|
385
372
|
# flush any outstanding log messages during shutdown
|
386
373
|
org.apache.logging.log4j.LogManager.shutdown
|
387
374
|
|
@@ -514,15 +501,4 @@ class LogStash::Runner < Clamp::StrictCommand
|
|
514
501
|
@settings.get_value(key)
|
515
502
|
end
|
516
503
|
|
517
|
-
def verify_deprecated_setting(old_setting, new_setting)
|
518
|
-
if @settings.set?(old_setting)
|
519
|
-
logger.warn("Deprecated setting `#{old_setting}` please use `#{new_setting}`")
|
520
|
-
if @settings.set?(new_setting)
|
521
|
-
signal_usage_error("Both `#{new_setting}` and the deprecated `#{old_setting}` settings are set, please remove `#{old_setting}`")
|
522
|
-
else
|
523
|
-
@settings.set_value(new_setting, @settings.get_value(old_setting))
|
524
|
-
end
|
525
|
-
end
|
526
|
-
end
|
527
|
-
|
528
504
|
end
|
data/lib/logstash/settings.rb
CHANGED
@@ -368,7 +368,7 @@ module LogStash
|
|
368
368
|
case value
|
369
369
|
when ::Range
|
370
370
|
value
|
371
|
-
when ::
|
371
|
+
when ::Fixnum
|
372
372
|
value..value
|
373
373
|
when ::String
|
374
374
|
first, last = value.split(PORT_SEPARATOR)
|
@@ -481,11 +481,11 @@ module LogStash
|
|
481
481
|
|
482
482
|
class Bytes < Coercible
|
483
483
|
def initialize(name, default=nil, strict=true)
|
484
|
-
super(name, ::
|
484
|
+
super(name, ::Fixnum, default, strict=true) { |value| valid?(value) }
|
485
485
|
end
|
486
486
|
|
487
487
|
def valid?(value)
|
488
|
-
value.is_a?(
|
488
|
+
value.is_a?(Fixnum) && value >= 0
|
489
489
|
end
|
490
490
|
|
491
491
|
def coerce(value)
|
@@ -508,11 +508,11 @@ module LogStash
|
|
508
508
|
|
509
509
|
class TimeValue < Coercible
|
510
510
|
def initialize(name, default, strict=true, &validator_proc)
|
511
|
-
super(name, ::
|
511
|
+
super(name, ::Fixnum, default, strict, &validator_proc)
|
512
512
|
end
|
513
513
|
|
514
514
|
def coerce(value)
|
515
|
-
return value if value.is_a?(::
|
515
|
+
return value if value.is_a?(::Fixnum)
|
516
516
|
Util::TimeValue.from_value(value).to_nanos
|
517
517
|
end
|
518
518
|
end
|
@@ -10,11 +10,11 @@ module LogStash
|
|
10
10
|
@metric = metric
|
11
11
|
end
|
12
12
|
|
13
|
-
def resolve(
|
13
|
+
def resolve(pipelines, pipeline_configs)
|
14
14
|
actions = []
|
15
15
|
|
16
16
|
pipeline_configs.each do |pipeline_config|
|
17
|
-
pipeline =
|
17
|
+
pipeline = pipelines[pipeline_config.pipeline_id]
|
18
18
|
|
19
19
|
if pipeline.nil?
|
20
20
|
actions << LogStash::PipelineAction::Create.new(pipeline_config, @metric)
|
@@ -25,12 +25,12 @@ module LogStash
|
|
25
25
|
end
|
26
26
|
end
|
27
27
|
|
28
|
-
|
28
|
+
running_pipelines = pipeline_configs.collect(&:pipeline_id)
|
29
29
|
|
30
30
|
# If one of the running pipeline is not in the pipeline_configs, we assume that we need to
|
31
31
|
# stop it.
|
32
|
-
|
33
|
-
.select { |pipeline_id| !
|
32
|
+
pipelines.keys
|
33
|
+
.select { |pipeline_id| !running_pipelines.include?(pipeline_id) }
|
34
34
|
.each { |pipeline_id| actions << LogStash::PipelineAction::Stop.new(pipeline_id) }
|
35
35
|
|
36
36
|
actions.sort # See logstash/pipeline_action.rb
|
@@ -7,7 +7,7 @@ module LogStash::Util::DurationFormatter
|
|
7
7
|
# a format that a human can understand. This is currently used by
|
8
8
|
# the API.
|
9
9
|
#
|
10
|
-
# @param [
|
10
|
+
# @param [Fixnum] Duration in milliseconds
|
11
11
|
# @return [String] Duration in human format
|
12
12
|
def self.human_format(duration)
|
13
13
|
ChronicDuration.output(duration / 1000, CHRONIC_OPTIONS)
|