logstash-core 6.8.23-java → 7.0.0.alpha1-java

Sign up to get free protection for your applications and to get access to all the features.
Files changed (80) hide show
  1. checksums.yaml +4 -4
  2. data/lib/logstash/agent.rb +69 -85
  3. data/lib/logstash/api/modules/stats.rb +1 -1
  4. data/lib/logstash/compiler/lscl.rb +7 -7
  5. data/lib/logstash/config/config_ast.rb +1 -1
  6. data/lib/logstash/config/mixin.rb +1 -1
  7. data/lib/logstash/config/modules_common.rb +3 -3
  8. data/lib/logstash/dependency_report.rb +1 -2
  9. data/lib/logstash/environment.rb +4 -9
  10. data/lib/logstash/event.rb +1 -24
  11. data/lib/logstash/filter_delegator.rb +69 -2
  12. data/lib/logstash/filters/base.rb +2 -0
  13. data/lib/logstash/instrument/metric_store.rb +1 -1
  14. data/lib/logstash/instrument/periodic_poller/dlq.rb +7 -5
  15. data/lib/logstash/instrument/periodic_poller/jvm.rb +3 -3
  16. data/lib/logstash/instrument/periodic_poller/pq.rb +8 -6
  17. data/lib/logstash/instrument/periodic_pollers.rb +3 -3
  18. data/lib/logstash/java_pipeline.rb +11 -38
  19. data/lib/logstash/modules/kibana_config.rb +1 -1
  20. data/lib/logstash/modules/logstash_config.rb +1 -1
  21. data/lib/logstash/patches/resolv.rb +32 -17
  22. data/lib/logstash/pipeline.rb +11 -28
  23. data/lib/logstash/pipeline_action/base.rb +1 -1
  24. data/lib/logstash/pipeline_action/create.rb +13 -7
  25. data/lib/logstash/pipeline_action/reload.rb +12 -35
  26. data/lib/logstash/pipeline_action/stop.rb +6 -4
  27. data/lib/logstash/pipeline_settings.rb +1 -2
  28. data/lib/logstash/plugins/registry.rb +2 -5
  29. data/lib/logstash/runner.rb +0 -24
  30. data/lib/logstash/settings.rb +5 -5
  31. data/lib/logstash/state_resolver.rb +5 -5
  32. data/lib/logstash/util/duration_formatter.rb +1 -1
  33. data/lib/logstash/util/safe_uri.rb +0 -1
  34. data/lib/logstash/util/substitution_variables.rb +1 -22
  35. data/lib/logstash/util/thread_dump.rb +1 -1
  36. data/lib/logstash/util.rb +1 -11
  37. data/lib/logstash-core/version.rb +3 -1
  38. data/locales/en.yml +7 -16
  39. data/logstash-core.gemspec +11 -2
  40. data/spec/logstash/acked_queue_concurrent_stress_spec.rb +2 -2
  41. data/spec/logstash/agent/converge_spec.rb +31 -25
  42. data/spec/logstash/agent/metrics_spec.rb +1 -1
  43. data/spec/logstash/agent_spec.rb +7 -6
  44. data/spec/logstash/compiler/compiler_spec.rb +0 -28
  45. data/spec/logstash/config/config_ast_spec.rb +0 -15
  46. data/spec/logstash/config/mixin_spec.rb +2 -3
  47. data/spec/logstash/converge_result_spec.rb +1 -1
  48. data/spec/logstash/environment_spec.rb +4 -4
  49. data/spec/logstash/event_spec.rb +2 -10
  50. data/spec/logstash/filter_delegator_spec.rb +12 -2
  51. data/spec/logstash/filters/base_spec.rb +9 -45
  52. data/spec/logstash/instrument/periodic_poller/cgroup_spec.rb +2 -0
  53. data/spec/logstash/instrument/wrapped_write_client_spec.rb +1 -1
  54. data/spec/logstash/java_filter_delegator_spec.rb +11 -1
  55. data/spec/logstash/legacy_ruby_event_spec.rb +5 -6
  56. data/spec/logstash/patches_spec.rb +3 -1
  57. data/spec/logstash/pipeline_action/create_spec.rb +8 -14
  58. data/spec/logstash/pipeline_action/reload_spec.rb +9 -16
  59. data/spec/logstash/pipeline_action/stop_spec.rb +3 -4
  60. data/spec/logstash/queue_factory_spec.rb +1 -2
  61. data/spec/logstash/runner_spec.rb +0 -2
  62. data/spec/logstash/settings/array_coercible_spec.rb +1 -1
  63. data/spec/logstash/settings/bytes_spec.rb +2 -2
  64. data/spec/logstash/settings/port_range_spec.rb +1 -1
  65. data/spec/logstash/settings_spec.rb +0 -10
  66. data/spec/logstash/state_resolver_spec.rb +22 -26
  67. data/spec/logstash/util/safe_uri_spec.rb +0 -40
  68. data/spec/logstash/util/secretstore_spec.rb +1 -1
  69. data/spec/logstash/util/time_value_spec.rb +1 -1
  70. data/spec/logstash/util/wrapped_acked_queue_spec.rb +1 -1
  71. data/spec/logstash/webserver_spec.rb +5 -9
  72. data/spec/support/matchers.rb +19 -25
  73. data/spec/support/shared_contexts.rb +3 -3
  74. data/versions-gem-copy.yml +9 -9
  75. metadata +31 -44
  76. data/lib/logstash/patches/resolv_9270.rb +0 -2903
  77. data/lib/logstash/pipelines_registry.rb +0 -166
  78. data/lib/logstash/util/lazy_singleton.rb +0 -33
  79. data/spec/logstash/jruby_version_spec.rb +0 -15
  80. data/spec/logstash/pipelines_registry_spec.rb +0 -220
@@ -11,12 +11,14 @@ module LogStash module Instrument module PeriodicPoller
11
11
  end
12
12
 
13
13
  def collect
14
- pipelines = @agent.running_user_defined_pipelines
15
- pipelines.each do |_, pipeline|
16
- unless pipeline.nil?
17
- pipeline.collect_stats
18
- end
14
+ pipelines = @agent.with_running_user_defined_pipelines {|pipelines| pipelines}
15
+ unless pipelines.nil?
16
+ pipelines.each {|_, pipeline|
17
+ unless pipeline.nil?
18
+ pipeline.collect_stats
19
+ end
20
+ }
19
21
  end
20
22
  end
21
23
  end
22
- end end end
24
+ end; end; end
@@ -11,12 +11,12 @@ module LogStash module Instrument
11
11
  class PeriodicPollers
12
12
  attr_reader :metric
13
13
 
14
- def initialize(metric, queue_type, agent)
14
+ def initialize(metric, queue_type, pipelines)
15
15
  @metric = metric
16
16
  @periodic_pollers = [PeriodicPoller::Os.new(metric),
17
17
  PeriodicPoller::JVM.new(metric),
18
- PeriodicPoller::PersistentQueue.new(metric, queue_type, agent),
19
- PeriodicPoller::DeadLetterQueue.new(metric, agent)]
18
+ PeriodicPoller::PersistentQueue.new(metric, queue_type, pipelines),
19
+ PeriodicPoller::DeadLetterQueue.new(metric, pipelines)]
20
20
  end
21
21
 
22
22
  def start
@@ -1,7 +1,6 @@
1
1
  # encoding: utf-8
2
2
  require "thread"
3
3
  require "concurrent"
4
- require "logstash/event"
5
4
  require "logstash/filters/base"
6
5
  require "logstash/inputs/base"
7
6
  require "logstash/outputs/base"
@@ -39,23 +38,9 @@ module LogStash; class JavaPipeline < JavaBasePipeline
39
38
  @flushRequested = java.util.concurrent.atomic.AtomicBoolean.new(false)
40
39
  @shutdownRequested = java.util.concurrent.atomic.AtomicBoolean.new(false)
41
40
  @outputs_registered = Concurrent::AtomicBoolean.new(false)
42
-
43
- # @finished_execution signals that the pipeline thread has finished its execution
44
- # regardless of any exceptions; it will always be true when the thread completes
45
41
  @finished_execution = Concurrent::AtomicBoolean.new(false)
46
-
47
- # @finished_run signals that the run methods called in the pipeline thread was completed
48
- # without errors and it will NOT be set if the run method exits from an exception; this
49
- # is by design and necessary for the wait_until_started semantic
50
- @finished_run = Concurrent::AtomicBoolean.new(false)
51
-
52
- @thread = nil
53
42
  end # def initialize
54
43
 
55
- def finished_execution?
56
- @finished_execution.true?
57
- end
58
-
59
44
  def ready?
60
45
  @ready.value
61
46
  end
@@ -97,18 +82,15 @@ module LogStash; class JavaPipeline < JavaBasePipeline
97
82
  @logger.debug("Starting pipeline", default_logging_keys)
98
83
 
99
84
  @finished_execution.make_false
100
- @finished_run.make_false
101
85
 
102
86
  @thread = Thread.new do
103
87
  begin
104
88
  LogStash::Util.set_thread_name("pipeline.#{pipeline_id}")
105
89
  run
106
- @finished_run.make_true
90
+ @finished_execution.make_true
107
91
  rescue => e
108
92
  close
109
93
  logger.error("Pipeline aborted due to error", default_logging_keys(:exception => e, :backtrace => e.backtrace))
110
- ensure
111
- @finished_execution.make_true
112
94
  end
113
95
  end
114
96
 
@@ -123,14 +105,15 @@ module LogStash; class JavaPipeline < JavaBasePipeline
123
105
 
124
106
  def wait_until_started
125
107
  while true do
126
- if @finished_run.true?
127
- # it completed run without exception
108
+ # This should be changed with an appropriate FSM
109
+ # It's an edge case, if we have a pipeline with
110
+ # a generator { count => 1 } its possible that `Thread#alive?` doesn't return true
111
+ # because the execution of the thread was successful and complete
112
+ if @finished_execution.true?
128
113
  return true
129
114
  elsif thread.nil? || !thread.alive?
130
- # some exception occurred and the thread is dead
131
115
  return false
132
116
  elsif running?
133
- # fully initialized and running
134
117
  return true
135
118
  else
136
119
  sleep 0.01
@@ -232,11 +215,11 @@ module LogStash; class JavaPipeline < JavaBasePipeline
232
215
 
233
216
  pipeline_workers.times do |t|
234
217
  thread = Thread.new do
235
- Util.set_thread_name("[#{pipeline_id}]>worker#{t}")
236
218
  org.logstash.execution.WorkerLoop.new(
237
219
  lir_execution, filter_queue_client, @events_filtered, @events_consumed,
238
220
  @flushRequested, @flushing, @shutdownRequested, @drain_queue).run
239
221
  end
222
+ Util.set_thread_name("[#{pipeline_id}]>worker#{t}")
240
223
  @worker_threads << thread
241
224
  end
242
225
 
@@ -257,13 +240,7 @@ module LogStash; class JavaPipeline < JavaBasePipeline
257
240
  end
258
241
 
259
242
  def wait_inputs
260
- @input_threads.each do |thread|
261
- if thread.class == Java::JavaObject
262
- thread.to_java.join
263
- else
264
- thread.join
265
- end
266
- end
243
+ @input_threads.each(&:join)
267
244
  end
268
245
 
269
246
  def start_inputs
@@ -285,11 +262,7 @@ module LogStash; class JavaPipeline < JavaBasePipeline
285
262
  end
286
263
 
287
264
  def start_input(plugin)
288
- if plugin.class == LogStash::JavaInputDelegator
289
- @input_threads << plugin.start
290
- else
291
- @input_threads << Thread.new { inputworker(plugin) }
292
- end
265
+ @input_threads << Thread.new { inputworker(plugin) }
293
266
  end
294
267
 
295
268
  def inputworker(plugin)
@@ -335,7 +308,7 @@ module LogStash; class JavaPipeline < JavaBasePipeline
335
308
 
336
309
  stop_inputs
337
310
 
338
- # We make this call blocking, so we know for sure when the method return the shtudown is
311
+ # We make this call blocking, so we know for sure when the method return the shutdown is
339
312
  # stopped
340
313
  wait_for_workers
341
314
  clear_pipeline_metrics
@@ -399,7 +372,7 @@ module LogStash; class JavaPipeline < JavaBasePipeline
399
372
  end
400
373
 
401
374
  def plugin_threads_info
402
- input_threads = @input_threads.select {|t| t.class == Thread && t.alive? }
375
+ input_threads = @input_threads.select {|t| t.alive? }
403
376
  worker_threads = @worker_threads.select {|t| t.alive? }
404
377
  (input_threads + worker_threads).map {|t| Util.thread_info(t) }
405
378
  end
@@ -8,7 +8,7 @@ module LogStash module Modules class KibanaConfig
8
8
  include LogStash::Util::Loggable
9
9
 
10
10
  ALLOWED_DIRECTORIES = ["search", "visualization"]
11
- attr_reader :index_name # not used when importing via kibana but for BWC with ElastsearchConfig
11
+ attr_reader :index_name # not used when importing via kibana but for BWC with ElasticsearchConfig
12
12
 
13
13
  # We name it `modul` here because `module` has meaning in Ruby.
14
14
  def initialize(modul, settings)
@@ -39,7 +39,7 @@ module LogStash module Modules class LogStashConfig
39
39
  def get_setting(setting_class)
40
40
  raw_value = @settings[setting_class.name]
41
41
  # If we dont check for NIL, the Settings class will try to coerce the value
42
- # and most of the it will fails when a NIL value is explicitely set.
42
+ # and most of the it will fails when a NIL value is explicitly set.
43
43
  # This will be fixed once we wrap the plugins settings into a Settings class
44
44
  setting_class.set(raw_value) unless raw_value.nil?
45
45
  setting_class.value
@@ -1,25 +1,40 @@
1
1
  require "resolv"
2
2
 
3
- # ref:
4
- # https://github.com/logstash-plugins/logstash-filter-dns/issues/51
5
- # https://github.com/jruby/jruby/pull/5722
3
+ # ref: https://github.com/logstash-plugins/logstash-filter-dns/issues/40
6
4
  #
7
- # JRuby versions starting at 9.2.0.0 have a bug in resolv.rb with a leak between the
8
- # DNS.allocate_request_id/DNS.free_request_id methods.
5
+ # JRuby 9k versions prior to 9.1.16.0 have a bug which crashes IP address
6
+ # resolution after 64k unique IP addresses resolutions.
9
7
  #
10
- # We are opting to load a patched full resolv.rb instead of monkey patching the
11
- # offending methods because we do not yet know in which upcoming version of JRuby
12
- # this will be fixed and we want to avoid potential conflicting monkey patches.
13
- # A spec which will break on JRuby upgrade will redirect here
14
- # to make a manual verification and eventually remove that patch here once the fix is
15
- # made in the JRuby version of resolv.rb.
8
+ # Note that the oldest JRuby version in LS 6 is 9.1.13.0 and
9
+ # JRuby 1.7.25 and 1.7.27 (the 2 versions used across LS 5) are not affected by this bug.
16
10
 
17
- if Gem::Version.new(JRUBY_VERSION) >= Gem::Version.new("9.2.0.0")
18
- # save verbose level and mute the "warning: already initialized constant"
19
- warn_level = $VERBOSE
20
- $VERBOSE = nil
11
+ # make sure we abort if a known correct JRuby version is installed
12
+ # to avoid having an unnecessary legacy patch being applied in the future.
13
+ raise("Unnecessary patch on resolv.rb for JRuby version 9.1.16+") if Gem::Version.new(JRUBY_VERSION) >= Gem::Version.new("9.1.16.0")
21
14
 
22
- require_relative "resolv_9270"
15
+ # The code below is copied from JRuby 9.1.16.0 resolv.rb:
16
+ # https://github.com/jruby/jruby/blob/9.1.16.0/lib/ruby/stdlib/resolv.rb#L775-L784
17
+ #
18
+ # JRuby is Copyright (c) 2007-2017 The JRuby project, and is released
19
+ # under a tri EPL/GPL/LGPL license.
20
+ # Full license available at https://github.com/jruby/jruby/blob/9.1.16.0/COPYING
23
21
 
24
- $VERBOSE = warn_level
22
+ class Resolv
23
+ class DNS
24
+ class Requester
25
+ class UnconnectedUDP
26
+ def sender(msg, data, host, port=Port)
27
+ sock = @socks_hash[host.index(':') ? "::" : "0.0.0.0"]
28
+ return nil if !sock
29
+ service = [IPAddr.new(host), port]
30
+ id = DNS.allocate_request_id(service[0], service[1])
31
+ request = msg.encode
32
+ request[0,2] = [id].pack('n')
33
+ return @senders[[service, id]] =
34
+ Sender.new(request, data, sock, host, port)
35
+ end
36
+ end
37
+ end
38
+ end
25
39
  end
40
+
@@ -107,23 +107,8 @@ module LogStash; class Pipeline < BasePipeline
107
107
  @flushing = Concurrent::AtomicReference.new(false)
108
108
  @outputs_registered = Concurrent::AtomicBoolean.new(false)
109
109
  @worker_shutdown = java.util.concurrent.atomic.AtomicBoolean.new(false)
110
-
111
- # @finished_execution signals that the pipeline thread has finished its execution
112
- # regardless of any exceptions; it will always be true when the thread completes
113
- @finished_execution = Concurrent::AtomicBoolean.new(false)
114
-
115
- # @finished_run signals that the run methods called in the pipeline thread was completed
116
- # without errors and it will NOT be set if the run method exits from an exception; this
117
- # is by design and necessary for the wait_until_started semantic
118
- @finished_run = Concurrent::AtomicBoolean.new(false)
119
-
120
- @thread = nil
121
110
  end # def initialize
122
111
 
123
- def finished_execution?
124
- @finished_execution.true?
125
- end
126
-
127
112
  def ready?
128
113
  @ready.value
129
114
  end
@@ -167,19 +152,16 @@ module LogStash; class Pipeline < BasePipeline
167
152
  "pipeline.batch.size" => settings.get("pipeline.batch.size"),
168
153
  "pipeline.batch.delay" => settings.get("pipeline.batch.delay")))
169
154
 
170
- @finished_execution.make_false
171
- @finished_run.make_false
155
+ @finished_execution = Concurrent::AtomicBoolean.new(false)
172
156
 
173
157
  @thread = Thread.new do
174
158
  begin
175
159
  LogStash::Util.set_thread_name("pipeline.#{pipeline_id}")
176
160
  run
177
- @finished_run.make_true
161
+ @finished_execution.make_true
178
162
  rescue => e
179
163
  close
180
164
  @logger.error("Pipeline aborted due to error", default_logging_keys(:exception => e, :backtrace => e.backtrace))
181
- ensure
182
- @finished_execution.make_true
183
165
  end
184
166
  end
185
167
 
@@ -194,14 +176,15 @@ module LogStash; class Pipeline < BasePipeline
194
176
 
195
177
  def wait_until_started
196
178
  while true do
197
- if @finished_run.true?
198
- # it completed run without exception
179
+ # This should be changed with an appropriate FSM
180
+ # It's an edge case, if we have a pipeline with
181
+ # a generator { count => 1 } its possible that `Thread#alive?` doesn't return true
182
+ # because the execution of the thread was successful and complete
183
+ if @finished_execution.true?
199
184
  return true
200
- elsif thread.nil? || !thread.alive?
201
- # some exception occured and the thread is dead
185
+ elsif !thread.alive?
202
186
  return false
203
187
  elsif running?
204
- # fully initialized and running
205
188
  return true
206
189
  else
207
190
  sleep 0.01
@@ -300,9 +283,9 @@ module LogStash; class Pipeline < BasePipeline
300
283
 
301
284
  pipeline_workers.times do |t|
302
285
  thread = Thread.new(batch_size, batch_delay, self) do |_b_size, _b_delay, _pipeline|
303
- Util.set_thread_name("[#{pipeline_id}]>worker#{t}")
304
286
  _pipeline.worker_loop(_b_size, _b_delay)
305
287
  end
288
+ Util.set_thread_name("[#{pipeline_id}]>worker#{t}")
306
289
  @worker_threads << thread
307
290
  end
308
291
 
@@ -463,7 +446,7 @@ module LogStash; class Pipeline < BasePipeline
463
446
 
464
447
  stop_inputs
465
448
 
466
- # We make this call blocking, so we know for sure when the method return the shtudown is
449
+ # We make this call blocking, so we know for sure when the method return the shutdown is
467
450
  # stopped
468
451
  wait_for_workers
469
452
  clear_pipeline_metrics
@@ -542,7 +525,7 @@ module LogStash; class Pipeline < BasePipeline
542
525
 
543
526
  # Calculate the uptime in milliseconds
544
527
  #
545
- # @return [Integer] Uptime in milliseconds, 0 if the pipeline is not started
528
+ # @return [Fixnum] Uptime in milliseconds, 0 if the pipeline is not started
546
529
  def uptime
547
530
  return 0 if started_at.nil?
548
531
  ((Time.now.to_f - started_at.to_f) * 1000.0).to_i
@@ -12,7 +12,7 @@ module LogStash module PipelineAction
12
12
  end
13
13
  alias_method :to_s, :inspect
14
14
 
15
- def execute(agent, pipelines_registry)
15
+ def execute(agent, pipelines)
16
16
  raise "`#execute` Not implemented!"
17
17
  end
18
18
 
@@ -3,7 +3,6 @@ require "logstash/pipeline_action/base"
3
3
  require "logstash/pipeline"
4
4
  require "logstash/java_pipeline"
5
5
 
6
-
7
6
  module LogStash module PipelineAction
8
7
  class Create < Base
9
8
  include LogStash::Util::Loggable
@@ -31,26 +30,33 @@ module LogStash module PipelineAction
31
30
 
32
31
  # The execute assume that the thread safety access of the pipeline
33
32
  # is managed by the caller.
34
- def execute(agent, pipelines_registry)
35
- new_pipeline =
33
+ def execute(agent, pipelines)
34
+ pipeline =
36
35
  if @pipeline_config.settings.get_value("pipeline.java_execution")
37
36
  LogStash::JavaPipeline.new(@pipeline_config, @metric, agent)
38
37
  else
39
38
  agent.exclusive do
40
39
  # The Ruby pipeline initialization is not thread safe because of the module level
41
40
  # shared state in LogsStash::Config::AST. When using multiple pipelines this gets
42
- # executed simultaneously in different threads and we need to synchronize this initialization.
41
+ # executed simultaneously in different threads and we need to synchonize this initialization.
43
42
  LogStash::Pipeline.new(@pipeline_config, @metric, agent)
44
43
  end
45
44
  end
46
45
 
47
- success = pipelines_registry.create_pipeline(pipeline_id, new_pipeline) do
48
- new_pipeline.start # block until the pipeline is correctly started or crashed
46
+ status = nil
47
+ pipelines.compute(pipeline_id) do |id,value|
48
+ if value
49
+ LogStash::ConvergeResult::ActionResult.create(self, true)
50
+ end
51
+ status = pipeline.start # block until the pipeline is correctly started or crashed
52
+ pipeline # The pipeline is successfully started we can add it to the map
49
53
  end
50
54
 
51
- LogStash::ConvergeResult::ActionResult.create(self, success)
55
+
56
+ LogStash::ConvergeResult::ActionResult.create(self, status)
52
57
  end
53
58
 
59
+
54
60
  def to_s
55
61
  "PipelineAction::Create<#{pipeline_id}>"
56
62
  end
@@ -20,12 +20,8 @@ module LogStash module PipelineAction
20
20
  "PipelineAction::Reload<#{pipeline_id}>"
21
21
  end
22
22
 
23
- def execute(agent, pipelines_registry)
24
- old_pipeline = pipelines_registry.get_pipeline(pipeline_id)
25
-
26
- if old_pipeline.nil?
27
- return LogStash::ConvergeResult::FailedAction.new("Cannot reload pipeline, because the pipeline does not exist")
28
- end
23
+ def execute(agent, pipelines)
24
+ old_pipeline = pipelines[pipeline_id]
29
25
 
30
26
  if !old_pipeline.reloadable?
31
27
  return LogStash::ConvergeResult::FailedAction.new("Cannot reload pipeline, because the existing pipeline is not reloadable")
@@ -38,8 +34,8 @@ module LogStash module PipelineAction
38
34
  else
39
35
  agent.exclusive do
40
36
  # The Ruby pipeline initialization is not thread safe because of the module level
41
- # shared state in LogsStash::Config::AST. When using multiple pipelines this gets
42
- # executed simultaneously in different threads and we need to synchronize this initialization.
37
+ # shared state in LogsStash::Config::AST. When using multiple pipelines this can gets
38
+ # executed simultaneously in different threads and we need to synchonize this initialization.
43
39
  LogStash::BasePipeline.new(@pipeline_config)
44
40
  end
45
41
  end
@@ -53,35 +49,16 @@ module LogStash module PipelineAction
53
49
 
54
50
  logger.info("Reloading pipeline", "pipeline.id" => pipeline_id)
55
51
 
56
- success = pipelines_registry.reload_pipeline(pipeline_id) do
57
- # important NOT to explicitly return from block here
58
- # the block must emit a success boolean value
52
+ pipelines.compute(pipeline_id) do |_,pipeline|
53
+ status = Stop.new(pipeline_id).execute(agent, pipelines)
59
54
 
60
- # First shutdown old pipeline
61
- old_pipeline.shutdown { LogStash::ShutdownWatcher.start(old_pipeline) }
62
- old_pipeline.thread.join
63
-
64
- # Then create a new pipeline
65
- new_pipeline =
66
- if @pipeline_config.settings.get_value("pipeline.java_execution")
67
- LogStash::JavaPipeline.new(@pipeline_config, @metric, agent)
68
- else
69
- agent.exclusive do
70
- # The Ruby pipeline initialization is not thread safe because of the module level
71
- # shared state in LogsStash::Config::AST. When using multiple pipelines this gets
72
- # executed simultaneously in different threads and we need to synchronize this initialization.
73
- LogStash::Pipeline.new(@pipeline_config, @metric, agent)
74
- end
75
- end
76
-
77
- success = new_pipeline.start # block until the pipeline is correctly started or crashed
78
-
79
- # return success and new_pipeline to registry reload_pipeline
80
- [success, new_pipeline]
55
+ if status
56
+ return Create.new(@pipeline_config, @metric).execute(agent, pipelines)
57
+ else
58
+ return status
59
+ end
60
+ pipeline
81
61
  end
82
-
83
- LogStash::ConvergeResult::ActionResult.create(self, success)
84
62
  end
85
-
86
63
  end
87
64
  end end
@@ -9,13 +9,15 @@ module LogStash module PipelineAction
9
9
  @pipeline_id = pipeline_id
10
10
  end
11
11
 
12
- def execute(agent, pipelines_registry)
13
- pipelines_registry.terminate_pipeline(pipeline_id) do |pipeline|
12
+ def execute(agent, pipelines)
13
+ pipelines.compute(pipeline_id) do |_,pipeline|
14
14
  pipeline.shutdown { LogStash::ShutdownWatcher.start(pipeline) }
15
15
  pipeline.thread.join
16
+ nil # delete the pipeline
16
17
  end
17
-
18
- LogStash::ConvergeResult::SuccessfulAction.new
18
+ # If we reach this part of the code we have succeeded because
19
+ # the shutdown call will block.
20
+ return LogStash::ConvergeResult::SuccessfulAction.new
19
21
  end
20
22
 
21
23
  def to_s
@@ -28,7 +28,6 @@ module LogStash
28
28
  "queue.checkpoint.acks",
29
29
  "queue.checkpoint.interval",
30
30
  "queue.checkpoint.writes",
31
- "queue.checkpoint.retry",
32
31
  "queue.drain",
33
32
  "queue.max_bytes",
34
33
  "queue.max_events",
@@ -47,7 +46,7 @@ module LogStash
47
46
 
48
47
  def register(setting)
49
48
  unless SETTINGS_WHITE_LIST.include?(setting.name)
50
- raise ArgumentError.new("Only pipeline related settings can be registed in a PipelineSettings object. Received \"#{setting.name}\". Allowed settings: #{SETTINGS_WHITE_LIST}")
49
+ raise ArgumentError.new("Only pipeline related settings can be registered in a PipelineSettings object. Received \"#{setting.name}\". Allowed settings: #{SETTINGS_WHITE_LIST}")
51
50
  end
52
51
  super(setting)
53
52
  end
@@ -262,14 +262,11 @@ module LogStash module Plugins
262
262
  # @param name [String] plugin name
263
263
  # @return [Boolean] true if klass is a valid plugin for name
264
264
  def is_a_plugin?(klass, name)
265
- (klass.class == Java::JavaClass && klass.simple_name.downcase == name.gsub('_','')) ||
266
- (klass.ancestors.include?(LogStash::Plugin) && klass.respond_to?(:config_name) && klass.config_name == name)
265
+ klass.ancestors.include?(LogStash::Plugin) && klass.respond_to?(:config_name) && klass.config_name == name
267
266
  end
268
267
 
269
268
  def add_plugin(type, name, klass)
270
- if klass.respond_to?("javaClass", true)
271
- @registry[key_for(type, name)] = PluginSpecification.new(type, name, klass.javaClass)
272
- elsif !exists?(type, name)
269
+ if !exists?(type, name)
273
270
  specification_klass = type == :universal ? UniversalPluginSpecification : PluginSpecification
274
271
  @registry[key_for(type, name)] = specification_klass.new(type, name, klass)
275
272
  else
@@ -276,17 +276,6 @@ class LogStash::Runner < Clamp::StrictCommand
276
276
  LogStash::PLUGIN_REGISTRY.hooks.register_emitter(self.class, @dispatcher)
277
277
 
278
278
  @settings.validate_all
279
-
280
- if @settings.registered?("xpack.monitoring.enabled")
281
- verify_deprecated_setting("xpack.monitoring.elasticsearch.url", "xpack.monitoring.elasticsearch.hosts")
282
- verify_deprecated_setting("xpack.monitoring.elasticsearch.ssl.ca", "xpack.monitoring.elasticsearch.ssl.certificate_authority")
283
- end
284
-
285
- if @settings.registered?("xpack.management.enabled")
286
- verify_deprecated_setting("xpack.management.elasticsearch.url", "xpack.management.elasticsearch.hosts")
287
- verify_deprecated_setting("xpack.management.elasticsearch.ssl.ca", "xpack.management.elasticsearch.ssl.certificate_authority")
288
- end
289
-
290
279
  @dispatcher.fire(:before_bootstrap_checks)
291
280
 
292
281
  return start_shell(setting("interactive"), binding) if setting("interactive")
@@ -380,8 +369,6 @@ class LogStash::Runner < Clamp::StrictCommand
380
369
 
381
370
  @agent.shutdown
382
371
 
383
- logger.info("Logstash shut down.")
384
-
385
372
  # flush any outstanding log messages during shutdown
386
373
  org.apache.logging.log4j.LogManager.shutdown
387
374
 
@@ -514,15 +501,4 @@ class LogStash::Runner < Clamp::StrictCommand
514
501
  @settings.get_value(key)
515
502
  end
516
503
 
517
- def verify_deprecated_setting(old_setting, new_setting)
518
- if @settings.set?(old_setting)
519
- logger.warn("Deprecated setting `#{old_setting}` please use `#{new_setting}`")
520
- if @settings.set?(new_setting)
521
- signal_usage_error("Both `#{new_setting}` and the deprecated `#{old_setting}` settings are set, please remove `#{old_setting}`")
522
- else
523
- @settings.set_value(new_setting, @settings.get_value(old_setting))
524
- end
525
- end
526
- end
527
-
528
504
  end
@@ -368,7 +368,7 @@ module LogStash
368
368
  case value
369
369
  when ::Range
370
370
  value
371
- when ::Integer
371
+ when ::Fixnum
372
372
  value..value
373
373
  when ::String
374
374
  first, last = value.split(PORT_SEPARATOR)
@@ -481,11 +481,11 @@ module LogStash
481
481
 
482
482
  class Bytes < Coercible
483
483
  def initialize(name, default=nil, strict=true)
484
- super(name, ::Integer, default, strict=true) { |value| valid?(value) }
484
+ super(name, ::Fixnum, default, strict=true) { |value| valid?(value) }
485
485
  end
486
486
 
487
487
  def valid?(value)
488
- value.is_a?(::Integer) && value >= 0
488
+ value.is_a?(Fixnum) && value >= 0
489
489
  end
490
490
 
491
491
  def coerce(value)
@@ -508,11 +508,11 @@ module LogStash
508
508
 
509
509
  class TimeValue < Coercible
510
510
  def initialize(name, default, strict=true, &validator_proc)
511
- super(name, ::Integer, default, strict, &validator_proc)
511
+ super(name, ::Fixnum, default, strict, &validator_proc)
512
512
  end
513
513
 
514
514
  def coerce(value)
515
- return value if value.is_a?(::Integer)
515
+ return value if value.is_a?(::Fixnum)
516
516
  Util::TimeValue.from_value(value).to_nanos
517
517
  end
518
518
  end
@@ -10,11 +10,11 @@ module LogStash
10
10
  @metric = metric
11
11
  end
12
12
 
13
- def resolve(pipelines_registry, pipeline_configs)
13
+ def resolve(pipelines, pipeline_configs)
14
14
  actions = []
15
15
 
16
16
  pipeline_configs.each do |pipeline_config|
17
- pipeline = pipelines_registry.get_pipeline(pipeline_config.pipeline_id)
17
+ pipeline = pipelines[pipeline_config.pipeline_id]
18
18
 
19
19
  if pipeline.nil?
20
20
  actions << LogStash::PipelineAction::Create.new(pipeline_config, @metric)
@@ -25,12 +25,12 @@ module LogStash
25
25
  end
26
26
  end
27
27
 
28
- configured_pipelines = pipeline_configs.collect(&:pipeline_id)
28
+ running_pipelines = pipeline_configs.collect(&:pipeline_id)
29
29
 
30
30
  # If one of the running pipeline is not in the pipeline_configs, we assume that we need to
31
31
  # stop it.
32
- pipelines_registry.running_pipelines.keys
33
- .select { |pipeline_id| !configured_pipelines.include?(pipeline_id) }
32
+ pipelines.keys
33
+ .select { |pipeline_id| !running_pipelines.include?(pipeline_id) }
34
34
  .each { |pipeline_id| actions << LogStash::PipelineAction::Stop.new(pipeline_id) }
35
35
 
36
36
  actions.sort # See logstash/pipeline_action.rb
@@ -7,7 +7,7 @@ module LogStash::Util::DurationFormatter
7
7
  # a format that a human can understand. This is currently used by
8
8
  # the API.
9
9
  #
10
- # @param [Integer] Duration in milliseconds
10
+ # @param [Fixnum] Duration in milliseconds
11
11
  # @return [String] Duration in human format
12
12
  def self.human_format(duration)
13
13
  ChronicDuration.output(duration / 1000, CHRONIC_OPTIONS)
@@ -26,7 +26,6 @@ class LogStash::Util::SafeURI
26
26
  else
27
27
  raise ArgumentError, "Expected a string, java.net.URI, or URI, got a #{arg.class} creating a URL"
28
28
  end
29
- raise ArgumentError, "URI is not valid - host is not specified" if @uri.host.nil?
30
29
  end
31
30
 
32
31
  def to_s