logstash-core 6.8.23-java → 7.0.0.alpha1-java

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. checksums.yaml +4 -4
  2. data/lib/logstash/agent.rb +69 -85
  3. data/lib/logstash/api/modules/stats.rb +1 -1
  4. data/lib/logstash/compiler/lscl.rb +7 -7
  5. data/lib/logstash/config/config_ast.rb +1 -1
  6. data/lib/logstash/config/mixin.rb +1 -1
  7. data/lib/logstash/config/modules_common.rb +3 -3
  8. data/lib/logstash/dependency_report.rb +1 -2
  9. data/lib/logstash/environment.rb +4 -9
  10. data/lib/logstash/event.rb +1 -24
  11. data/lib/logstash/filter_delegator.rb +69 -2
  12. data/lib/logstash/filters/base.rb +2 -0
  13. data/lib/logstash/instrument/metric_store.rb +1 -1
  14. data/lib/logstash/instrument/periodic_poller/dlq.rb +7 -5
  15. data/lib/logstash/instrument/periodic_poller/jvm.rb +3 -3
  16. data/lib/logstash/instrument/periodic_poller/pq.rb +8 -6
  17. data/lib/logstash/instrument/periodic_pollers.rb +3 -3
  18. data/lib/logstash/java_pipeline.rb +11 -38
  19. data/lib/logstash/modules/kibana_config.rb +1 -1
  20. data/lib/logstash/modules/logstash_config.rb +1 -1
  21. data/lib/logstash/patches/resolv.rb +32 -17
  22. data/lib/logstash/pipeline.rb +11 -28
  23. data/lib/logstash/pipeline_action/base.rb +1 -1
  24. data/lib/logstash/pipeline_action/create.rb +13 -7
  25. data/lib/logstash/pipeline_action/reload.rb +12 -35
  26. data/lib/logstash/pipeline_action/stop.rb +6 -4
  27. data/lib/logstash/pipeline_settings.rb +1 -2
  28. data/lib/logstash/plugins/registry.rb +2 -5
  29. data/lib/logstash/runner.rb +0 -24
  30. data/lib/logstash/settings.rb +5 -5
  31. data/lib/logstash/state_resolver.rb +5 -5
  32. data/lib/logstash/util/duration_formatter.rb +1 -1
  33. data/lib/logstash/util/safe_uri.rb +0 -1
  34. data/lib/logstash/util/substitution_variables.rb +1 -22
  35. data/lib/logstash/util/thread_dump.rb +1 -1
  36. data/lib/logstash/util.rb +1 -11
  37. data/lib/logstash-core/version.rb +3 -1
  38. data/locales/en.yml +7 -16
  39. data/logstash-core.gemspec +11 -2
  40. data/spec/logstash/acked_queue_concurrent_stress_spec.rb +2 -2
  41. data/spec/logstash/agent/converge_spec.rb +31 -25
  42. data/spec/logstash/agent/metrics_spec.rb +1 -1
  43. data/spec/logstash/agent_spec.rb +7 -6
  44. data/spec/logstash/compiler/compiler_spec.rb +0 -28
  45. data/spec/logstash/config/config_ast_spec.rb +0 -15
  46. data/spec/logstash/config/mixin_spec.rb +2 -3
  47. data/spec/logstash/converge_result_spec.rb +1 -1
  48. data/spec/logstash/environment_spec.rb +4 -4
  49. data/spec/logstash/event_spec.rb +2 -10
  50. data/spec/logstash/filter_delegator_spec.rb +12 -2
  51. data/spec/logstash/filters/base_spec.rb +9 -45
  52. data/spec/logstash/instrument/periodic_poller/cgroup_spec.rb +2 -0
  53. data/spec/logstash/instrument/wrapped_write_client_spec.rb +1 -1
  54. data/spec/logstash/java_filter_delegator_spec.rb +11 -1
  55. data/spec/logstash/legacy_ruby_event_spec.rb +5 -6
  56. data/spec/logstash/patches_spec.rb +3 -1
  57. data/spec/logstash/pipeline_action/create_spec.rb +8 -14
  58. data/spec/logstash/pipeline_action/reload_spec.rb +9 -16
  59. data/spec/logstash/pipeline_action/stop_spec.rb +3 -4
  60. data/spec/logstash/queue_factory_spec.rb +1 -2
  61. data/spec/logstash/runner_spec.rb +0 -2
  62. data/spec/logstash/settings/array_coercible_spec.rb +1 -1
  63. data/spec/logstash/settings/bytes_spec.rb +2 -2
  64. data/spec/logstash/settings/port_range_spec.rb +1 -1
  65. data/spec/logstash/settings_spec.rb +0 -10
  66. data/spec/logstash/state_resolver_spec.rb +22 -26
  67. data/spec/logstash/util/safe_uri_spec.rb +0 -40
  68. data/spec/logstash/util/secretstore_spec.rb +1 -1
  69. data/spec/logstash/util/time_value_spec.rb +1 -1
  70. data/spec/logstash/util/wrapped_acked_queue_spec.rb +1 -1
  71. data/spec/logstash/webserver_spec.rb +5 -9
  72. data/spec/support/matchers.rb +19 -25
  73. data/spec/support/shared_contexts.rb +3 -3
  74. data/versions-gem-copy.yml +9 -9
  75. metadata +31 -44
  76. data/lib/logstash/patches/resolv_9270.rb +0 -2903
  77. data/lib/logstash/pipelines_registry.rb +0 -166
  78. data/lib/logstash/util/lazy_singleton.rb +0 -33
  79. data/spec/logstash/jruby_version_spec.rb +0 -15
  80. data/spec/logstash/pipelines_registry_spec.rb +0 -220
@@ -11,12 +11,14 @@ module LogStash module Instrument module PeriodicPoller
11
11
  end
12
12
 
13
13
  def collect
14
- pipelines = @agent.running_user_defined_pipelines
15
- pipelines.each do |_, pipeline|
16
- unless pipeline.nil?
17
- pipeline.collect_stats
18
- end
14
+ pipelines = @agent.with_running_user_defined_pipelines {|pipelines| pipelines}
15
+ unless pipelines.nil?
16
+ pipelines.each {|_, pipeline|
17
+ unless pipeline.nil?
18
+ pipeline.collect_stats
19
+ end
20
+ }
19
21
  end
20
22
  end
21
23
  end
22
- end end end
24
+ end; end; end
@@ -11,12 +11,12 @@ module LogStash module Instrument
11
11
  class PeriodicPollers
12
12
  attr_reader :metric
13
13
 
14
- def initialize(metric, queue_type, agent)
14
+ def initialize(metric, queue_type, pipelines)
15
15
  @metric = metric
16
16
  @periodic_pollers = [PeriodicPoller::Os.new(metric),
17
17
  PeriodicPoller::JVM.new(metric),
18
- PeriodicPoller::PersistentQueue.new(metric, queue_type, agent),
19
- PeriodicPoller::DeadLetterQueue.new(metric, agent)]
18
+ PeriodicPoller::PersistentQueue.new(metric, queue_type, pipelines),
19
+ PeriodicPoller::DeadLetterQueue.new(metric, pipelines)]
20
20
  end
21
21
 
22
22
  def start
@@ -1,7 +1,6 @@
1
1
  # encoding: utf-8
2
2
  require "thread"
3
3
  require "concurrent"
4
- require "logstash/event"
5
4
  require "logstash/filters/base"
6
5
  require "logstash/inputs/base"
7
6
  require "logstash/outputs/base"
@@ -39,23 +38,9 @@ module LogStash; class JavaPipeline < JavaBasePipeline
39
38
  @flushRequested = java.util.concurrent.atomic.AtomicBoolean.new(false)
40
39
  @shutdownRequested = java.util.concurrent.atomic.AtomicBoolean.new(false)
41
40
  @outputs_registered = Concurrent::AtomicBoolean.new(false)
42
-
43
- # @finished_execution signals that the pipeline thread has finished its execution
44
- # regardless of any exceptions; it will always be true when the thread completes
45
41
  @finished_execution = Concurrent::AtomicBoolean.new(false)
46
-
47
- # @finished_run signals that the run methods called in the pipeline thread was completed
48
- # without errors and it will NOT be set if the run method exits from an exception; this
49
- # is by design and necessary for the wait_until_started semantic
50
- @finished_run = Concurrent::AtomicBoolean.new(false)
51
-
52
- @thread = nil
53
42
  end # def initialize
54
43
 
55
- def finished_execution?
56
- @finished_execution.true?
57
- end
58
-
59
44
  def ready?
60
45
  @ready.value
61
46
  end
@@ -97,18 +82,15 @@ module LogStash; class JavaPipeline < JavaBasePipeline
97
82
  @logger.debug("Starting pipeline", default_logging_keys)
98
83
 
99
84
  @finished_execution.make_false
100
- @finished_run.make_false
101
85
 
102
86
  @thread = Thread.new do
103
87
  begin
104
88
  LogStash::Util.set_thread_name("pipeline.#{pipeline_id}")
105
89
  run
106
- @finished_run.make_true
90
+ @finished_execution.make_true
107
91
  rescue => e
108
92
  close
109
93
  logger.error("Pipeline aborted due to error", default_logging_keys(:exception => e, :backtrace => e.backtrace))
110
- ensure
111
- @finished_execution.make_true
112
94
  end
113
95
  end
114
96
 
@@ -123,14 +105,15 @@ module LogStash; class JavaPipeline < JavaBasePipeline
123
105
 
124
106
  def wait_until_started
125
107
  while true do
126
- if @finished_run.true?
127
- # it completed run without exception
108
+ # This should be changed with an appropriate FSM
109
+ # It's an edge case, if we have a pipeline with
110
+ # a generator { count => 1 } its possible that `Thread#alive?` doesn't return true
111
+ # because the execution of the thread was successful and complete
112
+ if @finished_execution.true?
128
113
  return true
129
114
  elsif thread.nil? || !thread.alive?
130
- # some exception occurred and the thread is dead
131
115
  return false
132
116
  elsif running?
133
- # fully initialized and running
134
117
  return true
135
118
  else
136
119
  sleep 0.01
@@ -232,11 +215,11 @@ module LogStash; class JavaPipeline < JavaBasePipeline
232
215
 
233
216
  pipeline_workers.times do |t|
234
217
  thread = Thread.new do
235
- Util.set_thread_name("[#{pipeline_id}]>worker#{t}")
236
218
  org.logstash.execution.WorkerLoop.new(
237
219
  lir_execution, filter_queue_client, @events_filtered, @events_consumed,
238
220
  @flushRequested, @flushing, @shutdownRequested, @drain_queue).run
239
221
  end
222
+ Util.set_thread_name("[#{pipeline_id}]>worker#{t}")
240
223
  @worker_threads << thread
241
224
  end
242
225
 
@@ -257,13 +240,7 @@ module LogStash; class JavaPipeline < JavaBasePipeline
257
240
  end
258
241
 
259
242
  def wait_inputs
260
- @input_threads.each do |thread|
261
- if thread.class == Java::JavaObject
262
- thread.to_java.join
263
- else
264
- thread.join
265
- end
266
- end
243
+ @input_threads.each(&:join)
267
244
  end
268
245
 
269
246
  def start_inputs
@@ -285,11 +262,7 @@ module LogStash; class JavaPipeline < JavaBasePipeline
285
262
  end
286
263
 
287
264
  def start_input(plugin)
288
- if plugin.class == LogStash::JavaInputDelegator
289
- @input_threads << plugin.start
290
- else
291
- @input_threads << Thread.new { inputworker(plugin) }
292
- end
265
+ @input_threads << Thread.new { inputworker(plugin) }
293
266
  end
294
267
 
295
268
  def inputworker(plugin)
@@ -335,7 +308,7 @@ module LogStash; class JavaPipeline < JavaBasePipeline
335
308
 
336
309
  stop_inputs
337
310
 
338
- # We make this call blocking, so we know for sure when the method return the shtudown is
311
+ # We make this call blocking, so we know for sure when the method return the shutdown is
339
312
  # stopped
340
313
  wait_for_workers
341
314
  clear_pipeline_metrics
@@ -399,7 +372,7 @@ module LogStash; class JavaPipeline < JavaBasePipeline
399
372
  end
400
373
 
401
374
  def plugin_threads_info
402
- input_threads = @input_threads.select {|t| t.class == Thread && t.alive? }
375
+ input_threads = @input_threads.select {|t| t.alive? }
403
376
  worker_threads = @worker_threads.select {|t| t.alive? }
404
377
  (input_threads + worker_threads).map {|t| Util.thread_info(t) }
405
378
  end
@@ -8,7 +8,7 @@ module LogStash module Modules class KibanaConfig
8
8
  include LogStash::Util::Loggable
9
9
 
10
10
  ALLOWED_DIRECTORIES = ["search", "visualization"]
11
- attr_reader :index_name # not used when importing via kibana but for BWC with ElastsearchConfig
11
+ attr_reader :index_name # not used when importing via kibana but for BWC with ElasticsearchConfig
12
12
 
13
13
  # We name it `modul` here because `module` has meaning in Ruby.
14
14
  def initialize(modul, settings)
@@ -39,7 +39,7 @@ module LogStash module Modules class LogStashConfig
39
39
  def get_setting(setting_class)
40
40
  raw_value = @settings[setting_class.name]
41
41
  # If we dont check for NIL, the Settings class will try to coerce the value
42
- # and most of the it will fails when a NIL value is explicitely set.
42
+ # and most of the it will fails when a NIL value is explicitly set.
43
43
  # This will be fixed once we wrap the plugins settings into a Settings class
44
44
  setting_class.set(raw_value) unless raw_value.nil?
45
45
  setting_class.value
@@ -1,25 +1,40 @@
1
1
  require "resolv"
2
2
 
3
- # ref:
4
- # https://github.com/logstash-plugins/logstash-filter-dns/issues/51
5
- # https://github.com/jruby/jruby/pull/5722
3
+ # ref: https://github.com/logstash-plugins/logstash-filter-dns/issues/40
6
4
  #
7
- # JRuby versions starting at 9.2.0.0 have a bug in resolv.rb with a leak between the
8
- # DNS.allocate_request_id/DNS.free_request_id methods.
5
+ # JRuby 9k versions prior to 9.1.16.0 have a bug which crashes IP address
6
+ # resolution after 64k unique IP addresses resolutions.
9
7
  #
10
- # We are opting to load a patched full resolv.rb instead of monkey patching the
11
- # offending methods because we do not yet know in which upcoming version of JRuby
12
- # this will be fixed and we want to avoid potential conflicting monkey patches.
13
- # A spec which will break on JRuby upgrade will redirect here
14
- # to make a manual verification and eventually remove that patch here once the fix is
15
- # made in the JRuby version of resolv.rb.
8
+ # Note that the oldest JRuby version in LS 6 is 9.1.13.0 and
9
+ # JRuby 1.7.25 and 1.7.27 (the 2 versions used across LS 5) are not affected by this bug.
16
10
 
17
- if Gem::Version.new(JRUBY_VERSION) >= Gem::Version.new("9.2.0.0")
18
- # save verbose level and mute the "warning: already initialized constant"
19
- warn_level = $VERBOSE
20
- $VERBOSE = nil
11
+ # make sure we abort if a known correct JRuby version is installed
12
+ # to avoid having an unnecessary legacy patch being applied in the future.
13
+ raise("Unnecessary patch on resolv.rb for JRuby version 9.1.16+") if Gem::Version.new(JRUBY_VERSION) >= Gem::Version.new("9.1.16.0")
21
14
 
22
- require_relative "resolv_9270"
15
+ # The code below is copied from JRuby 9.1.16.0 resolv.rb:
16
+ # https://github.com/jruby/jruby/blob/9.1.16.0/lib/ruby/stdlib/resolv.rb#L775-L784
17
+ #
18
+ # JRuby is Copyright (c) 2007-2017 The JRuby project, and is released
19
+ # under a tri EPL/GPL/LGPL license.
20
+ # Full license available at https://github.com/jruby/jruby/blob/9.1.16.0/COPYING
23
21
 
24
- $VERBOSE = warn_level
22
+ class Resolv
23
+ class DNS
24
+ class Requester
25
+ class UnconnectedUDP
26
+ def sender(msg, data, host, port=Port)
27
+ sock = @socks_hash[host.index(':') ? "::" : "0.0.0.0"]
28
+ return nil if !sock
29
+ service = [IPAddr.new(host), port]
30
+ id = DNS.allocate_request_id(service[0], service[1])
31
+ request = msg.encode
32
+ request[0,2] = [id].pack('n')
33
+ return @senders[[service, id]] =
34
+ Sender.new(request, data, sock, host, port)
35
+ end
36
+ end
37
+ end
38
+ end
25
39
  end
40
+
@@ -107,23 +107,8 @@ module LogStash; class Pipeline < BasePipeline
107
107
  @flushing = Concurrent::AtomicReference.new(false)
108
108
  @outputs_registered = Concurrent::AtomicBoolean.new(false)
109
109
  @worker_shutdown = java.util.concurrent.atomic.AtomicBoolean.new(false)
110
-
111
- # @finished_execution signals that the pipeline thread has finished its execution
112
- # regardless of any exceptions; it will always be true when the thread completes
113
- @finished_execution = Concurrent::AtomicBoolean.new(false)
114
-
115
- # @finished_run signals that the run methods called in the pipeline thread was completed
116
- # without errors and it will NOT be set if the run method exits from an exception; this
117
- # is by design and necessary for the wait_until_started semantic
118
- @finished_run = Concurrent::AtomicBoolean.new(false)
119
-
120
- @thread = nil
121
110
  end # def initialize
122
111
 
123
- def finished_execution?
124
- @finished_execution.true?
125
- end
126
-
127
112
  def ready?
128
113
  @ready.value
129
114
  end
@@ -167,19 +152,16 @@ module LogStash; class Pipeline < BasePipeline
167
152
  "pipeline.batch.size" => settings.get("pipeline.batch.size"),
168
153
  "pipeline.batch.delay" => settings.get("pipeline.batch.delay")))
169
154
 
170
- @finished_execution.make_false
171
- @finished_run.make_false
155
+ @finished_execution = Concurrent::AtomicBoolean.new(false)
172
156
 
173
157
  @thread = Thread.new do
174
158
  begin
175
159
  LogStash::Util.set_thread_name("pipeline.#{pipeline_id}")
176
160
  run
177
- @finished_run.make_true
161
+ @finished_execution.make_true
178
162
  rescue => e
179
163
  close
180
164
  @logger.error("Pipeline aborted due to error", default_logging_keys(:exception => e, :backtrace => e.backtrace))
181
- ensure
182
- @finished_execution.make_true
183
165
  end
184
166
  end
185
167
 
@@ -194,14 +176,15 @@ module LogStash; class Pipeline < BasePipeline
194
176
 
195
177
  def wait_until_started
196
178
  while true do
197
- if @finished_run.true?
198
- # it completed run without exception
179
+ # This should be changed with an appropriate FSM
180
+ # It's an edge case, if we have a pipeline with
181
+ # a generator { count => 1 } its possible that `Thread#alive?` doesn't return true
182
+ # because the execution of the thread was successful and complete
183
+ if @finished_execution.true?
199
184
  return true
200
- elsif thread.nil? || !thread.alive?
201
- # some exception occured and the thread is dead
185
+ elsif !thread.alive?
202
186
  return false
203
187
  elsif running?
204
- # fully initialized and running
205
188
  return true
206
189
  else
207
190
  sleep 0.01
@@ -300,9 +283,9 @@ module LogStash; class Pipeline < BasePipeline
300
283
 
301
284
  pipeline_workers.times do |t|
302
285
  thread = Thread.new(batch_size, batch_delay, self) do |_b_size, _b_delay, _pipeline|
303
- Util.set_thread_name("[#{pipeline_id}]>worker#{t}")
304
286
  _pipeline.worker_loop(_b_size, _b_delay)
305
287
  end
288
+ Util.set_thread_name("[#{pipeline_id}]>worker#{t}")
306
289
  @worker_threads << thread
307
290
  end
308
291
 
@@ -463,7 +446,7 @@ module LogStash; class Pipeline < BasePipeline
463
446
 
464
447
  stop_inputs
465
448
 
466
- # We make this call blocking, so we know for sure when the method return the shtudown is
449
+ # We make this call blocking, so we know for sure when the method return the shutdown is
467
450
  # stopped
468
451
  wait_for_workers
469
452
  clear_pipeline_metrics
@@ -542,7 +525,7 @@ module LogStash; class Pipeline < BasePipeline
542
525
 
543
526
  # Calculate the uptime in milliseconds
544
527
  #
545
- # @return [Integer] Uptime in milliseconds, 0 if the pipeline is not started
528
+ # @return [Fixnum] Uptime in milliseconds, 0 if the pipeline is not started
546
529
  def uptime
547
530
  return 0 if started_at.nil?
548
531
  ((Time.now.to_f - started_at.to_f) * 1000.0).to_i
@@ -12,7 +12,7 @@ module LogStash module PipelineAction
12
12
  end
13
13
  alias_method :to_s, :inspect
14
14
 
15
- def execute(agent, pipelines_registry)
15
+ def execute(agent, pipelines)
16
16
  raise "`#execute` Not implemented!"
17
17
  end
18
18
 
@@ -3,7 +3,6 @@ require "logstash/pipeline_action/base"
3
3
  require "logstash/pipeline"
4
4
  require "logstash/java_pipeline"
5
5
 
6
-
7
6
  module LogStash module PipelineAction
8
7
  class Create < Base
9
8
  include LogStash::Util::Loggable
@@ -31,26 +30,33 @@ module LogStash module PipelineAction
31
30
 
32
31
  # The execute assume that the thread safety access of the pipeline
33
32
  # is managed by the caller.
34
- def execute(agent, pipelines_registry)
35
- new_pipeline =
33
+ def execute(agent, pipelines)
34
+ pipeline =
36
35
  if @pipeline_config.settings.get_value("pipeline.java_execution")
37
36
  LogStash::JavaPipeline.new(@pipeline_config, @metric, agent)
38
37
  else
39
38
  agent.exclusive do
40
39
  # The Ruby pipeline initialization is not thread safe because of the module level
41
40
  # shared state in LogsStash::Config::AST. When using multiple pipelines this gets
42
- # executed simultaneously in different threads and we need to synchronize this initialization.
41
+ # executed simultaneously in different threads and we need to synchonize this initialization.
43
42
  LogStash::Pipeline.new(@pipeline_config, @metric, agent)
44
43
  end
45
44
  end
46
45
 
47
- success = pipelines_registry.create_pipeline(pipeline_id, new_pipeline) do
48
- new_pipeline.start # block until the pipeline is correctly started or crashed
46
+ status = nil
47
+ pipelines.compute(pipeline_id) do |id,value|
48
+ if value
49
+ LogStash::ConvergeResult::ActionResult.create(self, true)
50
+ end
51
+ status = pipeline.start # block until the pipeline is correctly started or crashed
52
+ pipeline # The pipeline is successfully started we can add it to the map
49
53
  end
50
54
 
51
- LogStash::ConvergeResult::ActionResult.create(self, success)
55
+
56
+ LogStash::ConvergeResult::ActionResult.create(self, status)
52
57
  end
53
58
 
59
+
54
60
  def to_s
55
61
  "PipelineAction::Create<#{pipeline_id}>"
56
62
  end
@@ -20,12 +20,8 @@ module LogStash module PipelineAction
20
20
  "PipelineAction::Reload<#{pipeline_id}>"
21
21
  end
22
22
 
23
- def execute(agent, pipelines_registry)
24
- old_pipeline = pipelines_registry.get_pipeline(pipeline_id)
25
-
26
- if old_pipeline.nil?
27
- return LogStash::ConvergeResult::FailedAction.new("Cannot reload pipeline, because the pipeline does not exist")
28
- end
23
+ def execute(agent, pipelines)
24
+ old_pipeline = pipelines[pipeline_id]
29
25
 
30
26
  if !old_pipeline.reloadable?
31
27
  return LogStash::ConvergeResult::FailedAction.new("Cannot reload pipeline, because the existing pipeline is not reloadable")
@@ -38,8 +34,8 @@ module LogStash module PipelineAction
38
34
  else
39
35
  agent.exclusive do
40
36
  # The Ruby pipeline initialization is not thread safe because of the module level
41
- # shared state in LogsStash::Config::AST. When using multiple pipelines this gets
42
- # executed simultaneously in different threads and we need to synchronize this initialization.
37
+ # shared state in LogsStash::Config::AST. When using multiple pipelines this can gets
38
+ # executed simultaneously in different threads and we need to synchonize this initialization.
43
39
  LogStash::BasePipeline.new(@pipeline_config)
44
40
  end
45
41
  end
@@ -53,35 +49,16 @@ module LogStash module PipelineAction
53
49
 
54
50
  logger.info("Reloading pipeline", "pipeline.id" => pipeline_id)
55
51
 
56
- success = pipelines_registry.reload_pipeline(pipeline_id) do
57
- # important NOT to explicitly return from block here
58
- # the block must emit a success boolean value
52
+ pipelines.compute(pipeline_id) do |_,pipeline|
53
+ status = Stop.new(pipeline_id).execute(agent, pipelines)
59
54
 
60
- # First shutdown old pipeline
61
- old_pipeline.shutdown { LogStash::ShutdownWatcher.start(old_pipeline) }
62
- old_pipeline.thread.join
63
-
64
- # Then create a new pipeline
65
- new_pipeline =
66
- if @pipeline_config.settings.get_value("pipeline.java_execution")
67
- LogStash::JavaPipeline.new(@pipeline_config, @metric, agent)
68
- else
69
- agent.exclusive do
70
- # The Ruby pipeline initialization is not thread safe because of the module level
71
- # shared state in LogsStash::Config::AST. When using multiple pipelines this gets
72
- # executed simultaneously in different threads and we need to synchronize this initialization.
73
- LogStash::Pipeline.new(@pipeline_config, @metric, agent)
74
- end
75
- end
76
-
77
- success = new_pipeline.start # block until the pipeline is correctly started or crashed
78
-
79
- # return success and new_pipeline to registry reload_pipeline
80
- [success, new_pipeline]
55
+ if status
56
+ return Create.new(@pipeline_config, @metric).execute(agent, pipelines)
57
+ else
58
+ return status
59
+ end
60
+ pipeline
81
61
  end
82
-
83
- LogStash::ConvergeResult::ActionResult.create(self, success)
84
62
  end
85
-
86
63
  end
87
64
  end end
@@ -9,13 +9,15 @@ module LogStash module PipelineAction
9
9
  @pipeline_id = pipeline_id
10
10
  end
11
11
 
12
- def execute(agent, pipelines_registry)
13
- pipelines_registry.terminate_pipeline(pipeline_id) do |pipeline|
12
+ def execute(agent, pipelines)
13
+ pipelines.compute(pipeline_id) do |_,pipeline|
14
14
  pipeline.shutdown { LogStash::ShutdownWatcher.start(pipeline) }
15
15
  pipeline.thread.join
16
+ nil # delete the pipeline
16
17
  end
17
-
18
- LogStash::ConvergeResult::SuccessfulAction.new
18
+ # If we reach this part of the code we have succeeded because
19
+ # the shutdown call will block.
20
+ return LogStash::ConvergeResult::SuccessfulAction.new
19
21
  end
20
22
 
21
23
  def to_s
@@ -28,7 +28,6 @@ module LogStash
28
28
  "queue.checkpoint.acks",
29
29
  "queue.checkpoint.interval",
30
30
  "queue.checkpoint.writes",
31
- "queue.checkpoint.retry",
32
31
  "queue.drain",
33
32
  "queue.max_bytes",
34
33
  "queue.max_events",
@@ -47,7 +46,7 @@ module LogStash
47
46
 
48
47
  def register(setting)
49
48
  unless SETTINGS_WHITE_LIST.include?(setting.name)
50
- raise ArgumentError.new("Only pipeline related settings can be registed in a PipelineSettings object. Received \"#{setting.name}\". Allowed settings: #{SETTINGS_WHITE_LIST}")
49
+ raise ArgumentError.new("Only pipeline related settings can be registered in a PipelineSettings object. Received \"#{setting.name}\". Allowed settings: #{SETTINGS_WHITE_LIST}")
51
50
  end
52
51
  super(setting)
53
52
  end
@@ -262,14 +262,11 @@ module LogStash module Plugins
262
262
  # @param name [String] plugin name
263
263
  # @return [Boolean] true if klass is a valid plugin for name
264
264
  def is_a_plugin?(klass, name)
265
- (klass.class == Java::JavaClass && klass.simple_name.downcase == name.gsub('_','')) ||
266
- (klass.ancestors.include?(LogStash::Plugin) && klass.respond_to?(:config_name) && klass.config_name == name)
265
+ klass.ancestors.include?(LogStash::Plugin) && klass.respond_to?(:config_name) && klass.config_name == name
267
266
  end
268
267
 
269
268
  def add_plugin(type, name, klass)
270
- if klass.respond_to?("javaClass", true)
271
- @registry[key_for(type, name)] = PluginSpecification.new(type, name, klass.javaClass)
272
- elsif !exists?(type, name)
269
+ if !exists?(type, name)
273
270
  specification_klass = type == :universal ? UniversalPluginSpecification : PluginSpecification
274
271
  @registry[key_for(type, name)] = specification_klass.new(type, name, klass)
275
272
  else
@@ -276,17 +276,6 @@ class LogStash::Runner < Clamp::StrictCommand
276
276
  LogStash::PLUGIN_REGISTRY.hooks.register_emitter(self.class, @dispatcher)
277
277
 
278
278
  @settings.validate_all
279
-
280
- if @settings.registered?("xpack.monitoring.enabled")
281
- verify_deprecated_setting("xpack.monitoring.elasticsearch.url", "xpack.monitoring.elasticsearch.hosts")
282
- verify_deprecated_setting("xpack.monitoring.elasticsearch.ssl.ca", "xpack.monitoring.elasticsearch.ssl.certificate_authority")
283
- end
284
-
285
- if @settings.registered?("xpack.management.enabled")
286
- verify_deprecated_setting("xpack.management.elasticsearch.url", "xpack.management.elasticsearch.hosts")
287
- verify_deprecated_setting("xpack.management.elasticsearch.ssl.ca", "xpack.management.elasticsearch.ssl.certificate_authority")
288
- end
289
-
290
279
  @dispatcher.fire(:before_bootstrap_checks)
291
280
 
292
281
  return start_shell(setting("interactive"), binding) if setting("interactive")
@@ -380,8 +369,6 @@ class LogStash::Runner < Clamp::StrictCommand
380
369
 
381
370
  @agent.shutdown
382
371
 
383
- logger.info("Logstash shut down.")
384
-
385
372
  # flush any outstanding log messages during shutdown
386
373
  org.apache.logging.log4j.LogManager.shutdown
387
374
 
@@ -514,15 +501,4 @@ class LogStash::Runner < Clamp::StrictCommand
514
501
  @settings.get_value(key)
515
502
  end
516
503
 
517
- def verify_deprecated_setting(old_setting, new_setting)
518
- if @settings.set?(old_setting)
519
- logger.warn("Deprecated setting `#{old_setting}` please use `#{new_setting}`")
520
- if @settings.set?(new_setting)
521
- signal_usage_error("Both `#{new_setting}` and the deprecated `#{old_setting}` settings are set, please remove `#{old_setting}`")
522
- else
523
- @settings.set_value(new_setting, @settings.get_value(old_setting))
524
- end
525
- end
526
- end
527
-
528
504
  end
@@ -368,7 +368,7 @@ module LogStash
368
368
  case value
369
369
  when ::Range
370
370
  value
371
- when ::Integer
371
+ when ::Fixnum
372
372
  value..value
373
373
  when ::String
374
374
  first, last = value.split(PORT_SEPARATOR)
@@ -481,11 +481,11 @@ module LogStash
481
481
 
482
482
  class Bytes < Coercible
483
483
  def initialize(name, default=nil, strict=true)
484
- super(name, ::Integer, default, strict=true) { |value| valid?(value) }
484
+ super(name, ::Fixnum, default, strict=true) { |value| valid?(value) }
485
485
  end
486
486
 
487
487
  def valid?(value)
488
- value.is_a?(::Integer) && value >= 0
488
+ value.is_a?(Fixnum) && value >= 0
489
489
  end
490
490
 
491
491
  def coerce(value)
@@ -508,11 +508,11 @@ module LogStash
508
508
 
509
509
  class TimeValue < Coercible
510
510
  def initialize(name, default, strict=true, &validator_proc)
511
- super(name, ::Integer, default, strict, &validator_proc)
511
+ super(name, ::Fixnum, default, strict, &validator_proc)
512
512
  end
513
513
 
514
514
  def coerce(value)
515
- return value if value.is_a?(::Integer)
515
+ return value if value.is_a?(::Fixnum)
516
516
  Util::TimeValue.from_value(value).to_nanos
517
517
  end
518
518
  end
@@ -10,11 +10,11 @@ module LogStash
10
10
  @metric = metric
11
11
  end
12
12
 
13
- def resolve(pipelines_registry, pipeline_configs)
13
+ def resolve(pipelines, pipeline_configs)
14
14
  actions = []
15
15
 
16
16
  pipeline_configs.each do |pipeline_config|
17
- pipeline = pipelines_registry.get_pipeline(pipeline_config.pipeline_id)
17
+ pipeline = pipelines[pipeline_config.pipeline_id]
18
18
 
19
19
  if pipeline.nil?
20
20
  actions << LogStash::PipelineAction::Create.new(pipeline_config, @metric)
@@ -25,12 +25,12 @@ module LogStash
25
25
  end
26
26
  end
27
27
 
28
- configured_pipelines = pipeline_configs.collect(&:pipeline_id)
28
+ running_pipelines = pipeline_configs.collect(&:pipeline_id)
29
29
 
30
30
  # If one of the running pipeline is not in the pipeline_configs, we assume that we need to
31
31
  # stop it.
32
- pipelines_registry.running_pipelines.keys
33
- .select { |pipeline_id| !configured_pipelines.include?(pipeline_id) }
32
+ pipelines.keys
33
+ .select { |pipeline_id| !running_pipelines.include?(pipeline_id) }
34
34
  .each { |pipeline_id| actions << LogStash::PipelineAction::Stop.new(pipeline_id) }
35
35
 
36
36
  actions.sort # See logstash/pipeline_action.rb
@@ -7,7 +7,7 @@ module LogStash::Util::DurationFormatter
7
7
  # a format that a human can understand. This is currently used by
8
8
  # the API.
9
9
  #
10
- # @param [Integer] Duration in milliseconds
10
+ # @param [Fixnum] Duration in milliseconds
11
11
  # @return [String] Duration in human format
12
12
  def self.human_format(duration)
13
13
  ChronicDuration.output(duration / 1000, CHRONIC_OPTIONS)
@@ -26,7 +26,6 @@ class LogStash::Util::SafeURI
26
26
  else
27
27
  raise ArgumentError, "Expected a string, java.net.URI, or URI, got a #{arg.class} creating a URL"
28
28
  end
29
- raise ArgumentError, "URI is not valid - host is not specified" if @uri.host.nil?
30
29
  end
31
30
 
32
31
  def to_s