logstash-core 6.6.0-java → 6.6.1-java

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: cbe096068df3326925926019e2137eb56527f92c91c9d14861616d85e19ac46b
4
- data.tar.gz: 8a5a795fbb82efc19a01232f8ef173d668948a33f75aaa749df73e0cc075e206
3
+ metadata.gz: 6ae11e5a947442d4fe74bd284d674355a78112f5aff290403930bb35665ad724
4
+ data.tar.gz: 30c09cc64470baa5bb29f806cff27614fe64f83a205b6704740db4be569dd3a9
5
5
  SHA512:
6
- metadata.gz: 99c5e85d59c293bd4479ca1d4805cb0dff72b225ed97aa7351060c452c1e73e495a24bf3fbc4142df0e67f01bb1eaa90893af1cd8c30d22f6a61d4fed2a0cfd0
7
- data.tar.gz: 977df2140b2d9596ad4d08e784bd03cbb063c2c40b9a158771eaf91b6b4f09f80e0804dce33f801c002dfe2cf4664b8a544f588565c4aa492fa2f83c7c964339
6
+ metadata.gz: 81f35a278ce95f4cb063524c5333f4ce09f1de1a14fbefdc09a4b72f62cea73ec519a90b5b2b27cd720bd802bf40001656e6f90a294440c9898738567104dbdd
7
+ data.tar.gz: 74fdec603be67ac988e3b786e890733fd8254caa78c7d03b2c99afc1a8bb25779b0fefc9cb5675fdfec1fa06b3e124c6af880c0ee0197320d14e176d1370c742
@@ -8,6 +8,7 @@ require "logstash/webserver"
8
8
  require "logstash/config/source_loader"
9
9
  require "logstash/pipeline_action"
10
10
  require "logstash/state_resolver"
11
+ require "logstash/pipelines_registry"
11
12
  require "stud/trap"
12
13
  require "uri"
13
14
  require "socket"
@@ -19,7 +20,7 @@ class LogStash::Agent
19
20
  include LogStash::Util::Loggable
20
21
  STARTED_AT = Time.now.freeze
21
22
 
22
- attr_reader :metric, :name, :settings, :webserver, :dispatcher, :ephemeral_id, :pipelines, :pipeline_bus
23
+ attr_reader :metric, :name, :settings, :dispatcher, :ephemeral_id, :pipeline_bus
23
24
  attr_accessor :logger
24
25
 
25
26
  # initialize method for LogStash::Agent
@@ -36,11 +37,12 @@ class LogStash::Agent
36
37
  # Mutex to synchonize in the exclusive method
37
38
  # Initial usage for the Ruby pipeline initialization which is not thread safe
38
39
  @exclusive_lock = Mutex.new
40
+ @webserver_control_lock = Mutex.new
39
41
 
40
42
  # Special bus object for inter-pipelines communications. Used by the `pipeline` input/output
41
43
  @pipeline_bus = org.logstash.plugins.pipeline.PipelineBus.new
42
44
 
43
- @pipelines = java.util.concurrent.ConcurrentHashMap.new();
45
+ @pipelines_registry = LogStash::PipelinesRegistry.new
44
46
 
45
47
  @name = setting("node.name")
46
48
  @http_host = setting("http.host")
@@ -118,14 +120,17 @@ class LogStash::Agent
118
120
  converge_state_and_update unless stopped?
119
121
  end
120
122
  else
121
- return 1 if clean_state?
123
+ # exit with error status if the initial converge_state_and_update did not create any pipeline
124
+ return 1 if @pipelines_registry.empty?
122
125
 
123
126
  while !Stud.stop?
124
- if clean_state? || running_user_defined_pipelines?
125
- sleep(0.5)
126
- else
127
- break
128
- end
127
+ # exit if all pipelines are terminated and none are reloading
128
+ break if no_pipeline?
129
+
130
+ # exit if there are no user defined pipelines (not system pipeline) and none are reloading
131
+ break if !running_user_defined_pipelines?
132
+
133
+ sleep(0.5)
129
134
  end
130
135
  end
131
136
 
@@ -139,11 +144,11 @@ class LogStash::Agent
139
144
  end
140
145
 
141
146
  def running?
142
- @running.value
147
+ @running.true?
143
148
  end
144
149
 
145
150
  def stopped?
146
- !@running.value
151
+ @running.false?
147
152
  end
148
153
 
149
154
  def converge_state_and_update
@@ -237,43 +242,48 @@ class LogStash::Agent
237
242
  @id_path ||= ::File.join(settings.get("path.data"), "uuid")
238
243
  end
239
244
 
245
+ #
246
+ # Backward compatibility proxies to the PipelineRegistry
247
+ #
248
+
240
249
  def get_pipeline(pipeline_id)
241
- pipelines.get(pipeline_id)
250
+ @pipelines_registry.get_pipeline(pipeline_id)
242
251
  end
243
252
 
244
253
  def pipelines_count
245
- pipelines.size
254
+ @pipelines_registry.size
246
255
  end
247
256
 
248
257
  def running_pipelines
249
- pipelines.select {|id,pipeline| running_pipeline?(id) }
250
- end
258
+ @pipelines_registry.running_pipelines
259
+ end
251
260
 
252
261
  def non_running_pipelines
253
- pipelines.select {|id,pipeline| !running_pipeline?(id) }
262
+ @pipelines_registry.non_running_pipelines
254
263
  end
255
264
 
256
265
  def running_pipelines?
257
- running_pipelines_count > 0
266
+ @pipelines_registry.running_pipelines.any?
258
267
  end
259
268
 
260
269
  def running_pipelines_count
261
- running_pipelines.size
270
+ @pipelines_registry.running_pipelines.size
262
271
  end
263
272
 
264
273
  def running_user_defined_pipelines?
265
- !running_user_defined_pipelines.empty?
274
+ @pipelines_registry.running_user_defined_pipelines.any?
266
275
  end
267
276
 
268
277
  def running_user_defined_pipelines
269
- pipelines.select {|id, pipeline| running_pipeline?(id) && !pipeline.system? }
278
+ @pipelines_registry.running_user_defined_pipelines
270
279
  end
271
280
 
272
- def with_running_user_defined_pipelines
273
- yield running_user_defined_pipelines
281
+ def no_pipeline?
282
+ @pipelines_registry.running_pipelines.empty?
274
283
  end
275
284
 
276
285
  private
286
+
277
287
  def transition_to_stopped
278
288
  @running.make_false
279
289
  end
@@ -298,7 +308,7 @@ class LogStash::Agent
298
308
  converge_result = LogStash::ConvergeResult.new(pipeline_actions.size)
299
309
 
300
310
  pipeline_actions.map do |action|
301
- Thread.new do
311
+ Thread.new(action, converge_result) do |action, converge_result|
302
312
  java.lang.Thread.currentThread().setName("Converge #{action}");
303
313
  # We execute every task we need to converge the current state of pipelines
304
314
  # for every task we will record the action result, that will help us
@@ -314,34 +324,35 @@ class LogStash::Agent
314
324
  # that we currently have.
315
325
  begin
316
326
  logger.debug("Executing action", :action => action)
317
- action_result = action.execute(self, pipelines)
327
+ action_result = action.execute(self, @pipelines_registry)
318
328
  converge_result.add(action, action_result)
319
329
 
320
330
  unless action_result.successful?
321
- logger.error("Failed to execute action", :id => action.pipeline_id,
322
- :action_type => action_result.class, :message => action_result.message,
323
- :backtrace => action_result.backtrace)
331
+ logger.error("Failed to execute action",
332
+ :id => action.pipeline_id,
333
+ :action_type => action_result.class,
334
+ :message => action_result.message,
335
+ :backtrace => action_result.backtrace
336
+ )
324
337
  end
325
- rescue SystemExit => e
326
- converge_result.add(action, e)
327
- rescue Exception => e
338
+ rescue SystemExit, Exception => e
328
339
  logger.error("Failed to execute action", :action => action, :exception => e.class.name, :message => e.message, :backtrace => e.backtrace)
329
340
  converge_result.add(action, e)
330
341
  end
331
342
  end
332
343
  end.each(&:join)
333
344
 
334
- if logger.trace?
335
- logger.trace("Converge results", :success => converge_result.success?,
336
- :failed_actions => converge_result.failed_actions.collect { |a, r| "id: #{a.pipeline_id}, action_type: #{a.class}, message: #{r.message}" },
337
- :successful_actions => converge_result.successful_actions.collect { |a, r| "id: #{a.pipeline_id}, action_type: #{a.class}" })
338
- end
345
+ logger.trace? && logger.trace("Converge results",
346
+ :success => converge_result.success?,
347
+ :failed_actions => converge_result.failed_actions.collect { |a, r| "id: #{a.pipeline_id}, action_type: #{a.class}, message: #{r.message}" },
348
+ :successful_actions => converge_result.successful_actions.collect { |a, r| "id: #{a.pipeline_id}, action_type: #{a.class}" }
349
+ )
339
350
 
340
351
  converge_result
341
352
  end
342
353
 
343
354
  def resolve_actions(pipeline_configs)
344
- @state_resolver.resolve(@pipelines, pipeline_configs)
355
+ @state_resolver.resolve(@pipelines_registry, pipeline_configs)
345
356
  end
346
357
 
347
358
  def dispatch_events(converge_results)
@@ -358,20 +369,24 @@ class LogStash::Agent
358
369
  end
359
370
 
360
371
  def start_webserver
361
- options = {:http_host => @http_host, :http_ports => @http_port, :http_environment => @http_environment }
362
- @webserver = LogStash::WebServer.new(@logger, self, options)
363
- @webserver_thread = Thread.new(@webserver) do |webserver|
364
- LogStash::Util.set_thread_name("Api Webserver")
365
- webserver.run
372
+ @webserver_control_lock.synchronize do
373
+ options = {:http_host => @http_host, :http_ports => @http_port, :http_environment => @http_environment }
374
+ @webserver = LogStash::WebServer.new(@logger, self, options)
375
+ @webserver_thread = Thread.new(@webserver) do |webserver|
376
+ LogStash::Util.set_thread_name("Api Webserver")
377
+ webserver.run
378
+ end
366
379
  end
367
380
  end
368
381
 
369
382
  def stop_webserver
370
- if @webserver
371
- @webserver.stop
372
- if @webserver_thread.join(5).nil?
373
- @webserver_thread.kill
374
- @webserver_thread.join
383
+ @webserver_control_lock.synchronize do
384
+ if @webserver
385
+ @webserver.stop
386
+ if @webserver_thread.join(5).nil?
387
+ @webserver_thread.kill
388
+ @webserver_thread.join
389
+ end
375
390
  end
376
391
  end
377
392
  end
@@ -399,7 +414,7 @@ class LogStash::Agent
399
414
  end
400
415
 
401
416
  def shutdown_pipelines
402
- logger.debug("Shutting down all pipelines", :pipelines_count => pipelines_count)
417
+ logger.debug("Shutting down all pipelines", :pipelines_count => running_pipelines_count)
403
418
 
404
419
  # In this context I could just call shutdown, but I've decided to
405
420
  # use the stop action implementation for that so we have the same code.
@@ -408,16 +423,6 @@ class LogStash::Agent
408
423
  converge_state(pipeline_actions)
409
424
  end
410
425
 
411
- def running_pipeline?(pipeline_id)
412
- pipeline = get_pipeline(pipeline_id)
413
- return false unless pipeline
414
- thread = pipeline.thread
415
- thread.is_a?(Thread) && thread.alive?
416
- end
417
-
418
- def clean_state?
419
- pipelines.empty?
420
- end
421
426
 
422
427
  def setting(key)
423
428
  @settings.get(key)
@@ -10,13 +10,11 @@ module LogStash module Instrument module PeriodicPoller
10
10
  end
11
11
 
12
12
  def collect
13
- pipelines = @agent.with_running_user_defined_pipelines {|pipelines| pipelines}
14
- unless pipelines.nil?
15
- pipelines.each {|_, pipeline|
16
- unless pipeline.nil?
17
- pipeline.collect_dlq_stats
18
- end
19
- }
13
+ pipelines = @agent.running_user_defined_pipelines
14
+ pipelines.each do |_, pipeline|
15
+ unless pipeline.nil?
16
+ pipeline.collect_dlq_stats
17
+ end
20
18
  end
21
19
  end
22
20
  end
@@ -11,14 +11,12 @@ module LogStash module Instrument module PeriodicPoller
11
11
  end
12
12
 
13
13
  def collect
14
- pipelines = @agent.with_running_user_defined_pipelines {|pipelines| pipelines}
15
- unless pipelines.nil?
16
- pipelines.each {|_, pipeline|
17
- unless pipeline.nil?
18
- pipeline.collect_stats
19
- end
20
- }
14
+ pipelines = @agent.running_user_defined_pipelines
15
+ pipelines.each do |_, pipeline|
16
+ unless pipeline.nil?
17
+ pipeline.collect_stats
18
+ end
21
19
  end
22
20
  end
23
21
  end
24
- end; end; end
22
+ end end end
@@ -11,12 +11,12 @@ module LogStash module Instrument
11
11
  class PeriodicPollers
12
12
  attr_reader :metric
13
13
 
14
- def initialize(metric, queue_type, pipelines)
14
+ def initialize(metric, queue_type, agent)
15
15
  @metric = metric
16
16
  @periodic_pollers = [PeriodicPoller::Os.new(metric),
17
17
  PeriodicPoller::JVM.new(metric),
18
- PeriodicPoller::PersistentQueue.new(metric, queue_type, pipelines),
19
- PeriodicPoller::DeadLetterQueue.new(metric, pipelines)]
18
+ PeriodicPoller::PersistentQueue.new(metric, queue_type, agent),
19
+ PeriodicPoller::DeadLetterQueue.new(metric, agent)]
20
20
  end
21
21
 
22
22
  def start
@@ -41,9 +41,23 @@ module LogStash; class JavaPipeline < JavaBasePipeline
41
41
  @flushRequested = java.util.concurrent.atomic.AtomicBoolean.new(false)
42
42
  @shutdownRequested = java.util.concurrent.atomic.AtomicBoolean.new(false)
43
43
  @outputs_registered = Concurrent::AtomicBoolean.new(false)
44
+
45
+ # @finished_execution signals that the pipeline thread has finished its execution
46
+ # regardless of any exceptions; it will always be true when the thread completes
44
47
  @finished_execution = Concurrent::AtomicBoolean.new(false)
48
+
49
+ # @finished_run signals that the run methods called in the pipeline thread was completed
50
+ # without errors and it will NOT be set if the run method exits from an exception; this
51
+ # is by design and necessary for the wait_until_started semantic
52
+ @finished_run = Concurrent::AtomicBoolean.new(false)
53
+
54
+ @thread = nil
45
55
  end # def initialize
46
56
 
57
+ def finished_execution?
58
+ @finished_execution.true?
59
+ end
60
+
47
61
  def ready?
48
62
  @ready.value
49
63
  end
@@ -85,15 +99,18 @@ module LogStash; class JavaPipeline < JavaBasePipeline
85
99
  @logger.debug("Starting pipeline", default_logging_keys)
86
100
 
87
101
  @finished_execution.make_false
102
+ @finished_run.make_false
88
103
 
89
104
  @thread = Thread.new do
90
105
  begin
91
106
  LogStash::Util.set_thread_name("pipeline.#{pipeline_id}")
92
107
  run
93
- @finished_execution.make_true
108
+ @finished_run.make_true
94
109
  rescue => e
95
110
  close
96
111
  logger.error("Pipeline aborted due to error", default_logging_keys(:exception => e, :backtrace => e.backtrace))
112
+ ensure
113
+ @finished_execution.make_true
97
114
  end
98
115
  end
99
116
 
@@ -108,15 +125,14 @@ module LogStash; class JavaPipeline < JavaBasePipeline
108
125
 
109
126
  def wait_until_started
110
127
  while true do
111
- # This should be changed with an appropriate FSM
112
- # It's an edge case, if we have a pipeline with
113
- # a generator { count => 1 } its possible that `Thread#alive?` doesn't return true
114
- # because the execution of the thread was successful and complete
115
- if @finished_execution.true?
128
+ if @finished_run.true?
129
+ # it completed run without exception
116
130
  return true
117
131
  elsif thread.nil? || !thread.alive?
132
+ # some exception occurred and the thread is dead
118
133
  return false
119
134
  elsif running?
135
+ # fully initialized and running
120
136
  return true
121
137
  else
122
138
  sleep 0.01
@@ -107,8 +107,23 @@ module LogStash; class Pipeline < BasePipeline
107
107
  @flushing = Concurrent::AtomicReference.new(false)
108
108
  @outputs_registered = Concurrent::AtomicBoolean.new(false)
109
109
  @worker_shutdown = java.util.concurrent.atomic.AtomicBoolean.new(false)
110
+
111
+ # @finished_execution signals that the pipeline thread has finished its execution
112
+ # regardless of any exceptions; it will always be true when the thread completes
113
+ @finished_execution = Concurrent::AtomicBoolean.new(false)
114
+
115
+ # @finished_run signals that the run methods called in the pipeline thread was completed
116
+ # without errors and it will NOT be set if the run method exits from an exception; this
117
+ # is by design and necessary for the wait_until_started semantic
118
+ @finished_run = Concurrent::AtomicBoolean.new(false)
119
+
120
+ @thread = nil
110
121
  end # def initialize
111
122
 
123
+ def finished_execution?
124
+ @finished_execution.true?
125
+ end
126
+
112
127
  def ready?
113
128
  @ready.value
114
129
  end
@@ -152,16 +167,19 @@ module LogStash; class Pipeline < BasePipeline
152
167
  "pipeline.batch.size" => settings.get("pipeline.batch.size"),
153
168
  "pipeline.batch.delay" => settings.get("pipeline.batch.delay")))
154
169
 
155
- @finished_execution = Concurrent::AtomicBoolean.new(false)
170
+ @finished_execution.make_false
171
+ @finished_run.make_false
156
172
 
157
173
  @thread = Thread.new do
158
174
  begin
159
175
  LogStash::Util.set_thread_name("pipeline.#{pipeline_id}")
160
176
  run
161
- @finished_execution.make_true
177
+ @finished_run.make_true
162
178
  rescue => e
163
179
  close
164
180
  @logger.error("Pipeline aborted due to error", default_logging_keys(:exception => e, :backtrace => e.backtrace))
181
+ ensure
182
+ @finished_execution.make_true
165
183
  end
166
184
  end
167
185
 
@@ -176,15 +194,14 @@ module LogStash; class Pipeline < BasePipeline
176
194
 
177
195
  def wait_until_started
178
196
  while true do
179
- # This should be changed with an appropriate FSM
180
- # It's an edge case, if we have a pipeline with
181
- # a generator { count => 1 } its possible that `Thread#alive?` doesn't return true
182
- # because the execution of the thread was successful and complete
183
- if @finished_execution.true?
197
+ if @finished_run.true?
198
+ # it completed run without exception
184
199
  return true
185
- elsif !thread.alive?
200
+ elsif thread.nil? || !thread.alive?
201
+ # some exception occured and the thread is dead
186
202
  return false
187
203
  elsif running?
204
+ # fully initialized and running
188
205
  return true
189
206
  else
190
207
  sleep 0.01
@@ -12,7 +12,7 @@ module LogStash module PipelineAction
12
12
  end
13
13
  alias_method :to_s, :inspect
14
14
 
15
- def execute(agent, pipelines)
15
+ def execute(agent, pipelines_registry)
16
16
  raise "`#execute` Not implemented!"
17
17
  end
18
18