logstash-core 5.0.2-java → 5.1.1.1-java

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (84) hide show
  1. checksums.yaml +4 -4
  2. data/gemspec_jars.rb +9 -0
  3. data/lib/logstash-core/logstash-core.jar +0 -0
  4. data/lib/logstash-core/logstash-core.rb +22 -0
  5. data/lib/logstash-core/version.rb +1 -1
  6. data/lib/logstash-core_jars.rb +20 -0
  7. data/lib/logstash/agent.rb +65 -14
  8. data/lib/logstash/api/commands/default_metadata.rb +2 -1
  9. data/lib/logstash/api/commands/stats.rb +3 -2
  10. data/lib/logstash/config/file.rb +0 -1
  11. data/lib/logstash/config/loader.rb +1 -0
  12. data/lib/logstash/config/mixin.rb +2 -6
  13. data/lib/logstash/environment.rb +25 -2
  14. data/lib/logstash/event_dispatcher.rb +40 -0
  15. data/lib/logstash/filter_delegator.rb +1 -1
  16. data/lib/logstash/filters/base.rb +10 -2
  17. data/lib/logstash/instrument/metric_store.rb +0 -1
  18. data/lib/logstash/instrument/metric_type/base.rb +0 -1
  19. data/lib/logstash/instrument/namespaced_null_metric.rb +54 -0
  20. data/lib/logstash/instrument/null_metric.rb +55 -46
  21. data/lib/logstash/instrument/periodic_poller/jvm.rb +26 -3
  22. data/lib/logstash/instrument/periodic_poller/load_average.rb +47 -0
  23. data/lib/logstash/instrument/snapshot.rb +0 -1
  24. data/lib/logstash/java_integration.rb +0 -1
  25. data/lib/logstash/logging/logger.rb +37 -4
  26. data/lib/logstash/outputs/base.rb +1 -1
  27. data/lib/logstash/patches.rb +1 -0
  28. data/lib/logstash/patches/exception_to_json.rb +5 -0
  29. data/lib/logstash/pipeline.rb +50 -17
  30. data/lib/logstash/plugin.rb +14 -48
  31. data/lib/logstash/plugins/hooks_registry.rb +57 -0
  32. data/lib/logstash/plugins/registry.rb +208 -45
  33. data/lib/logstash/runner.rb +10 -5
  34. data/lib/logstash/settings.rb +101 -9
  35. data/lib/logstash/universal_plugin.rb +13 -0
  36. data/lib/logstash/util/byte_value.rb +60 -0
  37. data/lib/logstash/util/loggable.rb +14 -2
  38. data/lib/logstash/util/safe_uri.rb +1 -0
  39. data/lib/logstash/util/time_value.rb +70 -0
  40. data/lib/logstash/util/wrapped_acked_queue.rb +347 -0
  41. data/lib/logstash/util/wrapped_synchronous_queue.rb +17 -33
  42. data/lib/logstash/version.rb +1 -1
  43. data/locales/en.yml +1 -1
  44. data/logstash-core.gemspec +13 -18
  45. data/spec/api/lib/api/node_stats_spec.rb +3 -1
  46. data/spec/api/lib/api/support/resource_dsl_methods.rb +14 -6
  47. data/spec/api/spec_helper.rb +1 -0
  48. data/spec/conditionals_spec.rb +3 -2
  49. data/spec/logstash/agent_spec.rb +142 -62
  50. data/spec/logstash/environment_spec.rb +38 -0
  51. data/spec/logstash/event_dispatcher_spec.rb +76 -0
  52. data/spec/logstash/filter_delegator_spec.rb +2 -1
  53. data/spec/logstash/instrument/namespaced_null_metric_spec.rb +33 -0
  54. data/spec/logstash/instrument/null_metric_spec.rb +9 -5
  55. data/spec/logstash/instrument/periodic_poller/jvm_spec.rb +40 -0
  56. data/spec/logstash/instrument/periodic_poller/load_average_spec.rb +91 -0
  57. data/spec/logstash/output_delegator_spec.rb +2 -1
  58. data/spec/logstash/patches_spec.rb +15 -4
  59. data/spec/logstash/pipeline_pq_file_spec.rb +131 -0
  60. data/spec/logstash/pipeline_spec.rb +21 -17
  61. data/spec/logstash/plugin_spec.rb +4 -16
  62. data/spec/logstash/plugins/hooks_registry_spec.rb +60 -0
  63. data/spec/logstash/plugins/registry_spec.rb +22 -14
  64. data/spec/logstash/settings/bytes_spec.rb +53 -0
  65. data/spec/logstash/settings/time_value_spec.rb +31 -0
  66. data/spec/logstash/settings/writable_directory_spec.rb +125 -0
  67. data/spec/logstash/settings_spec.rb +39 -0
  68. data/spec/logstash/util/byte_value_spec.rb +33 -0
  69. data/spec/logstash/util/time_value_spec.rb +59 -0
  70. data/spec/logstash/util/wrapped_synchronous_queue_spec.rb +2 -2
  71. data/spec/logstash/webserver_spec.rb +4 -7
  72. data/spec/support/helpers.rb +8 -0
  73. data/spec/support/mocks_classes.rb +61 -31
  74. metadata +73 -20
  75. data/lib/jars.rb +0 -7
  76. data/lib/logstash/config/registry.rb +0 -13
  77. data/lib/logstash/inputs/metrics.rb +0 -47
  78. data/spec/logstash/inputs/metrics_spec.rb +0 -51
  79. data/vendor/jars/com/fasterxml/jackson/core/jackson-core/2.7.4/jackson-core-2.7.4.jar +0 -0
  80. data/vendor/jars/com/fasterxml/jackson/core/jackson-databind/2.7.4/jackson-databind-2.7.4.jar +0 -0
  81. data/vendor/jars/org/apache/logging/log4j/log4j-1.2-api/2.6.2/log4j-1.2-api-2.6.2.jar +0 -0
  82. data/vendor/jars/org/apache/logging/log4j/log4j-api/2.6.2/log4j-api-2.6.2.jar +0 -0
  83. data/vendor/jars/org/apache/logging/log4j/log4j-core/2.6.2/log4j-core-2.6.2.jar +0 -0
  84. data/vendor/jars/org/logstash/logstash-core/5.0.2/logstash-core-5.0.2.jar +0 -0
@@ -5,8 +5,8 @@ module LogStash; module Util
5
5
  java_import java.util.concurrent.SynchronousQueue
6
6
  java_import java.util.concurrent.TimeUnit
7
7
 
8
- def initialize()
9
- @queue = java.util.concurrent.SynchronousQueue.new()
8
+ def initialize
9
+ @queue = java.util.concurrent.SynchronousQueue.new
10
10
  end
11
11
 
12
12
  # Push an object to the queue if the queue is full
@@ -30,7 +30,7 @@ module LogStash; module Util
30
30
 
31
31
  # Blocking
32
32
  def take
33
- @queue.take()
33
+ @queue.take
34
34
  end
35
35
 
36
36
  # Block for X millis
@@ -42,16 +42,20 @@ module LogStash; module Util
42
42
  WriteClient.new(self)
43
43
  end
44
44
 
45
- def read_client()
45
+ def read_client
46
46
  ReadClient.new(self)
47
47
  end
48
48
 
49
+ def close
50
+ # ignore
51
+ end
52
+
49
53
  class ReadClient
50
54
  # We generally only want one thread at a time able to access pop/take/poll operations
51
55
  # from this queue. We also depend on this to be able to block consumers while we snapshot
52
56
  # in-flight buffers
53
57
 
54
- def initialize(queue, batch_size = 125, wait_for = 5)
58
+ def initialize(queue, batch_size = 125, wait_for = 250)
55
59
  @queue = queue
56
60
  @mutex = Mutex.new
57
61
  # Note that @infilght_batches as a central mechanism for tracking inflight
@@ -64,6 +68,10 @@ module LogStash; module Util
64
68
  @wait_for = wait_for
65
69
  end
66
70
 
71
+ def close
72
+ # noop, compat with acked queue read client
73
+ end
74
+
67
75
  def set_batch_dimensions(batch_size, wait_for)
68
76
  @batch_size = batch_size
69
77
  @wait_for = wait_for
@@ -145,8 +153,6 @@ module LogStash; module Util
145
153
 
146
154
  class ReadBatch
147
155
  def initialize(queue, size, wait)
148
- @shutdown_signal_received = false
149
- @flush_signal_received = false
150
156
  @originals = Hash.new
151
157
 
152
158
  # TODO: disabled for https://github.com/elastic/logstash/issues/6055 - will have to properly refactor
@@ -210,14 +216,6 @@ module LogStash; module Util
210
216
  # @cancelled.size
211
217
  end
212
218
 
213
- def shutdown_signal_received?
214
- @shutdown_signal_received
215
- end
216
-
217
- def flush_signal_received?
218
- @flush_signal_received
219
- end
220
-
221
219
  private
222
220
 
223
221
  def iterating?
@@ -231,24 +229,10 @@ module LogStash; module Util
231
229
 
232
230
  def take_originals_from_queue(queue, size, wait)
233
231
  size.times do |t|
234
- event = (t == 0) ? queue.take : queue.poll(wait)
235
- if event.nil?
236
- # queue poll timed out
237
- next
238
- elsif event.is_a?(LogStash::SignalEvent)
239
- # We MUST break here. If a batch consumes two SHUTDOWN events
240
- # then another worker may have its SHUTDOWN 'stolen', thus blocking
241
- # the pipeline.
242
- @shutdown_signal_received = event.shutdown?
243
-
244
- # See comment above
245
- # We should stop doing work after flush as well.
246
- @flush_signal_received = event.flush?
247
-
248
- break
249
- else
250
- @originals[event] = true
251
- end
232
+ event = queue.poll(wait)
233
+ return if event.nil? # queue poll timed out
234
+
235
+ @originals[event] = true
252
236
  end
253
237
  end
254
238
  end
@@ -11,4 +11,4 @@
11
11
  # eventually this file should be in the root logstash lib fir and dependencies in logstash-core should be
12
12
  # fixed.
13
13
 
14
- LOGSTASH_VERSION = "5.0.2"
14
+ LOGSTASH_VERSION = "5.1.1.1"
data/locales/en.yml CHANGED
@@ -246,7 +246,7 @@ en:
246
246
  rubyshell: |+
247
247
  Drop to shell instead of running as normal.
248
248
  Valid shells are "irb" and "pry"
249
- node_name: |+
249
+ name: |+
250
250
  Specify the name of this logstash instance, if no value is given
251
251
  it will default to the current hostname.
252
252
  agent: |+
@@ -11,13 +11,16 @@ Gem::Specification.new do |gem|
11
11
  gem.homepage = "http://www.elastic.co/guide/en/logstash/current/index.html"
12
12
  gem.license = "Apache License (2.0)"
13
13
 
14
- gem.files = Dir.glob(["logstash-core.gemspec", "lib/**/*.rb", "spec/**/*.rb", "locales/*", "lib/logstash/api/init.ru", "vendor/jars/**/*.jar"])
14
+ gem.files = Dir.glob(["logstash-core.gemspec", "gemspec_jars.rb", "lib/**/*.rb", "spec/**/*.rb", "locales/*", "lib/logstash/api/init.ru", "lib/logstash-core/logstash-core.jar"])
15
15
  gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
16
16
  gem.name = "logstash-core"
17
- gem.require_paths = ["lib", "vendor/jars"]
17
+ gem.require_paths = ["lib"]
18
18
  gem.version = LOGSTASH_CORE_VERSION
19
19
 
20
- gem.add_runtime_dependency "logstash-core-event-java", "5.0.2"
20
+ gem.platform = "java"
21
+
22
+ gem.add_runtime_dependency "logstash-core-event-java", LOGSTASH_CORE_VERSION
23
+ gem.add_runtime_dependency "logstash-core-queue-jruby", LOGSTASH_CORE_VERSION
21
24
 
22
25
  gem.add_runtime_dependency "pry", "~> 0.10.1" #(Ruby license)
23
26
  gem.add_runtime_dependency "stud", "~> 0.0.19" #(Apache 2.0 license)
@@ -43,21 +46,13 @@ Gem::Specification.new do |gem|
43
46
  gem.add_runtime_dependency "rubyzip", "~> 1.1.7"
44
47
  gem.add_runtime_dependency "thread_safe", "~> 0.3.5" #(Apache 2.0 license)
45
48
 
46
- if RUBY_PLATFORM == 'java'
47
- gem.platform = RUBY_PLATFORM
48
- gem.add_runtime_dependency "jrjackson", "~> 0.4.0" #(Apache 2.0 license)
49
- else
50
- gem.add_runtime_dependency "oj" #(MIT-style license)
51
- end
52
-
53
- if RUBY_ENGINE == "rbx"
54
- # rubinius puts the ruby stdlib into gems.
55
- gem.add_runtime_dependency "rubysl"
49
+ gem.add_runtime_dependency "jrjackson", "~> 0.4.0" #(Apache 2.0 license)
56
50
 
57
- # Include racc to make the xml tests pass.
58
- # https://github.com/rubinius/rubinius/issues/2632#issuecomment-26954565
59
- gem.add_runtime_dependency "racc"
60
- end
51
+ gem.add_runtime_dependency "jar-dependencies"
52
+ # as of Feb 3rd 2016, the ruby-maven gem is resolved to version 3.3.3 and that version
53
+ # has an rdoc problem that causes a bundler exception. 3.3.9 is the current latest version
54
+ # which does not have this problem.
55
+ gem.add_runtime_dependency "ruby-maven", "~> 3.3.9"
61
56
 
62
- gem.add_runtime_dependency 'jar-dependencies', '~> 0.3.4'
57
+ eval(File.read(File.expand_path("../gemspec_jars.rb", __FILE__)))
63
58
  end
@@ -14,6 +14,7 @@ describe LogStash::Api::Modules::NodeStats do
14
14
  # DSL describing response structure
15
15
  root_structure = {
16
16
  "jvm"=>{
17
+ "uptime_in_millis" => Numeric,
17
18
  "threads"=>{
18
19
  "count"=>Numeric,
19
20
  "peak_count"=>Numeric
@@ -68,7 +69,8 @@ describe LogStash::Api::Modules::NodeStats do
68
69
  },
69
70
  "cpu"=>{
70
71
  "total_in_millis"=>Numeric,
71
- "percent"=>Numeric
72
+ "percent"=>Numeric,
73
+ "load_average" => { "1m" => Numeric }
72
74
  }
73
75
  },
74
76
  "pipeline" => {
@@ -25,16 +25,16 @@ module ResourceDSLMethods
25
25
  def test_api(expected, path)
26
26
  context "GET #{path}" do
27
27
  let(:payload) { LogStash::Json.load(last_response.body) }
28
-
28
+
29
29
  before(:all) do
30
30
  do_request { get path }
31
- end
32
-
31
+ end
32
+
33
33
  it "should respond OK" do
34
34
  expect(last_response).to be_ok
35
35
  end
36
36
 
37
-
37
+
38
38
  describe "the default metadata" do
39
39
  it "should include the host" do
40
40
  expect(payload["host"]).to eql(Socket.gethostname)
@@ -47,11 +47,19 @@ module ResourceDSLMethods
47
47
  it "should include the http address" do
48
48
  expect(payload["http_address"]).to eql("#{Socket.gethostname}:#{::LogStash::WebServer::DEFAULT_PORTS.first}")
49
49
  end
50
+
51
+ it "should include the node name" do
52
+ expect(payload["name"]).to eql(@runner.agent.name)
53
+ end
54
+
55
+ it "should include the node id" do
56
+ expect(payload["id"]).to eql(@runner.agent.id)
57
+ end
50
58
  end
51
-
59
+
52
60
  hash_to_mapping(expected).each do |resource_path,klass|
53
61
  dotted = resource_path.join(".")
54
-
62
+
55
63
  it "should set '#{dotted}' at '#{path}' to be a '#{klass}'" do
56
64
  expect(last_response).to be_ok # fail early if need be
57
65
  resource_path_value = resource_path.reduce(payload) do |acc,v|
@@ -5,6 +5,7 @@ require "stud/task"
5
5
  require "logstash/devutils/rspec/spec_helper"
6
6
  $LOAD_PATH.unshift(File.expand_path(File.dirname(__FILE__)))
7
7
  require "lib/api/support/resource_dsl_methods"
8
+ require_relative "../support/mocks_classes"
8
9
  require 'rspec/expectations'
9
10
  require "logstash/settings"
10
11
  require 'rack/test'
@@ -26,6 +26,8 @@ describe "conditionals in output" do
26
26
  extend ConditionalFanciness
27
27
 
28
28
  class DummyNullOutput < LogStash::Outputs::Base
29
+ config_name "dummynull"
30
+
29
31
  def register
30
32
  end
31
33
  def multi_receive(events)
@@ -33,7 +35,7 @@ describe "conditionals in output" do
33
35
  end
34
36
 
35
37
  before do
36
- LogStash::Registry.instance.register("logstash/outputs/dummynull", DummyNullOutput)
38
+ LogStash::PLUGIN_REGISTRY.add(:output, "dummynull", DummyNullOutput)
37
39
  end
38
40
 
39
41
  describe "simple" do
@@ -436,5 +438,4 @@ describe "conditionals in filter" do
436
438
  expect(subject[2].get("cond2")).to eq("true")
437
439
  end
438
440
  end
439
-
440
441
  end
@@ -3,10 +3,11 @@ require "spec_helper"
3
3
  require "stud/temporary"
4
4
  require "logstash/inputs/generator"
5
5
  require_relative "../support/mocks_classes"
6
+ require "fileutils"
7
+ require_relative "../support/helpers"
6
8
 
7
9
  describe LogStash::Agent do
8
10
 
9
- let(:logger) { double("logger") }
10
11
  let(:agent_settings) { LogStash::SETTINGS }
11
12
  let(:agent_args) { {} }
12
13
  let(:pipeline_settings) { agent_settings.clone }
@@ -14,15 +15,12 @@ describe LogStash::Agent do
14
15
  let(:config_file) { Stud::Temporary.pathname }
15
16
  let(:config_file_txt) { "input { generator { count => 100000 } } output { }" }
16
17
 
17
- subject { LogStash::Agent.new(agent_settings) }
18
+ subject { LogStash::Agent.new(agent_settings) }
18
19
 
19
20
  before :each do
20
- [:info, :warn, :error, :fatal, :debug].each do |level|
21
- allow(logger).to receive(level)
22
- end
23
- [:info?, :warn?, :error?, :fatal?, :debug?].each do |level|
24
- allow(logger).to receive(level)
25
- end
21
+ # This MUST run first, before `subject` is invoked to ensure clean state
22
+ clear_data_dir
23
+
26
24
  File.open(config_file, "w") { |f| f.puts config_file_txt }
27
25
  agent_args.each do |key, value|
28
26
  agent_settings.set(key, value)
@@ -31,7 +29,6 @@ describe LogStash::Agent do
31
29
  pipeline_args.each do |key, value|
32
30
  pipeline_settings.set(key, value)
33
31
  end
34
- #subject.logger = logger
35
32
  end
36
33
 
37
34
  after :each do
@@ -40,7 +37,7 @@ describe LogStash::Agent do
40
37
  end
41
38
 
42
39
  it "fallback to hostname when no name is provided" do
43
- expect(LogStash::Agent.new.node_name).to eq(Socket.gethostname)
40
+ expect(LogStash::Agent.new.name).to eq(Socket.gethostname)
44
41
  end
45
42
 
46
43
  describe "register_pipeline" do
@@ -58,13 +55,27 @@ describe LogStash::Agent do
58
55
  it "should delegate settings to new pipeline" do
59
56
  expect(LogStash::Pipeline).to receive(:new) do |arg1, arg2|
60
57
  expect(arg1).to eq(config_string)
61
- expect(arg2.to_hash).to include(agent_args)
58
+ expect(arg2.to_hash).to include(agent_args)
62
59
  end
63
60
  subject.register_pipeline(pipeline_id, agent_settings)
64
61
  end
65
62
  end
66
63
 
67
- describe "#execute" do
64
+ describe "#id" do
65
+ let(:config_file_txt) { "" }
66
+ let(:id_file_data) { File.open(subject.id_path) {|f| f.read } }
67
+
68
+ it "should return a UUID" do
69
+ expect(subject.id).to be_a(String)
70
+ expect(subject.id.size).to be > 0
71
+ end
72
+
73
+ it "should write out the persistent UUID" do
74
+ expect(id_file_data).to eql(subject.id)
75
+ end
76
+ end
77
+
78
+ describe "#execute" do
68
79
  let(:config_file_txt) { "input { generator { count => 100000 } } output { }" }
69
80
 
70
81
  before :each do
@@ -102,7 +113,7 @@ describe LogStash::Agent do
102
113
  end
103
114
  end
104
115
 
105
- context "when calling reload_state!" do
116
+ context "when calling reload_pipeline!" do
106
117
  context "with a config that contains reload incompatible plugins" do
107
118
  let(:second_pipeline_config) { "input { stdin {} } filter { } output { }" }
108
119
 
@@ -111,7 +122,7 @@ describe LogStash::Agent do
111
122
  sleep 0.01 until subject.running_pipelines? && subject.pipelines.values.first.ready?
112
123
  expect(subject).to_not receive(:upgrade_pipeline)
113
124
  File.open(config_file, "w") { |f| f.puts second_pipeline_config }
114
- subject.send(:reload_state!)
125
+ subject.send(:"reload_pipeline!", "main")
115
126
  sleep 0.1
116
127
  Stud.stop!(t)
117
128
  t.join
@@ -127,7 +138,45 @@ describe LogStash::Agent do
127
138
  sleep 0.01 until subject.running_pipelines? && subject.pipelines.values.first.ready?
128
139
  expect(subject).to receive(:upgrade_pipeline).once.and_call_original
129
140
  File.open(config_file, "w") { |f| f.puts second_pipeline_config }
130
- subject.send(:reload_state!)
141
+ subject.send(:"reload_pipeline!", "main")
142
+ sleep 0.1
143
+ Stud.stop!(t)
144
+ t.join
145
+
146
+ subject.shutdown
147
+ end
148
+ end
149
+
150
+ end
151
+ context "when calling reload_state!" do
152
+ context "with a pipeline with auto reloading turned off" do
153
+ let(:second_pipeline_config) { "input { generator { } } filter { } output { }" }
154
+ let(:pipeline_args) { { "config.reload.automatic" => false } }
155
+
156
+ it "does not try to reload the pipeline" do
157
+ t = Thread.new { subject.execute }
158
+ sleep 0.01 until subject.running_pipelines? && subject.pipelines.values.first.ready?
159
+ expect(subject).to_not receive(:reload_pipeline!)
160
+ File.open(config_file, "w") { |f| f.puts second_pipeline_config }
161
+ subject.reload_state!
162
+ sleep 0.1
163
+ Stud.stop!(t)
164
+ t.join
165
+
166
+ subject.shutdown
167
+ end
168
+ end
169
+
170
+ context "with a pipeline with auto reloading turned on" do
171
+ let(:second_pipeline_config) { "input { generator { } } filter { } output { }" }
172
+ let(:pipeline_args) { { "config.reload.automatic" => true } }
173
+
174
+ it "tries to reload the pipeline" do
175
+ t = Thread.new { subject.execute }
176
+ sleep 0.01 until subject.running_pipelines? && subject.pipelines.values.first.ready?
177
+ expect(subject).to receive(:reload_pipeline!).once.and_call_original
178
+ File.open(config_file, "w") { |f| f.puts second_pipeline_config }
179
+ subject.reload_state!
131
180
  sleep 0.1
132
181
  Stud.stop!(t)
133
182
  t.join
@@ -155,9 +204,9 @@ describe LogStash::Agent do
155
204
  context "if state is clean" do
156
205
  it "should periodically reload_state" do
157
206
  allow(subject).to receive(:clean_state?).and_return(false)
158
- expect(subject).to receive(:reload_state!).at_least(3).times
159
207
  t = Thread.new { subject.execute }
160
208
  sleep 0.01 until subject.running_pipelines? && subject.pipelines.values.first.ready?
209
+ expect(subject).to receive(:reload_state!).at_least(2).times
161
210
  sleep 0.1
162
211
  Stud.stop!(t)
163
212
  t.join
@@ -205,7 +254,8 @@ describe LogStash::Agent do
205
254
  let(:second_pipeline_config) { "input { generator {} } filter { } output { }" }
206
255
  let(:pipeline_args) { {
207
256
  "config.string" => first_pipeline_config,
208
- "pipeline.workers" => 4
257
+ "pipeline.workers" => 4,
258
+ "config.reload.automatic" => true
209
259
  } }
210
260
 
211
261
  before(:each) do
@@ -216,14 +266,14 @@ describe LogStash::Agent do
216
266
  it "upgrades the state" do
217
267
  expect(subject).to receive(:fetch_config).and_return(second_pipeline_config)
218
268
  expect(subject).to receive(:upgrade_pipeline).with(pipeline_id, kind_of(LogStash::Pipeline))
219
- subject.send(:reload_state!)
269
+ subject.reload_state!
220
270
  end
221
271
  end
222
272
  context "when fetching the same state" do
223
273
  it "doesn't upgrade the state" do
224
274
  expect(subject).to receive(:fetch_config).and_return(first_pipeline_config)
225
275
  expect(subject).to_not receive(:upgrade_pipeline)
226
- subject.send(:reload_state!)
276
+ subject.reload_state!
227
277
  end
228
278
  end
229
279
  end
@@ -268,6 +318,10 @@ describe LogStash::Agent do
268
318
  subject.register_pipeline(pipeline_id, pipeline_settings)
269
319
  end
270
320
 
321
+ after(:each) do
322
+ subject.shutdown
323
+ end
324
+
271
325
  context "when the upgrade fails" do
272
326
  before :each do
273
327
  allow(subject).to receive(:fetch_config).and_return(new_pipeline_config)
@@ -276,14 +330,14 @@ describe LogStash::Agent do
276
330
  end
277
331
 
278
332
  it "leaves the state untouched" do
279
- subject.send(:reload_state!)
333
+ subject.send(:"reload_pipeline!", pipeline_id)
280
334
  expect(subject.pipelines[pipeline_id].config_str).to eq(pipeline_config)
281
335
  end
282
336
 
283
337
  context "and current state is empty" do
284
338
  it "should not start a pipeline" do
285
339
  expect(subject).to_not receive(:start_pipeline)
286
- subject.send(:reload_state!)
340
+ subject.send(:"reload_pipeline!", pipeline_id)
287
341
  end
288
342
  end
289
343
  end
@@ -293,15 +347,16 @@ describe LogStash::Agent do
293
347
  before :each do
294
348
  allow(subject).to receive(:fetch_config).and_return(new_config)
295
349
  allow(subject).to receive(:stop_pipeline)
350
+ allow(subject).to receive(:start_pipeline)
296
351
  end
297
352
  it "updates the state" do
298
- subject.send(:reload_state!)
353
+ subject.send(:"reload_pipeline!", pipeline_id)
299
354
  expect(subject.pipelines[pipeline_id].config_str).to eq(new_config)
300
355
  end
301
356
  it "starts the pipeline" do
302
357
  expect(subject).to receive(:stop_pipeline)
303
358
  expect(subject).to receive(:start_pipeline)
304
- subject.send(:reload_state!)
359
+ subject.send(:"reload_pipeline!", pipeline_id)
305
360
  end
306
361
  end
307
362
  end
@@ -330,41 +385,45 @@ describe LogStash::Agent do
330
385
 
331
386
 
332
387
  context "metrics after config reloading" do
333
- let(:config) { "input { generator { } } output { dummyoutput { } }" }
334
- let(:config_path) do
388
+ let!(:config) { "input { generator { } } output { dummyoutput { } }" }
389
+ let!(:config_path) do
335
390
  f = Stud::Temporary.file
336
391
  f.write(config)
392
+ f.fsync
337
393
  f.close
338
394
  f.path
339
395
  end
340
- let(:interval) { 0.2 }
341
396
  let(:pipeline_args) do
342
397
  {
343
- "pipeline.workers" => 4,
398
+ "pipeline.workers" => 2,
344
399
  "path.config" => config_path
345
400
  }
346
401
  end
347
402
 
348
403
  let(:agent_args) do
349
- super.merge({ "config.reload.automatic" => true,
350
- "config.reload.interval" => interval,
351
- "metric.collect" => true })
404
+ {
405
+ "config.reload.automatic" => false,
406
+ "pipeline.batch.size" => 1,
407
+ "metric.collect" => true
408
+ }
352
409
  end
353
410
 
354
411
  # We need to create theses dummy classes to know how many
355
412
  # events where actually generated by the pipeline and successfully send to the output.
356
413
  # Theses values are compared with what we store in the metric store.
357
- let!(:dummy_output) { DummyOutput.new }
358
- let!(:dummy_output2) { DummyOutput.new }
359
- class DummyOutput2 < LogStash::Outputs::Base; end
414
+ class DummyOutput2 < LogStash::Outputs::DroppingDummyOutput; end
415
+
416
+ let!(:dummy_output) { LogStash::Outputs::DroppingDummyOutput.new }
417
+ let!(:dummy_output2) { DummyOutput2.new }
418
+ let(:initial_generator_threshold) { 1000 }
360
419
 
361
420
  before :each do
362
- allow(DummyOutput).to receive(:new).at_least(:once).with(anything).and_return(dummy_output)
421
+ allow(LogStash::Outputs::DroppingDummyOutput).to receive(:new).at_least(:once).with(anything).and_return(dummy_output)
363
422
  allow(DummyOutput2).to receive(:new).at_least(:once).with(anything).and_return(dummy_output2)
364
423
 
365
424
  allow(LogStash::Plugin).to receive(:lookup).with("input", "generator").and_return(LogStash::Inputs::Generator)
366
425
  allow(LogStash::Plugin).to receive(:lookup).with("codec", "plain").and_return(LogStash::Codecs::Plain)
367
- allow(LogStash::Plugin).to receive(:lookup).with("output", "dummyoutput").and_return(DummyOutput)
426
+ allow(LogStash::Plugin).to receive(:lookup).with("output", "dummyoutput").and_return(LogStash::Outputs::DroppingDummyOutput)
368
427
  allow(LogStash::Plugin).to receive(:lookup).with("output", "dummyoutput2").and_return(DummyOutput2)
369
428
 
370
429
  @abort_on_exception = Thread.abort_on_exception
@@ -375,7 +434,8 @@ describe LogStash::Agent do
375
434
  subject.execute
376
435
  end
377
436
 
378
- sleep(2)
437
+ # wait for some events to reach the dummy_output
438
+ sleep(0.01) until dummy_output.events_received > initial_generator_threshold
379
439
  end
380
440
 
381
441
  after :each do
@@ -392,43 +452,34 @@ describe LogStash::Agent do
392
452
  let(:new_config_generator_counter) { 500 }
393
453
  let(:new_config) { "input { generator { count => #{new_config_generator_counter} } } output { dummyoutput2 {} }" }
394
454
  before :each do
395
- # We know that the store has more events coming in.
396
- i = 0
397
- while dummy_output.events.size <= new_config_generator_counter
398
- i += 1
399
- raise "Waiting too long!" if i > 20
400
- sleep(0.1)
401
- end
402
-
403
455
 
404
- # Also force a flush to disk to make sure ruby reload it.
405
456
  File.open(config_path, "w") do |f|
406
457
  f.write(new_config)
407
458
  f.fsync
408
459
  end
409
460
 
410
- sleep(interval * 3) # Give time to reload the config
461
+ subject.send(:"reload_pipeline!", "main")
411
462
 
412
- # be eventually consistent.
413
- sleep(0.01) while dummy_output2.events.size < new_config_generator_counter
463
+ # wait until pipeline restarts
464
+ sleep(0.01) until dummy_output2.events_received > 0
414
465
  end
415
466
 
416
467
  it "resets the pipeline metric collector" do
417
468
  snapshot = subject.metric.collector.snapshot_metric
418
469
  value = snapshot.metric_store.get_with_path("/stats/pipelines")[:stats][:pipelines][:main][:events][:in].value
419
- expect(value).to eq(new_config_generator_counter)
470
+ expect(value).to be <= new_config_generator_counter
420
471
  end
421
472
 
422
473
  it "does not reset the global event count" do
423
474
  snapshot = subject.metric.collector.snapshot_metric
424
475
  value = snapshot.metric_store.get_with_path("/stats/events")[:stats][:events][:in].value
425
- expect(value).to be > new_config_generator_counter
476
+ expect(value).to be > initial_generator_threshold
426
477
  end
427
478
 
428
479
  it "increases the successful reload count" do
429
480
  snapshot = subject.metric.collector.snapshot_metric
430
481
  value = snapshot.metric_store.get_with_path("/stats/pipelines")[:stats][:pipelines][:main][:reloads][:successes].value
431
- expect(value).to be(1)
482
+ expect(value).to eq(1)
432
483
  end
433
484
 
434
485
  it "does not set the failure reload timestamp" do
@@ -455,28 +506,19 @@ describe LogStash::Agent do
455
506
  let(:new_config) { "input { generator { count => " }
456
507
  let(:new_config_generator_counter) { 500 }
457
508
  before :each do
458
- # We know that the store has more events coming in.
459
- i = 0
460
- while dummy_output.events.size <= new_config_generator_counter
461
- i += 1
462
- raise "Waiting too long!" if i > 20
463
- sleep(0.1)
464
- end
465
509
 
466
-
467
- # Also force a flush to disk to make sure ruby reload it.
468
510
  File.open(config_path, "w") do |f|
469
511
  f.write(new_config)
470
512
  f.fsync
471
513
  end
472
514
 
473
- sleep(interval * 3) # Give time to reload the config
515
+ subject.send(:"reload_pipeline!", "main")
474
516
  end
475
517
 
476
518
  it "does not increase the successful reload count" do
477
519
  snapshot = subject.metric.collector.snapshot_metric
478
520
  value = snapshot.metric_store.get_with_path("/stats/pipelines")[:stats][:pipelines][:main][:reloads][:successes].value
479
- expect(value).to be(0)
521
+ expect(value).to eq(0)
480
522
  end
481
523
 
482
524
  it "does not set the successful reload timestamp" do
@@ -504,5 +546,43 @@ describe LogStash::Agent do
504
546
  expect(value).to be > 0
505
547
  end
506
548
  end
549
+
550
+ context "when reloading a config that raises exception on pipeline.run" do
551
+ let(:new_config) { "input { generator { count => 10000 } }" }
552
+ let(:new_config_generator_counter) { 500 }
553
+
554
+ class BrokenGenerator < LogStash::Inputs::Generator
555
+ def register
556
+ raise ArgumentError
557
+ end
558
+ end
559
+
560
+ before :each do
561
+
562
+ allow(LogStash::Plugin).to receive(:lookup).with("input", "generator").and_return(BrokenGenerator)
563
+
564
+ File.open(config_path, "w") do |f|
565
+ f.write(new_config)
566
+ f.fsync
567
+ end
568
+
569
+ end
570
+
571
+ it "does not increase the successful reload count" do
572
+ expect { subject.send(:"reload_pipeline!", "main") }.to_not change {
573
+ snapshot = subject.metric.collector.snapshot_metric
574
+ reload_metrics = snapshot.metric_store.get_with_path("/stats/pipelines")[:stats][:pipelines][:main][:reloads]
575
+ reload_metrics[:successes].value
576
+ }
577
+ end
578
+
579
+ it "increases the failured reload count" do
580
+ expect { subject.send(:"reload_pipeline!", "main") }.to change {
581
+ snapshot = subject.metric.collector.snapshot_metric
582
+ reload_metrics = snapshot.metric_store.get_with_path("/stats/pipelines")[:stats][:pipelines][:main][:reloads]
583
+ reload_metrics[:failures].value
584
+ }.by(1)
585
+ end
586
+ end
507
587
  end
508
588
  end