logstash-core 5.0.2-java → 5.1.1.1-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/gemspec_jars.rb +9 -0
- data/lib/logstash-core/logstash-core.jar +0 -0
- data/lib/logstash-core/logstash-core.rb +22 -0
- data/lib/logstash-core/version.rb +1 -1
- data/lib/logstash-core_jars.rb +20 -0
- data/lib/logstash/agent.rb +65 -14
- data/lib/logstash/api/commands/default_metadata.rb +2 -1
- data/lib/logstash/api/commands/stats.rb +3 -2
- data/lib/logstash/config/file.rb +0 -1
- data/lib/logstash/config/loader.rb +1 -0
- data/lib/logstash/config/mixin.rb +2 -6
- data/lib/logstash/environment.rb +25 -2
- data/lib/logstash/event_dispatcher.rb +40 -0
- data/lib/logstash/filter_delegator.rb +1 -1
- data/lib/logstash/filters/base.rb +10 -2
- data/lib/logstash/instrument/metric_store.rb +0 -1
- data/lib/logstash/instrument/metric_type/base.rb +0 -1
- data/lib/logstash/instrument/namespaced_null_metric.rb +54 -0
- data/lib/logstash/instrument/null_metric.rb +55 -46
- data/lib/logstash/instrument/periodic_poller/jvm.rb +26 -3
- data/lib/logstash/instrument/periodic_poller/load_average.rb +47 -0
- data/lib/logstash/instrument/snapshot.rb +0 -1
- data/lib/logstash/java_integration.rb +0 -1
- data/lib/logstash/logging/logger.rb +37 -4
- data/lib/logstash/outputs/base.rb +1 -1
- data/lib/logstash/patches.rb +1 -0
- data/lib/logstash/patches/exception_to_json.rb +5 -0
- data/lib/logstash/pipeline.rb +50 -17
- data/lib/logstash/plugin.rb +14 -48
- data/lib/logstash/plugins/hooks_registry.rb +57 -0
- data/lib/logstash/plugins/registry.rb +208 -45
- data/lib/logstash/runner.rb +10 -5
- data/lib/logstash/settings.rb +101 -9
- data/lib/logstash/universal_plugin.rb +13 -0
- data/lib/logstash/util/byte_value.rb +60 -0
- data/lib/logstash/util/loggable.rb +14 -2
- data/lib/logstash/util/safe_uri.rb +1 -0
- data/lib/logstash/util/time_value.rb +70 -0
- data/lib/logstash/util/wrapped_acked_queue.rb +347 -0
- data/lib/logstash/util/wrapped_synchronous_queue.rb +17 -33
- data/lib/logstash/version.rb +1 -1
- data/locales/en.yml +1 -1
- data/logstash-core.gemspec +13 -18
- data/spec/api/lib/api/node_stats_spec.rb +3 -1
- data/spec/api/lib/api/support/resource_dsl_methods.rb +14 -6
- data/spec/api/spec_helper.rb +1 -0
- data/spec/conditionals_spec.rb +3 -2
- data/spec/logstash/agent_spec.rb +142 -62
- data/spec/logstash/environment_spec.rb +38 -0
- data/spec/logstash/event_dispatcher_spec.rb +76 -0
- data/spec/logstash/filter_delegator_spec.rb +2 -1
- data/spec/logstash/instrument/namespaced_null_metric_spec.rb +33 -0
- data/spec/logstash/instrument/null_metric_spec.rb +9 -5
- data/spec/logstash/instrument/periodic_poller/jvm_spec.rb +40 -0
- data/spec/logstash/instrument/periodic_poller/load_average_spec.rb +91 -0
- data/spec/logstash/output_delegator_spec.rb +2 -1
- data/spec/logstash/patches_spec.rb +15 -4
- data/spec/logstash/pipeline_pq_file_spec.rb +131 -0
- data/spec/logstash/pipeline_spec.rb +21 -17
- data/spec/logstash/plugin_spec.rb +4 -16
- data/spec/logstash/plugins/hooks_registry_spec.rb +60 -0
- data/spec/logstash/plugins/registry_spec.rb +22 -14
- data/spec/logstash/settings/bytes_spec.rb +53 -0
- data/spec/logstash/settings/time_value_spec.rb +31 -0
- data/spec/logstash/settings/writable_directory_spec.rb +125 -0
- data/spec/logstash/settings_spec.rb +39 -0
- data/spec/logstash/util/byte_value_spec.rb +33 -0
- data/spec/logstash/util/time_value_spec.rb +59 -0
- data/spec/logstash/util/wrapped_synchronous_queue_spec.rb +2 -2
- data/spec/logstash/webserver_spec.rb +4 -7
- data/spec/support/helpers.rb +8 -0
- data/spec/support/mocks_classes.rb +61 -31
- metadata +73 -20
- data/lib/jars.rb +0 -7
- data/lib/logstash/config/registry.rb +0 -13
- data/lib/logstash/inputs/metrics.rb +0 -47
- data/spec/logstash/inputs/metrics_spec.rb +0 -51
- data/vendor/jars/com/fasterxml/jackson/core/jackson-core/2.7.4/jackson-core-2.7.4.jar +0 -0
- data/vendor/jars/com/fasterxml/jackson/core/jackson-databind/2.7.4/jackson-databind-2.7.4.jar +0 -0
- data/vendor/jars/org/apache/logging/log4j/log4j-1.2-api/2.6.2/log4j-1.2-api-2.6.2.jar +0 -0
- data/vendor/jars/org/apache/logging/log4j/log4j-api/2.6.2/log4j-api-2.6.2.jar +0 -0
- data/vendor/jars/org/apache/logging/log4j/log4j-core/2.6.2/log4j-core-2.6.2.jar +0 -0
- data/vendor/jars/org/logstash/logstash-core/5.0.2/logstash-core-5.0.2.jar +0 -0
|
@@ -5,8 +5,8 @@ module LogStash; module Util
|
|
|
5
5
|
java_import java.util.concurrent.SynchronousQueue
|
|
6
6
|
java_import java.util.concurrent.TimeUnit
|
|
7
7
|
|
|
8
|
-
def initialize
|
|
9
|
-
@queue = java.util.concurrent.SynchronousQueue.new
|
|
8
|
+
def initialize
|
|
9
|
+
@queue = java.util.concurrent.SynchronousQueue.new
|
|
10
10
|
end
|
|
11
11
|
|
|
12
12
|
# Push an object to the queue if the queue is full
|
|
@@ -30,7 +30,7 @@ module LogStash; module Util
|
|
|
30
30
|
|
|
31
31
|
# Blocking
|
|
32
32
|
def take
|
|
33
|
-
@queue.take
|
|
33
|
+
@queue.take
|
|
34
34
|
end
|
|
35
35
|
|
|
36
36
|
# Block for X millis
|
|
@@ -42,16 +42,20 @@ module LogStash; module Util
|
|
|
42
42
|
WriteClient.new(self)
|
|
43
43
|
end
|
|
44
44
|
|
|
45
|
-
def read_client
|
|
45
|
+
def read_client
|
|
46
46
|
ReadClient.new(self)
|
|
47
47
|
end
|
|
48
48
|
|
|
49
|
+
def close
|
|
50
|
+
# ignore
|
|
51
|
+
end
|
|
52
|
+
|
|
49
53
|
class ReadClient
|
|
50
54
|
# We generally only want one thread at a time able to access pop/take/poll operations
|
|
51
55
|
# from this queue. We also depend on this to be able to block consumers while we snapshot
|
|
52
56
|
# in-flight buffers
|
|
53
57
|
|
|
54
|
-
def initialize(queue, batch_size = 125, wait_for =
|
|
58
|
+
def initialize(queue, batch_size = 125, wait_for = 250)
|
|
55
59
|
@queue = queue
|
|
56
60
|
@mutex = Mutex.new
|
|
57
61
|
# Note that @infilght_batches as a central mechanism for tracking inflight
|
|
@@ -64,6 +68,10 @@ module LogStash; module Util
|
|
|
64
68
|
@wait_for = wait_for
|
|
65
69
|
end
|
|
66
70
|
|
|
71
|
+
def close
|
|
72
|
+
# noop, compat with acked queue read client
|
|
73
|
+
end
|
|
74
|
+
|
|
67
75
|
def set_batch_dimensions(batch_size, wait_for)
|
|
68
76
|
@batch_size = batch_size
|
|
69
77
|
@wait_for = wait_for
|
|
@@ -145,8 +153,6 @@ module LogStash; module Util
|
|
|
145
153
|
|
|
146
154
|
class ReadBatch
|
|
147
155
|
def initialize(queue, size, wait)
|
|
148
|
-
@shutdown_signal_received = false
|
|
149
|
-
@flush_signal_received = false
|
|
150
156
|
@originals = Hash.new
|
|
151
157
|
|
|
152
158
|
# TODO: disabled for https://github.com/elastic/logstash/issues/6055 - will have to properly refactor
|
|
@@ -210,14 +216,6 @@ module LogStash; module Util
|
|
|
210
216
|
# @cancelled.size
|
|
211
217
|
end
|
|
212
218
|
|
|
213
|
-
def shutdown_signal_received?
|
|
214
|
-
@shutdown_signal_received
|
|
215
|
-
end
|
|
216
|
-
|
|
217
|
-
def flush_signal_received?
|
|
218
|
-
@flush_signal_received
|
|
219
|
-
end
|
|
220
|
-
|
|
221
219
|
private
|
|
222
220
|
|
|
223
221
|
def iterating?
|
|
@@ -231,24 +229,10 @@ module LogStash; module Util
|
|
|
231
229
|
|
|
232
230
|
def take_originals_from_queue(queue, size, wait)
|
|
233
231
|
size.times do |t|
|
|
234
|
-
event =
|
|
235
|
-
if event.nil?
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
elsif event.is_a?(LogStash::SignalEvent)
|
|
239
|
-
# We MUST break here. If a batch consumes two SHUTDOWN events
|
|
240
|
-
# then another worker may have its SHUTDOWN 'stolen', thus blocking
|
|
241
|
-
# the pipeline.
|
|
242
|
-
@shutdown_signal_received = event.shutdown?
|
|
243
|
-
|
|
244
|
-
# See comment above
|
|
245
|
-
# We should stop doing work after flush as well.
|
|
246
|
-
@flush_signal_received = event.flush?
|
|
247
|
-
|
|
248
|
-
break
|
|
249
|
-
else
|
|
250
|
-
@originals[event] = true
|
|
251
|
-
end
|
|
232
|
+
event = queue.poll(wait)
|
|
233
|
+
return if event.nil? # queue poll timed out
|
|
234
|
+
|
|
235
|
+
@originals[event] = true
|
|
252
236
|
end
|
|
253
237
|
end
|
|
254
238
|
end
|
data/lib/logstash/version.rb
CHANGED
data/locales/en.yml
CHANGED
|
@@ -246,7 +246,7 @@ en:
|
|
|
246
246
|
rubyshell: |+
|
|
247
247
|
Drop to shell instead of running as normal.
|
|
248
248
|
Valid shells are "irb" and "pry"
|
|
249
|
-
|
|
249
|
+
name: |+
|
|
250
250
|
Specify the name of this logstash instance, if no value is given
|
|
251
251
|
it will default to the current hostname.
|
|
252
252
|
agent: |+
|
data/logstash-core.gemspec
CHANGED
|
@@ -11,13 +11,16 @@ Gem::Specification.new do |gem|
|
|
|
11
11
|
gem.homepage = "http://www.elastic.co/guide/en/logstash/current/index.html"
|
|
12
12
|
gem.license = "Apache License (2.0)"
|
|
13
13
|
|
|
14
|
-
gem.files = Dir.glob(["logstash-core.gemspec", "lib/**/*.rb", "spec/**/*.rb", "locales/*", "lib/logstash/api/init.ru", "
|
|
14
|
+
gem.files = Dir.glob(["logstash-core.gemspec", "gemspec_jars.rb", "lib/**/*.rb", "spec/**/*.rb", "locales/*", "lib/logstash/api/init.ru", "lib/logstash-core/logstash-core.jar"])
|
|
15
15
|
gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
|
|
16
16
|
gem.name = "logstash-core"
|
|
17
|
-
gem.require_paths = ["lib"
|
|
17
|
+
gem.require_paths = ["lib"]
|
|
18
18
|
gem.version = LOGSTASH_CORE_VERSION
|
|
19
19
|
|
|
20
|
-
gem.
|
|
20
|
+
gem.platform = "java"
|
|
21
|
+
|
|
22
|
+
gem.add_runtime_dependency "logstash-core-event-java", LOGSTASH_CORE_VERSION
|
|
23
|
+
gem.add_runtime_dependency "logstash-core-queue-jruby", LOGSTASH_CORE_VERSION
|
|
21
24
|
|
|
22
25
|
gem.add_runtime_dependency "pry", "~> 0.10.1" #(Ruby license)
|
|
23
26
|
gem.add_runtime_dependency "stud", "~> 0.0.19" #(Apache 2.0 license)
|
|
@@ -43,21 +46,13 @@ Gem::Specification.new do |gem|
|
|
|
43
46
|
gem.add_runtime_dependency "rubyzip", "~> 1.1.7"
|
|
44
47
|
gem.add_runtime_dependency "thread_safe", "~> 0.3.5" #(Apache 2.0 license)
|
|
45
48
|
|
|
46
|
-
|
|
47
|
-
gem.platform = RUBY_PLATFORM
|
|
48
|
-
gem.add_runtime_dependency "jrjackson", "~> 0.4.0" #(Apache 2.0 license)
|
|
49
|
-
else
|
|
50
|
-
gem.add_runtime_dependency "oj" #(MIT-style license)
|
|
51
|
-
end
|
|
52
|
-
|
|
53
|
-
if RUBY_ENGINE == "rbx"
|
|
54
|
-
# rubinius puts the ruby stdlib into gems.
|
|
55
|
-
gem.add_runtime_dependency "rubysl"
|
|
49
|
+
gem.add_runtime_dependency "jrjackson", "~> 0.4.0" #(Apache 2.0 license)
|
|
56
50
|
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
51
|
+
gem.add_runtime_dependency "jar-dependencies"
|
|
52
|
+
# as of Feb 3rd 2016, the ruby-maven gem is resolved to version 3.3.3 and that version
|
|
53
|
+
# has an rdoc problem that causes a bundler exception. 3.3.9 is the current latest version
|
|
54
|
+
# which does not have this problem.
|
|
55
|
+
gem.add_runtime_dependency "ruby-maven", "~> 3.3.9"
|
|
61
56
|
|
|
62
|
-
|
|
57
|
+
eval(File.read(File.expand_path("../gemspec_jars.rb", __FILE__)))
|
|
63
58
|
end
|
|
@@ -14,6 +14,7 @@ describe LogStash::Api::Modules::NodeStats do
|
|
|
14
14
|
# DSL describing response structure
|
|
15
15
|
root_structure = {
|
|
16
16
|
"jvm"=>{
|
|
17
|
+
"uptime_in_millis" => Numeric,
|
|
17
18
|
"threads"=>{
|
|
18
19
|
"count"=>Numeric,
|
|
19
20
|
"peak_count"=>Numeric
|
|
@@ -68,7 +69,8 @@ describe LogStash::Api::Modules::NodeStats do
|
|
|
68
69
|
},
|
|
69
70
|
"cpu"=>{
|
|
70
71
|
"total_in_millis"=>Numeric,
|
|
71
|
-
"percent"=>Numeric
|
|
72
|
+
"percent"=>Numeric,
|
|
73
|
+
"load_average" => { "1m" => Numeric }
|
|
72
74
|
}
|
|
73
75
|
},
|
|
74
76
|
"pipeline" => {
|
|
@@ -25,16 +25,16 @@ module ResourceDSLMethods
|
|
|
25
25
|
def test_api(expected, path)
|
|
26
26
|
context "GET #{path}" do
|
|
27
27
|
let(:payload) { LogStash::Json.load(last_response.body) }
|
|
28
|
-
|
|
28
|
+
|
|
29
29
|
before(:all) do
|
|
30
30
|
do_request { get path }
|
|
31
|
-
end
|
|
32
|
-
|
|
31
|
+
end
|
|
32
|
+
|
|
33
33
|
it "should respond OK" do
|
|
34
34
|
expect(last_response).to be_ok
|
|
35
35
|
end
|
|
36
36
|
|
|
37
|
-
|
|
37
|
+
|
|
38
38
|
describe "the default metadata" do
|
|
39
39
|
it "should include the host" do
|
|
40
40
|
expect(payload["host"]).to eql(Socket.gethostname)
|
|
@@ -47,11 +47,19 @@ module ResourceDSLMethods
|
|
|
47
47
|
it "should include the http address" do
|
|
48
48
|
expect(payload["http_address"]).to eql("#{Socket.gethostname}:#{::LogStash::WebServer::DEFAULT_PORTS.first}")
|
|
49
49
|
end
|
|
50
|
+
|
|
51
|
+
it "should include the node name" do
|
|
52
|
+
expect(payload["name"]).to eql(@runner.agent.name)
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
it "should include the node id" do
|
|
56
|
+
expect(payload["id"]).to eql(@runner.agent.id)
|
|
57
|
+
end
|
|
50
58
|
end
|
|
51
|
-
|
|
59
|
+
|
|
52
60
|
hash_to_mapping(expected).each do |resource_path,klass|
|
|
53
61
|
dotted = resource_path.join(".")
|
|
54
|
-
|
|
62
|
+
|
|
55
63
|
it "should set '#{dotted}' at '#{path}' to be a '#{klass}'" do
|
|
56
64
|
expect(last_response).to be_ok # fail early if need be
|
|
57
65
|
resource_path_value = resource_path.reduce(payload) do |acc,v|
|
data/spec/api/spec_helper.rb
CHANGED
|
@@ -5,6 +5,7 @@ require "stud/task"
|
|
|
5
5
|
require "logstash/devutils/rspec/spec_helper"
|
|
6
6
|
$LOAD_PATH.unshift(File.expand_path(File.dirname(__FILE__)))
|
|
7
7
|
require "lib/api/support/resource_dsl_methods"
|
|
8
|
+
require_relative "../support/mocks_classes"
|
|
8
9
|
require 'rspec/expectations'
|
|
9
10
|
require "logstash/settings"
|
|
10
11
|
require 'rack/test'
|
data/spec/conditionals_spec.rb
CHANGED
|
@@ -26,6 +26,8 @@ describe "conditionals in output" do
|
|
|
26
26
|
extend ConditionalFanciness
|
|
27
27
|
|
|
28
28
|
class DummyNullOutput < LogStash::Outputs::Base
|
|
29
|
+
config_name "dummynull"
|
|
30
|
+
|
|
29
31
|
def register
|
|
30
32
|
end
|
|
31
33
|
def multi_receive(events)
|
|
@@ -33,7 +35,7 @@ describe "conditionals in output" do
|
|
|
33
35
|
end
|
|
34
36
|
|
|
35
37
|
before do
|
|
36
|
-
LogStash::
|
|
38
|
+
LogStash::PLUGIN_REGISTRY.add(:output, "dummynull", DummyNullOutput)
|
|
37
39
|
end
|
|
38
40
|
|
|
39
41
|
describe "simple" do
|
|
@@ -436,5 +438,4 @@ describe "conditionals in filter" do
|
|
|
436
438
|
expect(subject[2].get("cond2")).to eq("true")
|
|
437
439
|
end
|
|
438
440
|
end
|
|
439
|
-
|
|
440
441
|
end
|
data/spec/logstash/agent_spec.rb
CHANGED
|
@@ -3,10 +3,11 @@ require "spec_helper"
|
|
|
3
3
|
require "stud/temporary"
|
|
4
4
|
require "logstash/inputs/generator"
|
|
5
5
|
require_relative "../support/mocks_classes"
|
|
6
|
+
require "fileutils"
|
|
7
|
+
require_relative "../support/helpers"
|
|
6
8
|
|
|
7
9
|
describe LogStash::Agent do
|
|
8
10
|
|
|
9
|
-
let(:logger) { double("logger") }
|
|
10
11
|
let(:agent_settings) { LogStash::SETTINGS }
|
|
11
12
|
let(:agent_args) { {} }
|
|
12
13
|
let(:pipeline_settings) { agent_settings.clone }
|
|
@@ -14,15 +15,12 @@ describe LogStash::Agent do
|
|
|
14
15
|
let(:config_file) { Stud::Temporary.pathname }
|
|
15
16
|
let(:config_file_txt) { "input { generator { count => 100000 } } output { }" }
|
|
16
17
|
|
|
17
|
-
|
|
18
|
+
subject { LogStash::Agent.new(agent_settings) }
|
|
18
19
|
|
|
19
20
|
before :each do
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
[:info?, :warn?, :error?, :fatal?, :debug?].each do |level|
|
|
24
|
-
allow(logger).to receive(level)
|
|
25
|
-
end
|
|
21
|
+
# This MUST run first, before `subject` is invoked to ensure clean state
|
|
22
|
+
clear_data_dir
|
|
23
|
+
|
|
26
24
|
File.open(config_file, "w") { |f| f.puts config_file_txt }
|
|
27
25
|
agent_args.each do |key, value|
|
|
28
26
|
agent_settings.set(key, value)
|
|
@@ -31,7 +29,6 @@ describe LogStash::Agent do
|
|
|
31
29
|
pipeline_args.each do |key, value|
|
|
32
30
|
pipeline_settings.set(key, value)
|
|
33
31
|
end
|
|
34
|
-
#subject.logger = logger
|
|
35
32
|
end
|
|
36
33
|
|
|
37
34
|
after :each do
|
|
@@ -40,7 +37,7 @@ describe LogStash::Agent do
|
|
|
40
37
|
end
|
|
41
38
|
|
|
42
39
|
it "fallback to hostname when no name is provided" do
|
|
43
|
-
expect(LogStash::Agent.new.
|
|
40
|
+
expect(LogStash::Agent.new.name).to eq(Socket.gethostname)
|
|
44
41
|
end
|
|
45
42
|
|
|
46
43
|
describe "register_pipeline" do
|
|
@@ -58,13 +55,27 @@ describe LogStash::Agent do
|
|
|
58
55
|
it "should delegate settings to new pipeline" do
|
|
59
56
|
expect(LogStash::Pipeline).to receive(:new) do |arg1, arg2|
|
|
60
57
|
expect(arg1).to eq(config_string)
|
|
61
|
-
|
|
58
|
+
expect(arg2.to_hash).to include(agent_args)
|
|
62
59
|
end
|
|
63
60
|
subject.register_pipeline(pipeline_id, agent_settings)
|
|
64
61
|
end
|
|
65
62
|
end
|
|
66
63
|
|
|
67
|
-
describe "#
|
|
64
|
+
describe "#id" do
|
|
65
|
+
let(:config_file_txt) { "" }
|
|
66
|
+
let(:id_file_data) { File.open(subject.id_path) {|f| f.read } }
|
|
67
|
+
|
|
68
|
+
it "should return a UUID" do
|
|
69
|
+
expect(subject.id).to be_a(String)
|
|
70
|
+
expect(subject.id.size).to be > 0
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
it "should write out the persistent UUID" do
|
|
74
|
+
expect(id_file_data).to eql(subject.id)
|
|
75
|
+
end
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
describe "#execute" do
|
|
68
79
|
let(:config_file_txt) { "input { generator { count => 100000 } } output { }" }
|
|
69
80
|
|
|
70
81
|
before :each do
|
|
@@ -102,7 +113,7 @@ describe LogStash::Agent do
|
|
|
102
113
|
end
|
|
103
114
|
end
|
|
104
115
|
|
|
105
|
-
context "when calling
|
|
116
|
+
context "when calling reload_pipeline!" do
|
|
106
117
|
context "with a config that contains reload incompatible plugins" do
|
|
107
118
|
let(:second_pipeline_config) { "input { stdin {} } filter { } output { }" }
|
|
108
119
|
|
|
@@ -111,7 +122,7 @@ describe LogStash::Agent do
|
|
|
111
122
|
sleep 0.01 until subject.running_pipelines? && subject.pipelines.values.first.ready?
|
|
112
123
|
expect(subject).to_not receive(:upgrade_pipeline)
|
|
113
124
|
File.open(config_file, "w") { |f| f.puts second_pipeline_config }
|
|
114
|
-
subject.send(:
|
|
125
|
+
subject.send(:"reload_pipeline!", "main")
|
|
115
126
|
sleep 0.1
|
|
116
127
|
Stud.stop!(t)
|
|
117
128
|
t.join
|
|
@@ -127,7 +138,45 @@ describe LogStash::Agent do
|
|
|
127
138
|
sleep 0.01 until subject.running_pipelines? && subject.pipelines.values.first.ready?
|
|
128
139
|
expect(subject).to receive(:upgrade_pipeline).once.and_call_original
|
|
129
140
|
File.open(config_file, "w") { |f| f.puts second_pipeline_config }
|
|
130
|
-
subject.send(:
|
|
141
|
+
subject.send(:"reload_pipeline!", "main")
|
|
142
|
+
sleep 0.1
|
|
143
|
+
Stud.stop!(t)
|
|
144
|
+
t.join
|
|
145
|
+
|
|
146
|
+
subject.shutdown
|
|
147
|
+
end
|
|
148
|
+
end
|
|
149
|
+
|
|
150
|
+
end
|
|
151
|
+
context "when calling reload_state!" do
|
|
152
|
+
context "with a pipeline with auto reloading turned off" do
|
|
153
|
+
let(:second_pipeline_config) { "input { generator { } } filter { } output { }" }
|
|
154
|
+
let(:pipeline_args) { { "config.reload.automatic" => false } }
|
|
155
|
+
|
|
156
|
+
it "does not try to reload the pipeline" do
|
|
157
|
+
t = Thread.new { subject.execute }
|
|
158
|
+
sleep 0.01 until subject.running_pipelines? && subject.pipelines.values.first.ready?
|
|
159
|
+
expect(subject).to_not receive(:reload_pipeline!)
|
|
160
|
+
File.open(config_file, "w") { |f| f.puts second_pipeline_config }
|
|
161
|
+
subject.reload_state!
|
|
162
|
+
sleep 0.1
|
|
163
|
+
Stud.stop!(t)
|
|
164
|
+
t.join
|
|
165
|
+
|
|
166
|
+
subject.shutdown
|
|
167
|
+
end
|
|
168
|
+
end
|
|
169
|
+
|
|
170
|
+
context "with a pipeline with auto reloading turned on" do
|
|
171
|
+
let(:second_pipeline_config) { "input { generator { } } filter { } output { }" }
|
|
172
|
+
let(:pipeline_args) { { "config.reload.automatic" => true } }
|
|
173
|
+
|
|
174
|
+
it "tries to reload the pipeline" do
|
|
175
|
+
t = Thread.new { subject.execute }
|
|
176
|
+
sleep 0.01 until subject.running_pipelines? && subject.pipelines.values.first.ready?
|
|
177
|
+
expect(subject).to receive(:reload_pipeline!).once.and_call_original
|
|
178
|
+
File.open(config_file, "w") { |f| f.puts second_pipeline_config }
|
|
179
|
+
subject.reload_state!
|
|
131
180
|
sleep 0.1
|
|
132
181
|
Stud.stop!(t)
|
|
133
182
|
t.join
|
|
@@ -155,9 +204,9 @@ describe LogStash::Agent do
|
|
|
155
204
|
context "if state is clean" do
|
|
156
205
|
it "should periodically reload_state" do
|
|
157
206
|
allow(subject).to receive(:clean_state?).and_return(false)
|
|
158
|
-
expect(subject).to receive(:reload_state!).at_least(3).times
|
|
159
207
|
t = Thread.new { subject.execute }
|
|
160
208
|
sleep 0.01 until subject.running_pipelines? && subject.pipelines.values.first.ready?
|
|
209
|
+
expect(subject).to receive(:reload_state!).at_least(2).times
|
|
161
210
|
sleep 0.1
|
|
162
211
|
Stud.stop!(t)
|
|
163
212
|
t.join
|
|
@@ -205,7 +254,8 @@ describe LogStash::Agent do
|
|
|
205
254
|
let(:second_pipeline_config) { "input { generator {} } filter { } output { }" }
|
|
206
255
|
let(:pipeline_args) { {
|
|
207
256
|
"config.string" => first_pipeline_config,
|
|
208
|
-
"pipeline.workers" => 4
|
|
257
|
+
"pipeline.workers" => 4,
|
|
258
|
+
"config.reload.automatic" => true
|
|
209
259
|
} }
|
|
210
260
|
|
|
211
261
|
before(:each) do
|
|
@@ -216,14 +266,14 @@ describe LogStash::Agent do
|
|
|
216
266
|
it "upgrades the state" do
|
|
217
267
|
expect(subject).to receive(:fetch_config).and_return(second_pipeline_config)
|
|
218
268
|
expect(subject).to receive(:upgrade_pipeline).with(pipeline_id, kind_of(LogStash::Pipeline))
|
|
219
|
-
subject.
|
|
269
|
+
subject.reload_state!
|
|
220
270
|
end
|
|
221
271
|
end
|
|
222
272
|
context "when fetching the same state" do
|
|
223
273
|
it "doesn't upgrade the state" do
|
|
224
274
|
expect(subject).to receive(:fetch_config).and_return(first_pipeline_config)
|
|
225
275
|
expect(subject).to_not receive(:upgrade_pipeline)
|
|
226
|
-
subject.
|
|
276
|
+
subject.reload_state!
|
|
227
277
|
end
|
|
228
278
|
end
|
|
229
279
|
end
|
|
@@ -268,6 +318,10 @@ describe LogStash::Agent do
|
|
|
268
318
|
subject.register_pipeline(pipeline_id, pipeline_settings)
|
|
269
319
|
end
|
|
270
320
|
|
|
321
|
+
after(:each) do
|
|
322
|
+
subject.shutdown
|
|
323
|
+
end
|
|
324
|
+
|
|
271
325
|
context "when the upgrade fails" do
|
|
272
326
|
before :each do
|
|
273
327
|
allow(subject).to receive(:fetch_config).and_return(new_pipeline_config)
|
|
@@ -276,14 +330,14 @@ describe LogStash::Agent do
|
|
|
276
330
|
end
|
|
277
331
|
|
|
278
332
|
it "leaves the state untouched" do
|
|
279
|
-
subject.send(:
|
|
333
|
+
subject.send(:"reload_pipeline!", pipeline_id)
|
|
280
334
|
expect(subject.pipelines[pipeline_id].config_str).to eq(pipeline_config)
|
|
281
335
|
end
|
|
282
336
|
|
|
283
337
|
context "and current state is empty" do
|
|
284
338
|
it "should not start a pipeline" do
|
|
285
339
|
expect(subject).to_not receive(:start_pipeline)
|
|
286
|
-
subject.send(:
|
|
340
|
+
subject.send(:"reload_pipeline!", pipeline_id)
|
|
287
341
|
end
|
|
288
342
|
end
|
|
289
343
|
end
|
|
@@ -293,15 +347,16 @@ describe LogStash::Agent do
|
|
|
293
347
|
before :each do
|
|
294
348
|
allow(subject).to receive(:fetch_config).and_return(new_config)
|
|
295
349
|
allow(subject).to receive(:stop_pipeline)
|
|
350
|
+
allow(subject).to receive(:start_pipeline)
|
|
296
351
|
end
|
|
297
352
|
it "updates the state" do
|
|
298
|
-
subject.send(:
|
|
353
|
+
subject.send(:"reload_pipeline!", pipeline_id)
|
|
299
354
|
expect(subject.pipelines[pipeline_id].config_str).to eq(new_config)
|
|
300
355
|
end
|
|
301
356
|
it "starts the pipeline" do
|
|
302
357
|
expect(subject).to receive(:stop_pipeline)
|
|
303
358
|
expect(subject).to receive(:start_pipeline)
|
|
304
|
-
subject.send(:
|
|
359
|
+
subject.send(:"reload_pipeline!", pipeline_id)
|
|
305
360
|
end
|
|
306
361
|
end
|
|
307
362
|
end
|
|
@@ -330,41 +385,45 @@ describe LogStash::Agent do
|
|
|
330
385
|
|
|
331
386
|
|
|
332
387
|
context "metrics after config reloading" do
|
|
333
|
-
let(:config) { "input { generator { } } output { dummyoutput { } }" }
|
|
334
|
-
let(:config_path) do
|
|
388
|
+
let!(:config) { "input { generator { } } output { dummyoutput { } }" }
|
|
389
|
+
let!(:config_path) do
|
|
335
390
|
f = Stud::Temporary.file
|
|
336
391
|
f.write(config)
|
|
392
|
+
f.fsync
|
|
337
393
|
f.close
|
|
338
394
|
f.path
|
|
339
395
|
end
|
|
340
|
-
let(:interval) { 0.2 }
|
|
341
396
|
let(:pipeline_args) do
|
|
342
397
|
{
|
|
343
|
-
"pipeline.workers" =>
|
|
398
|
+
"pipeline.workers" => 2,
|
|
344
399
|
"path.config" => config_path
|
|
345
400
|
}
|
|
346
401
|
end
|
|
347
402
|
|
|
348
403
|
let(:agent_args) do
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
404
|
+
{
|
|
405
|
+
"config.reload.automatic" => false,
|
|
406
|
+
"pipeline.batch.size" => 1,
|
|
407
|
+
"metric.collect" => true
|
|
408
|
+
}
|
|
352
409
|
end
|
|
353
410
|
|
|
354
411
|
# We need to create theses dummy classes to know how many
|
|
355
412
|
# events where actually generated by the pipeline and successfully send to the output.
|
|
356
413
|
# Theses values are compared with what we store in the metric store.
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
414
|
+
class DummyOutput2 < LogStash::Outputs::DroppingDummyOutput; end
|
|
415
|
+
|
|
416
|
+
let!(:dummy_output) { LogStash::Outputs::DroppingDummyOutput.new }
|
|
417
|
+
let!(:dummy_output2) { DummyOutput2.new }
|
|
418
|
+
let(:initial_generator_threshold) { 1000 }
|
|
360
419
|
|
|
361
420
|
before :each do
|
|
362
|
-
allow(
|
|
421
|
+
allow(LogStash::Outputs::DroppingDummyOutput).to receive(:new).at_least(:once).with(anything).and_return(dummy_output)
|
|
363
422
|
allow(DummyOutput2).to receive(:new).at_least(:once).with(anything).and_return(dummy_output2)
|
|
364
423
|
|
|
365
424
|
allow(LogStash::Plugin).to receive(:lookup).with("input", "generator").and_return(LogStash::Inputs::Generator)
|
|
366
425
|
allow(LogStash::Plugin).to receive(:lookup).with("codec", "plain").and_return(LogStash::Codecs::Plain)
|
|
367
|
-
allow(LogStash::Plugin).to receive(:lookup).with("output", "dummyoutput").and_return(
|
|
426
|
+
allow(LogStash::Plugin).to receive(:lookup).with("output", "dummyoutput").and_return(LogStash::Outputs::DroppingDummyOutput)
|
|
368
427
|
allow(LogStash::Plugin).to receive(:lookup).with("output", "dummyoutput2").and_return(DummyOutput2)
|
|
369
428
|
|
|
370
429
|
@abort_on_exception = Thread.abort_on_exception
|
|
@@ -375,7 +434,8 @@ describe LogStash::Agent do
|
|
|
375
434
|
subject.execute
|
|
376
435
|
end
|
|
377
436
|
|
|
378
|
-
|
|
437
|
+
# wait for some events to reach the dummy_output
|
|
438
|
+
sleep(0.01) until dummy_output.events_received > initial_generator_threshold
|
|
379
439
|
end
|
|
380
440
|
|
|
381
441
|
after :each do
|
|
@@ -392,43 +452,34 @@ describe LogStash::Agent do
|
|
|
392
452
|
let(:new_config_generator_counter) { 500 }
|
|
393
453
|
let(:new_config) { "input { generator { count => #{new_config_generator_counter} } } output { dummyoutput2 {} }" }
|
|
394
454
|
before :each do
|
|
395
|
-
# We know that the store has more events coming in.
|
|
396
|
-
i = 0
|
|
397
|
-
while dummy_output.events.size <= new_config_generator_counter
|
|
398
|
-
i += 1
|
|
399
|
-
raise "Waiting too long!" if i > 20
|
|
400
|
-
sleep(0.1)
|
|
401
|
-
end
|
|
402
|
-
|
|
403
455
|
|
|
404
|
-
# Also force a flush to disk to make sure ruby reload it.
|
|
405
456
|
File.open(config_path, "w") do |f|
|
|
406
457
|
f.write(new_config)
|
|
407
458
|
f.fsync
|
|
408
459
|
end
|
|
409
460
|
|
|
410
|
-
|
|
461
|
+
subject.send(:"reload_pipeline!", "main")
|
|
411
462
|
|
|
412
|
-
#
|
|
413
|
-
sleep(0.01)
|
|
463
|
+
# wait until pipeline restarts
|
|
464
|
+
sleep(0.01) until dummy_output2.events_received > 0
|
|
414
465
|
end
|
|
415
466
|
|
|
416
467
|
it "resets the pipeline metric collector" do
|
|
417
468
|
snapshot = subject.metric.collector.snapshot_metric
|
|
418
469
|
value = snapshot.metric_store.get_with_path("/stats/pipelines")[:stats][:pipelines][:main][:events][:in].value
|
|
419
|
-
expect(value).to
|
|
470
|
+
expect(value).to be <= new_config_generator_counter
|
|
420
471
|
end
|
|
421
472
|
|
|
422
473
|
it "does not reset the global event count" do
|
|
423
474
|
snapshot = subject.metric.collector.snapshot_metric
|
|
424
475
|
value = snapshot.metric_store.get_with_path("/stats/events")[:stats][:events][:in].value
|
|
425
|
-
expect(value).to be >
|
|
476
|
+
expect(value).to be > initial_generator_threshold
|
|
426
477
|
end
|
|
427
478
|
|
|
428
479
|
it "increases the successful reload count" do
|
|
429
480
|
snapshot = subject.metric.collector.snapshot_metric
|
|
430
481
|
value = snapshot.metric_store.get_with_path("/stats/pipelines")[:stats][:pipelines][:main][:reloads][:successes].value
|
|
431
|
-
expect(value).to
|
|
482
|
+
expect(value).to eq(1)
|
|
432
483
|
end
|
|
433
484
|
|
|
434
485
|
it "does not set the failure reload timestamp" do
|
|
@@ -455,28 +506,19 @@ describe LogStash::Agent do
|
|
|
455
506
|
let(:new_config) { "input { generator { count => " }
|
|
456
507
|
let(:new_config_generator_counter) { 500 }
|
|
457
508
|
before :each do
|
|
458
|
-
# We know that the store has more events coming in.
|
|
459
|
-
i = 0
|
|
460
|
-
while dummy_output.events.size <= new_config_generator_counter
|
|
461
|
-
i += 1
|
|
462
|
-
raise "Waiting too long!" if i > 20
|
|
463
|
-
sleep(0.1)
|
|
464
|
-
end
|
|
465
509
|
|
|
466
|
-
|
|
467
|
-
# Also force a flush to disk to make sure ruby reload it.
|
|
468
510
|
File.open(config_path, "w") do |f|
|
|
469
511
|
f.write(new_config)
|
|
470
512
|
f.fsync
|
|
471
513
|
end
|
|
472
514
|
|
|
473
|
-
|
|
515
|
+
subject.send(:"reload_pipeline!", "main")
|
|
474
516
|
end
|
|
475
517
|
|
|
476
518
|
it "does not increase the successful reload count" do
|
|
477
519
|
snapshot = subject.metric.collector.snapshot_metric
|
|
478
520
|
value = snapshot.metric_store.get_with_path("/stats/pipelines")[:stats][:pipelines][:main][:reloads][:successes].value
|
|
479
|
-
expect(value).to
|
|
521
|
+
expect(value).to eq(0)
|
|
480
522
|
end
|
|
481
523
|
|
|
482
524
|
it "does not set the successful reload timestamp" do
|
|
@@ -504,5 +546,43 @@ describe LogStash::Agent do
|
|
|
504
546
|
expect(value).to be > 0
|
|
505
547
|
end
|
|
506
548
|
end
|
|
549
|
+
|
|
550
|
+
context "when reloading a config that raises exception on pipeline.run" do
|
|
551
|
+
let(:new_config) { "input { generator { count => 10000 } }" }
|
|
552
|
+
let(:new_config_generator_counter) { 500 }
|
|
553
|
+
|
|
554
|
+
class BrokenGenerator < LogStash::Inputs::Generator
|
|
555
|
+
def register
|
|
556
|
+
raise ArgumentError
|
|
557
|
+
end
|
|
558
|
+
end
|
|
559
|
+
|
|
560
|
+
before :each do
|
|
561
|
+
|
|
562
|
+
allow(LogStash::Plugin).to receive(:lookup).with("input", "generator").and_return(BrokenGenerator)
|
|
563
|
+
|
|
564
|
+
File.open(config_path, "w") do |f|
|
|
565
|
+
f.write(new_config)
|
|
566
|
+
f.fsync
|
|
567
|
+
end
|
|
568
|
+
|
|
569
|
+
end
|
|
570
|
+
|
|
571
|
+
it "does not increase the successful reload count" do
|
|
572
|
+
expect { subject.send(:"reload_pipeline!", "main") }.to_not change {
|
|
573
|
+
snapshot = subject.metric.collector.snapshot_metric
|
|
574
|
+
reload_metrics = snapshot.metric_store.get_with_path("/stats/pipelines")[:stats][:pipelines][:main][:reloads]
|
|
575
|
+
reload_metrics[:successes].value
|
|
576
|
+
}
|
|
577
|
+
end
|
|
578
|
+
|
|
579
|
+
it "increases the failured reload count" do
|
|
580
|
+
expect { subject.send(:"reload_pipeline!", "main") }.to change {
|
|
581
|
+
snapshot = subject.metric.collector.snapshot_metric
|
|
582
|
+
reload_metrics = snapshot.metric_store.get_with_path("/stats/pipelines")[:stats][:pipelines][:main][:reloads]
|
|
583
|
+
reload_metrics[:failures].value
|
|
584
|
+
}.by(1)
|
|
585
|
+
end
|
|
586
|
+
end
|
|
507
587
|
end
|
|
508
588
|
end
|