logstash-core 5.4.3-java → 5.5.0-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/logstash-core/logstash-core.jar +0 -0
- data/lib/logstash-core/version.rb +1 -1
- data/lib/logstash/api/commands/hot_threads_reporter.rb +2 -2
- data/lib/logstash/api/commands/node.rb +0 -1
- data/lib/logstash/api/commands/stats.rb +0 -1
- data/lib/logstash/config/mixin.rb +5 -43
- data/lib/logstash/config/modules_common.rb +71 -0
- data/lib/logstash/elasticsearch_client.rb +120 -0
- data/lib/logstash/environment.rb +14 -3
- data/lib/logstash/errors.rb +1 -0
- data/lib/logstash/execution_context.rb +11 -3
- data/lib/logstash/inputs/base.rb +2 -0
- data/lib/logstash/instrument/global_metrics.rb +13 -0
- data/lib/logstash/instrument/metric_type/mean.rb +5 -0
- data/lib/logstash/instrument/periodic_poller/jvm.rb +5 -5
- data/lib/logstash/logging/logger.rb +26 -1
- data/lib/logstash/modules/cli_parser.rb +74 -0
- data/lib/logstash/modules/elasticsearch_config.rb +22 -0
- data/lib/logstash/modules/elasticsearch_resource.rb +10 -0
- data/lib/logstash/modules/file_reader.rb +36 -0
- data/lib/logstash/modules/importer.rb +37 -0
- data/lib/logstash/modules/kibana_base_resource.rb +10 -0
- data/lib/logstash/modules/kibana_config.rb +104 -0
- data/lib/logstash/modules/kibana_resource.rb +10 -0
- data/lib/logstash/modules/logstash_config.rb +48 -0
- data/lib/logstash/modules/resource_base.rb +37 -0
- data/lib/logstash/modules/scaffold.rb +44 -0
- data/lib/logstash/namespace.rb +1 -0
- data/lib/logstash/outputs/base.rb +2 -0
- data/lib/logstash/pipeline.rb +18 -4
- data/lib/logstash/plugin.rb +1 -0
- data/lib/logstash/plugins/registry.rb +5 -0
- data/lib/logstash/runner.rb +42 -2
- data/lib/logstash/settings.rb +7 -1
- data/lib/logstash/timestamp.rb +4 -0
- data/lib/logstash/util/dead_letter_queue_manager.rb +61 -0
- data/lib/logstash/util/safe_uri.rb +130 -11
- data/lib/logstash/util/thread_dump.rb +3 -1
- data/lib/logstash/util/wrapped_acked_queue.rb +24 -6
- data/lib/logstash/util/wrapped_synchronous_queue.rb +19 -5
- data/lib/logstash/version.rb +1 -1
- data/locales/en.yml +46 -0
- data/logstash-core.gemspec +7 -2
- data/spec/{api/lib/commands/stats.rb → logstash/api/commands/stats_spec.rb} +7 -2
- data/spec/{api/lib → logstash/api}/errors_spec.rb +1 -1
- data/spec/{api/lib/api → logstash/api/modules}/logging_spec.rb +1 -10
- data/spec/{api/lib/api → logstash/api/modules}/node_plugins_spec.rb +2 -3
- data/spec/{api/lib/api → logstash/api/modules}/node_spec.rb +6 -7
- data/spec/{api/lib/api → logstash/api/modules}/node_stats_spec.rb +2 -2
- data/spec/{api/lib/api → logstash/api/modules}/plugins_spec.rb +4 -3
- data/spec/{api/lib/api → logstash/api/modules}/root_spec.rb +3 -3
- data/spec/{api/lib → logstash/api}/rack_app_spec.rb +0 -0
- data/spec/logstash/config/mixin_spec.rb +2 -2
- data/spec/logstash/execution_context_spec.rb +20 -1
- data/spec/logstash/filter_delegator_spec.rb +2 -1
- data/spec/logstash/inputs/base_spec.rb +1 -1
- data/spec/logstash/output_delegator_spec.rb +2 -1
- data/spec/logstash/outputs/base_spec.rb +1 -1
- data/spec/logstash/pipeline_dlq_commit_spec.rb +107 -0
- data/spec/logstash/pipeline_pq_file_spec.rb +1 -1
- data/spec/logstash/plugin_spec.rb +1 -1
- data/spec/logstash/plugins/registry_spec.rb +22 -5
- data/spec/logstash/runner_spec.rb +122 -19
- data/spec/logstash/settings_spec.rb +91 -0
- data/spec/logstash/timestamp_spec.rb +6 -0
- data/spec/support/helpers.rb +80 -1
- data/spec/support/matchers.rb +13 -0
- data/spec/support/shared_contexts.rb +38 -0
- data/spec/support/shared_examples.rb +1 -1
- metadata +95 -40
- data/spec/api/lib/api/support/resource_dsl_methods.rb +0 -87
- data/spec/api/spec_helper.rb +0 -111
@@ -14,7 +14,8 @@ describe LogStash::FilterDelegator do
|
|
14
14
|
let(:collector) { [] }
|
15
15
|
let(:metric) { LogStash::Instrument::NamespacedNullMetric.new(collector, :null) }
|
16
16
|
let(:events) { [LogStash::Event.new, LogStash::Event.new] }
|
17
|
-
let(:default_execution_context) { LogStash::ExecutionContext.new(:main
|
17
|
+
let(:default_execution_context) { LogStash::ExecutionContext.new(:main, filter_id, "filter",
|
18
|
+
LogStash::Util::DummyDeadLetterQueueWriter.new) }
|
18
19
|
|
19
20
|
before :each do
|
20
21
|
allow(metric).to receive(:namespace).with(anything).and_return(metric)
|
@@ -64,7 +64,7 @@ describe "LogStash::Inputs::Base#decorate" do
|
|
64
64
|
end
|
65
65
|
|
66
66
|
context "execution context" do
|
67
|
-
let(:default_execution_context) { LogStash::ExecutionContext.new(:main) }
|
67
|
+
let(:default_execution_context) { LogStash::ExecutionContext.new(:main, "id", "input", LogStash::Util::DummyDeadLetterQueueWriter.new) }
|
68
68
|
let(:klass) { LogStash::Inputs::NOOP }
|
69
69
|
|
70
70
|
subject(:instance) { klass.new({}) }
|
@@ -9,7 +9,8 @@ describe LogStash::OutputDelegator do
|
|
9
9
|
let(:plugin_args) { {"id" => "foo", "arg1" => "val1"} }
|
10
10
|
let(:collector) { [] }
|
11
11
|
let(:metric) { LogStash::Instrument::NamespacedNullMetric.new(collector, :null) }
|
12
|
-
let(:default_execution_context) { LogStash::ExecutionContext.new(:main
|
12
|
+
let(:default_execution_context) { LogStash::ExecutionContext.new(:main, "foo", "output",
|
13
|
+
LogStash::Util::DummyDeadLetterQueueWriter.new) }
|
13
14
|
|
14
15
|
subject { described_class.new(logger, out_klass, metric, default_execution_context, ::LogStash::OutputDelegatorStrategyRegistry.instance, plugin_args) }
|
15
16
|
|
@@ -80,7 +80,7 @@ describe "LogStash::Outputs::Base#new" do
|
|
80
80
|
end
|
81
81
|
|
82
82
|
context "execution context" do
|
83
|
-
let(:default_execution_context) { LogStash::ExecutionContext.new(:main) }
|
83
|
+
let(:default_execution_context) { LogStash::ExecutionContext.new(:main, "id", "output", LogStash::Util::DummyDeadLetterQueueWriter.new) }
|
84
84
|
let(:klass) { LogStash::Outputs::NOOPSingle }
|
85
85
|
|
86
86
|
subject(:instance) { klass.new(params.dup) }
|
@@ -0,0 +1,107 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require "tmpdir"
|
3
|
+
require "spec_helper"
|
4
|
+
require "logstash/codecs/plain"
|
5
|
+
require_relative "../support/mocks_classes"
|
6
|
+
|
7
|
+
class SingleGeneratorInput < LogStash::Inputs::Base
|
8
|
+
config_name "singlegenerator"
|
9
|
+
milestone 2
|
10
|
+
|
11
|
+
def register
|
12
|
+
end
|
13
|
+
|
14
|
+
def run(queue)
|
15
|
+
queue << LogStash::Event.new
|
16
|
+
end
|
17
|
+
|
18
|
+
def close
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
class DLQCommittingFilter < LogStash::Filters::Base
|
23
|
+
config_name "dlq_commit"
|
24
|
+
milestone 2
|
25
|
+
|
26
|
+
def register()
|
27
|
+
end
|
28
|
+
|
29
|
+
def filter(event)
|
30
|
+
execution_context.dlq_writer.write(event, "my reason")
|
31
|
+
end
|
32
|
+
|
33
|
+
def threadsafe?() true; end
|
34
|
+
|
35
|
+
def close() end
|
36
|
+
end
|
37
|
+
|
38
|
+
describe LogStash::Pipeline do
|
39
|
+
let(:pipeline_settings_obj) { LogStash::SETTINGS }
|
40
|
+
let(:pipeline_settings) do
|
41
|
+
{
|
42
|
+
"pipeline.workers" => 2,
|
43
|
+
"pipeline.id" => pipeline_id,
|
44
|
+
"dead_letter_queue.enable" => enable_dlq,
|
45
|
+
"path.dead_letter_queue" => Dir.mktmpdir
|
46
|
+
}
|
47
|
+
end
|
48
|
+
let(:metric) { LogStash::Instrument::Metric.new(LogStash::Instrument::Collector.new) }
|
49
|
+
let(:test_config) {
|
50
|
+
<<-eos
|
51
|
+
input { singlegenerator { id => input_id } }
|
52
|
+
|
53
|
+
filter { dlq_commit { id => filter_id } }
|
54
|
+
|
55
|
+
output { dummyoutput { id => output_id } }
|
56
|
+
eos
|
57
|
+
}
|
58
|
+
|
59
|
+
subject { LogStash::Pipeline.new(test_config, pipeline_settings_obj, metric) }
|
60
|
+
|
61
|
+
before(:each) do
|
62
|
+
pipeline_settings.each {|k, v| pipeline_settings_obj.set(k, v) }
|
63
|
+
allow(LogStash::Plugin).to receive(:lookup).with("input", "singlegenerator").and_return(SingleGeneratorInput)
|
64
|
+
allow(LogStash::Plugin).to receive(:lookup).with("codec", "plain").and_return(LogStash::Codecs::Plain)
|
65
|
+
allow(LogStash::Plugin).to receive(:lookup).with("filter", "dlq_commit").and_return(DLQCommittingFilter)
|
66
|
+
allow(LogStash::Plugin).to receive(:lookup).with("output", "dummyoutput").and_return(::LogStash::Outputs::DummyOutput)
|
67
|
+
end
|
68
|
+
|
69
|
+
after(:each) do
|
70
|
+
FileUtils.remove_entry pipeline_settings["path.dead_letter_queue"]
|
71
|
+
end
|
72
|
+
|
73
|
+
context "dlq is enabled" do
|
74
|
+
let(:enable_dlq) { true }
|
75
|
+
let(:pipeline_id) { "test-dlq" }
|
76
|
+
|
77
|
+
it "retrieves proper pipeline-level DLQ writer" do
|
78
|
+
expect(LogStash::Util::PluginDeadLetterQueueWriter).to receive(:new).with(anything, "input_id", "singlegenerator").and_call_original
|
79
|
+
expect(LogStash::Util::PluginDeadLetterQueueWriter).to receive(:new).with(anything, "filter_id", "dlq_commit").and_call_original
|
80
|
+
expect(LogStash::Util::PluginDeadLetterQueueWriter).to receive(:new).with(anything, "output_id", "dummyoutput").and_call_original
|
81
|
+
expect_any_instance_of(org.logstash.common.io.DeadLetterQueueWriter).to receive(:close).and_call_original
|
82
|
+
subject.run
|
83
|
+
dlq_path = java.nio.file.Paths.get(pipeline_settings_obj.get("path.dead_letter_queue"), pipeline_id)
|
84
|
+
dlq_reader = org.logstash.common.io.DeadLetterQueueReader.new(dlq_path)
|
85
|
+
entry = dlq_reader.pollEntry(40)
|
86
|
+
expect(entry).to_not be_nil
|
87
|
+
expect(entry.reason).to eq("my reason")
|
88
|
+
end
|
89
|
+
end
|
90
|
+
|
91
|
+
context "dlq is disabled" do
|
92
|
+
let(:enable_dlq) { false }
|
93
|
+
let(:pipeline_id) { "test-without-dlq" }
|
94
|
+
|
95
|
+
it "does not write to the DLQ" do
|
96
|
+
expect(LogStash::Util::PluginDeadLetterQueueWriter).to receive(:new).with(anything, "input_id", "singlegenerator").and_call_original
|
97
|
+
expect(LogStash::Util::PluginDeadLetterQueueWriter).to receive(:new).with(anything, "filter_id", "dlq_commit").and_call_original
|
98
|
+
expect(LogStash::Util::PluginDeadLetterQueueWriter).to receive(:new).with(anything, "output_id", "dummyoutput").and_call_original
|
99
|
+
expect(LogStash::Util::DummyDeadLetterQueueWriter).to receive(:new).and_call_original
|
100
|
+
expect_any_instance_of(LogStash::Util::DummyDeadLetterQueueWriter).to receive(:close).and_call_original
|
101
|
+
subject.run
|
102
|
+
dlq_path = java.nio.file.Paths.get(pipeline_settings_obj.get("path.dead_letter_queue"), pipeline_id)
|
103
|
+
expect(java.nio.file.Files.exists(dlq_path)).to eq(false)
|
104
|
+
end
|
105
|
+
end
|
106
|
+
|
107
|
+
end
|
@@ -36,7 +36,7 @@ class PipelinePqFileOutput < LogStash::Outputs::Base
|
|
36
36
|
end
|
37
37
|
|
38
38
|
describe LogStash::Pipeline do
|
39
|
-
let(:pipeline_settings_obj) { LogStash::SETTINGS }
|
39
|
+
let(:pipeline_settings_obj) { LogStash::SETTINGS.clone }
|
40
40
|
let(:pipeline_id) { "main" }
|
41
41
|
|
42
42
|
let(:multiline_id) { "my-multiline" }
|
@@ -48,7 +48,7 @@ describe LogStash::Plugin do
|
|
48
48
|
|
49
49
|
it "can be set and get" do
|
50
50
|
expect(subject.execution_context).to be_nil
|
51
|
-
execution_context = LogStash::ExecutionContext.new(:main)
|
51
|
+
execution_context = LogStash::ExecutionContext.new(:main, "id", "type", nil)
|
52
52
|
subject.execution_context = execution_context
|
53
53
|
expect(subject.execution_context).to eq(execution_context)
|
54
54
|
end
|
@@ -53,13 +53,30 @@ describe LogStash::Plugins::Registry do
|
|
53
53
|
end
|
54
54
|
|
55
55
|
context "when loading plugin manually configured" do
|
56
|
-
|
57
|
-
class SimplePlugin
|
58
|
-
end
|
56
|
+
let(:simple_plugin) { Class.new }
|
59
57
|
|
58
|
+
it "should return the plugin" do
|
60
59
|
expect { registry.lookup("filter", "simple_plugin") }.to raise_error(LoadError)
|
61
|
-
registry.add(:filter, "simple_plugin",
|
62
|
-
expect(registry.lookup("filter", "simple_plugin")).to eq(
|
60
|
+
registry.add(:filter, "simple_plugin", simple_plugin)
|
61
|
+
expect(registry.lookup("filter", "simple_plugin")).to eq(simple_plugin)
|
62
|
+
end
|
63
|
+
|
64
|
+
it "doesn't add multiple time the same plugin" do
|
65
|
+
plugin1 = Class.new
|
66
|
+
plugin2 = Class.new
|
67
|
+
|
68
|
+
registry.add(:filter, "simple_plugin", plugin1)
|
69
|
+
registry.add(:filter, "simple_plugin", plugin2)
|
70
|
+
|
71
|
+
expect(registry.plugins_with_type(:filter)).to include(plugin1)
|
72
|
+
expect(registry.plugins_with_type(:filter).size).to eq(1)
|
73
|
+
end
|
74
|
+
|
75
|
+
it "allow you find plugin by type" do
|
76
|
+
registry.add(:filter, "simple_plugin", simple_plugin)
|
77
|
+
|
78
|
+
expect(registry.plugins_with_type(:filter)).to include(simple_plugin)
|
79
|
+
expect(registry.plugins_with_type(:modules)).to match([])
|
63
80
|
end
|
64
81
|
end
|
65
82
|
end
|
@@ -6,7 +6,11 @@ require "stud/trap"
|
|
6
6
|
require "stud/temporary"
|
7
7
|
require "logstash/util/java_version"
|
8
8
|
require "logstash/logging/json"
|
9
|
+
require "logstash/config/modules_common"
|
10
|
+
require "logstash/elasticsearch_client"
|
9
11
|
require "json"
|
12
|
+
require_relative "../support/helpers"
|
13
|
+
require_relative "../support/matchers"
|
10
14
|
|
11
15
|
class NullRunner
|
12
16
|
def run(args); end
|
@@ -32,13 +36,6 @@ describe LogStash::Runner do
|
|
32
36
|
end
|
33
37
|
end
|
34
38
|
|
35
|
-
after :each do
|
36
|
-
LogStash::SETTINGS.reset
|
37
|
-
end
|
38
|
-
|
39
|
-
after :all do
|
40
|
-
end
|
41
|
-
|
42
39
|
describe "argument precedence" do
|
43
40
|
let(:config) { "input {} output {}" }
|
44
41
|
let(:cli_args) { ["-e", config, "-w", "20"] }
|
@@ -48,10 +45,6 @@ describe LogStash::Runner do
|
|
48
45
|
allow(LogStash::SETTINGS).to receive(:read_yaml).and_return(settings_yml_hash)
|
49
46
|
end
|
50
47
|
|
51
|
-
after :each do
|
52
|
-
LogStash::SETTINGS.reset
|
53
|
-
end
|
54
|
-
|
55
48
|
it "favors the last occurence of an option" do
|
56
49
|
expect(LogStash::Agent).to receive(:new) do |settings|
|
57
50
|
expect(settings.get("config.string")).to eq(config)
|
@@ -142,6 +135,7 @@ describe LogStash::Runner do
|
|
142
135
|
context "with a good configuration" do
|
143
136
|
let(:pipeline_string) { "input { } filter { } output { }" }
|
144
137
|
it "should exit successfully" do
|
138
|
+
expect(logger).not_to receive(:fatal)
|
145
139
|
expect(subject.run(args)).to eq(0)
|
146
140
|
end
|
147
141
|
end
|
@@ -165,38 +159,56 @@ describe LogStash::Runner do
|
|
165
159
|
allow(pipeline).to receive(:run).and_return(task)
|
166
160
|
allow(pipeline).to receive(:shutdown)
|
167
161
|
end
|
168
|
-
|
162
|
+
|
169
163
|
context "when :path.data is defined by the user" do
|
170
164
|
let(:test_data_path) { "/tmp/ls-test-data" }
|
171
165
|
let(:test_queue_path) { test_data_path + "/" + "queue" }
|
172
|
-
|
166
|
+
let(:test_dlq_path) { test_data_path + "/" + "dead_letter_queue" }
|
167
|
+
|
173
168
|
it "should set data paths" do
|
174
169
|
expect(LogStash::Agent).to receive(:new) do |settings|
|
175
170
|
expect(settings.get("path.data")).to eq(test_data_path)
|
176
171
|
expect(settings.get("path.queue")).to eq(test_queue_path)
|
172
|
+
expect(settings.get("path.dead_letter_queue")).to eq(test_dlq_path)
|
177
173
|
end
|
178
|
-
|
174
|
+
|
179
175
|
args = ["--path.data", test_data_path, "-e", pipeline_string]
|
180
176
|
subject.run("bin/logstash", args)
|
181
177
|
end
|
182
|
-
|
178
|
+
|
183
179
|
context "and path.queue is manually set" do
|
184
180
|
let(:queue_override_path) { "/tmp/queue-override_path" }
|
185
|
-
|
181
|
+
|
186
182
|
it "should set data paths" do
|
187
183
|
expect(LogStash::Agent).to receive(:new) do |settings|
|
188
184
|
expect(settings.get("path.data")).to eq(test_data_path)
|
189
185
|
expect(settings.get("path.queue")).to eq(queue_override_path)
|
190
186
|
end
|
191
|
-
|
187
|
+
|
192
188
|
LogStash::SETTINGS.set("path.queue", queue_override_path)
|
193
|
-
|
189
|
+
|
190
|
+
args = ["--path.data", test_data_path, "-e", pipeline_string]
|
191
|
+
subject.run("bin/logstash", args)
|
192
|
+
end
|
193
|
+
end
|
194
|
+
|
195
|
+
context "and path.dead_letter_queue is manually set" do
|
196
|
+
let(:queue_override_path) { "/tmp/queue-override_path" }
|
197
|
+
|
198
|
+
it "should set data paths" do
|
199
|
+
expect(LogStash::Agent).to receive(:new) do |settings|
|
200
|
+
expect(settings.get("path.data")).to eq(test_data_path)
|
201
|
+
expect(settings.get("path.dead_letter_queue")).to eq(queue_override_path)
|
202
|
+
end
|
203
|
+
|
204
|
+
LogStash::SETTINGS.set("path.dead_letter_queue", queue_override_path)
|
205
|
+
|
194
206
|
args = ["--path.data", test_data_path, "-e", pipeline_string]
|
195
207
|
subject.run("bin/logstash", args)
|
196
208
|
end
|
197
209
|
end
|
198
210
|
end
|
199
|
-
|
211
|
+
|
200
212
|
context "when :http.host is defined by the user" do
|
201
213
|
it "should pass the value to the webserver" do
|
202
214
|
expect(LogStash::Agent).to receive(:new) do |settings|
|
@@ -303,6 +315,97 @@ describe LogStash::Runner do
|
|
303
315
|
end
|
304
316
|
end
|
305
317
|
|
318
|
+
describe "logstash modules" do
|
319
|
+
describe "--config.test_and_exit" do
|
320
|
+
subject { LogStash::Runner.new("") }
|
321
|
+
let(:args) { ["-t", "--modules", module_string] }
|
322
|
+
|
323
|
+
context "with a good configuration" do
|
324
|
+
let(:module_string) { "cef" }
|
325
|
+
it "should exit successfully" do
|
326
|
+
skip("Skipped until cef module is added back to the codebase as explained in #7455")
|
327
|
+
expect(logger).not_to receive(:fatal)
|
328
|
+
expect(subject.run(args)).to eq(0)
|
329
|
+
end
|
330
|
+
end
|
331
|
+
|
332
|
+
context "with a bad configuration" do
|
333
|
+
let(:module_string) { "rlwekjhrewlqrkjh" }
|
334
|
+
it "should fail by returning a bad exit code" do
|
335
|
+
expect(logger).to receive(:fatal)
|
336
|
+
expect(subject.run(args)).to eq(1)
|
337
|
+
end
|
338
|
+
end
|
339
|
+
end
|
340
|
+
|
341
|
+
describe "--modules" do
|
342
|
+
let(:args) { ["--modules", module_string] }
|
343
|
+
let(:agent_instance) { double("agent") }
|
344
|
+
context "with an available module specified but no connection to elasticsearch" do
|
345
|
+
let(:module_string) { "cef" }
|
346
|
+
before do
|
347
|
+
expect(logger).to receive(:fatal) do |msg, hash|
|
348
|
+
expect(msg).to eq("An unexpected error occurred!")
|
349
|
+
expect(hash).to be_a_config_loading_error_hash(
|
350
|
+
/Failed to import module configurations to Elasticsearch. Module: cef/)
|
351
|
+
end
|
352
|
+
end
|
353
|
+
it "should log fatally and return a bad exit code" do
|
354
|
+
skip("Skipped until cef module is added back to the codebase as explained in #7455")
|
355
|
+
expect(subject.run("bin/logstash", args)).to eq(1)
|
356
|
+
end
|
357
|
+
end
|
358
|
+
|
359
|
+
context "with an available module specified and a mocked connection to elasticsearch" do
|
360
|
+
let(:module_string) { "cef" }
|
361
|
+
let(:client) { double(:client) }
|
362
|
+
let(:response) { double(:response) }
|
363
|
+
before do
|
364
|
+
allow(response).to receive(:status).and_return(404)
|
365
|
+
allow(client).to receive(:head).and_return(response)
|
366
|
+
allow(client).to receive(:can_connect?).and_return(true)
|
367
|
+
allow(agent_instance).to receive(:register_pipeline)
|
368
|
+
allow(agent_instance).to receive(:execute)
|
369
|
+
allow(agent_instance).to receive(:shutdown)
|
370
|
+
allow(LogStash::ElasticsearchClient).to receive(:build).and_return(client)
|
371
|
+
|
372
|
+
expect(client).to receive(:put).at_least(15).times do |path, content|
|
373
|
+
LogStash::ElasticsearchClient::Response.new(201, "", {})
|
374
|
+
end
|
375
|
+
expect(LogStash::Agent).to receive(:new) do |settings|
|
376
|
+
pipelines = LogStash::Config::ModulesCommon.pipeline_configs(settings)
|
377
|
+
expect(pipelines).not_to be_empty
|
378
|
+
cef_pipeline = pipelines.first
|
379
|
+
expect(cef_pipeline).to include("pipeline_id", "config_string")
|
380
|
+
expect(cef_pipeline["pipeline_id"]).to include('cef')
|
381
|
+
expect(cef_pipeline["config_string"]).to include('index => "cef-')
|
382
|
+
agent_instance
|
383
|
+
end
|
384
|
+
expect(logger).not_to receive(:fatal)
|
385
|
+
expect(logger).not_to receive(:error)
|
386
|
+
end
|
387
|
+
it "should not terminate logstash" do
|
388
|
+
skip("Skipped until cef module is added back to the codebase as explained in #7455")
|
389
|
+
expect(subject.run("bin/logstash", args)).to be_nil
|
390
|
+
end
|
391
|
+
end
|
392
|
+
|
393
|
+
context "with an unavailable module specified" do
|
394
|
+
let(:module_string) { "fancypants" }
|
395
|
+
before do
|
396
|
+
expect(logger).to receive(:fatal) do |msg, hash|
|
397
|
+
expect(msg).to eq("An unexpected error occurred!")
|
398
|
+
expect(hash).to be_a_config_loading_error_hash(
|
399
|
+
/The modules specified are not available yet. Specified modules: \["fancypants"\] Available modules:/)
|
400
|
+
end
|
401
|
+
end
|
402
|
+
it "should log fatally and return a bad exit code" do
|
403
|
+
expect(subject.run("bin/logstash", args)).to eq(1)
|
404
|
+
end
|
405
|
+
end
|
406
|
+
end
|
407
|
+
end
|
408
|
+
|
306
409
|
describe "--log.level" do
|
307
410
|
before :each do
|
308
411
|
allow_any_instance_of(subject).to receive(:show_version)
|
@@ -1,5 +1,6 @@
|
|
1
1
|
# encoding: utf-8
|
2
2
|
require "spec_helper"
|
3
|
+
require "logstash/util/environment_variables"
|
3
4
|
require "logstash/settings"
|
4
5
|
require "fileutils"
|
5
6
|
|
@@ -146,4 +147,94 @@ describe LogStash::Settings do
|
|
146
147
|
end
|
147
148
|
end
|
148
149
|
end
|
150
|
+
|
151
|
+
describe "#from_yaml" do
|
152
|
+
|
153
|
+
context "env placeholders in flat logstash.yml" do
|
154
|
+
|
155
|
+
after do
|
156
|
+
ENV.delete('SOME_LOGSTASH_SPEC_ENV_VAR')
|
157
|
+
ENV.delete('some.logstash.spec.env.var')
|
158
|
+
end
|
159
|
+
|
160
|
+
subject do
|
161
|
+
settings = described_class.new
|
162
|
+
settings.register(LogStash::Setting::String.new("interpolated", "missing"))
|
163
|
+
settings.register(LogStash::Setting::String.new("with_dot", "missing"))
|
164
|
+
settings
|
165
|
+
end
|
166
|
+
|
167
|
+
let(:values) {{
|
168
|
+
"interpolated" => "${SOME_LOGSTASH_SPEC_ENV_VAR}",
|
169
|
+
"with_dot" => "${some.logstash.spec.env.var}"
|
170
|
+
}}
|
171
|
+
let(:yaml_path) do
|
172
|
+
p = Stud::Temporary.pathname
|
173
|
+
FileUtils.mkdir_p(p)
|
174
|
+
|
175
|
+
::File.open(::File.join(p, "logstash.yml"), "w+") do |f|
|
176
|
+
f.write(YAML.dump(values))
|
177
|
+
end
|
178
|
+
p
|
179
|
+
end
|
180
|
+
|
181
|
+
it "can interpolate environment into settings" do
|
182
|
+
expect(subject.get('interpolated')).to eq("missing")
|
183
|
+
expect(subject.get('with_dot')).to eq("missing")
|
184
|
+
ENV['SOME_LOGSTASH_SPEC_ENV_VAR'] = "correct_setting"
|
185
|
+
ENV['some.logstash.spec.env.var'] = "correct_setting_for_dotted"
|
186
|
+
subject.from_yaml(yaml_path)
|
187
|
+
expect(subject.get('interpolated')).to eq("correct_setting")
|
188
|
+
expect(subject.get('with_dot')).to eq("correct_setting_for_dotted")
|
189
|
+
end
|
190
|
+
end
|
191
|
+
end
|
192
|
+
|
193
|
+
context "env placeholders in nested logstash.yml" do
|
194
|
+
|
195
|
+
before do
|
196
|
+
ENV['lsspecdomain'] = "domain1"
|
197
|
+
ENV['lsspecdomain2'] = "domain2"
|
198
|
+
end
|
199
|
+
|
200
|
+
after do
|
201
|
+
ENV.delete('lsspecdomain')
|
202
|
+
ENV.delete('lsspecdomain2')
|
203
|
+
end
|
204
|
+
|
205
|
+
subject do
|
206
|
+
settings = described_class.new
|
207
|
+
settings.register(LogStash::Setting::ArrayCoercible.new("host", String, []))
|
208
|
+
settings.register(LogStash::Setting::ArrayCoercible.new("modules", Hash, []))
|
209
|
+
settings
|
210
|
+
end
|
211
|
+
|
212
|
+
let(:values) {{
|
213
|
+
"host" => ["dev1.${lsspecdomain}", "dev2.${lsspecdomain}"],
|
214
|
+
"modules" => [
|
215
|
+
{"name" => "${lsspecdomain}", "testing" => "${lsspecdomain}"},
|
216
|
+
{"name" => "${lsspecdomain2}", "testing" => "${lsspecdomain2}"}
|
217
|
+
]
|
218
|
+
}}
|
219
|
+
let(:yaml_path) do
|
220
|
+
p = Stud::Temporary.pathname
|
221
|
+
FileUtils.mkdir_p(p)
|
222
|
+
|
223
|
+
::File.open(::File.join(p, "logstash.yml"), "w+") do |f|
|
224
|
+
f.write(YAML.dump(values))
|
225
|
+
end
|
226
|
+
p
|
227
|
+
end
|
228
|
+
|
229
|
+
it "can interpolate environment into settings" do
|
230
|
+
expect(subject.get('host')).to match_array([])
|
231
|
+
expect(subject.get('modules')).to match_array([])
|
232
|
+
subject.from_yaml(yaml_path)
|
233
|
+
expect(subject.get('host')).to match_array(["dev1.domain1", "dev2.domain1"])
|
234
|
+
expect(subject.get('modules')).to match_array([
|
235
|
+
{"name" => "domain1", "testing" => "domain1"},
|
236
|
+
{"name" => "domain2", "testing" => "domain2"}
|
237
|
+
])
|
238
|
+
end
|
239
|
+
end
|
149
240
|
end
|